From 9b569dedbc9375a7a74a8c83df55ecca025d13a7 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Tue, 30 Sep 2025 11:25:49 +0200 Subject: [PATCH 1/3] tests: improve logging --- tests/test_inputs_schema_gen.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/test_inputs_schema_gen.py b/tests/test_inputs_schema_gen.py index 00091b4c..e583bdd1 100644 --- a/tests/test_inputs_schema_gen.py +++ b/tests/test_inputs_schema_gen.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 """Tests for cwl-inputs-schema-gen.""" +import logging from pathlib import Path import pytest @@ -8,11 +9,13 @@ from ruamel.yaml import YAML from cwl_utils.inputs_schema_gen import cwl_to_jsonschema -from cwl_utils.loghandler import _logger as _cwlutilslogger from cwl_utils.parser import load_document_by_uri from .util import get_path +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger() + TEST_PARAMS = [ # Packed Case ( @@ -51,10 +54,10 @@ def test_cwl_inputs_to_jsonschema(tool_path: Path, inputs_path: Path) -> None: cwl_obj = load_document_by_uri(tool_path.as_uri()) - _cwlutilslogger.info(f"Generating schema for {tool_path.name}") + logger.info(f"Generating schema for {tool_path.name}") json_schema = cwl_to_jsonschema(cwl_obj) - _cwlutilslogger.info( + logger.info( f"Testing {inputs_path.name} against schema generated for input {tool_path.name}" ) @@ -65,7 +68,7 @@ def test_cwl_inputs_to_jsonschema(tool_path: Path, inputs_path: Path) -> None: try: validate(input_obj, json_schema) except (ValidationError, SchemaError) as err: - _cwlutilslogger.error( + logger.error( f"Validation failed for {inputs_path.name} " f"against schema generated for input {tool_path.name}" ) @@ -79,10 +82,10 @@ def test_cwl_inputs_to_jsonschema_fails() -> None: cwl_obj = load_document_by_uri(tool_path.as_uri()) - _cwlutilslogger.info(f"Generating schema for {tool_path.name}") + logger.info(f"Generating schema for {tool_path.name}") json_schema = cwl_to_jsonschema(cwl_obj) - _cwlutilslogger.info( + logger.info( f"Testing {inputs_path.name} against schema generated for input {tool_path.name}" ) From dda7c3a714aec8bd961e3e2b55a9a53ee78171c3 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Tue, 30 Sep 2025 12:01:36 +0200 Subject: [PATCH 2/3] helper utility to test cwl-inputs-schema-gen --- tests/jschema_validate.py | 71 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100755 tests/jschema_validate.py diff --git a/tests/jschema_validate.py b/tests/jschema_validate.py new file mode 100755 index 00000000..f855116c --- /dev/null +++ b/tests/jschema_validate.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 + +import argparse +import os +import sys +from pathlib import Path + +from jsonschema.validators import validate +from ruamel.yaml import YAML + +from cwl_utils.inputs_schema_gen import cwl_to_jsonschema +from cwl_utils.parser import load_document_by_uri + + +def main() -> None: + parser = argparse.ArgumentParser(description="test cwl-inputs-schema-gen.") + parser.add_argument( + "--outdir", + type=str, + default=os.path.abspath("."), + help="Output directory. This is present only for cwltest's usage, and it is ignored.", + ) + parser.add_argument( + "--quiet", action="store_true", help="Only print warnings and errors." + ) + parser.add_argument("--version", action="store_true", help="Print version and exit") + parser.add_argument( + "workflow", + type=str, + nargs="?", + default=None, + metavar="cwl_document", + help="path or URL to a CWL Workflow, " "CommandLineTool, or ExpressionTool.", + ) + parser.add_argument( + "job_order", + nargs=argparse.REMAINDER, + metavar="inputs_object", + help="path or URL to a YAML or JSON " + "formatted description of the required input values for the given " + "`cwl_document`.", + ) + + args = parser.parse_args(sys.argv[1:]) + + if args.version: + print(f"{sys.argv[1]} 0.0.1") + return + + if len(args.job_order) < 1: + job_order = {} + job_order_name = "empty inputs" + else: + yaml = YAML() + job_order_name = args.job_order[0] + job_order = yaml.load(Path(job_order_name)) + + validate( + job_order, + cwl_to_jsonschema(load_document_by_uri(args.workflow)), + ) + + if not args.quiet: + print( + f"Validation of the JSON schema generated from {args.workflow} " + "using {job_order_name} suceeded." + ) + + +if __name__ == "__main__": + main() From 5e8d628944334ae83ede5937328d2c412c1c5648 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Tue, 30 Sep 2025 14:13:45 +0200 Subject: [PATCH 3/3] test jsonschema with bad inputs --- testdata/echo-tool.cwl | 17 +++++++++ testdata/empty.json | 1 + testdata/null-expression-echo-job.json | 3 ++ testdata/null-expression1-job.json | 3 ++ testdata/null-expression2-tool.cwl | 14 +++++++ tests/test_inputs_schema_gen.py | 53 +++++++++++++++++++++++++- 6 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 testdata/echo-tool.cwl create mode 100644 testdata/empty.json create mode 100644 testdata/null-expression-echo-job.json create mode 100644 testdata/null-expression1-job.json create mode 100644 testdata/null-expression2-tool.cwl diff --git a/testdata/echo-tool.cwl b/testdata/echo-tool.cwl new file mode 100644 index 00000000..5a9835b8 --- /dev/null +++ b/testdata/echo-tool.cwl @@ -0,0 +1,17 @@ +#!/usr/bin/env cwl-runner + +class: CommandLineTool +cwlVersion: v1.2 +inputs: + in: + type: Any + inputBinding: {} +outputs: + out: + type: string + outputBinding: + glob: out.txt + loadContents: true + outputEval: $(self[0].contents) +baseCommand: echo +stdout: out.txt diff --git a/testdata/empty.json b/testdata/empty.json new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/testdata/empty.json @@ -0,0 +1 @@ +{} diff --git a/testdata/null-expression-echo-job.json b/testdata/null-expression-echo-job.json new file mode 100644 index 00000000..9047fcfe --- /dev/null +++ b/testdata/null-expression-echo-job.json @@ -0,0 +1,3 @@ +{ + "in": null +} diff --git a/testdata/null-expression1-job.json b/testdata/null-expression1-job.json new file mode 100644 index 00000000..b8ffcd56 --- /dev/null +++ b/testdata/null-expression1-job.json @@ -0,0 +1,3 @@ +{ + "i1": null +} \ No newline at end of file diff --git a/testdata/null-expression2-tool.cwl b/testdata/null-expression2-tool.cwl new file mode 100644 index 00000000..2b09ec04 --- /dev/null +++ b/testdata/null-expression2-tool.cwl @@ -0,0 +1,14 @@ +#!/usr/bin/env cwl-runner + +class: ExpressionTool +requirements: + - class: InlineJavascriptRequirement +cwlVersion: v1.2 + +inputs: + i1: Any + +outputs: + output: int + +expression: "$({'output': (inputs.i1 == 'the-default' ? 1 : 2)})" \ No newline at end of file diff --git a/tests/test_inputs_schema_gen.py b/tests/test_inputs_schema_gen.py index e583bdd1..77f6577d 100644 --- a/tests/test_inputs_schema_gen.py +++ b/tests/test_inputs_schema_gen.py @@ -75,7 +75,7 @@ def test_cwl_inputs_to_jsonschema(tool_path: Path, inputs_path: Path) -> None: raise SchemaError(f"{inputs_path.name} failed schema validation") from err -def test_cwl_inputs_to_jsonschema_fails() -> None: +def test_cwl_inputs_to_jsonschema_single_fail() -> None: """Compare tool schema of param 1 against input schema of param 2.""" tool_path: Path = TEST_PARAMS[0][0] inputs_path: Path = TEST_PARAMS[3][1] @@ -96,3 +96,54 @@ def test_cwl_inputs_to_jsonschema_fails() -> None: # We expect this to fail with pytest.raises(ValidationError): validate(input_obj, json_schema) + + +BAD_TEST_PARAMS = [ + # Any without defaults cannot be unspecified + ( + get_path("testdata/null-expression2-tool.cwl"), + get_path("testdata/empty.json"), + "'i1' is a required property", + ), + # null to Any type without a default value. + ( + get_path("testdata/null-expression2-tool.cwl"), + get_path("testdata/null-expression1-job.json"), + "None is not valid under any of the given schemas", + ), + # Any without defaults, unspecified + ( + get_path("testdata/echo-tool.cwl"), + get_path("testdata/null-expression-echo-job.json"), + "None is not valid under any of the given schemas", + ), + # JSON null provided for required input + ( + get_path("testdata/echo-tool.cwl"), + get_path("testdata/null-expression1-job.json"), + "'in' is a required property", + ), +] + + +@pytest.mark.parametrize("tool_path,inputs_path,exception_message", BAD_TEST_PARAMS) +def test_cwl_inputs_to_jsonschema_fails( + tool_path: Path, + inputs_path: Path, + exception_message: str, +) -> None: + cwl_obj = load_document_by_uri(tool_path.as_uri()) + + logger.info(f"Generating schema for {tool_path.name}") + json_schema = cwl_to_jsonschema(cwl_obj) + + logger.info( + f"Testing {inputs_path.name} against schema generated for input {tool_path.name}" + ) + + yaml = YAML() + + input_obj = yaml.load(inputs_path) + + with pytest.raises(ValidationError, match=exception_message): + validate(input_obj, json_schema)