From 21bba73680d9b07855e273594073efd52fa92b6a Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Fri, 3 Oct 2025 10:48:48 +0200 Subject: [PATCH 1/2] Drop Python 3.9 support --- .github/workflows/ci-tests.yml | 2 +- CONTRIBUTING.md | 2 +- Makefile | 2 +- cwl_utils/cwl_v1_0_expression_refactor.py | 157 ++++++++++---------- cwl_utils/cwl_v1_1_expression_refactor.py | 163 ++++++++++----------- cwl_utils/cwl_v1_2_expression_refactor.py | 169 +++++++++++----------- cwl_utils/expression.py | 22 +-- cwl_utils/expression_refactor.py | 9 +- cwl_utils/file_formats.py | 9 +- cwl_utils/graph_split.py | 3 +- cwl_utils/image_puller.py | 5 +- cwl_utils/inputs_schema_gen.py | 20 +-- cwl_utils/pack.py | 10 +- cwl_utils/parser/__init__.py | 24 +-- cwl_utils/parser/cwl_v1_0_utils.py | 40 ++--- cwl_utils/parser/cwl_v1_1_utils.py | 40 ++--- cwl_utils/parser/cwl_v1_2_utils.py | 42 +++--- cwl_utils/parser/utils.py | 64 ++++---- cwl_utils/sandboxjs.py | 14 +- cwl_utils/singularity.py | 3 +- cwl_utils/utils.py | 18 +-- mypy-requirements.txt | 1 - mypy-stubs/cwlformat/formatter.pyi | 2 +- mypy-stubs/rdflib/collection.pyi | 3 +- mypy-stubs/rdflib/compare.pyi | 1 - mypy-stubs/rdflib/graph.pyi | 135 +++++++++-------- mypy-stubs/rdflib/namespace/__init__.pyi | 2 +- mypy-stubs/rdflib/paths.pyi | 4 +- mypy-stubs/rdflib/plugin.pyi | 2 +- mypy-stubs/rdflib/query.pyi | 18 +-- mypy-stubs/rdflib/resource.pyi | 9 +- mypy-stubs/rdflib/term.pyi | 5 +- pyproject.toml | 3 +- requirements.txt | 1 - tests/test_format.py | 3 +- tox.ini | 53 ++++--- 36 files changed, 514 insertions(+), 546 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index ffa28360..04277a31 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -20,7 +20,7 @@ jobs: strategy: matrix: py-ver-major: [3] - py-ver-minor: [9, 10, 11, 12, 13, 14] + py-ver-minor: [10, 11, 12, 13, 14] step: [lint, unit, bandit, mypy] env: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 618e350c..d3a06883 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ Style guide: - PEP-8 / format with ``black`` via ``make format`` -- Python 3.9+ compatible code +- Python 3.10+ compatible code - PEP-484 type hints It is suggested that you run `git config blame.ignoreRevsFile .git-blame-ignore-revs` diff --git a/Makefile b/Makefile index fe931959..4b94c56c 100644 --- a/Makefile +++ b/Makefile @@ -170,7 +170,7 @@ shellcheck: FORCE shellcheck release-test.sh pyupgrade: $(PYSOURCES) - pyupgrade --exit-zero-even-if-changed --py39-plus $^ + pyupgrade --exit-zero-even-if-changed --py310-plus $^ auto-walrus $^ release-test: FORCE diff --git a/cwl_utils/cwl_v1_0_expression_refactor.py b/cwl_utils/cwl_v1_0_expression_refactor.py index f4d8fc23..5c96f353 100755 --- a/cwl_utils/cwl_v1_0_expression_refactor.py +++ b/cwl_utils/cwl_v1_0_expression_refactor.py @@ -6,7 +6,7 @@ import hashlib import uuid from collections.abc import Mapping, MutableSequence, Sequence -from typing import Any, Optional, Union, cast +from typing import Any, Optional, cast from ruamel import yaml from schema_salad.sourceline import SourceLine @@ -49,8 +49,8 @@ def escape_expression_field(contents: str) -> str: def clean_type_ids( - cwltype: Union[cwl.ArraySchema, cwl.InputRecordSchema], -) -> Union[cwl.ArraySchema, cwl.InputRecordSchema]: + cwltype: cwl.ArraySchema | cwl.InputRecordSchema, +) -> cwl.ArraySchema | cwl.InputRecordSchema: """Simplify type identifiers.""" result = copy.deepcopy(cwltype) if isinstance(result, cwl.ArraySchema): @@ -74,8 +74,8 @@ def clean_type_ids( def get_expression( - string: str, inputs: CWLObjectType, self: Optional[CWLOutputType] -) -> Optional[str]: + string: str, inputs: CWLObjectType, self: CWLOutputType | None +) -> str | None: """ Find and return a normalized CWL expression, if any. @@ -132,7 +132,7 @@ def get_expression( def etool_to_cltool( - etool: cwl.ExpressionTool, expressionLib: Optional[list[str]] = None + etool: cwl.ExpressionTool, expressionLib: list[str] | None = None ) -> cwl.CommandLineTool: """Convert a ExpressionTool to a CommandLineTool.""" inputs = yaml.comments.CommentedSeq() # preserve the order @@ -201,12 +201,12 @@ def etool_to_cltool( def traverse( - process: Union[cwl.CommandLineTool, cwl.ExpressionTool, cwl.Workflow], + process: cwl.CommandLineTool | cwl.ExpressionTool | cwl.Workflow, replace_etool: bool, inside: bool, skip_command_line1: bool, skip_command_line2: bool, -) -> tuple[Union[cwl.CommandLineTool, cwl.ExpressionTool, cwl.Workflow], bool]: +) -> tuple[cwl.CommandLineTool | cwl.ExpressionTool | cwl.Workflow, bool]: """Convert the given process and any subprocesses.""" if not inside and isinstance(process, cwl.CommandLineTool): process = expand_stream_shortcuts(process) @@ -314,18 +314,16 @@ def load_step( def generate_etool_from_expr( expr: str, - target: Union[cwl.CommandInputParameter, cwl.InputParameter], + target: cwl.CommandInputParameter | cwl.InputParameter, no_inputs: bool = False, - self_type: Optional[ - Union[ - cwl.InputParameter, - cwl.CommandInputParameter, - list[Union[cwl.InputParameter, cwl.CommandInputParameter]], - ] - ] = None, # if the "self" input should be a different type than the "result" output - extra_processes: Optional[ - Sequence[Union[cwl.Workflow, cwl.WorkflowStep, cwl.CommandLineTool]] - ] = None, + self_type: None | ( + cwl.InputParameter + | cwl.CommandInputParameter + | list[cwl.InputParameter | cwl.CommandInputParameter] + ) = None, # if the "self" input should be a different type than the "result" output + extra_processes: None | ( + Sequence[cwl.Workflow | cwl.WorkflowStep | cwl.CommandLineTool] + ) = None, ) -> cwl.ExpressionTool: """Convert a CWL Expression into an ExpressionTool.""" inputs = yaml.comments.CommentedSeq() @@ -333,10 +331,11 @@ def generate_etool_from_expr( if not self_type: self_type = target if isinstance(self_type, list): - new_type: Union[ - list[Union[cwl.ArraySchema, cwl.InputRecordSchema]], - Union[cwl.ArraySchema, cwl.InputRecordSchema], - ] = [clean_type_ids(t.type_) for t in self_type if t.type_] + new_type: ( + list[cwl.ArraySchema | cwl.InputRecordSchema] + | cwl.ArraySchema + | cwl.InputRecordSchema + ) = [clean_type_ids(t.type_) for t in self_type if t.type_] elif self_type.type_: new_type = clean_type_ids(self_type.type_) else: @@ -406,8 +405,8 @@ def generate_etool_from_expr( def get_input_for_id( - name: str, tool: Union[cwl.CommandLineTool, cwl.Workflow] -) -> Optional[cwl.CommandInputParameter]: + name: str, tool: cwl.CommandLineTool | cwl.Workflow +) -> cwl.CommandInputParameter | None: """Determine the CommandInputParameter for the given input name.""" name = name.split("/")[-1] @@ -426,9 +425,9 @@ def get_input_for_id( def find_expressionLib( processes: Sequence[ - Union[cwl.CommandLineTool, cwl.Workflow, cwl.ExpressionTool, cwl.WorkflowStep] + cwl.CommandLineTool | cwl.Workflow | cwl.ExpressionTool | cwl.WorkflowStep ], -) -> Optional[list[str]]: +) -> list[str] | None: """ Return the expressionLib from the highest priority InlineJavascriptRequirement. @@ -447,30 +446,30 @@ def replace_expr_with_etool( expr: str, name: str, workflow: cwl.Workflow, - target: Union[cwl.CommandInputParameter, cwl.InputParameter], - source: Optional[Union[str, list[Any]]], + target: cwl.CommandInputParameter | cwl.InputParameter, + source: str | list[Any] | None, replace_etool: bool = False, - extra_process: Optional[ - Union[cwl.Workflow, cwl.WorkflowStep, cwl.CommandLineTool] - ] = None, - source_type: Optional[cwl.CommandInputParameter] = None, + extra_process: None | ( + cwl.Workflow | cwl.WorkflowStep | cwl.CommandLineTool + ) = None, + source_type: cwl.CommandInputParameter | None = None, ) -> None: """Modify the given workflow, replacing the expr with an standalone ExpressionTool.""" - extra_processes: list[ - Union[cwl.Workflow, cwl.WorkflowStep, cwl.CommandLineTool] - ] = [workflow] + extra_processes: list[cwl.Workflow | cwl.WorkflowStep | cwl.CommandLineTool] = [ + workflow + ] if extra_process: extra_processes.append(extra_process) etool: cwl.ExpressionTool = generate_etool_from_expr( expr, target, source is None, source_type, extra_processes ) if replace_etool: - processes: list[Union[cwl.WorkflowStep, cwl.Workflow, cwl.CommandLineTool]] = [ + processes: list[cwl.WorkflowStep | cwl.Workflow | cwl.CommandLineTool] = [ workflow ] if extra_process: processes.append(extra_process) - final_tool: Union[cwl.ExpressionTool, cwl.CommandLineTool] = etool_to_cltool( + final_tool: cwl.ExpressionTool | cwl.CommandLineTool = etool_to_cltool( etool, find_expressionLib(processes) ) else: @@ -515,10 +514,10 @@ def replace_wf_input_ref_with_step_output( def empty_inputs( - process_or_step: Union[ - cwl.CommandLineTool, cwl.WorkflowStep, cwl.ExpressionTool, cwl.Workflow - ], - parent: Optional[cwl.Workflow] = None, + process_or_step: ( + cwl.CommandLineTool | cwl.WorkflowStep | cwl.ExpressionTool | cwl.Workflow + ), + parent: cwl.Workflow | None = None, ) -> dict[str, Any]: """Produce a mock input object for the given inputs.""" result = {} @@ -653,20 +652,20 @@ def process_workflow_reqs_and_hints( # and connecting all workflow inputs to the generated step modified = False inputs = empty_inputs(workflow) - generated_res_reqs: list[tuple[str, Union[int, str]]] = [] - generated_iwdr_reqs: list[tuple[str, Union[int, str]]] = [] - generated_envVar_reqs: list[tuple[str, Union[int, str]]] = [] + generated_res_reqs: list[tuple[str, int | str]] = [] + generated_iwdr_reqs: list[tuple[str, int | str]] = [] + generated_envVar_reqs: list[tuple[str, int | str]] = [] prop_reqs: tuple[ - Union[ - type[cwl.EnvVarRequirement], - type[cwl.ResourceRequirement], - type[cwl.InitialWorkDirRequirement], - ], + ( + type[cwl.EnvVarRequirement] + | type[cwl.ResourceRequirement] + | type[cwl.InitialWorkDirRequirement] + ), ..., ] = () - resourceReq: Optional[cwl.ResourceRequirement] = None - envVarReq: Optional[cwl.EnvVarRequirement] = None - iwdr: Optional[cwl.InitialWorkDirRequirement] = None + resourceReq: cwl.ResourceRequirement | None = None + envVarReq: cwl.EnvVarRequirement | None = None + iwdr: cwl.InitialWorkDirRequirement | None = None if workflow.requirements is not None: for req in cast(list[cwl.ProcessRequirement], workflow.requirements): if req and isinstance(req, cwl.EnvVarRequirement): @@ -986,8 +985,8 @@ def process_level_reqs( target_process = step.run inputs = empty_inputs(process) generated_res_reqs: list[tuple[str, str]] = [] - generated_iwdr_reqs: list[tuple[str, Union[int, str], Any]] = [] - generated_envVar_reqs: list[tuple[str, Union[int, str]]] = [] + generated_iwdr_reqs: list[tuple[str, int | str, Any]] = [] + generated_envVar_reqs: list[tuple[str, int | str]] = [] if not step.id: return False step_name = step.id.split("#", 1)[-1] @@ -1410,7 +1409,7 @@ def traverse_CommandLineTool( inp.linkMerge = None if replace_etool: processes = [parent] - final_etool: Union[cwl.CommandLineTool, cwl.ExpressionTool] = ( + final_etool: cwl.CommandLineTool | cwl.ExpressionTool = ( etool_to_cltool(etool, find_expressionLib(processes)) ) else: @@ -1528,7 +1527,7 @@ def simplify_step_id(uri: str) -> str: def remove_JSReq( - process: Union[cwl.CommandLineTool, cwl.WorkflowStep, cwl.Workflow], + process: cwl.CommandLineTool | cwl.WorkflowStep | cwl.Workflow, skip_command_line1: bool, ) -> None: """Since the InlineJavascriptRequirement is longer needed, remove it.""" @@ -1559,7 +1558,7 @@ def replace_step_clt_expr_with_etool( target: cwl.InputParameter, step: cwl.WorkflowStep, replace_etool: bool, - self_name: Optional[str] = None, + self_name: str | None = None, ) -> None: """Convert a step level CWL Expression to a sibling expression step.""" etool_inputs = cltool_inputs_to_etool_inputs(step.run) @@ -1568,7 +1567,7 @@ def replace_step_clt_expr_with_etool( ) if replace_etool: processes = [workflow] - etool: Union[cwl.ExpressionTool, cwl.CommandLineTool] = etool_to_cltool( + etool: cwl.ExpressionTool | cwl.CommandLineTool = etool_to_cltool( temp_etool, find_expressionLib(processes) ) else: @@ -1594,8 +1593,8 @@ def replace_clt_hintreq_expr_with_etool( target: cwl.InputParameter, step: cwl.WorkflowStep, replace_etool: bool, - self_name: Optional[str] = None, -) -> Union[cwl.CommandLineTool, cwl.ExpressionTool]: + self_name: str | None = None, +) -> cwl.CommandLineTool | cwl.ExpressionTool: """Factor out an expression inside a CommandLineTool req or hint into a sibling step.""" # Same as replace_step_clt_expr_with_etool or different? etool_inputs = cltool_inputs_to_etool_inputs(step.run) @@ -1604,7 +1603,7 @@ def replace_clt_hintreq_expr_with_etool( ) if replace_etool: processes = [workflow] - etool: Union[cwl.CommandLineTool, cwl.ExpressionTool] = etool_to_cltool( + etool: cwl.CommandLineTool | cwl.ExpressionTool = etool_to_cltool( temp_etool, find_expressionLib(processes) ) else: @@ -1691,12 +1690,12 @@ def cltool_step_outputs_to_workflow_outputs( def generate_etool_from_expr2( expr: str, target: cwl.InputParameter, - inputs: Sequence[Union[cwl.InputParameter, cwl.CommandInputParameter]], - self_name: Optional[str] = None, - process: Optional[Union[cwl.CommandLineTool, cwl.ExpressionTool]] = None, - extra_processes: Optional[ - Sequence[Union[cwl.Workflow, cwl.WorkflowStep, cwl.CommandLineTool]] - ] = None, + inputs: Sequence[cwl.InputParameter | cwl.CommandInputParameter], + self_name: str | None = None, + process: cwl.CommandLineTool | cwl.ExpressionTool | None = None, + extra_processes: None | ( + Sequence[cwl.Workflow | cwl.WorkflowStep | cwl.CommandLineTool] + ) = None, ) -> cwl.ExpressionTool: """Generate an ExpressionTool to achieve the same result as the given expression.""" outputs = yaml.comments.CommentedSeq() @@ -1723,7 +1722,7 @@ def generate_etool_from_expr2( ) hints = None procs: list[ - Union[cwl.CommandLineTool, cwl.ExpressionTool, cwl.Workflow, cwl.WorkflowStep] + cwl.CommandLineTool | cwl.ExpressionTool | cwl.Workflow | cwl.WorkflowStep ] = [] if process: procs.append(process) @@ -1811,9 +1810,9 @@ def traverse_step( if not target: raise WorkflowException("target not found") input_source_id = None - source_type: Optional[ - Union[list[cwl.InputParameter], cwl.InputParameter] - ] = None + source_type: None | (list[cwl.InputParameter] | cwl.InputParameter) = ( + None + ) if inp.source: if isinstance(inp.source, MutableSequence): input_source_id = [] @@ -1935,14 +1934,14 @@ def replace_step_valueFrom_expr_with_etool( expr: str, name: str, workflow: cwl.Workflow, - target: Union[cwl.CommandInputParameter, cwl.InputParameter], + target: cwl.CommandInputParameter | cwl.InputParameter, step: cwl.WorkflowStep, step_inp: cwl.WorkflowStepInput, - original_process: Union[cwl.CommandLineTool, cwl.ExpressionTool], + original_process: cwl.CommandLineTool | cwl.ExpressionTool, original_step_ins: list[cwl.WorkflowStepInput], - source: Optional[Union[str, list[str]]], + source: str | list[str] | None, replace_etool: bool, - source_type: Optional[Union[cwl.InputParameter, list[cwl.InputParameter]]] = None, + source_type: cwl.InputParameter | list[cwl.InputParameter] | None = None, ) -> None: """Replace a WorkflowStep level 'valueFrom' expression with a sibling ExpressionTool step.""" if not step_inp.id: @@ -1965,15 +1964,13 @@ def replace_step_valueFrom_expr_with_etool( ) if replace_etool: processes: list[ - Union[ - cwl.Workflow, cwl.CommandLineTool, cwl.ExpressionTool, cwl.WorkflowStep - ] + (cwl.Workflow | cwl.CommandLineTool | cwl.ExpressionTool | cwl.WorkflowStep) ] = [ workflow, step, ] cltool = etool_to_cltool(temp_etool, find_expressionLib(processes)) - etool: Union[cwl.ExpressionTool, cwl.CommandLineTool] = cltool + etool: cwl.ExpressionTool | cwl.CommandLineTool = cltool else: etool = temp_etool wf_step_inputs = copy.deepcopy(original_step_ins) diff --git a/cwl_utils/cwl_v1_1_expression_refactor.py b/cwl_utils/cwl_v1_1_expression_refactor.py index b77ad7ed..4f9ca0ba 100755 --- a/cwl_utils/cwl_v1_1_expression_refactor.py +++ b/cwl_utils/cwl_v1_1_expression_refactor.py @@ -6,7 +6,7 @@ import hashlib import uuid from collections.abc import Mapping, MutableSequence, Sequence -from typing import Any, Optional, Union, cast +from typing import Any, Optional, cast from ruamel import yaml from schema_salad.sourceline import SourceLine @@ -49,8 +49,8 @@ def escape_expression_field(contents: str) -> str: def clean_type_ids( - cwltype: Union[cwl.ArraySchema, cwl.InputRecordSchema], -) -> Union[cwl.ArraySchema, cwl.InputRecordSchema]: + cwltype: cwl.ArraySchema | cwl.InputRecordSchema, +) -> cwl.ArraySchema | cwl.InputRecordSchema: """Simplify type identifiers.""" result = copy.deepcopy(cwltype) if isinstance(result, cwl.ArraySchema): @@ -74,8 +74,8 @@ def clean_type_ids( def get_expression( - string: str, inputs: CWLObjectType, self: Optional[CWLOutputType] -) -> Optional[str]: + string: str, inputs: CWLObjectType, self: CWLOutputType | None +) -> str | None: """ Find and return a normalized CWL expression, if any. @@ -132,7 +132,7 @@ def get_expression( def etool_to_cltool( - etool: cwl.ExpressionTool, expressionLib: Optional[list[str]] = None + etool: cwl.ExpressionTool, expressionLib: list[str] | None = None ) -> cwl.CommandLineTool: """Convert a ExpressionTool to a CommandLineTool.""" inputs = yaml.comments.CommentedSeq() # preserve the order @@ -201,12 +201,12 @@ def etool_to_cltool( def traverse( - process: Union[cwl.CommandLineTool, cwl.ExpressionTool, cwl.Workflow], + process: cwl.CommandLineTool | cwl.ExpressionTool | cwl.Workflow, replace_etool: bool, inside: bool, skip_command_line1: bool, skip_command_line2: bool, -) -> tuple[Union[cwl.CommandLineTool, cwl.ExpressionTool, cwl.Workflow], bool]: +) -> tuple[cwl.CommandLineTool | cwl.ExpressionTool | cwl.Workflow, bool]: """Convert the given process and any subprocesses.""" if not inside and isinstance(process, cwl.CommandLineTool): process = expand_stream_shortcuts(process) @@ -314,18 +314,16 @@ def load_step( def generate_etool_from_expr( expr: str, - target: Union[cwl.CommandInputParameter, cwl.WorkflowInputParameter], + target: cwl.CommandInputParameter | cwl.WorkflowInputParameter, no_inputs: bool = False, - self_type: Optional[ - Union[ - cwl.WorkflowInputParameter, - cwl.CommandInputParameter, - list[Union[cwl.WorkflowInputParameter, cwl.CommandInputParameter]], - ] - ] = None, # if the "self" input should be a different type than the "result" output - extra_processes: Optional[ - Sequence[Union[cwl.Workflow, cwl.WorkflowStep, cwl.CommandLineTool]] - ] = None, + self_type: None | ( + cwl.WorkflowInputParameter + | cwl.CommandInputParameter + | list[cwl.WorkflowInputParameter | cwl.CommandInputParameter] + ) = None, # if the "self" input should be a different type than the "result" output + extra_processes: None | ( + Sequence[cwl.Workflow | cwl.WorkflowStep | cwl.CommandLineTool] + ) = None, ) -> cwl.ExpressionTool: """Convert a CWL Expression into an ExpressionTool.""" inputs = yaml.comments.CommentedSeq() @@ -333,10 +331,11 @@ def generate_etool_from_expr( if not self_type: self_type = target if isinstance(self_type, list): - new_type: Union[ - list[Union[cwl.ArraySchema, cwl.InputRecordSchema]], - Union[cwl.ArraySchema, cwl.InputRecordSchema], - ] = [clean_type_ids(t.type_) for t in self_type] + new_type: ( + list[cwl.ArraySchema | cwl.InputRecordSchema] + | cwl.ArraySchema + | cwl.InputRecordSchema + ) = [clean_type_ids(t.type_) for t in self_type] else: new_type = clean_type_ids(self_type.type_) inputs.append( @@ -404,8 +403,8 @@ def generate_etool_from_expr( def get_input_for_id( - name: str, tool: Union[cwl.CommandLineTool, cwl.Workflow] -) -> Optional[cwl.CommandInputParameter]: + name: str, tool: cwl.CommandLineTool | cwl.Workflow +) -> cwl.CommandInputParameter | None: """Determine the CommandInputParameter for the given input name.""" name = name.split("/")[-1] @@ -424,9 +423,9 @@ def get_input_for_id( def find_expressionLib( processes: Sequence[ - Union[cwl.CommandLineTool, cwl.Workflow, cwl.ExpressionTool, cwl.WorkflowStep] + cwl.CommandLineTool | cwl.Workflow | cwl.ExpressionTool | cwl.WorkflowStep ], -) -> Optional[list[str]]: +) -> list[str] | None: """ Return the expressionLib from the highest priority InlineJavascriptRequirement. @@ -445,30 +444,30 @@ def replace_expr_with_etool( expr: str, name: str, workflow: cwl.Workflow, - target: Union[cwl.CommandInputParameter, cwl.WorkflowInputParameter], - source: Optional[Union[str, list[Any]]], + target: cwl.CommandInputParameter | cwl.WorkflowInputParameter, + source: str | list[Any] | None, replace_etool: bool = False, - extra_process: Optional[ - Union[cwl.Workflow, cwl.WorkflowStep, cwl.CommandLineTool] - ] = None, - source_type: Optional[cwl.CommandInputParameter] = None, + extra_process: None | ( + cwl.Workflow | cwl.WorkflowStep | cwl.CommandLineTool + ) = None, + source_type: cwl.CommandInputParameter | None = None, ) -> None: """Modify the given workflow, replacing the expr with an standalone ExpressionTool.""" - extra_processes: list[ - Union[cwl.Workflow, cwl.WorkflowStep, cwl.CommandLineTool] - ] = [workflow] + extra_processes: list[cwl.Workflow | cwl.WorkflowStep | cwl.CommandLineTool] = [ + workflow + ] if extra_process: extra_processes.append(extra_process) etool: cwl.ExpressionTool = generate_etool_from_expr( expr, target, source is None, source_type, extra_processes ) if replace_etool: - processes: list[Union[cwl.WorkflowStep, cwl.Workflow, cwl.CommandLineTool]] = [ + processes: list[cwl.WorkflowStep | cwl.Workflow | cwl.CommandLineTool] = [ workflow ] if extra_process: processes.append(extra_process) - final_tool: Union[cwl.ExpressionTool, cwl.CommandLineTool] = etool_to_cltool( + final_tool: cwl.ExpressionTool | cwl.CommandLineTool = etool_to_cltool( etool, find_expressionLib(processes) ) else: @@ -513,10 +512,10 @@ def replace_wf_input_ref_with_step_output( def empty_inputs( - process_or_step: Union[ - cwl.CommandLineTool, cwl.WorkflowStep, cwl.ExpressionTool, cwl.Workflow - ], - parent: Optional[cwl.Workflow] = None, + process_or_step: ( + cwl.CommandLineTool | cwl.WorkflowStep | cwl.ExpressionTool | cwl.Workflow + ), + parent: cwl.Workflow | None = None, ) -> dict[str, Any]: """Produce a mock input object for the given inputs.""" result = {} @@ -653,20 +652,20 @@ def process_workflow_reqs_and_hints( # and connecting all workflow inputs to the generated step modified = False inputs = empty_inputs(workflow) - generated_res_reqs: list[tuple[str, Union[int, str]]] = [] - generated_iwdr_reqs: list[tuple[str, Union[int, str]]] = [] - generated_envVar_reqs: list[tuple[str, Union[int, str]]] = [] + generated_res_reqs: list[tuple[str, int | str]] = [] + generated_iwdr_reqs: list[tuple[str, int | str]] = [] + generated_envVar_reqs: list[tuple[str, int | str]] = [] prop_reqs: tuple[ - Union[ - type[cwl.EnvVarRequirement], - type[cwl.ResourceRequirement], - type[cwl.InitialWorkDirRequirement], - ], + ( + type[cwl.EnvVarRequirement] + | type[cwl.ResourceRequirement] + | type[cwl.InitialWorkDirRequirement] + ), ..., ] = () - resourceReq: Optional[cwl.ResourceRequirement] = None - envVarReq: Optional[cwl.EnvVarRequirement] = None - iwdr: Optional[cwl.InitialWorkDirRequirement] = None + resourceReq: cwl.ResourceRequirement | None = None + envVarReq: cwl.EnvVarRequirement | None = None + iwdr: cwl.InitialWorkDirRequirement | None = None if workflow.requirements is not None: for req in cast(list[cwl.ProcessRequirement], workflow.requirements): if req and isinstance(req, cwl.EnvVarRequirement): @@ -986,8 +985,8 @@ def process_level_reqs( target_process = step.run inputs = empty_inputs(process) generated_res_reqs: list[tuple[str, str]] = [] - generated_iwdr_reqs: list[tuple[str, Union[int, str], Any]] = [] - generated_envVar_reqs: list[tuple[str, Union[int, str]]] = [] + generated_iwdr_reqs: list[tuple[str, int | str, Any]] = [] + generated_envVar_reqs: list[tuple[str, int | str]] = [] if not step.id: return False step_name = step.id.split("#", 1)[-1] @@ -1410,7 +1409,7 @@ def traverse_CommandLineTool( inp.linkMerge = None if replace_etool: processes = [parent] - final_etool: Union[cwl.CommandLineTool, cwl.ExpressionTool] = ( + final_etool: cwl.CommandLineTool | cwl.ExpressionTool = ( etool_to_cltool(etool, find_expressionLib(processes)) ) else: @@ -1528,7 +1527,7 @@ def simplify_step_id(uri: str) -> str: def remove_JSReq( - process: Union[cwl.CommandLineTool, cwl.WorkflowStep, cwl.Workflow], + process: cwl.CommandLineTool | cwl.WorkflowStep | cwl.Workflow, skip_command_line1: bool, ) -> None: """Since the InlineJavascriptRequirement is longer needed, remove it.""" @@ -1559,7 +1558,7 @@ def replace_step_clt_expr_with_etool( target: cwl.WorkflowInputParameter, step: cwl.WorkflowStep, replace_etool: bool, - self_name: Optional[str] = None, + self_name: str | None = None, ) -> None: """Convert a step level CWL Expression to a sibling expression step.""" etool_inputs = cltool_inputs_to_etool_inputs(step.run) @@ -1568,7 +1567,7 @@ def replace_step_clt_expr_with_etool( ) if replace_etool: processes = [workflow] - etool: Union[cwl.ExpressionTool, cwl.CommandLineTool] = etool_to_cltool( + etool: cwl.ExpressionTool | cwl.CommandLineTool = etool_to_cltool( temp_etool, find_expressionLib(processes) ) else: @@ -1594,8 +1593,8 @@ def replace_clt_hintreq_expr_with_etool( target: cwl.WorkflowInputParameter, step: cwl.WorkflowStep, replace_etool: bool, - self_name: Optional[str] = None, -) -> Union[cwl.CommandLineTool, cwl.ExpressionTool]: + self_name: str | None = None, +) -> cwl.CommandLineTool | cwl.ExpressionTool: """Factor out an expression inside a CommandLineTool req or hint into a sibling step.""" # Same as replace_step_clt_expr_with_etool or different? etool_inputs = cltool_inputs_to_etool_inputs(step.run) @@ -1604,7 +1603,7 @@ def replace_clt_hintreq_expr_with_etool( ) if replace_etool: processes = [workflow] - etool: Union[cwl.CommandLineTool, cwl.ExpressionTool] = etool_to_cltool( + etool: cwl.CommandLineTool | cwl.ExpressionTool = etool_to_cltool( temp_etool, find_expressionLib(processes) ) else: @@ -1690,13 +1689,13 @@ def cltool_step_outputs_to_workflow_outputs( def generate_etool_from_expr2( expr: str, - target: Union[cwl.CommandInputParameter, cwl.WorkflowInputParameter], - inputs: Sequence[Union[cwl.WorkflowInputParameter, cwl.CommandInputParameter]], - self_name: Optional[str] = None, - process: Optional[Union[cwl.CommandLineTool, cwl.ExpressionTool]] = None, - extra_processes: Optional[ - Sequence[Union[cwl.Workflow, cwl.WorkflowStep, cwl.CommandLineTool]] - ] = None, + target: cwl.CommandInputParameter | cwl.WorkflowInputParameter, + inputs: Sequence[cwl.WorkflowInputParameter | cwl.CommandInputParameter], + self_name: str | None = None, + process: cwl.CommandLineTool | cwl.ExpressionTool | None = None, + extra_processes: None | ( + Sequence[cwl.Workflow | cwl.WorkflowStep | cwl.CommandLineTool] + ) = None, ) -> cwl.ExpressionTool: """Generate an ExpressionTool to achieve the same result as the given expression.""" outputs = yaml.comments.CommentedSeq() @@ -1723,7 +1722,7 @@ def generate_etool_from_expr2( ) hints = None procs: list[ - Union[cwl.CommandLineTool, cwl.ExpressionTool, cwl.Workflow, cwl.WorkflowStep] + cwl.CommandLineTool | cwl.ExpressionTool | cwl.Workflow | cwl.WorkflowStep ] = [] if process: procs.append(process) @@ -1811,9 +1810,9 @@ def traverse_step( if not target: raise WorkflowException("target not found") input_source_id = None - source_type: Optional[ - Union[list[cwl.WorkflowInputParameter], cwl.WorkflowInputParameter] - ] = None + source_type: None | ( + list[cwl.WorkflowInputParameter] | cwl.WorkflowInputParameter + ) = None if inp.source: if isinstance(inp.source, MutableSequence): input_source_id = [] @@ -1927,16 +1926,16 @@ def replace_step_valueFrom_expr_with_etool( expr: str, name: str, workflow: cwl.Workflow, - target: Union[cwl.CommandInputParameter, cwl.WorkflowInputParameter], + target: cwl.CommandInputParameter | cwl.WorkflowInputParameter, step: cwl.WorkflowStep, step_inp: cwl.WorkflowStepInput, - original_process: Union[cwl.CommandLineTool, cwl.ExpressionTool], + original_process: cwl.CommandLineTool | cwl.ExpressionTool, original_step_ins: list[cwl.WorkflowStepInput], - source: Optional[Union[str, list[str]]], + source: str | list[str] | None, replace_etool: bool, - source_type: Optional[ - Union[cwl.WorkflowInputParameter, list[cwl.WorkflowInputParameter]] - ] = None, + source_type: None | ( + cwl.WorkflowInputParameter | list[cwl.WorkflowInputParameter] + ) = None, ) -> None: """Replace a WorkflowStep level 'valueFrom' expression with a sibling ExpressionTool step.""" if not step_inp.id: @@ -1959,15 +1958,13 @@ def replace_step_valueFrom_expr_with_etool( ) if replace_etool: processes: list[ - Union[ - cwl.Workflow, cwl.CommandLineTool, cwl.ExpressionTool, cwl.WorkflowStep - ] + (cwl.Workflow | cwl.CommandLineTool | cwl.ExpressionTool | cwl.WorkflowStep) ] = [ workflow, step, ] cltool = etool_to_cltool(temp_etool, find_expressionLib(processes)) - etool: Union[cwl.ExpressionTool, cwl.CommandLineTool] = cltool + etool: cwl.ExpressionTool | cwl.CommandLineTool = cltool else: etool = temp_etool wf_step_inputs = copy.deepcopy(original_step_ins) diff --git a/cwl_utils/cwl_v1_2_expression_refactor.py b/cwl_utils/cwl_v1_2_expression_refactor.py index 236cd105..1b5398b3 100755 --- a/cwl_utils/cwl_v1_2_expression_refactor.py +++ b/cwl_utils/cwl_v1_2_expression_refactor.py @@ -6,7 +6,7 @@ import hashlib import uuid from collections.abc import Mapping, MutableSequence, Sequence -from typing import Any, Optional, Union, cast +from typing import Any, Optional, cast from ruamel import yaml from schema_salad.sourceline import SourceLine @@ -49,8 +49,8 @@ def escape_expression_field(contents: str) -> str: def clean_type_ids( - cwltype: Union[cwl.ArraySchema, cwl.InputRecordSchema], -) -> Union[cwl.ArraySchema, cwl.InputRecordSchema]: + cwltype: cwl.ArraySchema | cwl.InputRecordSchema, +) -> cwl.ArraySchema | cwl.InputRecordSchema: """Simplify type identifiers.""" result = copy.deepcopy(cwltype) if isinstance(result, cwl.ArraySchema): @@ -74,8 +74,8 @@ def clean_type_ids( def get_expression( - string: str, inputs: CWLObjectType, self: Optional[CWLOutputType] -) -> Optional[str]: + string: str, inputs: CWLObjectType, self: CWLOutputType | None +) -> str | None: """ Find and return a normalized CWL expression, if any. @@ -132,7 +132,7 @@ def get_expression( def etool_to_cltool( - etool: cwl.ExpressionTool, expressionLib: Optional[list[str]] = None + etool: cwl.ExpressionTool, expressionLib: list[str] | None = None ) -> cwl.CommandLineTool: """Convert a ExpressionTool to a CommandLineTool.""" inputs = yaml.comments.CommentedSeq() # preserve the order @@ -201,12 +201,12 @@ def etool_to_cltool( def traverse( - process: Union[cwl.CommandLineTool, cwl.ExpressionTool, cwl.Workflow], + process: cwl.CommandLineTool | cwl.ExpressionTool | cwl.Workflow, replace_etool: bool, inside: bool, skip_command_line1: bool, skip_command_line2: bool, -) -> tuple[Union[cwl.CommandLineTool, cwl.ExpressionTool, cwl.Workflow], bool]: +) -> tuple[cwl.CommandLineTool | cwl.ExpressionTool | cwl.Workflow, bool]: """Convert the given process and any subprocesses.""" if not inside and isinstance(process, cwl.CommandLineTool): process = expand_stream_shortcuts(process) @@ -314,18 +314,16 @@ def load_step( def generate_etool_from_expr( expr: str, - target: Union[cwl.CommandInputParameter, cwl.WorkflowInputParameter], + target: cwl.CommandInputParameter | cwl.WorkflowInputParameter, no_inputs: bool = False, - self_type: Optional[ - Union[ - cwl.WorkflowInputParameter, - cwl.CommandInputParameter, - list[Union[cwl.WorkflowInputParameter, cwl.CommandInputParameter]], - ] - ] = None, # if the "self" input should be a different type than the "result" output - extra_processes: Optional[ - Sequence[Union[cwl.Workflow, cwl.WorkflowStep, cwl.CommandLineTool]] - ] = None, + self_type: None | ( + cwl.WorkflowInputParameter + | cwl.CommandInputParameter + | list[cwl.WorkflowInputParameter | cwl.CommandInputParameter] + ) = None, # if the "self" input should be a different type than the "result" output + extra_processes: None | ( + Sequence[cwl.Workflow | cwl.WorkflowStep | cwl.CommandLineTool] + ) = None, ) -> cwl.ExpressionTool: """Convert a CWL Expression into an ExpressionTool.""" inputs = yaml.comments.CommentedSeq() @@ -333,10 +331,11 @@ def generate_etool_from_expr( if not self_type: self_type = target if isinstance(self_type, list): - new_type: Union[ - list[Union[cwl.ArraySchema, cwl.InputRecordSchema]], - Union[cwl.ArraySchema, cwl.InputRecordSchema], - ] = [clean_type_ids(t.type_) for t in self_type] + new_type: ( + list[cwl.ArraySchema | cwl.InputRecordSchema] + | cwl.ArraySchema + | cwl.InputRecordSchema + ) = [clean_type_ids(t.type_) for t in self_type] else: new_type = clean_type_ids(self_type.type_) inputs.append( @@ -404,8 +403,8 @@ def generate_etool_from_expr( def get_input_for_id( - name: str, tool: Union[cwl.CommandLineTool, cwl.Workflow] -) -> Optional[cwl.CommandInputParameter]: + name: str, tool: cwl.CommandLineTool | cwl.Workflow +) -> cwl.CommandInputParameter | None: """Determine the CommandInputParameter for the given input name.""" name = name.split("/")[-1] @@ -424,9 +423,9 @@ def get_input_for_id( def find_expressionLib( processes: Sequence[ - Union[cwl.CommandLineTool, cwl.Workflow, cwl.ExpressionTool, cwl.WorkflowStep] + cwl.CommandLineTool | cwl.Workflow | cwl.ExpressionTool | cwl.WorkflowStep ], -) -> Optional[list[str]]: +) -> list[str] | None: """ Return the expressionLib from the highest priority InlineJavascriptRequirement. @@ -445,30 +444,30 @@ def replace_expr_with_etool( expr: str, name: str, workflow: cwl.Workflow, - target: Union[cwl.CommandInputParameter, cwl.WorkflowInputParameter], - source: Optional[Union[str, list[Any]]], + target: cwl.CommandInputParameter | cwl.WorkflowInputParameter, + source: str | list[Any] | None, replace_etool: bool = False, - extra_process: Optional[ - Union[cwl.Workflow, cwl.WorkflowStep, cwl.CommandLineTool] - ] = None, - source_type: Optional[cwl.CommandInputParameter] = None, + extra_process: None | ( + cwl.Workflow | cwl.WorkflowStep | cwl.CommandLineTool + ) = None, + source_type: cwl.CommandInputParameter | None = None, ) -> None: """Modify the given workflow, replacing the expr with an standalone ExpressionTool.""" - extra_processes: list[ - Union[cwl.Workflow, cwl.WorkflowStep, cwl.CommandLineTool] - ] = [workflow] + extra_processes: list[cwl.Workflow | cwl.WorkflowStep | cwl.CommandLineTool] = [ + workflow + ] if extra_process: extra_processes.append(extra_process) etool: cwl.ExpressionTool = generate_etool_from_expr( expr, target, source is None, source_type, extra_processes ) if replace_etool: - processes: list[Union[cwl.WorkflowStep, cwl.Workflow, cwl.CommandLineTool]] = [ + processes: list[cwl.WorkflowStep | cwl.Workflow | cwl.CommandLineTool] = [ workflow ] if extra_process: processes.append(extra_process) - final_tool: Union[cwl.ExpressionTool, cwl.CommandLineTool] = etool_to_cltool( + final_tool: cwl.ExpressionTool | cwl.CommandLineTool = etool_to_cltool( etool, find_expressionLib(processes) ) else: @@ -513,10 +512,10 @@ def replace_wf_input_ref_with_step_output( def empty_inputs( - process_or_step: Union[ - cwl.CommandLineTool, cwl.WorkflowStep, cwl.ExpressionTool, cwl.Workflow - ], - parent: Optional[cwl.Workflow] = None, + process_or_step: ( + cwl.CommandLineTool | cwl.WorkflowStep | cwl.ExpressionTool | cwl.Workflow + ), + parent: cwl.Workflow | None = None, ) -> dict[str, Any]: """Produce a mock input object for the given inputs.""" result = {} @@ -748,20 +747,20 @@ def process_workflow_reqs_and_hints( # and connecting all workflow inputs to the generated step modified = False inputs = empty_inputs(workflow) - generated_res_reqs: list[tuple[str, Union[int, str]]] = [] - generated_iwdr_reqs: list[tuple[str, Union[int, str]]] = [] - generated_envVar_reqs: list[tuple[str, Union[int, str]]] = [] + generated_res_reqs: list[tuple[str, int | str]] = [] + generated_iwdr_reqs: list[tuple[str, int | str]] = [] + generated_envVar_reqs: list[tuple[str, int | str]] = [] prop_reqs: tuple[ - Union[ - type[cwl.EnvVarRequirement], - type[cwl.ResourceRequirement], - type[cwl.InitialWorkDirRequirement], - ], + ( + type[cwl.EnvVarRequirement] + | type[cwl.ResourceRequirement] + | type[cwl.InitialWorkDirRequirement] + ), ..., ] = () - resourceReq: Optional[cwl.ResourceRequirement] = None - envVarReq: Optional[cwl.EnvVarRequirement] = None - iwdr: Optional[cwl.InitialWorkDirRequirement] = None + resourceReq: cwl.ResourceRequirement | None = None + envVarReq: cwl.EnvVarRequirement | None = None + iwdr: cwl.InitialWorkDirRequirement | None = None if workflow.requirements is not None: for req in cast(list[cwl.ProcessRequirement], workflow.requirements): if req and isinstance(req, cwl.EnvVarRequirement): @@ -1081,8 +1080,8 @@ def process_level_reqs( target_process = step.run inputs = empty_inputs(process) generated_res_reqs: list[tuple[str, str]] = [] - generated_iwdr_reqs: list[tuple[str, Union[int, str], Any]] = [] - generated_envVar_reqs: list[tuple[str, Union[int, str]]] = [] + generated_iwdr_reqs: list[tuple[str, int | str, Any]] = [] + generated_envVar_reqs: list[tuple[str, int | str]] = [] if not step.id: return False step_name = step.id.split("#", 1)[-1] @@ -1505,7 +1504,7 @@ def traverse_CommandLineTool( inp.linkMerge = None if replace_etool: processes = [parent] - final_etool: Union[cwl.CommandLineTool, cwl.ExpressionTool] = ( + final_etool: cwl.CommandLineTool | cwl.ExpressionTool = ( etool_to_cltool(etool, find_expressionLib(processes)) ) else: @@ -1623,7 +1622,7 @@ def simplify_step_id(uri: str) -> str: def remove_JSReq( - process: Union[cwl.CommandLineTool, cwl.WorkflowStep, cwl.Workflow], + process: cwl.CommandLineTool | cwl.WorkflowStep | cwl.Workflow, skip_command_line1: bool, ) -> None: """Since the InlineJavascriptRequirement is longer needed, remove it.""" @@ -1654,7 +1653,7 @@ def replace_step_clt_expr_with_etool( target: cwl.WorkflowInputParameter, step: cwl.WorkflowStep, replace_etool: bool, - self_name: Optional[str] = None, + self_name: str | None = None, ) -> None: """Convert a step level CWL Expression to a sibling expression step.""" etool_inputs = cltool_inputs_to_etool_inputs(step.run) @@ -1663,7 +1662,7 @@ def replace_step_clt_expr_with_etool( ) if replace_etool: processes = [workflow] - etool: Union[cwl.ExpressionTool, cwl.CommandLineTool] = etool_to_cltool( + etool: cwl.ExpressionTool | cwl.CommandLineTool = etool_to_cltool( temp_etool, find_expressionLib(processes) ) else: @@ -1689,8 +1688,8 @@ def replace_clt_hintreq_expr_with_etool( target: cwl.WorkflowInputParameter, step: cwl.WorkflowStep, replace_etool: bool, - self_name: Optional[str] = None, -) -> Union[cwl.CommandLineTool, cwl.ExpressionTool]: + self_name: str | None = None, +) -> cwl.CommandLineTool | cwl.ExpressionTool: """Factor out an expression inside a CommandLineTool req or hint into a sibling step.""" # Same as replace_step_clt_expr_with_etool or different? etool_inputs = cltool_inputs_to_etool_inputs(step.run) @@ -1699,7 +1698,7 @@ def replace_clt_hintreq_expr_with_etool( ) if replace_etool: processes = [workflow] - etool: Union[cwl.CommandLineTool, cwl.ExpressionTool] = etool_to_cltool( + etool: cwl.CommandLineTool | cwl.ExpressionTool = etool_to_cltool( temp_etool, find_expressionLib(processes) ) else: @@ -1785,13 +1784,13 @@ def cltool_step_outputs_to_workflow_outputs( def generate_etool_from_expr2( expr: str, - target: Union[cwl.CommandInputParameter, cwl.WorkflowInputParameter], - inputs: Sequence[Union[cwl.WorkflowInputParameter, cwl.CommandInputParameter]], - self_name: Optional[str] = None, - process: Optional[Union[cwl.CommandLineTool, cwl.ExpressionTool]] = None, - extra_processes: Optional[ - Sequence[Union[cwl.Workflow, cwl.WorkflowStep, cwl.CommandLineTool]] - ] = None, + target: cwl.CommandInputParameter | cwl.WorkflowInputParameter, + inputs: Sequence[cwl.WorkflowInputParameter | cwl.CommandInputParameter], + self_name: str | None = None, + process: cwl.CommandLineTool | cwl.ExpressionTool | None = None, + extra_processes: None | ( + Sequence[cwl.Workflow | cwl.WorkflowStep | cwl.CommandLineTool] + ) = None, ) -> cwl.ExpressionTool: """Generate an ExpressionTool to achieve the same result as the given expression.""" outputs = yaml.comments.CommentedSeq() @@ -1818,7 +1817,7 @@ def generate_etool_from_expr2( ) hints = None procs: list[ - Union[cwl.CommandLineTool, cwl.ExpressionTool, cwl.Workflow, cwl.WorkflowStep] + cwl.CommandLineTool | cwl.ExpressionTool | cwl.Workflow | cwl.WorkflowStep ] = [] if process: procs.append(process) @@ -1906,9 +1905,9 @@ def traverse_step( if not target: raise WorkflowException("target not found") input_source_id = None - source_type: Optional[ - Union[list[cwl.WorkflowInputParameter], cwl.WorkflowInputParameter] - ] = None + source_type: None | ( + list[cwl.WorkflowInputParameter] | cwl.WorkflowInputParameter + ) = None if inp.source: if isinstance(inp.source, MutableSequence): input_source_id = [] @@ -2030,16 +2029,16 @@ def replace_step_valueFrom_expr_with_etool( expr: str, name: str, workflow: cwl.Workflow, - target: Union[cwl.CommandInputParameter, cwl.WorkflowInputParameter], + target: cwl.CommandInputParameter | cwl.WorkflowInputParameter, step: cwl.WorkflowStep, step_inp: cwl.WorkflowStepInput, - original_process: Union[cwl.CommandLineTool, cwl.ExpressionTool], + original_process: cwl.CommandLineTool | cwl.ExpressionTool, original_step_ins: list[cwl.WorkflowStepInput], - source: Optional[Union[str, list[str]]], + source: str | list[str] | None, replace_etool: bool, - source_type: Optional[ - Union[cwl.WorkflowInputParameter, list[cwl.WorkflowInputParameter]] - ] = None, + source_type: None | ( + cwl.WorkflowInputParameter | list[cwl.WorkflowInputParameter] + ) = None, ) -> None: """Replace a WorkflowStep level 'valueFrom' expression with a sibling ExpressionTool step.""" if not step_inp.id: @@ -2062,15 +2061,13 @@ def replace_step_valueFrom_expr_with_etool( ) if replace_etool: processes: list[ - Union[ - cwl.Workflow, cwl.CommandLineTool, cwl.ExpressionTool, cwl.WorkflowStep - ] + (cwl.Workflow | cwl.CommandLineTool | cwl.ExpressionTool | cwl.WorkflowStep) ] = [ workflow, step, ] cltool = etool_to_cltool(temp_etool, find_expressionLib(processes)) - etool: Union[cwl.ExpressionTool, cwl.CommandLineTool] = cltool + etool: cwl.ExpressionTool | cwl.CommandLineTool = cltool else: etool = temp_etool wf_step_inputs = copy.deepcopy(original_step_ins) @@ -2140,15 +2137,13 @@ def replace_step_when_expr_with_etool( ) if replace_etool: processes: list[ - Union[ - cwl.Workflow, cwl.CommandLineTool, cwl.ExpressionTool, cwl.WorkflowStep - ] + (cwl.Workflow | cwl.CommandLineTool | cwl.ExpressionTool | cwl.WorkflowStep) ] = [ workflow, step, ] cltool = etool_to_cltool(temp_etool, find_expressionLib(processes)) - etool: Union[cwl.ExpressionTool, cwl.CommandLineTool] = cltool + etool: cwl.ExpressionTool | cwl.CommandLineTool = cltool else: etool = temp_etool wf_step_inputs = copy.deepcopy(original_step_ins) diff --git a/cwl_utils/expression.py b/cwl_utils/expression.py index 302f1cd4..8833e866 100644 --- a/cwl_utils/expression.py +++ b/cwl_utils/expression.py @@ -5,7 +5,7 @@ import inspect import json from collections.abc import Awaitable, MutableMapping -from typing import Any, Optional, Union, cast +from typing import Any, Union, cast from schema_salad.utils import json_dumps @@ -20,7 +20,7 @@ def _convert_dumper(string: str) -> str: return f"{json.dumps(string)} + " -def scanner(scan: str) -> Optional[tuple[int, int]]: +def scanner(scan: str) -> tuple[int, int] | None: """Find JS relevant punctuation in a string.""" DEFAULT = 0 DOLLAR = 1 @@ -106,7 +106,7 @@ def evaluator( jslib: str, fullJS: bool, **kwargs: Any, -) -> Optional[CWLOutputType]: +) -> CWLOutputType | None: js_engine = js_engine or get_js_engine() expression_parse_exception = None @@ -174,9 +174,9 @@ def interpolate( strip_whitespace: bool = True, escaping_behavior: int = 2, convert_to_expression: bool = False, - js_engine: Optional[JSEngine] = None, + js_engine: JSEngine | None = None, **kwargs: Any, -) -> Optional[CWLOutputType]: +) -> CWLOutputType | None: """ Interpolate and evaluate. @@ -266,18 +266,18 @@ def needs_parsing(snippet: Any) -> bool: def do_eval( - ex: Optional[CWLOutputType], + ex: CWLOutputType | None, jobinput: CWLObjectType, requirements: list[CWLObjectType], - outdir: Optional[str], - tmpdir: Optional[str], - resources: dict[str, Union[float, int]], - context: Optional[CWLOutputType] = None, + outdir: str | None, + tmpdir: str | None, + resources: dict[str, float | int], + context: CWLOutputType | None = None, timeout: float = default_timeout, strip_whitespace: bool = True, cwlVersion: str = "", **kwargs: Any, -) -> Optional[CWLOutputType]: +) -> CWLOutputType | None: """ Evaluate the given CWL expression, in context. diff --git a/cwl_utils/expression_refactor.py b/cwl_utils/expression_refactor.py index aea3913b..526cc86e 100755 --- a/cwl_utils/expression_refactor.py +++ b/cwl_utils/expression_refactor.py @@ -6,9 +6,9 @@ import logging import shutil import sys -from collections.abc import MutableMapping, MutableSequence +from collections.abc import Callable, MutableMapping, MutableSequence from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, Optional, Union +from typing import Any, Optional, Protocol, Union from ruamel.yaml.main import YAML from ruamel.yaml.scalarstring import walk_tree @@ -22,11 +22,6 @@ from cwl_utils.loghandler import _logger as _cwlutilslogger from cwl_utils.parser import cwl_v1_0, cwl_v1_1, cwl_v1_2 -if TYPE_CHECKING: - from typing_extensions import Protocol -else: - Protocol = object - _logger = logging.getLogger("cwl-expression-refactor") # pylint: disable=invalid-name defaultStreamHandler = logging.StreamHandler() # pylint: disable=invalid-name _logger.addHandler(defaultStreamHandler) diff --git a/cwl_utils/file_formats.py b/cwl_utils/file_formats.py index 86e50128..2eea6b21 100644 --- a/cwl_utils/file_formats.py +++ b/cwl_utils/file_formats.py @@ -5,7 +5,6 @@ For more information, please visit https://www.commonwl.org/user_guide/16-file-formats/ """ -from typing import Optional, Union from rdflib import OWL, RDFS, Graph, URIRef from schema_salad.exceptions import ValidationException @@ -15,7 +14,7 @@ def formatSubclassOf( - fmt: str, cls: str, ontology: Optional[Graph], visited: set[str] + fmt: str, cls: str, ontology: Graph | None, visited: set[str] ) -> bool: """Determine if `fmt` is a subclass of `cls`.""" if URIRef(fmt) == URIRef(cls): @@ -50,9 +49,9 @@ def formatSubclassOf( def check_format( - actual_file: Union[CWLObjectType, list[CWLObjectType]], - input_formats: Union[list[str], str], - ontology: Optional[Graph], + actual_file: CWLObjectType | list[CWLObjectType], + input_formats: list[str] | str, + ontology: Graph | None, ) -> None: """Confirm that the format present is valid for the allowed formats.""" for afile in aslist(actual_file): diff --git a/cwl_utils/graph_split.py b/cwl_utils/graph_split.py index 3a1b346c..e1caacfb 100755 --- a/cwl_utils/graph_split.py +++ b/cwl_utils/graph_split.py @@ -20,7 +20,6 @@ from typing import ( IO, Any, - Union, cast, ) @@ -167,7 +166,7 @@ def rewrite( document[key] = value[len(doc_id) + 2 :] elif key == "out" and isinstance(value, list): - def rewrite_id(entry: Any) -> Union[MutableMapping[Any, Any], str]: + def rewrite_id(entry: Any) -> MutableMapping[Any, Any] | str: if isinstance(entry, MutableMapping): if entry["id"].startswith(this_id): assert isinstance(this_id, str) # nosec B101 diff --git a/cwl_utils/image_puller.py b/cwl_utils/image_puller.py index 5fae0781..da5452cc 100644 --- a/cwl_utils/image_puller.py +++ b/cwl_utils/image_puller.py @@ -5,7 +5,6 @@ import subprocess # nosec from abc import ABC, abstractmethod from pathlib import Path -from typing import Optional, Union from .singularity import get_version as get_singularity_version from .singularity import is_version_2_6 as is_singularity_version_2_6 @@ -19,7 +18,7 @@ class ImagePuller(ABC): def __init__( self, req: str, - save_directory: Optional[Union[str, Path]], + save_directory: str | Path | None, cmd: str, force_pull: bool, ) -> None: @@ -109,7 +108,7 @@ def get_image_name(self) -> str: def save_docker_image(self) -> None: """Pull down the Docker software container image and save it in the Singularity image format.""" - save_directory: Union[str, Path] + save_directory: str | Path if self.save_directory: save_directory = self.save_directory if ( diff --git a/cwl_utils/inputs_schema_gen.py b/cwl_utils/inputs_schema_gen.py index 072a33f7..3836f169 100644 --- a/cwl_utils/inputs_schema_gen.py +++ b/cwl_utils/inputs_schema_gen.py @@ -10,17 +10,11 @@ import sys from copy import deepcopy from pathlib import Path -from typing import Any, Optional, Union +from typing import Any, TypeGuard, Union from urllib.parse import urlparse import requests -# Get typeguard from extensions if we're running in python3.8 -if sys.version_info[:2] < (3, 10): - from typing_extensions import TypeGuard # Not in 3.8 typing module -else: - from typing import TypeGuard - from cwl_utils.loghandler import _logger as _cwlutilslogger from cwl_utils.parser import ( CommandLineTool, @@ -85,14 +79,14 @@ class JSONSchemaProperty: def __init__( self, name: str, - type_: Union[InputType, list[InputType], str, Any], - description: Optional[str] = "", - required: Optional[bool] = False, + type_: InputType | list[InputType] | str | Any, + description: str | None = "", + required: bool | None = False, ): """Initialise the JSONSchemaProperty object.""" # Initialise values self.name: str = name - self.type_: Union[InputType, list[InputType], str, Any] = type_ + self.type_: InputType | list[InputType] | str | Any = type_ self.description = description self.required = required self.type_dict = self.generate_type_dict() @@ -320,7 +314,7 @@ def generate_definition_from_schema(schema: InputRecordSchema) -> dict[str, Any] } -def cwl_to_jsonschema(cwl_obj: Union[Workflow, CommandLineTool]) -> Any: +def cwl_to_jsonschema(cwl_obj: Workflow | CommandLineTool) -> Any: """ cwl_obj: A CWL Object. @@ -490,7 +484,7 @@ def _get_all_ref_attributes(json_object: dict[str, Any]) -> list[Any]: def get_property_dependencies( property_dict: dict[str, Any], input_json_schema: dict[str, Any], - existing_property_dependencies: Optional[list[Any]] = None, + existing_property_dependencies: list[Any] | None = None, ) -> list[str]: """Recursively collect all dependencies for a property.""" # Initialise return list diff --git a/cwl_utils/pack.py b/cwl_utils/pack.py index 30721891..e0c4eb97 100644 --- a/cwl_utils/pack.py +++ b/cwl_utils/pack.py @@ -20,7 +20,7 @@ import urllib.parse import urllib.request from collections.abc import ItemsView -from typing import TYPE_CHECKING, Any, Optional, Union, cast +from typing import TYPE_CHECKING, Any, Union, cast from packaging import version @@ -34,7 +34,7 @@ def get_inner_dict( cwl: dict[str, Any], path: list[dict[str, Any]] -) -> Optional[dict[str, Any]]: +) -> dict[str, Any] | None: if len(path) == 0: return cwl @@ -56,7 +56,7 @@ def pack_process( cwl: dict[str, Any], base_url: urllib.parse.ParseResult, cwl_version: str, - parent_user_defined_types: Optional[dict[str, Any]] = None, + parent_user_defined_types: dict[str, Any] | None = None, ) -> dict[str, Any]: cwl = listify_everything(cwl) cwl = normalize_sources(cwl) @@ -144,7 +144,7 @@ def _normalize(s: str) -> str: def load_schemadefs( cwl: dict[str, Any], base_url: urllib.parse.ParseResult, - parent_user_defined_types: Optional[dict[str, Any]] = None, + parent_user_defined_types: dict[str, Any] | None = None, ) -> tuple[dict[str, Any], dict[str, Any]]: """Internalize any SchemaDefRequirement, and remove it.""" user_defined_types = schemadef.build_user_defined_type_dict(cwl, base_url) @@ -196,7 +196,7 @@ def resolve_steps( cwl: dict[str, Any], base_url: urllib.parse.ParseResult, cwl_version: str, - parent_user_defined_types: Optional[dict[str, Any]] = None, + parent_user_defined_types: dict[str, Any] | None = None, ) -> dict[str, Any]: """Load and pack all "run" sections of the workflow steps.""" if isinstance(cwl, str): diff --git a/cwl_utils/parser/__init__.py b/cwl_utils/parser/__init__.py index 3d0868e6..384bd728 100644 --- a/cwl_utils/parser/__init__.py +++ b/cwl_utils/parser/__init__.py @@ -226,7 +226,7 @@ class NoType(ABC): """Type union for a CWL v1.x _Loader.""" -def _get_id_from_graph(yaml: MutableMapping[str, Any], id_: Optional[str]) -> Any: +def _get_id_from_graph(yaml: MutableMapping[str, Any], id_: str | None) -> Any: if id_ is None: id_ = "main" for el in yaml["$graph"]: @@ -238,7 +238,7 @@ def _get_id_from_graph(yaml: MutableMapping[str, Any], id_: Optional[str]) -> An ) -def cwl_version(yaml: Any) -> Optional[str]: +def cwl_version(yaml: Any) -> str | None: """ Return the cwlVersion of a YAML object. @@ -254,8 +254,8 @@ def cwl_version(yaml: Any) -> Optional[str]: def load_document_by_uri( - path: Union[str, Path], - loadingOptions: Optional[LoadingOptions] = None, + path: str | Path, + loadingOptions: LoadingOptions | None = None, load_all: bool = False, ) -> Any: """Load a CWL object from a URI or a path.""" @@ -298,9 +298,9 @@ def load_document_by_uri( def load_document( doc: Any, - baseuri: Optional[str] = None, - loadingOptions: Optional[LoadingOptions] = None, - id_: Optional[str] = None, + baseuri: str | None = None, + loadingOptions: LoadingOptions | None = None, + id_: str | None = None, load_all: bool = False, ) -> Any: """Load a CWL object from a serialized YAML string or a YAML object.""" @@ -314,8 +314,8 @@ def load_document( def load_document_by_string( string: str, uri: str, - loadingOptions: Optional[LoadingOptions] = None, - id_: Optional[str] = None, + loadingOptions: LoadingOptions | None = None, + id_: str | None = None, load_all: bool = False, ) -> Any: """Load a CWL object from a serialized YAML string.""" @@ -327,8 +327,8 @@ def load_document_by_string( def load_document_by_yaml( yaml: Any, uri: str, - loadingOptions: Optional[LoadingOptions] = None, - id_: Optional[str] = None, + loadingOptions: LoadingOptions | None = None, + id_: str | None = None, load_all: bool = False, ) -> Any: """Load a CWL object from a YAML object.""" @@ -366,7 +366,7 @@ def load_document_by_yaml( def save( - val: Optional[Union[Saveable, MutableSequence[Saveable]]], + val: Saveable | MutableSequence[Saveable] | None, top: bool = True, base_url: str = "", relative_uris: bool = True, diff --git a/cwl_utils/parser/cwl_v1_0_utils.py b/cwl_utils/parser/cwl_v1_0_utils.py index c6fae5f6..62cf9956 100644 --- a/cwl_utils/parser/cwl_v1_0_utils.py +++ b/cwl_utils/parser/cwl_v1_0_utils.py @@ -5,7 +5,7 @@ from collections import namedtuple from collections.abc import MutableMapping, MutableSequence from io import StringIO -from typing import IO, Any, Optional, Union, cast +from typing import IO, Any, Union, cast from urllib.parse import urldefrag from schema_salad.exceptions import ValidationException @@ -87,10 +87,10 @@ def _compare_type(type1: Any, type2: Any) -> bool: def _inputfile_load( - doc: Union[str, MutableMapping[str, Any], MutableSequence[Any]], + doc: str | MutableMapping[str, Any] | MutableSequence[Any], baseuri: str, loadingOptions: cwl.LoadingOptions, - addl_metadata_fields: Optional[MutableSequence[str]] = None, + addl_metadata_fields: MutableSequence[str] | None = None, ) -> tuple[Any, cwl.LoadingOptions]: loader = cwl.CWLInputFileLoader if isinstance(doc, str): @@ -181,7 +181,7 @@ def can_assign_src_to_sink(src: Any, sink: Any, strict: bool = False) -> bool: def check_all_types( src_dict: dict[str, Any], - sinks: MutableSequence[Union[cwl.WorkflowStepInput, cwl.WorkflowOutputParameter]], + sinks: MutableSequence[cwl.WorkflowStepInput | cwl.WorkflowOutputParameter], type_dict: dict[str, Any], ) -> dict[str, list[SrcSink]]: """Given a list of sinks, check if their types match with the types of their sources.""" @@ -228,8 +228,8 @@ def check_all_types( def check_types( srctype: Any, sinktype: Any, - linkMerge: Optional[str], - valueFrom: Optional[str] = None, + linkMerge: str | None, + valueFrom: str | None = None, ) -> str: """ Check if the source and sink types are correct. @@ -304,8 +304,8 @@ def convert_stdstreams_to_files(clt: cwl.CommandLineTool) -> None: def load_inputfile( doc: Any, - baseuri: Optional[str] = None, - loadingOptions: Optional[cwl.LoadingOptions] = None, + baseuri: str | None = None, + loadingOptions: cwl.LoadingOptions | None = None, ) -> Any: """Load a CWL v1.0 input file from a serialized YAML string or a YAML object.""" if baseuri is None: @@ -324,7 +324,7 @@ def load_inputfile( def load_inputfile_by_string( string: Any, uri: str, - loadingOptions: Optional[cwl.LoadingOptions] = None, + loadingOptions: cwl.LoadingOptions | None = None, ) -> Any: """Load a CWL v1.0 input file from a serialized YAML string.""" yaml = yaml_no_ts() @@ -345,7 +345,7 @@ def load_inputfile_by_string( def load_inputfile_by_yaml( yaml: Any, uri: str, - loadingOptions: Optional[cwl.LoadingOptions] = None, + loadingOptions: cwl.LoadingOptions | None = None, ) -> Any: """Load a CWL v1.0 input file from a YAML object.""" add_lc_filename(yaml, uri) @@ -421,13 +421,13 @@ def type_for_step_output( def type_for_source( - process: Union[cwl.CommandLineTool, cwl.Workflow, cwl.ExpressionTool], - sourcenames: Union[str, list[str]], - parent: Optional[cwl.Workflow] = None, - linkMerge: Optional[str] = None, + process: cwl.CommandLineTool | cwl.Workflow | cwl.ExpressionTool, + sourcenames: str | list[str], + parent: cwl.Workflow | None = None, + linkMerge: str | None = None, ) -> Any: """Determine the type for the given sourcenames.""" - scatter_context: list[Optional[tuple[int, str]]] = [] + scatter_context: list[tuple[int, str] | None] = [] params = param_for_source_id(process, sourcenames, parent, scatter_context) if not isinstance(params, list): new_type = params.type_ @@ -473,11 +473,11 @@ def type_for_source( def param_for_source_id( - process: Union[cwl.CommandLineTool, cwl.Workflow, cwl.ExpressionTool], - sourcenames: Union[str, list[str]], - parent: Optional[cwl.Workflow] = None, - scatter_context: Optional[list[Optional[tuple[int, str]]]] = None, -) -> Union[list[cwl.InputParameter], cwl.InputParameter]: + process: cwl.CommandLineTool | cwl.Workflow | cwl.ExpressionTool, + sourcenames: str | list[str], + parent: cwl.Workflow | None = None, + scatter_context: list[tuple[int, str] | None] | None = None, +) -> list[cwl.InputParameter] | cwl.InputParameter: """Find the process input parameter that matches one of the given sourcenames.""" if isinstance(sourcenames, str): sourcenames = [sourcenames] diff --git a/cwl_utils/parser/cwl_v1_1_utils.py b/cwl_utils/parser/cwl_v1_1_utils.py index 2c2cce2f..cb3f3304 100644 --- a/cwl_utils/parser/cwl_v1_1_utils.py +++ b/cwl_utils/parser/cwl_v1_1_utils.py @@ -5,7 +5,7 @@ from collections import namedtuple from collections.abc import MutableMapping, MutableSequence from io import StringIO -from typing import IO, Any, Optional, Union, cast +from typing import IO, Any, Union, cast from urllib.parse import urldefrag from schema_salad.exceptions import ValidationException @@ -87,10 +87,10 @@ def _compare_type(type1: Any, type2: Any) -> bool: def _inputfile_load( - doc: Union[str, MutableMapping[str, Any], MutableSequence[Any]], + doc: str | MutableMapping[str, Any] | MutableSequence[Any], baseuri: str, loadingOptions: cwl.LoadingOptions, - addl_metadata_fields: Optional[MutableSequence[str]] = None, + addl_metadata_fields: MutableSequence[str] | None = None, ) -> tuple[Any, cwl.LoadingOptions]: loader = cwl.CWLInputFileLoader if isinstance(doc, str): @@ -181,7 +181,7 @@ def can_assign_src_to_sink(src: Any, sink: Any, strict: bool = False) -> bool: def check_all_types( src_dict: dict[str, Any], - sinks: MutableSequence[Union[cwl.WorkflowStepInput, cwl.WorkflowOutputParameter]], + sinks: MutableSequence[cwl.WorkflowStepInput | cwl.WorkflowOutputParameter], type_dict: dict[str, Any], ) -> dict[str, list[SrcSink]]: """Given a list of sinks, check if their types match with the types of their sources.""" @@ -228,8 +228,8 @@ def check_all_types( def check_types( srctype: Any, sinktype: Any, - linkMerge: Optional[str], - valueFrom: Optional[str] = None, + linkMerge: str | None, + valueFrom: str | None = None, ) -> str: """ Check if the source and sink types are correct. @@ -320,8 +320,8 @@ def convert_stdstreams_to_files(clt: cwl.CommandLineTool) -> None: def load_inputfile( doc: Any, - baseuri: Optional[str] = None, - loadingOptions: Optional[cwl.LoadingOptions] = None, + baseuri: str | None = None, + loadingOptions: cwl.LoadingOptions | None = None, ) -> Any: """Load a CWL v1.1 input file from a serialized YAML string or a YAML object.""" if baseuri is None: @@ -340,7 +340,7 @@ def load_inputfile( def load_inputfile_by_string( string: Any, uri: str, - loadingOptions: Optional[cwl.LoadingOptions] = None, + loadingOptions: cwl.LoadingOptions | None = None, ) -> Any: """Load a CWL v1.1 input file from a serialized YAML string.""" yaml = yaml_no_ts() @@ -361,7 +361,7 @@ def load_inputfile_by_string( def load_inputfile_by_yaml( yaml: Any, uri: str, - loadingOptions: Optional[cwl.LoadingOptions] = None, + loadingOptions: cwl.LoadingOptions | None = None, ) -> Any: """Load a CWL v1.1 input file from a YAML object.""" add_lc_filename(yaml, uri) @@ -437,13 +437,13 @@ def type_for_step_output( def type_for_source( - process: Union[cwl.CommandLineTool, cwl.Workflow, cwl.ExpressionTool], - sourcenames: Union[str, list[str]], - parent: Optional[cwl.Workflow] = None, - linkMerge: Optional[str] = None, + process: cwl.CommandLineTool | cwl.Workflow | cwl.ExpressionTool, + sourcenames: str | list[str], + parent: cwl.Workflow | None = None, + linkMerge: str | None = None, ) -> Any: """Determine the type for the given sourcenames.""" - scatter_context: list[Optional[tuple[int, str]]] = [] + scatter_context: list[tuple[int, str] | None] = [] params = param_for_source_id(process, sourcenames, parent, scatter_context) if not isinstance(params, list): new_type = params.type_ @@ -489,11 +489,11 @@ def type_for_source( def param_for_source_id( - process: Union[cwl.CommandLineTool, cwl.Workflow, cwl.ExpressionTool], - sourcenames: Union[str, list[str]], - parent: Optional[cwl.Workflow] = None, - scatter_context: Optional[list[Optional[tuple[int, str]]]] = None, -) -> Union[list[cwl.WorkflowInputParameter], cwl.WorkflowInputParameter]: + process: cwl.CommandLineTool | cwl.Workflow | cwl.ExpressionTool, + sourcenames: str | list[str], + parent: cwl.Workflow | None = None, + scatter_context: list[tuple[int, str] | None] | None = None, +) -> list[cwl.WorkflowInputParameter] | cwl.WorkflowInputParameter: """Find the process input parameter that matches one of the given sourcenames.""" if isinstance(sourcenames, str): sourcenames = [sourcenames] diff --git a/cwl_utils/parser/cwl_v1_2_utils.py b/cwl_utils/parser/cwl_v1_2_utils.py index 544c6080..5cfcc1da 100644 --- a/cwl_utils/parser/cwl_v1_2_utils.py +++ b/cwl_utils/parser/cwl_v1_2_utils.py @@ -5,7 +5,7 @@ from collections import namedtuple from collections.abc import MutableMapping, MutableSequence from io import StringIO -from typing import IO, Any, Optional, Union, cast +from typing import IO, Any, Union, cast from urllib.parse import urldefrag from schema_salad.exceptions import ValidationException @@ -106,10 +106,10 @@ def _is_conditional_step( def _inputfile_load( - doc: Union[str, MutableMapping[str, Any], MutableSequence[Any]], + doc: str | MutableMapping[str, Any] | MutableSequence[Any], baseuri: str, loadingOptions: cwl.LoadingOptions, - addl_metadata_fields: Optional[MutableSequence[str]] = None, + addl_metadata_fields: MutableSequence[str] | None = None, ) -> tuple[Any, cwl.LoadingOptions]: loader = cwl.CWLInputFileLoader if isinstance(doc, str): @@ -200,7 +200,7 @@ def can_assign_src_to_sink(src: Any, sink: Any, strict: bool = False) -> bool: def check_all_types( src_dict: dict[str, Any], - sinks: MutableSequence[Union[cwl.WorkflowStepInput, cwl.WorkflowOutputParameter]], + sinks: MutableSequence[cwl.WorkflowStepInput | cwl.WorkflowOutputParameter], param_to_step: dict[str, cwl.WorkflowStep], type_dict: dict[str, Any], ) -> dict[str, list[SrcSink]]: @@ -306,8 +306,8 @@ def check_all_types( def check_types( srctype: Any, sinktype: Any, - linkMerge: Optional[str], - valueFrom: Optional[str] = None, + linkMerge: str | None, + valueFrom: str | None = None, ) -> str: """ Check if the source and sink types are correct. @@ -403,8 +403,8 @@ def convert_stdstreams_to_files(clt: cwl.CommandLineTool) -> None: def load_inputfile( doc: Any, - baseuri: Optional[str] = None, - loadingOptions: Optional[cwl.LoadingOptions] = None, + baseuri: str | None = None, + loadingOptions: cwl.LoadingOptions | None = None, ) -> Any: """Load a CWL v1.2 input file from a serialized YAML string or a YAML object.""" if baseuri is None: @@ -423,7 +423,7 @@ def load_inputfile( def load_inputfile_by_string( string: Any, uri: str, - loadingOptions: Optional[cwl.LoadingOptions] = None, + loadingOptions: cwl.LoadingOptions | None = None, ) -> Any: """Load a CWL v1.2 input file from a serialized YAML string.""" yaml = yaml_no_ts() @@ -444,7 +444,7 @@ def load_inputfile_by_string( def load_inputfile_by_yaml( yaml: Any, uri: str, - loadingOptions: Optional[cwl.LoadingOptions] = None, + loadingOptions: cwl.LoadingOptions | None = None, ) -> Any: """Load a CWL v1.2 input file from a YAML object.""" add_lc_filename(yaml, uri) @@ -520,14 +520,14 @@ def type_for_step_output( def type_for_source( - process: Union[cwl.CommandLineTool, cwl.Workflow, cwl.ExpressionTool], - sourcenames: Union[str, list[str]], - parent: Optional[cwl.Workflow] = None, - linkMerge: Optional[str] = None, - pickValue: Optional[str] = None, + process: cwl.CommandLineTool | cwl.Workflow | cwl.ExpressionTool, + sourcenames: str | list[str], + parent: cwl.Workflow | None = None, + linkMerge: str | None = None, + pickValue: str | None = None, ) -> Any: """Determine the type for the given sourcenames.""" - scatter_context: list[Optional[tuple[int, str]]] = [] + scatter_context: list[tuple[int, str] | None] = [] params = param_for_source_id(process, sourcenames, parent, scatter_context) if not isinstance(params, list): new_type = params.type_ @@ -580,11 +580,11 @@ def type_for_source( def param_for_source_id( - process: Union[cwl.CommandLineTool, cwl.Workflow, cwl.ExpressionTool], - sourcenames: Union[str, list[str]], - parent: Optional[cwl.Workflow] = None, - scatter_context: Optional[list[Optional[tuple[int, str]]]] = None, -) -> Union[list[cwl.WorkflowInputParameter], cwl.WorkflowInputParameter]: + process: cwl.CommandLineTool | cwl.Workflow | cwl.ExpressionTool, + sourcenames: str | list[str], + parent: cwl.Workflow | None = None, + scatter_context: list[tuple[int, str] | None] | None = None, +) -> list[cwl.WorkflowInputParameter] | cwl.WorkflowInputParameter: """Find the process input parameter that matches one of the given sourcenames.""" if isinstance(sourcenames, str): sourcenames = [sourcenames] diff --git a/cwl_utils/parser/utils.py b/cwl_utils/parser/utils.py index 9d5b60f2..b51f1e64 100644 --- a/cwl_utils/parser/utils.py +++ b/cwl_utils/parser/utils.py @@ -45,8 +45,8 @@ def convert_stdstreams_to_files(process: Process) -> None: def load_inputfile_by_uri( version: str, - path: Union[str, Path], - loadingOptions: Optional[LoadingOptions] = None, + path: str | Path, + loadingOptions: LoadingOptions | None = None, ) -> Any: """Load a CWL input file from a URI or a path.""" if isinstance(path, str): @@ -82,8 +82,8 @@ def load_inputfile_by_uri( def load_inputfile( version: str, doc: Any, - baseuri: Optional[str] = None, - loadingOptions: Optional[LoadingOptions] = None, + baseuri: str | None = None, + loadingOptions: LoadingOptions | None = None, ) -> Any: """Load a CWL input file from a serialized YAML string or a YAML object.""" if baseuri is None: @@ -97,7 +97,7 @@ def load_inputfile_by_string( version: str, string: str, uri: str, - loadingOptions: Optional[LoadingOptions] = None, + loadingOptions: LoadingOptions | None = None, ) -> Any: """Load a CWL input file from a serialized YAML string.""" yaml = yaml_no_ts() @@ -109,7 +109,7 @@ def load_inputfile_by_yaml( version: str, yaml: Any, uri: str, - loadingOptions: Optional[LoadingOptions] = None, + loadingOptions: LoadingOptions | None = None, ) -> Any: """Load a CWL input file from a YAML object.""" if version == "v1.0": @@ -329,10 +329,10 @@ def static_checker(workflow: cwl_utils.parser.Workflow) -> None: def type_for_source( process: Process, - sourcenames: Union[str, list[str]], - parent: Optional[Workflow] = None, - linkMerge: Optional[str] = None, - pickValue: Optional[str] = None, + sourcenames: str | list[str], + parent: Workflow | None = None, + linkMerge: str | None = None, + pickValue: str | None = None, ) -> Any: """Determine the type for the given sourcenames.""" if process.cwlVersion == "v1.0": @@ -421,28 +421,28 @@ def type_for_step_output(step: WorkflowStep, sourcename: str, cwlVersion: str) - def param_for_source_id( - process: Union[ - cwl_utils.parser.CommandLineTool, - cwl_utils.parser.Workflow, - cwl_utils.parser.ExpressionTool, - ], - sourcenames: Union[str, list[str]], - parent: Optional[cwl_utils.parser.Workflow] = None, - scatter_context: Optional[list[Optional[tuple[int, str]]]] = None, -) -> Union[ - Union[ - list[cwl_utils.parser.cwl_v1_0.InputParameter], - cwl_utils.parser.cwl_v1_0.InputParameter, - ], - Union[ - list[cwl_utils.parser.cwl_v1_1.WorkflowInputParameter], - cwl_utils.parser.cwl_v1_1.WorkflowInputParameter, - ], - Union[ - list[cwl_utils.parser.cwl_v1_2.WorkflowInputParameter], - cwl_utils.parser.cwl_v1_2.WorkflowInputParameter, - ], -]: + process: ( + cwl_utils.parser.CommandLineTool + | cwl_utils.parser.Workflow + | cwl_utils.parser.ExpressionTool + ), + sourcenames: str | list[str], + parent: cwl_utils.parser.Workflow | None = None, + scatter_context: list[tuple[int, str] | None] | None = None, +) -> ( + ( + list[cwl_utils.parser.cwl_v1_0.InputParameter] + | cwl_utils.parser.cwl_v1_0.InputParameter + ) + | ( + list[cwl_utils.parser.cwl_v1_1.WorkflowInputParameter] + | cwl_utils.parser.cwl_v1_1.WorkflowInputParameter + ) + | ( + list[cwl_utils.parser.cwl_v1_2.WorkflowInputParameter] + | cwl_utils.parser.cwl_v1_2.WorkflowInputParameter + ) +): if process.cwlVersion == "v1.0": return cwl_utils.parser.cwl_v1_0_utils.param_for_source_id( cast( diff --git a/cwl_utils/sandboxjs.py b/cwl_utils/sandboxjs.py index c67f8aca..e4175711 100644 --- a/cwl_utils/sandboxjs.py +++ b/cwl_utils/sandboxjs.py @@ -13,7 +13,7 @@ from collections.abc import Awaitable, Mapping, MutableMapping, MutableSequence from importlib.resources import files from io import BytesIO -from typing import Any, Deque, Optional, Union, cast +from typing import Any, Deque, cast from schema_salad.utils import json_dumps @@ -64,7 +64,7 @@ class JSEngine(ABC): @abstractmethod def eval( self, scan: str, jslib: str = "", **kwargs: Any - ) -> Union[CWLOutputType, Awaitable[CWLOutputType]]: ... + ) -> CWLOutputType | Awaitable[CWLOutputType]: ... @abstractmethod def regex_eval( @@ -73,7 +73,7 @@ def regex_eval( remaining_string: str, current_value: CWLOutputType, **kwargs: Any, - ) -> Union[CWLOutputType, Awaitable[CWLOutputType]]: ... + ) -> CWLOutputType | Awaitable[CWLOutputType]: ... class NodeJSEngine(JSEngine): @@ -148,7 +148,7 @@ def exec_js_process( js_text: str, timeout: float = default_timeout, js_console: bool = False, - context: Optional[str] = None, + context: str | None = None, force_docker_pull: bool = False, container_engine: str = "docker", ) -> tuple[int, str, str]: @@ -273,7 +273,7 @@ def new_js_proc( ) -> "subprocess.Popen[str]": """Return a subprocess ready to submit javascript to.""" required_node_version, docker = (False,) * 2 - nodejs = None # type: Optional[subprocess.Popen[str]] + nodejs: subprocess.Popen[str] | None = None trynodes = ("nodejs", "node") for n in trynodes: try: @@ -308,7 +308,7 @@ def new_js_proc( nodeimg = f"docker.io/library/{nodeimg}" if not self.have_node_slim: - singularity_cache: Optional[str] = None + singularity_cache: str | None = None if container_engine in ("docker", "podman"): dockerimgs = subprocess.check_output( # nosec [container_engine, "images", "-q", nodeimg], @@ -515,7 +515,7 @@ def regex_eval( if not m: return current_value next_segment_str = m.group(1) - key: Optional[Union[str, int]] = None + key: str | int | None = None if next_segment_str[0] == ".": key = next_segment_str[1:] elif next_segment_str[1] in ("'", '"'): diff --git a/cwl_utils/singularity.py b/cwl_utils/singularity.py index 09234182..745650ee 100644 --- a/cwl_utils/singularity.py +++ b/cwl_utils/singularity.py @@ -2,7 +2,6 @@ import re from subprocess import check_output # nosec -from typing import Optional from .loghandler import _logger @@ -10,7 +9,7 @@ # This is a list containing major and minor versions as integer. # (The number of minor version digits can vary among different distributions, # therefore we need a list here.) -_SINGULARITY_VERSION: Optional[list[int]] = None +_SINGULARITY_VERSION: list[int] | None = None # Cached flavor / distribution of singularity # Can be singularity, singularity-ce or apptainer _SINGULARITY_FLAVOR: str = "" diff --git a/cwl_utils/utils.py b/cwl_utils/utils.py index 4bf2cf10..83e2bfc0 100644 --- a/cwl_utils/utils.py +++ b/cwl_utils/utils.py @@ -10,7 +10,7 @@ from collections.abc import MutableMapping, MutableSequence from copy import deepcopy from io import StringIO -from typing import Any, Optional, Union +from typing import Any from urllib.parse import urlparse from ruamel.yaml.main import YAML @@ -29,7 +29,7 @@ fast_yaml = YAML(typ="safe") -_USERNS: Optional[bool] = None +_USERNS: bool | None = None def _is_github_symbolic_link(base_url: urllib.parse.ParseResult, contents: str) -> bool: @@ -55,8 +55,8 @@ def _is_github_symbolic_link(base_url: urllib.parse.ParseResult, contents: str) def bytes2str_in_dicts( - inp: Union[MutableMapping[str, Any], MutableSequence[Any], Any], -) -> Union[str, MutableSequence[Any], MutableMapping[str, Any]]: + inp: MutableMapping[str, Any] | MutableSequence[Any] | Any, +) -> str | MutableSequence[Any] | MutableMapping[str, Any]: """ Convert any present byte string to unicode string, inplace. @@ -124,9 +124,7 @@ def load_linked_file( return _node, new_url -def normalize_to_map( - obj: Union[list[Any], dict[str, Any]], key_field: str -) -> dict[str, Any]: +def normalize_to_map(obj: list[Any] | dict[str, Any], key_field: str) -> dict[str, Any]: """From https://github.com/rabix/sbpack/blob/b8404a0859ffcbe1edae6d8f934e51847b003320/sbpack/lib.py .""" if isinstance(obj, dict): return deepcopy(obj) @@ -146,7 +144,7 @@ def normalize_to_map( def normalize_to_list( - obj: Union[list[Any], dict[str, Any]], key_field: str, value_field: Optional[str] + obj: list[Any] | dict[str, Any], key_field: str, value_field: str | None ) -> list[Any]: """From https://github.com/rabix/sbpack/blob/b8404a0859ffcbe1edae6d8f934e51847b003320/sbpack/lib.py .""" if isinstance(obj, list): @@ -260,8 +258,8 @@ def to_pascal_case(name: str) -> str: def sanitise_schema_field( - schema_field_item: Union[dict[str, Any], str], -) -> Union[dict[str, Any], str]: + schema_field_item: dict[str, Any] | str, +) -> dict[str, Any] | str: """ Schemas need to be resolved before converted to JSON properties. diff --git a/mypy-requirements.txt b/mypy-requirements.txt index 523eb6a1..4019c3cc 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -1,5 +1,4 @@ mypy==1.18.2 -typing_extensions types-requests types-jsonschema types-setuptools>=57.4.0 diff --git a/mypy-stubs/cwlformat/formatter.pyi b/mypy-stubs/cwlformat/formatter.pyi index a9f59ed4..dbd248fa 100644 --- a/mypy-stubs/cwlformat/formatter.pyi +++ b/mypy-stubs/cwlformat/formatter.pyi @@ -1,3 +1,3 @@ from typing import Any, Dict -def stringify_dict(as_dict: Dict[str, Any]) -> str: ... +def stringify_dict(as_dict: dict[str, Any]) -> str: ... diff --git a/mypy-stubs/rdflib/collection.pyi b/mypy-stubs/rdflib/collection.pyi index 0bee9842..a04649b6 100644 --- a/mypy-stubs/rdflib/collection.pyi +++ b/mypy-stubs/rdflib/collection.pyi @@ -1,4 +1,5 @@ -from typing import Any, Iterator +from collections.abc import Iterator +from typing import Any from rdflib.graph import Graph from rdflib.term import Node diff --git a/mypy-stubs/rdflib/compare.pyi b/mypy-stubs/rdflib/compare.pyi index ebfad917..6b56e905 100644 --- a/mypy-stubs/rdflib/compare.pyi +++ b/mypy-stubs/rdflib/compare.pyi @@ -1,6 +1,5 @@ from rdflib.graph import ConjunctiveGraph, Graph - class IsomorphicGraph(ConjunctiveGraph): pass diff --git a/mypy-stubs/rdflib/graph.pyi b/mypy-stubs/rdflib/graph.pyi index 23c2e6e1..c554cc82 100644 --- a/mypy-stubs/rdflib/graph.pyi +++ b/mypy-stubs/rdflib/graph.pyi @@ -1,16 +1,15 @@ import pathlib +from collections.abc import Iterable, Iterator from typing import ( IO, Any, - Iterable, - Iterator, List, Optional, - Set, Tuple, Union, overload, ) +from builtins import set as _set from rdflib import query from rdflib.collection import Collection @@ -26,9 +25,9 @@ class Graph(Node): def __init__( self, store: str = ..., - identifier: Optional[Any] = ..., - namespace_manager: Optional[Any] = ..., - base: Optional[Any] = ..., + identifier: Any | None = ..., + namespace_manager: Any | None = ..., + base: Any | None = ..., ) -> None: ... store: Any = ... identifier: Any = ... @@ -44,59 +43,59 @@ class Graph(Node): def remove(self, triple: Any) -> None: ... def triples( self, - triple: Tuple[ - Optional[Union[str, Identifier]], - Optional[Union[str, Identifier]], - Optional[Identifier], + triple: tuple[ + str | Identifier | None, + str | Identifier | None, + Identifier | None, ], - ) -> Iterator[Tuple[Identifier, Identifier, Identifier]]: ... + ) -> Iterator[tuple[Identifier, Identifier, Identifier]]: ... def __getitem__( self, item: slice | Path | Node ) -> Iterator[ - Tuple[Identifier, Identifier, Identifier] | Tuple[Identifier, identifier] | Node + tuple[Identifier, Identifier, Identifier] | tuple[Identifier, identifier] | Node ]: ... def __contains__(self, triple: Any) -> bool: ... def __add__(self, other: Any) -> Graph: ... def set(self, triple: Any) -> None: ... def subjects( - self, predicate: Optional[Any] = ..., object: Optional[Any] = ... + self, predicate: Any | None = ..., object: Any | None = ... ) -> Iterable[Node]: ... def predicates( - self, subject: Optional[Any] = ..., object: Optional[Any] = ... + self, subject: Any | None = ..., object: Any | None = ... ) -> Iterable[Node]: ... def objects( - self, subject: Optional[Any] = ..., predicate: Optional[Any] = ... + self, subject: Any | None = ..., predicate: Any | None = ... ) -> Iterable[Identifier]: ... - def subject_predicates(self, object: Optional[Any] = ...) -> None: ... - def subject_objects(self, predicate: Optional[Any] = ...) -> None: ... - def predicate_objects(self, subject: Optional[Any] = ...) -> None: ... - def triples_choices(self, triple: Any, context: Optional[Any] = ...) -> None: ... + def subject_predicates(self, object: Any | None = ...) -> None: ... + def subject_objects(self, predicate: Any | None = ...) -> None: ... + def predicate_objects(self, subject: Any | None = ...) -> None: ... + def triples_choices(self, triple: Any, context: Any | None = ...) -> None: ... def value( self, - subject: Optional[Any] = ..., + subject: Any | None = ..., predicate: Any = ..., - object: Optional[Any] = ..., - default: Optional[Any] = ..., + object: Any | None = ..., + default: Any | None = ..., any: bool = ..., ) -> Any: ... def label(self, subject: Any, default: str = ...) -> Any: ... def preferredLabel( self, subject: Any, - lang: Optional[Any] = ..., - default: Optional[Any] = ..., + lang: Any | None = ..., + default: Any | None = ..., labelProperties: Any = ..., - ) -> List[Tuple[Any, Any]]: ... + ) -> list[tuple[Any, Any]]: ... def comment(self, subject: Any, default: str = ...) -> Any: ... def items(self, list: Any) -> Iterator[Any]: ... def transitiveClosure( - self, func: Any, arg: Any, seen: Optional[Any] = ... + self, func: Any, arg: Any, seen: Any | None = ... ) -> Iterator[Any]: ... def transitive_objects( - self, subject: Any, property: Any, remember: Optional[Any] = ... + self, subject: Any, property: Any, remember: Any | None = ... ) -> Iterator[Any]: ... def transitive_subjects( - self, predicate: Any, object: Any, remember: Optional[Any] = ... + self, predicate: Any, object: Any, remember: Any | None = ... ) -> Iterator[Any]: ... def seq(self, subject: Any) -> Seq | None: ... def qname(self, uri: Any) -> Any: ... @@ -104,7 +103,7 @@ class Graph(Node): def bind( self, prefix: Any, namespace: Any, override: bool = ..., replace: bool = ... ) -> Any: ... - def namespaces(self) -> Iterator[Tuple[Any, Any]]: ... + def namespaces(self) -> Iterator[tuple[Any, Any]]: ... def absolutize(self, uri: Any, defrag: int = ...) -> Any: ... # no destination and non-None positional encoding @@ -113,7 +112,7 @@ class Graph(Node): self, destination: None, format: str, - base: Optional[str], + base: str | None, encoding: str, **args: Any, ) -> bytes: ... @@ -124,7 +123,7 @@ class Graph(Node): self, destination: None = ..., format: str = ..., - base: Optional[str] = ..., + base: str | None = ..., *, encoding: str, **args: Any, @@ -136,7 +135,7 @@ class Graph(Node): self, destination: None = ..., format: str = ..., - base: Optional[str] = ..., + base: str | None = ..., encoding: None = ..., **args: Any, ) -> str: ... @@ -145,10 +144,10 @@ class Graph(Node): @overload def serialize( self, - destination: Union[str, pathlib.PurePath, IO[bytes]], + destination: str | pathlib.PurePath | IO[bytes], format: str = ..., - base: Optional[str] = ..., - encoding: Optional[str] = ..., + base: str | None = ..., + encoding: str | None = ..., **args: Any, ) -> "Graph": ... @@ -156,32 +155,32 @@ class Graph(Node): @overload def serialize( self, - destination: Optional[Union[str, pathlib.PurePath, IO[bytes]]] = ..., + destination: str | pathlib.PurePath | IO[bytes] | None = ..., format: str = ..., - base: Optional[str] = ..., - encoding: Optional[str] = ..., + base: str | None = ..., + encoding: str | None = ..., **args: Any, ) -> Union[bytes, str, "Graph"]: ... def parse( self, - source: Optional[Any] = ..., - publicID: Optional[Any] = ..., - format: Optional[str] = ..., - location: Optional[Any] = ..., - file: Optional[Any] = ..., - data: Optional[Any] = ..., + source: Any | None = ..., + publicID: Any | None = ..., + format: str | None = ..., + location: Any | None = ..., + file: Any | None = ..., + data: Any | None = ..., **args: Any, ) -> "Graph": ... def load( - self, source: Any, publicID: Optional[Any] = ..., format: str = ... + self, source: Any, publicID: Any | None = ..., format: str = ... ) -> "Graph": ... def query( self, query_object: Any, processor: str = ..., result: str = ..., - initNs: Optional[Any] = ..., - initBindings: Optional[Any] = ..., + initNs: Any | None = ..., + initBindings: Any | None = ..., use_store_provided: bool = ..., **kwargs: Any, ) -> query.Result: ... @@ -189,26 +188,26 @@ class Graph(Node): self, update_object: Any, processor: str = ..., - initNs: Optional[Any] = ..., - initBindings: Optional[Any] = ..., + initNs: Any | None = ..., + initBindings: Any | None = ..., use_store_provided: bool = ..., **kwargs: Any, ) -> Any: ... def n3(self) -> str: ... def isomorphic(self, other: Any) -> bool: ... def connected(self) -> bool: ... - def all_nodes(self) -> Set[Any]: ... + def all_nodes(self) -> _set[Node]: ... def collection(self, identifier: Any) -> Collection: ... def resource(self, identifier: Any) -> Resource: ... def skolemize( self, - new_graph: Optional[Any] = ..., - bnode: Optional[Any] = ..., - authority: Optional[Any] = ..., - basepath: Optional[Any] = ..., + new_graph: Any | None = ..., + bnode: Any | None = ..., + authority: Any | None = ..., + basepath: Any | None = ..., ) -> Graph: ... def de_skolemize( - self, new_graph: Optional[Any] = ..., uriref: Optional[Any] = ... + self, new_graph: Any | None = ..., uriref: Any | None = ... ) -> Graph: ... class ConjunctiveGraph(Graph): @@ -218,32 +217,32 @@ class ConjunctiveGraph(Graph): def __init__( self, store: str = ..., - identifier: Optional[Any] = ..., - default_graph_base: Optional[Any] = ..., + identifier: Any | None = ..., + default_graph_base: Any | None = ..., ) -> None: ... def add(self, triple_or_quad: Any) -> None: ... def addN(self, quads: Any) -> None: ... def remove(self, triple_or_quad: Any) -> None: ... # def triples(self, triple_or_quad: Tuple[Optional[Union[str, BNode]], Optional[Union[str, BNode]], Optional[BNode]], context: Tuple[Optional[Union[str, BNode]], Optional[Union[str, BNode]], Optional[BNode]]) -> Iterator[Tuple[Identifier, Identifier, Identifier]]: ... - def quads(self, triple_or_quad: Optional[Any] = ...) -> None: ... - def triples_choices(self, triple: Any, context: Optional[Any] = ...) -> None: ... - def contexts(self, triple: Optional[Any] = ...) -> None: ... + def quads(self, triple_or_quad: Any | None = ...) -> None: ... + def triples_choices(self, triple: Any, context: Any | None = ...) -> None: ... + def contexts(self, triple: Any | None = ...) -> None: ... def get_context( self, identifier: Node | str | None, quoted: bool = ..., - base: Optional[str] = ..., + base: str | None = ..., ) -> Graph: ... def remove_context(self, context: Any) -> None: ... - def context_id(self, uri: Any, context_id: Optional[Any] = ...) -> Any: ... + def context_id(self, uri: Any, context_id: Any | None = ...) -> Any: ... def parse( self, - source: Optional[Any] = ..., - publicID: Optional[Any] = ..., - format: Optional[str] = ..., - location: Optional[Any] = ..., - file: Optional[Any] = ..., - data: Optional[Any] = ..., + source: Any | None = ..., + publicID: Any | None = ..., + format: str | None = ..., + location: Any | None = ..., + file: Any | None = ..., + data: Any | None = ..., **args: Any, ) -> Graph: ... diff --git a/mypy-stubs/rdflib/namespace/__init__.pyi b/mypy-stubs/rdflib/namespace/__init__.pyi index 63af9297..6f1b96f9 100644 --- a/mypy-stubs/rdflib/namespace/__init__.pyi +++ b/mypy-stubs/rdflib/namespace/__init__.pyi @@ -59,7 +59,7 @@ SPLIT_START_CATEGORIES = NAME_START_CATEGORIES + ["Nd"] XMLNS = "http://www.w3.org/XML/1998/namespace" -def split_uri(uri: Any, split_start: Any = ...) -> Tuple[str, str]: ... +def split_uri(uri: Any, split_start: Any = ...) -> tuple[str, str]: ... from rdflib.namespace._CSVW import CSVW from rdflib.namespace._DC import DC diff --git a/mypy-stubs/rdflib/paths.pyi b/mypy-stubs/rdflib/paths.pyi index 9bea1795..62190987 100644 --- a/mypy-stubs/rdflib/paths.pyi +++ b/mypy-stubs/rdflib/paths.pyi @@ -1,5 +1,5 @@ -from collections.abc import Generator -from typing import Any, Callable, Union +from collections.abc import Callable, Generator +from typing import Any, Union from rdflib.term import Node as Node from rdflib.term import URIRef as URIRef diff --git a/mypy-stubs/rdflib/plugin.pyi b/mypy-stubs/rdflib/plugin.pyi index 9b94e99f..cef006f0 100644 --- a/mypy-stubs/rdflib/plugin.pyi +++ b/mypy-stubs/rdflib/plugin.pyi @@ -6,5 +6,5 @@ def register(name: str, kind: Any, module_path: str, class_name: str) -> None: . PluginT = TypeVar("PluginT") -def get(name: str, kind: Type[PluginT]) -> Type[PluginT]: ... +def get(name: str, kind: type[PluginT]) -> type[PluginT]: ... def plugins(name: Any | None = ..., kind: Any | None = ...) -> None: ... diff --git a/mypy-stubs/rdflib/query.pyi b/mypy-stubs/rdflib/query.pyi index 73f4008b..a72bf711 100644 --- a/mypy-stubs/rdflib/query.pyi +++ b/mypy-stubs/rdflib/query.pyi @@ -1,12 +1,12 @@ -from typing import IO, Any, Dict, Iterator, List, Mapping, Optional, Tuple, overload +from collections.abc import Iterator, Mapping +from typing import IO, Any, Dict, List, Optional, SupportsIndex, Tuple, overload from rdflib import URIRef, Variable from rdflib.term import Identifier -from typing_extensions import SupportsIndex -class ResultRow(Tuple["Identifier", ...]): +class ResultRow(tuple["Identifier", ...]): def __new__( - cls, values: Mapping[Variable, Identifier], labels: List[Variable] + cls, values: Mapping[Variable, Identifier], labels: list[Variable] ) -> ResultRow: ... def __getattr__(self, name: str) -> Identifier: ... @overload @@ -14,9 +14,9 @@ class ResultRow(Tuple["Identifier", ...]): @overload def __getitem__(self, __x: SupportsIndex) -> Identifier: ... @overload - def __getitem__(self, __x: slice) -> Tuple[Identifier, ...]: ... + def __getitem__(self, __x: slice) -> tuple[Identifier, ...]: ... def get(self, name: str, default: Any | None = ...) -> Identifier: ... - def asdict(self) -> Dict[str, Identifier]: ... + def asdict(self) -> dict[str, Identifier]: ... class Result: type: Any @@ -31,12 +31,12 @@ class Result: source: IO[Any] | None = ..., format: str | None = ..., content_type: str | None = ..., - **kwargs: Any + **kwargs: Any, ) -> Result: ... def serialize( self, destination: str | IO[Any] | None = ..., encoding: str = ..., format: str = ..., - **args: Any - ) -> Optional[bytes]: ... + **args: Any, + ) -> bytes | None: ... diff --git a/mypy-stubs/rdflib/resource.pyi b/mypy-stubs/rdflib/resource.pyi index e520dbe6..815bbb79 100644 --- a/mypy-stubs/rdflib/resource.pyi +++ b/mypy-stubs/rdflib/resource.pyi @@ -1,4 +1,5 @@ -from typing import Any, Iterable, Iterator, Tuple +from collections.abc import Iterable, Iterator +from typing import Any, Tuple from _typeshed import Incomplete from rdflib.graph import Graph, Seq @@ -14,9 +15,9 @@ class Resource: def subjects(self, predicate: Any | None = ...) -> Iterable[Node]: ... def predicates(self, o: Incomplete | None = ...) -> Iterable[Node]: ... def objects(self, predicate: Any | None = ...) -> Iterable[Node]: ... - def subject_predicates(self) -> Iterator[Tuple[Node, Node]]: ... - def subject_objects(self) -> Iterator[Tuple[Node, Node]]: ... - def predicate_objects(self) -> Iterator[Tuple[Node, Node]]: ... + def subject_predicates(self) -> Iterator[tuple[Node, Node]]: ... + def subject_objects(self) -> Iterator[tuple[Node, Node]]: ... + def predicate_objects(self) -> Iterator[tuple[Node, Node]]: ... def value( self, p: Node, o: Node | None = ..., default: Any | None = ..., any: bool = ... ) -> Any: ... diff --git a/mypy-stubs/rdflib/term.pyi b/mypy-stubs/rdflib/term.pyi index 0830bdc2..83a595b5 100644 --- a/mypy-stubs/rdflib/term.pyi +++ b/mypy-stubs/rdflib/term.pyi @@ -1,9 +1,10 @@ -from typing import Any, Callable, Union +from collections.abc import Callable +from typing import Any, Union class Node: ... class Identifier(Node, str): - def __new__(cls, value: Union[Any, str, None]) -> "Identifier": ... + def __new__(cls, value: Any | str | None) -> "Identifier": ... def eq(self, other: Any) -> bool: ... def neq(self, other: Any) -> bool: ... diff --git a/pyproject.toml b/pyproject.toml index 1b67965f..cc5336d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,6 @@ classifiers = [ "Operating System :: MacOS :: MacOS X", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", @@ -30,7 +29,7 @@ classifiers = [ "Topic :: System :: Distributed Computing", "Typing :: Typed", ] -requires-python = ">=3.9,<3.15" +requires-python = ">=3.10,<3.15" dynamic = ["version", "dependencies"] [project.urls] diff --git a/requirements.txt b/requirements.txt index 604f77ea..6ae186b8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,3 @@ rdflib requests schema-salad >= 8.8.20250205075315,<9 ruamel.yaml >= 0.17.6, < 0.19 -typing_extensions;python_version<'3.10' diff --git a/tests/test_format.py b/tests/test_format.py index c0320d07..190d1a3a 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -3,7 +3,6 @@ import xml.sax from pathlib import Path -from typing import Optional import requests from pytest import raises @@ -20,7 +19,7 @@ from .util import get_path -def _create_file(format_: Optional[str] = None) -> CWLObjectType: +def _create_file(format_: str | None = None) -> CWLObjectType: obj: CWLObjectType = { "class": "File", "basename": "example.txt", diff --git a/tox.ini b/tox.ini index 5dfb41a2..ce89ee91 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,9 @@ [tox] envlist = - py3{9,10,11,12,13,14}-lint, - py3{9,10,11,12,13,14}-unit, - py3{9,10,11,12,13,14}-bandit, - py3{9,10,11,12,13,14}-mypy, + py3{10,11,12,13,14}-lint, + py3{10,11,12,13,14}-unit, + py3{10,11,12,13,14}-bandit, + py3{10,11,12,13,14}-mypy, py312-lint-readme, py312-pydocstyle isolated_build = True @@ -11,7 +11,6 @@ skip_missing_interpreters = True [gh-actions] python = - 3.9: py39 3.10: py310 3.11: py311 3.12: py312 @@ -20,10 +19,10 @@ python = [testenv] description = - py3{9,10,11,12,13,14}-unit: Run the unit tests - py3{9,10,11,12,13,14}-lint: Lint the Python code - py3{9,10,11,12,13,14}-bandit: Search for common security issues - py3{9,10,11,12,13,14}-mypy: Check for type safety + py3{10,11,12,13,14}-unit: Run the unit tests + py3{10,11,12,13,14}-lint: Lint the Python code + py3{10,11,12,13,14}-bandit: Search for common security issues + py3{10,11,12,13,14}-mypy: Check for type safety py312-pydocstyle: docstring style checker py312-lint-readme: Lint the README.rst->.md conversion @@ -31,34 +30,34 @@ passenv = CI GITHUB_* deps = - py3{9,10,11,12,13,14}-{unit,mypy}: -rrequirements.txt - py3{9,10,11,12,13,14}-{unit,mypy}: -rtest-requirements.txt - py3{9,10,11,12,13,14}-lint: -rlint-requirements.txt - py3{9,10,11,12,13,14}-bandit: bandit - py3{9,10,11,12,13,14}-mypy: -rmypy-requirements.txt + py3{10,11,12,13,14}-{unit,mypy}: -rrequirements.txt + py3{10,11,12,13,14}-{unit,mypy}: -rtest-requirements.txt + py3{10,11,12,13,14}-lint: -rlint-requirements.txt + py3{10,11,12,13,14}-bandit: bandit + py3{10,11,12,13,14}-mypy: -rmypy-requirements.txt setenv = - py3{9,10,11,12,13,14}-unit: LC_ALL = C.UTF-8 + py3{10,11,12,13,14}-unit: LC_ALL = C.UTF-8 commands = - py3{9,10,11,12,13,14}-unit: python -m pip install -U pip setuptools wheel - py3{9,10,11,12,13,14}-unit: make coverage-report coverage.xml PYTEST_EXTRA={posargs} - py3{9,10,11,12,13,14}-bandit: bandit --recursive cwl_utils - py3{9,10,11,12,13,14}-lint: make flake8 - py3{9,10,11,12,13,14}-lint: make format-check - py3{9,10,11,12,13,14}-mypy: make mypy + py3{10,11,12,13,14}-unit: python -m pip install -U pip setuptools wheel + py3{10,11,12,13,14}-unit: make coverage-report coverage.xml PYTEST_EXTRA={posargs} + py3{10,11,12,13,14}-bandit: bandit --recursive cwl_utils + py3{10,11,12,13,14}-lint: make flake8 + py3{10,11,12,13,14}-lint: make format-check + py3{10,11,12,13,14}-mypy: make mypy allowlist_externals = - py3{9,10,11,12,13,14}-lint: flake8 - py3{9,10,11,12,13,14}-lint: black - py3{9,10,11,12,13,14}-{mypy,shellcheck,lint,unit}: make + py3{10,11,12,13,14}-lint: flake8 + py3{10,11,12,13,14}-lint: black + py3{10,11,12,13,14}-{mypy,shellcheck,lint,unit}: make skip_install = - py3{9,10,11,12,13,14}-lint: true - py3{9,10,11,12,13,14}-bandit: true + py3{10,11,12,13,14}-lint: true + py3{10,11,12,13,14}-bandit: true extras = - py3{9,10,11,12,13,14}-unit: pretty + py3{10,11,12,13,14}-unit: pretty [testenv:py312-pydocstyle] allowlist_externals = make From d2b5156f94a9f527832723b73239e41587cce82c Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Fri, 3 Oct 2025 13:42:31 +0200 Subject: [PATCH 2/2] structural pattern matching --- cwl_utils/cwl_v1_0_expression_refactor.py | 275 +++++++-------- cwl_utils/cwl_v1_1_expression_refactor.py | 277 +++++++-------- cwl_utils/cwl_v1_2_expression_refactor.py | 277 +++++++-------- cwl_utils/expression.py | 116 +++--- cwl_utils/expression_refactor.py | 41 +-- cwl_utils/graph_split.py | 89 ++--- cwl_utils/inputs_schema_gen.py | 133 ++++--- cwl_utils/parser/__init__.py | 123 ++++--- cwl_utils/parser/cwl_v1_0_utils.py | 156 +++++---- cwl_utils/parser/cwl_v1_1_utils.py | 156 +++++---- cwl_utils/parser/cwl_v1_2_utils.py | 158 ++++----- cwl_utils/parser/utils.py | 407 +++++++++++----------- cwl_utils/sandboxjs.py | 63 ++-- cwl_utils/utils.py | 90 +++-- 14 files changed, 1205 insertions(+), 1156 deletions(-) diff --git a/cwl_utils/cwl_v1_0_expression_refactor.py b/cwl_utils/cwl_v1_0_expression_refactor.py index 5c96f353..5b5a5e92 100755 --- a/cwl_utils/cwl_v1_0_expression_refactor.py +++ b/cwl_utils/cwl_v1_0_expression_refactor.py @@ -208,89 +208,91 @@ def traverse( skip_command_line2: bool, ) -> tuple[cwl.CommandLineTool | cwl.ExpressionTool | cwl.Workflow, bool]: """Convert the given process and any subprocesses.""" - if not inside and isinstance(process, cwl.CommandLineTool): - process = expand_stream_shortcuts(process) - wf_inputs = [] - wf_outputs = [] - step_inputs = [] - step_outputs = [] - if process.inputs: - for inp in process.inputs: - inp_id = inp.id.split("#")[-1] - step_inputs.append( - cwl.WorkflowStepInput( - id=inp_id, - source=inp_id, - extension_fields=inp.extension_fields, - loadingOptions=inp.loadingOptions, + match process: + case cwl.CommandLineTool() if not inside: + process = expand_stream_shortcuts(process) + wf_inputs = [] + wf_outputs = [] + step_inputs = [] + step_outputs = [] + if process.inputs: + for inp in process.inputs: + inp_id = inp.id.split("#")[-1] + step_inputs.append( + cwl.WorkflowStepInput( + id=inp_id, + source=inp_id, + extension_fields=inp.extension_fields, + loadingOptions=inp.loadingOptions, + ) ) - ) - wf_inputs.append( - cwl.InputParameter( - id=inp_id, - label=inp.label, - secondaryFiles=inp.secondaryFiles, - streamable=inp.streamable, - doc=inp.doc, - format=inp.format, - default=inp.default, - type_=inp.type_, - extension_fields=inp.extension_fields, - loadingOptions=inp.loadingOptions, + wf_inputs.append( + cwl.InputParameter( + id=inp_id, + label=inp.label, + secondaryFiles=inp.secondaryFiles, + streamable=inp.streamable, + doc=inp.doc, + format=inp.format, + default=inp.default, + type_=inp.type_, + extension_fields=inp.extension_fields, + loadingOptions=inp.loadingOptions, + ) ) - ) - if process.outputs: - for outp in process.outputs: - outp_id = outp.id.split("#")[-1] - step_outputs.append(outp_id) - wf_outputs.append( - cwl.WorkflowOutputParameter( - id=outp_id, - label=outp.label, - secondaryFiles=outp.secondaryFiles, - streamable=outp.streamable, - doc=outp.doc, - format=outp.format, - outputSource=f"main/{outp_id}", - type_=outp.type_, - extension_fields=outp.extension_fields, - loadingOptions=outp.loadingOptions, + if process.outputs: + for outp in process.outputs: + outp_id = outp.id.split("#")[-1] + step_outputs.append(outp_id) + wf_outputs.append( + cwl.WorkflowOutputParameter( + id=outp_id, + label=outp.label, + secondaryFiles=outp.secondaryFiles, + streamable=outp.streamable, + doc=outp.doc, + format=outp.format, + outputSource=f"main/{outp_id}", + type_=outp.type_, + extension_fields=outp.extension_fields, + loadingOptions=outp.loadingOptions, + ) ) - ) - step = cwl.WorkflowStep( - id="#main", - in_=step_inputs, - out=step_outputs, - run=copy.deepcopy(process), - ) - workflow = cwl.Workflow( - inputs=wf_inputs, - outputs=wf_outputs, - steps=[step], - cwlVersion=process.cwlVersion, - ) - result, modified = traverse_workflow( - workflow, replace_etool, skip_command_line1, skip_command_line2 - ) - if modified: - return result, True - else: + step = cwl.WorkflowStep( + id="#main", + in_=step_inputs, + out=step_outputs, + run=copy.deepcopy(process), + ) + workflow = cwl.Workflow( + inputs=wf_inputs, + outputs=wf_outputs, + steps=[step], + cwlVersion=process.cwlVersion, + ) + result, modified = traverse_workflow( + workflow, replace_etool, skip_command_line1, skip_command_line2 + ) + if modified: + return result, True + else: + return process, False + case cwl.ExpressionTool() if replace_etool: + expression = get_expression(process.expression, empty_inputs(process), None) + # Why call get_expression on an ExpressionTool? + # It normalizes the form of $() CWL expressions into the ${} style + if expression: + process2 = copy.deepcopy(process) + process2.expression = expression + else: + process2 = process + return etool_to_cltool(process2), True + case cwl.Workflow(): + return traverse_workflow( + process, replace_etool, skip_command_line1, skip_command_line2 + ) + case _: return process, False - if isinstance(process, cwl.ExpressionTool) and replace_etool: - expression = get_expression(process.expression, empty_inputs(process), None) - # Why call get_expression on an ExpressionTool? - # It normalizes the form of $() CWL expressions into the ${} style - if expression: - process2 = copy.deepcopy(process) - process2.expression = expression - else: - process2 = process - return etool_to_cltool(process2), True - if isinstance(process, cwl.Workflow): - return traverse_workflow( - process, replace_etool, skip_command_line1, skip_command_line2 - ) - return process, False def load_step( @@ -668,8 +670,8 @@ def process_workflow_reqs_and_hints( iwdr: cwl.InitialWorkDirRequirement | None = None if workflow.requirements is not None: for req in cast(list[cwl.ProcessRequirement], workflow.requirements): - if req and isinstance(req, cwl.EnvVarRequirement): - if req.envDef: + match req: + case cwl.EnvVarRequirement() if req.envDef: for index, envDef in enumerate(req.envDef): if envDef.envValue: expression = get_expression(envDef.envValue, inputs, None) @@ -699,36 +701,33 @@ def process_workflow_reqs_and_hints( newEnvDef.envValue = f"$(inputs._envDef{index})" envVarReq.envDef[index] = newEnvDef generated_envVar_reqs.append((etool_id, index)) - if req and isinstance(req, cwl.ResourceRequirement): - for attr in cwl.ResourceRequirement.attrs: - this_attr = getattr(req, attr, None) - if this_attr: - expression = get_expression(this_attr, inputs, None) - if expression: - modified = True - target = cwl.InputParameter(id=None, type_="long") - etool_id = ( - "_expression_workflow_ResourceRequirement_{}".format( + case cwl.ResourceRequirement(): + for attr in cwl.ResourceRequirement.attrs: + this_attr = getattr(req, attr, None) + if this_attr: + expression = get_expression(this_attr, inputs, None) + if expression: + modified = True + target = cwl.InputParameter(id=None, type_="long") + etool_id = "_expression_workflow_ResourceRequirement_{}".format( attr ) - ) - replace_expr_with_etool( - expression, - etool_id, - workflow, - target, - None, - replace_etool, - ) - if not resourceReq: - resourceReq = cwl.ResourceRequirement( - loadingOptions=workflow.loadingOptions, + replace_expr_with_etool( + expression, + etool_id, + workflow, + target, + None, + replace_etool, ) - prop_reqs += (cwl.ResourceRequirement,) - setattr(resourceReq, attr, f"$(inputs._{attr})") - generated_res_reqs.append((etool_id, attr)) - if req and isinstance(req, cwl.InitialWorkDirRequirement): - if req.listing: + if not resourceReq: + resourceReq = cwl.ResourceRequirement( + loadingOptions=workflow.loadingOptions, + ) + prop_reqs += (cwl.ResourceRequirement,) + setattr(resourceReq, attr, f"$(inputs._{attr})") + generated_res_reqs.append((etool_id, attr)) + case cwl.InitialWorkDirRequirement() if req.listing: if isinstance(req.listing, str): expression = get_expression(req.listing, inputs, None) if expression: @@ -991,8 +990,8 @@ def process_level_reqs( return False step_name = step.id.split("#", 1)[-1] for req_index, req in enumerate(process.requirements): - if req and isinstance(req, cwl.EnvVarRequirement): - if req.envDef: + match req: + case cwl.EnvVarRequirement() if req.envDef: for env_index, envDef in enumerate(req.envDef): if envDef.envValue: expression = get_expression(envDef.envValue, inputs, None) @@ -1015,33 +1014,35 @@ def process_level_reqs( env_index ].envValue = f"$(inputs._envDef{env_index})" generated_envVar_reqs.append((etool_id, env_index)) - if req and isinstance(req, cwl.ResourceRequirement): - for attr in cwl.ResourceRequirement.attrs: - this_attr = getattr(req, attr, None) - if this_attr: - expression = get_expression(this_attr, inputs, None) - if expression: - modified = True - target = cwl.InputParameter(id=None, type_="long") - etool_id = "_expression_{}_ResourceRequirement_{}".format( - step_name, attr - ) - replace_clt_hintreq_expr_with_etool( - expression, etool_id, parent, target, step, replace_etool - ) - setattr( - target_process.requirements[req_index], - attr, - f"$(inputs._{attr})", - ) - generated_res_reqs.append((etool_id, attr)) - - if ( - not skip_command_line2 - and req - and isinstance(req, cwl.InitialWorkDirRequirement) - ): - if req.listing: + case cwl.ResourceRequirement(): + for attr in cwl.ResourceRequirement.attrs: + this_attr = getattr(req, attr, None) + if this_attr: + expression = get_expression(this_attr, inputs, None) + if expression: + modified = True + target = cwl.InputParameter(id=None, type_="long") + etool_id = "_expression_{}_ResourceRequirement_{}".format( + step_name, attr + ) + replace_clt_hintreq_expr_with_etool( + expression, + etool_id, + parent, + target, + step, + replace_etool, + ) + setattr( + target_process.requirements[req_index], + attr, + f"$(inputs._{attr})", + ) + generated_res_reqs.append((etool_id, attr)) + + case cwl.InitialWorkDirRequirement() if ( + not skip_command_line2 and req.listing + ): if isinstance(req.listing, str): expression = get_expression(req.listing, inputs, None) if expression: diff --git a/cwl_utils/cwl_v1_1_expression_refactor.py b/cwl_utils/cwl_v1_1_expression_refactor.py index 4f9ca0ba..286862af 100755 --- a/cwl_utils/cwl_v1_1_expression_refactor.py +++ b/cwl_utils/cwl_v1_1_expression_refactor.py @@ -208,89 +208,91 @@ def traverse( skip_command_line2: bool, ) -> tuple[cwl.CommandLineTool | cwl.ExpressionTool | cwl.Workflow, bool]: """Convert the given process and any subprocesses.""" - if not inside and isinstance(process, cwl.CommandLineTool): - process = expand_stream_shortcuts(process) - wf_inputs = [] - wf_outputs = [] - step_inputs = [] - step_outputs = [] - if process.inputs: - for inp in process.inputs: - inp_id = inp.id.split("#")[-1] - step_inputs.append( - cwl.WorkflowStepInput( - id=inp_id, - source=inp_id, - extension_fields=inp.extension_fields, - loadingOptions=inp.loadingOptions, + match process: + case cwl.CommandLineTool() if not inside: + process = expand_stream_shortcuts(process) + wf_inputs = [] + wf_outputs = [] + step_inputs = [] + step_outputs = [] + if process.inputs: + for inp in process.inputs: + inp_id = inp.id.split("#")[-1] + step_inputs.append( + cwl.WorkflowStepInput( + id=inp_id, + source=inp_id, + extension_fields=inp.extension_fields, + loadingOptions=inp.loadingOptions, + ) ) - ) - wf_inputs.append( - cwl.WorkflowInputParameter( - id=inp_id, - label=inp.label, - secondaryFiles=inp.secondaryFiles, - streamable=inp.streamable, - doc=inp.doc, - format=inp.format, - default=inp.default, - type_=inp.type_, - extension_fields=inp.extension_fields, - loadingOptions=inp.loadingOptions, + wf_inputs.append( + cwl.WorkflowInputParameter( + id=inp_id, + label=inp.label, + secondaryFiles=inp.secondaryFiles, + streamable=inp.streamable, + doc=inp.doc, + format=inp.format, + default=inp.default, + type_=inp.type_, + extension_fields=inp.extension_fields, + loadingOptions=inp.loadingOptions, + ) ) - ) - if process.outputs: - for outp in process.outputs: - outp_id = outp.id.split("#")[-1] - step_outputs.append(outp_id) - wf_outputs.append( - cwl.WorkflowOutputParameter( - id=outp_id, - label=outp.label, - secondaryFiles=outp.secondaryFiles, - streamable=outp.streamable, - doc=outp.doc, - format=outp.format, - outputSource=f"main/{outp_id}", - type_=outp.type_, - extension_fields=outp.extension_fields, - loadingOptions=outp.loadingOptions, + if process.outputs: + for outp in process.outputs: + outp_id = outp.id.split("#")[-1] + step_outputs.append(outp_id) + wf_outputs.append( + cwl.WorkflowOutputParameter( + id=outp_id, + label=outp.label, + secondaryFiles=outp.secondaryFiles, + streamable=outp.streamable, + doc=outp.doc, + format=outp.format, + outputSource=f"main/{outp_id}", + type_=outp.type_, + extension_fields=outp.extension_fields, + loadingOptions=outp.loadingOptions, + ) ) - ) - step = cwl.WorkflowStep( - id="#main", - in_=step_inputs, - out=step_outputs, - run=copy.deepcopy(process), - ) - workflow = cwl.Workflow( - inputs=wf_inputs, - outputs=wf_outputs, - steps=[step], - cwlVersion=process.cwlVersion, - ) - result, modified = traverse_workflow( - workflow, replace_etool, skip_command_line1, skip_command_line2 - ) - if modified: - return result, True - else: + step = cwl.WorkflowStep( + id="#main", + in_=step_inputs, + out=step_outputs, + run=copy.deepcopy(process), + ) + workflow = cwl.Workflow( + inputs=wf_inputs, + outputs=wf_outputs, + steps=[step], + cwlVersion=process.cwlVersion, + ) + result, modified = traverse_workflow( + workflow, replace_etool, skip_command_line1, skip_command_line2 + ) + if modified: + return result, True + else: + return process, False + case cwl.ExpressionTool() if replace_etool: + expression = get_expression(process.expression, empty_inputs(process), None) + # Why call get_expression on an ExpressionTool? + # It normalizes the form of $() CWL expressions into the ${} style + if expression: + process2 = copy.deepcopy(process) + process2.expression = expression + else: + process2 = process + return etool_to_cltool(process2), True + case cwl.Workflow(): + return traverse_workflow( + process, replace_etool, skip_command_line1, skip_command_line2 + ) + case _: return process, False - if isinstance(process, cwl.ExpressionTool) and replace_etool: - expression = get_expression(process.expression, empty_inputs(process), None) - # Why call get_expression on an ExpressionTool? - # It normalizes the form of $() CWL expressions into the ${} style - if expression: - process2 = copy.deepcopy(process) - process2.expression = expression - else: - process2 = process - return etool_to_cltool(process2), True - if isinstance(process, cwl.Workflow): - return traverse_workflow( - process, replace_etool, skip_command_line1, skip_command_line2 - ) - return process, False def load_step( @@ -668,8 +670,8 @@ def process_workflow_reqs_and_hints( iwdr: cwl.InitialWorkDirRequirement | None = None if workflow.requirements is not None: for req in cast(list[cwl.ProcessRequirement], workflow.requirements): - if req and isinstance(req, cwl.EnvVarRequirement): - if req.envDef: + match req: + case cwl.EnvVarRequirement() if req.envDef: for index, envDef in enumerate(req.envDef): if envDef.envValue: expression = get_expression(envDef.envValue, inputs, None) @@ -699,36 +701,35 @@ def process_workflow_reqs_and_hints( newEnvDef.envValue = f"$(inputs._envDef{index})" envVarReq.envDef[index] = newEnvDef generated_envVar_reqs.append((etool_id, index)) - if req and isinstance(req, cwl.ResourceRequirement): - for attr in cwl.ResourceRequirement.attrs: - this_attr = getattr(req, attr, None) - if this_attr: - expression = get_expression(this_attr, inputs, None) - if expression: - modified = True - target = cwl.WorkflowInputParameter(id=None, type_="long") - etool_id = ( - "_expression_workflow_ResourceRequirement_{}".format( + case cwl.ResourceRequirement(): + for attr in cwl.ResourceRequirement.attrs: + this_attr = getattr(req, attr, None) + if this_attr: + expression = get_expression(this_attr, inputs, None) + if expression: + modified = True + target = cwl.WorkflowInputParameter( + id=None, type_="long" + ) + etool_id = "_expression_workflow_ResourceRequirement_{}".format( attr ) - ) - replace_expr_with_etool( - expression, - etool_id, - workflow, - target, - None, - replace_etool, - ) - if not resourceReq: - resourceReq = cwl.ResourceRequirement( - loadingOptions=workflow.loadingOptions, + replace_expr_with_etool( + expression, + etool_id, + workflow, + target, + None, + replace_etool, ) - prop_reqs += (cwl.ResourceRequirement,) - setattr(resourceReq, attr, f"$(inputs._{attr})") - generated_res_reqs.append((etool_id, attr)) - if req and isinstance(req, cwl.InitialWorkDirRequirement): - if req.listing: + if not resourceReq: + resourceReq = cwl.ResourceRequirement( + loadingOptions=workflow.loadingOptions, + ) + prop_reqs += (cwl.ResourceRequirement,) + setattr(resourceReq, attr, f"$(inputs._{attr})") + generated_res_reqs.append((etool_id, attr)) + case cwl.InitialWorkDirRequirement() if req.listing: if isinstance(req.listing, str): expression = get_expression(req.listing, inputs, None) if expression: @@ -991,8 +992,8 @@ def process_level_reqs( return False step_name = step.id.split("#", 1)[-1] for req_index, req in enumerate(process.requirements): - if req and isinstance(req, cwl.EnvVarRequirement): - if req.envDef: + match req: + case cwl.EnvVarRequirement() if req.envDef: for env_index, envDef in enumerate(req.envDef): if envDef.envValue: expression = get_expression(envDef.envValue, inputs, None) @@ -1015,33 +1016,35 @@ def process_level_reqs( env_index ].envValue = f"$(inputs._envDef{env_index})" generated_envVar_reqs.append((etool_id, env_index)) - if req and isinstance(req, cwl.ResourceRequirement): - for attr in cwl.ResourceRequirement.attrs: - this_attr = getattr(req, attr, None) - if this_attr: - expression = get_expression(this_attr, inputs, None) - if expression: - modified = True - target = cwl.WorkflowInputParameter(id=None, type_="long") - etool_id = "_expression_{}_ResourceRequirement_{}".format( - step_name, attr - ) - replace_clt_hintreq_expr_with_etool( - expression, etool_id, parent, target, step, replace_etool - ) - setattr( - target_process.requirements[req_index], - attr, - f"$(inputs._{attr})", - ) - generated_res_reqs.append((etool_id, attr)) - - if ( - not skip_command_line2 - and req - and isinstance(req, cwl.InitialWorkDirRequirement) - ): - if req.listing: + case cwl.ResourceRequirement(): + for attr in cwl.ResourceRequirement.attrs: + this_attr = getattr(req, attr, None) + if this_attr: + expression = get_expression(this_attr, inputs, None) + if expression: + modified = True + target = cwl.WorkflowInputParameter(id=None, type_="long") + etool_id = "_expression_{}_ResourceRequirement_{}".format( + step_name, attr + ) + replace_clt_hintreq_expr_with_etool( + expression, + etool_id, + parent, + target, + step, + replace_etool, + ) + setattr( + target_process.requirements[req_index], + attr, + f"$(inputs._{attr})", + ) + generated_res_reqs.append((etool_id, attr)) + + case cwl.InitialWorkDirRequirement() if ( + not skip_command_line2 and req.listing + ): if isinstance(req.listing, str): expression = get_expression(req.listing, inputs, None) if expression: diff --git a/cwl_utils/cwl_v1_2_expression_refactor.py b/cwl_utils/cwl_v1_2_expression_refactor.py index 1b5398b3..48c8c706 100755 --- a/cwl_utils/cwl_v1_2_expression_refactor.py +++ b/cwl_utils/cwl_v1_2_expression_refactor.py @@ -208,89 +208,91 @@ def traverse( skip_command_line2: bool, ) -> tuple[cwl.CommandLineTool | cwl.ExpressionTool | cwl.Workflow, bool]: """Convert the given process and any subprocesses.""" - if not inside and isinstance(process, cwl.CommandLineTool): - process = expand_stream_shortcuts(process) - wf_inputs = [] - wf_outputs = [] - step_inputs = [] - step_outputs = [] - if process.inputs: - for inp in process.inputs: - inp_id = inp.id.split("#")[-1] - step_inputs.append( - cwl.WorkflowStepInput( - id=inp_id, - source=inp_id, - extension_fields=inp.extension_fields, - loadingOptions=inp.loadingOptions, + match process: + case cwl.CommandLineTool() if not inside: + process = expand_stream_shortcuts(process) + wf_inputs = [] + wf_outputs = [] + step_inputs = [] + step_outputs = [] + if process.inputs: + for inp in process.inputs: + inp_id = inp.id.split("#")[-1] + step_inputs.append( + cwl.WorkflowStepInput( + id=inp_id, + source=inp_id, + extension_fields=inp.extension_fields, + loadingOptions=inp.loadingOptions, + ) ) - ) - wf_inputs.append( - cwl.WorkflowInputParameter( - id=inp_id, - label=inp.label, - secondaryFiles=inp.secondaryFiles, - streamable=inp.streamable, - doc=inp.doc, - format=inp.format, - default=inp.default, - type_=inp.type_, - extension_fields=inp.extension_fields, - loadingOptions=inp.loadingOptions, + wf_inputs.append( + cwl.WorkflowInputParameter( + id=inp_id, + label=inp.label, + secondaryFiles=inp.secondaryFiles, + streamable=inp.streamable, + doc=inp.doc, + format=inp.format, + default=inp.default, + type_=inp.type_, + extension_fields=inp.extension_fields, + loadingOptions=inp.loadingOptions, + ) ) - ) - if process.outputs: - for outp in process.outputs: - outp_id = outp.id.split("#")[-1] - step_outputs.append(outp_id) - wf_outputs.append( - cwl.WorkflowOutputParameter( - id=outp_id, - label=outp.label, - secondaryFiles=outp.secondaryFiles, - streamable=outp.streamable, - doc=outp.doc, - format=outp.format, - outputSource=f"main/{outp_id}", - type_=outp.type_, - extension_fields=outp.extension_fields, - loadingOptions=outp.loadingOptions, + if process.outputs: + for outp in process.outputs: + outp_id = outp.id.split("#")[-1] + step_outputs.append(outp_id) + wf_outputs.append( + cwl.WorkflowOutputParameter( + id=outp_id, + label=outp.label, + secondaryFiles=outp.secondaryFiles, + streamable=outp.streamable, + doc=outp.doc, + format=outp.format, + outputSource=f"main/{outp_id}", + type_=outp.type_, + extension_fields=outp.extension_fields, + loadingOptions=outp.loadingOptions, + ) ) - ) - step = cwl.WorkflowStep( - id="#main", - in_=step_inputs, - out=step_outputs, - run=copy.deepcopy(process), - ) - workflow = cwl.Workflow( - inputs=wf_inputs, - outputs=wf_outputs, - steps=[step], - cwlVersion=process.cwlVersion, - ) - result, modified = traverse_workflow( - workflow, replace_etool, skip_command_line1, skip_command_line2 - ) - if modified: - return result, True - else: + step = cwl.WorkflowStep( + id="#main", + in_=step_inputs, + out=step_outputs, + run=copy.deepcopy(process), + ) + workflow = cwl.Workflow( + inputs=wf_inputs, + outputs=wf_outputs, + steps=[step], + cwlVersion=process.cwlVersion, + ) + result, modified = traverse_workflow( + workflow, replace_etool, skip_command_line1, skip_command_line2 + ) + if modified: + return result, True + else: + return process, False + case cwl.ExpressionTool() if replace_etool: + expression = get_expression(process.expression, empty_inputs(process), None) + # Why call get_expression on an ExpressionTool? + # It normalizes the form of $() CWL expressions into the ${} style + if expression: + process2 = copy.deepcopy(process) + process2.expression = expression + else: + process2 = process + return etool_to_cltool(process2), True + case cwl.Workflow(): + return traverse_workflow( + process, replace_etool, skip_command_line1, skip_command_line2 + ) + case _: return process, False - if isinstance(process, cwl.ExpressionTool) and replace_etool: - expression = get_expression(process.expression, empty_inputs(process), None) - # Why call get_expression on an ExpressionTool? - # It normalizes the form of $() CWL expressions into the ${} style - if expression: - process2 = copy.deepcopy(process) - process2.expression = expression - else: - process2 = process - return etool_to_cltool(process2), True - if isinstance(process, cwl.Workflow): - return traverse_workflow( - process, replace_etool, skip_command_line1, skip_command_line2 - ) - return process, False def load_step( @@ -763,8 +765,8 @@ def process_workflow_reqs_and_hints( iwdr: cwl.InitialWorkDirRequirement | None = None if workflow.requirements is not None: for req in cast(list[cwl.ProcessRequirement], workflow.requirements): - if req and isinstance(req, cwl.EnvVarRequirement): - if req.envDef: + match req: + case cwl.EnvVarRequirement() if req.envDef: for index, envDef in enumerate(req.envDef): if envDef.envValue: expression = get_expression(envDef.envValue, inputs, None) @@ -794,36 +796,35 @@ def process_workflow_reqs_and_hints( newEnvDef.envValue = f"$(inputs._envDef{index})" envVarReq.envDef[index] = newEnvDef generated_envVar_reqs.append((etool_id, index)) - if req and isinstance(req, cwl.ResourceRequirement): - for attr in cwl.ResourceRequirement.attrs: - this_attr = getattr(req, attr, None) - if this_attr: - expression = get_expression(this_attr, inputs, None) - if expression: - modified = True - target = cwl.WorkflowInputParameter(id=None, type_="long") - etool_id = ( - "_expression_workflow_ResourceRequirement_{}".format( + case cwl.ResourceRequirement(): + for attr in cwl.ResourceRequirement.attrs: + this_attr = getattr(req, attr, None) + if this_attr: + expression = get_expression(this_attr, inputs, None) + if expression: + modified = True + target = cwl.WorkflowInputParameter( + id=None, type_="long" + ) + etool_id = "_expression_workflow_ResourceRequirement_{}".format( attr ) - ) - replace_expr_with_etool( - expression, - etool_id, - workflow, - target, - None, - replace_etool, - ) - if not resourceReq: - resourceReq = cwl.ResourceRequirement( - loadingOptions=workflow.loadingOptions, + replace_expr_with_etool( + expression, + etool_id, + workflow, + target, + None, + replace_etool, ) - prop_reqs += (cwl.ResourceRequirement,) - setattr(resourceReq, attr, f"$(inputs._{attr})") - generated_res_reqs.append((etool_id, attr)) - if req and isinstance(req, cwl.InitialWorkDirRequirement): - if req.listing: + if not resourceReq: + resourceReq = cwl.ResourceRequirement( + loadingOptions=workflow.loadingOptions, + ) + prop_reqs += (cwl.ResourceRequirement,) + setattr(resourceReq, attr, f"$(inputs._{attr})") + generated_res_reqs.append((etool_id, attr)) + case cwl.InitialWorkDirRequirement() if req.listing: if isinstance(req.listing, str): expression = get_expression(req.listing, inputs, None) if expression: @@ -1086,8 +1087,8 @@ def process_level_reqs( return False step_name = step.id.split("#", 1)[-1] for req_index, req in enumerate(process.requirements): - if req and isinstance(req, cwl.EnvVarRequirement): - if req.envDef: + match req: + case cwl.EnvVarRequirement() if req.envDef: for env_index, envDef in enumerate(req.envDef): if envDef.envValue: expression = get_expression(envDef.envValue, inputs, None) @@ -1110,33 +1111,35 @@ def process_level_reqs( env_index ].envValue = f"$(inputs._envDef{env_index})" generated_envVar_reqs.append((etool_id, env_index)) - if req and isinstance(req, cwl.ResourceRequirement): - for attr in cwl.ResourceRequirement.attrs: - this_attr = getattr(req, attr, None) - if this_attr: - expression = get_expression(this_attr, inputs, None) - if expression: - modified = True - target = cwl.WorkflowInputParameter(id=None, type_="long") - etool_id = "_expression_{}_ResourceRequirement_{}".format( - step_name, attr - ) - replace_clt_hintreq_expr_with_etool( - expression, etool_id, parent, target, step, replace_etool - ) - setattr( - target_process.requirements[req_index], - attr, - f"$(inputs._{attr})", - ) - generated_res_reqs.append((etool_id, attr)) - - if ( - not skip_command_line2 - and req - and isinstance(req, cwl.InitialWorkDirRequirement) - ): - if req.listing: + case cwl.ResourceRequirement(): + for attr in cwl.ResourceRequirement.attrs: + this_attr = getattr(req, attr, None) + if this_attr: + expression = get_expression(this_attr, inputs, None) + if expression: + modified = True + target = cwl.WorkflowInputParameter(id=None, type_="long") + etool_id = "_expression_{}_ResourceRequirement_{}".format( + step_name, attr + ) + replace_clt_hintreq_expr_with_etool( + expression, + etool_id, + parent, + target, + step, + replace_etool, + ) + setattr( + target_process.requirements[req_index], + attr, + f"$(inputs._{attr})", + ) + generated_res_reqs.append((etool_id, attr)) + + case cwl.InitialWorkDirRequirement() if ( + not skip_command_line2 and req.listing + ): if isinstance(req.listing, str): expression = get_expression(req.listing, inputs, None) if expression: diff --git a/cwl_utils/expression.py b/cwl_utils/expression.py index 8833e866..b72d09b4 100644 --- a/cwl_utils/expression.py +++ b/cwl_utils/expression.py @@ -5,6 +5,7 @@ import inspect import json from collections.abc import Awaitable, MutableMapping +from enum import Enum from typing import Any, Union, cast from schema_salad.utils import json_dumps @@ -20,8 +21,9 @@ def _convert_dumper(string: str) -> str: return f"{json.dumps(string)} + " -def scanner(scan: str) -> tuple[int, int] | None: - """Find JS relevant punctuation in a string.""" +class t(Enum): + """Tokens.""" + DEFAULT = 0 DOLLAR = 1 PAREN = 2 @@ -30,67 +32,71 @@ def scanner(scan: str) -> tuple[int, int] | None: DOUBLE_QUOTE = 5 BACKSLASH = 6 + +def scanner(scan: str) -> tuple[int, int] | None: + """Find JS relevant punctuation in a string.""" i = 0 - stack = [DEFAULT] + stack: list[t] = [t.DEFAULT] start = 0 while i < len(scan): - state = stack[-1] c = scan[i] - if state == DEFAULT: - if c == "$": - stack.append(DOLLAR) - elif c == "\\": - stack.append(BACKSLASH) - elif state == BACKSLASH: - stack.pop() - if stack[-1] == DEFAULT: - return i - 1, i + 1 - elif state == DOLLAR: - if c == "(": - start = i - 1 - stack.append(PAREN) - elif c == "{": - start = i - 1 - stack.append(BRACE) - else: - stack.pop() - i -= 1 - elif state == PAREN: - if c == "(": - stack.append(PAREN) - elif c == ")": - stack.pop() - if stack[-1] == DOLLAR: - return start, i + 1 - elif c == "'": - stack.append(SINGLE_QUOTE) - elif c == '"': - stack.append(DOUBLE_QUOTE) - elif state == BRACE: - if c == "{": - stack.append(BRACE) - elif c == "}": - stack.pop() - if stack[-1] == DOLLAR: - return start, i + 1 - elif c == "'": - stack.append(SINGLE_QUOTE) - elif c == '"': - stack.append(DOUBLE_QUOTE) - elif state == SINGLE_QUOTE: - if c == "'": - stack.pop() - elif c == "\\": - stack.append(BACKSLASH) - elif state == DOUBLE_QUOTE: - if c == '"': + state = stack[-1] + match state: + case t.DEFAULT: + if c == "$": + stack.append(t.DOLLAR) + elif c == "\\": + stack.append(t.BACKSLASH) + case t.BACKSLASH: stack.pop() - elif c == "\\": - stack.append(BACKSLASH) + if stack[-1] == t.DEFAULT: + return i - 1, i + 1 + case t.DOLLAR: + if c == "(": + start = i - 1 + stack.append(t.PAREN) + elif c == "{": + start = i - 1 + stack.append(t.BRACE) + else: + stack.pop() + i -= 1 + case t.PAREN: + if c == "(": + stack.append(t.PAREN) + elif c == ")": + stack.pop() + if stack[-1] == t.DOLLAR: + return start, i + 1 + elif c == "'": + stack.append(t.SINGLE_QUOTE) + elif c == '"': + stack.append(t.DOUBLE_QUOTE) + case t.BRACE: + if c == "{": + stack.append(t.BRACE) + elif c == "}": + stack.pop() + if stack[-1] == t.DOLLAR: + return start, i + 1 + elif c == "'": + stack.append(t.SINGLE_QUOTE) + elif c == '"': + stack.append(t.DOUBLE_QUOTE) + case t.SINGLE_QUOTE: + if c == "'": + stack.pop() + elif c == "\\": + stack.append(t.BACKSLASH) + case t.DOUBLE_QUOTE: + if c == '"': + stack.pop() + elif c == "\\": + stack.append(t.BACKSLASH) i += 1 - if len(stack) > 1 and not (len(stack) == 2 and stack[1] in (BACKSLASH, DOLLAR)): + if len(stack) > 1 and not (len(stack) == 2 and stack[1] in (t.BACKSLASH, t.DOLLAR)): raise SubstitutionError( "Substitution error, unfinished block starting at position {}: '{}' stack was {}".format( start, scan[start:], stack diff --git a/cwl_utils/expression_refactor.py b/cwl_utils/expression_refactor.py index 526cc86e..eca02ece 100755 --- a/cwl_utils/expression_refactor.py +++ b/cwl_utils/expression_refactor.py @@ -103,27 +103,28 @@ def refactor(args: argparse.Namespace) -> int: _logger.info("Processing %s.", document) with open(document) as doc_handle: result = yaml.load(doc_handle) - version = result["cwlVersion"] uri = Path(document).resolve().as_uri() - if version == "v1.0": - top = cwl_v1_0.load_document_by_yaml(result, uri) - traverse: Callable[[Any, bool, bool, bool, bool], tuple[Any, bool]] = ( - cwl_v1_0_expression_refactor.traverse - ) - save: saveCWL = cwl_v1_0.save - elif version == "v1.1": - top = cwl_v1_1.load_document_by_yaml(result, uri) - traverse = cwl_v1_1_expression_refactor.traverse - save = cwl_v1_1.save - elif version == "v1.2": - top = cwl_v1_2.load_document_by_yaml(result, uri) - traverse = cwl_v1_2_expression_refactor.traverse - save = cwl_v1_2.save - else: - _logger.error( - "Sorry, %s is not a supported CWL version by this tool.", version - ) - return -1 + match result["cwlVersion"]: + case "v1.0": + top = cwl_v1_0.load_document_by_yaml(result, uri) + traverse: Callable[[Any, bool, bool, bool, bool], tuple[Any, bool]] = ( + cwl_v1_0_expression_refactor.traverse + ) + save: saveCWL = cwl_v1_0.save + case "v1.1": + top = cwl_v1_1.load_document_by_yaml(result, uri) + traverse = cwl_v1_1_expression_refactor.traverse + save = cwl_v1_1.save + case "v1.2": + top = cwl_v1_2.load_document_by_yaml(result, uri) + traverse = cwl_v1_2_expression_refactor.traverse + save = cwl_v1_2.save + case _: + _logger.error( + "Sorry, %s is not a supported CWL version by this tool.", + result["cwlVersion"], + ) + return -1 try: result, modified = traverse( top, not args.etools, False, args.skip_some1, args.skip_some2 diff --git a/cwl_utils/graph_split.py b/cwl_utils/graph_split.py index e1caacfb..e2ba20ef 100755 --- a/cwl_utils/graph_split.py +++ b/cwl_utils/graph_split.py @@ -148,6 +148,19 @@ def my_represent_none( yaml_dump(entry, output_handle, pretty) +def rewrite_id(entry: Any, this_id: str | None) -> MutableMapping[Any, Any] | str: + if isinstance(entry, MutableMapping): + if entry["id"].startswith(this_id): + assert isinstance(this_id, str) # nosec B101 + entry["id"] = cast(str, entry["id"])[len(this_id) + 1 :] + return entry + elif isinstance(entry, str): + if this_id and entry.startswith(this_id): + return entry[len(this_id) + 1 :] + return entry + raise Exception(f"{entry} is neither a dictionary nor string.") + + def rewrite( document: Any, doc_id: str, output_dir: Path, pretty: bool = False ) -> set[str]: @@ -160,50 +173,40 @@ def rewrite( this_id = document["id"] if "id" in document else None for key, value in document.items(): with SourceLine(document, key, Exception): - if key == "run" and isinstance(value, str) and value[0] == "#": - document[key] = f"{re.sub('.cwl$', '', value[1:])}.cwl" - elif key in ("id", "outputSource") and value.startswith("#" + doc_id): - document[key] = value[len(doc_id) + 2 :] - elif key == "out" and isinstance(value, list): - - def rewrite_id(entry: Any) -> MutableMapping[Any, Any] | str: - if isinstance(entry, MutableMapping): - if entry["id"].startswith(this_id): - assert isinstance(this_id, str) # nosec B101 - entry["id"] = cast(str, entry["id"])[len(this_id) + 1 :] - return entry - elif isinstance(entry, str): - if this_id and entry.startswith(this_id): - return entry[len(this_id) + 1 :] - return entry - raise Exception(f"{entry} is neither a dictionary nor string.") - - document[key][:] = [rewrite_id(entry) for entry in value] - elif key in ("source", "scatter", "items", "format"): - if ( - isinstance(value, str) - and value.startswith("#") - and "/" in value - ): - referrant_file, sub = value[1:].split("/", 1) - if referrant_file == doc_id: - document[key] = sub - else: - document[key] = f"{referrant_file}#{sub}" - elif isinstance(value, list): - new_sources = list() - for entry in value: - if entry.startswith("#" + doc_id): - new_sources.append(entry[len(doc_id) + 2 :]) + match key: + case "run" if isinstance(value, str) and value[0] == "#": + document[key] = f"{re.sub('.cwl$', '', value[1:])}.cwl" + case "id" | "outputSource" if value.startswith("#" + doc_id): + document[key] = value[len(doc_id) + 2 :] + case "out" if isinstance(value, list): + document[key][:] = [ + rewrite_id(entry, this_id) for entry in value + ] + case "source" | "scatter" | "items" | "format": + if ( + isinstance(value, str) + and value.startswith("#") + and "/" in value + ): + referrant_file, sub = value[1:].split("/", 1) + if referrant_file == doc_id: + document[key] = sub else: - new_sources.append(entry) - document[key] = new_sources - elif key == "$import": - rewrite_import(document) - elif key == "class" and value == "SchemaDefRequirement": - return rewrite_schemadef(document, output_dir, pretty) - else: - imports.update(rewrite(value, doc_id, output_dir, pretty)) + document[key] = f"{referrant_file}#{sub}" + elif isinstance(value, list): + new_sources = list() + for entry in value: + if entry.startswith("#" + doc_id): + new_sources.append(entry[len(doc_id) + 2 :]) + else: + new_sources.append(entry) + document[key] = new_sources + case "$import": + rewrite_import(document) + case "class" if value == "SchemaDefRequirement": + return rewrite_schemadef(document, output_dir, pretty) + case _: + imports.update(rewrite(value, doc_id, output_dir, pretty)) return imports diff --git a/cwl_utils/inputs_schema_gen.py b/cwl_utils/inputs_schema_gen.py index 3836f169..deba943c 100644 --- a/cwl_utils/inputs_schema_gen.py +++ b/cwl_utils/inputs_schema_gen.py @@ -21,13 +21,14 @@ Directory, File, InputArraySchema, - InputArraySchemaTypes, InputEnumSchema, - InputEnumSchemaTypes, InputRecordSchema, InputRecordSchemaTypes, Workflow, WorkflowInputParameter, + cwl_v1_0, + cwl_v1_1, + cwl_v1_2, load_document_by_uri, ) from cwl_utils.utils import ( @@ -124,74 +125,90 @@ def generate_type_dict_from_type(self, type_item: Any) -> dict[str, Any]: """ # Primitive types should have a 1-1 mapping # Between an CWL Input Parameter type and a JSON schema type - if isinstance(type_item, str): - if type_item in PRIMITIVE_TYPES_MAPPING.keys(): - return {"type": PRIMITIVE_TYPES_MAPPING[type_item]} - elif type_item in ["stdin"]: + + match type_item: + case str() as key if key in PRIMITIVE_TYPES_MAPPING.keys(): + return {"type": PRIMITIVE_TYPES_MAPPING[key]} + case "stdin": return {"$ref": "#/definitions/File"} - elif type_item in ["File", "Directory", "Any"]: + case "File" | "Directory" | "Any": return {"$ref": f"#/definitions/{type_item}"} # When item is a record schema type - elif is_uri(type_item): + case str() if is_uri(type_item): return { "$ref": f"#/definitions/{to_pascal_case(get_value_from_uri(type_item))}" } - else: + case str(): raise ValueError(f"Unknown type: {type_item}") - elif isinstance(type_item, InputArraySchemaTypes): - return { - "type": "array", - "items": self.generate_type_dict_from_type(type_item.items), - } - elif isinstance(type_item, InputEnumSchemaTypes): - return { - "type": "string", - "enum": list( - map( - lambda symbol_iter: get_value_from_uri(symbol_iter), - type_item.symbols, - ) - ), - } - elif isinstance(type_item, InputRecordSchemaTypes): - if type_item.fields is None: - return {"type": "object"} - if not isinstance(type_item.fields, list): - _cwlutilslogger.error( - "Expected fields of InputRecordSchemaType to be a list" - ) - raise TypeError - return { - "type": "object", - "properties": { - get_value_from_uri(prop.name): self.generate_type_dict_from_type( - prop.type_ - ) - for prop in type_item.fields - }, - } - elif isinstance(type_item, dict): - # Nested import - # {'$import': '../relative/path/to/schema'} - if "$import" in type_item.keys(): + case ( + cwl_v1_0.InputArraySchema() + | cwl_v1_1.InputArraySchema() + | cwl_v1_2.InputArraySchema() + ): + return { + "type": "array", + "items": self.generate_type_dict_from_type(type_item.items), + } + case ( + cwl_v1_0.InputEnumSchema() + | cwl_v1_1.InputEnumSchema() + | cwl_v1_2.InputEnumSchema() + ): + return { + "type": "string", + "enum": list( + map( + lambda symbol_iter: get_value_from_uri(symbol_iter), + type_item.symbols, + ) + ), + } + case ( + cwl_v1_0.InputRecordSchema(fields=f) + | cwl_v1_1.InputRecordSchema(fields=f) + | cwl_v1_2.InputRecordSchema(fields=f) + ): + match f: + case None: + return {"type": "object"} + case list() as fields: + return { + "type": "object", + "properties": { + get_value_from_uri( + prop.name + ): self.generate_type_dict_from_type(prop.type_) + for prop in fields + }, + } + case _: + _cwlutilslogger.error( + "Expected fields of InputRecordSchemaType to be a list" + ) + raise TypeError + case {"$import": uri}: # This path is a relative path to import return { - "$ref": f"#/definitions/{to_pascal_case(get_value_from_uri(type_item['$import']))}" + "$ref": f"#/definitions/{to_pascal_case(get_value_from_uri(uri))}" } - else: + # Nested import + # {'$import': '../relative/path/to/schema'} + case dict(): raise ValueError(f"Unknown type: {type_item}") - elif isinstance(type_item, list): - # Nested schema - return { - "oneOf": list( - map( - lambda type_iter: self.generate_type_dict_from_type(type_iter), - type_item, + case list(): + # Nested schema + return { + "oneOf": list( + map( + lambda type_iter: self.generate_type_dict_from_type( + type_iter + ), + type_item, + ) ) - ) - } - else: - raise ValueError(f"Unknown type: {type_item}") + } + case _: + raise ValueError(f"Unknown type: {type_item}") def generate_type_dict_from_type_list( self, type_: list[InputType] diff --git a/cwl_utils/parser/__init__.py b/cwl_utils/parser/__init__.py index 384bd728..8d99f5b1 100644 --- a/cwl_utils/parser/__init__.py +++ b/cwl_utils/parser/__init__.py @@ -273,24 +273,25 @@ def load_document_by_uri( base_uri = path.resolve().parent.as_uri() id_ = path.resolve().name.split("#")[1] if "#" in path.resolve().name else None - if isinstance(loadingOptions, cwl_v1_0.LoadingOptions): - loadingOptions = cwl_v1_0.LoadingOptions( - fileuri=real_uri, baseuri=base_uri, copyfrom=loadingOptions - ) - elif isinstance(loadingOptions, cwl_v1_1.LoadingOptions): - loadingOptions = cwl_v1_1.LoadingOptions( - fileuri=real_uri, baseuri=base_uri, copyfrom=loadingOptions - ) - elif isinstance(loadingOptions, cwl_v1_2.LoadingOptions): - loadingOptions = cwl_v1_2.LoadingOptions( - fileuri=real_uri, baseuri=base_uri, copyfrom=loadingOptions - ) - elif loadingOptions is None: - loadingOptions = cwl_v1_2.LoadingOptions(fileuri=real_uri, baseuri=base_uri) - else: - raise ValidationException( - f"Unsupported loadingOptions type: {type(loadingOptions)}" - ) + match loadingOptions: + case cwl_v1_0.LoadingOptions(): + loadingOptions = cwl_v1_0.LoadingOptions( + fileuri=real_uri, baseuri=base_uri, copyfrom=loadingOptions + ) + case cwl_v1_1.LoadingOptions(): + loadingOptions = cwl_v1_1.LoadingOptions( + fileuri=real_uri, baseuri=base_uri, copyfrom=loadingOptions + ) + case cwl_v1_2.LoadingOptions(): + loadingOptions = cwl_v1_2.LoadingOptions( + fileuri=real_uri, baseuri=base_uri, copyfrom=loadingOptions + ) + case None: + loadingOptions = cwl_v1_2.LoadingOptions(fileuri=real_uri, baseuri=base_uri) + case _: + raise ValidationException( + f"Unsupported loadingOptions type: {type(loadingOptions)}" + ) doc = loadingOptions.fetcher.fetch_text(real_uri) return load_document_by_string(doc, real_uri, loadingOptions, id_, load_all) @@ -336,24 +337,25 @@ def load_document_by_yaml( if "$graph" in yaml and not load_all: yaml = _get_id_from_graph(yaml, id_) yaml["cwlVersion"] = version - if version == "v1.0": - result = cwl_v1_0.load_document_by_yaml( - yaml, uri, cast(Optional[cwl_v1_0.LoadingOptions], loadingOptions) - ) - elif version == "v1.1": - result = cwl_v1_1.load_document_by_yaml( - yaml, uri, cast(Optional[cwl_v1_1.LoadingOptions], loadingOptions) - ) - elif version == "v1.2": - result = cwl_v1_2.load_document_by_yaml( - yaml, uri, cast(Optional[cwl_v1_2.LoadingOptions], loadingOptions) - ) - elif version is None: - raise ValidationException("could not get the cwlVersion") - else: - raise ValidationException( - f"Version error. Did not recognise {version} as a CWL version" - ) + match version: + case "v1.0": + result = cwl_v1_0.load_document_by_yaml( + yaml, uri, cast(Optional[cwl_v1_0.LoadingOptions], loadingOptions) + ) + case "v1.1": + result = cwl_v1_1.load_document_by_yaml( + yaml, uri, cast(Optional[cwl_v1_1.LoadingOptions], loadingOptions) + ) + case "v1.2": + result = cwl_v1_2.load_document_by_yaml( + yaml, uri, cast(Optional[cwl_v1_2.LoadingOptions], loadingOptions) + ) + case None: + raise ValidationException("could not get the cwlVersion") + case _: + raise ValidationException( + f"Version error. Did not recognise {version} as a CWL version" + ) if isinstance(result, MutableSequence): lst = [] @@ -372,33 +374,30 @@ def save( relative_uris: bool = True, ) -> Any: """Convert a CWL Python object into a JSON/YAML serializable object.""" - if ( - isinstance(val, cwl_v1_0.Saveable) - or isinstance(val, cwl_v1_1.Saveable) - or isinstance(val, cwl_v1_2.Saveable) - ): - return val.save(top=top, base_url=base_url, relative_uris=relative_uris) - if isinstance(val, MutableSequence): - lst = [ - save(v, top=top, base_url=base_url, relative_uris=relative_uris) - for v in val - ] - if top and all(is_process(v) for v in val): - vers = [ - e.get("cwlVersion") for i, e in enumerate(lst) if is_process(val[i]) + match val: + case cwl_v1_0.Saveable() | cwl_v1_1.Saveable() | cwl_v1_2.Saveable(): + return val.save(top=top, base_url=base_url, relative_uris=relative_uris) + case MutableSequence(): + lst = [ + save(v, top=top, base_url=base_url, relative_uris=relative_uris) + for v in val ] - latest = max( - (v for v in vers if v is not None), key=cast(Any, version_split) - ) - return {"cwlVersion": latest, "$graph": lst} - return lst - if isinstance(val, MutableMapping): - newdict = {} - for key in val: - newdict[key] = save( - val[key], top=False, base_url=base_url, relative_uris=relative_uris - ) - return newdict + if top and all(is_process(v) for v in val): + vers = [ + e.get("cwlVersion") for i, e in enumerate(lst) if is_process(val[i]) + ] + latest = max( + (v for v in vers if v is not None), key=cast(Any, version_split) + ) + return {"cwlVersion": latest, "$graph": lst} + return lst + case MutableMapping(): + newdict = {} + for key in val: + newdict[key] = save( + val[key], top=False, base_url=base_url, relative_uris=relative_uris + ) + return newdict return val diff --git a/cwl_utils/parser/cwl_v1_0_utils.py b/cwl_utils/parser/cwl_v1_0_utils.py index 62cf9956..050e1960 100644 --- a/cwl_utils/parser/cwl_v1_0_utils.py +++ b/cwl_utils/parser/cwl_v1_0_utils.py @@ -63,27 +63,28 @@ def _compare_records( def _compare_type(type1: Any, type2: Any) -> bool: - if isinstance(type1, cwl.ArraySchema) and isinstance(type2, cwl.ArraySchema): - return _compare_type(type1.items, type2.items) - elif isinstance(type1, cwl.RecordSchema) and isinstance(type2, cwl.RecordSchema): - fields1 = { - cwl.shortname(field.name): field.type_ for field in (type1.fields or {}) - } - fields2 = { - cwl.shortname(field.name): field.type_ for field in (type2.fields or {}) - } - if fields1.keys() != fields2.keys(): - return False - return all(_compare_type(fields1[k], fields2[k]) for k in fields1.keys()) - elif isinstance(type1, MutableSequence) and isinstance(type2, MutableSequence): - if len(type1) != len(type2): - return False - for t1 in type1: - if not any(_compare_type(t1, t2) for t2 in type2): + match (type1, type1): + case cwl.ArraySchema() as t1, cwl.ArraySchema() as t2: + return _compare_type(t1.items, t2.items) + case cwl.RecordSchema(), cwl.RecordSchema(): + fields1 = { + cwl.shortname(field.name): field.type_ for field in (type1.fields or {}) + } + fields2 = { + cwl.shortname(field.name): field.type_ for field in (type2.fields or {}) + } + if fields1.keys() != fields2.keys(): return False - return True - else: - return bool(type1 == type2) + return all(_compare_type(fields1[k], fields2[k]) for k in fields1.keys()) + case MutableSequence(), MutableSequence(): + if len(type1) != len(type2): + return False + for t1 in type1: + if not any(_compare_type(t1, t2) for t2 in type2): + return False + return True + case _: + return bool(type1 == type2) def _inputfile_load( @@ -93,55 +94,59 @@ def _inputfile_load( addl_metadata_fields: MutableSequence[str] | None = None, ) -> tuple[Any, cwl.LoadingOptions]: loader = cwl.CWLInputFileLoader - if isinstance(doc, str): - url = loadingOptions.fetcher.urljoin(baseuri, doc) - if url in loadingOptions.idx: + match doc: + case str(): + url = loadingOptions.fetcher.urljoin(baseuri, doc) + if url in loadingOptions.idx: + return loadingOptions.idx[url] + doc_url, frg = urldefrag(url) + text = loadingOptions.fetcher.fetch_text(doc_url) + textIO = StringIO(text) + textIO.name = str(doc_url) + yaml = yaml_no_ts() + result = yaml.load(textIO) + add_lc_filename(result, doc_url) + loadingOptions = cwl.LoadingOptions( + copyfrom=loadingOptions, fileuri=doc_url + ) + _inputfile_load( + result, + doc_url, + loadingOptions, + ) return loadingOptions.idx[url] - doc_url, frg = urldefrag(url) - text = loadingOptions.fetcher.fetch_text(doc_url) - textIO = StringIO(text) - textIO.name = str(doc_url) - yaml = yaml_no_ts() - result = yaml.load(textIO) - add_lc_filename(result, doc_url) - loadingOptions = cwl.LoadingOptions(copyfrom=loadingOptions, fileuri=doc_url) - _inputfile_load( - result, - doc_url, - loadingOptions, - ) - return loadingOptions.idx[url] - - if isinstance(doc, MutableMapping): - addl_metadata = {} - if addl_metadata_fields is not None: - for mf in addl_metadata_fields: - if mf in doc: - addl_metadata[mf] = doc[mf] - - loadingOptions = cwl.LoadingOptions( - copyfrom=loadingOptions, - baseuri=baseuri, - addl_metadata=addl_metadata, - ) + case MutableMapping(): + addl_metadata = {} + if addl_metadata_fields is not None: + for mf in addl_metadata_fields: + if mf in doc: + addl_metadata[mf] = doc[mf] + + loadingOptions = cwl.LoadingOptions( + copyfrom=loadingOptions, + baseuri=baseuri, + addl_metadata=addl_metadata, + ) - loadingOptions.idx[baseuri] = ( - loader.load(doc, baseuri, loadingOptions, docRoot=baseuri), - loadingOptions, - ) + loadingOptions.idx[baseuri] = ( + loader.load(doc, baseuri, loadingOptions, docRoot=baseuri), + loadingOptions, + ) - return loadingOptions.idx[baseuri] + return loadingOptions.idx[baseuri] - if isinstance(doc, MutableSequence): - loadingOptions.idx[baseuri] = ( - loader.load(doc, baseuri, loadingOptions), - loadingOptions, - ) - return loadingOptions.idx[baseuri] + case MutableSequence(): + loadingOptions.idx[baseuri] = ( + loader.load(doc, baseuri, loadingOptions), + loadingOptions, + ) + return loadingOptions.idx[baseuri] - raise ValidationException( - "Expected URI string, MutableMapping or MutableSequence, got %s" % type(doc) - ) + case _: + raise ValidationException( + "Expected URI string, MutableMapping or MutableSequence, got %s" + % type(doc) + ) def can_assign_src_to_sink(src: Any, sink: Any, strict: bool = False) -> bool: @@ -187,14 +192,15 @@ def check_all_types( """Given a list of sinks, check if their types match with the types of their sources.""" validation: dict[str, list[SrcSink]] = {"warning": [], "exception": []} for sink in sinks: - if isinstance(sink, cwl.WorkflowOutputParameter): - sourceName = "outputSource" - sourceField = sink.outputSource - elif isinstance(sink, cwl.WorkflowStepInput): - sourceName = "source" - sourceField = sink.source - else: - continue + match sink: + case cwl.WorkflowOutputParameter(): + sourceName = "outputSource" + sourceField = sink.outputSource + case cwl.WorkflowStepInput(): + sourceName = "source" + sourceField = sink.source + case _: + continue if sourceField is not None: if isinstance(sourceField, MutableSequence): linkMerge = sink.linkMerge or ( @@ -218,10 +224,8 @@ def check_all_types( linkMerge, getattr(sink, "valueFrom", None), ) - if check_result == "warning": - validation["warning"].append(SrcSink(src, sink, linkMerge, None)) - elif check_result == "exception": - validation["exception"].append(SrcSink(src, sink, linkMerge, None)) + if check_result in ("warning", "exception"): + validation[check_result].append(SrcSink(src, sink, linkMerge, None)) return validation diff --git a/cwl_utils/parser/cwl_v1_1_utils.py b/cwl_utils/parser/cwl_v1_1_utils.py index cb3f3304..96fd88f8 100644 --- a/cwl_utils/parser/cwl_v1_1_utils.py +++ b/cwl_utils/parser/cwl_v1_1_utils.py @@ -63,27 +63,28 @@ def _compare_records( def _compare_type(type1: Any, type2: Any) -> bool: - if isinstance(type1, cwl.ArraySchema) and isinstance(type2, cwl.ArraySchema): - return _compare_type(type1.items, type2.items) - elif isinstance(type1, cwl.RecordSchema) and isinstance(type2, cwl.RecordSchema): - fields1 = { - cwl.shortname(field.name): field.type_ for field in (type1.fields or {}) - } - fields2 = { - cwl.shortname(field.name): field.type_ for field in (type2.fields or {}) - } - if fields1.keys() != fields2.keys(): - return False - return all(_compare_type(fields1[k], fields2[k]) for k in fields1.keys()) - elif isinstance(type1, MutableSequence) and isinstance(type2, MutableSequence): - if len(type1) != len(type2): - return False - for t1 in type1: - if not any(_compare_type(t1, t2) for t2 in type2): + match (type1, type1): + case cwl.ArraySchema() as t1, cwl.ArraySchema() as t2: + return _compare_type(t1.items, t2.items) + case cwl.RecordSchema(), cwl.RecordSchema(): + fields1 = { + cwl.shortname(field.name): field.type_ for field in (type1.fields or {}) + } + fields2 = { + cwl.shortname(field.name): field.type_ for field in (type2.fields or {}) + } + if fields1.keys() != fields2.keys(): return False - return True - else: - return bool(type1 == type2) + return all(_compare_type(fields1[k], fields2[k]) for k in fields1.keys()) + case MutableSequence(), MutableSequence(): + if len(type1) != len(type2): + return False + for t1 in type1: + if not any(_compare_type(t1, t2) for t2 in type2): + return False + return True + case _: + return bool(type1 == type2) def _inputfile_load( @@ -93,55 +94,59 @@ def _inputfile_load( addl_metadata_fields: MutableSequence[str] | None = None, ) -> tuple[Any, cwl.LoadingOptions]: loader = cwl.CWLInputFileLoader - if isinstance(doc, str): - url = loadingOptions.fetcher.urljoin(baseuri, doc) - if url in loadingOptions.idx: + match doc: + case str(): + url = loadingOptions.fetcher.urljoin(baseuri, doc) + if url in loadingOptions.idx: + return loadingOptions.idx[url] + doc_url, frg = urldefrag(url) + text = loadingOptions.fetcher.fetch_text(doc_url) + textIO = StringIO(text) + textIO.name = str(doc_url) + yaml = yaml_no_ts() + result = yaml.load(textIO) + add_lc_filename(result, doc_url) + loadingOptions = cwl.LoadingOptions( + copyfrom=loadingOptions, fileuri=doc_url + ) + _inputfile_load( + result, + doc_url, + loadingOptions, + ) return loadingOptions.idx[url] - doc_url, frg = urldefrag(url) - text = loadingOptions.fetcher.fetch_text(doc_url) - textIO = StringIO(text) - textIO.name = str(doc_url) - yaml = yaml_no_ts() - result = yaml.load(textIO) - add_lc_filename(result, doc_url) - loadingOptions = cwl.LoadingOptions(copyfrom=loadingOptions, fileuri=doc_url) - _inputfile_load( - result, - doc_url, - loadingOptions, - ) - return loadingOptions.idx[url] - - if isinstance(doc, MutableMapping): - addl_metadata = {} - if addl_metadata_fields is not None: - for mf in addl_metadata_fields: - if mf in doc: - addl_metadata[mf] = doc[mf] - - loadingOptions = cwl.LoadingOptions( - copyfrom=loadingOptions, - baseuri=baseuri, - addl_metadata=addl_metadata, - ) + case MutableMapping(): + addl_metadata = {} + if addl_metadata_fields is not None: + for mf in addl_metadata_fields: + if mf in doc: + addl_metadata[mf] = doc[mf] + + loadingOptions = cwl.LoadingOptions( + copyfrom=loadingOptions, + baseuri=baseuri, + addl_metadata=addl_metadata, + ) - loadingOptions.idx[baseuri] = ( - loader.load(doc, baseuri, loadingOptions, docRoot=baseuri), - loadingOptions, - ) + loadingOptions.idx[baseuri] = ( + loader.load(doc, baseuri, loadingOptions, docRoot=baseuri), + loadingOptions, + ) - return loadingOptions.idx[baseuri] + return loadingOptions.idx[baseuri] - if isinstance(doc, MutableSequence): - loadingOptions.idx[baseuri] = ( - loader.load(doc, baseuri, loadingOptions), - loadingOptions, - ) - return loadingOptions.idx[baseuri] + case MutableSequence(): + loadingOptions.idx[baseuri] = ( + loader.load(doc, baseuri, loadingOptions), + loadingOptions, + ) + return loadingOptions.idx[baseuri] - raise ValidationException( - "Expected URI string, MutableMapping or MutableSequence, got %s" % type(doc) - ) + case _: + raise ValidationException( + "Expected URI string, MutableMapping or MutableSequence, got %s" + % type(doc) + ) def can_assign_src_to_sink(src: Any, sink: Any, strict: bool = False) -> bool: @@ -187,14 +192,15 @@ def check_all_types( """Given a list of sinks, check if their types match with the types of their sources.""" validation: dict[str, list[SrcSink]] = {"warning": [], "exception": []} for sink in sinks: - if isinstance(sink, cwl.WorkflowOutputParameter): - sourceName = "outputSource" - sourceField = sink.outputSource - elif isinstance(sink, cwl.WorkflowStepInput): - sourceName = "source" - sourceField = sink.source - else: - continue + match sink: + case cwl.WorkflowOutputParameter(): + sourceName = "outputSource" + sourceField = sink.outputSource + case cwl.WorkflowStepInput(): + sourceName = "source" + sourceField = sink.source + case _: + continue if sourceField is not None: if isinstance(sourceField, MutableSequence): linkMerge = sink.linkMerge or ( @@ -218,10 +224,8 @@ def check_all_types( linkMerge, getattr(sink, "valueFrom", None), ) - if check_result == "warning": - validation["warning"].append(SrcSink(src, sink, linkMerge, None)) - elif check_result == "exception": - validation["exception"].append(SrcSink(src, sink, linkMerge, None)) + if check_result in ("warning", "exception"): + validation[check_result].append(SrcSink(src, sink, linkMerge, None)) return validation diff --git a/cwl_utils/parser/cwl_v1_2_utils.py b/cwl_utils/parser/cwl_v1_2_utils.py index 5cfcc1da..36928969 100644 --- a/cwl_utils/parser/cwl_v1_2_utils.py +++ b/cwl_utils/parser/cwl_v1_2_utils.py @@ -63,27 +63,28 @@ def _compare_records( def _compare_type(type1: Any, type2: Any) -> bool: - if isinstance(type1, cwl.ArraySchema) and isinstance(type2, cwl.ArraySchema): - return _compare_type(type1.items, type2.items) - elif isinstance(type1, cwl.RecordSchema) and isinstance(type2, cwl.RecordSchema): - fields1 = { - cwl.shortname(field.name): field.type_ for field in (type1.fields or {}) - } - fields2 = { - cwl.shortname(field.name): field.type_ for field in (type2.fields or {}) - } - if fields1.keys() != fields2.keys(): - return False - return all(_compare_type(fields1[k], fields2[k]) for k in fields1.keys()) - elif isinstance(type1, MutableSequence) and isinstance(type2, MutableSequence): - if len(type1) != len(type2): - return False - for t1 in type1: - if not any(_compare_type(t1, t2) for t2 in type2): + match (type1, type1): + case cwl.ArraySchema() as t1, cwl.ArraySchema() as t2: + return _compare_type(t1.items, t2.items) + case cwl.RecordSchema(), cwl.RecordSchema(): + fields1 = { + cwl.shortname(field.name): field.type_ for field in (type1.fields or {}) + } + fields2 = { + cwl.shortname(field.name): field.type_ for field in (type2.fields or {}) + } + if fields1.keys() != fields2.keys(): return False - return True - else: - return bool(type1 == type2) + return all(_compare_type(fields1[k], fields2[k]) for k in fields1.keys()) + case MutableSequence(), MutableSequence(): + if len(type1) != len(type2): + return False + for t1 in type1: + if not any(_compare_type(t1, t2) for t2 in type2): + return False + return True + case _: + return bool(type1 == type2) def _is_all_output_method_loop_step( @@ -112,55 +113,59 @@ def _inputfile_load( addl_metadata_fields: MutableSequence[str] | None = None, ) -> tuple[Any, cwl.LoadingOptions]: loader = cwl.CWLInputFileLoader - if isinstance(doc, str): - url = loadingOptions.fetcher.urljoin(baseuri, doc) - if url in loadingOptions.idx: + match doc: + case str(): + url = loadingOptions.fetcher.urljoin(baseuri, doc) + if url in loadingOptions.idx: + return loadingOptions.idx[url] + doc_url, frg = urldefrag(url) + text = loadingOptions.fetcher.fetch_text(doc_url) + textIO = StringIO(text) + textIO.name = str(doc_url) + yaml = yaml_no_ts() + result = yaml.load(textIO) + add_lc_filename(result, doc_url) + loadingOptions = cwl.LoadingOptions( + copyfrom=loadingOptions, fileuri=doc_url + ) + _inputfile_load( + result, + doc_url, + loadingOptions, + ) return loadingOptions.idx[url] - doc_url, frg = urldefrag(url) - text = loadingOptions.fetcher.fetch_text(doc_url) - textIO = StringIO(text) - textIO.name = str(doc_url) - yaml = yaml_no_ts() - result = yaml.load(textIO) - add_lc_filename(result, doc_url) - loadingOptions = cwl.LoadingOptions(copyfrom=loadingOptions, fileuri=doc_url) - _inputfile_load( - result, - doc_url, - loadingOptions, - ) - return loadingOptions.idx[url] - - if isinstance(doc, MutableMapping): - addl_metadata = {} - if addl_metadata_fields is not None: - for mf in addl_metadata_fields: - if mf in doc: - addl_metadata[mf] = doc[mf] - - loadingOptions = cwl.LoadingOptions( - copyfrom=loadingOptions, - baseuri=baseuri, - addl_metadata=addl_metadata, - ) + case MutableMapping(): + addl_metadata = {} + if addl_metadata_fields is not None: + for mf in addl_metadata_fields: + if mf in doc: + addl_metadata[mf] = doc[mf] + + loadingOptions = cwl.LoadingOptions( + copyfrom=loadingOptions, + baseuri=baseuri, + addl_metadata=addl_metadata, + ) - loadingOptions.idx[baseuri] = ( - loader.load(doc, baseuri, loadingOptions, docRoot=baseuri), - loadingOptions, - ) + loadingOptions.idx[baseuri] = ( + loader.load(doc, baseuri, loadingOptions, docRoot=baseuri), + loadingOptions, + ) - return loadingOptions.idx[baseuri] + return loadingOptions.idx[baseuri] - if isinstance(doc, MutableSequence): - loadingOptions.idx[baseuri] = ( - loader.load(doc, baseuri, loadingOptions), - loadingOptions, - ) - return loadingOptions.idx[baseuri] + case MutableSequence(): + loadingOptions.idx[baseuri] = ( + loader.load(doc, baseuri, loadingOptions), + loadingOptions, + ) + return loadingOptions.idx[baseuri] - raise ValidationException( - "Expected URI string, MutableMapping or MutableSequence, got %s" % type(doc) - ) + case _: + raise ValidationException( + "Expected URI string, MutableMapping or MutableSequence, got %s" + % type(doc) + ) def can_assign_src_to_sink(src: Any, sink: Any, strict: bool = False) -> bool: @@ -210,14 +215,15 @@ def check_all_types( extra_message = ( "pickValue is %s" % sink.pickValue if sink.pickValue is not None else None ) - if isinstance(sink, cwl.WorkflowOutputParameter): - sourceName = "outputSource" - sourceField = sink.outputSource - elif isinstance(sink, cwl.WorkflowStepInput): - sourceName = "source" - sourceField = sink.source - else: - continue + match sink: + case cwl.WorkflowOutputParameter(): + sourceName = "outputSource" + sourceField = sink.outputSource + case cwl.WorkflowStepInput(): + sourceName = "source" + sourceField = sink.source + case _: + continue if sourceField is not None: if isinstance(sourceField, MutableSequence): linkMerge = sink.linkMerge or ( @@ -292,12 +298,8 @@ def check_all_types( linkMerge, getattr(sink, "valueFrom", None), ) - if check_result == "warning": - validation["warning"].append( - SrcSink(src, sink, linkMerge, extra_message) - ) - elif check_result == "exception": - validation["exception"].append( + if check_result in ("warning", "exception"): + validation[check_result].append( SrcSink(src, sink, linkMerge, extra_message) ) return validation diff --git a/cwl_utils/parser/utils.py b/cwl_utils/parser/utils.py index b51f1e64..70c5f82d 100644 --- a/cwl_utils/parser/utils.py +++ b/cwl_utils/parser/utils.py @@ -35,12 +35,13 @@ def convert_stdstreams_to_files(process: Process) -> None: """Convert stdin, stdout and stderr type shortcuts to files.""" - if isinstance(process, cwl_v1_0.CommandLineTool): - cwl_v1_0_utils.convert_stdstreams_to_files(process) - elif isinstance(process, cwl_v1_1.CommandLineTool): - cwl_v1_1_utils.convert_stdstreams_to_files(process) - elif isinstance(process, cwl_v1_2.CommandLineTool): - cwl_v1_2_utils.convert_stdstreams_to_files(process) + match process: + case cwl_v1_0.CommandLineTool(): + cwl_v1_0_utils.convert_stdstreams_to_files(process) + case cwl_v1_1.CommandLineTool(): + cwl_v1_1_utils.convert_stdstreams_to_files(process) + case cwl_v1_2.CommandLineTool(): + cwl_v1_2_utils.convert_stdstreams_to_files(process) def load_inputfile_by_uri( @@ -64,16 +65,17 @@ def load_inputfile_by_uri( baseuri = str(real_path) if loadingOptions is None: - if version == "v1.0": - loadingOptions = cwl_v1_0.LoadingOptions(fileuri=baseuri) - elif version == "v1.1": - loadingOptions = cwl_v1_1.LoadingOptions(fileuri=baseuri) - elif version == "v1.2": - loadingOptions = cwl_v1_2.LoadingOptions(fileuri=baseuri) - else: - raise ValidationException( - f"Version error. Did not recognise {version} as a CWL version" - ) + match version: + case "v1.0": + loadingOptions = cwl_v1_0.LoadingOptions(fileuri=baseuri) + case "v1.1": + loadingOptions = cwl_v1_1.LoadingOptions(fileuri=baseuri) + case "v1.2": + loadingOptions = cwl_v1_2.LoadingOptions(fileuri=baseuri) + case _: + raise ValidationException( + f"Version error. Did not recognise {version} as a CWL version" + ) doc = loadingOptions.fetcher.fetch_text(real_path) return load_inputfile_by_string(version, doc, baseuri, loadingOptions) @@ -112,26 +114,25 @@ def load_inputfile_by_yaml( loadingOptions: LoadingOptions | None = None, ) -> Any: """Load a CWL input file from a YAML object.""" - if version == "v1.0": - result = cwl_v1_0_utils.load_inputfile_by_yaml( - yaml, uri, cast(Optional[cwl_v1_0.LoadingOptions], loadingOptions) - ) - elif version == "v1.1": - result = cwl_v1_1_utils.load_inputfile_by_yaml( - yaml, uri, cast(Optional[cwl_v1_1.LoadingOptions], loadingOptions) - ) - elif version == "v1.2": - result = cwl_v1_2_utils.load_inputfile_by_yaml( - yaml, uri, cast(Optional[cwl_v1_2.LoadingOptions], loadingOptions) - ) - elif version is None: - raise ValidationException("could not get the cwlVersion") - else: - raise ValidationException( - f"Version error. Did not recognise {version} as a CWL version" - ) - - return result + match version: + case "v1.0": + return cwl_v1_0_utils.load_inputfile_by_yaml( + yaml, uri, cast(Optional[cwl_v1_0.LoadingOptions], loadingOptions) + ) + case "v1.1": + return cwl_v1_1_utils.load_inputfile_by_yaml( + yaml, uri, cast(Optional[cwl_v1_1.LoadingOptions], loadingOptions) + ) + case "v1.2": + return cwl_v1_2_utils.load_inputfile_by_yaml( + yaml, uri, cast(Optional[cwl_v1_2.LoadingOptions], loadingOptions) + ) + case None: + raise ValidationException("could not get the cwlVersion") + case _: + raise ValidationException( + f"Version error. Did not recognise {version} as a CWL version" + ) def load_step( @@ -170,23 +171,24 @@ def static_checker(workflow: cwl_utils.parser.Workflow) -> None: ) if step.out is not None: # FIXME: the correct behaviour here would be to create WorkflowStepOutput directly at load time - if workflow.cwlVersion == "v1.0": - step_outs = [ - cwl_v1_0.WorkflowStepOutput(s) if isinstance(s, str) else s - for s in step.out - ] - elif workflow.cwlVersion == "v1.1": - step_outs = [ - cwl_v1_1.WorkflowStepOutput(s) if isinstance(s, str) else s - for s in step.out - ] - elif workflow.cwlVersion == "v1.2": - step_outs = [ - cwl_v1_2.WorkflowStepOutput(s) if isinstance(s, str) else s - for s in step.out - ] - else: - raise Exception(f"Unsupported CWL version {workflow.cwlVersion}") + match workflow.cwlVersion: + case "v1.0": + step_outs = [ + cwl_v1_0.WorkflowStepOutput(s) if isinstance(s, str) else s + for s in step.out + ] + case "v1.1": + step_outs = [ + cwl_v1_1.WorkflowStepOutput(s) if isinstance(s, str) else s + for s in step.out + ] + case "v1.2": + step_outs = [ + cwl_v1_2.WorkflowStepOutput(s) if isinstance(s, str) else s + for s in step.out + ] + case _: + raise Exception(f"Unsupported CWL version {workflow.cwlVersion}") step_outputs.extend(step_outs) param_to_step.update({s.id: step for s in step_outs}) type_dict.update( @@ -208,32 +210,33 @@ def static_checker(workflow: cwl_utils.parser.Workflow) -> None: parser: ModuleType step_inputs_val: dict[str, Any] workflow_outputs_val: dict[str, Any] - if workflow.cwlVersion == "v1.0": - parser = cwl_v1_0 - step_inputs_val = cwl_v1_0_utils.check_all_types( - src_dict, step_inputs, type_dict - ) - workflow_outputs_val = cwl_v1_0_utils.check_all_types( - src_dict, workflow.outputs, type_dict - ) - elif workflow.cwlVersion == "v1.1": - parser = cwl_v1_1 - step_inputs_val = cwl_v1_1_utils.check_all_types( - src_dict, step_inputs, type_dict - ) - workflow_outputs_val = cwl_v1_1_utils.check_all_types( - src_dict, workflow.outputs, type_dict - ) - elif workflow.cwlVersion == "v1.2": - parser = cwl_v1_2 - step_inputs_val = cwl_v1_2_utils.check_all_types( - src_dict, step_inputs, param_to_step, type_dict - ) - workflow_outputs_val = cwl_v1_2_utils.check_all_types( - src_dict, workflow.outputs, param_to_step, type_dict - ) - else: - raise Exception(f"Unsupported CWL version {workflow.cwlVersion}") + match workflow.cwlVersion: + case "v1.0": + parser = cwl_v1_0 + step_inputs_val = cwl_v1_0_utils.check_all_types( + src_dict, step_inputs, type_dict + ) + workflow_outputs_val = cwl_v1_0_utils.check_all_types( + src_dict, workflow.outputs, type_dict + ) + case "v1.1": + parser = cwl_v1_1 + step_inputs_val = cwl_v1_1_utils.check_all_types( + src_dict, step_inputs, type_dict + ) + workflow_outputs_val = cwl_v1_1_utils.check_all_types( + src_dict, workflow.outputs, type_dict + ) + case "v1.2": + parser = cwl_v1_2 + step_inputs_val = cwl_v1_2_utils.check_all_types( + src_dict, step_inputs, param_to_step, type_dict + ) + workflow_outputs_val = cwl_v1_2_utils.check_all_types( + src_dict, workflow.outputs, param_to_step, type_dict + ) + case _ as cwlVersion: + raise Exception(f"Unsupported CWL version {cwlVersion}") warnings = step_inputs_val["warning"] + workflow_outputs_val["warning"] exceptions = step_inputs_val["exception"] + workflow_outputs_val["exception"] @@ -335,89 +338,92 @@ def type_for_source( pickValue: str | None = None, ) -> Any: """Determine the type for the given sourcenames.""" - if process.cwlVersion == "v1.0": - return cwl_v1_0_utils.type_for_source( - cast( - Union[ - cwl_v1_0.CommandLineTool, - cwl_v1_0.Workflow, - cwl_v1_0.ExpressionTool, - ], - process, - ), - sourcenames, - cast(Optional[cwl_v1_0.Workflow], parent), - linkMerge, - ) - elif process.cwlVersion == "v1.1": - return cwl_v1_1_utils.type_for_source( - cast( - Union[ - cwl_v1_1.CommandLineTool, - cwl_v1_1.Workflow, - cwl_v1_1.ExpressionTool, - ], - process, - ), - sourcenames, - cast(Optional[cwl_v1_1.Workflow], parent), - linkMerge, - ) - elif process.cwlVersion == "v1.2": - return cwl_v1_2_utils.type_for_source( - cast( - Union[ - cwl_v1_2.CommandLineTool, - cwl_v1_2.Workflow, - cwl_v1_2.ExpressionTool, - ], - process, - ), - sourcenames, - cast(Optional[cwl_v1_2.Workflow], parent), - linkMerge, - pickValue, - ) - elif process.cwlVersion is None: - raise ValidationException("could not get the cwlVersion") - else: - raise ValidationException( - f"Version error. Did not recognise {process.cwlVersion} as a CWL version" - ) + match process.cwlVersion: + case "v1.0": + return cwl_v1_0_utils.type_for_source( + cast( + Union[ + cwl_v1_0.CommandLineTool, + cwl_v1_0.Workflow, + cwl_v1_0.ExpressionTool, + ], + process, + ), + sourcenames, + cast(Optional[cwl_v1_0.Workflow], parent), + linkMerge, + ) + case "v1.1": + return cwl_v1_1_utils.type_for_source( + cast( + Union[ + cwl_v1_1.CommandLineTool, + cwl_v1_1.Workflow, + cwl_v1_1.ExpressionTool, + ], + process, + ), + sourcenames, + cast(Optional[cwl_v1_1.Workflow], parent), + linkMerge, + ) + case "v1.2": + return cwl_v1_2_utils.type_for_source( + cast( + Union[ + cwl_v1_2.CommandLineTool, + cwl_v1_2.Workflow, + cwl_v1_2.ExpressionTool, + ], + process, + ), + sourcenames, + cast(Optional[cwl_v1_2.Workflow], parent), + linkMerge, + pickValue, + ) + case None: + raise ValidationException("could not get the cwlVersion") + case _ as cwlVersion: + raise ValidationException( + f"Version error. Did not recognise {cwlVersion} as a CWL version" + ) def type_for_step_input( step: WorkflowStep, in_: WorkflowStepInput, cwlVersion: str ) -> Any: """Determine the type for the given step output.""" - if cwlVersion == "v1.0": - return cwl_v1_0_utils.type_for_step_input( - cast(cwl_v1_0.WorkflowStep, step), cast(cwl_v1_0.WorkflowStepInput, in_) - ) - elif cwlVersion == "v1.1": - return cwl_v1_1_utils.type_for_step_input( - cast(cwl_v1_1.WorkflowStep, step), cast(cwl_v1_1.WorkflowStepInput, in_) - ) - elif cwlVersion == "v1.2": - return cwl_v1_2_utils.type_for_step_input( - cast(cwl_v1_2.WorkflowStep, step), cast(cwl_v1_2.WorkflowStepInput, in_) - ) + match cwlVersion: + case "v1.0": + return cwl_v1_0_utils.type_for_step_input( + cast(cwl_v1_0.WorkflowStep, step), cast(cwl_v1_0.WorkflowStepInput, in_) + ) + case "v1.1": + return cwl_v1_1_utils.type_for_step_input( + cast(cwl_v1_1.WorkflowStep, step), cast(cwl_v1_1.WorkflowStepInput, in_) + ) + case "v1.2": + return cwl_v1_2_utils.type_for_step_input( + cast(cwl_v1_2.WorkflowStep, step), cast(cwl_v1_2.WorkflowStepInput, in_) + ) def type_for_step_output(step: WorkflowStep, sourcename: str, cwlVersion: str) -> Any: """Determine the type for the given step output.""" - if cwlVersion == "v1.0": - return cwl_v1_0_utils.type_for_step_output( - cast(cwl_v1_0.WorkflowStep, step), sourcename - ) - elif cwlVersion == "v1.1": - return cwl_v1_1_utils.type_for_step_output( - cast(cwl_v1_1.WorkflowStep, step), sourcename - ) - elif cwlVersion == "v1.2": - return cwl_v1_2_utils.type_for_step_output( - cast(cwl_v1_2.WorkflowStep, step), sourcename - ) + match cwlVersion: + case "v1.0": + return cwl_v1_0_utils.type_for_step_output( + cast(cwl_v1_0.WorkflowStep, step), sourcename + ) + case "v1.1": + return cwl_v1_1_utils.type_for_step_output( + cast(cwl_v1_1.WorkflowStep, step), sourcename + ) + case "v1.2": + return cwl_v1_2_utils.type_for_step_output( + cast(cwl_v1_2.WorkflowStep, step), sourcename + ) def param_for_source_id( @@ -443,51 +449,52 @@ def param_for_source_id( | cwl_utils.parser.cwl_v1_2.WorkflowInputParameter ) ): - if process.cwlVersion == "v1.0": - return cwl_utils.parser.cwl_v1_0_utils.param_for_source_id( - cast( - Union[ - cwl_utils.parser.cwl_v1_0.CommandLineTool, - cwl_utils.parser.cwl_v1_0.Workflow, - cwl_utils.parser.cwl_v1_0.ExpressionTool, - ], - process, - ), - sourcenames, - cast(cwl_utils.parser.cwl_v1_0.Workflow, parent), - scatter_context, - ) - elif process.cwlVersion == "v1.1": - return cwl_utils.parser.cwl_v1_1_utils.param_for_source_id( - cast( - Union[ - cwl_utils.parser.cwl_v1_1.CommandLineTool, - cwl_utils.parser.cwl_v1_1.Workflow, - cwl_utils.parser.cwl_v1_1.ExpressionTool, - ], - process, - ), - sourcenames, - cast(cwl_utils.parser.cwl_v1_1.Workflow, parent), - scatter_context, - ) - elif process.cwlVersion == "v1.2": - return cwl_utils.parser.cwl_v1_2_utils.param_for_source_id( - cast( - Union[ - cwl_utils.parser.cwl_v1_2.CommandLineTool, - cwl_utils.parser.cwl_v1_2.Workflow, - cwl_utils.parser.cwl_v1_2.ExpressionTool, - ], - process, - ), - sourcenames, - cast(cwl_utils.parser.cwl_v1_2.Workflow, parent), - scatter_context, - ) - elif process.cwlVersion is None: - raise ValidationException("could not get the cwlVersion") - else: - raise ValidationException( - f"Version error. Did not recognise {process.cwlVersion} as a CWL version" - ) + match process.cwlVersion: + case "v1.0": + return cwl_utils.parser.cwl_v1_0_utils.param_for_source_id( + cast( + Union[ + cwl_utils.parser.cwl_v1_0.CommandLineTool, + cwl_utils.parser.cwl_v1_0.Workflow, + cwl_utils.parser.cwl_v1_0.ExpressionTool, + ], + process, + ), + sourcenames, + cast(cwl_utils.parser.cwl_v1_0.Workflow, parent), + scatter_context, + ) + case "v1.1": + return cwl_utils.parser.cwl_v1_1_utils.param_for_source_id( + cast( + Union[ + cwl_utils.parser.cwl_v1_1.CommandLineTool, + cwl_utils.parser.cwl_v1_1.Workflow, + cwl_utils.parser.cwl_v1_1.ExpressionTool, + ], + process, + ), + sourcenames, + cast(cwl_utils.parser.cwl_v1_1.Workflow, parent), + scatter_context, + ) + case "v1.2": + return cwl_utils.parser.cwl_v1_2_utils.param_for_source_id( + cast( + Union[ + cwl_utils.parser.cwl_v1_2.CommandLineTool, + cwl_utils.parser.cwl_v1_2.Workflow, + cwl_utils.parser.cwl_v1_2.ExpressionTool, + ], + process, + ), + sourcenames, + cast(cwl_utils.parser.cwl_v1_2.Workflow, parent), + scatter_context, + ) + case None: + raise ValidationException("could not get the cwlVersion") + case _: + raise ValidationException( + f"Version error. Did not recognise {process.cwlVersion} as a CWL version" + ) diff --git a/cwl_utils/sandboxjs.py b/cwl_utils/sandboxjs.py index e4175711..83ad3b9a 100644 --- a/cwl_utils/sandboxjs.py +++ b/cwl_utils/sandboxjs.py @@ -309,39 +309,40 @@ def new_js_proc( if not self.have_node_slim: singularity_cache: str | None = None - if container_engine in ("docker", "podman"): - dockerimgs = subprocess.check_output( # nosec - [container_engine, "images", "-q", nodeimg], - text=True, - ) - elif container_engine == "singularity": - singularity_cache = os.environ.get("CWL_SINGULARITY_CACHE") - if singularity_cache: - singularityimgs = glob.glob( - singularity_cache + "/node_alpine.sif" + match container_engine: + case "docker" | "podman": + dockerimgs = subprocess.check_output( # nosec + [container_engine, "images", "-q", nodeimg], + text=True, ) - else: - singularityimgs = glob.glob( - os.getcwd() + "/node_alpine.sif" + case "singularity": + singularity_cache = os.environ.get("CWL_SINGULARITY_CACHE") + if singularity_cache: + singularityimgs = glob.glob( + singularity_cache + "/node_alpine.sif" + ) + else: + singularityimgs = glob.glob( + os.getcwd() + "/node_alpine.sif" + ) + if singularityimgs: + nodeimg = singularityimgs[0] + case "udocker": + matches = re.search( + re.escape(nodeimg), + subprocess.check_output( # nosec + [container_engine, "images"], + text=True, + ), + ) + if matches: + dockerimgs = matches[0] + else: + dockerimgs = "" + case _: + raise Exception( + f"Unknown container_engine: {container_engine}." ) - if singularityimgs: - nodeimg = singularityimgs[0] - elif container_engine == "udocker": - matches = re.search( - re.escape(nodeimg), - subprocess.check_output( # nosec - [container_engine, "images"], - text=True, - ), - ) - if matches: - dockerimgs = matches[0] - else: - dockerimgs = "" - else: - raise Exception( - f"Unknown container_engine: {container_engine}." - ) # if output is an empty string need_singularity = ( container_engine == "singularity" and not singularityimgs diff --git a/cwl_utils/utils.py b/cwl_utils/utils.py index 83e2bfc0..86b68dfc 100644 --- a/cwl_utils/utils.py +++ b/cwl_utils/utils.py @@ -21,7 +21,7 @@ from cwl_utils.loghandler import _logger # Type hinting -from cwl_utils.parser import InputRecordSchemaTypes +from cwl_utils.parser import cwl_v1_0, cwl_v1_1, cwl_v1_2 # Load as 1.2 files from cwl_utils.parser.cwl_v1_2 import InputArraySchema as InputArraySchemaV1_2 @@ -338,61 +338,59 @@ def sanitise_schema_field( # Copy schema field schema_field_item = deepcopy(schema_field_item) required = True - - if isinstance(schema_field_item, InputRecordSchemaTypes): - return schema_field_item - - if isinstance(schema_field_item.get("type"), list): - if "null" in schema_field_item.get("type", []): - required = False - schema_field_item["type"] = list( - filter( - lambda type_item: type_item != "null", schema_field_item.get("type", []) - ) - ) - if len(schema_field_item["type"]) == 1: - schema_field_item["type"] = schema_field_item["type"][0] - else: - # Recursively get items + match schema_field_item: + case ( + cwl_v1_0.InputRecordSchema() + | cwl_v1_1.InputRecordSchema() + | cwl_v1_2.InputRecordSchema() + ): + return schema_field_item + case {"type": list() as field_item_type}: + if "null" in field_item_type: + required = False schema_field_item["type"] = list( - map( - lambda field_subtypes: sanitise_schema_field(field_subtypes), - schema_field_item.get("type", []), - ) + filter(lambda type_item: type_item != "null", field_item_type) ) + if len(schema_field_item["type"]) == 1: + schema_field_item["type"] = schema_field_item["type"][0] + else: + # Recursively get items + schema_field_item["type"] = list( + map( + lambda field_subtypes: sanitise_schema_field(field_subtypes), + schema_field_item.get("type", []), + ) + ) - if isinstance(schema_field_item.get("type"), str): - if schema_field_item.get("type", "").endswith("?"): - required = False - schema_field_item["type"] = schema_field_item.get("type", "").replace( - "?", "" - ) + case {"type": str() as field_item_type}: + if field_item_type.endswith("?"): + required = False + schema_field_item["type"] = schema_field_item.get("type", "").replace( + "?", "" + ) - if schema_field_item.get("type", "").endswith("[]"): - # Strip list - schema_field_item["type"] = schema_field_item.get("type", "").replace( - "[]", "" - ) - # Convert to array - schema_field_item["type"] = InputArraySchemaV1_2( - type_="array", items=schema_field_item.get("type", "") - ) + if schema_field_item.get("type", "").endswith("[]"): + # Strip list + schema_field_item["type"] = schema_field_item.get("type", "").replace( + "[]", "" + ) + # Convert to array + schema_field_item["type"] = InputArraySchemaV1_2( + type_="array", items=schema_field_item.get("type", "") + ) - if isinstance(schema_field_item.get("type"), dict): - # Likely an enum - if schema_field_item.get("type", {}).get("type", "") == "enum": + case {"type": {"type": "enum", **rest}}: schema_field_item["type"] = InputEnumSchemaV1_2( type_="enum", - symbols=schema_field_item.get("type", {}).get("symbols", ""), + symbols=rest.get("symbols", ""), ) - elif schema_field_item.get("type", {}).get("type", "") == "array": + case {"type": {"type": "array", **rest}}: schema_field_item["type"] = InputArraySchemaV1_2( - type_="array", items=schema_field_item.get("type", {}).get("items", "") + type_="array", items=rest.get("items", "") ) - elif "$import" in schema_field_item.get("type", {}).keys(): - # Leave import as is - pass - else: + case {"type": {"$import": _}}: + pass # Leave import as is + case {"type": dict()}: raise ValueError(f"Unknown type: {schema_field_item.get('type')}") if not required: