From d9ea096d98822632cf3adf4c7e92c16495f26167 Mon Sep 17 00:00:00 2001 From: Fabian Peschel Date: Wed, 6 Dec 2023 12:27:28 +0100 Subject: [PATCH 01/11] fix pass manager config for dynamic jobs --- src/jobflow/core/job.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/jobflow/core/job.py b/src/jobflow/core/job.py index f11e1917..19b7fcd1 100644 --- a/src/jobflow/core/job.py +++ b/src/jobflow/core/job.py @@ -1354,11 +1354,7 @@ def pass_manager_config( A job, flow, or list of jobs/flows. manager_config A manager config to pass on. - metadata - Metadata to pass on. """ - from copy import deepcopy - all_jobs: list[Job] = [] def get_jobs(arg): @@ -1378,4 +1374,4 @@ def get_jobs(arg): # update manager config for ajob in all_jobs: - ajob.config.manager_config = deepcopy(manager_config) + ajob.config.manager_config = manager_config | ajob.config.manager_config From 13b834f7ee41e9b9f95f2a0394924cf5c72ef7ec Mon Sep 17 00:00:00 2001 From: Fabian Peschel Date: Thu, 7 Dec 2023 13:10:27 +0100 Subject: [PATCH 02/11] fix tests with removing decorator --- src/jobflow/core/job.py | 4 ++-- tests/core/test_job.py | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/jobflow/core/job.py b/src/jobflow/core/job.py index 19b7fcd1..64a26b48 100644 --- a/src/jobflow/core/job.py +++ b/src/jobflow/core/job.py @@ -1266,7 +1266,6 @@ def apply_schema(output: Any, schema: type[BaseModel] | None): return schema(**output) -@job(config=JobConfig(resolve_references=False, on_missing_references=OnMissing.NONE)) def store_inputs(inputs: Any) -> Any: """ Job to store inputs. @@ -1316,7 +1315,8 @@ def prepare_replace( # add a job with same UUID as the current job to store the outputs of the # flow; this job will inherit the metadata and output schema of the current # job - store_output_job = store_inputs(replace.output) + new_config = JobConfig(resolve_references=False, on_missing_references=OnMissing.NONE) + store_output_job = Job(store_inputs, function_args=(replace.output,), config=new_config) store_output_job.set_uuid(current_job.uuid) store_output_job.index = current_job.index + 1 store_output_job.metadata = current_job.metadata diff --git a/tests/core/test_job.py b/tests/core/test_job.py index a231e400..27d02350 100644 --- a/tests/core/test_job.py +++ b/tests/core/test_job.py @@ -934,15 +934,16 @@ def add_schema_replace(a, b): def test_store_inputs(memory_jobstore): - from jobflow.core.job import Job, OutputReference, store_inputs + from jobflow.core.job import Job, OutputReference, store_inputs, JobConfig, OnMissing - test_job = store_inputs(1) + configs = JobConfig(resolve_references=False, on_missing_references=OnMissing.NONE) + test_job = Job(store_inputs, (1,), config=configs) test_job.run(memory_jobstore) output = memory_jobstore.query_one({"uuid": test_job.uuid}, ["output"])["output"] assert output == 1 ref = OutputReference("abcd") - test_job = store_inputs(ref) + test_job = Job(store_inputs, (ref,), config=configs) test_job.run(memory_jobstore) output = memory_jobstore.query_one({"uuid": test_job.uuid}, ["output"])["output"] assert OutputReference.from_dict(output) == ref From 1933d34daf63495e19a72b3c4ad050f4631f859c Mon Sep 17 00:00:00 2001 From: Fabian Peschel Date: Fri, 8 Dec 2023 10:49:43 +0100 Subject: [PATCH 03/11] add doc --- src/jobflow/core/job.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/jobflow/core/job.py b/src/jobflow/core/job.py index 64a26b48..1766a49b 100644 --- a/src/jobflow/core/job.py +++ b/src/jobflow/core/job.py @@ -1347,6 +1347,7 @@ def pass_manager_config( ): """ Pass the manager config on to any jobs in the jobs array. + Merge with already specified manager config. Parameters ---------- From 1619bb0c77b1c40ae510e4ae70521dad829b18fe Mon Sep 17 00:00:00 2001 From: Fabian Peschel Date: Fri, 8 Dec 2023 12:03:21 +0100 Subject: [PATCH 04/11] apply black --- src/jobflow/core/job.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/jobflow/core/job.py b/src/jobflow/core/job.py index 1766a49b..7293cf37 100644 --- a/src/jobflow/core/job.py +++ b/src/jobflow/core/job.py @@ -1315,8 +1315,12 @@ def prepare_replace( # add a job with same UUID as the current job to store the outputs of the # flow; this job will inherit the metadata and output schema of the current # job - new_config = JobConfig(resolve_references=False, on_missing_references=OnMissing.NONE) - store_output_job = Job(store_inputs, function_args=(replace.output,), config=new_config) + new_config = JobConfig( + resolve_references=False, on_missing_references=OnMissing.NONE + ) + store_output_job = Job( + store_inputs, function_args=(replace.output,), config=new_config + ) store_output_job.set_uuid(current_job.uuid) store_output_job.index = current_job.index + 1 store_output_job.metadata = current_job.metadata From 50321d73e8a9dc2297d7de68cb520f23b9b4e4a0 Mon Sep 17 00:00:00 2001 From: Fabian Peschel Date: Tue, 12 Dec 2023 11:31:48 +0100 Subject: [PATCH 05/11] apply black and reformat import --- tests/core/test_job.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/core/test_job.py b/tests/core/test_job.py index 27d02350..7829b548 100644 --- a/tests/core/test_job.py +++ b/tests/core/test_job.py @@ -934,7 +934,13 @@ def add_schema_replace(a, b): def test_store_inputs(memory_jobstore): - from jobflow.core.job import Job, OutputReference, store_inputs, JobConfig, OnMissing + from jobflow.core.job import ( + Job, + OutputReference, + store_inputs, + JobConfig, + OnMissing, + ) configs = JobConfig(resolve_references=False, on_missing_references=OnMissing.NONE) test_job = Job(store_inputs, (1,), config=configs) From d413017b384d16f04704e0234d9bb94379e8c1bd Mon Sep 17 00:00:00 2001 From: Fabian Peschel Date: Wed, 28 Feb 2024 19:50:45 +0100 Subject: [PATCH 06/11] add type hint --- src/jobflow/core/job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jobflow/core/job.py b/src/jobflow/core/job.py index cd622ab8..7caeecf1 100644 --- a/src/jobflow/core/job.py +++ b/src/jobflow/core/job.py @@ -65,7 +65,7 @@ class JobConfig(MSONable): response_manager_config: dict = field(default_factory=dict) -def job(method: Callable = None, **job_kwargs): +def job(method: Callable = None, **job_kwargs) -> Callable[..., Job]: """ Wrap a function to produce a :obj:`Job`. From e6e92d0aa78f2e412fa6bff63ab1d4f02fc3a60b Mon Sep 17 00:00:00 2001 From: Fabian Peschel Date: Wed, 13 Mar 2024 13:12:28 +0100 Subject: [PATCH 07/11] remove enum value keywords --- src/jobflow/core/job.py | 4 ++-- src/jobflow/core/reference.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/jobflow/core/job.py b/src/jobflow/core/job.py index 6d036a2b..84e84b01 100644 --- a/src/jobflow/core/job.py +++ b/src/jobflow/core/job.py @@ -624,7 +624,7 @@ def run(self, store: jobflow.JobStore) -> Response: try: output = jsanitize( - response.output, strict=True, enum_values=True, allow_bson=True + response.output, strict=True, allow_bson=True ) except AttributeError as err: raise RuntimeError( @@ -1106,7 +1106,7 @@ def as_dict(self) -> dict: # fireworks can't serialize functions and classes, so explicitly serialize to # the job recursively using monty to avoid issues - return jsanitize(d, strict=True, enum_values=True, allow_bson=True) + return jsanitize(d, strict=True, allow_bson=True) def __setattr__(self, key, value): """Handle setting attributes. Implements a special case for job name.""" diff --git a/src/jobflow/core/reference.py b/src/jobflow/core/reference.py index 10d774b3..07e1b1c9 100644 --- a/src/jobflow/core/reference.py +++ b/src/jobflow/core/reference.py @@ -394,7 +394,7 @@ def find_and_get_references(arg: Any) -> tuple[OutputReference, ...]: # argument is a primitive, we won't find a reference here return () - arg = jsanitize(arg, strict=True, enum_values=True, allow_bson=True) + arg = jsanitize(arg, strict=True, allow_bson=True) # recursively find any reference classes locations = find_key_value(arg, "@class", "OutputReference") @@ -458,7 +458,7 @@ def find_and_resolve_references( return arg # serialize the argument to a dictionary - encoded_arg = jsanitize(arg, strict=True, enum_values=True, allow_bson=True) + encoded_arg = jsanitize(arg, strict=True, allow_bson=True) # recursively find any reference classes locations = find_key_value(encoded_arg, "@class", "OutputReference") From 978e66f0cfe4e1f7c7470f8e9ddf5a5dc9dd445e Mon Sep 17 00:00:00 2001 From: Fabian Peschel Date: Wed, 13 Mar 2024 13:15:50 +0100 Subject: [PATCH 08/11] fix ruff --- src/jobflow/core/job.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/jobflow/core/job.py b/src/jobflow/core/job.py index 84e84b01..378f43d2 100644 --- a/src/jobflow/core/job.py +++ b/src/jobflow/core/job.py @@ -623,9 +623,7 @@ def run(self, store: jobflow.JobStore) -> Response: pass_manager_config(response.replace, passed_config) try: - output = jsanitize( - response.output, strict=True, allow_bson=True - ) + output = jsanitize(response.output, strict=True, allow_bson=True) except AttributeError as err: raise RuntimeError( "Job output contained an object that is not MSONable and therefore " From 2c19c67ac1089e34830bdcf3e06764b95db20830 Mon Sep 17 00:00:00 2001 From: Fabian Peschel Date: Tue, 2 Apr 2024 16:22:33 +0200 Subject: [PATCH 09/11] fix typing --- src/jobflow/managers/local.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jobflow/managers/local.py b/src/jobflow/managers/local.py index 002fd212..9bb2479c 100644 --- a/src/jobflow/managers/local.py +++ b/src/jobflow/managers/local.py @@ -16,7 +16,7 @@ def run_locally( flow: jobflow.Flow | jobflow.Job | list[jobflow.Job], log: bool = True, - store: jobflow.JobStore = None, + store: jobflow.JobStore | None = None, create_folders: bool = False, root_dir: str | Path | None = None, ensure_success: bool = False, From 4ac1d9dc75ba86fd94e555e548dad48d9b9275a7 Mon Sep 17 00:00:00 2001 From: Fabian Peschel Date: Wed, 3 Apr 2024 10:56:02 +0200 Subject: [PATCH 10/11] fix ruff --- src/jobflow/core/job.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/jobflow/core/job.py b/src/jobflow/core/job.py index e5d2cae2..1b6c2e08 100644 --- a/src/jobflow/core/job.py +++ b/src/jobflow/core/job.py @@ -1349,6 +1349,7 @@ def pass_manager_config( ): """ Pass the manager config on to any jobs in the jobs array. + Merge with already specified manager config. Parameters From 65fd80961c3352bc5e3cad330b157e02170c6075 Mon Sep 17 00:00:00 2001 From: Fabian Peschel Date: Fri, 5 Apr 2024 11:24:38 +0200 Subject: [PATCH 11/11] fix @job annotation --- src/jobflow/core/job.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/jobflow/core/job.py b/src/jobflow/core/job.py index a5d3315c..4d0c9b3b 100644 --- a/src/jobflow/core/job.py +++ b/src/jobflow/core/job.py @@ -78,7 +78,9 @@ def job(method: Callable = None, **job_kwargs) -> Callable[..., Callable[..., Jo pass -def job(method: Callable = None, **job_kwargs): +def job( + method: Callable = None, **job_kwargs +) -> Callable[..., Job] | Callable[..., Callable[..., Job]]: """ Wrap a function to produce a :obj:`Job`.