From a5be134abef81b482f631ea8feb26573ac599c3a Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 19 Oct 2022 10:27:48 -0500 Subject: [PATCH 01/15] Move dataset.py from modeldata to core package. Move module and its Dataset, DatasetType, (abstract) DatasetManager, and DatasetUser classes to dmod.core, as these classes are likely needed in a variety of places where the full modeldata package is not (e.g., the GUI). --- .../dmod/modeldata/data => core/dmod/core}/dataset.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename python/lib/{modeldata/dmod/modeldata/data => core/dmod/core}/dataset.py (99%) diff --git a/python/lib/modeldata/dmod/modeldata/data/dataset.py b/python/lib/core/dmod/core/dataset.py similarity index 99% rename from python/lib/modeldata/dmod/modeldata/data/dataset.py rename to python/lib/core/dmod/core/dataset.py index aac52024b..e92bb5bbe 100644 --- a/python/lib/modeldata/dmod/modeldata/data/dataset.py +++ b/python/lib/core/dmod/core/dataset.py @@ -1,10 +1,10 @@ from abc import ABC, abstractmethod -from dmod.core.meta_data import ContinuousRestriction, DataCategory, DataDomain, DataFormat, DiscreteRestriction, \ +from .meta_data import ContinuousRestriction, DataCategory, DataDomain, DataFormat, DiscreteRestriction, \ StandardDatasetIndex, TimeRange -from dmod.core.exception import DmodRuntimeError +from .exception import DmodRuntimeError from datetime import datetime, timedelta -from dmod.core.serializable import Serializable, ResultIndicator +from .serializable import Serializable, ResultIndicator from enum import Enum from numbers import Number from typing import Any, Callable, Dict, FrozenSet, List, Optional, Set, Tuple, Type, Union From ea866b8883cf8ea017db7684f99ecbc4a59b9387 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 19 Oct 2022 10:39:37 -0500 Subject: [PATCH 02/15] Updates to dmod.core package README.md summary. --- python/lib/core/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/lib/core/README.md b/python/lib/core/README.md index 5f7d8d8b4..e555ed540 100644 --- a/python/lib/core/README.md +++ b/python/lib/core/README.md @@ -1,2 +1,4 @@ # About -Python package for core types depended upon by multiple other DMOD Python packages, which must be located in an isolated "core" package to avoid circular dependencies. +Python package for core DMOD types, both concrete and abstract, that are depended upon by other DMOD Python packages and themselves have no dependencies outside of Python and its standard library. + +Classes belong here if placing them in a more specialized package would cause undesired consequences, such as circular dependencies or transitive dependency on otherwise unnecessary packages. From 23920d1cdcd06871a916b1a3ef3577f428991a73 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 19 Oct 2022 10:41:36 -0500 Subject: [PATCH 03/15] Move unit test module for Dataset class. Moving unit tests for Dataset to dmod.core after class moved to this package. --- python/lib/{modeldata => core}/dmod/test/test_dataset.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename python/lib/{modeldata => core}/dmod/test/test_dataset.py (96%) diff --git a/python/lib/modeldata/dmod/test/test_dataset.py b/python/lib/core/dmod/test/test_dataset.py similarity index 96% rename from python/lib/modeldata/dmod/test/test_dataset.py rename to python/lib/core/dmod/test/test_dataset.py index 9b4548fb4..0548b395c 100644 --- a/python/lib/modeldata/dmod/test/test_dataset.py +++ b/python/lib/core/dmod/test/test_dataset.py @@ -1,8 +1,8 @@ import unittest -from ..modeldata.data.dataset import Dataset, DatasetType -from dmod.core.meta_data import DataCategory, DataDomain, DataFormat, DiscreteRestriction, TimeRange +from ..core.dataset import Dataset, DatasetType +from ..core.meta_data import DataCategory, DataDomain, DataFormat, DiscreteRestriction, TimeRange from datetime import datetime, timedelta -from typing import Any, Dict, Optional, Union +from typing import Optional, Union class TestDataset(unittest.TestCase): From f40349a53e25a1a187779f165fbf202bb01185e1 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 19 Oct 2022 10:42:31 -0500 Subject: [PATCH 04/15] Add init file to core package test directory. Adding __init__.py to correct issue with unit tests not always being loaded correctly. --- python/lib/core/dmod/test/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 python/lib/core/dmod/test/__init__.py diff --git a/python/lib/core/dmod/test/__init__.py b/python/lib/core/dmod/test/__init__.py new file mode 100644 index 000000000..e69de29bb From f76036ac9c5a05af803a38528706cf1fbf6e4e5d Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 19 Oct 2022 10:44:26 -0500 Subject: [PATCH 05/15] Fix incorrect import in AllocationParadigm test. --- python/lib/core/dmod/test/test_allocationparadigm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lib/core/dmod/test/test_allocationparadigm.py b/python/lib/core/dmod/test/test_allocationparadigm.py index 060e45f09..410fe2bb3 100644 --- a/python/lib/core/dmod/test/test_allocationparadigm.py +++ b/python/lib/core/dmod/test/test_allocationparadigm.py @@ -1,5 +1,5 @@ import unittest -from dmod.scheduler.job.job import AllocationParadigm +from ..core.execution import AllocationParadigm from dmod.communication import SchedulerRequestMessage From 32531396163054364ac50417ed2a855c3df7205e Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 19 Oct 2022 10:46:49 -0500 Subject: [PATCH 06/15] Fix location of DataRequirement unit tests. --- .../lib/{modeldata => core}/dmod/test/test_data_requirement.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename python/lib/{modeldata => core}/dmod/test/test_data_requirement.py (94%) diff --git a/python/lib/modeldata/dmod/test/test_data_requirement.py b/python/lib/core/dmod/test/test_data_requirement.py similarity index 94% rename from python/lib/modeldata/dmod/test/test_data_requirement.py rename to python/lib/core/dmod/test/test_data_requirement.py index f0716f65a..4cbf4e2c5 100644 --- a/python/lib/modeldata/dmod/test/test_data_requirement.py +++ b/python/lib/core/dmod/test/test_data_requirement.py @@ -1,5 +1,5 @@ import unittest -from dmod.core.meta_data import DataDomain, DataFormat, DataRequirement, DataCategory, DiscreteRestriction +from ..core.meta_data import DataDomain, DataFormat, DataRequirement, DataCategory, DiscreteRestriction class TestDataRequirement(unittest.TestCase): From caa723c85979f3597421a8738ae6be983709a887 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 19 Oct 2022 10:50:33 -0500 Subject: [PATCH 07/15] Bump core package to 0.3.0. --- python/lib/core/dmod/core/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lib/core/dmod/core/_version.py b/python/lib/core/dmod/core/_version.py index a9fdc5cfe..290d7c60d 100644 --- a/python/lib/core/dmod/core/_version.py +++ b/python/lib/core/dmod/core/_version.py @@ -1 +1 @@ -__version__ = '0.2.0' \ No newline at end of file +__version__ = '0.3.0' \ No newline at end of file From 3c7f211e8f513c288d7ee8ca2c2ccc4d170a9f1d Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 19 Oct 2022 10:59:20 -0500 Subject: [PATCH 08/15] Update dataset manager to get serial file name. --- python/lib/core/dmod/core/dataset.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/python/lib/core/dmod/core/dataset.py b/python/lib/core/dmod/core/dataset.py index e92bb5bbe..b1e5ca2aa 100644 --- a/python/lib/core/dmod/core/dataset.py +++ b/python/lib/core/dmod/core/dataset.py @@ -616,6 +616,26 @@ class DatasetManager(ABC): ::method:`transform` creates a new dataset from an existing one, but transforms the data to another format. """ + _SERIALIZED_OBJ_NAME_TEMPLATE = "{}_serialized.json" + """ The name of the file/object for serialized versions of datasets, within a dataset's bucket. """ + + @classmethod + def get_serial_dataset_filename(cls, dataset_name: str) -> str: + """ + Get the standard file basename for persisting the serialized state of a ::class:`Dataset` of the given name. + + Parameters + ---------- + dataset_name: str + The name of the dataset in question. + + Returns + ------- + str + The file basename for persisting the serialized state of a dataset with the given name. + """ + return cls._SERIALIZED_OBJ_NAME_TEMPLATE.format(dataset_name) + def __init__(self, uuid: Optional[UUID] = None, datasets: Optional[Dict[str, Dataset]] = None, *args, **kwargs): self._uuid = uuid4() if uuid is None else uuid self._datasets = datasets if datasets is not None else dict() From 041b9aa0d5e437135cd1f4edace4bc56aebadf2f Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 19 Oct 2022 11:00:55 -0500 Subject: [PATCH 09/15] Move dmod.modeldata to new dmod.core dep version. --- python/lib/modeldata/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lib/modeldata/setup.py b/python/lib/modeldata/setup.py index c536a6b03..d00c51756 100644 --- a/python/lib/modeldata/setup.py +++ b/python/lib/modeldata/setup.py @@ -14,7 +14,7 @@ author_email='', url='', license='', - install_requires=['numpy>=1.20.1', 'pandas', 'geopandas', 'dmod-communication>=0.4.2', 'dmod-core>=0.1.0', 'minio', + install_requires=['numpy>=1.20.1', 'pandas', 'geopandas', 'dmod-communication>=0.4.2', 'dmod-core>=0.3.0', 'minio', 'aiohttp<=3.7.4', 'hypy@git+https://github.com/NOAA-OWP/hypy@master#egg=hypy&subdirectory=python'], packages=find_namespace_packages(exclude=('tests', 'schemas', 'ssl', 'src')) ) From 378cdfa3f235080b07d8b2838477d284c4d54e40 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 19 Oct 2022 11:02:20 -0500 Subject: [PATCH 10/15] Update filesystem_manager.py dataset package. --- python/lib/modeldata/dmod/modeldata/data/filesystem_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lib/modeldata/dmod/modeldata/data/filesystem_manager.py b/python/lib/modeldata/dmod/modeldata/data/filesystem_manager.py index 51044b005..e57b54e4d 100644 --- a/python/lib/modeldata/dmod/modeldata/data/filesystem_manager.py +++ b/python/lib/modeldata/dmod/modeldata/data/filesystem_manager.py @@ -4,7 +4,7 @@ from pathlib import Path from typing import Optional, List, Set, Union, Any, Tuple -from .dataset import Dataset, DatasetManager, DatasetType +from dmod.core.dataset import Dataset, DatasetManager, DatasetType from dmod.core.exception import DmodRuntimeError from dmod.core.meta_data import DataCategory, DataDomain From 987dd7df6dc16c91242803e351aa9d664d23bb35 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 19 Oct 2022 11:03:51 -0500 Subject: [PATCH 11/15] Update object_store_manager.py dataset package. Also removing class member _SERIALIZED_OBJ_NAME_TEMPLATE that is now located in superclass. --- .../lib/modeldata/dmod/modeldata/data/object_store_manager.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/python/lib/modeldata/dmod/modeldata/data/object_store_manager.py b/python/lib/modeldata/dmod/modeldata/data/object_store_manager.py index 084492805..dffb5a8cf 100644 --- a/python/lib/modeldata/dmod/modeldata/data/object_store_manager.py +++ b/python/lib/modeldata/dmod/modeldata/data/object_store_manager.py @@ -4,7 +4,7 @@ import minio.retention from dmod.core.meta_data import DataCategory, DataDomain -from .dataset import Dataset, DatasetManager, DatasetType +from dmod.core.dataset import Dataset, DatasetManager, DatasetType from datetime import datetime, timedelta from minio import Minio from minio.api import ObjectWriteResult @@ -21,8 +21,6 @@ class ObjectStoreDatasetManager(DatasetManager): _SUPPORTED_TYPES = {DatasetType.OBJECT_STORE} """ Supported dataset types set, which is always ::class:`ObjectStoreDataset` for this manager subtype. """ - _SERIALIZED_OBJ_NAME_TEMPLATE = "{}_serialized.json" - """ The name of the file/object for serialized versions of datasets, within a dataset's bucket. """ def __init__(self, obj_store_host_str: str, access_key: Optional[str] = None, secret_key: Optional[str] = None, secure_connection: bool = False, *args, **kwargs): From e437669e8880834c61e635abb4fbf9317efec60d Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 19 Oct 2022 11:04:31 -0500 Subject: [PATCH 12/15] Bump dmod.modeldata package version to 0.9.0. --- python/lib/modeldata/dmod/modeldata/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lib/modeldata/dmod/modeldata/_version.py b/python/lib/modeldata/dmod/modeldata/_version.py index ccf9e6286..1658609d0 100644 --- a/python/lib/modeldata/dmod/modeldata/_version.py +++ b/python/lib/modeldata/dmod/modeldata/_version.py @@ -1 +1 @@ -__version__ = '0.8.0' \ No newline at end of file +__version__ = '0.9.0' \ No newline at end of file From 70287cc7e9fe78b36c83c7d3ab5f34e3ab14d556 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 19 Oct 2022 11:06:25 -0500 Subject: [PATCH 13/15] Update dataservice DMOD dependency versions. Bump dependency on dmod.core to 0.3.0 and on dmod.modeldata to 0.9.0. --- python/services/dataservice/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/services/dataservice/setup.py b/python/services/dataservice/setup.py index 32dc10aab..cd74d404d 100644 --- a/python/services/dataservice/setup.py +++ b/python/services/dataservice/setup.py @@ -14,7 +14,7 @@ author_email='', url='', license='', - install_requires=['dmod-core>=0.1.1', 'dmod-communication>=0.7.1', 'dmod-scheduler>=0.7.0', 'dmod-modeldata>=0.8.0', + install_requires=['dmod-core>=0.3.0', 'dmod-communication>=0.7.1', 'dmod-scheduler>=0.7.0', 'dmod-modeldata>=0.9.0', 'redis'], packages=find_namespace_packages(exclude=('tests', 'test', 'deprecated', 'conf', 'schemas', 'ssl', 'src')) ) From 8c5a8889145350805e878b9eaa0495b74f8dd013 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 19 Oct 2022 11:07:15 -0500 Subject: [PATCH 14/15] Fix/update Dataset import in dataservice tests. --- python/services/dataservice/dmod/test/test_service_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/services/dataservice/dmod/test/test_service_manager.py b/python/services/dataservice/dmod/test/test_service_manager.py index 23341dd6e..0c0f45e68 100644 --- a/python/services/dataservice/dmod/test/test_service_manager.py +++ b/python/services/dataservice/dmod/test/test_service_manager.py @@ -4,7 +4,7 @@ import os from ..dataservice.service import ServiceManager from dmod.communication.client import get_or_create_eventloop -from dmod.modeldata.data.dataset import Dataset +from dmod.core.dataset import Dataset from dmod.scheduler.job import RequestedJob from pathlib import Path from typing import Any, Dict, List, Optional From 7a78ade068f325c108530a1405424d28ff6e2b28 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 19 Oct 2022 11:07:34 -0500 Subject: [PATCH 15/15] Bump dmod.dataservice package version to 0.4.0. --- python/services/dataservice/dmod/dataservice/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/services/dataservice/dmod/dataservice/_version.py b/python/services/dataservice/dmod/dataservice/_version.py index 290d7c60d..222c11cfd 100644 --- a/python/services/dataservice/dmod/dataservice/_version.py +++ b/python/services/dataservice/dmod/dataservice/_version.py @@ -1 +1 @@ -__version__ = '0.3.0' \ No newline at end of file +__version__ = '0.4.0' \ No newline at end of file