diff --git a/python/lib/core/README.md b/python/lib/core/README.md index 5f7d8d8b4..e555ed540 100644 --- a/python/lib/core/README.md +++ b/python/lib/core/README.md @@ -1,2 +1,4 @@ # About -Python package for core types depended upon by multiple other DMOD Python packages, which must be located in an isolated "core" package to avoid circular dependencies. +Python package for core DMOD types, both concrete and abstract, that are depended upon by other DMOD Python packages and themselves have no dependencies outside of Python and its standard library. + +Classes belong here if placing them in a more specialized package would cause undesired consequences, such as circular dependencies or transitive dependency on otherwise unnecessary packages. diff --git a/python/lib/core/dmod/core/_version.py b/python/lib/core/dmod/core/_version.py index a9fdc5cfe..290d7c60d 100644 --- a/python/lib/core/dmod/core/_version.py +++ b/python/lib/core/dmod/core/_version.py @@ -1 +1 @@ -__version__ = '0.2.0' \ No newline at end of file +__version__ = '0.3.0' \ No newline at end of file diff --git a/python/lib/modeldata/dmod/modeldata/data/dataset.py b/python/lib/core/dmod/core/dataset.py similarity index 97% rename from python/lib/modeldata/dmod/modeldata/data/dataset.py rename to python/lib/core/dmod/core/dataset.py index aac52024b..b1e5ca2aa 100644 --- a/python/lib/modeldata/dmod/modeldata/data/dataset.py +++ b/python/lib/core/dmod/core/dataset.py @@ -1,10 +1,10 @@ from abc import ABC, abstractmethod -from dmod.core.meta_data import ContinuousRestriction, DataCategory, DataDomain, DataFormat, DiscreteRestriction, \ +from .meta_data import ContinuousRestriction, DataCategory, DataDomain, DataFormat, DiscreteRestriction, \ StandardDatasetIndex, TimeRange -from dmod.core.exception import DmodRuntimeError +from .exception import DmodRuntimeError from datetime import datetime, timedelta -from dmod.core.serializable import Serializable, ResultIndicator +from .serializable import Serializable, ResultIndicator from enum import Enum from numbers import Number from typing import Any, Callable, Dict, FrozenSet, List, Optional, Set, Tuple, Type, Union @@ -616,6 +616,26 @@ class DatasetManager(ABC): ::method:`transform` creates a new dataset from an existing one, but transforms the data to another format. """ + _SERIALIZED_OBJ_NAME_TEMPLATE = "{}_serialized.json" + """ The name of the file/object for serialized versions of datasets, within a dataset's bucket. """ + + @classmethod + def get_serial_dataset_filename(cls, dataset_name: str) -> str: + """ + Get the standard file basename for persisting the serialized state of a ::class:`Dataset` of the given name. + + Parameters + ---------- + dataset_name: str + The name of the dataset in question. + + Returns + ------- + str + The file basename for persisting the serialized state of a dataset with the given name. + """ + return cls._SERIALIZED_OBJ_NAME_TEMPLATE.format(dataset_name) + def __init__(self, uuid: Optional[UUID] = None, datasets: Optional[Dict[str, Dataset]] = None, *args, **kwargs): self._uuid = uuid4() if uuid is None else uuid self._datasets = datasets if datasets is not None else dict() diff --git a/python/lib/core/dmod/test/__init__.py b/python/lib/core/dmod/test/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/lib/core/dmod/test/test_allocationparadigm.py b/python/lib/core/dmod/test/test_allocationparadigm.py index 060e45f09..410fe2bb3 100644 --- a/python/lib/core/dmod/test/test_allocationparadigm.py +++ b/python/lib/core/dmod/test/test_allocationparadigm.py @@ -1,5 +1,5 @@ import unittest -from dmod.scheduler.job.job import AllocationParadigm +from ..core.execution import AllocationParadigm from dmod.communication import SchedulerRequestMessage diff --git a/python/lib/modeldata/dmod/test/test_data_requirement.py b/python/lib/core/dmod/test/test_data_requirement.py similarity index 94% rename from python/lib/modeldata/dmod/test/test_data_requirement.py rename to python/lib/core/dmod/test/test_data_requirement.py index f0716f65a..4cbf4e2c5 100644 --- a/python/lib/modeldata/dmod/test/test_data_requirement.py +++ b/python/lib/core/dmod/test/test_data_requirement.py @@ -1,5 +1,5 @@ import unittest -from dmod.core.meta_data import DataDomain, DataFormat, DataRequirement, DataCategory, DiscreteRestriction +from ..core.meta_data import DataDomain, DataFormat, DataRequirement, DataCategory, DiscreteRestriction class TestDataRequirement(unittest.TestCase): diff --git a/python/lib/modeldata/dmod/test/test_dataset.py b/python/lib/core/dmod/test/test_dataset.py similarity index 96% rename from python/lib/modeldata/dmod/test/test_dataset.py rename to python/lib/core/dmod/test/test_dataset.py index 9b4548fb4..0548b395c 100644 --- a/python/lib/modeldata/dmod/test/test_dataset.py +++ b/python/lib/core/dmod/test/test_dataset.py @@ -1,8 +1,8 @@ import unittest -from ..modeldata.data.dataset import Dataset, DatasetType -from dmod.core.meta_data import DataCategory, DataDomain, DataFormat, DiscreteRestriction, TimeRange +from ..core.dataset import Dataset, DatasetType +from ..core.meta_data import DataCategory, DataDomain, DataFormat, DiscreteRestriction, TimeRange from datetime import datetime, timedelta -from typing import Any, Dict, Optional, Union +from typing import Optional, Union class TestDataset(unittest.TestCase): diff --git a/python/lib/modeldata/dmod/modeldata/_version.py b/python/lib/modeldata/dmod/modeldata/_version.py index ccf9e6286..1658609d0 100644 --- a/python/lib/modeldata/dmod/modeldata/_version.py +++ b/python/lib/modeldata/dmod/modeldata/_version.py @@ -1 +1 @@ -__version__ = '0.8.0' \ No newline at end of file +__version__ = '0.9.0' \ No newline at end of file diff --git a/python/lib/modeldata/dmod/modeldata/data/filesystem_manager.py b/python/lib/modeldata/dmod/modeldata/data/filesystem_manager.py index 51044b005..e57b54e4d 100644 --- a/python/lib/modeldata/dmod/modeldata/data/filesystem_manager.py +++ b/python/lib/modeldata/dmod/modeldata/data/filesystem_manager.py @@ -4,7 +4,7 @@ from pathlib import Path from typing import Optional, List, Set, Union, Any, Tuple -from .dataset import Dataset, DatasetManager, DatasetType +from dmod.core.dataset import Dataset, DatasetManager, DatasetType from dmod.core.exception import DmodRuntimeError from dmod.core.meta_data import DataCategory, DataDomain diff --git a/python/lib/modeldata/dmod/modeldata/data/object_store_manager.py b/python/lib/modeldata/dmod/modeldata/data/object_store_manager.py index 084492805..dffb5a8cf 100644 --- a/python/lib/modeldata/dmod/modeldata/data/object_store_manager.py +++ b/python/lib/modeldata/dmod/modeldata/data/object_store_manager.py @@ -4,7 +4,7 @@ import minio.retention from dmod.core.meta_data import DataCategory, DataDomain -from .dataset import Dataset, DatasetManager, DatasetType +from dmod.core.dataset import Dataset, DatasetManager, DatasetType from datetime import datetime, timedelta from minio import Minio from minio.api import ObjectWriteResult @@ -21,8 +21,6 @@ class ObjectStoreDatasetManager(DatasetManager): _SUPPORTED_TYPES = {DatasetType.OBJECT_STORE} """ Supported dataset types set, which is always ::class:`ObjectStoreDataset` for this manager subtype. """ - _SERIALIZED_OBJ_NAME_TEMPLATE = "{}_serialized.json" - """ The name of the file/object for serialized versions of datasets, within a dataset's bucket. """ def __init__(self, obj_store_host_str: str, access_key: Optional[str] = None, secret_key: Optional[str] = None, secure_connection: bool = False, *args, **kwargs): diff --git a/python/lib/modeldata/setup.py b/python/lib/modeldata/setup.py index c536a6b03..d00c51756 100644 --- a/python/lib/modeldata/setup.py +++ b/python/lib/modeldata/setup.py @@ -14,7 +14,7 @@ author_email='', url='', license='', - install_requires=['numpy>=1.20.1', 'pandas', 'geopandas', 'dmod-communication>=0.4.2', 'dmod-core>=0.1.0', 'minio', + install_requires=['numpy>=1.20.1', 'pandas', 'geopandas', 'dmod-communication>=0.4.2', 'dmod-core>=0.3.0', 'minio', 'aiohttp<=3.7.4', 'hypy@git+https://github.com/NOAA-OWP/hypy@master#egg=hypy&subdirectory=python'], packages=find_namespace_packages(exclude=('tests', 'schemas', 'ssl', 'src')) ) diff --git a/python/services/dataservice/dmod/dataservice/_version.py b/python/services/dataservice/dmod/dataservice/_version.py index 290d7c60d..222c11cfd 100644 --- a/python/services/dataservice/dmod/dataservice/_version.py +++ b/python/services/dataservice/dmod/dataservice/_version.py @@ -1 +1 @@ -__version__ = '0.3.0' \ No newline at end of file +__version__ = '0.4.0' \ No newline at end of file diff --git a/python/services/dataservice/dmod/test/test_service_manager.py b/python/services/dataservice/dmod/test/test_service_manager.py index 23341dd6e..0c0f45e68 100644 --- a/python/services/dataservice/dmod/test/test_service_manager.py +++ b/python/services/dataservice/dmod/test/test_service_manager.py @@ -4,7 +4,7 @@ import os from ..dataservice.service import ServiceManager from dmod.communication.client import get_or_create_eventloop -from dmod.modeldata.data.dataset import Dataset +from dmod.core.dataset import Dataset from dmod.scheduler.job import RequestedJob from pathlib import Path from typing import Any, Dict, List, Optional diff --git a/python/services/dataservice/setup.py b/python/services/dataservice/setup.py index 32dc10aab..cd74d404d 100644 --- a/python/services/dataservice/setup.py +++ b/python/services/dataservice/setup.py @@ -14,7 +14,7 @@ author_email='', url='', license='', - install_requires=['dmod-core>=0.1.1', 'dmod-communication>=0.7.1', 'dmod-scheduler>=0.7.0', 'dmod-modeldata>=0.8.0', + install_requires=['dmod-core>=0.3.0', 'dmod-communication>=0.7.1', 'dmod-scheduler>=0.7.0', 'dmod-modeldata>=0.9.0', 'redis'], packages=find_namespace_packages(exclude=('tests', 'test', 'deprecated', 'conf', 'schemas', 'ssl', 'src')) )