From f4657abaa4cc34c3dcce34bd42632cb49e1a7f82 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Thu, 28 Mar 2024 09:54:54 -0400 Subject: [PATCH 01/16] Extend Reader with RepeatableReader interface. Adding extended interface for reader that can reset for repeated data reads. --- python/lib/core/dmod/core/common/reader.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/python/lib/core/dmod/core/common/reader.py b/python/lib/core/dmod/core/common/reader.py index d0e09d4be..2b7ddb476 100644 --- a/python/lib/core/dmod/core/common/reader.py +++ b/python/lib/core/dmod/core/common/reader.py @@ -5,3 +5,12 @@ class Reader(Protocol): def read(self, size: int = -1, /) -> bytes: """EOF if empty b''.""" + + +class RepeatableReader(Reader): + """ + Extension of ::class:`Reader` that provides a reset mechanism that allows its data can be read multiple times. + """ + + def reset(self): + """ Reset such that ::method:`read` returns to the start of the data this instance reads. """ From bf7e7dd09d4504f20295d8375bb5799bb3d46a1d Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Thu, 28 Mar 2024 09:56:58 -0400 Subject: [PATCH 02/16] Implement proper truth values for ResultIndicator. --- python/lib/core/dmod/core/serializable.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/python/lib/core/dmod/core/serializable.py b/python/lib/core/dmod/core/serializable.py index 02da62148..e4f48b42b 100644 --- a/python/lib/core/dmod/core/serializable.py +++ b/python/lib/core/dmod/core/serializable.py @@ -366,6 +366,17 @@ class ResultIndicator(Serializable, ABC): reason: str = Field(description="A very short, high-level summary of the result.") message: str = Field("", description="An optional, more detailed explanation of the result, which by default is an empty string.") + def __bool__(self) -> bool: + """ + Implementation of truth value testing for instances, which directly depends on the value of ``success``. + + Returns + ------- + bool + The current value of the instance's ::attribute:`success` attribute. + """ + return self.success + class BasicResultIndicator(ResultIndicator): """ From f13bbe30fafb4cdb2397fc40688c051c9134a928 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 2 Apr 2024 09:56:57 -0400 Subject: [PATCH 03/16] Fix/improve GeoPackageHydrofabric. - Modify GeoPackageHydrofabric to reflect current format for NextGen hydrofabrics, which uses 'divide_id' instead of 'id' for catchment id within divides layer. - Modify hydrofabric test data to have consistent catchment id column. - Modify GeoPackageHydrofabric in type hinting for from_file class method to also accept a bytes object, which the function is already functionally compatible with. - Remove GeoPackageHydrofabric use of fiona and replacing with pyogrio. --- data/example_hydrofabric_2/hydrofabric.gpkg | Bin 274432 -> 278528 bytes .../hydrofabric/geopackage_hydrofabric.py | 20 ++++++++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/data/example_hydrofabric_2/hydrofabric.gpkg b/data/example_hydrofabric_2/hydrofabric.gpkg index 7622813ef8d027daaa3be4ffea8d15308fb6bc4c..38a5b4a917bfc6f30cc773f15e0f55bb2c5e7e10 100644 GIT binary patch delta 1810 zcmcIkYitx%6uxKf?B3bgoqHC$+t&!~#+s&qr7ci!i?}VNfDaOCstFiU6wovTC=d$; zYIj;1{9yuhw1W@>ga{^DWV6wc-~$t5f{mteea5sB&>)qVVpB?sQ15KFu^>Ot#F?DC zn{&SNnD09?v2*@dY|1`;N;x59k+6x&=$JA4zH%~ilEW+z}6|dlByol$q z7r)13+^9uXwM_&c+L-9Y8Gdaczr^?RYVOv$@Mzqyv_w~U7(g-KC@u)h3753KQhOBII*!zP05co>@!yktFUUaG^|`o{XT z>$WPnW=oxJ9*NfGh_2qW_pXydlae$X6rCCXf5sp16MPe2!)9!V8!JUaAbaR)s;o ztOqBa{<(FVHYj;=Bxo9SS!VOzt6Zr#$RCdW^cjD2(Q=qaZC89r}D{aBfPN~4$FvJp(p{bHnFp*d} z99JOlst?ZFzvsgUFBaaQTO66rCgnJwsZ<;p@NejMa(y2AS|0fP$K`$Cx8)d!)71c@ z2-(M0ga6oaVHMzi?+nQPdzcm_SivvwCeF1kEs96tMv-PmyI_r^DQ-O&-W(cd&078r z)NmDlB8;CU{5QOppSAf(IL&<_m%yc|cVI!cQsRmP!w&h@sxAmv`p598Ps|nyr!`2y zI>iXb)u-NfRsiM0pQ$_9a8&K$Y;ZRmI z3dVHH7=vu9DN23T-kY$-iSVrX^TE(lkv)MiumfWfHzdgJFlw#sF?!g(*hh1~E^VdB zWw|O`ApD}Xz_V1@BsbBSaKZPI_gilPS=M0`t8Ez#OkWjI+I6e+e!0m0Y`VM#XzSAT zR%w!ET8CZ~$EqZ)vij2hf_CgH`YV&jM|L_2hZaii?bePlcu*)dkzGa&*k(R5XsE`#MwX@P&{}P- z9)6yUZ*HCW&2X8`3*>et1cXLMKn;{#Y?6GIkZ-}A=nF}m6;#gkx-tlE-FD&Q|xH80tN%k7ehQw}t*UiY#7ajxBUh7Q%_$R=OT$}I(DXR6Yqg)0K(e`owJtF{aV delta 1163 zcmb7CZ){Ul6o2R4`}$sA_wMOr-PS$B)>&B=*R^G^v5wjO31^M4~ux(|txS-Yf`0r@bGoH;uXm-)5oyl|IQ%DL9*d=)}M9cU;1|xQMrK z4rjB)>D?$kJ(!)tI=42C9qcwc%XTn_mN5;kWix?$0Fe4P=r3sqgc<_jP)~SkWJ@Tr zrK!GoYg1}2Sni!_f99)~<4e^cJ&7MHsVZJ8=C0zdkt~;tE`?s_3dGs1ki}s1zb*t?LT&Lr7(!I$Imjry5 z{l?zVN?j8wSMJNd$feTXbeY^Hz2daE!v^d!jIgyk2U89;`9h7XM|EZBSw8kBRPaa> z%_y5>z1=qWrF^Q2<2HI1p0rlC(F*{6>wJVVP@X72hMfh7HQhqHK&dV;z29D7vLdat z78IXc7L7vkDV|(|a?Yx$gU6%9!B0PceQHvylFM3JAgMom9;v4K2f;P4f@|V(LKLnT zMh|~4LDFX03s!iD%yd`CupqESt<~A39HPfaJuGO8TCLFSdfPC9R6PKrrtu+=p{_w5 zo*||D;wd}qnIWC|`6!$`R^nUot1OD&o5qYtPSzaYhtG>8{J}i&SOat9Y=>@-g-SI6 qUu5Un^RypAqk!q-B8cW2Q^wSa9>YgT}z&nEUs^z~yyy8Fa`cj_& diff --git a/python/lib/modeldata/dmod/modeldata/hydrofabric/geopackage_hydrofabric.py b/python/lib/modeldata/dmod/modeldata/hydrofabric/geopackage_hydrofabric.py index 0f955b1eb..9f171810b 100644 --- a/python/lib/modeldata/dmod/modeldata/hydrofabric/geopackage_hydrofabric.py +++ b/python/lib/modeldata/dmod/modeldata/hydrofabric/geopackage_hydrofabric.py @@ -1,4 +1,4 @@ -import fiona +import pyogrio import geopandas as gpd import hashlib from pandas.util import hash_pandas_object @@ -316,7 +316,7 @@ class GeoPackageHydrofabric(Hydrofabric): #_FLOWPATHS_TO_NEX_COL = 'toid' _DIVIDES_LAYER_NAME = 'divides' - _DIVIDES_CAT_ID_COL = 'id' + _DIVIDES_CAT_ID_COL = 'divide_id' _DIVIDES_TO_NEX_COL = 'toid' _NEXUS_LAYER_NAME = 'nexus' @@ -324,14 +324,17 @@ class GeoPackageHydrofabric(Hydrofabric): _NEXUS_TO_CAT_COL = 'toid' @classmethod - def from_file(cls, geopackage_file: Union[str, Path], vpu: Optional[int] = None, is_conus: bool = False) -> 'GeoPackageHydrofabric': + def from_file(cls, geopackage_file: Union[str, Path, bytes], vpu: Optional[int] = None, is_conus: bool = False) -> 'GeoPackageHydrofabric': """ - Initialize a new instance from a GeoPackage file. + Initialize a new instance from a GeoPackage file or contents of such a file (as ``bytes``). + + Note that while a warning may appear because of implementation details in ``pyogrio``, this should work + perfectly well if passed raw bytes from a file. Parameters ---------- - geopackage_file: Union[str, Path] - The source file for data from which to instantiate. + geopackage_file: Union[str, Path, bytes] + The source file for data, or raw data from such a file, from which to instantiate. vpu: Optional[int] The VPU of the hydrofabric to create, if it is known (defaults to ``None``). is_conus: bool @@ -342,7 +345,10 @@ def from_file(cls, geopackage_file: Union[str, Path], vpu: Optional[int] = None, GeoPackageHydrofabric A new instance of this type. """ - layer_names = fiona.listlayers(geopackage_file) + # TODO: see if fiona dependency can be removed + # pyogrio's function returns an ndarry of ndarrays, with inner layer info array containing layer name and type + # We only need a list of layer names, though + layer_names = [layer_info[0] for layer_info in pyogrio.list_layers(geopackage_file)] return cls(layer_names=layer_names, layer_dataframes={ln: gpd.read_file(geopackage_file, layer=ln, engine="pyogrio") for ln in layer_names}, vpu=vpu, From b254647d8ec28f2db07702f7b8181976bf34fff0 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 2 Apr 2024 09:59:20 -0400 Subject: [PATCH 04/16] Remove dmod.modeldata fiona dependency. --- .../dmod/modeldata/hydrofabric/geopackage_hydrofabric.py | 1 - python/lib/modeldata/setup.py | 1 - 2 files changed, 2 deletions(-) diff --git a/python/lib/modeldata/dmod/modeldata/hydrofabric/geopackage_hydrofabric.py b/python/lib/modeldata/dmod/modeldata/hydrofabric/geopackage_hydrofabric.py index 9f171810b..f7cd63ef5 100644 --- a/python/lib/modeldata/dmod/modeldata/hydrofabric/geopackage_hydrofabric.py +++ b/python/lib/modeldata/dmod/modeldata/hydrofabric/geopackage_hydrofabric.py @@ -345,7 +345,6 @@ def from_file(cls, geopackage_file: Union[str, Path, bytes], vpu: Optional[int] GeoPackageHydrofabric A new instance of this type. """ - # TODO: see if fiona dependency can be removed # pyogrio's function returns an ndarry of ndarrays, with inner layer info array containing layer name and type # We only need a list of layer names, though layer_names = [layer_info[0] for layer_info in pyogrio.list_layers(geopackage_file)] diff --git a/python/lib/modeldata/setup.py b/python/lib/modeldata/setup.py index b9f5ac01b..d551006d1 100644 --- a/python/lib/modeldata/setup.py +++ b/python/lib/modeldata/setup.py @@ -23,7 +23,6 @@ install_requires=[ "numpy>=1.20.1", "pandas", - "fiona", "geopandas", "dmod-communication>=0.4.2", "dmod-core>=0.13.1", From bd9614af964f8baf01b410bafb69b1cbf2ee00b9 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 2 Apr 2024 09:59:36 -0400 Subject: [PATCH 05/16] Remove project fiona dependency. --- requirements.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 60251124d..4d16cb986 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,6 @@ Deprecated cryptography flask pandas -fiona geopandas gitpython python-dotenv @@ -42,7 +41,6 @@ attrs Pillow Jinja2 click -Fiona cligj munch six From d9037af60f722ce904a33a06aa375b9d2db8c9f6 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 2 Apr 2024 10:00:02 -0400 Subject: [PATCH 06/16] Bump dmod.modeldata to 0.10.0. --- python/lib/modeldata/dmod/modeldata/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lib/modeldata/dmod/modeldata/_version.py b/python/lib/modeldata/dmod/modeldata/_version.py index 9272695b3..9d1bb721b 100644 --- a/python/lib/modeldata/dmod/modeldata/_version.py +++ b/python/lib/modeldata/dmod/modeldata/_version.py @@ -1 +1 @@ -__version__ = '0.9.5' +__version__ = '0.10.0' From aa6f20cdf493e9e2b874711f4d33f9e3291d37d2 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 2 Apr 2024 10:03:43 -0400 Subject: [PATCH 07/16] Add/fix data formats and indices. - Add NGEN_GEOPACKAGE_HYDROFABRIC_V2 format for hydrofabrics. - Add HYDROFABRIC_VERSION and HYDROFABRIC_REGION indices for use with NGEN_GEOPACKAGE_HYDROFABRIC_V2. - Add EMPTY format to represent an empty dataset. - Add GENERIC format to represent dataset for more easily dealing with datasets for which the true domain/format has not yet been determined. - Tweaking capitalization in indices mapping and fields for AORC_CSV to match format produced by current ngen-forcing regridding tool --- python/lib/core/dmod/core/meta_data.py | 40 ++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/python/lib/core/dmod/core/meta_data.py b/python/lib/core/dmod/core/meta_data.py index 012d8d74e..0c34e38b3 100644 --- a/python/lib/core/dmod/core/meta_data.py +++ b/python/lib/core/dmod/core/meta_data.py @@ -42,6 +42,10 @@ class StandardDatasetIndex(str, PydanticEnum): """ Index for the name of a data file within a dataset. """ COMPOSITE_SOURCE_ID = (9, str, "COMPOSITE_SOURCE_ID") """ Index for DATA_ID values of source dataset(s) when dataset is composite format and derives from others. """ + HYDROFABRIC_VERSION = (10, str, "HYDROFABRIC_VERSION") + """ Version string for version of the hydrofabric to use (e.g., 2.0.1). """ + HYDROFABRIC_REGION = (11, str, "HYDROFABRIC_REGION") + """ Region string (e.g., conus, vpu01) for the applicable region of the hydrofabric. """ def __new__(cls, index: int, ty: type, name: str): o = str.__new__(cls, name) @@ -90,8 +94,8 @@ class DataFormat(PydanticEnum): index that can be used to distinguish the collections, so that the right data can be identified. """ AORC_CSV = (0, - {StandardDatasetIndex.CATCHMENT_ID: None, StandardDatasetIndex.TIME: ""}, - {"": datetime, "APCP_surface": float, "DLWRF_surface": float, "DSWRF_surface": float, + {StandardDatasetIndex.CATCHMENT_ID: None, StandardDatasetIndex.TIME: "Time"}, + {"Time": datetime, "APCP_surface": float, "DLWRF_surface": float, "DSWRF_surface": float, "PRES_surface": float, "SPFH_2maboveground": float, "TMP_2maboveground": float, "UGRD_10maboveground": float, "VGRD_10maboveground": float, "precip_rate": float}, True @@ -184,6 +188,38 @@ class DataFormat(PydanticEnum): T_ROUTE_CONFIG = (13, {StandardDatasetIndex.DATA_ID: None, StandardDatasetIndex.HYDROFABRIC_ID: None}, None, False) """ Format for t-route application configuration. """ + NGEN_GEOPACKAGE_HYDROFABRIC_V2 = (14, + {StandardDatasetIndex.CATCHMENT_ID: "divide_id", + StandardDatasetIndex.HYDROFABRIC_ID: None, + StandardDatasetIndex.HYDROFABRIC_REGION: None, + StandardDatasetIndex.HYDROFABRIC_VERSION: None}, + {"fid": int, "divide_id": str, "geom": Any, "toid": str, "type": str, + "ds_id": float, "areasqkm": float, "id": str, "lengthkm": float, + "tot_drainage_areasqkm": float, "has_flowline": bool}, + ) + """ GeoPackage hydrofabric format v2 used by NextGen (id is catchment id). """ + + EMPTY = (15, {}, None, False) + """ + "Format" for an empty dataset that, having no data (yet), doesn't have (or need) an applicable defined structure. + + The intent of this is for simplicity when creating dataset. This format represents a type of dataset that doesn't, + and importantly, **cannot** yet truly have a more specific format that matches its contents. A key implication is + an expectation is that the domain of the dataset (including the format) **must** be changed as soon as any data is + added to the dataset. + """ + + GENERIC = (16, {}, None, False) + """ + Format without any indications or restrictions on the defined structure of contained data. + + This value is very much like ``EMPTY`` except that it is applicable to non-empty datasets. It represents absolutely + nothing about the structure of any contents, and thus that absolutely anything can be contained or added. In + practice, the main intended difference from ``EMPTY`` is that datasets in this format will not be required to update + their data domain at the time new data is added (while not applicable to ``EMPTY``, the same is true when any data + is removed). + """ + @classmethod def can_format_fulfill(cls, needed: 'DataFormat', alternate: 'DataFormat') -> bool: """ From 66741315987d409b988130de2594d61e98e49301 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 2 Apr 2024 10:05:57 -0400 Subject: [PATCH 08/16] Improve ContinuousRestriction. - Add docstring for member attributes. - Remove duplicate __hash__ implementation. - Add __eq__ implementation. --- python/lib/core/dmod/core/meta_data.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/python/lib/core/dmod/core/meta_data.py b/python/lib/core/dmod/core/meta_data.py index 0c34e38b3..2c1fcb7c4 100644 --- a/python/lib/core/dmod/core/meta_data.py +++ b/python/lib/core/dmod/core/meta_data.py @@ -361,7 +361,9 @@ class ContinuousRestriction(Serializable): variable: StandardDatasetIndex begin: datetime + """ An inclusive beginning value. """ end: datetime + """ An exclusive end value. """ datetime_pattern: Optional[str] subclass: str = None """ @@ -475,9 +477,6 @@ def convert_truncated_serial_form(cls, truncated_json_obj: dict, datetime_format return json_copy - def __hash__(self) -> int: - return hash((self.variable.name, self.begin, self.end)) - def contains(self, other: 'ContinuousRestriction') -> bool: """ Whether this object contains all the values of the given object and the two are of the same index. @@ -528,6 +527,11 @@ def __init__( if allow_reorder: self.values.sort() + def __eq__(self, other): + if not isinstance(other, DiscreteRestriction): + return False + return self.variable == other.variable and sorted(self.values) == sorted(other.values) + def __hash__(self) -> int: return hash((self.variable.name, *self.values)) From 3b10b39b30f112c60fce9d0a979befda95a3b831 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 2 Apr 2024 10:11:50 -0400 Subject: [PATCH 09/16] Improve DataDomain. - Expand class docstring. - Update validators for restrictions to allow for no restrictions when the data foramt is EMPTY or GENERIC. --- python/lib/core/dmod/core/meta_data.py | 31 +++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/python/lib/core/dmod/core/meta_data.py b/python/lib/core/dmod/core/meta_data.py index 2c1fcb7c4..71a26715d 100644 --- a/python/lib/core/dmod/core/meta_data.py +++ b/python/lib/core/dmod/core/meta_data.py @@ -591,7 +591,22 @@ def is_all_possible_values(self) -> bool: class DataDomain(Serializable): """ - A domain for a dataset, with domain-defining values contained by one or more discrete and/or continuous components. + A domain for some collection of data, with defining values contained by discrete and/or continuous components. + + A definition for the domain of some kind of collection of data. The collection may be something more concrete, like + a ::class:`Dataset` instance, or more abstract, like forcing data sufficient to run a requested model execution. + + The definition consists of details on the structure and content of the data within the collection. Structure is + represented by a ::class:`DataFormat` attribute, and contents are represented by collections of + ::class:`ContinuousRestriction` and ::class:`DiscreteRestriction` objects. + + While a domain may have any number of continuous or discrete restrictions individually, combined it must have at + least one, or validation will fail. + + There is a notion of whether a domain "contains" certain described data. This described data can be a simple + description of some data index and values it, fundamentally the definition of ::class:`ContinuousRestriction` and + ::class:`DiscreteRestriction` objects. The described data can also be more complex, like another fully defined + domain. A function is provided by the type for performing such tests. """ data_format: DataFormat = Field( description="The format for the data in this domain, which contains details like the indices and other data fields." @@ -662,12 +677,17 @@ def handle_type_map(t): @root_validator() def validate_sufficient_restrictions(cls, values): + data_format = values.get("data_format") + if data_format == DataFormat.EMPTY or data_format == DataFormat.GENERIC: + return values continuous_restrictions = values.get("continuous_restrictions", {}) discrete_restrictions = values.get("discrete_restrictions", {}) - if len(continuous_restrictions) + len(discrete_restrictions) == 0: - msg = "Cannot create {} without at least one finite continuous or discrete restriction" - raise RuntimeError(msg.format(cls.__name__)) - return values + if len(continuous_restrictions) + len(discrete_restrictions) > 0: + return values + raise RuntimeError(f"Cannot create {cls.__name__} without at least one finite continuous or discrete " + f"restriction, except when data format is {DataFormat.GENERIC.name} or " + f"{DataFormat.EMPTY.name} (provided value was: " + f"{'None' if data_format is None else data_format.name})") @classmethod def factory_init_from_restriction_collections(cls, data_format: DataFormat, **kwargs) -> 'DataDomain': @@ -876,6 +896,7 @@ def dict( return serial + class DataCategory(PydanticEnum): """ The general category values for different data. From 55654ac20566f0dd3e9c96863d38e432341977fb Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 2 Apr 2024 10:13:45 -0400 Subject: [PATCH 10/16] Improve placement of Dataset __hash__ within file. --- python/lib/core/dmod/core/dataset.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/python/lib/core/dmod/core/dataset.py b/python/lib/core/dmod/core/dataset.py index f1c2d9c17..3b172face 100644 --- a/python/lib/core/dmod/core/dataset.py +++ b/python/lib/core/dmod/core/dataset.py @@ -198,6 +198,19 @@ def cond_eq(a, b): and cond_eq(self.uuid, other.uuid) ) + def __hash__(self): + members = [ + self.__class__.__name__, + self.name, + self.category.name, + str(hash(self.data_domain)), + self.access_location, + str(self.is_read_only), + str(hash(self.created_on)), + ] + description = ",".join(members) + return hash(description) + @property def manager(self) -> Optional[DatasetManager]: """ @@ -242,19 +255,6 @@ def set_manager(self, value: Union[DatasetManager, None]): self._manager = value self.manager_uuid = value.uuid - def __hash__(self): - members = [ - self.__class__.__name__, - self.name, - self.category.name, - str(hash(self.data_domain)), - self.access_location, - str(self.is_read_only), - str(hash(self.created_on)), - ] - description = ",".join(members) - return hash(description) - def _set_expires(self, new_expires: datetime): """ "Private" function to set the ::attribute:`expires` property. From 4b7063a6b38d0adeefd88df5fcb1ea88148e6f39 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 2 Apr 2024 10:14:38 -0400 Subject: [PATCH 11/16] Fix DatasetManager is_managed_dataset function. Fixing function to actually have return statement. Also removing outdated TODO within the class. --- python/lib/core/dmod/core/dataset.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/python/lib/core/dmod/core/dataset.py b/python/lib/core/dmod/core/dataset.py index 3b172face..67db8795e 100644 --- a/python/lib/core/dmod/core/dataset.py +++ b/python/lib/core/dmod/core/dataset.py @@ -529,8 +529,6 @@ def __init__(self, uuid: Optional[UUID] = None, datasets: Optional[Dict[str, Dat self._errors = [] """ A property attribute to hold errors encountered during operations. """ - # TODO: implement functions and routines for scrubbing temporary datasets as needed - @abstractmethod def add_data(self, dataset_name: str, dest: str, data: Optional[Union[bytes, Reader]] = None, source: Optional[str] = None, is_temp: bool = False, **kwargs) -> bool: @@ -848,7 +846,7 @@ def is_managed_dataset(self, dataset: Dataset) -> bool: if dataset.manager is None and self.uuid == dataset.manager_uuid: dataset.set_manager(self) - return + return dataset.manager_uuid == self.uuid def link_user(self, user: DatasetUser, dataset: Dataset) -> bool: """ From e30d7a0b0c2bcfa45b7f376c1ab7b670e4e32219 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 2 Apr 2024 10:14:52 -0400 Subject: [PATCH 12/16] Bump dmod.core to 0.14.0. --- python/lib/core/dmod/core/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lib/core/dmod/core/_version.py b/python/lib/core/dmod/core/_version.py index deea98b42..ef9199407 100644 --- a/python/lib/core/dmod/core/_version.py +++ b/python/lib/core/dmod/core/_version.py @@ -1 +1 @@ -__version__ = '0.13.1' +__version__ = '0.14.0' From 6553964a7186b604597049eba0a0fcd99c8ba48d Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 3 Apr 2024 16:07:52 -0400 Subject: [PATCH 13/16] Fix subsetting for new geopackage cat id column. --- .../dmod/modeldata/hydrofabric/geopackage_hydrofabric.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/lib/modeldata/dmod/modeldata/hydrofabric/geopackage_hydrofabric.py b/python/lib/modeldata/dmod/modeldata/hydrofabric/geopackage_hydrofabric.py index f7cd63ef5..8bf6d70e7 100644 --- a/python/lib/modeldata/dmod/modeldata/hydrofabric/geopackage_hydrofabric.py +++ b/python/lib/modeldata/dmod/modeldata/hydrofabric/geopackage_hydrofabric.py @@ -447,8 +447,8 @@ def get_subset_hydrofabric(self, subset: SubsetDefinition) -> 'GeoPackageHydrofa # Value[1]: callable no arg function, returning collection of ids for records/rows to include in subset subset_query_setups: Dict[str, Tuple[str, Callable[[], Iterable[str]]]] = { 'flowpaths': ('realized_catchment', lambda: subset.catchment_ids), - 'divides': ('id', lambda: subset.catchment_ids), - 'nexus': ('id', lambda: subset.nexus_ids), + 'divides': (self._DIVIDES_CAT_ID_COL, lambda: subset.catchment_ids), + 'nexus': (self._NEXUS_NEX_ID_COL, lambda: subset.nexus_ids), 'flowpath_attributes': ('id', lambda: new_dfs['flowpaths']['id']), 'flowpath_edge_list': ('id', lambda: new_dfs['flowpaths']['id']), 'crosswalk': ('id', lambda: new_dfs['flowpaths']['id']), From fd4034d5f1715ed88df9e90b64ca848f13678450 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 3 Apr 2024 16:18:42 -0400 Subject: [PATCH 14/16] Add Seeker protocol definition. --- python/lib/core/dmod/core/common/reader.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/python/lib/core/dmod/core/common/reader.py b/python/lib/core/dmod/core/common/reader.py index 2b7ddb476..ebe6ab719 100644 --- a/python/lib/core/dmod/core/common/reader.py +++ b/python/lib/core/dmod/core/common/reader.py @@ -1,4 +1,5 @@ from typing_extensions import Protocol, runtime_checkable +from os import SEEK_SET @runtime_checkable @@ -7,6 +8,12 @@ def read(self, size: int = -1, /) -> bytes: """EOF if empty b''.""" +@runtime_checkable +class Seeker(Protocol): + def seek(self, offset: int, whence: int = SEEK_SET): + """ Change the position to the given offset. """ + + class RepeatableReader(Reader): """ Extension of ::class:`Reader` that provides a reset mechanism that allows its data can be read multiple times. From 59989188f4f0e8a6287870d820e49654ebe73f74 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 3 Apr 2024 16:23:34 -0400 Subject: [PATCH 15/16] Remove RepeatableReader and create ReadSeeker. --- python/lib/core/dmod/core/common/reader.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/python/lib/core/dmod/core/common/reader.py b/python/lib/core/dmod/core/common/reader.py index ebe6ab719..a89ffb705 100644 --- a/python/lib/core/dmod/core/common/reader.py +++ b/python/lib/core/dmod/core/common/reader.py @@ -10,14 +10,11 @@ def read(self, size: int = -1, /) -> bytes: @runtime_checkable class Seeker(Protocol): - def seek(self, offset: int, whence: int = SEEK_SET): - """ Change the position to the given offset. """ + def seek(self, offset: int, whence: int = SEEK_SET) -> int: + """ Change the position to the given offset, returning the absolute position. """ -class RepeatableReader(Reader): +class ReadSeeker(Reader, Seeker): """ - Extension of ::class:`Reader` that provides a reset mechanism that allows its data can be read multiple times. + A :class:`Reader` capable of changing the position from which it is reading. """ - - def reset(self): - """ Reset such that ::method:`read` returns to the start of the data this instance reads. """ From c552c3b8bb82f730bfae0fa53009dc113838bc61 Mon Sep 17 00:00:00 2001 From: Chris Tubbs Date: Thu, 4 Apr 2024 09:39:26 -0500 Subject: [PATCH 16/16] Update python/lib/core/dmod/core/common/reader.py Co-authored-by: Austin Raney --- python/lib/core/dmod/core/common/reader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/lib/core/dmod/core/common/reader.py b/python/lib/core/dmod/core/common/reader.py index a89ffb705..ea9deec52 100644 --- a/python/lib/core/dmod/core/common/reader.py +++ b/python/lib/core/dmod/core/common/reader.py @@ -14,7 +14,8 @@ def seek(self, offset: int, whence: int = SEEK_SET) -> int: """ Change the position to the given offset, returning the absolute position. """ -class ReadSeeker(Reader, Seeker): +@runtime_checkable +class ReadSeeker(Reader, Seeker, Protocol): """ A :class:`Reader` capable of changing the position from which it is reading. """