From 758e056b0e72352683333e86482edea1355aebbf Mon Sep 17 00:00:00 2001 From: carolynmichael19 Date: Wed, 3 Dec 2025 14:35:03 +0000 Subject: [PATCH 1/4] add glacier info attribute to datacube --- dtcg/datacube/update_metadata.py | 1 + dtcg/integration/oggm_bindings.py | 6 ++++++ dtcg/tests/datacube/test_metadata_mapping.py | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/dtcg/datacube/update_metadata.py b/dtcg/datacube/update_metadata.py index 1ad4f78..80a0d2a 100644 --- a/dtcg/datacube/update_metadata.py +++ b/dtcg/datacube/update_metadata.py @@ -126,6 +126,7 @@ def _update_shared_metadata(dataset: xr.Dataset, ds_name: str) -> None: "Components (DTC) Early Development Actions." ), "date_created": datetime.now().isoformat(), + "glacier_info": dataset.attrs.get("glacier_info", {}) } if ds_name == "L1": shared_metadata.update({ diff --git a/dtcg/integration/oggm_bindings.py b/dtcg/integration/oggm_bindings.py index c40787f..b389861 100644 --- a/dtcg/integration/oggm_bindings.py +++ b/dtcg/integration/oggm_bindings.py @@ -962,6 +962,12 @@ def get_eolis_data(self, gdir): """Get gridded data enhanced with CryoTEMPO-EOLIS data.""" with xr.open_dataset(gdir.get_filepath("gridded_data")) as datacube: datacube = datacube.load() + keywords = ('rgi', 'glacier', 'name', 'terminus') + glacier_attributes = { + key: val for key, val in gdir.__dict__.items() + if any(k in key for k in keywords) + } + datacube.attrs.update({"glacier_info": glacier_attributes}) self.datacube_manager.retrieve_prepare_eolis_gridded_data( oggm_ds=datacube, grid=gdir.grid diff --git a/dtcg/tests/datacube/test_metadata_mapping.py b/dtcg/tests/datacube/test_metadata_mapping.py index d799d69..b0ff1b0 100644 --- a/dtcg/tests/datacube/test_metadata_mapping.py +++ b/dtcg/tests/datacube/test_metadata_mapping.py @@ -82,7 +82,7 @@ def test_shared_metadata_attributes_and_crs( mapper = MetadataMapper(temp_metadata_file) result = mapper.update_metadata(test_dataset.copy(), "L1") - for attr in ["Conventions", "title", "summary", "comment", "date_created"]: + for attr in ["Conventions", "title", "summary", "comment", "date_created", "glacier_info"]: assert attr in result.attrs assert result.rio.crs is not None From d15731568ee2984710bc3bb72bb4333fddd5b61e Mon Sep 17 00:00:00 2001 From: gampnico <45390064+gampnico@users.noreply.github.com> Date: Thu, 8 Jan 2026 12:44:00 +0000 Subject: [PATCH 2/4] fix(tests): update tests for the new year --- dtcg/datacube/desp.py | 2 +- dtcg/tests/datacube/test_desp.py | 26 ++++++++++++++++++-------- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/dtcg/datacube/desp.py b/dtcg/datacube/desp.py index 923f041..8f70d60 100644 --- a/dtcg/datacube/desp.py +++ b/dtcg/datacube/desp.py @@ -188,7 +188,7 @@ def process_desp_era5_data( temperature = temperature.compute().data precipitation = precipitation.compute().data - + height = height.compute().data # OK, ready to write diff --git a/dtcg/tests/datacube/test_desp.py b/dtcg/tests/datacube/test_desp.py index 94e11e8..1cdde8c 100644 --- a/dtcg/tests/datacube/test_desp.py +++ b/dtcg/tests/datacube/test_desp.py @@ -40,6 +40,7 @@ def check_desp_api_key(): _has_desp_access = check_desp_api_key() +current_year = int(date.today().year) class Test_DatacubeDespERA5: @@ -72,11 +73,11 @@ def test_get_desp_datastream(self, conftest_boilerplate, monkeypatch): reason="No access to DESP. Check your .netrc file has a valid API key.", ) @pytest.mark.parametrize("arg_frequency", ["monthly", "daily"]) - def test_process_desp_era5_data(self, arg_frequency, hef_gdir): + def test_process_desp_era5_data(self, arg_frequency, hef_gdir, current_year): gdir = hef_gdir - arg_y0, arg_y1 = (2023, 2024) + arg_y0, arg_y1 = (current_year - 2, current_year - 1) desp.process_desp_era5_data( gdir=gdir, frequency=arg_frequency, y0=arg_y0, y1=arg_y1 ) @@ -88,8 +89,8 @@ def test_process_desp_era5_data(self, arg_frequency, hef_gdir): else: assert ds.yr_0 == arg_y0 if not arg_y1: - # This may fail each new year's day - assert ds.yr_1 == int(date.today().year) + # This may fail on new year's day or if January data unavailable. + assert ds.yr_1 == current_year else: assert ds.yr_1 == arg_y1 @@ -98,9 +99,15 @@ def test_process_desp_era5_data(self, arg_frequency, hef_gdir): reason="No access to DESP. Check your .netrc file has a valid API key.", ) @pytest.mark.parametrize( - "arg_years", [(None, 1941), (2023, None), (2023, 2024), (None, None)] + "arg_years", + [ + (None, 1941), + (current_year - 2, None), + (current_year - 2, current_year - 1), + (None, None), + ], ) - def test_process_desp_era5_data_years(self, arg_years, hef_gdir): + def test_process_desp_era5_data_years(self, arg_years, hef_gdir, current_year): gdir = hef_gdir arg_y0, arg_y1 = arg_years @@ -115,7 +122,10 @@ def test_process_desp_era5_data_years(self, arg_years, hef_gdir): else: assert ds.yr_0 == arg_y0 if not arg_y1: - # This may fail each new year's day - assert ds.yr_1 == int(date.today().year) + if int(date.today().month) != 1: + assert ds.yr_1 == current_year + else: + # Data not uploaded until mid-January + assert ds.yr_1 == current_year - 1 else: assert ds.yr_1 == arg_y1 From eedbb25bcd5f2f11f2aea615d26268f401af57c5 Mon Sep 17 00:00:00 2001 From: gampnico <45390064+gampnico@users.noreply.github.com> Date: Thu, 8 Jan 2026 12:45:45 +0000 Subject: [PATCH 3/4] refactor(datacube): rename glacier_info, add attribute keywords --- dtcg/datacube/update_metadata.py | 78 +++++++++++--------- dtcg/integration/oggm_bindings.py | 14 +++- dtcg/tests/conftest.py | 15 ++++ dtcg/tests/datacube/test_metadata_mapping.py | 40 ++++++---- 4 files changed, 95 insertions(+), 52 deletions(-) diff --git a/dtcg/datacube/update_metadata.py b/dtcg/datacube/update_metadata.py index 80a0d2a..726e245 100644 --- a/dtcg/datacube/update_metadata.py +++ b/dtcg/datacube/update_metadata.py @@ -126,52 +126,64 @@ def _update_shared_metadata(dataset: xr.Dataset, ds_name: str) -> None: "Components (DTC) Early Development Actions." ), "date_created": datetime.now().isoformat(), - "glacier_info": dataset.attrs.get("glacier_info", {}) + "glacier_attributes": dataset.attrs.get("glacier_attributes", {}), } if ds_name == "L1": - shared_metadata.update({ - "title": "Datacube of glacier-domain variables.", - "summary": ( - "Resampled glacier-domain variables from multiple sources. " - "Generated for the DTC Glaciers project." - ), - }) + shared_metadata.update( + { + "title": "Datacube of glacier-domain variables.", + "summary": ( + "Resampled glacier-domain variables from multiple sources. " + "Generated for the DTC Glaciers project." + ), + } + ) elif ds_name == "L2": - shared_metadata.update({ - "title": "Datacube of observation-informed modelled variables.", - "summary": ( - "Observation-informed modelled variables. " - "Generated for the DTC Glaciers project." - ), - }) - - dataset.attrs.clear() # clear old metadata + shared_metadata.update( + { + "title": "Datacube of observation-informed modelled variables.", + "summary": ( + "Observation-informed modelled variables. " + "Generated for the DTC Glaciers project." + ), + } + ) + + dataset.attrs.clear() # clear old metadata dataset.attrs.update(shared_metadata) if "x" in dataset.dims: # update coordinate metadata - dataset["x"].attrs.update({ - "standard_name": "projection_x_coordinate", - "long_name": "x coordinate of projection", - "units": "m", - }) + dataset["x"].attrs.update( + { + "standard_name": "projection_x_coordinate", + "long_name": "x coordinate of projection", + "units": "m", + } + ) if "y" in dataset.dims: - dataset["y"].attrs.update({ - "standard_name": "projection_y_coordinate", - "long_name": "y coordinate of projection", - "units": "m", - }) + dataset["y"].attrs.update( + { + "standard_name": "projection_y_coordinate", + "long_name": "y coordinate of projection", + "units": "m", + } + ) if "t" in dataset.dims: # assuming unix epoch - dataset["t"].attrs.update({ - "standard_name": "time", - "long_name": "time since the unix epoch", - "units": "seconds since 1970-01-01 00:00:00", - }) + dataset["t"].attrs.update( + { + "standard_name": "time", + "long_name": "time since the unix epoch", + "units": "seconds since 1970-01-01 00:00:00", + } + ) - def update_metadata(self: MetadataMapper, dataset: xr.Dataset, ds_name: str) -> xr.Dataset: + def update_metadata( + self: MetadataMapper, dataset: xr.Dataset, ds_name: str + ) -> xr.Dataset: """Apply variable and shared metadata to an xarray Dataset. Parameters diff --git a/dtcg/integration/oggm_bindings.py b/dtcg/integration/oggm_bindings.py index b389861..8c451cc 100644 --- a/dtcg/integration/oggm_bindings.py +++ b/dtcg/integration/oggm_bindings.py @@ -962,12 +962,20 @@ def get_eolis_data(self, gdir): """Get gridded data enhanced with CryoTEMPO-EOLIS data.""" with xr.open_dataset(gdir.get_filepath("gridded_data")) as datacube: datacube = datacube.load() - keywords = ('rgi', 'glacier', 'name', 'terminus') + keywords = ( + "rgi", + "glacier", + "name", + "terminus", + "cenl", # latitude, longitude + "glims", + ) glacier_attributes = { - key: val for key, val in gdir.__dict__.items() + key: val + for key, val in gdir.__dict__.items() if any(k in key for k in keywords) } - datacube.attrs.update({"glacier_info": glacier_attributes}) + datacube.attrs.update({"glacier_attributes": glacier_attributes}) self.datacube_manager.retrieve_prepare_eolis_gridded_data( oggm_ds=datacube, grid=gdir.grid diff --git a/dtcg/tests/conftest.py b/dtcg/tests/conftest.py index f6a90d4..4a5cbca 100644 --- a/dtcg/tests/conftest.py +++ b/dtcg/tests/conftest.py @@ -30,6 +30,7 @@ def test_foobar(self, conftest_mock_grid): ... """ +from datetime import date from pathlib import Path from types import ModuleType from typing import Any @@ -55,6 +56,20 @@ def conftest_sample_data_path(pytestconfig): yield test_data_path +@pytest.fixture(name="current_year", scope="function", autouse=False) +def conftest_get_current_year(): + """Yield current year. + + Yields + ------ + int + Current year. + """ + year = int(date.today().year) + assert isinstance(year, int) + yield year + + @pytest.fixture(name="outline_shapefile", scope="function", autouse=False) def conftest_outline_shapefile(sample_data_path): """Yield sample glacier outline. diff --git a/dtcg/tests/datacube/test_metadata_mapping.py b/dtcg/tests/datacube/test_metadata_mapping.py index b0ff1b0..e9519da 100644 --- a/dtcg/tests/datacube/test_metadata_mapping.py +++ b/dtcg/tests/datacube/test_metadata_mapping.py @@ -57,8 +57,7 @@ def fixture_test_dataset(self): ds = xr.Dataset( {"var1": (["y", "x"], data)}, coords={"x": np.arange(3), "y": np.arange(3)}, - attrs={ - "pyproj_srs": "+proj=longlat +datum=WGS84 +no_defs +type=crs"} + attrs={"pyproj_srs": "+proj=longlat +datum=WGS84 +no_defs +type=crs"}, ) return ds @@ -67,8 +66,7 @@ def test_load_metadata(self, temp_metadata_file): assert "var1" in mapper.metadata_mappings assert isinstance(mapper.metadata_mappings["var1"], dict) - def test_apply_metadata_to_variables( - self, temp_metadata_file, test_dataset): + def test_apply_metadata_to_variables(self, temp_metadata_file, test_dataset): mapper = MetadataMapper(temp_metadata_file) result = mapper.update_metadata(test_dataset.copy(), "L1") @@ -76,30 +74,40 @@ def test_apply_metadata_to_variables( for key, val in expected.items(): assert result["var1"].attrs[key] == val - def test_shared_metadata_attributes_and_crs( - self, temp_metadata_file, test_dataset): + def test_shared_metadata_attributes_and_crs(self, temp_metadata_file, test_dataset): # Ensure CRS is preserved or written correctly mapper = MetadataMapper(temp_metadata_file) result = mapper.update_metadata(test_dataset.copy(), "L1") - for attr in ["Conventions", "title", "summary", "comment", "date_created", "glacier_info"]: + for attr in [ + "Conventions", + "title", + "summary", + "comment", + "date_created", + "glacier_attributes", + ]: assert attr in result.attrs assert result.rio.crs is not None - assert CRS.from_user_input(result.rio.crs).equals( - CRS(test_dataset.pyproj_srs)) + assert CRS.from_user_input(result.rio.crs).equals(CRS(test_dataset.pyproj_srs)) def test_warns_on_unmapped_variables(self, temp_metadata_file): - ds = xr.Dataset({ - "var1": (["x", "y"], [[1.0, 2.0], [3.0, 4.0]]), - "var2": (["x", "y"], [[4.0, 5.0], [6.0, 2.0]]), - "var3": (["x", "y"], [[4.0, 5.0], [6.0, 3.0]])}, - attrs={"pyproj_srs": CRS(3413).to_proj4()}) + ds = xr.Dataset( + { + "var1": (["x", "y"], [[1.0, 2.0], [3.0, 4.0]]), + "var2": (["x", "y"], [[4.0, 5.0], [6.0, 2.0]]), + "var3": (["x", "y"], [[4.0, 5.0], [6.0, 3.0]]), + }, + attrs={"pyproj_srs": CRS(3413).to_proj4()}, + ) mapper = MetadataMapper(temp_metadata_file) with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") mapper.update_metadata(ds, "L1") - assert ("Metadata mapping is missing for the following variables: " - "['var2', 'var3']" in str(w[0].message)) + assert ( + "Metadata mapping is missing for the following variables: " + "['var2', 'var3']" in str(w[0].message) + ) From 175d2250ec26160d6f3415adb7ae9bd3be4dca61 Mon Sep 17 00:00:00 2001 From: gampnico <45390064+gampnico@users.noreply.github.com> Date: Thu, 8 Jan 2026 12:46:46 +0000 Subject: [PATCH 4/4] build: update version number --- docs/source/conf.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 7d2ca88..caa27c9 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -30,7 +30,7 @@ def setup(app): project = "DTCG" copyright = f"{date.today().year}, DTCG Contributors" author = "DTCG Contributors" -release = "0.6.0" +release = "0.6.3" version = os.environ.get("READTHEDOCS_VERSION", "latest") # -- General configuration --------------------------------------------------- diff --git a/pyproject.toml b/pyproject.toml index fcf8621..2c8d8d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "dtcg" -version = "0.6.0" +version = "0.6.3" authors = [ { name = "DTCG Contributors", email = "" }, ]