From f8d9b28d16a074b3756f291503e37f09758abfe1 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 16 Dec 2025 13:08:19 +0000 Subject: [PATCH 01/13] switch from property to method --- activestorage/active.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/activestorage/active.py b/activestorage/active.py index 7ffe141c..a7da2277 100644 --- a/activestorage/active.py +++ b/activestorage/active.py @@ -362,19 +362,25 @@ def method(self, value): self._method = value - @property - def mean(self): + + def mean(self, axis=None): self._method = "mean" + if axis is not None: + self.axis = axis return self - @property - def min(self): + + def min(self, axis=None): self._method = "min" + if axis is not None: + self.axis = axis return self - @property - def max(self): + + def max(self, axis=None): self._method = "max" + if axis is not None: + self.axis = axis return self @property From 59d54b3843c418b81a77cff2a0529ac6ed296988 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 16 Dec 2025 13:08:45 +0000 Subject: [PATCH 02/13] start fixing tests --- tests/test_bigger_data.py | 12 ++++++------ tests/unit/test_active_axis.py | 23 +++++++++++++++++++++-- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/tests/test_bigger_data.py b/tests/test_bigger_data.py index a305dec1..99446489 100644 --- a/tests/test_bigger_data.py +++ b/tests/test_bigger_data.py @@ -136,7 +136,7 @@ def test_cl_mean(tmp_path): active = Active(ncfile, "cl", storage_type=utils.get_storage_type()) active._version = 2 active.components = True - result2 = active.mean[4:5, 1:2] + result2 = active.mean()[4:5, 1:2] print(result2, ncfile) # expect {'sum': array([[[[264.]]]], dtype=float32), 'n': array([[[[12]]]])} # check for typing and structure @@ -151,7 +151,7 @@ def test_cl_min(tmp_path): ncfile = save_cl_file_with_a(tmp_path) active = Active(ncfile, "cl", storage_type=utils.get_storage_type()) active._version = 2 - result2 = active.min[4:5, 1:2] + result2 = active.min()[4:5, 1:2] np.testing.assert_array_equal(result2, np.array([[[[22.]]]], dtype="float32")) @@ -160,7 +160,7 @@ def test_cl_max(tmp_path): ncfile = save_cl_file_with_a(tmp_path) active = Active(ncfile, "cl", storage_type=utils.get_storage_type()) active._version = 2 - result2 = active.max[4:5, 1:2] + result2 = active.max()[4:5, 1:2] np.testing.assert_array_equal(result2, np.array([[[[22.]]]], dtype="float32")) @@ -169,7 +169,7 @@ def test_cl_global_max(tmp_path): ncfile = save_cl_file_with_a(tmp_path) active = Active(ncfile, "cl", storage_type=utils.get_storage_type()) active._version = 2 - result2 = active.max[:] + result2 = active.max()[:] np.testing.assert_array_equal(result2, np.array([[[[22.]]]], dtype="float32")) @@ -192,7 +192,7 @@ def test_ps(tmp_path): active = Active(ncfile, "ps", storage_type=utils.get_storage_type()) active._version = 2 active.components = True - result2 = active.mean[4:5, 1:2] + result2 = active.mean()[4:5, 1:2] print(result2, ncfile) # expect {'sum': array([[[22.]]]), 'n': array([[[4]]])} # check for typing and structure @@ -381,7 +381,7 @@ def test_daily_data_masked_two_stats(test_data_path): # first a mean active = Active(uri, "ta", storage_type=utils.get_storage_type()) active._version = 2 - result2 = active.min[:] + result2 = active.min()[:] assert result2 == 245.0020751953125 # then recycle Active object for something else diff --git a/tests/unit/test_active_axis.py b/tests/unit/test_active_axis.py index 3bbc085f..05712394 100644 --- a/tests/unit/test_active_axis.py +++ b/tests/unit/test_active_axis.py @@ -76,14 +76,33 @@ def test_active_axis_format_1(): active1 = Active(rfile, ncvar, axis=[0, 2]) active2 = Active(rfile, ncvar, axis=(-1, -3)) - x1 = active2.mean[...] - x2 = active2.mean[...] + x1 = active2.mean()[...] + x2 = active2.mean()[...] assert x1.shape == x2.shape assert (x1.mask == x2.mask).all() assert np.ma.allclose(x1, x2) +def test_active_axis_format_new_api(): + """Unit test for class:Active axis format with Numpy-style API.""" + active1 = Active(rfile, ncvar) + active2 = Active(rfile, ncvar) + + x1 = active2.mean(axis=[0, 2])[...] + assert active2.axis == [0, 2] + x2 = active2.mean(axis=(-1, -3))[...] + assert active2.axis == (-1, -3) + + assert x1.shape == x2.shape + assert (x1.mask == x2.mask).all() + assert np.ma.allclose(x1, x2) + + xmin = active2.min(axis=[0, 2])[...] + xmax = active2.min(axis=[0, 2])[...] + assert xmin == xmax == [[[198.82859802246094]]] + + def test_active_axis_format_2(): """Unit test for class:Active axis format.""" # Disallow out-of-range axes From 554fe6fd88d38b13ac6c275e94ca0b46d2115b77 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 16 Dec 2025 13:12:16 +0000 Subject: [PATCH 03/13] more test fixing --- tests/test_real_https.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_real_https.py b/tests/test_real_https.py index 5a718c35..2bb6f7cf 100644 --- a/tests/test_real_https.py +++ b/tests/test_real_https.py @@ -15,7 +15,7 @@ def test_https(): active = Active(test_file_uri, "cl", storage_type="https") active._version = 1 - result = active.min[0:3, 4:6, 7:9] + result = active.min()[0:3, 4:6, 7:9] print("Result is", result) assert result == np.array([0.6909787], dtype="float32") @@ -26,7 +26,7 @@ def test_https_100years(): test_file_uri = "https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/CMIP/MOHC/UKESM1-1-LL/historical/r1i1p1f2/Amon/pr/gn/latest/pr_Amon_UKESM1-1-LL_historical_r1i1p1f2_gn_195001-201412.nc" active = Active(test_file_uri, "pr") active._version = 1 - result = active.min[0:3, 4:6, 7:9] + result = active.min()[0:3, 4:6, 7:9] print("Result is", result) assert result == np.array([5.4734613e-07], dtype="float32") @@ -43,7 +43,7 @@ def test_https_reductionist(): with pytest.raises(activestorage.reductionist.ReductionistError): active = Active(test_file_uri, "cl") active._version = 2 - result = active.min[0:3, 4:6, 7:9] + result = active.min()[0:3, 4:6, 7:9] print("Result is", result) assert result == np.array([0.6909787], dtype="float32") @@ -57,7 +57,7 @@ def test_https_implicit_storage(): active = Active(test_file_uri, "cl") active._version = 1 - result = active.min[0:3, 4:6, 7:9] + result = active.min()[0:3, 4:6, 7:9] print("Result is", result) assert result == np.array([0.6909787], dtype="float32") @@ -73,7 +73,7 @@ def test_https_implicit_storage_file_not_found(): with pytest.raises(FileNotFoundError): active = Active(test_file_uri, "cl") active._version = 1 - result = active.min[0:3, 4:6, 7:9] + result = active.min()[0:3, 4:6, 7:9] def test_https_implicit_storage_wrong_url(): @@ -98,7 +98,7 @@ def test_https_dataset(): active = Active(av, storage_type="https") active._version = 1 - result = active.min[0:3, 4:6, 7:9] + result = active.min()[0:3, 4:6, 7:9] print("Result is", result) assert result == np.array([0.6909787], dtype="float32") @@ -114,6 +114,6 @@ def test_https_dataset_implicit_storage(): active = Active(av) active._version = 1 - result = active.min[0:3, 4:6, 7:9] + result = active.min()[0:3, 4:6, 7:9] print("Result is", result) assert result == np.array([0.6909787], dtype="float32") From 8c82b698f8fc45498b879b9d6a3641536e8b1c78 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 16 Dec 2025 13:14:07 +0000 Subject: [PATCH 04/13] more test fixing --- tests/unit/test_active.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_active.py b/tests/unit/test_active.py index 30453e5b..9c30b12f 100644 --- a/tests/unit/test_active.py +++ b/tests/unit/test_active.py @@ -112,7 +112,7 @@ def test_activevariable_pyfive_with_attributed_min(): ncvar = "TREFHT" ds = pyfive.File(uri)[ncvar] av = Active(ds) - av_slice_min = av.min[3:5] + av_slice_min = av.min()[3:5] assert av_slice_min == np.array(258.62814, dtype="float32") # test with Numpy np_slice_min = np.min(ds[3:5]) @@ -125,7 +125,7 @@ def test_activevariable_pyfive_with_attributed_mean(): ds = pyfive.File(uri)[ncvar] av = Active(ds) av.components = True - av_slice_min = av.mean[3:5] + av_slice_min = av.mean()[3:5] actual_mean = av_slice_min["sum"] / av_slice_min["n"] assert actual_mean == np.array(283.39508056640625, dtype="float32") # test with Numpy From 934b01380e52e06071cc59f0b42d0f9f548a8d03 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 16 Dec 2025 13:15:47 +0000 Subject: [PATCH 05/13] final test fixing --- tests/test_real_s3.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_real_s3.py b/tests/test_real_s3.py index 70e774f1..8821a9a7 100644 --- a/tests/test_real_s3.py +++ b/tests/test_real_s3.py @@ -39,7 +39,7 @@ def test_s3_dataset(): storage_options=storage_options, active_storage_url=active_storage_url) active._version = 2 - result = active.min[0:3, 4:6, 7:9] # standardized slice + result = active.min()[0:3, 4:6, 7:9] # standardized slice print("Result is", result) assert result == 5098.625 @@ -49,7 +49,7 @@ def test_s3_dataset(): storage_options=storage_options, active_storage_url=active_storage_url) active._version = 2 - result = active.min[0:3, 4:6, 7:9] # standardized slice + result = active.min()[0:3, 4:6, 7:9] # standardized slice print("Result is", result) assert result == 5098.625 @@ -63,7 +63,7 @@ def test_s3_dataset(): storage_options=storage_options, active_storage_url=active_storage_url) active._version = 2 - result = active.min[0:3, 4:6, 7:9] # standardized slice + result = active.min()[0:3, 4:6, 7:9] # standardized slice print("Result is", result) assert result == 5098.625 @@ -72,6 +72,6 @@ def test_s3_dataset(): storage_options=storage_options, active_storage_url=active_storage_url) active._version = 2 - result = active.min[0:3, 4:6, 7:9] # standardized slice + result = active.min()[0:3, 4:6, 7:9] # standardized slice print("Result is", result) assert result == 5098.625 From 1f47bcbdd6c648610996cca43a67f55429a2717c Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 16 Dec 2025 13:33:41 +0000 Subject: [PATCH 06/13] real s3 axis test --- tests/test_real_s3_with_axes.py | 54 +++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 tests/test_real_s3_with_axes.py diff --git a/tests/test_real_s3_with_axes.py b/tests/test_real_s3_with_axes.py new file mode 100644 index 00000000..b81c221f --- /dev/null +++ b/tests/test_real_s3_with_axes.py @@ -0,0 +1,54 @@ +import os +import numpy as np + +from activestorage.active import Active + + +S3_BUCKET = "bnl" + +def build_active(): + """Run an integration test with real data off S3.""" + storage_options = { + 'key': "f2d55c6dcfc7618b2c34e00b58df3cef", + 'secret': "$/'#M{0{/4rVhp%n^(XeX$q@y#&(NM3W1->~N.Q6VP.5[@bLpi='nt]AfH)>78pT", + 'client_kwargs': {'endpoint_url': "https://uor-aces-o.s3-ext.jc.rl.ac.uk"}, # final proxy + } + active_storage_url = "https://reductionist.jasmin.ac.uk/" # Wacasoft new Reductionist + bigger_file = "da193a_25_6hr_t_pt_cordex__198807-198807.nc" # m01s30i111 ## older 3GB 30 chunks + + test_file_uri = os.path.join( + S3_BUCKET, + bigger_file + ) + print("S3 Test file path:", test_file_uri) + active = Active(test_file_uri, 'm01s30i111', storage_type="s3", # 'm01s06i247_4', storage_type="s3", + storage_options=storage_options, + active_storage_url=active_storage_url) + + active._version = 2 + + return active + + +def test_no_axis(): + active = build_active() + result = active.min()[:] + assert result == [[[[164.8125]]]] + + +def test_no_axis_2(): + active = build_active() + result = active.min(axis=())[:] + assert result == [[[[164.8125]]]] + + +def test_axis_0_1(): + active = build_active() + result = active.min(axis=(0, 1))[:] + assert result == [[[[164.8125]]]] + + +def test_no_axis_1(): + active = build_active() + result = active.min(axis=(1))[:] + assert result == [[[[164.8125]]]] From b88a55251896135bc8a0dd1f2b672061c2ffd6d7 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 16 Dec 2025 13:33:52 +0000 Subject: [PATCH 07/13] turn on axis --- activestorage/reductionist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/activestorage/reductionist.py b/activestorage/reductionist.py index 66240dd5..4cc4089d 100644 --- a/activestorage/reductionist.py +++ b/activestorage/reductionist.py @@ -10,7 +10,7 @@ import numpy as np import requests -REDUCTIONIST_AXIS_READY = False +REDUCTIONIST_AXIS_READY = True DEBUG = 0 From 23acfde900c13cf47c56c66078674065df832afb Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 16 Dec 2025 13:50:22 +0000 Subject: [PATCH 08/13] add axis to mock expected --- tests/unit/test_reductionist.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/unit/test_reductionist.py b/tests/unit/test_reductionist.py index 4be797db..3b06670e 100644 --- a/tests/unit/test_reductionist.py +++ b/tests/unit/test_reductionist.py @@ -149,6 +149,8 @@ def test_reduce_chunk_compression(mock_request, compression, filters): "id": filter.codec_id, "element_size": filter.elementsize } for filter in filters], + "axis": + axis, } mock_request.assert_called_once_with(session, expected_url, expected_data) @@ -258,6 +260,8 @@ def test_reduce_chunk_missing(mock_request, missing): ]], "missing": api_arg, + "axis": + axis, } mock_request.assert_called_once_with(session, expected_url, expected_data) From 29fcaee95e2e0a530761a6d424b6db704493686b Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 16 Dec 2025 14:40:39 +0000 Subject: [PATCH 09/13] fix bug --- activestorage/active.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/activestorage/active.py b/activestorage/active.py index a7da2277..220212b7 100644 --- a/activestorage/active.py +++ b/activestorage/active.py @@ -272,6 +272,7 @@ def __load_nc_file(self): nc = load_from_https(self.uri) self.filename = self.uri self.ds = nc[ncvar] + print("Loaded dataset", self.ds) def __get_missing_attributes(self): if self.ds is None: @@ -366,21 +367,21 @@ def method(self, value): def mean(self, axis=None): self._method = "mean" if axis is not None: - self.axis = axis + self._axis = axis return self def min(self, axis=None): self._method = "min" if axis is not None: - self.axis = axis + self._axis = axis return self def max(self, axis=None): self._method = "max" if axis is not None: - self.axis = axis + self._axis = axis return self @property From 7298051e7fd75e41f2ea3e208639b1989db77749 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 16 Dec 2025 14:40:54 +0000 Subject: [PATCH 10/13] turn on some screen printing --- activestorage/reductionist.py | 1 + 1 file changed, 1 insertion(+) diff --git a/activestorage/reductionist.py b/activestorage/reductionist.py index 4cc4089d..e7cff1c0 100644 --- a/activestorage/reductionist.py +++ b/activestorage/reductionist.py @@ -88,6 +88,7 @@ def reduce_chunk(session, chunk_selection, axis, storage_type=storage_type) + print(f"Reductionist request data dictionary: {request_data}") if DEBUG: print(f"Reductionist request data dictionary: {request_data}") api_operation = "sum" if operation == "mean" else operation or "select" From 5ef6146dfca3c3aceab9d8b26b54cb03a7c094e3 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 16 Dec 2025 14:41:13 +0000 Subject: [PATCH 11/13] ix test --- tests/unit/test_active_axis.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/unit/test_active_axis.py b/tests/unit/test_active_axis.py index 05712394..d3d62cf1 100644 --- a/tests/unit/test_active_axis.py +++ b/tests/unit/test_active_axis.py @@ -89,18 +89,19 @@ def test_active_axis_format_new_api(): active1 = Active(rfile, ncvar) active2 = Active(rfile, ncvar) - x1 = active2.mean(axis=[0, 2])[...] - assert active2.axis == [0, 2] + x1 = active2.mean(axis=(0, 2))[...] + assert active2._axis == (0, 2) x2 = active2.mean(axis=(-1, -3))[...] - assert active2.axis == (-1, -3) + assert active2._axis == (-1, -3) assert x1.shape == x2.shape assert (x1.mask == x2.mask).all() assert np.ma.allclose(x1, x2) - xmin = active2.min(axis=[0, 2])[...] - xmax = active2.min(axis=[0, 2])[...] - assert xmin == xmax == [[[198.82859802246094]]] + xmin = active2.min(axis=(0, 2))[...] + xmax = active2.max(axis=(0, 2))[...] + assert xmin[0][0][0] == 209.44680786132812 + assert xmax[0][0][0] == 255.54661560058594 def test_active_axis_format_2(): From f7f69499c66c16d48120d91ba5a4298e961816eb Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 16 Dec 2025 14:41:32 +0000 Subject: [PATCH 12/13] final version of not working Reductionist test --- tests/test_real_s3_with_axes.py | 36 +++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/tests/test_real_s3_with_axes.py b/tests/test_real_s3_with_axes.py index b81c221f..364f83c1 100644 --- a/tests/test_real_s3_with_axes.py +++ b/tests/test_real_s3_with_axes.py @@ -30,25 +30,57 @@ def build_active(): return active +## Active loads a 4dim dataset +## Loaded dataset +## default axis arg (when axis=None): 'axis': (0, 1, 2, 3) + def test_no_axis(): + """ + Fails: it should pass: 'axis': (0, 1, 2, 3) default + are fine! + + activestorage.reductionist.ReductionistError: Reductionist error: HTTP 400: {"error": {"message": "request data is not valid", "caused_by": ["__all__: Validation error: Number of reduction axes must be less than length of shape - to reduce over all axes omit the axis field completely [{}]"]}} + """ active = build_active() result = active.min()[:] assert result == [[[[164.8125]]]] def test_no_axis_2(): + """ + Fails: it should pass: 'axis': (0, 1, 2, 3) default + are fine! + + activestorage.reductionist.ReductionistError: Reductionist error: HTTP 400: {"error": {"message": "request data is not valid", "caused_by": ["__all__: Validation error: Number of reduction axes must be less than length of shape - to reduce over all axes omit the axis field completely [{}]"]}} + """ active = build_active() result = active.min(axis=())[:] assert result == [[[[164.8125]]]] +def test_axis_0(): + """Fails: activestorage.reductionist.ReductionistError: Reductionist error: HTTP 502: -""" + active = build_active() + result = active.min(axis=(0, ))[:] + assert result == [[[[164.8125]]]] + + def test_axis_0_1(): + """Fails: activestorage.reductionist.ReductionistError: Reductionist error: HTTP 502: -""" active = build_active() result = active.min(axis=(0, 1))[:] assert result == [[[[164.8125]]]] -def test_no_axis_1(): +def test_axis_1(): + """Fails: activestorage.reductionist.ReductionistError: Reductionist error: HTTP 502: -""" active = build_active() - result = active.min(axis=(1))[:] + result = active.min(axis=(1, ))[:] assert result == [[[[164.8125]]]] + + +def test_axis_0_1_2(): + """Passes fine.""" + active = build_active() + result = active.min(axis=(0, 1, 2))[:] + assert result[0][0][0][0] == 171.05126953125 From 72f12ac1f22a40e30167094dfcd97e7f5911a3a1 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 16 Dec 2025 15:01:41 +0000 Subject: [PATCH 13/13] too many blank lines --- activestorage/active.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/activestorage/active.py b/activestorage/active.py index 220212b7..6d1bfe59 100644 --- a/activestorage/active.py +++ b/activestorage/active.py @@ -363,21 +363,18 @@ def method(self, value): self._method = value - def mean(self, axis=None): self._method = "mean" if axis is not None: self._axis = axis return self - def min(self, axis=None): self._method = "min" if axis is not None: self._axis = axis return self - def max(self, axis=None): self._method = "max" if axis is not None: