update cloud_optimised lib and impacted dependencies and method

.github/workflows/test.yml

@@ -19,7 +19,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.10.14"
+          python-version: "3.11"
       - name: Set up Poetry
         uses: snok/install-poetry@v1
         with:

.pre-commit-config.yaml

@@ -3,7 +3,7 @@ repos:
     rev: 24.8.0
     hooks:
       - id: black
-        language_version: python3.10
+        language_version: python3.11
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.6.0
     hooks:

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.10-slim
+FROM python:3.11-slim
 
 WORKDIR /app
 RUN useradd -l -m -s /bin/bash appuser

data_access_service/core/api.py

@@ -26,6 +26,7 @@
     STR_LATITUDE_LOWER_CASE,
     STR_LONGITUDE_UPPER_CASE,
     STR_LONGITUDE_LOWER_CASE,
+    STR_TIME_UPPER_CASE,
 )
 from data_access_service.core.descriptor import Depth, Descriptor, Coordinate
 from urllib.parse import unquote_plus
@@ -753,9 +754,14 @@ def get_dataset(
                     lon_mapped = self.map_column_names(
                         uuid, key, [STR_LONGITUDE_UPPER_CASE]
                     )
+                    # we need the actual time field name to create temporal filter for DataQuery from cloud_optimised lib
+                    time_mapped = self.map_column_names(
+                        uuid, key, [STR_TIME_UPPER_CASE]
+                    )
 
                     lat_varname = lat_mapped[0] if lat_mapped else None
                     lon_varname = lon_mapped[0] if lon_mapped else None
+                    time_varname = time_mapped[0] if time_mapped else None
 
                     # Accuracy to nanoseconds
                     result = ds.get_data(
@@ -769,6 +775,7 @@ def get_dataset(
                         self.map_column_names(uuid, key, columns),
                         lat_varname=lat_varname,
                         lon_varname=lon_varname,
+                        time_varname=time_varname,
                     )
 
                     return ddf.from_pandas(

environment.yml

@@ -3,7 +3,7 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-  - python=3.10.14
+  - python=3.11
   - pip<24.1
   - virtualenv==20.28.1 # https://github.com/python-poetry/poetry/issues/10056#issuecomment-2594269592
   - pip:

pyproject.toml

@@ -7,7 +7,7 @@ readme = "README.md"
 
 [tool.poetry.dependencies]
 # Dependency from a GitHub repository, use tag = "v0.1.32" or branch = "xxx"
-aodn_cloud_optimised = { git = "https://github.com/aodn/aodn_cloud_optimised.git", tag = "v0.1.53" }
+aodn_cloud_optimised = { git = "https://github.com/aodn/aodn_cloud_optimised.git", tag = "v0.1.64" }
 black = "^24.8.0"
 blosc = "^1.11.3"
 boto3 = "^1.28.0"
@@ -25,8 +25,8 @@ jsonify = "^0.5"
 logging = "^0.4.9.6"
 matplotlib = "3.10.0"
 pandas = "^2.2.3"
-pyarrow = "17.0.0"
-python = "^3.10.14"
+pyarrow = "22.0.0"
+python = "^3.11"
 python-levenshtein = "^0.25.1"
 psutil = "^7.0.0"
 pre-commit = "^3.8.0"
@@ -41,6 +41,7 @@ coverage = "^7.8.0"
 anyio = "4.9.0"
 duckdb = "^1.3.2"
 apscheduler = "^3.10.4"
+tqdm = ">=4.67.1"
 
 [tool.poetry.group.test.dependencies]
 pytest = "8.4.1"

tests/core/test_with_s3.py

@@ -64,23 +64,29 @@ def setup(self):
         os.environ["PROFILE"] = EnvType.TESTING.value
 
     @pytest.fixture(scope="class")
-    def localstack(self, request) -> Generator[LocalStackContainer, Any, None]:
+    def localstack(self) -> Generator[LocalStackContainer, None, None]:
         """
         Start LocalStack container with SQS and S3 services.
-        using with scope cause the call to localstack.stop() happen
-        automatically
         """
-        with LocalStackContainer(image="localstack/localstack:4.3.0") as localstack:
-            localstack.start()
-            time = wait_for_logs(localstack, "Ready.")
+        container = None
+        try:
+            container = LocalStackContainer(image="localstack/localstack:4.3.0")
+            container.start()
+
+            time = wait_for_logs(container, "Ready.")
             log.info(
-                f"Create localstack S3 at port {localstack.get_url()}, time = {time}"
+                f"Create localstack S3 at port {container.get_url()}, time = {time}"
             )
 
-            # Tier down automatically
-            yield localstack
+            yield container
 
-            log.info(f"Close localstack S3 at port {localstack.get_url()}")
+        finally:
+            if container is not None:
+                try:
+                    log.info(f"Stopping localstack S3 at port {container.get_url()}")
+                    container.stop()
+                except Exception as e:
+                    log.warning(f"Error stopping localstack: {e}")
 
     @pytest.fixture(scope="class")
     def aws_clients(self, localstack):
@@ -150,7 +156,8 @@ def wrapped_get_mapper(path, **kwargs):
         # Other test may have call this get_s3_filesystem() this function is cached and
         # may use different ENDPOINT_URL other than the one above
         # so we need to clear it now before the next call happens
-        DataQuery.get_s3_filesystem.cache_clear()
+        # commented this line as current co lib did not support it.
+        # DataQuery.get_s3_filesystem.cache_clear()
         # Force a load so its cache value use the test value
         DataQuery.get_s3_filesystem()