Enable specification of local archive without public access

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 ---
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v6.0.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
@@ -12,57 +12,54 @@ repos:
       - id: check-ast
       - id: check-added-large-files
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.6.2
+    rev: v0.14.9
     hooks:
       - id: ruff
         args: [--fix]
   - repo: https://github.com/psf/black
-    rev: 24.8.0
+    rev: 25.12.0
     hooks:
       - id: black
         language_version: python3
+        additional_dependencies: ["black[jupyter]"]
   - repo: https://github.com/keewis/blackdoc
-    rev: v0.3.9
+    rev: v0.4.6
     hooks:
       - id: blackdoc
         additional_dependencies: ["black[jupyter]"]
-  - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: "v4.0.0-alpha.8"
-    hooks:
-      - id: prettier
-        types_or: [yaml, html, css, scss, javascript, json] # markdown to avoid conflicts with mdformat
+        files: \.(rst)$
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.3.0
+    rev: v2.4.1
     hooks:
       - id: codespell
         types_or: [python, markdown, rst]
         additional_dependencies: [tomli]
-  - repo: https://github.com/asottile/pyupgrade
-    rev: v3.17.0
-    hooks:
-      - id: pyupgrade
+  # - repo: https://github.com/asottile/pyupgrade
+  #   rev: v3.20.0
+  #   hooks:
+  #     - id: pyupgrade
   - repo: https://github.com/MarcoGorelli/madforhooks
     rev: 0.4.1
     hooks:
       # - id: conda-env-sorter # conflicts with prettier
       - id: check-execution-order
   - repo: https://github.com/executablebooks/mdformat
-    rev: 0.7.17
+    rev: 1.0.0
     hooks:
       - id: mdformat
         additional_dependencies: [mdformat-gfm, mdformat-black]
         exclude: ^.github/ # this avoid the hook to wrongly reformats md issue template files
   - repo: https://github.com/kynan/nbstripout
-    rev: 0.7.1
+    rev: 0.8.1
     hooks:
       - id: nbstripout
         args: [--keep-output]
   - repo: https://github.com/nbQA-dev/nbQA
-    rev: 1.8.7
+    rev: 1.9.1
     hooks:
       - id: nbqa-black
-      - id: nbqa-ruff
-        args: [--fix, "--ignore=E402,B018"]
+      # - id: nbqa-ruff
+      #  args: [--fix, "--ignore=E402,B018"]
   - repo: https://github.com/pre-commit/pygrep-hooks
     rev: "v1.10.0"
     hooks:

CODE_OF_CONDUCT.md

@@ -58,7 +58,7 @@ representative at an online or offline event.
 
 Instances of abusive, harassing, or otherwise unacceptable behavior may be
 reported to the community leaders responsible for enforcement at
-\[INSERT CONTACT METHOD\].
+[INSERT CONTACT METHOD].
 All complaints will be reviewed and investigated promptly and fairly.
 
 All community leaders are obligated to respect the privacy and security of the

README.md

@@ -17,7 +17,7 @@
 | Code Quality      | [![Codefactor](https://www.codefactor.io/repository/github/ghiggi/radar_api/badge?style=flat)](https://www.codefactor.io/repository/github/ghiggi/radar_api) [![Codebeat](https://codebeat.co/badges/57498d71-f042-473f-bb8e-9b45e50572d8?style=flat)](https://codebeat.co/projects/github-com-ghiggi-radar_api-main) [![Codacy](https://app.codacy.com/project/badge/Grade/bee842cb10004ad8bb9288256f2fc8af?style=flat)](https://app.codacy.com/gh/ghiggi/radar_api/dashboard?utm_source=gh&utm_medium=referral&utm_content=&utm_campaign=Badge_grade) [![Codescene](https://codescene.io/projects/63299/status-badges/average-code-health?style=flat)](https://codescene.io/projects/63299) |
 | License           | [![License](https://img.shields.io/github/license/ghiggi/radar_api?style=flat)](https://github.com/ghiggi/radar_api/blob/main/LICENSE)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
 | Community         | [![Discourse](https://img.shields.io/badge/Slack-radar_api-green.svg?logo=slack&style=flat)](https://openradar.discourse.group/) [![GitHub Discussions](https://img.shields.io/badge/GitHub-Discussions-green?logo=github&style=flat)](https://github.com/ghiggi/radar_api/discussions)                                                                                                                                                                                                                                                                                                                                                                                                       |
-| Citation          | [![DOI](https://zenodo.org/badge/922589509.svg?style=flat)](https://doi.org/10.5281/zenodo.14743651)                                                                                                                                                                                                                                                                                                                                                                                    </div>                                                                                                                                                                                                |
+| Citation          | [![DOI](https://zenodo.org/badge/922589509.svg?style=flat)](https://doi.org/10.5281/zenodo.14743651) </div>                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   |
 
 [**Documentation: https://radar-api.readthedocs.io**](https://radar-api.readthedocs.io/)
 

pyproject.toml

@@ -25,7 +25,6 @@ classifiers = [
     "Programming Language :: Python",
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3 :: Only",
-    "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
@@ -46,7 +45,7 @@ dependencies = [
     "pandas",
     "tqdm",
 ]
-requires-python = ">=3.10"
+requires-python = ">=3.11"
 dynamic = ["version"]
 
 [tool.setuptools]
@@ -60,6 +59,7 @@ write_to = "radar_api/_version.py"
 
 [project.optional-dependencies]
 dev = ["pre-commit", "loghub",
+       "xarray", "xradar",
        "black[jupyter]", "blackdoc", "codespell", "ruff",
        "pytest", "pytest-cov", "pytest-mock", "pytest-check", "pytest-sugar",
        "pytest-watcher", "deepdiff",
@@ -186,9 +186,8 @@ ignore = [
     "B006", # {} defaults in function arguments
     "PT011", # pytest raised error must be checked if match the expected error msg
     "PERF203",
-    "UP038",
+    "PLC0415", # import not at the top-level file
     # "PD011", # suggest values --> to_numpy
-    "PD901",
     "PD013", # suggest melt instead of stack
     "PLW2901",
     "PLW0603",

radar_api/__init__.py

@@ -39,6 +39,7 @@
 from radar_api.info import group_filepaths
 from radar_api.io import (
     available_networks,
+    available_products,
     available_radars,
 )
 from radar_api.readers import (
@@ -52,17 +53,18 @@
 
 
 __all__ = [
-    "available_radars",
     "available_networks",
+    "available_products",
+    "available_radars",
     "config",
     "define_configs",
-    "read_configs",
+    "download_files",
     "find_files",
     "group_filepaths",
-    "open_datatree",
     "open_dataset",
+    "open_datatree",
     "open_pyart",
-    "download_files",
+    "read_configs",
 ]
 
 # Get version

radar_api/checks.py

@@ -27,7 +27,6 @@
 import datetime
 import os
 import pathlib
-import sys
 
 import numpy as np
 
@@ -37,9 +36,7 @@
 
 def get_current_utc_time():
     """Return current UTC time."""
-    if sys.version_info >= (3, 11):
-        return datetime.datetime.now(datetime.UTC).replace(tzinfo=None)
-    return datetime.datetime.utcnow()
+    return datetime.datetime.now(datetime.UTC).replace(tzinfo=None)
 
 
 def check_protocol(protocol):
@@ -85,7 +82,7 @@ def check_radar(radar, network):
     if not isinstance(radar, str):
         raise TypeError("Specify 'radar' as a string.")
     check_network(network)
-    valid_radars = available_radars()
+    valid_radars = available_radars(only_online=False)
     if radar not in valid_radars:
         raise ValueError(f"Invalid {network} radar {radar}. Available radars: {valid_radars}")
     return radar
@@ -98,12 +95,29 @@ def check_network(network):
     if not isinstance(network, str):
         raise TypeError("Specify 'network' as a string.")
 
-    valid_networks = available_networks()
+    valid_networks = available_networks(only_online=False)
     if network not in valid_networks:
         raise ValueError(f"Invalid network {network}. Available networks: {valid_networks}")
     return network
 
 
+def check_product(network, product=None):
+    """Check product validity.
+
+    If only one product available for that network, return that.
+    """
+    from radar_api.io import available_products
+
+    valid_products = available_products(network, only_online=False)
+    if product is None:
+        if len(valid_products) == 1:
+            return valid_products[0]
+        raise ValueError(f"{network} has {valid_products} products available. 'product' must be specified.")
+    if product not in valid_products:
+        raise ValueError(f"Invalid product {product} for {network}. Available products: {valid_products}")
+    return product
+
+
 def check_time(time):
     """Check time validity.
 

radar_api/configs.py

@@ -27,6 +27,7 @@
 """RADAR-API configurations settings."""
 import os
 
+import radar_api
 from radar_api.utils.yaml import read_yaml, write_yaml
 
 
@@ -75,6 +76,11 @@ def define_configs(
 
     print(f"The RADAR-API config file has been {action_msg} successfully!")
 
+    # Now read the config file and set it as the active configuration
+    # - This avoid the need to restart a python session to take effect !
+    config_dict = read_configs()
+    radar_api.config.update(config_dict)
+
 
 def read_configs() -> dict[str, str]:
     """Reads the RADAR-API configuration file and returns a dictionary with the configuration settings.
@@ -110,8 +116,6 @@ def read_configs() -> dict[str, str]:
 ####--------------------------------------------------------------------------.
 def _get_config_key(key):
     """Return the config key."""
-    import radar_api
-
     value = radar_api.config.get(key, None)
     if value is None:
         raise ValueError(f"The '{key}' is not specified in the RADAR-API configuration file.")
@@ -120,8 +124,6 @@ def _get_config_key(key):
 
 def get_base_dir(base_dir=None):
     """Return the RADAR-API base directory."""
-    import radar_api
-
     if base_dir is None:
         base_dir = radar_api.config.get("base_dir")
     if base_dir is None:

radar_api/download.py

@@ -41,6 +41,7 @@
     check_base_dir,
     check_download_protocol,
     check_network,
+    check_product,
     check_radar,
     check_start_end_time,
 )
@@ -115,13 +116,13 @@ def _select_missing_fpaths(local_fpaths, bucket_fpaths):
     return local_fpaths, bucket_fpaths
 
 
-def define_local_filepath(filename, network, radar, base_dir=None):
+def define_local_filepath(filename, network, product, radar, base_dir=None):
     """Define filepath where to save file locally on disk."""
     base_dir = get_base_dir(base_dir)
     base_dir = check_base_dir(base_dir)
     # Get directory pattern
-    directory_pattern = get_directory_pattern(protocol="local", network=network)
-    info_dict = get_info_from_filepath(filename, network=network)
+    directory_pattern = get_directory_pattern(protocol="local", network=network, product=product)
+    info_dict = get_info_from_filepath(filename, network=network, product=product)
     time = info_dict["start_time"]
     # Define local directory path
     parser = Parser(directory_pattern)
@@ -133,10 +134,16 @@ def define_local_filepath(filename, network, radar, base_dir=None):
     return filepath
 
 
-def _get_local_from_bucket_fpaths(base_dir, network, radar, bucket_fpaths):
+def _get_local_from_bucket_fpaths(base_dir, network, product, radar, bucket_fpaths):
     """Convert cloud bucket filepaths to local storage filepaths."""
     fpaths = [
-        define_local_filepath(filename=os.path.basename(fpath), network=network, radar=radar, base_dir=base_dir)
+        define_local_filepath(
+            filename=os.path.basename(fpath),
+            network=network,
+            product=product,
+            radar=radar,
+            base_dir=base_dir,
+        )
         for fpath in bucket_fpaths
     ]
     return fpaths
@@ -236,6 +243,7 @@ def download_files(
     radar,
     start_time,
     end_time,
+    product=None,
     n_threads=20,
     force_download=False,
     check_data_integrity=True,
@@ -256,6 +264,11 @@ def download_files(
     network : str
         The name of the radar network.
         See `radar_api.available_network()` for available radar networks.
+    product: str
+        The product acronym. The default is None.
+        It must be specified if for a given network, multiple products are available
+        through radar_api.
+        See `radar_api.available_products(network)` for available products.
     start_time : datetime.datetime
         The start (inclusive) time of the interval period for retrieving the filepaths.
     end_time : datetime.datetime
@@ -294,6 +307,7 @@ def download_files(
     base_dir = check_base_dir(base_dir)
     network = check_network(network)
     radar = check_radar(radar=radar, network=network)
+    product = check_product(network=network, product=product)
     start_time, end_time = check_start_end_time(start_time, end_time)
 
     # Initialize timing
@@ -323,6 +337,7 @@ def download_files(
             fs_args=fs_args,
             radar=radar,
             network=network,
+            product=product,
             start_time=start_time,
             end_time=end_time,
             base_dir=None,
@@ -339,6 +354,7 @@ def download_files(
             base_dir=base_dir,
             network=network,
             radar=radar,
+            product=product,
             bucket_fpaths=bucket_fpaths,
         )
 

radar_api/etc/network/FMI.yaml → radar_api/etc/network/FMI/PVOL.yaml

@@ -1,5 +1,6 @@
 network: "FMI"
 description: "Finnish Meteorological Institute radars"
+public_data: True
 cloud_directory_pattern: "s3://fmi-opendata-radar-volume-hdf5/{time:%Y}/{time:%m}/{time:%d}/{radar:s}"
 local_directory_pattern: "{base_dir}/FMI/{time:%Y}/{time:%m}/{time:%d}/{time:%H}/{radar:s}"
 filename_patterns:

radar_api/etc/network/IDEAM.yaml → radar_api/etc/network/IDEAM/IDEAM_L2.yaml

@@ -1,5 +1,6 @@
 network: "IDEAM"
 description: "Colombian weather radar network"
+public_data: True
 cloud_directory_pattern: "s3://s3-radaresideam/l2_data/{time:%Y}/{time:%m}/{time:%d}/{radar:s}"
 local_directory_pattern: "{base_dir}/IDEAM/{time:%Y}/{time:%m}/{time:%d}/{time:%H}/{radar:s}"
 filename_patterns:

radar_api/etc/network/MCH_CSCS/POL.yaml

@@ -0,0 +1,10 @@
+network: MCH_CSCS
+description: MeteoSwiss Rad4Alp network
+public_data: False
+cloud_directory_pattern: null
+local_directory_pattern: "/store_new/mch/msrad/radar/swiss/data/{time:%Y}/{time:%y}{time:%j}/ML{radar:1s}{time:%y}{time:%j}.zip"
+filename_patterns:
+  - "{radar_acronym:3s}{start_time:%y%j%H%M}0U.{volume_identifier:3s}"
+pyart_reader: read_metranet
+xradar_reader: null
+xradar_engine: null

radar_api/etc/network/MCH_LTE/HYM.yaml

@@ -0,0 +1,10 @@
+network: MCH_LTE
+description: MeteoSwiss Rad4Alp network
+public_data: False
+cloud_directory_pattern: null
+local_directory_pattern: "{base_dir}/MCH/{time:%Y}/{time:%m}/{time:%d}/{time:%H}/YM{radar:1s}"
+filename_patterns:
+  - "YM{radar_acronym:1s}{start_time:%y%j%H%M}0L.8{sweep_identifier:2s}"
+pyart_reader: read_metranet
+xradar_reader: null
+xradar_engine: null

radar_api/etc/network/MCH_LTE/HZT.yaml

@@ -0,0 +1,10 @@
+network: MCH_LTE
+description: MeteoSwiss Rad4Alp network
+public_data: False
+cloud_directory_pattern: null
+local_directory_pattern: "{base_dir}/MCH/{time:%Y}/{time:%m}/{time:%d}/{time:%H}/HZT"
+filename_patterns:
+  - "HZT{start_time:%y%j%H%M}0L.{volume_identifier:3s}" # volume_identifier=800
+pyart_reader: read_metranet
+xradar_reader: null
+xradar_engine: null