From 06cd31f75764d50cf19e470f33d7bf8f4d979748 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Thu, 25 Sep 2025 23:33:25 +0300 Subject: [PATCH 01/71] New ArgoSensor class --- argopy/__init__.py | 8 +- argopy/related/__init__.py | 8 +- argopy/related/about_sensors.py | 387 ++++++++++++++++++++++++++++++++ 3 files changed, 395 insertions(+), 8 deletions(-) create mode 100644 argopy/related/about_sensors.py diff --git a/argopy/__init__.py b/argopy/__init__.py index caadf8b1a..c610b68e9 100644 --- a/argopy/__init__.py +++ b/argopy/__init__.py @@ -41,7 +41,7 @@ from .utils import clear_cache, lscache # noqa: E402 from .utils import MonitoredThreadPoolExecutor # noqa: E402, F401 from .utils import monitor_status as status # noqa: E402 -from .related import TopoFetcher, OceanOPSDeployments, ArgoNVSReferenceTables, ArgoDocs, ArgoDOI # noqa: E402 +from .related import TopoFetcher, OceanOPSDeployments, ArgoNVSReferenceTables, ArgoDocs, ArgoDOI, ArgoSensor # noqa: E402 from .extensions import CanyonMED # noqa: E402 @@ -69,6 +69,7 @@ "ArgoDocs", # Class "TopoFetcher", # Class "ArgoDOI", # Class + "ArgoSensor", # Class # Advanced Argo data stores: "ArgoFloat", # Class @@ -83,9 +84,6 @@ "stores", "tutorial", - # Argo xarray accessor extensions - "CanyonMED", - - # Constants + # Constants: "__version__" ) diff --git a/argopy/related/__init__.py b/argopy/related/__init__.py index 465bc5de2..ca9ba81be 100644 --- a/argopy/related/__init__.py +++ b/argopy/related/__init__.py @@ -4,22 +4,24 @@ from .argo_documentation import ArgoDocs from .doi_snapshot import ArgoDOI from .euroargo_api import get_coriolis_profile_id, get_ea_profile_page +from .about_sensors import ArgoSensor from .utils import load_dict, mapp_dict # Should come last # __all__ = ( - # Classes: + # Classes : "TopoFetcher", "OceanOPSDeployments", "ArgoNVSReferenceTables", "ArgoDocs", "ArgoDOI", + "ArgoSensor", - # Functions: + # Functions : "get_coriolis_profile_id", "get_ea_profile_page", - # Utilities: + # Utilities : "load_dict", "mapp_dict", ) diff --git a/argopy/related/about_sensors.py b/argopy/related/about_sensors.py new file mode 100644 index 000000000..03e64591d --- /dev/null +++ b/argopy/related/about_sensors.py @@ -0,0 +1,387 @@ +from typing import List + +import pandas as pd + +# from ..errors import InvalidOption +from ..stores import ArgoFloat, ArgoIndex, httpstore +from ..utils import check_wmo, Chunker, to_list +from ..errors import ( + DataNotFound, + InvalidDataset, + InvalidDatasetStructure, + OptionValueError, +) +from ..options import OPTIONS +from . import ArgoNVSReferenceTables +import numpy as np + + +class ArgoSensor: + """ + + Notes + ----- + We keep this class in line with: + - https://github.com/OneArgo/ArgoVocabs/issues/156 + - https://github.com/OneArgo/ArgoVocabs/issues/157 + """ + + def __init__( + self, + model: str = None, + cache: bool = True, + cachedir: str = "", + timeout: int = 0, + ): + """Create an ArgoSensor helper class instance + + Parameters + ---------- + model: str, optional + A sensor model to use, by default None. + + Allowed values can be obtained with: + ``ArgoSensor().reference_model_list['altLabel']`` + + cache : bool, optional, default: False + Use cache or not for fetched data + cachedir: str, optional, default: OPTIONS['cachedir'] + Folder where to store cached files. + timeout: int, optional, default: OPTIONS['api_timeout'] + Time out in seconds to connect to web API + """ + self._cache = bool(cache) + self._cachedir = OPTIONS["cachedir"] if cachedir == "" else cachedir + self.timeout = OPTIONS["api_timeout"] if timeout == 0 else timeout + self.fs = httpstore(cache=self._cache, cachedir=self._cachedir) + + self._r25 = None + self._r27 = None + + if model is not None: + try: + df = self.search_model(model, strict=True) + except DataNotFound: + raise DataNotFound( + f"No sensor model named '{model}', as per ArgoSensor().reference_model_list['altLabel'] values, based on Ref. Table 27." + ) + + if df.shape[0] == 1: + self._model = model + self._model_r27 = df.iloc[0].to_dict() + else: + raise InvalidDatasetStructure( + f"Found multiple sensor models with '{model}'. Refine your sensor model name to only one value in: {to_list(df['altLabel'].values)}" + ) + + else: + self._model = None + self._model_r27 = None + + @property + def model(self): + if self._model is not None: + return self._model + else: + raise InvalidDataset( + "No model name available for an ArgoSensor instance not created with a specific sensor model" + ) + + @property + def model_long_name(self): + if self._model_r27 is not None: + return self._model_r27["prefLabel"] + else: + raise InvalidDataset( + "No model long name available for an ArgoSensor instance not created with a specific sensor model" + ) + + @property + def model_definition(self): + if self._model_r27 is not None: + return self._model_r27["definition"] + else: + raise InvalidDataset( + "No model definition available for an ArgoSensor instance not created with a specific sensor model" + ) + + @property + def model_deprecated(self): + if self._model_r27 is not None: + return self._model_r27["deprecated"] + else: + raise InvalidDataset( + "No model deprecation available for an ArgoSensor instance not created with a specific sensor model" + ) + + @property + def model_uri(self): + if self._model_r27 is not None: + return self._model_r27["id"] + else: + raise InvalidDataset( + "No model URI available for an ArgoSensor instance not created with a specific sensor model" + ) + + def __repr__(self): + if self._model_r27 is not None: + summary = [f""] + summary.append(f"➀ {self.model_long_name}") + if self.model_deprecated: + summary.append("β›” This model is deprecated !") + else: + summary.append("βœ… This model is not deprecated.") + summary.append(f"πŸ”— {self.model_uri}") + summary.append(f"❝{self.model_definition}❞") + else: + summary = [""] + summary.append("This instance was not created with a sensor model name, you still have access to the following:") + summary.append("πŸ‘‰ attributes: ") + for attr in ['reference_model', 'reference_model_name', 'reference_sensor', 'reference_sensor_type']: + summary.append(f"β•°β”ˆβž€ ArgoSensor().{attr}") + + summary.append("πŸ‘‰ methods: ") + for meth in ['search_model', 'search_model_name', 'search_wmo_with', 'search_sn_with', 'search_wmo_sn_with', 'iterfloats_with']: + summary.append(f"β•°β”ˆβž€ ArgoSensor().{meth}()") + return "\n".join(summary) + + @property + def reference_model(self) -> pd.DataFrame: + """Return the official reference table for Argo sensor models + + Return the Argo Reference table R27 'SENSOR_MODEL': + + > Terms listing models of sensors mounted on Argo floats. + """ + if self._r27 is None: + self._r27 = ArgoNVSReferenceTables(cache=self._cache, cachedir=self._cachedir).tbl("R27") + return self._r27 + + @property + def reference_model_name(self) -> List[str]: + """Return the official list of Argo sensor models + + Return a sorted list of strings with altLabel from Argo Reference table R27 'SENSOR_MODEL'. + + Notes + ----- + Argo netCDF variable ``SENSOR_MODEL`` is populated with values from this list. + """ + return sorted(to_list(self.reference_model["altLabel"].values)) + + @property + def reference_sensor(self) -> pd.DataFrame: + """Return the official list of Argo sensor types + + Return the Argo Reference table R25 'SENSOR': + + > Terms describing sensor types mounted on Argo floats. + """ + if self._r25 is None: + self._r25 = ArgoNVSReferenceTables(cache=self._cache, cachedir=self._cachedir).tbl("R25") + return self._r25 + + @property + def reference_sensor_type(self) -> List[str]: + """Return the official list of Argo sensor types + + Return a sorted list of strings with altLabel from Argo Reference table R25 'SENSOR'. + + Notes + ----- + Argo netCDF variable ``SENSOR`` is populated with values from this list. + """ + return sorted(to_list(self.reference_sensor["altLabel"].values)) + + def search_model(self, model: str, strict: bool = False) -> pd.DataFrame: + """Return references of Argo sensor models matching a string + + Look for occurrences in Argo Reference table R27 altLabel values and return a dataframe with matching row(s). + """ + if strict: + data = self.reference_model[ + self.reference_model["altLabel"].apply(lambda x: x == model) + ] + else: + data = self.reference_model[ + self.reference_model["altLabel"].apply(lambda x: model in x) + ] + if data.shape[0] == 0: + if strict: + raise DataNotFound(f"No sensor models matching '{model}'. You may try to search with strict=False.") + else: + raise DataNotFound(f"No sensor model names with '{model}' string occurrence.") + else: + return data + + def search_model_name(self, model: str = None, strict: bool = False) -> List[str]: + """Return a list of Argo sensor model names matching a string + + Notes + ----- + Argo netCDF variable SENSOR_MODEL is populated by such R27 altLabel names. + """ + if model is None: + if self._model is not None: + model = self._model + else: + raise OptionValueError( + "You must provide a sensor model name or create an ArgoSensor instance with an exact sensor model name to use this method" + ) + df = self.search_model(model=model, strict=strict) + return sorted(to_list(df["altLabel"].values)) + + def search_wmo_with(self, model: str): + """Return the list of WMOs with a given sensor model + + Notes + ----- + Based on a fleet-monitoring API request to `platformCodes/multi-lines-search` on `sensorModels` field. + """ + api_point = f"{OPTIONS['fleetmonitoring']}/platformCodes/multi-lines-search" + payload = [ + { + "nested": False, + "path": "string", + "searchValueType": "Text", + "values": [model], + "field": "sensorModels", + } + ] + wmos = self.fs.post(api_point, json_data=payload) + if wmos is None or len(wmos) == 0: + raise DataNotFound(f"No floats matching sensor model name '{model}'") + return check_wmo(wmos) + + def _floats_api( + self, + model: str, + preprocess=None, + preprocess_opts={}, + postprocess=None, + postprocess_opts={}, + progress=False, + errors="raise", + ): + """Fetch and process JSON data returned from the fleet-monitoring API for a list of float WMOs + + Process float metadata (calibrations, sensors, cycles, configs, ...) for all WMOs with a given sensor model name + + Notes + ----- + Based on a fleet-monitoring API request to `/floats/{wmo}`. + """ + wmos = self.search_wmo_with(model) + + URI = [] + for wmo in wmos: + URI.append(f"{OPTIONS['fleetmonitoring']}/floats/{wmo}") + + sns = self.fs.open_mfjson( + URI, + preprocess=preprocess, + preprocess_opts=preprocess_opts, + progress=progress, + errors=errors, + ) + + return postprocess(sns, **postprocess_opts) + + def search_sn_with(self, model: str, progress=False, errors="raise"): + """Return serial number of sensor models with a given string in name""" + + def preprocess(jsdata, model_name: str = ""): + sn = np.unique( + [s["serial"] for s in jsdata["sensors"] if model_name in s["model"]] + ) + return sn + + def postprocess(data, **kwargs): + S = [] + for row in data: + for sensor in row: + S.append(sensor) + return np.sort(np.array(S)) + + return self._floats_api( + model, + preprocess=preprocess, + preprocess_opts={"model_name": model}, + postprocess=postprocess, + progress=progress, + errors=errors, + ) + + def search_wmo_sn_with(self, model: str, progress=False, errors="raise"): + """Return a dictionary of float WMOs with their sensor serial numbers""" + + def preprocess(jsdata, model_name: str = ""): + sn = np.unique( + [s["serial"] for s in jsdata["sensors"] if model_name in s["model"]] + ) + return [jsdata["wmo"], sn] + + def postprocess(data, **kwargs): + S = {} + for wmo, sn in data: + S[check_wmo(wmo)[0]] = to_list(sn) + return S + + return self._floats_api( + model, + preprocess=preprocess, + preprocess_opts={"model_name": model}, + postprocess=postprocess, + progress=progress, + errors=errors, + ) + + def iterfloats_with(self, model: str, chunksize: int = None): + """Iterate over ArgoFloat equipped with a given sensor model + + By default, iterate over a single float, otherwise use the `chunksize` argument to iterate over chunk of floats. + + Parameters + ---------- + model: str + + chunksize: int, optional + Maximum chunk size + + Eg: A value of 5 will create chunks with as many as 5 WMOs each. + + Returns + ------- + Iterator of :class:`ArgoFloat` + + Examples + -------- + .. code-block:: python + :caption: Example of iteration + + for float in ArgoSensor().iterfloats_with('SBE41CP'): + float # is a ArgoFloat instance + + """ + # from .. import ArgoFloat # Prevent circular import + + wmos = self.search_wmo_with(model=model) + + idx = ArgoIndex( + index_file="core", + cache=self.cache, + ) + + if chunksize is not None: + chk_opts = {} + chk_opts.update({"chunks": {"wmo": "auto"}}) + chk_opts.update({"chunksize": {"wmo": chunksize}}) + chunked = Chunker( + {"wmo": self.search_wmo_with(model=model)}, **chk_opts + ).fit_transform() + for grp in chunked: + yield [ArgoFloat(wmo, idx=idx) for wmo in grp] + + else: + for wmo in wmos: + yield ArgoFloat(wmo, idx=idx) From 7b57b9ad7d31bc8c89dd68a6e5d707b3547e0905 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Thu, 25 Sep 2025 23:33:48 +0300 Subject: [PATCH 02/71] Update reference_tables.py fix bug whereby deprecated columns was not casted as bool --- argopy/related/reference_tables.py | 1 + 1 file changed, 1 insertion(+) diff --git a/argopy/related/reference_tables.py b/argopy/related/reference_tables.py index 5139f5f76..6ec96cc7d 100644 --- a/argopy/related/reference_tables.py +++ b/argopy/related/reference_tables.py @@ -109,6 +109,7 @@ def _jsConcept2df(self, data): content["deprecated"].append(k["owl:deprecated"]) content["id"].append(k["@id"]) df = pd.DataFrame.from_dict(content) + df['deprecated'] = df.apply(lambda x: True if x['deprecated']=='true' else False, axis=1) df.name = Collection_name return df From bac763d4033b37edd2dd701de70011a6748604c8 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Thu, 25 Sep 2025 23:34:46 +0300 Subject: [PATCH 03/71] Update options.py - New fleetmonitoring option to point toward the API - Refactor validate_http with a more appropriate name --- argopy/options.py | 264 +++++++++++++++++++++++++--------------------- 1 file changed, 145 insertions(+), 119 deletions(-) diff --git a/argopy/options.py b/argopy/options.py index 78fe7c1db..e45b58940 100644 --- a/argopy/options.py +++ b/argopy/options.py @@ -27,7 +27,7 @@ HAS_BOTO3 = False -from .errors import OptionValueError, GdacPathError, ErddapPathError +from .errors import OptionValueError, GdacPathError, ErddapPathError, APIServerError # Define a logger @@ -50,6 +50,7 @@ PARALLEL = "parallel" PARALLEL_DEFAULT_METHOD = "parallel_default_method" LON = "longitude_convention" +API_FLEETMONITORING = "fleetmonitoring" # Define the list of available options and default values: OPTIONS = { @@ -69,6 +70,7 @@ PARALLEL: False, PARALLEL_DEFAULT_METHOD: "thread", LON: "180", + API_FLEETMONITORING: "https://fleetmonitoring.euro-argo.eu", } DEFAULT = OPTIONS.copy() @@ -78,7 +80,7 @@ _USER_LEVEL_LIST = frozenset(["standard", "expert", "research"]) -# Define how to validate options: + def _positive_integer(value): return isinstance(value, int) and value > 0 @@ -91,7 +93,7 @@ def validate_gdac(this_path): return False -def validate_http(this_path): +def validate_erddap(this_path): if this_path != "-": return check_erddap_path(this_path, errors="raise") else: @@ -117,10 +119,148 @@ def validate_parallel_method(method): return False +def validate_fleetmonitoring(this_path): + if this_path != "-": + return check_fleetmonitoring_path(this_path, errors="raise") + else: + log.debug("OPTIONS['%s'] is not defined" % API_FLEETMONITORING) + return False + + +def PARALLEL_SETUP(parallel): + parallel = VALIDATE("parallel", parallel) + if isinstance(parallel, bool): + if parallel: + return True, OPTIONS["parallel_default_method"] + else: + return False, "sequential" + else: + return True, parallel + + +def check_erddap_path(path, errors="ignore"): + """Check if a url points to a valid ERDDAP server""" + fs = fsspec.filesystem("http", ssl=False) + check1 = fs.exists(path + "/info/index.json") + if check1: + return True + elif errors == "raise": + raise ErddapPathError(f"This url is not a valid ERDDAP server:\n{path}") + elif errors == "warn": + warnings.warn(f"This url is not a valid ERDDAP server:\n{path}") + return False + else: + return False + + +def check_fleetmonitoring_path(path, errors='ignore'): + """Check if a url points to a Euro-Argo valid FleetMonitoring server""" + fs = fsspec.filesystem("http", ssl=False) + try: + fs.open_json(path + "/get-version") + return True + except Exception as e: + if errors == "raise": + raise APIServerError(f"This url is not a valid Euro-Argo FleetMonitoring server:\n{path}") + elif errors == "warn": + warnings.warn(f"This url is not a valid Euro-Argo FleetMonitoring server:\n{path}") + return False + else: + return False + + +def check_gdac_option( + path, errors: str = "ignore", ignore_knowns: bool = True +): # noqa: C901 + """Check if a path has the expected GDAC structure + + Expected GDAC structure:: + + . + └── dac + β”œβ”€β”€ aoml + β”œβ”€β”€ ... + β”œβ”€β”€ coriolis + β”œβ”€β”€ ... + β”œβ”€β”€ meds + └── nmdis + + Examples:: + + >>> check_gdac_path("https://data-argo.ifremer.fr") # True + >>> check_gdac_path("https://usgodae.org/pub/outgoing/argo") # True + >>> check_gdac_path("ftp://ftp.ifremer.fr/ifremer/argo") # True + >>> check_gdac_path("/home/ref-argo/gdac") # True + >>> check_gdac_path("s3://argo-gdac-sandbox/pub") # True + + >>> check_gdac_path("https://www.ifremer.fr") # False + >>> check_gdac_path("ftp://usgodae.org/pub/outgoing") # False + + Parameters + ---------- + path: str + Path name to check, including access protocol + errors: str, default="ignore" + Determine how check procedure error are handled: "ignore", "raise" or "warn" + ignore_knowns: bool, default=False + Should the checking procedure be by-passed for the internal list of known GDACs. + Set this to True to check if a known GDACs is connected or not. + + Returns + ------- + checked: boolean + + See also + -------- + :class:`argopy.stores.gdacfs`, :meth:`argopy.utils.list_gdac_servers` + + """ + from .utils import ( + list_gdac_servers, + ) # import here, otherwise raises circular import + + if path in list_gdac_servers() and ignore_knowns: + return True + else: + + from .stores import gdacfs # import here, otherwise raises circular import + + try: + fs = gdacfs(path) + except GdacPathError: + if errors == "raise": + raise + elif errors == "warn": + warnings.warn("Can't get address info (GAIerror) on '%s'" % path) + return False + else: + return False + + check1 = fs.exists("dac") + if check1: + return True + + elif errors == "raise": + raise GdacPathError( + "This path is not GDAC compliant (no legitimate sub-folder `dac`):\n%s" + % path + ) + + elif errors == "warn": + warnings.warn( + "This path is not GDAC compliant (no legitimate sub-folder `dac`):\n%s" + % path + ) + return False + + else: + return False + + _VALIDATORS = { DATA_SOURCE: _DATA_SOURCE_LIST.__contains__, GDAC: validate_gdac, - ERDDAP: validate_http, + ERDDAP: validate_erddap, DATASET: _DATASET_LIST.__contains__, CACHE_FOLDER: lambda x: os.access(x, os.W_OK), CACHE_EXPIRATION: lambda x: isinstance(x, int) and x > 0, @@ -134,6 +274,7 @@ def validate_parallel_method(method): PARALLEL: validate_parallel, PARALLEL_DEFAULT_METHOD: validate_parallel_method, LON: lambda x: x in ['180', '360'], + API_FLEETMONITORING: validate_fleetmonitoring, } @@ -148,17 +289,6 @@ def VALIDATE(key, val): raise ValueError(f"option '{key}' has no validation method") -def PARALLEL_SETUP(parallel): - parallel = VALIDATE("parallel", parallel) - if isinstance(parallel, bool): - if parallel: - return True, OPTIONS["parallel_default_method"] - else: - return False, "sequential" - else: - return True, parallel - - class set_options: """Set options for argopy @@ -285,107 +415,3 @@ def __exit__(self, type, value, traceback): def reset_options(): """Reset all options to default values""" set_options(**DEFAULT) - - -def check_erddap_path(path, errors="ignore"): - """Check if an url points to an ERDDAP server""" - fs = fsspec.filesystem("http", ssl=False) - check1 = fs.exists(path + "/info/index.json") - if check1: - return True - elif errors == "raise": - raise ErddapPathError("This url is not a valid ERDDAP server:\n%s" % path) - - elif errors == "warn": - warnings.warn("This url is not a valid ERDDAP server:\n%s" % path) - return False - else: - return False - - -def check_gdac_option( - path, errors: str = "ignore", ignore_knowns: bool = True -): # noqa: C901 - """Check if a path has the expected GDAC structure - - Expected GDAC structure:: - - . - └── dac - β”œβ”€β”€ aoml - β”œβ”€β”€ ... - β”œβ”€β”€ coriolis - β”œβ”€β”€ ... - β”œβ”€β”€ meds - └── nmdis - - Examples:: - - >>> check_gdac_path("https://data-argo.ifremer.fr") # True - >>> check_gdac_path("https://usgodae.org/pub/outgoing/argo") # True - >>> check_gdac_path("ftp://ftp.ifremer.fr/ifremer/argo") # True - >>> check_gdac_path("/home/ref-argo/gdac") # True - >>> check_gdac_path("s3://argo-gdac-sandbox/pub") # True - - >>> check_gdac_path("https://www.ifremer.fr") # False - >>> check_gdac_path("ftp://usgodae.org/pub/outgoing") # False - - Parameters - ---------- - path: str - Path name to check, including access protocol - errors: str, default="ignore" - Determine how check procedure error are handled: "ignore", "raise" or "warn" - ignore_knowns: bool, default=False - Should the checking procedure be by-passed for the internal list of known GDACs. - Set this to True to check if a known GDACs is connected or not. - - Returns - ------- - checked: boolean - - See also - -------- - :class:`argopy.stores.gdacfs`, :meth:`argopy.utils.list_gdac_servers` - - """ - from .utils import ( - list_gdac_servers, - ) # import here, otherwise raises circular import - - if path in list_gdac_servers() and ignore_knowns: - return True - else: - - from .stores import gdacfs # import here, otherwise raises circular import - - try: - fs = gdacfs(path) - except GdacPathError: - if errors == "raise": - raise - elif errors == "warn": - warnings.warn("Can't get address info (GAIerror) on '%s'" % path) - return False - else: - return False - - check1 = fs.exists("dac") - if check1: - return True - - elif errors == "raise": - raise GdacPathError( - "This path is not GDAC compliant (no legitimate sub-folder `dac`):\n%s" - % path - ) - - elif errors == "warn": - warnings.warn( - "This path is not GDAC compliant (no legitimate sub-folder `dac`):\n%s" - % path - ) - return False - - else: - return False From a17c997f7ae967513d314fa3b00c580cfbdbefc0 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Sat, 27 Sep 2025 14:01:00 +0300 Subject: [PATCH 04/71] Determine sensor type from sensor model using AVTT mapping --- argopy/related/__init__.py | 2 +- .../related/{about_sensors.py => sensors.py} | 137 ++++++++++++++---- .../nvs_R25_R27/NVS_R25_R27_mappings_1.txt | 125 ++++++++++++++++ .../nvs_R25_R27/NVS_R25_R27_mappings_2.txt | 25 ++++ .../nvs_R25_R27/NVS_R25_R27_mappings_2b.txt | 1 + .../nvs_R25_R27/NVS_R25_R27_mappings_3.txt | 1 + .../nvs_R25_R27/NVS_R25_R27_mappings_3b.txt | 10 ++ .../NVS_R25_R27_mappings_4_cndc.txt | 42 ++++++ .../NVS_R25_R27_mappings_4_ido_doxy.txt | 4 + .../NVS_R25_R27_mappings_4_pres.txt | 42 ++++++ .../NVS_R25_R27_mappings_4_temp.txt | 42 ++++++ argopy/static/assets/nvs_R25_R27/README.md | 8 + 12 files changed, 406 insertions(+), 33 deletions(-) rename argopy/related/{about_sensors.py => sensors.py} (69%) create mode 100644 argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_1.txt create mode 100644 argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_2.txt create mode 100644 argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_2b.txt create mode 100644 argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_3.txt create mode 100644 argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_3b.txt create mode 100644 argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_4_cndc.txt create mode 100644 argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_4_ido_doxy.txt create mode 100644 argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_4_pres.txt create mode 100644 argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_4_temp.txt create mode 100644 argopy/static/assets/nvs_R25_R27/README.md diff --git a/argopy/related/__init__.py b/argopy/related/__init__.py index ca9ba81be..633dd295a 100644 --- a/argopy/related/__init__.py +++ b/argopy/related/__init__.py @@ -4,7 +4,7 @@ from .argo_documentation import ArgoDocs from .doi_snapshot import ArgoDOI from .euroargo_api import get_coriolis_profile_id, get_ea_profile_page -from .about_sensors import ArgoSensor +from .sensors import ArgoSensor from .utils import load_dict, mapp_dict # Should come last # diff --git a/argopy/related/about_sensors.py b/argopy/related/sensors.py similarity index 69% rename from argopy/related/about_sensors.py rename to argopy/related/sensors.py index 03e64591d..18d277d3c 100644 --- a/argopy/related/about_sensors.py +++ b/argopy/related/sensors.py @@ -1,9 +1,10 @@ from typing import List - import pandas as pd +import numpy as np +from pathlib import Path # from ..errors import InvalidOption -from ..stores import ArgoFloat, ArgoIndex, httpstore +from ..stores import ArgoFloat, ArgoIndex, httpstore, filestore from ..utils import check_wmo, Chunker, to_list from ..errors import ( DataNotFound, @@ -12,8 +13,8 @@ OptionValueError, ) from ..options import OPTIONS +from ..utils import path2assets from . import ArgoNVSReferenceTables -import numpy as np class ArgoSensor: @@ -21,7 +22,8 @@ class ArgoSensor: Notes ----- - We keep this class in line with: + Related NVS issues: + - https://github.com/OneArgo/ADMT/issues/112 - https://github.com/OneArgo/ArgoVocabs/issues/156 - https://github.com/OneArgo/ArgoVocabs/issues/157 """ @@ -41,8 +43,10 @@ def __init__( A sensor model to use, by default None. Allowed values can be obtained with: - ``ArgoSensor().reference_model_list['altLabel']`` + ``ArgoSensor().reference_model_name`` + Other Parameters + ---------------- cache : bool, optional, default: False Use cache or not for fetched data cachedir: str, optional, default: OPTIONS['cachedir'] @@ -55,15 +59,16 @@ def __init__( self.timeout = OPTIONS["api_timeout"] if timeout == 0 else timeout self.fs = httpstore(cache=self._cache, cachedir=self._cachedir) - self._r25 = None - self._r27 = None + self._r25 = None # will be loaded when necessary + self._r27 = None # will be loaded when necessary + self._load_mappers() # Load r25 model to r27 type mapping dictionary if model is not None: try: df = self.search_model(model, strict=True) except DataNotFound: raise DataNotFound( - f"No sensor model named '{model}', as per ArgoSensor().reference_model_list['altLabel'] values, based on Ref. Table 27." + f"No sensor model named '{model}', as per ArgoSensor().reference_model_name values, based on Ref. Table 27." ) if df.shape[0] == 1: @@ -78,55 +83,102 @@ def __init__( self._model = None self._model_r27 = None + def _load_mappers(self): + """Load the NVS R25 to R27 key mappings""" + df = [] + for p in Path(path2assets).joinpath("nvs_R25_R27").glob("NVS_R25_R27_mappings_*.txt"): + df.append(filestore().read_csv(p, header=None, names=["origin", "model", "?", "destination", "type", "??"])) + df = pd.concat(df) + df = df.reset_index(drop=True) + self.r27_to_r25 = {} + df.apply(lambda row: self.r27_to_r25.update({row['model'].strip(): row['type'].strip()}), axis=1) + @property - def model(self): + def model(self) -> str: if self._model is not None: return self._model else: raise InvalidDataset( - "No model name available for an ArgoSensor instance not created with a specific sensor model" + "The 'model' property is not available for an ArgoSensor instance not created with a specific sensor model" ) @property - def model_long_name(self): + def model_long_name(self) -> str: if self._model_r27 is not None: return self._model_r27["prefLabel"] else: raise InvalidDataset( - "No model long name available for an ArgoSensor instance not created with a specific sensor model" + "The 'model_long_name' property is not available for an ArgoSensor instance not created with a specific sensor model" ) @property - def model_definition(self): + def model_definition(self) -> str: if self._model_r27 is not None: return self._model_r27["definition"] else: raise InvalidDataset( - "No model definition available for an ArgoSensor instance not created with a specific sensor model" + "The 'model_definition' property is not available for an ArgoSensor instance not created with a specific sensor model" ) @property - def model_deprecated(self): + def model_deprecated(self) -> bool: if self._model_r27 is not None: return self._model_r27["deprecated"] else: raise InvalidDataset( - "No model deprecation available for an ArgoSensor instance not created with a specific sensor model" + "The 'model_deprecated' property is not available for an ArgoSensor instance not created with a specific sensor model" ) @property - def model_uri(self): + def model_uri(self) -> str: if self._model_r27 is not None: return self._model_r27["id"] else: raise InvalidDataset( - "No model URI available for an ArgoSensor instance not created with a specific sensor model" + "The 'model_uri' property is not available for an ArgoSensor instance not created with a specific sensor model" + ) + + @property + def type(self) -> str: + if self._model_r27 is not None: + if self.model in self.r27_to_r25: + return self.r27_to_r25[self.model] + else: + return "?" + else: + raise InvalidDataset( + "The 'type' property is not available for an ArgoSensor instance not created with a specific sensor model" + ) + + @property + def type_long_name(self) -> str: + if self.type is not "?": + sensor = self.reference_sensor[ + self.reference_sensor["altLabel"].apply(lambda x: x == self.type) + ].iloc[0].to_dict() + return sensor['prefLabel'] + else: + raise InvalidDataset( + "The 'type_long_name' property is not available for an ArgoSensor instance not created with a specific sensor model" + ) + + @property + def type_definition(self) -> str: + if self.type is not "?": + sensor = self.reference_sensor[ + self.reference_sensor["altLabel"].apply(lambda x: x == self.type) + ].iloc[0].to_dict() + return sensor['definition'] + else: + raise InvalidDataset( + "The 'type_definition' property is not available for an ArgoSensor instance not created with a specific sensor model" ) def __repr__(self): if self._model_r27 is not None: - summary = [f""] - summary.append(f"➀ {self.model_long_name}") + summary = [f""] + summary.append(f"TYPE➀ {self.type_long_name}") + summary.append(f"MODEL➀ {self.model_long_name}") if self.model_deprecated: summary.append("β›” This model is deprecated !") else: @@ -135,14 +187,28 @@ def __repr__(self): summary.append(f"❝{self.model_definition}❞") else: summary = [""] - summary.append("This instance was not created with a sensor model name, you still have access to the following:") + summary.append( + "This instance was not created with a sensor model name, you still have access to the following:" + ) summary.append("πŸ‘‰ attributes: ") - for attr in ['reference_model', 'reference_model_name', 'reference_sensor', 'reference_sensor_type']: - summary.append(f"β•°β”ˆβž€ ArgoSensor().{attr}") + for attr in [ + "reference_model", + "reference_model_name", + "reference_sensor", + "reference_sensor_type", + ]: + summary.append(f" β•°β”ˆβž€ ArgoSensor().{attr}") summary.append("πŸ‘‰ methods: ") - for meth in ['search_model', 'search_model_name', 'search_wmo_with', 'search_sn_with', 'search_wmo_sn_with', 'iterfloats_with']: - summary.append(f"β•°β”ˆβž€ ArgoSensor().{meth}()") + for meth in [ + "search_model", + "search_model_name", + "search_wmo_with", + "search_sn_with", + "search_wmo_sn_with", + "iterfloats_with", + ]: + summary.append(f" β•°β”ˆβž€ ArgoSensor().{meth}()") return "\n".join(summary) @property @@ -154,7 +220,9 @@ def reference_model(self) -> pd.DataFrame: > Terms listing models of sensors mounted on Argo floats. """ if self._r27 is None: - self._r27 = ArgoNVSReferenceTables(cache=self._cache, cachedir=self._cachedir).tbl("R27") + self._r27 = ArgoNVSReferenceTables( + cache=self._cache, cachedir=self._cachedir + ).tbl("R27") return self._r27 @property @@ -178,7 +246,9 @@ def reference_sensor(self) -> pd.DataFrame: > Terms describing sensor types mounted on Argo floats. """ if self._r25 is None: - self._r25 = ArgoNVSReferenceTables(cache=self._cache, cachedir=self._cachedir).tbl("R25") + self._r25 = ArgoNVSReferenceTables( + cache=self._cache, cachedir=self._cachedir + ).tbl("R25") return self._r25 @property @@ -208,9 +278,13 @@ def search_model(self, model: str, strict: bool = False) -> pd.DataFrame: ] if data.shape[0] == 0: if strict: - raise DataNotFound(f"No sensor models matching '{model}'. You may try to search with strict=False.") + raise DataNotFound( + f"No sensor models matching '{model}'. You may try to search with strict=False." + ) else: - raise DataNotFound(f"No sensor model names with '{model}' string occurrence.") + raise DataNotFound( + f"No sensor model names with '{model}' string occurrence." + ) else: return data @@ -363,13 +437,12 @@ def iterfloats_with(self, model: str, chunksize: int = None): float # is a ArgoFloat instance """ - # from .. import ArgoFloat # Prevent circular import - wmos = self.search_wmo_with(model=model) idx = ArgoIndex( index_file="core", - cache=self.cache, + cache=self._cache, + cachedir=self._cachedir, ) if chunksize is not None: diff --git a/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_1.txt b/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_1.txt new file mode 100644 index 000000000..d671729c8 --- /dev/null +++ b/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_1.txt @@ -0,0 +1,125 @@ +R27, AANDERAA_OPTODE , MIN, R25, OPTODE_DOXY , I +R27, AANDERAA_OPTODE_3830 , MIN, R25, OPTODE_DOXY , I +R27, AANDERAA_OPTODE_3835 , MIN, R25, OPTODE_DOXY , I +R27, AANDERAA_OPTODE_3930 , MIN, R25, OPTODE_DOXY , I +R27, AANDERAA_OPTODE_4330 , MIN, R25, OPTODE_DOXY , I +R27, AANDERAA_OPTODE_4330F , MIN, R25, OPTODE_DOXY , I +R27, AANDERAA_OPTODE_4831 , MIN, R25, OPTODE_DOXY , I +R27, AANDERAA_OPTODE_4831F , MIN, R25, OPTODE_DOXY , I +R27, AMETEK , MIN, R25, CTD_PRES , I +R27, AMETEK_3000PSIA , MIN, R25, CTD_PRES , I +R27, ARO_FT , MIN, R25, OPTODE_DOXY , I +R27, AROD_FT , MIN, R25, OPTODE_DOXY , I +R27, C_ROVER , MIN, R25, TRANSMISSOMETER_CP660 , I +R27, CTD_F01 , MIN, R25, CTD_PRES , I +R27, CTD_F01 , MIN, R25, CTD_TEMP , I +R27, CTD_F01 , MIN, R25, CTD_CNDC , I +R27, CYCLOPS_7_FLUOROMETER , MIN, R25, FLUOROMETER_CHLA , I +R27, CYCLOPS_7_FLUOROMETER , MIN, R25, BACKSCATTERINGMETER_TURBIDITY , I +R27, DRUCK , MIN, R25, CTD_PRES , I +R27, DRUCK_2900PSIA , MIN, R25, CTD_PRES , I +R27, DURA , MIN, R25, TRANSISTOR_PH , I +R27, ECO_FL , MIN, R25, FLUOROMETER_CHLA , I +R27, ECO_FLBB , MIN, R25, FLUOROMETER_CHLA , I +R27, ECO_FLBB_2K , MIN, R25, FLUOROMETER_CHLA , I +R27, ECO_FLBB_AP2 , MIN, R25, FLUOROMETER_CHLA , I +R27, ECO_FLBB2 , MIN, R25, FLUOROMETER_CHLA , I +R27, ECO_FLBBCD , MIN, R25, FLUOROMETER_CHLA , I +R27, ECO_FLBBCD , MIN, R25, FLUOROMETER_CDOM , I +R27, ECO_FLBBCD_AP2 , MIN, R25, FLUOROMETER_CHLA , I +R27, ECO_FLBBCD_AP2 , MIN, R25, FLUOROMETER_CDOM , I +R27, ECO_FLBBFL , MIN, R25, FLUOROMETER_CHLA , I +R27, ECO_FLBBFL_AP2 , MIN, R25, FLUOROMETER_CHLA , I +R27, ECO_FLNTU , MIN, R25, FLUOROMETER_CHLA , I +R27, ECO_FLNTU , MIN, R25, BACKSCATTERINGMETER_TURBIDITY , I +R27, ECO_NTU , MIN, R25, BACKSCATTERINGMETER_TURBIDITY , I +R27, EM , MIN, R25, EM , I +R27, FLOATCLOCK , MIN, R25, FLOATCLOCK_MTIME , I +R27, FSI , MIN, R25, CTD_PRES , I +R27, FSI , MIN, R25, CTD_TEMP , I +R27, FSI , MIN, R25, CTD_CNDC , I +R27, GDF , MIN, R25, TRANSISTOR_PH , I +R27, ISUS , MIN, R25, SPECTROPHOTOMETER_NITRATE , I +R27, ISUS_V3 , MIN, R25, SPECTROPHOTOMETER_NITRATE , I +R27, KELLER_PA8 , MIN, R25, CTD_PRES , I +R27, KISTLER , MIN, R25, CTD_PRES , I +R27, KISTLER_10153PSIA , MIN, R25, CTD_PRES , I +R27, KISTLER_2900PSIA , MIN, R25, CTD_PRES , I +R27, MCOMS_FLBB2 , MIN, R25, FLUOROMETER_CHLA , I +R27, MCOMS_FLBBCD , MIN, R25, FLUOROMETER_CHLA , I +R27, MCOMS_FLBBCD , MIN, R25, FLUOROMETER_CDOM , I +R27, MENSOR , MIN, R25, CTD_PRES , I +R27, MP40_C_2000_G , MIN, R25, CTD_PRES , I +R27, OPUS_DS , MIN, R25, SPECTROPHOTOMETER_NITRATE , I +R27, OPUS_DS , MIN, R25, SPECTROPHOTOMETER_BISULFIDE , I +R27, PAINE , MIN, R25, CTD_PRES , I +R27, PAINE_1500PSIA , MIN, R25, CTD_PRES , I +R27, PAINE_1600PSIA , MIN, R25, CTD_PRES , I +R27, PAINE_2000PSIA , MIN, R25, CTD_PRES , I +R27, PAINE_2900PSIA , MIN, R25, CTD_PRES , I +R27, PAINE_3000PSIA , MIN, R25, CTD_PRES , I +R27, PAL_UW , MIN, R25, ACOUSTIC , I +R27, RAFOS , MIN, R25, ACOUSTIC_GEOLOCATION , I +R27, RAMSES_ACC , MIN, R25, RADIOMETER_PAR , I +R27, RBR , MIN, R25, CTD_PRES , I +R27, RBR , MIN, R25, CTD_TEMP , I +R27, RBR , MIN, R25, CTD_CNDC , I +R27, RBR_ARGO , MIN, R25, CTD_PRES , I +R27, RBR_ARGO , MIN, R25, CTD_TEMP , I +R27, RBR_ARGO , MIN, R25, CTD_CNDC , I +R27, RBR_ARGO3 , MIN, R25, CTD_PRES , I +R27, RBR_ARGO3 , MIN, R25, CTD_TEMP , I +R27, RBR_ARGO3 , MIN, R25, CTD_CNDC , I +R27, RBR_ARGO3_DEEP4K , MIN, R25, CTD_PRES , I +R27, RBR_ARGO3_DEEP4K , MIN, R25, CTD_TEMP , I +R27, RBR_ARGO3_DEEP4K , MIN, R25, CTD_CNDC , I +R27, RBR_ARGO3_DEEP6K , MIN, R25, CTD_PRES , I +R27, RBR_ARGO3_DEEP6K , MIN, R25, CTD_TEMP , I +R27, RBR_ARGO3_DEEP6K , MIN, R25, CTD_CNDC , I +R27, RBR_CODA_T_ODO , MIN, R25, OPTODE_DOXY , I +R27, RBR_PRES , MIN, R25, CTD_PRES , I +R27, RBR_PRES_A , MIN, R25, CTD_PRES , I +R27, SATLANTIC_OCR504_ICSW , MIN, R25, RADIOMETER_PAR , I +R27, SATLANTIC_OCR507_ICSW , MIN, R25, RADIOMETER_PAR , I +R27, SATLANTIC_OCR507_ICSWR10W , MIN, R25, RADIOMETER_PAR , I +R27, SATLANTIC_PAR , MIN, R25, RADIOMETER_PAR , I +R27, SBE , MIN, R25, CTD_PRES , I +R27, SBE , MIN, R25, CTD_TEMP , I +R27, SBE , MIN, R25, CTD_CNDC , I +R27, SBE_STS , MIN, R25, STS_CNDC , I +R27, SBE_STS , MIN, R25, STS_TEMP , I +R27, SBE37 , MIN, R25, CTD_PRES , I +R27, SBE37 , MIN, R25, CTD_TEMP , I +R27, SBE37 , MIN, R25, CTD_CNDC , I +R27, SBE41 , MIN, R25, CTD_PRES , I +R27, SBE41 , MIN, R25, CTD_TEMP , I +R27, SBE41 , MIN, R25, CTD_CNDC , I +R27, SBE41_IDO , MIN, R25, CTD_PRES , I +R27, SBE41_IDO , MIN, R25, CTD_TEMP , I +R27, SBE41_IDO , MIN, R25, CTD_CNDC , I +R27, SBE41_IDO , MIN, R25, IDO_DOXY , I +R27, SBE41CP , MIN, R25, CTD_PRES , I +R27, SBE41CP , MIN, R25, CTD_TEMP , I +R27, SBE41CP , MIN, R25, CTD_CNDC , I +R27, SBE41CP_IDO , MIN, R25, CTD_PRES , I +R27, SBE41CP_IDO , MIN, R25, CTD_TEMP , I +R27, SBE41CP_IDO , MIN, R25, CTD_CNDC , I +R27, SBE41CP_IDO , MIN, R25, IDO_DOXY , I +R27, SBE41N , MIN, R25, CTD_PRES , I +R27, SBE41N , MIN, R25, CTD_TEMP , I +R27, SBE41N , MIN, R25, CTD_CNDC , I +R27, SBE43_IDO , MIN, R25, IDO_DOXY , I +R27, SBE43F_IDO , MIN, R25, IDO_DOXY , I +R27, SBE43I , MIN, R25, IDO_DOXY , I +R27, SBE61 , MIN, R25, CTD_PRES , I +R27, SBE61 , MIN, R25, CTD_TEMP , I +R27, SBE61 , MIN, R25, CTD_CNDC , I +R27, SBE63_OPTODE , MIN, R25, OPTODE_DOXY , I +R27, SBE83_OPTODE , MIN, R25, OPTODE_DOXY , I +R27, SEAFET , MIN, R25, TRANSISTOR_PH , I +R27, SEAPOINT_TURBIDITY_METER , MIN, R25, BACKSCATTERINGMETER_TURBIDITY , I +R27, SEASCAN_SSTD , MIN, R25, CTD_PRES , I +R27, SUNA , MIN, R25, SPECTROPHOTOMETER_NITRATE , I +R27, SUNA , MIN, R25, SPECTROPHOTOMETER_BISULFIDE , I +R27, SUNA_V2 , MIN, R25, SPECTROPHOTOMETER_NITRATE , I +R27, SUNA_V2 , MIN, R25, SPECTROPHOTOMETER_BISULFIDE , I \ No newline at end of file diff --git a/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_2.txt b/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_2.txt new file mode 100644 index 000000000..dd79316c2 --- /dev/null +++ b/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_2.txt @@ -0,0 +1,25 @@ +R27, ECO_BB3 , MIN, R25, BACKSCATTERINGMETER_BBP470 , I +R27, ECO_FLBB2 , MIN, R25, BACKSCATTERINGMETER_BBP470 , I +R27, ECO_BB3 , MIN, R25, BACKSCATTERINGMETER_BBP532 , I +R27, ECO_FLBB2 , MIN, R25, BACKSCATTERINGMETER_BBP532 , I +R27, MCOMS_FLBB2 , MIN, R25, BACKSCATTERINGMETER_BBP532 , I +R27, ECO_BB3 , MIN, R25, BACKSCATTERINGMETER_BBP700 , I +R27, ECO_FLBB , MIN, R25, BACKSCATTERINGMETER_BBP700 , I +R27, ECO_FLBB2 , MIN, R25, BACKSCATTERINGMETER_BBP700 , I +R27, ECO_FLBB_2K , MIN, R25, BACKSCATTERINGMETER_BBP700 , I +R27, ECO_FLBB_AP2 , MIN, R25, BACKSCATTERINGMETER_BBP700 , I +R27, ECO_FLBBCD , MIN, R25, BACKSCATTERINGMETER_BBP700 , I +R27, ECO_FLBBCD_AP2 , MIN, R25, BACKSCATTERINGMETER_BBP700 , I +R27, ECO_FLBBFL , MIN, R25, BACKSCATTERINGMETER_BBP700 , I +R27, ECO_FLBBFL_AP2 , MIN, R25, BACKSCATTERINGMETER_BBP700 , I +R27, MCOMS_FLBB2 , MIN, R25, BACKSCATTERINGMETER_BBP700 , I +R27, MCOMS_FLBBCD , MIN, R25, BACKSCATTERINGMETER_BBP700 , I +R27, SATLANTIC_OCR504_ICSW , MIN, R25, RADIOMETER_DOWN_IRR380 , I +R27, SATLANTIC_OCR504_ICSW , MIN, R25, RADIOMETER_DOWN_IRR412 , I +R27, SATLANTIC_OCR504_ICSW , MIN, R25, RADIOMETER_DOWN_IRR443 , I +R27, SATLANTIC_OCR504_ICSW , MIN, R25, RADIOMETER_DOWN_IRR490 , I +R27, SATLANTIC_OCR504_ICSW , MIN, R25, RADIOMETER_DOWN_IRR555 , I +R27, SATLANTIC_OCR504_R10W , MIN, R25, RADIOMETER_UP_RAD412 , I +R27, SATLANTIC_OCR504_R10W , MIN, R25, RADIOMETER_UP_RAD443 , I +R27, SATLANTIC_OCR504_R10W , MIN, R25, RADIOMETER_UP_RAD490 , I +R27, SATLANTIC_OCR504_R10W , MIN, R25, RADIOMETER_UP_RAD555 , I diff --git a/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_2b.txt b/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_2b.txt new file mode 100644 index 000000000..3b7098ff8 --- /dev/null +++ b/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_2b.txt @@ -0,0 +1 @@ +R27, RAMSES_ACC, MIN, R25,RADIOMETER_DOWN_IRR , I \ No newline at end of file diff --git a/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_3.txt b/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_3.txt new file mode 100644 index 000000000..8358155ff --- /dev/null +++ b/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_3.txt @@ -0,0 +1 @@ +R27, DRUCK_10153PSIA , MIN, R25, CTD_PRES , I \ No newline at end of file diff --git a/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_3b.txt b/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_3b.txt new file mode 100644 index 000000000..2ae333bb5 --- /dev/null +++ b/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_3b.txt @@ -0,0 +1,10 @@ +R27, DSB301-10-C85 , MIN, R25, CTD_PRES , I +R27, OCR504_ICSW , MIN, R25, RADIOMETER_DOWN_IRR380 , I +R27, OCR504_ICSW , MIN, R25, RADIOMETER_DOWN_IRR412 , I +R27, OCR504_ICSW , MIN, R25, RADIOMETER_DOWN_IRR443 , I +R27, OCR504_ICSW , MIN, R25, RADIOMETER_DOWN_IRR490 , I +R27, OCR504_ICSW , MIN, R25, RADIOMETER_DOWN_IRR555 , I +R27, OCR504_R10W , MIN, R25, RADIOMETER_UP_RAD412 , I +R27, OCR504_R10W , MIN, R25, RADIOMETER_UP_RAD443 , I +R27, OCR504_R10W , MIN, R25, RADIOMETER_UP_RAD490 , I +R27, OCR504_R10W , MIN, R25, RADIOMETER_UP_RAD555 , I \ No newline at end of file diff --git a/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_4_cndc.txt b/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_4_cndc.txt new file mode 100644 index 000000000..a5012c2dc --- /dev/null +++ b/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_4_cndc.txt @@ -0,0 +1,42 @@ +R27, SBE41_V2.5, MIN, R25,CTD_CNDC, I +R27, SBE41_V2.6, MIN, R25,CTD_CNDC, I +R27, SBE41_V3, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V1, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V1.1, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V1.2, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V1.2a, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V1.3, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V1.3b, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V1.4, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V1.5, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V1.7, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V1.8, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V1.9, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V1.9a, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V2, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V3, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V3.0a, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V3.0c, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V4.4.0, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V5.0.1, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V5.3.0, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V5.3.1, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V5.3.2, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V7.2.3, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V7.2.5, MIN, R25,CTD_CNDC, I +R27, SBE41N_V5.3.0, MIN, R25,CTD_CNDC, I +R27, SBE41N_V5.3.4, MIN, R25,CTD_CNDC, I +R27, SBE41N_V5.4.0, MIN, R25,CTD_CNDC, I +R27, SBE61_V4.5.2, MIN, R25,CTD_CNDC, I +R27, SBE61_V4.5.3, MIN, R25,CTD_CNDC, I +R27, SBE61_V5.0.0, MIN, R25,CTD_CNDC, I +R27, SBE61_V5.0.1, MIN, R25,CTD_CNDC, I +R27, SBE61_V5.0.10, MIN, R25,CTD_CNDC, I +R27, SBE61_V5.0.12, MIN, R25,CTD_CNDC, I +R27, SBE61_V5.0.2, MIN, R25,CTD_CNDC, I +R27, SBE61_V5.0.3, MIN, R25,CTD_CNDC, I +R27, SBE61_V5.0.9, MIN, R25,CTD_CNDC, I +R27, SBE41_IDO_V1.0c, MIN, R25,CTD_CNDC, I +R27, SBE41_IDO_V2.0, MIN, R25,CTD_CNDC, I +R27, SBE41_IDO_V3.0, MIN, R25,CTD_CNDC, I +R27, SBE41CP_IDO_V2.0b, MIN, R25,CTD_CNDC, I \ No newline at end of file diff --git a/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_4_ido_doxy.txt b/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_4_ido_doxy.txt new file mode 100644 index 000000000..9f41276bc --- /dev/null +++ b/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_4_ido_doxy.txt @@ -0,0 +1,4 @@ +R27, SBE41_IDO_V1.0c, MIN, R25,IDO_DOXY, I +R27, SBE41_IDO_V2.0, MIN, R25,IDO_DOXY, I +R27, SBE41_IDO_V3.0, MIN, R25,IDO_DOXY, I +R27, SBE41CP_IDO_V2.0b, MIN, R25,IDO_DOXY, I \ No newline at end of file diff --git a/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_4_pres.txt b/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_4_pres.txt new file mode 100644 index 000000000..765950382 --- /dev/null +++ b/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_4_pres.txt @@ -0,0 +1,42 @@ +R27, SBE41_V2.5, MIN, R25,CTD_PRES, I +R27, SBE41_V2.6, MIN, R25,CTD_PRES, I +R27, SBE41_V3, MIN, R25,CTD_PRES, I +R27, SBE41CP_V1, MIN, R25,CTD_PRES, I +R27, SBE41CP_V1.1, MIN, R25,CTD_PRES, I +R27, SBE41CP_V1.2, MIN, R25,CTD_PRES, I +R27, SBE41CP_V1.2a, MIN, R25,CTD_PRES, I +R27, SBE41CP_V1.3, MIN, R25,CTD_PRES, I +R27, SBE41CP_V1.3b, MIN, R25,CTD_PRES, I +R27, SBE41CP_V1.4, MIN, R25,CTD_PRES, I +R27, SBE41CP_V1.5, MIN, R25,CTD_PRES, I +R27, SBE41CP_V1.7, MIN, R25,CTD_PRES, I +R27, SBE41CP_V1.8, MIN, R25,CTD_PRES, I +R27, SBE41CP_V1.9, MIN, R25,CTD_PRES, I +R27, SBE41CP_V1.9a, MIN, R25,CTD_PRES, I +R27, SBE41CP_V2, MIN, R25,CTD_PRES, I +R27, SBE41CP_V3, MIN, R25,CTD_PRES, I +R27, SBE41CP_V3.0a, MIN, R25,CTD_PRES, I +R27, SBE41CP_V3.0c, MIN, R25,CTD_PRES, I +R27, SBE41CP_V4.4.0, MIN, R25,CTD_PRES, I +R27, SBE41CP_V5.0.1, MIN, R25,CTD_PRES, I +R27, SBE41CP_V5.3.0, MIN, R25,CTD_PRES, I +R27, SBE41CP_V5.3.1, MIN, R25,CTD_PRES, I +R27, SBE41CP_V5.3.2, MIN, R25,CTD_PRES, I +R27, SBE41CP_V7.2.3, MIN, R25,CTD_PRES, I +R27, SBE41CP_V7.2.5, MIN, R25,CTD_PRES, I +R27, SBE41N_V5.3.0, MIN, R25,CTD_PRES, I +R27, SBE41N_V5.3.4, MIN, R25,CTD_PRES, I +R27, SBE41N_V5.4.0, MIN, R25,CTD_PRES, I +R27, SBE61_V4.5.2, MIN, R25,CTD_PRES, I +R27, SBE61_V4.5.3, MIN, R25,CTD_PRES, I +R27, SBE61_V5.0.0, MIN, R25,CTD_PRES, I +R27, SBE61_V5.0.1, MIN, R25,CTD_PRES, I +R27, SBE61_V5.0.10, MIN, R25,CTD_PRES, I +R27, SBE61_V5.0.12, MIN, R25,CTD_PRES, I +R27, SBE61_V5.0.2, MIN, R25,CTD_PRES, I +R27, SBE61_V5.0.3, MIN, R25,CTD_PRES, I +R27, SBE61_V5.0.9, MIN, R25,CTD_PRES, I +R27, SBE41_IDO_V1.0c, MIN, R25,CTD_PRES, I +R27, SBE41_IDO_V2.0, MIN, R25,CTD_PRES, I +R27, SBE41_IDO_V3.0, MIN, R25,CTD_PRES, I +R27, SBE41CP_IDO_V2.0b, MIN, R25,CTD_PRES, I \ No newline at end of file diff --git a/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_4_temp.txt b/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_4_temp.txt new file mode 100644 index 000000000..05430e673 --- /dev/null +++ b/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_4_temp.txt @@ -0,0 +1,42 @@ +R27, SBE41_V2.5, MIN, R25,CTD_TEMP, I +R27, SBE41_V2.6, MIN, R25,CTD_TEMP, I +R27, SBE41_V3, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V1, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V1.1, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V1.2, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V1.2a, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V1.3, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V1.3b, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V1.4, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V1.5, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V1.7, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V1.8, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V1.9, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V1.9a, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V2, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V3, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V3.0a, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V3.0c, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V4.4.0, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V5.0.1, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V5.3.0, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V5.3.1, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V5.3.2, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V7.2.3, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V7.2.5, MIN, R25,CTD_TEMP, I +R27, SBE41N_V5.3.0, MIN, R25,CTD_TEMP, I +R27, SBE41N_V5.3.4, MIN, R25,CTD_TEMP, I +R27, SBE41N_V5.4.0, MIN, R25,CTD_TEMP, I +R27, SBE61_V4.5.2, MIN, R25,CTD_TEMP, I +R27, SBE61_V4.5.3, MIN, R25,CTD_TEMP, I +R27, SBE61_V5.0.0, MIN, R25,CTD_TEMP, I +R27, SBE61_V5.0.1, MIN, R25,CTD_TEMP, I +R27, SBE61_V5.0.10, MIN, R25,CTD_TEMP, I +R27, SBE61_V5.0.12, MIN, R25,CTD_TEMP, I +R27, SBE61_V5.0.2, MIN, R25,CTD_TEMP, I +R27, SBE61_V5.0.3, MIN, R25,CTD_TEMP, I +R27, SBE61_V5.0.9, MIN, R25,CTD_TEMP, I +R27, SBE41_IDO_V1.0c, MIN, R25,CTD_TEMP, I +R27, SBE41_IDO_V2.0, MIN, R25,CTD_TEMP, I +R27, SBE41_IDO_V3.0, MIN, R25,CTD_TEMP, I +R27, SBE41CP_IDO_V2.0b, MIN, R25,CTD_TEMP, I \ No newline at end of file diff --git a/argopy/static/assets/nvs_R25_R27/README.md b/argopy/static/assets/nvs_R25_R27/README.md new file mode 100644 index 000000000..3dbedb6d1 --- /dev/null +++ b/argopy/static/assets/nvs_R25_R27/README.md @@ -0,0 +1,8 @@ +# Sensor models and types + +This folder hold mapping data to get R25 sensor types from R27 sensor models. + +Relevant issues from ADMT/AVTT: +- https://github.com/OneArgo/ADMT/issues/112 +- https://github.com/OneArgo/ArgoVocabs/issues/156 +- https://github.com/OneArgo/ArgoVocabs/issues/157 From 1a79496448c820f48d098d54e184ec33dbda38d8 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Sat, 27 Sep 2025 14:56:45 +0300 Subject: [PATCH 05/71] Update sensors.py --- argopy/related/sensors.py | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/argopy/related/sensors.py b/argopy/related/sensors.py index 18d277d3c..50f997ddc 100644 --- a/argopy/related/sensors.py +++ b/argopy/related/sensors.py @@ -18,22 +18,10 @@ class ArgoSensor: - """ - - Notes - ----- - Related NVS issues: - - https://github.com/OneArgo/ADMT/issues/112 - - https://github.com/OneArgo/ArgoVocabs/issues/156 - - https://github.com/OneArgo/ArgoVocabs/issues/157 - """ - def __init__( self, model: str = None, - cache: bool = True, - cachedir: str = "", - timeout: int = 0, + **kwargs, ): """Create an ArgoSensor helper class instance @@ -53,10 +41,24 @@ def __init__( Folder where to store cached files. timeout: int, optional, default: OPTIONS['api_timeout'] Time out in seconds to connect to web API + + Examples + -------- + .. code-block:: python + :caption: ? + + from argopy import ArgoSensor + + Notes + ----- + Related NVS issues: + - https://github.com/OneArgo/ADMT/issues/112 + - https://github.com/OneArgo/ArgoVocabs/issues/156 + - https://github.com/OneArgo/ArgoVocabs/issues/157 """ - self._cache = bool(cache) - self._cachedir = OPTIONS["cachedir"] if cachedir == "" else cachedir - self.timeout = OPTIONS["api_timeout"] if timeout == 0 else timeout + self._cache = kwargs.get('cache', True) + self._cachedir = kwargs.get('cachedir', OPTIONS["cachedir"]) + self.timeout = kwargs.get('timeout', OPTIONS["api_timeout"]) self.fs = httpstore(cache=self._cache, cachedir=self._cachedir) self._r25 = None # will be loaded when necessary From a3185459d12681fb1225ea98ec9ac004a6cb86a4 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Sat, 27 Sep 2025 16:01:20 +0300 Subject: [PATCH 06/71] Refactor ArgoSensor handling of model and type info from r27 and r25 now uses dataclassese --- argopy/related/sensors.py | 156 +++++++++++++++++++------------------- 1 file changed, 77 insertions(+), 79 deletions(-) diff --git a/argopy/related/sensors.py b/argopy/related/sensors.py index 50f997ddc..54c7481ba 100644 --- a/argopy/related/sensors.py +++ b/argopy/related/sensors.py @@ -2,6 +2,9 @@ import pandas as pd import numpy as np from pathlib import Path +from dataclasses import dataclass +from typing import Optional, ClassVar, Literal,Union +import warnings # from ..errors import InvalidOption from ..stores import ArgoFloat, ArgoIndex, httpstore, filestore @@ -17,6 +20,51 @@ from . import ArgoNVSReferenceTables +@dataclass +class NVSrow: + """This class makes it easier to work with a single :class:`pd.DataFrame` row as an object""" + name: str = "" + long_name: str = "" + definition: str = "" + uri: str = "" + deprecated: bool = None + + reftable : ClassVar[str] + """Reference table""" + + def __init__(self, row: pd.Series): + if not isinstance(row, pd.Series) and isinstance(row, pd.DataFrame): + row = row.iloc[0] + row = row.to_dict() + self.name = row['altLabel'] + self.long_name = row['prefLabel'] + self.definition = row['definition'] + self.deprecated = row['deprecated'] + self.uri = row['id'] + + @staticmethod + def from_series(obj: pd.Series) -> 'NVSrow': + return NVSrow(obj) + + +class SensorType(NVSrow): + """One single sensor type data from a R25 row""" + reftable = 'R25' + + @staticmethod + def from_series(obj: pd.Series) -> 'SensorType': + return SensorType(obj) + + +class SensorModel(NVSrow): + """One single sensor model data from a R27 row""" + reftable = 'R27' + + @staticmethod + def from_series(obj: pd.Series) -> 'SensorModel': + return SensorModel(obj) + + class ArgoSensor: def __init__( self, @@ -74,16 +122,16 @@ def __init__( ) if df.shape[0] == 1: - self._model = model - self._model_r27 = df.iloc[0].to_dict() + self._model = SensorModel.from_series(df) + self._type = self._model_to_type(self._model, errors='ignore') else: raise InvalidDatasetStructure( - f"Found multiple sensor models with '{model}'. Refine your sensor model name to only one value in: {to_list(df['altLabel'].values)}" + f"Found multiple sensor models with '{model}'. Restrict your sensor model name to only one value in: {to_list(df['altLabel'].values)}" ) else: self._model = None - self._model_r27 = None + self._type = None def _load_mappers(self): """Load the NVS R25 to R27 key mappings""" @@ -95,9 +143,22 @@ def _load_mappers(self): self.r27_to_r25 = {} df.apply(lambda row: self.r27_to_r25.update({row['model'].strip(): row['type'].strip()}), axis=1) + def _model_to_type(self, model: Union[str, SensorModel] = None, errors : Literal['raise', 'ignore'] = 'raise') -> Optional[SensorType]: + """Read a sensor type for a given sensor model""" + model_name = model.name if isinstance(model, SensorModel) else model + if model_name in self.r27_to_r25: + sensor_type = self.r27_to_r25[model_name] + row = self.reference_sensor[ + self.reference_sensor["altLabel"].apply(lambda x: x == sensor_type) + ].iloc[0] + return SensorType.from_series(row) + elif errors == 'raise': + raise DataNotFound(f"Can't determine the type of sensor model '{model_name}' (no matching key in self.r27_to_r25 mapper)") + return None + @property - def model(self) -> str: - if self._model is not None: + def model(self) -> SensorModel: + if isinstance(self._model, SensorModel): return self._model else: raise InvalidDataset( @@ -105,88 +166,25 @@ def model(self) -> str: ) @property - def model_long_name(self) -> str: - if self._model_r27 is not None: - return self._model_r27["prefLabel"] - else: - raise InvalidDataset( - "The 'model_long_name' property is not available for an ArgoSensor instance not created with a specific sensor model" - ) - - @property - def model_definition(self) -> str: - if self._model_r27 is not None: - return self._model_r27["definition"] - else: - raise InvalidDataset( - "The 'model_definition' property is not available for an ArgoSensor instance not created with a specific sensor model" - ) - - @property - def model_deprecated(self) -> bool: - if self._model_r27 is not None: - return self._model_r27["deprecated"] - else: - raise InvalidDataset( - "The 'model_deprecated' property is not available for an ArgoSensor instance not created with a specific sensor model" - ) - - @property - def model_uri(self) -> str: - if self._model_r27 is not None: - return self._model_r27["id"] - else: - raise InvalidDataset( - "The 'model_uri' property is not available for an ArgoSensor instance not created with a specific sensor model" - ) - - @property - def type(self) -> str: - if self._model_r27 is not None: - if self.model in self.r27_to_r25: - return self.r27_to_r25[self.model] - else: - return "?" + def type(self) -> SensorType: + if isinstance(self._type, SensorType): + return self._type else: raise InvalidDataset( "The 'type' property is not available for an ArgoSensor instance not created with a specific sensor model" ) - @property - def type_long_name(self) -> str: - if self.type is not "?": - sensor = self.reference_sensor[ - self.reference_sensor["altLabel"].apply(lambda x: x == self.type) - ].iloc[0].to_dict() - return sensor['prefLabel'] - else: - raise InvalidDataset( - "The 'type_long_name' property is not available for an ArgoSensor instance not created with a specific sensor model" - ) - - @property - def type_definition(self) -> str: - if self.type is not "?": - sensor = self.reference_sensor[ - self.reference_sensor["altLabel"].apply(lambda x: x == self.type) - ].iloc[0].to_dict() - return sensor['definition'] - else: - raise InvalidDataset( - "The 'type_definition' property is not available for an ArgoSensor instance not created with a specific sensor model" - ) - def __repr__(self): - if self._model_r27 is not None: - summary = [f""] - summary.append(f"TYPE➀ {self.type_long_name}") - summary.append(f"MODEL➀ {self.model_long_name}") - if self.model_deprecated: + if isinstance(self._model, SensorModel): + summary = [f""] + summary.append(f"TYPE➀ {self.type.long_name}") + summary.append(f"MODEL➀ {self.model.long_name}") + if self.model.deprecated: summary.append("β›” This model is deprecated !") else: summary.append("βœ… This model is not deprecated.") - summary.append(f"πŸ”— {self.model_uri}") - summary.append(f"❝{self.model_definition}❞") + summary.append(f"πŸ”— {self.model.uri}") + summary.append(f"❝{self.model.definition}❞") else: summary = [""] summary.append( From 546e6b53b4db78c2b0a72472c32b6ab83529ce55 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Sat, 27 Sep 2025 16:30:19 +0300 Subject: [PATCH 07/71] Refactor search methods --- argopy/related/sensors.py | 89 +++++++++++++++++++++++++++------------ 1 file changed, 62 insertions(+), 27 deletions(-) diff --git a/argopy/related/sensors.py b/argopy/related/sensors.py index 54c7481ba..f54b04b97 100644 --- a/argopy/related/sensors.py +++ b/argopy/related/sensors.py @@ -5,6 +5,7 @@ from dataclasses import dataclass from typing import Optional, ClassVar, Literal,Union import warnings +import logging # from ..errors import InvalidOption from ..stores import ArgoFloat, ArgoIndex, httpstore, filestore @@ -20,6 +21,9 @@ from . import ArgoNVSReferenceTables +log = logging.getLogger("argopy.related.sensors") + + @dataclass class NVSrow: """This class makes it easier to work with a single :class:`pd.DataFrame` row as an object""" @@ -97,6 +101,38 @@ def __init__( from argopy import ArgoSensor + # Return the reference table R27 with the list of sensor models + ArgoSensor().reference_model + + # Return all R27 referenced sensor models with some string in their name + ArgoSensor().search_model('RBR') + ArgoSensor().search_model('RBR', output='name') # Return list of names instead of dataframe + ArgoSensor().search_model('SBE41CP', strict=False) + ArgoSensor().search_model('SBE41CP', strict=True) + + # Return list of WMOs equipped with a sensor model name having some string + ArgoSensor().search('RBR', output='wmo') + + # Return list of sensor serial number for a sensor model name having some string + ArgoSensor().search('SBE', output='sn') + ArgoSensor().search('SBE', output='sn', progress=True) + + # Return dict of WMOs with sensor serial number for a sensor model name having some string + ArgoSensor().search('SBE', output='wmo_sn') + ArgoSensor().search('SBE', output='wmo_sn', progress=True) + + # Loop through each (or chunks) of ArgoFloat instances for floats equipped with a sensor model name having some string + for af in ArgoSensor().iterfloats_with("RAFOS"): + print(af.WMO) + + # Same loop, show casing how to use the metadata attribute of an ArgoFloat instance: + model = "RAFOS" + for af in ArgoSensor().iterfloats_with(model): + models = af.metadata['sensors'] + for s in models: + if model in s['model']: + print(af.WMO, s['maker'], s['model'], s['serial']) + Notes ----- Related NVS issues: @@ -263,7 +299,7 @@ def reference_sensor_type(self) -> List[str]: """ return sorted(to_list(self.reference_sensor["altLabel"].values)) - def search_model(self, model: str, strict: bool = False) -> pd.DataFrame: + def search_model(self, model: str, strict: bool = False, output: Literal['table', 'name'] = 'table') -> pd.DataFrame: """Return references of Argo sensor models matching a string Look for occurrences in Argo Reference table R27 altLabel values and return a dataframe with matching row(s). @@ -286,26 +322,12 @@ def search_model(self, model: str, strict: bool = False) -> pd.DataFrame: f"No sensor model names with '{model}' string occurrence." ) else: - return data - - def search_model_name(self, model: str = None, strict: bool = False) -> List[str]: - """Return a list of Argo sensor model names matching a string - - Notes - ----- - Argo netCDF variable SENSOR_MODEL is populated by such R27 altLabel names. - """ - if model is None: - if self._model is not None: - model = self._model + if output == 'name': + return sorted(to_list(data["altLabel"].values)) else: - raise OptionValueError( - "You must provide a sensor model name or create an ArgoSensor instance with an exact sensor model name to use this method" - ) - df = self.search_model(model=model, strict=strict) - return sorted(to_list(df["altLabel"].values)) + return data - def search_wmo_with(self, model: str): + def _search_wmo_with(self, model: str, errors="raise"): """Return the list of WMOs with a given sensor model Notes @@ -324,7 +346,10 @@ def search_wmo_with(self, model: str): ] wmos = self.fs.post(api_point, json_data=payload) if wmos is None or len(wmos) == 0: - raise DataNotFound(f"No floats matching sensor model name '{model}'") + if errors == 'raise': + raise DataNotFound(f"No floats matching sensor model name '{model}'") + else: + log.error(f"No floats matching sensor model name '{model}'") return check_wmo(wmos) def _floats_api( @@ -343,9 +368,9 @@ def _floats_api( Notes ----- - Based on a fleet-monitoring API request to `/floats/{wmo}`. + Based on fleet-monitoring API requests to `/floats/{wmo}`. """ - wmos = self.search_wmo_with(model) + wmos = self._search_wmo_with(model) URI = [] for wmo in wmos: @@ -361,7 +386,7 @@ def _floats_api( return postprocess(sns, **postprocess_opts) - def search_sn_with(self, model: str, progress=False, errors="raise"): + def _search_sn_with(self, model: str, progress=False, errors="raise"): """Return serial number of sensor models with a given string in name""" def preprocess(jsdata, model_name: str = ""): @@ -386,14 +411,14 @@ def postprocess(data, **kwargs): errors=errors, ) - def search_wmo_sn_with(self, model: str, progress=False, errors="raise"): + def _search_wmo_sn_with(self, model: str, progress=False, errors="raise"): """Return a dictionary of float WMOs with their sensor serial numbers""" def preprocess(jsdata, model_name: str = ""): sn = np.unique( [s["serial"] for s in jsdata["sensors"] if model_name in s["model"]] ) - return [jsdata["wmo"], sn] + return [jsdata["wmo"], [str(s) for s in sn]] def postprocess(data, **kwargs): S = {} @@ -410,6 +435,16 @@ def postprocess(data, **kwargs): errors=errors, ) + def search(self, model: str, output: Literal['wmo', 'sn', 'wmo_sn'] = 'wmo', progress=False, errors='raise'): + if output == 'wmo': + return self._search_wmo_with(model=model, errors=errors) + elif output == 'sn': + return self._search_sn_with(model=model, progress=progress, errors=errors) + elif output == 'wmo_sn': + return self._search_wmo_sn_with(model=model, progress=progress, errors=errors) + else: + raise OptionValueError("'output' option value must be in: 'wmo', 'sn', or 'wmo_sn'") + def iterfloats_with(self, model: str, chunksize: int = None): """Iterate over ArgoFloat equipped with a given sensor model @@ -437,7 +472,7 @@ def iterfloats_with(self, model: str, chunksize: int = None): float # is a ArgoFloat instance """ - wmos = self.search_wmo_with(model=model) + wmos = self._search_wmo_with(model=model) idx = ArgoIndex( index_file="core", @@ -450,7 +485,7 @@ def iterfloats_with(self, model: str, chunksize: int = None): chk_opts.update({"chunks": {"wmo": "auto"}}) chk_opts.update({"chunksize": {"wmo": chunksize}}) chunked = Chunker( - {"wmo": self.search_wmo_with(model=model)}, **chk_opts + {"wmo": self._search_wmo_with(model=model)}, **chk_opts ).fit_transform() for grp in chunked: yield [ArgoFloat(wmo, idx=idx) for wmo in grp] From 5d6a242ae76371e1bcb49ae49a4c60564cdd93e4 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Wed, 1 Oct 2025 14:00:18 +0200 Subject: [PATCH 08/71] Update sensors.py --- argopy/related/sensors.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/argopy/related/sensors.py b/argopy/related/sensors.py index f54b04b97..faeab9b95 100644 --- a/argopy/related/sensors.py +++ b/argopy/related/sensors.py @@ -50,6 +50,9 @@ def __init__(self, row: pd.Series): def from_series(obj: pd.Series) -> 'NVSrow': return NVSrow(obj) + def __eq__(self, obj): + return self.name == obj + class SensorType(NVSrow): """One single sensor type data from a R25 row""" @@ -103,8 +106,13 @@ def __init__( # Return the reference table R27 with the list of sensor models ArgoSensor().reference_model + ArgoSensor().reference_model_name # Only the list of names (used to fill 'SENSOR_MODEL') + + # Return the reference table R25 with the list of sensor types + ArgoSensor().reference_sensor + ArgoSensor().reference_sensor_type # Only the list of types (used to fill 'SENSOR') - # Return all R27 referenced sensor models with some string in their name + # Return all (R27) referenced sensor models with some string in their name ArgoSensor().search_model('RBR') ArgoSensor().search_model('RBR', output='name') # Return list of names instead of dataframe ArgoSensor().search_model('SBE41CP', strict=False) @@ -114,8 +122,8 @@ def __init__( ArgoSensor().search('RBR', output='wmo') # Return list of sensor serial number for a sensor model name having some string - ArgoSensor().search('SBE', output='sn') - ArgoSensor().search('SBE', output='sn', progress=True) + ArgoSensor().search('RBR_ARGO3_DEEP6K', output='sn') + ArgoSensor().search('RBR_ARGO3_DEEP6K', output='sn', progress=True) # Return dict of WMOs with sensor serial number for a sensor model name having some string ArgoSensor().search('SBE', output='wmo_sn') @@ -159,7 +167,7 @@ def __init__( if df.shape[0] == 1: self._model = SensorModel.from_series(df) - self._type = self._model_to_type(self._model, errors='ignore') + self._type = self.model_to_type(self._model, errors='ignore') else: raise InvalidDatasetStructure( f"Found multiple sensor models with '{model}'. Restrict your sensor model name to only one value in: {to_list(df['altLabel'].values)}" @@ -179,11 +187,11 @@ def _load_mappers(self): self.r27_to_r25 = {} df.apply(lambda row: self.r27_to_r25.update({row['model'].strip(): row['type'].strip()}), axis=1) - def _model_to_type(self, model: Union[str, SensorModel] = None, errors : Literal['raise', 'ignore'] = 'raise') -> Optional[SensorType]: + def model_to_type(self, model: Union[str, SensorModel] = None, errors : Literal['raise', 'ignore'] = 'raise') -> Optional[SensorType]: """Read a sensor type for a given sensor model""" model_name = model.name if isinstance(model, SensorModel) else model - if model_name in self.r27_to_r25: - sensor_type = self.r27_to_r25[model_name] + sensor_type = self.r27_to_r25.get(model_name, None) + if sensor_type is not None: row = self.reference_sensor[ self.reference_sensor["altLabel"].apply(lambda x: x == sensor_type) ].iloc[0] From 78656bc23fb645d9fd7110cd593957f8d6eb878a Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Fri, 3 Oct 2025 21:54:19 +0200 Subject: [PATCH 09/71] Update sensors.py --- argopy/related/sensors.py | 220 +++++++++++++++++++++++++------------- 1 file changed, 146 insertions(+), 74 deletions(-) diff --git a/argopy/related/sensors.py b/argopy/related/sensors.py index faeab9b95..bf99e5d7b 100644 --- a/argopy/related/sensors.py +++ b/argopy/related/sensors.py @@ -2,14 +2,12 @@ import pandas as pd import numpy as np from pathlib import Path -from dataclasses import dataclass -from typing import Optional, ClassVar, Literal,Union -import warnings +from typing import Optional, Literal, Union import logging # from ..errors import InvalidOption from ..stores import ArgoFloat, ArgoIndex, httpstore, filestore -from ..utils import check_wmo, Chunker, to_list +from ..utils import check_wmo, Chunker, to_list, NVSrow from ..errors import ( DataNotFound, InvalidDataset, @@ -24,51 +22,62 @@ log = logging.getLogger("argopy.related.sensors") -@dataclass -class NVSrow: - """This class makes it easier to work with a single :class:`pd.DataFrame` row as an object""" - name: str = "" - long_name: str = "" - definition: str = "" - uri: str = "" - deprecated: bool = None - - reftable : ClassVar[str] - """Reference table""" - - def __init__(self, row: pd.Series): - if not isinstance(row, pd.Series) and isinstance(row, pd.DataFrame): - row = row.iloc[0] - row = row.to_dict() - self.name = row['altLabel'] - self.long_name = row['prefLabel'] - self.definition = row['definition'] - self.deprecated = row['deprecated'] - self.uri = row['id'] +class SensorType(NVSrow): + """One single sensor type data from a R25-"Argo sensor types" row - @staticmethod - def from_series(obj: pd.Series) -> 'NVSrow': - return NVSrow(obj) + Examples + -------- + .. code-block:: python + :caption: Search methods - def __eq__(self, obj): - return self.name == obj + from argopy import ArgoNVSReferenceTables + df = ArgoNVSReferenceTables().tbl(25) -class SensorType(NVSrow): - """One single sensor type data from a R25 row""" - reftable = 'R25' + sensor_type = 'CTD' + st = SensorType.from_series(df[df["altLabel"].apply(lambda x: x == sensor_type)].iloc[0]) + + st.name + st.long_name + st.definition + st.deprecated + st.uri + + """ + + reftable = "R25" @staticmethod - def from_series(obj: pd.Series) -> 'SensorType': + def from_series(obj: pd.Series) -> "SensorType": return SensorType(obj) class SensorModel(NVSrow): - """One single sensor model data from a R27 row""" - reftable = 'R27' + """One single sensor model data from a R27-"Argo sensor models" row + + Examples + -------- + .. code-block:: python + :caption: Search methods + + from argopy import ArgoNVSReferenceTables + + df = ArgoNVSReferenceTables().tbl(27) + + sensor_model = 'AANDERAA_OPTODE_4330F' + sm = SensorModel.from_series(df[df["altLabel"].apply(lambda x: x == sensor_model)].iloc[0]) + + sm.name + sm.long_name + sm.definition + sm.deprecated + sm.uri + """ + + reftable = "R27" @staticmethod - def from_series(obj: pd.Series) -> 'SensorModel': + def from_series(obj: pd.Series) -> "SensorModel": return SensorModel(obj) @@ -78,12 +87,12 @@ def __init__( model: str = None, **kwargs, ): - """Create an ArgoSensor helper class instance + """Argo sensors helper class Parameters ---------- model: str, optional - A sensor model to use, by default None. + An exact sensor model name, by default None. Allowed values can be obtained with: ``ArgoSensor().reference_model_name`` @@ -100,40 +109,50 @@ def __init__( Examples -------- .. code-block:: python - :caption: ? + :caption: Access and search reference tables 25 and 27 from argopy import ArgoSensor - # Return the reference table R27 with the list of sensor models + # Return reference table R27-"Argo sensor models" with the list of sensor models ArgoSensor().reference_model - ArgoSensor().reference_model_name # Only the list of names (used to fill 'SENSOR_MODEL') + ArgoSensor().reference_model_name # Only the list of names (used to fill 'SENSOR_MODEL' parameter) - # Return the reference table R25 with the list of sensor types + # Return reference table R25-"Argo sensor types" with the list of sensor types ArgoSensor().reference_sensor - ArgoSensor().reference_sensor_type # Only the list of types (used to fill 'SENSOR') + ArgoSensor().reference_sensor_type # Only the list of types (used to fill 'SENSOR' parameter) - # Return all (R27) referenced sensor models with some string in their name + # Search for all (R27) referenced sensor models with some string in their name ArgoSensor().search_model('RBR') - ArgoSensor().search_model('RBR', output='name') # Return list of names instead of dataframe + ArgoSensor().search_model('RBR', output='name') # Return a list of names instead of a DataFrame ArgoSensor().search_model('SBE41CP', strict=False) - ArgoSensor().search_model('SBE41CP', strict=True) + ArgoSensor().search_model('SBE41CP', strict=True) # Exact string match required + + .. code-block:: python + :caption: Search the Argo dataset for some sensor models and retrieve a list of WMOs or sensor serial numbers, or both - # Return list of WMOs equipped with a sensor model name having some string + from argopy import ArgoSensor + + # Search for sensor model name(s) having some string and return a list of WMOs equipped with it/them ArgoSensor().search('RBR', output='wmo') - # Return list of sensor serial number for a sensor model name having some string + # Search for sensor model name(s) having some string and return a list of sensor serial numbers in Argo ArgoSensor().search('RBR_ARGO3_DEEP6K', output='sn') ArgoSensor().search('RBR_ARGO3_DEEP6K', output='sn', progress=True) - # Return dict of WMOs with sensor serial number for a sensor model name having some string + # Search for sensor model name(s) having some string and return a list of tuples with WMOs and serial numbers for those equipped with this model ArgoSensor().search('SBE', output='wmo_sn') ArgoSensor().search('SBE', output='wmo_sn', progress=True) - # Loop through each (or chunks) of ArgoFloat instances for floats equipped with a sensor model name having some string - for af in ArgoSensor().iterfloats_with("RAFOS"): + .. code-block:: python + :caption: Search the Argo dataset for some sensor models and easily loop through `ArgoFloat` instances for each floats + + from argopy import ArgoSensor + + model = "RAFOS" + for af in ArgoSensor().iterfloats_with(model): print(af.WMO) - # Same loop, show casing how to use the metadata attribute of an ArgoFloat instance: + # Example for how to use the metadata attribute of an ArgoFloat instance: model = "RAFOS" for af in ArgoSensor().iterfloats_with(model): models = af.metadata['sensors'] @@ -141,16 +160,30 @@ def __init__( if model in s['model']: print(af.WMO, s['maker'], s['model'], s['serial']) + .. code-block:: python + :caption: Use an exact sensor model name to create an instance + + from argopy import ArgoSensor + + sensor = ArgoSensor('RBR_ARGO3_DEEP6K') + + sensor.model + sensor.type + + sensor.search(output='wmo') + sensor.search(output='sn') + sensor.search(output='wmo_sn') + Notes ----- - Related NVS issues: + Related ADMT/AVTT issues: - https://github.com/OneArgo/ADMT/issues/112 - https://github.com/OneArgo/ArgoVocabs/issues/156 - https://github.com/OneArgo/ArgoVocabs/issues/157 """ - self._cache = kwargs.get('cache', True) - self._cachedir = kwargs.get('cachedir', OPTIONS["cachedir"]) - self.timeout = kwargs.get('timeout', OPTIONS["api_timeout"]) + self._cache = kwargs.get("cache", True) + self._cachedir = kwargs.get("cachedir", OPTIONS["cachedir"]) + self.timeout = kwargs.get("timeout", OPTIONS["api_timeout"]) self.fs = httpstore(cache=self._cache, cachedir=self._cachedir) self._r25 = None # will be loaded when necessary @@ -167,7 +200,7 @@ def __init__( if df.shape[0] == 1: self._model = SensorModel.from_series(df) - self._type = self.model_to_type(self._model, errors='ignore') + self._type = self.model_to_type(self._model, errors="ignore") else: raise InvalidDatasetStructure( f"Found multiple sensor models with '{model}'. Restrict your sensor model name to only one value in: {to_list(df['altLabel'].values)}" @@ -180,14 +213,31 @@ def __init__( def _load_mappers(self): """Load the NVS R25 to R27 key mappings""" df = [] - for p in Path(path2assets).joinpath("nvs_R25_R27").glob("NVS_R25_R27_mappings_*.txt"): - df.append(filestore().read_csv(p, header=None, names=["origin", "model", "?", "destination", "type", "??"])) + for p in ( + Path(path2assets).joinpath("nvs_R25_R27").glob("NVS_R25_R27_mappings_*.txt") + ): + df.append( + filestore().read_csv( + p, + header=None, + names=["origin", "model", "?", "destination", "type", "??"], + ) + ) df = pd.concat(df) df = df.reset_index(drop=True) self.r27_to_r25 = {} - df.apply(lambda row: self.r27_to_r25.update({row['model'].strip(): row['type'].strip()}), axis=1) + df.apply( + lambda row: self.r27_to_r25.update( + {row["model"].strip(): row["type"].strip()} + ), + axis=1, + ) - def model_to_type(self, model: Union[str, SensorModel] = None, errors : Literal['raise', 'ignore'] = 'raise') -> Optional[SensorType]: + def model_to_type( + self, + model: Union[str, SensorModel] = None, + errors: Literal["raise", "ignore"] = "raise", + ) -> Optional[SensorType]: """Read a sensor type for a given sensor model""" model_name = model.name if isinstance(model, SensorModel) else model sensor_type = self.r27_to_r25.get(model_name, None) @@ -196,8 +246,10 @@ def model_to_type(self, model: Union[str, SensorModel] = None, errors : Literal[ self.reference_sensor["altLabel"].apply(lambda x: x == sensor_type) ].iloc[0] return SensorType.from_series(row) - elif errors == 'raise': - raise DataNotFound(f"Can't determine the type of sensor model '{model_name}' (no matching key in self.r27_to_r25 mapper)") + elif errors == "raise": + raise DataNotFound( + f"Can't determine the type of sensor model '{model_name}' (no matching key in self.r27_to_r25 mapper)" + ) return None @property @@ -307,7 +359,12 @@ def reference_sensor_type(self) -> List[str]: """ return sorted(to_list(self.reference_sensor["altLabel"].values)) - def search_model(self, model: str, strict: bool = False, output: Literal['table', 'name'] = 'table') -> pd.DataFrame: + def search_model( + self, + model: str, + strict: bool = False, + output: Literal["table", "name"] = "table", + ) -> pd.DataFrame: """Return references of Argo sensor models matching a string Look for occurrences in Argo Reference table R27 altLabel values and return a dataframe with matching row(s). @@ -330,7 +387,7 @@ def search_model(self, model: str, strict: bool = False, output: Literal['table' f"No sensor model names with '{model}' string occurrence." ) else: - if output == 'name': + if output == "name": return sorted(to_list(data["altLabel"].values)) else: return data @@ -354,7 +411,7 @@ def _search_wmo_with(self, model: str, errors="raise"): ] wmos = self.fs.post(api_point, json_data=payload) if wmos is None or len(wmos) == 0: - if errors == 'raise': + if errors == "raise": raise DataNotFound(f"No floats matching sensor model name '{model}'") else: log.error(f"No floats matching sensor model name '{model}'") @@ -443,17 +500,29 @@ def postprocess(data, **kwargs): errors=errors, ) - def search(self, model: str, output: Literal['wmo', 'sn', 'wmo_sn'] = 'wmo', progress=False, errors='raise'): - if output == 'wmo': + def search( + self, + model: str = None, + output: Literal["wmo", "sn", "wmo_sn"] = "wmo", + progress=False, + errors="raise", + ): + if model is None and self.model is not None: + model = self.model.name + if output == "wmo": return self._search_wmo_with(model=model, errors=errors) - elif output == 'sn': + elif output == "sn": return self._search_sn_with(model=model, progress=progress, errors=errors) - elif output == 'wmo_sn': - return self._search_wmo_sn_with(model=model, progress=progress, errors=errors) + elif output == "wmo_sn": + return self._search_wmo_sn_with( + model=model, progress=progress, errors=errors + ) else: - raise OptionValueError("'output' option value must be in: 'wmo', 'sn', or 'wmo_sn'") + raise OptionValueError( + "'output' option value must be in: 'wmo', 'sn', or 'wmo_sn'" + ) - def iterfloats_with(self, model: str, chunksize: int = None): + def iterfloats_with(self, model: str = None, chunksize: int = None): """Iterate over ArgoFloat equipped with a given sensor model By default, iterate over a single float, otherwise use the `chunksize` argument to iterate over chunk of floats. @@ -480,6 +549,9 @@ def iterfloats_with(self, model: str, chunksize: int = None): float # is a ArgoFloat instance """ + if model is None and self.model is not None: + model = self.model.name + wmos = self._search_wmo_with(model=model) idx = ArgoIndex( From f23cf0575bc7f5eb5d0c7b5be4485d72a3068342 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Fri, 3 Oct 2025 21:54:39 +0200 Subject: [PATCH 10/71] Refactor NVSrow class to utils.accessories --- argopy/utils/__init__.py | 3 +- argopy/utils/accessories.py | 71 +++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/argopy/utils/__init__.py b/argopy/utils/__init__.py index 882595e9d..954844e65 100644 --- a/argopy/utils/__init__.py +++ b/argopy/utils/__init__.py @@ -36,7 +36,7 @@ from .caching import clear_cache, lscache from .monitored_threadpool import MyThreadPoolExecutor as MonitoredThreadPoolExecutor from .chunking import Chunker -from .accessories import Registry, float_wmo +from .accessories import Registry, float_wmo, NVSrow from .locals import ( # noqa: F401 show_versions, show_options, @@ -117,6 +117,7 @@ # Accessories classes (specific objects): "Registry", "float_wmo", + "NVSrow", # Locals (environments, versions, systems): "show_versions", "show_options", diff --git a/argopy/utils/accessories.py b/argopy/utils/accessories.py index be0bd7313..95f8e1b7e 100644 --- a/argopy/utils/accessories.py +++ b/argopy/utils/accessories.py @@ -3,6 +3,9 @@ import warnings import logging import copy +from dataclasses import dataclass +from typing import ClassVar +import pandas as pd from .checkers import check_wmo, is_wmo @@ -256,3 +259,71 @@ def __copy__(self): def copy(self): """Return a shallow copy of the registry""" return self.__copy__() + + +@dataclass +class NVSrow: + """This proto makes it easier to work with a single NVS table row from a :class:`pd.DataFrame` + + Examples + -------- + .. code-block:: python + :caption: Use this prototype to create a NVS table row class + + class SensorType(NVSrow): + reftable = "R25" + @staticmethod + def from_series(obj: pd.Series) -> "SensorType": + return SensorType(obj) + + .. code-block:: python + :caption: Then use the row class + + from argopy import ArgoNVSReferenceTables + + df = ArgoNVSReferenceTables().tbl(25) + + st = SensorType.from_series(df[df["altLabel"].apply(lambda x: x == 'CTD')].iloc[0]) + + st.name + st.long_name + st.definition + st.deprecated + st.uri + + """ + + name: str = "" + long_name: str = "" + definition: str = "" + uri: str = "" + deprecated: bool = None + + reftable: ClassVar[str] + """Reference table""" + + def __init__(self, row: pd.Series): + if not isinstance(row, pd.Series) and isinstance(row, pd.DataFrame): + row = row.iloc[0] + row = row.to_dict() + self.name = row["altLabel"] + self.long_name = row["prefLabel"] + self.definition = row["definition"] + self.deprecated = row["deprecated"] + self.uri = row["id"] + + @staticmethod + def from_series(obj: pd.Series) -> "NVSrow": + return NVSrow(obj) + + def __eq__(self, obj): + return self.name == obj + + def __repr__(self): + summary = [f"<{self.reftable}.row>"] + summary.append(f"%12s: {self.name}" % "name") + summary.append(f"%12s: {self.long_name}" % "long_name") + summary.append(f"%12s: {self.uri}" % "uri") + summary.append(f"%12s: {self.deprecated}" % "deprecated") + summary.append(f'%12s: "{self.definition}"' % "definition") + return "\n".join(summary) From 65fc83b17d2991a11f465a11087e8877ea28f1fc Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Fri, 3 Oct 2025 22:22:26 +0200 Subject: [PATCH 11/71] ArgoSensor docstrings --- argopy/related/sensors.py | 62 ++++++++++++++++++++++++++++++++------- 1 file changed, 52 insertions(+), 10 deletions(-) diff --git a/argopy/related/sensors.py b/argopy/related/sensors.py index bf99e5d7b..06f938d66 100644 --- a/argopy/related/sensors.py +++ b/argopy/related/sensors.py @@ -299,9 +299,7 @@ def __repr__(self): for meth in [ "search_model", "search_model_name", - "search_wmo_with", - "search_sn_with", - "search_wmo_sn_with", + "search", "iterfloats_with", ]: summary.append(f" β•°β”ˆβž€ ArgoSensor().{meth}()") @@ -398,6 +396,11 @@ def _search_wmo_with(self, model: str, errors="raise"): Notes ----- Based on a fleet-monitoring API request to `platformCodes/multi-lines-search` on `sensorModels` field. + + Documentation: + + https://fleetmonitoring.euro-argo.eu/swagger-ui.html#!/platform-code-controller/getPlatformCodesMultiLinesSearchUsingPOST + """ api_point = f"{OPTIONS['fleetmonitoring']}/platformCodes/multi-lines-search" payload = [ @@ -427,13 +430,13 @@ def _floats_api( progress=False, errors="raise", ): - """Fetch and process JSON data returned from the fleet-monitoring API for a list of float WMOs - - Process float metadata (calibrations, sensors, cycles, configs, ...) for all WMOs with a given sensor model name + """Search floats with a sensor model and then fetch and process JSON data returned from the fleet-monitoring API for each floats Notes ----- - Based on fleet-monitoring API requests to `/floats/{wmo}`. + Based on a POST request to the fleet-monitoring API requests to `/floats/{wmo}`. + + `Endpoint documentation `_. """ wmos = self._search_wmo_with(model) @@ -507,6 +510,44 @@ def search( progress=False, errors="raise", ): + """Search for Argo floats equipped with a sensor model name + + All information are retrieved using the `Euro-Argo fleet-monitoring API `_. + + Parameters + ---------- + model: str, optional + A string to search in the `sensorModels` field of the Euro-Argo fleet-monitoring API `platformCodes/multi-lines-search` endpoint. + + output: str, Literal["wmo", "sn", "wmo_sn"], default "wmo" + Define the output to return: + + - "wmo": a list of WMO numbers (integers) + - "sn": a list of sensor serial numbers (strings) + - "wmo_sn": a list of dictionary with WMO as key and serial numbers as values + + progress: bool, default False + Define whether to display a progress bar or not + + errors: str, default "raise" + + Returns + ------- + List[int], List[str], Dict + + Notes + ----- + The list of WMOs equipped with a given sensor model is retrieved using the Euro-Argo fleet-monitoring API and a request to the `platformCodes/multi-lines-search` endpoint using the `sensorModels` search field. + + Sensor serial numbers are given by float meta-data retrieved using the Euro-Argo fleet-monitoring API and a request to the `/floats/{wmo}` endpoint: + + See Also + -------- + `Endpoint 'platformCodes/multi-lines-search' documentation `_. + + `Endpoint '/floats/{wmo}' documentation `_. + + """ if model is None and self.model is not None: model = self.model.name if output == "wmo": @@ -523,13 +564,14 @@ def search( ) def iterfloats_with(self, model: str = None, chunksize: int = None): - """Iterate over ArgoFloat equipped with a given sensor model + """Iterator over :class:`ArgoFloat` equipped with a given sensor model By default, iterate over a single float, otherwise use the `chunksize` argument to iterate over chunk of floats. Parameters ---------- model: str + A string to search in the `sensorModels` field of the Euro-Argo fleet-monitoring API `platformCodes/multi-lines-search` endpoint. chunksize: int, optional Maximum chunk size @@ -545,8 +587,8 @@ def iterfloats_with(self, model: str = None, chunksize: int = None): .. code-block:: python :caption: Example of iteration - for float in ArgoSensor().iterfloats_with('SBE41CP'): - float # is a ArgoFloat instance + for af in ArgoSensor().iterfloats_with('SBE41CP'): + af # is a ArgoFloat instance """ if model is None and self.model is not None: From b25f252d12de95cbbb2257099b5154f0931105a2 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Sat, 4 Oct 2025 00:46:56 +0200 Subject: [PATCH 12/71] Update sensors.py - add CLI support - add dataframe export --- argopy/related/sensors.py | 148 ++++++++++++++++++++++++++++++++++---- 1 file changed, 133 insertions(+), 15 deletions(-) diff --git a/argopy/related/sensors.py b/argopy/related/sensors.py index 06f938d66..b5f013425 100644 --- a/argopy/related/sensors.py +++ b/argopy/related/sensors.py @@ -2,7 +2,7 @@ import pandas as pd import numpy as np from pathlib import Path -from typing import Optional, Literal, Union +from typing import Optional, Literal, Union, Callable import logging # from ..errors import InvalidOption @@ -132,6 +132,9 @@ def __init__( from argopy import ArgoSensor + # Search for sensor model name(s) having some string and return a DataFrame with full sensor information from floats equipped + ArgoSensor().search('RBR', output='df') + # Search for sensor model name(s) having some string and return a list of WMOs equipped with it/them ArgoSensor().search('RBR', output='wmo') @@ -174,6 +177,14 @@ def __init__( sensor.search(output='sn') sensor.search(output='wmo_sn') + .. code-block:: bash + :caption: Get clean search results from the command-line + + python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('RBR', output='wmo')" + + python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('RBR', output='sn')" + + Notes ----- Related ADMT/AVTT issues: @@ -248,10 +259,33 @@ def model_to_type( return SensorType.from_series(row) elif errors == "raise": raise DataNotFound( - f"Can't determine the type of sensor model '{model_name}' (no matching key in self.r27_to_r25 mapper)" + f"Can't determine the type of sensor model '{model_name}' (no matching key in ArgoSensor().r27_to_r25 mapper)" ) return None + def type_to_model(self, type: Union[str, SensorType], + errors: Literal["raise", "ignore"] = "raise",) -> Optional[List[str]]: + """Read all sensor models for a given sensor type""" + sensor_type = type.name if isinstance(type, SensorType) else type + result = [] + for key, val in self.r27_to_r25.items(): + if sensor_type.lower() in val.lower(): + row = self.reference_model[ + self.reference_model["altLabel"].apply(lambda x: x == key) + ].iloc[0] + result.append(SensorModel.from_series(row).name) + if len(result) == 0: + if errors == "raise": + raise DataNotFound( + f"Can't find any sensor model for this type '{sensor_type}' (no matching key in ArgoSensor().r27_to_r25 mapper)" + ) + else: + return None + else: + return result + + + @property def model(self) -> SensorModel: if isinstance(self._model, SensorModel): @@ -388,7 +422,7 @@ def search_model( if output == "name": return sorted(to_list(data["altLabel"].values)) else: - return data + return data.reset_index(drop=True) def _search_wmo_with(self, model: str, errors="raise"): """Return the list of WMOs with a given sensor model @@ -414,10 +448,15 @@ def _search_wmo_with(self, model: str, errors="raise"): ] wmos = self.fs.post(api_point, json_data=payload) if wmos is None or len(wmos) == 0: + try: + search_hint = self.search_model(model, output='name', strict=False) + msg = f"No floats matching this sensor model name '{model}'. Possible hint: %s" % ("; ".join(search_hint)) + except DataNotFound: + msg = f"No floats matching this sensor model name '{model}'" if errors == "raise": - raise DataNotFound(f"No floats matching sensor model name '{model}'") + raise DataNotFound(msg) else: - log.error(f"No floats matching sensor model name '{model}'") + log.error(msg) return check_wmo(wmos) def _floats_api( @@ -467,7 +506,8 @@ def postprocess(data, **kwargs): S = [] for row in data: for sensor in row: - S.append(sensor) + if sensor is not None: + S.append(sensor) return np.sort(np.array(S)) return self._floats_api( @@ -503,10 +543,69 @@ def postprocess(data, **kwargs): errors=errors, ) + def _to_dataframe(self, model: str, progress=False, errors="raise") -> pd.DataFrame: + """Return a DataFrame with WMO, sensor type, model, maker, sn, units, accuracy and resolution + + Parameters + ---------- + model: str, optional + A string to search in the `sensorModels` field of the Euro-Argo fleet-monitoring API `platformCodes/multi-lines-search` endpoint. + + """ + if model is None and self.model is not None: + model = self.model.name + + def preprocess(jsdata, model_name: str = ""): + output = [] + for s in jsdata["sensors"]: + if model_name in s["model"]: + this = [jsdata["wmo"]] + [ + this.append(s[key]) + for key in [ + "id", + "maker", + "model", + "serial", + "units", + "accuracy", + "resolution", + ] + ] + output.append(this) + return output + + def postprocess(data, **kwargs): + d = [] + for this in data: + for wmo, sid, maker, model, sn, units, accuracy, resolution in this: + d.append( + { + "WMO": wmo, + "Type": sid, + "Model": model, + "Maker": maker, + "SerialNumber": sn, + "Units": units, + "Accuracy": accuracy, + "Resolution": resolution, + } + ) + return pd.DataFrame(d).sort_values(by="WMO").reset_index(drop=True) + + return self._floats_api( + model, + preprocess=preprocess, + preprocess_opts={"model_name": model}, + postprocess=postprocess, + progress=progress, + errors=errors, + ) + def search( self, model: str = None, - output: Literal["wmo", "sn", "wmo_sn"] = "wmo", + output: Literal["wmo", "sn", "wmo_sn", "df"] = "wmo", progress=False, errors="raise", ): @@ -519,9 +618,10 @@ def search( model: str, optional A string to search in the `sensorModels` field of the Euro-Argo fleet-monitoring API `platformCodes/multi-lines-search` endpoint. - output: str, Literal["wmo", "sn", "wmo_sn"], default "wmo" + output: str, Literal["wmo", "sn", "wmo_sn", "df"], default "wmo" Define the output to return: + - "df": a :class:`pandas.DataFrame` with WMO, sensor type/model/maker and serial number - "wmo": a list of WMO numbers (integers) - "sn": a list of sensor serial numbers (strings) - "wmo_sn": a list of dictionary with WMO as key and serial numbers as values @@ -533,7 +633,7 @@ def search( Returns ------- - List[int], List[str], Dict + :class:`pandas.DataFrame`, List[int], List[str], Dict Notes ----- @@ -541,16 +641,16 @@ def search( Sensor serial numbers are given by float meta-data retrieved using the Euro-Argo fleet-monitoring API and a request to the `/floats/{wmo}` endpoint: - See Also - -------- - `Endpoint 'platformCodes/multi-lines-search' documentation `_. + `Endpoint platformCodes/multi-lines-search documentation `_. - `Endpoint '/floats/{wmo}' documentation `_. + `Endpoint /floats/{wmo} documentation `_. """ if model is None and self.model is not None: model = self.model.name - if output == "wmo": + if output == "df": + return self._to_dataframe(model=model, progress=progress, errors=errors) + elif output == "wmo": return self._search_wmo_with(model=model, errors=errors) elif output == "sn": return self._search_sn_with(model=model, progress=progress, errors=errors) @@ -560,7 +660,7 @@ def search( ) else: raise OptionValueError( - "'output' option value must be in: 'wmo', 'sn', or 'wmo_sn'" + "'output' option value must be in: 'wmo', 'sn', 'wmo_sn' or 'df'" ) def iterfloats_with(self, model: str = None, chunksize: int = None): @@ -615,3 +715,21 @@ def iterfloats_with(self, model: str = None, chunksize: int = None): else: for wmo in wmos: yield ArgoFloat(wmo, idx=idx) + + def cli_search(self, model: str, output: str = "wmo"): + """Quick sensor lookups from the terminal + + This function is a command-line-friendly output for float search (e.g., for piping to other tools). + + Examples + -------- + .. code-block:: bash + :caption: Example of search results from the command-line + + python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('RBR', output='wmo')" + + python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('RBR', output='sn')" + + """ + results = self.search(model, output=output) + print("\n".join(map(str, results))) From d1d6df01bcfaa21d4b0e2a9f159ff9bd8ce99a53 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Sat, 4 Oct 2025 00:47:03 +0200 Subject: [PATCH 13/71] Update accessories.py --- argopy/utils/accessories.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/argopy/utils/accessories.py b/argopy/utils/accessories.py index 95f8e1b7e..69eadb9c6 100644 --- a/argopy/utils/accessories.py +++ b/argopy/utils/accessories.py @@ -325,5 +325,5 @@ def __repr__(self): summary.append(f"%12s: {self.long_name}" % "long_name") summary.append(f"%12s: {self.uri}" % "uri") summary.append(f"%12s: {self.deprecated}" % "deprecated") - summary.append(f'%12s: "{self.definition}"' % "definition") + summary.append("%12s: %s" % ("definition", self.definition)) return "\n".join(summary) From db8c765685a172d8de10d173bc5a9be0b6f77a2e Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Sat, 4 Oct 2025 00:51:02 +0200 Subject: [PATCH 14/71] documentation --- docs/advanced-tools/metadata/index.rst | 3 +- docs/advanced-tools/metadata/sensors.rst | 158 +++++++++++++++++++++++ docs/api-hidden.rst | 11 ++ docs/api.rst | 1 + docs/whats-new.rst | 7 +- 5 files changed, 178 insertions(+), 2 deletions(-) create mode 100644 docs/advanced-tools/metadata/sensors.rst diff --git a/docs/advanced-tools/metadata/index.rst b/docs/advanced-tools/metadata/index.rst index bf8a00887..2cf51db55 100644 --- a/docs/advanced-tools/metadata/index.rst +++ b/docs/advanced-tools/metadata/index.rst @@ -13,6 +13,7 @@ Here we document **argopy** tools related to: * :doc:`the ADMT collection of documentation manuals ` * :doc:`the global deployment plan from Ocean-OPS ` * :doc:`GDAC snapshot with DOIs ` +* :doc:`Argo sensor models and types ` Note that data more specifically used in quality control are described in the dedicated :ref:`data_qc` documentation section (e.g. topography, altimetry, in-situ reference data from Argo float and GOSHIP, etc ...). @@ -24,4 +25,4 @@ topography, altimetry, in-situ reference data from Argo float and GOSHIP, etc .. Argo reference tables (NVS) admt_documentation deployment_plan - gdac_doi + argosensor diff --git a/docs/advanced-tools/metadata/sensors.rst b/docs/advanced-tools/metadata/sensors.rst new file mode 100644 index 000000000..69054ae21 --- /dev/null +++ b/docs/advanced-tools/metadata/sensors.rst @@ -0,0 +1,158 @@ +.. currentmodule:: argopy.related +.. _argosensor: + +ArgoSensor +========== + +**Simplify Argo float sensor queries with standardized metadata access.** + +The :class:`ArgoSensor` class provides direct access to Argo's sensor metadata through Reference Tables R25 (sensor types) and R27 (sensor models), combined with the Euro-Argo fleet-monitoring API. This enables users to: + +- Search for floats equipped with specific sensor models +- Retrieve sensor serial numbers across the global array + +.. _argosensor-reference-tables: + +Access and Search Reference Tables 25 and 27 +-------------------------------------------- + +**Purpose**: Explore the official Argo vocabularies for sensor types (R25) and models (R27). + +.. csv-table:: Attributes and Methods + :header: "Attribute/Method", "Description" + :widths: 30, 70 + + ``reference_model``, "Returns Reference Table **R27** (``SENSOR_MODEL``) as a ``pandas.DataFrame``" + ``reference_model_name``, "List of all sensor model names (e.g., ``'SBE41CP'``)" + ``reference_sensor``, "Returns Reference Table **R25** (``SENSOR``) as a ``pandas.DataFrame``" + ``reference_sensor_type``, "List of all sensor types (e.g., ``'CTD'``, ``'OPTODE'``)" + ``search_model(model, strict)``, "Search R27 for models matching a string (exact or fuzzy)" + ``model_to_type(model_name)``, "Returns AVTT mapping on sensor type for a given model name" + ``type_to_model(sensor_type)``, "Returns AVTT mapping on model names for a given sensor type" + +**Example**: + +.. code-block:: python + + from argopy import ArgoSensor + + # List all CTD sensor models (R27) + ArgoSensor().reference_model + + # Fuzzy search (default): + ArgoSensor().search_model('RBR', strict=False) + + # Exact search: + ArgoSensor().search_model('SBE61_V5.0.12', strict=True) + + # List the sensor type of a given model: + ArgoSensor().model_to_type('SBE61') + + # List all possible model names of a given sensor type: + ArgoSensor().type_to_model('FLUOROMETER_CDOM') + + +.. _argosensor-search-floats: + +Search the Argo Array for Floats Equipped with a Given Sensor Model +------------------------------------------------------------------- + +**Purpose**: Find WMOs, serial numbers, or any other sensor information floats equipped with a specific sensor model. + +.. csv-table:: Attributes and Methods + :header: "Attribute/Method", "Description" + :widths: 30, 70 + + ``search(model, output)``, "Search for floats with a sensor model" + "", "``output='wmo'``: Returns list of WMOs (e.g., ``[1901234, 1901235]``)" + "", "``output='sn'``: Returns list of serial numbers (e.g., ``['1234', '5678']``)" + "", "``output='wmo_sn'``: Returns dict ``{WMO: [serial1, serial2]}``" + "", "``output='df'``: Returns a :class:`pandas.DataFrame` with WMO, sensor type, model, maker, sn, units, accuracy and resolution + +**Example**: + +.. code-block:: python + + from argopy import ArgoSensor + + # Get WMOs for all floats with "AANDERAA_OPTODE_4831" + wmos = ArgoSensor().search("KISTLER_10153PSIA", output="df") + + # Get WMOs for all floats with "AANDERAA_OPTODE_4831" + wmos = ArgoSensor().search("AANDERAA_OPTODE_4831", output="wmo") + + # Get serial numbers for the "SBE43F_IDO" model + serials = ArgoSensor().search("SBE43F_IDO", output="sn") + + # Get WMO-serial pairs for "RBR_ARGO3_DEEP6K" + wmo_sn = sensor_tool.search("RBR_ARGO3_DEEP6K", output="wmo_sn") + for wmo, sns in list(wmo_sn.items()): # First float + print(f"Float {wmo}: Serials {sns}") + + +Using an Exact Sensor Model Name to Create an Instance +------------------------------------------------------ + +**Purpose**: Initialize an :class:`ArgoSensor` instance for a specific model to access its metadata and methods directly. + +.. csv-table:: Attributes and Methods + :header: "Attribute/Method", "Description" + :widths: 30, 70 + + ``ArgoSensor(model)``, "Create an instance for an exact model name (e.g., ``'SBE41CP'``)" + ``model``, "Returns a ``SensorModel`` object (name, long_name, definition, URI, deprecated)" + ``type``, "Returns a ``SensorType`` object (name, long_name, definition, URI, deprecated)" + ``search(output)``, "Inherits search methods but defaults to the instance's model" + +**Example**: + +.. code-block:: python + + # Create an instance for the "SBE43F_IDO" sensor model: + sbe_sensor = ArgoSensor("SBE43F_IDO") + + # Access model metadata + print(f"Model: {sbe_sensor.model.name}") + print(f"Type: {sbe_sensor.type.name}") + print(f"Definition: {sbe_sensor.model.definition}") + + # Search for floats with this model + df = sbe_sensor.search(output="df") + + # WMO Type Model Maker SerialNumber Units Accuracy Resolution + # 0 1901328 IDO_DOXY SBE43F_IDO SBE 577 micro moles 5.0 None + # 1 1901329 IDO_DOXY SBE43F_IDO SBE 579 micro moles 5.0 None + # 2 2900114 IDO_DOXY SBE43F_IDO SBE 0164 micromole/kg NaN None + # 3 2900115 IDO_DOXY SBE43F_IDO SBE 0188 micromole/kg NaN None + # 4 2900116 IDO_DOXY SBE43F_IDO SBE 0179 micromole/kg NaN None + + +Loop Through ArgoFloat Instances for Each Float +----------------------------------------------- + +**Purpose**: Iterate over ``ArgoFloat`` instances for floats matching a sensor model. + +.. csv-table:: Attributes and Methods + :header: "Attribute/Method", "Description" + :widths: 30, 70 + + ``iterfloats_with(model)``, "Yields :class:`ArgoFloat` instances for floats with the specified sensor model" + "", "Use ``chunksize`` to process floats in batches" + +**Example**: + +.. code-block:: python + + from argopy import ArgoSensor + # Loop through all floats with "RAFOS" sensors + for afloat in ArgoSensor().iterfloats_with("RAFOS"): + print(f"Float {afloat.WMO}: Platform type = {afloat.metadata['platform_type']}") + + # Access sensor metadata + for sensor in afloat.metadata["sensors"]: + if "RAFOS" in sensor["model"]: + print(f" - Maker: {sensor['maker']}, Serial: {sensor['serial']}") + + # Output: + # Float 1901234: Platform type = APF11 + # - Maker: Webb Research, Serial: RAFOS-001 diff --git a/docs/api-hidden.rst b/docs/api-hidden.rst index 920f1e2fc..09b85be0a 100644 --- a/docs/api-hidden.rst +++ b/docs/api-hidden.rst @@ -177,6 +177,17 @@ argopy.related.ArgoDOI.doi argopy.related.doi_snapshot.DOIrecord + argopy.related.ArgoSensor + argopy.related.ArgoSensor.reference_model + argopy.related.ArgoSensor.reference_model_name + argopy.related.ArgoSensor.reference_sensor + argopy.related.ArgoSensor.reference_sensor_type + argopy.related.ArgoSensor.search_model + argopy.related.ArgoSensor.model_to_type + argopy.related.ArgoSensor.type_to_model + argopy.related.ArgoSensor.search + argopy.related.ArgoSensor.iterfloats_with + argopy.plot argopy.plot.dashboard argopy.plot.bar_plot diff --git a/docs/api.rst b/docs/api.rst index 1df9917e1..7823e169c 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -167,6 +167,7 @@ Argo meta/related data OceanOPSDeployments CTDRefDataFetcher TopoFetcher + ArgoSensor .. _Module Visualisation: diff --git a/docs/whats-new.rst b/docs/whats-new.rst index 3108a4d2c..3a8adc507 100644 --- a/docs/whats-new.rst +++ b/docs/whats-new.rst @@ -10,8 +10,13 @@ What's New Coming up next (unreleased) --------------------------- +Features and front-end API +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- **New** :class:`ArgoSensor` class to work with Argo sensor models, types and floats equipped with. (:pr:`532`) by |gmaze|. + Internals ---------- +^^^^^^^^^ - **New post method for the** :class:`stores.httpstore` by |gmaze|. From 8c7f9160e55811c707f709e4529718bd51a10ec9 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Sat, 4 Oct 2025 22:49:15 +0200 Subject: [PATCH 15/71] Update reference_tables.py - refactor ArgoNVSReferenceTables to inherit from a more general NVScollection class, not limited to Argo tables --- argopy/related/reference_tables.py | 238 ++++++++++++++++------------- 1 file changed, 135 insertions(+), 103 deletions(-) diff --git a/argopy/related/reference_tables.py b/argopy/related/reference_tables.py index 6ec96cc7d..28854f33f 100644 --- a/argopy/related/reference_tables.py +++ b/argopy/related/reference_tables.py @@ -5,68 +5,8 @@ from ..options import OPTIONS -class ArgoNVSReferenceTables: - """Argo Reference Tables - - Utility function to retrieve Argo Reference Tables from a NVS server. - - By default, this relies on: https://vocab.nerc.ac.uk/collection - - Examples - -------- - Methods: - - >>> R = ArgoNVSReferenceTables() - >>> R.search('sensor') - >>> R.tbl(3) - >>> R.tbl('R09') - - Properties: - - >>> R.all_tbl_name - >>> R.all_tbl - >>> R.valid_ref - - """ - - valid_ref = [ - "R01", - - "R03", - "R04", - "R05", - "R06", - "R07", - "R08", - "R09", - "R10", - "R11", - "R12", - "R13", - "R15", - "R16", - - "R18", - "R19", - "R20", - "R21", - "R22", - "R23", - "R24", - "R25", - "R26", - "R27", - "R28", - - "R40", - - "RD2", - "RMC", - "RP2", - "RR2", - "RTV", - ] - """List of all available Reference Tables""" +class NVScollection: + """ A class to handle any NVS collection table """ def __init__( self, @@ -74,20 +14,18 @@ def __init__( cache: bool = True, cachedir: str = "", ): - """Argo Reference Tables from NVS""" - + """Reference Tables from NVS collection""" cachedir = OPTIONS["cachedir"] if cachedir == "" else cachedir self.fs = httpstore(cache=cache, cachedir=cachedir) self.nvs = nvs + @property + def valid_ref(self): + df = self._FullCollection() + return df['ID'].to_list() + def _valid_ref(self, rtid): - if rtid not in self.valid_ref: - rtid = "R%0.2d" % rtid - if rtid not in self.valid_ref: - raise ValueError( - "Invalid Argo Reference Table, should be one in: %s" - % ", ".join(self.valid_ref) - ) + """No validation""" return rtid def _jsConcept2df(self, data): @@ -122,6 +60,25 @@ def _jsCollection(self, data): rtid = k["@id"] return (name, desc, rtid) + def _jsFullCollection(self, data): + """Return all skos:Collection information as data""" + result = [] + for k in data["@graph"]: + if k["@type"] == "skos:Collection": + title = k["dc:title"] + name = k["dc:alternative"] + desc = k["dc:description"] + url = k["@id"] + tid = k['@id'].split('/')[-3] + result.append((tid, title, name, desc, url)) + return result + + @lru_cache + def _FullCollection(self): + url = f"{self.nvs}/?_profile=nvs&_mediatype=application/ld+json" + js = self.fs.open_json(url) + return pd.DataFrame(self._jsFullCollection(js), columns=['ID', 'title', 'name', 'description', 'url']) + def get_url(self, rtid, fmt="ld+json"): """Return URL toward a given reference table for a given format @@ -152,7 +109,7 @@ def get_url(self, rtid, fmt="ld+json"): @lru_cache def tbl(self, rtid): - """Return an Argo Reference table + """Return a Reference table Parameters ---------- @@ -168,8 +125,9 @@ def tbl(self, rtid): df = self._jsConcept2df(js) return df + @lru_cache def tbl_name(self, rtid): - """Return name of an Argo Reference table + """Return name of a Reference table Parameters ---------- @@ -184,37 +142,9 @@ def tbl_name(self, rtid): js = self.fs.open_json(self.get_url(rtid)) return self._jsCollection(js) - def search(self, txt, where="all"): - """Search for string in tables title and/or description - - Parameters - ---------- - txt: str - where: str, default='all' - Where to search, can be: 'title', 'description', 'all' - - Returns - ------- - list of table id matching the search - """ - results = [] - for tbl_id in self.all_tbl_name: - title = self.tbl_name(tbl_id)[0] - description = self.tbl_name(tbl_id)[1] - if where == "title": - if txt.lower() in title.lower(): - results.append(tbl_id) - elif where == "description": - if txt.lower() in description.lower(): - results.append(tbl_id) - elif where == "all": - if txt.lower() in description.lower() or txt.lower() in title.lower(): - results.append(tbl_id) - return results - @property def all_tbl(self): - """Return all Argo Reference tables + """Return all Reference tables Returns ------- @@ -230,7 +160,7 @@ def all_tbl(self): @property def all_tbl_name(self): - """Return names of all Argo Reference tables + """Return names of all Reference tables Returns ------- @@ -246,3 +176,105 @@ def all_tbl_name(self): ] all_tables = collections.OrderedDict(sorted(all_tables.items())) return all_tables + + def search(self, txt, where="all"): + """Search for string in tables title and/or description + + Parameters + ---------- + txt: str + where: str, default='all' + Where to search, can be: 'title', 'description', 'all' + + Returns + ------- + list of table id matching the search + """ + results = [] + for tbl_id in self.all_tbl_name: + title = self.tbl_name(tbl_id)[0] + description = self.tbl_name(tbl_id)[1] + if where == "title": + if txt.lower() in title.lower(): + results.append(tbl_id) + elif where == "description": + if txt.lower() in description.lower(): + results.append(tbl_id) + elif where == "all": + if txt.lower() in description.lower() or txt.lower() in title.lower(): + results.append(tbl_id) + return results + + +class ArgoNVSReferenceTables(NVScollection): + """Argo Reference Tables + + Utility function to retrieve Argo Reference Tables from a NVS server. + + By default, this relies on: https://vocab.nerc.ac.uk/collection + + Examples + -------- + Methods: + + >>> R = ArgoNVSReferenceTables() + >>> R.search('sensor') + >>> R.tbl(3) + >>> R.tbl('R09') + + Properties: + + >>> R.all_tbl_name + >>> R.all_tbl + >>> R.valid_ref + + """ + + valid_ref = [ + "R01", + + "R03", + "R04", + "R05", + "R06", + "R07", + "R08", + "R09", + "R10", + "R11", + "R12", + "R13", + "R15", + "R16", + + "R18", + "R19", + "R20", + "R21", + "R22", + "R23", + "R24", + "R25", + "R26", + "R27", + "R28", + + "R40", + + "RD2", + "RMC", + "RP2", + "RR2", + "RTV", + ] + """List of all available Reference Tables""" + + def _valid_ref(self, rtid): + if rtid not in self.valid_ref: + rtid = "R%0.2d" % rtid + if rtid not in self.valid_ref: + raise ValueError( + "Invalid Argo Reference Table, should be one in: %s" + % ", ".join(self.valid_ref) + ) + return rtid From 14ca098345719b65815353c6b241a4b0c58a38b0 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Sat, 4 Oct 2025 22:49:45 +0200 Subject: [PATCH 16/71] more docstrings --- argopy/related/__init__.py | 2 +- argopy/related/sensors.py | 261 ++++++++++++++++++++++++++++-------- argopy/utils/accessories.py | 14 +- 3 files changed, 215 insertions(+), 62 deletions(-) diff --git a/argopy/related/__init__.py b/argopy/related/__init__.py index 633dd295a..9ad2e95bc 100644 --- a/argopy/related/__init__.py +++ b/argopy/related/__init__.py @@ -4,7 +4,7 @@ from .argo_documentation import ArgoDocs from .doi_snapshot import ArgoDOI from .euroargo_api import get_coriolis_profile_id, get_ea_profile_page -from .sensors import ArgoSensor +from .sensors import ArgoSensor, SensorType, SensorModel from .utils import load_dict, mapp_dict # Should come last # diff --git a/argopy/related/sensors.py b/argopy/related/sensors.py index b5f013425..695f30ee5 100644 --- a/argopy/related/sensors.py +++ b/argopy/related/sensors.py @@ -2,7 +2,7 @@ import pandas as pd import numpy as np from pathlib import Path -from typing import Optional, Literal, Union, Callable +from typing import Optional, Literal, Union, Dict import logging # from ..errors import InvalidOption @@ -28,14 +28,15 @@ class SensorType(NVSrow): Examples -------- .. code-block:: python - :caption: Search methods from argopy import ArgoNVSReferenceTables + sensor_type = 'CTD' + df = ArgoNVSReferenceTables().tbl(25) + df_match = df[df["altLabel"].apply(lambda x: x == sensor_type)].iloc[0] - sensor_type = 'CTD' - st = SensorType.from_series(df[df["altLabel"].apply(lambda x: x == sensor_type)].iloc[0]) + st = SensorType.from_series(df_match) st.name st.long_name @@ -49,6 +50,7 @@ class SensorType(NVSrow): @staticmethod def from_series(obj: pd.Series) -> "SensorType": + """Create a :class:`SensorType` from a a R25-"Argo sensor models" row""" return SensorType(obj) @@ -58,14 +60,15 @@ class SensorModel(NVSrow): Examples -------- .. code-block:: python - :caption: Search methods from argopy import ArgoNVSReferenceTables + sensor_model = 'AANDERAA_OPTODE_4330F' + df = ArgoNVSReferenceTables().tbl(27) + df_match = df[df["altLabel"].apply(lambda x: x == sensor_model)].iloc[0] - sensor_model = 'AANDERAA_OPTODE_4330F' - sm = SensorModel.from_series(df[df["altLabel"].apply(lambda x: x == sensor_model)].iloc[0]) + sm = SensorModel.from_series(df_match) sm.name sm.long_name @@ -78,24 +81,35 @@ class SensorModel(NVSrow): @staticmethod def from_series(obj: pd.Series) -> "SensorModel": + """Create a :class:`SensorModel` from a R27-"Argo sensor models" row""" return SensorModel(obj) class ArgoSensor: + """ + Argo sensors helper class + + The :class:`ArgoSensor` class provides direct access to Argo's sensor metadata through Reference Tables `Sensor Types (R25) `_ and `Sensor Models (R27) `_, combined with the `Euro-Argo fleet-monitoring API `_. This enables users to: + + - navigate reference tables 25 and 27, + - search for/iterate over floats equipped with specific sensor models, + - retrieve sensor serial numbers across the global array. + + """ def __init__( self, model: str = None, **kwargs, ): - """Argo sensors helper class + """Create an instance of :class:`ArgoSensor` Parameters ---------- model: str, optional - An exact sensor model name, by default None. + An exact sensor model name, by default set to None because this is optional. - Allowed values can be obtained with: - ``ArgoSensor().reference_model_name`` + Allowed possible values can be obtained with: + :class:`ArgoSensor.reference_model_name` Other Parameters ---------------- @@ -109,45 +123,45 @@ def __init__( Examples -------- .. code-block:: python - :caption: Access and search reference tables 25 and 27 + :caption: Access and search reference tables from argopy import ArgoSensor - # Return reference table R27-"Argo sensor models" with the list of sensor models + # Reference table R27-"Argo sensor models" with the list of sensor models ArgoSensor().reference_model ArgoSensor().reference_model_name # Only the list of names (used to fill 'SENSOR_MODEL' parameter) - # Return reference table R25-"Argo sensor types" with the list of sensor types + # Reference table R25-"Argo sensor types" with the list of sensor types ArgoSensor().reference_sensor ArgoSensor().reference_sensor_type # Only the list of types (used to fill 'SENSOR' parameter) - # Search for all (R27) referenced sensor models with some string in their name + # Search for all referenced sensor models with some string in their name ArgoSensor().search_model('RBR') ArgoSensor().search_model('RBR', output='name') # Return a list of names instead of a DataFrame ArgoSensor().search_model('SBE41CP', strict=False) ArgoSensor().search_model('SBE41CP', strict=True) # Exact string match required .. code-block:: python - :caption: Search the Argo dataset for some sensor models and retrieve a list of WMOs or sensor serial numbers, or both + :caption: Search for Argo floats with some sensor models from argopy import ArgoSensor - # Search for sensor model name(s) having some string and return a DataFrame with full sensor information from floats equipped - ArgoSensor().search('RBR', output='df') - - # Search for sensor model name(s) having some string and return a list of WMOs equipped with it/them + # Search and return a list of WMOs equipped with it/them ArgoSensor().search('RBR', output='wmo') - # Search for sensor model name(s) having some string and return a list of sensor serial numbers in Argo + # Search and return a list of sensor serial numbers in Argo ArgoSensor().search('RBR_ARGO3_DEEP6K', output='sn') ArgoSensor().search('RBR_ARGO3_DEEP6K', output='sn', progress=True) - # Search for sensor model name(s) having some string and return a list of tuples with WMOs and serial numbers for those equipped with this model + # Search and return a list of tuples with WMOs and serial numbers for those equipped with this model ArgoSensor().search('SBE', output='wmo_sn') ArgoSensor().search('SBE', output='wmo_sn', progress=True) + # Search and return a DataFrame with full sensor information from floats equipped + ArgoSensor().search('RBR', output='df') + .. code-block:: python - :caption: Search the Argo dataset for some sensor models and easily loop through `ArgoFloat` instances for each floats + :caption: Easily loop through `ArgoFloat` instances for each floats equipped with a sensor model from argopy import ArgoSensor @@ -178,7 +192,7 @@ def __init__( sensor.search(output='wmo_sn') .. code-block:: bash - :caption: Get clean search results from the command-line + :caption: Get clean search results from the command-line with :class:`ArgoSensor.cli_search` python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('RBR', output='wmo')" @@ -187,7 +201,7 @@ def __init__( Notes ----- - Related ADMT/AVTT issues: + Related ADMT/AVTT work: - https://github.com/OneArgo/ADMT/issues/112 - https://github.com/OneArgo/ArgoVocabs/issues/156 - https://github.com/OneArgo/ArgoVocabs/issues/157 @@ -222,7 +236,10 @@ def __init__( self._type = None def _load_mappers(self): - """Load the NVS R25 to R27 key mappings""" + """Load from static assets file the NVS R25 to R27 key mappings + + These mapping files were download from https://github.com/OneArgo/ArgoVocabs/issues/156. + """ df = [] for p in ( Path(path2assets).joinpath("nvs_R25_R27").glob("NVS_R25_R27_mappings_*.txt") @@ -236,20 +253,58 @@ def _load_mappers(self): ) df = pd.concat(df) df = df.reset_index(drop=True) - self.r27_to_r25 = {} + self._r27_to_r25 = {} df.apply( - lambda row: self.r27_to_r25.update( + lambda row: self._r27_to_r25.update( {row["model"].strip(): row["type"].strip()} ), axis=1, ) + @property + def r27_to_r25(self) -> Dict: + """Dictionary mapping of R27 to R25 + + This mapping is from files for group 1, 2, 2b, 3, 3b and 4(s) downloaded on 2025/10/03 from https://github.com/OneArgo/ArgoVocabs/issues/156. + + Returns + ------- + Dict + + Notes + ----- + If you think a key is missing or that mapping files bundled with argopy are out of date, please raise an issue at https://github.com/euroargodev/argopy/issues. + """ + if self._r27_to_r25 is None: + self._load_mappers() + return self._r27_to_r25 + def model_to_type( self, model: Union[str, SensorModel] = None, errors: Literal["raise", "ignore"] = "raise", ) -> Optional[SensorType]: - """Read a sensor type for a given sensor model""" + """Get a sensor type for a given sensor model + + All valid sensor model name can be obtained with :attr:`ArgoSensor.reference_model_name`. + + Mapping between sensor model name (R27) and sensor type (R25) are from AVTT work at https://github.com/OneArgo/ArgoVocabs/issues/156. + + Parameters + ---------- + model : Union[str, SensorModel] + The model to read the sensor type for. + errors : Literal["raise", "ignore"] = "raise" + How to handle possible errors. If set to "ignore", the method will return None. + + Returns + ------- + :class:`SensorType` or None + + See Also + -------- + :attr:`ArgoSensor.type_to_model` + """ model_name = model.name if isinstance(model, SensorModel) else model sensor_type = self.r27_to_r25.get(model_name, None) if sensor_type is not None: @@ -265,7 +320,27 @@ def model_to_type( def type_to_model(self, type: Union[str, SensorType], errors: Literal["raise", "ignore"] = "raise",) -> Optional[List[str]]: - """Read all sensor models for a given sensor type""" + """Get all sensor model names of a given sensor type + + All valid sensor types can be obtained with :attr:`ArgoSensor.reference_sensor_type` + + Mapping between sensor model name (R27) and sensor type (R25) are from AVTT work at https://github.com/OneArgo/ArgoVocabs/issues/156. + + Parameters + ---------- + type : Union[str, SensorType] + The sensor type to read the sensor model name for. + errors : Literal["raise", "ignore"] = "raise" + How to handle possible errors. If set to "ignore", the method will return None. + + Returns + ------- + List[str] + + See Also + -------- + :attr:`ArgoSensor.model_to_type` + """ sensor_type = type.name if isinstance(type, SensorType) else type result = [] for key, val in self.r27_to_r25.items(): @@ -284,10 +359,20 @@ def type_to_model(self, type: Union[str, SensorType], else: return result - - @property def model(self) -> SensorModel: + """ :class:`SensorModel` of this class instance + + Only available for a class instance created with an explicit sensor model name. + + Returns + ------- + :class:`SensorModel` + + Raises + ------ + :class:`InvalidDataset` + """ if isinstance(self._model, SensorModel): return self._model else: @@ -297,6 +382,18 @@ def model(self) -> SensorModel: @property def type(self) -> SensorType: + """ :class:`SensorType` of this class instance sensor model + + Only available for a class instance created with an explicit sensor model name. + + Returns: + ------- + :class:`SensorType` + + Raises + ------ + :class:`InvalidDataset` + """ if isinstance(self._type, SensorType): return self._type else: @@ -341,11 +438,15 @@ def __repr__(self): @property def reference_model(self) -> pd.DataFrame: - """Return the official reference table for Argo sensor models + """Official reference table for Argo sensor models (R27) - Return the Argo Reference table R27 'SENSOR_MODEL': + Returns + ------- + :class:`pandas.DataFrame` - > Terms listing models of sensors mounted on Argo floats. + See Also + -------- + :class:`ArgoNVSReferenceTables` """ if self._r27 is None: self._r27 = ArgoNVSReferenceTables( @@ -355,23 +456,35 @@ def reference_model(self) -> pd.DataFrame: @property def reference_model_name(self) -> List[str]: - """Return the official list of Argo sensor models + """Official list of Argo sensor models (R27) - Return a sorted list of strings with altLabel from Argo Reference table R27 'SENSOR_MODEL'. + Return a sorted list of strings with altLabel from Argo Reference table R27 on 'SENSOR_MODEL'. + + Returns + ------- + List[str] Notes ----- Argo netCDF variable ``SENSOR_MODEL`` is populated with values from this list. + + See Also + -------- + :attr:`ArgoSensor.reference_model` """ return sorted(to_list(self.reference_model["altLabel"].values)) @property def reference_sensor(self) -> pd.DataFrame: - """Return the official list of Argo sensor types + """Official reference table for Argo sensor types (R25) - Return the Argo Reference table R25 'SENSOR': + Returns + ------- + :class:`pandas.DataFrame` - > Terms describing sensor types mounted on Argo floats. + See Also + -------- + :class:`ArgoNVSReferenceTables` """ if self._r25 is None: self._r25 = ArgoNVSReferenceTables( @@ -381,13 +494,21 @@ def reference_sensor(self) -> pd.DataFrame: @property def reference_sensor_type(self) -> List[str]: - """Return the official list of Argo sensor types + """Official list of Argo sensor types (R25) - Return a sorted list of strings with altLabel from Argo Reference table R25 'SENSOR'. + Return a sorted list of strings with altLabel from Argo Reference table R25 on 'SENSOR'. + + Returns + ------- + List[str] Notes ----- Argo netCDF variable ``SENSOR`` is populated with values from this list. + + See Also + -------- + :attr:`ArgoSensor.reference_sensor` """ return sorted(to_list(self.reference_sensor["altLabel"].values)) @@ -395,19 +516,36 @@ def search_model( self, model: str, strict: bool = False, - output: Literal["table", "name"] = "table", + output: Literal["df", "name"] = "df", ) -> pd.DataFrame: """Return references of Argo sensor models matching a string - Look for occurrences in Argo Reference table R27 altLabel values and return a dataframe with matching row(s). + Look for occurrences in Argo Reference table R27 `altLabel` and return a :class:`pandas.DataFrame` with matching row(s). + + Parameters + ---------- + model : str + The model to search for. + strict : bool, optional, default: False + Is the model string a strict match or an occurrence in table look up. + output : str, Literal["table", "name"], default "table" + Is the output a :class:`pandas.DataFrame` with matching rows from :attr:`ArgoSensor.reference_model`, or a list of string. + + Returns + ------- + :class:`pandas.DataFrame`, sorted(List[str]) + + See Also + -------- + :class:`ArgoSensor.reference_model` """ if strict: data = self.reference_model[ - self.reference_model["altLabel"].apply(lambda x: x == model) + self.reference_model["altLabel"].apply(lambda x: x == model.upper()) ] else: data = self.reference_model[ - self.reference_model["altLabel"].apply(lambda x: model in x) + self.reference_model["altLabel"].apply(lambda x: model.upper() in x) ] if data.shape[0] == 0: if strict: @@ -616,15 +754,15 @@ def search( Parameters ---------- model: str, optional - A string to search in the `sensorModels` field of the Euro-Argo fleet-monitoring API `platformCodes/multi-lines-search` endpoint. + A string to search in the ``sensorModels`` field of the Euro-Argo fleet-monitoring API ``platformCodes/multi-lines-search`` endpoint. output: str, Literal["wmo", "sn", "wmo_sn", "df"], default "wmo" Define the output to return: - - "df": a :class:`pandas.DataFrame` with WMO, sensor type/model/maker and serial number - - "wmo": a list of WMO numbers (integers) - - "sn": a list of sensor serial numbers (strings) - - "wmo_sn": a list of dictionary with WMO as key and serial numbers as values + - ``wmo``: a list of WMO numbers (integers) + - ``sn``: a list of sensor serial numbers (strings) + - ``wmo_sn``: a list of dictionary with WMO as key and serial numbers as values + - ``df``: a :class:`pandas.DataFrame` with WMO, sensor type/model/maker and serial number progress: bool, default False Define whether to display a progress bar or not @@ -637,13 +775,13 @@ def search( Notes ----- - The list of WMOs equipped with a given sensor model is retrieved using the Euro-Argo fleet-monitoring API and a request to the `platformCodes/multi-lines-search` endpoint using the `sensorModels` search field. + The list of WMOs equipped with a given sensor model is retrieved using the Euro-Argo fleet-monitoring API and a request to the ``platformCodes/multi-lines-search`` endpoint using the ``sensorModels`` search field. - Sensor serial numbers are given by float meta-data retrieved using the Euro-Argo fleet-monitoring API and a request to the `/floats/{wmo}` endpoint: + Sensor serial numbers are given by float meta-data retrieved using the Euro-Argo fleet-monitoring API and a request to the ``/floats/{wmo}`` endpoint: - `Endpoint platformCodes/multi-lines-search documentation `_. + - `Documentation for endpoint: platformCodes/multi-lines-search `_. - `Endpoint /floats/{wmo} documentation `_. + - `Documentation for endpoint: /floats/{wmo} `_. """ if model is None and self.model is not None: @@ -664,7 +802,7 @@ def search( ) def iterfloats_with(self, model: str = None, chunksize: int = None): - """Iterator over :class:`ArgoFloat` equipped with a given sensor model + """Iterator over :class:`argopy.ArgoFloat` equipped with a given sensor model By default, iterate over a single float, otherwise use the `chunksize` argument to iterate over chunk of floats. @@ -680,15 +818,20 @@ def iterfloats_with(self, model: str = None, chunksize: int = None): Returns ------- - Iterator of :class:`ArgoFloat` + Iterator of :class:`argopy.ArgoFloat` Examples -------- .. code-block:: python :caption: Example of iteration - for af in ArgoSensor().iterfloats_with('SBE41CP'): - af # is a ArgoFloat instance + from argopy import ArgoSensor + + for afloat in ArgoSensor().iterfloats_with("SATLANTIC_PAR"): + print(f"\n-Float {afloat.WMO}: Platform description = {afloat.metadata['platform']['description']}") + for sensor in afloat.metadata["sensors"]: + if "SATLANTIC_PAR" in sensor["model"]: + print(f" - Sensor Maker: {sensor['maker']}, Serial: {sensor['serial']}") """ if model is None and self.model is not None: diff --git a/argopy/utils/accessories.py b/argopy/utils/accessories.py index 69eadb9c6..ec5f8d767 100644 --- a/argopy/utils/accessories.py +++ b/argopy/utils/accessories.py @@ -282,8 +282,9 @@ def from_series(obj: pd.Series) -> "SensorType": from argopy import ArgoNVSReferenceTables df = ArgoNVSReferenceTables().tbl(25) + row = df[df["altLabel"].apply(lambda x: x == 'CTD')].iloc[0] - st = SensorType.from_series(df[df["altLabel"].apply(lambda x: x == 'CTD')].iloc[0]) + st = SensorType.from_series(row) st.name st.long_name @@ -294,13 +295,22 @@ def from_series(obj: pd.Series) -> "SensorType": """ name: str = "" + """From 'altLabel' column""" + long_name: str = "" + """From 'prefLabel' column""" + definition: str = "" + """From 'definition' column""" + uri: str = "" + """From 'ID' column""" + deprecated: bool = None + """From 'deprecated' column""" reftable: ClassVar[str] - """Reference table""" + """Reference table this row is based on""" def __init__(self, row: pd.Series): if not isinstance(row, pd.Series) and isinstance(row, pd.DataFrame): From 118a5b216488272f46055984b2df1a5b2ecfc756 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Sat, 4 Oct 2025 22:50:42 +0200 Subject: [PATCH 17/71] add documentation --- docs/advanced-tools/metadata/index.rst | 4 +- docs/advanced-tools/metadata/sensors.rst | 233 +++++++++++++---------- docs/api-hidden.rst | 6 + 3 files changed, 141 insertions(+), 102 deletions(-) diff --git a/docs/advanced-tools/metadata/index.rst b/docs/advanced-tools/metadata/index.rst index 2cf51db55..e8703200c 100644 --- a/docs/advanced-tools/metadata/index.rst +++ b/docs/advanced-tools/metadata/index.rst @@ -13,7 +13,7 @@ Here we document **argopy** tools related to: * :doc:`the ADMT collection of documentation manuals ` * :doc:`the global deployment plan from Ocean-OPS ` * :doc:`GDAC snapshot with DOIs ` -* :doc:`Argo sensor models and types ` +* :doc:`Argo sensor models and types ` Note that data more specifically used in quality control are described in the dedicated :ref:`data_qc` documentation section (e.g. topography, altimetry, in-situ reference data from Argo float and GOSHIP, etc ...). @@ -25,4 +25,4 @@ topography, altimetry, in-situ reference data from Argo float and GOSHIP, etc .. Argo reference tables (NVS) admt_documentation deployment_plan - argosensor + Argo sensor models and types diff --git a/docs/advanced-tools/metadata/sensors.rst b/docs/advanced-tools/metadata/sensors.rst index 69054ae21..1e476c3ee 100644 --- a/docs/advanced-tools/metadata/sensors.rst +++ b/docs/advanced-tools/metadata/sensors.rst @@ -1,158 +1,191 @@ .. currentmodule:: argopy.related .. _argosensor: -ArgoSensor -========== +Argo sensor: models and types +============================= -**Simplify Argo float sensor queries with standardized metadata access.** +The :class:`ArgoSensor` class provides direct access to Argo's sensor metadata through Reference Tables `Sensor Types (R25) `_ and `Sensor Models (R27) `_, combined with the `Euro-Argo fleet-monitoring API `_. This enables users to: -The :class:`ArgoSensor` class provides direct access to Argo's sensor metadata through Reference Tables R25 (sensor types) and R27 (sensor models), combined with the Euro-Argo fleet-monitoring API. This enables users to: +- Navigate reference tables, +- Search for floats equipped with specific sensor models, +- Retrieve sensor serial numbers across the global array. -- Search for floats equipped with specific sensor models -- Retrieve sensor serial numbers across the global array +.. contents:: + :local: .. _argosensor-reference-tables: -Access and Search Reference Tables 25 and 27 --------------------------------------------- +Work with reference tables on sensors +------------------------------------- + +With the :class:`ArgoSensor` class, you can work with official Argo vocabularies for `Sensor Types (R25) `_ and `Sensor Models (R27) `_. With these methods, it is easy to look for a specific sensor model name, which can then be used to :ref:`look for floats ` or :ref:`create a ArgoSensor class ` directly. + +.. list-table:: :class:`ArgoSensor` attributes and methods for reference tables + :header-rows: 1 + :stub-columns: 1 + + * - Attribute/Method + - Description + * - :attr:`ArgoSensor.reference_model` + - Returns Reference Table **Sensor Models (R27)** as a :class:`pandas.DataFrame` + * - :attr:`ArgoSensor.reference_model_name` + - List of all sensor model names (e.g., ``'SBE41CP'``) + * - :attr:`ArgoSensor.reference_sensor` + - Returns Reference Table **Sensor Types (R25)** as a :class:`pandas.DataFrame` + * - :attr:`ArgoSensor.reference_sensor_type` + - List of all sensor types (e.g., ``'CTD'``, ``'OPTODE'``) + * - :attr:`ArgoSensor.r27_to_r25` + - Dictionary mapping of R27 to R25 + * - :meth:`ArgoSensor.search_model` + - Search R27 for models matching a string (exact or fuzzy) + * - :meth:`ArgoSensor.model_to_type` + - Returns AVTT mapping on sensor type for a given model name + * - :meth:`ArgoSensor.type_to_model` + - Returns AVTT mapping on model names for a given sensor type + +Examples +^^^^^^^^ + +- List all CTD sensor models (R27): + +.. ipython:: python + :okwarning: -**Purpose**: Explore the official Argo vocabularies for sensor types (R25) and models (R27). + from argopy import ArgoSensor -.. csv-table:: Attributes and Methods - :header: "Attribute/Method", "Description" - :widths: 30, 70 + ArgoSensor().reference_model - ``reference_model``, "Returns Reference Table **R27** (``SENSOR_MODEL``) as a ``pandas.DataFrame``" - ``reference_model_name``, "List of all sensor model names (e.g., ``'SBE41CP'``)" - ``reference_sensor``, "Returns Reference Table **R25** (``SENSOR``) as a ``pandas.DataFrame``" - ``reference_sensor_type``, "List of all sensor types (e.g., ``'CTD'``, ``'OPTODE'``)" - ``search_model(model, strict)``, "Search R27 for models matching a string (exact or fuzzy)" - ``model_to_type(model_name)``, "Returns AVTT mapping on sensor type for a given model name" - ``type_to_model(sensor_type)``, "Returns AVTT mapping on model names for a given sensor type" +- Fuzzy search (default): -**Example**: +.. ipython:: python + :okwarning: -.. code-block:: python + ArgoSensor().search_model('SBE61_V5.0.1', strict=False) - from argopy import ArgoSensor +- Exact search: - # List all CTD sensor models (R27) - ArgoSensor().reference_model +.. ipython:: python + :okwarning: - # Fuzzy search (default): - ArgoSensor().search_model('RBR', strict=False) + ArgoSensor().search_model('SBE61_V5.0.1', strict=True) - # Exact search: - ArgoSensor().search_model('SBE61_V5.0.12', strict=True) +- List the sensor type of a given model: - # List the sensor type of a given model: - ArgoSensor().model_to_type('SBE61') +.. ipython:: python + :okwarning: + + ArgoSensor().model_to_type('RBR_ARGO3_DEEP6K') + +- List all possible model names of a given sensor type: + +.. ipython:: python + :okwarning: - # List all possible model names of a given sensor type: ArgoSensor().type_to_model('FLUOROMETER_CDOM') .. _argosensor-search-floats: -Search the Argo Array for Floats Equipped with a Given Sensor Model -------------------------------------------------------------------- +Search for Argo floats equipped with a given sensor model +--------------------------------------------------------- -**Purpose**: Find WMOs, serial numbers, or any other sensor information floats equipped with a specific sensor model. +In this section we show how to find WMOs, serial numbers for floats equipped with a specific sensor model using the :meth:`ArgoSensor.search` method. -.. csv-table:: Attributes and Methods - :header: "Attribute/Method", "Description" - :widths: 30, 70 +The method takes a model sensor name as input, and possibly 4 values to `output` to determine how to format results: - ``search(model, output)``, "Search for floats with a sensor model" - "", "``output='wmo'``: Returns list of WMOs (e.g., ``[1901234, 1901235]``)" - "", "``output='sn'``: Returns list of serial numbers (e.g., ``['1234', '5678']``)" - "", "``output='wmo_sn'``: Returns dict ``{WMO: [serial1, serial2]}``" - "", "``output='df'``: Returns a :class:`pandas.DataFrame` with WMO, sensor type, model, maker, sn, units, accuracy and resolution +- ``output='wmo'`` returns a list of WMOs (e.g., ``[1901234, 1901235]``) +- ``output='sn'`` returns a list of serial numbers (e.g., ``['1234', '5678']``) +- ``output='wmo_sn'`` returns a dict mapping serial numbers to float WMOs (e.g. ``{WMO: [serial1, serial2]}``) +- ``output='df'`` returns a :class:`pandas.DataFrame` with WMO, sensor type, model, maker, serial number, units, accuracy and resolution. -**Example**: +Examples +^^^^^^^^ -.. code-block:: python +- Get WMOs for all floats equiped with the "AANDERAA_OPTODE_4831" model: - from argopy import ArgoSensor +.. ipython:: python + :okwarning: - # Get WMOs for all floats with "AANDERAA_OPTODE_4831" - wmos = ArgoSensor().search("KISTLER_10153PSIA", output="df") + ArgoSensor().search("AANDERAA_OPTODE_4831") # output='wmo' is default - # Get WMOs for all floats with "AANDERAA_OPTODE_4831" - wmos = ArgoSensor().search("AANDERAA_OPTODE_4831", output="wmo") - # Get serial numbers for the "SBE43F_IDO" model - serials = ArgoSensor().search("SBE43F_IDO", output="sn") +- Get sensor serial numbers for floats equiped with the "SBE43F_IDO" model: - # Get WMO-serial pairs for "RBR_ARGO3_DEEP6K" - wmo_sn = sensor_tool.search("RBR_ARGO3_DEEP6K", output="wmo_sn") - for wmo, sns in list(wmo_sn.items()): # First float - print(f"Float {wmo}: Serials {sns}") +.. ipython:: python + :okwarning: + ArgoSensor().search("SBE43F_IDO", output="sn") -Using an Exact Sensor Model Name to Create an Instance ------------------------------------------------------- +- Get WMO-serial dictionnary for floats equiped with the "RBR_ARGO3_DEEP6K" model: -**Purpose**: Initialize an :class:`ArgoSensor` instance for a specific model to access its metadata and methods directly. +.. ipython:: python + :okwarning: + + wmo_sn = ArgoSensor().search("RBR_ARGO3_DEEP6K", output="wmo_sn") + wmo_sn + +.. _argosensor-exact-sensor: + +Use an exact sensor model name to create an instance +---------------------------------------------------- + +You can initialize an :class:`ArgoSensor` instance for a specific model to access its metadata and methods directly. .. csv-table:: Attributes and Methods :header: "Attribute/Method", "Description" :widths: 30, 70 - ``ArgoSensor(model)``, "Create an instance for an exact model name (e.g., ``'SBE41CP'``)" - ``model``, "Returns a ``SensorModel`` object (name, long_name, definition, URI, deprecated)" - ``type``, "Returns a ``SensorType`` object (name, long_name, definition, URI, deprecated)" - ``search(output)``, "Inherits search methods but defaults to the instance's model" + :class:`ArgoSensor`, "Create an instance for an exact model name (e.g., ``'SBE41CP'``)" + :attr:`ArgoSensor.model`, "Returns a :class:`SensorModel` object (name, long_name, definition, URI, deprecated)" + :attr:`ArgoSensor.type`, "Returns a :class:`SensorType` object (name, long_name, definition, URI, deprecated)" + :meth:`ArgoSensor.search`, "Inherits search methods but defaults to the instance's model" + +Examples +^^^^^^^^ + +As an example, let's create an instance for the "SBE43F_IDO" sensor model: + +.. ipython:: python + :okwarning: -**Example**: + sensor = ArgoSensor("SBE43F_IDO") + sensor -.. code-block:: python +You can then access model metadata: - # Create an instance for the "SBE43F_IDO" sensor model: - sbe_sensor = ArgoSensor("SBE43F_IDO") +.. ipython:: python + :okwarning: - # Access model metadata - print(f"Model: {sbe_sensor.model.name}") - print(f"Type: {sbe_sensor.type.name}") - print(f"Definition: {sbe_sensor.model.definition}") + sensor.model - # Search for floats with this model - df = sbe_sensor.search(output="df") +.. ipython:: python + :okwarning: - # WMO Type Model Maker SerialNumber Units Accuracy Resolution - # 0 1901328 IDO_DOXY SBE43F_IDO SBE 577 micro moles 5.0 None - # 1 1901329 IDO_DOXY SBE43F_IDO SBE 579 micro moles 5.0 None - # 2 2900114 IDO_DOXY SBE43F_IDO SBE 0164 micromole/kg NaN None - # 3 2900115 IDO_DOXY SBE43F_IDO SBE 0188 micromole/kg NaN None - # 4 2900116 IDO_DOXY SBE43F_IDO SBE 0179 micromole/kg NaN None + sensor.type +And you can look for floats equiped: + +.. ipython:: python + :okwarning: + + df = sensor.search(output="df") + df Loop Through ArgoFloat Instances for Each Float ----------------------------------------------- -**Purpose**: Iterate over ``ArgoFloat`` instances for floats matching a sensor model. +The :meth:`ArgoSensor.iterfloats_with` will yields :class:`argopy.ArgoFloat` instances for floats with the specified sensor model (use ``chunksize`` to process floats in batches). -.. csv-table:: Attributes and Methods - :header: "Attribute/Method", "Description" - :widths: 30, 70 - - ``iterfloats_with(model)``, "Yields :class:`ArgoFloat` instances for floats with the specified sensor model" - "", "Use ``chunksize`` to process floats in batches" +Example +^^^^^^^ -**Example**: +Loop through all floats with "SATLANTIC_PAR" sensors: -.. code-block:: python +.. ipython:: python + :okwarning: - from argopy import ArgoSensor - # Loop through all floats with "RAFOS" sensors - for afloat in ArgoSensor().iterfloats_with("RAFOS"): - print(f"Float {afloat.WMO}: Platform type = {afloat.metadata['platform_type']}") - - # Access sensor metadata - for sensor in afloat.metadata["sensors"]: - if "RAFOS" in sensor["model"]: - print(f" - Maker: {sensor['maker']}, Serial: {sensor['serial']}") - - # Output: - # Float 1901234: Platform type = APF11 - # - Maker: Webb Research, Serial: RAFOS-001 + for afloat in ArgoSensor().iterfloats_with("SATLANTIC_PAR"): + print(f"\n-Float {afloat.WMO}: Platform description = {afloat.metadata['platform']['description']}") + for sensor in afloat.metadata["sensors"]: + if "SATLANTIC_PAR" in sensor["model"]: + print(f" - Sensor Maker: {sensor['maker']}, Serial: {sensor['serial']}") \ No newline at end of file diff --git a/docs/api-hidden.rst b/docs/api-hidden.rst index 09b85be0a..84b678733 100644 --- a/docs/api-hidden.rst +++ b/docs/api-hidden.rst @@ -182,11 +182,17 @@ argopy.related.ArgoSensor.reference_model_name argopy.related.ArgoSensor.reference_sensor argopy.related.ArgoSensor.reference_sensor_type + argopy.related.ArgoSensor.model + argopy.related.ArgoSensor.type + argopy.related.ArgoSensor.r27_to_r25 argopy.related.ArgoSensor.search_model argopy.related.ArgoSensor.model_to_type argopy.related.ArgoSensor.type_to_model argopy.related.ArgoSensor.search argopy.related.ArgoSensor.iterfloats_with + argopy.related.ArgoSensor.cli_search + argopy.related.SensorModel + argopy.related.SensorType argopy.plot argopy.plot.dashboard From b37affa1ff90af583f93e7c10f0aa711bdd99ae8 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Sat, 4 Oct 2025 23:19:36 +0200 Subject: [PATCH 18/71] more documentation --- argopy/related/sensors.py | 19 +++++++---- docs/advanced-tools/metadata/sensors.rst | 40 +++++++++++++++++------- 2 files changed, 41 insertions(+), 18 deletions(-) diff --git a/argopy/related/sensors.py b/argopy/related/sensors.py index 695f30ee5..d8f050a4b 100644 --- a/argopy/related/sensors.py +++ b/argopy/related/sensors.py @@ -96,6 +96,7 @@ class ArgoSensor: - retrieve sensor serial numbers across the global array. """ + def __init__( self, model: str = None, @@ -318,8 +319,11 @@ def model_to_type( ) return None - def type_to_model(self, type: Union[str, SensorType], - errors: Literal["raise", "ignore"] = "raise",) -> Optional[List[str]]: + def type_to_model( + self, + type: Union[str, SensorType], + errors: Literal["raise", "ignore"] = "raise", + ) -> Optional[List[str]]: """Get all sensor model names of a given sensor type All valid sensor types can be obtained with :attr:`ArgoSensor.reference_sensor_type` @@ -361,7 +365,7 @@ def type_to_model(self, type: Union[str, SensorType], @property def model(self) -> SensorModel: - """ :class:`SensorModel` of this class instance + """:class:`SensorModel` of this class instance Only available for a class instance created with an explicit sensor model name. @@ -382,7 +386,7 @@ def model(self) -> SensorModel: @property def type(self) -> SensorType: - """ :class:`SensorType` of this class instance sensor model + """:class:`SensorType` of this class instance sensor model Only available for a class instance created with an explicit sensor model name. @@ -587,8 +591,11 @@ def _search_wmo_with(self, model: str, errors="raise"): wmos = self.fs.post(api_point, json_data=payload) if wmos is None or len(wmos) == 0: try: - search_hint = self.search_model(model, output='name', strict=False) - msg = f"No floats matching this sensor model name '{model}'. Possible hint: %s" % ("; ".join(search_hint)) + search_hint = self.search_model(model, output="name", strict=False) + msg = ( + f"No floats matching this sensor model name '{model}'. Possible hint: %s" + % ("; ".join(search_hint)) + ) except DataNotFound: msg = f"No floats matching this sensor model name '{model}'" if errors == "raise": diff --git a/docs/advanced-tools/metadata/sensors.rst b/docs/advanced-tools/metadata/sensors.rst index 1e476c3ee..85c7528c1 100644 --- a/docs/advanced-tools/metadata/sensors.rst +++ b/docs/advanced-tools/metadata/sensors.rst @@ -69,7 +69,7 @@ Examples ArgoSensor().search_model('SBE61_V5.0.1', strict=True) -- List the sensor type of a given model: +- Get the sensor type of a given model: .. ipython:: python :okwarning: @@ -91,17 +91,17 @@ Search for Argo floats equipped with a given sensor model In this section we show how to find WMOs, serial numbers for floats equipped with a specific sensor model using the :meth:`ArgoSensor.search` method. -The method takes a model sensor name as input, and possibly 4 values to `output` to determine how to format results: +The method takes a model sensor name as input, and possibly 4 values to the ``output`` option to determine how to format results: - ``output='wmo'`` returns a list of WMOs (e.g., ``[1901234, 1901235]``) - ``output='sn'`` returns a list of serial numbers (e.g., ``['1234', '5678']``) -- ``output='wmo_sn'`` returns a dict mapping serial numbers to float WMOs (e.g. ``{WMO: [serial1, serial2]}``) +- ``output='wmo_sn'`` returns a dictionary mapping float WMOs to serial numbers (e.g. ``{WMO: [serial1, serial2]}``) - ``output='df'`` returns a :class:`pandas.DataFrame` with WMO, sensor type, model, maker, serial number, units, accuracy and resolution. Examples ^^^^^^^^ -- Get WMOs for all floats equiped with the "AANDERAA_OPTODE_4831" model: +- Get WMOs for all floats equipped with the "AANDERAA_OPTODE_4831" model: .. ipython:: python :okwarning: @@ -109,14 +109,14 @@ Examples ArgoSensor().search("AANDERAA_OPTODE_4831") # output='wmo' is default -- Get sensor serial numbers for floats equiped with the "SBE43F_IDO" model: +- Get sensor serial numbers for floats equipped with the "SBE43F_IDO" model: .. ipython:: python :okwarning: ArgoSensor().search("SBE43F_IDO", output="sn") -- Get WMO-serial dictionnary for floats equiped with the "RBR_ARGO3_DEEP6K" model: +- Get WMO/serial-number dictionary for floats equipped with the "RBR_ARGO3_DEEP6K" model: .. ipython:: python :okwarning: @@ -129,9 +129,9 @@ Examples Use an exact sensor model name to create an instance ---------------------------------------------------- -You can initialize an :class:`ArgoSensor` instance for a specific model to access its metadata and methods directly. +You can initialize an :class:`ArgoSensor` instance with a specific model to access its metadata and methods directly. -.. csv-table:: Attributes and Methods +.. csv-table:: :class:`ArgoSensor` Attributes and Methods for a specific model :header: "Attribute/Method", "Description" :widths: 30, 70 @@ -163,7 +163,7 @@ You can then access model metadata: sensor.type -And you can look for floats equiped: +And you can look for floats equipped: .. ipython:: python :okwarning: @@ -171,7 +171,7 @@ And you can look for floats equiped: df = sensor.search(output="df") df -Loop Through ArgoFloat Instances for Each Float +Loop through ArgoFloat instances for each float ----------------------------------------------- The :meth:`ArgoSensor.iterfloats_with` will yields :class:`argopy.ArgoFloat` instances for floats with the specified sensor model (use ``chunksize`` to process floats in batches). @@ -179,7 +179,7 @@ The :meth:`ArgoSensor.iterfloats_with` will yields :class:`argopy.ArgoFloat` ins Example ^^^^^^^ -Loop through all floats with "SATLANTIC_PAR" sensors: +Loop through all floats with "SATLANTIC_PAR" sensor: .. ipython:: python :okwarning: @@ -188,4 +188,20 @@ Loop through all floats with "SATLANTIC_PAR" sensors: print(f"\n-Float {afloat.WMO}: Platform description = {afloat.metadata['platform']['description']}") for sensor in afloat.metadata["sensors"]: if "SATLANTIC_PAR" in sensor["model"]: - print(f" - Sensor Maker: {sensor['maker']}, Serial: {sensor['serial']}") \ No newline at end of file + print(f" - Sensor Maker: {sensor['maker']}, Serial: {sensor['serial']}") + +Quick float and sensor lookups from the command line +---------------------------------------------------- + +The :meth:`ArgoSensor.cli_search` function is available to search for Argo floats equipped with a given sensor model from the command line and retrieve a `CLI `_ friendly list of WMOs or serial numbers. + +This could be useful for piping to other tools. + +Here is an example with the "SATLANTIC_PAR" sensor: + +.. code-block:: bash + :caption: Call :meth:`ArgoSensor.search` from the command line + + python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('SATLANTIC_PAR', output='wmo')" + + python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('SATLANTIC_PAR', output='sn')" From 163ed70232f7782c109ee2b1d461e67821212e7e Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Sun, 5 Oct 2025 01:23:37 +0200 Subject: [PATCH 19/71] Improve type checking --- argopy/related/sensors.py | 75 ++++++++++++++++++++----------------- argopy/utils/accessories.py | 2 +- 2 files changed, 42 insertions(+), 35 deletions(-) diff --git a/argopy/related/sensors.py b/argopy/related/sensors.py index d8f050a4b..97594dea0 100644 --- a/argopy/related/sensors.py +++ b/argopy/related/sensors.py @@ -2,9 +2,12 @@ import pandas as pd import numpy as np from pathlib import Path -from typing import Optional, Literal, Union, Dict +from typing import TYPE_CHECKING, Optional, Literal, Union, Any, Iterator import logging +if TYPE_CHECKING: + SearchOutputOptions = Literal["wmo", "sn", "wmo_sn", "df"] + # from ..errors import InvalidOption from ..stores import ArgoFloat, ArgoIndex, httpstore, filestore from ..utils import check_wmo, Chunker, to_list, NVSrow @@ -99,7 +102,7 @@ class ArgoSensor: def __init__( self, - model: str = None, + model: str | None = None, **kwargs, ): """Create an instance of :class:`ArgoSensor` @@ -212,10 +215,12 @@ def __init__( self.timeout = kwargs.get("timeout", OPTIONS["api_timeout"]) self.fs = httpstore(cache=self._cache, cachedir=self._cachedir) - self._r25 = None # will be loaded when necessary - self._r27 = None # will be loaded when necessary + self._r25: pd.DataFrame | None = None # will be loaded when necessary + self._r27: pd.DataFrame | None = None # will be loaded when necessary self._load_mappers() # Load r25 model to r27 type mapping dictionary + self._model: SensorModel | None = None + self._type: SensorType | None = None if model is not None: try: df = self.search_model(model, strict=True) @@ -225,17 +230,13 @@ def __init__( ) if df.shape[0] == 1: - self._model = SensorModel.from_series(df) + self._model = SensorModel.from_series(df.iloc[0]) self._type = self.model_to_type(self._model, errors="ignore") else: raise InvalidDatasetStructure( f"Found multiple sensor models with '{model}'. Restrict your sensor model name to only one value in: {to_list(df['altLabel'].values)}" ) - else: - self._model = None - self._type = None - def _load_mappers(self): """Load from static assets file the NVS R25 to R27 key mappings @@ -254,7 +255,7 @@ def _load_mappers(self): ) df = pd.concat(df) df = df.reset_index(drop=True) - self._r27_to_r25 = {} + self._r27_to_r25: dict[str, str] = {} df.apply( lambda row: self._r27_to_r25.update( {row["model"].strip(): row["type"].strip()} @@ -263,14 +264,14 @@ def _load_mappers(self): ) @property - def r27_to_r25(self) -> Dict: + def r27_to_r25(self) -> dict[str, str]: """Dictionary mapping of R27 to R25 This mapping is from files for group 1, 2, 2b, 3, 3b and 4(s) downloaded on 2025/10/03 from https://github.com/OneArgo/ArgoVocabs/issues/156. Returns ------- - Dict + dict[str, str] Notes ----- @@ -282,7 +283,7 @@ def r27_to_r25(self) -> Dict: def model_to_type( self, - model: Union[str, SensorModel] = None, + model: Union[str, SensorModel, None] = None, errors: Literal["raise", "ignore"] = "raise", ) -> Optional[SensorType]: """Get a sensor type for a given sensor model @@ -306,7 +307,7 @@ def model_to_type( -------- :attr:`ArgoSensor.type_to_model` """ - model_name = model.name if isinstance(model, SensorModel) else model + model_name: str = model.name if isinstance(model, SensorModel) else model sensor_type = self.r27_to_r25.get(model_name, None) if sensor_type is not None: row = self.reference_sensor[ @@ -323,7 +324,7 @@ def type_to_model( self, type: Union[str, SensorType], errors: Literal["raise", "ignore"] = "raise", - ) -> Optional[List[str]]: + ) -> list[str] | None: """Get all sensor model names of a given sensor type All valid sensor types can be obtained with :attr:`ArgoSensor.reference_sensor_type` @@ -405,7 +406,7 @@ def type(self) -> SensorType: "The 'type' property is not available for an ArgoSensor instance not created with a specific sensor model" ) - def __repr__(self): + def __repr__(self) -> str: if isinstance(self._model, SensorModel): summary = [f""] summary.append(f"TYPE➀ {self.type.long_name}") @@ -459,14 +460,14 @@ def reference_model(self) -> pd.DataFrame: return self._r27 @property - def reference_model_name(self) -> List[str]: + def reference_model_name(self) -> list[str]: """Official list of Argo sensor models (R27) Return a sorted list of strings with altLabel from Argo Reference table R27 on 'SENSOR_MODEL'. Returns ------- - List[str] + list[str] Notes ----- @@ -497,14 +498,14 @@ def reference_sensor(self) -> pd.DataFrame: return self._r25 @property - def reference_sensor_type(self) -> List[str]: + def reference_sensor_type(self) -> list[str]: """Official list of Argo sensor types (R25) Return a sorted list of strings with altLabel from Argo Reference table R25 on 'SENSOR'. Returns ------- - List[str] + list[str] Notes ----- @@ -521,7 +522,7 @@ def search_model( model: str, strict: bool = False, output: Literal["df", "name"] = "df", - ) -> pd.DataFrame: + ) -> pd.DataFrame | list[str]: """Return references of Argo sensor models matching a string Look for occurrences in Argo Reference table R27 `altLabel` and return a :class:`pandas.DataFrame` with matching row(s). @@ -537,7 +538,7 @@ def search_model( Returns ------- - :class:`pandas.DataFrame`, sorted(List[str]) + :class:`pandas.DataFrame`, list[str] See Also -------- @@ -566,7 +567,7 @@ def search_model( else: return data.reset_index(drop=True) - def _search_wmo_with(self, model: str, errors="raise"): + def _search_wmo_with(self, model: str, errors="raise") -> list[int]: """Return the list of WMOs with a given sensor model Notes @@ -591,7 +592,9 @@ def _search_wmo_with(self, model: str, errors="raise"): wmos = self.fs.post(api_point, json_data=payload) if wmos is None or len(wmos) == 0: try: - search_hint = self.search_model(model, output="name", strict=False) + search_hint: list[str] = self.search_model( + model, output="name", strict=False + ) msg = ( f"No floats matching this sensor model name '{model}'. Possible hint: %s" % ("; ".join(search_hint)) @@ -613,7 +616,7 @@ def _floats_api( postprocess_opts={}, progress=False, errors="raise", - ): + ) -> Any: """Search floats with a sensor model and then fetch and process JSON data returned from the fleet-monitoring API for each floats Notes @@ -638,7 +641,7 @@ def _floats_api( return postprocess(sns, **postprocess_opts) - def _search_sn_with(self, model: str, progress=False, errors="raise"): + def _search_sn_with(self, model: str, progress=False, errors="raise") -> list[str]: """Return serial number of sensor models with a given string in name""" def preprocess(jsdata, model_name: str = ""): @@ -664,7 +667,9 @@ def postprocess(data, **kwargs): errors=errors, ) - def _search_wmo_sn_with(self, model: str, progress=False, errors="raise"): + def _search_wmo_sn_with( + self, model: str, progress=False, errors="raise" + ) -> dict[int, str]: """Return a dictionary of float WMOs with their sensor serial numbers""" def preprocess(jsdata, model_name: str = ""): @@ -706,7 +711,7 @@ def preprocess(jsdata, model_name: str = ""): if model_name in s["model"]: this = [jsdata["wmo"]] [ - this.append(s[key]) + this.append(s[key]) # type: ignore for key in [ "id", "maker", @@ -749,11 +754,11 @@ def postprocess(data, **kwargs): def search( self, - model: str = None, - output: Literal["wmo", "sn", "wmo_sn", "df"] = "wmo", + model: str | None = None, + output: SearchOutputOptions = "wmo", progress=False, errors="raise", - ): + ) -> list[int] | list[str] | dict[int, str] | pd.DataFrame: """Search for Argo floats equipped with a sensor model name All information are retrieved using the `Euro-Argo fleet-monitoring API `_. @@ -778,7 +783,7 @@ def search( Returns ------- - :class:`pandas.DataFrame`, List[int], List[str], Dict + list[int], list[str], dict[int, str], :class:`pandas.DataFrame` Notes ----- @@ -808,7 +813,9 @@ def search( "'output' option value must be in: 'wmo', 'sn', 'wmo_sn' or 'df'" ) - def iterfloats_with(self, model: str = None, chunksize: int = None): + def iterfloats_with( + self, model: str | None = None, chunksize: int | None = None + ) -> Iterator[ArgoFloat]: """Iterator over :class:`argopy.ArgoFloat` equipped with a given sensor model By default, iterate over a single float, otherwise use the `chunksize` argument to iterate over chunk of floats. @@ -866,7 +873,7 @@ def iterfloats_with(self, model: str = None, chunksize: int = None): for wmo in wmos: yield ArgoFloat(wmo, idx=idx) - def cli_search(self, model: str, output: str = "wmo"): + def cli_search(self, model: str, output: SearchOutputOptions = "wmo") -> str: # type: ignore """Quick sensor lookups from the terminal This function is a command-line-friendly output for float search (e.g., for piping to other tools). diff --git a/argopy/utils/accessories.py b/argopy/utils/accessories.py index ec5f8d767..273eb3430 100644 --- a/argopy/utils/accessories.py +++ b/argopy/utils/accessories.py @@ -312,7 +312,7 @@ def from_series(obj: pd.Series) -> "SensorType": reftable: ClassVar[str] """Reference table this row is based on""" - def __init__(self, row: pd.Series): + def __init__(self, row: pd.Series | pd.DataFrame): if not isinstance(row, pd.Series) and isinstance(row, pd.DataFrame): row = row.iloc[0] row = row.to_dict() From b5528e50c3417b4337694d5ad9abcce14ca2d39a Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Sun, 5 Oct 2025 01:36:46 +0200 Subject: [PATCH 20/71] more typing --- argopy/related/sensors.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/argopy/related/sensors.py b/argopy/related/sensors.py index 97594dea0..41bd1e131 100644 --- a/argopy/related/sensors.py +++ b/argopy/related/sensors.py @@ -2,11 +2,10 @@ import pandas as pd import numpy as np from pathlib import Path -from typing import TYPE_CHECKING, Optional, Literal, Union, Any, Iterator +from typing import Optional, Literal, Union, Any, Iterator import logging -if TYPE_CHECKING: - SearchOutputOptions = Literal["wmo", "sn", "wmo_sn", "df"] +SearchOutputOptions = Literal["wmo", "sn", "wmo_sn", "df"] # from ..errors import InvalidOption from ..stores import ArgoFloat, ArgoIndex, httpstore, filestore @@ -53,7 +52,7 @@ class SensorType(NVSrow): @staticmethod def from_series(obj: pd.Series) -> "SensorType": - """Create a :class:`SensorType` from a a R25-"Argo sensor models" row""" + """Create a :class:`SensorType` from a R25-"Argo sensor models" row""" return SensorType(obj) @@ -283,9 +282,9 @@ def r27_to_r25(self) -> dict[str, str]: def model_to_type( self, - model: Union[str, SensorModel, None] = None, + model: str | SensorModel | None = None, errors: Literal["raise", "ignore"] = "raise", - ) -> Optional[SensorType]: + ) -> SensorType | None: """Get a sensor type for a given sensor model All valid sensor model name can be obtained with :attr:`ArgoSensor.reference_model_name`. @@ -294,14 +293,14 @@ def model_to_type( Parameters ---------- - model : Union[str, SensorModel] + model : str | :class:`SensorModel` The model to read the sensor type for. errors : Literal["raise", "ignore"] = "raise" How to handle possible errors. If set to "ignore", the method will return None. Returns ------- - :class:`SensorType` or None + :class:`SensorType` | None See Also -------- @@ -322,7 +321,7 @@ def model_to_type( def type_to_model( self, - type: Union[str, SensorType], + type: str | SensorType, errors: Literal["raise", "ignore"] = "raise", ) -> list[str] | None: """Get all sensor model names of a given sensor type @@ -333,14 +332,14 @@ def type_to_model( Parameters ---------- - type : Union[str, SensorType] + type : str, :class:`SensorType` The sensor type to read the sensor model name for. errors : Literal["raise", "ignore"] = "raise" How to handle possible errors. If set to "ignore", the method will return None. Returns ------- - List[str] + list[str] See Also -------- @@ -533,7 +532,7 @@ def search_model( The model to search for. strict : bool, optional, default: False Is the model string a strict match or an occurrence in table look up. - output : str, Literal["table", "name"], default "table" + output : str, Literal["df", "name"], default "df" Is the output a :class:`pandas.DataFrame` with matching rows from :attr:`ArgoSensor.reference_model`, or a list of string. Returns From c04ffe6b5988f3a11822acd2e52d263fc1eadc21 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Sun, 5 Oct 2025 11:44:32 +0200 Subject: [PATCH 21/71] Update whats-new.rst --- docs/whats-new.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/whats-new.rst b/docs/whats-new.rst index 3a8adc507..4a855236f 100644 --- a/docs/whats-new.rst +++ b/docs/whats-new.rst @@ -13,7 +13,7 @@ Coming up next (unreleased) Features and front-end API ^^^^^^^^^^^^^^^^^^^^^^^^^^ -- **New** :class:`ArgoSensor` class to work with Argo sensor models, types and floats equipped with. (:pr:`532`) by |gmaze|. +- **New** :class:`ArgoSensor` class to work with Argo sensor models, types and floats equipped with. Check the new :ref:`Argo sensor: models and types` documentation section. (:pr:`532`) by |gmaze|. Internals ^^^^^^^^^ From f61821826b860455eed352e38fd01baa7e2a274e Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Mon, 6 Oct 2025 10:43:37 +0200 Subject: [PATCH 22/71] Update reference_tables.py - add possibly to pass fs instance to NVS --- argopy/related/reference_tables.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/argopy/related/reference_tables.py b/argopy/related/reference_tables.py index 28854f33f..3f7137187 100644 --- a/argopy/related/reference_tables.py +++ b/argopy/related/reference_tables.py @@ -11,12 +11,15 @@ class NVScollection: def __init__( self, nvs="https://vocab.nerc.ac.uk/collection", - cache: bool = True, - cachedir: str = "", + **kwargs, ): """Reference Tables from NVS collection""" - cachedir = OPTIONS["cachedir"] if cachedir == "" else cachedir - self.fs = httpstore(cache=cache, cachedir=cachedir) + self.fs = kwargs.get("fs", None) + if self.fs is None: + self._cache = kwargs.get("cache", True) + self._cachedir = kwargs.get("cachedir", OPTIONS["cachedir"]) + self._timeout = kwargs.get("timeout", OPTIONS["api_timeout"]) + self.fs = httpstore(cache=self._cache, cachedir=self._cachedir, timeout=self._timeout) self.nvs = nvs @property From fc98ed0533a13f41c0bb6a6b615f1a8fa4125df3 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Mon, 6 Oct 2025 10:43:55 +0200 Subject: [PATCH 23/71] improved arguments def --- argopy/related/sensors.py | 29 +++++++++++++---------------- argopy/utils/accessories.py | 1 - 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/argopy/related/sensors.py b/argopy/related/sensors.py index 41bd1e131..529b670d3 100644 --- a/argopy/related/sensors.py +++ b/argopy/related/sensors.py @@ -1,13 +1,12 @@ -from typing import List import pandas as pd import numpy as np from pathlib import Path -from typing import Optional, Literal, Union, Any, Iterator +from typing import Literal, Any, Iterator import logging SearchOutputOptions = Literal["wmo", "sn", "wmo_sn", "df"] +ErrorOptions = Literal["raise", "ignore"] -# from ..errors import InvalidOption from ..stores import ArgoFloat, ArgoIndex, httpstore, filestore from ..utils import check_wmo, Chunker, to_list, NVSrow from ..errors import ( @@ -99,6 +98,8 @@ class ArgoSensor: """ + __slots__ = ["_cache", "_cachedir", "_timeout", "fs", "_r25", "_r27", "_r27_to_r25", "_model", "_type"] + def __init__( self, model: str | None = None, @@ -211,8 +212,8 @@ def __init__( """ self._cache = kwargs.get("cache", True) self._cachedir = kwargs.get("cachedir", OPTIONS["cachedir"]) - self.timeout = kwargs.get("timeout", OPTIONS["api_timeout"]) - self.fs = httpstore(cache=self._cache, cachedir=self._cachedir) + self._timeout = kwargs.get("timeout", OPTIONS["api_timeout"]) + self.fs = httpstore(cache=self._cache, cachedir=self._cachedir, timeout=self._timeout) self._r25: pd.DataFrame | None = None # will be loaded when necessary self._r27: pd.DataFrame | None = None # will be loaded when necessary @@ -453,9 +454,7 @@ def reference_model(self) -> pd.DataFrame: :class:`ArgoNVSReferenceTables` """ if self._r27 is None: - self._r27 = ArgoNVSReferenceTables( - cache=self._cache, cachedir=self._cachedir - ).tbl("R27") + self._r27 = ArgoNVSReferenceTables(fs=self.fs).tbl("R27") return self._r27 @property @@ -491,9 +490,7 @@ def reference_sensor(self) -> pd.DataFrame: :class:`ArgoNVSReferenceTables` """ if self._r25 is None: - self._r25 = ArgoNVSReferenceTables( - cache=self._cache, cachedir=self._cachedir - ).tbl("R25") + self._r25 = ArgoNVSReferenceTables(fs=self.fs).tbl("R25") return self._r25 @property @@ -566,7 +563,7 @@ def search_model( else: return data.reset_index(drop=True) - def _search_wmo_with(self, model: str, errors="raise") -> list[int]: + def _search_wmo_with(self, model: str, errors : ErrorOptions = "raise") -> list[int]: """Return the list of WMOs with a given sensor model Notes @@ -640,7 +637,7 @@ def _floats_api( return postprocess(sns, **postprocess_opts) - def _search_sn_with(self, model: str, progress=False, errors="raise") -> list[str]: + def _search_sn_with(self, model: str, progress=False, errors : ErrorOptions = "raise") -> list[str]: """Return serial number of sensor models with a given string in name""" def preprocess(jsdata, model_name: str = ""): @@ -692,7 +689,7 @@ def postprocess(data, **kwargs): errors=errors, ) - def _to_dataframe(self, model: str, progress=False, errors="raise") -> pd.DataFrame: + def _to_dataframe(self, model: str, progress=False, errors : ErrorOptions = "raise") -> pd.DataFrame: """Return a DataFrame with WMO, sensor type, model, maker, sn, units, accuracy and resolution Parameters @@ -755,8 +752,8 @@ def search( self, model: str | None = None, output: SearchOutputOptions = "wmo", - progress=False, - errors="raise", + progress : bool = False, + errors : ErrorOptions = "raise", ) -> list[int] | list[str] | dict[int, str] | pd.DataFrame: """Search for Argo floats equipped with a sensor model name diff --git a/argopy/utils/accessories.py b/argopy/utils/accessories.py index 273eb3430..511c0df8f 100644 --- a/argopy/utils/accessories.py +++ b/argopy/utils/accessories.py @@ -293,7 +293,6 @@ def from_series(obj: pd.Series) -> "SensorType": st.uri """ - name: str = "" """From 'altLabel' column""" From 1fd79eeba8b06255d7c1da7b2d42936e6518bb15 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Fri, 10 Oct 2025 17:29:32 +0200 Subject: [PATCH 24/71] Update options.py add new options for NVS and RBR servers --- argopy/options.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/argopy/options.py b/argopy/options.py index e45b58940..2881ae53a 100644 --- a/argopy/options.py +++ b/argopy/options.py @@ -51,6 +51,9 @@ PARALLEL_DEFAULT_METHOD = "parallel_default_method" LON = "longitude_convention" API_FLEETMONITORING = "fleetmonitoring" +RBR_API_KEY = "rbr_api_key" +API_RBR = "rbr_api" +NVS = "nvs" # Define the list of available options and default values: OPTIONS = { @@ -71,6 +74,9 @@ PARALLEL_DEFAULT_METHOD: "thread", LON: "180", API_FLEETMONITORING: "https://fleetmonitoring.euro-argo.eu", + RBR_API_KEY: os.environ.get("RBR_API_KEY"), # Contact RBR at argo@rbr-global.com if you do not have an authorization key. + API_RBR: "https://oem-lookup.rbr-global.com/api/v1", + NVS: "https://vocab.nerc.ac.uk/collection", } DEFAULT = OPTIONS.copy() @@ -127,6 +133,14 @@ def validate_fleetmonitoring(this_path): return False +def validate_rbr(this_path): + if this_path != "-": + return True # todo: check with RBR how to get the API endpoint for its status + else: + log.debug("OPTIONS['%s'] is not defined" % API_RBR) + return False + + def PARALLEL_SETUP(parallel): parallel = VALIDATE("parallel", parallel) if isinstance(parallel, bool): @@ -271,10 +285,14 @@ def check_gdac_option( USER: lambda x: isinstance(x, str) or x is None, PASSWORD: lambda x: isinstance(x, str) or x is None, ARGOVIS_API_KEY: lambda x: isinstance(x, str) or x is None, + RBR_API_KEY: lambda x: isinstance(x, str) or x is None, PARALLEL: validate_parallel, PARALLEL_DEFAULT_METHOD: validate_parallel_method, LON: lambda x: x in ['180', '360'], API_FLEETMONITORING: validate_fleetmonitoring, + API_RBR: validate_rbr, + NVS: lambda x: isinstance(x, str) or x is None, + } @@ -336,6 +354,13 @@ class set_options: You can get a free key at https://argovis-keygen.colorado.edu + rbr_api_key: str, default: None + The API key to use when fetching data from the RBR OEM lookup API. + + RBR sensor owners can get a key by contacting RBR at argo@rbr-global.com. + + This key can also be used from: https://oem-lookup.rbr-global.com + parallel: bool, str, :class:`distributed.Client`, default: False Set whether to use parallelisation or not, and possibly which method to use. From 7c471d164c416be02606cf2e2a6e9349057293e2 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Fri, 10 Oct 2025 17:30:08 +0200 Subject: [PATCH 25/71] refactor ArgoSensor to related/sensors --- argopy/related/__init__.py | 2 +- argopy/related/sensors/__init__.py | 0 argopy/related/{ => sensors}/sensors.py | 77 +++++++++++++++++++++---- 3 files changed, 66 insertions(+), 13 deletions(-) create mode 100644 argopy/related/sensors/__init__.py rename argopy/related/{ => sensors}/sensors.py (92%) diff --git a/argopy/related/__init__.py b/argopy/related/__init__.py index 9ad2e95bc..3bb5042ad 100644 --- a/argopy/related/__init__.py +++ b/argopy/related/__init__.py @@ -4,7 +4,7 @@ from .argo_documentation import ArgoDocs from .doi_snapshot import ArgoDOI from .euroargo_api import get_coriolis_profile_id, get_ea_profile_page -from .sensors import ArgoSensor, SensorType, SensorModel +from .sensors.sensors import ArgoSensor, SensorType, SensorModel from .utils import load_dict, mapp_dict # Should come last # diff --git a/argopy/related/sensors/__init__.py b/argopy/related/sensors/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/argopy/related/sensors.py b/argopy/related/sensors/sensors.py similarity index 92% rename from argopy/related/sensors.py rename to argopy/related/sensors/sensors.py index 529b670d3..9df48ed91 100644 --- a/argopy/related/sensors.py +++ b/argopy/related/sensors/sensors.py @@ -4,22 +4,22 @@ from typing import Literal, Any, Iterator import logging -SearchOutputOptions = Literal["wmo", "sn", "wmo_sn", "df"] -ErrorOptions = Literal["raise", "ignore"] - -from ..stores import ArgoFloat, ArgoIndex, httpstore, filestore -from ..utils import check_wmo, Chunker, to_list, NVSrow -from ..errors import ( +from ...stores import ArgoFloat, ArgoIndex, httpstore, filestore +from ...utils import check_wmo, Chunker, to_list, NVSrow +from ...errors import ( DataNotFound, InvalidDataset, InvalidDatasetStructure, OptionValueError, ) -from ..options import OPTIONS -from ..utils import path2assets -from . import ArgoNVSReferenceTables +from ...options import OPTIONS +from ...utils import path2assets +from .. import ArgoNVSReferenceTables +SearchOutputOptions = Literal["wmo", "sn", "wmo_sn", "df"] +ErrorOptions = Literal["raise", "ignore"] + log = logging.getLogger("argopy.related.sensors") @@ -85,16 +85,23 @@ def from_series(obj: pd.Series) -> "SensorModel": """Create a :class:`SensorModel` from a R27-"Argo sensor models" row""" return SensorModel(obj) + def __contains__(self, string) -> bool: + return string.lower() in self.name.lower() or string.lower() in self.long_name.lower() class ArgoSensor: """ Argo sensors helper class - The :class:`ArgoSensor` class provides direct access to Argo's sensor metadata through Reference Tables `Sensor Types (R25) `_ and `Sensor Models (R27) `_, combined with the `Euro-Argo fleet-monitoring API `_. This enables users to: + The :class:`ArgoSensor` class aims to provide direct access to Argo's sensor metadata from: + + - NVS Reference Tables `Sensor Types (R25) `_ and `Sensor Models (R27) `_ + - `Euro-Argo fleet-monitoring API `_ + + This enables users to: - navigate reference tables 25 and 27, - search for/iterate over floats equipped with specific sensor models, - - retrieve sensor serial numbers across the global array. + - retrieve sensor serial numbers across the global array, """ @@ -139,6 +146,10 @@ def __init__( ArgoSensor().reference_sensor ArgoSensor().reference_sensor_type # Only the list of types (used to fill 'SENSOR' parameter) + # Reference table R26-"Argo sensor manufacturers" with the list of sensor maker + ArgoSensor().reference_manufaturer + ArgoSensor().reference_manufaturer_name # Only the list of makers (used to fill 'SENSOR_MAKER' parameter) + # Search for all referenced sensor models with some string in their name ArgoSensor().search_model('RBR') ArgoSensor().search_model('RBR', output='name') # Return a list of names instead of a DataFrame @@ -213,9 +224,11 @@ def __init__( self._cache = kwargs.get("cache", True) self._cachedir = kwargs.get("cachedir", OPTIONS["cachedir"]) self._timeout = kwargs.get("timeout", OPTIONS["api_timeout"]) - self.fs = httpstore(cache=self._cache, cachedir=self._cachedir, timeout=self._timeout) + fs_kargs = {"cache": self._cache, "cachedir": self._cachedir, "timeout": self._timeout} + self.fs = httpstore(**fs_kargs) self._r25: pd.DataFrame | None = None # will be loaded when necessary + self._r26: pd.DataFrame | None = None # will be loaded when necessary self._r27: pd.DataFrame | None = None # will be loaded when necessary self._load_mappers() # Load r25 model to r27 type mapping dictionary @@ -232,11 +245,15 @@ def __init__( if df.shape[0] == 1: self._model = SensorModel.from_series(df.iloc[0]) self._type = self.model_to_type(self._model, errors="ignore") + # if "RBR" in self._model: + # Add the RBR OEM API Authorization key for this sensor: + # fs_kargs.update(client_kwargs={'headers': {'Authorization': OPTIONS.get('rbr_api_key') }}) else: raise InvalidDatasetStructure( f"Found multiple sensor models with '{model}'. Restrict your sensor model name to only one value in: {to_list(df['altLabel'].values)}" ) + def _load_mappers(self): """Load from static assets file the NVS R25 to R27 key mappings @@ -513,6 +530,42 @@ def reference_sensor_type(self) -> list[str]: """ return sorted(to_list(self.reference_sensor["altLabel"].values)) + @property + def reference_manufacturer(self) -> pd.DataFrame: + """Official reference table for Argo sensor manufacturers (R26) + + Returns + ------- + :class:`pandas.DataFrame` + + See Also + -------- + :class:`ArgoNVSReferenceTables` + """ + if self._r26 is None: + self._r26 = ArgoNVSReferenceTables(fs=self.fs).tbl("R26") + return self._r26 + + @property + def reference_manufacturer_name(self) -> list[str]: + """Official list of Argo sensor maker (R26) + + Return a sorted list of strings with altLabel from Argo Reference table R26 on 'SENSOR_MAKER'. + + Returns + ------- + list[str] + + Notes + ----- + Argo netCDF variable ``SENSOR_MAKER`` is populated with values from this list. + + See Also + -------- + :attr:`ArgoSensor.reference_manufacturer` + """ + return sorted(to_list(self.reference_manufacturer["altLabel"].values)) + def search_model( self, model: str, From 4c0f1c05f6852bdd1299952034c0752b5a59df6c Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Fri, 10 Oct 2025 17:30:30 +0200 Subject: [PATCH 26/71] Update reference_tables.py use the OPTION by default --- argopy/related/reference_tables.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/argopy/related/reference_tables.py b/argopy/related/reference_tables.py index 3f7137187..d447d5ccb 100644 --- a/argopy/related/reference_tables.py +++ b/argopy/related/reference_tables.py @@ -10,17 +10,17 @@ class NVScollection: def __init__( self, - nvs="https://vocab.nerc.ac.uk/collection", **kwargs, ): """Reference Tables from NVS collection""" + self.nvs = kwargs.get("nvs", OPTIONS["nvs"]) + self.fs = kwargs.get("fs", None) if self.fs is None: self._cache = kwargs.get("cache", True) self._cachedir = kwargs.get("cachedir", OPTIONS["cachedir"]) self._timeout = kwargs.get("timeout", OPTIONS["api_timeout"]) self.fs = httpstore(cache=self._cache, cachedir=self._cachedir, timeout=self._timeout) - self.nvs = nvs @property def valid_ref(self): From 25ee4cb229d4b456ec15f8a57e8599fbaed32b32 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Fri, 10 Oct 2025 17:30:56 +0200 Subject: [PATCH 27/71] Update filesystems.py fix httpstore custom header setting --- argopy/stores/filesystems.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/argopy/stores/filesystems.py b/argopy/stores/filesystems.py index e6487e66d..e3e65ed1c 100644 --- a/argopy/stores/filesystems.py +++ b/argopy/stores/filesystems.py @@ -71,6 +71,10 @@ def new_fs( "headers": {"Argopy-Version": __version__}, } # Passed to aiohttp.ClientSession if "client_kwargs" in kwargs: + if "headers" in kwargs['client_kwargs']: + for header_key, header_val in kwargs['client_kwargs']['headers'].items(): + client_kwargs['headers'].update({header_key: header_val}) + kwargs['client_kwargs'].pop("headers") client_kwargs = {**client_kwargs, **kwargs["client_kwargs"]} kwargs.pop("client_kwargs") default_fsspec_kwargs = { From 67590d7975bf5225325c695c51e4e2d7d8e56081 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Fri, 10 Oct 2025 17:31:14 +0200 Subject: [PATCH 28/71] Update http.py docstring with example of custom header --- argopy/stores/implementations/http.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/argopy/stores/implementations/http.py b/argopy/stores/implementations/http.py index 18222498c..15f9a2cbd 100644 --- a/argopy/stores/implementations/http.py +++ b/argopy/stores/implementations/http.py @@ -42,12 +42,23 @@ class httpstore(ArgoStoreProto): This store intends to make argopy safer to failures from http requests and to provide higher levels methods to work with our datasets. Key methods are: + - :class:`httpstore.open` + - :class:`httpstore.exists` - :class:`httpstore.download_url` - :class:`httpstore.open_dataset` - - :class:`httpstore.open_json` - :class:`httpstore.open_mfdataset` + - :class:`httpstore.open_json` - :class:`httpstore.open_mfjson` - :class:`httpstore.read_csv` + - :class:`httpstore.post` + + Examples + -------- + .. code-block:: python + :caption: How to add a specific header key + + from argopy.stores import httpstore + fs = httpstore(client_kwargs={'headers': {'Authorization': 'Token'}}) """ From 4a9c8c61cd80ed5390e2b21eb8f267a19d0ed3f3 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Fri, 10 Oct 2025 17:31:39 +0200 Subject: [PATCH 29/71] Add urnparser for NVS and schema --- argopy/utils/__init__.py | 2 +- argopy/utils/format.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/argopy/utils/__init__.py b/argopy/utils/__init__.py index 954844e65..dc791bf9a 100644 --- a/argopy/utils/__init__.py +++ b/argopy/utils/__init__.py @@ -60,7 +60,7 @@ filter_param_by_data_mode, split_data_mode, ) -from .format import argo_split_path, format_oneline, UriCName, redact, dirfs_relpath +from .format import argo_split_path, format_oneline, UriCName, redact, dirfs_relpath, urnparser from .loggers import warnUnless, log_argopy_callerstack from .carbon import GreenCoding, Github from . import optical_modeling diff --git a/argopy/utils/format.py b/argopy/utils/format.py index 46ea02e99..88ba1a562 100644 --- a/argopy/utils/format.py +++ b/argopy/utils/format.py @@ -398,3 +398,12 @@ def cname(self) -> str: cname = self.dataset_id + ";" + cname return cname + + +def urnparser(urn): + """Parsing RFC 8141 compliant uniform resource names (URN) from NVS""" + pp = urn.split(":") + if len(pp) == 4 and pp[0] == 'SDN': + return {'listid': pp[1], 'version': pp[2], 'termid': pp[3]} + else: + raise ValueError("NVS URNs must follow the pattern: 'SDN:{listid}:{version}:{termid}' or 'SDN:{listid}::{termid}' for NVS2.0") From 8384bc6962381ee09a8dbd649b90f8b8c398241f Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Fri, 10 Oct 2025 17:32:26 +0200 Subject: [PATCH 30/71] new class OemArgoSensorMetaData 1st attempt for OEM-based sensor meta data --- argopy/related/sensors/oem_metadata.py | 265 ++++++++++++++++++++ argopy/related/sensors/oem_metadata_repr.py | 157 ++++++++++++ argopy/static/css/argopy.css | 11 + argopy/static/css/oemsensor.css | 79 ++++++ 4 files changed, 512 insertions(+) create mode 100644 argopy/related/sensors/oem_metadata.py create mode 100644 argopy/related/sensors/oem_metadata_repr.py create mode 100644 argopy/static/css/argopy.css create mode 100644 argopy/static/css/oemsensor.css diff --git a/argopy/related/sensors/oem_metadata.py b/argopy/related/sensors/oem_metadata.py new file mode 100644 index 000000000..b6e3f503c --- /dev/null +++ b/argopy/related/sensors/oem_metadata.py @@ -0,0 +1,265 @@ +import json +from dataclasses import dataclass, field +from typing import List, Dict, Optional, Any +from jsonschema import validate, ValidationError + +from ...stores import httpstore +from ...options import OPTIONS +from ...utils import urnparser +from .oem_metadata_repr import NotebookCellDisplay + + +@dataclass +class SensorInfo: + created_by: str + date_creation: str # ISO 8601 datetime string + link: str + format_version: str + contents: str + sensor_described: str + + +@dataclass +class Context: + SDN_R03: str = "http://vocab.nerc.ac.uk/collection/R03/current/" + SDN_R25: str = "http://vocab.nerc.ac.uk/collection/R25/current/" + SDN_R26: str = "http://vocab.nerc.ac.uk/collection/R26/current/" + SDN_R27: str = "http://vocab.nerc.ac.uk/collection/R27/current/" + SDN_L22: str = "http://vocab.nerc.ac.uk/collection/L22/current/" + + +@dataclass +class Sensor: + SENSOR: str # SDN:R25::CTD_PRES + SENSOR_MAKER: str # SDN:R26::RBR + SENSOR_MODEL: str # SDN:R27::RBR_PRES_A + SENSOR_FIRMWARE_VERSION: str # wrong key used by RBR, https://github.com/euroargodev/sensor_metadata_json/issues/20 + # SENSOR_MODEL_FIRMWARE: str # Correct schema key + SENSOR_SERIAL_NO: str + sensor_vendorinfo: Optional[Dict[str, Any]] = None + + @property + def SENSOR_uri(self): + urnparts = urnparser(self.SENSOR) + return f"{OPTIONS['nvs']}/{urnparts['listid']}/current/{urnparts['termid']}" + + @property + def SENSOR_MAKER_uri(self): + urnparts = urnparser(self.SENSOR_MAKER) + return f"{OPTIONS['nvs']}/{urnparts['listid']}/current/{urnparts['termid']}" + + @property + def SENSOR_MODEL_uri(self): + urnparts = urnparser(self.SENSOR_MODEL) + return f"{OPTIONS['nvs']}/{urnparts['listid']}/current/{urnparts['termid']}" + + +@dataclass +class Parameter: + PARAMETER: str # SDN:R03::PRES + PARAMETER_SENSOR: str # SDN:R25::CTD_PRES + PARAMETER_UNITS: str + PARAMETER_ACCURACY: str + PARAMETER_RESOLUTION: str + PREDEPLOYMENT_CALIB_EQUATION: str + PREDEPLOYMENT_CALIB_COEFFICIENT_LIST: Dict[str, str] + PREDEPLOYMENT_CALIB_COMMENT: str + PREDEPLOYMENT_CALIB_DATE: str + parameter_vendorinfo: Optional[Dict[str, Any]] = None + predeployment_vendorinfo: Optional[Dict[str, Any]] = None + + @property + def PARAMETER_uri(self): + urnparts = urnparser(self.PARAMETER) + return f"{OPTIONS['nvs']}/{urnparts['listid']}/current/{urnparts['termid']}" + + @property + def PARAMETER_SENSOR_uri(self): + urnparts = urnparser(self.PARAMETER_SENSOR) + return f"{OPTIONS['nvs']}/{urnparts['listid']}/current/{urnparts['termid']}" + + +def fetch_rbr_data(sn: str, **kwargs): + if kwargs.get("fs", None) is None: + _cache = kwargs.get("cache", True) + _cachedir = kwargs.get("cachedir", OPTIONS["cachedir"]) + _timeout = kwargs.get("timeout", OPTIONS["api_timeout"]) + fs_kargs = {"cache": _cache, "cachedir": _cachedir, "timeout": _timeout} + _api_key = kwargs.get("rbr_api_key", OPTIONS["rbr_api_key"]) + fs_kargs.update(client_kwargs={"headers": {"Authorization": _api_key}}) + fs = httpstore(**fs_kargs) + uri = f"{OPTIONS['rbr_api']}/instruments/{sn}/argometadatajson" + return fs.open_json(uri) + + +class OemArgoSensorMetaData: + """Argo sensor meta-data from OEM + + OEM : Original Equipment Manufacturer + + Comply to schema from https://github.com/euroargodev/sensor_metadata_json + + Examples + -------- + .. code-block:: python + + OemArgoSensorMetaData() + + OemArgoSensorMetaData(validate=True) # Run json schema validation compliance automatically + + OemArgoSensorMetaData().from_dict(jsdata) # Use any compliant json data from OEM + + OemArgoSensorMetaData.from_rbr(208380) # Direct call to the RBR api + + + """ + + _schema_src = "https://raw.githubusercontent.com/euroargodev/sensor_metadata_json/refs/heads/main/schemas/argo.sensor.schema.json" + """URI of the argo sensor JSON schema""" + + def __init__( + self, + json_data: Optional[Dict[str, Any]] = None, + validate: bool = False, + **kwargs, + ): + self._run_validation = validate + if self._run_validation: + self.schema = self._read_schema() + + self.sensor_info: Optional[SensorInfo] = None + self.context: Optional[Context] = None + self.sensors: List[Sensor] = field(default_factory=list) + self.parameters: List[Parameter] = field(default_factory=list) + self.instrument_vendorinfo: Optional[Dict[str, Any]] = None + + if kwargs.get("fs", None) is not None: + self._fs = kwargs.get("fs") + else: + self._cache = kwargs.get("cache", True) + self._cachedir = kwargs.get("cachedir", OPTIONS["cachedir"]) + self._timeout = kwargs.get("timeout", OPTIONS["api_timeout"]) + fs_kargs = { + "cache": self._cache, + "cachedir": self._cachedir, + "timeout": self._timeout, + } + self._fs = httpstore(**fs_kargs) + + if json_data: + self.from_dict(json_data) + + def __repr__(self): + sensor_count = len(self.sensors) + parameter_count = len(self.parameters) + sensor_described = ( + self.sensor_info.sensor_described if self.sensor_info else "N/A" + ) + created_by = self.sensor_info.created_by if self.sensor_info else "N/A" + date_creation = self.sensor_info.date_creation if self.sensor_info else "N/A" + + summary = [f""] + summary.append(f"created_by: '{created_by}'") + summary.append(f"date_creation: '{date_creation}'") + summary.append(f"sensors: {sensor_count} {[urnparser(s.SENSOR)['termid'] for s in self.sensors]}") + summary.append(f"parameters: {parameter_count} {[urnparser(s.PARAMETER)['termid'] for s in self.parameters]}") + summary.append( + f"instrument_vendorinfo: {'Present' if self.instrument_vendorinfo else 'None'}" + ) + return "\n".join(summary) + + def _repr_html_(self): + return NotebookCellDisplay(self).html + + def _ipython_display_(self): + from IPython.display import display, HTML + + display(HTML(NotebookCellDisplay(self).html)) + + def _read_schema(self) -> Dict[str, Any]: + """Load the JSON schema for validation.""" + schema = httpstore().open_json(self.schema_src) + return schema + + def from_dict(self, data: Dict[str, Any]): + """Load data from a dictionary and validate it.""" + if self._run_validation: + try: + validate(instance=data, schema=self.schema) + except ValidationError as e: + raise ValueError(f"Json schema Validation error: {e.message}") + + self.sensor_info = SensorInfo(**data["sensor_info"]) + self.context = Context( + **{ + k.replace("::", "").replace(":", "_"): v + for k, v in data["@context"].items() + } + ) + self.sensors = [Sensor(**sensor) for sensor in data["SENSORS"]] + self.parameters = [Parameter(**param) for param in data["PARAMETERS"]] + self.instrument_vendorinfo = data.get("instrument_vendorinfo") + + return self + + def to_dict(self) -> Dict[str, Any]: + """Convert the object back to a dictionary, following schema""" + return { + "sensor_info": { + "created_by": self.sensor_info.created_by, + "date_creation": self.sensor_info.date_creation, + "link": self.sensor_info.link, + "format_version": self.sensor_info.format_version, + "contents": self.sensor_info.contents, + "sensor_described": self.sensor_info.sensor_described, + }, + "@context": { + "SDN:R03::": self.context.SDN_R03, + "SDN:R25::": self.context.SDN_R25, + "SDN:R26::": self.context.SDN_R26, + "SDN:R27::": self.context.SDN_R27, + "SDN:L22::": self.context.SDN_L22, + }, + "SENSORS": [ + { + "SENSOR": sensor.SENSOR, + "SENSOR_MAKER": sensor.SENSOR_MAKER, + "SENSOR_MODEL": sensor.SENSOR_MODEL, + "SENSOR_MODEL_FIRMWARE": getattr( + sensor, "SENSOR_MODEL_FIRMWARE", sensor.SENSOR_FIRMWARE_VERSION + ), + "SENSOR_SERIAL_NO": sensor.SENSOR_SERIAL_NO, + "sensor_vendorinfo": sensor.sensor_vendorinfo, + } + for sensor in self.sensors + ], + "PARAMETERS": [ + { + "PARAMETER": param.PARAMETER, + "PARAMETER_SENSOR": param.PARAMETER_SENSOR, + "PARAMETER_UNITS": param.PARAMETER_UNITS, + "PARAMETER_ACCURACY": param.PARAMETER_ACCURACY, + "PARAMETER_RESOLUTION": param.PARAMETER_RESOLUTION, + "PREDEPLOYMENT_CALIB_EQUATION": param.PREDEPLOYMENT_CALIB_EQUATION, + "PREDEPLOYMENT_CALIB_COEFFICIENT_LIST": param.PREDEPLOYMENT_CALIB_COEFFICIENT_LIST, + "PREDEPLOYMENT_CALIB_COMMENT": param.PREDEPLOYMENT_CALIB_COMMENT, + "PREDEPLOYMENT_CALIB_DATE": param.PREDEPLOYMENT_CALIB_DATE, + "parameter_vendorinfo": param.parameter_vendorinfo, + "predeployment_vendorinfo": param.predeployment_vendorinfo, + } + for param in self.parameters + ], + "instrument_vendorinfo": self.instrument_vendorinfo, + } + + def save_to_json_file(self, file_path: str) -> None: + """Save the object to a JSON file.""" + with open(file_path, "w") as f: + json.dump(self.to_dict(), f, indent=2) + + @classmethod + def from_rbr(cls, serial_number: str, **kwargs): + """Fetch sensor metadata from RBR API and return an OemArgoSensorMetaData instance""" + # Use your HTTP store or API client to fetch data + data = fetch_rbr_data(serial_number, **kwargs) + return cls(json_data=data, **kwargs) diff --git a/argopy/related/sensors/oem_metadata_repr.py b/argopy/related/sensors/oem_metadata_repr.py new file mode 100644 index 000000000..ddee1e2fd --- /dev/null +++ b/argopy/related/sensors/oem_metadata_repr.py @@ -0,0 +1,157 @@ +from IPython.display import display, HTML +from functools import lru_cache +import importlib + +try: + from importlib.resources import files # New in version 3.9 +except ImportError: + from pathlib import Path + + files = lambda x: Path( # noqa: E731 + importlib.util.find_spec(x).submodule_search_locations[0] + ) + +from ...utils import urnparser + + +STATIC_FILES = ( + ("argopy.static.css", "argopy.css"), + ("argopy.static.css", "oemsensor.css"), +) + +@lru_cache(None) +def _load_static_files(): + """Lazily load the resource files into memory the first time they are needed""" + return [ + files(package).joinpath(resource).read_text(encoding="utf-8") + for package, resource in STATIC_FILES + ] + +class NotebookCellDisplay: + + def __init__(self, obj): + self.OEMsensor = obj + + @property + def css_style(self): + return "\n".join(_load_static_files()) + + @property + def html(self): + + # --- Header --- + header_html = f""" +

Argo Sensor Metadata: {getattr(self.OEMsensor.sensor_info, 'sensor_described', 'N/A')}

+

Created by: {getattr(self.OEMsensor.sensor_info, 'created_by', 'N/A')} | + Date: {getattr(self.OEMsensor.sensor_info, 'date_creation', 'N/A')}

+ """ + + # --- Sensors --- + sensors_html = """ +

List of sensors:

+ + + + + + + + + + + + """ + + for sensor in self.OEMsensor.sensors: + sensors_html += f""" + + + + + + + + """ + + sensors_html += "
SensorMakerModelFirmwareSerial No
{sensor.SENSOR}{sensor.SENSOR_MAKER}{sensor.SENSOR_MODEL}{getattr(sensor, 'SENSOR_MODEL_FIRMWARE', getattr(sensor, 'SENSOR_FIRMWARE_VERSION', 'N/A'))}{sensor.SENSOR_SERIAL_NO}
" + + # --- Parameters --- + parameters_html = """ +

List of parameters:

+ + + + + + + + + + + + + """ + + for i, param in enumerate(self.OEMsensor.parameters): + details_html = f""" + + + + """ + + parameters_html += f""" + + + + + + + + + {details_html} + """ + + parameters_html += "
ParameterSensorUnitsAccuracyResolutionCalibration details
+
+

Calibration Equation: {param.PREDEPLOYMENT_CALIB_EQUATION}

+

Calibration Coefficients: {param.PREDEPLOYMENT_CALIB_COEFFICIENT_LIST}

+

Calibration Comment: {param.PREDEPLOYMENT_CALIB_COMMENT}

+

Calibration Date: {param.PREDEPLOYMENT_CALIB_DATE}

+
+
{param.PARAMETER}{param.PARAMETER_SENSOR}{param.PARAMETER_UNITS}{param.PARAMETER_ACCURACY}{param.PARAMETER_RESOLUTION}Click for more
" + + # --- JavaScript for Toggle --- + js = """ + + """ + + # --- Vendor Info --- + vendor_html = "" + if self.OEMsensor.instrument_vendorinfo: + vendor_html = f""" +

Instrument Vendor Info:

+ 
{self.OEMsensor.instrument_vendorinfo}
+ """ + + # --- Combine All HTML --- + full_html = f""" + \n + {header_html}\n + {sensors_html}\n + {parameters_html}\n + {js}\n + {vendor_html} + """ + return full_html + + def _repr_html_(self): + return self.html diff --git a/argopy/static/css/argopy.css b/argopy/static/css/argopy.css new file mode 100644 index 000000000..ae1d26703 --- /dev/null +++ b/argopy/static/css/argopy.css @@ -0,0 +1,11 @@ +/* CSS stylesheet for argopy style +* +*/ + +:root { + --argopy_cyan:rgb(18, 235, 229); + --argopy_blue:rgb(16, 137, 182); + --argopy_darkblue:rgb(10, 89, 162); + --argopy_yellow:rgb(229, 174, 41); + --argopy_darkyellow:rgb(224, 158, 37); +} diff --git a/argopy/static/css/oemsensor.css b/argopy/static/css/oemsensor.css new file mode 100644 index 000000000..29f483696 --- /dev/null +++ b/argopy/static/css/oemsensor.css @@ -0,0 +1,79 @@ +/* CSS stylesheet for displaying OemArgoSensorMetaData object in a Jupyter notebook cell +* +*/ +.oemsensor { + font-family: Arial, sans-serif; + line-height: 1em; + overflow: hidden; +} +h1.oemsensor { + font-size: 0.9em; + font-weight: bold; + margin-bottom: 0.4em; + border-bottom: 1px solid var(--argopy_darkyellow); + padding-bottom: 0.2em; + color: var(--argopy_darkblue); +} +h2.oemsensor { + font-size: 0.9em !important; + font-weight: bold; + margin-top: 0.2em; + margin-bottom: 0.2em; + color: var(--argopy_darkblue); +} +p.oemsensor { + font-size: small; + color: var(--argopy_darkblue); +} +table.oemsensor { + width: 100%; + border-collapse: collapse; + margin-bottom: 0.8em; + font-size: 0.9em; + color: var(--argopy_blue); +} +table.oemsensor thead{ + color: var(--argopy_darkblue); +} +th.oemsensor, td.oemsensor { + padding: 0.4em 0.6em; + text-align: left; + border: 1px solid #eee; +} +th.oemsensor { + background-color: #f8f8f8; + font-weight: bold; +} +tr.parameter-row.oemsensor td.calibration { + cursor: pointer; +} +tr.parameter-row.oemsensor:hover { + background-color: var(--argopy_yellow); +} + +td.oemsensor a:link a:visited a:active{ + color: var(--argopy_blue); +} + +.parameter-details.oemsensor { + display: none; + background-color: #f9f9f9; +} +.parameter-details.oemsensor td { + padding: 0.6em; +} +.details-content.oemsensor { + margin: 0; + padding: 0.4em 0; +} +.details-content.oemsensor p { + margin: 0.2em 0; +} +pre.oemsensor { + background-color: #f8f8f8; + padding: 0.6em; + border-radius: 3px; + font-size: 0.85em; + overflow-x: auto; + color: var(--argopy_blue); +} From 68e18034605beef69c21ca81c41ad8c5eac02fc7 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Sun, 12 Oct 2025 21:33:47 +0200 Subject: [PATCH 31/71] more refactoring and docstrings --- argopy/related/sensors/oem_metadata.py | 178 +++++++++++++------- argopy/related/sensors/oem_metadata_repr.py | 114 +++++++++++-- argopy/related/utils.py | 18 +- argopy/static/css/oemsensor.css | 8 +- argopy/utils/format.py | 4 +- 5 files changed, 244 insertions(+), 78 deletions(-) diff --git a/argopy/related/sensors/oem_metadata.py b/argopy/related/sensors/oem_metadata.py index b6e3f503c..bffc8bbb3 100644 --- a/argopy/related/sensors/oem_metadata.py +++ b/argopy/related/sensors/oem_metadata.py @@ -6,7 +6,7 @@ from ...stores import httpstore from ...options import OPTIONS from ...utils import urnparser -from .oem_metadata_repr import NotebookCellDisplay +from .oem_metadata_repr import OemMetaDataDisplay, ParameterDisplay @dataclass @@ -53,6 +53,17 @@ def SENSOR_MODEL_uri(self): urnparts = urnparser(self.SENSOR_MODEL) return f"{OPTIONS['nvs']}/{urnparts['listid']}/current/{urnparts['termid']}" + def __repr__(self): + summary = [f""] + summary.append(f" SENSOR: {self.SENSOR} ({self.SENSOR_uri})") + summary.append(f" SENSOR_MAKER: {self.SENSOR_MAKER} ({self.SENSOR_MAKER_uri})") + summary.append(f" SENSOR_MODEL: {self.SENSOR_MODEL} ({self.SENSOR_MODEL_uri})") + summary.append(f" SENSOR_FIRMWARE_VERSION: {self.SENSOR_FIRMWARE_VERSION}") + summary.append(f" sensor_vendorinfo:") + for key in self.sensor_vendorinfo.keys(): + summary.append(f" - {key}: {self.sensor_vendorinfo[key]}") + return "\n".join(summary) + @dataclass class Parameter: @@ -78,42 +89,50 @@ def PARAMETER_SENSOR_uri(self): urnparts = urnparser(self.PARAMETER_SENSOR) return f"{OPTIONS['nvs']}/{urnparts['listid']}/current/{urnparts['termid']}" + def __repr__(self): + summary = [f""] + summary.append(f" PARAMETER: {self.PARAMETER} ({self.PARAMETER_uri})") + summary.append(f" PARAMETER_SENSOR: {self.PARAMETER_SENSOR} ({self.PARAMETER_SENSOR_uri})") + for key in ['UNITS', 'ACCURACY', 'RESOLUTION']: + p = f"PARAMETER_{key}" + summary.append(f" {key}: {getattr(self, p, 'N/A')}") + for key in ['EQUATION', 'COEFFICIENT', 'COMMENT', 'DATE']: + p = f"PREDEPLOYMENT_CALIB_{key}" + summary.append(f" {key}: {getattr(self, p, 'N/A')}") + for key in ['parameter_vendorinfo', 'predeployment_vendorinfo']: + summary.append(f" {key}: {getattr(self, key, 'N/A')}") + return "\n".join(summary) -def fetch_rbr_data(sn: str, **kwargs): - if kwargs.get("fs", None) is None: - _cache = kwargs.get("cache", True) - _cachedir = kwargs.get("cachedir", OPTIONS["cachedir"]) - _timeout = kwargs.get("timeout", OPTIONS["api_timeout"]) - fs_kargs = {"cache": _cache, "cachedir": _cachedir, "timeout": _timeout} - _api_key = kwargs.get("rbr_api_key", OPTIONS["rbr_api_key"]) - fs_kargs.update(client_kwargs={"headers": {"Authorization": _api_key}}) - fs = httpstore(**fs_kargs) - uri = f"{OPTIONS['rbr_api']}/instruments/{sn}/argometadatajson" - return fs.open_json(uri) + def _repr_html_(self): + return ParameterDisplay(self).html + def _ipython_display_(self): + from IPython.display import display, HTML + display(HTML(ParameterDisplay(self).html)) -class OemArgoSensorMetaData: - """Argo sensor meta-data from OEM +class ArgoSensorMetaDataOem: + """Argo sensor meta-data - from OEM - OEM : Original Equipment Manufacturer + A class helper to work with meta-data structure complying to schema from https://github.com/euroargodev/sensor_metadata_json - Comply to schema from https://github.com/euroargodev/sensor_metadata_json + Such meta-data structures are expected to come from sensor manufacturer (web-api or file). + + OEM : Original Equipment Manufacturer Examples -------- .. code-block:: python - OemArgoSensorMetaData() + ArgoSensorMetaData() - OemArgoSensorMetaData(validate=True) # Run json schema validation compliance automatically + ArgoSensorMetaData(validate=True) # Run json schema validation compliance when necessary - OemArgoSensorMetaData().from_dict(jsdata) # Use any compliant json data from OEM + ArgoSensorMetaData().from_dict(jsdata) # Use any compliant json data - OemArgoSensorMetaData.from_rbr(208380) # Direct call to the RBR api + ArgoSensorMetaData().from_rbr(208380) # Direct call to the RBR api """ - _schema_src = "https://raw.githubusercontent.com/euroargodev/sensor_metadata_json/refs/heads/main/schemas/argo.sensor.schema.json" """URI of the argo sensor JSON schema""" @@ -123,16 +142,6 @@ def __init__( validate: bool = False, **kwargs, ): - self._run_validation = validate - if self._run_validation: - self.schema = self._read_schema() - - self.sensor_info: Optional[SensorInfo] = None - self.context: Optional[Context] = None - self.sensors: List[Sensor] = field(default_factory=list) - self.parameters: List[Parameter] = field(default_factory=list) - self.instrument_vendorinfo: Optional[Dict[str, Any]] = None - if kwargs.get("fs", None) is not None: self._fs = kwargs.get("fs") else: @@ -146,39 +155,69 @@ def __init__( } self._fs = httpstore(**fs_kargs) + self._run_validation = validate + self.schema = self._read_schema() # requires a self._fs instance + + self.sensor_info: Optional[SensorInfo] = None + self.context: Optional[Context] = None + self.sensors: List[Sensor] = field(default_factory=list) + self.parameters: List[Parameter] = field(default_factory=list) + self.instrument_vendorinfo: Optional[Dict[str, Any]] = None + if json_data: self.from_dict(json_data) + def _empty_str(self): + summary = [f""] + summary.append("This object has no sensor info. You can use one of the following methods:") + for meth in [ + "from_rbr(serial_number)", + "from_dict(dict_or_json_data)", + ]: + summary.append(f" β•°β”ˆβž€ ArgoSensorMetaDataOem().{meth}") + return summary + def __repr__(self): - sensor_count = len(self.sensors) - parameter_count = len(self.parameters) - sensor_described = ( - self.sensor_info.sensor_described if self.sensor_info else "N/A" - ) - created_by = self.sensor_info.created_by if self.sensor_info else "N/A" - date_creation = self.sensor_info.date_creation if self.sensor_info else "N/A" - - summary = [f""] - summary.append(f"created_by: '{created_by}'") - summary.append(f"date_creation: '{date_creation}'") - summary.append(f"sensors: {sensor_count} {[urnparser(s.SENSOR)['termid'] for s in self.sensors]}") - summary.append(f"parameters: {parameter_count} {[urnparser(s.PARAMETER)['termid'] for s in self.parameters]}") - summary.append( - f"instrument_vendorinfo: {'Present' if self.instrument_vendorinfo else 'None'}" - ) + if self.sensor_info: + + sensor_described = ( + self.sensor_info.sensor_described if self.sensor_info else "N/A" + ) + created_by = self.sensor_info.created_by if self.sensor_info else "N/A" + date_creation = self.sensor_info.date_creation if self.sensor_info else "N/A" + sensor_count = len(self.sensors) if self.sensor_info else 0 + parameter_count = len(self.parameters) if self.sensor_info else 0 + + summary = [f""] + summary.append(f"created_by: '{created_by}'") + summary.append(f"date_creation: '{date_creation}'") + summary.append(f"sensors: {sensor_count} {[urnparser(s.SENSOR)['termid'] for s in self.sensors]}") + summary.append(f"parameters: {parameter_count} {[urnparser(s.PARAMETER)['termid'] for s in self.parameters]}") + summary.append(f"instrument_vendorinfo: {self.instrument_vendorinfo}") + + else: + summary = self._empty_str() + return "\n".join(summary) def _repr_html_(self): - return NotebookCellDisplay(self).html + if self.sensor_info: + return OemMetaDataDisplay(self).html + else: + return self._empty_str() def _ipython_display_(self): from IPython.display import display, HTML - display(HTML(NotebookCellDisplay(self).html)) + if self.sensor_info: + display(HTML(OemMetaDataDisplay(self).html)) + else: + display("\n".join(self._empty_str())) def _read_schema(self) -> Dict[str, Any]: """Load the JSON schema for validation.""" - schema = httpstore().open_json(self.schema_src) + # todo: implement static asset backup to load schema + schema = self._fs.open_json(self._schema_src) return schema def from_dict(self, data: Dict[str, Any]): @@ -252,14 +291,37 @@ def to_dict(self) -> Dict[str, Any]: "instrument_vendorinfo": self.instrument_vendorinfo, } - def save_to_json_file(self, file_path: str) -> None: - """Save the object to a JSON file.""" + def to_json_file(self, file_path: str) -> None: + """Save meta-data to a JSON file + + Notes + ----- + The output json file is compliant with the Argo sensor meta-data JSON schema :attr:`ArgoSensorMetaDataOem.schema` + """ with open(file_path, "w") as f: json.dump(self.to_dict(), f, indent=2) - @classmethod - def from_rbr(cls, serial_number: str, **kwargs): - """Fetch sensor metadata from RBR API and return an OemArgoSensorMetaData instance""" - # Use your HTTP store or API client to fetch data - data = fetch_rbr_data(serial_number, **kwargs) - return cls(json_data=data, **kwargs) + def from_rbr(self, serial_number: str, **kwargs): + """Fetch sensor metadata from RBR API and return an ArgoSensorMetaDataOem instance + + Parameters + ---------- + serial_number : str + Sensor serial number from RBR + kwargs : dict + Additional keyword arguments passed to the constructor of ArgoSensorMetaDataOem + + Notes + ----- + The instance :class:`httpstore` is automatically updated to use the OPTIONS value for ``rbr_api_key``. + """ + # Ensure that the instance httpstore has the appropriate authorization key: + fss = self._fs.fs.fs if getattr(self._fs, 'cache') else self._fs.fs + headers = fss.client_kwargs.get("headers", {}) + headers.update({"Authorization": kwargs.get("rbr_api_key", OPTIONS["rbr_api_key"])}) + fss._session = None # Reset fsspec aiohttp.ClientSession + + uri = f"{OPTIONS['rbr_api']}/instruments/{serial_number}/argometadatajson" + data = self._fs.open_json(uri) + + return self.from_dict(data) diff --git a/argopy/related/sensors/oem_metadata_repr.py b/argopy/related/sensors/oem_metadata_repr.py index ddee1e2fd..814347b37 100644 --- a/argopy/related/sensors/oem_metadata_repr.py +++ b/argopy/related/sensors/oem_metadata_repr.py @@ -27,7 +27,17 @@ def _load_static_files(): for package, resource in STATIC_FILES ] -class NotebookCellDisplay: + +def urn_html(this_urn): + x = urnparser(this_urn) + if x.get('version') is not "": + return f"{x.get('termid', '?')} ({x.get('listid', '?')}, {x.get('version', 'n/a')})" + else: + return f"{x.get('termid', '?')} ({x.get('listid', '?')})" + + + +class OemMetaDataDisplay: def __init__(self, obj): self.OEMsensor = obj @@ -62,12 +72,12 @@ def html(self): """ - for sensor in self.OEMsensor.sensors: + for ii, sensor in enumerate(self.OEMsensor.sensors): sensors_html += f""" - {sensor.SENSOR} - {sensor.SENSOR_MAKER} - {sensor.SENSOR_MODEL} + {urn_html(sensor.SENSOR)} + {urn_html(sensor.SENSOR_MAKER)} + {urn_html(sensor.SENSOR_MODEL)} {getattr(sensor, 'SENSOR_MODEL_FIRMWARE', getattr(sensor, 'SENSOR_FIRMWARE_VERSION', 'N/A'))} {sensor.SENSOR_SERIAL_NO} @@ -92,15 +102,15 @@ def html(self): """ - for i, param in enumerate(self.OEMsensor.parameters): + for ip, param in enumerate(self.OEMsensor.parameters): details_html = f""" - +
-

Calibration Equation: {param.PREDEPLOYMENT_CALIB_EQUATION}

-

Calibration Coefficients: {param.PREDEPLOYMENT_CALIB_COEFFICIENT_LIST}

-

Calibration Comment: {param.PREDEPLOYMENT_CALIB_COMMENT}

-

Calibration Date: {param.PREDEPLOYMENT_CALIB_DATE}

+

Calibration Equation:
{param.PREDEPLOYMENT_CALIB_EQUATION}

+

Calibration Coefficients:
{param.PREDEPLOYMENT_CALIB_COEFFICIENT_LIST}

+

Calibration Comment:
{param.PREDEPLOYMENT_CALIB_COMMENT}

+

Calibration Date:
{param.PREDEPLOYMENT_CALIB_DATE}

@@ -108,12 +118,12 @@ def html(self): parameters_html += f""" - {param.PARAMETER} - {param.PARAMETER_SENSOR} + {urn_html(param.PARAMETER)} + {urn_html(param.PARAMETER_SENSOR)} {param.PARAMETER_UNITS} {param.PARAMETER_ACCURACY} {param.PARAMETER_RESOLUTION} - Click for more + Click for more {details_html} """ @@ -155,3 +165,79 @@ def html(self): def _repr_html_(self): return self.html + + +class ParameterDisplay: + + def __init__(self, obj): + self.data = obj + + @property + def css_style(self): + return "\n".join(_load_static_files()) + + @property + def html(self): + + # --- Header --- + header_html = f"

Argo Sensor Metadata for Parameter: {urn_html(self.data.PARAMETER)}

" + + info = " | ".join([f"{p} {v}" for p, v in self.data.parameter_vendorinfo.items()]) + header_html += f"

{info}

" + + # --- Parameter details --- + html = """ + + + + + + + + + + + + """ + html += f""" + + + + + + + """ + + html += f""" + + + + """ + html += "
SensorUnitsAccuracyResolution
{urn_html(self.data.PARAMETER_SENSOR)}{self.data.PARAMETER_UNITS}{self.data.PARAMETER_ACCURACY}{self.data.PARAMETER_RESOLUTION}
+
+

Calibration Equation:
{self.data.PREDEPLOYMENT_CALIB_EQUATION}

+

Calibration Coefficients:
{self.data.PREDEPLOYMENT_CALIB_COEFFICIENT_LIST}

+

Calibration Comment:
{self.data.PREDEPLOYMENT_CALIB_COMMENT}

+

Calibration Date:
{self.data.PREDEPLOYMENT_CALIB_DATE}

+
+
" + + # --- Vendor Info --- + vendor_html = "" + if self.data.predeployment_vendorinfo: + vendor_html = f""" +

Pre-deployment Vendor Info:

+ 
{self.data.predeployment_vendorinfo}
+ """ + + # --- Combine All HTML --- + full_html = f""" + \n + {header_html}\n + {html}\n + {vendor_html} + """ + return full_html + + def _repr_html_(self): + return self.html \ No newline at end of file diff --git a/argopy/related/utils.py b/argopy/related/utils.py index ad75d4f81..587aad618 100644 --- a/argopy/related/utils.py +++ b/argopy/related/utils.py @@ -2,6 +2,8 @@ import os import json import logging +from typing import Literal + from . import ArgoNVSReferenceTables @@ -11,7 +13,21 @@ ).submodule_search_locations[0] -def load_dict(ptype): +def load_dict(ptype : Literal["profilers", "institutions"] | None = None) -> dict: + """Load a dictionary from a ArgoNVSReferenceTables instance + + Otherwise, fall back on static JSON asset files. + + List of possible dictionaries: + + - "profilers": key are R08 ID, values are prefLabel + - "institutions": key are R04 altLabel, values are prefLabel + + Parameters + ---------- + ptype: str, Literal["profilers", "institutions"], default None + + """ if ptype == "profilers": try: nvs = ArgoNVSReferenceTables(cache=True) diff --git a/argopy/static/css/oemsensor.css b/argopy/static/css/oemsensor.css index 29f483696..3f829d82a 100644 --- a/argopy/static/css/oemsensor.css +++ b/argopy/static/css/oemsensor.css @@ -47,11 +47,11 @@ th.oemsensor { tr.parameter-row.oemsensor td.calibration { cursor: pointer; } -tr.parameter-row.oemsensor:hover { - background-color: var(--argopy_yellow); -} +/*tr.parameter-row.oemsensor:hover {*/ +/* background-color: var(--argopy_yellow);*/ +/*}*/ -td.oemsensor a:link a:visited a:active{ +td.oemsensor a:link a:visited a:active a:hover a:active{ color: var(--argopy_blue); } diff --git a/argopy/utils/format.py b/argopy/utils/format.py index 88ba1a562..b4c4f467c 100644 --- a/argopy/utils/format.py +++ b/argopy/utils/format.py @@ -401,7 +401,9 @@ def cname(self) -> str: def urnparser(urn): - """Parsing RFC 8141 compliant uniform resource names (URN) from NVS""" + """Parsing RFC 8141 compliant uniform resource names (URN) from NVS + SDN stands for SeaDataNet + """ pp = urn.split(":") if len(pp) == 4 and pp[0] == 'SDN': return {'listid': pp[1], 'version': pp[2], 'termid': pp[3]} From 9cda4f0b0c74efb1a2bfdb5dbd8210d240a7c70c Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Mon, 13 Oct 2025 10:53:15 +0200 Subject: [PATCH 32/71] Update oem_metadata.py --- argopy/related/sensors/oem_metadata.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/argopy/related/sensors/oem_metadata.py b/argopy/related/sensors/oem_metadata.py index bffc8bbb3..b3b4b1ca8 100644 --- a/argopy/related/sensors/oem_metadata.py +++ b/argopy/related/sensors/oem_metadata.py @@ -110,6 +110,7 @@ def _ipython_display_(self): from IPython.display import display, HTML display(HTML(ParameterDisplay(self).html)) + class ArgoSensorMetaDataOem: """Argo sensor meta-data - from OEM @@ -302,14 +303,12 @@ def to_json_file(self, file_path: str) -> None: json.dump(self.to_dict(), f, indent=2) def from_rbr(self, serial_number: str, **kwargs): - """Fetch sensor metadata from RBR API and return an ArgoSensorMetaDataOem instance + """Fetch sensor metadata from RBR API Parameters ---------- serial_number : str Sensor serial number from RBR - kwargs : dict - Additional keyword arguments passed to the constructor of ArgoSensorMetaDataOem Notes ----- From 8cdd694e591a5b49ff2a0373c664a1560950514d Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Tue, 14 Oct 2025 16:18:17 +0200 Subject: [PATCH 33/71] Update local.py New open_subprocess for local file store, allowing to call on the system "open" for any file --- argopy/stores/implementations/local.py | 97 ++++++++++++++++++++++---- 1 file changed, 85 insertions(+), 12 deletions(-) diff --git a/argopy/stores/implementations/local.py b/argopy/stores/implementations/local.py index 9bbd7b077..6cdee593f 100644 --- a/argopy/stores/implementations/local.py +++ b/argopy/stores/implementations/local.py @@ -11,6 +11,8 @@ from pathlib import Path import warnings from netCDF4 import Dataset +from subprocess import Popen, PIPE +from platform import system from ...options import OPTIONS from ...errors import InvalidMethod, DataNotFound @@ -30,7 +32,9 @@ class filestore(ArgoStoreProto): protocol = "file" - def open_json(self, url, errors: Literal['raise', 'silent', 'ignore'] = 'raise', **kwargs) -> Any: + def open_json( + self, url, errors: Literal["raise", "silent", "ignore"] = "raise", **kwargs + ) -> Any: """Open and process a json document from a path Steps performed: @@ -88,12 +92,12 @@ def open_json(self, url, errors: Literal['raise', 'silent', 'ignore'] = 'raise', return js def open_dataset( - self, - path, - errors: Literal["raise", "ignore", "silent"] = "raise", - lazy: bool = False, - xr_opts: dict = {}, - **kwargs, + self, + path, + errors: Literal["raise", "ignore", "silent"] = "raise", + lazy: bool = False, + xr_opts: dict = {}, + **kwargs, ) -> xr.Dataset: """Create a :class:`xarray.Dataset` from a local path pointing to a netcdf file @@ -114,6 +118,7 @@ def open_dataset( ------- :class:`xarray.Dataset` """ + def load_in_memory(path, errors="raise", xr_opts={}): """ Returns @@ -155,7 +160,9 @@ def load_lazily(path, errors="raise", xr_opts={}, akoverwrite: bool = False): "backend_kwargs": { "consolidated": False, "storage_options": { - "fo": self.ak.to_kerchunk(path, overwrite=akoverwrite), # codespell:ignore + "fo": self.ak.to_kerchunk( + path, overwrite=akoverwrite + ), # codespell:ignore "remote_protocol": fsspec.core.split_protocol(path)[0], }, }, @@ -197,7 +204,7 @@ def load_lazily(path, errors="raise", xr_opts={}, akoverwrite: bool = False): else: target = target if isinstance(target, bytes) else target.getbuffer() - ds = Dataset(None, memory=target, diskless=True, mode='r') + ds = Dataset(None, memory=target, diskless=True, mode="r") self.register(path) return ds @@ -395,9 +402,9 @@ def open_mfdataset( results, dim=concat_dim, data_vars="minimal", # Only data variables in which the dimension already appears are included. - coords="all", # All coordinate variables will be concatenated, except those corresponding - # to other dimensions. - compat="override", # skip comparing and pick variable from first dataset, + coords="all", # All coordinate variables will be concatenated, except those corresponding + # to other dimensions. + compat="override", # skip comparing and pick variable from first dataset, ) return ds else: @@ -421,3 +428,69 @@ def read_csv(self, path, **kwargs): with self.open(path) as of: df = pd.read_csv(of, **kwargs) return df + + def open_subprocess(self, filepath): + """Open file with the default program in a subprocess. + + As being called in a subprocess, it will not block the current one. + This method runs the command on a subprocess using the default system shell. + + Parameters + ---------- + filepath: str + Path to the file to open + + Returns + ------- + subprocess + The pointer the subprocess returned by the Popen call + + Examples + -------- + The subprocess will run in background so you won't be able to bring back + the result code, but the current log can be obtained via the Popen's PIPE: + + .. code-block:: python + + from argopy.stores import filestore + + # Run as subprocess + subproc = filestore().open_subprocess("sensor_certificate.pdf") + + # Get the stdout for the subprocess + print(subproc.stdout) + + # Get the stderr for the subprocess + print(subproc.stderr) + + Notes + ----- + Source: https://gist.github.com/ytturi/0c23ad5ab89154d24c340c2b1cc3432b + """ + + def get_open_command(filepath): + """ + Get the console-like command to open the file + for the current platform: + - Windows: "start {{ filepath }}" + - OS X: "open {{ filepath }}" + - Linux based (wdg): "wdg-open {{ filepath }}" + :param filepath: Path to the file to be opened + :type filepath: string + :return: Command to run from a shell + :rtype: string + """ + OSNAME = system().lower() + if "windows" in OSNAME: + opener = "start" + elif "osx" in OSNAME or "darwin" in OSNAME: + opener = "open" + else: + opener = "xdg-open" + return "{opener} {filepath}".format(opener=opener, filepath=filepath) + + subproc = Popen( + get_open_command(filepath), stdout=PIPE, stderr=PIPE, shell=True + ) + subproc.wait() + return subproc From ec493a772d8581d62d0dfb4e715cd2c333995787 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Tue, 14 Oct 2025 16:18:43 +0200 Subject: [PATCH 34/71] Implement sensor schema validation --- argopy/errors.py | 2 +- argopy/related/sensors/oem_metadata.py | 212 +++++++++++++++++--- argopy/related/sensors/oem_metadata_repr.py | 60 ++++-- argopy/related/sensors/sensors.py | 2 +- 4 files changed, 224 insertions(+), 52 deletions(-) diff --git a/argopy/errors.py b/argopy/errors.py index 2cc2705ef..9d55b0f34 100644 --- a/argopy/errors.py +++ b/argopy/errors.py @@ -85,7 +85,7 @@ class InvalidDataset(ValueError): class InvalidDatasetStructure(ValueError): - """Raise when the xarray dataset is not as expected.""" + """Raise when an internal dataset structure is not as expected.""" pass diff --git a/argopy/related/sensors/oem_metadata.py b/argopy/related/sensors/oem_metadata.py index b3b4b1ca8..a1308039a 100644 --- a/argopy/related/sensors/oem_metadata.py +++ b/argopy/related/sensors/oem_metadata.py @@ -1,14 +1,23 @@ import json from dataclasses import dataclass, field -from typing import List, Dict, Optional, Any -from jsonschema import validate, ValidationError - -from ...stores import httpstore +from typing import List, Dict, Optional, Any, Literal +from pathlib import Path +from zipfile import ZipFile +from referencing import Registry, Resource +import jsonschema +import logging +import warnings + +from ...stores import httpstore, filestore from ...options import OPTIONS from ...utils import urnparser +from ...errors import InvalidDatasetStructure from .oem_metadata_repr import OemMetaDataDisplay, ParameterDisplay +log = logging.getLogger("argopy.related.sensors") + + @dataclass class SensorInfo: created_by: str @@ -33,9 +42,13 @@ class Sensor: SENSOR: str # SDN:R25::CTD_PRES SENSOR_MAKER: str # SDN:R26::RBR SENSOR_MODEL: str # SDN:R27::RBR_PRES_A - SENSOR_FIRMWARE_VERSION: str # wrong key used by RBR, https://github.com/euroargodev/sensor_metadata_json/issues/20 - # SENSOR_MODEL_FIRMWARE: str # Correct schema key SENSOR_SERIAL_NO: str + + # FIRMWARE VERSION attributes are temporarily optional to handle the wrong key used by RBR + # see https://github.com/euroargodev/sensor_metadata_json/issues/20 + SENSOR_FIRMWARE_VERSION: str = None # RBR + SENSOR_MODEL_FIRMWARE: str = None # Correct schema key + sensor_vendorinfo: Optional[Dict[str, Any]] = None @property @@ -53,15 +66,37 @@ def SENSOR_MODEL_uri(self): urnparts = urnparser(self.SENSOR_MODEL) return f"{OPTIONS['nvs']}/{urnparts['listid']}/current/{urnparts['termid']}" + def _attr2str(self, x): + """Return a class attribute, or 'n/a' if it's None, {} or "".""" + value = getattr(self, x, None) + if type(value) is str: + return value if value and value.strip() else 'n/a' + elif type(value) is dict: + if len(value.keys()) == 0: + return 'n/a' + else: + return value + else: + return value + def __repr__(self): + + def key2str(d, x): + """Return a dict value as a string, or 'n/a' if it's None or empty.""" + value = d.get(x, None) + return value if value and value.strip() else 'n/a' + summary = [f""] summary.append(f" SENSOR: {self.SENSOR} ({self.SENSOR_uri})") summary.append(f" SENSOR_MAKER: {self.SENSOR_MAKER} ({self.SENSOR_MAKER_uri})") summary.append(f" SENSOR_MODEL: {self.SENSOR_MODEL} ({self.SENSOR_MODEL_uri})") - summary.append(f" SENSOR_FIRMWARE_VERSION: {self.SENSOR_FIRMWARE_VERSION}") + if getattr(self, "SENSOR_MODEL_FIRMWARE", None) is None: + summary.append(f" SENSOR_FIRMWARE_VERSION: {self._attr2str('SENSOR_FIRMWARE_VERSION')} (but should be 'SENSOR_MODEL_FIRMWARE') ") + else: + summary.append(f" SENSOR_MODEL_FIRMWARE: {self._attr2str('SENSOR_MODEL_FIRMWARE')}") summary.append(f" sensor_vendorinfo:") for key in self.sensor_vendorinfo.keys(): - summary.append(f" - {key}: {self.sensor_vendorinfo[key]}") + summary.append(f" - {key}: {key2str(self.sensor_vendorinfo, key)}") return "\n".join(summary) @@ -89,18 +124,45 @@ def PARAMETER_SENSOR_uri(self): urnparts = urnparser(self.PARAMETER_SENSOR) return f"{OPTIONS['nvs']}/{urnparts['listid']}/current/{urnparts['termid']}" + def _attr2str(self, x): + """Return a class attribute, or 'n/a' if it's None, {} or "".""" + value = getattr(self, x, None) + if type(value) is str: + return value if value and value.strip() else 'n/a' + elif type(value) is dict: + if len(value.keys()) == 0: + return 'n/a' + else: + return value + else: + return value + + @property + def _has_calibration_data(self): + s = "".join([str(self._attr2str(key)) for key in + ['PREDEPLOYMENT_CALIB_EQUATION', + 'PREDEPLOYMENT_CALIB_COEFFICIENT_LIST', + 'PREDEPLOYMENT_CALIB_COMMENT', + 'PREDEPLOYMENT_CALIB_DATE'] if self._attr2str(key) != 'n/a']) + return len(s) > 0 + def __repr__(self): + summary = [f""] summary.append(f" PARAMETER: {self.PARAMETER} ({self.PARAMETER_uri})") summary.append(f" PARAMETER_SENSOR: {self.PARAMETER_SENSOR} ({self.PARAMETER_SENSOR_uri})") + for key in ['UNITS', 'ACCURACY', 'RESOLUTION']: p = f"PARAMETER_{key}" - summary.append(f" {key}: {getattr(self, p, 'N/A')}") + summary.append(f" {key}: {self._attr2str(p)}") + + summary.append(f" PREDEPLOYMENT CALIBRATION:") for key in ['EQUATION', 'COEFFICIENT', 'COMMENT', 'DATE']: p = f"PREDEPLOYMENT_CALIB_{key}" - summary.append(f" {key}: {getattr(self, p, 'N/A')}") + summary.append(f" - {key}: {self._attr2str(p)}") + for key in ['parameter_vendorinfo', 'predeployment_vendorinfo']: - summary.append(f" {key}: {getattr(self, key, 'N/A')}") + summary.append(f" {key}: {self._attr2str(p)}") return "\n".join(summary) def _repr_html_(self): @@ -132,15 +194,15 @@ class ArgoSensorMetaDataOem: ArgoSensorMetaData().from_rbr(208380) # Direct call to the RBR api - """ - _schema_src = "https://raw.githubusercontent.com/euroargodev/sensor_metadata_json/refs/heads/main/schemas/argo.sensor.schema.json" - """URI of the argo sensor JSON schema""" + _schema_root = "https://raw.githubusercontent.com/euroargodev/sensor_metadata_json/refs/heads/main/schemas" + """URI root to argo JSON schema""" def __init__( self, json_data: Optional[Dict[str, Any]] = None, validate: bool = False, + validation_error: Literal["warn", "raise", "ignore"] = "warn", **kwargs, ): if kwargs.get("fs", None) is not None: @@ -157,6 +219,7 @@ def __init__( self._fs = httpstore(**fs_kargs) self._run_validation = validate + self._validation_error = validation_error self.schema = self._read_schema() # requires a self._fs instance self.sensor_info: Optional[SensorInfo] = None @@ -164,6 +227,8 @@ def __init__( self.sensors: List[Sensor] = field(default_factory=list) self.parameters: List[Parameter] = field(default_factory=list) self.instrument_vendorinfo: Optional[Dict[str, Any]] = None + self._serial_number = None + self._local_certificates = None if json_data: self.from_dict(json_data) @@ -182,10 +247,10 @@ def __repr__(self): if self.sensor_info: sensor_described = ( - self.sensor_info.sensor_described if self.sensor_info else "N/A" + self.sensor_info.sensor_described if self.sensor_info else "n/a" ) - created_by = self.sensor_info.created_by if self.sensor_info else "N/A" - date_creation = self.sensor_info.date_creation if self.sensor_info else "N/A" + created_by = self.sensor_info.created_by if self.sensor_info else "n/a" + date_creation = self.sensor_info.date_creation if self.sensor_info else "n/a" sensor_count = len(self.sensors) if self.sensor_info else 0 parameter_count = len(self.parameters) if self.sensor_info else 0 @@ -215,19 +280,43 @@ def _ipython_display_(self): else: display("\n".join(self._empty_str())) - def _read_schema(self) -> Dict[str, Any]: - """Load the JSON schema for validation.""" - # todo: implement static asset backup to load schema - schema = self._fs.open_json(self._schema_src) + def _read_schema(self, ref="argo.sensor.schema.json") -> Dict[str, Any]: + """Load a JSON schema for validation.""" + # todo: implement static asset backup to load schema offline + uri = f"{self._schema_root}/{ref}" + schema = self._fs.open_json(uri) return schema + def validate(self, data): + """Validate meta-data against the Argo sensor json schema""" + # Set a method to resolve references to subschemas + registry = Registry(retrieve=lambda x: Resource.from_contents(self._read_schema(x))) + + # Select the validator based on $schema property in schema + # (validators correspond to various drafts of JSON Schema) + validator = jsonschema.validators.validator_for(self.schema) + + # Create the validator using the registry and associated resolver + v = validator(self.schema, registry=registry) + + try: + v.validate(data) + except Exception as error: + if self._validation_error == "raise": + raise error + elif self._validation_error == "warn": + warnings.warn(str(error)) + else: + log.error(error) + + # Create a list of errors, if any + errors = list(v.evolve(schema=self.schema).iter_errors(v, data)) + return errors + def from_dict(self, data: Dict[str, Any]): - """Load data from a dictionary and validate it.""" + """Load data from a dictionary and possibly validate""" if self._run_validation: - try: - validate(instance=data, schema=self.schema) - except ValidationError as e: - raise ValueError(f"Json schema Validation error: {e.message}") + self.validate(data) self.sensor_info = SensorInfo(**data["sensor_info"]) self.context = Context( @@ -305,6 +394,8 @@ def to_json_file(self, file_path: str) -> None: def from_rbr(self, serial_number: str, **kwargs): """Fetch sensor metadata from RBR API + We also download certificates if available + Parameters ---------- serial_number : str @@ -314,13 +405,78 @@ def from_rbr(self, serial_number: str, **kwargs): ----- The instance :class:`httpstore` is automatically updated to use the OPTIONS value for ``rbr_api_key``. """ + self._serial_number = serial_number + # Ensure that the instance httpstore has the appropriate authorization key: fss = self._fs.fs.fs if getattr(self._fs, 'cache') else self._fs.fs headers = fss.client_kwargs.get("headers", {}) headers.update({"Authorization": kwargs.get("rbr_api_key", OPTIONS["rbr_api_key"])}) fss._session = None # Reset fsspec aiohttp.ClientSession - uri = f"{OPTIONS['rbr_api']}/instruments/{serial_number}/argometadatajson" + uri = f"{OPTIONS['rbr_api']}/instruments/{self._serial_number}/argometadatajson" data = self._fs.open_json(uri) + obj = self.from_dict(data) + + # Download RBR zip archive with calibration certificates in PDFs: + obj = obj.certificates_rbr(action='download', quiet=True) + + return obj + + def certificates_rbr(self, action: Literal["download", "open"] = "download", **kwargs): + """Download RBR zip archive with calibration certificates in PDFs - return self.from_dict(data) + Certificate PDF files are written to the OPTIONS['cachedir'] folder + + """ + cdir = Path(OPTIONS['cachedir']).joinpath("RBR_certificates") + cdir.mkdir(parents=True, exist_ok=True) + local_zip_path = cdir.joinpath(f"RBRcertificates_{self._serial_number}.zip") + lfs = filestore() + quiet = kwargs.get('quiet', False) + + # Check if we can continue: + if self._serial_number is not None: + new = False + + # Trigger download if necessary: + if not lfs.exists(local_zip_path): + new = True + certif_uri = f"{OPTIONS['rbr_api']}/instruments/{self._serial_number}/certificates" + with open(local_zip_path, 'wb') as local_zip: + with self._fs.open(certif_uri) as remote_zip: + local_zip.write(remote_zip.read()) + + # Expand locally: + with ZipFile(local_zip_path, "r") as local_zip: + local_zip.testzip() + local_zip.extractall(cdir) + + # List PDF certificates: + with ZipFile(local_zip_path, "r") as local_zip: + local_zip.testzip() + info = local_zip.infolist() + certificates = [] + for doc in info: + certificates.append(Path(cdir).joinpath(doc.filename)) + self.local_certificates = certificates + + if not quiet: + for f in self.local_certificates: + if new: + s = f"One RBR certificate file written to: {f}" + else: + s = f"One RBR certificate file already in: {f}" + print(s) + else: + raise InvalidDatasetStructure(f"You must load meta-data for a given RBR sensor serial number first. Use the 'from_rbr' method.") + + if action == 'download': + return self + elif action == 'open': + subp = [] + for f in self.local_certificates: + subp.append(lfs.open_subprocess(str(f))) + if not quiet: + return subp + else: + raise ValueError(f"Unknown action {action}") \ No newline at end of file diff --git a/argopy/related/sensors/oem_metadata_repr.py b/argopy/related/sensors/oem_metadata_repr.py index 814347b37..5232c8806 100644 --- a/argopy/related/sensors/oem_metadata_repr.py +++ b/argopy/related/sensors/oem_metadata_repr.py @@ -1,6 +1,6 @@ -from IPython.display import display, HTML from functools import lru_cache import importlib +from numpy.random import randint try: from importlib.resources import files # New in version 3.9 @@ -30,7 +30,7 @@ def _load_static_files(): def urn_html(this_urn): x = urnparser(this_urn) - if x.get('version') is not "": + if x.get('version') != "": return f"{x.get('termid', '?')} ({x.get('listid', '?')}, {x.get('version', 'n/a')})" else: return f"{x.get('termid', '?')} ({x.get('listid', '?')})" @@ -48,6 +48,9 @@ def css_style(self): @property def html(self): + # Generate a dummy random id to be used in html elements for this output only + # This avoids messing up javascript actions between notebook cells + uid = f"{randint(low=1e7):8d}".strip() # --- Header --- header_html = f""" @@ -73,13 +76,19 @@ def html(self): """ for ii, sensor in enumerate(self.OEMsensor.sensors): + + if getattr(sensor, "SENSOR_MODEL_FIRMWARE", None) is None: + firmware = f"{sensor._attr2str('SENSOR_FIRMWARE_VERSION')}" + else: + firmware = f"{sensor._attr2str('SENSOR_MODEL_FIRMWARE')}" + sensors_html += f""" {urn_html(sensor.SENSOR)} {urn_html(sensor.SENSOR_MAKER)} {urn_html(sensor.SENSOR_MODEL)} - {getattr(sensor, 'SENSOR_MODEL_FIRMWARE', getattr(sensor, 'SENSOR_FIRMWARE_VERSION', 'N/A'))} - {sensor.SENSOR_SERIAL_NO} + {firmware} + {sensor._attr2str('SENSOR_SERIAL_NO')} """ @@ -104,26 +113,32 @@ def html(self): for ip, param in enumerate(self.OEMsensor.parameters): details_html = f""" - +
-

Calibration Equation:
{param.PREDEPLOYMENT_CALIB_EQUATION}

-

Calibration Coefficients:
{param.PREDEPLOYMENT_CALIB_COEFFICIENT_LIST}

-

Calibration Comment:
{param.PREDEPLOYMENT_CALIB_COMMENT}

-

Calibration Date:
{param.PREDEPLOYMENT_CALIB_DATE}

+

Calibration Equation:
{param._attr2str('PREDEPLOYMENT_CALIB_EQUATION')}

+

Calibration Coefficients:
{param._attr2str('PREDEPLOYMENT_CALIB_COEFFICIENT_LIST')}

+

Calibration Comment:
{param._attr2str('PREDEPLOYMENT_CALIB_COMMENT')}

+

Calibration Date:
{param._attr2str('PREDEPLOYMENT_CALIB_DATE')}

""" + # param.PREDEPLOYMENT_CALIB_EQUATION + if param._has_calibration_data: + line = 'Click for more' % (uid, ip) + else: + line = f"n/a" + parameters_html += f""" {urn_html(param.PARAMETER)} {urn_html(param.PARAMETER_SENSOR)} - {param.PARAMETER_UNITS} - {param.PARAMETER_ACCURACY} - {param.PARAMETER_RESOLUTION} - Click for more + {param._attr2str('PARAMETER_UNITS')} + {param._attr2str('PARAMETER_ACCURACY')} + {param._attr2str('PARAMETER_RESOLUTION')} + {line} {details_html} """ @@ -182,8 +197,9 @@ def html(self): # --- Header --- header_html = f"

Argo Sensor Metadata for Parameter: {urn_html(self.data.PARAMETER)}

" - info = " | ".join([f"{p} {v}" for p, v in self.data.parameter_vendorinfo.items()]) - header_html += f"

{info}

" + if self.data.parameter_vendorinfo is not None: + info = " | ".join([f"{p} {v}" for p, v in self.data.parameter_vendorinfo.items()]) + header_html += f"

{info}

" # --- Parameter details --- html = """ @@ -202,9 +218,9 @@ def html(self): html += f""" {urn_html(self.data.PARAMETER_SENSOR)} - {self.data.PARAMETER_UNITS} - {self.data.PARAMETER_ACCURACY} - {self.data.PARAMETER_RESOLUTION} + {self.data._attr2str('PARAMETER_UNITS')} + {self.data._attr2str('PARAMETER_ACCURACY')} + {self.data._attr2str('PARAMETER_RESOLUTION')} """ @@ -212,10 +228,10 @@ def html(self):
-

Calibration Equation:
{self.data.PREDEPLOYMENT_CALIB_EQUATION}

-

Calibration Coefficients:
{self.data.PREDEPLOYMENT_CALIB_COEFFICIENT_LIST}

-

Calibration Comment:
{self.data.PREDEPLOYMENT_CALIB_COMMENT}

-

Calibration Date:
{self.data.PREDEPLOYMENT_CALIB_DATE}

+

Calibration Equation:
{self.data._attr2str('PREDEPLOYMENT_CALIB_EQUATION')}

+

Calibration Coefficients:
{self.data._attr2str('PREDEPLOYMENT_CALIB_COEFFICIENT_LIST')}

+

Calibration Comment:
{self.data._attr2str('PREDEPLOYMENT_CALIB_COMMENT')}

+

Calibration Date:
{self.data._attr2str('PREDEPLOYMENT_CALIB_DATE')}

diff --git a/argopy/related/sensors/sensors.py b/argopy/related/sensors/sensors.py index 9df48ed91..802f0e8c0 100644 --- a/argopy/related/sensors/sensors.py +++ b/argopy/related/sensors/sensors.py @@ -105,7 +105,7 @@ class ArgoSensor: """ - __slots__ = ["_cache", "_cachedir", "_timeout", "fs", "_r25", "_r27", "_r27_to_r25", "_model", "_type"] + __slots__ = ["_cache", "_cachedir", "_timeout", "fs", "_r25", "_r26", "_r27", "_r27_to_r25", "_model", "_type"] def __init__( self, From 8888272206d378da09b8cb2289a2a54717590f76 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Tue, 14 Oct 2025 22:46:50 +0200 Subject: [PATCH 35/71] Add new column with URN to ArgoNVSReferenceTables --- argopy/related/reference_tables.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/argopy/related/reference_tables.py b/argopy/related/reference_tables.py index d447d5ccb..eddb69426 100644 --- a/argopy/related/reference_tables.py +++ b/argopy/related/reference_tables.py @@ -3,6 +3,7 @@ import collections from ..stores import httpstore from ..options import OPTIONS +from ..utils import urnparser class NVScollection: @@ -38,16 +39,18 @@ def _jsConcept2df(self, data): "prefLabel": [], "definition": [], "deprecated": [], + "urn": [], "id": [], } for k in data["@graph"]: if k["@type"] == "skos:Collection": Collection_name = k["dc:alternative"] elif k["@type"] == "skos:Concept": - content["altLabel"].append(k["skos:altLabel"]) + content["altLabel"].append(urnparser(k['skos:notation'])['termid']) content["prefLabel"].append(k["skos:prefLabel"]["@value"]) content["definition"].append(k["skos:definition"]["@value"]) content["deprecated"].append(k["owl:deprecated"]) + content["urn"].append(k['skos:notation']) content["id"].append(k["@id"]) df = pd.DataFrame.from_dict(content) df['deprecated'] = df.apply(lambda x: True if x['deprecated']=='true' else False, axis=1) From 7dc641f46fba2734a46c7204f564cf6ec6a807df Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Tue, 14 Oct 2025 22:47:36 +0200 Subject: [PATCH 36/71] Update oem_metadata.py - validate json by default - new from_example method for demo --- argopy/related/sensors/oem_metadata.py | 29 ++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/argopy/related/sensors/oem_metadata.py b/argopy/related/sensors/oem_metadata.py index a1308039a..eccbf0fc9 100644 --- a/argopy/related/sensors/oem_metadata.py +++ b/argopy/related/sensors/oem_metadata.py @@ -1,22 +1,24 @@ -import json from dataclasses import dataclass, field from typing import List, Dict, Optional, Any, Literal from pathlib import Path from zipfile import ZipFile from referencing import Registry, Resource import jsonschema +import json import logging import warnings +from fsspec import filesystem from ...stores import httpstore, filestore from ...options import OPTIONS -from ...utils import urnparser +from ...utils import urnparser, path2assets from ...errors import InvalidDatasetStructure from .oem_metadata_repr import OemMetaDataDisplay, ParameterDisplay log = logging.getLogger("argopy.related.sensors") +SENSOR_JS_EXAMPLES = filestore().open_json(Path(path2assets).joinpath("sensor_metadata_examples.json"))['data']['uri'] @dataclass class SensorInfo: @@ -190,9 +192,13 @@ class ArgoSensorMetaDataOem: ArgoSensorMetaData(validate=True) # Run json schema validation compliance when necessary - ArgoSensorMetaData().from_dict(jsdata) # Use any compliant json data + ArgoSensorMetaData().from_rbr(208380) # Direct call to the RBR api with a serial number + + ArgoSensorMetaData().list_examples - ArgoSensorMetaData().from_rbr(208380) # Direct call to the RBR api + ArgoSensorMetaData().from_examples('WETLABS-ECO_FLBBAP2-8589') + + ArgoSensorMetaData().from_dict(jsdata) # Use any compliant json data """ _schema_root = "https://raw.githubusercontent.com/euroargodev/sensor_metadata_json/refs/heads/main/schemas" @@ -201,7 +207,7 @@ class ArgoSensorMetaDataOem: def __init__( self, json_data: Optional[Dict[str, Any]] = None, - validate: bool = False, + validate: bool = True, validation_error: Literal["warn", "raise", "ignore"] = "warn", **kwargs, ): @@ -479,4 +485,15 @@ def certificates_rbr(self, action: Literal["download", "open"] = "download", **k if not quiet: return subp else: - raise ValueError(f"Unknown action {action}") \ No newline at end of file + raise ValueError(f"Unknown action {action}") + + @property + def list_examples(self): + """List of example names""" + return [k for k in SENSOR_JS_EXAMPLES.keys()] + + def from_examples(self, eg: str = None, **kwargs): + if eg not in self.list_examples: + raise ValueError(f"Unknown sensor example: '{eg}'. \n Use one in: {self.list_examples}") + data = self._fs.open_json(SENSOR_JS_EXAMPLES[eg]) + return self.from_dict(data) From 0c0fdb4066a116a980f4b3ed0b45a41196bb6dcf Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Tue, 14 Oct 2025 22:47:55 +0200 Subject: [PATCH 37/71] Update oem_metadata_repr.py improve notebook html display --- argopy/related/sensors/oem_metadata_repr.py | 53 +++++++++++++++------ 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/argopy/related/sensors/oem_metadata_repr.py b/argopy/related/sensors/oem_metadata_repr.py index 5232c8806..68ceb38fd 100644 --- a/argopy/related/sensors/oem_metadata_repr.py +++ b/argopy/related/sensors/oem_metadata_repr.py @@ -112,12 +112,23 @@ def html(self): """ for ip, param in enumerate(self.OEMsensor.parameters): + PREDEPLOYMENT_CALIB_EQUATION = param._attr2str('PREDEPLOYMENT_CALIB_EQUATION').split(';') + PREDEPLOYMENT_CALIB_EQUATION = [p.replace("=", " = ") for p in PREDEPLOYMENT_CALIB_EQUATION] + PREDEPLOYMENT_CALIB_EQUATION = "
\t".join(PREDEPLOYMENT_CALIB_EQUATION) + + PREDEPLOYMENT_CALIB_COEFFICIENT_LIST = param._attr2str('PREDEPLOYMENT_CALIB_COEFFICIENT_LIST') + s = [] + if isinstance(PREDEPLOYMENT_CALIB_COEFFICIENT_LIST, dict): + for key, value in PREDEPLOYMENT_CALIB_COEFFICIENT_LIST.items(): + s.append(f"{key} = {value}") + PREDEPLOYMENT_CALIB_COEFFICIENT_LIST = "
\t".join(s) + details_html = f"""
-

Calibration Equation:
{param._attr2str('PREDEPLOYMENT_CALIB_EQUATION')}

-

Calibration Coefficients:
{param._attr2str('PREDEPLOYMENT_CALIB_COEFFICIENT_LIST')}

+

Calibration Equation:
{PREDEPLOYMENT_CALIB_EQUATION}

+

Calibration Coefficients:
{PREDEPLOYMENT_CALIB_COEFFICIENT_LIST}

Calibration Comment:
{param._attr2str('PREDEPLOYMENT_CALIB_COMMENT')}

Calibration Date:
{param._attr2str('PREDEPLOYMENT_CALIB_DATE')}

@@ -193,12 +204,13 @@ def css_style(self): @property def html(self): + param = self.data # --- Header --- - header_html = f"

Argo Sensor Metadata for Parameter: {urn_html(self.data.PARAMETER)}

" + header_html = f"

Argo Sensor Metadata for Parameter: {urn_html(param.PARAMETER)}

" - if self.data.parameter_vendorinfo is not None: - info = " | ".join([f"{p} {v}" for p, v in self.data.parameter_vendorinfo.items()]) + if param.parameter_vendorinfo is not None: + info = " | ".join([f"{p} {v}" for p, v in param.parameter_vendorinfo.items()]) header_html += f"

{info}

" # --- Parameter details --- @@ -217,21 +229,32 @@ def html(self): """ html += f""" - {urn_html(self.data.PARAMETER_SENSOR)} - {self.data._attr2str('PARAMETER_UNITS')} - {self.data._attr2str('PARAMETER_ACCURACY')} - {self.data._attr2str('PARAMETER_RESOLUTION')} + {urn_html(param.PARAMETER_SENSOR)} + {param._attr2str('PARAMETER_UNITS')} + {param._attr2str('PARAMETER_ACCURACY')} + {param._attr2str('PARAMETER_RESOLUTION')} """ + PREDEPLOYMENT_CALIB_EQUATION = param._attr2str('PREDEPLOYMENT_CALIB_EQUATION').split(';') + PREDEPLOYMENT_CALIB_EQUATION = [p.replace("=", " = ") for p in PREDEPLOYMENT_CALIB_EQUATION] + PREDEPLOYMENT_CALIB_EQUATION = "
\t".join(PREDEPLOYMENT_CALIB_EQUATION) + + PREDEPLOYMENT_CALIB_COEFFICIENT_LIST = param._attr2str('PREDEPLOYMENT_CALIB_COEFFICIENT_LIST') + s = [] + if isinstance(PREDEPLOYMENT_CALIB_COEFFICIENT_LIST, dict): + for key, value in PREDEPLOYMENT_CALIB_COEFFICIENT_LIST.items(): + s.append(f"{key} = {value}") + PREDEPLOYMENT_CALIB_COEFFICIENT_LIST = "
\t".join(s) + html += f"""
-

Calibration Equation:
{self.data._attr2str('PREDEPLOYMENT_CALIB_EQUATION')}

-

Calibration Coefficients:
{self.data._attr2str('PREDEPLOYMENT_CALIB_COEFFICIENT_LIST')}

-

Calibration Comment:
{self.data._attr2str('PREDEPLOYMENT_CALIB_COMMENT')}

-

Calibration Date:
{self.data._attr2str('PREDEPLOYMENT_CALIB_DATE')}

+

Calibration Equation:
{PREDEPLOYMENT_CALIB_EQUATION}

+

Calibration Coefficients:
{PREDEPLOYMENT_CALIB_COEFFICIENT_LIST}

+

Calibration Comment:
{param._attr2str('PREDEPLOYMENT_CALIB_COMMENT')}

+

Calibration Date:
{param._attr2str('PREDEPLOYMENT_CALIB_DATE')}

@@ -240,10 +263,10 @@ def html(self): # --- Vendor Info --- vendor_html = "" - if self.data.predeployment_vendorinfo: + if param.predeployment_vendorinfo: vendor_html = f"""

Pre-deployment Vendor Info:

- 
{self.data.predeployment_vendorinfo}
+ 
{param.predeployment_vendorinfo}
""" # --- Combine All HTML --- From ad8c4eaceb00a8aa2d73ad2cbe7bce9d0f0b29f2 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Tue, 14 Oct 2025 22:48:19 +0200 Subject: [PATCH 38/71] Add JSON schema and list of sensor examples in assets --- .../static/assets/schema/argo.MRV.schema.json | 34 ++ .../static/assets/schema/argo.RBR.schema.json | 82 ++++ .../static/assets/schema/argo.SBE.schema.json | 381 ++++++++++++++++++ .../assets/schema/argo.TRIOS.schema.json | 50 +++ .../assets/schema/argo.float.schema.json | 145 +++++++ .../assets/schema/argo.platform.schema.json | 224 ++++++++++ .../assets/schema/argo.sensor.schema.json | 206 ++++++++++ .../assets/schema/argo.vendors.schema.json | 356 ++++++++++++++++ .../assets/sensor_metadata_examples.json | 53 +++ 9 files changed, 1531 insertions(+) create mode 100644 argopy/static/assets/schema/argo.MRV.schema.json create mode 100644 argopy/static/assets/schema/argo.RBR.schema.json create mode 100644 argopy/static/assets/schema/argo.SBE.schema.json create mode 100644 argopy/static/assets/schema/argo.TRIOS.schema.json create mode 100644 argopy/static/assets/schema/argo.float.schema.json create mode 100644 argopy/static/assets/schema/argo.platform.schema.json create mode 100644 argopy/static/assets/schema/argo.sensor.schema.json create mode 100644 argopy/static/assets/schema/argo.vendors.schema.json create mode 100644 argopy/static/assets/sensor_metadata_examples.json diff --git a/argopy/static/assets/schema/argo.MRV.schema.json b/argopy/static/assets/schema/argo.MRV.schema.json new file mode 100644 index 000000000..e3efab379 --- /dev/null +++ b/argopy/static/assets/schema/argo.MRV.schema.json @@ -0,0 +1,34 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "http://schema.nerc.ac.uk/schemas/sensor/0.1.0/argo.MRV.schema.json", + "title": "JSON Schema for Argo Program Sensors", + "description": "A JSON Schema used to populate Argo float sensor and parameter metadata elements specific to RBR. ", + "version": { + "const": "0.1" + }, + "type": "object", + "$defs": { + "platform_vendorinfo": { + "type": "object", + "version": { + "$ref": "#/version" + }, + "properties": { + "vendor_schema": true, + "version": { + "$ref": "#/version" + }, + "MRV_foo7": { + "type": "number" + }, + "MRV_foo8": { + "type": "string" + } + }, + "required": [ + "MRV_foo7" + ], + "additionalProperties": false + } + } +} \ No newline at end of file diff --git a/argopy/static/assets/schema/argo.RBR.schema.json b/argopy/static/assets/schema/argo.RBR.schema.json new file mode 100644 index 000000000..f5e6c05cc --- /dev/null +++ b/argopy/static/assets/schema/argo.RBR.schema.json @@ -0,0 +1,82 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "http://schema.nerc.ac.uk/schemas/sensor/0.1.0/argo.RBR.schema.json", + "title": "JSON Schema for Argo Program Sensors", + "description": "A JSON Schema used to populate Argo float sensor and parameter metadata elements specific to RBR. ", + "version": { + "const": "0.1" + }, + "type": "object", + "$defs": { + "sensor_vendorinfo": { + "type": "object", + "properties": { + "vendor_schema": true, + "version": { + "$ref": "#/version" + }, + "RBR_foo1": { + "type": "number" + }, + "RBR_foo2": { + "type": "string" + } + }, + "required": [ + "RBR_foo1", + "RBR_foo2" + ], + "additionalProperties": false + }, + "parameter_vendorinfo": { + "type": "object", + "properties": { + "vendor_schema": true, + "version": { + "$ref": "#/version" + }, + "RBR_foo3": { + "type": "number" + }, + "RBR_foo4": { + "type": "string" + } + }, + "additionalProperties": false + }, + "predeployment_vendorinfo": { + "type": "object", + "properties": { + "vendor_schema": true, + "version": { + "$ref": "#/version" + }, + "certificate": { + "type": "string", + "format": "uri", + "desription": "Link to calibration certificate" + } + }, + "required": [ + "certificate" + ], + "additionalProperties": false + }, + "instrument_vendorinfo": { + "type": "object", + "properties": { + "vendor_schema": true, + "version": { + "$ref": "#/version" + }, + "RBR_foo7": { + "type": "number" + }, + "RBR_foo8": { + "type": "string" + } + }, + "additionalProperties": false + } + } +} \ No newline at end of file diff --git a/argopy/static/assets/schema/argo.SBE.schema.json b/argopy/static/assets/schema/argo.SBE.schema.json new file mode 100644 index 000000000..449c6afc8 --- /dev/null +++ b/argopy/static/assets/schema/argo.SBE.schema.json @@ -0,0 +1,381 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "http://schema.nerc.ac.uk/schemas/sensor/0.1/argo.SBE.schema.json", + "title": "JSON Schema for Argo Program Sensors", + "description": "A JSON Schema used to populate Argo float sensor configuration and parameter metadata elements specifice to SBE. ", + "version": { + "const": "0.2" + }, + "type": "object", + "$defs": { + "sensor_vendorinfo": { + "type": "object", + "properties": { + "vendor_schema": true, + "version": { + "$ref": "#/version" + }, + "SBE_manufacturing_date": { + "type": "string", + "format": "date-time" + }, + "SBE_ISFET_SERIAL_NO": { + "type": "string" + }, + "SBE_REFERENCE_SERIAL_NO": { + "type": "string" + }, + "SBE_pHType": { + "type": "string", + "enum": [ + "Float", + "Deep", + "LOV", + "Stem,", + "Glider pH" + ] + }, + "CONFIG_EcoCdomFluorescenceExcitationWavelength_nm": { + "type": "number", + "const": 370 + }, + "CONFIG_EcoCdomFluorescenceExcitationBandwidth_nm": { + "type": "number", + "const": 15 + }, + "CONFIG_EcoCdomFluorescenceEmissionWavelength_nm": { + "type": "number", + "const": 460 + }, + "CONFIG_EcoCdomFluorescenceEmissionBandwidth_nm": { + "type": "number", + "const": 120 + }, + "CONFIG_EcoChlaFluorescenceExcitationWavelength_nm": { + "type": "number", + "enum": [ + 435, + 470 + ] + }, + "CONFIG_EcoChlaFluorescenceExcitationBandwidth_nm": { + "type": "number", + "enum": [ + 23, + 24 + ] + }, + "CONFIG_EcoChlaFluorescenceEmissionWavelength_nm": { + "type": "number", + "const": 695 + }, + "CONFIG_EcoChlaFluorescenceEmissionBandwidth_nm": { + "type": "number", + "const": 70 + }, + "CONFIG_EcoChlaFluorescenceExcitationWavelength1_nm": { + "type": "number", + "const": 470 + }, + "CONFIG_EcoChlaFluorescenceExcitationBandwidth1_nm": { + "type": "number", + "const": 23 + }, + "CONFIG_EcoChlaFluorescenceEmissionWavelength1_nm": { + "type": "number", + "const": 695 + }, + "CONFIG_EcoChlaFluorescenceEmissionBandwidth1_nm": { + "type": "number", + "const": 70 + }, + "CONFIG_EcoChlaFluorescenceExcitationWavelength2_nm": { + "type": "number", + "const": 435 + }, + "CONFIG_EcoChlaFluorescenceExcitationBandwidth2_nm": { + "type": "number", + "const": 15 + }, + "CONFIG_EcoChlaFluorescenceEmissionWavelength2_nm": { + "type": "number", + "const": 695 + }, + "CONFIG_EcoChlaFluorescenceEmissionBandwidth2_nm": { + "type": "number", + "const": 70 + }, + "CONFIG_EcoBetaWavelength_nm": { + "type": "number", + "enum": [ + 412, + 440, + 470, + 488, + 510, + 532, + 595, + 650, + 676, + 700, + 715 + ] + }, + "CONFIG_EcoBetaBandwidth_nm": { + "type": "number", + "enum": [ + 20, + 25 + ] + }, + "CONFIG_EcoBetaAngle_angularDeg": { + "type": "number", + "enum": [ + 124, + 142, + 150 + ] + }, + "CONFIG_EcoBetaWavelength1_nm": { + "type": "number", + "enum": [ + 412, + 440, + 470, + 488, + 510, + 532, + 595, + 650, + 676, + 700, + 715 + ] + }, + "CONFIG_EcoBetaBandwidth1_nm": { + "type": "number", + "const": 25 + }, + "CONFIG_EcoBetaAngle_angularDeg1": { + "type": "number", + "enum": [ + 124, + 142, + 150 + ] + }, + "CONFIG_EcoBetaWavelength2_nm": { + "type": "number", + "enum": [ + 412, + 440, + 470, + 488, + 510, + 532, + 595, + 650, + 676, + 700, + 715 + ] + }, + "CONFIG_EcoBetaBandwidth2_nm": { + "type": "number", + "const": 25 + }, + "CONFIG_EcoBetaAngle_angularDeg2": { + "type": "number", + "enum": [ + 124, + 142, + 150 + ] + }, + "CONFIG_OcrDownIrrWavelength1_nm": { + "type": "integer", + "enum": [ + 380 + ], + "maximum": 900 + }, + "CONFIG_OcrDownIrrBandwidth1_nm": { + "type": "number", + "enum": [ + 10 + ] + }, + "CONFIG_OcrDownIrrWavelength2_nm": { + "type": "number", + "enum": [ + 412, + 443 + ], + "maximum": 900 + }, + "CONFIG_OcrDownIrrBandwidth2_nm": { + "type": "number", + "enum": [ + 10 + ] + }, + "CONFIG_OcrDownIrrWavelength3_nm": { + "type": "number", + "enum": [ + 490 + ], + "maximum": 900 + }, + "CONFIG_OcrDownIrrBandwidth3_nm": { + "type": "number", + "enum": [ + 10 + ] + }, + "CONFIG_OcrDownIrrWavelength4_nm": { + "type": "number", + "enum": [ + 555 + ] + }, + "CONFIG_OcrDownIrrBandwidth4_nm": { + "type": "number", + "enum": [ + 10 + ] + }, + "CONFIG_SunaApfFrameOutputPixelBegin_NUMBER": { + "type": "number", + "minimum": 1, + "maximum": 256 + }, + "CONFIG_SunaApfFrameOutputPixelEnd_NUMBER": { + "type": "number", + "minimum": 1, + "maximum": 256 + }, + "CONFIG_EcoVerticalPressureOffset_dbar": { + "type": "number", + "minimum": -2.0, + "maximum": 2.0 + }, + "CONFIG_OcrVerticalPressureOffset_dbar": { + "type": "number", + "minimum": -2.0, + "maximum": 2.0 + }, + "CONFIG_OptodeVerticalPressureOffset_dbar": { + "type": "number", + "minimum": -2.0, + "maximum": 2.0 + }, + "CONFIG_SunaVerticalPressureOffset_dbar": { + "type": "number", + "minimum": -2.0, + "maximum": 2.0 + } + }, + "required": [ + "vendor_schema", + "version" + ], + "additionalProperties": false + }, + "parameter_vendorinfo": { + "type": "object", + "version": { + "$ref": "#/version" + }, + "properties": { + "vendor_schema": true, + "version": { + "$ref": "#/version" + }, + "SBE_calfile": { + "type": "string" + }, + "SBE_optional3": { + "type": "number" + }, + "SBE_optional4": { + "type": "string" + } + }, + "required": [ + "vendor_schema", + "version" + ], + "additionalProperties": false + }, + "predeployment_vendorinfo": { + "type": "object", + "version": { + "$ref": "#/version" + }, + "properties": { + "vendor_schema": true, + "version": { + "$ref": "#/version" + }, + "PREDEPLOYMENT_SEAFET_K0_CALIB_DATE": { + "type": "string", + "format": "date-time" + }, + "PREDEPLOYMENT_SEAFET_K2_CALIB_DATE": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "vendor_schema", + "version" + ], + "additionalProperties": false + }, + "instrument_vendorinfo": { + "type": "object", + "version": { + "$ref": "#/version" + }, + "properties": { + "vendor_schema": true, + "version": { + "$ref": "#/version" + }, + "SBE_part_number": { + "type": "string" + } + }, + "required": [ + "vendor_schema", + "version" + ], + "additionalProperties": false + }, + "platform_vendorinfo": { + "type": "object", + "version": { + "$ref": "#/version" + }, + "properties": { + "vendor_schema": true, + "version": { + "$ref": "#/version" + }, + "SBE_manufacturing_date": { + "type": "string", + "format": "date-time" + }, + "SBE_part_number": { + "type": "string" + } + }, + "required": [ + "vendor_schema", + "version", + "SBE_manufacturing_date" + ], + "additionalProperties": false + } + } +} \ No newline at end of file diff --git a/argopy/static/assets/schema/argo.TRIOS.schema.json b/argopy/static/assets/schema/argo.TRIOS.schema.json new file mode 100644 index 000000000..ccfe2087d --- /dev/null +++ b/argopy/static/assets/schema/argo.TRIOS.schema.json @@ -0,0 +1,50 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "http://schema.nerc.ac.uk/schemas/sensor/0.1.0/argo.TRIOS.schema.json", + "title": "JSON Schema for TRIOS-specific sensor metadata", + "description": "A JSON Schema used to populate Argo sensor and parameter metadata elements specific to RBR. ", + "version": { + "const": "0.1" + }, + "type": "object", + "$defs": { + "sensor_vendorinfo": { + "type": "object", + "version": { + "$ref": "#/version" + }, + "properties": { + "vendor_schema": true, + "version": { + "$ref": "#/version" + }, + "TRIOS_RAMSESType": { + "type": "string" + } + }, + "required": [ + "TRIOS_RAMSESType" + ], + "additionalProperties": false + }, + "parameter_vendorinfo": { + "type": "object", + "version": { + "$ref": "#/version" + }, + "properties": { + "vendor_schema": true, + "version": { + "$ref": "#/version" + }, + "TRIOS_calfile": { + "type": "string" + } + }, + "required": [ + "TRIOS_calfile" + ], + "additionalProperties": false + } + } +} \ No newline at end of file diff --git a/argopy/static/assets/schema/argo.float.schema.json b/argopy/static/assets/schema/argo.float.schema.json new file mode 100644 index 000000000..119765932 --- /dev/null +++ b/argopy/static/assets/schema/argo.float.schema.json @@ -0,0 +1,145 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "./argo.float.schema.json", + "title": "JSON Schema for Argo Program float = platform + sensors", + "description": "A JSON Schema used to describe an Argo float. A float consists of a single platform and any number of sensors. See Argo User's Manual. Metada format Version 3.41.1", + "format_version": { + "const": "0.4.0" + }, + "type": "object", + "properties": { + "float_info": { + "type": "object", + "properties": { + "created_by": { + "type": "string", + "description": "The entity primarily responsible for making the content of this JSON metadata resource" + }, + "date_creation": { + "type": "string", + "format": "date-time", + "description": "Date and time of creation of the resource. RFC 3339 format compliant with ISO 8601" + }, + "link": { + "const": "./argo.float.schema.json", + "description": "Link to JSON Schema that validates this JSON metadata resource" + }, + "format_version": { + "$ref": "#/format_version", + "description": "Schema version for this resource. Must match /format_version." + }, + "contents": { + "type": "string", + "description": "Description of this JSON metadata respirce." + }, + "float_described": { + "type": "string", + "description": "Identification of float described by this JSON metadata resouce: Ideally: float--." + } + }, + "required": [ + "created_by", + "date_creation", + "link", + "format_version", + "contents", + "float_described" + ] + }, + "@context": { + "type": "object", + "properties": { + "SDN:R03::": { + "const": "http://vocab.nerc.ac.uk/collection/R03/current/" + }, + "SDN:R08::": { + "const": "http://vocab.nerc.ac.uk/collection/R08/current/" + }, + "SDN:R09::": { + "const": "http://vocab.nerc.ac.uk/collection/R09/current/" + }, + "SDN:R10::": { + "const": "http://vocab.nerc.ac.uk/collection/R10/current/" + }, + "SDN:R22::": { + "const": "http://vocab.nerc.ac.uk/collection/R22/current/" + }, + "SDN:R23::": { + "const": "http://vocab.nerc.ac.uk/collection/R23/current/" + }, + "SDN:R24::": { + "const": "http://vocab.nerc.ac.uk/collection/R24/current/" + }, + "SDN:R25::": { + "const": "http://vocab.nerc.ac.uk/collection/R25/current/" + }, + "SDN:R26::": { + "const": "http://vocab.nerc.ac.uk/collection/R26/current/" + }, + "SDN:R27::": { + "const": "http://vocab.nerc.ac.uk/collection/R27/current/" + }, + "SDN:R28::": { + "const": "http://vocab.nerc.ac.uk/collection/R28/current/" + } + }, + "description": "Mapping from SDN NVS term IDs to validation URL. Must be present in JSON float metadata resource.", + "required": [ + "SDN:R03::", + "SDN:R08::", + "SDN:R09::", + "SDN:R10::", + "SDN:R22::", + "SDN:R23::", + "SDN:R24::", + "SDN:R25::", + "SDN:R26::", + "SDN:R27::", + "SDN:R28::" + ], + "additionalProperties": false + }, + "files_merged": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Array of one platform JSON filename and multiple sensors JSON filename merged to create this JSON float metadata resource" + }, + "platform_info": { + "$ref": "./argo.platform.schema.json#/properties/platform_info", + "description": "Schema-valid platform metadata instance, typically from JSON PLATFORM file" + }, + "sensor_info_list": { + "type": "array", + "items": { + "$ref": "./argo.sensor.schema.json#/properties/sensor_info" + }, + "description": "Array of schema-valid sensor_info metadata instances, typically from JSON SENSOR files." + }, + "PLATFORM": { + "$ref": "./argo.platform.schema.json#/properties/PLATFORM", + "description": "Schema-valid PLATFORM metadata instance, typically from JSON PLATFORM file." + }, + "SENSORS": { + "$ref": "./argo.sensor.schema.json#/properties/SENSORS", + "description": "Array of schema-valid SENSOR metadata instances, typically from JSON SENSOR files." + }, + "PARAMETERS": { + "$ref": "./argo.sensor.schema.json#/properties/PARAMETERS", + "description": "Array of schema-valid PARAMETER metadata instances corresponding to SENSORS above. Typically from JSON SENSOR files." + } + }, + "required": [ + "@context", + "float_info", + "files_merged", + "platform_info", + "sensor_info_list", + "PLATFORM", + "SENSORS", + "PARAMETERS" + ], + "uniqueItems": true, + "additionalProperties": false +} \ No newline at end of file diff --git a/argopy/static/assets/schema/argo.platform.schema.json b/argopy/static/assets/schema/argo.platform.schema.json new file mode 100644 index 000000000..31165e322 --- /dev/null +++ b/argopy/static/assets/schema/argo.platform.schema.json @@ -0,0 +1,224 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "./argo.platform.schema.json", + "title": "JSON Schema for Argo Program Platforms", + "description": "A JSON Schema used to describe an Argo platform. The platform includes the hull, controller, buoyancy engine, positioning and transmission systems, but not the sensors. See Argo User's Manual. Metada format Version 3.41.1", + "format_version": { + "const": "0.4.0" + }, + "type": "object", + "properties": { + "platform_info": { + "type": "object", + "properties": { + "created_by": { + "type": "string" + }, + "date_creation": { + "type": "string", + "format": "date-time" + }, + "link": { + "const": "./argo.platform.schema.json" + }, + "format_version": { + "$ref": "#/format_version" + }, + "contents": { + "type": "string" + }, + "platform_described": { + "type": "string" + } + }, + "required": [ + "created_by", + "date_creation", + "format_version", + "contents", + "platform_described" + ] + }, + "@context": { + "type": "object", + "properties": { + "SDN:R08::": { + "const": "http://vocab.nerc.ac.uk/collection/R08/current/" + }, + "SDN:R09::": { + "const": "http://vocab.nerc.ac.uk/collection/R09/current/" + }, + "SDN:R10::": { + "const": "http://vocab.nerc.ac.uk/collection/R10/current/" + }, + "SDN:R22::": { + "const": "http://vocab.nerc.ac.uk/collection/R22/current/" + }, + "SDN:R23::": { + "const": "http://vocab.nerc.ac.uk/collection/R23/current/" + }, + "SDN:R24::": { + "const": "http://vocab.nerc.ac.uk/collection/R24/current/" + }, + "SDN:R28::": { + "const": "http://vocab.nerc.ac.uk/collection/R28/current/" + } + }, + "required": [ + "SDN:R08::", + "SDN:R09::", + "SDN:R10::", + "SDN:R22::", + "SDN:R23::", + "SDN:R24::", + "SDN:R28::" + ], + "additionalProperties": false + }, + "PLATFORM": { + "type": "object", + "properties": { + "PLATFORM_FAMILY": { + "type": "string", + "format": "uri", + "pattern": "^SDN:R22::", + "description": "PLATFORM_FAMILY string must be valid in current Argo reference table R22", + "validation-uri": "https://vocab.nerc.ac.uk/collection/R22/current/" + }, + "PLATFORM_TYPE": { + "type": "string", + "format": "uri", + "pattern": "^SDN:R23::", + "description": "PLATFORM_TYPE string must be valid in current Argo reference table R23", + "validation-uri": "https://vocab.nerc.ac.uk/collection/R23/current/" + }, + "PLATFORM_MAKER": { + "type": "string", + "format": "uri", + "pattern": "^SDN:R24::", + "description": "PLATFORM_MAKER (manufacturer) string must be valid in current Argo reference table R24", + "validation-uri": "https://vocab.nerc.ac.uk/collection/R24/current/" + }, + "POSITIONING_SYSTEM": { + "type": "array", + "items": { + "type": "string", + "format": "uri", + "pattern": "^SDN:R09::", + "description": "POSITIONING_SYSTEM is a list of positioning systems. Each entry must be valid in current Argo reference table R09", + "validation-uri": "https://vocab.nerc.ac.uk/collection/R09/current/" + } + }, + "PTT": { + "type": "string", + "description": "Transmission identifier of the float. Comma separated list for multi-beacon transmission." + }, + "TRANS_SYSTEM": { + "type": "array", + "items": { + "type": "string", + "format": "uri", + "pattern": "^SDN:R10::", + "description": "TRANS_SYSTEM is a list of telecommunication systems. Each entry must be valid in the current reference table R10", + "validation-uri": "https://vocab.nerc.ac.uk/collection/R10/current/" + } + }, + "TRANS_SYSTEM_ID": { + "type": "array", + "items": { + "type": "string", + "description": "TRANS_SYSTEM_ID is the Program identifier of the telecommunication subscription. DACs can use N/A or alternative of their choice when not applicable" + } + }, + "TRANS_FREQUENCY": { + "type": "array", + "items": { + "type": "string", + "description": "Frequency of transmission from the float (hertz)." + } + }, + "FIRMWARE_VERSION": { + "type": "string", + "description": "Firmware version of the platform, defined by the platform_maker. This is specified as per the format on the manufacturer\u2019s manual." + }, + "MANUAL_VERSION": { + "type": "string", + "description": "The version date or number for the manual for the platform, defined by the platform_maker." + }, + "FLOAT_SERIAL_NO": { + "type": "string", + "description": "This field should contain only the serial number of the platform, defined by the platform_maker." + }, + "WMO_INST_TYPE": { + "type": "string", + "format": "uri", + "pattern": "^SDN:R08::", + "description": "WMO_INST_TYPE string is a subset of instrument type codes from the World Meteorological Organization (WMO) Common Code Table C-3 (CCT C-3) 1770. String must be valid in current Argo reference table R08", + "validation-uri": "https://vocab.nerc.ac.uk/collection/R08/current/" + }, + "BATTERY_TYPE": { + "type": "string", + "pattern": "(?i)^(?:\\s*\\w+\\s+)?(?:Alkaline|Lithium)\\s+\\d+(?:\\.\\d+)?\\s*V(?:\\s*\\+\\s*\\w+\\s+(?:Alkaline|Lithium)\\s+\\d+(?:\\.\\d+)?\\s*V)*$", + "description": "Describes the type of battery packs in the float. BATTERY_TYPE = 'Manufacturer Alkaline x V' or 'Manufacturer Lithium x V + Manufacturer Alkaline x V, e.g.,ELECTROCHEM Lithium 15 V" + }, + "BATTERY_PACKS": { + "type": "string", + "pattern": "(?i)^[U]|(\\s*\\d+(?:DD|C|D)\\s+(?:Li|Alk|Hyb)(?:\\s*\\+\\s*\\d+(?:DD|C|D)\\s+(?:Li|Alk|Hyb))*)$", + "description": "Describes the configuration of battery packs in the float, number and type. Example: \u201c4DD Li + 1C Alk. Pattern above enforces rules in reference table 30" + }, + "CONTROLLER_BOARD_TYPE_PRIMARY": { + "type": "string", + "format": "uri", + "pattern": "^SDN:R28::\\w+(\\s+\\[.*\\])?$", + "description": "Describes the type of primary controller board. first part of string: see reference table 28.0 Free text between optional []", + "validation-uri": "https://vocab.nerc.ac.uk/collection/R28/current/" + }, + "CONTROLLER_BOARD_SERIAL_NO_PRIMARY": { + "type": "string", + "description": "The serial number for the primary controller board." + }, + "CONTROLLER_BOARD_TYPE_SECONDARY": { + "type": "string", + "format": "uri", + "pattern": "^SDN:R28::\\w+(\\s+\\[\\w+\\])?$", + "description": "Optional: Describes the type of secondary controller board. first part of string: see reference table 28.0. Free text between optional []", + "validation-uri": "https://vocab.nerc.ac.uk/collection/R28/current/" + }, + "CONTROLLER_BOARD_SERIAL_NO_SECONDARY": { + "type": "string", + "description": "Optional: The serial number for the secondary controller board." + }, + "SPECIAL_FEATURES": { + "type": "string", + "description": "Additional float features can be specified here such as algorithms used by the float. Indicates there are AUX data files. See reference table 32." + }, + "platform_vendorinfo": { + "$ref": "./argo.vendors.schema.json#/$defs/platform_vendorinfo" + } + }, + "required": [ + "POSITIONING_SYSTEM", + "PTT", + "TRANS_SYSTEM", + "PLATFORM_FAMILY", + "PLATFORM_TYPE", + "PLATFORM_MAKER", + "FLOAT_SERIAL_NO", + "FIRMWARE_VERSION", + "MANUAL_VERSION", + "BATTERY_TYPE", + "CONTROLLER_BOARD_TYPE_PRIMARY", + "CONTROLLER_BOARD_SERIAL_NO_PRIMARY" + ], + "additionalProperties": false, + "uniqueItems": true + } + }, + "required": [ + "@context", + "platform_info", + "PLATFORM" + ], + "uniqueItems": true, + "additionalProperties": false +} \ No newline at end of file diff --git a/argopy/static/assets/schema/argo.sensor.schema.json b/argopy/static/assets/schema/argo.sensor.schema.json new file mode 100644 index 000000000..fcca8ea70 --- /dev/null +++ b/argopy/static/assets/schema/argo.sensor.schema.json @@ -0,0 +1,206 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "./argo.sensor.schema.json", + "title": "JSON Schema for Argo Program Sensors", + "description": "A JSON Schema used to populate Argo float sensor and parameter metadata elements. See Argo User's Manual. Metada format Version 3.41.1", + "format_version": { + "const": "0.4.0" + }, + "type": "object", + "properties": { + "sensor_info": { + "type": "object", + "properties": { + "created_by": { + "type": "string" + }, + "date_creation": { + "type": "string", + "format": "date-time" + }, + "link": { + "const": "./argo.sensor.schema.json" + }, + "format_version": { + "$ref": "#/format_version" + }, + "contents": { + "type": "string" + }, + "sensor_described": { + "type": "string" + } + }, + "required": [ + "created_by", + "date_creation", + "link", + "format_version", + "contents", + "sensor_described" + ] + }, + "@context": { + "type": "object", + "properties": { + "SDN:R03::": { + "const": "http://vocab.nerc.ac.uk/collection/R03/current/" + }, + "SDN:R25::": { + "const": "http://vocab.nerc.ac.uk/collection/R25/current/" + }, + "SDN:R26::": { + "const": "http://vocab.nerc.ac.uk/collection/R26/current/" + }, + "SDN:R27::": { + "const": "http://vocab.nerc.ac.uk/collection/R27/current/" + }, + "SDN:L22::": { + "const": "http://vocab.nerc.ac.uk/collection/L22/current/" + } + }, + "required": [ + "SDN:R03::", + "SDN:R25::", + "SDN:R26::", + "SDN:R27::" + ], + "additionalProperties": false + }, + "SENSORS": { + "type": "array", + "items": { + "type": "object", + "properties": { + "SENSOR": { + "type": "string", + "format": "uri", + "pattern": "^SDN:R25::", + "description": "SENSOR string must be valid in current Argo reference table R25", + "validation-uri": "https://vocab.nerc.ac.uk/collection/R25/current/" + }, + "SENSOR_MAKER": { + "type": "string", + "format": "uri", + "pattern": "^SDN:R26::", + "description": "SENSOR_MAKER string must be valid in current Argo reference table R26", + "validation-uri": "https://vocab.nerc.ac.uk/collection/R26/current/" + }, + "SENSOR_MODEL": { + "type": "string", + "format": "uri", + "pattern": "^SDN:(R27|L22)::", + "description": "SENSOR_MODEL string must be valid in current Argo reference table R27 or L22", + "validation-uri": "https://vocab.nerc.ac.uk/collection/R27/current/" + }, + "SENSOR_MODEL_FIRMWARE": { + "type": "string", + "description": "Firmware version of this sensor_model, defined by the sensor_maker" + }, + "SENSOR_SERIAL_NO": { + "type": "string", + "description": "Serial number of the sensor" + }, + "sensor_vendorinfo": { + "$ref": "./argo.vendors.schema.json#/$defs/sensor_vendorinfo" + } + }, + "required": [ + "SENSOR", + "SENSOR_MAKER", + "SENSOR_MODEL", + "SENSOR_SERIAL_NO", + "SENSOR_MODEL_FIRMWARE" + ], + "additionalProperties": false + }, + "minItems": 1, + "uniqueItems": true + }, + "PARAMETERS": { + "type": "array", + "items": { + "type": "object", + "properties": { + "PARAMETER": { + "type": "string", + "format": "uri", + "pattern": "^SDN:R03::", + "description": "PARAMETER string must be valid in current Argo reference table R03", + "validation_uri": "http://vocab.nerc.ac.uk/collection/R03/current/" + }, + "PARAMETER_SENSOR": { + "type": "string", + "format": "uri", + "pattern": "^SDN:R25::", + "description": "PARAMETER_SENSOR string must be valid in current Argo reference table R25 and must be listed as a SENSOR in the SENSORS section above", + "validation_uri": "http://vocab.nerc.ac.uk/collection/R25/current/" + }, + "PARAMETER_UNITS": { + "type": "string" + }, + "PARAMETER_ACCURACY": { + "type": "string" + }, + "PARAMETER_RESOLUTION": { + "type": "string" + }, + "PREDEPLOYMENT_CALIB_EQUATION": { + "type": "string", + "description": "Calibration equation for this parameter." + }, + "PREDEPLOYMENT_CALIB_COEFFICIENT_LIST": { + "type": "object", + "description": "Calibation coefficients are listed individually here, string-encoded to guarantee precision. They will be compiled to create the Argo PREDEPLOYMENT_CALIB_COEFFCIENT string" + }, + "PREDEPLOYMENT_CALIB_COMMENT": { + "type": "string" + }, + "PREDEPLOYMENT_CALIB_DATE": { + "anyOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "string", + "const": " " + } + ] + }, + "parameter_vendorinfo": { + "$ref": "./argo.vendors.schema.json#/$defs/parameter_vendorinfo" + }, + "predeployment_vendorinfo": { + "$ref": "./argo.vendors.schema.json#/$defs/predeployment_vendorinfo" + } + }, + "required": [ + "PARAMETER", + "PARAMETER_SENSOR", + "PARAMETER_UNITS", + "PARAMETER_ACCURACY", + "PARAMETER_RESOLUTION", + "PREDEPLOYMENT_CALIB_EQUATION", + "PREDEPLOYMENT_CALIB_COEFFICIENT_LIST", + "PREDEPLOYMENT_CALIB_COMMENT", + "PREDEPLOYMENT_CALIB_DATE" + ], + "additionalProperties": false + }, + "minItems": 1, + "uniqueItems": true + }, + "instrument_vendorinfo": { + "$ref": "./argo.vendors.schema.json#/$defs/instrument_vendorinfo" + } + }, + "required": [ + "@context", + "sensor_info", + "SENSORS", + "PARAMETERS" + ], + "uniqueItems": true, + "additionalProperties": false +} \ No newline at end of file diff --git a/argopy/static/assets/schema/argo.vendors.schema.json b/argopy/static/assets/schema/argo.vendors.schema.json new file mode 100644 index 000000000..b6d4579ba --- /dev/null +++ b/argopy/static/assets/schema/argo.vendors.schema.json @@ -0,0 +1,356 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "http://schema.nerc.ac.uk/schemas/sensor/0.1.0/argo.vendors.schema.json", + "title": "JSON Schema for Argo Program Sensors : Vendor-specific information", + "description": "A JSON Schema used to populate vendor-specific Argo float sensor and parameter metadata elements.", + "version": "0.1", + "type": "object", + "$defs": { + "sensor_vendorinfo": { + "type": "object", + "properties": { + "vendor_schema": { + "type": "string", + "enum": [ + "None", + "SBE", + "RBR", + "TRIOS" + ], + "default": "None" + }, + "version": { + "type": "string" + } + }, + "allOf": [ + { + "if": { + "properties": { + "vendor_schema": { + "const": "None" + } + } + }, + "then": { + "additionalProperties": { + "__comment__": { + "description": "sensor_vendorinfo is optional. When vendor schema is unspecified or None, content is uncontrolled", + "type": "string" + } + } + } + }, + { + "if": { + "properties": { + "vendor_schema": { + "const": "SBE" + } + }, + "required": [ + "vendor_schema" + ] + }, + "then": { + "$ref": "./argo.SBE.schema.json#/$defs/sensor_vendorinfo" + } + }, + { + "if": { + "properties": { + "vendor_schema": { + "const": "RBR" + } + }, + "required": [ + "vendor_schema" + ] + }, + "then": { + "$ref": "./argo.RBR.schema.json#/$defs/sensor_vendorinfo" + } + }, + { + "if": { + "properties": { + "vendor_schema": { + "const": "TRIOS" + } + }, + "required": [ + "vendor_schema" + ] + }, + "then": { + "$ref": "./argo.TRIOS.schema.json#/$defs/sensor_vendorinfo" + } + } + ] + }, + "parameter_vendorinfo": { + "type": "object", + "properties": { + "vendor_schema": { + "type": "string", + "enum": [ + "None", + "SBE", + "RBR", + "TRIOS" + ], + "default": "None" + }, + "version": { + "type": "string" + } + }, + "allOf": [ + { + "if": { + "properties": { + "vendor_schema": { + "const": "None" + } + } + }, + "then": { + "additionalProperties": { + "__comment__": { + "description": "parameter_vendorinfo is optional. When vendor schema is unspecified or None, content is uncontrolled", + "type": "string" + } + } + } + }, + { + "if": { + "properties": { + "vendor_schema": { + "const": "SBE" + } + }, + "required": [ + "vendor_schema" + ] + }, + "then": { + "$ref": "./argo.SBE.schema.json#/$defs/parameter_vendorinfo" + } + }, + { + "if": { + "properties": { + "vendor_schema": { + "const": "RBR" + } + }, + "required": [ + "vendor_schema" + ] + }, + "then": { + "$ref": "./argo.RBR.schema.json#/$defs/parameter_vendorinfo" + } + }, + { + "if": { + "properties": { + "vendor_schema": { + "const": "TRIOS" + } + }, + "required": [ + "vendor_schema" + ] + }, + "then": { + "$ref": "./argo.TRIOS.schema.json#/$defs/parameter_vendorinfo" + } + } + ] + }, + "predeployment_vendorinfo": { + "type": "object", + "properties": { + "vendor_schema": { + "type": "string", + "enum": [ + "None", + "RBR", + "SBE" + ], + "default": "None" + }, + "version": { + "type": "string" + } + }, + "allOf": [ + { + "if": { + "properties": { + "vendor_schema": { + "const": "None" + } + } + }, + "then": { + "additionalProperties": { + "__comment__": { + "description": "predeployment_vendorinfo is optional. When vendor schema is unspecified or None, content is uncontrolled", + "type": "string" + } + } + } + }, + { + "if": { + "properties": { + "vendor_schema": { + "const": "RBR" + } + }, + "required": [ + "vendor_schema" + ] + }, + "then": { + "$ref": "./argo.RBR.schema.json#/$defs/predeployment_vendorinfo" + } + }, + { + "if": { + "properties": { + "vendor_schema": { + "const": "SBE" + } + }, + "required": [ + "vendor_schema" + ] + }, + "then": { + "$ref": "./argo.SBE.schema.json#/$defs/predeployment_vendorinfo" + } + } + ] + }, + "instrument_vendorinfo": { + "type": "object", + "properties": { + "vendor_schema": { + "type": "string", + "enum": [ + "None", + "SBE" + ], + "default": "None" + }, + "version": { + "type": "string" + } + }, + "allOf": [ + { + "if": { + "properties": { + "vendor_schema": { + "const": "None" + } + } + }, + "then": { + "additionalProperties": { + "__comment__": { + "description": "instrument_vendorinfo is optional. When vendor_schema is unspecified or None, content is uncontrolled", + "type": "string" + } + } + } + }, + { + "if": { + "properties": { + "vendor_schema": { + "const": "SBE" + } + }, + "required": [ + "vendor_schema" + ] + }, + "then": { + "$ref": "./argo.SBE.schema.json#/$defs/instrument_vendorinfo" + } + } + ] + }, + "platform_vendorinfo": { + "type": "object", + "properties": { + "vendor_schema": { + "type": "string", + "enum": [ + "None", + "SBE", + "MRV" + ], + "default": "None" + }, + "version": { + "type": "string" + } + }, + "allOf": [ + { + "if": { + "properties": { + "vendor_schema": { + "const": "None" + } + } + }, + "then": { + "additionalProperties": { + "__comment__": { + "description": "instrument_vendorinfo is optional. When vendor_schema is unspecified or None, content is uncontrolled", + "type": "string" + } + } + } + }, + { + "if": { + "properties": { + "vendor_schema": { + "const": "SBE" + } + }, + "required": [ + "vendor_schema" + ] + }, + "then": { + "$ref": "./argo.SBE.schema.json#/$defs/platform_vendorinfo" + } + }, + { + "if": { + "properties": { + "vendor_schema": { + "const": "MRV" + } + }, + "required": [ + "vendor_schema" + ] + }, + "then": { + "$ref": "./argo.MRV.schema.json#/$defs/platform_vendorinfo" + } + } + ] + } + } +} \ No newline at end of file diff --git a/argopy/static/assets/sensor_metadata_examples.json b/argopy/static/assets/sensor_metadata_examples.json new file mode 100644 index 000000000..1f22020f9 --- /dev/null +++ b/argopy/static/assets/sensor_metadata_examples.json @@ -0,0 +1,53 @@ +{ + "name": "Sensor json meta-data examples", + "long_name": "Examples of Argo sensor meta-data following a schema", + "last_update": "2025-10-14T19:59:01.825972+00:00", + "data": { + "list": [ + "AANDERAA-AANDERAA_OPTODE_4330-3901", + "RBR-RBR_ARGO3-205908", + "SATLANTIC-SATLANTIC_OCR504_ICSW-42139", + "SATLANTIC-SATLANTIC_OCR504_ICSW-42442", + "SATLANTIC-SUNA_V2-1527", + "SBE-SBE41CP-11643", + "SBE-SBE63_OPTODE-2739", + "SBE-SEAFET-11341", + "SBE-SEAFET-17682", + "SBE-SEAFET-17683", + "SBE-SEAFET-18950", + "TRIOS-RAMSES_ACC_VIS-01600040", + "WETLABS-ECO_FLBBAP2-8589", + "WETLABS-ECO_FLBBBBRT2K-8635", + "WETLABS-ECO_FLBBCD-3666", + "WETLABS-ECO_FLBBCDRT2K-6666", + "WETLABS-ECO_FLBBCDRT2K-8690", + "WETLABS-ECO_FLBBFLRT2K-8660", + "WETLABS-ECO_FLBBRT2K-8462", + "WETLABS-MCOMS_FLBBCD-0157", + "WETLABS-MCOMS_MCOMS_FLBBCD-0498" + ], + "uri": { + "AANDERAA-AANDERAA_OPTODE_4330-3901": "https://raw.githubusercontent.com/euroargodev/sensor_metadata_json/refs/heads/main/json_sensors/sensor-AANDERAA-AANDERAA_OPTODE_4330-3901.json", + "RBR-RBR_ARGO3-205908": "https://raw.githubusercontent.com/euroargodev/sensor_metadata_json/refs/heads/main/json_sensors/sensor-RBR-RBR_ARGO3-205908.json", + "SATLANTIC-SATLANTIC_OCR504_ICSW-42139": "https://raw.githubusercontent.com/euroargodev/sensor_metadata_json/refs/heads/main/json_sensors/sensor-SATLANTIC-SATLANTIC_OCR504_ICSW-42139.json", + "SATLANTIC-SATLANTIC_OCR504_ICSW-42442": "https://raw.githubusercontent.com/euroargodev/sensor_metadata_json/refs/heads/main/json_sensors/sensor-SATLANTIC-SATLANTIC_OCR504_ICSW-42442.json", + "SATLANTIC-SUNA_V2-1527": "https://raw.githubusercontent.com/euroargodev/sensor_metadata_json/refs/heads/main/json_sensors/sensor-SATLANTIC-SUNA_V2-1527.json", + "SBE-SBE41CP-11643": "https://raw.githubusercontent.com/euroargodev/sensor_metadata_json/refs/heads/main/json_sensors/sensor-SBE-SBE41CP-11643.json", + "SBE-SBE63_OPTODE-2739": "https://raw.githubusercontent.com/euroargodev/sensor_metadata_json/refs/heads/main/json_sensors/sensor-SBE-SBE63_OPTODE-2739.json", + "SBE-SEAFET-11341": "https://raw.githubusercontent.com/euroargodev/sensor_metadata_json/refs/heads/main/json_sensors/sensor-SBE-SEAFET-11341.json", + "SBE-SEAFET-17682": "https://raw.githubusercontent.com/euroargodev/sensor_metadata_json/refs/heads/main/json_sensors/sensor-SBE-SEAFET-17682.json", + "SBE-SEAFET-17683": "https://raw.githubusercontent.com/euroargodev/sensor_metadata_json/refs/heads/main/json_sensors/sensor-SBE-SEAFET-17683.json", + "SBE-SEAFET-18950": "https://raw.githubusercontent.com/euroargodev/sensor_metadata_json/refs/heads/main/json_sensors/sensor-SBE-SEAFET-18950.json", + "TRIOS-RAMSES_ACC_VIS-01600040": "https://raw.githubusercontent.com/euroargodev/sensor_metadata_json/refs/heads/main/json_sensors/sensor-TRIOS-RAMSES_ACC_VIS-01600040.json", + "WETLABS-ECO_FLBBAP2-8589": "https://raw.githubusercontent.com/euroargodev/sensor_metadata_json/refs/heads/main/json_sensors/sensor-WETLABS-ECO_FLBBAP2-8589.json", + "WETLABS-ECO_FLBBBBRT2K-8635": "https://raw.githubusercontent.com/euroargodev/sensor_metadata_json/refs/heads/main/json_sensors/sensor-WETLABS-ECO_FLBBBBRT2K-8635.json", + "WETLABS-ECO_FLBBCD-3666": "https://raw.githubusercontent.com/euroargodev/sensor_metadata_json/refs/heads/main/json_sensors/sensor-WETLABS-ECO_FLBBCD-3666.json", + "WETLABS-ECO_FLBBCDRT2K-6666": "https://raw.githubusercontent.com/euroargodev/sensor_metadata_json/refs/heads/main/json_sensors/sensor-WETLABS-ECO_FLBBCDRT2K-6666.json", + "WETLABS-ECO_FLBBCDRT2K-8690": "https://raw.githubusercontent.com/euroargodev/sensor_metadata_json/refs/heads/main/json_sensors/sensor-WETLABS-ECO_FLBBCDRT2K-8690.json", + "WETLABS-ECO_FLBBFLRT2K-8660": "https://raw.githubusercontent.com/euroargodev/sensor_metadata_json/refs/heads/main/json_sensors/sensor-WETLABS-ECO_FLBBFLRT2K-8660.json", + "WETLABS-ECO_FLBBRT2K-8462": "https://raw.githubusercontent.com/euroargodev/sensor_metadata_json/refs/heads/main/json_sensors/sensor-WETLABS-ECO_FLBBRT2K-8462.json", + "WETLABS-MCOMS_FLBBCD-0157": "https://raw.githubusercontent.com/euroargodev/sensor_metadata_json/refs/heads/main/json_sensors/sensor-WETLABS-MCOMS_FLBBCD-0157.json", + "WETLABS-MCOMS_MCOMS_FLBBCD-0498": "https://raw.githubusercontent.com/euroargodev/sensor_metadata_json/refs/heads/main/json_sensors/sensor-WETLABS-MCOMS_MCOMS_FLBBCD-0498.json" + } + } +} \ No newline at end of file From 14286410ec2dea855924a27602f73d233db9ed75 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Tue, 14 Oct 2025 23:09:58 +0200 Subject: [PATCH 39/71] Fall back on static assets version for schema --- argopy/related/sensors/oem_metadata.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/argopy/related/sensors/oem_metadata.py b/argopy/related/sensors/oem_metadata.py index eccbf0fc9..b4d64b6fc 100644 --- a/argopy/related/sensors/oem_metadata.py +++ b/argopy/related/sensors/oem_metadata.py @@ -7,7 +7,7 @@ import json import logging import warnings -from fsspec import filesystem +import pandas as pd from ...stores import httpstore, filestore from ...options import OPTIONS @@ -287,10 +287,21 @@ def _ipython_display_(self): display("\n".join(self._empty_str())) def _read_schema(self, ref="argo.sensor.schema.json") -> Dict[str, Any]: - """Load a JSON schema for validation.""" - # todo: implement static asset backup to load schema offline + """Load a JSON schema for validation + + Fall back on static version if online resource not available + """ uri = f"{self._schema_root}/{ref}" - schema = self._fs.open_json(uri) + try: + schema = self._fs.open_json(uri) + except: + # Fall back on static assets version + local_uri = Path(path2assets).joinpath("schema").joinpath(ref) + fs = filestore() + updated = pd.Timestamp(fs.info(local_uri)['mtime'], unit='s') + warnings.warn(f"\nCan't get '{ref}' schema from the official online resource ({uri}).\nFall back on a static version packaged with this release at {updated}.") + schema = fs.open_json(local_uri) + return schema def validate(self, data): @@ -311,7 +322,7 @@ def validate(self, data): if self._validation_error == "raise": raise error elif self._validation_error == "warn": - warnings.warn(str(error)) + warnings.warn(f"\nJSON schema validation error: {str(error)}") else: log.error(error) From d6ee9997b3db4b86544b515e63f06c5e45d6e3ca Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Mon, 20 Oct 2025 10:02:08 +0200 Subject: [PATCH 40/71] update docstrings and formating --- argopy/related/reference_tables.py | 2 +- argopy/related/sensors/oem_metadata.py | 110 +++++++++++++++++-------- argopy/related/sensors/sensors.py | 6 +- 3 files changed, 79 insertions(+), 39 deletions(-) diff --git a/argopy/related/reference_tables.py b/argopy/related/reference_tables.py index eddb69426..10f4a1d44 100644 --- a/argopy/related/reference_tables.py +++ b/argopy/related/reference_tables.py @@ -280,7 +280,7 @@ def _valid_ref(self, rtid): rtid = "R%0.2d" % rtid if rtid not in self.valid_ref: raise ValueError( - "Invalid Argo Reference Table, should be one in: %s" + "Invalid Argo Reference Table, must be one in: %s" % ", ".join(self.valid_ref) ) return rtid diff --git a/argopy/related/sensors/oem_metadata.py b/argopy/related/sensors/oem_metadata.py index b4d64b6fc..b38bd5388 100644 --- a/argopy/related/sensors/oem_metadata.py +++ b/argopy/related/sensors/oem_metadata.py @@ -1,6 +1,6 @@ from dataclasses import dataclass, field from typing import List, Dict, Optional, Any, Literal -from pathlib import Path +from pathlib import Path from zipfile import ZipFile from referencing import Registry, Resource import jsonschema @@ -18,7 +18,10 @@ log = logging.getLogger("argopy.related.sensors") -SENSOR_JS_EXAMPLES = filestore().open_json(Path(path2assets).joinpath("sensor_metadata_examples.json"))['data']['uri'] +SENSOR_JS_EXAMPLES = filestore().open_json( + Path(path2assets).joinpath("sensor_metadata_examples.json") +)["data"]["uri"] + @dataclass class SensorInfo: @@ -72,10 +75,10 @@ def _attr2str(self, x): """Return a class attribute, or 'n/a' if it's None, {} or "".""" value = getattr(self, x, None) if type(value) is str: - return value if value and value.strip() else 'n/a' + return value if value and value.strip() else "n/a" elif type(value) is dict: if len(value.keys()) == 0: - return 'n/a' + return "n/a" else: return value else: @@ -86,16 +89,20 @@ def __repr__(self): def key2str(d, x): """Return a dict value as a string, or 'n/a' if it's None or empty.""" value = d.get(x, None) - return value if value and value.strip() else 'n/a' + return value if value and value.strip() else "n/a" summary = [f""] summary.append(f" SENSOR: {self.SENSOR} ({self.SENSOR_uri})") summary.append(f" SENSOR_MAKER: {self.SENSOR_MAKER} ({self.SENSOR_MAKER_uri})") summary.append(f" SENSOR_MODEL: {self.SENSOR_MODEL} ({self.SENSOR_MODEL_uri})") if getattr(self, "SENSOR_MODEL_FIRMWARE", None) is None: - summary.append(f" SENSOR_FIRMWARE_VERSION: {self._attr2str('SENSOR_FIRMWARE_VERSION')} (but should be 'SENSOR_MODEL_FIRMWARE') ") + summary.append( + f" SENSOR_FIRMWARE_VERSION: {self._attr2str('SENSOR_FIRMWARE_VERSION')} (but should be 'SENSOR_MODEL_FIRMWARE') " + ) else: - summary.append(f" SENSOR_MODEL_FIRMWARE: {self._attr2str('SENSOR_MODEL_FIRMWARE')}") + summary.append( + f" SENSOR_MODEL_FIRMWARE: {self._attr2str('SENSOR_MODEL_FIRMWARE')}" + ) summary.append(f" sensor_vendorinfo:") for key in self.sensor_vendorinfo.keys(): summary.append(f" - {key}: {key2str(self.sensor_vendorinfo, key)}") @@ -130,10 +137,10 @@ def _attr2str(self, x): """Return a class attribute, or 'n/a' if it's None, {} or "".""" value = getattr(self, x, None) if type(value) is str: - return value if value and value.strip() else 'n/a' + return value if value and value.strip() else "n/a" elif type(value) is dict: if len(value.keys()) == 0: - return 'n/a' + return "n/a" else: return value else: @@ -141,29 +148,38 @@ def _attr2str(self, x): @property def _has_calibration_data(self): - s = "".join([str(self._attr2str(key)) for key in - ['PREDEPLOYMENT_CALIB_EQUATION', - 'PREDEPLOYMENT_CALIB_COEFFICIENT_LIST', - 'PREDEPLOYMENT_CALIB_COMMENT', - 'PREDEPLOYMENT_CALIB_DATE'] if self._attr2str(key) != 'n/a']) + s = "".join( + [ + str(self._attr2str(key)) + for key in [ + "PREDEPLOYMENT_CALIB_EQUATION", + "PREDEPLOYMENT_CALIB_COEFFICIENT_LIST", + "PREDEPLOYMENT_CALIB_COMMENT", + "PREDEPLOYMENT_CALIB_DATE", + ] + if self._attr2str(key) != "n/a" + ] + ) return len(s) > 0 def __repr__(self): summary = [f""] summary.append(f" PARAMETER: {self.PARAMETER} ({self.PARAMETER_uri})") - summary.append(f" PARAMETER_SENSOR: {self.PARAMETER_SENSOR} ({self.PARAMETER_SENSOR_uri})") + summary.append( + f" PARAMETER_SENSOR: {self.PARAMETER_SENSOR} ({self.PARAMETER_SENSOR_uri})" + ) - for key in ['UNITS', 'ACCURACY', 'RESOLUTION']: + for key in ["UNITS", "ACCURACY", "RESOLUTION"]: p = f"PARAMETER_{key}" summary.append(f" {key}: {self._attr2str(p)}") summary.append(f" PREDEPLOYMENT CALIBRATION:") - for key in ['EQUATION', 'COEFFICIENT', 'COMMENT', 'DATE']: + for key in ["EQUATION", "COEFFICIENT", "COMMENT", "DATE"]: p = f"PREDEPLOYMENT_CALIB_{key}" summary.append(f" - {key}: {self._attr2str(p)}") - for key in ['parameter_vendorinfo', 'predeployment_vendorinfo']: + for key in ["parameter_vendorinfo", "predeployment_vendorinfo"]: summary.append(f" {key}: {self._attr2str(p)}") return "\n".join(summary) @@ -172,6 +188,7 @@ def _repr_html_(self): def _ipython_display_(self): from IPython.display import display, HTML + display(HTML(ParameterDisplay(self).html)) @@ -201,6 +218,7 @@ class ArgoSensorMetaDataOem: ArgoSensorMetaData().from_dict(jsdata) # Use any compliant json data """ + _schema_root = "https://raw.githubusercontent.com/euroargodev/sensor_metadata_json/refs/heads/main/schemas" """URI root to argo JSON schema""" @@ -241,7 +259,9 @@ def __init__( def _empty_str(self): summary = [f""] - summary.append("This object has no sensor info. You can use one of the following methods:") + summary.append( + "This object has no sensor info. You can use one of the following methods:" + ) for meth in [ "from_rbr(serial_number)", "from_dict(dict_or_json_data)", @@ -256,15 +276,21 @@ def __repr__(self): self.sensor_info.sensor_described if self.sensor_info else "n/a" ) created_by = self.sensor_info.created_by if self.sensor_info else "n/a" - date_creation = self.sensor_info.date_creation if self.sensor_info else "n/a" + date_creation = ( + self.sensor_info.date_creation if self.sensor_info else "n/a" + ) sensor_count = len(self.sensors) if self.sensor_info else 0 parameter_count = len(self.parameters) if self.sensor_info else 0 summary = [f""] summary.append(f"created_by: '{created_by}'") summary.append(f"date_creation: '{date_creation}'") - summary.append(f"sensors: {sensor_count} {[urnparser(s.SENSOR)['termid'] for s in self.sensors]}") - summary.append(f"parameters: {parameter_count} {[urnparser(s.PARAMETER)['termid'] for s in self.parameters]}") + summary.append( + f"sensors: {sensor_count} {[urnparser(s.SENSOR)['termid'] for s in self.sensors]}" + ) + summary.append( + f"parameters: {parameter_count} {[urnparser(s.PARAMETER)['termid'] for s in self.parameters]}" + ) summary.append(f"instrument_vendorinfo: {self.instrument_vendorinfo}") else: @@ -298,8 +324,10 @@ def _read_schema(self, ref="argo.sensor.schema.json") -> Dict[str, Any]: # Fall back on static assets version local_uri = Path(path2assets).joinpath("schema").joinpath(ref) fs = filestore() - updated = pd.Timestamp(fs.info(local_uri)['mtime'], unit='s') - warnings.warn(f"\nCan't get '{ref}' schema from the official online resource ({uri}).\nFall back on a static version packaged with this release at {updated}.") + updated = pd.Timestamp(fs.info(local_uri)["mtime"], unit="s") + warnings.warn( + f"\nCan't get '{ref}' schema from the official online resource ({uri}).\nFall back on a static version packaged with this release at {updated}." + ) schema = fs.open_json(local_uri) return schema @@ -307,7 +335,9 @@ def _read_schema(self, ref="argo.sensor.schema.json") -> Dict[str, Any]: def validate(self, data): """Validate meta-data against the Argo sensor json schema""" # Set a method to resolve references to subschemas - registry = Registry(retrieve=lambda x: Resource.from_contents(self._read_schema(x))) + registry = Registry( + retrieve=lambda x: Resource.from_contents(self._read_schema(x)) + ) # Select the validator based on $schema property in schema # (validators correspond to various drafts of JSON Schema) @@ -425,9 +455,11 @@ def from_rbr(self, serial_number: str, **kwargs): self._serial_number = serial_number # Ensure that the instance httpstore has the appropriate authorization key: - fss = self._fs.fs.fs if getattr(self._fs, 'cache') else self._fs.fs + fss = self._fs.fs.fs if getattr(self._fs, "cache") else self._fs.fs headers = fss.client_kwargs.get("headers", {}) - headers.update({"Authorization": kwargs.get("rbr_api_key", OPTIONS["rbr_api_key"])}) + headers.update( + {"Authorization": kwargs.get("rbr_api_key", OPTIONS["rbr_api_key"])} + ) fss._session = None # Reset fsspec aiohttp.ClientSession uri = f"{OPTIONS['rbr_api']}/instruments/{self._serial_number}/argometadatajson" @@ -435,21 +467,23 @@ def from_rbr(self, serial_number: str, **kwargs): obj = self.from_dict(data) # Download RBR zip archive with calibration certificates in PDFs: - obj = obj.certificates_rbr(action='download', quiet=True) + obj = obj.certificates_rbr(action="download", quiet=True) return obj - def certificates_rbr(self, action: Literal["download", "open"] = "download", **kwargs): + def certificates_rbr( + self, action: Literal["download", "open"] = "download", **kwargs + ): """Download RBR zip archive with calibration certificates in PDFs Certificate PDF files are written to the OPTIONS['cachedir'] folder """ - cdir = Path(OPTIONS['cachedir']).joinpath("RBR_certificates") + cdir = Path(OPTIONS["cachedir"]).joinpath("RBR_certificates") cdir.mkdir(parents=True, exist_ok=True) local_zip_path = cdir.joinpath(f"RBRcertificates_{self._serial_number}.zip") lfs = filestore() - quiet = kwargs.get('quiet', False) + quiet = kwargs.get("quiet", False) # Check if we can continue: if self._serial_number is not None: @@ -459,7 +493,7 @@ def certificates_rbr(self, action: Literal["download", "open"] = "download", **k if not lfs.exists(local_zip_path): new = True certif_uri = f"{OPTIONS['rbr_api']}/instruments/{self._serial_number}/certificates" - with open(local_zip_path, 'wb') as local_zip: + with open(local_zip_path, "wb") as local_zip: with self._fs.open(certif_uri) as remote_zip: local_zip.write(remote_zip.read()) @@ -485,11 +519,13 @@ def certificates_rbr(self, action: Literal["download", "open"] = "download", **k s = f"One RBR certificate file already in: {f}" print(s) else: - raise InvalidDatasetStructure(f"You must load meta-data for a given RBR sensor serial number first. Use the 'from_rbr' method.") + raise InvalidDatasetStructure( + f"You must load meta-data for a given RBR sensor serial number first. Use the 'from_rbr' method." + ) - if action == 'download': + if action == "download": return self - elif action == 'open': + elif action == "open": subp = [] for f in self.local_certificates: subp.append(lfs.open_subprocess(str(f))) @@ -505,6 +541,8 @@ def list_examples(self): def from_examples(self, eg: str = None, **kwargs): if eg not in self.list_examples: - raise ValueError(f"Unknown sensor example: '{eg}'. \n Use one in: {self.list_examples}") + raise ValueError( + f"Unknown sensor example: '{eg}'. \n Use one in: {self.list_examples}" + ) data = self._fs.open_json(SENSOR_JS_EXAMPLES[eg]) return self.from_dict(data) diff --git a/argopy/related/sensors/sensors.py b/argopy/related/sensors/sensors.py index 802f0e8c0..67b8e720a 100644 --- a/argopy/related/sensors/sensors.py +++ b/argopy/related/sensors/sensors.py @@ -89,8 +89,10 @@ def __contains__(self, string) -> bool: return string.lower() in self.name.lower() or string.lower() in self.long_name.lower() class ArgoSensor: - """ - Argo sensors helper class + """Argo sensor 'package' helper class + + A :class:`ArgoSensor` class instance shall represent one float sensor 'package' + The :class:`ArgoSensor` class aims to provide direct access to Argo's sensor metadata from: From 71282ac7919cb100d92e3df303c9d41d8963c695 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Mon, 20 Oct 2025 10:03:41 +0200 Subject: [PATCH 41/71] Update accessories.py NVSrow directly from ref table row lookup --- argopy/utils/accessories.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/argopy/utils/accessories.py b/argopy/utils/accessories.py index 511c0df8f..578a31700 100644 --- a/argopy/utils/accessories.py +++ b/argopy/utils/accessories.py @@ -8,6 +8,7 @@ import pandas as pd from .checkers import check_wmo, is_wmo +from .format import urnparser log = logging.getLogger("argopy.utils.accessories") @@ -265,6 +266,8 @@ def copy(self): class NVSrow: """This proto makes it easier to work with a single NVS table row from a :class:`pd.DataFrame` + It will turn :class:`pd.DataFrame` columns into class attributes + Examples -------- .. code-block:: python @@ -303,7 +306,10 @@ def from_series(obj: pd.Series) -> "SensorType": """From 'definition' column""" uri: str = "" - """From 'ID' column""" + """From 'ID' column, typically a link toward NVS specific row entry""" + + urn: str = "" + """From 'urn' column""" deprecated: bool = None """From 'deprecated' column""" @@ -320,18 +326,25 @@ def __init__(self, row: pd.Series | pd.DataFrame): self.definition = row["definition"] self.deprecated = row["deprecated"] self.uri = row["id"] + self.urn = row["urn"] @staticmethod def from_series(obj: pd.Series) -> "NVSrow": return NVSrow(obj) + @staticmethod + def from_df(df: pd.DataFrame, txt: str, column: str = 'altLabel') -> "NVSrow": + row = df[df[column].apply(lambda x: str(x) == str(txt))].iloc[0] + return NVSrow(row) + def __eq__(self, obj): return self.name == obj def __repr__(self): - summary = [f"<{self.reftable}.row>"] + summary = [f"<{getattr(self, 'reftable', 'n/a')}.{self.urn}>"] summary.append(f"%12s: {self.name}" % "name") summary.append(f"%12s: {self.long_name}" % "long_name") + summary.append(f"%12s: {self.urn}" % "urn") summary.append(f"%12s: {self.uri}" % "uri") summary.append(f"%12s: {self.deprecated}" % "deprecated") summary.append("%12s: %s" % ("definition", self.definition)) From e723fc4b601055cb5c685b9260d64a697c3d7c01 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Mon, 20 Oct 2025 11:39:13 +0200 Subject: [PATCH 42/71] Update data_types.json --- argopy/static/assets/data_types.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/argopy/static/assets/data_types.json b/argopy/static/assets/data_types.json index e775e2335..834b37c27 100644 --- a/argopy/static/assets/data_types.json +++ b/argopy/static/assets/data_types.json @@ -1,7 +1,7 @@ { "name": "data_types", "long_name": "Expected data types of Argo variables", - "last_update": "2025-08-20T06:50:55.965456+00:00", + "last_update": "2025-10-19T19:06:04.934857+00:00", "data": { "str": [ "ANOMALY", @@ -94,6 +94,7 @@ "NB_SAMPLE_MCOMS_DATA_MODE", "NB_SAMPLE_OCR_DATA_MODE", "NB_SAMPLE_OPTODE_DATA_MODE", + "NB_SAMPLE_SFET_DATA_MODE", "NB_SAMPLE_STM_DATA_MODE", "NB_SAMPLE_SUNA_DATA_MODE", "NITRATE_DATA_MODE", @@ -170,6 +171,7 @@ "PROFILE_NB_SAMPLE_OCR_QC", "PROFILE_NB_SAMPLE_OPTODE_QC", "PROFILE_NB_SAMPLE_QC", + "PROFILE_NB_SAMPLE_SFET_QC", "PROFILE_NB_SAMPLE_STM_QC", "PROFILE_NB_SAMPLE_SUNA_QC", "PROFILE_NITRATE_QC", @@ -287,6 +289,8 @@ "int": [ "CONFIG_MISSION_NUMBER", "CYCLE_NUMBER", + "CYCLE_NUMBER_INDEX", + "CYCLE_NUMBER_INDEX_ADJUSTED", "JULD_ADJUSTED_STATUS", "JULD_ASCENT_END_STATUS", "JULD_ASCENT_START_STATUS", From 10985f01693e6412e38071c0c43d069e7d0637cb Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Thu, 23 Oct 2025 21:53:53 +0200 Subject: [PATCH 43/71] Update oem_metadata.py --- argopy/related/sensors/oem_metadata.py | 63 ++++++++++++++++++-------- 1 file changed, 44 insertions(+), 19 deletions(-) diff --git a/argopy/related/sensors/oem_metadata.py b/argopy/related/sensors/oem_metadata.py index b38bd5388..186cdc11a 100644 --- a/argopy/related/sensors/oem_metadata.py +++ b/argopy/related/sensors/oem_metadata.py @@ -74,7 +74,9 @@ def SENSOR_MODEL_uri(self): def _attr2str(self, x): """Return a class attribute, or 'n/a' if it's None, {} or "".""" value = getattr(self, x, None) - if type(value) is str: + if value is None: + return "n/a" + elif type(value) is str: return value if value and value.strip() else "n/a" elif type(value) is dict: if len(value.keys()) == 0: @@ -91,7 +93,7 @@ def key2str(d, x): value = d.get(x, None) return value if value and value.strip() else "n/a" - summary = [f""] + summary = [f"<{self.SENSOR}><{self.SENSOR_SERIAL_NO}>"] summary.append(f" SENSOR: {self.SENSOR} ({self.SENSOR_uri})") summary.append(f" SENSOR_MAKER: {self.SENSOR_MAKER} ({self.SENSOR_MAKER_uri})") summary.append(f" SENSOR_MODEL: {self.SENSOR_MODEL} ({self.SENSOR_MODEL_uri})") @@ -103,9 +105,12 @@ def key2str(d, x): summary.append( f" SENSOR_MODEL_FIRMWARE: {self._attr2str('SENSOR_MODEL_FIRMWARE')}" ) - summary.append(f" sensor_vendorinfo:") - for key in self.sensor_vendorinfo.keys(): - summary.append(f" - {key}: {key2str(self.sensor_vendorinfo, key)}") + if getattr(self, "sensor_vendorinfo", None) is not None: + summary.append(f" sensor_vendorinfo:") + for key in self.sensor_vendorinfo.keys(): + summary.append(f" - {key}: {key2str(self.sensor_vendorinfo, key)}") + else: + summary.append(f" sensor_vendorinfo: None") return "\n".join(summary) @@ -113,13 +118,16 @@ def key2str(d, x): class Parameter: PARAMETER: str # SDN:R03::PRES PARAMETER_SENSOR: str # SDN:R25::CTD_PRES - PARAMETER_UNITS: str PARAMETER_ACCURACY: str PARAMETER_RESOLUTION: str - PREDEPLOYMENT_CALIB_EQUATION: str PREDEPLOYMENT_CALIB_COEFFICIENT_LIST: Dict[str, str] - PREDEPLOYMENT_CALIB_COMMENT: str - PREDEPLOYMENT_CALIB_DATE: str + + # Made optional to accommodate errors in OEM data + PARAMETER_UNITS: str = None + PREDEPLOYMENT_CALIB_EQUATION: str = None + PREDEPLOYMENT_CALIB_COMMENT: str = None + PREDEPLOYMENT_CALIB_DATE: str = None + parameter_vendorinfo: Optional[Dict[str, Any]] = None predeployment_vendorinfo: Optional[Dict[str, Any]] = None @@ -136,7 +144,9 @@ def PARAMETER_SENSOR_uri(self): def _attr2str(self, x): """Return a class attribute, or 'n/a' if it's None, {} or "".""" value = getattr(self, x, None) - if type(value) is str: + if value is None: + return "n/a" + elif type(value) is str: return value if value and value.strip() else "n/a" elif type(value) is dict: if len(value.keys()) == 0: @@ -164,7 +174,7 @@ def _has_calibration_data(self): def __repr__(self): - summary = [f""] + summary = [f"<{self.PARAMETER}>"] summary.append(f" PARAMETER: {self.PARAMETER} ({self.PARAMETER_uri})") summary.append( f" PARAMETER_SENSOR: {self.PARAMETER_SENSOR} ({self.PARAMETER_SENSOR_uri})" @@ -180,7 +190,10 @@ def __repr__(self): summary.append(f" - {key}: {self._attr2str(p)}") for key in ["parameter_vendorinfo", "predeployment_vendorinfo"]: - summary.append(f" {key}: {self._attr2str(p)}") + if getattr(self, key, None) is not None: + summary.append(f" {key}: {self._attr2str(p)}") + else: + summary.append(f" {key}: None") return "\n".join(summary) def _repr_html_(self): @@ -374,7 +387,7 @@ def from_dict(self, data: Dict[str, Any]): ) self.sensors = [Sensor(**sensor) for sensor in data["SENSORS"]] self.parameters = [Parameter(**param) for param in data["PARAMETERS"]] - self.instrument_vendorinfo = data.get("instrument_vendorinfo") + self.instrument_vendorinfo = data.get("instrument_vendorinfo", None) return self @@ -429,20 +442,22 @@ def to_dict(self) -> Dict[str, Any]: } def to_json_file(self, file_path: str) -> None: - """Save meta-data to a JSON file + """Save meta-data to a JSON file - in dev. Notes ----- - The output json file is compliant with the Argo sensor meta-data JSON schema :attr:`ArgoSensorMetaDataOem.schema` + The output json file should be compliant with the Argo sensor meta-data JSON schema :attr:`ArgoSensorMetaDataOem.schema` """ with open(file_path, "w") as f: json.dump(self.to_dict(), f, indent=2) def from_rbr(self, serial_number: str, **kwargs): - """Fetch sensor metadata from RBR API + """Fetch sensor metadata from RBR-GLOBAL API We also download certificates if available + TODO: Check mark if the sensor is ok with dynamic correction or not + Parameters ---------- serial_number : str @@ -466,18 +481,28 @@ def from_rbr(self, serial_number: str, **kwargs): data = self._fs.open_json(uri) obj = self.from_dict(data) - # Download RBR zip archive with calibration certificates in PDFs: - obj = obj.certificates_rbr(action="download", quiet=True) + # Also download RBR zip archive with calibration certificates in PDFs: + obj = obj.__certificates_rbr(action="download", quiet=True) + + # Finally reset httpstore parameters: + headers = fss.client_kwargs.get("headers") + headers.pop("Authorization", None) + fss._session = None # Reset fsspec aiohttp.ClientSession return obj - def certificates_rbr( + def __certificates_rbr( self, action: Literal["download", "open"] = "download", **kwargs ): """Download RBR zip archive with calibration certificates in PDFs Certificate PDF files are written to the OPTIONS['cachedir'] folder + Notes + ----- + We keep this method very private because it is expected to be called only by the self.from_rbr() method. + This ensures that the httpstore has the appropriate authorization key. + """ cdir = Path(OPTIONS["cachedir"]).joinpath("RBR_certificates") cdir.mkdir(parents=True, exist_ok=True) From 0f6c10e1f68b77a58dfd853c5bdef54ff8a47078 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Thu, 23 Oct 2025 21:53:57 +0200 Subject: [PATCH 44/71] Update oem_metadata_repr.py --- argopy/related/sensors/oem_metadata_repr.py | 27 +++++++++++++-------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/argopy/related/sensors/oem_metadata_repr.py b/argopy/related/sensors/oem_metadata_repr.py index 68ceb38fd..c18230e9b 100644 --- a/argopy/related/sensors/oem_metadata_repr.py +++ b/argopy/related/sensors/oem_metadata_repr.py @@ -112,16 +112,23 @@ def html(self): """ for ip, param in enumerate(self.OEMsensor.parameters): - PREDEPLOYMENT_CALIB_EQUATION = param._attr2str('PREDEPLOYMENT_CALIB_EQUATION').split(';') - PREDEPLOYMENT_CALIB_EQUATION = [p.replace("=", " = ") for p in PREDEPLOYMENT_CALIB_EQUATION] - PREDEPLOYMENT_CALIB_EQUATION = "
\t".join(PREDEPLOYMENT_CALIB_EQUATION) - - PREDEPLOYMENT_CALIB_COEFFICIENT_LIST = param._attr2str('PREDEPLOYMENT_CALIB_COEFFICIENT_LIST') - s = [] - if isinstance(PREDEPLOYMENT_CALIB_COEFFICIENT_LIST, dict): - for key, value in PREDEPLOYMENT_CALIB_COEFFICIENT_LIST.items(): - s.append(f"{key} = {value}") - PREDEPLOYMENT_CALIB_COEFFICIENT_LIST = "
\t".join(s) + if getattr(param, 'PREDEPLOYMENT_CALIB_EQUATION', None) is not None: + PREDEPLOYMENT_CALIB_EQUATION = param._attr2str('PREDEPLOYMENT_CALIB_EQUATION').split(';') + PREDEPLOYMENT_CALIB_EQUATION = [p.replace("=", " = ") for p in PREDEPLOYMENT_CALIB_EQUATION] + PREDEPLOYMENT_CALIB_EQUATION = "
\t".join(PREDEPLOYMENT_CALIB_EQUATION) + else: + PREDEPLOYMENT_CALIB_EQUATION = "This information is missing, but it should not !" + + if getattr(param, 'PREDEPLOYMENT_CALIB_COEFFICIENT_LIST', None) is not None: + PREDEPLOYMENT_CALIB_COEFFICIENT_LIST = param._attr2str('PREDEPLOYMENT_CALIB_COEFFICIENT_LIST') + s = [] + if isinstance(PREDEPLOYMENT_CALIB_COEFFICIENT_LIST, dict): + for key, value in PREDEPLOYMENT_CALIB_COEFFICIENT_LIST.items(): + s.append(f"{key} = {value}") + PREDEPLOYMENT_CALIB_COEFFICIENT_LIST = "
\t".join(s) + else: + PREDEPLOYMENT_CALIB_COEFFICIENT_LIST = "This information is missing, but it should not !" + details_html = f""" From b51526cbbd51e63990fb28089e0710b411962cd4 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Thu, 23 Oct 2025 21:54:02 +0200 Subject: [PATCH 45/71] Update format.py --- argopy/utils/format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/argopy/utils/format.py b/argopy/utils/format.py index b4c4f467c..cc1facb2f 100644 --- a/argopy/utils/format.py +++ b/argopy/utils/format.py @@ -408,4 +408,4 @@ def urnparser(urn): if len(pp) == 4 and pp[0] == 'SDN': return {'listid': pp[1], 'version': pp[2], 'termid': pp[3]} else: - raise ValueError("NVS URNs must follow the pattern: 'SDN:{listid}:{version}:{termid}' or 'SDN:{listid}::{termid}' for NVS2.0") + raise ValueError(f"This NVS URN '{urn}' does not follow the pattern: 'SDN:{listid}:{version}:{termid}' or 'SDN:{listid}::{termid}' for NVS2.0") From 42a52076f70a42630c07016a9539744638514254 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Thu, 23 Oct 2025 23:17:00 +0200 Subject: [PATCH 46/71] New SeaBird webAPI access for sensor metadata --- argopy/options.py | 17 ++++ argopy/related/sensors/oem_metadata.py | 93 +++++++++++++++------ argopy/related/sensors/oem_metadata_repr.py | 6 +- 3 files changed, 87 insertions(+), 29 deletions(-) diff --git a/argopy/options.py b/argopy/options.py index 2881ae53a..43bdd9598 100644 --- a/argopy/options.py +++ b/argopy/options.py @@ -54,6 +54,7 @@ RBR_API_KEY = "rbr_api_key" API_RBR = "rbr_api" NVS = "nvs" +API_SEABIRD = "seabird_api" # Define the list of available options and default values: OPTIONS = { @@ -77,6 +78,7 @@ RBR_API_KEY: os.environ.get("RBR_API_KEY"), # Contact RBR at argo@rbr-global.com if you do not have an authorization key. API_RBR: "https://oem-lookup.rbr-global.com/api/v1", NVS: "https://vocab.nerc.ac.uk/collection", + API_SEABIRD: "https://instrument.seabirdhub.com/api/argo-calibration", } DEFAULT = OPTIONS.copy() @@ -141,6 +143,14 @@ def validate_rbr(this_path): return False +def validate_seabird(this_path): + if this_path != "-": + return True # todo: check with Seabird how to get the API endpoint for its status + else: + log.debug("OPTIONS['%s'] is not defined" % API_RBR) + return False + + def PARALLEL_SETUP(parallel): parallel = VALIDATE("parallel", parallel) if isinstance(parallel, bool): @@ -291,6 +301,7 @@ def check_gdac_option( LON: lambda x: x in ['180', '360'], API_FLEETMONITORING: validate_fleetmonitoring, API_RBR: validate_rbr, + API_SEABIRD: validate_seabird, NVS: lambda x: isinstance(x, str) or x is None, } @@ -361,6 +372,12 @@ class set_options: This key can also be used from: https://oem-lookup.rbr-global.com + rbr_api: str + Server address of the RBR "RBRargo Product Lookup" web-API + + seabird_api: str + Server address of the Seabird-Scientific "Instrument Metadata Portal" web-API + parallel: bool, str, :class:`distributed.Client`, default: False Set whether to use parallelisation or not, and possibly which method to use. diff --git a/argopy/related/sensors/oem_metadata.py b/argopy/related/sensors/oem_metadata.py index 186cdc11a..31e87b0b3 100644 --- a/argopy/related/sensors/oem_metadata.py +++ b/argopy/related/sensors/oem_metadata.py @@ -30,8 +30,19 @@ class SensorInfo: link: str format_version: str contents: str - sensor_described: str + # Made optional to accommodate errors in OEM data + sensor_described: str = None + + def _attr2str(self, x): + """Return a class attribute, or 'n/a' if it's None or "".""" + value = getattr(self, x, None) + if value is None: + return "n/a" + elif type(value) is str: + return value if value and value.strip() else "n/a" + else: + return value @dataclass class Context: @@ -270,7 +281,7 @@ def __init__( if json_data: self.from_dict(json_data) - def _empty_str(self): + def _repr_hint(self): summary = [f""] summary.append( "This object has no sensor info. You can use one of the following methods:" @@ -285,19 +296,18 @@ def _empty_str(self): def __repr__(self): if self.sensor_info: - sensor_described = ( - self.sensor_info.sensor_described if self.sensor_info else "n/a" - ) - created_by = self.sensor_info.created_by if self.sensor_info else "n/a" - date_creation = ( - self.sensor_info.date_creation if self.sensor_info else "n/a" - ) - sensor_count = len(self.sensors) if self.sensor_info else 0 - parameter_count = len(self.parameters) if self.sensor_info else 0 + sensor_described = self.sensor_info._attr2str('sensor_described') + created_by = self.sensor_info._attr2str('created_by') + date_creation = self.sensor_info._attr2str('date_creation') + link = self.sensor_info._attr2str('link') + + sensor_count = len(self.sensors) + parameter_count = len(self.parameters) - summary = [f""] + summary = [f"<{sensor_described}>"] summary.append(f"created_by: '{created_by}'") summary.append(f"date_creation: '{date_creation}'") + summary.append(f"link: '{link}'") summary.append( f"sensors: {sensor_count} {[urnparser(s.SENSOR)['termid'] for s in self.sensors]}" ) @@ -307,7 +317,7 @@ def __repr__(self): summary.append(f"instrument_vendorinfo: {self.instrument_vendorinfo}") else: - summary = self._empty_str() + summary = self._repr_hint() return "\n".join(summary) @@ -315,7 +325,7 @@ def _repr_html_(self): if self.sensor_info: return OemMetaDataDisplay(self).html else: - return self._empty_str() + return self._repr_hint() def _ipython_display_(self): from IPython.display import display, HTML @@ -323,7 +333,7 @@ def _ipython_display_(self): if self.sensor_info: display(HTML(OemMetaDataDisplay(self).html)) else: - display("\n".join(self._empty_str())) + display("\n".join(self._repr_hint())) def _read_schema(self, ref="argo.sensor.schema.json") -> Dict[str, Any]: """Load a JSON schema for validation @@ -379,12 +389,15 @@ def from_dict(self, data: Dict[str, Any]): self.validate(data) self.sensor_info = SensorInfo(**data["sensor_info"]) - self.context = Context( - **{ - k.replace("::", "").replace(":", "_"): v - for k, v in data["@context"].items() - } - ) + if data.get("@context", None) is not None: + self.context = Context( + **{ + k.replace("::", "").replace(":", "_"): v + for k, v in data["@context"].items() + } + ) + else: + self.context = Context() self.sensors = [Sensor(**sensor) for sensor in data["SENSORS"]] self.parameters = [Parameter(**param) for param in data["PARAMETERS"]] self.instrument_vendorinfo = data.get("instrument_vendorinfo", None) @@ -451,8 +464,8 @@ def to_json_file(self, file_path: str) -> None: with open(file_path, "w") as f: json.dump(self.to_dict(), f, indent=2) - def from_rbr(self, serial_number: str, **kwargs): - """Fetch sensor metadata from RBR-GLOBAL API + def from_rbr(self, serial_number: str, **kwargs) -> 'ArgoSensorMetaDataOem': + """Fetch sensor metadata from "RBRargo Product Lookup" web-API We also download certificates if available @@ -482,7 +495,7 @@ def from_rbr(self, serial_number: str, **kwargs): obj = self.from_dict(data) # Also download RBR zip archive with calibration certificates in PDFs: - obj = obj.__certificates_rbr(action="download", quiet=True) + obj = obj._certificates_rbr(action="download", quiet=True) # Finally reset httpstore parameters: headers = fss.client_kwargs.get("headers") @@ -491,7 +504,7 @@ def from_rbr(self, serial_number: str, **kwargs): return obj - def __certificates_rbr( + def _certificates_rbr( self, action: Literal["download", "open"] = "download", **kwargs ): """Download RBR zip archive with calibration certificates in PDFs @@ -500,7 +513,7 @@ def __certificates_rbr( Notes ----- - We keep this method very private because it is expected to be called only by the self.from_rbr() method. + We keep this method private because it is expected to be called only by the self.from_rbr() method. This ensures that the httpstore has the appropriate authorization key. """ @@ -559,6 +572,34 @@ def __certificates_rbr( else: raise ValueError(f"Unknown action {action}") + def from_seabird(self, serial_number: str, sensor_model: str, **kwargs) -> 'ArgoSensorMetaDataOem': + """Fetch sensor metadata from Seabird-Scientific "Instrument Metadata Portal" web-API + + Parameters + ---------- + serial_number : str + Sensor serial number from RBR + """ + # The Seabird api requires a sensor model (R27) with a serial number. + # This could easily be done if we get the s/n by searching float metadata. + # In other word, it's easy to go from a sensor_model to a serial_number + # But it is much more complicated to from a serial_number to a sensor_model. + # so, the time being, we'll ask users to specify a sensor_model. + + url = f"{OPTIONS['seabird_api']}?SENSOR_SERIAL_NO={serial_number}&SENSOR_MODEL={sensor_model}" + data = self._fs.open_json(url) + + # Temporary fix for errors in SBE api output: + data.update({'sensor_info': data['json_info']}) + data['sensor_info'].update({'link': data['sensor_info']['created_by']}) + data['sensor_info'].update({'created_by': 'Sea-Bird Instrument Metadata Portal'}) + data['sensor_info'].update({'sensor_described': f"{sensor_model}-{serial_number}"}) + data.pop('json_info', None) + + # Create and return an instance + obj = self.from_dict(data) + return obj + @property def list_examples(self): """List of example names""" diff --git a/argopy/related/sensors/oem_metadata_repr.py b/argopy/related/sensors/oem_metadata_repr.py index c18230e9b..78f1c65a6 100644 --- a/argopy/related/sensors/oem_metadata_repr.py +++ b/argopy/related/sensors/oem_metadata_repr.py @@ -54,9 +54,9 @@ def html(self): # --- Header --- header_html = f""" -

Argo Sensor Metadata: {getattr(self.OEMsensor.sensor_info, 'sensor_described', 'N/A')}

-

Created by: {getattr(self.OEMsensor.sensor_info, 'created_by', 'N/A')} | - Date: {getattr(self.OEMsensor.sensor_info, 'date_creation', 'N/A')}

+

Argo Sensor Metadata: {self.OEMsensor.sensor_info._attr2str('sensor_described')}

+

Created by: {self.OEMsensor.sensor_info._attr2str('created_by')} | + Date: {self.OEMsensor.sensor_info._attr2str('date_creation')}

""" # --- Sensors --- From 754e836dfaf1ebc12ebfd58cc5db5c1f138900aa Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Fri, 24 Oct 2025 11:32:02 +0200 Subject: [PATCH 47/71] Update sensors.py - support multiple model search - more type and options checking --- argopy/related/sensors/sensors.py | 285 ++++++++++++++++++++++++------ 1 file changed, 230 insertions(+), 55 deletions(-) diff --git a/argopy/related/sensors/sensors.py b/argopy/related/sensors/sensors.py index 67b8e720a..78f497955 100644 --- a/argopy/related/sensors/sensors.py +++ b/argopy/related/sensors/sensors.py @@ -3,8 +3,12 @@ from pathlib import Path from typing import Literal, Any, Iterator import logging +import concurrent.futures from ...stores import ArgoFloat, ArgoIndex, httpstore, filestore +from ...stores.filesystems import ( + tqdm, +) # Safe import, return a lambda if tqdm not available from ...utils import check_wmo, Chunker, to_list, NVSrow from ...errors import ( DataNotFound, @@ -17,8 +21,13 @@ from .. import ArgoNVSReferenceTables -SearchOutputOptions = Literal["wmo", "sn", "wmo_sn", "df"] -ErrorOptions = Literal["raise", "ignore"] +# Define allowed values as a tuple +SearchOutput = ("wmo", "sn", "wmo_sn", "df") +Error = ("raise", "ignore", "silent") + +# Define Literal types using tuples +SearchOutputOptions = Literal[*SearchOutput] +ErrorOptions = Literal[*Error] log = logging.getLogger("argopy.related.sensors") @@ -86,7 +95,11 @@ def from_series(obj: pd.Series) -> "SensorModel": return SensorModel(obj) def __contains__(self, string) -> bool: - return string.lower() in self.name.lower() or string.lower() in self.long_name.lower() + return ( + string.lower() in self.name.lower() + or string.lower() in self.long_name.lower() + ) + class ArgoSensor: """Argo sensor 'package' helper class @@ -107,7 +120,18 @@ class ArgoSensor: """ - __slots__ = ["_cache", "_cachedir", "_timeout", "fs", "_r25", "_r26", "_r27", "_r27_to_r25", "_model", "_type"] + __slots__ = [ + "_cache", + "_cachedir", + "_timeout", + "_fs", + "_r25", + "_r26", + "_r27", + "_r27_to_r25", + "_model", + "_type", + ] def __init__( self, @@ -177,8 +201,12 @@ def __init__( # Search and return a DataFrame with full sensor information from floats equipped ArgoSensor().search('RBR', output='df') + # Search by model, can take a list of string, not necessarily a single value: + ArgoSensor().search(['ECO_FLBBCD_AP2', 'ECO_FLBBCD']) + + .. code-block:: python - :caption: Easily loop through `ArgoFloat` instances for each floats equipped with a sensor model + :caption: Easily loop through `ArgoFloat` instances for each float equipped with a sensor model from argopy import ArgoSensor @@ -226,8 +254,12 @@ def __init__( self._cache = kwargs.get("cache", True) self._cachedir = kwargs.get("cachedir", OPTIONS["cachedir"]) self._timeout = kwargs.get("timeout", OPTIONS["api_timeout"]) - fs_kargs = {"cache": self._cache, "cachedir": self._cachedir, "timeout": self._timeout} - self.fs = httpstore(**fs_kargs) + fs_kargs = { + "cache": self._cache, + "cachedir": self._cachedir, + "timeout": self._timeout, + } + self._fs = httpstore(**fs_kargs) self._r25: pd.DataFrame | None = None # will be loaded when necessary self._r26: pd.DataFrame | None = None # will be loaded when necessary @@ -248,14 +280,13 @@ def __init__( self._model = SensorModel.from_series(df.iloc[0]) self._type = self.model_to_type(self._model, errors="ignore") # if "RBR" in self._model: - # Add the RBR OEM API Authorization key for this sensor: - # fs_kargs.update(client_kwargs={'headers': {'Authorization': OPTIONS.get('rbr_api_key') }}) + # Add the RBR OEM API Authorization key for this sensor: + # fs_kargs.update(client_kwargs={'headers': {'Authorization': OPTIONS.get('rbr_api_key') }}) else: raise InvalidDatasetStructure( f"Found multiple sensor models with '{model}'. Restrict your sensor model name to only one value in: {to_list(df['altLabel'].values)}" ) - def _load_mappers(self): """Load from static assets file the NVS R25 to R27 key mappings @@ -447,13 +478,14 @@ def __repr__(self) -> str: "reference_model_name", "reference_sensor", "reference_sensor_type", + "reference_manufacturer", + "reference_manufacture_name", ]: summary.append(f" β•°β”ˆβž€ ArgoSensor().{attr}") summary.append("πŸ‘‰ methods: ") for meth in [ "search_model", - "search_model_name", "search", "iterfloats_with", ]: @@ -473,7 +505,7 @@ def reference_model(self) -> pd.DataFrame: :class:`ArgoNVSReferenceTables` """ if self._r27 is None: - self._r27 = ArgoNVSReferenceTables(fs=self.fs).tbl("R27") + self._r27 = ArgoNVSReferenceTables(fs=self._fs).tbl("R27") return self._r27 @property @@ -509,7 +541,7 @@ def reference_sensor(self) -> pd.DataFrame: :class:`ArgoNVSReferenceTables` """ if self._r25 is None: - self._r25 = ArgoNVSReferenceTables(fs=self.fs).tbl("R25") + self._r25 = ArgoNVSReferenceTables(fs=self._fs).tbl("R25") return self._r25 @property @@ -545,7 +577,7 @@ def reference_manufacturer(self) -> pd.DataFrame: :class:`ArgoNVSReferenceTables` """ if self._r26 is None: - self._r26 = ArgoNVSReferenceTables(fs=self.fs).tbl("R26") + self._r26 = ArgoNVSReferenceTables(fs=self._fs).tbl("R26") return self._r26 @property @@ -618,8 +650,8 @@ def search_model( else: return data.reset_index(drop=True) - def _search_wmo_with(self, model: str, errors : ErrorOptions = "raise") -> list[int]: - """Return the list of WMOs with a given sensor model + def _search_wmo_with(self, model: str, errors: ErrorOptions = "raise") -> list[int]: + """Return the list of WMOs equipped with a given sensor model Notes ----- @@ -629,32 +661,45 @@ def _search_wmo_with(self, model: str, errors : ErrorOptions = "raise") -> list[ https://fleetmonitoring.euro-argo.eu/swagger-ui.html#!/platform-code-controller/getPlatformCodesMultiLinesSearchUsingPOST + Notes + ----- + No option checking, to be done by caller """ + models = to_list(model) + # Security Issue: models string should be sanitized ? api_point = f"{OPTIONS['fleetmonitoring']}/platformCodes/multi-lines-search" payload = [ { "nested": False, "path": "string", "searchValueType": "Text", - "values": [model], + "values": models, "field": "sensorModels", } ] - wmos = self.fs.post(api_point, json_data=payload) + wmos = self._fs.post(api_point, json_data=payload) if wmos is None or len(wmos) == 0: - try: - search_hint: list[str] = self.search_model( - model, output="name", strict=False - ) + # Handle failed search: + search_hint: list[str] = [] + for model in models: + try: + hint: list[str] = self.search_model( + model, output="name", strict=False + ) + search_hint.extend(hint) + except DataNotFound: + pass + if len(search_hint) > 0: msg = ( - f"No floats matching this sensor model name '{model}'. Possible hint: %s" + f"No floats matching this sensor model name {models}. Possible hint: %s" % ("; ".join(search_hint)) ) - except DataNotFound: - msg = f"No floats matching this sensor model name '{model}'" + else: + msg = f"Unknown sensor models: {models}" + if errors == "raise": raise DataNotFound(msg) - else: + elif errors == "ignore": log.error(msg) return check_wmo(wmos) @@ -666,7 +711,7 @@ def _floats_api( postprocess=None, postprocess_opts={}, progress=False, - errors="raise", + errors: ErrorOptions = "raise", ) -> Any: """Search floats with a sensor model and then fetch and process JSON data returned from the fleet-monitoring API for each floats @@ -675,6 +720,10 @@ def _floats_api( Based on a POST request to the fleet-monitoring API requests to `/floats/{wmo}`. `Endpoint documentation `_. + + Notes + ----- + No option checking, to be done by caller """ wmos = self._search_wmo_with(model) @@ -682,7 +731,7 @@ def _floats_api( for wmo in wmos: URI.append(f"{OPTIONS['fleetmonitoring']}/floats/{wmo}") - sns = self.fs.open_mfjson( + sns = self._fs.open_mfjson( URI, preprocess=preprocess, preprocess_opts=preprocess_opts, @@ -692,8 +741,14 @@ def _floats_api( return postprocess(sns, **postprocess_opts) - def _search_sn_with(self, model: str, progress=False, errors : ErrorOptions = "raise") -> list[str]: - """Return serial number of sensor models with a given string in name""" + def _search_sn_with( + self, model: str, progress=False, errors: ErrorOptions = "raise" + ) -> list[str]: + """Return serial number of sensor models with a given string in name + Notes + ----- + No option checking, to be done by caller + """ def preprocess(jsdata, model_name: str = ""): sn = np.unique( @@ -719,9 +774,14 @@ def postprocess(data, **kwargs): ) def _search_wmo_sn_with( - self, model: str, progress=False, errors="raise" + self, model: str, progress=False, errors: ErrorOptions = "raise" ) -> dict[int, str]: - """Return a dictionary of float WMOs with their sensor serial numbers""" + """Return a dictionary of float WMOs with their sensor serial numbers + + Notes + ----- + No option checking, to be done by caller + """ def preprocess(jsdata, model_name: str = ""): sn = np.unique( @@ -744,7 +804,9 @@ def postprocess(data, **kwargs): errors=errors, ) - def _to_dataframe(self, model: str, progress=False, errors : ErrorOptions = "raise") -> pd.DataFrame: + def _to_dataframe( + self, model: str, progress=False, errors: ErrorOptions = "raise" + ) -> pd.DataFrame: """Return a DataFrame with WMO, sensor type, model, maker, sn, units, accuracy and resolution Parameters @@ -752,6 +814,9 @@ def _to_dataframe(self, model: str, progress=False, errors : ErrorOptions = "rai model: str, optional A string to search in the `sensorModels` field of the Euro-Argo fleet-monitoring API `platformCodes/multi-lines-search` endpoint. + Notes + ----- + No option checking, to be done by caller """ if model is None and self.model is not None: model = self.model.name @@ -803,21 +868,112 @@ def postprocess(data, **kwargs): errors=errors, ) + def _search_single( + self, + model: str, + output: SearchOutputOptions = "wmo", + progress: bool = False, + errors: ErrorOptions = "raise", + ) -> list[int] | list[str] | dict[int, str] | pd.DataFrame: + """Run a single model search""" + + if output == "df": + return self._to_dataframe(model=model, progress=progress, errors=errors) + elif output == "sn": + return self._search_sn_with(model=model, progress=progress, errors=errors) + elif output == "wmo_sn": + return self._search_wmo_sn_with( + model=model, progress=progress, errors=errors + ) + else: + return self._search_wmo_with(model=model, errors=errors) + + def _search_multi( + self, + models: list[str], + output: SearchOutputOptions = "wmo", + progress: bool = False, + errors: ErrorOptions = "raise", + max_workers: int | None = None, + ) -> list[int] | list[str] | dict[int, str] | pd.DataFrame: + """Run a multiple models search in parallel with multithreading""" + # Remove duplicates: + models = list(set(models)) + + ConcurrentExecutor = concurrent.futures.ThreadPoolExecutor( + max_workers=max_workers + ) + failed = [] + if output in ["wmo", "sn", "df"]: + results = [] + elif output == "wmo_sn": + results = {} + + with ConcurrentExecutor as executor: + future_to_model = { + executor.submit( + self._search_single, + model, + output=output, + errors=errors, + ): model + for model in models + } + futures = concurrent.futures.as_completed(future_to_model) + if progress: + futures = tqdm( + futures, total=len(models), disable="disable" in [progress] + ) + + for future in futures: + data = None + try: + data = future.result() + except Exception: + failed.append(future_to_model[future]) + if errors == "ignore": + log.error( + "Ignored error with this url: %s" % future_to_model[future] + ) + elif errors == "silent": + pass + else: + raise + finally: + # Gather results according to final output format: + if data is not None: + if output in ["wmo", "sn"]: + results.extend(data) + elif output == "df": + results.append(data) + else: + for wmo in data.keys(): + results.update({wmo: data[wmo]}) + + results = [r for r in results if r is not None] # Only keep non-empty results + if len(results) > 0: + if output == "df": + return pd.concat(results, axis=0).reset_index(drop=True) + else: + return results + raise DataNotFound(models) + def search( self, - model: str | None = None, + model: str | list[str] | None = None, output: SearchOutputOptions = "wmo", - progress : bool = False, - errors : ErrorOptions = "raise", + progress: bool = False, + errors: ErrorOptions = "raise", + **kwargs, ) -> list[int] | list[str] | dict[int, str] | pd.DataFrame: """Search for Argo floats equipped with a sensor model name - All information are retrieved using the `Euro-Argo fleet-monitoring API `_. + All information are retrieved with one or more requests to the `Euro-Argo fleet-monitoring API `_. Parameters ---------- - model: str, optional - A string to search in the ``sensorModels`` field of the Euro-Argo fleet-monitoring API ``platformCodes/multi-lines-search`` endpoint. + model: str, list[str], optional + One or more models string to search. output: str, Literal["wmo", "sn", "wmo_sn", "df"], default "wmo" Define the output to return: @@ -838,30 +994,49 @@ def search( Notes ----- - The list of WMOs equipped with a given sensor model is retrieved using the Euro-Argo fleet-monitoring API and a request to the ``platformCodes/multi-lines-search`` endpoint using the ``sensorModels`` search field. + Whatever the output format, the first step is to retrieve a list of WMOs equipped with one or more sensor models. + This is done using the Euro-Argo fleet-monitoring API and a request to the ``platformCodes/multi-lines-search`` endpoint using the ``sensorModels`` search field. - Sensor serial numbers are given by float meta-data retrieved using the Euro-Argo fleet-monitoring API and a request to the ``/floats/{wmo}`` endpoint: + Then and if necessary (all output format but 'wmo'), the corresponding list of sensor serial numbers are retrieved using one request per float to the Euro-Argo fleet-monitoring API ``/floats/{wmo}`` endpoint. - - `Documentation for endpoint: platformCodes/multi-lines-search `_. + Web-api documentation: + - `Documentation for endpoint: platformCodes/multi-lines-search `_. - `Documentation for endpoint: /floats/{wmo} `_. """ - if model is None and self.model is not None: - model = self.model.name - if output == "df": - return self._to_dataframe(model=model, progress=progress, errors=errors) - elif output == "wmo": - return self._search_wmo_with(model=model, errors=errors) - elif output == "sn": - return self._search_sn_with(model=model, progress=progress, errors=errors) - elif output == "wmo_sn": - return self._search_wmo_sn_with( - model=model, progress=progress, errors=errors + if output not in SearchOutput: + raise OptionValueError( + f"Invalid 'output' option value '{output}', must be in: {SearchOutput}" ) - else: + if errors not in Error: raise OptionValueError( - "'output' option value must be in: 'wmo', 'sn', 'wmo_sn' or 'df'" + f"Invalid 'errors' option value '{errors}', must be in: {Error}" + ) + + if model is None: + if self.model is not None: + return self._search_single( + model=self.model.name, + output=output, + progress=progress, + errors=errors, + ) + else: + raise OptionValueError("You must specify at list one model to search !") + + models = to_list(model) + if len(models) == 1: + return self._search_single( + model=model, output=output, progress=progress, errors=errors + ) + else: + return self._search_multi( + models=models, + output=output, + progress=progress, + errors=errors, + **kwargs, ) def iterfloats_with( From 76894572f19cc27339cb0dbfa610e6b8e13bc4ed Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Mon, 27 Oct 2025 19:09:38 +0100 Subject: [PATCH 48/71] Some refactoring and improvements --- argopy/related/__init__.py | 3 +- argopy/related/sensors/references.py | 427 ++++++++++++++ .../{sensors.py => sensors_deprecated.py} | 0 argopy/related/sensors/spec.py | 556 ++++++++++++++++++ argopy/stores/implementations/http.py | 6 +- argopy/utils/__init__.py | 4 +- argopy/utils/format.py | 34 +- 7 files changed, 1026 insertions(+), 4 deletions(-) create mode 100644 argopy/related/sensors/references.py rename argopy/related/sensors/{sensors.py => sensors_deprecated.py} (100%) create mode 100644 argopy/related/sensors/spec.py diff --git a/argopy/related/__init__.py b/argopy/related/__init__.py index 3bb5042ad..ce582f7fe 100644 --- a/argopy/related/__init__.py +++ b/argopy/related/__init__.py @@ -4,7 +4,8 @@ from .argo_documentation import ArgoDocs from .doi_snapshot import ArgoDOI from .euroargo_api import get_coriolis_profile_id, get_ea_profile_page -from .sensors.sensors import ArgoSensor, SensorType, SensorModel +from .sensors.spec import ArgoSensor +from .sensors.sensors_deprecated import SensorType, SensorModel from .utils import load_dict, mapp_dict # Should come last # diff --git a/argopy/related/sensors/references.py b/argopy/related/sensors/references.py new file mode 100644 index 000000000..019eba705 --- /dev/null +++ b/argopy/related/sensors/references.py @@ -0,0 +1,427 @@ +import pandas as pd +from functools import lru_cache +from pathlib import Path +import xarray as xr +from typing import Any, Callable, Literal, NoReturn +import inspect +from abc import ABC, abstractmethod +import logging +import fnmatch + +from ...options import OPTIONS +from ...stores import httpstore, filestore +from ...related import ArgoNVSReferenceTables +from ...utils import to_list, NVSrow, path2assets, register_accessor +from ...errors import DataNotFound + + +log = logging.getLogger("argopy.related.sensors.ref") + + +# Define allowed values as a tuple +Error = ("raise", "ignore", "silent") + +# Define Literal types using tuples +ErrorOptions = Literal[*Error] + + +class SensorType(NVSrow): + """One single sensor type data from a R25-"Argo sensor types" row + + Examples + -------- + .. code-block:: python + + from argopy import ArgoNVSReferenceTables + + sensor_type = 'CTD' + + df = ArgoNVSReferenceTables().tbl(25) + df_match = df[df["altLabel"].apply(lambda x: x == sensor_type)].iloc[0] + + st = SensorType.from_series(df_match) + + st.name + st.long_name + st.definition + st.deprecated + st.uri + + """ + + reftable = "R25" + + @staticmethod + def from_series(obj: pd.Series) -> "SensorType": + """Create a :class:`SensorType` from a R25-"Argo sensor models" row""" + return SensorType(obj) + + +class SensorModel(NVSrow): + """One single sensor model data from a R27-"Argo sensor models" row + + Examples + -------- + .. code-block:: python + + from argopy import ArgoNVSReferenceTables + + sensor_model = 'AANDERAA_OPTODE_4330F' + + df = ArgoNVSReferenceTables().tbl(27) + df_match = df[df["altLabel"].apply(lambda x: x == sensor_model)].iloc[0] + + sm = SensorModel.from_series(df_match) + + sm.name + sm.long_name + sm.definition + sm.deprecated + sm.uri + """ + + reftable = "R27" + + @staticmethod + def from_series(obj: pd.Series) -> "SensorModel": + """Create a :class:`SensorModel` from a R27-"Argo sensor models" row""" + return SensorModel(obj) + + def __contains__(self, string) -> bool: + return ( + string.lower() in self.name.lower() + or string.lower() in self.long_name.lower() + ) + + +class SensorReferenceHolder(ABC): + """Parent class to hold R25, R26, R27 and R27_to_R25 mapping data""" + + _r25: pd.DataFrame | None = None + """NVS Reference table for Argo sensor types (R25)""" + + _r26: pd.DataFrame | None = None + """NVS Reference table for Argo sensor manufacturer (R26)""" + + _r27: pd.DataFrame | None = None + """NVS Reference table for Argo sensor models (R27)""" + + _r27_to_r25: dict[str, str] | None = None + """Dictionary mapping of R27 to R25""" + + def __call__(self, *args, **kwargs) -> NoReturn: + raise ValueError( + "A SensorReference instance cannot be called directly." + ) + + def __init__(self, obj): + self._obj = obj # An instance of SensorReferences, possibly with a filesystem + if getattr(obj, '_fs', None) is None: + self._fs = httpstore( + cache=True, + cachedir=OPTIONS["cachedir"], + timeout=OPTIONS["api_timeout"], + ) + else: + self._fs = obj._fs + + @property + def r25(self): + """NVS Reference table for Argo sensor types (R25)""" + if self._r25 is None: + self._r25 = ArgoNVSReferenceTables(fs=self._fs).tbl("R25") + return self._r25 + + @property + def r26(self): + """NVS Reference table for Argo sensor manufacturer (R26)""" + if self._r26 is None: + self._r26 = ArgoNVSReferenceTables(fs=self._fs).tbl("R26") + return self._r26 + + @property + def r27(self): + """NVS Reference table for Argo sensor models (R27)""" + if self._r27 is None: + self._r27 = ArgoNVSReferenceTables(fs=self._fs).tbl("R27") + return self._r27 + + def _load_mappers(self): + """Load from static assets file the NVS R25 to R27 key mappings + + These mapping files were download from https://github.com/OneArgo/ArgoVocabs/issues/156. + """ + df = [] + for p in ( + Path(path2assets).joinpath("nvs_R25_R27").glob("NVS_R25_R27_mappings_*.txt") + ): + df.append( + filestore().read_csv( + p, + header=None, + names=["origin", "model", "?", "destination", "type", "??"], + ) + ) + df = pd.concat(df) + df = df.reset_index(drop=True) + self._r27_to_r25: dict[str, str] = {} + df.apply( + lambda row: self._r27_to_r25.update( + {row["model"].strip(): row["type"].strip()} + ), + axis=1, + ) + + @property + def r27_to_r25(self): + """Dictionary mapping of R27 to R25""" + if self._r27_to_r25 is None: + self._load_mappers() + return self._r27_to_r25 + + @abstractmethod + def to_dataframe(self): + raise NotImplementedError + + @abstractmethod + def hint(self): + raise NotImplementedError + + +class SensorReferenceR27(SensorReferenceHolder): + """Argo sensor models""" + + def to_dataframe(self) -> pd.DataFrame: + """Official reference table for Argo sensor models (R27) + + Returns + ------- + :class:`pandas.DataFrame` + + See Also + -------- + :class:`ArgoNVSReferenceTables` + """ + return self.r27 + + def hint(self) -> list[str]: + """Official list of Argo sensor models (R27) + + Return a sorted list of strings with altLabel from Argo Reference table R27 on 'SENSOR_MODEL'. + + Returns + ------- + list[str] + + Notes + ----- + Argo netCDF variable ``SENSOR_MODEL`` is populated with values from this list. + """ + return sorted(to_list(self.r27["altLabel"].values)) + + def to_type( + self, + model: str | SensorModel | None = None, + errors: ErrorOptions = "raise", + ) -> SensorType | None: + """Get a sensor type for a given sensor model + + All valid sensor model name can be obtained with :meth:`ArgoSensor.ref.mode.to_list()`. + + Mapping between sensor model name (R27) and sensor type (R25) are from AVTT work at https://github.com/OneArgo/ArgoVocabs/issues/156. + + Parameters + ---------- + model : str | :class:`SensorModel` + The model to read the sensor type for. + errors : Literal["raise", "ignore", "silent"] = "raise" + How to handle possible errors. If set to "ignore", the method will return None. + + Returns + ------- + :class:`SensorType` | None + """ + model_name: str = model.name if isinstance(model, SensorModel) else model + sensor_type = self.r27_to_r25.get(model_name, None) + if sensor_type is not None: + row = self.r25[ + self.r25["altLabel"].apply(lambda x: x == sensor_type) + ].iloc[0] + return SensorType.from_series(row) + elif errors == "raise": + raise DataNotFound( + f"Can't determine the type of sensor model '{model_name}' (no matching key in r27_to_r25 mapper)" + ) + elif errors == "silent": + log.error( + f"Can't determine the type of sensor model '{model_name}' (no matching key in r27_to_r25 mapper)" + ) + return None + + def search( + self, + model: str, + output: Literal["df", "name"] = "df", + ) -> pd.DataFrame | list[str]: + """Return Argo sensor model references matching a string + + Look for occurrences in Argo Reference table R27 `altLabel` and return a :class:`pandas.DataFrame` with matching row(s). + + Parameters + ---------- + model : str + The model to search for. You can use wildcards: "SBE41CP*" "*DEEP*", "RBR*", or an exact name like "RBR_ARGO3_DEEP6". + output : str, Literal["df", "name"], default "df" + Is the output a :class:`pandas.DataFrame` with matching rows from :attr:`ArgoSensor.reference_model`, or a list of string. + + Returns + ------- + :class:`pandas.DataFrame`, list[str] + + Raises + ------ + :class:`DataNotFound` + """ + match = fnmatch.filter(self.r27["altLabel"], model.upper()) + data = self.r27[self.r27["altLabel"].apply(lambda x: x in match)] + + if data.shape[0] == 0: + raise DataNotFound( + f"'{model}' is not a valid sensor model name. You can use wildcard for search, e.g. 'SBE61*'." + ) + else: + if output == "name": + return sorted(to_list(data["altLabel"].values)) + else: + return data.reset_index(drop=True) + + + +class SensorReferenceR25(SensorReferenceHolder): + """Argo sensor types""" + + def to_dataframe(self) -> pd.DataFrame: + """Official reference table for Argo sensor types (R25) + + Returns + ------- + :class:`pandas.DataFrame` + """ + return self.r25 + + def hint(self) -> list[str]: + """Official list of Argo sensor types (R25) + + Return a sorted list of strings with altLabel from Argo Reference table R25 on 'SENSOR'. + + Returns + ------- + list[str] + + Notes + ----- + Argo netCDF variable ``SENSOR`` is populated with values from this list. + """ + return sorted(to_list(self.r25["altLabel"].values)) + + def to_model( + self, + type: str | SensorType, + errors: Literal["raise", "ignore"] = "raise", + ) -> list[str] | None: + """Get all sensor model names of a given sensor type + + All valid sensor types can be obtained with :attr:`ArgoSensor.reference_sensor_type` + + Mapping between sensor model name (R27) and sensor type (R25) are from AVTT work at https://github.com/OneArgo/ArgoVocabs/issues/156. + + Parameters + ---------- + type : str, :class:`SensorType` + The sensor type to read the sensor model name for. + errors : Literal["raise", "ignore"] = "raise" + How to handle possible errors. If set to "ignore", the method will return None. + + Returns + ------- + list[str] + """ + sensor_type = type.name if isinstance(type, SensorType) else type + result = [] + for key, val in self.r27_to_r25.items(): + if sensor_type.lower() in val.lower(): + row = self.r27[ + self.r27["altLabel"].apply(lambda x: x == key) + ].iloc[0] + result.append(SensorModel.from_series(row).name) + if len(result) == 0: + if errors == "raise": + raise DataNotFound( + f"Can't find any sensor model for this type '{sensor_type}' (no matching key in r27_to_r25 mapper)" + ) + else: + return None + else: + return result + + +class SensorReferenceR26(SensorReferenceHolder): + """Argo sensor manufacturer""" + + def to_dataframe(self) -> pd.DataFrame: + """Official reference table for Argo sensor manufacturers (R26) + + Returns + ------- + :class:`pandas.DataFrame` + """ + return self.r26 + + def hint(self) -> list[str]: + """Official list of Argo sensor maker (R26) + + Return a sorted list of strings with altLabel from Argo Reference table R26 on 'SENSOR_MAKER'. + + Returns + ------- + list[str] + + Notes + ----- + Argo netCDF variable ``SENSOR_MAKER`` is populated with values from this list. + """ + return sorted(to_list(self.r26["altLabel"].values)) + + +class SensorReferences: + + __slots__ = ["_obj", "_fs"] + + def __init__(self, obj): + self._obj = obj # An instance of ArgoSensor, possibly with a filesystem + if getattr(obj, '_fs', None) is None: + self._fs = httpstore( + cache=True, + cachedir=OPTIONS["cachedir"], + timeout=OPTIONS["api_timeout"], + ) + else: + self._fs = obj._fs + + def __call__(self, *args, **kwargs) -> NoReturn: + raise ValueError( + "ArgoSensor.ref cannot be called directly." + ) + + +@register_accessor('type', SensorReferences) +class SensorExtension(SensorReferenceR25): + _name = "ref.type" + +@register_accessor('manufacturer', SensorReferences) +class ManufacturerExtension(SensorReferenceR26): + _name = "ref.manufacturer" + +@register_accessor('model', SensorReferences) +class ModelExtension(SensorReferenceR27): + _name = "ref.model" diff --git a/argopy/related/sensors/sensors.py b/argopy/related/sensors/sensors_deprecated.py similarity index 100% rename from argopy/related/sensors/sensors.py rename to argopy/related/sensors/sensors_deprecated.py diff --git a/argopy/related/sensors/spec.py b/argopy/related/sensors/spec.py new file mode 100644 index 000000000..d13a4efd2 --- /dev/null +++ b/argopy/related/sensors/spec.py @@ -0,0 +1,556 @@ +import pandas as pd +import numpy as np +from pathlib import Path +from typing import Literal, Any, Iterator, Callable +import logging +import concurrent.futures +import xarray as xr +import logging +import warnings + + +from ...stores import ArgoFloat, ArgoIndex, httpstore, filestore +from ...stores.filesystems import ( + tqdm, +) # Safe import, return a lambda if tqdm not available +from ...utils import check_wmo, Chunker, to_list, NVSrow, ppliststr, is_wmo +from ...errors import ( + DataNotFound, + InvalidDataset, + InvalidDatasetStructure, + OptionValueError, +) +from ...options import OPTIONS +from ...utils import path2assets, register_accessor +from .. import ArgoNVSReferenceTables + +from .references import SensorReferences + + +log = logging.getLogger("argopy.related.sensors") + +# Define allowed values as a tuple +SearchOutput = ("wmo", "sn", "wmo_sn", "df") +Error = ("raise", "ignore", "silent") + +# Define Literal types using tuples +SearchOutputOptions = Literal[*SearchOutput] +ErrorOptions = Literal[*Error] + + +class ArgoSensor: + def __init__(self, *args, **kwargs) -> None: + if kwargs.get("fs", None) is None: + self._fs = httpstore( + cache=kwargs.get("cache", True), + cachedir=kwargs.get("cachedir", OPTIONS["cachedir"]), + timeout=kwargs.get("timeout", OPTIONS["api_timeout"]), + ) + else: + self._fs = kwargs["fs"] + + def _search_wmo_with( + self, model: str | list[str], errors: ErrorOptions = "raise" + ) -> list[int]: + """Return the list of WMOs equipped with a given sensor model + + Notes + ----- + Based on a fleet-monitoring API request to `platformCodes/multi-lines-search` on `sensorModels` field. + + Documentation: + + https://fleetmonitoring.euro-argo.eu/swagger-ui.html#!/platform-code-controller/getPlatformCodesMultiLinesSearchUsingPOST + + Notes + ----- + No option checking, to be done by caller + """ + models = to_list(model) + api_endpoint = f"{OPTIONS['fleetmonitoring']}/platformCodes/multi-lines-search" + payload = [ + { + "nested": False, + "path": "string", + "searchValueType": "Text", + "values": models, + "field": "sensorModels", + } + ] + wmos = self._fs.post(api_endpoint, json_data=payload) + if wmos is None or len(wmos) == 0: + if len(models) == 1: + msg = f"Model is valid but no floats returned with this sensor model name: '{models[0]}'" + else: + msg = f"Models are valid but no floats returned with any of these sensor model names: {ppliststr(models)}" + if errors == "raise": + raise DataNotFound(msg) + elif errors == "ignore": + log.error(msg) + return sorted(check_wmo(wmos)) + + def _floats_api( + self, + model_or_wmo: str | int, + preprocess: Callable = None, + preprocess_opts: dict = {}, + postprocess: Callable = None, + postprocess_opts: dict = {}, + progress: bool = False, + errors: ErrorOptions = "raise", + ) -> Any: + """Search floats with a sensor model and then fetch and process JSON data returned from the fleet-monitoring API for each floats + + Notes + ----- + Based on a POST request to the fleet-monitoring API requests to `/floats/{wmo}`. + + `Endpoint documentation `_. + + Notes + ----- + No option checking, to be done by caller + """ + if preprocess_opts is None: + preprocess_opts = {} + + try: + is_wmo(model_or_wmo) + WMOs = check_wmo(model_or_wmo) + except ValueError: + WMOs = self._search_wmo_with(model_or_wmo) + + URI = [] + for wmo in WMOs: + URI.append(f"{OPTIONS['fleetmonitoring']}/floats/{wmo}") + + sns = self._fs.open_mfjson( + URI, + preprocess=preprocess, + preprocess_opts=preprocess_opts, + progress=progress, + errors=errors, + progress_unit="float", + progress_desc="Fetching floats metadata", + ) + + if postprocess is not None: + return postprocess(sns, **postprocess_opts) + else: + return sns + + def _search_sn_with( + self, model: str, progress: bool = False, errors: ErrorOptions = "raise", **kwargs + ) -> list[str]: + """Return serial number of sensor models with a given string in name + + Notes + ----- + No option checking, to be done by caller + """ + + def preprocess(jsdata, model_name: str = ""): + sn = np.unique( + [s["serial"] for s in jsdata["sensors"] if model_name in s["model"]] + ) + return sn + + def postprocess(data, **kwargs): + S = [] + for row in data: + for sensor in row: + if sensor is not None: + S.append(sensor) + return np.sort(np.array(S)) + + return self._floats_api( + model if kwargs.get("wmo", None) is None else kwargs["wmo"], + preprocess=preprocess, + preprocess_opts={"model_name": model}, + postprocess=postprocess, + progress=progress, + errors=errors, + ) + + def _search_wmo_sn_with( + self, model: str, progress: bool = False, errors: ErrorOptions = "raise", **kwargs + ) -> dict[int, str]: + """Return a dictionary of float WMOs with their sensor serial numbers + + Notes + ----- + No option checking, to be done by caller + """ + + def preprocess(jsdata, model_name: str = ""): + sn = np.unique( + [s["serial"] for s in jsdata["sensors"] if model_name in s["model"]] + ) + return [jsdata["wmo"], [str(s) for s in sn]] + + def postprocess(data, **kwargs): + S = {} + for wmo, sn in data: + S.update({check_wmo(wmo)[0]: to_list(sn)}) + return S + + results = self._floats_api( + model if kwargs.get("wmo", None) is None else kwargs["wmo"], + preprocess=preprocess, + preprocess_opts={"model_name": model}, + postprocess=postprocess, + progress=progress, + errors=errors, + ) + return dict(sorted(results.items())) + + def _to_dataframe( + self, model: str, progress: bool = False, errors: ErrorOptions = "raise", **kwargs + ) -> pd.DataFrame: + """Return a DataFrame with WMO, sensor type, model, maker, sn, units, accuracy and resolution + + Parameters + ---------- + model: str, optional + A string to search in the `sensorModels` field of the Euro-Argo fleet-monitoring API `platformCodes/multi-lines-search` endpoint. + + Notes + ----- + No option checking, to be done by caller + """ + if model is None and self.model is not None: + model = self.model.name + + def preprocess(jsdata, model_name: str = ""): + output = [] + for s in jsdata["sensors"]: + if model_name in s["model"]: + this = [jsdata["wmo"]] + [ + this.append(s[key]) # type: ignore + for key in [ + "id", + "maker", + "model", + "serial", + "units", + "accuracy", + "resolution", + ] + ] + output.append(this) + return output + + def postprocess(data, **kwargs): + d = [] + for this in data: + for wmo, sid, maker, model, sn, units, accuracy, resolution in this: + d.append( + { + "WMO": wmo, + "Type": sid, + "Model": model, + "Maker": maker, + "SerialNumber": sn if sn != 'n/a' else None, + "Units": units, + "Accuracy": accuracy, + "Resolution": resolution, + } + ) + return pd.DataFrame(d).sort_values(by="WMO").reset_index(drop=True) + + df = self._floats_api( + model if kwargs.get("wmo", None) is None else kwargs["wmo"], + preprocess=preprocess, + preprocess_opts={"model_name": model}, + postprocess=postprocess, + progress=progress, + errors=errors, + ) + return df.sort_values(by='WMO', axis=0).reset_index(drop=True) + + def _search_single( + self, + model: str, + output: SearchOutputOptions = "wmo", + progress: bool = False, + errors: ErrorOptions = "raise", + ) -> list[int] | list[str] | dict[int, str] | pd.DataFrame: + """Run a single model search""" + + if output == "df": + return self._to_dataframe(model=model, progress=progress, errors=errors) + elif output == "sn": + return self._search_sn_with(model=model, progress=progress, errors=errors) + elif output == "wmo_sn": + return self._search_wmo_sn_with( + model=model, progress=progress, errors=errors + ) + else: + return self._search_wmo_with(model=model, errors=errors) + + def _search_multi( + self, + models: list[str], + output: SearchOutputOptions = "wmo", + progress: bool = False, + errors: ErrorOptions = "raise", + max_workers: int | None = None, + ) -> list[int] | list[str] | dict[int, str] | pd.DataFrame: + """Run a multiple models search in parallel with multithreading""" + # Remove duplicates: + models = list(set(models)) + + if output == "wmo": + # Quite simple if we only need WMOs: + return self._search_wmo_with(model=models, errors=errors) + else: + # Otherwise we need the list of WMOs to get metadata for + # Even if we can request the fleetmonitoring API with multiple models at once, here we need + # the tuple (model, wmo) to use multithreading and existing private search methods. + lookup = [] + for model in models: + wmos = self._search_wmo_with(model=model, errors=errors) + lookup.extend([(model, wmo) for wmo in wmos]) + + # For all other output format, we use multithreading to process all floats: + if output == "sn": + func = self._search_sn_with + elif output == "wmo_sn": + func = self._search_wmo_sn_with + else: + func = self._to_dataframe + + ConcurrentExecutor = concurrent.futures.ThreadPoolExecutor( + max_workers=max_workers + ) + failed = [] + if output in ["sn", "df"]: + results = [] + elif output == "wmo_sn": + results = {} + + with ConcurrentExecutor as executor: + future_to_wmo = { + executor.submit( + func, + pair[0], + errors=errors, + wmo=pair[1], + ): pair + for pair in lookup + } + futures = concurrent.futures.as_completed(future_to_wmo) + if progress: + futures = tqdm( + futures, + total=len(lookup), + disable="disable" in [progress], + unit="float", + desc=f"Fetching sensor meta-data for {len(lookup)} floats...", + ) + + for future in futures: + data = None + try: + data = future.result() + except Exception: + failed.append(future_to_wmo[future]) + if errors == "ignore": + log.error( + "Ignored error with this float: %s" % future_to_wmo[future] + ) + elif errors == "silent": + pass + else: + raise + finally: + # Gather results according to final output format: + if data is not None: + if output == "sn": + results.extend(data) + elif output == "df": + results.append(data) + elif output == "wmo_sn": + for wmo in data.keys(): + if wmo in results: + # Update existing key: + results[wmo] = results[wmo].extend(data[wmo]) + else: + # Create new key: + results.update({wmo: data[wmo]}) + + + if output != "wmo_sn": + # Only keep non-empty results: + results = [r for r in results if r is not None] + else: + results = dict(sorted(results.items())) + + if len(results) > 0: + if output == "df": + results = [r for r in results if r.shape[0]>0] + return pd.concat(results, axis=0).sort_values(by='WMO', axis=0).reset_index(drop=True) + else: + return results + raise DataNotFound(ppliststr(models)) + + def search( + self, + model: str | list[str] | None = None, + output: SearchOutputOptions = "wmo", + progress: bool = True, + errors: ErrorOptions = "raise", + strict: bool = True, + **kwargs, + ) -> list[int] | list[str] | dict[int, str] | pd.DataFrame: + """Search for Argo floats equipped with one or more sensor model name(s) + + All information are retrieved from the `Euro-Argo fleet-monitoring API `_. + + Since this method can return a large number of data, model names are expected to be exact. You can use the :meth:`ArgoSensor.ref.model.search` method to search for exact model names. + + Parameters + ---------- + model: str, list[str], optional + One or more exact model names to search data for. + + output: str, Literal["wmo", "sn", "wmo_sn", "df"], default "wmo" + Define the output to return: + + - ``wmo``: a list of WMO numbers (integers) + - ``sn``: a list of sensor serial numbers (strings) + - ``wmo_sn``: a list of dictionary with WMO as key and serial numbers as values + - ``df``: a :class:`pandas.DataFrame` with WMO, sensor type/model/maker and serial number + + progress: bool, default True + Display a progress bar or not + + errors: str, Literal["raise", "ignore", "silent"], default "raise" + Raise an error, log it or do nothing if the search return nothing. + + Returns + ------- + list[int], list[str], dict[int, str], :class:`pandas.DataFrame` + + See Also + -------- + :meth:`ArgoSensor.ref.model.search` + + Notes + ----- + Whatever the output format, the first step is to retrieve a list of WMOs equipped with one or more sensor models. + This is done using the Euro-Argo fleet-monitoring API and a request to the ``platformCodes/multi-lines-search`` endpoint using the ``sensorModels`` search field. + + Then if necessary (all output format but 'wmo'), the corresponding list of sensor serial numbers are retrieved using one request per float to the Euro-Argo fleet-monitoring API ``/floats/{wmo}`` endpoint. + + Web-api documentation: + + - `Documentation for endpoint: platformCodes/multi-lines-search `_. + - `Documentation for endpoint: /floats/{wmo} `_. + + """ + if output not in SearchOutput: + raise OptionValueError( + f"Invalid 'output' option value '{output}', must be in: {SearchOutput}" + ) + if errors not in Error: + raise OptionValueError( + f"Invalid 'errors' option value '{errors}', must be in: {Error}" + ) + + if model is None: + if self.model is not None: + return self._search_single( + model=self.model.name, + output=output, + progress=progress, + errors=errors, + ) + else: + raise OptionValueError("You must specify at list one model to search !") + + models = to_list(model) + + def get_hints(these_models: str | list[str]): + these_models = to_list(these_models) + search_hint: list[str] = [] + for model in these_models: + model = f"*{model.upper()}*" # Use wildcards to get all possible hints + try: + hint: list[str] = self.ref.model.search( + model, + output="name", + ) + search_hint.extend(hint) + except DataNotFound: + pass + if len(search_hint) == 0: + search_hint = ["No match !"] + output = ppliststr( + search_hint, last="or", n=20 if len(search_hint) > 20 else None + ) + return output + + # Model names validation: + valid_models: list[str] = [] + invalid_models: list[str] = [] + for model in models: + if "*" in model: + raise OptionValueError( + f"This method expect exact model names but got a '*' in '{model}'. Possible hints are: {get_hints(model)}" + ) + try: + hint = self.ref.model.search(model, output="name") + valid_models.extend(hint) + except DataNotFound: + raise OptionValueError( + f"The model '{model}' is not exact. Possible hints are: {get_hints(model)}" + ) + + if len(valid_models) == 0: + msg = f"Unknown sensor model name(s) {ppliststr(invalid_models)}. We expect exact sensor names such as: {get_hints(invalid_models)}." + raise OptionValueError(msg) + + if len(valid_models) == 1: + return self._search_single( + model=valid_models[0], output=output, progress=progress, errors=errors + ) + else: + return self._search_multi( + models=valid_models, + output=output, + progress=progress, + errors=errors, + **kwargs, + ) + + +@register_accessor("ref", ArgoSensor) +class References(SensorReferences): + """An :class:`ArgoSensor` extension dedicated to reference tables appropriate for sensors + + Examples + -------- + .. code-block:: python + + from argopy import ArgoSensor + + ArgoSensor.ref.model.to_dataframe() # Return reference table R27 with the list of sensor models as a DataFrame + ArgoSensor.ref.model.hint() # Return list of sensor model names (possible values for 'SENSOR_MODEL' parameter) + ArgoSensor.ref.model.to_type('SBE61') # Return sensor type (R25) of a given model (R27) + + ArgoSensor.ref.sensor.to_dataframe() # Return reference table R25 with the list of sensor types as a DataFrame + ArgoSensor.ref.sensor.hint() # Return list of sensor types (possible values for 'SENSOR' parameter) + ArgoSensor.ref.sensor.to_model('FLUOROMETER_CDOM') # Return all possible model names (R27) for a given sensor type (R25) + + ArgoSensor.ref.manufacturer.to_dataframe() # Return reference table R26 with the list of manufacturer as a DataFrame + ArgoSensor.ref.manufacturer.hint() # Return list of manufacturer names (possible values for 'SENSOR_MAKER' parameter) + + ArgoSensor.ref.model.search('RBR') # Search for all (R27) referenced sensor models with some string in their name, return a DataFrame + ArgoSensor.ref.model.search('RBR', output='name') # Return a list of names instead + ArgoSensor.ref.model.search('SBE41CP', strict=False) + ArgoSensor.ref.model.search('SBE41CP', strict=True) # Exact string match required + """ + + _name = "ref" diff --git a/argopy/stores/implementations/http.py b/argopy/stores/implementations/http.py index 86dc178b6..231ddd478 100644 --- a/argopy/stores/implementations/http.py +++ b/argopy/stores/implementations/http.py @@ -1125,7 +1125,11 @@ def open_mfjson( futures = concurrent.futures.as_completed(future_to_url) if progress: futures = tqdm( - futures, total=len(urls), disable="disable" in [progress] + futures, + total=len(urls), + disable="disable" in [progress], + unit = kwargs.pop("progress_unit", "it"), + desc = kwargs.pop("progress_desc", None), ) for future in futures: diff --git a/argopy/utils/__init__.py b/argopy/utils/__init__.py index e482490d7..ca43cb118 100644 --- a/argopy/utils/__init__.py +++ b/argopy/utils/__init__.py @@ -60,7 +60,7 @@ filter_param_by_data_mode, split_data_mode, ) -from .format import argo_split_path, format_oneline, UriCName, redact, dirfs_relpath, urnparser +from .format import argo_split_path, format_oneline, UriCName, redact, dirfs_relpath, urnparser, ppliststr from .loggers import warnUnless, log_argopy_callerstack from .carbon import GreenCoding, Github from . import optical_modeling @@ -146,6 +146,8 @@ "dirfs_relpath", "UriCName", "redact", + "urnparser", + "ppliststr", # Loggers: "warnUnless", "log_argopy_callerstack", diff --git a/argopy/utils/format.py b/argopy/utils/format.py index cc1facb2f..7f1932f4b 100644 --- a/argopy/utils/format.py +++ b/argopy/utils/format.py @@ -408,4 +408,36 @@ def urnparser(urn): if len(pp) == 4 and pp[0] == 'SDN': return {'listid': pp[1], 'version': pp[2], 'termid': pp[3]} else: - raise ValueError(f"This NVS URN '{urn}' does not follow the pattern: 'SDN:{listid}:{version}:{termid}' or 'SDN:{listid}::{termid}' for NVS2.0") + raise ValueError(f"This NVS URN '{urn}' does not follow the pattern: 'SDN:listid:version:termid' or 'SDN:listid::termid' for NVS2.0") + + +def ppliststr(l: list[str], last : str = 'and', n : int | None = None) -> str: + """Pretty print a list of strings + + Examples + -------- + .. code-block:: python + + ppliststr(['a', 'b', 'c', 'd']) -> "'a', 'b', 'c' and 'd'" + ppliststr(['a', 'b'], last='or') -> "'a' or 'b'" + ppliststr(['a', 'b', 'c', 'd'], n=3) -> "'a', 'b', 'c' and 'd'" + + """ + n = n if n is not None else len(l) + if n == 0: + return "" + + s, ii, m = "", 0, len(l) + while ii < m: + item = l[ii] + if ii == n: + s += f" {last} more ..." + break + if ii == 0: + s += f"'{item}'" + elif ii == len(l) - 1: + s += f" {last} '{item}'" + else: + s += f", '{item}'" + ii += 1 + return s \ No newline at end of file From 85b2e2f9f66f913730332d0289f828b18546b1ff Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Tue, 28 Oct 2025 09:28:58 +0100 Subject: [PATCH 49/71] continue refactoring --- argopy/related/sensors/spec.py | 142 ++++++++++++++++++++++++++++++--- 1 file changed, 132 insertions(+), 10 deletions(-) diff --git a/argopy/related/sensors/spec.py b/argopy/related/sensors/spec.py index d13a4efd2..a1a033c30 100644 --- a/argopy/related/sensors/spec.py +++ b/argopy/related/sensors/spec.py @@ -24,7 +24,7 @@ from ...utils import path2assets, register_accessor from .. import ArgoNVSReferenceTables -from .references import SensorReferences +from .references import SensorReferences, SensorModel, SensorType log = logging.getLogger("argopy.related.sensors") @@ -39,7 +39,15 @@ class ArgoSensor: - def __init__(self, *args, **kwargs) -> None: + + __slots__ = [ + "_fs", + "_cache", # To cache extensions, not the option for filesystems + "_model", + "_type", + ] + + def __init__(self, model: str | None = None, *args, **kwargs) -> None: if kwargs.get("fs", None) is None: self._fs = httpstore( cache=kwargs.get("cache", True), @@ -49,6 +57,105 @@ def __init__(self, *args, **kwargs) -> None: else: self._fs = kwargs["fs"] + self._model: SensorModel | None = None + self._type: SensorType | None = None + if model is not None: + try: + df = self.ref.model.search(model) + except DataNotFound: + raise DataNotFound( + f"No sensor model named '{model}', as per ArgoSensor().ref.model.hint() values, based on Ref. Table 27." + ) + + if df.shape[0] == 1: + self._model = SensorModel.from_series(df.iloc[0]) + self._type = self.ref.model.to_type(self._model, errors="ignore") + # if "RBR" in self._model: + # Add the RBR OEM API Authorization key for this sensor: + # fs_kargs.update(client_kwargs={'headers': {'Authorization': OPTIONS.get('rbr_api_key') }}) + else: + raise InvalidDatasetStructure( + f"Found multiple sensor models with '{model}'. Restrict your sensor model name to only one value in: {to_list(df['altLabel'].values)}" + ) + + @property + def model(self) -> SensorModel: + """:class:`SensorModel` of this class instance + + Only available for a class instance created with an explicit sensor model name. + + Returns + ------- + :class:`SensorModel` + + Raises + ------ + :class:`InvalidDataset` + """ + if isinstance(self._model, SensorModel): + return self._model + else: + raise InvalidDataset( + "The 'model' property is not available for an ArgoSensor instance not created with a specific sensor model" + ) + + @property + def type(self) -> SensorType: + """:class:`SensorType` of this class instance sensor model + + Only available for a class instance created with an explicit sensor model name. + + Returns: + ------- + :class:`SensorType` + + Raises + ------ + :class:`InvalidDataset` + """ + if isinstance(self._type, SensorType): + return self._type + else: + raise InvalidDataset( + "The 'type' property is not available for an ArgoSensor instance not created with a specific sensor model" + ) + + def __repr__(self) -> str: + if isinstance(self._model, SensorModel): + summary = [f""] + summary.append(f"TYPE➀ {self.type.long_name}") + summary.append(f"MODEL➀ {self.model.long_name}") + if self.model.deprecated: + summary.append("β›” This model is deprecated !") + else: + summary.append("βœ… This model is not deprecated.") + summary.append(f"πŸ”— {self.model.uri}") + summary.append(f"❝{self.model.definition}❞") + else: + summary = [""] + summary.append( + "This instance was not created with a sensor model name, you still have access to the following:" + ) + summary.append("πŸ‘‰ attributes: ") + for attr in [ + "reference_model", + "reference_model_name", + "reference_sensor", + "reference_sensor_type", + "reference_manufacturer", + "reference_manufacture_name", + ]: + summary.append(f" β•°β”ˆβž€ ArgoSensor().{attr}") + + summary.append("πŸ‘‰ methods: ") + for meth in [ + "search_model", + "search", + "iterfloats_with", + ]: + summary.append(f" β•°β”ˆβž€ ArgoSensor().{meth}()") + return "\n".join(summary) + def _search_wmo_with( self, model: str | list[str], errors: ErrorOptions = "raise" ) -> list[int]: @@ -140,7 +247,11 @@ def _floats_api( return sns def _search_sn_with( - self, model: str, progress: bool = False, errors: ErrorOptions = "raise", **kwargs + self, + model: str, + progress: bool = False, + errors: ErrorOptions = "raise", + **kwargs, ) -> list[str]: """Return serial number of sensor models with a given string in name @@ -173,7 +284,11 @@ def postprocess(data, **kwargs): ) def _search_wmo_sn_with( - self, model: str, progress: bool = False, errors: ErrorOptions = "raise", **kwargs + self, + model: str, + progress: bool = False, + errors: ErrorOptions = "raise", + **kwargs, ) -> dict[int, str]: """Return a dictionary of float WMOs with their sensor serial numbers @@ -205,7 +320,11 @@ def postprocess(data, **kwargs): return dict(sorted(results.items())) def _to_dataframe( - self, model: str, progress: bool = False, errors: ErrorOptions = "raise", **kwargs + self, + model: str, + progress: bool = False, + errors: ErrorOptions = "raise", + **kwargs, ) -> pd.DataFrame: """Return a DataFrame with WMO, sensor type, model, maker, sn, units, accuracy and resolution @@ -251,7 +370,7 @@ def postprocess(data, **kwargs): "Type": sid, "Model": model, "Maker": maker, - "SerialNumber": sn if sn != 'n/a' else None, + "SerialNumber": sn if sn != "n/a" else None, "Units": units, "Accuracy": accuracy, "Resolution": resolution, @@ -267,7 +386,7 @@ def postprocess(data, **kwargs): progress=progress, errors=errors, ) - return df.sort_values(by='WMO', axis=0).reset_index(drop=True) + return df.sort_values(by="WMO", axis=0).reset_index(drop=True) def _search_single( self, @@ -380,7 +499,6 @@ def _search_multi( # Create new key: results.update({wmo: data[wmo]}) - if output != "wmo_sn": # Only keep non-empty results: results = [r for r in results if r is not None] @@ -389,8 +507,12 @@ def _search_multi( if len(results) > 0: if output == "df": - results = [r for r in results if r.shape[0]>0] - return pd.concat(results, axis=0).sort_values(by='WMO', axis=0).reset_index(drop=True) + results = [r for r in results if r.shape[0] > 0] + return ( + pd.concat(results, axis=0) + .sort_values(by="WMO", axis=0) + .reset_index(drop=True) + ) else: return results raise DataNotFound(ppliststr(models)) From 13575b1023b20b8e814564999421361be18846ef Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Tue, 28 Oct 2025 10:02:38 +0100 Subject: [PATCH 50/71] Refactor manufacturer as maker --- argopy/related/sensors/references.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/argopy/related/sensors/references.py b/argopy/related/sensors/references.py index 019eba705..0c12fc64b 100644 --- a/argopy/related/sensors/references.py +++ b/argopy/related/sensors/references.py @@ -101,7 +101,7 @@ class SensorReferenceHolder(ABC): """NVS Reference table for Argo sensor types (R25)""" _r26: pd.DataFrame | None = None - """NVS Reference table for Argo sensor manufacturer (R26)""" + """NVS Reference table for Argo sensor maker (R26)""" _r27: pd.DataFrame | None = None """NVS Reference table for Argo sensor models (R27)""" @@ -134,7 +134,7 @@ def r25(self): @property def r26(self): - """NVS Reference table for Argo sensor manufacturer (R26)""" + """NVS Reference table for Argo sensor maker (R26)""" if self._r26 is None: self._r26 = ArgoNVSReferenceTables(fs=self._fs).tbl("R26") return self._r26 @@ -366,10 +366,10 @@ def to_model( class SensorReferenceR26(SensorReferenceHolder): - """Argo sensor manufacturer""" + """Argo sensor maker""" def to_dataframe(self) -> pd.DataFrame: - """Official reference table for Argo sensor manufacturers (R26) + """Official reference table for Argo sensor makers (R26) Returns ------- @@ -418,9 +418,9 @@ def __call__(self, *args, **kwargs) -> NoReturn: class SensorExtension(SensorReferenceR25): _name = "ref.type" -@register_accessor('manufacturer', SensorReferences) -class ManufacturerExtension(SensorReferenceR26): - _name = "ref.manufacturer" +@register_accessor('maker', SensorReferences) +class MakerExtension(SensorReferenceR26): + _name = "ref.maker" @register_accessor('model', SensorReferences) class ModelExtension(SensorReferenceR27): From f84e19637f4bdb15bc7dddcbce07660dfaae598a Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Tue, 28 Oct 2025 10:03:13 +0100 Subject: [PATCH 51/71] refactor .model to .vacabulary + docstrigns --- argopy/related/sensors/spec.py | 186 +++++++++++++++++++++++++++------ 1 file changed, 153 insertions(+), 33 deletions(-) diff --git a/argopy/related/sensors/spec.py b/argopy/related/sensors/spec.py index a1a033c30..9dafa09ee 100644 --- a/argopy/related/sensors/spec.py +++ b/argopy/related/sensors/spec.py @@ -30,10 +30,12 @@ log = logging.getLogger("argopy.related.sensors") # Define allowed values as a tuple +# (for argument validation) SearchOutput = ("wmo", "sn", "wmo_sn", "df") Error = ("raise", "ignore", "silent") # Define Literal types using tuples +# (for typing) SearchOutputOptions = Literal[*SearchOutput] ErrorOptions = Literal[*Error] @@ -43,11 +45,128 @@ class ArgoSensor: __slots__ = [ "_fs", "_cache", # To cache extensions, not the option for filesystems - "_model", + "_vocabulary", "_type", ] def __init__(self, model: str | None = None, *args, **kwargs) -> None: + """Create an instance of :class:`ArgoSensor` + + Parameters + ---------- + model: str, optional + An exact sensor model name, None by default because this is optional. + + Possible values can be obtained from :meth:`ArgoSensor.ref.model.hint`. + + Other Parameters + ---------------- + cache : bool, optional, default: True + Use cache or not for fetched data + cachedir: str, optional, default: OPTIONS['cachedir'] + Folder where to store cached files. + timeout: int, optional, default: OPTIONS['api_timeout'] + Time out in seconds to connect to web API. + + Examples + -------- + .. code-block:: python + :caption: Access reference tables for 'SENSOR_MODEL'/R27, 'SENSOR'/R25 and 'SENSOR_MAKER'/R26 + + from argopy import ArgoSensor + + ArgoSensor().ref.model.to_dataframe() # Return reference table R27 with the list of sensor models as a DataFrame + ArgoSensor().ref.model.to_list() # Return list of sensor model names (possible values for 'SENSOR_MODEL' parameter) + ArgoSensor().ref.model.to_type('SBE61') # Return sensor type (R25) of a given model (R27) + + ArgoSensor().ref.type.to_dataframe() # Return reference table R25 with the list of sensor types as a DataFrame + ArgoSensor().ref.type.to_list() # Return list of sensor types (possible values for 'SENSOR' parameter) + ArgoSensor().ref.type.to_model('FLUOROMETER_CDOM') # Return all possible model names (R27) for a given sensor type (R25) + + ArgoSensor().ref.maker.to_dataframe() # Return reference table R26 with the list of manufacturer as a DataFrame + ArgoSensor().ref.maker.to_list() # Return list of manufacturer names (possible values for 'SENSOR_MAKER' parameter) + + .. code-block:: python + :caption: Search in 'SENSOR_MODEL'/R27 reference table + + from argopy import ArgoSensor + + ArgoSensor().ref.model.search('RBR') # Search and return a DataFrame + ArgoSensor().ref.model.search('RBR', output='name') # Search and return a list of names instead + ArgoSensor().ref.model.search('SBE61*') # Use of wildcards + ArgoSensor().ref.model.search('*Deep*') # search is case insensitive + + .. code-block:: python + :caption: Search for all Argo floats equipped with one or more sensor model + + from argopy import ArgoSensor + + sensors = ArgoSensor() + + # Search and return a list of WMOs equipped + sensors.search('SBE61_V5.0.2') + + # Search and return a list of sensor serial numbers in Argo + sensors.search('ECO_FLBBCD_AP2', output='sn') + sensors.search('ECO_FLBBCD_AP2', output='sn', progress=False) + + # Search and return a list of tuples with WMOs and serial numbers for those equipped with this model + ArgoSensor().search('SBE', output='wmo_sn') + ArgoSensor().search('SBE', output='wmo_sn', progress=True) + + # Search and return a DataFrame with full sensor information from floats equipped + ArgoSensor().search('RBR', output='df') + + # Search by model, can take a list of string, not necessarily a single value: + ArgoSensor().search(['ECO_FLBBCD_AP2', 'ECO_FLBBCD']) + + + .. code-block:: python + :caption: Easily loop through `ArgoFloat` instances for each float equipped with a sensor model + + from argopy import ArgoSensor + + model = "RAFOS" + for af in ArgoSensor().iterfloats_with(model): + print(af.WMO) + + # Example for how to use the metadata attribute of an ArgoFloat instance: + model = "RAFOS" + for af in ArgoSensor().iterfloats_with(model): + models = af.metadata['sensors'] + for s in models: + if model in s['model']: + print(af.WMO, s['maker'], s['model'], s['serial']) + + .. code-block:: python + :caption: Use an exact sensor model name to create an instance + + from argopy import ArgoSensor + + sensor = ArgoSensor('RBR_ARGO3_DEEP6K') + + sensor.vocabulary + sensor.type + + sensor.search(output='wmo') + sensor.search(output='sn') + sensor.search(output='wmo_sn') + + .. code-block:: bash + :caption: Get clean search results from the command-line with :class:`ArgoSensor.cli_search` + + python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('RBR', output='wmo')" + + python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('RBR', output='sn')" + + + Notes + ----- + Related ADMT/AVTT work: + - https://github.com/OneArgo/ADMT/issues/112 + - https://github.com/OneArgo/ArgoVocabs/issues/156 + - https://github.com/OneArgo/ArgoVocabs/issues/157 + """ if kwargs.get("fs", None) is None: self._fs = httpstore( cache=kwargs.get("cache", True), @@ -57,7 +176,7 @@ def __init__(self, model: str | None = None, *args, **kwargs) -> None: else: self._fs = kwargs["fs"] - self._model: SensorModel | None = None + self._vocabulary: SensorModel | None = None self._type: SensorType | None = None if model is not None: try: @@ -68,9 +187,9 @@ def __init__(self, model: str | None = None, *args, **kwargs) -> None: ) if df.shape[0] == 1: - self._model = SensorModel.from_series(df.iloc[0]) - self._type = self.ref.model.to_type(self._model, errors="ignore") - # if "RBR" in self._model: + self._vocabulary = SensorModel.from_series(df.iloc[0]) + self._type = self.ref.model.to_type(self._vocabulary, errors="ignore") + # if "RBR" in self._vocabulary: # Add the RBR OEM API Authorization key for this sensor: # fs_kargs.update(client_kwargs={'headers': {'Authorization': OPTIONS.get('rbr_api_key') }}) else: @@ -79,10 +198,10 @@ def __init__(self, model: str | None = None, *args, **kwargs) -> None: ) @property - def model(self) -> SensorModel: - """:class:`SensorModel` of this class instance + def vocabulary(self) -> SensorModel: + """Argo reference "SENSOR_MODEL" vocabulary for this sensor model - Only available for a class instance created with an explicit sensor model name. + ! Only available for a class instance created with an explicit sensor model name. Returns ------- @@ -92,18 +211,18 @@ def model(self) -> SensorModel: ------ :class:`InvalidDataset` """ - if isinstance(self._model, SensorModel): - return self._model + if isinstance(self._vocabulary, SensorModel): + return self._vocabulary else: raise InvalidDataset( - "The 'model' property is not available for an ArgoSensor instance not created with a specific sensor model" + "The 'vocabulary' property is not available for an ArgoSensor instance not created with a specific sensor model" ) @property def type(self) -> SensorType: - """:class:`SensorType` of this class instance sensor model + """Argo reference "SENSOR" vocabulary for this sensor model - Only available for a class instance created with an explicit sensor model name. + ! Only available for a class instance created with an explicit sensor model name. Returns: ------- @@ -121,35 +240,36 @@ def type(self) -> SensorType: ) def __repr__(self) -> str: - if isinstance(self._model, SensorModel): - summary = [f""] + if isinstance(self._vocabulary, SensorModel): + summary = [f""] summary.append(f"TYPE➀ {self.type.long_name}") - summary.append(f"MODEL➀ {self.model.long_name}") - if self.model.deprecated: + summary.append(f"MODEL➀ {self.vocabulary.long_name}") + if self.vocabulary.deprecated: summary.append("β›” This model is deprecated !") else: summary.append("βœ… This model is not deprecated.") - summary.append(f"πŸ”— {self.model.uri}") - summary.append(f"❝{self.model.definition}❞") + summary.append(f"πŸ”— {self.vocabulary.uri}") + summary.append(f"❝{self.vocabulary.definition}❞") else: summary = [""] summary.append( "This instance was not created with a sensor model name, you still have access to the following:" ) - summary.append("πŸ‘‰ attributes: ") + summary.append("πŸ‘‰ extensions: ") for attr in [ - "reference_model", - "reference_model_name", - "reference_sensor", - "reference_sensor_type", - "reference_manufacturer", - "reference_manufacture_name", + "ref.model.to_dataframe()", + "ref.model.hint()", + "ref.model.search", + "ref.type.to_dataframe()", + "ref.type.hint()", + "ref.maker.to_dataframe()", + "ref.maker.hint()", ]: summary.append(f" β•°β”ˆβž€ ArgoSensor().{attr}") summary.append("πŸ‘‰ methods: ") for meth in [ - "search_model", + "search_vocabulary", "search", "iterfloats_with", ]: @@ -337,8 +457,8 @@ def _to_dataframe( ----- No option checking, to be done by caller """ - if model is None and self.model is not None: - model = self.model.name + if model is None and self.vocabulary is not None: + model = self.vocabulary.name def preprocess(jsdata, model_name: str = ""): output = [] @@ -582,9 +702,9 @@ def search( ) if model is None: - if self.model is not None: + if self.vocabulary is not None: return self._search_single( - model=self.model.name, + model=self.vocabulary.name, output=output, progress=progress, errors=errors, @@ -666,8 +786,8 @@ class References(SensorReferences): ArgoSensor.ref.sensor.hint() # Return list of sensor types (possible values for 'SENSOR' parameter) ArgoSensor.ref.sensor.to_model('FLUOROMETER_CDOM') # Return all possible model names (R27) for a given sensor type (R25) - ArgoSensor.ref.manufacturer.to_dataframe() # Return reference table R26 with the list of manufacturer as a DataFrame - ArgoSensor.ref.manufacturer.hint() # Return list of manufacturer names (possible values for 'SENSOR_MAKER' parameter) + ArgoSensor.ref.maker.to_dataframe() # Return reference table R26 with the list of manufacturer as a DataFrame + ArgoSensor.ref.maker.hint() # Return list of manufacturer names (possible values for 'SENSOR_MAKER' parameter) ArgoSensor.ref.model.search('RBR') # Search for all (R27) referenced sensor models with some string in their name, return a DataFrame ArgoSensor.ref.model.search('RBR', output='name') # Return a list of names instead From 370ee36c6db5dbc3f6b6ef3c9c2d35df7d7c1ab1 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Tue, 28 Oct 2025 12:30:11 +0100 Subject: [PATCH 52/71] refactor iterfloats --- argopy/related/sensors/spec.py | 291 ++++++++++++++++++++++----------- argopy/stores/float/spec.py | 4 +- 2 files changed, 201 insertions(+), 94 deletions(-) diff --git a/argopy/related/sensors/spec.py b/argopy/related/sensors/spec.py index 9dafa09ee..e3337265c 100644 --- a/argopy/related/sensors/spec.py +++ b/argopy/related/sensors/spec.py @@ -33,139 +33,166 @@ # (for argument validation) SearchOutput = ("wmo", "sn", "wmo_sn", "df") Error = ("raise", "ignore", "silent") +Ds = ("core", "deep", "bgc") # Define Literal types using tuples # (for typing) SearchOutputOptions = Literal[*SearchOutput] ErrorOptions = Literal[*Error] +DsOptions = Literal[*Ds] class ArgoSensor: + """Argo sensor(s) helper class - __slots__ = [ - "_fs", - "_cache", # To cache extensions, not the option for filesystems - "_vocabulary", - "_type", - ] + The :class:`ArgoSensor` class aims to provide direct access to Argo's sensor metadata from: - def __init__(self, model: str | None = None, *args, **kwargs) -> None: - """Create an instance of :class:`ArgoSensor` + - NVS Reference Tables `Sensor Models (R27) `_, `Sensor Types (R25) `_ and `Sensor Manufacturers (R26) `_ + - `Euro-Argo fleet-monitoring API `_ - Parameters - ---------- - model: str, optional - An exact sensor model name, None by default because this is optional. + This enables users to: - Possible values can be obtained from :meth:`ArgoSensor.ref.model.hint`. + - navigate reference tables 27, 25 and 26, + - retrieve sensor serial numbers across the global array, + - search for/iterate over floats equipped with specific sensor models. - Other Parameters - ---------------- - cache : bool, optional, default: True - Use cache or not for fetched data - cachedir: str, optional, default: OPTIONS['cachedir'] - Folder where to store cached files. - timeout: int, optional, default: OPTIONS['api_timeout'] - Time out in seconds to connect to web API. - Examples - -------- - .. code-block:: python - :caption: Access reference tables for 'SENSOR_MODEL'/R27, 'SENSOR'/R25 and 'SENSOR_MAKER'/R26 + Examples + -------- + .. code-block:: python + :caption: Access reference tables for 'SENSOR_MODEL'/R27, 'SENSOR'/R25 and 'SENSOR_MAKER'/R26 - from argopy import ArgoSensor + from argopy import ArgoSensor - ArgoSensor().ref.model.to_dataframe() # Return reference table R27 with the list of sensor models as a DataFrame - ArgoSensor().ref.model.to_list() # Return list of sensor model names (possible values for 'SENSOR_MODEL' parameter) - ArgoSensor().ref.model.to_type('SBE61') # Return sensor type (R25) of a given model (R27) + ArgoSensor().ref.model.to_dataframe() # Return reference table R27 with the list of sensor models as a DataFrame + ArgoSensor().ref.model.to_list() # Return list of sensor model names (possible values for 'SENSOR_MODEL' parameter) + ArgoSensor().ref.model.to_type('SBE61') # Return sensor type (R25) of a given model (R27) - ArgoSensor().ref.type.to_dataframe() # Return reference table R25 with the list of sensor types as a DataFrame - ArgoSensor().ref.type.to_list() # Return list of sensor types (possible values for 'SENSOR' parameter) - ArgoSensor().ref.type.to_model('FLUOROMETER_CDOM') # Return all possible model names (R27) for a given sensor type (R25) + ArgoSensor().ref.type.to_dataframe() # Return reference table R25 with the list of sensor types as a DataFrame + ArgoSensor().ref.type.to_list() # Return list of sensor types (possible values for 'SENSOR' parameter) + ArgoSensor().ref.type.to_model('FLUOROMETER_CDOM') # Return all possible model names (R27) for a given sensor type (R25) - ArgoSensor().ref.maker.to_dataframe() # Return reference table R26 with the list of manufacturer as a DataFrame - ArgoSensor().ref.maker.to_list() # Return list of manufacturer names (possible values for 'SENSOR_MAKER' parameter) + ArgoSensor().ref.maker.to_dataframe() # Return reference table R26 with the list of manufacturer as a DataFrame + ArgoSensor().ref.maker.to_list() # Return list of manufacturer names (possible values for 'SENSOR_MAKER' parameter) - .. code-block:: python - :caption: Search in 'SENSOR_MODEL'/R27 reference table + .. code-block:: python + :caption: Search in 'SENSOR_MODEL'/R27 reference table - from argopy import ArgoSensor + from argopy import ArgoSensor - ArgoSensor().ref.model.search('RBR') # Search and return a DataFrame - ArgoSensor().ref.model.search('RBR', output='name') # Search and return a list of names instead - ArgoSensor().ref.model.search('SBE61*') # Use of wildcards - ArgoSensor().ref.model.search('*Deep*') # search is case insensitive + ArgoSensor().ref.model.search('RBR') # Search and return a DataFrame + ArgoSensor().ref.model.search('RBR', output='name') # Search and return a list of names instead + ArgoSensor().ref.model.search('SBE61*') # Use of wildcards + ArgoSensor().ref.model.search('*Deep*') # Search is case-insensitive - .. code-block:: python - :caption: Search for all Argo floats equipped with one or more sensor model + .. code-block:: python + :caption: Search for all Argo floats equipped with one or more exact sensor model(s) - from argopy import ArgoSensor + from argopy import ArgoSensor - sensors = ArgoSensor() + sensors = ArgoSensor() - # Search and return a list of WMOs equipped - sensors.search('SBE61_V5.0.2') + # Search and return a list of WMOs equipped + sensors.search('SBE61_V5.0.2') - # Search and return a list of sensor serial numbers in Argo - sensors.search('ECO_FLBBCD_AP2', output='sn') - sensors.search('ECO_FLBBCD_AP2', output='sn', progress=False) + # Search and return a list of sensor serial numbers in Argo + sensors.search('ECO_FLBBCD_AP2', output='sn') - # Search and return a list of tuples with WMOs and serial numbers for those equipped with this model - ArgoSensor().search('SBE', output='wmo_sn') - ArgoSensor().search('SBE', output='wmo_sn', progress=True) + # Search and return a list of tuples with WMOs and sensors serial number + sensors.search('SBE', output='wmo_sn') - # Search and return a DataFrame with full sensor information from floats equipped - ArgoSensor().search('RBR', output='df') + # Search and return a DataFrame with full sensor information from floats equipped + sensors.search('RBR', output='df') - # Search by model, can take a list of string, not necessarily a single value: - ArgoSensor().search(['ECO_FLBBCD_AP2', 'ECO_FLBBCD']) + # Search multiple models at once + sensors.search(['ECO_FLBBCD_AP2', 'ECO_FLBBCD']) - .. code-block:: python - :caption: Easily loop through `ArgoFloat` instances for each float equipped with a sensor model + .. code-block:: python + :caption: Easily loop through :class:`ArgoFloat` instances for each float equipped with a sensor model - from argopy import ArgoSensor + from argopy import ArgoSensor - model = "RAFOS" - for af in ArgoSensor().iterfloats_with(model): - print(af.WMO) + sensors = ArgoSensor() + + # Trivial example: + model = "RAFOS" + for af in sensors.iterfloats_with(model): + print(af.WMO) + + # Example to gather all platform types for all WMOs equipped with a list of sensor models + model = ['ECO_FLBBCD_AP2', 'ECO_FLBBCD'] + results = {} + for af in sensors.iterfloats_with(model, ds='bgc'): + if 'meta' in af.ls_dataset(): + platform_type = af.metadata['platform']['type'] # e.g. 'PROVOR_V_JUMBO' + if platform_type in results.keys(): + results[platform_type].extend([af.WMO]) + else: + results.update({platform_type: [af.WMO]}) + else: + print(f"No meta file for float {af.WMO}") + print(results.keys()) - # Example for how to use the metadata attribute of an ArgoFloat instance: - model = "RAFOS" - for af in ArgoSensor().iterfloats_with(model): - models = af.metadata['sensors'] - for s in models: - if model in s['model']: - print(af.WMO, s['maker'], s['model'], s['serial']) + .. code-block:: python + :caption: Use an exact sensor model name to create an instance - .. code-block:: python - :caption: Use an exact sensor model name to create an instance + from argopy import ArgoSensor - from argopy import ArgoSensor + sensor = ArgoSensor('RBR_ARGO3_DEEP6K') - sensor = ArgoSensor('RBR_ARGO3_DEEP6K') + sensor.vocabulary + sensor.type - sensor.vocabulary - sensor.type + sensor.search(output='wmo') + sensor.search(output='sn') + sensor.search(output='wmo_sn') - sensor.search(output='wmo') - sensor.search(output='sn') - sensor.search(output='wmo_sn') + .. code-block:: bash + :caption: Get clean search results from the command-line with :class:`ArgoSensor.cli_search` - .. code-block:: bash - :caption: Get clean search results from the command-line with :class:`ArgoSensor.cli_search` + python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('RBR', output='wmo')" - python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('RBR', output='wmo')" + python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('RBR', output='sn')" - python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('RBR', output='sn')" + Notes + ----- + Related ADMT/AVTT work: + - https://github.com/OneArgo/ADMT/issues/112 + - https://github.com/OneArgo/ArgoVocabs/issues/156 + - https://github.com/OneArgo/ArgoVocabs/issues/157 + + """ + + __slots__ = [ + "_vocabulary", # R27 row for an instance + "_type", # R25 row for an instance + "_fs", # http filesystem, extensions shall use it as well + "_cache", # To cache extensions, not the option for filesystems + ] + + def __init__(self, model: str | None = None, *args, **kwargs) -> None: + """Create an instance of :class:`ArgoSensor` + + Parameters + ---------- + model: str, optional + An exact sensor model name, None by default because this is optional. + + Otherwise, possible values can be obtained from :meth:`ArgoSensor.ref.model.hint`. + + Other Parameters + ---------------- + fs: :class:`stores.httpstore`, default: None + The http filesystem to use. If None is provided, we instantiate a new one based on `cache`, `cachedir` and `timeout` options. + cache : bool, optional, default: True + Use cache or not for fetched data. Used only if `fs` is None. + cachedir: str, optional, default: OPTIONS['cachedir'] + Folder where to store cached files. Used only if `fs` is None. + timeout: int, optional, default: OPTIONS['api_timeout'] + Time out in seconds to connect to web API. Used only if `fs` is None. - Notes - ----- - Related ADMT/AVTT work: - - https://github.com/OneArgo/ADMT/issues/112 - - https://github.com/OneArgo/ArgoVocabs/issues/156 - - https://github.com/OneArgo/ArgoVocabs/issues/157 """ if kwargs.get("fs", None) is None: self._fs = httpstore( @@ -259,9 +286,11 @@ def __repr__(self) -> str: for attr in [ "ref.model.to_dataframe()", "ref.model.hint()", + "ref.model.to_type", "ref.model.search", "ref.type.to_dataframe()", "ref.type.hint()", + "ref.type.to_model", "ref.maker.to_dataframe()", "ref.maker.hint()", ]: @@ -269,7 +298,6 @@ def __repr__(self) -> str: summary.append("πŸ‘‰ methods: ") for meth in [ - "search_vocabulary", "search", "iterfloats_with", ]: @@ -694,11 +722,11 @@ def search( """ if output not in SearchOutput: raise OptionValueError( - f"Invalid 'output' option value '{output}', must be in: {SearchOutput}" + f"Invalid 'output' option value '{output}', must be {ppliststr(SearchOutput, last='or')}" ) if errors not in Error: raise OptionValueError( - f"Invalid 'errors' option value '{errors}', must be in: {Error}" + f"Invalid 'errors' option value '{errors}', must be in: {ppliststr(Error, last='or')}" ) if model is None: @@ -767,6 +795,85 @@ def get_hints(these_models: str | list[str]): **kwargs, ) + def iterfloats_with( + self, + model: str | list[str] | None = None, + chunksize: int | None = None, + # ds: DsOptions = 'core', + **kwargs + ) -> Iterator[ArgoFloat]: + """Iterator over :class:`argopy.ArgoFloat` equipped with a given sensor model + + By default, iterate over a single float, otherwise use the `chunksize` argument to iterate over chunk of floats. + + Parameters + ---------- + model: str + A string to search in the `sensorModels` field of the Euro-Argo fleet-monitoring API `platformCodes/multi-lines-search` endpoint. + + chunksize: int, optional + Maximum chunk size + + Eg: A value of 5 will create chunks with as many as 5 WMOs each. + + Returns + ------- + Iterator of :class:`argopy.ArgoFloat` + + Examples + -------- + .. code-block:: python + :caption: Example of iteration + + sensors = ArgoSensor() + + for afloat in sensors.iterfloats_with("RAFOS"): + print(afloat.WMO) + + """ + if model is None: + if self.vocabulary is not None: + model = self.vocabulary.name, + else: + raise OptionValueError("You must specify at list one model to search !") + + models = to_list(model) + WMOs = self.search(model=models, progress=False) + + # Hidden option because I'm not 100% sure this will be needed. + # ds: str, Literal['core', 'bgc', 'deep'], default='core' + # The Argo mission for this collection of floats. + # This will be used to create an :class:`ArgoIndex` shared by all :class:`ArgoFloat` instances. + ds = kwargs.get('ds', 'core') + if ds not in Ds: + raise OptionValueError( + f"Invalid 'ds' option value '{ds}', must be {ppliststr(Ds)}" + ) + else: + if ds == 'deep': + ds = 'core' + elif ds == 'bgc': + ds = 'bgc-b' + + idx = ArgoIndex( + index_file = ds, + fs = self._fs, + ) + + if chunksize is not None: + chk_opts = {} + chk_opts.update({"chunks": {"wmo": "auto"}}) + chk_opts.update({"chunksize": {"wmo": chunksize}}) + chunked = Chunker( + {"wmo": WMOs}, **chk_opts + ).fit_transform() + for grp in chunked: + yield [ArgoFloat(wmo, idx=idx) for wmo in grp] + + else: + for wmo in WMOs: + yield ArgoFloat(wmo, idx=idx) + @register_accessor("ref", ArgoSensor) class References(SensorReferences): diff --git a/argopy/stores/float/spec.py b/argopy/stores/float/spec.py index af19a64a7..13f732cb2 100644 --- a/argopy/stores/float/spec.py +++ b/argopy/stores/float/spec.py @@ -9,7 +9,7 @@ import logging import numpy as np -from ...errors import InvalidOption +from ...errors import InvalidOption, OptionValueError from ...plot import dashboard from ...utils import check_wmo, argo_split_path, shortcut2gdac from ...options import OPTIONS @@ -308,7 +308,7 @@ def open_dataset(self, name: str = "prof", cast: bool = True, **kwargs) -> xr.Da """ if name not in self.ls_dataset(): - raise ValueError( + raise OptionValueError( "Dataset '%s' not found. Available dataset for this float are: %s" % (name, self.ls_dataset().keys()) ) From e18061f21af93e081c3ad04c889078557b7b9882 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Tue, 28 Oct 2025 13:22:33 +0100 Subject: [PATCH 53/71] end of ArgoSensor refactoring --- argopy/related/sensors/__init__.py | 3 + argopy/related/sensors/references.py | 6 +- argopy/related/sensors/sensors.py | 166 +++ argopy/related/sensors/sensors_deprecated.py | 1118 ------------------ argopy/related/sensors/spec.py | 232 +--- 5 files changed, 229 insertions(+), 1296 deletions(-) create mode 100644 argopy/related/sensors/sensors.py delete mode 100644 argopy/related/sensors/sensors_deprecated.py diff --git a/argopy/related/sensors/__init__.py b/argopy/related/sensors/__init__.py index e69de29bb..4618ffdde 100644 --- a/argopy/related/sensors/__init__.py +++ b/argopy/related/sensors/__init__.py @@ -0,0 +1,3 @@ +from .sensors import ArgoSensor + +__all__ = ["ArgoSensor"] \ No newline at end of file diff --git a/argopy/related/sensors/references.py b/argopy/related/sensors/references.py index 0c12fc64b..b25c5db78 100644 --- a/argopy/related/sensors/references.py +++ b/argopy/related/sensors/references.py @@ -1,9 +1,6 @@ import pandas as pd -from functools import lru_cache from pathlib import Path -import xarray as xr -from typing import Any, Callable, Literal, NoReturn -import inspect +from typing import Literal, NoReturn from abc import ABC, abstractmethod import logging import fnmatch @@ -296,7 +293,6 @@ def search( return data.reset_index(drop=True) - class SensorReferenceR25(SensorReferenceHolder): """Argo sensor types""" diff --git a/argopy/related/sensors/sensors.py b/argopy/related/sensors/sensors.py new file mode 100644 index 000000000..ff931e54b --- /dev/null +++ b/argopy/related/sensors/sensors.py @@ -0,0 +1,166 @@ +from ...utils import register_accessor +from .spec import ArgoSensorSpec +from .references import SensorReferences + + +class ArgoSensor(ArgoSensorSpec): + """Argo sensor(s) helper class + + The :class:`ArgoSensor` class aims to provide direct access to Argo's sensor metadata from: + + - NVS Reference Tables `Sensor Models (R27) `_, `Sensor Types (R25) `_ and `Sensor Manufacturers (R26) `_ + - `Euro-Argo fleet-monitoring API `_ + + This enables users to: + + - navigate reference tables 27, 25 and 26, + - retrieve sensor serial numbers across the global array, + - search for/iterate over floats equipped with specific sensor models. + + + Examples + -------- + .. code-block:: python + :caption: Access reference tables for 'SENSOR_MODEL'/R27, 'SENSOR'/R25 and 'SENSOR_MAKER'/R26 + + from argopy import ArgoSensor + + ArgoSensor().ref.model.to_dataframe() # Return reference table R27 with the list of sensor models as a DataFrame + ArgoSensor().ref.model.to_list() # Return list of sensor model names (possible values for 'SENSOR_MODEL' parameter) + ArgoSensor().ref.model.to_type('SBE61') # Return sensor type (R25) of a given model (R27) + + ArgoSensor().ref.type.to_dataframe() # Return reference table R25 with the list of sensor types as a DataFrame + ArgoSensor().ref.type.to_list() # Return list of sensor types (possible values for 'SENSOR' parameter) + ArgoSensor().ref.type.to_model('FLUOROMETER_CDOM') # Return all possible model names (R27) for a given sensor type (R25) + + ArgoSensor().ref.maker.to_dataframe() # Return reference table R26 with the list of manufacturer as a DataFrame + ArgoSensor().ref.maker.to_list() # Return list of manufacturer names (possible values for 'SENSOR_MAKER' parameter) + + .. code-block:: python + :caption: Search in 'SENSOR_MODEL'/R27 reference table + + from argopy import ArgoSensor + + ArgoSensor().ref.model.search('RBR') # Search and return a DataFrame + ArgoSensor().ref.model.search('RBR', output='name') # Search and return a list of names instead + ArgoSensor().ref.model.search('SBE61*') # Use of wildcards + ArgoSensor().ref.model.search('*Deep*') # Search is case-insensitive + + .. code-block:: python + :caption: Search for all Argo floats equipped with one or more exact sensor model(s) + + from argopy import ArgoSensor + + sensors = ArgoSensor() + + # Search and return a list of WMOs equipped + sensors.search('SBE61_V5.0.2') + + # Search and return a list of sensor serial numbers in Argo + sensors.search('ECO_FLBBCD_AP2', output='sn') + + # Search and return a list of tuples with WMOs and sensors serial number + sensors.search('SBE', output='wmo_sn') + + # Search and return a DataFrame with full sensor information from floats equipped + sensors.search('RBR', output='df') + + # Search multiple models at once + sensors.search(['ECO_FLBBCD_AP2', 'ECO_FLBBCD']) + + + .. code-block:: python + :caption: Easily loop through :class:`ArgoFloat` instances for each float equipped with a sensor model + + from argopy import ArgoSensor + + sensors = ArgoSensor() + + # Trivial example: + model = "RAFOS" + for af in sensors.iterfloats_with(model): + print(af.WMO) + + # Example to gather all platform types for all WMOs equipped with a list of sensor models + models = ['ECO_FLBBCD_AP2', 'ECO_FLBBCD'] + results = {} + for af in sensors.iterfloats_with(models): + if 'meta' in af.ls_dataset(): + platform_type = af.metadata['platform']['type'] # e.g. 'PROVOR_V_JUMBO' + if platform_type in results.keys(): + results[platform_type].extend([af.WMO]) + else: + results.update({platform_type: [af.WMO]}) + else: + print(f"No meta file for float {af.WMO}") + print(results.keys()) + + .. code-block:: python + :caption: Use an exact sensor model name to create an instance + + from argopy import ArgoSensor + + sensor = ArgoSensor('RBR_ARGO3_DEEP6K') + + sensor.vocabulary + sensor.type + + sensor.search(output='wmo') + sensor.search(output='sn') + sensor.search(output='wmo_sn') + + .. code-block:: bash + :caption: Get serialised search results from the command-line with :class:`ArgoSensor.cli_search` + + python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('RBR', output='wmo')" + python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('RBR', output='sn')" + python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('RBR', output='wmo_sn')" + python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('RBR', output='df')" + + + Notes + ----- + Related ADMT/AVTT work: + - https://github.com/OneArgo/ADMT/issues/112 + - https://github.com/OneArgo/ArgoVocabs/issues/156 + - https://github.com/OneArgo/ArgoVocabs/issues/157 + + """ + def __init__(self, **kwargs): + super().__init__(**kwargs) + + +@register_accessor("ref", ArgoSensor) +class References(SensorReferences): + """An :class:`ArgoSensor` extension dedicated to reference tables appropriate for sensors + + Examples + -------- + .. code-block:: python + :caption: Access reference tables for 'SENSOR_MODEL'/R27, 'SENSOR'/R25 and 'SENSOR_MAKER'/R26 + + from argopy import ArgoSensor + + ArgoSensor().ref.model.to_dataframe() # Return reference table R27 with the list of sensor models as a DataFrame + ArgoSensor().ref.model.to_list() # Return list of sensor model names (possible values for 'SENSOR_MODEL' parameter) + ArgoSensor().ref.model.to_type('SBE61') # Return sensor type (R25) of a given model (R27) + + ArgoSensor().ref.type.to_dataframe() # Return reference table R25 with the list of sensor types as a DataFrame + ArgoSensor().ref.type.to_list() # Return list of sensor types (possible values for 'SENSOR' parameter) + ArgoSensor().ref.type.to_model('FLUOROMETER_CDOM') # Return all possible model names (R27) for a given sensor type (R25) + + ArgoSensor().ref.maker.to_dataframe() # Return reference table R26 with the list of manufacturer as a DataFrame + ArgoSensor().ref.maker.to_list() # Return list of manufacturer names (possible values for 'SENSOR_MAKER' parameter) + + .. code-block:: python + :caption: Search in 'SENSOR_MODEL'/R27 reference table + + from argopy import ArgoSensor + + ArgoSensor().ref.model.search('RBR') # Search and return a DataFrame + ArgoSensor().ref.model.search('RBR', output='name') # Search and return a list of names instead + ArgoSensor().ref.model.search('SBE61*') # Use of wildcards + ArgoSensor().ref.model.search('*Deep*') # Search is case-insensitive + + """ + _name = "ref" diff --git a/argopy/related/sensors/sensors_deprecated.py b/argopy/related/sensors/sensors_deprecated.py deleted file mode 100644 index 78f497955..000000000 --- a/argopy/related/sensors/sensors_deprecated.py +++ /dev/null @@ -1,1118 +0,0 @@ -import pandas as pd -import numpy as np -from pathlib import Path -from typing import Literal, Any, Iterator -import logging -import concurrent.futures - -from ...stores import ArgoFloat, ArgoIndex, httpstore, filestore -from ...stores.filesystems import ( - tqdm, -) # Safe import, return a lambda if tqdm not available -from ...utils import check_wmo, Chunker, to_list, NVSrow -from ...errors import ( - DataNotFound, - InvalidDataset, - InvalidDatasetStructure, - OptionValueError, -) -from ...options import OPTIONS -from ...utils import path2assets -from .. import ArgoNVSReferenceTables - - -# Define allowed values as a tuple -SearchOutput = ("wmo", "sn", "wmo_sn", "df") -Error = ("raise", "ignore", "silent") - -# Define Literal types using tuples -SearchOutputOptions = Literal[*SearchOutput] -ErrorOptions = Literal[*Error] - -log = logging.getLogger("argopy.related.sensors") - - -class SensorType(NVSrow): - """One single sensor type data from a R25-"Argo sensor types" row - - Examples - -------- - .. code-block:: python - - from argopy import ArgoNVSReferenceTables - - sensor_type = 'CTD' - - df = ArgoNVSReferenceTables().tbl(25) - df_match = df[df["altLabel"].apply(lambda x: x == sensor_type)].iloc[0] - - st = SensorType.from_series(df_match) - - st.name - st.long_name - st.definition - st.deprecated - st.uri - - """ - - reftable = "R25" - - @staticmethod - def from_series(obj: pd.Series) -> "SensorType": - """Create a :class:`SensorType` from a R25-"Argo sensor models" row""" - return SensorType(obj) - - -class SensorModel(NVSrow): - """One single sensor model data from a R27-"Argo sensor models" row - - Examples - -------- - .. code-block:: python - - from argopy import ArgoNVSReferenceTables - - sensor_model = 'AANDERAA_OPTODE_4330F' - - df = ArgoNVSReferenceTables().tbl(27) - df_match = df[df["altLabel"].apply(lambda x: x == sensor_model)].iloc[0] - - sm = SensorModel.from_series(df_match) - - sm.name - sm.long_name - sm.definition - sm.deprecated - sm.uri - """ - - reftable = "R27" - - @staticmethod - def from_series(obj: pd.Series) -> "SensorModel": - """Create a :class:`SensorModel` from a R27-"Argo sensor models" row""" - return SensorModel(obj) - - def __contains__(self, string) -> bool: - return ( - string.lower() in self.name.lower() - or string.lower() in self.long_name.lower() - ) - - -class ArgoSensor: - """Argo sensor 'package' helper class - - A :class:`ArgoSensor` class instance shall represent one float sensor 'package' - - - The :class:`ArgoSensor` class aims to provide direct access to Argo's sensor metadata from: - - - NVS Reference Tables `Sensor Types (R25) `_ and `Sensor Models (R27) `_ - - `Euro-Argo fleet-monitoring API `_ - - This enables users to: - - - navigate reference tables 25 and 27, - - search for/iterate over floats equipped with specific sensor models, - - retrieve sensor serial numbers across the global array, - - """ - - __slots__ = [ - "_cache", - "_cachedir", - "_timeout", - "_fs", - "_r25", - "_r26", - "_r27", - "_r27_to_r25", - "_model", - "_type", - ] - - def __init__( - self, - model: str | None = None, - **kwargs, - ): - """Create an instance of :class:`ArgoSensor` - - Parameters - ---------- - model: str, optional - An exact sensor model name, by default set to None because this is optional. - - Allowed possible values can be obtained with: - :class:`ArgoSensor.reference_model_name` - - Other Parameters - ---------------- - cache : bool, optional, default: False - Use cache or not for fetched data - cachedir: str, optional, default: OPTIONS['cachedir'] - Folder where to store cached files. - timeout: int, optional, default: OPTIONS['api_timeout'] - Time out in seconds to connect to web API - - Examples - -------- - .. code-block:: python - :caption: Access and search reference tables - - from argopy import ArgoSensor - - # Reference table R27-"Argo sensor models" with the list of sensor models - ArgoSensor().reference_model - ArgoSensor().reference_model_name # Only the list of names (used to fill 'SENSOR_MODEL' parameter) - - # Reference table R25-"Argo sensor types" with the list of sensor types - ArgoSensor().reference_sensor - ArgoSensor().reference_sensor_type # Only the list of types (used to fill 'SENSOR' parameter) - - # Reference table R26-"Argo sensor manufacturers" with the list of sensor maker - ArgoSensor().reference_manufaturer - ArgoSensor().reference_manufaturer_name # Only the list of makers (used to fill 'SENSOR_MAKER' parameter) - - # Search for all referenced sensor models with some string in their name - ArgoSensor().search_model('RBR') - ArgoSensor().search_model('RBR', output='name') # Return a list of names instead of a DataFrame - ArgoSensor().search_model('SBE41CP', strict=False) - ArgoSensor().search_model('SBE41CP', strict=True) # Exact string match required - - .. code-block:: python - :caption: Search for Argo floats with some sensor models - - from argopy import ArgoSensor - - # Search and return a list of WMOs equipped with it/them - ArgoSensor().search('RBR', output='wmo') - - # Search and return a list of sensor serial numbers in Argo - ArgoSensor().search('RBR_ARGO3_DEEP6K', output='sn') - ArgoSensor().search('RBR_ARGO3_DEEP6K', output='sn', progress=True) - - # Search and return a list of tuples with WMOs and serial numbers for those equipped with this model - ArgoSensor().search('SBE', output='wmo_sn') - ArgoSensor().search('SBE', output='wmo_sn', progress=True) - - # Search and return a DataFrame with full sensor information from floats equipped - ArgoSensor().search('RBR', output='df') - - # Search by model, can take a list of string, not necessarily a single value: - ArgoSensor().search(['ECO_FLBBCD_AP2', 'ECO_FLBBCD']) - - - .. code-block:: python - :caption: Easily loop through `ArgoFloat` instances for each float equipped with a sensor model - - from argopy import ArgoSensor - - model = "RAFOS" - for af in ArgoSensor().iterfloats_with(model): - print(af.WMO) - - # Example for how to use the metadata attribute of an ArgoFloat instance: - model = "RAFOS" - for af in ArgoSensor().iterfloats_with(model): - models = af.metadata['sensors'] - for s in models: - if model in s['model']: - print(af.WMO, s['maker'], s['model'], s['serial']) - - .. code-block:: python - :caption: Use an exact sensor model name to create an instance - - from argopy import ArgoSensor - - sensor = ArgoSensor('RBR_ARGO3_DEEP6K') - - sensor.model - sensor.type - - sensor.search(output='wmo') - sensor.search(output='sn') - sensor.search(output='wmo_sn') - - .. code-block:: bash - :caption: Get clean search results from the command-line with :class:`ArgoSensor.cli_search` - - python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('RBR', output='wmo')" - - python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('RBR', output='sn')" - - - Notes - ----- - Related ADMT/AVTT work: - - https://github.com/OneArgo/ADMT/issues/112 - - https://github.com/OneArgo/ArgoVocabs/issues/156 - - https://github.com/OneArgo/ArgoVocabs/issues/157 - """ - self._cache = kwargs.get("cache", True) - self._cachedir = kwargs.get("cachedir", OPTIONS["cachedir"]) - self._timeout = kwargs.get("timeout", OPTIONS["api_timeout"]) - fs_kargs = { - "cache": self._cache, - "cachedir": self._cachedir, - "timeout": self._timeout, - } - self._fs = httpstore(**fs_kargs) - - self._r25: pd.DataFrame | None = None # will be loaded when necessary - self._r26: pd.DataFrame | None = None # will be loaded when necessary - self._r27: pd.DataFrame | None = None # will be loaded when necessary - self._load_mappers() # Load r25 model to r27 type mapping dictionary - - self._model: SensorModel | None = None - self._type: SensorType | None = None - if model is not None: - try: - df = self.search_model(model, strict=True) - except DataNotFound: - raise DataNotFound( - f"No sensor model named '{model}', as per ArgoSensor().reference_model_name values, based on Ref. Table 27." - ) - - if df.shape[0] == 1: - self._model = SensorModel.from_series(df.iloc[0]) - self._type = self.model_to_type(self._model, errors="ignore") - # if "RBR" in self._model: - # Add the RBR OEM API Authorization key for this sensor: - # fs_kargs.update(client_kwargs={'headers': {'Authorization': OPTIONS.get('rbr_api_key') }}) - else: - raise InvalidDatasetStructure( - f"Found multiple sensor models with '{model}'. Restrict your sensor model name to only one value in: {to_list(df['altLabel'].values)}" - ) - - def _load_mappers(self): - """Load from static assets file the NVS R25 to R27 key mappings - - These mapping files were download from https://github.com/OneArgo/ArgoVocabs/issues/156. - """ - df = [] - for p in ( - Path(path2assets).joinpath("nvs_R25_R27").glob("NVS_R25_R27_mappings_*.txt") - ): - df.append( - filestore().read_csv( - p, - header=None, - names=["origin", "model", "?", "destination", "type", "??"], - ) - ) - df = pd.concat(df) - df = df.reset_index(drop=True) - self._r27_to_r25: dict[str, str] = {} - df.apply( - lambda row: self._r27_to_r25.update( - {row["model"].strip(): row["type"].strip()} - ), - axis=1, - ) - - @property - def r27_to_r25(self) -> dict[str, str]: - """Dictionary mapping of R27 to R25 - - This mapping is from files for group 1, 2, 2b, 3, 3b and 4(s) downloaded on 2025/10/03 from https://github.com/OneArgo/ArgoVocabs/issues/156. - - Returns - ------- - dict[str, str] - - Notes - ----- - If you think a key is missing or that mapping files bundled with argopy are out of date, please raise an issue at https://github.com/euroargodev/argopy/issues. - """ - if self._r27_to_r25 is None: - self._load_mappers() - return self._r27_to_r25 - - def model_to_type( - self, - model: str | SensorModel | None = None, - errors: Literal["raise", "ignore"] = "raise", - ) -> SensorType | None: - """Get a sensor type for a given sensor model - - All valid sensor model name can be obtained with :attr:`ArgoSensor.reference_model_name`. - - Mapping between sensor model name (R27) and sensor type (R25) are from AVTT work at https://github.com/OneArgo/ArgoVocabs/issues/156. - - Parameters - ---------- - model : str | :class:`SensorModel` - The model to read the sensor type for. - errors : Literal["raise", "ignore"] = "raise" - How to handle possible errors. If set to "ignore", the method will return None. - - Returns - ------- - :class:`SensorType` | None - - See Also - -------- - :attr:`ArgoSensor.type_to_model` - """ - model_name: str = model.name if isinstance(model, SensorModel) else model - sensor_type = self.r27_to_r25.get(model_name, None) - if sensor_type is not None: - row = self.reference_sensor[ - self.reference_sensor["altLabel"].apply(lambda x: x == sensor_type) - ].iloc[0] - return SensorType.from_series(row) - elif errors == "raise": - raise DataNotFound( - f"Can't determine the type of sensor model '{model_name}' (no matching key in ArgoSensor().r27_to_r25 mapper)" - ) - return None - - def type_to_model( - self, - type: str | SensorType, - errors: Literal["raise", "ignore"] = "raise", - ) -> list[str] | None: - """Get all sensor model names of a given sensor type - - All valid sensor types can be obtained with :attr:`ArgoSensor.reference_sensor_type` - - Mapping between sensor model name (R27) and sensor type (R25) are from AVTT work at https://github.com/OneArgo/ArgoVocabs/issues/156. - - Parameters - ---------- - type : str, :class:`SensorType` - The sensor type to read the sensor model name for. - errors : Literal["raise", "ignore"] = "raise" - How to handle possible errors. If set to "ignore", the method will return None. - - Returns - ------- - list[str] - - See Also - -------- - :attr:`ArgoSensor.model_to_type` - """ - sensor_type = type.name if isinstance(type, SensorType) else type - result = [] - for key, val in self.r27_to_r25.items(): - if sensor_type.lower() in val.lower(): - row = self.reference_model[ - self.reference_model["altLabel"].apply(lambda x: x == key) - ].iloc[0] - result.append(SensorModel.from_series(row).name) - if len(result) == 0: - if errors == "raise": - raise DataNotFound( - f"Can't find any sensor model for this type '{sensor_type}' (no matching key in ArgoSensor().r27_to_r25 mapper)" - ) - else: - return None - else: - return result - - @property - def model(self) -> SensorModel: - """:class:`SensorModel` of this class instance - - Only available for a class instance created with an explicit sensor model name. - - Returns - ------- - :class:`SensorModel` - - Raises - ------ - :class:`InvalidDataset` - """ - if isinstance(self._model, SensorModel): - return self._model - else: - raise InvalidDataset( - "The 'model' property is not available for an ArgoSensor instance not created with a specific sensor model" - ) - - @property - def type(self) -> SensorType: - """:class:`SensorType` of this class instance sensor model - - Only available for a class instance created with an explicit sensor model name. - - Returns: - ------- - :class:`SensorType` - - Raises - ------ - :class:`InvalidDataset` - """ - if isinstance(self._type, SensorType): - return self._type - else: - raise InvalidDataset( - "The 'type' property is not available for an ArgoSensor instance not created with a specific sensor model" - ) - - def __repr__(self) -> str: - if isinstance(self._model, SensorModel): - summary = [f""] - summary.append(f"TYPE➀ {self.type.long_name}") - summary.append(f"MODEL➀ {self.model.long_name}") - if self.model.deprecated: - summary.append("β›” This model is deprecated !") - else: - summary.append("βœ… This model is not deprecated.") - summary.append(f"πŸ”— {self.model.uri}") - summary.append(f"❝{self.model.definition}❞") - else: - summary = [""] - summary.append( - "This instance was not created with a sensor model name, you still have access to the following:" - ) - summary.append("πŸ‘‰ attributes: ") - for attr in [ - "reference_model", - "reference_model_name", - "reference_sensor", - "reference_sensor_type", - "reference_manufacturer", - "reference_manufacture_name", - ]: - summary.append(f" β•°β”ˆβž€ ArgoSensor().{attr}") - - summary.append("πŸ‘‰ methods: ") - for meth in [ - "search_model", - "search", - "iterfloats_with", - ]: - summary.append(f" β•°β”ˆβž€ ArgoSensor().{meth}()") - return "\n".join(summary) - - @property - def reference_model(self) -> pd.DataFrame: - """Official reference table for Argo sensor models (R27) - - Returns - ------- - :class:`pandas.DataFrame` - - See Also - -------- - :class:`ArgoNVSReferenceTables` - """ - if self._r27 is None: - self._r27 = ArgoNVSReferenceTables(fs=self._fs).tbl("R27") - return self._r27 - - @property - def reference_model_name(self) -> list[str]: - """Official list of Argo sensor models (R27) - - Return a sorted list of strings with altLabel from Argo Reference table R27 on 'SENSOR_MODEL'. - - Returns - ------- - list[str] - - Notes - ----- - Argo netCDF variable ``SENSOR_MODEL`` is populated with values from this list. - - See Also - -------- - :attr:`ArgoSensor.reference_model` - """ - return sorted(to_list(self.reference_model["altLabel"].values)) - - @property - def reference_sensor(self) -> pd.DataFrame: - """Official reference table for Argo sensor types (R25) - - Returns - ------- - :class:`pandas.DataFrame` - - See Also - -------- - :class:`ArgoNVSReferenceTables` - """ - if self._r25 is None: - self._r25 = ArgoNVSReferenceTables(fs=self._fs).tbl("R25") - return self._r25 - - @property - def reference_sensor_type(self) -> list[str]: - """Official list of Argo sensor types (R25) - - Return a sorted list of strings with altLabel from Argo Reference table R25 on 'SENSOR'. - - Returns - ------- - list[str] - - Notes - ----- - Argo netCDF variable ``SENSOR`` is populated with values from this list. - - See Also - -------- - :attr:`ArgoSensor.reference_sensor` - """ - return sorted(to_list(self.reference_sensor["altLabel"].values)) - - @property - def reference_manufacturer(self) -> pd.DataFrame: - """Official reference table for Argo sensor manufacturers (R26) - - Returns - ------- - :class:`pandas.DataFrame` - - See Also - -------- - :class:`ArgoNVSReferenceTables` - """ - if self._r26 is None: - self._r26 = ArgoNVSReferenceTables(fs=self._fs).tbl("R26") - return self._r26 - - @property - def reference_manufacturer_name(self) -> list[str]: - """Official list of Argo sensor maker (R26) - - Return a sorted list of strings with altLabel from Argo Reference table R26 on 'SENSOR_MAKER'. - - Returns - ------- - list[str] - - Notes - ----- - Argo netCDF variable ``SENSOR_MAKER`` is populated with values from this list. - - See Also - -------- - :attr:`ArgoSensor.reference_manufacturer` - """ - return sorted(to_list(self.reference_manufacturer["altLabel"].values)) - - def search_model( - self, - model: str, - strict: bool = False, - output: Literal["df", "name"] = "df", - ) -> pd.DataFrame | list[str]: - """Return references of Argo sensor models matching a string - - Look for occurrences in Argo Reference table R27 `altLabel` and return a :class:`pandas.DataFrame` with matching row(s). - - Parameters - ---------- - model : str - The model to search for. - strict : bool, optional, default: False - Is the model string a strict match or an occurrence in table look up. - output : str, Literal["df", "name"], default "df" - Is the output a :class:`pandas.DataFrame` with matching rows from :attr:`ArgoSensor.reference_model`, or a list of string. - - Returns - ------- - :class:`pandas.DataFrame`, list[str] - - See Also - -------- - :class:`ArgoSensor.reference_model` - """ - if strict: - data = self.reference_model[ - self.reference_model["altLabel"].apply(lambda x: x == model.upper()) - ] - else: - data = self.reference_model[ - self.reference_model["altLabel"].apply(lambda x: model.upper() in x) - ] - if data.shape[0] == 0: - if strict: - raise DataNotFound( - f"No sensor models matching '{model}'. You may try to search with strict=False." - ) - else: - raise DataNotFound( - f"No sensor model names with '{model}' string occurrence." - ) - else: - if output == "name": - return sorted(to_list(data["altLabel"].values)) - else: - return data.reset_index(drop=True) - - def _search_wmo_with(self, model: str, errors: ErrorOptions = "raise") -> list[int]: - """Return the list of WMOs equipped with a given sensor model - - Notes - ----- - Based on a fleet-monitoring API request to `platformCodes/multi-lines-search` on `sensorModels` field. - - Documentation: - - https://fleetmonitoring.euro-argo.eu/swagger-ui.html#!/platform-code-controller/getPlatformCodesMultiLinesSearchUsingPOST - - Notes - ----- - No option checking, to be done by caller - """ - models = to_list(model) - # Security Issue: models string should be sanitized ? - api_point = f"{OPTIONS['fleetmonitoring']}/platformCodes/multi-lines-search" - payload = [ - { - "nested": False, - "path": "string", - "searchValueType": "Text", - "values": models, - "field": "sensorModels", - } - ] - wmos = self._fs.post(api_point, json_data=payload) - if wmos is None or len(wmos) == 0: - # Handle failed search: - search_hint: list[str] = [] - for model in models: - try: - hint: list[str] = self.search_model( - model, output="name", strict=False - ) - search_hint.extend(hint) - except DataNotFound: - pass - if len(search_hint) > 0: - msg = ( - f"No floats matching this sensor model name {models}. Possible hint: %s" - % ("; ".join(search_hint)) - ) - else: - msg = f"Unknown sensor models: {models}" - - if errors == "raise": - raise DataNotFound(msg) - elif errors == "ignore": - log.error(msg) - return check_wmo(wmos) - - def _floats_api( - self, - model: str, - preprocess=None, - preprocess_opts={}, - postprocess=None, - postprocess_opts={}, - progress=False, - errors: ErrorOptions = "raise", - ) -> Any: - """Search floats with a sensor model and then fetch and process JSON data returned from the fleet-monitoring API for each floats - - Notes - ----- - Based on a POST request to the fleet-monitoring API requests to `/floats/{wmo}`. - - `Endpoint documentation `_. - - Notes - ----- - No option checking, to be done by caller - """ - wmos = self._search_wmo_with(model) - - URI = [] - for wmo in wmos: - URI.append(f"{OPTIONS['fleetmonitoring']}/floats/{wmo}") - - sns = self._fs.open_mfjson( - URI, - preprocess=preprocess, - preprocess_opts=preprocess_opts, - progress=progress, - errors=errors, - ) - - return postprocess(sns, **postprocess_opts) - - def _search_sn_with( - self, model: str, progress=False, errors: ErrorOptions = "raise" - ) -> list[str]: - """Return serial number of sensor models with a given string in name - Notes - ----- - No option checking, to be done by caller - """ - - def preprocess(jsdata, model_name: str = ""): - sn = np.unique( - [s["serial"] for s in jsdata["sensors"] if model_name in s["model"]] - ) - return sn - - def postprocess(data, **kwargs): - S = [] - for row in data: - for sensor in row: - if sensor is not None: - S.append(sensor) - return np.sort(np.array(S)) - - return self._floats_api( - model, - preprocess=preprocess, - preprocess_opts={"model_name": model}, - postprocess=postprocess, - progress=progress, - errors=errors, - ) - - def _search_wmo_sn_with( - self, model: str, progress=False, errors: ErrorOptions = "raise" - ) -> dict[int, str]: - """Return a dictionary of float WMOs with their sensor serial numbers - - Notes - ----- - No option checking, to be done by caller - """ - - def preprocess(jsdata, model_name: str = ""): - sn = np.unique( - [s["serial"] for s in jsdata["sensors"] if model_name in s["model"]] - ) - return [jsdata["wmo"], [str(s) for s in sn]] - - def postprocess(data, **kwargs): - S = {} - for wmo, sn in data: - S[check_wmo(wmo)[0]] = to_list(sn) - return S - - return self._floats_api( - model, - preprocess=preprocess, - preprocess_opts={"model_name": model}, - postprocess=postprocess, - progress=progress, - errors=errors, - ) - - def _to_dataframe( - self, model: str, progress=False, errors: ErrorOptions = "raise" - ) -> pd.DataFrame: - """Return a DataFrame with WMO, sensor type, model, maker, sn, units, accuracy and resolution - - Parameters - ---------- - model: str, optional - A string to search in the `sensorModels` field of the Euro-Argo fleet-monitoring API `platformCodes/multi-lines-search` endpoint. - - Notes - ----- - No option checking, to be done by caller - """ - if model is None and self.model is not None: - model = self.model.name - - def preprocess(jsdata, model_name: str = ""): - output = [] - for s in jsdata["sensors"]: - if model_name in s["model"]: - this = [jsdata["wmo"]] - [ - this.append(s[key]) # type: ignore - for key in [ - "id", - "maker", - "model", - "serial", - "units", - "accuracy", - "resolution", - ] - ] - output.append(this) - return output - - def postprocess(data, **kwargs): - d = [] - for this in data: - for wmo, sid, maker, model, sn, units, accuracy, resolution in this: - d.append( - { - "WMO": wmo, - "Type": sid, - "Model": model, - "Maker": maker, - "SerialNumber": sn, - "Units": units, - "Accuracy": accuracy, - "Resolution": resolution, - } - ) - return pd.DataFrame(d).sort_values(by="WMO").reset_index(drop=True) - - return self._floats_api( - model, - preprocess=preprocess, - preprocess_opts={"model_name": model}, - postprocess=postprocess, - progress=progress, - errors=errors, - ) - - def _search_single( - self, - model: str, - output: SearchOutputOptions = "wmo", - progress: bool = False, - errors: ErrorOptions = "raise", - ) -> list[int] | list[str] | dict[int, str] | pd.DataFrame: - """Run a single model search""" - - if output == "df": - return self._to_dataframe(model=model, progress=progress, errors=errors) - elif output == "sn": - return self._search_sn_with(model=model, progress=progress, errors=errors) - elif output == "wmo_sn": - return self._search_wmo_sn_with( - model=model, progress=progress, errors=errors - ) - else: - return self._search_wmo_with(model=model, errors=errors) - - def _search_multi( - self, - models: list[str], - output: SearchOutputOptions = "wmo", - progress: bool = False, - errors: ErrorOptions = "raise", - max_workers: int | None = None, - ) -> list[int] | list[str] | dict[int, str] | pd.DataFrame: - """Run a multiple models search in parallel with multithreading""" - # Remove duplicates: - models = list(set(models)) - - ConcurrentExecutor = concurrent.futures.ThreadPoolExecutor( - max_workers=max_workers - ) - failed = [] - if output in ["wmo", "sn", "df"]: - results = [] - elif output == "wmo_sn": - results = {} - - with ConcurrentExecutor as executor: - future_to_model = { - executor.submit( - self._search_single, - model, - output=output, - errors=errors, - ): model - for model in models - } - futures = concurrent.futures.as_completed(future_to_model) - if progress: - futures = tqdm( - futures, total=len(models), disable="disable" in [progress] - ) - - for future in futures: - data = None - try: - data = future.result() - except Exception: - failed.append(future_to_model[future]) - if errors == "ignore": - log.error( - "Ignored error with this url: %s" % future_to_model[future] - ) - elif errors == "silent": - pass - else: - raise - finally: - # Gather results according to final output format: - if data is not None: - if output in ["wmo", "sn"]: - results.extend(data) - elif output == "df": - results.append(data) - else: - for wmo in data.keys(): - results.update({wmo: data[wmo]}) - - results = [r for r in results if r is not None] # Only keep non-empty results - if len(results) > 0: - if output == "df": - return pd.concat(results, axis=0).reset_index(drop=True) - else: - return results - raise DataNotFound(models) - - def search( - self, - model: str | list[str] | None = None, - output: SearchOutputOptions = "wmo", - progress: bool = False, - errors: ErrorOptions = "raise", - **kwargs, - ) -> list[int] | list[str] | dict[int, str] | pd.DataFrame: - """Search for Argo floats equipped with a sensor model name - - All information are retrieved with one or more requests to the `Euro-Argo fleet-monitoring API `_. - - Parameters - ---------- - model: str, list[str], optional - One or more models string to search. - - output: str, Literal["wmo", "sn", "wmo_sn", "df"], default "wmo" - Define the output to return: - - - ``wmo``: a list of WMO numbers (integers) - - ``sn``: a list of sensor serial numbers (strings) - - ``wmo_sn``: a list of dictionary with WMO as key and serial numbers as values - - ``df``: a :class:`pandas.DataFrame` with WMO, sensor type/model/maker and serial number - - progress: bool, default False - Define whether to display a progress bar or not - - errors: str, default "raise" - - Returns - ------- - list[int], list[str], dict[int, str], :class:`pandas.DataFrame` - - Notes - ----- - Whatever the output format, the first step is to retrieve a list of WMOs equipped with one or more sensor models. - This is done using the Euro-Argo fleet-monitoring API and a request to the ``platformCodes/multi-lines-search`` endpoint using the ``sensorModels`` search field. - - Then and if necessary (all output format but 'wmo'), the corresponding list of sensor serial numbers are retrieved using one request per float to the Euro-Argo fleet-monitoring API ``/floats/{wmo}`` endpoint. - - Web-api documentation: - - - `Documentation for endpoint: platformCodes/multi-lines-search `_. - - `Documentation for endpoint: /floats/{wmo} `_. - - """ - if output not in SearchOutput: - raise OptionValueError( - f"Invalid 'output' option value '{output}', must be in: {SearchOutput}" - ) - if errors not in Error: - raise OptionValueError( - f"Invalid 'errors' option value '{errors}', must be in: {Error}" - ) - - if model is None: - if self.model is not None: - return self._search_single( - model=self.model.name, - output=output, - progress=progress, - errors=errors, - ) - else: - raise OptionValueError("You must specify at list one model to search !") - - models = to_list(model) - if len(models) == 1: - return self._search_single( - model=model, output=output, progress=progress, errors=errors - ) - else: - return self._search_multi( - models=models, - output=output, - progress=progress, - errors=errors, - **kwargs, - ) - - def iterfloats_with( - self, model: str | None = None, chunksize: int | None = None - ) -> Iterator[ArgoFloat]: - """Iterator over :class:`argopy.ArgoFloat` equipped with a given sensor model - - By default, iterate over a single float, otherwise use the `chunksize` argument to iterate over chunk of floats. - - Parameters - ---------- - model: str - A string to search in the `sensorModels` field of the Euro-Argo fleet-monitoring API `platformCodes/multi-lines-search` endpoint. - - chunksize: int, optional - Maximum chunk size - - Eg: A value of 5 will create chunks with as many as 5 WMOs each. - - Returns - ------- - Iterator of :class:`argopy.ArgoFloat` - - Examples - -------- - .. code-block:: python - :caption: Example of iteration - - from argopy import ArgoSensor - - for afloat in ArgoSensor().iterfloats_with("SATLANTIC_PAR"): - print(f"\n-Float {afloat.WMO}: Platform description = {afloat.metadata['platform']['description']}") - for sensor in afloat.metadata["sensors"]: - if "SATLANTIC_PAR" in sensor["model"]: - print(f" - Sensor Maker: {sensor['maker']}, Serial: {sensor['serial']}") - - """ - if model is None and self.model is not None: - model = self.model.name - - wmos = self._search_wmo_with(model=model) - - idx = ArgoIndex( - index_file="core", - cache=self._cache, - cachedir=self._cachedir, - ) - - if chunksize is not None: - chk_opts = {} - chk_opts.update({"chunks": {"wmo": "auto"}}) - chk_opts.update({"chunksize": {"wmo": chunksize}}) - chunked = Chunker( - {"wmo": self._search_wmo_with(model=model)}, **chk_opts - ).fit_transform() - for grp in chunked: - yield [ArgoFloat(wmo, idx=idx) for wmo in grp] - - else: - for wmo in wmos: - yield ArgoFloat(wmo, idx=idx) - - def cli_search(self, model: str, output: SearchOutputOptions = "wmo") -> str: # type: ignore - """Quick sensor lookups from the terminal - - This function is a command-line-friendly output for float search (e.g., for piping to other tools). - - Examples - -------- - .. code-block:: bash - :caption: Example of search results from the command-line - - python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('RBR', output='wmo')" - - python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('RBR', output='sn')" - - """ - results = self.search(model, output=output) - print("\n".join(map(str, results))) diff --git a/argopy/related/sensors/spec.py b/argopy/related/sensors/spec.py index e3337265c..77fdcc1b4 100644 --- a/argopy/related/sensors/spec.py +++ b/argopy/related/sensors/spec.py @@ -1,19 +1,16 @@ import pandas as pd import numpy as np -from pathlib import Path from typing import Literal, Any, Iterator, Callable -import logging import concurrent.futures -import xarray as xr import logging -import warnings - +import json +import sys -from ...stores import ArgoFloat, ArgoIndex, httpstore, filestore +from ...stores import ArgoFloat, ArgoIndex, httpstore from ...stores.filesystems import ( tqdm, ) # Safe import, return a lambda if tqdm not available -from ...utils import check_wmo, Chunker, to_list, NVSrow, ppliststr, is_wmo +from ...utils import check_wmo, Chunker, to_list, ppliststr, is_wmo from ...errors import ( DataNotFound, InvalidDataset, @@ -21,15 +18,13 @@ OptionValueError, ) from ...options import OPTIONS -from ...utils import path2assets, register_accessor -from .. import ArgoNVSReferenceTables -from .references import SensorReferences, SensorModel, SensorType +from .references import SensorModel, SensorType log = logging.getLogger("argopy.related.sensors") -# Define allowed values as a tuple +# Define some options expected values as tuples # (for argument validation) SearchOutput = ("wmo", "sn", "wmo_sn", "df") Error = ("raise", "ignore", "silent") @@ -42,128 +37,7 @@ DsOptions = Literal[*Ds] -class ArgoSensor: - """Argo sensor(s) helper class - - The :class:`ArgoSensor` class aims to provide direct access to Argo's sensor metadata from: - - - NVS Reference Tables `Sensor Models (R27) `_, `Sensor Types (R25) `_ and `Sensor Manufacturers (R26) `_ - - `Euro-Argo fleet-monitoring API `_ - - This enables users to: - - - navigate reference tables 27, 25 and 26, - - retrieve sensor serial numbers across the global array, - - search for/iterate over floats equipped with specific sensor models. - - - Examples - -------- - .. code-block:: python - :caption: Access reference tables for 'SENSOR_MODEL'/R27, 'SENSOR'/R25 and 'SENSOR_MAKER'/R26 - - from argopy import ArgoSensor - - ArgoSensor().ref.model.to_dataframe() # Return reference table R27 with the list of sensor models as a DataFrame - ArgoSensor().ref.model.to_list() # Return list of sensor model names (possible values for 'SENSOR_MODEL' parameter) - ArgoSensor().ref.model.to_type('SBE61') # Return sensor type (R25) of a given model (R27) - - ArgoSensor().ref.type.to_dataframe() # Return reference table R25 with the list of sensor types as a DataFrame - ArgoSensor().ref.type.to_list() # Return list of sensor types (possible values for 'SENSOR' parameter) - ArgoSensor().ref.type.to_model('FLUOROMETER_CDOM') # Return all possible model names (R27) for a given sensor type (R25) - - ArgoSensor().ref.maker.to_dataframe() # Return reference table R26 with the list of manufacturer as a DataFrame - ArgoSensor().ref.maker.to_list() # Return list of manufacturer names (possible values for 'SENSOR_MAKER' parameter) - - .. code-block:: python - :caption: Search in 'SENSOR_MODEL'/R27 reference table - - from argopy import ArgoSensor - - ArgoSensor().ref.model.search('RBR') # Search and return a DataFrame - ArgoSensor().ref.model.search('RBR', output='name') # Search and return a list of names instead - ArgoSensor().ref.model.search('SBE61*') # Use of wildcards - ArgoSensor().ref.model.search('*Deep*') # Search is case-insensitive - - .. code-block:: python - :caption: Search for all Argo floats equipped with one or more exact sensor model(s) - - from argopy import ArgoSensor - - sensors = ArgoSensor() - - # Search and return a list of WMOs equipped - sensors.search('SBE61_V5.0.2') - - # Search and return a list of sensor serial numbers in Argo - sensors.search('ECO_FLBBCD_AP2', output='sn') - - # Search and return a list of tuples with WMOs and sensors serial number - sensors.search('SBE', output='wmo_sn') - - # Search and return a DataFrame with full sensor information from floats equipped - sensors.search('RBR', output='df') - - # Search multiple models at once - sensors.search(['ECO_FLBBCD_AP2', 'ECO_FLBBCD']) - - - .. code-block:: python - :caption: Easily loop through :class:`ArgoFloat` instances for each float equipped with a sensor model - - from argopy import ArgoSensor - - sensors = ArgoSensor() - - # Trivial example: - model = "RAFOS" - for af in sensors.iterfloats_with(model): - print(af.WMO) - - # Example to gather all platform types for all WMOs equipped with a list of sensor models - model = ['ECO_FLBBCD_AP2', 'ECO_FLBBCD'] - results = {} - for af in sensors.iterfloats_with(model, ds='bgc'): - if 'meta' in af.ls_dataset(): - platform_type = af.metadata['platform']['type'] # e.g. 'PROVOR_V_JUMBO' - if platform_type in results.keys(): - results[platform_type].extend([af.WMO]) - else: - results.update({platform_type: [af.WMO]}) - else: - print(f"No meta file for float {af.WMO}") - print(results.keys()) - - .. code-block:: python - :caption: Use an exact sensor model name to create an instance - - from argopy import ArgoSensor - - sensor = ArgoSensor('RBR_ARGO3_DEEP6K') - - sensor.vocabulary - sensor.type - - sensor.search(output='wmo') - sensor.search(output='sn') - sensor.search(output='wmo_sn') - - .. code-block:: bash - :caption: Get clean search results from the command-line with :class:`ArgoSensor.cli_search` - - python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('RBR', output='wmo')" - - python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('RBR', output='sn')" - - - Notes - ----- - Related ADMT/AVTT work: - - https://github.com/OneArgo/ADMT/issues/112 - - https://github.com/OneArgo/ArgoVocabs/issues/156 - - https://github.com/OneArgo/ArgoVocabs/issues/157 - - """ +class ArgoSensorSpec: __slots__ = [ "_vocabulary", # R27 row for an instance @@ -420,7 +294,7 @@ def postprocess(data, **kwargs): for sensor in row: if sensor is not None: S.append(sensor) - return np.sort(np.array(S)) + return np.sort(np.array(S)).tolist() return self._floats_api( model if kwargs.get("wmo", None) is None else kwargs["wmo"], @@ -671,9 +545,9 @@ def search( output: SearchOutputOptions = "wmo", progress: bool = True, errors: ErrorOptions = "raise", - strict: bool = True, + serialised: bool = False, **kwargs, - ) -> list[int] | list[str] | dict[int, str] | pd.DataFrame: + ) -> list[int] | list[str] | dict[int, str] | pd.DataFrame | str: """Search for Argo floats equipped with one or more sensor model name(s) All information are retrieved from the `Euro-Argo fleet-monitoring API `_. @@ -685,7 +559,7 @@ def search( model: str, list[str], optional One or more exact model names to search data for. - output: str, Literal["wmo", "sn", "wmo_sn", "df"], default "wmo" + output: str, Literal["wmo", "sn", "wmo_sn", "df"], default: "wmo" Define the output to return: - ``wmo``: a list of WMO numbers (integers) @@ -693,11 +567,14 @@ def search( - ``wmo_sn``: a list of dictionary with WMO as key and serial numbers as values - ``df``: a :class:`pandas.DataFrame` with WMO, sensor type/model/maker and serial number - progress: bool, default True + progress: bool, default: True Display a progress bar or not - errors: str, Literal["raise", "ignore", "silent"], default "raise" - Raise an error, log it or do nothing if the search return nothing. + errors: str, Literal["raise", "ignore", "silent"], default: "raise" + Raise an error, log it or do nothing for no search results. + + serialised: bool, default: False + Return a serialised output. This allows for search results to be saved in cross-language, human-readable formats like json. Returns ------- @@ -783,17 +660,56 @@ def get_hints(these_models: str | list[str]): raise OptionValueError(msg) if len(valid_models) == 1: - return self._search_single( + results = self._search_single( model=valid_models[0], output=output, progress=progress, errors=errors ) else: - return self._search_multi( + results = self._search_multi( models=valid_models, output=output, progress=progress, errors=errors, **kwargs, ) + if serialised: + # Serialise all output format to json + if output == 'df': + return results.to_json(indent=2) + return json.dumps(results, indent=2) + return results + + def cli_search(self, + model: str | list[str] | None = None, + output: SearchOutputOptions = "wmo") -> str: # type: ignore + """A command-line-friendly search for Argo floats equipped with one or more sensor model name(s) + + This function is intended to call from the command-line and return serialized results for easy piping to other tools. + + Parameters + ---------- + model: str, list[str], optional + One or more exact model names to search data for. + + output: str, Literal["wmo", "sn", "wmo_sn", "df"], default: "wmo" + Define the output to return: + + - ``wmo``: a list of WMO numbers (integers) + - ``sn``: a list of sensor serial numbers (strings) + - ``wmo_sn``: a list of dictionary with WMO as key and serial numbers as values + - ``df``: a dictionary with 'WMO', 'Type', 'Model', 'Maker', 'SerialNumber', 'Units', 'Accuracy', 'Resolution' keys. + + Examples + -------- + .. code-block:: bash + :caption: Example of search results from the command-line + + python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('RBR', output='wmo')" + python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('RBR', output='sn')" + python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('RBR', output='wmo_sn')" + python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('RBR', output='df')" + + """ + print(self.search(model, output=output, serialised=True, progress=False), file=sys.stdout) def iterfloats_with( self, @@ -840,7 +756,7 @@ def iterfloats_with( models = to_list(model) WMOs = self.search(model=models, progress=False) - # Hidden option because I'm not 100% sure this will be needed. + # 'ds' is a hidden option because I'm not 100% sure this will be needed. # ds: str, Literal['core', 'bgc', 'deep'], default='core' # The Argo mission for this collection of floats. # This will be used to create an :class:`ArgoIndex` shared by all :class:`ArgoFloat` instances. @@ -873,33 +789,3 @@ def iterfloats_with( else: for wmo in WMOs: yield ArgoFloat(wmo, idx=idx) - - -@register_accessor("ref", ArgoSensor) -class References(SensorReferences): - """An :class:`ArgoSensor` extension dedicated to reference tables appropriate for sensors - - Examples - -------- - .. code-block:: python - - from argopy import ArgoSensor - - ArgoSensor.ref.model.to_dataframe() # Return reference table R27 with the list of sensor models as a DataFrame - ArgoSensor.ref.model.hint() # Return list of sensor model names (possible values for 'SENSOR_MODEL' parameter) - ArgoSensor.ref.model.to_type('SBE61') # Return sensor type (R25) of a given model (R27) - - ArgoSensor.ref.sensor.to_dataframe() # Return reference table R25 with the list of sensor types as a DataFrame - ArgoSensor.ref.sensor.hint() # Return list of sensor types (possible values for 'SENSOR' parameter) - ArgoSensor.ref.sensor.to_model('FLUOROMETER_CDOM') # Return all possible model names (R27) for a given sensor type (R25) - - ArgoSensor.ref.maker.to_dataframe() # Return reference table R26 with the list of manufacturer as a DataFrame - ArgoSensor.ref.maker.hint() # Return list of manufacturer names (possible values for 'SENSOR_MAKER' parameter) - - ArgoSensor.ref.model.search('RBR') # Search for all (R27) referenced sensor models with some string in their name, return a DataFrame - ArgoSensor.ref.model.search('RBR', output='name') # Return a list of names instead - ArgoSensor.ref.model.search('SBE41CP', strict=False) - ArgoSensor.ref.model.search('SBE41CP', strict=True) # Exact string match required - """ - - _name = "ref" From c0f5fa874d0e86574f89e9bdf7da10da1b5d7733 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Tue, 28 Oct 2025 13:43:05 +0100 Subject: [PATCH 54/71] let's say more refactoring --- argopy/related/__init__.py | 5 +- argopy/related/sensors/__init__.py | 3 +- argopy/related/sensors/accessories.py | 86 +++++++++++++++++++ argopy/related/sensors/references.py | 119 +++++--------------------- argopy/related/sensors/sensors.py | 2 + argopy/related/sensors/spec.py | 13 +-- 6 files changed, 116 insertions(+), 112 deletions(-) create mode 100644 argopy/related/sensors/accessories.py diff --git a/argopy/related/__init__.py b/argopy/related/__init__.py index ce582f7fe..a11a7799e 100644 --- a/argopy/related/__init__.py +++ b/argopy/related/__init__.py @@ -4,9 +4,8 @@ from .argo_documentation import ArgoDocs from .doi_snapshot import ArgoDOI from .euroargo_api import get_coriolis_profile_id, get_ea_profile_page -from .sensors.spec import ArgoSensor -from .sensors.sensors_deprecated import SensorType, SensorModel -from .utils import load_dict, mapp_dict # Should come last +from .sensors import ArgoSensor, SensorType, SensorModel +from .utils import load_dict, mapp_dict # Must come last to avoid circular import, I know, not good # __all__ = ( diff --git a/argopy/related/sensors/__init__.py b/argopy/related/sensors/__init__.py index 4618ffdde..c5a3e8313 100644 --- a/argopy/related/sensors/__init__.py +++ b/argopy/related/sensors/__init__.py @@ -1,3 +1,4 @@ from .sensors import ArgoSensor +from .accessories import SensorType, SensorModel -__all__ = ["ArgoSensor"] \ No newline at end of file +__all__ = ["ArgoSensor", "SensorType", "SensorModel"] diff --git a/argopy/related/sensors/accessories.py b/argopy/related/sensors/accessories.py new file mode 100644 index 000000000..8db3344f3 --- /dev/null +++ b/argopy/related/sensors/accessories.py @@ -0,0 +1,86 @@ +from typing import Literal +import pandas as pd + +from ...utils import NVSrow + + +# Define some options expected values as tuples +# (for argument validation) +SearchOutput = ("wmo", "sn", "wmo_sn", "df") +Error = ("raise", "ignore", "silent") +Ds = ("core", "deep", "bgc") + +# Define Literal types using tuples +# (for typing) +SearchOutputOptions = Literal[*SearchOutput] +ErrorOptions = Literal[*Error] +DsOptions = Literal[*Ds] + + +class SensorType(NVSrow): + """One single sensor type data from a R25-"Argo sensor types" row + + Examples + -------- + .. code-block:: python + + from argopy import ArgoNVSReferenceTables + + sensor_type = 'CTD' + + df = ArgoNVSReferenceTables().tbl(25) + df_match = df[df["altLabel"].apply(lambda x: x == sensor_type)].iloc[0] + + st = SensorType.from_series(df_match) + + st.name + st.long_name + st.definition + st.deprecated + st.uri + + """ + + reftable = "R25" + + @staticmethod + def from_series(obj: pd.Series) -> "SensorType": + """Create a :class:`SensorType` from a R25-"Argo sensor models" row""" + return SensorType(obj) + + +class SensorModel(NVSrow): + """One single sensor model data from a R27-"Argo sensor models" row + + Examples + -------- + .. code-block:: python + + from argopy import ArgoNVSReferenceTables + + sensor_model = 'AANDERAA_OPTODE_4330F' + + df = ArgoNVSReferenceTables().tbl(27) + df_match = df[df["altLabel"].apply(lambda x: x == sensor_model)].iloc[0] + + sm = SensorModel.from_series(df_match) + + sm.name + sm.long_name + sm.definition + sm.deprecated + sm.uri + """ + + reftable = "R27" + + @staticmethod + def from_series(obj: pd.Series) -> "SensorModel": + """Create a :class:`SensorModel` from a R27-"Argo sensor models" row""" + return SensorModel(obj) + + def __contains__(self, string) -> bool: + return ( + string.lower() in self.name.lower() + or string.lower() in self.long_name.lower() + ) diff --git a/argopy/related/sensors/references.py b/argopy/related/sensors/references.py index b25c5db78..4dc27c46c 100644 --- a/argopy/related/sensors/references.py +++ b/argopy/related/sensors/references.py @@ -8,87 +8,14 @@ from ...options import OPTIONS from ...stores import httpstore, filestore from ...related import ArgoNVSReferenceTables -from ...utils import to_list, NVSrow, path2assets, register_accessor -from ...errors import DataNotFound +from ...utils import to_list, path2assets, register_accessor, ppliststr +from ...errors import DataNotFound, OptionValueError +from .accessories import SensorType, SensorModel +from .accessories import Error, ErrorOptions -log = logging.getLogger("argopy.related.sensors.ref") - - -# Define allowed values as a tuple -Error = ("raise", "ignore", "silent") - -# Define Literal types using tuples -ErrorOptions = Literal[*Error] - - -class SensorType(NVSrow): - """One single sensor type data from a R25-"Argo sensor types" row - - Examples - -------- - .. code-block:: python - - from argopy import ArgoNVSReferenceTables - - sensor_type = 'CTD' - - df = ArgoNVSReferenceTables().tbl(25) - df_match = df[df["altLabel"].apply(lambda x: x == sensor_type)].iloc[0] - - st = SensorType.from_series(df_match) - - st.name - st.long_name - st.definition - st.deprecated - st.uri - - """ - - reftable = "R25" - - @staticmethod - def from_series(obj: pd.Series) -> "SensorType": - """Create a :class:`SensorType` from a R25-"Argo sensor models" row""" - return SensorType(obj) - - -class SensorModel(NVSrow): - """One single sensor model data from a R27-"Argo sensor models" row - Examples - -------- - .. code-block:: python - - from argopy import ArgoNVSReferenceTables - - sensor_model = 'AANDERAA_OPTODE_4330F' - - df = ArgoNVSReferenceTables().tbl(27) - df_match = df[df["altLabel"].apply(lambda x: x == sensor_model)].iloc[0] - - sm = SensorModel.from_series(df_match) - - sm.name - sm.long_name - sm.definition - sm.deprecated - sm.uri - """ - - reftable = "R27" - - @staticmethod - def from_series(obj: pd.Series) -> "SensorModel": - """Create a :class:`SensorModel` from a R27-"Argo sensor models" row""" - return SensorModel(obj) - - def __contains__(self, string) -> bool: - return ( - string.lower() in self.name.lower() - or string.lower() in self.long_name.lower() - ) +log = logging.getLogger("argopy.related.sensors.ref") class SensorReferenceHolder(ABC): @@ -107,13 +34,11 @@ class SensorReferenceHolder(ABC): """Dictionary mapping of R27 to R25""" def __call__(self, *args, **kwargs) -> NoReturn: - raise ValueError( - "A SensorReference instance cannot be called directly." - ) + raise ValueError("A SensorReference instance cannot be called directly.") def __init__(self, obj): self._obj = obj # An instance of SensorReferences, possibly with a filesystem - if getattr(obj, '_fs', None) is None: + if getattr(obj, "_fs", None) is None: self._fs = httpstore( cache=True, cachedir=OPTIONS["cachedir"], @@ -232,18 +157,22 @@ def to_type( model : str | :class:`SensorModel` The model to read the sensor type for. errors : Literal["raise", "ignore", "silent"] = "raise" - How to handle possible errors. If set to "ignore", the method will return None. + How to handle possible errors. If set to "ignore", the method may return None. Returns ------- :class:`SensorType` | None """ + if errors not in Error: + raise OptionValueError( + f"Invalid 'errors' option value '{errors}', must be in: {ppliststr(Error, last='or')}" + ) model_name: str = model.name if isinstance(model, SensorModel) else model sensor_type = self.r27_to_r25.get(model_name, None) if sensor_type is not None: - row = self.r25[ - self.r25["altLabel"].apply(lambda x: x == sensor_type) - ].iloc[0] + row = self.r25[self.r25["altLabel"].apply(lambda x: x == sensor_type)].iloc[ + 0 + ] return SensorType.from_series(row) elif errors == "raise": raise DataNotFound( @@ -346,9 +275,7 @@ def to_model( result = [] for key, val in self.r27_to_r25.items(): if sensor_type.lower() in val.lower(): - row = self.r27[ - self.r27["altLabel"].apply(lambda x: x == key) - ].iloc[0] + row = self.r27[self.r27["altLabel"].apply(lambda x: x == key)].iloc[0] result.append(SensorModel.from_series(row).name) if len(result) == 0: if errors == "raise": @@ -395,7 +322,7 @@ class SensorReferences: def __init__(self, obj): self._obj = obj # An instance of ArgoSensor, possibly with a filesystem - if getattr(obj, '_fs', None) is None: + if getattr(obj, "_fs", None) is None: self._fs = httpstore( cache=True, cachedir=OPTIONS["cachedir"], @@ -405,19 +332,19 @@ def __init__(self, obj): self._fs = obj._fs def __call__(self, *args, **kwargs) -> NoReturn: - raise ValueError( - "ArgoSensor.ref cannot be called directly." - ) + raise ValueError("ArgoSensor.ref cannot be called directly.") -@register_accessor('type', SensorReferences) +@register_accessor("type", SensorReferences) class SensorExtension(SensorReferenceR25): _name = "ref.type" -@register_accessor('maker', SensorReferences) + +@register_accessor("maker", SensorReferences) class MakerExtension(SensorReferenceR26): _name = "ref.maker" -@register_accessor('model', SensorReferences) + +@register_accessor("model", SensorReferences) class ModelExtension(SensorReferenceR27): _name = "ref.model" diff --git a/argopy/related/sensors/sensors.py b/argopy/related/sensors/sensors.py index ff931e54b..b0a735e4b 100644 --- a/argopy/related/sensors/sensors.py +++ b/argopy/related/sensors/sensors.py @@ -126,6 +126,7 @@ class ArgoSensor(ArgoSensorSpec): - https://github.com/OneArgo/ArgoVocabs/issues/157 """ + def __init__(self, **kwargs): super().__init__(**kwargs) @@ -163,4 +164,5 @@ class References(SensorReferences): ArgoSensor().ref.model.search('*Deep*') # Search is case-insensitive """ + _name = "ref" diff --git a/argopy/related/sensors/spec.py b/argopy/related/sensors/spec.py index 77fdcc1b4..2b632e713 100644 --- a/argopy/related/sensors/spec.py +++ b/argopy/related/sensors/spec.py @@ -20,22 +20,11 @@ from ...options import OPTIONS from .references import SensorModel, SensorType +from .accessories import Error, ErrorOptions, Ds, DsOptions, SearchOutput, SearchOutputOptions log = logging.getLogger("argopy.related.sensors") -# Define some options expected values as tuples -# (for argument validation) -SearchOutput = ("wmo", "sn", "wmo_sn", "df") -Error = ("raise", "ignore", "silent") -Ds = ("core", "deep", "bgc") - -# Define Literal types using tuples -# (for typing) -SearchOutputOptions = Literal[*SearchOutput] -ErrorOptions = Literal[*Error] -DsOptions = Literal[*Ds] - class ArgoSensorSpec: From 804e3ce0cf18498989852c2ae48edbf13658d20e Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Wed, 29 Oct 2025 08:09:00 +0100 Subject: [PATCH 55/71] More separation of concerns prepare for ArgoFloat ability to format sensor metadata --- argopy/related/euroargo_api.py | 73 +++++++++++++++++ argopy/related/sensors/references.py | 76 ++++++++++++------ argopy/related/sensors/sensors.py | 5 +- argopy/related/sensors/spec.py | 114 +++++++++++---------------- argopy/related/sensors/utils.py | 45 +++++++++++ argopy/stores/float/spec.py | 14 ++++ 6 files changed, 228 insertions(+), 99 deletions(-) create mode 100644 argopy/related/sensors/utils.py diff --git a/argopy/related/euroargo_api.py b/argopy/related/euroargo_api.py index fe982fb6d..4f6784f81 100644 --- a/argopy/related/euroargo_api.py +++ b/argopy/related/euroargo_api.py @@ -1,4 +1,6 @@ import pandas as pd +from typing import Any, Callable, Literal + from ..options import OPTIONS from ..utils.checkers import check_wmo, check_cyc from ..stores import httpstore @@ -100,3 +102,74 @@ def get_ea_profile_page(WMO, CYC=None, **kwargs): df = get_coriolis_profile_id(WMO, CYC, **kwargs) url = "https://dataselection.euro-argo.eu/cycle/{}" return [url.format(this_id) for this_id in sorted(df["ID"])] + + +class EAfleetmonitoringAPI: + + def __init__(self, **kwargs) -> None: + """Create an instance of :class:`EAfleetmonitoringAPI` + + Parameters + ---------- + fs: :class:`stores.httpstore`, default: None + The http filesystem to use. If None is provided, we instantiate a new one based on `cache`, `cachedir` and `timeout` options. + cache : bool, optional, default: True + Use cache or not for fetched data. Used only if `fs` is None. + cachedir: str, optional, default: OPTIONS['cachedir'] + Folder where to store cached files. Used only if `fs` is None. + timeout: int, optional, default: OPTIONS['api_timeout'] + Time out in seconds to connect to web API. Used only if `fs` is None. + + """ + if kwargs.get("fs", None) is None: + self._fs = httpstore( + cache=kwargs.get("cache", True), + cachedir=kwargs.get("cachedir", OPTIONS["cachedir"]), + timeout=kwargs.get("timeout", OPTIONS["api_timeout"]), + ) + else: + self._fs = kwargs["fs"] + + + def floats( + self, + wmo: str | int | list[int] | list[str], + preprocess: Callable = None, + preprocess_opts: dict = {}, + postprocess: Callable = None, + postprocess_opts: dict = {}, + errors: Literal["raise", "ignore", "silent"] = "raise", + progress: bool = False, + ) -> Any: + """Call to 'floats' endpoint with one or more WMOs + + Notes + ----- + Based on a POST request to the fleet-monitoring API requests to `/floats/{wmo}`. + + `Endpoint documentation `_. + + Notes + ----- + No option checking, to be done by caller + """ + WMOs = check_wmo(wmo) + + URI = [] + for wmo in WMOs: + URI.append(f"{OPTIONS['fleetmonitoring']}/floats/{wmo}") + + sns = self._fs.open_mfjson( + URI, + preprocess=preprocess, + preprocess_opts=preprocess_opts, + progress=progress, + errors=errors, + progress_unit="float", + progress_desc="Fetching floats metadata", + ) + + if postprocess is not None: + return postprocess(sns, **postprocess_opts) + else: + return sns diff --git a/argopy/related/sensors/references.py b/argopy/related/sensors/references.py index 4dc27c46c..2d93fa8a5 100644 --- a/argopy/related/sensors/references.py +++ b/argopy/related/sensors/references.py @@ -85,17 +85,13 @@ def _load_mappers(self): ) ) df = pd.concat(df) + for col in ['origin', 'destination', 'type', 'model', '?', '??']: + df[col] = df[col].apply(lambda x: x.strip()) df = df.reset_index(drop=True) - self._r27_to_r25: dict[str, str] = {} - df.apply( - lambda row: self._r27_to_r25.update( - {row["model"].strip(): row["type"].strip()} - ), - axis=1, - ) + self._r27_to_r25 : pd.DataFrame | None = df @property - def r27_to_r25(self): + def r27_to_r25(self) -> pd.DataFrame: """Dictionary mapping of R27 to R25""" if self._r27_to_r25 is None: self._load_mappers() @@ -145,35 +141,47 @@ def to_type( self, model: str | SensorModel | None = None, errors: ErrorOptions = "raise", - ) -> SensorType | None: - """Get a sensor type for a given sensor model + obj: bool = False, + ) -> list[str] | list[SensorType] | None: + """Get all sensor types of a given sensor model - All valid sensor model name can be obtained with :meth:`ArgoSensor.ref.mode.to_list()`. + All valid sensor model names can be obtained with :meth:`ArgoSensor.ref.model.hint`. Mapping between sensor model name (R27) and sensor type (R25) are from AVTT work at https://github.com/OneArgo/ArgoVocabs/issues/156. Parameters ---------- model : str | :class:`SensorModel` - The model to read the sensor type for. + The sensor model to read the sensor type for. errors : Literal["raise", "ignore", "silent"] = "raise" How to handle possible errors. If set to "ignore", the method may return None. Returns ------- - :class:`SensorType` | None + list[str] | list[:class:`SensorType`] | None + + Raises + ------ + :class:`DataNotFound` """ if errors not in Error: raise OptionValueError( f"Invalid 'errors' option value '{errors}', must be in: {ppliststr(Error, last='or')}" ) model_name: str = model.name if isinstance(model, SensorModel) else model - sensor_type = self.r27_to_r25.get(model_name, None) - if sensor_type is not None: - row = self.r25[self.r25["altLabel"].apply(lambda x: x == sensor_type)].iloc[ - 0 - ] - return SensorType.from_series(row) + + match = fnmatch.filter(self.r27_to_r25["model"], model_name.upper()) + types = self.r27_to_r25[self.r27_to_r25["model"].apply(lambda x: x in match)]['type'].tolist() + + if len(types) > 0: + # Since 'types' comes from the mapping, we double-check values against the R25 entries: + types = [self.r25[self.r25["altLabel"].apply(lambda x: x == this_type)]['altLabel'].item() for this_type in types] + if not obj: + return types + else: + rows = [self.r25[self.r25["altLabel"].apply(lambda x: x == this_type)].iloc[0] for this_type in types] + return [SensorType.from_series(row) for row in rows] + elif errors == "raise": raise DataNotFound( f"Can't determine the type of sensor model '{model_name}' (no matching key in r27_to_r25 mapper)" @@ -253,10 +261,11 @@ def to_model( self, type: str | SensorType, errors: Literal["raise", "ignore"] = "raise", - ) -> list[str] | None: + obj: bool = False, + ) -> list[str] | list[SensorModel] | None: """Get all sensor model names of a given sensor type - All valid sensor types can be obtained with :attr:`ArgoSensor.reference_sensor_type` + All valid sensor types can be obtained with :meth:`ArgoSensor.ref.sensor.hint` Mapping between sensor model name (R27) and sensor type (R25) are from AVTT work at https://github.com/OneArgo/ArgoVocabs/issues/156. @@ -269,14 +278,29 @@ def to_model( Returns ------- - list[str] + list[str] | list[:class:`SensorModel`] | None + + Raises + ------ + :class:`DataNotFound` """ sensor_type = type.name if isinstance(type, SensorType) else type result = [] - for key, val in self.r27_to_r25.items(): - if sensor_type.lower() in val.lower(): - row = self.r27[self.r27["altLabel"].apply(lambda x: x == key)].iloc[0] - result.append(SensorModel.from_series(row).name) + + match = fnmatch.filter(self.r27_to_r25["type"], sensor_type.upper()) + models = self.r27_to_r25[self.r27_to_r25["type"].apply(lambda x: x in match)]['model'].tolist() + + if len(models) > 0: + # Since 'models' comes from the mapping, we double-check values against the R27 entries: + models = [self.r27[self.r27["altLabel"].apply(lambda x: x == model)]['altLabel'].item() for + model in models] + if not obj: + return models + else: + rows = [self.r27[self.r27["altLabel"].apply(lambda x: x == model)].iloc[0] for + model in models] + return [SensorModel.from_series(row) for row in rows] + if len(result) == 0: if errors == "raise": raise DataNotFound( diff --git a/argopy/related/sensors/sensors.py b/argopy/related/sensors/sensors.py index b0a735e4b..51afca33c 100644 --- a/argopy/related/sensors/sensors.py +++ b/argopy/related/sensors/sensors.py @@ -126,9 +126,8 @@ class ArgoSensor(ArgoSensorSpec): - https://github.com/OneArgo/ArgoVocabs/issues/157 """ - - def __init__(self, **kwargs): - super().__init__(**kwargs) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) @register_accessor("ref", ArgoSensor) diff --git a/argopy/related/sensors/spec.py b/argopy/related/sensors/spec.py index 2b632e713..e2e35b87a 100644 --- a/argopy/related/sensors/spec.py +++ b/argopy/related/sensors/spec.py @@ -18,9 +18,11 @@ OptionValueError, ) from ...options import OPTIONS +from ..euroargo_api import EAfleetmonitoringAPI from .references import SensorModel, SensorType from .accessories import Error, ErrorOptions, Ds, DsOptions, SearchOutput, SearchOutputOptions +from .utils import APISensorMetaDataProcessing log = logging.getLogger("argopy.related.sensors") @@ -67,7 +69,9 @@ def __init__(self, model: str | None = None, *args, **kwargs) -> None: self._fs = kwargs["fs"] self._vocabulary: SensorModel | None = None - self._type: SensorType | None = None + self._type: list[SensorType] | None = None + self._api = EAfleetmonitoringAPI(fs=self._fs) + if model is not None: try: df = self.ref.model.search(model) @@ -78,7 +82,7 @@ def __init__(self, model: str | None = None, *args, **kwargs) -> None: if df.shape[0] == 1: self._vocabulary = SensorModel.from_series(df.iloc[0]) - self._type = self.ref.model.to_type(self._vocabulary, errors="ignore") + self._type = self.ref.model.to_type(self._vocabulary, errors="ignore", obj=True) # if "RBR" in self._vocabulary: # Add the RBR OEM API Authorization key for this sensor: # fs_kargs.update(client_kwargs={'headers': {'Authorization': OPTIONS.get('rbr_api_key') }}) @@ -122,7 +126,7 @@ def type(self) -> SensorType: ------ :class:`InvalidDataset` """ - if isinstance(self._type, SensorType): + if len(self._type) > 0 and isinstance(self._type[0], SensorType): return self._type else: raise InvalidDataset( @@ -229,33 +233,20 @@ def _floats_api( ----- No option checking, to be done by caller """ - if preprocess_opts is None: - preprocess_opts = {} - try: is_wmo(model_or_wmo) WMOs = check_wmo(model_or_wmo) except ValueError: WMOs = self._search_wmo_with(model_or_wmo) - URI = [] - for wmo in WMOs: - URI.append(f"{OPTIONS['fleetmonitoring']}/floats/{wmo}") - - sns = self._fs.open_mfjson( - URI, - preprocess=preprocess, - preprocess_opts=preprocess_opts, - progress=progress, - errors=errors, - progress_unit="float", - progress_desc="Fetching floats metadata", - ) - - if postprocess is not None: - return postprocess(sns, **postprocess_opts) - else: - return sns + return self._api.floats(WMOs, + preprocess=preprocess, + preprocess_opts=preprocess_opts, + postprocess=postprocess, + postprocess_opts=postprocess_opts, + progress=progress, + errors=errors, + ) def _search_sn_with( self, @@ -309,10 +300,16 @@ def _search_wmo_sn_with( """ def preprocess(jsdata, model_name: str = ""): - sn = np.unique( - [s["serial"] for s in jsdata["sensors"] if model_name in s["model"]] - ) - return [jsdata["wmo"], [str(s) for s in sn]] + try: + x = [s["serial"] for s in jsdata["sensors"] if model_name in s["model"]] + x = [x for x in x if x is not None] + if len(x) == 0: + sn = ['n/a'] + else: + sn = np.unique(x).tolist() + return [jsdata["wmo"], [str(s) for s in sn]] + except: + log.error(f"Could not find sensor model {model_name}: {jsdata['sensors']}") def postprocess(data, **kwargs): S = {} @@ -351,53 +348,14 @@ def _to_dataframe( if model is None and self.vocabulary is not None: model = self.vocabulary.name - def preprocess(jsdata, model_name: str = ""): - output = [] - for s in jsdata["sensors"]: - if model_name in s["model"]: - this = [jsdata["wmo"]] - [ - this.append(s[key]) # type: ignore - for key in [ - "id", - "maker", - "model", - "serial", - "units", - "accuracy", - "resolution", - ] - ] - output.append(this) - return output - - def postprocess(data, **kwargs): - d = [] - for this in data: - for wmo, sid, maker, model, sn, units, accuracy, resolution in this: - d.append( - { - "WMO": wmo, - "Type": sid, - "Model": model, - "Maker": maker, - "SerialNumber": sn if sn != "n/a" else None, - "Units": units, - "Accuracy": accuracy, - "Resolution": resolution, - } - ) - return pd.DataFrame(d).sort_values(by="WMO").reset_index(drop=True) - - df = self._floats_api( + return self._floats_api( model if kwargs.get("wmo", None) is None else kwargs["wmo"], - preprocess=preprocess, + preprocess=APISensorMetaDataProcessing.preprocess_df, preprocess_opts={"model_name": model}, - postprocess=postprocess, + postprocess=APISensorMetaDataProcessing.postprocess_df, progress=progress, errors=errors, ) - return df.sort_values(by="WMO", axis=0).reset_index(drop=True) def _search_single( self, @@ -778,3 +736,19 @@ def iterfloats_with( else: for wmo in WMOs: yield ArgoFloat(wmo, idx=idx) + + def from_wmo(self, wmo: int | str) -> pd.DataFrame: + """Retrieve sensor metadata from a given float WMO number + + Parameters + ---------- + wmo: int | str + Float WMO number, only one value + + Returns + ------- + :class:`pandas.DataFrame` + """ + wmo = check_wmo(wmo) + df = self._to_dataframe('', wmo=wmo[0]) + return df.drop('WMO', axis=1).T \ No newline at end of file diff --git a/argopy/related/sensors/utils.py b/argopy/related/sensors/utils.py new file mode 100644 index 000000000..f386e683d --- /dev/null +++ b/argopy/related/sensors/utils.py @@ -0,0 +1,45 @@ +import pandas as pd + + +class APISensorMetaDataProcessing: + + @classmethod + def preprocess_df(cls, jsdata, model_name: str = "", **kwargs) -> list[str]: + """Get the list of sensor metadata for a given model""" + output = [] + for s in jsdata["sensors"]: + if model_name in s["model"]: + this = [jsdata["wmo"]] + [ + this.append(s[key]) # type: ignore + for key in [ + "id", + "maker", + "model", + "serial", + "units", + "accuracy", + "resolution", + ] + ] + output.append(this) + return output + + @classmethod + def postprocess_df(cls, data, **kwargs): + d = [] + for this in data: + for wmo, sid, maker, model, sn, units, accuracy, resolution in this: + d.append( + { + "WMO": wmo, + "Type": sid, + "Model": model, + "Maker": maker, + "SerialNumber": sn if sn != "n/a" else None, + "Units": units, + "Accuracy": accuracy, + "Resolution": resolution, + } + ) + return pd.DataFrame(d).sort_values(by="WMO").reset_index(drop=True) diff --git a/argopy/stores/float/spec.py b/argopy/stores/float/spec.py index 13f732cb2..50def2ed0 100644 --- a/argopy/stores/float/spec.py +++ b/argopy/stores/float/spec.py @@ -393,3 +393,17 @@ def __repr__(self): ) return "\n".join(summary) + + @property + def sensors(self): + """Return a :class:`pandas.DataFrame` describing float sensors + + ! This is experimental and may change in the future. + """ + # We can't call on: + # >>> ArgoSensor(fs=self.fs).from_wmo(self.WMO) + # because ArgoSensor import ArgoFloat for the iterator method and + # that would be circular import. + # We need to implement here a self.metadata -> dataframe transform + # like it's done in ArgoSensor from json data + pass \ No newline at end of file From 96bbfb225ce3646c1e26060bd9bc07f604c4dc27 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Wed, 29 Oct 2025 08:09:09 +0100 Subject: [PATCH 56/71] doc + docstring --- argopy/static/assets/nvs_R25_R27/README.md | 2 - docs/advanced-tools/metadata/sensors.rst | 149 ++++++++++++--------- docs/api-hidden.rst | 25 ++-- docs/whats-new.rst | 15 ++- 4 files changed, 110 insertions(+), 81 deletions(-) diff --git a/argopy/static/assets/nvs_R25_R27/README.md b/argopy/static/assets/nvs_R25_R27/README.md index 3dbedb6d1..55c3a1888 100644 --- a/argopy/static/assets/nvs_R25_R27/README.md +++ b/argopy/static/assets/nvs_R25_R27/README.md @@ -3,6 +3,4 @@ This folder hold mapping data to get R25 sensor types from R27 sensor models. Relevant issues from ADMT/AVTT: -- https://github.com/OneArgo/ADMT/issues/112 - https://github.com/OneArgo/ArgoVocabs/issues/156 -- https://github.com/OneArgo/ArgoVocabs/issues/157 diff --git a/docs/advanced-tools/metadata/sensors.rst b/docs/advanced-tools/metadata/sensors.rst index 85c7528c1..23b920661 100644 --- a/docs/advanced-tools/metadata/sensors.rst +++ b/docs/advanced-tools/metadata/sensors.rst @@ -1,14 +1,21 @@ .. currentmodule:: argopy.related .. _argosensor: -Argo sensor: models and types -============================= +Argo sensors +============ -The :class:`ArgoSensor` class provides direct access to Argo's sensor metadata through Reference Tables `Sensor Types (R25) `_ and `Sensor Models (R27) `_, combined with the `Euro-Argo fleet-monitoring API `_. This enables users to: +The :class:`ArgoSensor` class aims to provide user-friendly access to Argo's sensor metadata from: + +- NVS Reference Tables `Sensor Models (R27) `_, `Sensor Types (R25) `_ and `Sensor Manufacturers (R26) `_ +- the `Euro-Argo fleet-monitoring API `_ + +This enables users to: + +- navigate reference tables, +- search for floats equipped with specific sensor models, +- retrieve sensor serial numbers across the global array. +- search for/iterate over floats equipped with specific sensor models. -- Navigate reference tables, -- Search for floats equipped with specific sensor models, -- Retrieve sensor serial numbers across the global array. .. contents:: :local: @@ -18,70 +25,68 @@ The :class:`ArgoSensor` class provides direct access to Argo's sensor metadata t Work with reference tables on sensors ------------------------------------- -With the :class:`ArgoSensor` class, you can work with official Argo vocabularies for `Sensor Types (R25) `_ and `Sensor Models (R27) `_. With these methods, it is easy to look for a specific sensor model name, which can then be used to :ref:`look for floats ` or :ref:`create a ArgoSensor class ` directly. +With the :class:`ArgoSensor` class, you can work with official Argo vocabularies for `Sensor Models (R27) `_, `Sensor Types (R25) `_ and `Sensor Manufacturers (R26) `_. With these methods, it is easy to look for a specific sensor model name, which can then be used to :ref:`look for floats ` or :ref:`create a ArgoSensor class ` directly. -.. list-table:: :class:`ArgoSensor` attributes and methods for reference tables +.. list-table:: :class:`ArgoSensor` methods for reference tables :header-rows: 1 :stub-columns: 1 - * - Attribute/Method + * - Methods - Description - * - :attr:`ArgoSensor.reference_model` + * - :meth:`ArgoSensor.ref.model.to_dataframe` - Returns Reference Table **Sensor Models (R27)** as a :class:`pandas.DataFrame` - * - :attr:`ArgoSensor.reference_model_name` - - List of all sensor model names (e.g., ``'SBE41CP'``) - * - :attr:`ArgoSensor.reference_sensor` + * - :meth:`ArgoSensor.ref.type.to_dataframe` - Returns Reference Table **Sensor Types (R25)** as a :class:`pandas.DataFrame` - * - :attr:`ArgoSensor.reference_sensor_type` - - List of all sensor types (e.g., ``'CTD'``, ``'OPTODE'``) - * - :attr:`ArgoSensor.r27_to_r25` - - Dictionary mapping of R27 to R25 - * - :meth:`ArgoSensor.search_model` - - Search R27 for models matching a string (exact or fuzzy) - * - :meth:`ArgoSensor.model_to_type` - - Returns AVTT mapping on sensor type for a given model name - * - :meth:`ArgoSensor.type_to_model` - - Returns AVTT mapping on model names for a given sensor type + * - :meth:`ArgoSensor.ref.maker.to_dataframe` + - Returns Reference Table **Sensor Manufacturers (R26)** as a :class:`pandas.DataFrame` + + * - :meth:`ArgoSensor.ref.model.hint` + - List of all sensor model names (e.g., ``['AANDERAA_OPTODE', ..., 'RBR_ARGO3_DEEP6K', ..., 'SBE41CP', ...]``) + * - :meth:`ArgoSensor.ref.type.hint` + - List of all sensor types (e.g., ``[..., 'CTD_CNDC', ..., 'OPTODE_DOXY', ...]``) + * - :meth:`ArgoSensor.ref.maker.hint` + - List of all sensor makers (e.g., ``[..., 'DRUCK', ... 'SEASCAN', ... ]``) + + * - :meth:`ArgoSensor.ref.model.to_type` + - Returns sensor type of a given model + * - :meth:`ArgoSensor.ref.type.to_model` + - Returns model names of a given sensor type + + * - :meth:`ArgoSensor.ref.model.search` + - Search R27 for models matching a string, can use wildcard (eg: ``'SBE61*'``) Examples ^^^^^^^^ -- List all CTD sensor models (R27): +- List all CTD models from a given type: .. ipython:: python :okwarning: from argopy import ArgoSensor - ArgoSensor().reference_model + ArgoSensor().ref.type.to_model('CTD_CNDC') -- Fuzzy search (default): +- Get the sensor type(s) of a given model: .. ipython:: python :okwarning: - ArgoSensor().search_model('SBE61_V5.0.1', strict=False) + ArgoSensor().ref.model.to_type('RBR_ARGO3_DEEP6K') -- Exact search: +- Search all SBE61 versions with a wildcard: .. ipython:: python :okwarning: - ArgoSensor().search_model('SBE61_V5.0.1', strict=True) + ArgoSensor().ref.model.search('SBE61*') -- Get the sensor type of a given model: +- Get one model single model description (see also :attr:`ArgoSensor.vocabulary`): .. ipython:: python :okwarning: - ArgoSensor().model_to_type('RBR_ARGO3_DEEP6K') - -- List all possible model names of a given sensor type: - -.. ipython:: python - :okwarning: - - ArgoSensor().type_to_model('FLUOROMETER_CDOM') + ArgoSensor().ref.model.search('SBE61_V5.0.1') .. _argosensor-search-floats: @@ -89,14 +94,19 @@ Examples Search for Argo floats equipped with a given sensor model --------------------------------------------------------- -In this section we show how to find WMOs, serial numbers for floats equipped with a specific sensor model using the :meth:`ArgoSensor.search` method. +In this section we show how to find WMOs, and possibly more sensor metadata like serial numbers, for floats equipped with a specific sensor model. +This can be done using the :meth:`ArgoSensor.search` method. -The method takes a model sensor name as input, and possibly 4 values to the ``output`` option to determine how to format results: +The method takes one or more model sensor names as input, and return results in 4 different formats with the ``output`` option: - ``output='wmo'`` returns a list of WMOs (e.g., ``[1901234, 1901235]``) - ``output='sn'`` returns a list of serial numbers (e.g., ``['1234', '5678']``) -- ``output='wmo_sn'`` returns a dictionary mapping float WMOs to serial numbers (e.g. ``{WMO: [serial1, serial2]}``) -- ``output='df'`` returns a :class:`pandas.DataFrame` with WMO, sensor type, model, maker, serial number, units, accuracy and resolution. +- ``output='wmo_sn'`` returns a dictionary mapping float WMOs to serial numbers (e.g. ``{1900166: ['0325', '2657720242']}``) +- ``output='df'`` returns a :class:`pandas.DataFrame` with WMO, sensor type, model, maker, serial number, units, accuracy and resolution in columns. + +.. note:: + + This method can potentially lead to a scan of the entire Argo float collection, which would be very time consuming. Therefore the :meth:`ArgoSensor.search` method takes only exact sensor models and will raise an error if a wildcard is found. Examples ^^^^^^^^ @@ -116,28 +126,28 @@ Examples ArgoSensor().search("SBE43F_IDO", output="sn") -- Get WMO/serial-number dictionary for floats equipped with the "RBR_ARGO3_DEEP6K" model: +- Get everything for floats equipped with the "RBR_ARGO3_DEEP6K" model: .. ipython:: python :okwarning: - wmo_sn = ArgoSensor().search("RBR_ARGO3_DEEP6K", output="wmo_sn") - wmo_sn + ArgoSensor().search("RBR_ARGO3_DEEP6K", output="df") + .. _argosensor-exact-sensor: -Use an exact sensor model name to create an instance ----------------------------------------------------- +Use an exact sensor model name to create a specific ArgoSensor +-------------------------------------------------------------- -You can initialize an :class:`ArgoSensor` instance with a specific model to access its metadata and methods directly. +You can initialize an :class:`ArgoSensor` instance with a specific model to access more metadata. .. csv-table:: :class:`ArgoSensor` Attributes and Methods for a specific model :header: "Attribute/Method", "Description" :widths: 30, 70 - :class:`ArgoSensor`, "Create an instance for an exact model name (e.g., ``'SBE41CP'``)" - :attr:`ArgoSensor.model`, "Returns a :class:`SensorModel` object (name, long_name, definition, URI, deprecated)" - :attr:`ArgoSensor.type`, "Returns a :class:`SensorType` object (name, long_name, definition, URI, deprecated)" + :class:`ArgoSensor`, "Create an instance for an exact model name (e.g., ``'SBE43F_IDO'``)" + :attr:`ArgoSensor.vocabulary`, "Returns a :class:`SensorModel` object with R27 concept vocabulary (name, long_name, definition, URI, deprecated)" + :attr:`ArgoSensor.type`, "Returns a :class:`SensorType` object with R25 concept (name, long_name, definition, URI, deprecated)" :meth:`ArgoSensor.search`, "Inherits search methods but defaults to the instance's model" Examples @@ -151,19 +161,21 @@ As an example, let's create an instance for the "SBE43F_IDO" sensor model: sensor = ArgoSensor("SBE43F_IDO") sensor -You can then access model metadata: +You can then access this model metadata from the NVS vocabulary (Reference table R27): .. ipython:: python :okwarning: - sensor.model + sensor.vocabulary + +from Reference table R25: .. ipython:: python :okwarning: sensor.type -And you can look for floats equipped: +And you can look for floats equipped with it: .. ipython:: python :okwarning: @@ -179,29 +191,40 @@ The :meth:`ArgoSensor.iterfloats_with` will yields :class:`argopy.ArgoFloat` ins Example ^^^^^^^ -Loop through all floats with "SATLANTIC_PAR" sensor: +Let's try to gather all platform types of WMOs equipped with a list of sensor models: .. ipython:: python :okwarning: - for afloat in ArgoSensor().iterfloats_with("SATLANTIC_PAR"): - print(f"\n-Float {afloat.WMO}: Platform description = {afloat.metadata['platform']['description']}") - for sensor in afloat.metadata["sensors"]: - if "SATLANTIC_PAR" in sensor["model"]: - print(f" - Sensor Maker: {sensor['maker']}, Serial: {sensor['serial']}") + sensors = ArgoSensor() + + models = ['ECO_FLBBCD_AP2', 'ECO_FLBBCD'] + results = {} + for af in sensors.iterfloats_with(models): + if 'meta' in af.ls_dataset(): + platform_type = af.metadata['platform']['type'] # e.g. 'PROVOR_V_JUMBO' + if platform_type in results.keys(): + results[platform_type].extend([af.WMO]) + else: + results.update({platform_type: [af.WMO]}) + else: + print(f"No meta file for float {af.WMO}") + print(results.keys()) + Quick float and sensor lookups from the command line ---------------------------------------------------- The :meth:`ArgoSensor.cli_search` function is available to search for Argo floats equipped with a given sensor model from the command line and retrieve a `CLI `_ friendly list of WMOs or serial numbers. -This could be useful for piping to other tools. +You will take note that all output *format* are available to determine the information to retrieve, but that the actual standard output will be a serialized string for easy piping to other tools. -Here is an example with the "SATLANTIC_PAR" sensor: +Here is an example with the "SATLANTIC_PAR" sensor. .. code-block:: bash :caption: Call :meth:`ArgoSensor.search` from the command line python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('SATLANTIC_PAR', output='wmo')" - python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('SATLANTIC_PAR', output='sn')" + python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('SATLANTIC_PAR', output='wmo_sn')" + python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('SATLANTIC_PAR', output='df')" diff --git a/docs/api-hidden.rst b/docs/api-hidden.rst index 85414a568..2e5f5d721 100644 --- a/docs/api-hidden.rst +++ b/docs/api-hidden.rst @@ -178,21 +178,24 @@ argopy.related.doi_snapshot.DOIrecord argopy.related.ArgoSensor - argopy.related.ArgoSensor.reference_model - argopy.related.ArgoSensor.reference_model_name - argopy.related.ArgoSensor.reference_sensor - argopy.related.ArgoSensor.reference_sensor_type - argopy.related.ArgoSensor.model + argopy.related.ArgoSensor.ref.model.to_dataframe + argopy.related.ArgoSensor.ref.model.hint + argopy.related.ArgoSensor.ref.model.to_type + argopy.related.ArgoSensor.ref.model.search + argopy.related.ArgoSensor.ref.type.to_dataframe + argopy.related.ArgoSensor.ref.type.hint + argopy.related.ArgoSensor.ref.type.to_model + argopy.related.ArgoSensor.ref.maker.to_dataframe + argopy.related.ArgoSensor.ref.maker.hint + + argopy.related.ArgoSensor.vocabulary argopy.related.ArgoSensor.type - argopy.related.ArgoSensor.r27_to_r25 - argopy.related.ArgoSensor.search_model - argopy.related.ArgoSensor.model_to_type - argopy.related.ArgoSensor.type_to_model argopy.related.ArgoSensor.search argopy.related.ArgoSensor.iterfloats_with argopy.related.ArgoSensor.cli_search - argopy.related.SensorModel - argopy.related.SensorType + + argopy.related.sensors.SensorModel + argopy.related.sensors.SensorType argopy.plot argopy.plot.dashboard diff --git a/docs/whats-new.rst b/docs/whats-new.rst index 194e33a8d..23b98bf88 100644 --- a/docs/whats-new.rst +++ b/docs/whats-new.rst @@ -7,6 +7,16 @@ What's New |pypi dwn| |conda dwn| + +Coming up next (un-released) +---------------------------- + +Features and front-end API +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- **New** :class:`ArgoSensor` class to work with Argo sensor models, types and floats equipped with. Check the new :ref:`Argo sensor: models and types` documentation section. (:pr:`532`) by |gmaze|. + + v1.3.1 (22 Oct. 2025) --------------------- @@ -21,11 +31,6 @@ Features and front-end API .. _v1.3.1-internals: -Features and front-end API -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -- **New** :class:`ArgoSensor` class to work with Argo sensor models, types and floats equipped with. Check the new :ref:`Argo sensor: models and types` documentation section. (:pr:`532`) by |gmaze|. - Internals ^^^^^^^^^ From a7435baaaffb26375eb512082bb0d19972d7cccf Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Wed, 29 Oct 2025 10:44:19 +0100 Subject: [PATCH 57/71] doc and docstrings --- argopy/related/sensors/accessories.py | 20 +++ argopy/related/sensors/references.py | 32 ++-- argopy/related/sensors/sensors.py | 105 ++++++++----- argopy/related/sensors/spec.py | 38 ++--- argopy/stores/implementations/http.py | 12 +- .../argopy.ArgoSensor.iterfloats_with.rst | 6 + .../argopy.ArgoSensor.ref.maker.hint.rst | 6 + ...gopy.ArgoSensor.ref.maker.to_dataframe.rst | 6 + .../argopy.ArgoSensor.ref.model.hint.rst | 6 + .../argopy.ArgoSensor.ref.model.search.rst | 6 + ...gopy.ArgoSensor.ref.model.to_dataframe.rst | 6 + .../argopy.ArgoSensor.ref.model.to_type.rst | 6 + .../argopy.ArgoSensor.ref.type.hint.rst | 6 + ...rgopy.ArgoSensor.ref.type.to_dataframe.rst | 6 + .../argopy.ArgoSensor.ref.type.to_model.rst | 6 + .../metadata/generated/argopy.ArgoSensor.rst | 33 ++++ .../generated/argopy.ArgoSensor.search.rst | 6 + .../generated/argopy.ArgoSensor.type.rst | 6 + .../argopy.ArgoSensor.vocabulary.rst | 6 + docs/advanced-tools/metadata/index.rst | 4 +- docs/advanced-tools/metadata/sensors.rst | 144 ++++++++++-------- docs/api-hidden.rst | 36 ++--- docs/whats-new.rst | 2 +- 23 files changed, 343 insertions(+), 161 deletions(-) create mode 100644 docs/advanced-tools/metadata/generated/argopy.ArgoSensor.iterfloats_with.rst create mode 100644 docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.maker.hint.rst create mode 100644 docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.maker.to_dataframe.rst create mode 100644 docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.model.hint.rst create mode 100644 docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.model.search.rst create mode 100644 docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.model.to_dataframe.rst create mode 100644 docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.model.to_type.rst create mode 100644 docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.type.hint.rst create mode 100644 docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.type.to_dataframe.rst create mode 100644 docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.type.to_model.rst create mode 100644 docs/advanced-tools/metadata/generated/argopy.ArgoSensor.rst create mode 100644 docs/advanced-tools/metadata/generated/argopy.ArgoSensor.search.rst create mode 100644 docs/advanced-tools/metadata/generated/argopy.ArgoSensor.type.rst create mode 100644 docs/advanced-tools/metadata/generated/argopy.ArgoSensor.vocabulary.rst diff --git a/argopy/related/sensors/accessories.py b/argopy/related/sensors/accessories.py index 8db3344f3..870459e20 100644 --- a/argopy/related/sensors/accessories.py +++ b/argopy/related/sensors/accessories.py @@ -20,6 +20,9 @@ class SensorType(NVSrow): """One single sensor type data from a R25-"Argo sensor types" row + .. warning:: + This class is experimental and may change in a future release. + Examples -------- .. code-block:: python @@ -52,8 +55,24 @@ def from_series(obj: pd.Series) -> "SensorType": class SensorModel(NVSrow): """One single sensor model data from a R27-"Argo sensor models" row + .. warning:: + This class is experimental and may change in a future release. + Examples -------- + .. code-block:: python + + from argopy import ArgoSensor + + sm = ArgoSensor('AANDERAA_OPTODE_4330F').vocabulary + + sm.name + sm.long_name + sm.definition + sm.deprecated + sm.urn + sm.uri + .. code-block:: python from argopy import ArgoNVSReferenceTables @@ -69,6 +88,7 @@ class SensorModel(NVSrow): sm.long_name sm.definition sm.deprecated + sm.urn sm.uri """ diff --git a/argopy/related/sensors/references.py b/argopy/related/sensors/references.py index 2d93fa8a5..62ed3d375 100644 --- a/argopy/related/sensors/references.py +++ b/argopy/related/sensors/references.py @@ -110,7 +110,7 @@ class SensorReferenceR27(SensorReferenceHolder): """Argo sensor models""" def to_dataframe(self) -> pd.DataFrame: - """Official reference table for Argo sensor models (R27) + """Reference Table **Sensor Models (R27)** as a :class:`pandas.DataFrame` Returns ------- @@ -123,7 +123,7 @@ def to_dataframe(self) -> pd.DataFrame: return self.r27 def hint(self) -> list[str]: - """Official list of Argo sensor models (R27) + """List of Argo sensor models Return a sorted list of strings with altLabel from Argo Reference table R27 on 'SENSOR_MODEL'. @@ -139,7 +139,7 @@ def hint(self) -> list[str]: def to_type( self, - model: str | SensorModel | None = None, + model: str | SensorModel, errors: ErrorOptions = "raise", obj: bool = False, ) -> list[str] | list[SensorType] | None: @@ -151,14 +151,16 @@ def to_type( Parameters ---------- - model : str | :class:`SensorModel` + model : str | :class:`argopy.related.SensorModel` The sensor model to read the sensor type for. - errors : Literal["raise", "ignore", "silent"] = "raise" + errors : Literal["raise", "ignore", "silent"], optional, default: "raise" How to handle possible errors. If set to "ignore", the method may return None. + obj: bool, optional, default: False + Return a list of strings (False) or a list of :class:`argopy.related.SensorType` Returns ------- - list[str] | list[:class:`SensorType`] | None + list[str] | list[:class:`argopy.related.SensorType`] | None Raises ------ @@ -199,14 +201,14 @@ def search( ) -> pd.DataFrame | list[str]: """Return Argo sensor model references matching a string - Look for occurrences in Argo Reference table R27 `altLabel` and return a :class:`pandas.DataFrame` with matching row(s). + Look for occurrences in Argo Reference table R27 `altLabel` and return a subset of the :class:`pandas.DataFrame` with matching row(s). Parameters ---------- model : str The model to search for. You can use wildcards: "SBE41CP*" "*DEEP*", "RBR*", or an exact name like "RBR_ARGO3_DEEP6". output : str, Literal["df", "name"], default "df" - Is the output a :class:`pandas.DataFrame` with matching rows from :attr:`ArgoSensor.reference_model`, or a list of string. + Is the output a :class:`pandas.DataFrame` with matching rows from R27, or a list of string. Returns ------- @@ -234,7 +236,7 @@ class SensorReferenceR25(SensorReferenceHolder): """Argo sensor types""" def to_dataframe(self) -> pd.DataFrame: - """Official reference table for Argo sensor types (R25) + """Reference Table **Sensor Types (R25)** as a :class:`pandas.DataFrame` Returns ------- @@ -243,7 +245,7 @@ def to_dataframe(self) -> pd.DataFrame: return self.r25 def hint(self) -> list[str]: - """Official list of Argo sensor types (R25) + """List of Argo sensor types Return a sorted list of strings with altLabel from Argo Reference table R25 on 'SENSOR'. @@ -271,14 +273,16 @@ def to_model( Parameters ---------- - type : str, :class:`SensorType` + type : str, :class:`argopy.related.SensorType` The sensor type to read the sensor model name for. errors : Literal["raise", "ignore"] = "raise" How to handle possible errors. If set to "ignore", the method will return None. + obj: bool, optional, default: False + Return a list of strings (False) or a list of :class:`argopy.related.SensorModel` Returns ------- - list[str] | list[:class:`SensorModel`] | None + list[str] | list[:class:`argopy.related.SensorModel`] | None Raises ------ @@ -316,7 +320,7 @@ class SensorReferenceR26(SensorReferenceHolder): """Argo sensor maker""" def to_dataframe(self) -> pd.DataFrame: - """Official reference table for Argo sensor makers (R26) + """Reference Table **Sensor Makers (R26)** as a :class:`pandas.DataFrame` Returns ------- @@ -325,7 +329,7 @@ def to_dataframe(self) -> pd.DataFrame: return self.r26 def hint(self) -> list[str]: - """Official list of Argo sensor maker (R26) + """List of Argo sensor makers Return a sorted list of strings with altLabel from Argo Reference table R26 on 'SENSOR_MAKER'. diff --git a/argopy/related/sensors/sensors.py b/argopy/related/sensors/sensors.py index 51afca33c..68e680925 100644 --- a/argopy/related/sensors/sensors.py +++ b/argopy/related/sensors/sensors.py @@ -4,47 +4,57 @@ class ArgoSensor(ArgoSensorSpec): - """Argo sensor(s) helper class + """Argo sensor(s) The :class:`ArgoSensor` class aims to provide direct access to Argo's sensor metadata from: - - NVS Reference Tables `Sensor Models (R27) `_, `Sensor Types (R25) `_ and `Sensor Manufacturers (R26) `_ - - `Euro-Argo fleet-monitoring API `_ + - NVS Reference Tables `Sensor Models (R27) `_, `Sensor Types (R25) `_ and `Sensor Manufacturers (R26) `_, + - the `Euro-Argo fleet-monitoring API `_. This enables users to: - - navigate reference tables 27, 25 and 26, + - navigate reference tables, + - search for floats equipped with specific sensor models, - retrieve sensor serial numbers across the global array, - search for/iterate over floats equipped with specific sensor models. - Examples -------- .. code-block:: python - :caption: Access reference tables for 'SENSOR_MODEL'/R27, 'SENSOR'/R25 and 'SENSOR_MAKER'/R26 + :caption: Access reference tables for SENSOR_MODEL (R27), SENSOR (R25) and SENSOR_MAKER (R26) from argopy import ArgoSensor + sensor = ArgoSensor() + + sensor.ref.model.to_dataframe() # Return reference table R27 with the list of sensor models as a DataFrame + sensor.ref.model.hint() # Return list of sensor model names (possible values for 'SENSOR_MODEL' parameter) + + sensor.ref.type.to_dataframe() # Return reference table R25 with the list of sensor types as a DataFrame + sensor.ref.type.hint() # Return list of sensor types (possible values for 'SENSOR' parameter) + + sensor.ref.maker.to_dataframe() # Return reference table R26 with the list of manufacturer as a DataFrame + sensor.ref.maker.hint() # Return list of manufacturer names (possible values for 'SENSOR_MAKER' parameter) - ArgoSensor().ref.model.to_dataframe() # Return reference table R27 with the list of sensor models as a DataFrame - ArgoSensor().ref.model.to_list() # Return list of sensor model names (possible values for 'SENSOR_MODEL' parameter) - ArgoSensor().ref.model.to_type('SBE61') # Return sensor type (R25) of a given model (R27) + .. code-block:: python + :caption: Mapping between SENSOR_MODEL (R27) and SENSOR (R25) + + from argopy import ArgoSensor + sensor = ArgoSensor() - ArgoSensor().ref.type.to_dataframe() # Return reference table R25 with the list of sensor types as a DataFrame - ArgoSensor().ref.type.to_list() # Return list of sensor types (possible values for 'SENSOR' parameter) - ArgoSensor().ref.type.to_model('FLUOROMETER_CDOM') # Return all possible model names (R27) for a given sensor type (R25) + sensor.ref.model.to_type('SBE61') # Return sensor type (R25) of a given model (R27) - ArgoSensor().ref.maker.to_dataframe() # Return reference table R26 with the list of manufacturer as a DataFrame - ArgoSensor().ref.maker.to_list() # Return list of manufacturer names (possible values for 'SENSOR_MAKER' parameter) + sensor.ref.type.to_model('FLUOROMETER_CDOM') # Return all possible model names (R27) for a given sensor type (R25) .. code-block:: python - :caption: Search in 'SENSOR_MODEL'/R27 reference table + :caption: Search in SENSOR_MODEL (R27) reference table from argopy import ArgoSensor + sensor = ArgoSensor() - ArgoSensor().ref.model.search('RBR') # Search and return a DataFrame - ArgoSensor().ref.model.search('RBR', output='name') # Search and return a list of names instead - ArgoSensor().ref.model.search('SBE61*') # Use of wildcards - ArgoSensor().ref.model.search('*Deep*') # Search is case-insensitive + sensor.ref.model.search('RBR') # Search and return a DataFrame + sensor.ref.model.search('RBR', output='name') # Search and return a list of names instead + sensor.ref.model.search('SBE61*') # Use of wildcards + sensor.ref.model.search('*Deep*') # Search is case-insensitive .. code-block:: python :caption: Search for all Argo floats equipped with one or more exact sensor model(s) @@ -102,15 +112,21 @@ class ArgoSensor(ArgoSensorSpec): sensor = ArgoSensor('RBR_ARGO3_DEEP6K') - sensor.vocabulary - sensor.type + sensor.vocabulary # R25 concept + sensor.type # R27 concept + # Retrieve info from floats equipped with this model: sensor.search(output='wmo') sensor.search(output='sn') sensor.search(output='wmo_sn') + sensor.search(output='df') + + # Iterator: + for af in sensors.iterfloats_with(): + print(af.WMO) .. code-block:: bash - :caption: Get serialised search results from the command-line with :class:`ArgoSensor.cli_search` + :caption: Get serialized search results from the command-line with :class:`ArgoSensor.cli_search` python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('RBR', output='wmo')" python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('RBR', output='sn')" @@ -120,11 +136,8 @@ class ArgoSensor(ArgoSensorSpec): Notes ----- - Related ADMT/AVTT work: - - https://github.com/OneArgo/ADMT/issues/112 - - https://github.com/OneArgo/ArgoVocabs/issues/156 - - https://github.com/OneArgo/ArgoVocabs/issues/157 - + Ongoing related ADMT/AVTT work can be found here: + https://github.com/OneArgo/ArgoVocabs/issues?q=state%3Aopen%20label%3A%22R25%22%20OR%20label%3AR27%20OR%20label%3AR26 """ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -137,30 +150,40 @@ class References(SensorReferences): Examples -------- .. code-block:: python - :caption: Access reference tables for 'SENSOR_MODEL'/R27, 'SENSOR'/R25 and 'SENSOR_MAKER'/R26 + :caption: Access reference tables for SENSOR_MODEL (R27), SENSOR (R25) and SENSOR_MAKER (R26) from argopy import ArgoSensor + sensor = ArgoSensor() + + sensor.ref.model.to_dataframe() # Return reference table R27 with the list of sensor models as a DataFrame + sensor.ref.model.hint() # Return list of sensor model names (possible values for 'SENSOR_MODEL' parameter) + + sensor.ref.type.to_dataframe() # Return reference table R25 with the list of sensor types as a DataFrame + sensor.ref.type.hint() # Return list of sensor types (possible values for 'SENSOR' parameter) - ArgoSensor().ref.model.to_dataframe() # Return reference table R27 with the list of sensor models as a DataFrame - ArgoSensor().ref.model.to_list() # Return list of sensor model names (possible values for 'SENSOR_MODEL' parameter) - ArgoSensor().ref.model.to_type('SBE61') # Return sensor type (R25) of a given model (R27) + sensor.ref.maker.to_dataframe() # Return reference table R26 with the list of manufacturer as a DataFrame + sensor.ref.maker.hint() # Return list of manufacturer names (possible values for 'SENSOR_MAKER' parameter) + + .. code-block:: python + :caption: Mapping between SENSOR_MODEL (R27) and SENSOR (R25) + + from argopy import ArgoSensor + sensor = ArgoSensor() - ArgoSensor().ref.type.to_dataframe() # Return reference table R25 with the list of sensor types as a DataFrame - ArgoSensor().ref.type.to_list() # Return list of sensor types (possible values for 'SENSOR' parameter) - ArgoSensor().ref.type.to_model('FLUOROMETER_CDOM') # Return all possible model names (R27) for a given sensor type (R25) + sensor.ref.model.to_type('SBE61') # Return sensor type (R25) of a given model (R27) - ArgoSensor().ref.maker.to_dataframe() # Return reference table R26 with the list of manufacturer as a DataFrame - ArgoSensor().ref.maker.to_list() # Return list of manufacturer names (possible values for 'SENSOR_MAKER' parameter) + sensor.ref.type.to_model('FLUOROMETER_CDOM') # Return all possible model names (R27) for a given sensor type (R25) .. code-block:: python - :caption: Search in 'SENSOR_MODEL'/R27 reference table + :caption: Search in SENSOR_MODEL (R27) reference table from argopy import ArgoSensor + sensor = ArgoSensor() - ArgoSensor().ref.model.search('RBR') # Search and return a DataFrame - ArgoSensor().ref.model.search('RBR', output='name') # Search and return a list of names instead - ArgoSensor().ref.model.search('SBE61*') # Use of wildcards - ArgoSensor().ref.model.search('*Deep*') # Search is case-insensitive + sensor.ref.model.search('RBR') # Search and return a DataFrame + sensor.ref.model.search('RBR', output='name') # Search and return a list of names instead + sensor.ref.model.search('SBE61*') # Use of wildcards + sensor.ref.model.search('*Deep*') # Search is case-insensitive """ diff --git a/argopy/related/sensors/spec.py b/argopy/related/sensors/spec.py index e2e35b87a..140e7b516 100644 --- a/argopy/related/sensors/spec.py +++ b/argopy/related/sensors/spec.py @@ -42,14 +42,14 @@ def __init__(self, model: str | None = None, *args, **kwargs) -> None: Parameters ---------- - model: str, optional + model: str, optional, default: None An exact sensor model name, None by default because this is optional. Otherwise, possible values can be obtained from :meth:`ArgoSensor.ref.model.hint`. Other Parameters ---------------- - fs: :class:`stores.httpstore`, default: None + fs: :class:`stores.httpstore`, optional, default: None The http filesystem to use. If None is provided, we instantiate a new one based on `cache`, `cachedir` and `timeout` options. cache : bool, optional, default: True Use cache or not for fetched data. Used only if `fs` is None. @@ -95,11 +95,12 @@ def __init__(self, model: str | None = None, *args, **kwargs) -> None: def vocabulary(self) -> SensorModel: """Argo reference "SENSOR_MODEL" vocabulary for this sensor model - ! Only available for a class instance created with an explicit sensor model name. + .. note:: + Only available for a class instance created with an explicit sensor model name. Returns ------- - :class:`SensorModel` + :class:`argopy.related.SensorModel` Raises ------ @@ -116,11 +117,12 @@ def vocabulary(self) -> SensorModel: def type(self) -> SensorType: """Argo reference "SENSOR" vocabulary for this sensor model - ! Only available for a class instance created with an explicit sensor model name. + .. note:: + Only available for a class instance created with an explicit sensor model name. - Returns: + Returns ------- - :class:`SensorType` + :class:`argopy.related.SensorType` Raises ------ @@ -135,8 +137,8 @@ def type(self) -> SensorType: def __repr__(self) -> str: if isinstance(self._vocabulary, SensorModel): - summary = [f""] - summary.append(f"TYPE➀ {self.type.long_name}") + summary = [f""] + summary.append(f"TYPE➀ {ppliststr([t.long_name for t in self.type])}") summary.append(f"MODEL➀ {self.vocabulary.long_name}") if self.vocabulary.deprecated: summary.append("β›” This model is deprecated !") @@ -167,6 +169,7 @@ def __repr__(self) -> str: for meth in [ "search", "iterfloats_with", + "from_wmo", ]: summary.append(f" β•°β”ˆβž€ ArgoSensor().{meth}()") return "\n".join(summary) @@ -492,7 +495,7 @@ def search( output: SearchOutputOptions = "wmo", progress: bool = True, errors: ErrorOptions = "raise", - serialised: bool = False, + serialized: bool = False, **kwargs, ) -> list[int] | list[str] | dict[int, str] | pd.DataFrame | str: """Search for Argo floats equipped with one or more sensor model name(s) @@ -520,8 +523,8 @@ def search( errors: str, Literal["raise", "ignore", "silent"], default: "raise" Raise an error, log it or do nothing for no search results. - serialised: bool, default: False - Return a serialised output. This allows for search results to be saved in cross-language, human-readable formats like json. + serialized: bool, default: False + Return a serialized output. This allows for search results to be saved in cross-language, human-readable formats like json. Returns ------- @@ -618,8 +621,8 @@ def get_hints(these_models: str | list[str]): errors=errors, **kwargs, ) - if serialised: - # Serialise all output format to json + if serialized: + # Serialize all output format to json if output == 'df': return results.to_json(indent=2) return json.dumps(results, indent=2) @@ -630,7 +633,7 @@ def cli_search(self, output: SearchOutputOptions = "wmo") -> str: # type: ignore """A command-line-friendly search for Argo floats equipped with one or more sensor model name(s) - This function is intended to call from the command-line and return serialized results for easy piping to other tools. + This function is intended to be called from the command-line and return serialized results for easy piping to other tools. Parameters ---------- @@ -643,7 +646,7 @@ def cli_search(self, - ``wmo``: a list of WMO numbers (integers) - ``sn``: a list of sensor serial numbers (strings) - ``wmo_sn``: a list of dictionary with WMO as key and serial numbers as values - - ``df``: a dictionary with 'WMO', 'Type', 'Model', 'Maker', 'SerialNumber', 'Units', 'Accuracy', 'Resolution' keys. + - ``df``: a dictionary with all available metadata. Examples -------- @@ -656,7 +659,7 @@ def cli_search(self, python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('RBR', output='df')" """ - print(self.search(model, output=output, serialised=True, progress=False), file=sys.stdout) + print(self.search(model, output=output, serialized=True, progress=False), file=sys.stdout) def iterfloats_with( self, @@ -702,6 +705,7 @@ def iterfloats_with( models = to_list(model) WMOs = self.search(model=models, progress=False) + WMOs = np.unique(WMOs).tolist() # 'ds' is a hidden option because I'm not 100% sure this will be needed. # ds: str, Literal['core', 'bgc', 'deep'], default='core' diff --git a/argopy/stores/implementations/http.py b/argopy/stores/implementations/http.py index 231ddd478..bc18757e2 100644 --- a/argopy/stores/implementations/http.py +++ b/argopy/stores/implementations/http.py @@ -516,14 +516,14 @@ def finalize(obj_list, **kwargs): # log.info('drop_variables_not_in_all_datasets') ds_list = drop_variables_not_in_all_datasets(ds_list) - log.info("Nb of dataset to concat: %i" % len(ds_list)) + # log.info("Nb of dataset to concat: %i" % len(ds_list)) # log.debug(concat_dim) # for ds in ds_list: # log.debug(ds[concat_dim]) - log.info( - "Dataset sizes before concat: %s" - % [len(ds[concat_dim]) for ds in ds_list] - ) + # log.info( + # "Dataset sizes before concat: %s" + # % [len(ds[concat_dim]) for ds in ds_list] + # ) ds = xr.concat( ds_list, dim=concat_dim, @@ -531,7 +531,7 @@ def finalize(obj_list, **kwargs): coords="all", compat="override", # skip comparing and pick variable from first dataset ) - log.info("Dataset size after concat: %i" % len(ds[concat_dim])) + # log.info("Dataset size after concat: %i" % len(ds[concat_dim])) return ds, True else: ds_list = [v for v in dict(sorted(obj_list.items())).values()] diff --git a/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.iterfloats_with.rst b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.iterfloats_with.rst new file mode 100644 index 000000000..445cc47d0 --- /dev/null +++ b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.iterfloats_with.rst @@ -0,0 +1,6 @@ +ο»Ώargopy.ArgoSensor.iterfloats\_with +================================== + +.. currentmodule:: argopy + +.. automethod:: ArgoSensor.iterfloats_with \ No newline at end of file diff --git a/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.maker.hint.rst b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.maker.hint.rst new file mode 100644 index 000000000..bdde55df4 --- /dev/null +++ b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.maker.hint.rst @@ -0,0 +1,6 @@ +ο»Ώargopy.ArgoSensor.ref.maker.hint +================================ + +.. currentmodule:: argopy + +.. autoaccessormethod:: ArgoSensor.ref.maker.hint \ No newline at end of file diff --git a/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.maker.to_dataframe.rst b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.maker.to_dataframe.rst new file mode 100644 index 000000000..9f92f15b5 --- /dev/null +++ b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.maker.to_dataframe.rst @@ -0,0 +1,6 @@ +ο»Ώargopy.ArgoSensor.ref.maker.to_dataframe +======================================== + +.. currentmodule:: argopy + +.. autoaccessormethod:: ArgoSensor.ref.maker.to_dataframe \ No newline at end of file diff --git a/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.model.hint.rst b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.model.hint.rst new file mode 100644 index 000000000..98c08807f --- /dev/null +++ b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.model.hint.rst @@ -0,0 +1,6 @@ +ο»Ώargopy.ArgoSensor.ref.model.hint +================================ + +.. currentmodule:: argopy + +.. autoaccessormethod:: ArgoSensor.ref.model.hint \ No newline at end of file diff --git a/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.model.search.rst b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.model.search.rst new file mode 100644 index 000000000..8996008c9 --- /dev/null +++ b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.model.search.rst @@ -0,0 +1,6 @@ +ο»Ώargopy.ArgoSensor.ref.model.search +================================== + +.. currentmodule:: argopy + +.. autoaccessormethod:: ArgoSensor.ref.model.search \ No newline at end of file diff --git a/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.model.to_dataframe.rst b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.model.to_dataframe.rst new file mode 100644 index 000000000..2b275aa92 --- /dev/null +++ b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.model.to_dataframe.rst @@ -0,0 +1,6 @@ +ο»Ώargopy.ArgoSensor.ref.model.to_dataframe +======================================== + +.. currentmodule:: argopy + +.. autoaccessormethod:: ArgoSensor.ref.model.to_dataframe \ No newline at end of file diff --git a/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.model.to_type.rst b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.model.to_type.rst new file mode 100644 index 000000000..6dcfb6ab3 --- /dev/null +++ b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.model.to_type.rst @@ -0,0 +1,6 @@ +ο»Ώargopy.ArgoSensor.ref.model.to_type +=================================== + +.. currentmodule:: argopy + +.. autoaccessormethod:: ArgoSensor.ref.model.to_type \ No newline at end of file diff --git a/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.type.hint.rst b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.type.hint.rst new file mode 100644 index 000000000..7d2b24827 --- /dev/null +++ b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.type.hint.rst @@ -0,0 +1,6 @@ +ο»Ώargopy.ArgoSensor.ref.type.hint +=============================== + +.. currentmodule:: argopy + +.. autoaccessormethod:: ArgoSensor.ref.type.hint \ No newline at end of file diff --git a/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.type.to_dataframe.rst b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.type.to_dataframe.rst new file mode 100644 index 000000000..2f13e5f82 --- /dev/null +++ b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.type.to_dataframe.rst @@ -0,0 +1,6 @@ +ο»Ώargopy.ArgoSensor.ref.type.to_dataframe +======================================= + +.. currentmodule:: argopy + +.. autoaccessormethod:: ArgoSensor.ref.type.to_dataframe \ No newline at end of file diff --git a/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.type.to_model.rst b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.type.to_model.rst new file mode 100644 index 000000000..bd88d1488 --- /dev/null +++ b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.type.to_model.rst @@ -0,0 +1,6 @@ +ο»Ώargopy.ArgoSensor.ref.type.to_model +=================================== + +.. currentmodule:: argopy + +.. autoaccessormethod:: ArgoSensor.ref.type.to_model \ No newline at end of file diff --git a/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.rst b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.rst new file mode 100644 index 000000000..3096f4533 --- /dev/null +++ b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.rst @@ -0,0 +1,33 @@ +ο»Ώargopy.ArgoSensor +================= + +.. currentmodule:: argopy + +.. autoclass:: ArgoSensor + + + .. automethod:: __init__ + + + .. rubric:: Methods + + .. autosummary:: + + ~ArgoSensor.__init__ + ~ArgoSensor.cli_search + ~ArgoSensor.from_wmo + ~ArgoSensor.iterfloats_with + ~ArgoSensor.search + + + + + + .. rubric:: Attributes + + .. autosummary:: + + ~ArgoSensor.type + ~ArgoSensor.vocabulary + + \ No newline at end of file diff --git a/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.search.rst b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.search.rst new file mode 100644 index 000000000..0c4e38e15 --- /dev/null +++ b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.search.rst @@ -0,0 +1,6 @@ +ο»Ώargopy.ArgoSensor.search +======================== + +.. currentmodule:: argopy + +.. automethod:: ArgoSensor.search \ No newline at end of file diff --git a/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.type.rst b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.type.rst new file mode 100644 index 000000000..0fcae5eb8 --- /dev/null +++ b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.type.rst @@ -0,0 +1,6 @@ +ο»Ώargopy.ArgoSensor.type +====================== + +.. currentmodule:: argopy + +.. autoproperty:: ArgoSensor.type \ No newline at end of file diff --git a/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.vocabulary.rst b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.vocabulary.rst new file mode 100644 index 000000000..f38377c1d --- /dev/null +++ b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.vocabulary.rst @@ -0,0 +1,6 @@ +ο»Ώargopy.ArgoSensor.vocabulary +============================ + +.. currentmodule:: argopy + +.. autoproperty:: ArgoSensor.vocabulary \ No newline at end of file diff --git a/docs/advanced-tools/metadata/index.rst b/docs/advanced-tools/metadata/index.rst index e8703200c..5dc0a96c8 100644 --- a/docs/advanced-tools/metadata/index.rst +++ b/docs/advanced-tools/metadata/index.rst @@ -10,10 +10,10 @@ In **argopy** we consider Argo meta-data any *related* data that can complement Here we document **argopy** tools related to: * :doc:`Argo reference tables (NVS) ` +* :doc:`Argo sensors ` * :doc:`the ADMT collection of documentation manuals ` * :doc:`the global deployment plan from Ocean-OPS ` * :doc:`GDAC snapshot with DOIs ` -* :doc:`Argo sensor models and types ` Note that data more specifically used in quality control are described in the dedicated :ref:`data_qc` documentation section (e.g. topography, altimetry, in-situ reference data from Argo float and GOSHIP, etc ...). @@ -23,6 +23,6 @@ topography, altimetry, in-situ reference data from Argo float and GOSHIP, etc .. :hidden: Argo reference tables (NVS) + Argo sensors admt_documentation deployment_plan - Argo sensor models and types diff --git a/docs/advanced-tools/metadata/sensors.rst b/docs/advanced-tools/metadata/sensors.rst index 23b920661..c67ce7896 100644 --- a/docs/advanced-tools/metadata/sensors.rst +++ b/docs/advanced-tools/metadata/sensors.rst @@ -1,4 +1,4 @@ -.. currentmodule:: argopy.related +.. currentmodule:: argopy .. _argosensor: Argo sensors @@ -6,54 +6,61 @@ Argo sensors The :class:`ArgoSensor` class aims to provide user-friendly access to Argo's sensor metadata from: -- NVS Reference Tables `Sensor Models (R27) `_, `Sensor Types (R25) `_ and `Sensor Manufacturers (R26) `_ -- the `Euro-Argo fleet-monitoring API `_ +- NVS Reference Tables `Sensor Models (R27) `_, `Sensor Types (R25) `_ and `Sensor Manufacturers (R26) `_, +- the `Euro-Argo fleet-monitoring API `_. This enables users to: - navigate reference tables, - search for floats equipped with specific sensor models, -- retrieve sensor serial numbers across the global array. +- retrieve sensor serial numbers across the global array, - search for/iterate over floats equipped with specific sensor models. .. contents:: :local: + :depth: 1 .. _argosensor-reference-tables: -Work with reference tables on sensors -------------------------------------- +Working with reference tables +----------------------------- -With the :class:`ArgoSensor` class, you can work with official Argo vocabularies for `Sensor Models (R27) `_, `Sensor Types (R25) `_ and `Sensor Manufacturers (R26) `_. With these methods, it is easy to look for a specific sensor model name, which can then be used to :ref:`look for floats ` or :ref:`create a ArgoSensor class ` directly. -.. list-table:: :class:`ArgoSensor` methods for reference tables - :header-rows: 1 - :stub-columns: 1 +With the :class:`ArgoSensor` class, you can work with official Argo vocabularies for `Sensor Models (R27) `_, `Sensor Types (R25) `_ and `Sensor Manufacturers (R26) `_. With the following methods, it is easy to look for a specific sensor model name, which can then be used to :ref:`look for floats ` or :ref:`create a ArgoSensor class ` directly. - * - Methods - - Description - * - :meth:`ArgoSensor.ref.model.to_dataframe` - - Returns Reference Table **Sensor Models (R27)** as a :class:`pandas.DataFrame` - * - :meth:`ArgoSensor.ref.type.to_dataframe` - - Returns Reference Table **Sensor Types (R25)** as a :class:`pandas.DataFrame` - * - :meth:`ArgoSensor.ref.maker.to_dataframe` - - Returns Reference Table **Sensor Manufacturers (R26)** as a :class:`pandas.DataFrame` - * - :meth:`ArgoSensor.ref.model.hint` - - List of all sensor model names (e.g., ``['AANDERAA_OPTODE', ..., 'RBR_ARGO3_DEEP6K', ..., 'SBE41CP', ...]``) - * - :meth:`ArgoSensor.ref.type.hint` - - List of all sensor types (e.g., ``[..., 'CTD_CNDC', ..., 'OPTODE_DOXY', ...]``) - * - :meth:`ArgoSensor.ref.maker.hint` - - List of all sensor makers (e.g., ``[..., 'DRUCK', ... 'SEASCAN', ... ]``) +.. currentmodule:: argopy - * - :meth:`ArgoSensor.ref.model.to_type` - - Returns sensor type of a given model - * - :meth:`ArgoSensor.ref.type.to_model` - - Returns model names of a given sensor type +**Content** from reference tables: + +.. autosummary:: + :template: autosummary/accessor_method.rst + + ArgoSensor.ref.model.to_dataframe + ArgoSensor.ref.model.hint + + ArgoSensor.ref.type.to_dataframe + ArgoSensor.ref.type.hint + + ArgoSensor.ref.maker.to_dataframe + ArgoSensor.ref.maker.hint + +**Mapping** of sensor model vs type: + +.. autosummary:: + :template: autosummary/accessor_method.rst + + ArgoSensor.ref.model.to_type + ArgoSensor.ref.type.to_model + +Model **search**: + +.. autosummary:: + :template: autosummary/accessor_method.rst + + ArgoSensor.ref.model.search - * - :meth:`ArgoSensor.ref.model.search` - - Search R27 for models matching a string, can use wildcard (eg: ``'SBE61*'``) Examples ^^^^^^^^ @@ -65,14 +72,16 @@ Examples from argopy import ArgoSensor - ArgoSensor().ref.type.to_model('CTD_CNDC') + models = ArgoSensor().ref.type.to_model('CTD_CNDC') + print(models) - Get the sensor type(s) of a given model: .. ipython:: python :okwarning: - ArgoSensor().ref.model.to_type('RBR_ARGO3_DEEP6K') + types = ArgoSensor().ref.model.to_type('RBR_ARGO3_DEEP6K') + print(types) - Search all SBE61 versions with a wildcard: @@ -81,12 +90,12 @@ Examples ArgoSensor().ref.model.search('SBE61*') -- Get one model single model description (see also :attr:`ArgoSensor.vocabulary`): +- Get one single model description (see also :attr:`ArgoSensor.vocabulary`): .. ipython:: python :okwarning: - ArgoSensor().ref.model.search('SBE61_V5.0.1') + ArgoSensor().ref.model.search('SBE61_V5.0.1').T .. _argosensor-search-floats: @@ -124,7 +133,8 @@ Examples .. ipython:: python :okwarning: - ArgoSensor().search("SBE43F_IDO", output="sn") + serials = ArgoSensor().search("SBE43F_IDO", output="sn") + print(serials) - Get everything for floats equipped with the "RBR_ARGO3_DEEP6K" model: @@ -139,16 +149,16 @@ Examples Use an exact sensor model name to create a specific ArgoSensor -------------------------------------------------------------- -You can initialize an :class:`ArgoSensor` instance with a specific model to access more metadata. +You can initialize an :class:`ArgoSensor` instance with a specific model to access more metadata. In this use-case, you will have the following attributes and methods available: -.. csv-table:: :class:`ArgoSensor` Attributes and Methods for a specific model - :header: "Attribute/Method", "Description" - :widths: 30, 70 +.. currentmodule:: argopy - :class:`ArgoSensor`, "Create an instance for an exact model name (e.g., ``'SBE43F_IDO'``)" - :attr:`ArgoSensor.vocabulary`, "Returns a :class:`SensorModel` object with R27 concept vocabulary (name, long_name, definition, URI, deprecated)" - :attr:`ArgoSensor.type`, "Returns a :class:`SensorType` object with R25 concept (name, long_name, definition, URI, deprecated)" - :meth:`ArgoSensor.search`, "Inherits search methods but defaults to the instance's model" +.. autosummary:: + + ArgoSensor.vocabulary + ArgoSensor.type + ArgoSensor.search + ArgoSensor.iterfloats_with Examples ^^^^^^^^ @@ -168,14 +178,14 @@ You can then access this model metadata from the NVS vocabulary (Reference table sensor.vocabulary -from Reference table R25: +and from Reference table R25: .. ipython:: python :okwarning: sensor.type -And you can look for floats equipped with it: +You can also look for floats equipped with it: .. ipython:: python :okwarning: @@ -191,26 +201,36 @@ The :meth:`ArgoSensor.iterfloats_with` will yields :class:`argopy.ArgoFloat` ins Example ^^^^^^^ -Let's try to gather all platform types of WMOs equipped with a list of sensor models: +Let's try to gather all platform types and WMOs for floats equipped with a list of sensor models: -.. ipython:: python - :okwarning: +.. code-block:: python - sensors = ArgoSensor() - - models = ['ECO_FLBBCD_AP2', 'ECO_FLBBCD'] - results = {} - for af in sensors.iterfloats_with(models): - if 'meta' in af.ls_dataset(): - platform_type = af.metadata['platform']['type'] # e.g. 'PROVOR_V_JUMBO' - if platform_type in results.keys(): - results[platform_type].extend([af.WMO]) - else: - results.update({platform_type: [af.WMO]}) - else: - print(f"No meta file for float {af.WMO}") - print(results.keys()) + sensors = ArgoSensor() + models = ['ECO_FLBBCD_AP2', 'ECO_FLBBCD'] + results = {} + for af in sensors.iterfloats_with(models): + if 'meta' in af.ls_dataset(): + platform_type = af.metadata['platform']['type'] # e.g. 'PROVOR_V_JUMBO' + if platform_type in results.keys(): + results[platform_type].extend([af.WMO]) + else: + results.update({platform_type: [af.WMO]}) + else: + print(f"No meta file for float {af.WMO}") + + [f"{r:15s}: {len(results[r])} floats" for r in results.keys()] + +.. code-block:: python + + ['APEX : 37 floats', + 'SOLO_BGC_MRV : 19 floats', + 'PROVOR_V_JUMBO : 43 floats', + 'PROVOR_III : 206 floats', + 'PROVOR_V : 38 floats', + 'PROVOR : 18 floats', + 'PROVOR_IV : 21 floats', + 'SOLO_BGC : 4 floats'] Quick float and sensor lookups from the command line ---------------------------------------------------- diff --git a/docs/api-hidden.rst b/docs/api-hidden.rst index 2e5f5d721..1a1bd6ba1 100644 --- a/docs/api-hidden.rst +++ b/docs/api-hidden.rst @@ -178,24 +178,24 @@ argopy.related.doi_snapshot.DOIrecord argopy.related.ArgoSensor - argopy.related.ArgoSensor.ref.model.to_dataframe - argopy.related.ArgoSensor.ref.model.hint - argopy.related.ArgoSensor.ref.model.to_type - argopy.related.ArgoSensor.ref.model.search - argopy.related.ArgoSensor.ref.type.to_dataframe - argopy.related.ArgoSensor.ref.type.hint - argopy.related.ArgoSensor.ref.type.to_model - argopy.related.ArgoSensor.ref.maker.to_dataframe - argopy.related.ArgoSensor.ref.maker.hint - - argopy.related.ArgoSensor.vocabulary - argopy.related.ArgoSensor.type - argopy.related.ArgoSensor.search - argopy.related.ArgoSensor.iterfloats_with - argopy.related.ArgoSensor.cli_search - - argopy.related.sensors.SensorModel - argopy.related.sensors.SensorType + argopy.ArgoSensor.ref.model.to_dataframe + argopy.ArgoSensor.ref.model.hint + argopy.ArgoSensor.ref.model.to_type + argopy.ArgoSensor.ref.model.search + argopy.ArgoSensor.ref.type.to_dataframe + argopy.ArgoSensor.ref.type.hint + argopy.ArgoSensor.ref.type.to_model + argopy.ArgoSensor.ref.maker.to_dataframe + argopy.ArgoSensor.ref.maker.hint + + argopy.ArgoSensor.vocabulary + argopy.ArgoSensor.type + argopy.ArgoSensor.search + argopy.ArgoSensor.iterfloats_with + argopy.ArgoSensor.cli_search + + argopy.related.SensorModel + argopy.related.SensorType argopy.plot argopy.plot.dashboard diff --git a/docs/whats-new.rst b/docs/whats-new.rst index 23b98bf88..9055e7dd5 100644 --- a/docs/whats-new.rst +++ b/docs/whats-new.rst @@ -14,7 +14,7 @@ Coming up next (un-released) Features and front-end API ^^^^^^^^^^^^^^^^^^^^^^^^^^ -- **New** :class:`ArgoSensor` class to work with Argo sensor models, types and floats equipped with. Check the new :ref:`Argo sensor: models and types` documentation section. (:pr:`532`) by |gmaze|. +- **New** :class:`ArgoSensor` class to work with Argo sensor models, types and floats equipped with. Check the new :ref:`Argo sensors` documentation section. (:pr:`532`) by |gmaze|. v1.3.1 (22 Oct. 2025) From ae9aef961a04f1b827266e276c9b72a88999ce1b Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Wed, 29 Oct 2025 11:22:36 +0100 Subject: [PATCH 58/71] refactor OEM sensor metadata --- argopy/related/sensors/__init__.py | 3 +- argopy/related/sensors/oem/__init__.py | 3 + .../related/sensors/{ => oem}/oem_metadata.py | 64 ++++++++++++------- .../sensors/{ => oem}/oem_metadata_repr.py | 6 +- 4 files changed, 49 insertions(+), 27 deletions(-) create mode 100644 argopy/related/sensors/oem/__init__.py rename argopy/related/sensors/{ => oem}/oem_metadata.py (90%) rename argopy/related/sensors/{ => oem}/oem_metadata_repr.py (97%) diff --git a/argopy/related/sensors/__init__.py b/argopy/related/sensors/__init__.py index c5a3e8313..3aa6abed0 100644 --- a/argopy/related/sensors/__init__.py +++ b/argopy/related/sensors/__init__.py @@ -1,4 +1,5 @@ from .sensors import ArgoSensor from .accessories import SensorType, SensorModel +from .oem import OEMSensorMetaData -__all__ = ["ArgoSensor", "SensorType", "SensorModel"] +__all__ = ["ArgoSensor", "SensorType", "SensorModel", "OEMSensorMetaData"] diff --git a/argopy/related/sensors/oem/__init__.py b/argopy/related/sensors/oem/__init__.py new file mode 100644 index 000000000..c65357300 --- /dev/null +++ b/argopy/related/sensors/oem/__init__.py @@ -0,0 +1,3 @@ +from .oem_metadata import OEMSensorMetaData + +__all__ = ['OEMSensorMetaData'] \ No newline at end of file diff --git a/argopy/related/sensors/oem_metadata.py b/argopy/related/sensors/oem/oem_metadata.py similarity index 90% rename from argopy/related/sensors/oem_metadata.py rename to argopy/related/sensors/oem/oem_metadata.py index 31e87b0b3..10975e0ab 100644 --- a/argopy/related/sensors/oem_metadata.py +++ b/argopy/related/sensors/oem/oem_metadata.py @@ -9,14 +9,15 @@ import warnings import pandas as pd -from ...stores import httpstore, filestore -from ...options import OPTIONS -from ...utils import urnparser, path2assets -from ...errors import InvalidDatasetStructure +from ....stores import httpstore, filestore +from ....options import OPTIONS +from ....utils import urnparser, path2assets +from ....errors import InvalidDatasetStructure + from .oem_metadata_repr import OemMetaDataDisplay, ParameterDisplay -log = logging.getLogger("argopy.related.sensors") +log = logging.getLogger("argopy.related.sensors.oem") SENSOR_JS_EXAMPLES = filestore().open_json( Path(path2assets).joinpath("sensor_metadata_examples.json") @@ -202,7 +203,7 @@ def __repr__(self): for key in ["parameter_vendorinfo", "predeployment_vendorinfo"]: if getattr(self, key, None) is not None: - summary.append(f" {key}: {self._attr2str(p)}") + summary.append(f" {key}: {self._attr2str(key)}") else: summary.append(f" {key}: None") return "\n".join(summary) @@ -216,30 +217,34 @@ def _ipython_display_(self): display(HTML(ParameterDisplay(self).html)) -class ArgoSensorMetaDataOem: - """Argo sensor meta-data - from OEM +class OEMSensorMetaData: + """OEM sensor meta-data - A class helper to work with meta-data structure complying to schema from https://github.com/euroargodev/sensor_metadata_json + A class helper to work with sensor meta-data complying to schema from https://github.com/euroargodev/sensor_metadata_json Such meta-data structures are expected to come from sensor manufacturer (web-api or file). - OEM : Original Equipment Manufacturer + .. note:: + + OEM : Original Equipment Manufacturer Examples -------- .. code-block:: python - ArgoSensorMetaData() + OEMSensorMetaData() + + OEMSensorMetaData(validate=True) # Run json schema validation compliance when necessary - ArgoSensorMetaData(validate=True) # Run json schema validation compliance when necessary + OEMSensorMetaData().from_rbr(208380) # Direct call to RBR api with a serial number - ArgoSensorMetaData().from_rbr(208380) # Direct call to the RBR api with a serial number + OEMSensorMetaData().from_seabird(2444, 'SATLANTIC_OCR504_ICSW') # Direct call to Seabird api with a serial number and model name - ArgoSensorMetaData().list_examples + OEMSensorMetaData().list_examples - ArgoSensorMetaData().from_examples('WETLABS-ECO_FLBBAP2-8589') + OEMSensorMetaData().from_examples('WETLABS-ECO_FLBBAP2-8589') - ArgoSensorMetaData().from_dict(jsdata) # Use any compliant json data + OEMSensorMetaData().from_dict(jsdata) # Use any compliant json data """ @@ -290,7 +295,7 @@ def _repr_hint(self): "from_rbr(serial_number)", "from_dict(dict_or_json_data)", ]: - summary.append(f" β•°β”ˆβž€ ArgoSensorMetaDataOem().{meth}") + summary.append(f" β•°β”ˆβž€ OEMSensorMetaData().{meth}") return summary def __repr__(self): @@ -459,12 +464,12 @@ def to_json_file(self, file_path: str) -> None: Notes ----- - The output json file should be compliant with the Argo sensor meta-data JSON schema :attr:`ArgoSensorMetaDataOem.schema` + The output json file should be compliant with the Argo sensor meta-data JSON schema :attr:`OEMSensorMetaData.schema` """ with open(file_path, "w") as f: json.dump(self.to_dict(), f, indent=2) - def from_rbr(self, serial_number: str, **kwargs) -> 'ArgoSensorMetaDataOem': + def from_rbr(self, serial_number: str, **kwargs) -> 'OEMSensorMetaData': """Fetch sensor metadata from "RBRargo Product Lookup" web-API We also download certificates if available @@ -572,7 +577,7 @@ def _certificates_rbr( else: raise ValueError(f"Unknown action {action}") - def from_seabird(self, serial_number: str, sensor_model: str, **kwargs) -> 'ArgoSensorMetaDataOem': + def from_seabird(self, serial_number: str, sensor_model: str, **kwargs) -> 'OEMSensorMetaData': """Fetch sensor metadata from Seabird-Scientific "Instrument Metadata Portal" web-API Parameters @@ -582,9 +587,9 @@ def from_seabird(self, serial_number: str, sensor_model: str, **kwargs) -> 'Argo """ # The Seabird api requires a sensor model (R27) with a serial number. # This could easily be done if we get the s/n by searching float metadata. - # In other word, it's easy to go from a sensor_model to a serial_number - # But it is much more complicated to from a serial_number to a sensor_model. - # so, the time being, we'll ask users to specify a sensor_model. + # In other word, it's easy to go from a sensor_model to a serial_number. + # But it is much more complicated to go from a serial_number to a sensor_model. + # so, for the time being, we'll ask users to specify a sensor_model. url = f"{OPTIONS['seabird_api']}?SENSOR_SERIAL_NO={serial_number}&SENSOR_MODEL={sensor_model}" data = self._fs.open_json(url) @@ -596,6 +601,19 @@ def from_seabird(self, serial_number: str, sensor_model: str, **kwargs) -> 'Argo data['sensor_info'].update({'sensor_described': f"{sensor_model}-{serial_number}"}) data.pop('json_info', None) + # Check in vendor info for missing mandatory parameter attributes: + for ii, param in enumerate(data['PARAMETERS']): + vendorinfo = param['parameter_vendorinfo'] + if param.get('PARAMETER_UNITS', None) is None: + if 'units' in vendorinfo: + data['PARAMETERS'][ii]['PARAMETER_UNITS'] = vendorinfo['units'] + if param.get('PREDEPLOYMENT_CALIB_COMMENT', None) is None: + if 'units' in vendorinfo: + data['PARAMETERS'][ii]['PREDEPLOYMENT_CALIB_COMMENT'] = vendorinfo['comments'] + if param.get('PREDEPLOYMENT_CALIB_DATE', None) is None: + if 'units' in vendorinfo: + data['PARAMETERS'][ii]['PREDEPLOYMENT_CALIB_DATE'] = vendorinfo['calibration_date'] + # Create and return an instance obj = self.from_dict(data) return obj diff --git a/argopy/related/sensors/oem_metadata_repr.py b/argopy/related/sensors/oem/oem_metadata_repr.py similarity index 97% rename from argopy/related/sensors/oem_metadata_repr.py rename to argopy/related/sensors/oem/oem_metadata_repr.py index 78f1c65a6..67b01d7df 100644 --- a/argopy/related/sensors/oem_metadata_repr.py +++ b/argopy/related/sensors/oem/oem_metadata_repr.py @@ -11,7 +11,7 @@ importlib.util.find_spec(x).submodule_search_locations[0] ) -from ...utils import urnparser +from ....utils import urnparser STATIC_FILES = ( @@ -54,7 +54,7 @@ def html(self): # --- Header --- header_html = f""" -

Argo Sensor Metadata: {self.OEMsensor.sensor_info._attr2str('sensor_described')}

+

Argo Sensor OEM Metadata: {self.OEMsensor.sensor_info._attr2str('sensor_described')}

Created by: {self.OEMsensor.sensor_info._attr2str('created_by')} | Date: {self.OEMsensor.sensor_info._attr2str('date_creation')}

""" @@ -214,7 +214,7 @@ def html(self): param = self.data # --- Header --- - header_html = f"

Argo Sensor Metadata for Parameter: {urn_html(param.PARAMETER)}

" + header_html = f"

Argo Sensor OEM Metadata for Parameter: {urn_html(param.PARAMETER)}

" if param.parameter_vendorinfo is not None: info = " | ".join([f"{p} {v}" for p, v in param.parameter_vendorinfo.items()]) From ad513e397b071bdff238f23c7f54ad970faef8d3 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Wed, 29 Oct 2025 11:45:13 +0100 Subject: [PATCH 59/71] New display_style option for html vs text --- argopy/options.py | 10 +- argopy/related/sensors/oem/accessories.py | 199 ++++++++++++++++++ argopy/related/sensors/oem/oem_metadata.py | 225 ++------------------- 3 files changed, 223 insertions(+), 211 deletions(-) create mode 100644 argopy/related/sensors/oem/accessories.py diff --git a/argopy/options.py b/argopy/options.py index 43bdd9598..257094a41 100644 --- a/argopy/options.py +++ b/argopy/options.py @@ -55,6 +55,8 @@ API_RBR = "rbr_api" NVS = "nvs" API_SEABIRD = "seabird_api" +DISPLAY_STYLE = "display_style" + # Define the list of available options and default values: OPTIONS = { @@ -79,6 +81,7 @@ API_RBR: "https://oem-lookup.rbr-global.com/api/v1", NVS: "https://vocab.nerc.ac.uk/collection", API_SEABIRD: "https://instrument.seabirdhub.com/api/argo-calibration", + DISPLAY_STYLE: "html", } DEFAULT = OPTIONS.copy() @@ -86,8 +89,8 @@ _DATA_SOURCE_LIST = frozenset(["erddap", "argovis", "gdac"]) _DATASET_LIST = frozenset(["phy", "bgc", "ref", "bgc-s", "bgc-b"]) _USER_LEVEL_LIST = frozenset(["standard", "expert", "research"]) - - +_DISPLAY_STYLE_LIST = frozenset(["html", "text"]) +_LON_LIST = frozenset(['180', '360']) def _positive_integer(value): return isinstance(value, int) and value > 0 @@ -298,11 +301,12 @@ def check_gdac_option( RBR_API_KEY: lambda x: isinstance(x, str) or x is None, PARALLEL: validate_parallel, PARALLEL_DEFAULT_METHOD: validate_parallel_method, - LON: lambda x: x in ['180', '360'], + LON: _LON_LIST.__contains__, API_FLEETMONITORING: validate_fleetmonitoring, API_RBR: validate_rbr, API_SEABIRD: validate_seabird, NVS: lambda x: isinstance(x, str) or x is None, + DISPLAY_STYLE: _DISPLAY_STYLE_LIST.__contains__, } diff --git a/argopy/related/sensors/oem/accessories.py b/argopy/related/sensors/oem/accessories.py new file mode 100644 index 000000000..4cbe1600d --- /dev/null +++ b/argopy/related/sensors/oem/accessories.py @@ -0,0 +1,199 @@ +from dataclasses import dataclass +from typing import Dict, Optional, Any +from html import escape + +from ....options import OPTIONS +from ....utils import urnparser + +from .oem_metadata_repr import ParameterDisplay + + +@dataclass +class SensorInfo: + created_by: str + date_creation: str # ISO 8601 datetime string + link: str + format_version: str + contents: str + + # Made optional to accommodate errors in OEM data + sensor_described: str = None + + def _attr2str(self, x): + """Return a class attribute, or 'n/a' if it's None or "".""" + value = getattr(self, x, None) + if value is None: + return "n/a" + elif type(value) is str: + return value if value and value.strip() else "n/a" + else: + return value + + +@dataclass +class Context: + SDN_R03: str = "http://vocab.nerc.ac.uk/collection/R03/current/" + SDN_R25: str = "http://vocab.nerc.ac.uk/collection/R25/current/" + SDN_R26: str = "http://vocab.nerc.ac.uk/collection/R26/current/" + SDN_R27: str = "http://vocab.nerc.ac.uk/collection/R27/current/" + SDN_L22: str = "http://vocab.nerc.ac.uk/collection/L22/current/" + + +@dataclass +class Sensor: + SENSOR: str # SDN:R25::CTD_PRES + SENSOR_MAKER: str # SDN:R26::RBR + SENSOR_MODEL: str # SDN:R27::RBR_PRES_A + SENSOR_SERIAL_NO: str + + # FIRMWARE VERSION attributes are temporarily optional to handle the wrong key used by RBR + # see https://github.com/euroargodev/sensor_metadata_json/issues/20 + SENSOR_FIRMWARE_VERSION: str = None # RBR + SENSOR_MODEL_FIRMWARE: str = None # Correct schema key + + sensor_vendorinfo: Optional[Dict[str, Any]] = None + + @property + def SENSOR_uri(self): + urnparts = urnparser(self.SENSOR) + return f"{OPTIONS['nvs']}/{urnparts['listid']}/current/{urnparts['termid']}" + + @property + def SENSOR_MAKER_uri(self): + urnparts = urnparser(self.SENSOR_MAKER) + return f"{OPTIONS['nvs']}/{urnparts['listid']}/current/{urnparts['termid']}" + + @property + def SENSOR_MODEL_uri(self): + urnparts = urnparser(self.SENSOR_MODEL) + return f"{OPTIONS['nvs']}/{urnparts['listid']}/current/{urnparts['termid']}" + + def _attr2str(self, x): + """Return a class attribute, or 'n/a' if it's None, {} or "".""" + value = getattr(self, x, None) + if value is None: + return "n/a" + elif type(value) is str: + return value if value and value.strip() else "n/a" + elif type(value) is dict: + if len(value.keys()) == 0: + return "n/a" + else: + return value + else: + return value + + def __repr__(self): + + def key2str(d, x): + """Return a dict value as a string, or 'n/a' if it's None or empty.""" + value = d.get(x, None) + return value if value and value.strip() else "n/a" + + summary = [f"<{self.SENSOR}><{self.SENSOR_SERIAL_NO}>"] + summary.append(f" SENSOR: {self.SENSOR} ({self.SENSOR_uri})") + summary.append(f" SENSOR_MAKER: {self.SENSOR_MAKER} ({self.SENSOR_MAKER_uri})") + summary.append(f" SENSOR_MODEL: {self.SENSOR_MODEL} ({self.SENSOR_MODEL_uri})") + if getattr(self, "SENSOR_MODEL_FIRMWARE", None) is None: + summary.append( + f" SENSOR_FIRMWARE_VERSION: {self._attr2str('SENSOR_FIRMWARE_VERSION')} (but should be 'SENSOR_MODEL_FIRMWARE') " + ) + else: + summary.append( + f" SENSOR_MODEL_FIRMWARE: {self._attr2str('SENSOR_MODEL_FIRMWARE')}" + ) + if getattr(self, "sensor_vendorinfo", None) is not None: + summary.append(f" sensor_vendorinfo:") + for key in self.sensor_vendorinfo.keys(): + summary.append(f" - {key}: {key2str(self.sensor_vendorinfo, key)}") + else: + summary.append(f" sensor_vendorinfo: None") + return "\n".join(summary) + + +@dataclass +class Parameter: + PARAMETER: str # SDN:R03::PRES + PARAMETER_SENSOR: str # SDN:R25::CTD_PRES + PARAMETER_ACCURACY: str + PARAMETER_RESOLUTION: str + PREDEPLOYMENT_CALIB_COEFFICIENT_LIST: Dict[str, str] + + # Made optional to accommodate errors in OEM data + PARAMETER_UNITS: str = None + PREDEPLOYMENT_CALIB_EQUATION: str = None + PREDEPLOYMENT_CALIB_COMMENT: str = None + PREDEPLOYMENT_CALIB_DATE: str = None + + parameter_vendorinfo: Optional[Dict[str, Any]] = None + predeployment_vendorinfo: Optional[Dict[str, Any]] = None + + @property + def PARAMETER_uri(self): + urnparts = urnparser(self.PARAMETER) + return f"{OPTIONS['nvs']}/{urnparts['listid']}/current/{urnparts['termid']}" + + @property + def PARAMETER_SENSOR_uri(self): + urnparts = urnparser(self.PARAMETER_SENSOR) + return f"{OPTIONS['nvs']}/{urnparts['listid']}/current/{urnparts['termid']}" + + def _attr2str(self, x): + """Return a class attribute, or 'n/a' if it's None, {} or "".""" + value = getattr(self, x, None) + if value is None: + return "n/a" + elif type(value) is str: + return value if value and value.strip() else "n/a" + elif type(value) is dict: + if len(value.keys()) == 0: + return "n/a" + else: + return value + else: + return value + + @property + def _has_calibration_data(self): + s = "".join( + [ + str(self._attr2str(key)) + for key in [ + "PREDEPLOYMENT_CALIB_EQUATION", + "PREDEPLOYMENT_CALIB_COEFFICIENT_LIST", + "PREDEPLOYMENT_CALIB_COMMENT", + "PREDEPLOYMENT_CALIB_DATE", + ] + if self._attr2str(key) != "n/a" + ] + ) + return len(s) > 0 + + def __repr__(self): + + summary = [f"<{self.PARAMETER}>"] + summary.append(f" PARAMETER: {self.PARAMETER} ({self.PARAMETER_uri})") + summary.append( + f" PARAMETER_SENSOR: {self.PARAMETER_SENSOR} ({self.PARAMETER_SENSOR_uri})" + ) + + for key in ["UNITS", "ACCURACY", "RESOLUTION"]: + p = f"PARAMETER_{key}" + summary.append(f" {key}: {self._attr2str(p)}") + + summary.append(f" PREDEPLOYMENT CALIBRATION:") + for key in ["EQUATION", "COEFFICIENT", "COMMENT", "DATE"]: + p = f"PREDEPLOYMENT_CALIB_{key}" + summary.append(f" - {key}: {self._attr2str(p)}") + + for key in ["parameter_vendorinfo", "predeployment_vendorinfo"]: + if getattr(self, key, None) is not None: + summary.append(f" {key}: {self._attr2str(key)}") + else: + summary.append(f" {key}: None") + return "\n".join(summary) + + def _repr_html_(self): + if OPTIONS["display_style"] == "text": + return f"
{escape(repr(self))}
" + return ParameterDisplay(self).html diff --git a/argopy/related/sensors/oem/oem_metadata.py b/argopy/related/sensors/oem/oem_metadata.py index 10975e0ab..cf6d191b7 100644 --- a/argopy/related/sensors/oem/oem_metadata.py +++ b/argopy/related/sensors/oem/oem_metadata.py @@ -1,4 +1,4 @@ -from dataclasses import dataclass, field +from dataclasses import field from typing import List, Dict, Optional, Any, Literal from pathlib import Path from zipfile import ZipFile @@ -8,13 +8,15 @@ import logging import warnings import pandas as pd +from html import escape from ....stores import httpstore, filestore from ....options import OPTIONS from ....utils import urnparser, path2assets from ....errors import InvalidDatasetStructure -from .oem_metadata_repr import OemMetaDataDisplay, ParameterDisplay +from .oem_metadata_repr import OemMetaDataDisplay +from .accessories import SensorInfo, Context, Sensor, Parameter log = logging.getLogger("argopy.related.sensors.oem") @@ -24,199 +26,6 @@ )["data"]["uri"] -@dataclass -class SensorInfo: - created_by: str - date_creation: str # ISO 8601 datetime string - link: str - format_version: str - contents: str - - # Made optional to accommodate errors in OEM data - sensor_described: str = None - - def _attr2str(self, x): - """Return a class attribute, or 'n/a' if it's None or "".""" - value = getattr(self, x, None) - if value is None: - return "n/a" - elif type(value) is str: - return value if value and value.strip() else "n/a" - else: - return value - -@dataclass -class Context: - SDN_R03: str = "http://vocab.nerc.ac.uk/collection/R03/current/" - SDN_R25: str = "http://vocab.nerc.ac.uk/collection/R25/current/" - SDN_R26: str = "http://vocab.nerc.ac.uk/collection/R26/current/" - SDN_R27: str = "http://vocab.nerc.ac.uk/collection/R27/current/" - SDN_L22: str = "http://vocab.nerc.ac.uk/collection/L22/current/" - - -@dataclass -class Sensor: - SENSOR: str # SDN:R25::CTD_PRES - SENSOR_MAKER: str # SDN:R26::RBR - SENSOR_MODEL: str # SDN:R27::RBR_PRES_A - SENSOR_SERIAL_NO: str - - # FIRMWARE VERSION attributes are temporarily optional to handle the wrong key used by RBR - # see https://github.com/euroargodev/sensor_metadata_json/issues/20 - SENSOR_FIRMWARE_VERSION: str = None # RBR - SENSOR_MODEL_FIRMWARE: str = None # Correct schema key - - sensor_vendorinfo: Optional[Dict[str, Any]] = None - - @property - def SENSOR_uri(self): - urnparts = urnparser(self.SENSOR) - return f"{OPTIONS['nvs']}/{urnparts['listid']}/current/{urnparts['termid']}" - - @property - def SENSOR_MAKER_uri(self): - urnparts = urnparser(self.SENSOR_MAKER) - return f"{OPTIONS['nvs']}/{urnparts['listid']}/current/{urnparts['termid']}" - - @property - def SENSOR_MODEL_uri(self): - urnparts = urnparser(self.SENSOR_MODEL) - return f"{OPTIONS['nvs']}/{urnparts['listid']}/current/{urnparts['termid']}" - - def _attr2str(self, x): - """Return a class attribute, or 'n/a' if it's None, {} or "".""" - value = getattr(self, x, None) - if value is None: - return "n/a" - elif type(value) is str: - return value if value and value.strip() else "n/a" - elif type(value) is dict: - if len(value.keys()) == 0: - return "n/a" - else: - return value - else: - return value - - def __repr__(self): - - def key2str(d, x): - """Return a dict value as a string, or 'n/a' if it's None or empty.""" - value = d.get(x, None) - return value if value and value.strip() else "n/a" - - summary = [f"<{self.SENSOR}><{self.SENSOR_SERIAL_NO}>"] - summary.append(f" SENSOR: {self.SENSOR} ({self.SENSOR_uri})") - summary.append(f" SENSOR_MAKER: {self.SENSOR_MAKER} ({self.SENSOR_MAKER_uri})") - summary.append(f" SENSOR_MODEL: {self.SENSOR_MODEL} ({self.SENSOR_MODEL_uri})") - if getattr(self, "SENSOR_MODEL_FIRMWARE", None) is None: - summary.append( - f" SENSOR_FIRMWARE_VERSION: {self._attr2str('SENSOR_FIRMWARE_VERSION')} (but should be 'SENSOR_MODEL_FIRMWARE') " - ) - else: - summary.append( - f" SENSOR_MODEL_FIRMWARE: {self._attr2str('SENSOR_MODEL_FIRMWARE')}" - ) - if getattr(self, "sensor_vendorinfo", None) is not None: - summary.append(f" sensor_vendorinfo:") - for key in self.sensor_vendorinfo.keys(): - summary.append(f" - {key}: {key2str(self.sensor_vendorinfo, key)}") - else: - summary.append(f" sensor_vendorinfo: None") - return "\n".join(summary) - - -@dataclass -class Parameter: - PARAMETER: str # SDN:R03::PRES - PARAMETER_SENSOR: str # SDN:R25::CTD_PRES - PARAMETER_ACCURACY: str - PARAMETER_RESOLUTION: str - PREDEPLOYMENT_CALIB_COEFFICIENT_LIST: Dict[str, str] - - # Made optional to accommodate errors in OEM data - PARAMETER_UNITS: str = None - PREDEPLOYMENT_CALIB_EQUATION: str = None - PREDEPLOYMENT_CALIB_COMMENT: str = None - PREDEPLOYMENT_CALIB_DATE: str = None - - parameter_vendorinfo: Optional[Dict[str, Any]] = None - predeployment_vendorinfo: Optional[Dict[str, Any]] = None - - @property - def PARAMETER_uri(self): - urnparts = urnparser(self.PARAMETER) - return f"{OPTIONS['nvs']}/{urnparts['listid']}/current/{urnparts['termid']}" - - @property - def PARAMETER_SENSOR_uri(self): - urnparts = urnparser(self.PARAMETER_SENSOR) - return f"{OPTIONS['nvs']}/{urnparts['listid']}/current/{urnparts['termid']}" - - def _attr2str(self, x): - """Return a class attribute, or 'n/a' if it's None, {} or "".""" - value = getattr(self, x, None) - if value is None: - return "n/a" - elif type(value) is str: - return value if value and value.strip() else "n/a" - elif type(value) is dict: - if len(value.keys()) == 0: - return "n/a" - else: - return value - else: - return value - - @property - def _has_calibration_data(self): - s = "".join( - [ - str(self._attr2str(key)) - for key in [ - "PREDEPLOYMENT_CALIB_EQUATION", - "PREDEPLOYMENT_CALIB_COEFFICIENT_LIST", - "PREDEPLOYMENT_CALIB_COMMENT", - "PREDEPLOYMENT_CALIB_DATE", - ] - if self._attr2str(key) != "n/a" - ] - ) - return len(s) > 0 - - def __repr__(self): - - summary = [f"<{self.PARAMETER}>"] - summary.append(f" PARAMETER: {self.PARAMETER} ({self.PARAMETER_uri})") - summary.append( - f" PARAMETER_SENSOR: {self.PARAMETER_SENSOR} ({self.PARAMETER_SENSOR_uri})" - ) - - for key in ["UNITS", "ACCURACY", "RESOLUTION"]: - p = f"PARAMETER_{key}" - summary.append(f" {key}: {self._attr2str(p)}") - - summary.append(f" PREDEPLOYMENT CALIBRATION:") - for key in ["EQUATION", "COEFFICIENT", "COMMENT", "DATE"]: - p = f"PREDEPLOYMENT_CALIB_{key}" - summary.append(f" - {key}: {self._attr2str(p)}") - - for key in ["parameter_vendorinfo", "predeployment_vendorinfo"]: - if getattr(self, key, None) is not None: - summary.append(f" {key}: {self._attr2str(key)}") - else: - summary.append(f" {key}: None") - return "\n".join(summary) - - def _repr_html_(self): - return ParameterDisplay(self).html - - def _ipython_display_(self): - from IPython.display import display, HTML - - display(HTML(ParameterDisplay(self).html)) - - class OEMSensorMetaData: """OEM sensor meta-data @@ -234,7 +43,7 @@ class OEMSensorMetaData: OEMSensorMetaData() - OEMSensorMetaData(validate=True) # Run json schema validation compliance when necessary + OEMSensorMetaData(validate=True) # Use this option to run json schema validation compliance when necessary OEMSensorMetaData().from_rbr(208380) # Direct call to RBR api with a serial number @@ -293,6 +102,7 @@ def _repr_hint(self): ) for meth in [ "from_rbr(serial_number)", + "from_seabird(serial_number, model_name)", "from_dict(dict_or_json_data)", ]: summary.append(f" β•°β”ˆβž€ OEMSensorMetaData().{meth}") @@ -327,18 +137,17 @@ def __repr__(self): return "\n".join(summary) def _repr_html_(self): - if self.sensor_info: - return OemMetaDataDisplay(self).html - else: - return self._repr_hint() - - def _ipython_display_(self): - from IPython.display import display, HTML - - if self.sensor_info: - display(HTML(OemMetaDataDisplay(self).html)) - else: - display("\n".join(self._repr_hint())) + if OPTIONS["display_style"] == "text": + return f"
{escape(repr(self))}
" + return OemMetaDataDisplay(self).html + + # def _ipython_display_(self): + # from IPython.display import display, HTML + # + # if self.sensor_info: + # display(HTML(OemMetaDataDisplay(self).html)) + # else: + # display("\n".join(self._repr_hint())) def _read_schema(self, ref="argo.sensor.schema.json") -> Dict[str, Any]: """Load a JSON schema for validation From 54b8e454b8c9c5152d26a2a85f457648dca22bec Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Wed, 29 Oct 2025 12:39:57 +0100 Subject: [PATCH 60/71] fix dep. --- argopy/__init__.py | 3 ++- argopy/related/__init__.py | 3 ++- argopy/related/sensors/oem/oem_metadata.py | 15 ++++++++++++--- argopy/related/sensors/utils.py | 15 +++++++++++++++ argopy/utils/locals.py | 2 ++ ci/requirements/py3.11-all-free.yml | 2 ++ ci/requirements/py3.11-all-pinned.yml | 2 ++ ci/requirements/py3.12-all-free.yml | 2 ++ ci/requirements/py3.12-all-pinned.yml | 2 ++ 9 files changed, 41 insertions(+), 5 deletions(-) diff --git a/argopy/__init__.py b/argopy/__init__.py index c610b68e9..a17336706 100644 --- a/argopy/__init__.py +++ b/argopy/__init__.py @@ -41,7 +41,7 @@ from .utils import clear_cache, lscache # noqa: E402 from .utils import MonitoredThreadPoolExecutor # noqa: E402, F401 from .utils import monitor_status as status # noqa: E402 -from .related import TopoFetcher, OceanOPSDeployments, ArgoNVSReferenceTables, ArgoDocs, ArgoDOI, ArgoSensor # noqa: E402 +from .related import TopoFetcher, OceanOPSDeployments, ArgoNVSReferenceTables, ArgoDocs, ArgoDOI, ArgoSensor, OEMSensorMetaData # noqa: E402 from .extensions import CanyonMED # noqa: E402 @@ -70,6 +70,7 @@ "TopoFetcher", # Class "ArgoDOI", # Class "ArgoSensor", # Class + "OEMSensorMetaData", # Advanced Argo data stores: "ArgoFloat", # Class diff --git a/argopy/related/__init__.py b/argopy/related/__init__.py index a11a7799e..624dd1a2c 100644 --- a/argopy/related/__init__.py +++ b/argopy/related/__init__.py @@ -4,7 +4,7 @@ from .argo_documentation import ArgoDocs from .doi_snapshot import ArgoDOI from .euroargo_api import get_coriolis_profile_id, get_ea_profile_page -from .sensors import ArgoSensor, SensorType, SensorModel +from .sensors import ArgoSensor, SensorType, SensorModel, OEMSensorMetaData from .utils import load_dict, mapp_dict # Must come last to avoid circular import, I know, not good # @@ -16,6 +16,7 @@ "ArgoDocs", "ArgoDOI", "ArgoSensor", + "OEMSensorMetaData", # Functions : "get_coriolis_profile_id", diff --git a/argopy/related/sensors/oem/oem_metadata.py b/argopy/related/sensors/oem/oem_metadata.py index cf6d191b7..a31dfa3b8 100644 --- a/argopy/related/sensors/oem/oem_metadata.py +++ b/argopy/related/sensors/oem/oem_metadata.py @@ -2,8 +2,7 @@ from typing import List, Dict, Optional, Any, Literal from pathlib import Path from zipfile import ZipFile -from referencing import Registry, Resource -import jsonschema + import json import logging import warnings @@ -15,9 +14,14 @@ from ....utils import urnparser, path2assets from ....errors import InvalidDatasetStructure +from ..utils import has_jsonschema from .oem_metadata_repr import OemMetaDataDisplay from .accessories import SensorInfo, Context, Sensor, Parameter +if has_jsonschema: + from referencing import Registry, Resource + import jsonschema + log = logging.getLogger("argopy.related.sensors.oem") @@ -80,7 +84,12 @@ def __init__( } self._fs = httpstore(**fs_kargs) - self._run_validation = validate + if has_jsonschema: + self._run_validation = validate + else: + warnings.warn(f"Cannot run JSON validation without the 'jsonschema' library. Please install it manually. Fall back on setting `validate=False`. ") + self._run_validation = False + self._validation_error = validation_error self.schema = self._read_schema() # requires a self._fs instance diff --git a/argopy/related/sensors/utils.py b/argopy/related/sensors/utils.py index f386e683d..783a75324 100644 --- a/argopy/related/sensors/utils.py +++ b/argopy/related/sensors/utils.py @@ -1,4 +1,17 @@ import pandas as pd +import importlib + + +def _importorskip(modname): + try: + importlib.import_module(modname) # noqa: E402 + has = True + except ImportError: + has = False + return has + + +has_jsonschema = _importorskip("jsonschema") class APISensorMetaDataProcessing: @@ -43,3 +56,5 @@ def postprocess_df(cls, data, **kwargs): } ) return pd.DataFrame(d).sort_values(by="WMO").reset_index(drop=True) + + diff --git a/argopy/utils/locals.py b/argopy/utils/locals.py index 25818231a..28da1e879 100644 --- a/argopy/utils/locals.py +++ b/argopy/utils/locals.py @@ -181,6 +181,8 @@ def show_versions(file=sys.stdout, conda=False): # noqa: C901 ("s3fs", get_version), ("kerchunk", get_version), ("zarr", get_version), + ("pydantic", get_version), + ("jsonschema", get_version), ] ), "ext.perf": sorted( diff --git a/ci/requirements/py3.11-all-free.yml b/ci/requirements/py3.11-all-free.yml index e5c12c5d0..4a1e52fa6 100644 --- a/ci/requirements/py3.11-all-free.yml +++ b/ci/requirements/py3.11-all-free.yml @@ -23,8 +23,10 @@ dependencies: # EXT.FILES: - boto3 + - jsonschema - kerchunk - numcodecs +# - pydantic - s3fs - zarr diff --git a/ci/requirements/py3.11-all-pinned.yml b/ci/requirements/py3.11-all-pinned.yml index e9544d69c..f139face1 100644 --- a/ci/requirements/py3.11-all-pinned.yml +++ b/ci/requirements/py3.11-all-pinned.yml @@ -23,8 +23,10 @@ dependencies: # EXT.FILES: - boto3 = 1.38.27 + - jsonschema = 4.25.1 - kerchunk = 0.2.8 - numcodecs = 0.16.1 +# - pydantic = 2.12.0 - s3fs = 2025.5.1 - zarr = 3.0.10 diff --git a/ci/requirements/py3.12-all-free.yml b/ci/requirements/py3.12-all-free.yml index d594ec8ca..110b0dc2f 100644 --- a/ci/requirements/py3.12-all-free.yml +++ b/ci/requirements/py3.12-all-free.yml @@ -23,8 +23,10 @@ dependencies: # EXT.FILES: - boto3 + - jsonschema - kerchunk - numcodecs +# - pydantic - s3fs - zarr diff --git a/ci/requirements/py3.12-all-pinned.yml b/ci/requirements/py3.12-all-pinned.yml index 4094f8ee2..8d8cc9e61 100644 --- a/ci/requirements/py3.12-all-pinned.yml +++ b/ci/requirements/py3.12-all-pinned.yml @@ -23,8 +23,10 @@ dependencies: # EXT.FILES: - boto3 = 1.38.27 + - jsonschema - kerchunk = 0.2.8 - numcodecs = 0.16.1 +# - pydantic - s3fs = 2025.5.1 - zarr = 3.0.10 From 795298a163e1cf01b54c779700613d060be9b5e7 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Wed, 29 Oct 2025 15:03:31 +0100 Subject: [PATCH 61/71] Add OEMSensorMetaData to the documentation --- argopy/related/sensors/oem/oem_metadata.py | 28 +++- argopy/utils/locals.py | 1 - docs/advanced-tools/metadata/sensors.rst | 164 +++++++++++++++------ docs/api-hidden.rst | 7 + docs/api.rst | 1 + 5 files changed, 147 insertions(+), 54 deletions(-) diff --git a/argopy/related/sensors/oem/oem_metadata.py b/argopy/related/sensors/oem/oem_metadata.py index a31dfa3b8..bd179a74d 100644 --- a/argopy/related/sensors/oem/oem_metadata.py +++ b/argopy/related/sensors/oem/oem_metadata.py @@ -1,5 +1,5 @@ from dataclasses import field -from typing import List, Dict, Optional, Any, Literal +from typing import List, Dict, Optional, Any, Literal, Self from pathlib import Path from zipfile import ZipFile @@ -206,7 +206,7 @@ def validate(self, data): errors = list(v.evolve(schema=self.schema).iter_errors(v, data)) return errors - def from_dict(self, data: Dict[str, Any]): + def from_dict(self, data: Dict[str, Any]) -> Self: """Load data from a dictionary and possibly validate""" if self._run_validation: self.validate(data) @@ -287,7 +287,7 @@ def to_json_file(self, file_path: str) -> None: with open(file_path, "w") as f: json.dump(self.to_dict(), f, indent=2) - def from_rbr(self, serial_number: str, **kwargs) -> 'OEMSensorMetaData': + def from_rbr(self, serial_number: str, **kwargs) -> Self: """Fetch sensor metadata from "RBRargo Product Lookup" web-API We also download certificates if available @@ -301,7 +301,9 @@ def from_rbr(self, serial_number: str, **kwargs) -> 'OEMSensorMetaData': Notes ----- - The instance :class:`httpstore` is automatically updated to use the OPTIONS value for ``rbr_api_key``. + The instance :class:`httpstore` is automatically updated to use the ``rbr_api_key`` option. + + The "RBRargo Product Lookup" web-API location is set with: ``OPTIONS['rbr_api']`` """ self._serial_number = serial_number @@ -395,13 +397,27 @@ def _certificates_rbr( else: raise ValueError(f"Unknown action {action}") - def from_seabird(self, serial_number: str, sensor_model: str, **kwargs) -> 'OEMSensorMetaData': + def from_seabird(self, serial_number: str, sensor_model: str, **kwargs) -> Self: """Fetch sensor metadata from Seabird-Scientific "Instrument Metadata Portal" web-API Parameters ---------- serial_number : str Sensor serial number from RBR + + Notes + ----- + The "Instrument Metadata Portal" web-API location is set with: ``OPTIONS['seabird_api']`` + + .. warning:: + + Since the Seabird web-API does not yet return fully compliant json medatata, we internally fix some format errors to be able to use this source of information. In particular: + + - 'sensor_info' comes from the wrongly named 'json_info' attribute, + - If 'PARAMETER_UNITS' is empty, try to get value from the 'parameter_vendorinfo'/'units' attribute, + - If 'PREDEPLOYMENT_CALIB_COMMENT' is empty, try to get value from the 'parameter_vendorinfo'/'comments' attribute, + - If 'PREDEPLOYMENT_CALIB_DATE' is empty, try to get value from the 'parameter_vendorinfo'/'calibration_date' attribute. + """ # The Seabird api requires a sensor model (R27) with a serial number. # This could easily be done if we get the s/n by searching float metadata. @@ -441,7 +457,7 @@ def list_examples(self): """List of example names""" return [k for k in SENSOR_JS_EXAMPLES.keys()] - def from_examples(self, eg: str = None, **kwargs): + def from_examples(self, eg: str = None, **kwargs) -> Self: if eg not in self.list_examples: raise ValueError( f"Unknown sensor example: '{eg}'. \n Use one in: {self.list_examples}" diff --git a/argopy/utils/locals.py b/argopy/utils/locals.py index 28da1e879..a310fa91b 100644 --- a/argopy/utils/locals.py +++ b/argopy/utils/locals.py @@ -181,7 +181,6 @@ def show_versions(file=sys.stdout, conda=False): # noqa: C901 ("s3fs", get_version), ("kerchunk", get_version), ("zarr", get_version), - ("pydantic", get_version), ("jsonschema", get_version), ] ), diff --git a/docs/advanced-tools/metadata/sensors.rst b/docs/advanced-tools/metadata/sensors.rst index c67ce7896..4f3cb3509 100644 --- a/docs/advanced-tools/metadata/sensors.rst +++ b/docs/advanced-tools/metadata/sensors.rst @@ -4,28 +4,31 @@ Argo sensors ============ -The :class:`ArgoSensor` class aims to provide user-friendly access to Argo's sensor metadata from: +**Argopy** provides several classes to work with Argo sensors: -- NVS Reference Tables `Sensor Models (R27) `_, `Sensor Types (R25) `_ and `Sensor Manufacturers (R26) `_, -- the `Euro-Argo fleet-monitoring API `_. +- :class:`ArgoSensor`: provides user-friendly access to Argo's sensor metadata with search possibilities, +- :class:`OEMSensorMetaData`: provides facilitated access to manufacturers web-API and predeployment calibrations information. -This enables users to: +These should enable users to: -- navigate reference tables, +- navigate reference tables `Sensor Models (R27) `_, `Sensor Types (R25) `_ and `Sensor Manufacturers (R26) `_, - search for floats equipped with specific sensor models, - retrieve sensor serial numbers across the global array, -- search for/iterate over floats equipped with specific sensor models. +- search for/iterate over floats equipped with specific sensor models, +- retrieve sensor metadata directly from manufacturers. +.. note:: + + The :class:`ArgoSensor` get information using the `Euro-Argo fleet-monitoring API `_. .. contents:: :local: - :depth: 1 + :depth: 3 .. _argosensor-reference-tables: -Working with reference tables ------------------------------ - +Navigating reference tables +--------------------------- With the :class:`ArgoSensor` class, you can work with official Argo vocabularies for `Sensor Models (R27) `_, `Sensor Types (R25) `_ and `Sensor Manufacturers (R26) `_. With the following methods, it is easy to look for a specific sensor model name, which can then be used to :ref:`look for floats ` or :ref:`create a ArgoSensor class ` directly. @@ -100,8 +103,8 @@ Examples .. _argosensor-search-floats: -Search for Argo floats equipped with a given sensor model ---------------------------------------------------------- +Search for floats equipped with sensor models +--------------------------------------------- In this section we show how to find WMOs, and possibly more sensor metadata like serial numbers, for floats equipped with a specific sensor model. This can be done using the :meth:`ArgoSensor.search` method. @@ -143,13 +146,54 @@ Examples ArgoSensor().search("RBR_ARGO3_DEEP6K", output="df") +Advanced data retrieval with iterations on ArgoFloat instances +-------------------------------------------------------------- + +The :meth:`ArgoSensor.iterfloats_with` will yields :class:`argopy.ArgoFloat` instances for floats with the specified sensor model (use ``chunksize`` to process floats in batches). + +Example +^^^^^^^ + +Let's try to gather all platform types and WMOs for floats equipped with a list of sensor models: + +.. code-block:: python + + sensors = ArgoSensor() + + models = ['ECO_FLBBCD_AP2', 'ECO_FLBBCD'] + results = {} + for af in sensors.iterfloats_with(models): + if 'meta' in af.ls_dataset(): + platform_type = af.metadata['platform']['type'] # e.g. 'PROVOR_V_JUMBO' + if platform_type in results.keys(): + results[platform_type].extend([af.WMO]) + else: + results.update({platform_type: [af.WMO]}) + else: + print(f"No meta file for float {af.WMO}") + + [f"{r:15s}: {len(results[r])} floats" for r in results.keys()] + +.. code-block:: python + + ['APEX : 37 floats', + 'SOLO_BGC_MRV : 19 floats', + 'PROVOR_V_JUMBO : 43 floats', + 'PROVOR_III : 206 floats', + 'PROVOR_V : 38 floats', + 'PROVOR : 18 floats', + 'PROVOR_IV : 21 floats', + 'SOLO_BGC : 4 floats'] .. _argosensor-exact-sensor: -Use an exact sensor model name to create a specific ArgoSensor --------------------------------------------------------------- +Working with one Sensor model +----------------------------- + +Argo references +^^^^^^^^^^^^^^^ -You can initialize an :class:`ArgoSensor` instance with a specific model to access more metadata. In this use-case, you will have the following attributes and methods available: +To acces one sensor model complete list of referenced information, you can initialize a :class:`ArgoSensor` instance with a specific model. In this use-case, you will have the following attributes and methods available: .. currentmodule:: argopy @@ -160,9 +204,6 @@ You can initialize an :class:`ArgoSensor` instance with a specific model to acce ArgoSensor.search ArgoSensor.iterfloats_with -Examples -^^^^^^^^ - As an example, let's create an instance for the "SBE43F_IDO" sensor model: .. ipython:: python @@ -193,47 +234,76 @@ You can also look for floats equipped with it: df = sensor.search(output="df") df -Loop through ArgoFloat instances for each float ------------------------------------------------ +Manufacturers API +^^^^^^^^^^^^^^^^^ -The :meth:`ArgoSensor.iterfloats_with` will yields :class:`argopy.ArgoFloat` instances for floats with the specified sensor model (use ``chunksize`` to process floats in batches). +**Argopy** provides the experimental :class:`OEMSensorMetaData` class to deal with metadata provided by manufacturers. The :class:`OEMSensorMetaData` class provides methods to directly access some manufacturers web-API on your behalf. -Example -^^^^^^^ +Argo sensor makers are encouraged to provide the Argo community with all possible information about a sensor, in particular predeployment calibration metadata. -Let's try to gather all platform types and WMOs for floats equipped with a list of sensor models: +The ADMT has developed a JSON schema for this at https://github.com/euroargodev/sensor_metadata_json and an library for JSON validation at https://github.com/euroargodev/argo-metadata-validator. + +By default **argopy** will validate json data against the reference schema. + +Sensor metadata from RBR +"""""""""""""""""""""""" + +Thanks to the RBR web-API, you can access sensor metadata from a RBR sensor with the :meth:`OEMSensorMetaData.from_rbr` method and a sensor serial number: + +.. ipython:: python + :okwarning: + + from argopy import OEMSensorMetaData + + OEMSensorMetaData().from_rbr(208380) + +Note that the RBR web-API requires an authentication key (you can contact RBR at argo@rbr-global.com if you do not have an such a key). **Argopy** will try to get the key from the environment variable ``RBR_API_KEY`` or from the option ``rbr_api_key``. You can set the key temporarily in your code with: .. code-block:: python - sensors = ArgoSensor() + argopy.set_options(rbr_api_key="********") - models = ['ECO_FLBBCD_AP2', 'ECO_FLBBCD'] - results = {} - for af in sensors.iterfloats_with(models): - if 'meta' in af.ls_dataset(): - platform_type = af.metadata['platform']['type'] # e.g. 'PROVOR_V_JUMBO' - if platform_type in results.keys(): - results[platform_type].extend([af.WMO]) - else: - results.update({platform_type: [af.WMO]}) - else: - print(f"No meta file for float {af.WMO}") +Sensor metadata from Seabird +"""""""""""""""""""""""""""" - [f"{r:15s}: {len(results[r])} floats" for r in results.keys()] +Thanks to the Seabird web-API, you can access sensor metadata from a Seabird sensor with the :meth:`OEMSensorMetaData.from_seabird` method and a sensor serial number and a model name: + +.. ipython:: python + :okwarning: + + from argopy import OEMSensorMetaData + + OEMSensorMetaData().from_seabird(2444, 'SATLANTIC_OCR504_ICSW') + +Sensor metadata from elsewhere +"""""""""""""""""""""""""""""" + +If you have your own metadata, you can use :meth:`OEMSensorMetaData.from_dict`: .. code-block:: python - ['APEX : 37 floats', - 'SOLO_BGC_MRV : 19 floats', - 'PROVOR_V_JUMBO : 43 floats', - 'PROVOR_III : 206 floats', - 'PROVOR_V : 38 floats', - 'PROVOR : 18 floats', - 'PROVOR_IV : 21 floats', - 'SOLO_BGC : 4 floats'] + from argopy import OEMSensorMetaData + + jsdata = [...] + + OEMSensorMetaData().from_dict(jsdata) + +Examples +"""""""" + +Last, since this is still in active development on the manufacturer's side, we also added easy access to some examples from https://github.com/euroargodev/sensor_metadata_json: + +.. ipython:: python + :okwarning: + + OEMSensorMetaData().list_examples + + OEMSensorMetaData().from_examples('WETLABS-ECO_FLBBAP2-8589') + + -Quick float and sensor lookups from the command line ----------------------------------------------------- +From the command line +--------------------- The :meth:`ArgoSensor.cli_search` function is available to search for Argo floats equipped with a given sensor model from the command line and retrieve a `CLI `_ friendly list of WMOs or serial numbers. diff --git a/docs/api-hidden.rst b/docs/api-hidden.rst index 1a1bd6ba1..96777b0a8 100644 --- a/docs/api-hidden.rst +++ b/docs/api-hidden.rst @@ -194,6 +194,13 @@ argopy.ArgoSensor.iterfloats_with argopy.ArgoSensor.cli_search + argopy.related.OEMSensorMetaData + argopy.OEMSensorMetaData.from_rbr + argopy.OEMSensorMetaData.from_seabird + argopy.OEMSensorMetaData.from_dict + argopy.OEMSensorMetaData.from_examples + argopy.OEMSensorMetaData.list_examples + argopy.related.SensorModel argopy.related.SensorType diff --git a/docs/api.rst b/docs/api.rst index 7823e169c..4376c0682 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -168,6 +168,7 @@ Argo meta/related data CTDRefDataFetcher TopoFetcher ArgoSensor + OEMSensorMetaData .. _Module Visualisation: From 33f4588bcb48f75aec9f93fe913aaed955184b97 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Wed, 29 Oct 2025 17:46:00 +0100 Subject: [PATCH 62/71] Update oem_metadata.py keep track of raw data from OEM web-API --- argopy/related/sensors/oem/oem_metadata.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/argopy/related/sensors/oem/oem_metadata.py b/argopy/related/sensors/oem/oem_metadata.py index bd179a74d..bf312b6b9 100644 --- a/argopy/related/sensors/oem/oem_metadata.py +++ b/argopy/related/sensors/oem/oem_metadata.py @@ -316,8 +316,8 @@ def from_rbr(self, serial_number: str, **kwargs) -> Self: fss._session = None # Reset fsspec aiohttp.ClientSession uri = f"{OPTIONS['rbr_api']}/instruments/{self._serial_number}/argometadatajson" - data = self._fs.open_json(uri) - obj = self.from_dict(data) + self._data = self._fs.open_json(uri) + obj = self.from_dict(self._data) # Also download RBR zip archive with calibration certificates in PDFs: obj = obj._certificates_rbr(action="download", quiet=True) @@ -449,7 +449,8 @@ def from_seabird(self, serial_number: str, sensor_model: str, **kwargs) -> Self: data['PARAMETERS'][ii]['PREDEPLOYMENT_CALIB_DATE'] = vendorinfo['calibration_date'] # Create and return an instance - obj = self.from_dict(data) + self._data = data + obj = self.from_dict(self._data) return obj @property From abfeadafb51fb682bf6a4c1ae67a9ba18a1ea4a3 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Wed, 29 Oct 2025 17:46:03 +0100 Subject: [PATCH 63/71] Update accessories.py --- argopy/related/sensors/oem/accessories.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/argopy/related/sensors/oem/accessories.py b/argopy/related/sensors/oem/accessories.py index 4cbe1600d..5712e5b6a 100644 --- a/argopy/related/sensors/oem/accessories.py +++ b/argopy/related/sensors/oem/accessories.py @@ -48,8 +48,8 @@ class Sensor: # FIRMWARE VERSION attributes are temporarily optional to handle the wrong key used by RBR # see https://github.com/euroargodev/sensor_metadata_json/issues/20 - SENSOR_FIRMWARE_VERSION: str = None # RBR - SENSOR_MODEL_FIRMWARE: str = None # Correct schema key + SENSOR_FIRMWARE_VERSION: str = None # RBR, used in sensor schema 0.2.0 + SENSOR_MODEL_FIRMWARE: str = None # Correct schema key, used in sensor schema 0.4.0 sensor_vendorinfo: Optional[Dict[str, Any]] = None From d64170f31697809f03b4d4d92d8ecde89c23d24d Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Fri, 7 Nov 2025 09:45:36 +0100 Subject: [PATCH 64/71] Internal refactoring - removed relative import - refactor sensor-related dataclass items to utilities --- argopy/related/sensors/accessories.py | 2 +- argopy/related/sensors/oem/__init__.py | 2 +- argopy/related/sensors/oem/accessories.py | 199 ------------------ argopy/related/sensors/oem/oem_metadata.py | 17 +- .../related/sensors/oem/oem_metadata_repr.py | 89 +------- argopy/related/sensors/references.py | 16 +- argopy/related/sensors/sensors.py | 6 +- argopy/related/sensors/spec.py | 18 +- argopy/related/sensors/utils.py | 2 - argopy/stores/float/spec.py | 2 +- argopy/utils/schemas/__init__.py | 0 argopy/utils/schemas/sensors/__init__.py | 0 argopy/utils/schemas/sensors/repr.py | 137 ++++++++++++ argopy/utils/schemas/sensors/spec.py | 198 +++++++++++++++++ 14 files changed, 368 insertions(+), 320 deletions(-) create mode 100644 argopy/utils/schemas/__init__.py create mode 100644 argopy/utils/schemas/sensors/__init__.py create mode 100644 argopy/utils/schemas/sensors/repr.py create mode 100644 argopy/utils/schemas/sensors/spec.py diff --git a/argopy/related/sensors/accessories.py b/argopy/related/sensors/accessories.py index 870459e20..8a7072c38 100644 --- a/argopy/related/sensors/accessories.py +++ b/argopy/related/sensors/accessories.py @@ -1,7 +1,7 @@ from typing import Literal import pandas as pd -from ...utils import NVSrow +from argopy.utils import NVSrow # Define some options expected values as tuples diff --git a/argopy/related/sensors/oem/__init__.py b/argopy/related/sensors/oem/__init__.py index c65357300..c64dd8fe1 100644 --- a/argopy/related/sensors/oem/__init__.py +++ b/argopy/related/sensors/oem/__init__.py @@ -1,3 +1,3 @@ from .oem_metadata import OEMSensorMetaData -__all__ = ['OEMSensorMetaData'] \ No newline at end of file +__all__ = ['OEMSensorMetaData'] diff --git a/argopy/related/sensors/oem/accessories.py b/argopy/related/sensors/oem/accessories.py index 5712e5b6a..e69de29bb 100644 --- a/argopy/related/sensors/oem/accessories.py +++ b/argopy/related/sensors/oem/accessories.py @@ -1,199 +0,0 @@ -from dataclasses import dataclass -from typing import Dict, Optional, Any -from html import escape - -from ....options import OPTIONS -from ....utils import urnparser - -from .oem_metadata_repr import ParameterDisplay - - -@dataclass -class SensorInfo: - created_by: str - date_creation: str # ISO 8601 datetime string - link: str - format_version: str - contents: str - - # Made optional to accommodate errors in OEM data - sensor_described: str = None - - def _attr2str(self, x): - """Return a class attribute, or 'n/a' if it's None or "".""" - value = getattr(self, x, None) - if value is None: - return "n/a" - elif type(value) is str: - return value if value and value.strip() else "n/a" - else: - return value - - -@dataclass -class Context: - SDN_R03: str = "http://vocab.nerc.ac.uk/collection/R03/current/" - SDN_R25: str = "http://vocab.nerc.ac.uk/collection/R25/current/" - SDN_R26: str = "http://vocab.nerc.ac.uk/collection/R26/current/" - SDN_R27: str = "http://vocab.nerc.ac.uk/collection/R27/current/" - SDN_L22: str = "http://vocab.nerc.ac.uk/collection/L22/current/" - - -@dataclass -class Sensor: - SENSOR: str # SDN:R25::CTD_PRES - SENSOR_MAKER: str # SDN:R26::RBR - SENSOR_MODEL: str # SDN:R27::RBR_PRES_A - SENSOR_SERIAL_NO: str - - # FIRMWARE VERSION attributes are temporarily optional to handle the wrong key used by RBR - # see https://github.com/euroargodev/sensor_metadata_json/issues/20 - SENSOR_FIRMWARE_VERSION: str = None # RBR, used in sensor schema 0.2.0 - SENSOR_MODEL_FIRMWARE: str = None # Correct schema key, used in sensor schema 0.4.0 - - sensor_vendorinfo: Optional[Dict[str, Any]] = None - - @property - def SENSOR_uri(self): - urnparts = urnparser(self.SENSOR) - return f"{OPTIONS['nvs']}/{urnparts['listid']}/current/{urnparts['termid']}" - - @property - def SENSOR_MAKER_uri(self): - urnparts = urnparser(self.SENSOR_MAKER) - return f"{OPTIONS['nvs']}/{urnparts['listid']}/current/{urnparts['termid']}" - - @property - def SENSOR_MODEL_uri(self): - urnparts = urnparser(self.SENSOR_MODEL) - return f"{OPTIONS['nvs']}/{urnparts['listid']}/current/{urnparts['termid']}" - - def _attr2str(self, x): - """Return a class attribute, or 'n/a' if it's None, {} or "".""" - value = getattr(self, x, None) - if value is None: - return "n/a" - elif type(value) is str: - return value if value and value.strip() else "n/a" - elif type(value) is dict: - if len(value.keys()) == 0: - return "n/a" - else: - return value - else: - return value - - def __repr__(self): - - def key2str(d, x): - """Return a dict value as a string, or 'n/a' if it's None or empty.""" - value = d.get(x, None) - return value if value and value.strip() else "n/a" - - summary = [f"<{self.SENSOR}><{self.SENSOR_SERIAL_NO}>"] - summary.append(f" SENSOR: {self.SENSOR} ({self.SENSOR_uri})") - summary.append(f" SENSOR_MAKER: {self.SENSOR_MAKER} ({self.SENSOR_MAKER_uri})") - summary.append(f" SENSOR_MODEL: {self.SENSOR_MODEL} ({self.SENSOR_MODEL_uri})") - if getattr(self, "SENSOR_MODEL_FIRMWARE", None) is None: - summary.append( - f" SENSOR_FIRMWARE_VERSION: {self._attr2str('SENSOR_FIRMWARE_VERSION')} (but should be 'SENSOR_MODEL_FIRMWARE') " - ) - else: - summary.append( - f" SENSOR_MODEL_FIRMWARE: {self._attr2str('SENSOR_MODEL_FIRMWARE')}" - ) - if getattr(self, "sensor_vendorinfo", None) is not None: - summary.append(f" sensor_vendorinfo:") - for key in self.sensor_vendorinfo.keys(): - summary.append(f" - {key}: {key2str(self.sensor_vendorinfo, key)}") - else: - summary.append(f" sensor_vendorinfo: None") - return "\n".join(summary) - - -@dataclass -class Parameter: - PARAMETER: str # SDN:R03::PRES - PARAMETER_SENSOR: str # SDN:R25::CTD_PRES - PARAMETER_ACCURACY: str - PARAMETER_RESOLUTION: str - PREDEPLOYMENT_CALIB_COEFFICIENT_LIST: Dict[str, str] - - # Made optional to accommodate errors in OEM data - PARAMETER_UNITS: str = None - PREDEPLOYMENT_CALIB_EQUATION: str = None - PREDEPLOYMENT_CALIB_COMMENT: str = None - PREDEPLOYMENT_CALIB_DATE: str = None - - parameter_vendorinfo: Optional[Dict[str, Any]] = None - predeployment_vendorinfo: Optional[Dict[str, Any]] = None - - @property - def PARAMETER_uri(self): - urnparts = urnparser(self.PARAMETER) - return f"{OPTIONS['nvs']}/{urnparts['listid']}/current/{urnparts['termid']}" - - @property - def PARAMETER_SENSOR_uri(self): - urnparts = urnparser(self.PARAMETER_SENSOR) - return f"{OPTIONS['nvs']}/{urnparts['listid']}/current/{urnparts['termid']}" - - def _attr2str(self, x): - """Return a class attribute, or 'n/a' if it's None, {} or "".""" - value = getattr(self, x, None) - if value is None: - return "n/a" - elif type(value) is str: - return value if value and value.strip() else "n/a" - elif type(value) is dict: - if len(value.keys()) == 0: - return "n/a" - else: - return value - else: - return value - - @property - def _has_calibration_data(self): - s = "".join( - [ - str(self._attr2str(key)) - for key in [ - "PREDEPLOYMENT_CALIB_EQUATION", - "PREDEPLOYMENT_CALIB_COEFFICIENT_LIST", - "PREDEPLOYMENT_CALIB_COMMENT", - "PREDEPLOYMENT_CALIB_DATE", - ] - if self._attr2str(key) != "n/a" - ] - ) - return len(s) > 0 - - def __repr__(self): - - summary = [f"<{self.PARAMETER}>"] - summary.append(f" PARAMETER: {self.PARAMETER} ({self.PARAMETER_uri})") - summary.append( - f" PARAMETER_SENSOR: {self.PARAMETER_SENSOR} ({self.PARAMETER_SENSOR_uri})" - ) - - for key in ["UNITS", "ACCURACY", "RESOLUTION"]: - p = f"PARAMETER_{key}" - summary.append(f" {key}: {self._attr2str(p)}") - - summary.append(f" PREDEPLOYMENT CALIBRATION:") - for key in ["EQUATION", "COEFFICIENT", "COMMENT", "DATE"]: - p = f"PREDEPLOYMENT_CALIB_{key}" - summary.append(f" - {key}: {self._attr2str(p)}") - - for key in ["parameter_vendorinfo", "predeployment_vendorinfo"]: - if getattr(self, key, None) is not None: - summary.append(f" {key}: {self._attr2str(key)}") - else: - summary.append(f" {key}: None") - return "\n".join(summary) - - def _repr_html_(self): - if OPTIONS["display_style"] == "text": - return f"
{escape(repr(self))}
" - return ParameterDisplay(self).html diff --git a/argopy/related/sensors/oem/oem_metadata.py b/argopy/related/sensors/oem/oem_metadata.py index bf312b6b9..eba870146 100644 --- a/argopy/related/sensors/oem/oem_metadata.py +++ b/argopy/related/sensors/oem/oem_metadata.py @@ -9,14 +9,15 @@ import pandas as pd from html import escape -from ....stores import httpstore, filestore -from ....options import OPTIONS -from ....utils import urnparser, path2assets -from ....errors import InvalidDatasetStructure - -from ..utils import has_jsonschema -from .oem_metadata_repr import OemMetaDataDisplay -from .accessories import SensorInfo, Context, Sensor, Parameter +from argopy.stores import httpstore, filestore +from argopy.options import OPTIONS +from argopy.utils import urnparser, path2assets +from argopy.utils.schemas.sensors.spec import SensorInfo, Context, Sensor, Parameter +from argopy.errors import InvalidDatasetStructure + +from argopy.related.sensors.utils import has_jsonschema +from argopy.related.sensors.oem.oem_metadata_repr import OemMetaDataDisplay + if has_jsonschema: from referencing import Registry, Resource diff --git a/argopy/related/sensors/oem/oem_metadata_repr.py b/argopy/related/sensors/oem/oem_metadata_repr.py index 67b01d7df..e1d98fd52 100644 --- a/argopy/related/sensors/oem/oem_metadata_repr.py +++ b/argopy/related/sensors/oem/oem_metadata_repr.py @@ -11,7 +11,7 @@ importlib.util.find_spec(x).submodule_search_locations[0] ) -from ....utils import urnparser +from argopy.utils import urnparser STATIC_FILES = ( @@ -200,90 +200,3 @@ def _repr_html_(self): return self.html -class ParameterDisplay: - - def __init__(self, obj): - self.data = obj - - @property - def css_style(self): - return "\n".join(_load_static_files()) - - @property - def html(self): - param = self.data - - # --- Header --- - header_html = f"

Argo Sensor OEM Metadata for Parameter: {urn_html(param.PARAMETER)}

" - - if param.parameter_vendorinfo is not None: - info = " | ".join([f"{p} {v}" for p, v in param.parameter_vendorinfo.items()]) - header_html += f"

{info}

" - - # --- Parameter details --- - html = """ - - - - - - - - - - - - """ - html += f""" - - - - - - - """ - - PREDEPLOYMENT_CALIB_EQUATION = param._attr2str('PREDEPLOYMENT_CALIB_EQUATION').split(';') - PREDEPLOYMENT_CALIB_EQUATION = [p.replace("=", " = ") for p in PREDEPLOYMENT_CALIB_EQUATION] - PREDEPLOYMENT_CALIB_EQUATION = "
\t".join(PREDEPLOYMENT_CALIB_EQUATION) - - PREDEPLOYMENT_CALIB_COEFFICIENT_LIST = param._attr2str('PREDEPLOYMENT_CALIB_COEFFICIENT_LIST') - s = [] - if isinstance(PREDEPLOYMENT_CALIB_COEFFICIENT_LIST, dict): - for key, value in PREDEPLOYMENT_CALIB_COEFFICIENT_LIST.items(): - s.append(f"{key} = {value}") - PREDEPLOYMENT_CALIB_COEFFICIENT_LIST = "
\t".join(s) - - html += f""" - - - - """ - html += "
SensorUnitsAccuracyResolution
{urn_html(param.PARAMETER_SENSOR)}{param._attr2str('PARAMETER_UNITS')}{param._attr2str('PARAMETER_ACCURACY')}{param._attr2str('PARAMETER_RESOLUTION')}
-
-

Calibration Equation:
{PREDEPLOYMENT_CALIB_EQUATION}

-

Calibration Coefficients:
{PREDEPLOYMENT_CALIB_COEFFICIENT_LIST}

-

Calibration Comment:
{param._attr2str('PREDEPLOYMENT_CALIB_COMMENT')}

-

Calibration Date:
{param._attr2str('PREDEPLOYMENT_CALIB_DATE')}

-
-
" - - # --- Vendor Info --- - vendor_html = "" - if param.predeployment_vendorinfo: - vendor_html = f""" -

Pre-deployment Vendor Info:

- 
{param.predeployment_vendorinfo}
- """ - - # --- Combine All HTML --- - full_html = f""" - \n - {header_html}\n - {html}\n - {vendor_html} - """ - return full_html - - def _repr_html_(self): - return self.html \ No newline at end of file diff --git a/argopy/related/sensors/references.py b/argopy/related/sensors/references.py index 62ed3d375..c6240cca1 100644 --- a/argopy/related/sensors/references.py +++ b/argopy/related/sensors/references.py @@ -5,14 +5,14 @@ import logging import fnmatch -from ...options import OPTIONS -from ...stores import httpstore, filestore -from ...related import ArgoNVSReferenceTables -from ...utils import to_list, path2assets, register_accessor, ppliststr -from ...errors import DataNotFound, OptionValueError - -from .accessories import SensorType, SensorModel -from .accessories import Error, ErrorOptions +from argopy.options import OPTIONS +from argopy.stores import httpstore, filestore +from argopy.related import ArgoNVSReferenceTables +from argopy.utils import to_list, path2assets, register_accessor, ppliststr +from argopy.errors import DataNotFound, OptionValueError + +from argopy.related.sensors.accessories import SensorType, SensorModel +from argopy.related.sensors.accessories import Error, ErrorOptions log = logging.getLogger("argopy.related.sensors.ref") diff --git a/argopy/related/sensors/sensors.py b/argopy/related/sensors/sensors.py index 68e680925..863ba241f 100644 --- a/argopy/related/sensors/sensors.py +++ b/argopy/related/sensors/sensors.py @@ -1,6 +1,6 @@ -from ...utils import register_accessor -from .spec import ArgoSensorSpec -from .references import SensorReferences +from argopy.utils import register_accessor +from argopy.related.sensors.spec import ArgoSensorSpec +from argopy.related.sensors.references import SensorReferences class ArgoSensor(ArgoSensorSpec): diff --git a/argopy/related/sensors/spec.py b/argopy/related/sensors/spec.py index 140e7b516..1b48542e9 100644 --- a/argopy/related/sensors/spec.py +++ b/argopy/related/sensors/spec.py @@ -6,23 +6,23 @@ import json import sys -from ...stores import ArgoFloat, ArgoIndex, httpstore -from ...stores.filesystems import ( +from argopy.stores import ArgoFloat, ArgoIndex, httpstore +from argopy.stores.filesystems import ( tqdm, ) # Safe import, return a lambda if tqdm not available -from ...utils import check_wmo, Chunker, to_list, ppliststr, is_wmo -from ...errors import ( +from argopy.utils import check_wmo, Chunker, to_list, ppliststr, is_wmo +from argopy.errors import ( DataNotFound, InvalidDataset, InvalidDatasetStructure, OptionValueError, ) -from ...options import OPTIONS -from ..euroargo_api import EAfleetmonitoringAPI +from argopy.options import OPTIONS +from argopy.related.euroargo_api import EAfleetmonitoringAPI -from .references import SensorModel, SensorType -from .accessories import Error, ErrorOptions, Ds, DsOptions, SearchOutput, SearchOutputOptions -from .utils import APISensorMetaDataProcessing +from argopy.related.sensors.references import SensorModel, SensorType +from argopy.related.sensors.accessories import Error, ErrorOptions, Ds, DsOptions, SearchOutput, SearchOutputOptions +from argopy.related.sensors.utils import APISensorMetaDataProcessing log = logging.getLogger("argopy.related.sensors") diff --git a/argopy/related/sensors/utils.py b/argopy/related/sensors/utils.py index 783a75324..6d3dd6c44 100644 --- a/argopy/related/sensors/utils.py +++ b/argopy/related/sensors/utils.py @@ -56,5 +56,3 @@ def postprocess_df(cls, data, **kwargs): } ) return pd.DataFrame(d).sort_values(by="WMO").reset_index(drop=True) - - diff --git a/argopy/stores/float/spec.py b/argopy/stores/float/spec.py index 50def2ed0..5de5baf7a 100644 --- a/argopy/stores/float/spec.py +++ b/argopy/stores/float/spec.py @@ -126,7 +126,7 @@ def load_metadata_from_meta_file(self): """Method to load float meta-data""" data = {} - ds = self.open_dataset("meta") + ds = self.dataset["meta"] data.update( { "deployment": { diff --git a/argopy/utils/schemas/__init__.py b/argopy/utils/schemas/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/argopy/utils/schemas/sensors/__init__.py b/argopy/utils/schemas/sensors/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/argopy/utils/schemas/sensors/repr.py b/argopy/utils/schemas/sensors/repr.py new file mode 100644 index 000000000..4294439dc --- /dev/null +++ b/argopy/utils/schemas/sensors/repr.py @@ -0,0 +1,137 @@ +from functools import lru_cache +import importlib + + +try: + from importlib.resources import files # New in version 3.9 +except ImportError: + from pathlib import Path + + files = lambda x: Path( # noqa: E731 + importlib.util.find_spec(x).submodule_search_locations[0] + ) + +from argopy.utils import urnparser + + +STATIC_FILES = ( + ("argopy.static.css", "argopy.css"), + ("argopy.static.css", "oemsensor.css"), +) + + +@lru_cache(None) +def _load_static_files(): + """Lazily load the resource files into memory the first time they are needed""" + return [ + files(package).joinpath(resource).read_text(encoding="utf-8") + for package, resource in STATIC_FILES + ] + + +def urn_html(this_urn): + x = urnparser(this_urn) + if x.get("version") != "": + return f"{x.get('termid', '?')} ({x.get('listid', '?')}, {x.get('version', 'n/a')})" + else: + return f"{x.get('termid', '?')} ({x.get('listid', '?')})" + + +class ParameterDisplay: + + def __init__(self, obj): + self.data = obj + + @property + def css_style(self): + return "\n".join(_load_static_files()) + + @property + def html(self): + param = self.data + + # --- Header --- + header_html = f"

Argo Sensor OEM Metadata for Parameter: {urn_html(param.PARAMETER)}

" + + if param.parameter_vendorinfo is not None: + info = " | ".join( + [ + f"{p} {v}" + for p, v in param.parameter_vendorinfo.items() + ] + ) + header_html += f"

{info}

" + + # --- Parameter details --- + html = """ + + + + + + + + + + + + """ + html += f""" + + + + + + + """ + + PREDEPLOYMENT_CALIB_EQUATION = param._attr2str( + "PREDEPLOYMENT_CALIB_EQUATION" + ).split(";") + PREDEPLOYMENT_CALIB_EQUATION = [ + p.replace("=", " = ") for p in PREDEPLOYMENT_CALIB_EQUATION + ] + PREDEPLOYMENT_CALIB_EQUATION = "
\t".join(PREDEPLOYMENT_CALIB_EQUATION) + + PREDEPLOYMENT_CALIB_COEFFICIENT_LIST = param._attr2str( + "PREDEPLOYMENT_CALIB_COEFFICIENT_LIST" + ) + s = [] + if isinstance(PREDEPLOYMENT_CALIB_COEFFICIENT_LIST, dict): + for key, value in PREDEPLOYMENT_CALIB_COEFFICIENT_LIST.items(): + s.append(f"{key} = {value}") + PREDEPLOYMENT_CALIB_COEFFICIENT_LIST = "
\t".join(s) + + html += f""" + + + + """ + html += "
SensorUnitsAccuracyResolution
{urn_html(param.PARAMETER_SENSOR)}{param._attr2str('PARAMETER_UNITS')}{param._attr2str('PARAMETER_ACCURACY')}{param._attr2str('PARAMETER_RESOLUTION')}
+
+

Calibration Equation:
{PREDEPLOYMENT_CALIB_EQUATION}

+

Calibration Coefficients:
{PREDEPLOYMENT_CALIB_COEFFICIENT_LIST}

+

Calibration Comment:
{param._attr2str('PREDEPLOYMENT_CALIB_COMMENT')}

+

Calibration Date:
{param._attr2str('PREDEPLOYMENT_CALIB_DATE')}

+
+
" + + # --- Vendor Info --- + vendor_html = "" + if param.predeployment_vendorinfo: + vendor_html = f""" +

Pre-deployment Vendor Info:

+ 
{param.predeployment_vendorinfo}
+ """ + + # --- Combine All HTML --- + full_html = f""" + \n + {header_html}\n + {html}\n + {vendor_html} + """ + return full_html + + def _repr_html_(self): + return self.html diff --git a/argopy/utils/schemas/sensors/spec.py b/argopy/utils/schemas/sensors/spec.py new file mode 100644 index 000000000..6ea6884f1 --- /dev/null +++ b/argopy/utils/schemas/sensors/spec.py @@ -0,0 +1,198 @@ +from dataclasses import dataclass +from typing import Dict, Optional, Any +from html import escape + +from argopy.options import OPTIONS +from argopy.utils import urnparser +from argopy.utils.schemas.sensors.repr import ParameterDisplay + + +@dataclass +class SensorInfo: + created_by: str + date_creation: str # ISO 8601 datetime string + link: str + format_version: str + contents: str + + # Made optional to accommodate errors in OEM data + sensor_described: str = None + + def _attr2str(self, x): + """Return a class attribute, or 'n/a' if it's None or "".""" + value = getattr(self, x, None) + if value is None: + return "n/a" + elif type(value) is str: + return value if value and value.strip() else "n/a" + else: + return value + + +@dataclass +class Context: + SDN_R03: str = "http://vocab.nerc.ac.uk/collection/R03/current/" + SDN_R25: str = "http://vocab.nerc.ac.uk/collection/R25/current/" + SDN_R26: str = "http://vocab.nerc.ac.uk/collection/R26/current/" + SDN_R27: str = "http://vocab.nerc.ac.uk/collection/R27/current/" + SDN_L22: str = "http://vocab.nerc.ac.uk/collection/L22/current/" + + +@dataclass +class Sensor: + SENSOR: str # SDN:R25::CTD_PRES + SENSOR_MAKER: str # SDN:R26::RBR + SENSOR_MODEL: str # SDN:R27::RBR_PRES_A + SENSOR_SERIAL_NO: str + + # FIRMWARE VERSION attributes are temporarily optional to handle the wrong key used by RBR + # see https://github.com/euroargodev/sensor_metadata_json/issues/20 + SENSOR_FIRMWARE_VERSION: str = None # RBR, used in sensor schema 0.2.0 + SENSOR_MODEL_FIRMWARE: str = None # Correct schema key, used in sensor schema 0.4.0 + + sensor_vendorinfo: Optional[Dict[str, Any]] = None + + @property + def SENSOR_uri(self): + urnparts = urnparser(self.SENSOR) + return f"{OPTIONS['nvs']}/{urnparts['listid']}/current/{urnparts['termid']}" + + @property + def SENSOR_MAKER_uri(self): + urnparts = urnparser(self.SENSOR_MAKER) + return f"{OPTIONS['nvs']}/{urnparts['listid']}/current/{urnparts['termid']}" + + @property + def SENSOR_MODEL_uri(self): + urnparts = urnparser(self.SENSOR_MODEL) + return f"{OPTIONS['nvs']}/{urnparts['listid']}/current/{urnparts['termid']}" + + def _attr2str(self, x): + """Return a class attribute, or 'n/a' if it's None, {} or "".""" + value = getattr(self, x, None) + if value is None: + return "n/a" + elif type(value) is str: + return value if value and value.strip() else "n/a" + elif type(value) is dict: + if len(value.keys()) == 0: + return "n/a" + else: + return value + else: + return value + + def __repr__(self): + + def key2str(d, x): + """Return a dict value as a string, or 'n/a' if it's None or empty.""" + value = d.get(x, None) + return value if value and value.strip() else "n/a" + + summary = [f"<{self.SENSOR}><{self.SENSOR_SERIAL_NO}>"] + summary.append(f" SENSOR: {self.SENSOR} ({self.SENSOR_uri})") + summary.append(f" SENSOR_MAKER: {self.SENSOR_MAKER} ({self.SENSOR_MAKER_uri})") + summary.append(f" SENSOR_MODEL: {self.SENSOR_MODEL} ({self.SENSOR_MODEL_uri})") + if getattr(self, "SENSOR_MODEL_FIRMWARE", None) is None: + summary.append( + f" SENSOR_FIRMWARE_VERSION: {self._attr2str('SENSOR_FIRMWARE_VERSION')} (but should be 'SENSOR_MODEL_FIRMWARE') " + ) + else: + summary.append( + f" SENSOR_MODEL_FIRMWARE: {self._attr2str('SENSOR_MODEL_FIRMWARE')}" + ) + if getattr(self, "sensor_vendorinfo", None) is not None: + summary.append(f" sensor_vendorinfo:") + for key in self.sensor_vendorinfo.keys(): + summary.append(f" - {key}: {key2str(self.sensor_vendorinfo, key)}") + else: + summary.append(f" sensor_vendorinfo: None") + return "\n".join(summary) + + +@dataclass +class Parameter: + PARAMETER: str # SDN:R03::PRES + PARAMETER_SENSOR: str # SDN:R25::CTD_PRES + PARAMETER_ACCURACY: str + PARAMETER_RESOLUTION: str + PREDEPLOYMENT_CALIB_COEFFICIENT_LIST: Dict[str, str] + + # Made optional to accommodate errors in OEM data + PARAMETER_UNITS: str = None + PREDEPLOYMENT_CALIB_EQUATION: str = None + PREDEPLOYMENT_CALIB_COMMENT: str = None + PREDEPLOYMENT_CALIB_DATE: str = None + + parameter_vendorinfo: Optional[Dict[str, Any]] = None + predeployment_vendorinfo: Optional[Dict[str, Any]] = None + + @property + def PARAMETER_uri(self): + urnparts = urnparser(self.PARAMETER) + return f"{OPTIONS['nvs']}/{urnparts['listid']}/current/{urnparts['termid']}" + + @property + def PARAMETER_SENSOR_uri(self): + urnparts = urnparser(self.PARAMETER_SENSOR) + return f"{OPTIONS['nvs']}/{urnparts['listid']}/current/{urnparts['termid']}" + + def _attr2str(self, x): + """Return a class attribute, or 'n/a' if it's None, {} or "".""" + value = getattr(self, x, None) + if value is None: + return "n/a" + elif type(value) is str: + return value if value and value.strip() else "n/a" + elif type(value) is dict: + if len(value.keys()) == 0: + return "n/a" + else: + return value + else: + return value + + @property + def _has_calibration_data(self): + s = "".join( + [ + str(self._attr2str(key)) + for key in [ + "PREDEPLOYMENT_CALIB_EQUATION", + "PREDEPLOYMENT_CALIB_COEFFICIENT_LIST", + "PREDEPLOYMENT_CALIB_COMMENT", + "PREDEPLOYMENT_CALIB_DATE", + ] + if self._attr2str(key) != "n/a" + ] + ) + return len(s) > 0 + + def __repr__(self): + + summary = [f"<{self.PARAMETER}>"] + summary.append(f" PARAMETER: {self.PARAMETER} ({self.PARAMETER_uri})") + summary.append( + f" PARAMETER_SENSOR: {self.PARAMETER_SENSOR} ({self.PARAMETER_SENSOR_uri})" + ) + + for key in ["UNITS", "ACCURACY", "RESOLUTION"]: + p = f"PARAMETER_{key}" + summary.append(f" {key}: {self._attr2str(p)}") + + summary.append(f" PREDEPLOYMENT CALIBRATION:") + for key in ["EQUATION", "COEFFICIENT", "COMMENT", "DATE"]: + p = f"PREDEPLOYMENT_CALIB_{key}" + summary.append(f" - {key}: {self._attr2str(p)}") + + for key in ["parameter_vendorinfo", "predeployment_vendorinfo"]: + if getattr(self, key, None) is not None: + summary.append(f" {key}: {self._attr2str(key)}") + else: + summary.append(f" {key}: None") + return "\n".join(summary) + + def _repr_html_(self): + if OPTIONS["display_style"] == "text": + return f"
{escape(repr(self))}
" + return ParameterDisplay(self).html From a05c7f7db56af9d597f4c875253e0a287dfe7b98 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Fri, 7 Nov 2025 11:17:19 +0100 Subject: [PATCH 65/71] Update repr.py --- argopy/utils/schemas/sensors/repr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/argopy/utils/schemas/sensors/repr.py b/argopy/utils/schemas/sensors/repr.py index 4294439dc..29801669d 100644 --- a/argopy/utils/schemas/sensors/repr.py +++ b/argopy/utils/schemas/sensors/repr.py @@ -51,7 +51,7 @@ def html(self): param = self.data # --- Header --- - header_html = f"

Argo Sensor OEM Metadata for Parameter: {urn_html(param.PARAMETER)}

" + header_html = f"

Argo Sensor Metadata for Parameter: {urn_html(param.PARAMETER)}

" if param.parameter_vendorinfo is not None: info = " | ".join( From c119775173766b53bbfd770182b3405210dfd4ce Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Mon, 24 Nov 2025 19:01:52 +0100 Subject: [PATCH 66/71] Play with SensorMetaData --- argopy/related/sensors/__init__.py | 4 +- argopy/related/sensors/accessories.py | 275 +++++++++++++++++- .../related/sensors/oem/oem_metadata_repr.py | 2 +- argopy/static/assets/data_types.json | 5 +- argopy/utils/format.py | 2 +- 5 files changed, 280 insertions(+), 8 deletions(-) diff --git a/argopy/related/sensors/__init__.py b/argopy/related/sensors/__init__.py index 3aa6abed0..a6ffefde7 100644 --- a/argopy/related/sensors/__init__.py +++ b/argopy/related/sensors/__init__.py @@ -1,5 +1,5 @@ from .sensors import ArgoSensor -from .accessories import SensorType, SensorModel +from .accessories import SensorType, SensorModel, SensorMetaData from .oem import OEMSensorMetaData -__all__ = ["ArgoSensor", "SensorType", "SensorModel", "OEMSensorMetaData"] +__all__ = ["ArgoSensor", "SensorType", "SensorModel", "SensorMetaData", "OEMSensorMetaData"] diff --git a/argopy/related/sensors/accessories.py b/argopy/related/sensors/accessories.py index 8a7072c38..23ef3071b 100644 --- a/argopy/related/sensors/accessories.py +++ b/argopy/related/sensors/accessories.py @@ -1,7 +1,15 @@ -from typing import Literal +from typing import Literal, Any import pandas as pd +import xarray as xr +import numpy as np +from dataclasses import dataclass +from html import escape +from argopy.options import OPTIONS +from argopy.utils import ppliststr, to_list from argopy.utils import NVSrow +from argopy.utils.schemas.sensors.spec import Parameter, Sensor, SensorInfo +from argopy.related.sensors.oem.oem_metadata_repr import OemMetaDataDisplay # Define some options expected values as tuples @@ -104,3 +112,268 @@ def __contains__(self, string) -> bool: string.lower() in self.name.lower() or string.lower() in self.long_name.lower() ) + + +@dataclass +class SensorModelMetaData: + parameters: list[Parameter] + sensors: list[Sensor] + sensor_info: SensorInfo + instrument_vendorinfo: Any + + +class SensorMetaData: + """A placeholder for float sensors meta-data + + This is a design in active dev. No final specs. + + ..code-block:: python + meta = ArgoFloat(WMO).open_dataset('meta') + md = SensorMetaData(meta) + + md.param2sensor # Dictionary mapping PARAMETER to PARAMETER_SENSOR + md['DOXY'] # Dictionary with all PARAMETER meta-data + """ + + __slots__ = ( + "_obj", + "_param2sensor", + "_wmo", + "_data", + "_parameters", + "_sensors", + "_models", + "sensor_info", + "instrument_vendorinfo", + "sensors", + "parameters", + ) + + def __init__(self, obj): + self._obj: xr.Dataset = obj + self._param2sensor: dict[str, str] = None + self._wmo: int = self._obj["PLATFORM_NUMBER"].item() + self._data: dict[str, Any] = {} + + self._parameters: list[str] = sorted(list(set(self.param2sensor.keys()))) + self._sensors: list[str] = sorted(list(set(self.param2sensor.values()))) + self._models: list[str] = sorted( + set([self.sensor2dict(s)["SENSOR_MODEL"] for s in self._sensors]) + ) + + self.parameters: list[Parameter] = [self.to_schema(p) for p in self._parameters] + self.sensors: list[Sensor] = [self.to_schema(p) for p in self._sensors] + self.sensor_info: SensorInfo = SensorInfo( + **{ + "created_by": self._obj.attrs["institution"], + "date_creation": self._obj.attrs["history"] + .split(";")[0] + .strip() + .split(" ")[0], + "link": "./argo.sensor.schema.json", + "format_version": self._obj.attrs["Conventions"], + "contents": "", + "sensor_described": "", + } + ) + self.instrument_vendorinfo = None + + def __repr__(self): + summary = [f""] + n_model, n_param, n_sensor = ( + len(self._models), + len(self._parameters), + len(self._sensors), + ) + summary.append( + f"> {n_model} sensor models, equiped with {n_sensor} sensor types and providing {n_param} parameters" + ) + summary.append(f"models: {ppliststr(self._models)}") + summary.append(f"sensors: {ppliststr(self._sensors)}") + summary.append(f"parameters: {ppliststr(self._parameters)}") + return "\n".join(summary) + + @property + def param2sensor(self): + """Dictionary mapping PARAMETER to PARAMETER_SENSOR""" + if self._param2sensor is None: + + def get(parameter): + parameter = parameter.strip() + plist = list(self._obj["PARAMETER"].values) + iparams = {} + [iparams.update({param.strip(): plist.index(param)}) for param in plist] + if parameter in iparams.keys(): + # PARAMETER_SENSOR (N_PARAM): name, populated from R25, of the sensor measuring this parameter + val = ( + self._obj["PARAMETER_SENSOR"] + .isel({"N_PARAM": iparams[parameter]}) + .values[np.newaxis][0] + .strip() + ) + return val + return None + + result = {} + for param in self._obj["PARAMETER"]: + p = param.values[np.newaxis][0] + result.update({str(p).strip(): get(p)}) + self._param2sensor = result + return self._param2sensor + + def coef2dict(self, text: str) -> list[dict[str, Any]]: + """Transform a calibration coefficient string into a list of dictionaries""" + try: + result = [] + for coef_grp in text.split(";"): + grp = {} + for coef in coef_grp.split(","): + grp.update( + { + k.strip(): float(v) + for k, v in ( + pair.split("=") for pair in coef.strip().split(",") + ) + } + ) + result.append(grp) + return result + except: + return text.strip() + + def sensor2dict(self, sensor: str) -> dict[str, Any]: + """Extract one sensor data from a metadata xr.dataset""" + sensor = sensor.strip() + slist = list(self._obj["SENSOR"].values) + isensors = {} + [isensors.update({s.strip(): slist.index(s)}) for s in slist] + result = {} + if sensor in isensors.keys(): + for var in self._obj.data_vars: + if self._obj[var].dims == ("N_SENSOR",): + val = ( + self._obj[var] + .isel({"N_SENSOR": isensors[sensor]}) + .values[np.newaxis][0] + .strip() + ) + val = None if val in ["none", b""] else val + result.update({var: val}) + return result + return None + + def param2dict(self, parameter: str) -> dict[str, Any]: + """Extract one parameter data from a metadata xr.dataset""" + parameter = parameter.strip() + plist = list(self._obj["PARAMETER"].values) + iparams = {} + [iparams.update({param.strip(): plist.index(param)}) for param in plist] + result = {} + if parameter in iparams.keys(): + for var in self._obj.data_vars: + if self._obj[var].dims == ("N_PARAM",): + val = ( + self._obj[var] + .isel({"N_PARAM": iparams[parameter]}) + .values[np.newaxis][0] + .strip() + ) + if var == "PARAMETER_SENSOR": + val = self.sensor2dict(val) + if var == "PREDEPLOYMENT_CALIB_COEFFICIENT": + val = self.coef2dict(val) + val = None if val in ["none", b""] else val + result.update({var: val}) + return result + return None + + def to_schema(self, key: str) -> Parameter | Sensor: + """Return parameter or sensor data as a JSON-schema compliant object""" + + def param2sdn(param): + pd = self[param] + spd = pd.copy() + spd["PARAMETER"] = f"SDN:R03::{pd['PARAMETER']}" + spd["PARAMETER_SENSOR"] = f"SDN:R25::{pd['PARAMETER_SENSOR']['SENSOR']}" + spd["PREDEPLOYMENT_CALIB_COEFFICIENT_LIST"] = spd[ + "PREDEPLOYMENT_CALIB_COEFFICIENT" + ] + spd.pop("PREDEPLOYMENT_CALIB_COEFFICIENT") + spd["PREDEPLOYMENT_CALIB_DATE"] = "none" + return Parameter(**spd) + + def sensor2sdn(sensor): + pd = self[sensor] + spd = pd.copy() + spd["SENSOR"] = f"SDN:R25::{pd['SENSOR']}" + spd["SENSOR_MAKER"] = f"SDN:R26::{pd['SENSOR_MAKER']}" + spd["SENSOR_MODEL"] = f"SDN:R27::{pd['SENSOR_MODEL']}" + return Sensor(**spd) + + if key in self.param2sensor.keys(): + return param2sdn(key) + else: + return sensor2sdn(key) + + def __getitem__(self, key: str) -> dict[str, Any]: + """Get parameter or sensor data as dictionary""" + if self._data.get(key, None) is None: + if key in self.param2sensor.keys(): + self._data.update({key: self.param2dict(key)}) + elif key in self.param2sensor.values(): + self._data.update({key: self.sensor2dict(key)}) + else: + raise ValueError(f"Unknown parameter or sensor '{key}'") + return self._data[key] + + @property + def Models(self): + # Models = {'': {'sensors': [], 'parameters': []}} + Models = {} + for model in self._models: + # List of sensor types: + Models.update({model: {"sensors": [], "parameters": []}}) + + for sensor in self._sensors: + model = self[sensor]["SENSOR_MODEL"] + if self.to_schema(sensor) not in Models[model]["sensors"]: + Models[model]["sensors"].append(self.to_schema(sensor)) + + for parameter in self._parameters: + model = self[parameter]["PARAMETER_SENSOR"]["SENSOR_MODEL"] + if self.to_schema(parameter) not in Models[model]["parameters"]: + Models[model]["parameters"].append(self.to_schema(parameter)) + + return Models + + def display_model(self, model: str | None = None): + if model not in self.Models.keys(): + raise ValueError(f"Model name must be one in {ppliststr(self.Models.keys())}") + + this_model = SensorModelMetaData( + **{ + "sensors": self.Models[model]["sensors"], + "parameters": self.Models[model]["parameters"], + "sensor_info": SensorInfo( + **{ + "created_by": self._obj.attrs["institution"], + "date_creation": self._obj.attrs["history"] + .split(";")[0] + .strip() + .split(" ")[0], + "link": "./argo.sensor.schema.json", + "format_version": self._obj.attrs["Conventions"], + "contents": "", + "sensor_described": model, + } + ), + "instrument_vendorinfo": "", + } + ) + oem_like = OemMetaDataDisplay(this_model) + + if OPTIONS["display_style"] == "text": + return f"
{escape(repr(oem_like))}
" + + from IPython.display import display, HTML + display(HTML(oem_like.html)) diff --git a/argopy/related/sensors/oem/oem_metadata_repr.py b/argopy/related/sensors/oem/oem_metadata_repr.py index e1d98fd52..b856efc8b 100644 --- a/argopy/related/sensors/oem/oem_metadata_repr.py +++ b/argopy/related/sensors/oem/oem_metadata_repr.py @@ -54,7 +54,7 @@ def html(self): # --- Header --- header_html = f""" -

Argo Sensor OEM Metadata: {self.OEMsensor.sensor_info._attr2str('sensor_described')}

+

Argo Sensor Metadata: {self.OEMsensor.sensor_info._attr2str('sensor_described')}

Created by: {self.OEMsensor.sensor_info._attr2str('created_by')} | Date: {self.OEMsensor.sensor_info._attr2str('date_creation')}

""" diff --git a/argopy/static/assets/data_types.json b/argopy/static/assets/data_types.json index 0e5c4e83b..a5b880c84 100644 --- a/argopy/static/assets/data_types.json +++ b/argopy/static/assets/data_types.json @@ -1,7 +1,7 @@ { "name": "data_types", "long_name": "Expected data types of Argo variables", - "last_update": "2025-10-20T16:34:14.606570+00:00", + "last_update": "2025-11-11T20:51:33.505007+00:00", "data": { "str": [ "PLATFORM_NUMBER", @@ -73,6 +73,7 @@ "SENSOR_MAKER", "SENSOR_MODEL", "SENSOR_SERIAL_NO", + "SENSOR_FIRMWARE_VERSION", "PARAMETER_SENSOR", "PARAMETER_UNITS", "PARAMETER_ACCURACY", @@ -285,8 +286,6 @@ "PLATFORM_NUMBER", "WMO_INST_TYPE", "CYCLE_NUMBER", - "CYCLE_NUMBER_INDEX", - "CYCLE_NUMBER_INDEX_ADJUSTED", "CONFIG_MISSION_NUMBER", "JULD_STATUS", "JULD_ADJUSTED_STATUS", diff --git a/argopy/utils/format.py b/argopy/utils/format.py index 7f1932f4b..a31db1fa8 100644 --- a/argopy/utils/format.py +++ b/argopy/utils/format.py @@ -420,7 +420,7 @@ def ppliststr(l: list[str], last : str = 'and', n : int | None = None) -> str: ppliststr(['a', 'b', 'c', 'd']) -> "'a', 'b', 'c' and 'd'" ppliststr(['a', 'b'], last='or') -> "'a' or 'b'" - ppliststr(['a', 'b', 'c', 'd'], n=3) -> "'a', 'b', 'c' and 'd'" + ppliststr(['a', 'b', 'c', 'd'], n=3) -> "'a', 'b', 'c' and more ..." """ n = n if n is not None else len(l) From 44fc02e35b80c1f8bac17d7fe1657494fcbe20cf Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Mon, 24 Nov 2025 19:05:35 +0100 Subject: [PATCH 67/71] Update update_json_assets --- cli/update_json_assets | 1 + 1 file changed, 1 insertion(+) diff --git a/cli/update_json_assets b/cli/update_json_assets index d4dadbeb3..e53991474 100755 --- a/cli/update_json_assets +++ b/cli/update_json_assets @@ -112,6 +112,7 @@ def asset_data_types(): "SENSOR_MAKER", "SENSOR_MODEL", "SENSOR_SERIAL_NO", + "SENSOR_FIRMWARE_VERSION", "PARAMETER_SENSOR", "PARAMETER_UNITS", "PARAMETER_ACCURACY", From c61767854bc6fb82432e93d91c05515f587ab25c Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Mon, 5 Jan 2026 16:58:21 +0100 Subject: [PATCH 68/71] Fix merge with master branch --- argopy/utils/__init__.py | 14 ++-- argopy/utils/accessories.py | 43 ++++++++++++ argopy/utils/format.py | 130 +++++++++++++++++++++++++++++++++++- 3 files changed, 181 insertions(+), 6 deletions(-) diff --git a/argopy/utils/__init__.py b/argopy/utils/__init__.py index ca43cb118..52a81eb56 100644 --- a/argopy/utils/__init__.py +++ b/argopy/utils/__init__.py @@ -36,7 +36,7 @@ from .caching import clear_cache, lscache from .monitored_threadpool import MyThreadPoolExecutor as MonitoredThreadPoolExecutor from .chunking import Chunker -from .accessories import Registry, float_wmo, NVSrow +from .accessories import Registry, float_wmo, NVSrow, ListStrProperty from .locals import ( # noqa: F401 show_versions, show_options, @@ -51,16 +51,17 @@ conv_lon, toYearFraction, YearFraction_to_datetime, + point_in_polygon, ) -from .compute import linear_interpolation_remap, groupby_remap -from .transform import ( +from .computers import linear_interpolation_remap, groupby_remap +from .transformers import ( fill_variables_not_in_all_datasets, drop_variables_not_in_all_datasets, merge_param_with_param_adjusted, filter_param_by_data_mode, split_data_mode, ) -from .format import argo_split_path, format_oneline, UriCName, redact, dirfs_relpath, urnparser, ppliststr +from .format import argo_split_path, format_oneline, UriCName, redact, dirfs_relpath, urnparser, ppliststr, mono2multi, cfgnameparser, group_cycles_by_missions from .loggers import warnUnless, log_argopy_callerstack from .carbon import GreenCoding, Github from . import optical_modeling @@ -118,6 +119,7 @@ "Registry", "float_wmo", "NVSrow", + "ListStrProperty", # Locals (environments, versions, systems): "path2assets", "show_versions", @@ -131,6 +133,7 @@ "conv_lon", "toYearFraction", "YearFraction_to_datetime", + "point_in_polygon", # Computation with datasets: "linear_interpolation_remap", "groupby_remap", @@ -146,8 +149,11 @@ "dirfs_relpath", "UriCName", "redact", + "cfgnameparser", + "mono2multi", "urnparser", "ppliststr", + "group_cycles_by_missions", # Loggers: "warnUnless", "log_argopy_callerstack", diff --git a/argopy/utils/accessories.py b/argopy/utils/accessories.py index 578a31700..565386c0f 100644 --- a/argopy/utils/accessories.py +++ b/argopy/utils/accessories.py @@ -349,3 +349,46 @@ def __repr__(self): summary.append(f"%12s: {self.deprecated}" % "deprecated") summary.append("%12s: %s" % ("definition", self.definition)) return "\n".join(summary) + + +class ListStrProperty: + """ + Descriptor for a list of strings that is able to handle: + + - getting possible values, + - validating a value and + - print + + Examples + -------- + .. code-block:: python + + st = ListStrProperty(['a', 'b', 'c']) + st.values # ['a', 'b', 'c'] + 'a' in st # True + 'B' in st # True, validation is case-insensitive + 'z' in st # False + print(st) # 'a, b, c' + + """ + def __init__(self, possible_values: list[str]): + self.possible_values = possible_values + + @property + def values(self) -> list[str]: + """Return the list of possible values.""" + return self.possible_values + + def __contains__(self, value): + return self._is_valid(value) + + def _is_valid(self, value: str) -> bool: + """Check if the given value is in the list of possible values.""" + result = False + for txt in self.possible_values: + if value.lower() in txt.lower(): + result = True + return result + + def __repr__(self): + return ", ".join(self.possible_values) diff --git a/argopy/utils/format.py b/argopy/utils/format.py index a31db1fa8..0230ce8cf 100644 --- a/argopy/utils/format.py +++ b/argopy/utils/format.py @@ -8,7 +8,9 @@ import pandas as pd import numpy as np import warnings -from .checkers import check_cyc, check_wmo +from typing import Literal + +from argopy.utils.checkers import check_cyc, check_wmo log = logging.getLogger("argopy.utils.format") @@ -400,6 +402,130 @@ def cname(self) -> str: return cname +def cfgnameparser(name: str) -> dict[str, str]: + """Configuration parameter name parser + + Get parameter name and unit as dictionary out of R18 prefLabel. + + Unit is always lower case. + """ + assert name.split("_")[0] == 'CONFIG', "This is not a valid configuration parameter (see R18 prefLabel)" + unit = name.split("_")[-1] + label = "".join(name.split("_")[1:-1]) + return {'label': label, 'unit': unit.lower()} + + +def group_cycles_by_missions(cycles: dict[int, int], output: Literal['group', 'list'] = 'group') -> dict[int, str] | dict[int, list[int]]: + """ + + Parameters + ---------- + cycles: dict[int, int] + A dictionary mapping cycle (keys) on mission numbers (values). + output: Literal['group', 'list'], default='group' + + Returns + ------- + dict[int, str] | dict[int, list[int]] + A dictionary mapping mission numbers (keys) on group of cycle numbers (values). If output is set to 'group', values are a string (eg '1>3') and if output is set to 'list', values are the list of cycle numbers as integers. + + """ + is_suite = lambda x: list(range(np.min(x), np.max(x) + 1)) == sorted(x) + + def group_consecutive(lst): + if not lst: + return [] + + # Sort the list to ensure consecutive values are adjacent + lst = sorted(lst) + + groups = [[lst[0]]] + for i in range(1, len(lst)): + if lst[i] == groups[-1][-1] + 1: + groups[-1].append(lst[i]) + else: + groups.append([lst[i]]) + + return groups + + missions = np.unique(list(cycles.values())) + mission_cycles = {} + for m in missions: + mission_cycles.update({int(m): []}) + for cyc, mis in cycles.items(): + if mis == m: + mission_cycles[int(m)].append(cyc) + if output == 'list': + return mission_cycles + + else: + results = {} + for mis, cycs in mission_cycles.items(): + if len(cycs) == 1: + txt = f"{cycs[0]}" + elif is_suite(cycs): + txt = f"{np.min(cycs)}>{np.max(cycs)}" + else: + grps = group_consecutive(cycs) + summary = [] + for grp in grps: + summary.append(f"{np.min(grp)}>{np.max(grp)}") + txt = ",".join(summary) + results.update({int(mis): txt}) + return results + + +def mono2multi(flist : list[str], convention : str = 'core', sep :str = '/') -> list[str]: + """Convert a list of mono-profile files to a list of multi-profile files + + The multi-profile file name is based on an :class:`ArgoIndex` convention. + + Parameters + ---------- + flist: list[str] + A list of mono-profile files (relative GDAC paths), as output for :meth:`ArgoIndex.read_files`. + convention: str, optional, default = 'ar_index_global_prof' + The Argo index convention from which `flist` was extracted. Can be 'ar_index_global_prof' or 'argo_synthetic-profile_index'. + sep: str, optional, default = '/' + GDAC file system separator used in flist + + Returns + ------- + list(str) + """ + def _mono2multi(mono_path): + meta = argo_split_path(mono_path) + + if convention == "ar_index_global_prof": + return sep.join( + [ + meta["origin"], + "dac", + meta["dac"], + meta["wmo"], + "%s_prof.nc" % meta["wmo"], + ] + ) + + elif convention in ["argo_synthetic-profile_index"]: + return sep.join( + [ + meta["origin"], + "dac", + meta["dac"], + meta["wmo"], + "%s_Sprof.nc" % meta["wmo"], + ] + ) + + else: + raise ValueError("Method not available for this index (only 'ar_index_global_prof' and 'argo_synthetic-profile_index' allowed).") + + new_uri = [_mono2multi(uri)[2:] for uri in flist] + new_uri = list(set(new_uri)) + return new_uri + + def urnparser(urn): """Parsing RFC 8141 compliant uniform resource names (URN) from NVS SDN stands for SeaDataNet @@ -440,4 +566,4 @@ def ppliststr(l: list[str], last : str = 'and', n : int | None = None) -> str: else: s += f", '{item}'" ii += 1 - return s \ No newline at end of file + return s From 1d1ef8966ed947aefb3c292f0b99d5291e0e6fe1 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Mon, 5 Jan 2026 17:01:02 +0100 Subject: [PATCH 69/71] Update whats-new.rst --- docs/whats-new.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/whats-new.rst b/docs/whats-new.rst index 7c8f2c0af..306194e39 100644 --- a/docs/whats-new.rst +++ b/docs/whats-new.rst @@ -77,7 +77,7 @@ Breaking changes Energy ^^^^^^ -.. image:: https://img.shields.io/badge/Total%20carbon%20emitted%20by%20release%20v1.4.0%20%5BgCO2eq%5D-TBA-black?style=plastic&labelColor=grey +.. image:: https://img.shields.io/badge/Total%20carbon%20emitted%20by%20release%20v1.4.0%20%5BgCO2eq%5D-1666-black?style=plastic&labelColor=grey v1.3.1 (22 Oct. 2025) --------------------- From 08adaa13c903c8d05d36424274ac1f113be2bba2 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Mon, 5 Jan 2026 17:06:47 +0100 Subject: [PATCH 70/71] Fix merge with master branch --- argopy/utils/__init__.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/argopy/utils/__init__.py b/argopy/utils/__init__.py index 52a81eb56..b571bc447 100644 --- a/argopy/utils/__init__.py +++ b/argopy/utils/__init__.py @@ -36,7 +36,7 @@ from .caching import clear_cache, lscache from .monitored_threadpool import MyThreadPoolExecutor as MonitoredThreadPoolExecutor from .chunking import Chunker -from .accessories import Registry, float_wmo, NVSrow, ListStrProperty +from .accessories import Registry, float_wmo, ListStrProperty, NVSrow from .locals import ( # noqa: F401 show_versions, show_options, @@ -61,10 +61,12 @@ filter_param_by_data_mode, split_data_mode, ) +from .mappers import map_vars_to_dict from .format import argo_split_path, format_oneline, UriCName, redact, dirfs_relpath, urnparser, ppliststr, mono2multi, cfgnameparser, group_cycles_by_missions from .loggers import warnUnless, log_argopy_callerstack from .carbon import GreenCoding, Github from . import optical_modeling +from .carbonate import calculate_uncertainties, error_propagation import importlib path2assets = importlib.util.find_spec('argopy.static.assets').submodule_search_locations[0] @@ -118,8 +120,8 @@ # Accessories classes (specific objects): "Registry", "float_wmo", - "NVSrow", "ListStrProperty", + "NVSrow", # Locals (environments, versions, systems): "path2assets", "show_versions", @@ -143,6 +145,8 @@ "merge_param_with_param_adjusted", "filter_param_by_data_mode", "split_data_mode", + # Mapping out of datasets: + "map_vars_to_dict", # Formatters: "format_oneline", "argo_split_path", @@ -162,4 +166,7 @@ "Github", # Optical modeling "optical_modeling", + # Carbonate calculations + "calculate_uncertainties", + "error_propagation", ) From 32a3b8de14422bcf714bb821f90841b3ec257d90 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Mon, 5 Jan 2026 17:11:01 +0100 Subject: [PATCH 71/71] More Fix merge with master branch --- argopy/utils/__init__.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/argopy/utils/__init__.py b/argopy/utils/__init__.py index b571bc447..8704260da 100644 --- a/argopy/utils/__init__.py +++ b/argopy/utils/__init__.py @@ -62,7 +62,7 @@ split_data_mode, ) from .mappers import map_vars_to_dict -from .format import argo_split_path, format_oneline, UriCName, redact, dirfs_relpath, urnparser, ppliststr, mono2multi, cfgnameparser, group_cycles_by_missions +from .format import argo_split_path, format_oneline, UriCName, redact, dirfs_relpath, urnparser, ppliststr from .loggers import warnUnless, log_argopy_callerstack from .carbon import GreenCoding, Github from . import optical_modeling @@ -153,11 +153,8 @@ "dirfs_relpath", "UriCName", "redact", - "cfgnameparser", - "mono2multi", "urnparser", "ppliststr", - "group_cycles_by_missions", # Loggers: "warnUnless", "log_argopy_callerstack",