diff --git a/argopy/__init__.py b/argopy/__init__.py index caadf8b1a..a17336706 100644 --- a/argopy/__init__.py +++ b/argopy/__init__.py @@ -41,7 +41,7 @@ from .utils import clear_cache, lscache # noqa: E402 from .utils import MonitoredThreadPoolExecutor # noqa: E402, F401 from .utils import monitor_status as status # noqa: E402 -from .related import TopoFetcher, OceanOPSDeployments, ArgoNVSReferenceTables, ArgoDocs, ArgoDOI # noqa: E402 +from .related import TopoFetcher, OceanOPSDeployments, ArgoNVSReferenceTables, ArgoDocs, ArgoDOI, ArgoSensor, OEMSensorMetaData # noqa: E402 from .extensions import CanyonMED # noqa: E402 @@ -69,6 +69,8 @@ "ArgoDocs", # Class "TopoFetcher", # Class "ArgoDOI", # Class + "ArgoSensor", # Class + "OEMSensorMetaData", # Advanced Argo data stores: "ArgoFloat", # Class @@ -83,9 +85,6 @@ "stores", "tutorial", - # Argo xarray accessor extensions - "CanyonMED", - - # Constants + # Constants: "__version__" ) diff --git a/argopy/errors.py b/argopy/errors.py index 2cc2705ef..9d55b0f34 100644 --- a/argopy/errors.py +++ b/argopy/errors.py @@ -85,7 +85,7 @@ class InvalidDataset(ValueError): class InvalidDatasetStructure(ValueError): - """Raise when the xarray dataset is not as expected.""" + """Raise when an internal dataset structure is not as expected.""" pass diff --git a/argopy/options.py b/argopy/options.py index 78fe7c1db..257094a41 100644 --- a/argopy/options.py +++ b/argopy/options.py @@ -27,7 +27,7 @@ HAS_BOTO3 = False -from .errors import OptionValueError, GdacPathError, ErddapPathError +from .errors import OptionValueError, GdacPathError, ErddapPathError, APIServerError # Define a logger @@ -50,6 +50,13 @@ PARALLEL = "parallel" PARALLEL_DEFAULT_METHOD = "parallel_default_method" LON = "longitude_convention" +API_FLEETMONITORING = "fleetmonitoring" +RBR_API_KEY = "rbr_api_key" +API_RBR = "rbr_api" +NVS = "nvs" +API_SEABIRD = "seabird_api" +DISPLAY_STYLE = "display_style" + # Define the list of available options and default values: OPTIONS = { @@ -69,6 +76,12 @@ PARALLEL: False, PARALLEL_DEFAULT_METHOD: "thread", LON: "180", + API_FLEETMONITORING: "https://fleetmonitoring.euro-argo.eu", + RBR_API_KEY: os.environ.get("RBR_API_KEY"), # Contact RBR at argo@rbr-global.com if you do not have an authorization key. + API_RBR: "https://oem-lookup.rbr-global.com/api/v1", + NVS: "https://vocab.nerc.ac.uk/collection", + API_SEABIRD: "https://instrument.seabirdhub.com/api/argo-calibration", + DISPLAY_STYLE: "html", } DEFAULT = OPTIONS.copy() @@ -76,9 +89,9 @@ _DATA_SOURCE_LIST = frozenset(["erddap", "argovis", "gdac"]) _DATASET_LIST = frozenset(["phy", "bgc", "ref", "bgc-s", "bgc-b"]) _USER_LEVEL_LIST = frozenset(["standard", "expert", "research"]) +_DISPLAY_STYLE_LIST = frozenset(["html", "text"]) +_LON_LIST = frozenset(['180', '360']) - -# Define how to validate options: def _positive_integer(value): return isinstance(value, int) and value > 0 @@ -91,7 +104,7 @@ def validate_gdac(this_path): return False -def validate_http(this_path): +def validate_erddap(this_path): if this_path != "-": return check_erddap_path(this_path, errors="raise") else: @@ -117,10 +130,164 @@ def validate_parallel_method(method): return False +def validate_fleetmonitoring(this_path): + if this_path != "-": + return check_fleetmonitoring_path(this_path, errors="raise") + else: + log.debug("OPTIONS['%s'] is not defined" % API_FLEETMONITORING) + return False + + +def validate_rbr(this_path): + if this_path != "-": + return True # todo: check with RBR how to get the API endpoint for its status + else: + log.debug("OPTIONS['%s'] is not defined" % API_RBR) + return False + + +def validate_seabird(this_path): + if this_path != "-": + return True # todo: check with Seabird how to get the API endpoint for its status + else: + log.debug("OPTIONS['%s'] is not defined" % API_RBR) + return False + + +def PARALLEL_SETUP(parallel): + parallel = VALIDATE("parallel", parallel) + if isinstance(parallel, bool): + if parallel: + return True, OPTIONS["parallel_default_method"] + else: + return False, "sequential" + else: + return True, parallel + + +def check_erddap_path(path, errors="ignore"): + """Check if a url points to a valid ERDDAP server""" + fs = fsspec.filesystem("http", ssl=False) + check1 = fs.exists(path + "/info/index.json") + if check1: + return True + elif errors == "raise": + raise ErddapPathError(f"This url is not a valid ERDDAP server:\n{path}") + elif errors == "warn": + warnings.warn(f"This url is not a valid ERDDAP server:\n{path}") + return False + else: + return False + + +def check_fleetmonitoring_path(path, errors='ignore'): + """Check if a url points to a Euro-Argo valid FleetMonitoring server""" + fs = fsspec.filesystem("http", ssl=False) + try: + fs.open_json(path + "/get-version") + return True + except Exception as e: + if errors == "raise": + raise APIServerError(f"This url is not a valid Euro-Argo FleetMonitoring server:\n{path}") + elif errors == "warn": + warnings.warn(f"This url is not a valid Euro-Argo FleetMonitoring server:\n{path}") + return False + else: + return False + + +def check_gdac_option( + path, errors: str = "ignore", ignore_knowns: bool = True +): # noqa: C901 + """Check if a path has the expected GDAC structure + + Expected GDAC structure:: + + . + └── dac + ├── aoml + ├── ... + ├── coriolis + ├── ... + ├── meds + └── nmdis + + Examples:: + + >>> check_gdac_path("https://data-argo.ifremer.fr") # True + >>> check_gdac_path("https://usgodae.org/pub/outgoing/argo") # True + >>> check_gdac_path("ftp://ftp.ifremer.fr/ifremer/argo") # True + >>> check_gdac_path("/home/ref-argo/gdac") # True + >>> check_gdac_path("s3://argo-gdac-sandbox/pub") # True + + >>> check_gdac_path("https://www.ifremer.fr") # False + >>> check_gdac_path("ftp://usgodae.org/pub/outgoing") # False + + Parameters + ---------- + path: str + Path name to check, including access protocol + errors: str, default="ignore" + Determine how check procedure error are handled: "ignore", "raise" or "warn" + ignore_knowns: bool, default=False + Should the checking procedure be by-passed for the internal list of known GDACs. + Set this to True to check if a known GDACs is connected or not. + + Returns + ------- + checked: boolean + + See also + -------- + :class:`argopy.stores.gdacfs`, :meth:`argopy.utils.list_gdac_servers` + + """ + from .utils import ( + list_gdac_servers, + ) # import here, otherwise raises circular import + + if path in list_gdac_servers() and ignore_knowns: + return True + else: + + from .stores import gdacfs # import here, otherwise raises circular import + + try: + fs = gdacfs(path) + except GdacPathError: + if errors == "raise": + raise + elif errors == "warn": + warnings.warn("Can't get address info (GAIerror) on '%s'" % path) + return False + else: + return False + + check1 = fs.exists("dac") + if check1: + return True + + elif errors == "raise": + raise GdacPathError( + "This path is not GDAC compliant (no legitimate sub-folder `dac`):\n%s" + % path + ) + + elif errors == "warn": + warnings.warn( + "This path is not GDAC compliant (no legitimate sub-folder `dac`):\n%s" + % path + ) + return False + + else: + return False + + _VALIDATORS = { DATA_SOURCE: _DATA_SOURCE_LIST.__contains__, GDAC: validate_gdac, - ERDDAP: validate_http, + ERDDAP: validate_erddap, DATASET: _DATASET_LIST.__contains__, CACHE_FOLDER: lambda x: os.access(x, os.W_OK), CACHE_EXPIRATION: lambda x: isinstance(x, int) and x > 0, @@ -131,9 +298,16 @@ def validate_parallel_method(method): USER: lambda x: isinstance(x, str) or x is None, PASSWORD: lambda x: isinstance(x, str) or x is None, ARGOVIS_API_KEY: lambda x: isinstance(x, str) or x is None, + RBR_API_KEY: lambda x: isinstance(x, str) or x is None, PARALLEL: validate_parallel, PARALLEL_DEFAULT_METHOD: validate_parallel_method, - LON: lambda x: x in ['180', '360'], + LON: _LON_LIST.__contains__, + API_FLEETMONITORING: validate_fleetmonitoring, + API_RBR: validate_rbr, + API_SEABIRD: validate_seabird, + NVS: lambda x: isinstance(x, str) or x is None, + DISPLAY_STYLE: _DISPLAY_STYLE_LIST.__contains__, + } @@ -148,17 +322,6 @@ def VALIDATE(key, val): raise ValueError(f"option '{key}' has no validation method") -def PARALLEL_SETUP(parallel): - parallel = VALIDATE("parallel", parallel) - if isinstance(parallel, bool): - if parallel: - return True, OPTIONS["parallel_default_method"] - else: - return False, "sequential" - else: - return True, parallel - - class set_options: """Set options for argopy @@ -206,6 +369,19 @@ class set_options: You can get a free key at https://argovis-keygen.colorado.edu + rbr_api_key: str, default: None + The API key to use when fetching data from the RBR OEM lookup API. + + RBR sensor owners can get a key by contacting RBR at argo@rbr-global.com. + + This key can also be used from: https://oem-lookup.rbr-global.com + + rbr_api: str + Server address of the RBR "RBRargo Product Lookup" web-API + + seabird_api: str + Server address of the Seabird-Scientific "Instrument Metadata Portal" web-API + parallel: bool, str, :class:`distributed.Client`, default: False Set whether to use parallelisation or not, and possibly which method to use. @@ -285,107 +461,3 @@ def __exit__(self, type, value, traceback): def reset_options(): """Reset all options to default values""" set_options(**DEFAULT) - - -def check_erddap_path(path, errors="ignore"): - """Check if an url points to an ERDDAP server""" - fs = fsspec.filesystem("http", ssl=False) - check1 = fs.exists(path + "/info/index.json") - if check1: - return True - elif errors == "raise": - raise ErddapPathError("This url is not a valid ERDDAP server:\n%s" % path) - - elif errors == "warn": - warnings.warn("This url is not a valid ERDDAP server:\n%s" % path) - return False - else: - return False - - -def check_gdac_option( - path, errors: str = "ignore", ignore_knowns: bool = True -): # noqa: C901 - """Check if a path has the expected GDAC structure - - Expected GDAC structure:: - - . - └── dac - ├── aoml - ├── ... - ├── coriolis - ├── ... - ├── meds - └── nmdis - - Examples:: - - >>> check_gdac_path("https://data-argo.ifremer.fr") # True - >>> check_gdac_path("https://usgodae.org/pub/outgoing/argo") # True - >>> check_gdac_path("ftp://ftp.ifremer.fr/ifremer/argo") # True - >>> check_gdac_path("/home/ref-argo/gdac") # True - >>> check_gdac_path("s3://argo-gdac-sandbox/pub") # True - - >>> check_gdac_path("https://www.ifremer.fr") # False - >>> check_gdac_path("ftp://usgodae.org/pub/outgoing") # False - - Parameters - ---------- - path: str - Path name to check, including access protocol - errors: str, default="ignore" - Determine how check procedure error are handled: "ignore", "raise" or "warn" - ignore_knowns: bool, default=False - Should the checking procedure be by-passed for the internal list of known GDACs. - Set this to True to check if a known GDACs is connected or not. - - Returns - ------- - checked: boolean - - See also - -------- - :class:`argopy.stores.gdacfs`, :meth:`argopy.utils.list_gdac_servers` - - """ - from .utils import ( - list_gdac_servers, - ) # import here, otherwise raises circular import - - if path in list_gdac_servers() and ignore_knowns: - return True - else: - - from .stores import gdacfs # import here, otherwise raises circular import - - try: - fs = gdacfs(path) - except GdacPathError: - if errors == "raise": - raise - elif errors == "warn": - warnings.warn("Can't get address info (GAIerror) on '%s'" % path) - return False - else: - return False - - check1 = fs.exists("dac") - if check1: - return True - - elif errors == "raise": - raise GdacPathError( - "This path is not GDAC compliant (no legitimate sub-folder `dac`):\n%s" - % path - ) - - elif errors == "warn": - warnings.warn( - "This path is not GDAC compliant (no legitimate sub-folder `dac`):\n%s" - % path - ) - return False - - else: - return False diff --git a/argopy/related/__init__.py b/argopy/related/__init__.py index 465bc5de2..624dd1a2c 100644 --- a/argopy/related/__init__.py +++ b/argopy/related/__init__.py @@ -4,22 +4,25 @@ from .argo_documentation import ArgoDocs from .doi_snapshot import ArgoDOI from .euroargo_api import get_coriolis_profile_id, get_ea_profile_page -from .utils import load_dict, mapp_dict # Should come last +from .sensors import ArgoSensor, SensorType, SensorModel, OEMSensorMetaData +from .utils import load_dict, mapp_dict # Must come last to avoid circular import, I know, not good # __all__ = ( - # Classes: + # Classes : "TopoFetcher", "OceanOPSDeployments", "ArgoNVSReferenceTables", "ArgoDocs", "ArgoDOI", + "ArgoSensor", + "OEMSensorMetaData", - # Functions: + # Functions : "get_coriolis_profile_id", "get_ea_profile_page", - # Utilities: + # Utilities : "load_dict", "mapp_dict", ) diff --git a/argopy/related/euroargo_api.py b/argopy/related/euroargo_api.py index fe982fb6d..4f6784f81 100644 --- a/argopy/related/euroargo_api.py +++ b/argopy/related/euroargo_api.py @@ -1,4 +1,6 @@ import pandas as pd +from typing import Any, Callable, Literal + from ..options import OPTIONS from ..utils.checkers import check_wmo, check_cyc from ..stores import httpstore @@ -100,3 +102,74 @@ def get_ea_profile_page(WMO, CYC=None, **kwargs): df = get_coriolis_profile_id(WMO, CYC, **kwargs) url = "https://dataselection.euro-argo.eu/cycle/{}" return [url.format(this_id) for this_id in sorted(df["ID"])] + + +class EAfleetmonitoringAPI: + + def __init__(self, **kwargs) -> None: + """Create an instance of :class:`EAfleetmonitoringAPI` + + Parameters + ---------- + fs: :class:`stores.httpstore`, default: None + The http filesystem to use. If None is provided, we instantiate a new one based on `cache`, `cachedir` and `timeout` options. + cache : bool, optional, default: True + Use cache or not for fetched data. Used only if `fs` is None. + cachedir: str, optional, default: OPTIONS['cachedir'] + Folder where to store cached files. Used only if `fs` is None. + timeout: int, optional, default: OPTIONS['api_timeout'] + Time out in seconds to connect to web API. Used only if `fs` is None. + + """ + if kwargs.get("fs", None) is None: + self._fs = httpstore( + cache=kwargs.get("cache", True), + cachedir=kwargs.get("cachedir", OPTIONS["cachedir"]), + timeout=kwargs.get("timeout", OPTIONS["api_timeout"]), + ) + else: + self._fs = kwargs["fs"] + + + def floats( + self, + wmo: str | int | list[int] | list[str], + preprocess: Callable = None, + preprocess_opts: dict = {}, + postprocess: Callable = None, + postprocess_opts: dict = {}, + errors: Literal["raise", "ignore", "silent"] = "raise", + progress: bool = False, + ) -> Any: + """Call to 'floats' endpoint with one or more WMOs + + Notes + ----- + Based on a POST request to the fleet-monitoring API requests to `/floats/{wmo}`. + + `Endpoint documentation `_. + + Notes + ----- + No option checking, to be done by caller + """ + WMOs = check_wmo(wmo) + + URI = [] + for wmo in WMOs: + URI.append(f"{OPTIONS['fleetmonitoring']}/floats/{wmo}") + + sns = self._fs.open_mfjson( + URI, + preprocess=preprocess, + preprocess_opts=preprocess_opts, + progress=progress, + errors=errors, + progress_unit="float", + progress_desc="Fetching floats metadata", + ) + + if postprocess is not None: + return postprocess(sns, **postprocess_opts) + else: + return sns diff --git a/argopy/related/reference_tables.py b/argopy/related/reference_tables.py index 892329a59..b59a58e7f 100644 --- a/argopy/related/reference_tables.py +++ b/argopy/related/reference_tables.py @@ -5,96 +5,37 @@ from ..stores import httpstore, filestore from ..options import OPTIONS +from ..utils import urnparser from ..utils import path2assets VALID_REF = filestore(cache=True).open_json(Path(path2assets).joinpath("nvs_reference_tables.json"))['data']['valid_ref'] -class ArgoNVSReferenceTables: - """Argo Reference Tables - - Utility function to retrieve Argo Reference Tables from a NVS server. - - By default, this relies on: https://vocab.nerc.ac.uk/collection - - Examples - -------- - Methods: - - >>> R = ArgoNVSReferenceTables() - >>> R.search('sensor') - >>> R.tbl(3) - >>> R.tbl('R09') - - Properties: - - >>> R.all_tbl_name - >>> R.all_tbl - >>> R.valid_ref - - Notes - ----- - This class relies on a list of valid reference table ids that is updated on every argopy release. - - """ - valid_ref = VALID_REF.copy() - - """List of all available Reference Tables""" +class NVScollection: + """ A class to handle any NVS collection table """ def __init__( self, - nvs="https://vocab.nerc.ac.uk/collection", - cache: bool = True, - cachedir: str = "", + **kwargs, ): - """Argo Reference Tables from NVS""" - - cachedir = OPTIONS["cachedir"] if cachedir == "" else cachedir - self.fs = httpstore(cache=cache, cachedir=cachedir) - self.nvs = nvs + """Reference Tables from NVS collection""" + self.nvs = kwargs.get("nvs", OPTIONS["nvs"]) - def _valid_ref(self, rtid): - """ - Validate any rtid argument and return the corresponding valid ID from the list. + self.fs = kwargs.get("fs", None) + if self.fs is None: + self._cache = kwargs.get("cache", True) + self._cachedir = kwargs.get("cachedir", OPTIONS["cachedir"]) + self._timeout = kwargs.get("timeout", OPTIONS["api_timeout"]) + self.fs = httpstore(cache=self._cache, cachedir=self._cachedir, timeout=self._timeout) - Parameters - ---------- - rtid: Input reference ID. Can be a string (e.g., "R12", "12", "r12") or a number (e.g., 12). - - Returns: - str: Valid reference ID from the list, or None if not found. - """ - # Convert rtid to a string and standardize its format - if isinstance(rtid, (int, float)): - # If rtid is a number, format it as "RXX" - rtid_str = f"R{int(rtid):02d}" - else: - # If rtid is a string, convert to uppercase and standardize - rtid_str = str(rtid).strip().upper() - if rtid_str.startswith('R') and len(rtid_str) > 1: - # If it starts with 'R', ensure the numeric part is two digits - prefix = rtid_str[0] - suffix = rtid_str[1:] - try: - num = int(suffix) - rtid_str = f"{prefix}{num:02d}" - except ValueError: - pass # Keep the original string if conversion fails - elif ~rtid_str.startswith('R'): - try: - num = int(rtid_str) - rtid_str = f"R{num}" - except ValueError: - pass # Keep the original string if conversion fails + @property + def valid_ref(self): + df = self._FullCollection() + return df['ID'].to_list() - # Check if the standardized rtid_str is in the valid_refs list - if rtid_str in self.valid_ref: - return rtid_str - else: - raise ValueError( - f"Invalid Argo Reference Table '{rtid}', must be one in: {', '.join(self.valid_ref)}" - ) + def _valid_ref(self, rtid): + """No validation""" return rtid def _jsConcept2df(self, data): @@ -104,18 +45,21 @@ def _jsConcept2df(self, data): "prefLabel": [], "definition": [], "deprecated": [], + "urn": [], "id": [], } for k in data["@graph"]: if k["@type"] == "skos:Collection": Collection_name = k["dc:alternative"] elif k["@type"] == "skos:Concept": - content["altLabel"].append(k["skos:altLabel"]) + content["altLabel"].append(urnparser(k['skos:notation'])['termid']) content["prefLabel"].append(k["skos:prefLabel"]["@value"]) - content["definition"].append(k["skos:definition"]["@value"] if k["skos:definition"] != '' else None) + content["definition"].append(k["skos:definition"]["@value"]) content["deprecated"].append(k["owl:deprecated"]) + content["urn"].append(k['skos:notation']) content["id"].append(k["@id"]) df = pd.DataFrame.from_dict(content) + df['deprecated'] = df.apply(lambda x: True if x['deprecated']=='true' else False, axis=1) df.name = Collection_name return df @@ -128,6 +72,25 @@ def _jsCollection(self, data): rtid = k["@id"] return (name, desc, rtid) + def _jsFullCollection(self, data): + """Return all skos:Collection information as data""" + result = [] + for k in data["@graph"]: + if k["@type"] == "skos:Collection": + title = k["dc:title"] + name = k["dc:alternative"] + desc = k["dc:description"] + url = k["@id"] + tid = k['@id'].split('/')[-3] + result.append((tid, title, name, desc, url)) + return result + + @lru_cache + def _FullCollection(self): + url = f"{self.nvs}/?_profile=nvs&_mediatype=application/ld+json" + js = self.fs.open_json(url) + return pd.DataFrame(self._jsFullCollection(js), columns=['ID', 'title', 'name', 'description', 'url']) + def get_url(self, rtid, fmt="ld+json"): """Return URL toward a given reference table for a given format @@ -158,7 +121,7 @@ def get_url(self, rtid, fmt="ld+json"): @lru_cache def tbl(self, rtid): - """Return an Argo Reference table + """Return a Reference table Parameters ---------- @@ -174,8 +137,9 @@ def tbl(self, rtid): df = self._jsConcept2df(js) return df + @lru_cache def tbl_name(self, rtid): - """Return name of an Argo Reference table + """Return name of a Reference table Parameters ---------- @@ -190,6 +154,41 @@ def tbl_name(self, rtid): js = self.fs.open_json(self.get_url(rtid)) return self._jsCollection(js) + @property + def all_tbl(self): + """Return all Reference tables + + Returns + ------- + OrderedDict + Dictionary with all table short names as key and table content as class:`pandas.DataFrame` + """ + URLs = [self.get_url(rtid) for rtid in self.valid_ref] + df_list = self.fs.open_mfjson(URLs, preprocess=self._jsConcept2df) + all_tables = {} + [all_tables.update({t.name: t}) for t in df_list] + all_tables = collections.OrderedDict(sorted(all_tables.items())) + return all_tables + + @property + def all_tbl_name(self): + """Return names of all Reference tables + + Returns + ------- + OrderedDict + Dictionary with all table short names as key and table names as tuple('short name', 'description', 'NVS id link') + """ + URLs = [self.get_url(rtid) for rtid in self.valid_ref] + name_list = self.fs.open_mfjson(URLs, preprocess=self._jsCollection) + all_tables = {} + [ + all_tables.update({rtid.split("/")[-3]: (name, desc, rtid)}) + for name, desc, rtid in name_list + ] + all_tables = collections.OrderedDict(sorted(all_tables.items())) + return all_tables + def search(self, txt, where="all"): """Search for string in tables title and/or description @@ -218,37 +217,76 @@ def search(self, txt, where="all"): results.append(tbl_id) return results - @property - def all_tbl(self): - """Return all Argo Reference tables - Returns - ------- - OrderedDict - Dictionary with all table short names as key and table content as class:`pandas.DataFrame` +class ArgoNVSReferenceTables(NVScollection): + """Argo Reference Tables + + Utility function to retrieve Argo Reference Tables from a NVS server. + + By default, this relies on: https://vocab.nerc.ac.uk/collection + + Examples + -------- + Methods: + + >>> R = ArgoNVSReferenceTables() + >>> R.search('sensor') + >>> R.tbl(3) + >>> R.tbl('R09') + + Properties: + + >>> R.all_tbl_name + >>> R.all_tbl + >>> R.valid_ref + + Notes + ----- + This class relies on a list of valid reference table ids that is updated on every argopy release. + + """ + valid_ref = VALID_REF.copy() + + """List of all available Reference Tables""" + + def _valid_ref(self, rtid): """ - URLs = [self.get_url(rtid) for rtid in self.valid_ref] - df_list = self.fs.open_mfjson(URLs, preprocess=self._jsConcept2df) - all_tables = {} - [all_tables.update({t.name: t}) for t in df_list] - all_tables = collections.OrderedDict(sorted(all_tables.items())) - return all_tables + Validate any rtid argument and return the corresponding valid ID from the list. - @property - def all_tbl_name(self): - """Return names of all Argo Reference tables + Parameters + ---------- + rtid: Input reference ID. Can be a string (e.g., "R12", "12", "r12") or a number (e.g., 12). - Returns - ------- - OrderedDict - Dictionary with all table short names as key and table names as tuple('short name', 'description', 'NVS id link') + Returns: + str: Valid reference ID from the list, or None if not found. """ - URLs = [self.get_url(rtid) for rtid in self.valid_ref] - name_list = self.fs.open_mfjson(URLs, preprocess=self._jsCollection) - all_tables = {} - [ - all_tables.update({rtid.split("/")[-3]: (name, desc, rtid)}) - for name, desc, rtid in name_list - ] - all_tables = collections.OrderedDict(sorted(all_tables.items())) - return all_tables + # Convert rtid to a string and standardize its format + if isinstance(rtid, (int, float)): + # If rtid is a number, format it as "RXX" + rtid_str = f"R{int(rtid):02d}" + else: + # If rtid is a string, convert to uppercase and standardize + rtid_str = str(rtid).strip().upper() + if rtid_str.startswith('R') and len(rtid_str) > 1: + # If it starts with 'R', ensure the numeric part is two digits + prefix = rtid_str[0] + suffix = rtid_str[1:] + try: + num = int(suffix) + rtid_str = f"{prefix}{num:02d}" + except ValueError: + pass # Keep the original string if conversion fails + elif ~rtid_str.startswith('R'): + try: + num = int(rtid_str) + rtid_str = f"R{num}" + except ValueError: + pass # Keep the original string if conversion fails + + # Check if the standardized rtid_str is in the valid_refs list + if rtid_str in self.valid_ref: + return rtid_str + else: + raise ValueError( + f"Invalid Argo Reference Table '{rtid}', must be one in: {', '.join(self.valid_ref)}" + ) \ No newline at end of file diff --git a/argopy/related/sensors/__init__.py b/argopy/related/sensors/__init__.py new file mode 100644 index 000000000..a6ffefde7 --- /dev/null +++ b/argopy/related/sensors/__init__.py @@ -0,0 +1,5 @@ +from .sensors import ArgoSensor +from .accessories import SensorType, SensorModel, SensorMetaData +from .oem import OEMSensorMetaData + +__all__ = ["ArgoSensor", "SensorType", "SensorModel", "SensorMetaData", "OEMSensorMetaData"] diff --git a/argopy/related/sensors/accessories.py b/argopy/related/sensors/accessories.py new file mode 100644 index 000000000..23ef3071b --- /dev/null +++ b/argopy/related/sensors/accessories.py @@ -0,0 +1,379 @@ +from typing import Literal, Any +import pandas as pd +import xarray as xr +import numpy as np +from dataclasses import dataclass +from html import escape + +from argopy.options import OPTIONS +from argopy.utils import ppliststr, to_list +from argopy.utils import NVSrow +from argopy.utils.schemas.sensors.spec import Parameter, Sensor, SensorInfo +from argopy.related.sensors.oem.oem_metadata_repr import OemMetaDataDisplay + + +# Define some options expected values as tuples +# (for argument validation) +SearchOutput = ("wmo", "sn", "wmo_sn", "df") +Error = ("raise", "ignore", "silent") +Ds = ("core", "deep", "bgc") + +# Define Literal types using tuples +# (for typing) +SearchOutputOptions = Literal[*SearchOutput] +ErrorOptions = Literal[*Error] +DsOptions = Literal[*Ds] + + +class SensorType(NVSrow): + """One single sensor type data from a R25-"Argo sensor types" row + + .. warning:: + This class is experimental and may change in a future release. + + Examples + -------- + .. code-block:: python + + from argopy import ArgoNVSReferenceTables + + sensor_type = 'CTD' + + df = ArgoNVSReferenceTables().tbl(25) + df_match = df[df["altLabel"].apply(lambda x: x == sensor_type)].iloc[0] + + st = SensorType.from_series(df_match) + + st.name + st.long_name + st.definition + st.deprecated + st.uri + + """ + + reftable = "R25" + + @staticmethod + def from_series(obj: pd.Series) -> "SensorType": + """Create a :class:`SensorType` from a R25-"Argo sensor models" row""" + return SensorType(obj) + + +class SensorModel(NVSrow): + """One single sensor model data from a R27-"Argo sensor models" row + + .. warning:: + This class is experimental and may change in a future release. + + Examples + -------- + .. code-block:: python + + from argopy import ArgoSensor + + sm = ArgoSensor('AANDERAA_OPTODE_4330F').vocabulary + + sm.name + sm.long_name + sm.definition + sm.deprecated + sm.urn + sm.uri + + .. code-block:: python + + from argopy import ArgoNVSReferenceTables + + sensor_model = 'AANDERAA_OPTODE_4330F' + + df = ArgoNVSReferenceTables().tbl(27) + df_match = df[df["altLabel"].apply(lambda x: x == sensor_model)].iloc[0] + + sm = SensorModel.from_series(df_match) + + sm.name + sm.long_name + sm.definition + sm.deprecated + sm.urn + sm.uri + """ + + reftable = "R27" + + @staticmethod + def from_series(obj: pd.Series) -> "SensorModel": + """Create a :class:`SensorModel` from a R27-"Argo sensor models" row""" + return SensorModel(obj) + + def __contains__(self, string) -> bool: + return ( + string.lower() in self.name.lower() + or string.lower() in self.long_name.lower() + ) + + +@dataclass +class SensorModelMetaData: + parameters: list[Parameter] + sensors: list[Sensor] + sensor_info: SensorInfo + instrument_vendorinfo: Any + + +class SensorMetaData: + """A placeholder for float sensors meta-data + + This is a design in active dev. No final specs. + + ..code-block:: python + meta = ArgoFloat(WMO).open_dataset('meta') + md = SensorMetaData(meta) + + md.param2sensor # Dictionary mapping PARAMETER to PARAMETER_SENSOR + md['DOXY'] # Dictionary with all PARAMETER meta-data + """ + + __slots__ = ( + "_obj", + "_param2sensor", + "_wmo", + "_data", + "_parameters", + "_sensors", + "_models", + "sensor_info", + "instrument_vendorinfo", + "sensors", + "parameters", + ) + + def __init__(self, obj): + self._obj: xr.Dataset = obj + self._param2sensor: dict[str, str] = None + self._wmo: int = self._obj["PLATFORM_NUMBER"].item() + self._data: dict[str, Any] = {} + + self._parameters: list[str] = sorted(list(set(self.param2sensor.keys()))) + self._sensors: list[str] = sorted(list(set(self.param2sensor.values()))) + self._models: list[str] = sorted( + set([self.sensor2dict(s)["SENSOR_MODEL"] for s in self._sensors]) + ) + + self.parameters: list[Parameter] = [self.to_schema(p) for p in self._parameters] + self.sensors: list[Sensor] = [self.to_schema(p) for p in self._sensors] + self.sensor_info: SensorInfo = SensorInfo( + **{ + "created_by": self._obj.attrs["institution"], + "date_creation": self._obj.attrs["history"] + .split(";")[0] + .strip() + .split(" ")[0], + "link": "./argo.sensor.schema.json", + "format_version": self._obj.attrs["Conventions"], + "contents": "", + "sensor_described": "", + } + ) + self.instrument_vendorinfo = None + + def __repr__(self): + summary = [f""] + n_model, n_param, n_sensor = ( + len(self._models), + len(self._parameters), + len(self._sensors), + ) + summary.append( + f"> {n_model} sensor models, equiped with {n_sensor} sensor types and providing {n_param} parameters" + ) + summary.append(f"models: {ppliststr(self._models)}") + summary.append(f"sensors: {ppliststr(self._sensors)}") + summary.append(f"parameters: {ppliststr(self._parameters)}") + return "\n".join(summary) + + @property + def param2sensor(self): + """Dictionary mapping PARAMETER to PARAMETER_SENSOR""" + if self._param2sensor is None: + + def get(parameter): + parameter = parameter.strip() + plist = list(self._obj["PARAMETER"].values) + iparams = {} + [iparams.update({param.strip(): plist.index(param)}) for param in plist] + if parameter in iparams.keys(): + # PARAMETER_SENSOR (N_PARAM): name, populated from R25, of the sensor measuring this parameter + val = ( + self._obj["PARAMETER_SENSOR"] + .isel({"N_PARAM": iparams[parameter]}) + .values[np.newaxis][0] + .strip() + ) + return val + return None + + result = {} + for param in self._obj["PARAMETER"]: + p = param.values[np.newaxis][0] + result.update({str(p).strip(): get(p)}) + self._param2sensor = result + return self._param2sensor + + def coef2dict(self, text: str) -> list[dict[str, Any]]: + """Transform a calibration coefficient string into a list of dictionaries""" + try: + result = [] + for coef_grp in text.split(";"): + grp = {} + for coef in coef_grp.split(","): + grp.update( + { + k.strip(): float(v) + for k, v in ( + pair.split("=") for pair in coef.strip().split(",") + ) + } + ) + result.append(grp) + return result + except: + return text.strip() + + def sensor2dict(self, sensor: str) -> dict[str, Any]: + """Extract one sensor data from a metadata xr.dataset""" + sensor = sensor.strip() + slist = list(self._obj["SENSOR"].values) + isensors = {} + [isensors.update({s.strip(): slist.index(s)}) for s in slist] + result = {} + if sensor in isensors.keys(): + for var in self._obj.data_vars: + if self._obj[var].dims == ("N_SENSOR",): + val = ( + self._obj[var] + .isel({"N_SENSOR": isensors[sensor]}) + .values[np.newaxis][0] + .strip() + ) + val = None if val in ["none", b""] else val + result.update({var: val}) + return result + return None + + def param2dict(self, parameter: str) -> dict[str, Any]: + """Extract one parameter data from a metadata xr.dataset""" + parameter = parameter.strip() + plist = list(self._obj["PARAMETER"].values) + iparams = {} + [iparams.update({param.strip(): plist.index(param)}) for param in plist] + result = {} + if parameter in iparams.keys(): + for var in self._obj.data_vars: + if self._obj[var].dims == ("N_PARAM",): + val = ( + self._obj[var] + .isel({"N_PARAM": iparams[parameter]}) + .values[np.newaxis][0] + .strip() + ) + if var == "PARAMETER_SENSOR": + val = self.sensor2dict(val) + if var == "PREDEPLOYMENT_CALIB_COEFFICIENT": + val = self.coef2dict(val) + val = None if val in ["none", b""] else val + result.update({var: val}) + return result + return None + + def to_schema(self, key: str) -> Parameter | Sensor: + """Return parameter or sensor data as a JSON-schema compliant object""" + + def param2sdn(param): + pd = self[param] + spd = pd.copy() + spd["PARAMETER"] = f"SDN:R03::{pd['PARAMETER']}" + spd["PARAMETER_SENSOR"] = f"SDN:R25::{pd['PARAMETER_SENSOR']['SENSOR']}" + spd["PREDEPLOYMENT_CALIB_COEFFICIENT_LIST"] = spd[ + "PREDEPLOYMENT_CALIB_COEFFICIENT" + ] + spd.pop("PREDEPLOYMENT_CALIB_COEFFICIENT") + spd["PREDEPLOYMENT_CALIB_DATE"] = "none" + return Parameter(**spd) + + def sensor2sdn(sensor): + pd = self[sensor] + spd = pd.copy() + spd["SENSOR"] = f"SDN:R25::{pd['SENSOR']}" + spd["SENSOR_MAKER"] = f"SDN:R26::{pd['SENSOR_MAKER']}" + spd["SENSOR_MODEL"] = f"SDN:R27::{pd['SENSOR_MODEL']}" + return Sensor(**spd) + + if key in self.param2sensor.keys(): + return param2sdn(key) + else: + return sensor2sdn(key) + + def __getitem__(self, key: str) -> dict[str, Any]: + """Get parameter or sensor data as dictionary""" + if self._data.get(key, None) is None: + if key in self.param2sensor.keys(): + self._data.update({key: self.param2dict(key)}) + elif key in self.param2sensor.values(): + self._data.update({key: self.sensor2dict(key)}) + else: + raise ValueError(f"Unknown parameter or sensor '{key}'") + return self._data[key] + + @property + def Models(self): + # Models = {'': {'sensors': [], 'parameters': []}} + Models = {} + for model in self._models: + # List of sensor types: + Models.update({model: {"sensors": [], "parameters": []}}) + + for sensor in self._sensors: + model = self[sensor]["SENSOR_MODEL"] + if self.to_schema(sensor) not in Models[model]["sensors"]: + Models[model]["sensors"].append(self.to_schema(sensor)) + + for parameter in self._parameters: + model = self[parameter]["PARAMETER_SENSOR"]["SENSOR_MODEL"] + if self.to_schema(parameter) not in Models[model]["parameters"]: + Models[model]["parameters"].append(self.to_schema(parameter)) + + return Models + + def display_model(self, model: str | None = None): + if model not in self.Models.keys(): + raise ValueError(f"Model name must be one in {ppliststr(self.Models.keys())}") + + this_model = SensorModelMetaData( + **{ + "sensors": self.Models[model]["sensors"], + "parameters": self.Models[model]["parameters"], + "sensor_info": SensorInfo( + **{ + "created_by": self._obj.attrs["institution"], + "date_creation": self._obj.attrs["history"] + .split(";")[0] + .strip() + .split(" ")[0], + "link": "./argo.sensor.schema.json", + "format_version": self._obj.attrs["Conventions"], + "contents": "", + "sensor_described": model, + } + ), + "instrument_vendorinfo": "", + } + ) + oem_like = OemMetaDataDisplay(this_model) + + if OPTIONS["display_style"] == "text": + return f"
{escape(repr(oem_like))}
" + + from IPython.display import display, HTML + display(HTML(oem_like.html)) diff --git a/argopy/related/sensors/oem/__init__.py b/argopy/related/sensors/oem/__init__.py new file mode 100644 index 000000000..c64dd8fe1 --- /dev/null +++ b/argopy/related/sensors/oem/__init__.py @@ -0,0 +1,3 @@ +from .oem_metadata import OEMSensorMetaData + +__all__ = ['OEMSensorMetaData'] diff --git a/argopy/related/sensors/oem/accessories.py b/argopy/related/sensors/oem/accessories.py new file mode 100644 index 000000000..e69de29bb diff --git a/argopy/related/sensors/oem/oem_metadata.py b/argopy/related/sensors/oem/oem_metadata.py new file mode 100644 index 000000000..eba870146 --- /dev/null +++ b/argopy/related/sensors/oem/oem_metadata.py @@ -0,0 +1,468 @@ +from dataclasses import field +from typing import List, Dict, Optional, Any, Literal, Self +from pathlib import Path +from zipfile import ZipFile + +import json +import logging +import warnings +import pandas as pd +from html import escape + +from argopy.stores import httpstore, filestore +from argopy.options import OPTIONS +from argopy.utils import urnparser, path2assets +from argopy.utils.schemas.sensors.spec import SensorInfo, Context, Sensor, Parameter +from argopy.errors import InvalidDatasetStructure + +from argopy.related.sensors.utils import has_jsonschema +from argopy.related.sensors.oem.oem_metadata_repr import OemMetaDataDisplay + + +if has_jsonschema: + from referencing import Registry, Resource + import jsonschema + + +log = logging.getLogger("argopy.related.sensors.oem") + +SENSOR_JS_EXAMPLES = filestore().open_json( + Path(path2assets).joinpath("sensor_metadata_examples.json") +)["data"]["uri"] + + +class OEMSensorMetaData: + """OEM sensor meta-data + + A class helper to work with sensor meta-data complying to schema from https://github.com/euroargodev/sensor_metadata_json + + Such meta-data structures are expected to come from sensor manufacturer (web-api or file). + + .. note:: + + OEM : Original Equipment Manufacturer + + Examples + -------- + .. code-block:: python + + OEMSensorMetaData() + + OEMSensorMetaData(validate=True) # Use this option to run json schema validation compliance when necessary + + OEMSensorMetaData().from_rbr(208380) # Direct call to RBR api with a serial number + + OEMSensorMetaData().from_seabird(2444, 'SATLANTIC_OCR504_ICSW') # Direct call to Seabird api with a serial number and model name + + OEMSensorMetaData().list_examples + + OEMSensorMetaData().from_examples('WETLABS-ECO_FLBBAP2-8589') + + OEMSensorMetaData().from_dict(jsdata) # Use any compliant json data + + """ + + _schema_root = "https://raw.githubusercontent.com/euroargodev/sensor_metadata_json/refs/heads/main/schemas" + """URI root to argo JSON schema""" + + def __init__( + self, + json_data: Optional[Dict[str, Any]] = None, + validate: bool = True, + validation_error: Literal["warn", "raise", "ignore"] = "warn", + **kwargs, + ): + if kwargs.get("fs", None) is not None: + self._fs = kwargs.get("fs") + else: + self._cache = kwargs.get("cache", True) + self._cachedir = kwargs.get("cachedir", OPTIONS["cachedir"]) + self._timeout = kwargs.get("timeout", OPTIONS["api_timeout"]) + fs_kargs = { + "cache": self._cache, + "cachedir": self._cachedir, + "timeout": self._timeout, + } + self._fs = httpstore(**fs_kargs) + + if has_jsonschema: + self._run_validation = validate + else: + warnings.warn(f"Cannot run JSON validation without the 'jsonschema' library. Please install it manually. Fall back on setting `validate=False`. ") + self._run_validation = False + + self._validation_error = validation_error + self.schema = self._read_schema() # requires a self._fs instance + + self.sensor_info: Optional[SensorInfo] = None + self.context: Optional[Context] = None + self.sensors: List[Sensor] = field(default_factory=list) + self.parameters: List[Parameter] = field(default_factory=list) + self.instrument_vendorinfo: Optional[Dict[str, Any]] = None + self._serial_number = None + self._local_certificates = None + + if json_data: + self.from_dict(json_data) + + def _repr_hint(self): + summary = [f""] + summary.append( + "This object has no sensor info. You can use one of the following methods:" + ) + for meth in [ + "from_rbr(serial_number)", + "from_seabird(serial_number, model_name)", + "from_dict(dict_or_json_data)", + ]: + summary.append(f" ╰┈➤ OEMSensorMetaData().{meth}") + return summary + + def __repr__(self): + if self.sensor_info: + + sensor_described = self.sensor_info._attr2str('sensor_described') + created_by = self.sensor_info._attr2str('created_by') + date_creation = self.sensor_info._attr2str('date_creation') + link = self.sensor_info._attr2str('link') + + sensor_count = len(self.sensors) + parameter_count = len(self.parameters) + + summary = [f"<{sensor_described}>"] + summary.append(f"created_by: '{created_by}'") + summary.append(f"date_creation: '{date_creation}'") + summary.append(f"link: '{link}'") + summary.append( + f"sensors: {sensor_count} {[urnparser(s.SENSOR)['termid'] for s in self.sensors]}" + ) + summary.append( + f"parameters: {parameter_count} {[urnparser(s.PARAMETER)['termid'] for s in self.parameters]}" + ) + summary.append(f"instrument_vendorinfo: {self.instrument_vendorinfo}") + + else: + summary = self._repr_hint() + + return "\n".join(summary) + + def _repr_html_(self): + if OPTIONS["display_style"] == "text": + return f"
{escape(repr(self))}
" + return OemMetaDataDisplay(self).html + + # def _ipython_display_(self): + # from IPython.display import display, HTML + # + # if self.sensor_info: + # display(HTML(OemMetaDataDisplay(self).html)) + # else: + # display("\n".join(self._repr_hint())) + + def _read_schema(self, ref="argo.sensor.schema.json") -> Dict[str, Any]: + """Load a JSON schema for validation + + Fall back on static version if online resource not available + """ + uri = f"{self._schema_root}/{ref}" + try: + schema = self._fs.open_json(uri) + except: + # Fall back on static assets version + local_uri = Path(path2assets).joinpath("schema").joinpath(ref) + fs = filestore() + updated = pd.Timestamp(fs.info(local_uri)["mtime"], unit="s") + warnings.warn( + f"\nCan't get '{ref}' schema from the official online resource ({uri}).\nFall back on a static version packaged with this release at {updated}." + ) + schema = fs.open_json(local_uri) + + return schema + + def validate(self, data): + """Validate meta-data against the Argo sensor json schema""" + # Set a method to resolve references to subschemas + registry = Registry( + retrieve=lambda x: Resource.from_contents(self._read_schema(x)) + ) + + # Select the validator based on $schema property in schema + # (validators correspond to various drafts of JSON Schema) + validator = jsonschema.validators.validator_for(self.schema) + + # Create the validator using the registry and associated resolver + v = validator(self.schema, registry=registry) + + try: + v.validate(data) + except Exception as error: + if self._validation_error == "raise": + raise error + elif self._validation_error == "warn": + warnings.warn(f"\nJSON schema validation error: {str(error)}") + else: + log.error(error) + + # Create a list of errors, if any + errors = list(v.evolve(schema=self.schema).iter_errors(v, data)) + return errors + + def from_dict(self, data: Dict[str, Any]) -> Self: + """Load data from a dictionary and possibly validate""" + if self._run_validation: + self.validate(data) + + self.sensor_info = SensorInfo(**data["sensor_info"]) + if data.get("@context", None) is not None: + self.context = Context( + **{ + k.replace("::", "").replace(":", "_"): v + for k, v in data["@context"].items() + } + ) + else: + self.context = Context() + self.sensors = [Sensor(**sensor) for sensor in data["SENSORS"]] + self.parameters = [Parameter(**param) for param in data["PARAMETERS"]] + self.instrument_vendorinfo = data.get("instrument_vendorinfo", None) + + return self + + def to_dict(self) -> Dict[str, Any]: + """Convert the object back to a dictionary, following schema""" + return { + "sensor_info": { + "created_by": self.sensor_info.created_by, + "date_creation": self.sensor_info.date_creation, + "link": self.sensor_info.link, + "format_version": self.sensor_info.format_version, + "contents": self.sensor_info.contents, + "sensor_described": self.sensor_info.sensor_described, + }, + "@context": { + "SDN:R03::": self.context.SDN_R03, + "SDN:R25::": self.context.SDN_R25, + "SDN:R26::": self.context.SDN_R26, + "SDN:R27::": self.context.SDN_R27, + "SDN:L22::": self.context.SDN_L22, + }, + "SENSORS": [ + { + "SENSOR": sensor.SENSOR, + "SENSOR_MAKER": sensor.SENSOR_MAKER, + "SENSOR_MODEL": sensor.SENSOR_MODEL, + "SENSOR_MODEL_FIRMWARE": getattr( + sensor, "SENSOR_MODEL_FIRMWARE", sensor.SENSOR_FIRMWARE_VERSION + ), + "SENSOR_SERIAL_NO": sensor.SENSOR_SERIAL_NO, + "sensor_vendorinfo": sensor.sensor_vendorinfo, + } + for sensor in self.sensors + ], + "PARAMETERS": [ + { + "PARAMETER": param.PARAMETER, + "PARAMETER_SENSOR": param.PARAMETER_SENSOR, + "PARAMETER_UNITS": param.PARAMETER_UNITS, + "PARAMETER_ACCURACY": param.PARAMETER_ACCURACY, + "PARAMETER_RESOLUTION": param.PARAMETER_RESOLUTION, + "PREDEPLOYMENT_CALIB_EQUATION": param.PREDEPLOYMENT_CALIB_EQUATION, + "PREDEPLOYMENT_CALIB_COEFFICIENT_LIST": param.PREDEPLOYMENT_CALIB_COEFFICIENT_LIST, + "PREDEPLOYMENT_CALIB_COMMENT": param.PREDEPLOYMENT_CALIB_COMMENT, + "PREDEPLOYMENT_CALIB_DATE": param.PREDEPLOYMENT_CALIB_DATE, + "parameter_vendorinfo": param.parameter_vendorinfo, + "predeployment_vendorinfo": param.predeployment_vendorinfo, + } + for param in self.parameters + ], + "instrument_vendorinfo": self.instrument_vendorinfo, + } + + def to_json_file(self, file_path: str) -> None: + """Save meta-data to a JSON file - in dev. + + Notes + ----- + The output json file should be compliant with the Argo sensor meta-data JSON schema :attr:`OEMSensorMetaData.schema` + """ + with open(file_path, "w") as f: + json.dump(self.to_dict(), f, indent=2) + + def from_rbr(self, serial_number: str, **kwargs) -> Self: + """Fetch sensor metadata from "RBRargo Product Lookup" web-API + + We also download certificates if available + + TODO: Check mark if the sensor is ok with dynamic correction or not + + Parameters + ---------- + serial_number : str + Sensor serial number from RBR + + Notes + ----- + The instance :class:`httpstore` is automatically updated to use the ``rbr_api_key`` option. + + The "RBRargo Product Lookup" web-API location is set with: ``OPTIONS['rbr_api']`` + """ + self._serial_number = serial_number + + # Ensure that the instance httpstore has the appropriate authorization key: + fss = self._fs.fs.fs if getattr(self._fs, "cache") else self._fs.fs + headers = fss.client_kwargs.get("headers", {}) + headers.update( + {"Authorization": kwargs.get("rbr_api_key", OPTIONS["rbr_api_key"])} + ) + fss._session = None # Reset fsspec aiohttp.ClientSession + + uri = f"{OPTIONS['rbr_api']}/instruments/{self._serial_number}/argometadatajson" + self._data = self._fs.open_json(uri) + obj = self.from_dict(self._data) + + # Also download RBR zip archive with calibration certificates in PDFs: + obj = obj._certificates_rbr(action="download", quiet=True) + + # Finally reset httpstore parameters: + headers = fss.client_kwargs.get("headers") + headers.pop("Authorization", None) + fss._session = None # Reset fsspec aiohttp.ClientSession + + return obj + + def _certificates_rbr( + self, action: Literal["download", "open"] = "download", **kwargs + ): + """Download RBR zip archive with calibration certificates in PDFs + + Certificate PDF files are written to the OPTIONS['cachedir'] folder + + Notes + ----- + We keep this method private because it is expected to be called only by the self.from_rbr() method. + This ensures that the httpstore has the appropriate authorization key. + + """ + cdir = Path(OPTIONS["cachedir"]).joinpath("RBR_certificates") + cdir.mkdir(parents=True, exist_ok=True) + local_zip_path = cdir.joinpath(f"RBRcertificates_{self._serial_number}.zip") + lfs = filestore() + quiet = kwargs.get("quiet", False) + + # Check if we can continue: + if self._serial_number is not None: + new = False + + # Trigger download if necessary: + if not lfs.exists(local_zip_path): + new = True + certif_uri = f"{OPTIONS['rbr_api']}/instruments/{self._serial_number}/certificates" + with open(local_zip_path, "wb") as local_zip: + with self._fs.open(certif_uri) as remote_zip: + local_zip.write(remote_zip.read()) + + # Expand locally: + with ZipFile(local_zip_path, "r") as local_zip: + local_zip.testzip() + local_zip.extractall(cdir) + + # List PDF certificates: + with ZipFile(local_zip_path, "r") as local_zip: + local_zip.testzip() + info = local_zip.infolist() + certificates = [] + for doc in info: + certificates.append(Path(cdir).joinpath(doc.filename)) + self.local_certificates = certificates + + if not quiet: + for f in self.local_certificates: + if new: + s = f"One RBR certificate file written to: {f}" + else: + s = f"One RBR certificate file already in: {f}" + print(s) + else: + raise InvalidDatasetStructure( + f"You must load meta-data for a given RBR sensor serial number first. Use the 'from_rbr' method." + ) + + if action == "download": + return self + elif action == "open": + subp = [] + for f in self.local_certificates: + subp.append(lfs.open_subprocess(str(f))) + if not quiet: + return subp + else: + raise ValueError(f"Unknown action {action}") + + def from_seabird(self, serial_number: str, sensor_model: str, **kwargs) -> Self: + """Fetch sensor metadata from Seabird-Scientific "Instrument Metadata Portal" web-API + + Parameters + ---------- + serial_number : str + Sensor serial number from RBR + + Notes + ----- + The "Instrument Metadata Portal" web-API location is set with: ``OPTIONS['seabird_api']`` + + .. warning:: + + Since the Seabird web-API does not yet return fully compliant json medatata, we internally fix some format errors to be able to use this source of information. In particular: + + - 'sensor_info' comes from the wrongly named 'json_info' attribute, + - If 'PARAMETER_UNITS' is empty, try to get value from the 'parameter_vendorinfo'/'units' attribute, + - If 'PREDEPLOYMENT_CALIB_COMMENT' is empty, try to get value from the 'parameter_vendorinfo'/'comments' attribute, + - If 'PREDEPLOYMENT_CALIB_DATE' is empty, try to get value from the 'parameter_vendorinfo'/'calibration_date' attribute. + + """ + # The Seabird api requires a sensor model (R27) with a serial number. + # This could easily be done if we get the s/n by searching float metadata. + # In other word, it's easy to go from a sensor_model to a serial_number. + # But it is much more complicated to go from a serial_number to a sensor_model. + # so, for the time being, we'll ask users to specify a sensor_model. + + url = f"{OPTIONS['seabird_api']}?SENSOR_SERIAL_NO={serial_number}&SENSOR_MODEL={sensor_model}" + data = self._fs.open_json(url) + + # Temporary fix for errors in SBE api output: + data.update({'sensor_info': data['json_info']}) + data['sensor_info'].update({'link': data['sensor_info']['created_by']}) + data['sensor_info'].update({'created_by': 'Sea-Bird Instrument Metadata Portal'}) + data['sensor_info'].update({'sensor_described': f"{sensor_model}-{serial_number}"}) + data.pop('json_info', None) + + # Check in vendor info for missing mandatory parameter attributes: + for ii, param in enumerate(data['PARAMETERS']): + vendorinfo = param['parameter_vendorinfo'] + if param.get('PARAMETER_UNITS', None) is None: + if 'units' in vendorinfo: + data['PARAMETERS'][ii]['PARAMETER_UNITS'] = vendorinfo['units'] + if param.get('PREDEPLOYMENT_CALIB_COMMENT', None) is None: + if 'units' in vendorinfo: + data['PARAMETERS'][ii]['PREDEPLOYMENT_CALIB_COMMENT'] = vendorinfo['comments'] + if param.get('PREDEPLOYMENT_CALIB_DATE', None) is None: + if 'units' in vendorinfo: + data['PARAMETERS'][ii]['PREDEPLOYMENT_CALIB_DATE'] = vendorinfo['calibration_date'] + + # Create and return an instance + self._data = data + obj = self.from_dict(self._data) + return obj + + @property + def list_examples(self): + """List of example names""" + return [k for k in SENSOR_JS_EXAMPLES.keys()] + + def from_examples(self, eg: str = None, **kwargs) -> Self: + if eg not in self.list_examples: + raise ValueError( + f"Unknown sensor example: '{eg}'. \n Use one in: {self.list_examples}" + ) + data = self._fs.open_json(SENSOR_JS_EXAMPLES[eg]) + return self.from_dict(data) diff --git a/argopy/related/sensors/oem/oem_metadata_repr.py b/argopy/related/sensors/oem/oem_metadata_repr.py new file mode 100644 index 000000000..b856efc8b --- /dev/null +++ b/argopy/related/sensors/oem/oem_metadata_repr.py @@ -0,0 +1,202 @@ +from functools import lru_cache +import importlib +from numpy.random import randint + +try: + from importlib.resources import files # New in version 3.9 +except ImportError: + from pathlib import Path + + files = lambda x: Path( # noqa: E731 + importlib.util.find_spec(x).submodule_search_locations[0] + ) + +from argopy.utils import urnparser + + +STATIC_FILES = ( + ("argopy.static.css", "argopy.css"), + ("argopy.static.css", "oemsensor.css"), +) + +@lru_cache(None) +def _load_static_files(): + """Lazily load the resource files into memory the first time they are needed""" + return [ + files(package).joinpath(resource).read_text(encoding="utf-8") + for package, resource in STATIC_FILES + ] + + +def urn_html(this_urn): + x = urnparser(this_urn) + if x.get('version') != "": + return f"{x.get('termid', '?')} ({x.get('listid', '?')}, {x.get('version', 'n/a')})" + else: + return f"{x.get('termid', '?')} ({x.get('listid', '?')})" + + + +class OemMetaDataDisplay: + + def __init__(self, obj): + self.OEMsensor = obj + + @property + def css_style(self): + return "\n".join(_load_static_files()) + + @property + def html(self): + # Generate a dummy random id to be used in html elements for this output only + # This avoids messing up javascript actions between notebook cells + uid = f"{randint(low=1e7):8d}".strip() + + # --- Header --- + header_html = f""" +

Argo Sensor Metadata: {self.OEMsensor.sensor_info._attr2str('sensor_described')}

+

Created by: {self.OEMsensor.sensor_info._attr2str('created_by')} | + Date: {self.OEMsensor.sensor_info._attr2str('date_creation')}

+ """ + + # --- Sensors --- + sensors_html = """ +

List of sensors:

+ + + + + + + + + + + + """ + + for ii, sensor in enumerate(self.OEMsensor.sensors): + + if getattr(sensor, "SENSOR_MODEL_FIRMWARE", None) is None: + firmware = f"{sensor._attr2str('SENSOR_FIRMWARE_VERSION')}" + else: + firmware = f"{sensor._attr2str('SENSOR_MODEL_FIRMWARE')}" + + sensors_html += f""" + + + + + + + + """ + + sensors_html += "
SensorMakerModelFirmwareSerial No
{urn_html(sensor.SENSOR)}{urn_html(sensor.SENSOR_MAKER)}{urn_html(sensor.SENSOR_MODEL)}{firmware}{sensor._attr2str('SENSOR_SERIAL_NO')}
" + + # --- Parameters --- + parameters_html = """ +

List of parameters:

+ + + + + + + + + + + + + """ + + for ip, param in enumerate(self.OEMsensor.parameters): + if getattr(param, 'PREDEPLOYMENT_CALIB_EQUATION', None) is not None: + PREDEPLOYMENT_CALIB_EQUATION = param._attr2str('PREDEPLOYMENT_CALIB_EQUATION').split(';') + PREDEPLOYMENT_CALIB_EQUATION = [p.replace("=", " = ") for p in PREDEPLOYMENT_CALIB_EQUATION] + PREDEPLOYMENT_CALIB_EQUATION = "
\t".join(PREDEPLOYMENT_CALIB_EQUATION) + else: + PREDEPLOYMENT_CALIB_EQUATION = "This information is missing, but it should not !" + + if getattr(param, 'PREDEPLOYMENT_CALIB_COEFFICIENT_LIST', None) is not None: + PREDEPLOYMENT_CALIB_COEFFICIENT_LIST = param._attr2str('PREDEPLOYMENT_CALIB_COEFFICIENT_LIST') + s = [] + if isinstance(PREDEPLOYMENT_CALIB_COEFFICIENT_LIST, dict): + for key, value in PREDEPLOYMENT_CALIB_COEFFICIENT_LIST.items(): + s.append(f"{key} = {value}") + PREDEPLOYMENT_CALIB_COEFFICIENT_LIST = "
\t".join(s) + else: + PREDEPLOYMENT_CALIB_COEFFICIENT_LIST = "This information is missing, but it should not !" + + + details_html = f""" + + + + """ + + # param.PREDEPLOYMENT_CALIB_EQUATION + if param._has_calibration_data: + line = '' % (uid, ip) + else: + line = f"" + + parameters_html += f""" + + + + + + + {line} + + {details_html} + """ + + parameters_html += "
ParameterSensorUnitsAccuracyResolutionCalibration details
+
+

Calibration Equation:
{PREDEPLOYMENT_CALIB_EQUATION}

+

Calibration Coefficients:
{PREDEPLOYMENT_CALIB_COEFFICIENT_LIST}

+

Calibration Comment:
{param._attr2str('PREDEPLOYMENT_CALIB_COMMENT')}

+

Calibration Date:
{param._attr2str('PREDEPLOYMENT_CALIB_DATE')}

+
+
Click for moren/a
{urn_html(param.PARAMETER)}{urn_html(param.PARAMETER_SENSOR)}{param._attr2str('PARAMETER_UNITS')}{param._attr2str('PARAMETER_ACCURACY')}{param._attr2str('PARAMETER_RESOLUTION')}
" + + # --- JavaScript for Toggle --- + js = """ + + """ + + # --- Vendor Info --- + vendor_html = "" + if self.OEMsensor.instrument_vendorinfo: + vendor_html = f""" +

Instrument Vendor Info:

+ 
{self.OEMsensor.instrument_vendorinfo}
+ """ + + # --- Combine All HTML --- + full_html = f""" + \n + {header_html}\n + {sensors_html}\n + {parameters_html}\n + {js}\n + {vendor_html} + """ + return full_html + + def _repr_html_(self): + return self.html + + diff --git a/argopy/related/sensors/references.py b/argopy/related/sensors/references.py new file mode 100644 index 000000000..c6240cca1 --- /dev/null +++ b/argopy/related/sensors/references.py @@ -0,0 +1,378 @@ +import pandas as pd +from pathlib import Path +from typing import Literal, NoReturn +from abc import ABC, abstractmethod +import logging +import fnmatch + +from argopy.options import OPTIONS +from argopy.stores import httpstore, filestore +from argopy.related import ArgoNVSReferenceTables +from argopy.utils import to_list, path2assets, register_accessor, ppliststr +from argopy.errors import DataNotFound, OptionValueError + +from argopy.related.sensors.accessories import SensorType, SensorModel +from argopy.related.sensors.accessories import Error, ErrorOptions + + +log = logging.getLogger("argopy.related.sensors.ref") + + +class SensorReferenceHolder(ABC): + """Parent class to hold R25, R26, R27 and R27_to_R25 mapping data""" + + _r25: pd.DataFrame | None = None + """NVS Reference table for Argo sensor types (R25)""" + + _r26: pd.DataFrame | None = None + """NVS Reference table for Argo sensor maker (R26)""" + + _r27: pd.DataFrame | None = None + """NVS Reference table for Argo sensor models (R27)""" + + _r27_to_r25: dict[str, str] | None = None + """Dictionary mapping of R27 to R25""" + + def __call__(self, *args, **kwargs) -> NoReturn: + raise ValueError("A SensorReference instance cannot be called directly.") + + def __init__(self, obj): + self._obj = obj # An instance of SensorReferences, possibly with a filesystem + if getattr(obj, "_fs", None) is None: + self._fs = httpstore( + cache=True, + cachedir=OPTIONS["cachedir"], + timeout=OPTIONS["api_timeout"], + ) + else: + self._fs = obj._fs + + @property + def r25(self): + """NVS Reference table for Argo sensor types (R25)""" + if self._r25 is None: + self._r25 = ArgoNVSReferenceTables(fs=self._fs).tbl("R25") + return self._r25 + + @property + def r26(self): + """NVS Reference table for Argo sensor maker (R26)""" + if self._r26 is None: + self._r26 = ArgoNVSReferenceTables(fs=self._fs).tbl("R26") + return self._r26 + + @property + def r27(self): + """NVS Reference table for Argo sensor models (R27)""" + if self._r27 is None: + self._r27 = ArgoNVSReferenceTables(fs=self._fs).tbl("R27") + return self._r27 + + def _load_mappers(self): + """Load from static assets file the NVS R25 to R27 key mappings + + These mapping files were download from https://github.com/OneArgo/ArgoVocabs/issues/156. + """ + df = [] + for p in ( + Path(path2assets).joinpath("nvs_R25_R27").glob("NVS_R25_R27_mappings_*.txt") + ): + df.append( + filestore().read_csv( + p, + header=None, + names=["origin", "model", "?", "destination", "type", "??"], + ) + ) + df = pd.concat(df) + for col in ['origin', 'destination', 'type', 'model', '?', '??']: + df[col] = df[col].apply(lambda x: x.strip()) + df = df.reset_index(drop=True) + self._r27_to_r25 : pd.DataFrame | None = df + + @property + def r27_to_r25(self) -> pd.DataFrame: + """Dictionary mapping of R27 to R25""" + if self._r27_to_r25 is None: + self._load_mappers() + return self._r27_to_r25 + + @abstractmethod + def to_dataframe(self): + raise NotImplementedError + + @abstractmethod + def hint(self): + raise NotImplementedError + + +class SensorReferenceR27(SensorReferenceHolder): + """Argo sensor models""" + + def to_dataframe(self) -> pd.DataFrame: + """Reference Table **Sensor Models (R27)** as a :class:`pandas.DataFrame` + + Returns + ------- + :class:`pandas.DataFrame` + + See Also + -------- + :class:`ArgoNVSReferenceTables` + """ + return self.r27 + + def hint(self) -> list[str]: + """List of Argo sensor models + + Return a sorted list of strings with altLabel from Argo Reference table R27 on 'SENSOR_MODEL'. + + Returns + ------- + list[str] + + Notes + ----- + Argo netCDF variable ``SENSOR_MODEL`` is populated with values from this list. + """ + return sorted(to_list(self.r27["altLabel"].values)) + + def to_type( + self, + model: str | SensorModel, + errors: ErrorOptions = "raise", + obj: bool = False, + ) -> list[str] | list[SensorType] | None: + """Get all sensor types of a given sensor model + + All valid sensor model names can be obtained with :meth:`ArgoSensor.ref.model.hint`. + + Mapping between sensor model name (R27) and sensor type (R25) are from AVTT work at https://github.com/OneArgo/ArgoVocabs/issues/156. + + Parameters + ---------- + model : str | :class:`argopy.related.SensorModel` + The sensor model to read the sensor type for. + errors : Literal["raise", "ignore", "silent"], optional, default: "raise" + How to handle possible errors. If set to "ignore", the method may return None. + obj: bool, optional, default: False + Return a list of strings (False) or a list of :class:`argopy.related.SensorType` + + Returns + ------- + list[str] | list[:class:`argopy.related.SensorType`] | None + + Raises + ------ + :class:`DataNotFound` + """ + if errors not in Error: + raise OptionValueError( + f"Invalid 'errors' option value '{errors}', must be in: {ppliststr(Error, last='or')}" + ) + model_name: str = model.name if isinstance(model, SensorModel) else model + + match = fnmatch.filter(self.r27_to_r25["model"], model_name.upper()) + types = self.r27_to_r25[self.r27_to_r25["model"].apply(lambda x: x in match)]['type'].tolist() + + if len(types) > 0: + # Since 'types' comes from the mapping, we double-check values against the R25 entries: + types = [self.r25[self.r25["altLabel"].apply(lambda x: x == this_type)]['altLabel'].item() for this_type in types] + if not obj: + return types + else: + rows = [self.r25[self.r25["altLabel"].apply(lambda x: x == this_type)].iloc[0] for this_type in types] + return [SensorType.from_series(row) for row in rows] + + elif errors == "raise": + raise DataNotFound( + f"Can't determine the type of sensor model '{model_name}' (no matching key in r27_to_r25 mapper)" + ) + elif errors == "silent": + log.error( + f"Can't determine the type of sensor model '{model_name}' (no matching key in r27_to_r25 mapper)" + ) + return None + + def search( + self, + model: str, + output: Literal["df", "name"] = "df", + ) -> pd.DataFrame | list[str]: + """Return Argo sensor model references matching a string + + Look for occurrences in Argo Reference table R27 `altLabel` and return a subset of the :class:`pandas.DataFrame` with matching row(s). + + Parameters + ---------- + model : str + The model to search for. You can use wildcards: "SBE41CP*" "*DEEP*", "RBR*", or an exact name like "RBR_ARGO3_DEEP6". + output : str, Literal["df", "name"], default "df" + Is the output a :class:`pandas.DataFrame` with matching rows from R27, or a list of string. + + Returns + ------- + :class:`pandas.DataFrame`, list[str] + + Raises + ------ + :class:`DataNotFound` + """ + match = fnmatch.filter(self.r27["altLabel"], model.upper()) + data = self.r27[self.r27["altLabel"].apply(lambda x: x in match)] + + if data.shape[0] == 0: + raise DataNotFound( + f"'{model}' is not a valid sensor model name. You can use wildcard for search, e.g. 'SBE61*'." + ) + else: + if output == "name": + return sorted(to_list(data["altLabel"].values)) + else: + return data.reset_index(drop=True) + + +class SensorReferenceR25(SensorReferenceHolder): + """Argo sensor types""" + + def to_dataframe(self) -> pd.DataFrame: + """Reference Table **Sensor Types (R25)** as a :class:`pandas.DataFrame` + + Returns + ------- + :class:`pandas.DataFrame` + """ + return self.r25 + + def hint(self) -> list[str]: + """List of Argo sensor types + + Return a sorted list of strings with altLabel from Argo Reference table R25 on 'SENSOR'. + + Returns + ------- + list[str] + + Notes + ----- + Argo netCDF variable ``SENSOR`` is populated with values from this list. + """ + return sorted(to_list(self.r25["altLabel"].values)) + + def to_model( + self, + type: str | SensorType, + errors: Literal["raise", "ignore"] = "raise", + obj: bool = False, + ) -> list[str] | list[SensorModel] | None: + """Get all sensor model names of a given sensor type + + All valid sensor types can be obtained with :meth:`ArgoSensor.ref.sensor.hint` + + Mapping between sensor model name (R27) and sensor type (R25) are from AVTT work at https://github.com/OneArgo/ArgoVocabs/issues/156. + + Parameters + ---------- + type : str, :class:`argopy.related.SensorType` + The sensor type to read the sensor model name for. + errors : Literal["raise", "ignore"] = "raise" + How to handle possible errors. If set to "ignore", the method will return None. + obj: bool, optional, default: False + Return a list of strings (False) or a list of :class:`argopy.related.SensorModel` + + Returns + ------- + list[str] | list[:class:`argopy.related.SensorModel`] | None + + Raises + ------ + :class:`DataNotFound` + """ + sensor_type = type.name if isinstance(type, SensorType) else type + result = [] + + match = fnmatch.filter(self.r27_to_r25["type"], sensor_type.upper()) + models = self.r27_to_r25[self.r27_to_r25["type"].apply(lambda x: x in match)]['model'].tolist() + + if len(models) > 0: + # Since 'models' comes from the mapping, we double-check values against the R27 entries: + models = [self.r27[self.r27["altLabel"].apply(lambda x: x == model)]['altLabel'].item() for + model in models] + if not obj: + return models + else: + rows = [self.r27[self.r27["altLabel"].apply(lambda x: x == model)].iloc[0] for + model in models] + return [SensorModel.from_series(row) for row in rows] + + if len(result) == 0: + if errors == "raise": + raise DataNotFound( + f"Can't find any sensor model for this type '{sensor_type}' (no matching key in r27_to_r25 mapper)" + ) + else: + return None + else: + return result + + +class SensorReferenceR26(SensorReferenceHolder): + """Argo sensor maker""" + + def to_dataframe(self) -> pd.DataFrame: + """Reference Table **Sensor Makers (R26)** as a :class:`pandas.DataFrame` + + Returns + ------- + :class:`pandas.DataFrame` + """ + return self.r26 + + def hint(self) -> list[str]: + """List of Argo sensor makers + + Return a sorted list of strings with altLabel from Argo Reference table R26 on 'SENSOR_MAKER'. + + Returns + ------- + list[str] + + Notes + ----- + Argo netCDF variable ``SENSOR_MAKER`` is populated with values from this list. + """ + return sorted(to_list(self.r26["altLabel"].values)) + + +class SensorReferences: + + __slots__ = ["_obj", "_fs"] + + def __init__(self, obj): + self._obj = obj # An instance of ArgoSensor, possibly with a filesystem + if getattr(obj, "_fs", None) is None: + self._fs = httpstore( + cache=True, + cachedir=OPTIONS["cachedir"], + timeout=OPTIONS["api_timeout"], + ) + else: + self._fs = obj._fs + + def __call__(self, *args, **kwargs) -> NoReturn: + raise ValueError("ArgoSensor.ref cannot be called directly.") + + +@register_accessor("type", SensorReferences) +class SensorExtension(SensorReferenceR25): + _name = "ref.type" + + +@register_accessor("maker", SensorReferences) +class MakerExtension(SensorReferenceR26): + _name = "ref.maker" + + +@register_accessor("model", SensorReferences) +class ModelExtension(SensorReferenceR27): + _name = "ref.model" diff --git a/argopy/related/sensors/sensors.py b/argopy/related/sensors/sensors.py new file mode 100644 index 000000000..863ba241f --- /dev/null +++ b/argopy/related/sensors/sensors.py @@ -0,0 +1,190 @@ +from argopy.utils import register_accessor +from argopy.related.sensors.spec import ArgoSensorSpec +from argopy.related.sensors.references import SensorReferences + + +class ArgoSensor(ArgoSensorSpec): + """Argo sensor(s) + + The :class:`ArgoSensor` class aims to provide direct access to Argo's sensor metadata from: + + - NVS Reference Tables `Sensor Models (R27) `_, `Sensor Types (R25) `_ and `Sensor Manufacturers (R26) `_, + - the `Euro-Argo fleet-monitoring API `_. + + This enables users to: + + - navigate reference tables, + - search for floats equipped with specific sensor models, + - retrieve sensor serial numbers across the global array, + - search for/iterate over floats equipped with specific sensor models. + + Examples + -------- + .. code-block:: python + :caption: Access reference tables for SENSOR_MODEL (R27), SENSOR (R25) and SENSOR_MAKER (R26) + + from argopy import ArgoSensor + sensor = ArgoSensor() + + sensor.ref.model.to_dataframe() # Return reference table R27 with the list of sensor models as a DataFrame + sensor.ref.model.hint() # Return list of sensor model names (possible values for 'SENSOR_MODEL' parameter) + + sensor.ref.type.to_dataframe() # Return reference table R25 with the list of sensor types as a DataFrame + sensor.ref.type.hint() # Return list of sensor types (possible values for 'SENSOR' parameter) + + sensor.ref.maker.to_dataframe() # Return reference table R26 with the list of manufacturer as a DataFrame + sensor.ref.maker.hint() # Return list of manufacturer names (possible values for 'SENSOR_MAKER' parameter) + + .. code-block:: python + :caption: Mapping between SENSOR_MODEL (R27) and SENSOR (R25) + + from argopy import ArgoSensor + sensor = ArgoSensor() + + sensor.ref.model.to_type('SBE61') # Return sensor type (R25) of a given model (R27) + + sensor.ref.type.to_model('FLUOROMETER_CDOM') # Return all possible model names (R27) for a given sensor type (R25) + + .. code-block:: python + :caption: Search in SENSOR_MODEL (R27) reference table + + from argopy import ArgoSensor + sensor = ArgoSensor() + + sensor.ref.model.search('RBR') # Search and return a DataFrame + sensor.ref.model.search('RBR', output='name') # Search and return a list of names instead + sensor.ref.model.search('SBE61*') # Use of wildcards + sensor.ref.model.search('*Deep*') # Search is case-insensitive + + .. code-block:: python + :caption: Search for all Argo floats equipped with one or more exact sensor model(s) + + from argopy import ArgoSensor + + sensors = ArgoSensor() + + # Search and return a list of WMOs equipped + sensors.search('SBE61_V5.0.2') + + # Search and return a list of sensor serial numbers in Argo + sensors.search('ECO_FLBBCD_AP2', output='sn') + + # Search and return a list of tuples with WMOs and sensors serial number + sensors.search('SBE', output='wmo_sn') + + # Search and return a DataFrame with full sensor information from floats equipped + sensors.search('RBR', output='df') + + # Search multiple models at once + sensors.search(['ECO_FLBBCD_AP2', 'ECO_FLBBCD']) + + + .. code-block:: python + :caption: Easily loop through :class:`ArgoFloat` instances for each float equipped with a sensor model + + from argopy import ArgoSensor + + sensors = ArgoSensor() + + # Trivial example: + model = "RAFOS" + for af in sensors.iterfloats_with(model): + print(af.WMO) + + # Example to gather all platform types for all WMOs equipped with a list of sensor models + models = ['ECO_FLBBCD_AP2', 'ECO_FLBBCD'] + results = {} + for af in sensors.iterfloats_with(models): + if 'meta' in af.ls_dataset(): + platform_type = af.metadata['platform']['type'] # e.g. 'PROVOR_V_JUMBO' + if platform_type in results.keys(): + results[platform_type].extend([af.WMO]) + else: + results.update({platform_type: [af.WMO]}) + else: + print(f"No meta file for float {af.WMO}") + print(results.keys()) + + .. code-block:: python + :caption: Use an exact sensor model name to create an instance + + from argopy import ArgoSensor + + sensor = ArgoSensor('RBR_ARGO3_DEEP6K') + + sensor.vocabulary # R25 concept + sensor.type # R27 concept + + # Retrieve info from floats equipped with this model: + sensor.search(output='wmo') + sensor.search(output='sn') + sensor.search(output='wmo_sn') + sensor.search(output='df') + + # Iterator: + for af in sensors.iterfloats_with(): + print(af.WMO) + + .. code-block:: bash + :caption: Get serialized search results from the command-line with :class:`ArgoSensor.cli_search` + + python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('RBR', output='wmo')" + python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('RBR', output='sn')" + python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('RBR', output='wmo_sn')" + python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('RBR', output='df')" + + + Notes + ----- + Ongoing related ADMT/AVTT work can be found here: + https://github.com/OneArgo/ArgoVocabs/issues?q=state%3Aopen%20label%3A%22R25%22%20OR%20label%3AR27%20OR%20label%3AR26 + """ + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + +@register_accessor("ref", ArgoSensor) +class References(SensorReferences): + """An :class:`ArgoSensor` extension dedicated to reference tables appropriate for sensors + + Examples + -------- + .. code-block:: python + :caption: Access reference tables for SENSOR_MODEL (R27), SENSOR (R25) and SENSOR_MAKER (R26) + + from argopy import ArgoSensor + sensor = ArgoSensor() + + sensor.ref.model.to_dataframe() # Return reference table R27 with the list of sensor models as a DataFrame + sensor.ref.model.hint() # Return list of sensor model names (possible values for 'SENSOR_MODEL' parameter) + + sensor.ref.type.to_dataframe() # Return reference table R25 with the list of sensor types as a DataFrame + sensor.ref.type.hint() # Return list of sensor types (possible values for 'SENSOR' parameter) + + sensor.ref.maker.to_dataframe() # Return reference table R26 with the list of manufacturer as a DataFrame + sensor.ref.maker.hint() # Return list of manufacturer names (possible values for 'SENSOR_MAKER' parameter) + + .. code-block:: python + :caption: Mapping between SENSOR_MODEL (R27) and SENSOR (R25) + + from argopy import ArgoSensor + sensor = ArgoSensor() + + sensor.ref.model.to_type('SBE61') # Return sensor type (R25) of a given model (R27) + + sensor.ref.type.to_model('FLUOROMETER_CDOM') # Return all possible model names (R27) for a given sensor type (R25) + + .. code-block:: python + :caption: Search in SENSOR_MODEL (R27) reference table + + from argopy import ArgoSensor + sensor = ArgoSensor() + + sensor.ref.model.search('RBR') # Search and return a DataFrame + sensor.ref.model.search('RBR', output='name') # Search and return a list of names instead + sensor.ref.model.search('SBE61*') # Use of wildcards + sensor.ref.model.search('*Deep*') # Search is case-insensitive + + """ + + _name = "ref" diff --git a/argopy/related/sensors/spec.py b/argopy/related/sensors/spec.py new file mode 100644 index 000000000..1b48542e9 --- /dev/null +++ b/argopy/related/sensors/spec.py @@ -0,0 +1,758 @@ +import pandas as pd +import numpy as np +from typing import Literal, Any, Iterator, Callable +import concurrent.futures +import logging +import json +import sys + +from argopy.stores import ArgoFloat, ArgoIndex, httpstore +from argopy.stores.filesystems import ( + tqdm, +) # Safe import, return a lambda if tqdm not available +from argopy.utils import check_wmo, Chunker, to_list, ppliststr, is_wmo +from argopy.errors import ( + DataNotFound, + InvalidDataset, + InvalidDatasetStructure, + OptionValueError, +) +from argopy.options import OPTIONS +from argopy.related.euroargo_api import EAfleetmonitoringAPI + +from argopy.related.sensors.references import SensorModel, SensorType +from argopy.related.sensors.accessories import Error, ErrorOptions, Ds, DsOptions, SearchOutput, SearchOutputOptions +from argopy.related.sensors.utils import APISensorMetaDataProcessing + + +log = logging.getLogger("argopy.related.sensors") + + +class ArgoSensorSpec: + + __slots__ = [ + "_vocabulary", # R27 row for an instance + "_type", # R25 row for an instance + "_fs", # http filesystem, extensions shall use it as well + "_cache", # To cache extensions, not the option for filesystems + ] + + def __init__(self, model: str | None = None, *args, **kwargs) -> None: + """Create an instance of :class:`ArgoSensor` + + Parameters + ---------- + model: str, optional, default: None + An exact sensor model name, None by default because this is optional. + + Otherwise, possible values can be obtained from :meth:`ArgoSensor.ref.model.hint`. + + Other Parameters + ---------------- + fs: :class:`stores.httpstore`, optional, default: None + The http filesystem to use. If None is provided, we instantiate a new one based on `cache`, `cachedir` and `timeout` options. + cache : bool, optional, default: True + Use cache or not for fetched data. Used only if `fs` is None. + cachedir: str, optional, default: OPTIONS['cachedir'] + Folder where to store cached files. Used only if `fs` is None. + timeout: int, optional, default: OPTIONS['api_timeout'] + Time out in seconds to connect to web API. Used only if `fs` is None. + + """ + if kwargs.get("fs", None) is None: + self._fs = httpstore( + cache=kwargs.get("cache", True), + cachedir=kwargs.get("cachedir", OPTIONS["cachedir"]), + timeout=kwargs.get("timeout", OPTIONS["api_timeout"]), + ) + else: + self._fs = kwargs["fs"] + + self._vocabulary: SensorModel | None = None + self._type: list[SensorType] | None = None + self._api = EAfleetmonitoringAPI(fs=self._fs) + + if model is not None: + try: + df = self.ref.model.search(model) + except DataNotFound: + raise DataNotFound( + f"No sensor model named '{model}', as per ArgoSensor().ref.model.hint() values, based on Ref. Table 27." + ) + + if df.shape[0] == 1: + self._vocabulary = SensorModel.from_series(df.iloc[0]) + self._type = self.ref.model.to_type(self._vocabulary, errors="ignore", obj=True) + # if "RBR" in self._vocabulary: + # Add the RBR OEM API Authorization key for this sensor: + # fs_kargs.update(client_kwargs={'headers': {'Authorization': OPTIONS.get('rbr_api_key') }}) + else: + raise InvalidDatasetStructure( + f"Found multiple sensor models with '{model}'. Restrict your sensor model name to only one value in: {to_list(df['altLabel'].values)}" + ) + + @property + def vocabulary(self) -> SensorModel: + """Argo reference "SENSOR_MODEL" vocabulary for this sensor model + + .. note:: + Only available for a class instance created with an explicit sensor model name. + + Returns + ------- + :class:`argopy.related.SensorModel` + + Raises + ------ + :class:`InvalidDataset` + """ + if isinstance(self._vocabulary, SensorModel): + return self._vocabulary + else: + raise InvalidDataset( + "The 'vocabulary' property is not available for an ArgoSensor instance not created with a specific sensor model" + ) + + @property + def type(self) -> SensorType: + """Argo reference "SENSOR" vocabulary for this sensor model + + .. note:: + Only available for a class instance created with an explicit sensor model name. + + Returns + ------- + :class:`argopy.related.SensorType` + + Raises + ------ + :class:`InvalidDataset` + """ + if len(self._type) > 0 and isinstance(self._type[0], SensorType): + return self._type + else: + raise InvalidDataset( + "The 'type' property is not available for an ArgoSensor instance not created with a specific sensor model" + ) + + def __repr__(self) -> str: + if isinstance(self._vocabulary, SensorModel): + summary = [f""] + summary.append(f"TYPE➤ {ppliststr([t.long_name for t in self.type])}") + summary.append(f"MODEL➤ {self.vocabulary.long_name}") + if self.vocabulary.deprecated: + summary.append("⛔ This model is deprecated !") + else: + summary.append("✅ This model is not deprecated.") + summary.append(f"🔗 {self.vocabulary.uri}") + summary.append(f"❝{self.vocabulary.definition}❞") + else: + summary = [""] + summary.append( + "This instance was not created with a sensor model name, you still have access to the following:" + ) + summary.append("👉 extensions: ") + for attr in [ + "ref.model.to_dataframe()", + "ref.model.hint()", + "ref.model.to_type", + "ref.model.search", + "ref.type.to_dataframe()", + "ref.type.hint()", + "ref.type.to_model", + "ref.maker.to_dataframe()", + "ref.maker.hint()", + ]: + summary.append(f" ╰┈➤ ArgoSensor().{attr}") + + summary.append("👉 methods: ") + for meth in [ + "search", + "iterfloats_with", + "from_wmo", + ]: + summary.append(f" ╰┈➤ ArgoSensor().{meth}()") + return "\n".join(summary) + + def _search_wmo_with( + self, model: str | list[str], errors: ErrorOptions = "raise" + ) -> list[int]: + """Return the list of WMOs equipped with a given sensor model + + Notes + ----- + Based on a fleet-monitoring API request to `platformCodes/multi-lines-search` on `sensorModels` field. + + Documentation: + + https://fleetmonitoring.euro-argo.eu/swagger-ui.html#!/platform-code-controller/getPlatformCodesMultiLinesSearchUsingPOST + + Notes + ----- + No option checking, to be done by caller + """ + models = to_list(model) + api_endpoint = f"{OPTIONS['fleetmonitoring']}/platformCodes/multi-lines-search" + payload = [ + { + "nested": False, + "path": "string", + "searchValueType": "Text", + "values": models, + "field": "sensorModels", + } + ] + wmos = self._fs.post(api_endpoint, json_data=payload) + if wmos is None or len(wmos) == 0: + if len(models) == 1: + msg = f"Model is valid but no floats returned with this sensor model name: '{models[0]}'" + else: + msg = f"Models are valid but no floats returned with any of these sensor model names: {ppliststr(models)}" + if errors == "raise": + raise DataNotFound(msg) + elif errors == "ignore": + log.error(msg) + return sorted(check_wmo(wmos)) + + def _floats_api( + self, + model_or_wmo: str | int, + preprocess: Callable = None, + preprocess_opts: dict = {}, + postprocess: Callable = None, + postprocess_opts: dict = {}, + progress: bool = False, + errors: ErrorOptions = "raise", + ) -> Any: + """Search floats with a sensor model and then fetch and process JSON data returned from the fleet-monitoring API for each floats + + Notes + ----- + Based on a POST request to the fleet-monitoring API requests to `/floats/{wmo}`. + + `Endpoint documentation `_. + + Notes + ----- + No option checking, to be done by caller + """ + try: + is_wmo(model_or_wmo) + WMOs = check_wmo(model_or_wmo) + except ValueError: + WMOs = self._search_wmo_with(model_or_wmo) + + return self._api.floats(WMOs, + preprocess=preprocess, + preprocess_opts=preprocess_opts, + postprocess=postprocess, + postprocess_opts=postprocess_opts, + progress=progress, + errors=errors, + ) + + def _search_sn_with( + self, + model: str, + progress: bool = False, + errors: ErrorOptions = "raise", + **kwargs, + ) -> list[str]: + """Return serial number of sensor models with a given string in name + + Notes + ----- + No option checking, to be done by caller + """ + + def preprocess(jsdata, model_name: str = ""): + sn = np.unique( + [s["serial"] for s in jsdata["sensors"] if model_name in s["model"]] + ) + return sn + + def postprocess(data, **kwargs): + S = [] + for row in data: + for sensor in row: + if sensor is not None: + S.append(sensor) + return np.sort(np.array(S)).tolist() + + return self._floats_api( + model if kwargs.get("wmo", None) is None else kwargs["wmo"], + preprocess=preprocess, + preprocess_opts={"model_name": model}, + postprocess=postprocess, + progress=progress, + errors=errors, + ) + + def _search_wmo_sn_with( + self, + model: str, + progress: bool = False, + errors: ErrorOptions = "raise", + **kwargs, + ) -> dict[int, str]: + """Return a dictionary of float WMOs with their sensor serial numbers + + Notes + ----- + No option checking, to be done by caller + """ + + def preprocess(jsdata, model_name: str = ""): + try: + x = [s["serial"] for s in jsdata["sensors"] if model_name in s["model"]] + x = [x for x in x if x is not None] + if len(x) == 0: + sn = ['n/a'] + else: + sn = np.unique(x).tolist() + return [jsdata["wmo"], [str(s) for s in sn]] + except: + log.error(f"Could not find sensor model {model_name}: {jsdata['sensors']}") + + def postprocess(data, **kwargs): + S = {} + for wmo, sn in data: + S.update({check_wmo(wmo)[0]: to_list(sn)}) + return S + + results = self._floats_api( + model if kwargs.get("wmo", None) is None else kwargs["wmo"], + preprocess=preprocess, + preprocess_opts={"model_name": model}, + postprocess=postprocess, + progress=progress, + errors=errors, + ) + return dict(sorted(results.items())) + + def _to_dataframe( + self, + model: str, + progress: bool = False, + errors: ErrorOptions = "raise", + **kwargs, + ) -> pd.DataFrame: + """Return a DataFrame with WMO, sensor type, model, maker, sn, units, accuracy and resolution + + Parameters + ---------- + model: str, optional + A string to search in the `sensorModels` field of the Euro-Argo fleet-monitoring API `platformCodes/multi-lines-search` endpoint. + + Notes + ----- + No option checking, to be done by caller + """ + if model is None and self.vocabulary is not None: + model = self.vocabulary.name + + return self._floats_api( + model if kwargs.get("wmo", None) is None else kwargs["wmo"], + preprocess=APISensorMetaDataProcessing.preprocess_df, + preprocess_opts={"model_name": model}, + postprocess=APISensorMetaDataProcessing.postprocess_df, + progress=progress, + errors=errors, + ) + + def _search_single( + self, + model: str, + output: SearchOutputOptions = "wmo", + progress: bool = False, + errors: ErrorOptions = "raise", + ) -> list[int] | list[str] | dict[int, str] | pd.DataFrame: + """Run a single model search""" + + if output == "df": + return self._to_dataframe(model=model, progress=progress, errors=errors) + elif output == "sn": + return self._search_sn_with(model=model, progress=progress, errors=errors) + elif output == "wmo_sn": + return self._search_wmo_sn_with( + model=model, progress=progress, errors=errors + ) + else: + return self._search_wmo_with(model=model, errors=errors) + + def _search_multi( + self, + models: list[str], + output: SearchOutputOptions = "wmo", + progress: bool = False, + errors: ErrorOptions = "raise", + max_workers: int | None = None, + ) -> list[int] | list[str] | dict[int, str] | pd.DataFrame: + """Run a multiple models search in parallel with multithreading""" + # Remove duplicates: + models = list(set(models)) + + if output == "wmo": + # Quite simple if we only need WMOs: + return self._search_wmo_with(model=models, errors=errors) + else: + # Otherwise we need the list of WMOs to get metadata for + # Even if we can request the fleetmonitoring API with multiple models at once, here we need + # the tuple (model, wmo) to use multithreading and existing private search methods. + lookup = [] + for model in models: + wmos = self._search_wmo_with(model=model, errors=errors) + lookup.extend([(model, wmo) for wmo in wmos]) + + # For all other output format, we use multithreading to process all floats: + if output == "sn": + func = self._search_sn_with + elif output == "wmo_sn": + func = self._search_wmo_sn_with + else: + func = self._to_dataframe + + ConcurrentExecutor = concurrent.futures.ThreadPoolExecutor( + max_workers=max_workers + ) + failed = [] + if output in ["sn", "df"]: + results = [] + elif output == "wmo_sn": + results = {} + + with ConcurrentExecutor as executor: + future_to_wmo = { + executor.submit( + func, + pair[0], + errors=errors, + wmo=pair[1], + ): pair + for pair in lookup + } + futures = concurrent.futures.as_completed(future_to_wmo) + if progress: + futures = tqdm( + futures, + total=len(lookup), + disable="disable" in [progress], + unit="float", + desc=f"Fetching sensor meta-data for {len(lookup)} floats...", + ) + + for future in futures: + data = None + try: + data = future.result() + except Exception: + failed.append(future_to_wmo[future]) + if errors == "ignore": + log.error( + "Ignored error with this float: %s" % future_to_wmo[future] + ) + elif errors == "silent": + pass + else: + raise + finally: + # Gather results according to final output format: + if data is not None: + if output == "sn": + results.extend(data) + elif output == "df": + results.append(data) + elif output == "wmo_sn": + for wmo in data.keys(): + if wmo in results: + # Update existing key: + results[wmo] = results[wmo].extend(data[wmo]) + else: + # Create new key: + results.update({wmo: data[wmo]}) + + if output != "wmo_sn": + # Only keep non-empty results: + results = [r for r in results if r is not None] + else: + results = dict(sorted(results.items())) + + if len(results) > 0: + if output == "df": + results = [r for r in results if r.shape[0] > 0] + return ( + pd.concat(results, axis=0) + .sort_values(by="WMO", axis=0) + .reset_index(drop=True) + ) + else: + return results + raise DataNotFound(ppliststr(models)) + + def search( + self, + model: str | list[str] | None = None, + output: SearchOutputOptions = "wmo", + progress: bool = True, + errors: ErrorOptions = "raise", + serialized: bool = False, + **kwargs, + ) -> list[int] | list[str] | dict[int, str] | pd.DataFrame | str: + """Search for Argo floats equipped with one or more sensor model name(s) + + All information are retrieved from the `Euro-Argo fleet-monitoring API `_. + + Since this method can return a large number of data, model names are expected to be exact. You can use the :meth:`ArgoSensor.ref.model.search` method to search for exact model names. + + Parameters + ---------- + model: str, list[str], optional + One or more exact model names to search data for. + + output: str, Literal["wmo", "sn", "wmo_sn", "df"], default: "wmo" + Define the output to return: + + - ``wmo``: a list of WMO numbers (integers) + - ``sn``: a list of sensor serial numbers (strings) + - ``wmo_sn``: a list of dictionary with WMO as key and serial numbers as values + - ``df``: a :class:`pandas.DataFrame` with WMO, sensor type/model/maker and serial number + + progress: bool, default: True + Display a progress bar or not + + errors: str, Literal["raise", "ignore", "silent"], default: "raise" + Raise an error, log it or do nothing for no search results. + + serialized: bool, default: False + Return a serialized output. This allows for search results to be saved in cross-language, human-readable formats like json. + + Returns + ------- + list[int], list[str], dict[int, str], :class:`pandas.DataFrame` + + See Also + -------- + :meth:`ArgoSensor.ref.model.search` + + Notes + ----- + Whatever the output format, the first step is to retrieve a list of WMOs equipped with one or more sensor models. + This is done using the Euro-Argo fleet-monitoring API and a request to the ``platformCodes/multi-lines-search`` endpoint using the ``sensorModels`` search field. + + Then if necessary (all output format but 'wmo'), the corresponding list of sensor serial numbers are retrieved using one request per float to the Euro-Argo fleet-monitoring API ``/floats/{wmo}`` endpoint. + + Web-api documentation: + + - `Documentation for endpoint: platformCodes/multi-lines-search `_. + - `Documentation for endpoint: /floats/{wmo} `_. + + """ + if output not in SearchOutput: + raise OptionValueError( + f"Invalid 'output' option value '{output}', must be {ppliststr(SearchOutput, last='or')}" + ) + if errors not in Error: + raise OptionValueError( + f"Invalid 'errors' option value '{errors}', must be in: {ppliststr(Error, last='or')}" + ) + + if model is None: + if self.vocabulary is not None: + return self._search_single( + model=self.vocabulary.name, + output=output, + progress=progress, + errors=errors, + ) + else: + raise OptionValueError("You must specify at list one model to search !") + + models = to_list(model) + + def get_hints(these_models: str | list[str]): + these_models = to_list(these_models) + search_hint: list[str] = [] + for model in these_models: + model = f"*{model.upper()}*" # Use wildcards to get all possible hints + try: + hint: list[str] = self.ref.model.search( + model, + output="name", + ) + search_hint.extend(hint) + except DataNotFound: + pass + if len(search_hint) == 0: + search_hint = ["No match !"] + output = ppliststr( + search_hint, last="or", n=20 if len(search_hint) > 20 else None + ) + return output + + # Model names validation: + valid_models: list[str] = [] + invalid_models: list[str] = [] + for model in models: + if "*" in model: + raise OptionValueError( + f"This method expect exact model names but got a '*' in '{model}'. Possible hints are: {get_hints(model)}" + ) + try: + hint = self.ref.model.search(model, output="name") + valid_models.extend(hint) + except DataNotFound: + raise OptionValueError( + f"The model '{model}' is not exact. Possible hints are: {get_hints(model)}" + ) + + if len(valid_models) == 0: + msg = f"Unknown sensor model name(s) {ppliststr(invalid_models)}. We expect exact sensor names such as: {get_hints(invalid_models)}." + raise OptionValueError(msg) + + if len(valid_models) == 1: + results = self._search_single( + model=valid_models[0], output=output, progress=progress, errors=errors + ) + else: + results = self._search_multi( + models=valid_models, + output=output, + progress=progress, + errors=errors, + **kwargs, + ) + if serialized: + # Serialize all output format to json + if output == 'df': + return results.to_json(indent=2) + return json.dumps(results, indent=2) + return results + + def cli_search(self, + model: str | list[str] | None = None, + output: SearchOutputOptions = "wmo") -> str: # type: ignore + """A command-line-friendly search for Argo floats equipped with one or more sensor model name(s) + + This function is intended to be called from the command-line and return serialized results for easy piping to other tools. + + Parameters + ---------- + model: str, list[str], optional + One or more exact model names to search data for. + + output: str, Literal["wmo", "sn", "wmo_sn", "df"], default: "wmo" + Define the output to return: + + - ``wmo``: a list of WMO numbers (integers) + - ``sn``: a list of sensor serial numbers (strings) + - ``wmo_sn``: a list of dictionary with WMO as key and serial numbers as values + - ``df``: a dictionary with all available metadata. + + Examples + -------- + .. code-block:: bash + :caption: Example of search results from the command-line + + python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('RBR', output='wmo')" + python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('RBR', output='sn')" + python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('RBR', output='wmo_sn')" + python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('RBR', output='df')" + + """ + print(self.search(model, output=output, serialized=True, progress=False), file=sys.stdout) + + def iterfloats_with( + self, + model: str | list[str] | None = None, + chunksize: int | None = None, + # ds: DsOptions = 'core', + **kwargs + ) -> Iterator[ArgoFloat]: + """Iterator over :class:`argopy.ArgoFloat` equipped with a given sensor model + + By default, iterate over a single float, otherwise use the `chunksize` argument to iterate over chunk of floats. + + Parameters + ---------- + model: str + A string to search in the `sensorModels` field of the Euro-Argo fleet-monitoring API `platformCodes/multi-lines-search` endpoint. + + chunksize: int, optional + Maximum chunk size + + Eg: A value of 5 will create chunks with as many as 5 WMOs each. + + Returns + ------- + Iterator of :class:`argopy.ArgoFloat` + + Examples + -------- + .. code-block:: python + :caption: Example of iteration + + sensors = ArgoSensor() + + for afloat in sensors.iterfloats_with("RAFOS"): + print(afloat.WMO) + + """ + if model is None: + if self.vocabulary is not None: + model = self.vocabulary.name, + else: + raise OptionValueError("You must specify at list one model to search !") + + models = to_list(model) + WMOs = self.search(model=models, progress=False) + WMOs = np.unique(WMOs).tolist() + + # 'ds' is a hidden option because I'm not 100% sure this will be needed. + # ds: str, Literal['core', 'bgc', 'deep'], default='core' + # The Argo mission for this collection of floats. + # This will be used to create an :class:`ArgoIndex` shared by all :class:`ArgoFloat` instances. + ds = kwargs.get('ds', 'core') + if ds not in Ds: + raise OptionValueError( + f"Invalid 'ds' option value '{ds}', must be {ppliststr(Ds)}" + ) + else: + if ds == 'deep': + ds = 'core' + elif ds == 'bgc': + ds = 'bgc-b' + + idx = ArgoIndex( + index_file = ds, + fs = self._fs, + ) + + if chunksize is not None: + chk_opts = {} + chk_opts.update({"chunks": {"wmo": "auto"}}) + chk_opts.update({"chunksize": {"wmo": chunksize}}) + chunked = Chunker( + {"wmo": WMOs}, **chk_opts + ).fit_transform() + for grp in chunked: + yield [ArgoFloat(wmo, idx=idx) for wmo in grp] + + else: + for wmo in WMOs: + yield ArgoFloat(wmo, idx=idx) + + def from_wmo(self, wmo: int | str) -> pd.DataFrame: + """Retrieve sensor metadata from a given float WMO number + + Parameters + ---------- + wmo: int | str + Float WMO number, only one value + + Returns + ------- + :class:`pandas.DataFrame` + """ + wmo = check_wmo(wmo) + df = self._to_dataframe('', wmo=wmo[0]) + return df.drop('WMO', axis=1).T \ No newline at end of file diff --git a/argopy/related/sensors/utils.py b/argopy/related/sensors/utils.py new file mode 100644 index 000000000..6d3dd6c44 --- /dev/null +++ b/argopy/related/sensors/utils.py @@ -0,0 +1,58 @@ +import pandas as pd +import importlib + + +def _importorskip(modname): + try: + importlib.import_module(modname) # noqa: E402 + has = True + except ImportError: + has = False + return has + + +has_jsonschema = _importorskip("jsonschema") + + +class APISensorMetaDataProcessing: + + @classmethod + def preprocess_df(cls, jsdata, model_name: str = "", **kwargs) -> list[str]: + """Get the list of sensor metadata for a given model""" + output = [] + for s in jsdata["sensors"]: + if model_name in s["model"]: + this = [jsdata["wmo"]] + [ + this.append(s[key]) # type: ignore + for key in [ + "id", + "maker", + "model", + "serial", + "units", + "accuracy", + "resolution", + ] + ] + output.append(this) + return output + + @classmethod + def postprocess_df(cls, data, **kwargs): + d = [] + for this in data: + for wmo, sid, maker, model, sn, units, accuracy, resolution in this: + d.append( + { + "WMO": wmo, + "Type": sid, + "Model": model, + "Maker": maker, + "SerialNumber": sn if sn != "n/a" else None, + "Units": units, + "Accuracy": accuracy, + "Resolution": resolution, + } + ) + return pd.DataFrame(d).sort_values(by="WMO").reset_index(drop=True) diff --git a/argopy/related/utils.py b/argopy/related/utils.py index ad75d4f81..587aad618 100644 --- a/argopy/related/utils.py +++ b/argopy/related/utils.py @@ -2,6 +2,8 @@ import os import json import logging +from typing import Literal + from . import ArgoNVSReferenceTables @@ -11,7 +13,21 @@ ).submodule_search_locations[0] -def load_dict(ptype): +def load_dict(ptype : Literal["profilers", "institutions"] | None = None) -> dict: + """Load a dictionary from a ArgoNVSReferenceTables instance + + Otherwise, fall back on static JSON asset files. + + List of possible dictionaries: + + - "profilers": key are R08 ID, values are prefLabel + - "institutions": key are R04 altLabel, values are prefLabel + + Parameters + ---------- + ptype: str, Literal["profilers", "institutions"], default None + + """ if ptype == "profilers": try: nvs = ArgoNVSReferenceTables(cache=True) diff --git a/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_1.txt b/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_1.txt new file mode 100644 index 000000000..d671729c8 --- /dev/null +++ b/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_1.txt @@ -0,0 +1,125 @@ +R27, AANDERAA_OPTODE , MIN, R25, OPTODE_DOXY , I +R27, AANDERAA_OPTODE_3830 , MIN, R25, OPTODE_DOXY , I +R27, AANDERAA_OPTODE_3835 , MIN, R25, OPTODE_DOXY , I +R27, AANDERAA_OPTODE_3930 , MIN, R25, OPTODE_DOXY , I +R27, AANDERAA_OPTODE_4330 , MIN, R25, OPTODE_DOXY , I +R27, AANDERAA_OPTODE_4330F , MIN, R25, OPTODE_DOXY , I +R27, AANDERAA_OPTODE_4831 , MIN, R25, OPTODE_DOXY , I +R27, AANDERAA_OPTODE_4831F , MIN, R25, OPTODE_DOXY , I +R27, AMETEK , MIN, R25, CTD_PRES , I +R27, AMETEK_3000PSIA , MIN, R25, CTD_PRES , I +R27, ARO_FT , MIN, R25, OPTODE_DOXY , I +R27, AROD_FT , MIN, R25, OPTODE_DOXY , I +R27, C_ROVER , MIN, R25, TRANSMISSOMETER_CP660 , I +R27, CTD_F01 , MIN, R25, CTD_PRES , I +R27, CTD_F01 , MIN, R25, CTD_TEMP , I +R27, CTD_F01 , MIN, R25, CTD_CNDC , I +R27, CYCLOPS_7_FLUOROMETER , MIN, R25, FLUOROMETER_CHLA , I +R27, CYCLOPS_7_FLUOROMETER , MIN, R25, BACKSCATTERINGMETER_TURBIDITY , I +R27, DRUCK , MIN, R25, CTD_PRES , I +R27, DRUCK_2900PSIA , MIN, R25, CTD_PRES , I +R27, DURA , MIN, R25, TRANSISTOR_PH , I +R27, ECO_FL , MIN, R25, FLUOROMETER_CHLA , I +R27, ECO_FLBB , MIN, R25, FLUOROMETER_CHLA , I +R27, ECO_FLBB_2K , MIN, R25, FLUOROMETER_CHLA , I +R27, ECO_FLBB_AP2 , MIN, R25, FLUOROMETER_CHLA , I +R27, ECO_FLBB2 , MIN, R25, FLUOROMETER_CHLA , I +R27, ECO_FLBBCD , MIN, R25, FLUOROMETER_CHLA , I +R27, ECO_FLBBCD , MIN, R25, FLUOROMETER_CDOM , I +R27, ECO_FLBBCD_AP2 , MIN, R25, FLUOROMETER_CHLA , I +R27, ECO_FLBBCD_AP2 , MIN, R25, FLUOROMETER_CDOM , I +R27, ECO_FLBBFL , MIN, R25, FLUOROMETER_CHLA , I +R27, ECO_FLBBFL_AP2 , MIN, R25, FLUOROMETER_CHLA , I +R27, ECO_FLNTU , MIN, R25, FLUOROMETER_CHLA , I +R27, ECO_FLNTU , MIN, R25, BACKSCATTERINGMETER_TURBIDITY , I +R27, ECO_NTU , MIN, R25, BACKSCATTERINGMETER_TURBIDITY , I +R27, EM , MIN, R25, EM , I +R27, FLOATCLOCK , MIN, R25, FLOATCLOCK_MTIME , I +R27, FSI , MIN, R25, CTD_PRES , I +R27, FSI , MIN, R25, CTD_TEMP , I +R27, FSI , MIN, R25, CTD_CNDC , I +R27, GDF , MIN, R25, TRANSISTOR_PH , I +R27, ISUS , MIN, R25, SPECTROPHOTOMETER_NITRATE , I +R27, ISUS_V3 , MIN, R25, SPECTROPHOTOMETER_NITRATE , I +R27, KELLER_PA8 , MIN, R25, CTD_PRES , I +R27, KISTLER , MIN, R25, CTD_PRES , I +R27, KISTLER_10153PSIA , MIN, R25, CTD_PRES , I +R27, KISTLER_2900PSIA , MIN, R25, CTD_PRES , I +R27, MCOMS_FLBB2 , MIN, R25, FLUOROMETER_CHLA , I +R27, MCOMS_FLBBCD , MIN, R25, FLUOROMETER_CHLA , I +R27, MCOMS_FLBBCD , MIN, R25, FLUOROMETER_CDOM , I +R27, MENSOR , MIN, R25, CTD_PRES , I +R27, MP40_C_2000_G , MIN, R25, CTD_PRES , I +R27, OPUS_DS , MIN, R25, SPECTROPHOTOMETER_NITRATE , I +R27, OPUS_DS , MIN, R25, SPECTROPHOTOMETER_BISULFIDE , I +R27, PAINE , MIN, R25, CTD_PRES , I +R27, PAINE_1500PSIA , MIN, R25, CTD_PRES , I +R27, PAINE_1600PSIA , MIN, R25, CTD_PRES , I +R27, PAINE_2000PSIA , MIN, R25, CTD_PRES , I +R27, PAINE_2900PSIA , MIN, R25, CTD_PRES , I +R27, PAINE_3000PSIA , MIN, R25, CTD_PRES , I +R27, PAL_UW , MIN, R25, ACOUSTIC , I +R27, RAFOS , MIN, R25, ACOUSTIC_GEOLOCATION , I +R27, RAMSES_ACC , MIN, R25, RADIOMETER_PAR , I +R27, RBR , MIN, R25, CTD_PRES , I +R27, RBR , MIN, R25, CTD_TEMP , I +R27, RBR , MIN, R25, CTD_CNDC , I +R27, RBR_ARGO , MIN, R25, CTD_PRES , I +R27, RBR_ARGO , MIN, R25, CTD_TEMP , I +R27, RBR_ARGO , MIN, R25, CTD_CNDC , I +R27, RBR_ARGO3 , MIN, R25, CTD_PRES , I +R27, RBR_ARGO3 , MIN, R25, CTD_TEMP , I +R27, RBR_ARGO3 , MIN, R25, CTD_CNDC , I +R27, RBR_ARGO3_DEEP4K , MIN, R25, CTD_PRES , I +R27, RBR_ARGO3_DEEP4K , MIN, R25, CTD_TEMP , I +R27, RBR_ARGO3_DEEP4K , MIN, R25, CTD_CNDC , I +R27, RBR_ARGO3_DEEP6K , MIN, R25, CTD_PRES , I +R27, RBR_ARGO3_DEEP6K , MIN, R25, CTD_TEMP , I +R27, RBR_ARGO3_DEEP6K , MIN, R25, CTD_CNDC , I +R27, RBR_CODA_T_ODO , MIN, R25, OPTODE_DOXY , I +R27, RBR_PRES , MIN, R25, CTD_PRES , I +R27, RBR_PRES_A , MIN, R25, CTD_PRES , I +R27, SATLANTIC_OCR504_ICSW , MIN, R25, RADIOMETER_PAR , I +R27, SATLANTIC_OCR507_ICSW , MIN, R25, RADIOMETER_PAR , I +R27, SATLANTIC_OCR507_ICSWR10W , MIN, R25, RADIOMETER_PAR , I +R27, SATLANTIC_PAR , MIN, R25, RADIOMETER_PAR , I +R27, SBE , MIN, R25, CTD_PRES , I +R27, SBE , MIN, R25, CTD_TEMP , I +R27, SBE , MIN, R25, CTD_CNDC , I +R27, SBE_STS , MIN, R25, STS_CNDC , I +R27, SBE_STS , MIN, R25, STS_TEMP , I +R27, SBE37 , MIN, R25, CTD_PRES , I +R27, SBE37 , MIN, R25, CTD_TEMP , I +R27, SBE37 , MIN, R25, CTD_CNDC , I +R27, SBE41 , MIN, R25, CTD_PRES , I +R27, SBE41 , MIN, R25, CTD_TEMP , I +R27, SBE41 , MIN, R25, CTD_CNDC , I +R27, SBE41_IDO , MIN, R25, CTD_PRES , I +R27, SBE41_IDO , MIN, R25, CTD_TEMP , I +R27, SBE41_IDO , MIN, R25, CTD_CNDC , I +R27, SBE41_IDO , MIN, R25, IDO_DOXY , I +R27, SBE41CP , MIN, R25, CTD_PRES , I +R27, SBE41CP , MIN, R25, CTD_TEMP , I +R27, SBE41CP , MIN, R25, CTD_CNDC , I +R27, SBE41CP_IDO , MIN, R25, CTD_PRES , I +R27, SBE41CP_IDO , MIN, R25, CTD_TEMP , I +R27, SBE41CP_IDO , MIN, R25, CTD_CNDC , I +R27, SBE41CP_IDO , MIN, R25, IDO_DOXY , I +R27, SBE41N , MIN, R25, CTD_PRES , I +R27, SBE41N , MIN, R25, CTD_TEMP , I +R27, SBE41N , MIN, R25, CTD_CNDC , I +R27, SBE43_IDO , MIN, R25, IDO_DOXY , I +R27, SBE43F_IDO , MIN, R25, IDO_DOXY , I +R27, SBE43I , MIN, R25, IDO_DOXY , I +R27, SBE61 , MIN, R25, CTD_PRES , I +R27, SBE61 , MIN, R25, CTD_TEMP , I +R27, SBE61 , MIN, R25, CTD_CNDC , I +R27, SBE63_OPTODE , MIN, R25, OPTODE_DOXY , I +R27, SBE83_OPTODE , MIN, R25, OPTODE_DOXY , I +R27, SEAFET , MIN, R25, TRANSISTOR_PH , I +R27, SEAPOINT_TURBIDITY_METER , MIN, R25, BACKSCATTERINGMETER_TURBIDITY , I +R27, SEASCAN_SSTD , MIN, R25, CTD_PRES , I +R27, SUNA , MIN, R25, SPECTROPHOTOMETER_NITRATE , I +R27, SUNA , MIN, R25, SPECTROPHOTOMETER_BISULFIDE , I +R27, SUNA_V2 , MIN, R25, SPECTROPHOTOMETER_NITRATE , I +R27, SUNA_V2 , MIN, R25, SPECTROPHOTOMETER_BISULFIDE , I \ No newline at end of file diff --git a/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_2.txt b/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_2.txt new file mode 100644 index 000000000..dd79316c2 --- /dev/null +++ b/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_2.txt @@ -0,0 +1,25 @@ +R27, ECO_BB3 , MIN, R25, BACKSCATTERINGMETER_BBP470 , I +R27, ECO_FLBB2 , MIN, R25, BACKSCATTERINGMETER_BBP470 , I +R27, ECO_BB3 , MIN, R25, BACKSCATTERINGMETER_BBP532 , I +R27, ECO_FLBB2 , MIN, R25, BACKSCATTERINGMETER_BBP532 , I +R27, MCOMS_FLBB2 , MIN, R25, BACKSCATTERINGMETER_BBP532 , I +R27, ECO_BB3 , MIN, R25, BACKSCATTERINGMETER_BBP700 , I +R27, ECO_FLBB , MIN, R25, BACKSCATTERINGMETER_BBP700 , I +R27, ECO_FLBB2 , MIN, R25, BACKSCATTERINGMETER_BBP700 , I +R27, ECO_FLBB_2K , MIN, R25, BACKSCATTERINGMETER_BBP700 , I +R27, ECO_FLBB_AP2 , MIN, R25, BACKSCATTERINGMETER_BBP700 , I +R27, ECO_FLBBCD , MIN, R25, BACKSCATTERINGMETER_BBP700 , I +R27, ECO_FLBBCD_AP2 , MIN, R25, BACKSCATTERINGMETER_BBP700 , I +R27, ECO_FLBBFL , MIN, R25, BACKSCATTERINGMETER_BBP700 , I +R27, ECO_FLBBFL_AP2 , MIN, R25, BACKSCATTERINGMETER_BBP700 , I +R27, MCOMS_FLBB2 , MIN, R25, BACKSCATTERINGMETER_BBP700 , I +R27, MCOMS_FLBBCD , MIN, R25, BACKSCATTERINGMETER_BBP700 , I +R27, SATLANTIC_OCR504_ICSW , MIN, R25, RADIOMETER_DOWN_IRR380 , I +R27, SATLANTIC_OCR504_ICSW , MIN, R25, RADIOMETER_DOWN_IRR412 , I +R27, SATLANTIC_OCR504_ICSW , MIN, R25, RADIOMETER_DOWN_IRR443 , I +R27, SATLANTIC_OCR504_ICSW , MIN, R25, RADIOMETER_DOWN_IRR490 , I +R27, SATLANTIC_OCR504_ICSW , MIN, R25, RADIOMETER_DOWN_IRR555 , I +R27, SATLANTIC_OCR504_R10W , MIN, R25, RADIOMETER_UP_RAD412 , I +R27, SATLANTIC_OCR504_R10W , MIN, R25, RADIOMETER_UP_RAD443 , I +R27, SATLANTIC_OCR504_R10W , MIN, R25, RADIOMETER_UP_RAD490 , I +R27, SATLANTIC_OCR504_R10W , MIN, R25, RADIOMETER_UP_RAD555 , I diff --git a/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_2b.txt b/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_2b.txt new file mode 100644 index 000000000..3b7098ff8 --- /dev/null +++ b/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_2b.txt @@ -0,0 +1 @@ +R27, RAMSES_ACC, MIN, R25,RADIOMETER_DOWN_IRR , I \ No newline at end of file diff --git a/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_3.txt b/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_3.txt new file mode 100644 index 000000000..8358155ff --- /dev/null +++ b/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_3.txt @@ -0,0 +1 @@ +R27, DRUCK_10153PSIA , MIN, R25, CTD_PRES , I \ No newline at end of file diff --git a/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_3b.txt b/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_3b.txt new file mode 100644 index 000000000..2ae333bb5 --- /dev/null +++ b/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_3b.txt @@ -0,0 +1,10 @@ +R27, DSB301-10-C85 , MIN, R25, CTD_PRES , I +R27, OCR504_ICSW , MIN, R25, RADIOMETER_DOWN_IRR380 , I +R27, OCR504_ICSW , MIN, R25, RADIOMETER_DOWN_IRR412 , I +R27, OCR504_ICSW , MIN, R25, RADIOMETER_DOWN_IRR443 , I +R27, OCR504_ICSW , MIN, R25, RADIOMETER_DOWN_IRR490 , I +R27, OCR504_ICSW , MIN, R25, RADIOMETER_DOWN_IRR555 , I +R27, OCR504_R10W , MIN, R25, RADIOMETER_UP_RAD412 , I +R27, OCR504_R10W , MIN, R25, RADIOMETER_UP_RAD443 , I +R27, OCR504_R10W , MIN, R25, RADIOMETER_UP_RAD490 , I +R27, OCR504_R10W , MIN, R25, RADIOMETER_UP_RAD555 , I \ No newline at end of file diff --git a/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_4_cndc.txt b/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_4_cndc.txt new file mode 100644 index 000000000..a5012c2dc --- /dev/null +++ b/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_4_cndc.txt @@ -0,0 +1,42 @@ +R27, SBE41_V2.5, MIN, R25,CTD_CNDC, I +R27, SBE41_V2.6, MIN, R25,CTD_CNDC, I +R27, SBE41_V3, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V1, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V1.1, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V1.2, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V1.2a, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V1.3, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V1.3b, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V1.4, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V1.5, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V1.7, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V1.8, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V1.9, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V1.9a, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V2, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V3, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V3.0a, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V3.0c, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V4.4.0, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V5.0.1, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V5.3.0, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V5.3.1, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V5.3.2, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V7.2.3, MIN, R25,CTD_CNDC, I +R27, SBE41CP_V7.2.5, MIN, R25,CTD_CNDC, I +R27, SBE41N_V5.3.0, MIN, R25,CTD_CNDC, I +R27, SBE41N_V5.3.4, MIN, R25,CTD_CNDC, I +R27, SBE41N_V5.4.0, MIN, R25,CTD_CNDC, I +R27, SBE61_V4.5.2, MIN, R25,CTD_CNDC, I +R27, SBE61_V4.5.3, MIN, R25,CTD_CNDC, I +R27, SBE61_V5.0.0, MIN, R25,CTD_CNDC, I +R27, SBE61_V5.0.1, MIN, R25,CTD_CNDC, I +R27, SBE61_V5.0.10, MIN, R25,CTD_CNDC, I +R27, SBE61_V5.0.12, MIN, R25,CTD_CNDC, I +R27, SBE61_V5.0.2, MIN, R25,CTD_CNDC, I +R27, SBE61_V5.0.3, MIN, R25,CTD_CNDC, I +R27, SBE61_V5.0.9, MIN, R25,CTD_CNDC, I +R27, SBE41_IDO_V1.0c, MIN, R25,CTD_CNDC, I +R27, SBE41_IDO_V2.0, MIN, R25,CTD_CNDC, I +R27, SBE41_IDO_V3.0, MIN, R25,CTD_CNDC, I +R27, SBE41CP_IDO_V2.0b, MIN, R25,CTD_CNDC, I \ No newline at end of file diff --git a/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_4_ido_doxy.txt b/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_4_ido_doxy.txt new file mode 100644 index 000000000..9f41276bc --- /dev/null +++ b/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_4_ido_doxy.txt @@ -0,0 +1,4 @@ +R27, SBE41_IDO_V1.0c, MIN, R25,IDO_DOXY, I +R27, SBE41_IDO_V2.0, MIN, R25,IDO_DOXY, I +R27, SBE41_IDO_V3.0, MIN, R25,IDO_DOXY, I +R27, SBE41CP_IDO_V2.0b, MIN, R25,IDO_DOXY, I \ No newline at end of file diff --git a/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_4_pres.txt b/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_4_pres.txt new file mode 100644 index 000000000..765950382 --- /dev/null +++ b/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_4_pres.txt @@ -0,0 +1,42 @@ +R27, SBE41_V2.5, MIN, R25,CTD_PRES, I +R27, SBE41_V2.6, MIN, R25,CTD_PRES, I +R27, SBE41_V3, MIN, R25,CTD_PRES, I +R27, SBE41CP_V1, MIN, R25,CTD_PRES, I +R27, SBE41CP_V1.1, MIN, R25,CTD_PRES, I +R27, SBE41CP_V1.2, MIN, R25,CTD_PRES, I +R27, SBE41CP_V1.2a, MIN, R25,CTD_PRES, I +R27, SBE41CP_V1.3, MIN, R25,CTD_PRES, I +R27, SBE41CP_V1.3b, MIN, R25,CTD_PRES, I +R27, SBE41CP_V1.4, MIN, R25,CTD_PRES, I +R27, SBE41CP_V1.5, MIN, R25,CTD_PRES, I +R27, SBE41CP_V1.7, MIN, R25,CTD_PRES, I +R27, SBE41CP_V1.8, MIN, R25,CTD_PRES, I +R27, SBE41CP_V1.9, MIN, R25,CTD_PRES, I +R27, SBE41CP_V1.9a, MIN, R25,CTD_PRES, I +R27, SBE41CP_V2, MIN, R25,CTD_PRES, I +R27, SBE41CP_V3, MIN, R25,CTD_PRES, I +R27, SBE41CP_V3.0a, MIN, R25,CTD_PRES, I +R27, SBE41CP_V3.0c, MIN, R25,CTD_PRES, I +R27, SBE41CP_V4.4.0, MIN, R25,CTD_PRES, I +R27, SBE41CP_V5.0.1, MIN, R25,CTD_PRES, I +R27, SBE41CP_V5.3.0, MIN, R25,CTD_PRES, I +R27, SBE41CP_V5.3.1, MIN, R25,CTD_PRES, I +R27, SBE41CP_V5.3.2, MIN, R25,CTD_PRES, I +R27, SBE41CP_V7.2.3, MIN, R25,CTD_PRES, I +R27, SBE41CP_V7.2.5, MIN, R25,CTD_PRES, I +R27, SBE41N_V5.3.0, MIN, R25,CTD_PRES, I +R27, SBE41N_V5.3.4, MIN, R25,CTD_PRES, I +R27, SBE41N_V5.4.0, MIN, R25,CTD_PRES, I +R27, SBE61_V4.5.2, MIN, R25,CTD_PRES, I +R27, SBE61_V4.5.3, MIN, R25,CTD_PRES, I +R27, SBE61_V5.0.0, MIN, R25,CTD_PRES, I +R27, SBE61_V5.0.1, MIN, R25,CTD_PRES, I +R27, SBE61_V5.0.10, MIN, R25,CTD_PRES, I +R27, SBE61_V5.0.12, MIN, R25,CTD_PRES, I +R27, SBE61_V5.0.2, MIN, R25,CTD_PRES, I +R27, SBE61_V5.0.3, MIN, R25,CTD_PRES, I +R27, SBE61_V5.0.9, MIN, R25,CTD_PRES, I +R27, SBE41_IDO_V1.0c, MIN, R25,CTD_PRES, I +R27, SBE41_IDO_V2.0, MIN, R25,CTD_PRES, I +R27, SBE41_IDO_V3.0, MIN, R25,CTD_PRES, I +R27, SBE41CP_IDO_V2.0b, MIN, R25,CTD_PRES, I \ No newline at end of file diff --git a/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_4_temp.txt b/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_4_temp.txt new file mode 100644 index 000000000..05430e673 --- /dev/null +++ b/argopy/static/assets/nvs_R25_R27/NVS_R25_R27_mappings_4_temp.txt @@ -0,0 +1,42 @@ +R27, SBE41_V2.5, MIN, R25,CTD_TEMP, I +R27, SBE41_V2.6, MIN, R25,CTD_TEMP, I +R27, SBE41_V3, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V1, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V1.1, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V1.2, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V1.2a, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V1.3, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V1.3b, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V1.4, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V1.5, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V1.7, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V1.8, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V1.9, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V1.9a, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V2, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V3, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V3.0a, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V3.0c, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V4.4.0, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V5.0.1, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V5.3.0, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V5.3.1, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V5.3.2, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V7.2.3, MIN, R25,CTD_TEMP, I +R27, SBE41CP_V7.2.5, MIN, R25,CTD_TEMP, I +R27, SBE41N_V5.3.0, MIN, R25,CTD_TEMP, I +R27, SBE41N_V5.3.4, MIN, R25,CTD_TEMP, I +R27, SBE41N_V5.4.0, MIN, R25,CTD_TEMP, I +R27, SBE61_V4.5.2, MIN, R25,CTD_TEMP, I +R27, SBE61_V4.5.3, MIN, R25,CTD_TEMP, I +R27, SBE61_V5.0.0, MIN, R25,CTD_TEMP, I +R27, SBE61_V5.0.1, MIN, R25,CTD_TEMP, I +R27, SBE61_V5.0.10, MIN, R25,CTD_TEMP, I +R27, SBE61_V5.0.12, MIN, R25,CTD_TEMP, I +R27, SBE61_V5.0.2, MIN, R25,CTD_TEMP, I +R27, SBE61_V5.0.3, MIN, R25,CTD_TEMP, I +R27, SBE61_V5.0.9, MIN, R25,CTD_TEMP, I +R27, SBE41_IDO_V1.0c, MIN, R25,CTD_TEMP, I +R27, SBE41_IDO_V2.0, MIN, R25,CTD_TEMP, I +R27, SBE41_IDO_V3.0, MIN, R25,CTD_TEMP, I +R27, SBE41CP_IDO_V2.0b, MIN, R25,CTD_TEMP, I \ No newline at end of file diff --git a/argopy/static/assets/nvs_R25_R27/README.md b/argopy/static/assets/nvs_R25_R27/README.md new file mode 100644 index 000000000..55c3a1888 --- /dev/null +++ b/argopy/static/assets/nvs_R25_R27/README.md @@ -0,0 +1,6 @@ +# Sensor models and types + +This folder hold mapping data to get R25 sensor types from R27 sensor models. + +Relevant issues from ADMT/AVTT: +- https://github.com/OneArgo/ArgoVocabs/issues/156 diff --git a/argopy/static/css/argopy.css b/argopy/static/css/argopy.css new file mode 100644 index 000000000..ae1d26703 --- /dev/null +++ b/argopy/static/css/argopy.css @@ -0,0 +1,11 @@ +/* CSS stylesheet for argopy style +* +*/ + +:root { + --argopy_cyan:rgb(18, 235, 229); + --argopy_blue:rgb(16, 137, 182); + --argopy_darkblue:rgb(10, 89, 162); + --argopy_yellow:rgb(229, 174, 41); + --argopy_darkyellow:rgb(224, 158, 37); +} diff --git a/argopy/static/css/oemsensor.css b/argopy/static/css/oemsensor.css new file mode 100644 index 000000000..3f829d82a --- /dev/null +++ b/argopy/static/css/oemsensor.css @@ -0,0 +1,79 @@ +/* CSS stylesheet for displaying OemArgoSensorMetaData object in a Jupyter notebook cell +* +*/ +.oemsensor { + font-family: Arial, sans-serif; + line-height: 1em; + overflow: hidden; +} +h1.oemsensor { + font-size: 0.9em; + font-weight: bold; + margin-bottom: 0.4em; + border-bottom: 1px solid var(--argopy_darkyellow); + padding-bottom: 0.2em; + color: var(--argopy_darkblue); +} +h2.oemsensor { + font-size: 0.9em !important; + font-weight: bold; + margin-top: 0.2em; + margin-bottom: 0.2em; + color: var(--argopy_darkblue); +} +p.oemsensor { + font-size: small; + color: var(--argopy_darkblue); +} +table.oemsensor { + width: 100%; + border-collapse: collapse; + margin-bottom: 0.8em; + font-size: 0.9em; + color: var(--argopy_blue); +} +table.oemsensor thead{ + color: var(--argopy_darkblue); +} +th.oemsensor, td.oemsensor { + padding: 0.4em 0.6em; + text-align: left; + border: 1px solid #eee; +} +th.oemsensor { + background-color: #f8f8f8; + font-weight: bold; +} +tr.parameter-row.oemsensor td.calibration { + cursor: pointer; +} +/*tr.parameter-row.oemsensor:hover {*/ +/* background-color: var(--argopy_yellow);*/ +/*}*/ + +td.oemsensor a:link a:visited a:active a:hover a:active{ + color: var(--argopy_blue); +} + +.parameter-details.oemsensor { + display: none; + background-color: #f9f9f9; +} +.parameter-details.oemsensor td { + padding: 0.6em; +} +.details-content.oemsensor { + margin: 0; + padding: 0.4em 0; +} +.details-content.oemsensor p { + margin: 0.2em 0; +} +pre.oemsensor { + background-color: #f8f8f8; + padding: 0.6em; + border-radius: 3px; + font-size: 0.85em; + overflow-x: auto; + color: var(--argopy_blue); +} diff --git a/argopy/stores/float/spec.py b/argopy/stores/float/spec.py index 7c0aceb87..5de5baf7a 100644 --- a/argopy/stores/float/spec.py +++ b/argopy/stores/float/spec.py @@ -9,7 +9,7 @@ import logging import numpy as np -from ...errors import InvalidOption +from ...errors import InvalidOption, OptionValueError from ...plot import dashboard from ...utils import check_wmo, argo_split_path, shortcut2gdac from ...options import OPTIONS @@ -123,10 +123,10 @@ def load_metadata(self): raise NotImplementedError("Not implemented") def load_metadata_from_meta_file(self): - """Method to load float meta-data from the netcdf file""" + """Method to load float meta-data""" data = {} - ds = self.dataset("meta") + ds = self.dataset["meta"] data.update( { "deployment": { @@ -308,7 +308,7 @@ def open_dataset(self, name: str = "prof", cast: bool = True, **kwargs) -> xr.Da """ if name not in self.ls_dataset(): - raise ValueError( + raise OptionValueError( "Dataset '%s' not found. Available dataset for this float are: %s" % (name, self.ls_dataset().keys()) ) @@ -324,7 +324,7 @@ def open_dataset(self, name: str = "prof", cast: bool = True, **kwargs) -> xr.Da def dataset(self, name: str = "prof"): if name not in self._dataset: - self.open_dataset(name) # will commit this dataset to self._dataset dict + self._dataset[name] = self.open_dataset(name) return self._dataset[name] @property @@ -393,3 +393,17 @@ def __repr__(self): ) return "\n".join(summary) + + @property + def sensors(self): + """Return a :class:`pandas.DataFrame` describing float sensors + + ! This is experimental and may change in the future. + """ + # We can't call on: + # >>> ArgoSensor(fs=self.fs).from_wmo(self.WMO) + # because ArgoSensor import ArgoFloat for the iterator method and + # that would be circular import. + # We need to implement here a self.metadata -> dataframe transform + # like it's done in ArgoSensor from json data + pass \ No newline at end of file diff --git a/argopy/stores/implementations/http.py b/argopy/stores/implementations/http.py index 32f2425d7..bc18757e2 100644 --- a/argopy/stores/implementations/http.py +++ b/argopy/stores/implementations/http.py @@ -42,12 +42,23 @@ class httpstore(ArgoStoreProto): This store intends to make argopy safer to failures from http requests and to provide higher levels methods to work with our datasets. Key methods are: + - :class:`httpstore.open` + - :class:`httpstore.exists` - :class:`httpstore.download_url` - :class:`httpstore.open_dataset` - - :class:`httpstore.open_json` - :class:`httpstore.open_mfdataset` + - :class:`httpstore.open_json` - :class:`httpstore.open_mfjson` - :class:`httpstore.read_csv` + - :class:`httpstore.post` + + Examples + -------- + .. code-block:: python + :caption: How to add a specific header key + + from argopy.stores import httpstore + fs = httpstore(client_kwargs={'headers': {'Authorization': 'Token'}}) """ @@ -505,14 +516,14 @@ def finalize(obj_list, **kwargs): # log.info('drop_variables_not_in_all_datasets') ds_list = drop_variables_not_in_all_datasets(ds_list) - log.info("Nb of dataset to concat: %i" % len(ds_list)) + # log.info("Nb of dataset to concat: %i" % len(ds_list)) # log.debug(concat_dim) # for ds in ds_list: # log.debug(ds[concat_dim]) - log.info( - "Dataset sizes before concat: %s" - % [len(ds[concat_dim]) for ds in ds_list] - ) + # log.info( + # "Dataset sizes before concat: %s" + # % [len(ds[concat_dim]) for ds in ds_list] + # ) ds = xr.concat( ds_list, dim=concat_dim, @@ -520,7 +531,7 @@ def finalize(obj_list, **kwargs): coords="all", compat="override", # skip comparing and pick variable from first dataset ) - log.info("Dataset size after concat: %i" % len(ds[concat_dim])) + # log.info("Dataset size after concat: %i" % len(ds[concat_dim])) return ds, True else: ds_list = [v for v in dict(sorted(obj_list.items())).values()] @@ -1114,7 +1125,11 @@ def open_mfjson( futures = concurrent.futures.as_completed(future_to_url) if progress: futures = tqdm( - futures, total=len(urls), disable="disable" in [progress] + futures, + total=len(urls), + disable="disable" in [progress], + unit = kwargs.pop("progress_unit", "it"), + desc = kwargs.pop("progress_desc", None), ) for future in futures: diff --git a/argopy/stores/implementations/local.py b/argopy/stores/implementations/local.py index 9bbd7b077..6cdee593f 100644 --- a/argopy/stores/implementations/local.py +++ b/argopy/stores/implementations/local.py @@ -11,6 +11,8 @@ from pathlib import Path import warnings from netCDF4 import Dataset +from subprocess import Popen, PIPE +from platform import system from ...options import OPTIONS from ...errors import InvalidMethod, DataNotFound @@ -30,7 +32,9 @@ class filestore(ArgoStoreProto): protocol = "file" - def open_json(self, url, errors: Literal['raise', 'silent', 'ignore'] = 'raise', **kwargs) -> Any: + def open_json( + self, url, errors: Literal["raise", "silent", "ignore"] = "raise", **kwargs + ) -> Any: """Open and process a json document from a path Steps performed: @@ -88,12 +92,12 @@ def open_json(self, url, errors: Literal['raise', 'silent', 'ignore'] = 'raise', return js def open_dataset( - self, - path, - errors: Literal["raise", "ignore", "silent"] = "raise", - lazy: bool = False, - xr_opts: dict = {}, - **kwargs, + self, + path, + errors: Literal["raise", "ignore", "silent"] = "raise", + lazy: bool = False, + xr_opts: dict = {}, + **kwargs, ) -> xr.Dataset: """Create a :class:`xarray.Dataset` from a local path pointing to a netcdf file @@ -114,6 +118,7 @@ def open_dataset( ------- :class:`xarray.Dataset` """ + def load_in_memory(path, errors="raise", xr_opts={}): """ Returns @@ -155,7 +160,9 @@ def load_lazily(path, errors="raise", xr_opts={}, akoverwrite: bool = False): "backend_kwargs": { "consolidated": False, "storage_options": { - "fo": self.ak.to_kerchunk(path, overwrite=akoverwrite), # codespell:ignore + "fo": self.ak.to_kerchunk( + path, overwrite=akoverwrite + ), # codespell:ignore "remote_protocol": fsspec.core.split_protocol(path)[0], }, }, @@ -197,7 +204,7 @@ def load_lazily(path, errors="raise", xr_opts={}, akoverwrite: bool = False): else: target = target if isinstance(target, bytes) else target.getbuffer() - ds = Dataset(None, memory=target, diskless=True, mode='r') + ds = Dataset(None, memory=target, diskless=True, mode="r") self.register(path) return ds @@ -395,9 +402,9 @@ def open_mfdataset( results, dim=concat_dim, data_vars="minimal", # Only data variables in which the dimension already appears are included. - coords="all", # All coordinate variables will be concatenated, except those corresponding - # to other dimensions. - compat="override", # skip comparing and pick variable from first dataset, + coords="all", # All coordinate variables will be concatenated, except those corresponding + # to other dimensions. + compat="override", # skip comparing and pick variable from first dataset, ) return ds else: @@ -421,3 +428,69 @@ def read_csv(self, path, **kwargs): with self.open(path) as of: df = pd.read_csv(of, **kwargs) return df + + def open_subprocess(self, filepath): + """Open file with the default program in a subprocess. + + As being called in a subprocess, it will not block the current one. + This method runs the command on a subprocess using the default system shell. + + Parameters + ---------- + filepath: str + Path to the file to open + + Returns + ------- + subprocess + The pointer the subprocess returned by the Popen call + + Examples + -------- + The subprocess will run in background so you won't be able to bring back + the result code, but the current log can be obtained via the Popen's PIPE: + + .. code-block:: python + + from argopy.stores import filestore + + # Run as subprocess + subproc = filestore().open_subprocess("sensor_certificate.pdf") + + # Get the stdout for the subprocess + print(subproc.stdout) + + # Get the stderr for the subprocess + print(subproc.stderr) + + Notes + ----- + Source: https://gist.github.com/ytturi/0c23ad5ab89154d24c340c2b1cc3432b + """ + + def get_open_command(filepath): + """ + Get the console-like command to open the file + for the current platform: + - Windows: "start {{ filepath }}" + - OS X: "open {{ filepath }}" + - Linux based (wdg): "wdg-open {{ filepath }}" + :param filepath: Path to the file to be opened + :type filepath: string + :return: Command to run from a shell + :rtype: string + """ + OSNAME = system().lower() + if "windows" in OSNAME: + opener = "start" + elif "osx" in OSNAME or "darwin" in OSNAME: + opener = "open" + else: + opener = "xdg-open" + return "{opener} {filepath}".format(opener=opener, filepath=filepath) + + subproc = Popen( + get_open_command(filepath), stdout=PIPE, stderr=PIPE, shell=True + ) + subproc.wait() + return subproc diff --git a/argopy/utils/__init__.py b/argopy/utils/__init__.py index f3b2683de..8704260da 100644 --- a/argopy/utils/__init__.py +++ b/argopy/utils/__init__.py @@ -36,14 +36,13 @@ from .caching import clear_cache, lscache from .monitored_threadpool import MyThreadPoolExecutor as MonitoredThreadPoolExecutor from .chunking import Chunker -from .accessories import Registry, float_wmo, ListStrProperty +from .accessories import Registry, float_wmo, ListStrProperty, NVSrow from .locals import ( # noqa: F401 show_versions, show_options, modified_environ, get_sys_info, # noqa: F401 netcdf_and_hdf5_versions, # noqa: F401 - Asset, ) from .monitors import monitor_status, badge, fetch_status # noqa: F401 from .geo import ( @@ -63,7 +62,7 @@ split_data_mode, ) from .mappers import map_vars_to_dict -from .format import argo_split_path, format_oneline, UriCName, redact, dirfs_relpath +from .format import argo_split_path, format_oneline, UriCName, redact, dirfs_relpath, urnparser, ppliststr from .loggers import warnUnless, log_argopy_callerstack from .carbon import GreenCoding, Github from . import optical_modeling @@ -122,12 +121,12 @@ "Registry", "float_wmo", "ListStrProperty", + "NVSrow", # Locals (environments, versions, systems): "path2assets", "show_versions", "show_options", "modified_environ", - "Asset", # Monitors "monitor_status", # Geo (space/time data utilities) @@ -154,6 +153,8 @@ "dirfs_relpath", "UriCName", "redact", + "urnparser", + "ppliststr", # Loggers: "warnUnless", "log_argopy_callerstack", diff --git a/argopy/utils/accessories.py b/argopy/utils/accessories.py index 3a7624839..565386c0f 100644 --- a/argopy/utils/accessories.py +++ b/argopy/utils/accessories.py @@ -3,8 +3,12 @@ import warnings import logging import copy +from dataclasses import dataclass +from typing import ClassVar +import pandas as pd from .checkers import check_wmo, is_wmo +from .format import urnparser log = logging.getLogger("argopy.utils.accessories") @@ -258,6 +262,95 @@ def copy(self): return self.__copy__() +@dataclass +class NVSrow: + """This proto makes it easier to work with a single NVS table row from a :class:`pd.DataFrame` + + It will turn :class:`pd.DataFrame` columns into class attributes + + Examples + -------- + .. code-block:: python + :caption: Use this prototype to create a NVS table row class + + class SensorType(NVSrow): + reftable = "R25" + @staticmethod + def from_series(obj: pd.Series) -> "SensorType": + return SensorType(obj) + + .. code-block:: python + :caption: Then use the row class + + from argopy import ArgoNVSReferenceTables + + df = ArgoNVSReferenceTables().tbl(25) + row = df[df["altLabel"].apply(lambda x: x == 'CTD')].iloc[0] + + st = SensorType.from_series(row) + + st.name + st.long_name + st.definition + st.deprecated + st.uri + + """ + name: str = "" + """From 'altLabel' column""" + + long_name: str = "" + """From 'prefLabel' column""" + + definition: str = "" + """From 'definition' column""" + + uri: str = "" + """From 'ID' column, typically a link toward NVS specific row entry""" + + urn: str = "" + """From 'urn' column""" + + deprecated: bool = None + """From 'deprecated' column""" + + reftable: ClassVar[str] + """Reference table this row is based on""" + + def __init__(self, row: pd.Series | pd.DataFrame): + if not isinstance(row, pd.Series) and isinstance(row, pd.DataFrame): + row = row.iloc[0] + row = row.to_dict() + self.name = row["altLabel"] + self.long_name = row["prefLabel"] + self.definition = row["definition"] + self.deprecated = row["deprecated"] + self.uri = row["id"] + self.urn = row["urn"] + + @staticmethod + def from_series(obj: pd.Series) -> "NVSrow": + return NVSrow(obj) + + @staticmethod + def from_df(df: pd.DataFrame, txt: str, column: str = 'altLabel') -> "NVSrow": + row = df[df[column].apply(lambda x: str(x) == str(txt))].iloc[0] + return NVSrow(row) + + def __eq__(self, obj): + return self.name == obj + + def __repr__(self): + summary = [f"<{getattr(self, 'reftable', 'n/a')}.{self.urn}>"] + summary.append(f"%12s: {self.name}" % "name") + summary.append(f"%12s: {self.long_name}" % "long_name") + summary.append(f"%12s: {self.urn}" % "urn") + summary.append(f"%12s: {self.uri}" % "uri") + summary.append(f"%12s: {self.deprecated}" % "deprecated") + summary.append("%12s: %s" % ("definition", self.definition)) + return "\n".join(summary) + + class ListStrProperty: """ Descriptor for a list of strings that is able to handle: diff --git a/argopy/utils/format.py b/argopy/utils/format.py index 7bea675e4..0230ce8cf 100644 --- a/argopy/utils/format.py +++ b/argopy/utils/format.py @@ -524,3 +524,46 @@ def _mono2multi(mono_path): new_uri = [_mono2multi(uri)[2:] for uri in flist] new_uri = list(set(new_uri)) return new_uri + + +def urnparser(urn): + """Parsing RFC 8141 compliant uniform resource names (URN) from NVS + SDN stands for SeaDataNet + """ + pp = urn.split(":") + if len(pp) == 4 and pp[0] == 'SDN': + return {'listid': pp[1], 'version': pp[2], 'termid': pp[3]} + else: + raise ValueError(f"This NVS URN '{urn}' does not follow the pattern: 'SDN:listid:version:termid' or 'SDN:listid::termid' for NVS2.0") + + +def ppliststr(l: list[str], last : str = 'and', n : int | None = None) -> str: + """Pretty print a list of strings + + Examples + -------- + .. code-block:: python + + ppliststr(['a', 'b', 'c', 'd']) -> "'a', 'b', 'c' and 'd'" + ppliststr(['a', 'b'], last='or') -> "'a' or 'b'" + ppliststr(['a', 'b', 'c', 'd'], n=3) -> "'a', 'b', 'c' and more ..." + + """ + n = n if n is not None else len(l) + if n == 0: + return "" + + s, ii, m = "", 0, len(l) + while ii < m: + item = l[ii] + if ii == n: + s += f" {last} more ..." + break + if ii == 0: + s += f"'{item}'" + elif ii == len(l) - 1: + s += f" {last} '{item}'" + else: + s += f", '{item}'" + ii += 1 + return s diff --git a/argopy/utils/locals.py b/argopy/utils/locals.py index e7aed0295..53eba07ad 100644 --- a/argopy/utils/locals.py +++ b/argopy/utils/locals.py @@ -187,6 +187,7 @@ def show_versions(file=sys.stdout, conda=False): # noqa: C901 ("s3fs", get_version), ("kerchunk", get_version), ("zarr", get_version), + ("jsonschema", get_version), ] ), "ext.perf": sorted( diff --git a/argopy/utils/schemas/__init__.py b/argopy/utils/schemas/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/argopy/utils/schemas/sensors/__init__.py b/argopy/utils/schemas/sensors/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/argopy/utils/schemas/sensors/repr.py b/argopy/utils/schemas/sensors/repr.py new file mode 100644 index 000000000..29801669d --- /dev/null +++ b/argopy/utils/schemas/sensors/repr.py @@ -0,0 +1,137 @@ +from functools import lru_cache +import importlib + + +try: + from importlib.resources import files # New in version 3.9 +except ImportError: + from pathlib import Path + + files = lambda x: Path( # noqa: E731 + importlib.util.find_spec(x).submodule_search_locations[0] + ) + +from argopy.utils import urnparser + + +STATIC_FILES = ( + ("argopy.static.css", "argopy.css"), + ("argopy.static.css", "oemsensor.css"), +) + + +@lru_cache(None) +def _load_static_files(): + """Lazily load the resource files into memory the first time they are needed""" + return [ + files(package).joinpath(resource).read_text(encoding="utf-8") + for package, resource in STATIC_FILES + ] + + +def urn_html(this_urn): + x = urnparser(this_urn) + if x.get("version") != "": + return f"{x.get('termid', '?')} ({x.get('listid', '?')}, {x.get('version', 'n/a')})" + else: + return f"{x.get('termid', '?')} ({x.get('listid', '?')})" + + +class ParameterDisplay: + + def __init__(self, obj): + self.data = obj + + @property + def css_style(self): + return "\n".join(_load_static_files()) + + @property + def html(self): + param = self.data + + # --- Header --- + header_html = f"

Argo Sensor Metadata for Parameter: {urn_html(param.PARAMETER)}

" + + if param.parameter_vendorinfo is not None: + info = " | ".join( + [ + f"{p} {v}" + for p, v in param.parameter_vendorinfo.items() + ] + ) + header_html += f"

{info}

" + + # --- Parameter details --- + html = """ + + + + + + + + + + + + """ + html += f""" + + + + + + + """ + + PREDEPLOYMENT_CALIB_EQUATION = param._attr2str( + "PREDEPLOYMENT_CALIB_EQUATION" + ).split(";") + PREDEPLOYMENT_CALIB_EQUATION = [ + p.replace("=", " = ") for p in PREDEPLOYMENT_CALIB_EQUATION + ] + PREDEPLOYMENT_CALIB_EQUATION = "
\t".join(PREDEPLOYMENT_CALIB_EQUATION) + + PREDEPLOYMENT_CALIB_COEFFICIENT_LIST = param._attr2str( + "PREDEPLOYMENT_CALIB_COEFFICIENT_LIST" + ) + s = [] + if isinstance(PREDEPLOYMENT_CALIB_COEFFICIENT_LIST, dict): + for key, value in PREDEPLOYMENT_CALIB_COEFFICIENT_LIST.items(): + s.append(f"{key} = {value}") + PREDEPLOYMENT_CALIB_COEFFICIENT_LIST = "
\t".join(s) + + html += f""" + + + + """ + html += "
SensorUnitsAccuracyResolution
{urn_html(param.PARAMETER_SENSOR)}{param._attr2str('PARAMETER_UNITS')}{param._attr2str('PARAMETER_ACCURACY')}{param._attr2str('PARAMETER_RESOLUTION')}
+
+

Calibration Equation:
{PREDEPLOYMENT_CALIB_EQUATION}

+

Calibration Coefficients:
{PREDEPLOYMENT_CALIB_COEFFICIENT_LIST}

+

Calibration Comment:
{param._attr2str('PREDEPLOYMENT_CALIB_COMMENT')}

+

Calibration Date:
{param._attr2str('PREDEPLOYMENT_CALIB_DATE')}

+
+
" + + # --- Vendor Info --- + vendor_html = "" + if param.predeployment_vendorinfo: + vendor_html = f""" +

Pre-deployment Vendor Info:

+ 
{param.predeployment_vendorinfo}
+ """ + + # --- Combine All HTML --- + full_html = f""" + \n + {header_html}\n + {html}\n + {vendor_html} + """ + return full_html + + def _repr_html_(self): + return self.html diff --git a/argopy/utils/schemas/sensors/spec.py b/argopy/utils/schemas/sensors/spec.py new file mode 100644 index 000000000..6ea6884f1 --- /dev/null +++ b/argopy/utils/schemas/sensors/spec.py @@ -0,0 +1,198 @@ +from dataclasses import dataclass +from typing import Dict, Optional, Any +from html import escape + +from argopy.options import OPTIONS +from argopy.utils import urnparser +from argopy.utils.schemas.sensors.repr import ParameterDisplay + + +@dataclass +class SensorInfo: + created_by: str + date_creation: str # ISO 8601 datetime string + link: str + format_version: str + contents: str + + # Made optional to accommodate errors in OEM data + sensor_described: str = None + + def _attr2str(self, x): + """Return a class attribute, or 'n/a' if it's None or "".""" + value = getattr(self, x, None) + if value is None: + return "n/a" + elif type(value) is str: + return value if value and value.strip() else "n/a" + else: + return value + + +@dataclass +class Context: + SDN_R03: str = "http://vocab.nerc.ac.uk/collection/R03/current/" + SDN_R25: str = "http://vocab.nerc.ac.uk/collection/R25/current/" + SDN_R26: str = "http://vocab.nerc.ac.uk/collection/R26/current/" + SDN_R27: str = "http://vocab.nerc.ac.uk/collection/R27/current/" + SDN_L22: str = "http://vocab.nerc.ac.uk/collection/L22/current/" + + +@dataclass +class Sensor: + SENSOR: str # SDN:R25::CTD_PRES + SENSOR_MAKER: str # SDN:R26::RBR + SENSOR_MODEL: str # SDN:R27::RBR_PRES_A + SENSOR_SERIAL_NO: str + + # FIRMWARE VERSION attributes are temporarily optional to handle the wrong key used by RBR + # see https://github.com/euroargodev/sensor_metadata_json/issues/20 + SENSOR_FIRMWARE_VERSION: str = None # RBR, used in sensor schema 0.2.0 + SENSOR_MODEL_FIRMWARE: str = None # Correct schema key, used in sensor schema 0.4.0 + + sensor_vendorinfo: Optional[Dict[str, Any]] = None + + @property + def SENSOR_uri(self): + urnparts = urnparser(self.SENSOR) + return f"{OPTIONS['nvs']}/{urnparts['listid']}/current/{urnparts['termid']}" + + @property + def SENSOR_MAKER_uri(self): + urnparts = urnparser(self.SENSOR_MAKER) + return f"{OPTIONS['nvs']}/{urnparts['listid']}/current/{urnparts['termid']}" + + @property + def SENSOR_MODEL_uri(self): + urnparts = urnparser(self.SENSOR_MODEL) + return f"{OPTIONS['nvs']}/{urnparts['listid']}/current/{urnparts['termid']}" + + def _attr2str(self, x): + """Return a class attribute, or 'n/a' if it's None, {} or "".""" + value = getattr(self, x, None) + if value is None: + return "n/a" + elif type(value) is str: + return value if value and value.strip() else "n/a" + elif type(value) is dict: + if len(value.keys()) == 0: + return "n/a" + else: + return value + else: + return value + + def __repr__(self): + + def key2str(d, x): + """Return a dict value as a string, or 'n/a' if it's None or empty.""" + value = d.get(x, None) + return value if value and value.strip() else "n/a" + + summary = [f"<{self.SENSOR}><{self.SENSOR_SERIAL_NO}>"] + summary.append(f" SENSOR: {self.SENSOR} ({self.SENSOR_uri})") + summary.append(f" SENSOR_MAKER: {self.SENSOR_MAKER} ({self.SENSOR_MAKER_uri})") + summary.append(f" SENSOR_MODEL: {self.SENSOR_MODEL} ({self.SENSOR_MODEL_uri})") + if getattr(self, "SENSOR_MODEL_FIRMWARE", None) is None: + summary.append( + f" SENSOR_FIRMWARE_VERSION: {self._attr2str('SENSOR_FIRMWARE_VERSION')} (but should be 'SENSOR_MODEL_FIRMWARE') " + ) + else: + summary.append( + f" SENSOR_MODEL_FIRMWARE: {self._attr2str('SENSOR_MODEL_FIRMWARE')}" + ) + if getattr(self, "sensor_vendorinfo", None) is not None: + summary.append(f" sensor_vendorinfo:") + for key in self.sensor_vendorinfo.keys(): + summary.append(f" - {key}: {key2str(self.sensor_vendorinfo, key)}") + else: + summary.append(f" sensor_vendorinfo: None") + return "\n".join(summary) + + +@dataclass +class Parameter: + PARAMETER: str # SDN:R03::PRES + PARAMETER_SENSOR: str # SDN:R25::CTD_PRES + PARAMETER_ACCURACY: str + PARAMETER_RESOLUTION: str + PREDEPLOYMENT_CALIB_COEFFICIENT_LIST: Dict[str, str] + + # Made optional to accommodate errors in OEM data + PARAMETER_UNITS: str = None + PREDEPLOYMENT_CALIB_EQUATION: str = None + PREDEPLOYMENT_CALIB_COMMENT: str = None + PREDEPLOYMENT_CALIB_DATE: str = None + + parameter_vendorinfo: Optional[Dict[str, Any]] = None + predeployment_vendorinfo: Optional[Dict[str, Any]] = None + + @property + def PARAMETER_uri(self): + urnparts = urnparser(self.PARAMETER) + return f"{OPTIONS['nvs']}/{urnparts['listid']}/current/{urnparts['termid']}" + + @property + def PARAMETER_SENSOR_uri(self): + urnparts = urnparser(self.PARAMETER_SENSOR) + return f"{OPTIONS['nvs']}/{urnparts['listid']}/current/{urnparts['termid']}" + + def _attr2str(self, x): + """Return a class attribute, or 'n/a' if it's None, {} or "".""" + value = getattr(self, x, None) + if value is None: + return "n/a" + elif type(value) is str: + return value if value and value.strip() else "n/a" + elif type(value) is dict: + if len(value.keys()) == 0: + return "n/a" + else: + return value + else: + return value + + @property + def _has_calibration_data(self): + s = "".join( + [ + str(self._attr2str(key)) + for key in [ + "PREDEPLOYMENT_CALIB_EQUATION", + "PREDEPLOYMENT_CALIB_COEFFICIENT_LIST", + "PREDEPLOYMENT_CALIB_COMMENT", + "PREDEPLOYMENT_CALIB_DATE", + ] + if self._attr2str(key) != "n/a" + ] + ) + return len(s) > 0 + + def __repr__(self): + + summary = [f"<{self.PARAMETER}>"] + summary.append(f" PARAMETER: {self.PARAMETER} ({self.PARAMETER_uri})") + summary.append( + f" PARAMETER_SENSOR: {self.PARAMETER_SENSOR} ({self.PARAMETER_SENSOR_uri})" + ) + + for key in ["UNITS", "ACCURACY", "RESOLUTION"]: + p = f"PARAMETER_{key}" + summary.append(f" {key}: {self._attr2str(p)}") + + summary.append(f" PREDEPLOYMENT CALIBRATION:") + for key in ["EQUATION", "COEFFICIENT", "COMMENT", "DATE"]: + p = f"PREDEPLOYMENT_CALIB_{key}" + summary.append(f" - {key}: {self._attr2str(p)}") + + for key in ["parameter_vendorinfo", "predeployment_vendorinfo"]: + if getattr(self, key, None) is not None: + summary.append(f" {key}: {self._attr2str(key)}") + else: + summary.append(f" {key}: None") + return "\n".join(summary) + + def _repr_html_(self): + if OPTIONS["display_style"] == "text": + return f"
{escape(repr(self))}
" + return ParameterDisplay(self).html diff --git a/ci/requirements/py3.11-all-free.yml b/ci/requirements/py3.11-all-free.yml index 468d795d6..748e585d7 100644 --- a/ci/requirements/py3.11-all-free.yml +++ b/ci/requirements/py3.11-all-free.yml @@ -24,8 +24,10 @@ dependencies: # EXT.FILES: - boto3 + - jsonschema - kerchunk - numcodecs +# - pydantic - s3fs - zarr diff --git a/ci/requirements/py3.11-all-pinned.yml b/ci/requirements/py3.11-all-pinned.yml index bed18402b..bdae5c43c 100644 --- a/ci/requirements/py3.11-all-pinned.yml +++ b/ci/requirements/py3.11-all-pinned.yml @@ -24,8 +24,10 @@ dependencies: # EXT.FILES: - boto3 = 1.38.27 + - jsonschema = 4.25.1 - kerchunk = 0.2.8 - numcodecs = 0.16.1 +# - pydantic = 2.12.0 - s3fs = 2025.5.1 - zarr = 3.0.10 diff --git a/ci/requirements/py3.12-all-free.yml b/ci/requirements/py3.12-all-free.yml index 0859b8066..f6612a5d1 100644 --- a/ci/requirements/py3.12-all-free.yml +++ b/ci/requirements/py3.12-all-free.yml @@ -24,8 +24,10 @@ dependencies: # EXT.FILES: - boto3 + - jsonschema - kerchunk - numcodecs +# - pydantic - s3fs - zarr diff --git a/ci/requirements/py3.12-all-pinned.yml b/ci/requirements/py3.12-all-pinned.yml index 7adccf417..0b153a282 100644 --- a/ci/requirements/py3.12-all-pinned.yml +++ b/ci/requirements/py3.12-all-pinned.yml @@ -24,8 +24,10 @@ dependencies: # EXT.FILES: - boto3 = 1.38.27 + - jsonschema - kerchunk = 0.2.8 - numcodecs = 0.16.1 +# - pydantic - s3fs = 2025.5.1 - zarr = 3.0.10 diff --git a/cli/update_json_assets b/cli/update_json_assets index d4dadbeb3..e53991474 100755 --- a/cli/update_json_assets +++ b/cli/update_json_assets @@ -112,6 +112,7 @@ def asset_data_types(): "SENSOR_MAKER", "SENSOR_MODEL", "SENSOR_SERIAL_NO", + "SENSOR_FIRMWARE_VERSION", "PARAMETER_SENSOR", "PARAMETER_UNITS", "PARAMETER_ACCURACY", diff --git a/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.iterfloats_with.rst b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.iterfloats_with.rst new file mode 100644 index 000000000..445cc47d0 --- /dev/null +++ b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.iterfloats_with.rst @@ -0,0 +1,6 @@ +argopy.ArgoSensor.iterfloats\_with +================================== + +.. currentmodule:: argopy + +.. automethod:: ArgoSensor.iterfloats_with \ No newline at end of file diff --git a/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.maker.hint.rst b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.maker.hint.rst new file mode 100644 index 000000000..bdde55df4 --- /dev/null +++ b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.maker.hint.rst @@ -0,0 +1,6 @@ +argopy.ArgoSensor.ref.maker.hint +================================ + +.. currentmodule:: argopy + +.. autoaccessormethod:: ArgoSensor.ref.maker.hint \ No newline at end of file diff --git a/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.maker.to_dataframe.rst b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.maker.to_dataframe.rst new file mode 100644 index 000000000..9f92f15b5 --- /dev/null +++ b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.maker.to_dataframe.rst @@ -0,0 +1,6 @@ +argopy.ArgoSensor.ref.maker.to_dataframe +======================================== + +.. currentmodule:: argopy + +.. autoaccessormethod:: ArgoSensor.ref.maker.to_dataframe \ No newline at end of file diff --git a/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.model.hint.rst b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.model.hint.rst new file mode 100644 index 000000000..98c08807f --- /dev/null +++ b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.model.hint.rst @@ -0,0 +1,6 @@ +argopy.ArgoSensor.ref.model.hint +================================ + +.. currentmodule:: argopy + +.. autoaccessormethod:: ArgoSensor.ref.model.hint \ No newline at end of file diff --git a/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.model.search.rst b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.model.search.rst new file mode 100644 index 000000000..8996008c9 --- /dev/null +++ b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.model.search.rst @@ -0,0 +1,6 @@ +argopy.ArgoSensor.ref.model.search +================================== + +.. currentmodule:: argopy + +.. autoaccessormethod:: ArgoSensor.ref.model.search \ No newline at end of file diff --git a/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.model.to_dataframe.rst b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.model.to_dataframe.rst new file mode 100644 index 000000000..2b275aa92 --- /dev/null +++ b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.model.to_dataframe.rst @@ -0,0 +1,6 @@ +argopy.ArgoSensor.ref.model.to_dataframe +======================================== + +.. currentmodule:: argopy + +.. autoaccessormethod:: ArgoSensor.ref.model.to_dataframe \ No newline at end of file diff --git a/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.model.to_type.rst b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.model.to_type.rst new file mode 100644 index 000000000..6dcfb6ab3 --- /dev/null +++ b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.model.to_type.rst @@ -0,0 +1,6 @@ +argopy.ArgoSensor.ref.model.to_type +=================================== + +.. currentmodule:: argopy + +.. autoaccessormethod:: ArgoSensor.ref.model.to_type \ No newline at end of file diff --git a/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.type.hint.rst b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.type.hint.rst new file mode 100644 index 000000000..7d2b24827 --- /dev/null +++ b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.type.hint.rst @@ -0,0 +1,6 @@ +argopy.ArgoSensor.ref.type.hint +=============================== + +.. currentmodule:: argopy + +.. autoaccessormethod:: ArgoSensor.ref.type.hint \ No newline at end of file diff --git a/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.type.to_dataframe.rst b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.type.to_dataframe.rst new file mode 100644 index 000000000..2f13e5f82 --- /dev/null +++ b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.type.to_dataframe.rst @@ -0,0 +1,6 @@ +argopy.ArgoSensor.ref.type.to_dataframe +======================================= + +.. currentmodule:: argopy + +.. autoaccessormethod:: ArgoSensor.ref.type.to_dataframe \ No newline at end of file diff --git a/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.type.to_model.rst b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.type.to_model.rst new file mode 100644 index 000000000..bd88d1488 --- /dev/null +++ b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.ref.type.to_model.rst @@ -0,0 +1,6 @@ +argopy.ArgoSensor.ref.type.to_model +=================================== + +.. currentmodule:: argopy + +.. autoaccessormethod:: ArgoSensor.ref.type.to_model \ No newline at end of file diff --git a/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.rst b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.rst new file mode 100644 index 000000000..3096f4533 --- /dev/null +++ b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.rst @@ -0,0 +1,33 @@ +argopy.ArgoSensor +================= + +.. currentmodule:: argopy + +.. autoclass:: ArgoSensor + + + .. automethod:: __init__ + + + .. rubric:: Methods + + .. autosummary:: + + ~ArgoSensor.__init__ + ~ArgoSensor.cli_search + ~ArgoSensor.from_wmo + ~ArgoSensor.iterfloats_with + ~ArgoSensor.search + + + + + + .. rubric:: Attributes + + .. autosummary:: + + ~ArgoSensor.type + ~ArgoSensor.vocabulary + + \ No newline at end of file diff --git a/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.search.rst b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.search.rst new file mode 100644 index 000000000..0c4e38e15 --- /dev/null +++ b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.search.rst @@ -0,0 +1,6 @@ +argopy.ArgoSensor.search +======================== + +.. currentmodule:: argopy + +.. automethod:: ArgoSensor.search \ No newline at end of file diff --git a/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.type.rst b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.type.rst new file mode 100644 index 000000000..0fcae5eb8 --- /dev/null +++ b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.type.rst @@ -0,0 +1,6 @@ +argopy.ArgoSensor.type +====================== + +.. currentmodule:: argopy + +.. autoproperty:: ArgoSensor.type \ No newline at end of file diff --git a/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.vocabulary.rst b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.vocabulary.rst new file mode 100644 index 000000000..f38377c1d --- /dev/null +++ b/docs/advanced-tools/metadata/generated/argopy.ArgoSensor.vocabulary.rst @@ -0,0 +1,6 @@ +argopy.ArgoSensor.vocabulary +============================ + +.. currentmodule:: argopy + +.. autoproperty:: ArgoSensor.vocabulary \ No newline at end of file diff --git a/docs/advanced-tools/metadata/index.rst b/docs/advanced-tools/metadata/index.rst index bf8a00887..5dc0a96c8 100644 --- a/docs/advanced-tools/metadata/index.rst +++ b/docs/advanced-tools/metadata/index.rst @@ -10,6 +10,7 @@ In **argopy** we consider Argo meta-data any *related* data that can complement Here we document **argopy** tools related to: * :doc:`Argo reference tables (NVS) ` +* :doc:`Argo sensors ` * :doc:`the ADMT collection of documentation manuals ` * :doc:`the global deployment plan from Ocean-OPS ` * :doc:`GDAC snapshot with DOIs ` @@ -22,6 +23,6 @@ topography, altimetry, in-situ reference data from Argo float and GOSHIP, etc .. :hidden: Argo reference tables (NVS) + Argo sensors admt_documentation deployment_plan - gdac_doi diff --git a/docs/advanced-tools/metadata/sensors.rst b/docs/advanced-tools/metadata/sensors.rst new file mode 100644 index 000000000..4f3cb3509 --- /dev/null +++ b/docs/advanced-tools/metadata/sensors.rst @@ -0,0 +1,320 @@ +.. currentmodule:: argopy +.. _argosensor: + +Argo sensors +============ + +**Argopy** provides several classes to work with Argo sensors: + +- :class:`ArgoSensor`: provides user-friendly access to Argo's sensor metadata with search possibilities, +- :class:`OEMSensorMetaData`: provides facilitated access to manufacturers web-API and predeployment calibrations information. + +These should enable users to: + +- navigate reference tables `Sensor Models (R27) `_, `Sensor Types (R25) `_ and `Sensor Manufacturers (R26) `_, +- search for floats equipped with specific sensor models, +- retrieve sensor serial numbers across the global array, +- search for/iterate over floats equipped with specific sensor models, +- retrieve sensor metadata directly from manufacturers. + +.. note:: + + The :class:`ArgoSensor` get information using the `Euro-Argo fleet-monitoring API `_. + +.. contents:: + :local: + :depth: 3 + +.. _argosensor-reference-tables: + +Navigating reference tables +--------------------------- + +With the :class:`ArgoSensor` class, you can work with official Argo vocabularies for `Sensor Models (R27) `_, `Sensor Types (R25) `_ and `Sensor Manufacturers (R26) `_. With the following methods, it is easy to look for a specific sensor model name, which can then be used to :ref:`look for floats ` or :ref:`create a ArgoSensor class ` directly. + + +.. currentmodule:: argopy + +**Content** from reference tables: + +.. autosummary:: + :template: autosummary/accessor_method.rst + + ArgoSensor.ref.model.to_dataframe + ArgoSensor.ref.model.hint + + ArgoSensor.ref.type.to_dataframe + ArgoSensor.ref.type.hint + + ArgoSensor.ref.maker.to_dataframe + ArgoSensor.ref.maker.hint + +**Mapping** of sensor model vs type: + +.. autosummary:: + :template: autosummary/accessor_method.rst + + ArgoSensor.ref.model.to_type + ArgoSensor.ref.type.to_model + +Model **search**: + +.. autosummary:: + :template: autosummary/accessor_method.rst + + ArgoSensor.ref.model.search + + +Examples +^^^^^^^^ + +- List all CTD models from a given type: + +.. ipython:: python + :okwarning: + + from argopy import ArgoSensor + + models = ArgoSensor().ref.type.to_model('CTD_CNDC') + print(models) + +- Get the sensor type(s) of a given model: + +.. ipython:: python + :okwarning: + + types = ArgoSensor().ref.model.to_type('RBR_ARGO3_DEEP6K') + print(types) + +- Search all SBE61 versions with a wildcard: + +.. ipython:: python + :okwarning: + + ArgoSensor().ref.model.search('SBE61*') + +- Get one single model description (see also :attr:`ArgoSensor.vocabulary`): + +.. ipython:: python + :okwarning: + + ArgoSensor().ref.model.search('SBE61_V5.0.1').T + + +.. _argosensor-search-floats: + +Search for floats equipped with sensor models +--------------------------------------------- + +In this section we show how to find WMOs, and possibly more sensor metadata like serial numbers, for floats equipped with a specific sensor model. +This can be done using the :meth:`ArgoSensor.search` method. + +The method takes one or more model sensor names as input, and return results in 4 different formats with the ``output`` option: + +- ``output='wmo'`` returns a list of WMOs (e.g., ``[1901234, 1901235]``) +- ``output='sn'`` returns a list of serial numbers (e.g., ``['1234', '5678']``) +- ``output='wmo_sn'`` returns a dictionary mapping float WMOs to serial numbers (e.g. ``{1900166: ['0325', '2657720242']}``) +- ``output='df'`` returns a :class:`pandas.DataFrame` with WMO, sensor type, model, maker, serial number, units, accuracy and resolution in columns. + +.. note:: + + This method can potentially lead to a scan of the entire Argo float collection, which would be very time consuming. Therefore the :meth:`ArgoSensor.search` method takes only exact sensor models and will raise an error if a wildcard is found. + +Examples +^^^^^^^^ + +- Get WMOs for all floats equipped with the "AANDERAA_OPTODE_4831" model: + +.. ipython:: python + :okwarning: + + ArgoSensor().search("AANDERAA_OPTODE_4831") # output='wmo' is default + + +- Get sensor serial numbers for floats equipped with the "SBE43F_IDO" model: + +.. ipython:: python + :okwarning: + + serials = ArgoSensor().search("SBE43F_IDO", output="sn") + print(serials) + +- Get everything for floats equipped with the "RBR_ARGO3_DEEP6K" model: + +.. ipython:: python + :okwarning: + + ArgoSensor().search("RBR_ARGO3_DEEP6K", output="df") + +Advanced data retrieval with iterations on ArgoFloat instances +-------------------------------------------------------------- + +The :meth:`ArgoSensor.iterfloats_with` will yields :class:`argopy.ArgoFloat` instances for floats with the specified sensor model (use ``chunksize`` to process floats in batches). + +Example +^^^^^^^ + +Let's try to gather all platform types and WMOs for floats equipped with a list of sensor models: + +.. code-block:: python + + sensors = ArgoSensor() + + models = ['ECO_FLBBCD_AP2', 'ECO_FLBBCD'] + results = {} + for af in sensors.iterfloats_with(models): + if 'meta' in af.ls_dataset(): + platform_type = af.metadata['platform']['type'] # e.g. 'PROVOR_V_JUMBO' + if platform_type in results.keys(): + results[platform_type].extend([af.WMO]) + else: + results.update({platform_type: [af.WMO]}) + else: + print(f"No meta file for float {af.WMO}") + + [f"{r:15s}: {len(results[r])} floats" for r in results.keys()] + +.. code-block:: python + + ['APEX : 37 floats', + 'SOLO_BGC_MRV : 19 floats', + 'PROVOR_V_JUMBO : 43 floats', + 'PROVOR_III : 206 floats', + 'PROVOR_V : 38 floats', + 'PROVOR : 18 floats', + 'PROVOR_IV : 21 floats', + 'SOLO_BGC : 4 floats'] + +.. _argosensor-exact-sensor: + +Working with one Sensor model +----------------------------- + +Argo references +^^^^^^^^^^^^^^^ + +To acces one sensor model complete list of referenced information, you can initialize a :class:`ArgoSensor` instance with a specific model. In this use-case, you will have the following attributes and methods available: + +.. currentmodule:: argopy + +.. autosummary:: + + ArgoSensor.vocabulary + ArgoSensor.type + ArgoSensor.search + ArgoSensor.iterfloats_with + +As an example, let's create an instance for the "SBE43F_IDO" sensor model: + +.. ipython:: python + :okwarning: + + sensor = ArgoSensor("SBE43F_IDO") + sensor + +You can then access this model metadata from the NVS vocabulary (Reference table R27): + +.. ipython:: python + :okwarning: + + sensor.vocabulary + +and from Reference table R25: + +.. ipython:: python + :okwarning: + + sensor.type + +You can also look for floats equipped with it: + +.. ipython:: python + :okwarning: + + df = sensor.search(output="df") + df + +Manufacturers API +^^^^^^^^^^^^^^^^^ + +**Argopy** provides the experimental :class:`OEMSensorMetaData` class to deal with metadata provided by manufacturers. The :class:`OEMSensorMetaData` class provides methods to directly access some manufacturers web-API on your behalf. + +Argo sensor makers are encouraged to provide the Argo community with all possible information about a sensor, in particular predeployment calibration metadata. + +The ADMT has developed a JSON schema for this at https://github.com/euroargodev/sensor_metadata_json and an library for JSON validation at https://github.com/euroargodev/argo-metadata-validator. + +By default **argopy** will validate json data against the reference schema. + +Sensor metadata from RBR +"""""""""""""""""""""""" + +Thanks to the RBR web-API, you can access sensor metadata from a RBR sensor with the :meth:`OEMSensorMetaData.from_rbr` method and a sensor serial number: + +.. ipython:: python + :okwarning: + + from argopy import OEMSensorMetaData + + OEMSensorMetaData().from_rbr(208380) + +Note that the RBR web-API requires an authentication key (you can contact RBR at argo@rbr-global.com if you do not have an such a key). **Argopy** will try to get the key from the environment variable ``RBR_API_KEY`` or from the option ``rbr_api_key``. You can set the key temporarily in your code with: + +.. code-block:: python + + argopy.set_options(rbr_api_key="********") + +Sensor metadata from Seabird +"""""""""""""""""""""""""""" + +Thanks to the Seabird web-API, you can access sensor metadata from a Seabird sensor with the :meth:`OEMSensorMetaData.from_seabird` method and a sensor serial number and a model name: + +.. ipython:: python + :okwarning: + + from argopy import OEMSensorMetaData + + OEMSensorMetaData().from_seabird(2444, 'SATLANTIC_OCR504_ICSW') + +Sensor metadata from elsewhere +"""""""""""""""""""""""""""""" + +If you have your own metadata, you can use :meth:`OEMSensorMetaData.from_dict`: + +.. code-block:: python + + from argopy import OEMSensorMetaData + + jsdata = [...] + + OEMSensorMetaData().from_dict(jsdata) + +Examples +"""""""" + +Last, since this is still in active development on the manufacturer's side, we also added easy access to some examples from https://github.com/euroargodev/sensor_metadata_json: + +.. ipython:: python + :okwarning: + + OEMSensorMetaData().list_examples + + OEMSensorMetaData().from_examples('WETLABS-ECO_FLBBAP2-8589') + + + +From the command line +--------------------- + +The :meth:`ArgoSensor.cli_search` function is available to search for Argo floats equipped with a given sensor model from the command line and retrieve a `CLI `_ friendly list of WMOs or serial numbers. + +You will take note that all output *format* are available to determine the information to retrieve, but that the actual standard output will be a serialized string for easy piping to other tools. + +Here is an example with the "SATLANTIC_PAR" sensor. + +.. code-block:: bash + :caption: Call :meth:`ArgoSensor.search` from the command line + + python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('SATLANTIC_PAR', output='wmo')" + python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('SATLANTIC_PAR', output='sn')" + python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('SATLANTIC_PAR', output='wmo_sn')" + python -c "from argopy import ArgoSensor; ArgoSensor().cli_search('SATLANTIC_PAR', output='df')" diff --git a/docs/api-hidden.rst b/docs/api-hidden.rst index b7ea51f95..4d290c4b3 100644 --- a/docs/api-hidden.rst +++ b/docs/api-hidden.rst @@ -178,6 +178,33 @@ argopy.related.ArgoDOI.doi argopy.related.doi_snapshot.DOIrecord + argopy.related.ArgoSensor + argopy.ArgoSensor.ref.model.to_dataframe + argopy.ArgoSensor.ref.model.hint + argopy.ArgoSensor.ref.model.to_type + argopy.ArgoSensor.ref.model.search + argopy.ArgoSensor.ref.type.to_dataframe + argopy.ArgoSensor.ref.type.hint + argopy.ArgoSensor.ref.type.to_model + argopy.ArgoSensor.ref.maker.to_dataframe + argopy.ArgoSensor.ref.maker.hint + + argopy.ArgoSensor.vocabulary + argopy.ArgoSensor.type + argopy.ArgoSensor.search + argopy.ArgoSensor.iterfloats_with + argopy.ArgoSensor.cli_search + + argopy.related.OEMSensorMetaData + argopy.OEMSensorMetaData.from_rbr + argopy.OEMSensorMetaData.from_seabird + argopy.OEMSensorMetaData.from_dict + argopy.OEMSensorMetaData.from_examples + argopy.OEMSensorMetaData.list_examples + + argopy.related.SensorModel + argopy.related.SensorType + argopy.plot argopy.plot.dashboard argopy.plot.bar_plot diff --git a/docs/api.rst b/docs/api.rst index f5b6be735..4376c0682 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -64,138 +64,22 @@ Properties DataFetcher.uri -Dataset.argo (xarray accessor to work with Argo dataset) -======================================================== - -.. currentmodule:: xarray - -.. autosummary:: - :toctree: generated/ - :template: autosummary/accessor.rst - - Dataset.argo - -This accessor extends :py:class:`xarray.Dataset`. Proper use of this accessor should be like: - -.. code-block:: python - - >>> import xarray as xr # first import xarray - >>> import argopy # import argopy (the dataset 'argo' accessor is then registered) - >>> from argopy import DataFetcher - >>> ds = DataFetcher().float([6902766, 6902772, 6902914, 6902746]).load().data - >>> ds.argo - >>> ds.argo.filter_qc() - - -Data Transformation -------------------- - -.. autosummary:: - :toctree: generated/ - :template: autosummary/accessor_method.rst - - Dataset.argo.point2profile - Dataset.argo.profile2point - Dataset.argo.interp_std_levels - Dataset.argo.groupby_pressure_bins - Dataset.argo.datamode.compute - Dataset.argo.datamode.filter - Dataset.argo.datamode.merge - Dataset.argo.datamode.split - - -Data Filters ------------- - -.. autosummary:: - :toctree: generated/ - :template: autosummary/accessor_method.rst - - Dataset.argo.filter_qc - Dataset.argo.filter_scalib_pres - Dataset.argo.filter_researchmode - Dataset.argo.datamode.filter - - -Extensions ----------- -.. currentmodule:: argopy - -You can create your own extension to an Argo dataset for specific features. It should be registered by inheriting from :class:`argopy.extensions.ArgoAccessorExtension` and decorated with :class:`argopy.extensions.register_argo_accessor`. - -**argopy** comes with the following extensions: - -General purposes -^^^^^^^^^^^^^^^^ - -.. currentmodule:: xarray - -.. autosummary:: - :toctree: generated/ - :template: autosummary/accessor_method.rst - - Dataset.argo.teos10 - Dataset.argo.datamode - Dataset.argo.create_float_source - - -BGC specifics -^^^^^^^^^^^^^ - -.. currentmodule:: xarray - -.. autosummary:: - :toctree: generated/ - :template: autosummary/accessor_method.rst - - Dataset.argo.canyon_med - Dataset.argo.canyon_med.predict - Dataset.argo.canyon_med.input_list - Dataset.argo.canyon_med.output_list - - Dataset.argo.canyon_b - Dataset.argo.canyon_b.predict - Dataset.argo.canyon_b.input_list - Dataset.argo.canyon_b.output_list - - Dataset.argo.content - Dataset.argo.content.predict - Dataset.argo.content.input_list - Dataset.argo.content.output_list - - Dataset.argo.optic - Dataset.argo.optic.Zeu - Dataset.argo.optic.Zpd - Dataset.argo.optic.Z_iPAR_threshold - Dataset.argo.optic.DCM - -Misc ----- +Argo file stores +================ .. autosummary:: - :toctree: generated/ - :template: autosummary/accessor_method.rst - - Dataset.argo.index - Dataset.argo.domain - Dataset.argo.list_WMO_CYC - Dataset.argo.uid - Dataset.argo.cast_types - Dataset.argo.N_POINTS - Dataset.argo.to_zarr - Dataset.argo.reduce_profile - -ArgoFloat (data and meta-data fetcher for one float) -==================================================== + :toctree: generated/ -.. currentmodule:: argopy + gdacfs +ArgoFloat +--------- .. autosummary:: :toctree: generated/ ArgoFloat -**Extension: Plot** +List of extensions: .. currentmodule:: argopy @@ -205,6 +89,8 @@ ArgoFloat (data and meta-data fetcher for one float) ArgoFloat.plot +This extension provides the following **plotting methods** for one Argo float data: + .. autosummary:: :toctree: generated/ :template: autosummary/accessor_method.rst @@ -213,49 +99,8 @@ ArgoFloat (data and meta-data fetcher for one float) ArgoFloat.plot.map ArgoFloat.plot.scatter -**Extension: Configuration and launch configuration parameters** - -.. currentmodule:: argopy - -.. autosummary:: - :toctree: generated/ - :template: autosummary/accessor.rst - - ArgoFloat.config - -.. autosummary:: - :toctree: generated/ - :template: autosummary/accessor_method.rst - - ArgoFloat.config.n_params - ArgoFloat.config.parameters - ArgoFloat.config.n_missions - ArgoFloat.config.missions - ArgoFloat.config.cycles - ArgoFloat.config.for_cycles - ArgoFloat.config.to_dataframe - -.. currentmodule:: argopy - -.. autosummary:: - :toctree: generated/ - :template: autosummary/accessor.rst - - ArgoFloat.launchconfig - -.. autosummary:: - :toctree: generated/ - :template: autosummary/accessor_method.rst - - ArgoFloat.launchconfig.n_params - ArgoFloat.launchconfig.parameters - ArgoFloat.launchconfig.to_dataframe - - -ArgoIndex (explore Argo files index) -==================================== - -.. currentmodule:: argopy +ArgoIndex +--------- .. autosummary:: :toctree: generated/ @@ -288,9 +133,6 @@ List of extensions: ArgoIndex.query.parameter_data_mode ArgoIndex.query.profiler_type ArgoIndex.query.profiler_label - ArgoIndex.query.institution_code - ArgoIndex.query.institution_name - ArgoIndex.query.dac **Search on at least two properties** of a file record: @@ -312,8 +154,8 @@ List of extensions: ArgoIndex.plot.trajectory ArgoIndex.plot.bar -Other Argo related data -======================= +Argo meta/related data +====================== .. autosummary:: :toctree: generated/ @@ -325,14 +167,8 @@ Other Argo related data OceanOPSDeployments CTDRefDataFetcher TopoFetcher - -GDAC file stores -================ - -.. autosummary:: - :toctree: generated/ - - gdacfs + ArgoSensor + OEMSensorMetaData .. _Module Visualisation: @@ -364,6 +200,101 @@ All other visualisation functions are in the :mod:`argopy.plot` submodule: latlongrid +Dataset.argo (xarray accessor) +============================== + +.. currentmodule:: xarray + +.. autosummary:: + :toctree: generated/ + :template: autosummary/accessor.rst + + Dataset.argo + +This accessor extends :py:class:`xarray.Dataset`. Proper use of this accessor should be like: + +.. code-block:: python + + >>> import xarray as xr # first import xarray + >>> import argopy # import argopy (the dataset 'argo' accessor is then registered) + >>> from argopy import DataFetcher + >>> ds = DataFetcher().float([6902766, 6902772, 6902914, 6902746]).load().data + >>> ds.argo + >>> ds.argo.filter_qc() + + +Data Transformation +------------------- + +.. autosummary:: + :toctree: generated/ + :template: autosummary/accessor_method.rst + + Dataset.argo.point2profile + Dataset.argo.profile2point + Dataset.argo.interp_std_levels + Dataset.argo.groupby_pressure_bins + Dataset.argo.datamode.compute + Dataset.argo.datamode.filter + Dataset.argo.datamode.merge + Dataset.argo.datamode.split + + +Data Filters +------------ + +.. autosummary:: + :toctree: generated/ + :template: autosummary/accessor_method.rst + + Dataset.argo.filter_qc + Dataset.argo.filter_scalib_pres + Dataset.argo.filter_researchmode + Dataset.argo.datamode.filter + + +Extensions +---------- +.. currentmodule:: argopy + +You can create your own extension to an Argo dataset for specific features. It should be registered by inheriting from :class:`argopy.extensions.ArgoAccessorExtension` and decorated with :class:`argopy.extensions.register_argo_accessor`. + +**argopy** comes with the following extensions: + +.. currentmodule:: xarray + +.. autosummary:: + :toctree: generated/ + :template: autosummary/accessor_method.rst + + Dataset.argo.teos10 + Dataset.argo.create_float_source + Dataset.argo.canyon_med + Dataset.argo.datamode + Dataset.argo.optic + Dataset.argo.optic.Zeu + Dataset.argo.optic.Zpd + Dataset.argo.optic.Z_iPAR_threshold + Dataset.argo.optic.DCM + + +Misc +---- + +.. autosummary:: + :toctree: generated/ + :template: autosummary/accessor_method.rst + + Dataset.argo.index + Dataset.argo.domain + Dataset.argo.list_WMO_CYC + Dataset.argo.uid + Dataset.argo.cast_types + Dataset.argo.N_POINTS + Dataset.argo.to_zarr + Dataset.argo.reduce_profile + + Utilities ========= diff --git a/docs/whats-new.rst b/docs/whats-new.rst index 70f4095db..306194e39 100644 --- a/docs/whats-new.rst +++ b/docs/whats-new.rst @@ -7,14 +7,27 @@ What's New |pypi dwn| |conda dwn| -v1.4.0 (5 Jan. 2026) --------------------- -.. _v1.4.0-features: +Coming up next (un-released) +---------------------------- Features and front-end API ^^^^^^^^^^^^^^^^^^^^^^^^^^ +- **New** :class:`ArgoSensor` class to work with Argo sensor models, types and floats equipped with. Check the new :ref:`Argo sensors` documentation section. (:pr:`532`) by |gmaze|. + +Energy +^^^^^^ + +|eqco2_since_last_release| +|eqco2_baseline| + + +v1.4.0 (5 Jan. 2026) +-------------------- + +.. _v1.4.0-features: + .. currentmodule:: xarray - **Predict nutrients and carbonates in the global ocean with their uncertainties** with the two new classes :class:`Dataset.argo.canyon_b` and :class:`Dataset.argo.content`. The first class allows users to make predictions of the water-column nutrient concentrations (NO3, PO4, SiOH4) and carbonate system variables (AT, DIC, pHT, pCO2) using the :ref:`CANYON-B model (see doc.)` while the second class provides improved predictions of carbonate system variables using the :ref:`CONTENT (see doc.)` model. (:pr:`535` and :pr:`542`) by |fricour|. It goes as simply as: @@ -64,8 +77,7 @@ Breaking changes Energy ^^^^^^ -|eqco2_since_last_release| -|eqco2_baseline| +.. image:: https://img.shields.io/badge/Total%20carbon%20emitted%20by%20release%20v1.4.0%20%5BgCO2eq%5D-1666-black?style=plastic&labelColor=grey v1.3.1 (22 Oct. 2025) ---------------------