diff --git a/argopy/__init__.py b/argopy/__init__.py index caadf8b1a..54861b62d 100644 --- a/argopy/__init__.py +++ b/argopy/__init__.py @@ -22,27 +22,35 @@ log.addHandler(logging.NullHandler()) # Import facades: -from .fetchers import ArgoDataFetcher as DataFetcher # noqa: E402 -from .fetchers import ArgoIndexFetcher as IndexFetcher # noqa: E402 +from argopy.fetchers import ArgoDataFetcher as DataFetcher # noqa: E402 +from argopy.fetchers import ArgoIndexFetcher as IndexFetcher # noqa: E402 -from .xarray import ArgoAccessor # noqa: E402 +from argopy.xarray import ArgoAccessor # noqa: E402 # Other Import # from . import utils # noqa: E402 -from . import stores # noqa: E402 -from . import errors # noqa: E402 -from . import plot # noqa: E402 -from . import tutorial # noqa: E402 -from .plot import dashboard, ArgoColors # noqa: E402 -from .options import set_options, reset_options # noqa: E402 -from .data_fetchers import CTDRefDataFetcher # noqa: E402 -from .stores import ArgoIndex, ArgoFloat, gdacfs # noqa: E402 -from .utils import show_versions, show_options # noqa: E402 -from .utils import clear_cache, lscache # noqa: E402 -from .utils import MonitoredThreadPoolExecutor # noqa: E402, F401 -from .utils import monitor_status as status # noqa: E402 -from .related import TopoFetcher, OceanOPSDeployments, ArgoNVSReferenceTables, ArgoDocs, ArgoDOI # noqa: E402 -from .extensions import CanyonMED # noqa: E402 +# from argopy import errors # noqa: E402 +# from argopy import plot # noqa: E402 +import argopy.tutorial # noqa: E402 +from argopy.plot.dashboards import open_dashboard as dashboard +from argopy.plot.argo_colors import ArgoColors # noqa: E402 +from argopy.options import set_options, reset_options # noqa: E402 +from argopy.data_fetchers.erddap_refdata import Fetch_box as CTDRefDataFetcher # noqa: E402 +from argopy.stores.index.argo_index import ArgoIndex # noqa: E402 +from argopy.stores.float.argo_float import ArgoFloat # noqa: E402 +from argopy.stores.implementations.gdac import gdacfs # noqa: E402 +from argopy.utils.locals import show_versions, show_options # noqa: E402 +from argopy.utils.caching import clear_cache, lscache # noqa: E402 +from argopy.utils.monitored_threadpool import MyThreadPoolExecutor as MonitoredThreadPoolExecutor # noqa: E402, F401 +from argopy.utils.monitors import monitor_status as status # noqa: E402 +from argopy.related.topography import TopoFetcher +from argopy.related.ocean_ops_deployments import OceanOPSDeployments +from argopy.related.reference_tables import ArgoNVSReferenceTables +from argopy.related.argo_documentation import ArgoDocs +from argopy.related.doi_snapshot import ArgoDOI # noqa: E402 +# from argopy.extensions.canyon_med import CanyonMED # noqa: E402 +# from argopy.extensions.canyon_b import CanyonB # noqa: E402 +# from argopy.extensions.carbonate_content import CONTENT # noqa: E402 # @@ -77,14 +85,15 @@ # Submodules: # "utils", - "errors", - "plot", + # "errors", + # "plot", "ArgoColors", # Class - "stores", "tutorial", # Argo xarray accessor extensions - "CanyonMED", + # "CanyonMED", + # "CanyonB", + # "CONTENT", # Constants "__version__" diff --git a/argopy/data_fetchers/__init__.py b/argopy/data_fetchers/__init__.py index 350337973..f28a7cbab 100644 --- a/argopy/data_fetchers/__init__.py +++ b/argopy/data_fetchers/__init__.py @@ -1,19 +1,19 @@ -""" -This package contains implementations for data and index fetchers for specific data sources. Most of these fetchers are meant to be used and discovered automatically by the facades (in fetchers.py) and by utilities functions list_available_data_src() and list_available_index_src() -""" - -from .erddap_refdata import Fetch_box as CTDRefDataFetcher -from . import erddap_data -from . import erddap_index -from . import argovis_data -from . import gdac_data -from . import gdac_index - -__all__ = ( - "erddap_data", - "erddap_index", - "argovis_data", - "gdac_data", - "gdac_index", - "CTDRefDataFetcher", -) +# """ +# This package contains implementations for data and index fetchers for specific data sources. Most of these fetchers are meant to be used and discovered automatically by the facades (in fetchers.py) and by utilities functions list_available_data_src() and list_available_index_src() +# """ +# +# from argopy.data_fetchers.erddap_refdata import Fetch_box as CTDRefDataFetcher +# import argopy.data_fetchers.erddap_data +# import argopy.data_fetchers.erddap_index +# import argopy.data_fetchers.argovis_data +# import argopy.data_fetchers.gdac_data +# import argopy.data_fetchers.gdac_index +# +# __all__ = ( +# "erddap_data", +# "erddap_index", +# "argovis_data", +# "gdac_data", +# "gdac_index", +# "CTDRefDataFetcher", +# ) diff --git a/argopy/data_fetchers/argovis_data.py b/argopy/data_fetchers/argovis_data.py index ba3460bdf..af2b6328f 100755 --- a/argopy/data_fetchers/argovis_data.py +++ b/argopy/data_fetchers/argovis_data.py @@ -6,13 +6,14 @@ from abc import abstractmethod import warnings -from ..stores import httpstore -from ..options import OPTIONS, DEFAULT, PARALLEL_SETUP -from ..utils.chunking import Chunker -from ..errors import DataNotFound -from .. import __version__ -from .proto import ArgoDataFetcherProto -from .argovis_data_processors import pre_process, add_attributes +from argopy import __version__ +from argopy.errors import DataNotFound +from argopy.options import OPTIONS, DEFAULT, PARALLEL_SETUP +from argopy.stores.implementations.http import httpstore +from argopy.utils.chunking import Chunker +from argopy.data_fetchers.proto import ArgoDataFetcherProto +from argopy.data_fetchers.argovis_data_processors import pre_process, add_attributes + access_points = ["wmo", "box"] exit_formats = ["xarray"] diff --git a/argopy/data_fetchers/argovis_data_processors.py b/argopy/data_fetchers/argovis_data_processors.py index 30997c9b9..3503deefc 100644 --- a/argopy/data_fetchers/argovis_data_processors.py +++ b/argopy/data_fetchers/argovis_data_processors.py @@ -1,7 +1,7 @@ import pandas as pd import xarray as xr from typing import Any -from ..utils import list_bgc_s_parameters +from argopy.utils.lists import list_bgc_s_parameters def pre_process(profiles: Any, key_map: dict = None) -> pd.DataFrame: diff --git a/argopy/data_fetchers/erddap_data.py b/argopy/data_fetchers/erddap_data.py index 0e924c3c7..a34514425 100644 --- a/argopy/data_fetchers/erddap_data.py +++ b/argopy/data_fetchers/erddap_data.py @@ -20,14 +20,16 @@ from erddapy.erddapy import ERDDAP, parse_dates from erddapy.erddapy import _quote_string_constraints as quote_string_constraints -from ..options import OPTIONS, PARALLEL_SETUP -from ..utils.lists import list_bgc_s_variables, list_core_parameters -from ..errors import ErddapServerError, DataNotFound -from ..stores import httpstore, has_distributed, distributed -from ..stores.index import indexstore_pd as ArgoIndex -from ..utils import is_list_of_strings, to_list, Chunker -from .proto import ArgoDataFetcherProto -from .erddap_data_processors import pre_process +from argopy.options import OPTIONS, PARALLEL_SETUP +from argopy.errors import ErddapServerError, DataNotFound +from argopy.utils.lists import list_bgc_s_variables, list_core_parameters +from argopy.utils.checkers import is_list_of_strings, to_list +from argopy.utils.chunking import Chunker +from argopy.stores.implementations.http import httpstore +from argopy.stores.filesystems import has_distributed, distributed +from argopy.stores.index.implementations.pandas.index import indexstore as ArgoIndex +from argopy.data_fetchers.proto import ArgoDataFetcherProto +from argopy.data_fetchers.erddap_data_processors import pre_process log = logging.getLogger("argopy.erddap.data") diff --git a/argopy/data_fetchers/erddap_data_processors.py b/argopy/data_fetchers/erddap_data_processors.py index 94f05e824..9e03bcd0f 100644 --- a/argopy/data_fetchers/erddap_data_processors.py +++ b/argopy/data_fetchers/erddap_data_processors.py @@ -3,8 +3,9 @@ import xarray as xr import getpass from urllib.parse import urlparse -from ..utils import to_list, UriCName -from ..utils import list_bgc_s_parameters +from argopy.utils.format import UriCName +from argopy.utils.checkers import to_list +from argopy.utils.lists import list_bgc_s_parameters def pre_process( diff --git a/argopy/data_fetchers/erddap_index.py b/argopy/data_fetchers/erddap_index.py index a18c90aee..abca0e49e 100644 --- a/argopy/data_fetchers/erddap_index.py +++ b/argopy/data_fetchers/erddap_index.py @@ -18,10 +18,10 @@ from erddapy.erddapy import ERDDAP, parse_dates from erddapy.erddapy import _quote_string_constraints as quote_string_constraints -from ..utils.format import format_oneline -from ..related import load_dict, mapp_dict -from ..stores import httpstore -from ..options import OPTIONS +from argopy.options import OPTIONS +from argopy.utils.format import format_oneline +from argopy.related.utils import load_dict, mapp_dict +from argopy.stores.implementations.http import httpstore log = logging.getLogger("argopy.fetchers.erddap_index") diff --git a/argopy/data_fetchers/erddap_refdata.py b/argopy/data_fetchers/erddap_refdata.py index 50de58785..7576b1ab5 100644 --- a/argopy/data_fetchers/erddap_refdata.py +++ b/argopy/data_fetchers/erddap_refdata.py @@ -1,15 +1,15 @@ """ Fetcher to retrieve ship-based CTD reference data from Ifremer erddap """ - import xarray as xr import logging -from ..options import OPTIONS -from ..utils.chunking import Chunker -from ..utils.geo import conv_lon -from ..stores import httpstore_erddap_auth -from .erddap_data import ErddapArgoDataFetcher -from .erddap_data_processors import _add_attributes + +from argopy.options import OPTIONS +from argopy.utils.chunking import Chunker +from argopy.utils.geo import conv_lon +from argopy.stores.implementations.http_erddap import httpstore_erddap_auth +from argopy.data_fetchers.erddap_data import ErddapArgoDataFetcher +from argopy.data_fetchers.erddap_data_processors import _add_attributes # Load erddapy according to available version (breaking changes in v0.8.0) try: diff --git a/argopy/data_fetchers/gdac_data.py b/argopy/data_fetchers/gdac_data.py index ec3ae2aad..c5fcb8014 100644 --- a/argopy/data_fetchers/gdac_data.py +++ b/argopy/data_fetchers/gdac_data.py @@ -14,12 +14,13 @@ import logging from typing import Literal -from ..utils.format import argo_split_path -from ..options import OPTIONS, check_gdac_option, PARALLEL_SETUP -from ..errors import DataNotFound -from ..stores import ArgoIndex, has_distributed, distributed -from .proto import ArgoDataFetcherProto -from .gdac_data_processors import pre_process_multiprof +from argopy.utils.format import argo_split_path +from argopy.options import OPTIONS, check_gdac_option, PARALLEL_SETUP +from argopy.errors import DataNotFound +from argopy.stores.index.argo_index import ArgoIndex +from argopy.stores.filesystems import has_distributed, distributed +from argopy.data_fetchers.proto import ArgoDataFetcherProto +from argopy.data_fetchers.gdac_data_processors import pre_process_multiprof log = logging.getLogger("argopy.gdac.data") diff --git a/argopy/data_fetchers/gdac_index.py b/argopy/data_fetchers/gdac_index.py index 4a2038795..3c148c35c 100644 --- a/argopy/data_fetchers/gdac_index.py +++ b/argopy/data_fetchers/gdac_index.py @@ -11,10 +11,10 @@ import warnings import logging -from ..utils.format import format_oneline -from ..options import OPTIONS, check_gdac_option -from ..plot import dashboard -from ..stores import ArgoIndex +from argopy.utils.format import format_oneline +from argopy.options import OPTIONS, check_gdac_option +from argopy.plot.dashboards import open_dashboard as dashboard +from argopy.stores.index.argo_index import ArgoIndex log = logging.getLogger("argopy.gdac.index") diff --git a/argopy/data_fetchers/proto.py b/argopy/data_fetchers/proto.py index 016776ded..5c05e8a80 100644 --- a/argopy/data_fetchers/proto.py +++ b/argopy/data_fetchers/proto.py @@ -5,9 +5,10 @@ import hashlib import warnings import logging -from ..plot import dashboard -from ..utils.lists import list_standard_variables -from ..utils.format import UriCName, format_oneline + +from argopy.plot.dashboards import open_dashboard as dashboard +from argopy.utils.lists import list_standard_variables +from argopy.utils.format import UriCName, format_oneline log = logging.getLogger("argopy.fetcher.proto") diff --git a/argopy/extensions/__init__.py b/argopy/extensions/__init__.py index aeef46771..6474f03b9 100644 --- a/argopy/extensions/__init__.py +++ b/argopy/extensions/__init__.py @@ -1,17 +1,17 @@ -from .utils import register_argo_accessor, ArgoAccessorExtension -from .canyon_med import CanyonMED -from .canyon_b import CanyonB -from .carbonate_content import CONTENT -from .params_data_mode import ParamsDataMode -from .optical_modeling import OpticalModeling - +# from argopy.extensions.utils import register_argo_accessor, ArgoAccessorExtension +# from argopy.extensions.canyon_med import CanyonMED +# from argopy.extensions.canyon_b import CanyonB +# from argopy.extensions.carbonate_content import CONTENT +# from argopy.extensions.params_data_mode import ParamsDataMode +# from argopy.extensions.optical_modeling import OpticalModeling # -__all__ = ( - "register_argo_accessor", - "ArgoAccessorExtension", - "CanyonMED", - "CanyonB", - "CONTENT", - "ParamsDataMode", - "OpticalModeling", -) +# # +# __all__ = ( +# "register_argo_accessor", +# "ArgoAccessorExtension", +# "CanyonMED", +# "CanyonB", +# "CONTENT", +# "ParamsDataMode", +# "OpticalModeling", +# ) diff --git a/argopy/extensions/canyon_b.py b/argopy/extensions/canyon_b.py index b186990fe..66534693f 100644 --- a/argopy/extensions/canyon_b.py +++ b/argopy/extensions/canyon_b.py @@ -11,9 +11,11 @@ HAS_PYCO2SYS = False pyco2 = None -from ..errors import InvalidDatasetStructure, DataNotFound -from ..utils import path2assets, to_list, point_in_polygon -from . import register_argo_accessor, ArgoAccessorExtension +from argopy.errors import InvalidDatasetStructure, DataNotFound +from argopy.utils.locals import Asset +from argopy.utils.checkers import to_list +from argopy.utils.geo import point_in_polygon +from argopy.extensions.utils import register_argo_accessor, ArgoAccessorExtension @register_argo_accessor("canyon_b") @@ -98,9 +100,6 @@ def __init__(self, *args, **kwargs): if self._argo.N_POINTS == 0: raise DataNotFound("Empty dataset, no data to transform !") - self.path2coef = Path(path2assets).joinpath( - "canyon-b" - ) # Path to CANYON-B assets def get_param_attrs(self, param: str) -> dict: """ @@ -375,22 +374,16 @@ def load_weights(self, param: str) -> pd.DataFrame: Returns ------- - pd.DataFrame + :class:`pd.DataFrame` DataFrame containing the neural network weights for the specified parameter. """ if param in ["AT", "pCO2", "NO3", "PO4", "SiOH4"]: - weights = pd.read_csv( - self.path2coef.joinpath(f"wgts_{param}.txt"), header=None, sep="\t" - ) + weights = Asset.load(f"wgts_{param}.txt", header=None, sep="\t") elif param == "DIC": - weights = pd.read_csv( - self.path2coef.joinpath("wgts_CT.txt"), header=None, sep="\t" - ) + weights = Asset.load("wgts_CT.txt", header=None, sep="\t") else: - weights = pd.read_csv( - self.path2coef.joinpath("wgts_pH.txt"), header=None, sep="\t" - ) + weights = Asset.load("wgts_pH.txt", header=None, sep="\t") return weights diff --git a/argopy/extensions/canyon_med.py b/argopy/extensions/canyon_med.py index 72f78084d..5d4a8d758 100644 --- a/argopy/extensions/canyon_med.py +++ b/argopy/extensions/canyon_med.py @@ -4,9 +4,10 @@ import xarray as xr from typing import Union, List -from ..errors import InvalidDatasetStructure, DataNotFound -from ..utils import path2assets, to_list -from . import register_argo_accessor, ArgoAccessorExtension +from argopy.errors import InvalidDatasetStructure, DataNotFound +from argopy.utils.locals import Asset +from argopy.utils.checkers import to_list +from argopy.extensions.utils import register_argo_accessor, ArgoAccessorExtension nan_value = np.nan if not hasattr(np, 'NaN') else np.NaN @@ -78,7 +79,6 @@ def __init__(self, *args, **kwargs): raise DataNotFound("Empty dataset, no data to transform !") self.n_list = 5 - self.path2coef = Path(path2assets).joinpath("canyon-med") self._input = None # Private CANYON-MED input dataframe @property @@ -186,50 +186,21 @@ def isin_medsea(row): def load_normalisation_factors(self, param, subset="F"): suff = self.param2suff(param) - moy_sub = pd.read_table( - self.path2coef.joinpath("moy_%s_%s.txt" % (suff, subset)), - sep=" {3}", - header=None, - engine="python", - ).values - std_sub = pd.read_table( - self.path2coef.joinpath("std_%s_%s.txt" % (suff, subset)), - sep=" {3}", - header=None, - engine="python", - ).values + moy_sub = Asset.load(f"canyon-med:moy_{suff}_{subset}.txt", sep=" {3}", header=None, engine="python").values + std_sub = Asset.load(f"canyon-med:std_{suff}_{subset}.txt", sep=" {3}", header=None, engine="python").values + return moy_sub, std_sub def load_weights(self, param, subset, i): suff = self.param2suff(param) - b1 = pd.read_csv( - self.path2coef.joinpath("poids_%s_b1_%s_%i.txt" % (suff, subset, i)), - header=None, - ) - b2 = pd.read_csv( - self.path2coef.joinpath("poids_%s_b2_%s_%i.txt" % (suff, subset, i)), - header=None, - ) - b3 = pd.read_csv( - self.path2coef.joinpath("poids_%s_b3_%s_%i.txt" % (suff, subset, i)), - header=None, - ) - IW = pd.read_csv( - self.path2coef.joinpath("poids_%s_IW_%s_%i.txt" % (suff, subset, i)), - sep=r"\s+", - header=None, - ) - LW1 = pd.read_csv( - self.path2coef.joinpath("poids_%s_LW1_%s_%i.txt" % (suff, subset, i)), - sep=r"\s+", - header=None, - ) - LW2 = pd.read_csv( - self.path2coef.joinpath("poids_%s_LW2_%s_%i.txt" % (suff, subset, i)), - sep=r"\s+", - header=None, - ) + b1 = Asset.load(f"canyon-med:poids_{suff}_b1_{subset}_{i}.txt", header=None) + b2 = Asset.load(f"canyon-med:poids_{suff}_b2_{subset}_{i}.txt", header=None) + b3 = Asset.load(f"canyon-med:poids_{suff}_b3_{subset}_{i}.txt", header=None) + + IW = Asset.load(f"canyon-med:poids_{suff}_IW_{subset}_{i}.txt", header=None, sep=r"\s+") + LW1 = Asset.load(f"canyon-med:poids_{suff}_LW1_{subset}_{i}.txt", header=None, sep=r"\s+") + LW2 = Asset.load(f"canyon-med:poids_{suff}_LW2_{subset}_{i}.txt", header=None, sep=r"\s+") # Using float128 arrays avoid the error or warning "overflow encountered in exp" raised by the # activation function diff --git a/argopy/extensions/carbonate_content.py b/argopy/extensions/carbonate_content.py index 0c4b84861..d489c0621 100644 --- a/argopy/extensions/carbonate_content.py +++ b/argopy/extensions/carbonate_content.py @@ -10,11 +10,11 @@ HAS_PYCO2SYS = False pyco2 = None -from ..errors import InvalidDatasetStructure, DataNotFound -from . import register_argo_accessor, ArgoAccessorExtension +from argopy.errors import InvalidDatasetStructure, DataNotFound +from argopy.extensions.utils import register_argo_accessor, ArgoAccessorExtension # import carbonate utilities -from ..utils.carbonate import ( +from argopy.utils.carbonate import ( error_propagation, ChemistryConstants, CalculationOptions, diff --git a/argopy/extensions/optical_modeling.py b/argopy/extensions/optical_modeling.py index a50842ba4..04d207df1 100644 --- a/argopy/extensions/optical_modeling.py +++ b/argopy/extensions/optical_modeling.py @@ -1,8 +1,8 @@ from typing import Literal, Union import xarray as xr -from ..utils import optical_modeling as om -from . import register_argo_accessor, ArgoAccessorExtension +import argopy.utils.optical_modeling as om +from argopy.extensions.utils import register_argo_accessor, ArgoAccessorExtension @register_argo_accessor("optic") diff --git a/argopy/extensions/params_data_mode.py b/argopy/extensions/params_data_mode.py index a25c97de0..80e336c73 100644 --- a/argopy/extensions/params_data_mode.py +++ b/argopy/extensions/params_data_mode.py @@ -4,16 +4,17 @@ import time from typing import Union, List -from ..utils import to_list, list_core_parameters -from ..utils import ( +from argopy.errors import InvalidDatasetStructure +from argopy.utils.lists import list_core_parameters +from argopy.utils.checkers import to_list +from argopy.utils.transform import ( split_data_mode, merge_param_with_param_adjusted, filter_param_by_data_mode, ) -from ..stores import ArgoIndex -from ..stores.index.spec import ArgoIndexStoreProto -from ..errors import InvalidDatasetStructure -from . import register_argo_accessor, ArgoAccessorExtension +from argopy.stores.index.argo_index import ArgoIndex +from argopy.stores.index.spec import ArgoIndexStoreProto +from argopy.extensions.utils import register_argo_accessor, ArgoAccessorExtension log = logging.getLogger("argopy.xtensions.datamode") diff --git a/argopy/extensions/utils.py b/argopy/extensions/utils.py index 280082dd8..33b9603d2 100644 --- a/argopy/extensions/utils.py +++ b/argopy/extensions/utils.py @@ -1,5 +1,5 @@ -from ..xarray import xr, ArgoAccessor -from ..utils import register_accessor +from argopy.xarray import xr, ArgoAccessor +from argopy.utils.decorators import register_accessor def register_argo_accessor(name): diff --git a/argopy/fetchers.py b/argopy/fetchers.py index 280fdde49..1623f3e7f 100755 --- a/argopy/fetchers.py +++ b/argopy/fetchers.py @@ -19,25 +19,26 @@ import numpy as np import logging -from .options import OPTIONS, VALIDATE, PARALLEL_SETUP -from .errors import ( +from argopy.options import OPTIONS, VALIDATE, PARALLEL_SETUP +from argopy.errors import ( InvalidFetcherAccessPoint, InvalidFetcher, OptionValueError, DataNotFound, ) -from .related import ( +from argopy.related.euroargo_api import ( get_coriolis_profile_id, ) -from .utils.checkers import is_box, is_indexbox, check_wmo, check_cyc -from .utils.lists import ( +from argopy.utils.wmo import check_wmo +from argopy.utils.checkers import is_box, is_indexbox, check_cyc +from argopy.utils.lists import ( list_available_data_src, list_available_index_src, list_core_parameters, list_radiometry_parameters, list_bgc_s_parameters, ) -from .plot import plot_trajectory, bar_plot, open_sat_altim_report, scatter_plot +from argopy.plot.plot import plot_trajectory, bar_plot, open_sat_altim_report, scatter_plot AVAILABLE_DATA_SOURCES = list_available_data_src() diff --git a/argopy/plot/__init__.py b/argopy/plot/__init__.py index 8dc79942e..3f020928f 100644 --- a/argopy/plot/__init__.py +++ b/argopy/plot/__init__.py @@ -1,31 +1,31 @@ -""" -Plot Submodule - -See Also --------- -:func:`argopy.plot.scatter_map`, :func:`argopy.plot.scatter_plot`, :func:`argopy.plot.plot_trajectory`, :func:`argopy.plot.dashboard`, :class:`argopy.plot.ArgoColors`, :class:`argopy.ArgoColors`, :func:`argopy.plot.latlongrid` - - -""" -from .plot import plot_trajectory, bar_plot, open_sat_altim_report, scatter_map, scatter_plot -from .argo_colors import ArgoColors -from .dashboards import open_dashboard as dashboard -from .utils import latlongrid, ARGOPY_COLORS - - -__all__ = ( - # Also available on the argopy module level: - "dashboard", - "ArgoColors", - - # Plot: - "plot_trajectory", - "bar_plot", - "scatter_map", - "scatter_plot", - "open_sat_altim_report", - - # Utils: - "latlongrid", - "ARGOPY_COLORS", -) +# """ +# Plot Submodule +# +# See Also +# -------- +# :func:`argopy.plot.scatter_map`, :func:`argopy.plot.scatter_plot`, :func:`argopy.plot.plot_trajectory`, :func:`argopy.plot.dashboard`, :class:`argopy.plot.ArgoColors`, :class:`argopy.ArgoColors`, :func:`argopy.plot.latlongrid` +# +# +# """ +# from argopy.plot.plot import plot_trajectory, bar_plot, open_sat_altim_report, scatter_map, scatter_plot +# from argopy.plot.argo_colors import ArgoColors +# from argopy.plot.dashboards import open_dashboard as dashboard +# from argopy.plot.utils import latlongrid, ARGOPY_COLORS +# +# +# __all__ = ( +# # Also available on the argopy module level: +# "dashboard", +# "ArgoColors", +# +# # Plot: +# "plot_trajectory", +# "bar_plot", +# "scatter_map", +# "scatter_plot", +# "open_sat_altim_report", +# +# # Utils: +# "latlongrid", +# "ARGOPY_COLORS", +# ) diff --git a/argopy/plot/argo_colors.py b/argopy/plot/argo_colors.py index 8daddb8af..36da4765a 100644 --- a/argopy/plot/argo_colors.py +++ b/argopy/plot/argo_colors.py @@ -1,14 +1,15 @@ import numpy as np from packaging import version -from .utils import has_mpl, has_seaborn, ARGOPY_COLORS -from ..utils.loggers import warnUnless + +from argopy.utils.loggers import warnUnless +from argopy.plot.utils import has_mpl, has_seaborn, ARGOPY_COLORS if has_mpl: - from .utils import mpl, cm, mcolors, plt + from argopy.plot.utils import mpl, cm, mcolors, plt from matplotlib.colors import to_hex if has_seaborn: - from .utils import sns + from argopy.plot.utils import sns class ArgoColors: diff --git a/argopy/plot/dashboards.py b/argopy/plot/dashboards.py index 577ec0908..cd1b8e2e9 100644 --- a/argopy/plot/dashboards.py +++ b/argopy/plot/dashboards.py @@ -10,12 +10,13 @@ import warnings from packaging import version -from .utils import has_ipython -from ..utils.loggers import warnUnless -from ..related.euroargo_api import get_ea_profile_page -from ..utils import check_wmo, check_cyc -from ..errors import InvalidDashboard -from .. import __version__ as argopy_version +from argopy import __version__ as argopy_version +from argopy.errors import InvalidDashboard +from argopy.utils.loggers import warnUnless +from argopy.utils.checkers import check_cyc +from argopy.utils.wmo import check_wmo +from argopy.related.euroargo_api import get_ea_profile_page +from argopy.plot.utils import has_ipython if has_ipython: diff --git a/argopy/plot/plot.py b/argopy/plot/plot.py index d5489195c..0487b9b2c 100644 --- a/argopy/plot/plot.py +++ b/argopy/plot/plot.py @@ -8,26 +8,24 @@ # import warnings import logging -import os -import json import xarray as xr import pandas as pd import numpy as np from typing import Union -import importlib -from ..options import OPTIONS -from ..utils.loggers import warnUnless -from ..utils.checkers import check_wmo -from ..utils.geo import conv_lon -from ..utils.lists import subsample_list -from ..utils.casting import to_list -from ..errors import InvalidDatasetStructure +from argopy.options import OPTIONS +from argopy.errors import InvalidDatasetStructure +from argopy.utils.loggers import warnUnless +from argopy.utils.wmo import check_wmo +from argopy.utils.geo import conv_lon +from argopy.utils.lists import subsample_list +from argopy.utils.casting import to_list +from argopy.utils.locals import Asset -from .utils import STYLE, has_seaborn, has_mpl, has_cartopy, has_ipython, has_ipywidgets -from .utils import axes_style, latlongrid, land_feature -from .argo_colors import ArgoColors +from argopy.plot.utils import STYLE, has_seaborn, has_mpl, has_cartopy, has_ipython, has_ipywidgets +from argopy.plot.utils import axes_style, latlongrid, land_feature +from argopy.plot.argo_colors import ArgoColors if has_mpl: import matplotlib.pyplot as plt @@ -49,13 +47,6 @@ log = logging.getLogger("argopy.plot.plot") -path2assets = importlib.util.find_spec( - "argopy.static.assets" -).submodule_search_locations[0] - -with open(os.path.join(path2assets, "data_types.json"), "r") as f: - DATA_TYPES = json.load(f) - def open_sat_altim_report( WMO: Union[str, list] = None, embed: Union[str, None] = "dropdown", **kwargs @@ -741,7 +732,7 @@ def scatter_plot( """A quick-and-dirty parameter scatter plot for one variable""" warnUnless(has_mpl, "requires matplotlib installed") - if this_param in DATA_TYPES["data"]["str"]: + if this_param in Asset.load('data_types')["data"]["str"]: raise ValueError("scatter_plot does not support string data type (yet !)") if cmap is None: diff --git a/argopy/related/__init__.py b/argopy/related/__init__.py index 465bc5de2..33651556b 100644 --- a/argopy/related/__init__.py +++ b/argopy/related/__init__.py @@ -1,25 +1,25 @@ -from .topography import TopoFetcher -from .ocean_ops_deployments import OceanOPSDeployments -from .reference_tables import ArgoNVSReferenceTables -from .argo_documentation import ArgoDocs -from .doi_snapshot import ArgoDOI -from .euroargo_api import get_coriolis_profile_id, get_ea_profile_page -from .utils import load_dict, mapp_dict # Should come last - +# from argopy.related.topography import TopoFetcher +# from argopy.related.ocean_ops_deployments import OceanOPSDeployments +# from argopy.related.reference_tables import ArgoNVSReferenceTables +# from argopy.related.argo_documentation import ArgoDocs +# from argopy.related.doi_snapshot import ArgoDOI +# from argopy.related.euroargo_api import get_coriolis_profile_id, get_ea_profile_page +# from argopy.related.utils import load_dict, mapp_dict # Should come last # -__all__ = ( - # Classes: - "TopoFetcher", - "OceanOPSDeployments", - "ArgoNVSReferenceTables", - "ArgoDocs", - "ArgoDOI", - - # Functions: - "get_coriolis_profile_id", - "get_ea_profile_page", - - # Utilities: - "load_dict", - "mapp_dict", -) +# # +# __all__ = ( +# # Classes: +# "TopoFetcher", +# "OceanOPSDeployments", +# "ArgoNVSReferenceTables", +# "ArgoDocs", +# "ArgoDOI", +# +# # Functions: +# "get_coriolis_profile_id", +# "get_ea_profile_page", +# +# # Utilities: +# "load_dict", +# "mapp_dict", +# ) diff --git a/argopy/related/argo_documentation.py b/argopy/related/argo_documentation.py index e209d13ee..543a6f2eb 100644 --- a/argopy/related/argo_documentation.py +++ b/argopy/related/argo_documentation.py @@ -1,17 +1,15 @@ -import os -import json import pandas as pd from functools import lru_cache import requests -from ..stores import httpstore, memorystore -from ..options import OPTIONS -from .utils import path2assets +from argopy.options import OPTIONS +from argopy.stores.implementations.http import httpstore +from argopy.stores.implementations.memory import memorystore +from argopy.utils.locals import Asset # Load the ADMT documentation catalogue: -with open(os.path.join(path2assets, "admt_documentation_catalogue.json"), "rb") as f: - ADMT_CATALOGUE = json.load(f)['data']['catalogue'] +ADMT_CATALOGUE = Asset.load('admt_documentation_catalogue')['data']['catalogue'] class ArgoDocs: diff --git a/argopy/related/doi_snapshot.py b/argopy/related/doi_snapshot.py index e6979ac8d..1e8066e92 100644 --- a/argopy/related/doi_snapshot.py +++ b/argopy/related/doi_snapshot.py @@ -6,7 +6,7 @@ # from matplotlib.colors import to_hex # from IPython.display import IFrame -from ..stores import httpstore +from argopy.stores.implementations.http import httpstore class DOIrecord: diff --git a/argopy/related/euroargo_api.py b/argopy/related/euroargo_api.py index fe982fb6d..084f920fb 100644 --- a/argopy/related/euroargo_api.py +++ b/argopy/related/euroargo_api.py @@ -1,7 +1,9 @@ import pandas as pd -from ..options import OPTIONS -from ..utils.checkers import check_wmo, check_cyc -from ..stores import httpstore + +from argopy.options import OPTIONS +from argopy.utils.checkers import check_cyc +from argopy.utils.wmo import check_wmo +from argopy.stores.implementations.http import httpstore def get_coriolis_profile_id(WMO, CYC=None, **kwargs) -> pd.DataFrame: diff --git a/argopy/related/ocean_ops_deployments.py b/argopy/related/ocean_ops_deployments.py index 40d2f21d0..b98c57f49 100644 --- a/argopy/related/ocean_ops_deployments.py +++ b/argopy/related/ocean_ops_deployments.py @@ -1,8 +1,9 @@ import pandas as pd import numpy as np -from ..stores import httpstore -from ..errors import DataNotFound -from ..plot import scatter_map + +from argopy.errors import DataNotFound +from argopy.stores.implementations.http import httpstore +from argopy.plot.plot import scatter_map class OceanOPSDeployments: @@ -33,7 +34,6 @@ class OceanOPSDeployments: Import the class: - >>> from argopy.related import OceanOPSDeployments >>> from argopy import OceanOPSDeployments Possibly define the space/time box to work with: diff --git a/argopy/related/reference_tables.py b/argopy/related/reference_tables.py index 4470a0c7b..fdd2d81e0 100644 --- a/argopy/related/reference_tables.py +++ b/argopy/related/reference_tables.py @@ -1,14 +1,13 @@ import pandas as pd from functools import lru_cache import collections -from pathlib import Path -from ..stores import httpstore, filestore -from ..options import OPTIONS -from ..utils import path2assets +from argopy.options import OPTIONS +from argopy.stores.implementations.http import httpstore +from argopy.utils.locals import Asset -VALID_REF = filestore(cache=True).open_json(Path(path2assets).joinpath("nvs_reference_tables.json"))['data']['valid_ref'] +VALID_REF = Asset.load('nvs_reference_tables')['data']['valid_ref'] class ArgoNVSReferenceTables: diff --git a/argopy/related/topography.py b/argopy/related/topography.py index c03b7618b..53d1ff1ce 100644 --- a/argopy/related/topography.py +++ b/argopy/related/topography.py @@ -1,7 +1,8 @@ from typing import Union -from ..options import OPTIONS -from ..stores import httpstore -from ..utils.format import format_oneline + +from argopy.options import OPTIONS +from argopy.stores.implementations.http import httpstore +from argopy.utils.format import format_oneline class TopoFetcher: diff --git a/argopy/related/utils.py b/argopy/related/utils.py index ad75d4f81..da56e7415 100644 --- a/argopy/related/utils.py +++ b/argopy/related/utils.py @@ -1,14 +1,10 @@ -import importlib -import os -import json import logging -from . import ArgoNVSReferenceTables + +from argopy.utils.locals import Asset +from argopy.related.reference_tables import ArgoNVSReferenceTables log = logging.getLogger("argopy.related.utils") -path2assets = importlib.util.find_spec( - "argopy.static.assets" -).submodule_search_locations[0] def load_dict(ptype): @@ -21,8 +17,7 @@ def load_dict(ptype): profilers = dict(sorted(profilers.items())) return profilers except Exception: - with open(os.path.join(path2assets, "profilers.json"), "rb") as f: - jsdata = json.load(f) + jsdata = Asset.load('profilers') log.debug( "Failed to load the ArgoNVSReferenceTables R08 for profiler types, fall back on static assets last updated on %s" % jsdata["last_update"] @@ -37,8 +32,7 @@ def load_dict(ptype): institutions = dict(sorted(institutions.items())) return institutions except Exception: - with open(os.path.join(path2assets, "institutions.json"), "rb") as f: - jsdata = json.load(f) + jsdata = Asset.load('institutions') log.debug( "Failed to load the ArgoNVSReferenceTables R04 for institutions name, fall back on static assets last updated on %s" % jsdata["last_update"] diff --git a/argopy/stores/__init__.py b/argopy/stores/__init__.py index 1d5425827..c3236a714 100644 --- a/argopy/stores/__init__.py +++ b/argopy/stores/__init__.py @@ -1,31 +1,31 @@ -from .implementations.local import filestore -from .implementations.memory import memorystore -from .implementations.http import httpstore -from .implementations.http_erddap import httpstore_erddap -from .implementations.ftp import ftpstore -from .implementations.s3 import s3store -from .implementations.gdac import gdacfs - -from .index.argo_index import ArgoIndex -from .float.argo_float import ArgoFloat - -from .kerchunker import ArgoKerchunker - -from .filesystems import has_distributed, distributed # noqa: F401 -from .spec import ArgoStoreProto # noqa: F401 -from .implementations.http_erddap import httpstore_erddap_auth # noqa: F401 - - -__all__ = ( - # Classes: - "ArgoIndex", - "ArgoFloat", - "filestore", - "httpstore", - "httpstore_erddap", - "ftpstore", - "memorystore", - "s3store", - "ArgoKerchunker", - "gdacfs", -) +# from argopy.stores.implementations.local import filestore +# from argopy.stores.implementations.memory import memorystore +# from argopy.stores.implementations.http import httpstore +# from argopy.stores.implementations.http_erddap import httpstore_erddap +# from argopy.stores.implementations.ftp import ftpstore +# from argopy.stores.implementations.s3 import s3store +# from argopy.stores.implementations.gdac import gdacfs +# +# from argopy.stores.index.argo_index import ArgoIndex +# from argopy.stores.float.argo_float import ArgoFloat +# +# from argopy.stores.kerchunker import ArgoKerchunker +# +# from argopy.stores.filesystems import has_distributed, distributed # noqa: F401 +# from argopy.stores.spec import ArgoStoreProto # noqa: F401 +# from argopy.stores.implementations.http_erddap import httpstore_erddap_auth # noqa: F401 +# +# +# __all__ = ( +# # Classes: +# "ArgoIndex", +# "ArgoFloat", +# "filestore", +# "httpstore", +# "httpstore_erddap", +# "ftpstore", +# "memorystore", +# "s3store", +# "ArgoKerchunker", +# "gdacfs", +# ) diff --git a/argopy/stores/filesystems.py b/argopy/stores/filesystems.py index 59ac89f26..360586049 100644 --- a/argopy/stores/filesystems.py +++ b/argopy/stores/filesystems.py @@ -6,9 +6,9 @@ from typing import Union import sys -from ..options import OPTIONS -from ..utils.accessories import Registry -from .. import __version__ +from argopy import __version__ +from argopy.options import OPTIONS +from argopy.utils.accessories import Registry if importlib.util.find_spec("boto3") is not None: diff --git a/argopy/stores/float/argo_float.py b/argopy/stores/float/argo_float.py index d184a32fc..6d8f976f0 100644 --- a/argopy/stores/float/argo_float.py +++ b/argopy/stores/float/argo_float.py @@ -10,19 +10,19 @@ import logging import xarray as xr -from ...utils import isconnected -from .implementations.plot import ArgoFloatPlot +from argopy.utils.checkers import isconnected +from argopy.stores.float.implementations.plot import ArgoFloatPlot log = logging.getLogger("argopy.stores.ArgoFloat") if isconnected(): - from .implementations.online.float import FloatStore + from argopy.stores.float.implementations.online.float import FloatStore log.info("Using ONLINE Argo Float implementation") else: - from .implementations.offline.float import FloatStore + from argopy.stores.float.implementations.offline.float import FloatStore log.info("Using OFFLINE Argo Float implementation") diff --git a/argopy/stores/float/implementations/offline/__init__.py b/argopy/stores/float/implementations/offline/__init__.py index c8e86f9c1..e69de29bb 100644 --- a/argopy/stores/float/implementations/offline/__init__.py +++ b/argopy/stores/float/implementations/offline/__init__.py @@ -1 +0,0 @@ -from .float import FloatStore # noqa: F401 diff --git a/argopy/stores/float/implementations/offline/float.py b/argopy/stores/float/implementations/offline/float.py index 3caf916b2..5c3a78bd2 100644 --- a/argopy/stores/float/implementations/offline/float.py +++ b/argopy/stores/float/implementations/offline/float.py @@ -1,8 +1,8 @@ from pathlib import Path import logging -from .....errors import InvalidOption -from ...spec import FloatStoreProto +from argopy.errors import InvalidOption +from argopy.stores.float.spec import FloatStoreProto log = logging.getLogger("argopy.stores.offline.FloatStore") diff --git a/argopy/stores/float/implementations/online/__init__.py b/argopy/stores/float/implementations/online/__init__.py index c8e86f9c1..e69de29bb 100644 --- a/argopy/stores/float/implementations/online/__init__.py +++ b/argopy/stores/float/implementations/online/__init__.py @@ -1 +0,0 @@ -from .float import FloatStore # noqa: F401 diff --git a/argopy/stores/float/implementations/online/float.py b/argopy/stores/float/implementations/online/float.py index 3220eeb73..ae01484bf 100644 --- a/argopy/stores/float/implementations/online/float.py +++ b/argopy/stores/float/implementations/online/float.py @@ -1,8 +1,8 @@ import pandas as pd import logging -from .... import httpstore -from ...spec import FloatStoreProto +from argopy.stores.implementations.http import httpstore +from argopy.stores.float.spec import FloatStoreProto log = logging.getLogger("argopy.stores.online.FloatStore") diff --git a/argopy/stores/float/implementations/plot.py b/argopy/stores/float/implementations/plot.py index 190cc8348..d9d17d04c 100644 --- a/argopy/stores/float/implementations/plot.py +++ b/argopy/stores/float/implementations/plot.py @@ -1,9 +1,11 @@ import numpy as np from typing import Any -from ....plot import scatter_plot, scatter_map, ArgoColors -from ....utils import list_multiprofile_file_variables, list_bgc_s_variables, to_list -from ..extensions import ArgoFloatPlotProto +from argopy.plot.plot import scatter_plot, scatter_map +from argopy.plot.argo_colors import ArgoColors +from argopy.utils.lists import list_multiprofile_file_variables, list_bgc_s_variables +from argopy.utils.checkers import to_list +from argopy.stores.float.extensions import ArgoFloatPlotProto class ArgoFloatPlot(ArgoFloatPlotProto): diff --git a/argopy/stores/float/spec.py b/argopy/stores/float/spec.py index af19a64a7..db7849c04 100644 --- a/argopy/stores/float/spec.py +++ b/argopy/stores/float/spec.py @@ -9,11 +9,13 @@ import logging import numpy as np -from ...errors import InvalidOption -from ...plot import dashboard -from ...utils import check_wmo, argo_split_path, shortcut2gdac -from ...options import OPTIONS -from .. import ArgoIndex +from argopy.options import OPTIONS +from argopy.errors import InvalidOption +from argopy.plot.dashboards import open_dashboard as dashboard +from argopy.utils.wmo import check_wmo +from argopy.utils.format import argo_split_path +from argopy.utils.lists import shortcut2gdac +from argopy.stores.index.argo_index import ArgoIndex log = logging.getLogger("argopy.stores.ArgoFloat") diff --git a/argopy/stores/implementations/ftp.py b/argopy/stores/implementations/ftp.py index ad67fb057..7fb103785 100644 --- a/argopy/stores/implementations/ftp.py +++ b/argopy/stores/implementations/ftp.py @@ -9,11 +9,11 @@ from typing import Literal from netCDF4 import Dataset -from ...errors import InvalidMethod, DataNotFound -from ...utils.transform import drop_variables_not_in_all_datasets -from ..filesystems import has_distributed, distributed -from ..filesystems import tqdm -from .http import httpstore +from argopy.errors import InvalidMethod, DataNotFound +from argopy.utils.transform import drop_variables_not_in_all_datasets +from argopy.stores.filesystems import has_distributed, distributed +from argopy.stores.filesystems import tqdm +from argopy.stores.implementations.http import httpstore log = logging.getLogger("argopy.stores.implementation.ftp") diff --git a/argopy/stores/implementations/gdac.py b/argopy/stores/implementations/gdac.py index f0dcd68ca..006def067 100644 --- a/argopy/stores/implementations/gdac.py +++ b/argopy/stores/implementations/gdac.py @@ -6,10 +6,13 @@ from socket import gaierror import fsspec -from ...options import OPTIONS -from ...errors import GdacPathError -from ...utils.lists import shortcut2gdac -from .. import filestore, httpstore, ftpstore, s3store +from argopy.options import OPTIONS +from argopy.errors import GdacPathError +from argopy.utils.lists import shortcut2gdac +from argopy.stores.implementations.local import filestore +from argopy.stores.implementations.http import httpstore +from argopy.stores.implementations.ftp import ftpstore +from argopy.stores.implementations.s3 import s3store log = logging.getLogger("argopy.stores.gdac") diff --git a/argopy/stores/implementations/http.py b/argopy/stores/implementations/http.py index 32f2425d7..3e4985e0c 100644 --- a/argopy/stores/implementations/http.py +++ b/argopy/stores/implementations/http.py @@ -18,17 +18,18 @@ from netCDF4 import Dataset from urllib.parse import urlparse -from ...errors import InvalidMethod, DataNotFound -from ...utils import Registry, UriCName -from ...utils import has_aws_credentials -from ...utils import ( +from argopy.errors import InvalidMethod, DataNotFound +from argopy.utils.accessories import Registry +from argopy.utils.format import UriCName +from argopy.utils.checkers import has_aws_credentials +from argopy.utils.transform import ( drop_variables_not_in_all_datasets, fill_variables_not_in_all_datasets, ) -from ...utils.monitored_threadpool import MyThreadPoolExecutor as MyExecutor -from ..spec import ArgoStoreProto -from ..filesystems import has_distributed, distributed -from ..filesystems import tqdm +from argopy.utils.monitored_threadpool import MyThreadPoolExecutor as MyExecutor +from argopy.stores.spec import ArgoStoreProto +from argopy.stores.filesystems import has_distributed, distributed +from argopy.stores.filesystems import tqdm log = logging.getLogger("argopy.stores.implementation.http") diff --git a/argopy/stores/implementations/http_erddap.py b/argopy/stores/implementations/http_erddap.py index deee2a973..59ae61076 100644 --- a/argopy/stores/implementations/http_erddap.py +++ b/argopy/stores/implementations/http_erddap.py @@ -3,9 +3,9 @@ from urllib.parse import urlparse, parse_qs import copy -from ...options import OPTIONS -from ...errors import ErddapHTTPNotFound, ErddapHTTPUnauthorized -from .http import httpstore +from argopy.options import OPTIONS +from argopy.errors import ErddapHTTPNotFound, ErddapHTTPUnauthorized +from argopy.stores.implementations.http import httpstore log = logging.getLogger("argopy.stores.implementation.http_erddap") diff --git a/argopy/stores/implementations/local.py b/argopy/stores/implementations/local.py index 9bbd7b077..738584125 100644 --- a/argopy/stores/implementations/local.py +++ b/argopy/stores/implementations/local.py @@ -12,12 +12,12 @@ import warnings from netCDF4 import Dataset -from ...options import OPTIONS -from ...errors import InvalidMethod, DataNotFound +from argopy.options import OPTIONS +from argopy.errors import InvalidMethod, DataNotFound -from ..spec import ArgoStoreProto -from ..filesystems import has_distributed, distributed -from ..filesystems import tqdm +from argopy.stores.spec import ArgoStoreProto +from argopy.stores.filesystems import has_distributed, distributed +from argopy.stores.filesystems import tqdm log = logging.getLogger("argopy.stores.implementation.local") diff --git a/argopy/stores/implementations/memory.py b/argopy/stores/implementations/memory.py index 5706389af..43c1acdaa 100644 --- a/argopy/stores/implementations/memory.py +++ b/argopy/stores/implementations/memory.py @@ -1,5 +1,5 @@ -from .local import filestore -from ...errors import CacheFileNotFound, FileSystemHasNoCache +from argopy.errors import CacheFileNotFound, FileSystemHasNoCache +from argopy.stores.implementations.local import filestore class memorystore(filestore): diff --git a/argopy/stores/implementations/s3.py b/argopy/stores/implementations/s3.py index c259f91e1..4e73bcae8 100644 --- a/argopy/stores/implementations/s3.py +++ b/argopy/stores/implementations/s3.py @@ -1,4 +1,4 @@ -from .http import httpstore +from argopy.stores.implementations.http import httpstore class s3store(httpstore): @@ -13,7 +13,7 @@ class s3store(httpstore): In order to avoid a *no credentials found error*, you can use: - >>> from argopy.utils import has_aws_credentials + >>> from argopy.utils.checkers import has_aws_credentials >>> fs = s3store(anon=not has_aws_credentials()) """ diff --git a/argopy/stores/index/__init__.py b/argopy/stores/index/__init__.py index edf823711..e140e1394 100644 --- a/argopy/stores/index/__init__.py +++ b/argopy/stores/index/__init__.py @@ -1,13 +1,13 @@ -from .implementations.pyarrow.index import ( - indexstore as indexstore_pa, -) # noqa: F401 -from .implementations.pandas.index import ( - indexstore as indexstore_pd, -) # noqa: F401 - - -__all__ = ( - # Classes: - "indexstore_pa", - "indexstore_pd", -) +# from .implementations.pyarrow.index import ( +# indexstore as indexstore_pa, +# ) # noqa: F401 +# from .implementations.pandas.index import ( +# indexstore as indexstore_pd, +# ) # noqa: F401 +# +# +# __all__ = ( +# # Classes: +# "indexstore_pa", +# "indexstore_pd", +# ) diff --git a/argopy/stores/index/argo_index.py b/argopy/stores/index/argo_index.py index 46e05497c..eb1191084 100644 --- a/argopy/stores/index/argo_index.py +++ b/argopy/stores/index/argo_index.py @@ -2,9 +2,9 @@ if importlib.util.find_spec("pyarrow") is not None: - from .implementations.pyarrow.index import indexstore + from argopy.stores.index.implementations.pyarrow.index import indexstore else: - from .implementations.pandas.index import indexstore + from argopy.stores.index.implementations.pandas.index import indexstore class ArgoIndex(indexstore): diff --git a/argopy/stores/index/extensions.py b/argopy/stores/index/extensions.py index 4480a0373..c6792cd6f 100644 --- a/argopy/stores/index/extensions.py +++ b/argopy/stores/index/extensions.py @@ -2,9 +2,9 @@ from typing import NoReturn import logging -from ...utils import register_accessor -from ...errors import InvalidDatasetStructure -from ...utils import to_list +from argopy.errors import InvalidDatasetStructure +from argopy.utils.decorators import register_accessor +from argopy.utils.checkers import to_list log = logging.getLogger("argopy.stores.index.extensions") diff --git a/argopy/stores/index/implementations/index_s3.py b/argopy/stores/index/implementations/index_s3.py index d23064092..d93eb27e0 100644 --- a/argopy/stores/index/implementations/index_s3.py +++ b/argopy/stores/index/implementations/index_s3.py @@ -4,17 +4,17 @@ from decorator import decorator import warnings -from ....utils.checkers import ( +from argopy.errors import InvalidDatasetStructure +from argopy.utils.wmo import check_wmo +from argopy.utils.checkers import ( check_index_cols, - check_wmo, check_cyc, is_list_of_strings, has_aws_credentials, HAS_BOTO3, ) -from ....utils import redact -from ....errors import InvalidDatasetStructure -from ... import s3store +from argopy.utils.format import redact +from argopy.stores.implementations.s3 import s3store try: import pyarrow.csv as csv # noqa: F401 diff --git a/argopy/stores/index/implementations/pandas/__init__.py b/argopy/stores/index/implementations/pandas/__init__.py index eeff10bfc..a65574daf 100644 --- a/argopy/stores/index/implementations/pandas/__init__.py +++ b/argopy/stores/index/implementations/pandas/__init__.py @@ -1,2 +1,2 @@ -from .index import indexstore # noqa: F401 +# from .index import indexstore # noqa: F401 from .search_engine import SearchEngine # noqa: F401 # this import will register the .query accessor, don't remove diff --git a/argopy/stores/index/implementations/pandas/index.py b/argopy/stores/index/implementations/pandas/index.py index 60d7c9eea..03cdb3e01 100644 --- a/argopy/stores/index/implementations/pandas/index.py +++ b/argopy/stores/index/implementations/pandas/index.py @@ -5,10 +5,10 @@ from pathlib import Path from typing import List -from .....options import OPTIONS -from .....errors import DataNotFound, InvalidDatasetStructure -from .....utils import check_index_cols, conv_lon -from ...spec import ArgoIndexStoreProto +from argopy.options import OPTIONS +from argopy.errors import DataNotFound, InvalidDatasetStructure +from argopy.utils.checkers import check_index_cols, conv_lon +from argopy.stores.index.spec import ArgoIndexStoreProto log = logging.getLogger("argopy.stores.index.pd") diff --git a/argopy/stores/index/implementations/pandas/search_engine.py b/argopy/stores/index/implementations/pandas/search_engine.py index 00e7d8ce2..a8c6a73a3 100644 --- a/argopy/stores/index/implementations/pandas/search_engine.py +++ b/argopy/stores/index/implementations/pandas/search_engine.py @@ -3,12 +3,14 @@ import numpy as np from typing import List -from .....options import OPTIONS -from .....errors import InvalidDatasetStructure -from .....utils import is_indexbox, check_wmo, check_cyc, to_list, conv_lon -from ...extensions import register_ArgoIndex_accessor, ArgoIndexSearchEngine -from ..index_s3 import search_s3 -from .index import indexstore +from argopy.options import OPTIONS +from argopy.errors import InvalidDatasetStructure +from argopy.utils.checkers import is_indexbox, check_cyc, to_list, conv_lon +from argopy.utils.wmo import check_wmo +from argopy.stores.index.extensions import register_ArgoIndex_accessor, ArgoIndexSearchEngine +from argopy.stores.index.implementations.index_s3 import search_s3 +from argopy.stores.index.implementations.pandas.index import indexstore + log = logging.getLogger("argopy.stores.index.pd") diff --git a/argopy/stores/index/implementations/plot.py b/argopy/stores/index/implementations/plot.py index 802961612..c52460af9 100644 --- a/argopy/stores/index/implementations/plot.py +++ b/argopy/stores/index/implementations/plot.py @@ -1,7 +1,7 @@ from typing import Any -from ....plot import plot_trajectory, bar_plot -from ..extensions import ArgoIndexPlotProto +from argopy.plot.plot import plot_trajectory, bar_plot +from argopy.stores.index.extensions import ArgoIndexPlotProto class ArgoIndexPlot(ArgoIndexPlotProto): diff --git a/argopy/stores/index/implementations/pyarrow/__init__.py b/argopy/stores/index/implementations/pyarrow/__init__.py index eeff10bfc..a65574daf 100644 --- a/argopy/stores/index/implementations/pyarrow/__init__.py +++ b/argopy/stores/index/implementations/pyarrow/__init__.py @@ -1,2 +1,2 @@ -from .index import indexstore # noqa: F401 +# from .index import indexstore # noqa: F401 from .search_engine import SearchEngine # noqa: F401 # this import will register the .query accessor, don't remove diff --git a/argopy/stores/index/implementations/pyarrow/index.py b/argopy/stores/index/implementations/pyarrow/index.py index 2a4b4d74e..87709c664 100644 --- a/argopy/stores/index/implementations/pyarrow/index.py +++ b/argopy/stores/index/implementations/pyarrow/index.py @@ -15,10 +15,10 @@ except ModuleNotFoundError: pass -from .....options import OPTIONS -from .....errors import DataNotFound, InvalidDatasetStructure -from .....utils import check_index_cols, conv_lon -from ...spec import ArgoIndexStoreProto +from argopy.options import OPTIONS +from argopy.errors import DataNotFound, InvalidDatasetStructure +from argopy.utils.checkers import check_index_cols, conv_lon +from argopy.stores.index.spec import ArgoIndexStoreProto log = logging.getLogger("argopy.stores.index.pa") diff --git a/argopy/stores/index/implementations/pyarrow/search_engine.py b/argopy/stores/index/implementations/pyarrow/search_engine.py index 2079a8d0b..586482d3e 100644 --- a/argopy/stores/index/implementations/pyarrow/search_engine.py +++ b/argopy/stores/index/implementations/pyarrow/search_engine.py @@ -11,12 +11,14 @@ except ModuleNotFoundError: pass -from .....options import OPTIONS -from .....errors import InvalidDatasetStructure -from .....utils import is_indexbox, check_wmo, check_cyc, to_list, conv_lon -from ...extensions import register_ArgoIndex_accessor, ArgoIndexSearchEngine -from ..index_s3 import search_s3 -from .index import indexstore +from argopy.options import OPTIONS +from argopy.errors import InvalidDatasetStructure +from argopy.utils.checkers import is_indexbox, check_cyc, to_list, conv_lon +from argopy.utils.wmo import check_wmo +from argopy.stores.index.extensions import register_ArgoIndex_accessor, ArgoIndexSearchEngine +from argopy.stores.index.implementations.index_s3 import search_s3 +from argopy.stores.index.implementations.pyarrow.index import indexstore + log = logging.getLogger("argopy.stores.index.pa") diff --git a/argopy/stores/index/spec.py b/argopy/stores/index/spec.py index 6e5b966e7..526af14fa 100644 --- a/argopy/stores/index/spec.py +++ b/argopy/stores/index/spec.py @@ -16,16 +16,23 @@ else: from typing_extensions import Self -from ...options import OPTIONS -from ...errors import GdacPathError, S3PathError, InvalidDataset, OptionValueError -from ...utils import isconnected, has_aws_credentials -from ...utils import Registry -from ...utils import Chunker -from ...utils import shortcut2gdac +from argopy.options import OPTIONS +from argopy.errors import GdacPathError, S3PathError, InvalidDataset, OptionValueError +from argopy.utils.checkers import isconnected, has_aws_credentials +from argopy.utils.accessories import Registry +from argopy.utils.chunking import Chunker +from argopy.utils.lists import shortcut2gdac +from argopy.related.utils import load_dict -from .. import httpstore, memorystore, filestore, ftpstore, s3store -from .implementations.index_s3 import get_a_s3index -from .implementations.plot import ArgoIndexPlot +from argopy.stores.implementations.memory import memorystore +from argopy.stores.implementations.local import filestore +from argopy.stores.implementations.http import httpstore +from argopy.stores.implementations.ftp import ftpstore +from argopy.stores.implementations.s3 import s3store + + +from argopy.stores.index.implementations.index_s3 import get_a_s3index +from argopy.stores.index.implementations.plot import ArgoIndexPlot try: import pyarrow.csv as csv # noqa: F401 @@ -446,7 +453,7 @@ def sha_h5(self) -> str: def _r4(self): """Reference table 4 "Argo data centres and institutions" as a dictionary""" if self._load_dict is None: - from ...related import load_dict + # from argopy.related import load_dict self._load_dict = load_dict return self._load_dict('institutions') @@ -454,7 +461,7 @@ def _r4(self): def _r8(self): """Reference table 8 "Argo instrument types" as a dictionary""" if self._load_dict is None: - from ...related import load_dict + # from argopy.related import load_dict self._load_dict = load_dict return self._load_dict('profilers') @@ -688,7 +695,7 @@ def get_filename(s, index): else: log.debug("Converting [%s] to dataframe from scratch ..." % src) # Post-processing for user: - from ...related import mapp_dict + from argopy.related import mapp_dict if nrows is not None: df = df.loc[0 : nrows - 1].copy() @@ -1114,7 +1121,7 @@ def iterfloats(self, index=False, chunksize: int = None): float # is a ArgoFloat instance """ - from .. import ArgoFloat # Prevent circular import + from argopy.stores.float import ArgoFloat # Prevent circular import wmos = self.read_wmo(index=index) diff --git a/argopy/stores/kerchunker.py b/argopy/stores/kerchunker.py index 46a447e06..355c74a17 100644 --- a/argopy/stores/kerchunker.py +++ b/argopy/stores/kerchunker.py @@ -5,8 +5,10 @@ import logging from packaging import version -from ..utils import to_list -from . import memorystore, filestore +from argopy.utils.checkers import to_list +from argopy.stores.implementations.memory import memorystore +from argopy.stores.implementations.local import filestore + log = logging.getLogger("argopy.stores.kerchunk") diff --git a/argopy/stores/spec.py b/argopy/stores/spec.py index 213c03df2..7a20738ed 100644 --- a/argopy/stores/spec.py +++ b/argopy/stores/spec.py @@ -12,12 +12,12 @@ import logging -from ..options import OPTIONS -from ..errors import ( +from argopy.options import OPTIONS +from argopy.errors import ( FileSystemHasNoCache, CacheFileNotFound, ) -from .filesystems import new_fs +from argopy.stores.filesystems import new_fs log = logging.getLogger("argopy.stores.spec") diff --git a/argopy/tests/test_extensions_canyonb.py b/argopy/tests/test_extensions_canyonb.py index d7ae3ad98..95c7bd4f1 100644 --- a/argopy/tests/test_extensions_canyonb.py +++ b/argopy/tests/test_extensions_canyonb.py @@ -4,7 +4,7 @@ import pandas as pd import xarray as xr -from argopy import DataFetcher +from argopy.fetchers import ArgoDataFetcher from mocked_http import mocked_server_address from mocked_http import mocked_httpserver as mocked_erddapserver from utils import requires_pyco2sys @@ -27,7 +27,7 @@ def fetcher(): if USE_MOCKED_SERVER: defaults_args["server"] = mocked_server_address - return DataFetcher(**defaults_args).profile(5903248, 34) + return ArgoDataFetcher(**defaults_args).profile(5903248, 34) @pytest.mark.parametrize( diff --git a/argopy/tests/test_extensions_canyonmed.py b/argopy/tests/test_extensions_canyonmed.py index 00d0eca40..5d3e51364 100644 --- a/argopy/tests/test_extensions_canyonmed.py +++ b/argopy/tests/test_extensions_canyonmed.py @@ -1,7 +1,7 @@ import pytest import logging -from argopy import DataFetcher +from argopy.fetchers import ArgoDataFetcher from mocked_http import mocked_server_address from mocked_http import mocked_httpserver as mocked_erddapserver @@ -21,7 +21,7 @@ def fetcher(): if USE_MOCKED_SERVER: defaults_args['server'] = mocked_server_address - return DataFetcher(**defaults_args).profile(5903248, 34) + return ArgoDataFetcher(**defaults_args).profile(5903248, 34) @pytest.mark.parametrize("what", [ None, diff --git a/argopy/tests/test_extensions_carbonate_content.py b/argopy/tests/test_extensions_carbonate_content.py index 4e5ece3c9..396893a1f 100644 --- a/argopy/tests/test_extensions_carbonate_content.py +++ b/argopy/tests/test_extensions_carbonate_content.py @@ -5,7 +5,7 @@ import xarray as xr import pandas as pd -from argopy import DataFetcher +from argopy.fetchers import ArgoDataFetcher from mocked_http import mocked_server_address from mocked_http import mocked_httpserver as mocked_erddapserver from utils import requires_pyco2sys @@ -28,7 +28,7 @@ def fetcher(): if USE_MOCKED_SERVER: defaults_args["server"] = mocked_server_address - return DataFetcher(**defaults_args).profile(5903248, 34) + return ArgoDataFetcher(**defaults_args).profile(5903248, 34) def test_get_canyon_b_raw_predictions(fetcher, mocked_erddapserver): diff --git a/argopy/tests/test_extensions_optic.py b/argopy/tests/test_extensions_optic.py index 0808c53d6..cdd0678f9 100644 --- a/argopy/tests/test_extensions_optic.py +++ b/argopy/tests/test_extensions_optic.py @@ -1,7 +1,7 @@ import pytest import logging -from argopy import ArgoFloat +from argopy.stores.float.argo_float import ArgoFloat from mocked_http import mocked_server_address from mocked_http import mocked_httpserver as mocked_erddapserver diff --git a/argopy/tests/test_fetchers_dask_cluster.py b/argopy/tests/test_fetchers_dask_cluster.py index ae75af275..215e131b3 100644 --- a/argopy/tests/test_fetchers_dask_cluster.py +++ b/argopy/tests/test_fetchers_dask_cluster.py @@ -6,7 +6,7 @@ import logging -from argopy import DataFetcher +from argopy.fetchers import ArgoDataFetcher from collections import ChainMap import xarray as xr @@ -72,7 +72,7 @@ def create_fetcher(fetcher_args, access_point): def core(fargs, apts): try: - f = DataFetcher(**fargs) + f = ArgoDataFetcher(**fargs) if "float" in apts: f = f.float(apts["float"]) elif "profile" in apts: diff --git a/argopy/tests/test_fetchers_data_argovis.py b/argopy/tests/test_fetchers_data_argovis.py index 94fee7ac2..473145591 100644 --- a/argopy/tests/test_fetchers_data_argovis.py +++ b/argopy/tests/test_fetchers_data_argovis.py @@ -1,6 +1,6 @@ import logging -from argopy import DataFetcher as ArgoDataFetcher +from argopy.fetchers import ArgoDataFetcher from argopy.utils.checkers import is_list_of_strings import pytest diff --git a/argopy/tests/test_fetchers_data_erddap.py b/argopy/tests/test_fetchers_data_erddap.py index cb54a0d55..3c6007081 100644 --- a/argopy/tests/test_fetchers_data_erddap.py +++ b/argopy/tests/test_fetchers_data_erddap.py @@ -1,6 +1,6 @@ import logging -from argopy import DataFetcher as ArgoDataFetcher +from argopy.fetchers import ArgoDataFetcher from argopy.utils.checkers import is_list_of_strings import pytest diff --git a/argopy/tests/test_fetchers_data_erddap_bgc.py b/argopy/tests/test_fetchers_data_erddap_bgc.py index b4475ea7b..0b6d7cca5 100644 --- a/argopy/tests/test_fetchers_data_erddap_bgc.py +++ b/argopy/tests/test_fetchers_data_erddap_bgc.py @@ -1,9 +1,9 @@ import logging import numpy as np -from argopy import DataFetcher as ArgoDataFetcher +from argopy.fetchers import ArgoDataFetcher from argopy.utils.checkers import is_list_of_strings -from argopy.stores.index import indexstore_pd as ArgoIndex # make sure to work with the Pandas index store with erddap-bgc +from argopy.stores.index.implementations.pandas.index import indexstore as ArgoIndex # make sure to work with the Pandas index store with erddap-bgc import pytest import xarray as xr diff --git a/argopy/tests/test_fetchers_data_gdac.py b/argopy/tests/test_fetchers_data_gdac.py index 885f4f155..26f7e64a8 100644 --- a/argopy/tests/test_fetchers_data_gdac.py +++ b/argopy/tests/test_fetchers_data_gdac.py @@ -15,7 +15,7 @@ import importlib import argopy -from argopy import DataFetcher as ArgoDataFetcher +from argopy.fetchers import ArgoDataFetcher from argopy.errors import ( CacheFileNotFound, ) diff --git a/argopy/tests/test_fetchers_facade_data.py b/argopy/tests/test_fetchers_facade_data.py index 6d3895067..e3adce1f3 100644 --- a/argopy/tests/test_fetchers_facade_data.py +++ b/argopy/tests/test_fetchers_facade_data.py @@ -4,13 +4,13 @@ import pytest import argopy -from argopy import DataFetcher as ArgoDataFetcher +from argopy.fetchers import ArgoDataFetcher from argopy.errors import ( InvalidFetcherAccessPoint, InvalidFetcher, OptionValueError, ) -from argopy.utils import is_list_of_strings +from argopy.utils.checkers import is_list_of_strings from utils import ( requires_fetcher, requires_connection, diff --git a/argopy/tests/test_fetchers_facade_index.py b/argopy/tests/test_fetchers_facade_index.py index e689ec929..d84821987 100644 --- a/argopy/tests/test_fetchers_facade_index.py +++ b/argopy/tests/test_fetchers_facade_index.py @@ -2,7 +2,7 @@ import importlib import argopy -from argopy import IndexFetcher as ArgoIndexFetcher +from argopy.fetchers import ArgoIndexFetcher from argopy.errors import InvalidFetcherAccessPoint, OptionValueError from utils import ( # AVAILABLE_INDEX_SOURCES, diff --git a/argopy/tests/test_fetchers_index_erddap.py b/argopy/tests/test_fetchers_index_erddap.py index b42df83e2..a36bcf2a2 100644 --- a/argopy/tests/test_fetchers_index_erddap.py +++ b/argopy/tests/test_fetchers_index_erddap.py @@ -4,7 +4,7 @@ import tempfile import argopy -from argopy import IndexFetcher as ArgoIndexFetcher +from argopy.fetchers import ArgoIndexFetcher from argopy.errors import ( FileSystemHasNoCache, CacheFileNotFound diff --git a/argopy/tests/test_fetchers_index_gdac.py b/argopy/tests/test_fetchers_index_gdac.py index b7459a867..6f90b3ace 100644 --- a/argopy/tests/test_fetchers_index_gdac.py +++ b/argopy/tests/test_fetchers_index_gdac.py @@ -6,7 +6,7 @@ import logging import argopy -from argopy import IndexFetcher as ArgoIndexFetcher +from argopy.fetchers import ArgoIndexFetcher from argopy.errors import CacheFileNotFound, FileSystemHasNoCache, GdacPathError from argopy.utils.checkers import isconnected, is_list_of_strings diff --git a/argopy/tests/test_fetchers_proto.py b/argopy/tests/test_fetchers_proto.py index 0e7c04a6b..8041a0ac5 100644 --- a/argopy/tests/test_fetchers_proto.py +++ b/argopy/tests/test_fetchers_proto.py @@ -1,9 +1,9 @@ import pytest import logging - import xarray + from argopy.data_fetchers.proto import ArgoDataFetcherProto -from argopy.utils import to_list +from argopy.utils.checkers import to_list from mocked_http import mocked_httpserver, mocked_server_address log = logging.getLogger("argopy.tests.fetchers.proto") diff --git a/argopy/tests/test_options.py b/argopy/tests/test_options.py index df80564fb..6ab823407 100644 --- a/argopy/tests/test_options.py +++ b/argopy/tests/test_options.py @@ -2,12 +2,12 @@ import pytest import platform import argopy -from argopy.options import OPTIONS -from argopy.errors import OptionValueError, GdacPathError, ErddapPathError from utils import requires_gdac, create_read_only_folder from mocked_http import mocked_httpserver, mocked_server_address import logging +from argopy.options import OPTIONS +from argopy.errors import OptionValueError, GdacPathError, ErddapPathError log = logging.getLogger("argopy.tests.options") diff --git a/argopy/tests/test_plot_argo_colors.py b/argopy/tests/test_plot_argo_colors.py index 5613ff2b8..beaaa67a1 100644 --- a/argopy/tests/test_plot_argo_colors.py +++ b/argopy/tests/test_plot_argo_colors.py @@ -11,7 +11,7 @@ has_matplotlib, has_seaborn, ) -from argopy.plot import ArgoColors +from argopy.plot.argo_colors import ArgoColors if has_matplotlib: import matplotlib as mpl diff --git a/argopy/tests/test_plot_plot.py b/argopy/tests/test_plot_plot.py index 2962c4ab2..f3e7716aa 100644 --- a/argopy/tests/test_plot_plot.py +++ b/argopy/tests/test_plot_plot.py @@ -19,9 +19,9 @@ has_ipywidgets, ) -from argopy.plot import bar_plot, plot_trajectory, open_sat_altim_report, scatter_map +from argopy.plot.plot import bar_plot, plot_trajectory, open_sat_altim_report, scatter_map from argopy.errors import InvalidDatasetStructure -from argopy import DataFetcher as ArgoDataFetcher +from argopy.fetchers import ArgoDataFetcher from mocked_http import mocked_server_address if has_matplotlib: diff --git a/argopy/tests/test_related.py b/argopy/tests/test_related.py index 7437b46ce..76f47a18e 100644 --- a/argopy/tests/test_related.py +++ b/argopy/tests/test_related.py @@ -16,14 +16,13 @@ create_temp_folder, ) import argopy -from argopy.related import ( - TopoFetcher, - ArgoNVSReferenceTables, - OceanOPSDeployments, - ArgoDocs, - load_dict, mapp_dict, - get_coriolis_profile_id, get_ea_profile_page -) +from argopy.related.topography import TopoFetcher +from argopy.related.reference_tables import ArgoNVSReferenceTables +from argopy.related.ocean_ops_deployments import OceanOPSDeployments +from argopy.related.argo_documentation import ArgoDocs +from argopy.related.utils import load_dict, mapp_dict +from argopy.related.euroargo_api import get_ea_profile_page, get_coriolis_profile_id + from argopy.utils.checkers import ( is_list_of_strings, ) diff --git a/argopy/tests/test_stores_float_plot.py b/argopy/tests/test_stores_float_plot.py index b619e893e..3dfd36575 100644 --- a/argopy/tests/test_stores_float_plot.py +++ b/argopy/tests/test_stores_float_plot.py @@ -2,7 +2,7 @@ import logging import argopy -from argopy.stores import ArgoFloat +from argopy.stores.float.argo_float import ArgoFloat from utils import ( requires_gdac, diff --git a/argopy/tests/test_stores_fs_gdac.py b/argopy/tests/test_stores_fs_gdac.py index 14b545eae..53f73671a 100644 --- a/argopy/tests/test_stores_fs_gdac.py +++ b/argopy/tests/test_stores_fs_gdac.py @@ -7,7 +7,7 @@ import logging import argopy -from argopy.stores import gdacfs +from argopy.stores.implementations.gdac import gdacfs from mocked_http import mocked_httpserver, mocked_server_address from utils import patch_ftp diff --git a/argopy/tests/test_stores_fsspec.py b/argopy/tests/test_stores_fsspec.py index 1b0ba8fdc..f5071f47f 100644 --- a/argopy/tests/test_stores_fsspec.py +++ b/argopy/tests/test_stores_fsspec.py @@ -16,13 +16,11 @@ import netCDF4 import argopy -from argopy.stores import ( - filestore, - httpstore, - memorystore, - ftpstore, -) from argopy.stores.filesystems import new_fs +from argopy.stores.implementations.local import filestore +from argopy.stores.implementations.http import httpstore +from argopy.stores.implementations.memory import memorystore +from argopy.stores.implementations.ftp import ftpstore from argopy.options import OPTIONS from argopy.errors import ( FileSystemHasNoCache, diff --git a/argopy/tests/test_stores_index.py b/argopy/tests/test_stores_index.py index 56ca3a783..dd141c53e 100644 --- a/argopy/tests/test_stores_index.py +++ b/argopy/tests/test_stores_index.py @@ -16,9 +16,9 @@ OptionValueError, InvalidDatasetStructure, ) -from argopy.utils.checkers import is_list_of_strings, is_wmo -from argopy.stores.index import indexstore_pd -from argopy.stores import ArgoFloat +from argopy.utils.checkers import is_list_of_strings +from argopy.utils.wmo import is_wmo +from argopy.stores.float.argo_float import ArgoFloat from utils import create_temp_folder from mocked_http import mocked_httpserver, mocked_server_address from utils import patch_ftp @@ -560,7 +560,9 @@ def test_dateline_search(self): @skip_CORE class Test_IndexStore_pandas_CORE(IndexStore_test_proto): network = "core" - indexstore = indexstore_pd + from argopy.stores.index.implementations.pandas.index import indexstore + + indexstore = indexstore index_file = "ar_index_global_prof.txt" @@ -568,7 +570,9 @@ class Test_IndexStore_pandas_CORE(IndexStore_test_proto): @skip_BGCs class Test_IndexStore_pandas_BGC_synthetic(IndexStore_test_proto): network = "bgc" - indexstore = indexstore_pd + from argopy.stores.index.implementations.pandas.index import indexstore + + indexstore = indexstore index_file = "argo_synthetic-profile_index.txt" @@ -576,7 +580,9 @@ class Test_IndexStore_pandas_BGC_synthetic(IndexStore_test_proto): @skip_BGCb class Test_IndexStore_pandas_BGC_bio(IndexStore_test_proto): network = "bgc" - indexstore = indexstore_pd + from argopy.stores.index.implementations.pandas.index import indexstore + + indexstore = indexstore index_file = "argo_bio-profile_index.txt" @@ -589,9 +595,9 @@ class Test_IndexStore_pandas_BGC_bio(IndexStore_test_proto): @skip_CORE class Test_IndexStore_pyarrow_CORE(IndexStore_test_proto): network = "core" - from argopy.stores.index import indexstore_pa + from argopy.stores.index.implementations.pyarrow.index import indexstore - indexstore = indexstore_pa + indexstore = indexstore index_file = "ar_index_global_prof.txt" @@ -600,9 +606,9 @@ class Test_IndexStore_pyarrow_CORE(IndexStore_test_proto): @skip_BGCs class Test_IndexStore_pyarrow_BGC_bio(IndexStore_test_proto): network = "bgc" - from argopy.stores.index import indexstore_pa + from argopy.stores.index.implementations.pyarrow.index import indexstore - indexstore = indexstore_pa + indexstore = indexstore index_file = "argo_bio-profile_index.txt" @@ -611,7 +617,7 @@ class Test_IndexStore_pyarrow_BGC_bio(IndexStore_test_proto): @skip_BGCb class Test_IndexStore_pyarrow_BGC_synthetic(IndexStore_test_proto): network = "bgc" - from argopy.stores.index import indexstore_pa + from argopy.stores.index.implementations.pyarrow.index import indexstore - indexstore = indexstore_pa + indexstore = indexstore index_file = "argo_synthetic-profile_index.txt" diff --git a/argopy/tests/test_stores_index_plot.py b/argopy/tests/test_stores_index_plot.py index c69d11e9b..3a5aa3d87 100644 --- a/argopy/tests/test_stores_index_plot.py +++ b/argopy/tests/test_stores_index_plot.py @@ -1,9 +1,8 @@ import pytest import logging -import importlib import argopy -from argopy.stores import ArgoIndex +from argopy.stores.index.argo_index import ArgoIndex from utils import ( requires_gdac, diff --git a/argopy/tests/test_utils_accessories.py b/argopy/tests/test_utils_accessories.py index f786d8c1f..a8edd9ef4 100644 --- a/argopy/tests/test_utils_accessories.py +++ b/argopy/tests/test_utils_accessories.py @@ -1,31 +1,6 @@ import pytest -from argopy.utils.accessories import float_wmo, Registry - - -class Test_float_wmo(): - - def test_init(self): - assert isinstance(float_wmo(2901746), float_wmo) - assert isinstance(float_wmo(float_wmo(2901746)), float_wmo) - - def test_isvalid(self): - assert float_wmo(2901746).isvalid - assert not float_wmo(12, errors='ignore').isvalid - - def test_ppt(self): - assert isinstance(str(float_wmo(2901746)), str) - assert isinstance(repr(float_wmo(2901746)), str) - - def test_comparisons(self): - assert float_wmo(2901746) == float_wmo(2901746) - assert float_wmo(2901746) != float_wmo(2901745) - assert float_wmo(2901746) >= float_wmo(2901746) - assert float_wmo(2901746) > float_wmo(2901745) - assert float_wmo(2901746) <= float_wmo(2901746) - assert float_wmo(2901746) < float_wmo(2901747) - - def test_hashable(self): - assert isinstance(hash(float_wmo(2901746)), int) +from argopy.utils.wmo import float_wmo +from argopy.utils.accessories import Registry class Test_Registry(): @@ -83,4 +58,3 @@ def test_invalid_dtype(self, opts): Registry(opts[0][0], dtype=opts[1], invalid='warn').commit(opts[0][-1]) # Raise nothing: Registry(opts[0][0], dtype=opts[1], invalid='ignore').commit(opts[0][-1]) - diff --git a/argopy/tests/test_utils_caching.py b/argopy/tests/test_utils_caching.py index a4f84ffb6..409e55ac9 100644 --- a/argopy/tests/test_utils_caching.py +++ b/argopy/tests/test_utils_caching.py @@ -2,7 +2,7 @@ import pandas as pd import argopy import tempfile -from argopy import DataFetcher as ArgoDataFetcher +from argopy.fetchers import ArgoDataFetcher from utils import ( requires_gdac, ) diff --git a/argopy/tests/test_utils_checkers.py b/argopy/tests/test_utils_checkers.py index 73fc05b19..de76ab7f8 100644 --- a/argopy/tests/test_utils_checkers.py +++ b/argopy/tests/test_utils_checkers.py @@ -11,7 +11,6 @@ from argopy.errors import GdacPathError from argopy.utils.checkers import ( is_box, is_indexbox, - check_wmo, is_wmo, check_cyc, is_cyc, check_gdac_path, isconnected, urlhaskeyword, isAPIconnected, erddap_ds_exists, isalive @@ -118,42 +117,6 @@ def test_box_invalid_str(self): assert not is_indexbox(box, errors="ignore") -def test_is_wmo(): - assert is_wmo(12345) - assert is_wmo([12345]) - assert is_wmo([12345, 1234567]) - - with pytest.raises(ValueError): - is_wmo(1234, errors="raise") - with pytest.raises(ValueError): - is_wmo(-1234, errors="raise") - with pytest.raises(ValueError): - is_wmo(1234.12, errors="raise") - with pytest.raises(ValueError): - is_wmo(12345.7, errors="raise") - - with pytest.warns(UserWarning): - is_wmo(1234, errors="warn") - with pytest.warns(UserWarning): - is_wmo(-1234, errors="warn") - with pytest.warns(UserWarning): - is_wmo(1234.12, errors="warn") - with pytest.warns(UserWarning): - is_wmo(12345.7, errors="warn") - - assert not is_wmo(12, errors="ignore") - assert not is_wmo(-12, errors="ignore") - assert not is_wmo(1234.12, errors="ignore") - assert not is_wmo(12345.7, errors="ignore") - - -def test_check_wmo(): - assert check_wmo(12345) == [12345] - assert check_wmo([1234567]) == [1234567] - assert check_wmo([12345, 1234567]) == [12345, 1234567] - assert check_wmo(np.array((12345, 1234567), dtype='int')) == [12345, 1234567] - - def test_is_cyc(): assert is_cyc(123) assert is_cyc([123]) diff --git a/argopy/tests/test_utils_locals.py b/argopy/tests/test_utils_locals.py index 4ad32bfd1..9c34a1d38 100644 --- a/argopy/tests/test_utils_locals.py +++ b/argopy/tests/test_utils_locals.py @@ -1,8 +1,10 @@ import os + +import pandas as pd import pytest import io import argopy -from argopy.utils.locals import modified_environ +from argopy.utils.locals import modified_environ, Asset @pytest.mark.parametrize("conda", [False, True], @@ -20,3 +22,22 @@ def test_modified_environ(): assert os.environ['DUMMY_ENV_ARGOPY'] == 'toto' assert os.environ['DUMMY_ENV_ARGOPY'] == 'initial' os.environ.pop('DUMMY_ENV_ARGOPY') + + +class Test_Asset(): + assets = ['gdac_servers.json', 'data_types', 'schema:argo.sensor.schema.json', 'schema:argo.float.schema'] + assets_id = [f"{a}" for a in assets] + @pytest.mark.parametrize("asset", assets, indirect=False, ids=assets_id) + def test_load_json(self, asset): + data = Asset.load(asset) + assert isinstance(data, dict) + + assets = ['canyon-b:wgts_AT.txt'] + assets_id = [f"{a}" for a in assets] + @pytest.mark.parametrize("asset", assets, indirect=False, ids=assets_id) + def test_load_csv(self, asset): + data = Asset.load(asset) + assert isinstance(data, pd.DataFrame) + + data = Asset.load(asset, header=None, sep="\t") + assert isinstance(data, pd.DataFrame) diff --git a/argopy/tests/test_utils_transform.py b/argopy/tests/test_utils_transform.py index 37c28a657..17617d157 100644 --- a/argopy/tests/test_utils_transform.py +++ b/argopy/tests/test_utils_transform.py @@ -6,7 +6,7 @@ from argopy import tutorial from argopy.errors import InvalidDatasetStructure -from argopy.utils import ( +from argopy.utils.transform import ( fill_variables_not_in_all_datasets, drop_variables_not_in_all_datasets, merge_param_with_param_adjusted, diff --git a/argopy/tests/test_utils_wmo.py b/argopy/tests/test_utils_wmo.py new file mode 100644 index 000000000..beb159f45 --- /dev/null +++ b/argopy/tests/test_utils_wmo.py @@ -0,0 +1,65 @@ +import pytest +import numpy as np +from argopy.utils.wmo import float_wmo, is_wmo, check_wmo + + +def test_is_wmo(): + assert is_wmo(12345) + assert is_wmo([12345]) + assert is_wmo([12345, 1234567]) + + with pytest.raises(ValueError): + is_wmo(1234, errors="raise") + with pytest.raises(ValueError): + is_wmo(-1234, errors="raise") + with pytest.raises(ValueError): + is_wmo(1234.12, errors="raise") + with pytest.raises(ValueError): + is_wmo(12345.7, errors="raise") + + with pytest.warns(UserWarning): + is_wmo(1234, errors="warn") + with pytest.warns(UserWarning): + is_wmo(-1234, errors="warn") + with pytest.warns(UserWarning): + is_wmo(1234.12, errors="warn") + with pytest.warns(UserWarning): + is_wmo(12345.7, errors="warn") + + assert not is_wmo(12, errors="ignore") + assert not is_wmo(-12, errors="ignore") + assert not is_wmo(1234.12, errors="ignore") + assert not is_wmo(12345.7, errors="ignore") + + +def test_check_wmo(): + assert check_wmo(12345) == [12345] + assert check_wmo([1234567]) == [1234567] + assert check_wmo([12345, 1234567]) == [12345, 1234567] + assert check_wmo(np.array((12345, 1234567), dtype='int')) == [12345, 1234567] + + +class Test_float_wmo(): + + def test_init(self): + assert isinstance(float_wmo(2901746), float_wmo) + assert isinstance(float_wmo(float_wmo(2901746)), float_wmo) + + def test_isvalid(self): + assert float_wmo(2901746).isvalid + assert not float_wmo(12, errors='ignore').isvalid + + def test_ppt(self): + assert isinstance(str(float_wmo(2901746)), str) + assert isinstance(repr(float_wmo(2901746)), str) + + def test_comparisons(self): + assert float_wmo(2901746) == float_wmo(2901746) + assert float_wmo(2901746) != float_wmo(2901745) + assert float_wmo(2901746) >= float_wmo(2901746) + assert float_wmo(2901746) > float_wmo(2901745) + assert float_wmo(2901746) <= float_wmo(2901746) + assert float_wmo(2901746) < float_wmo(2901747) + + def test_hashable(self): + assert isinstance(hash(float_wmo(2901746)), int) diff --git a/argopy/tests/test_xarray_accessor.py b/argopy/tests/test_xarray_accessor.py index fdca2cc1f..bad52723f 100644 --- a/argopy/tests/test_xarray_accessor.py +++ b/argopy/tests/test_xarray_accessor.py @@ -6,7 +6,7 @@ import xarray as xr import argopy -from argopy import DataFetcher as ArgoDataFetcher +from argopy.fetchers import ArgoDataFetcher from argopy.errors import InvalidDatasetStructure, OptionValueError from utils import requires_gdac, _importorskip, _connectskip from mocked_http import mocked_server_address diff --git a/argopy/utils/__init__.py b/argopy/utils/__init__.py index 50c24af2d..f1aeb4822 100644 --- a/argopy/utils/__init__.py +++ b/argopy/utils/__init__.py @@ -1,162 +1,160 @@ -from .checkers import ( # noqa: F401 - is_box, - is_indexbox, - is_list_of_strings, - is_list_of_dicts, - is_list_of_datasets, - is_list_equal, - is_wmo, - check_wmo, - is_cyc, - check_cyc, - check_index_cols, - check_gdac_path, - isconnected, - urlhaskeyword, # noqa: F401 - isalive, - isAPIconnected, - erddap_ds_exists, - has_aws_credentials, -) -from .casting import DATA_TYPES, cast_Argo_variable_type, to_list -from .decorators import deprecated, doc_inherit, register_accessor -from .lists import ( - list_available_data_src, - list_available_index_src, - list_multiprofile_file_variables, - list_core_parameters, - list_standard_variables, - list_bgc_s_variables, - list_bgc_s_parameters, - list_radiometry_variables, - list_radiometry_parameters, - list_gdac_servers, - shortcut2gdac, -) -from .caching import clear_cache, lscache -from .monitored_threadpool import MyThreadPoolExecutor as MonitoredThreadPoolExecutor -from .chunking import Chunker -from .accessories import Registry, float_wmo -from .locals import ( # noqa: F401 - show_versions, - show_options, - modified_environ, - get_sys_info, # noqa: F401 - netcdf_and_hdf5_versions, # noqa: F401 -) -from .monitors import monitor_status, badge, fetch_status # noqa: F401 -from .geo import ( - wmo2box, - wrap_longitude, - conv_lon, - toYearFraction, - YearFraction_to_datetime, - point_in_polygon, -) -from .compute import linear_interpolation_remap, groupby_remap -from .transform import ( - fill_variables_not_in_all_datasets, - drop_variables_not_in_all_datasets, - merge_param_with_param_adjusted, - filter_param_by_data_mode, - split_data_mode, -) -from .format import argo_split_path, format_oneline, UriCName, redact, dirfs_relpath -from .loggers import warnUnless, log_argopy_callerstack -from .carbon import GreenCoding, Github -from . import optical_modeling -from .carbonate import calculate_uncertainties, error_propagation - -import importlib -path2assets = importlib.util.find_spec('argopy.static.assets').submodule_search_locations[0] - - -__all__ = ( - # Checkers: - "is_box", - "is_indexbox", - "is_list_of_strings", - "is_list_of_dicts", - "is_list_of_datasets", - "is_list_equal", - "is_wmo", - "check_wmo", - "is_cyc", - "check_cyc", - "check_index_cols", - "check_gdac_path", - "isconnected", - "isalive", - "isAPIconnected", - "erddap_ds_exists", - "has_aws_credentials", - # Data type casting: - "DATA_TYPES", - "cast_Argo_variable_type", - "to_list", - # Decorators: - "deprecated", - "doc_inherit", - "register_accessor", - # Lists: - "list_available_data_src", - "list_available_index_src", - "list_multiprofile_file_variables", - "list_standard_variables", - "list_core_parameters", - "list_bgc_s_variables", - "list_bgc_s_parameters", - "list_radiometry_variables", - "list_radiometry_parameters", - "list_gdac_servers", - "shortcut2gdac", - # Cache management: - "clear_cache", - "lscache", - # Computation and performances: - "MonitoredThreadPoolExecutor", - "Chunker", - # Accessories classes (specific objects): - "Registry", - "float_wmo", - # Locals (environments, versions, systems): - "path2assets", - "show_versions", - "show_options", - "modified_environ", - # Monitors - "monitor_status", - # Geo (space/time data utilities) - "wmo2box", - "wrap_longitude", - "conv_lon", - "toYearFraction", - "YearFraction_to_datetime", - "point_in_polygon", - # Computation with datasets: - "linear_interpolation_remap", - "groupby_remap", - # Transform datasets: - "fill_variables_not_in_all_datasets", - "drop_variables_not_in_all_datasets", - "merge_param_with_param_adjusted", - "filter_param_by_data_mode", - "split_data_mode", - # Formatters: - "format_oneline", - "argo_split_path", - "dirfs_relpath", - "UriCName", - "redact", - # Loggers: - "warnUnless", - "log_argopy_callerstack", - # Carbon - "GreenCoding", - "Github", - # Optical modeling - "optical_modeling", - # Carbonate calculations - "calculate_uncertainties", - "error_propagation", -) +# from argopy.utils.checkers import ( # noqa: F401 +# is_box, +# is_indexbox, +# is_list_of_strings, +# is_list_of_dicts, +# is_list_of_datasets, +# is_list_equal, +# is_wmo, +# check_wmo, +# is_cyc, +# check_cyc, +# check_index_cols, +# check_gdac_path, +# isconnected, +# urlhaskeyword, # noqa: F401 +# isalive, +# isAPIconnected, +# erddap_ds_exists, +# has_aws_credentials, +# ) +# from argopy.utils.casting import DATA_TYPES, cast_Argo_variable_type, to_list +# from argopy.utils.decorators import deprecated, doc_inherit, register_accessor +# from argopy.utils.lists import ( +# list_available_data_src, +# list_available_index_src, +# list_multiprofile_file_variables, +# list_core_parameters, +# list_standard_variables, +# list_bgc_s_variables, +# list_bgc_s_parameters, +# list_radiometry_variables, +# list_radiometry_parameters, +# list_gdac_servers, +# shortcut2gdac, +# ) +# from argopy.utils.caching import clear_cache, lscache +# from argopy.utils.monitored_threadpool import MyThreadPoolExecutor as MonitoredThreadPoolExecutor +# from argopy.utils.chunking import Chunker +# from argopy.utils.accessories import Registry, float_wmo +# from argopy.utils.locals import ( # noqa: F401 +# show_versions, +# show_options, +# modified_environ, +# get_sys_info, # noqa: F401 +# netcdf_and_hdf5_versions, # noqa: F401 +# Asset, +# ) +# from argopy.utils.monitors import monitor_status, badge, fetch_status # noqa: F401 +# from argopy.utils.geo import ( +# wmo2box, +# wrap_longitude, +# conv_lon, +# toYearFraction, +# YearFraction_to_datetime, +# point_in_polygon, +# ) +# from argopy.utils.compute import linear_interpolation_remap, groupby_remap +# from argopy.utils.transform import ( +# fill_variables_not_in_all_datasets, +# drop_variables_not_in_all_datasets, +# merge_param_with_param_adjusted, +# filter_param_by_data_mode, +# split_data_mode, +# ) +# from argopy.utils.format import argo_split_path, format_oneline, UriCName, redact, dirfs_relpath +# from argopy.utils.loggers import warnUnless, log_argopy_callerstack +# from argopy.utils.carbon import GreenCoding, Github +# from argopy.utils import optical_modeling +# from argopy.utils.carbonate import calculate_uncertainties, error_propagation +# +# +# __all__ = ( +# # Checkers: +# "is_box", +# "is_indexbox", +# "is_list_of_strings", +# "is_list_of_dicts", +# "is_list_of_datasets", +# "is_list_equal", +# "is_wmo", +# "check_wmo", +# "is_cyc", +# "check_cyc", +# "check_index_cols", +# "check_gdac_path", +# "isconnected", +# "isalive", +# "isAPIconnected", +# "erddap_ds_exists", +# "has_aws_credentials", +# # Data type casting: +# "DATA_TYPES", +# "cast_Argo_variable_type", +# "to_list", +# # Decorators: +# "deprecated", +# "doc_inherit", +# "register_accessor", +# # Lists: +# "list_available_data_src", +# "list_available_index_src", +# "list_multiprofile_file_variables", +# "list_standard_variables", +# "list_core_parameters", +# "list_bgc_s_variables", +# "list_bgc_s_parameters", +# "list_radiometry_variables", +# "list_radiometry_parameters", +# "list_gdac_servers", +# "shortcut2gdac", +# # Cache management: +# "clear_cache", +# "lscache", +# # Computation and performances: +# "MonitoredThreadPoolExecutor", +# "Chunker", +# # Accessories classes (specific objects): +# "Registry", +# "float_wmo", +# # Locals (environments, versions, systems, assets): +# "show_versions", +# "show_options", +# "modified_environ", +# "Asset", +# # Monitors +# "monitor_status", +# # Geo (space/time data utilities) +# "wmo2box", +# "wrap_longitude", +# "conv_lon", +# "toYearFraction", +# "YearFraction_to_datetime", +# "point_in_polygon", +# # Computation with datasets: +# "linear_interpolation_remap", +# "groupby_remap", +# # Transform datasets: +# "fill_variables_not_in_all_datasets", +# "drop_variables_not_in_all_datasets", +# "merge_param_with_param_adjusted", +# "filter_param_by_data_mode", +# "split_data_mode", +# # Formatters: +# "format_oneline", +# "argo_split_path", +# "dirfs_relpath", +# "UriCName", +# "redact", +# # Loggers: +# "warnUnless", +# "log_argopy_callerstack", +# # Carbon +# "GreenCoding", +# "Github", +# # Optical modeling +# "optical_modeling", +# # Carbonate calculations +# "calculate_uncertainties", +# "error_propagation", +# ) diff --git a/argopy/utils/accessories.py b/argopy/utils/accessories.py index be0bd7313..6f60ef04e 100644 --- a/argopy/utils/accessories.py +++ b/argopy/utils/accessories.py @@ -4,8 +4,6 @@ import logging import copy -from .checkers import check_wmo, is_wmo - log = logging.getLogger("argopy.utils.accessories") @@ -32,75 +30,6 @@ def __repr__(self): raise NotImplementedError("Not implemented") -class float_wmo(RegistryItem): - """Argo float WMO number object""" - - def __init__(self, WMO_number, errors="raise"): - """Create an Argo float WMO number object - - Parameters - ---------- - WMO_number: object - Anything that could be casted as an integer - errors: {'raise', 'warn', 'ignore'} - Possibly raises a ValueError exception or UserWarning, otherwise fails silently if WMO_number is not valid - - Returns - ------- - :class:`argopy.utilities.float_wmo` - """ - self.errors = errors - if isinstance(WMO_number, float_wmo): - item = WMO_number.value - else: - item = check_wmo(WMO_number, errors=self.errors)[ - 0 - ] # This will automatically validate item - self.item = item - - @property - def isvalid(self): - """Check if WMO number is valid""" - return is_wmo(self.item, errors=self.errors) - # return True # Because it was checked at instantiation - - @property - def value(self): - """Return WMO number as in integer""" - return int(self.item) - - def __str__(self): - # return "%s" % check_wmo(self.item)[0] - return "%s" % self.item - - def __repr__(self): - return f"WMO({self.item})" - - def __check_other__(self, other): - return check_wmo(other)[0] if type(other) is not float_wmo else other.item - - def __eq__(self, other): - return self.item.__eq__(self.__check_other__(other)) - - def __ne__(self, other): - return self.item.__ne__(self.__check_other__(other)) - - def __gt__(self, other): - return self.item.__gt__(self.__check_other__(other)) - - def __lt__(self, other): - return self.item.__lt__(self.__check_other__(other)) - - def __ge__(self, other): - return self.item.__ge__(self.__check_other__(other)) - - def __le__(self, other): - return self.item.__le__(self.__check_other__(other)) - - def __hash__(self): - return hash(self.item) - - class Registry(UserList): """A list manager that can validate item type @@ -177,14 +106,18 @@ def __init__( """ self.name = name self._invalid = invalid - if dtype == float_wmo or str(dtype).lower() == "wmo": + if str(dtype).lower() in ["float_wmo", "wmo"]: + from argopy.utils.wmo import float_wmo self._validator = self._wmo + self._dtype = "float_wmo" self.dtype = float_wmo elif hasattr(dtype, "isvalid"): self._validator = dtype.isvalid + self._dtype = str(dtype).lower() self.dtype = dtype else: self._validator = self._isinstance + self._dtype = str(dtype).lower() self.dtype = dtype # else: # raise ValueError("Unrecognised Registry data type '%s'" % dtype) @@ -209,8 +142,8 @@ def __repr__(self): def _process_items(self, items): if not isinstance(items, list): items = [items] - if self.dtype == float_wmo: - items = [float_wmo(item, errors=self._invalid) for item in items] + if self._dtype == "float_wmo": + items = [self.dtype(item, errors=self._invalid) for item in items] return items def commit(self, values): diff --git a/argopy/utils/caching.py b/argopy/utils/caching.py index 32513910d..e6f3023cd 100644 --- a/argopy/utils/caching.py +++ b/argopy/utils/caching.py @@ -6,8 +6,10 @@ import fsspec import pandas as pd from packaging import version -from ..options import OPTIONS -from ..errors import FileSystemHasNoCache + +from argopy.options import OPTIONS +from argopy.errors import FileSystemHasNoCache + log = logging.getLogger("argopy.utils.caching") diff --git a/argopy/utils/carbon.py b/argopy/utils/carbon.py index 368474ee9..51f258d86 100644 --- a/argopy/utils/carbon.py +++ b/argopy/utils/carbon.py @@ -7,8 +7,9 @@ from pathlib import Path import numpy as np -from ..errors import DataNotFound -from . import Registry +from argopy.errors import DataNotFound +from argopy.utils.accessories import Registry + log = logging.getLogger("argopy.utils.monitors") diff --git a/argopy/utils/casting.py b/argopy/utils/casting.py index fe7235bc5..79c255a27 100644 --- a/argopy/utils/casting.py +++ b/argopy/utils/casting.py @@ -1,22 +1,15 @@ import sys -import os import numpy as np import pandas as pd import xarray as xr -import importlib -import json import logging from copy import deepcopy +from argopy.utils.locals import Asset -log = logging.getLogger("argopy.utils.casting") - -path2assets = importlib.util.find_spec( - "argopy.static.assets" -).submodule_search_locations[0] -with open(os.path.join(path2assets, "data_types.json"), "r") as f: - DATA_TYPES = json.load(f) +log = logging.getLogger("argopy.utils.casting") +DATA_TYPES = Asset.load('data_types')['data'] def cast_Argo_variable_type(ds, overwrite=True): @@ -61,14 +54,14 @@ def cast_this_da(da, v): # print("Casting %s ..." % da.name) da.attrs["casted"] = 0 - if v in DATA_TYPES["data"]["str"] and da.dtype == "O": # Object + if v in DATA_TYPES["str"] and da.dtype == "O": # Object try: da = cast_this(da, str, exception_to_raise=UnicodeDecodeError) except UnicodeDecodeError: da = da.str.decode(encoding="unicode_escape") da = cast_this(da, str) - if v in DATA_TYPES["data"]["int"]: # and da.dtype == 'O': # Object + if v in DATA_TYPES["int"]: # and da.dtype == 'O': # Object if "conventions" in da.attrs: convname = "conventions" elif "convention" in da.attrs: @@ -90,7 +83,7 @@ def cast_this_da(da, v): da = cast_this(da, float) da = cast_this(da, int) - if v in DATA_TYPES["data"]["datetime"] and da.dtype == "O": # Object + if v in DATA_TYPES["datetime"] and da.dtype == "O": # Object if ( "conventions" in da.attrs and da.attrs["conventions"] == "YYYYMMDDHHMISS" diff --git a/argopy/utils/checkers.py b/argopy/utils/checkers.py index 908e66791..7c249eb06 100644 --- a/argopy/utils/checkers.py +++ b/argopy/utils/checkers.py @@ -11,11 +11,11 @@ import logging import importlib -from ..options import OPTIONS -from ..errors import InvalidDatasetStructure, GdacPathError, InvalidFetcher -from .lists import list_available_data_src, list_available_index_src, list_gdac_servers -from .casting import to_list -from .geo import conv_lon +from argopy.options import OPTIONS +from argopy.errors import InvalidDatasetStructure, GdacPathError, InvalidFetcher +from argopy.utils.lists import list_available_data_src, list_available_index_src, list_gdac_servers +from argopy.utils.casting import to_list +from argopy.utils.geo import conv_lon log = logging.getLogger("argopy.utils.checkers") @@ -310,80 +310,6 @@ def is_list_equal(lst1, lst2): ) -def check_wmo(lst, errors="raise"): - """Validate a WMO option and returned it as a list of integers - - Parameters - ---------- - wmo: int - WMO must be an integer or an iterable with elements that can be casted as integers - errors: {'raise', 'warn', 'ignore'} - Possibly raises a ValueError exception or UserWarning, otherwise fails silently. - - Returns - ------- - list(int) - """ - is_wmo(lst, errors=errors) - - # Make sure we deal with a list - lst = to_list(lst) - - # Then cast list elements as integers - return [abs(int(x)) for x in lst] - - -def is_wmo(lst, errors="raise"): # noqa: C901 - """Check if a WMO is valid - - Parameters - ---------- - wmo: int, list(int), array(int) - WMO must be a single or a list of 5/7 digit positive numbers - errors: {'raise', 'warn', 'ignore'} - Possibly raises a ValueError exception or UserWarning, otherwise fails silently. - - Returns - ------- - bool - True if wmo is indeed a list of integers - """ - - # Make sure we deal with a list - lst = to_list(lst) - - # Error message: - # msg = "WMO must be an integer or an iterable with elements that can be casted as integers" - msg = "WMO must be a single or a list of 5/7 digit positive numbers. Invalid: '{}'".format - - # Then try to cast list elements as integers, return True if ok - result = True - try: - for x in lst: - if not str(x).isdigit(): - result = False - - if (len(str(x)) != 5) and (len(str(x)) != 7): - result = False - - if int(x) <= 0: - result = False - - except Exception: - result = False - if errors == "raise": - raise ValueError(msg(x)) - elif errors == "warn": - warnings.warn(msg(x)) - - if not result: - if errors == "raise": - raise ValueError(msg(x)) - elif errors == "warn": - warnings.warn(msg(x)) - else: - return result - def check_cyc(lst, errors="raise"): """Validate a CYC option and returned it as a list of integers @@ -582,7 +508,7 @@ def check_gdac_path( return True else: - from ..stores import gdacfs # import here, otherwise raises circular import + from argopy.stores.implementations.gdac import gdacfs # import here, otherwise raises circular import try: fs = gdacfs(path) @@ -779,7 +705,7 @@ def erddap_ds_exists( erddap = OPTIONS["erddap"] # log.debug("from erddap_ds_exists: %s" % erddap) if isconnected(erddap, maxtry=maxtry): - from ..stores import httpstore # must import here to avoid circular import + from argopy.stores.implementations.http import httpstore # must import here to avoid circular import with httpstore(timeout=OPTIONS["api_timeout"]).open( "".join([erddap, "/info/index.json"]) diff --git a/argopy/utils/chunking.py b/argopy/utils/chunking.py index 8aa750c90..d56cdf191 100644 --- a/argopy/utils/chunking.py +++ b/argopy/utils/chunking.py @@ -1,8 +1,9 @@ import numpy as np import pandas as pd from functools import reduce -from ..errors import InvalidFetcherAccessPoint -from .checkers import is_box + +from argopy.errors import InvalidFetcherAccessPoint +from argopy.utils.checkers import is_box import collections diff --git a/argopy/utils/compute.py b/argopy/utils/compute.py index 402066b7b..977338b6b 100644 --- a/argopy/utils/compute.py +++ b/argopy/utils/compute.py @@ -7,7 +7,8 @@ import xarray as xr from packaging import version import logging -from ..errors import InvalidOption + +from argopy.errors import InvalidOption log = logging.getLogger("argopy.utils.compute") diff --git a/argopy/utils/format.py b/argopy/utils/format.py index 46ea02e99..58ad096aa 100644 --- a/argopy/utils/format.py +++ b/argopy/utils/format.py @@ -8,7 +8,8 @@ import pandas as pd import numpy as np import warnings -from .checkers import check_cyc, check_wmo +from argopy.utils.checkers import check_cyc +from argopy.utils.wmo import check_wmo log = logging.getLogger("argopy.utils.format") diff --git a/argopy/utils/geo.py b/argopy/utils/geo.py index ad376f5a1..96d63806d 100644 --- a/argopy/utils/geo.py +++ b/argopy/utils/geo.py @@ -1,7 +1,7 @@ import numpy as np import pandas as pd -from ..options import OPTIONS +from argopy.options import OPTIONS def wrap_longitude(grid_long): diff --git a/argopy/utils/lists.py b/argopy/utils/lists.py index 08174969d..be56ea152 100644 --- a/argopy/utils/lists.py +++ b/argopy/utils/lists.py @@ -1,14 +1,9 @@ import sys import warnings -import importlib -import os -import json -from ..options import OPTIONS from typing import List, Union -path2assets = importlib.util.find_spec( - "argopy.static.assets" -).submodule_search_locations[0] +from argopy.options import OPTIONS +from argopy.utils.locals import Asset def subsample_list(original_list, N): @@ -25,7 +20,7 @@ def list_available_data_src() -> dict: """List all available data sources""" sources = {} try: - from ..data_fetchers import erddap_data as Erddap_Fetchers + from argopy.data_fetchers import erddap_data as Erddap_Fetchers # Ensure we're loading the erddap data fetcher with the current options: Erddap_Fetchers.api_server_check = Erddap_Fetchers.api_server_check.replace( @@ -43,7 +38,7 @@ def list_available_data_src() -> dict: pass try: - from ..data_fetchers import argovis_data as ArgoVis_Fetchers + from argopy.data_fetchers import argovis_data as ArgoVis_Fetchers sources["argovis"] = ArgoVis_Fetchers except Exception: @@ -55,7 +50,7 @@ def list_available_data_src() -> dict: pass try: - from ..data_fetchers import gdac_data as GDAC_Fetchers + from argopy.data_fetchers import gdac_data as GDAC_Fetchers # Ensure we're loading the gdac data fetcher with the current options: GDAC_Fetchers.api_server_check = OPTIONS["gdac"] @@ -78,7 +73,7 @@ def list_available_index_src() -> dict: """List all available index sources""" sources = {} try: - from ..data_fetchers import erddap_index as Erddap_Fetchers + from argopy.data_fetchers import erddap_index as Erddap_Fetchers # Ensure we're loading the erddap data fetcher with the current options: Erddap_Fetchers.api_server_check = Erddap_Fetchers.api_server_check.replace( @@ -96,7 +91,7 @@ def list_available_index_src() -> dict: pass try: - from ..data_fetchers import gdac_index as GDAC_Fetchers + from argopy.data_fetchers import gdac_index as GDAC_Fetchers # Ensure we're loading the gdac data fetcher with the current options: GDAC_Fetchers.api_server_check = OPTIONS["gdac"] @@ -284,9 +279,7 @@ def list_bgc_s_variables() -> List[str]: :meth:`argopy.utils.list_radiometry_variables` :meth:`argopy.utils.list_radiometry_parameters`, """ - with open(os.path.join(path2assets, "variables_bgc_synthetic.json"), "r") as f: - vlist = json.load(f) - return vlist["data"]["variables"] + return Asset.load('variables_bgc_synthetic')["data"]["variables"] def list_bgc_s_parameters() -> List[str]: @@ -414,9 +407,7 @@ def list_gdac_servers() -> List[str]: :class:`argopy.gdacfs`, :meth:`argopy.utils.check_gdac_path`, :meth:`argopy.utils.shortcut2gdac` """ - with open(os.path.join(path2assets, "gdac_servers.json"), "r") as f: - vlist = json.load(f) - return vlist["data"]["paths"] + return Asset.load('gdac_servers')["data"]["paths"] def shortcut2gdac(short: str = None) -> Union[str, dict]: @@ -437,9 +428,7 @@ def shortcut2gdac(short: str = None) -> Union[str, dict]: :func:`argopy.utils.list_gdac_servers`, :class:`argopy.gdacfs`, :meth:`argopy.utils.check_gdac_path` """ - with open(os.path.join(path2assets, "gdac_servers.json"), "r") as f: - vlist = json.load(f) - shortcuts = vlist["data"]["shortcuts"] + shortcuts = Asset.load('gdac_servers')["data"]["shortcuts"] if short is not None: if short.lower().strip() in shortcuts.keys(): diff --git a/argopy/utils/locals.py b/argopy/utils/locals.py index 34bb407ad..4ec94cdce 100644 --- a/argopy/utils/locals.py +++ b/argopy/utils/locals.py @@ -4,20 +4,27 @@ import platform import locale import struct -import importlib from importlib.metadata import version import contextlib import copy import shutil import json -from ..options import OPTIONS +from typing import Any +import importlib +from pathlib import Path +import pandas as pd + +from argopy.options import OPTIONS +from argopy.stores.implementations.local import filestore PIP_INSTALLED = {} try: - reqs = subprocess.check_output([sys.executable, '-m', 'pip', 'list', '--format', 'json']) + reqs = subprocess.check_output( + [sys.executable, "-m", "pip", "list", "--format", "json"] + ) reqs = json.loads(reqs.decode()) - [PIP_INSTALLED.update({mod['name']: mod['version']}) for mod in reqs] + [PIP_INSTALLED.update({mod["name"]: mod["version"]}) for mod in reqs] except: # noqa E722 pass @@ -92,8 +99,8 @@ def netcdf_and_hdf5_versions(): def cli_version(cli_name): try: - a = subprocess.run([cli_name, '--version'], capture_output=True) - return a.stdout.decode().strip("\n").replace(cli_name, '').strip() + a = subprocess.run([cli_name, "--version"], capture_output=True) + return a.stdout.decode().strip("\n").replace(cli_name, "").strip() except: # noqa E722 if shutil.which(cli_name): return "- # installed" @@ -102,7 +109,7 @@ def cli_version(cli_name): def pip_version(pip_name): - version = '-' + version = "-" for name in [pip_name, pip_name.replace("_", "-"), pip_name.replace("-", "_")]: if name in PIP_INSTALLED: version = PIP_INSTALLED[name] @@ -110,7 +117,7 @@ def pip_version(pip_name): def get_version(module_name): - ver = '-' + ver = "-" try: ver = module_name.__version__ except AttributeError: @@ -124,8 +131,8 @@ def get_version(module_name): ver = cli_version(module_name) except: # noqa E722 pass - if sum([int(v == '0') for v in ver.split(".")]) == len(ver.split(".")): - ver = '-' + if sum([int(v == "0") for v in ver.split(".")]) == len(ver.split(".")): + ver = "-" return ver @@ -150,7 +157,6 @@ def show_versions(file=sys.stdout, conda=False): # noqa: C901 "core": sorted( [ ("argopy", get_version), - ("xarray", get_version), ("scipy", get_version), ("netCDF4", get_version), @@ -312,3 +318,76 @@ def show_options(file=sys.stdout): # noqa: C901 opts = dict(sorted(opts.items())) for k, v in opts.items(): print(f"{k}: {v}", file=file) + + +class Asset: + """Internal asset loader + + Assets are loaded using an instance of :class:`argopy.stores.filestore`. + + Notes + ----- + This is **single-instance** class, whereby a single instance will be created during a session, whatever the number of calls is made. This avoids to create too many, and unnecessary, instances of file stores. + + Examples + -------- + .. code-block:: python + :caption: Examples of asset files loading + + Asset.load('data_types') + Asset.load('data_types.json') + Asset.load('schema:argo.float.schema') + Asset.load('canyon-b:wgts_AT.txt', header=None, sep="\t") + """ + + _fs: Any = None + _instance: "Asset | None" = None + _initialized: bool = False + + def __new__(cls, *args: Any, **kwargs: Any) -> "Asset": + if cls._instance is None: + cls._instance = super().__new__(cls) + return cls._instance + + def __init__(self, *args, **kwargs) -> None: + if not self._initialized: + self._fs = filestore(cache=True, cachedir=OPTIONS["cachedir"]) + path2assets = importlib.util.find_spec( + "argopy.static.assets" + ).submodule_search_locations[0] + self._path = Path(path2assets) + self._initialized = True + + def _load(self, name: str, **kwargs) -> dict | pd.DataFrame: + suffix = Path(name).suffix + if suffix in [".csv", ".txt"]: + load = self._fs.read_csv + else: + load = self._fs.open_json + if suffix != ".json": # eg: '.schema' + name = f"{name}.json" + + name = name.strip() + name = name.split(":") + return load(self._path.joinpath(*name), **kwargs) + + @classmethod + def load(cls, name: str = None, **kwargs) -> Any: + """Load an asset file + + Parameters + ---------- + name: str + The *name* of the asset file to load. + If no suffix is indicated, it is assumed to be a JSON file with a `.json` extension. + If the asset is in sub-folders, use semicolons ':' as separator (eg: 'schema:argo.float.schema') + **kwargs: + All other arguments are passed down to the loading method. + + Notes + ----- + If the asset `name` has a `.txt` or `.csv` suffix, the :meth:`argopy.stores.filestore.read_csv` is used. + + For all other asset `name`, the :meth:`argopy.stores.filestore.load_json` is used by default. + """ + return cls()._load(name=name, **kwargs) diff --git a/argopy/utils/monitors.py b/argopy/utils/monitors.py index 9f92b35b8..4351b7a7d 100644 --- a/argopy/utils/monitors.py +++ b/argopy/utils/monitors.py @@ -4,8 +4,8 @@ import threading import logging -from .lists import list_available_data_src -from .checkers import isAPIconnected +from argopy.utils.lists import list_available_data_src +from argopy.utils.checkers import isAPIconnected try: importlib.import_module("matplotlib") # noqa: E402 diff --git a/argopy/utils/transform.py b/argopy/utils/transform.py index 66d4c9046..e260a4892 100644 --- a/argopy/utils/transform.py +++ b/argopy/utils/transform.py @@ -7,8 +7,8 @@ import logging from typing import List, Union -from ..errors import InvalidDatasetStructure -from .lists import list_core_parameters +from argopy.errors import InvalidDatasetStructure +from argopy.utils.lists import list_core_parameters log = logging.getLogger("argopy.utils.manip") diff --git a/argopy/utils/wmo.py b/argopy/utils/wmo.py new file mode 100644 index 000000000..d23eb0905 --- /dev/null +++ b/argopy/utils/wmo.py @@ -0,0 +1,147 @@ +import warnings +from argopy.utils.checkers import to_list +from argopy.utils.accessories import RegistryItem + + +def check_wmo(lst, errors="raise"): + """Validate a WMO option and returned it as a list of integers + + Parameters + ---------- + wmo: int + WMO must be an integer or an iterable with elements that can be casted as integers + errors: {'raise', 'warn', 'ignore'} + Possibly raises a ValueError exception or UserWarning, otherwise fails silently. + + Returns + ------- + list(int) + """ + is_wmo(lst, errors=errors) + + # Make sure we deal with a list + lst = to_list(lst) + + # Then cast list elements as integers + return [abs(int(x)) for x in lst] + + +def is_wmo(lst, errors="raise"): # noqa: C901 + """Check if a WMO is valid + + Parameters + ---------- + wmo: int, list(int), array(int) + WMO must be a single or a list of 5/7 digit positive numbers + errors: {'raise', 'warn', 'ignore'} + Possibly raises a ValueError exception or UserWarning, otherwise fails silently. + + Returns + ------- + bool + True if wmo is indeed a list of integers + """ + + # Make sure we deal with a list + lst = to_list(lst) + + # Error message: + # msg = "WMO must be an integer or an iterable with elements that can be casted as integers" + msg = "WMO must be a single or a list of 5/7 digit positive numbers. Invalid: '{}'".format + + # Then try to cast list elements as integers, return True if ok + result = True + try: + for x in lst: + if not str(x).isdigit(): + result = False + + if (len(str(x)) != 5) and (len(str(x)) != 7): + result = False + + if int(x) <= 0: + result = False + + except Exception: + result = False + if errors == "raise": + raise ValueError(msg(x)) + elif errors == "warn": + warnings.warn(msg(x)) + + if not result: + if errors == "raise": + raise ValueError(msg(x)) + elif errors == "warn": + warnings.warn(msg(x)) + else: + return result + + +class float_wmo(RegistryItem): + """Argo float WMO number object""" + + def __init__(self, WMO_number, errors="raise"): + """Create an Argo float WMO number object + + Parameters + ---------- + WMO_number: object + Anything that could be cast as an integer + errors: {'raise', 'warn', 'ignore'} + Possibly raises a ValueError exception or UserWarning, otherwise fails silently if WMO_number is not valid + + Returns + ------- + obj + """ + self.errors = errors + if isinstance(WMO_number, float_wmo): + item = WMO_number.value + else: + item = check_wmo(WMO_number, errors=self.errors)[ + 0 + ] # This will automatically validate item + self.item = item + + @property + def isvalid(self): + """Check if WMO number is valid""" + return is_wmo(self.item, errors=self.errors) + # return True # Because it was checked at instantiation + + @property + def value(self): + """Return WMO number as in integer""" + return int(self.item) + + def __str__(self): + # return "%s" % check_wmo(self.item)[0] + return "%s" % self.item + + def __repr__(self): + return f"WMO({self.item})" + + def __check_other__(self, other): + return check_wmo(other)[0] if type(other) is not float_wmo else other.item + + def __eq__(self, other): + return self.item.__eq__(self.__check_other__(other)) + + def __ne__(self, other): + return self.item.__ne__(self.__check_other__(other)) + + def __gt__(self, other): + return self.item.__gt__(self.__check_other__(other)) + + def __lt__(self, other): + return self.item.__lt__(self.__check_other__(other)) + + def __ge__(self, other): + return self.item.__ge__(self.__check_other__(other)) + + def __le__(self, other): + return self.item.__le__(self.__check_other__(other)) + + def __hash__(self): + return hash(self.item) diff --git a/argopy/xarray.py b/argopy/xarray.py index 9860dc0c1..c9d61b886 100644 --- a/argopy/xarray.py +++ b/argopy/xarray.py @@ -24,20 +24,19 @@ with_dask = False Delayed = lambda x: x # noqa: E731 - -from .utils import is_list_of_strings -from .utils import ( +from argopy.errors import InvalidDatasetStructure, OptionValueError, NoData, NoDataLeft +from argopy.utils.checkers import is_list_of_strings +from argopy.utils.casting import ( cast_Argo_variable_type, DATA_TYPES, to_list, ) -from .utils import ( +from argopy.utils.compute import ( linear_interpolation_remap, groupby_remap, ) -from .utils import list_core_parameters -from .utils import toYearFraction -from .errors import InvalidDatasetStructure, OptionValueError, NoData, NoDataLeft +from argopy.utils.lists import list_core_parameters +from argopy.utils.geo import toYearFraction log = logging.getLogger("argopy.xarray") diff --git a/docs/api-hidden.rst b/docs/api-hidden.rst index 61ac7b44c..65eec396e 100644 --- a/docs/api-hidden.rst +++ b/docs/api-hidden.rst @@ -80,6 +80,7 @@ argopy.utils.show_versions argopy.utils.show_options + argopy.utils.Asset argopy.utils.clear_cache argopy.utils.lscache