Merge pull request #146 from ocefpaf/summary

Add a summary method

Author: ocefpaf

gliderpy/__init__.py

@@ -1,5 +1,8 @@
 """Easier access to glider data."""
 
+import pandas as pd
+from pandas_flavor import register_dataframe_method
+
 try:
     from ._version import __version__
 except ImportError:
@@ -15,3 +18,36 @@
     "plot_transect",
     "plot_ts",
 ]
+
+
+def _num_profiles(df: pd.DataFrame) -> int:
+    """Compute the number of unique glider pofiles."""
+    return len(df[["latitude", "longitude"]].value_counts())
+
+
+def _days(df: pd.DataFrame) -> pd.Timedelta:
+    """Compute the glider days."""
+    return df.index.dropna()[-1].ceil("D") - df.index.dropna()[0].floor("D")
+
+
+def _deployment_lat(df: pd.DataFrame) -> dict:
+    """Return the glider deployment latitude."""
+    return df["latitude"].to_list()[0]
+
+
+def _deployment_lon(df: pd.DataFrame) -> dict:
+    """Return the glider deployment longitude."""
+    return df["longitude"].to_list()[0]
+
+
+@register_dataframe_method
+def summary(df: pd.DataFrame) -> pd.DataFrame:
+    """Return the summary for a set of gliders."""
+    summ = {
+        "num_profiles": _num_profiles(df),
+        "days": _days(df),
+        "deployment_lat": _deployment_lat(df),
+        "deployment_lon": _deployment_lon(df),
+    }
+
+    return pd.Series(summ)

gliderpy/fetchers.py

@@ -7,6 +7,7 @@
 
 import httpx
 import pandas as pd
+import stamina
 from erddapy import ERDDAP
 from erddapy.core.url import urlopen
 
@@ -28,14 +29,29 @@
 _server = "https://gliders.ioos.us/erddap"
 
 
+@stamina.retry(on=httpx.HTTPError, attempts=3)
+def _call_erddapy(glider_grab: "GliderDataFetcher") -> pd.DataFrame:
+    """Temporary workaround until we move optional stamina to erddapy."""
+    return glider_grab.fetcher.to_pandas()
+
+
 @functools.lru_cache(maxsize=128)
-def _to_pandas_multiple(glider_grab: "GliderDataFetcher") -> pd.DataFrame:
+def _to_pandas(
+    glider_grab: "GliderDataFetcher",
+    *,
+    query: OptionalBool = True,
+) -> pd.DataFrame:
     """Thin wrapper to cache results when multiple datasets are requested."""
     df_all = {}
     glider_grab_copy = copy(glider_grab)
-    for dataset_id in glider_grab_copy.datasets["Dataset ID"]:
+    if query:
+        dataset_ids = glider_grab_copy.datasets["Dataset ID"]
+    else:
+        dataset_ids = glider_grab_copy.dataset_ids
+
+    for dataset_id in dataset_ids:
         glider_grab_copy.fetcher.dataset_id = dataset_id
-        glider_df = glider_grab_copy.fetcher.to_pandas()
+        glider_df = _call_erddapy(glider_grab_copy)
         dataset_url = glider_grab_copy.fetcher.get_download_url().split("?")[0]
         glider_df = standardise_df(glider_df, dataset_url)
         df_all.update({dataset_id: glider_df})
@@ -81,7 +97,7 @@ def __init__(
             protocol="tabledap",
         )
         self.fetcher.variables = server_vars[server]
-        self.fetcher.dataset_id: OptionalStr = None
+        self.dataset_ids: OptionalList = None
         self.datasets: OptionalDF = None
 
     def to_pandas(self: "GliderDataFetcher") -> pd.DataFrame:
@@ -90,21 +106,20 @@ def to_pandas(self: "GliderDataFetcher") -> pd.DataFrame:
         :return: pandas a dataframe with datetime UTC as index,
                  multiple dataset_ids dataframes are stored in a dictionary
         """
-        if self.fetcher.dataset_id:
-            glider_df = self.fetcher.to_pandas()
-        elif not self.fetcher.dataset_id and self.datasets is not None:
-            glider_df = _to_pandas_multiple(self)
-            # We need to reset to avoid fetching a single dataset_id when
-            # making multiple requests.
-            self.fetcher.dataset_id = None
-            return glider_df
+        if self.dataset_ids is not None:
+            query = False  # Passing known dataset_ids
+        elif self.dataset_ids is None and self.datasets is not None:
+            query = True  # Passing an ERDDAP query
         else:
             msg = "Must provide a dataset_id or query terms to download data."
             raise ValueError(msg)
 
-        # Standardize variable names for the single dataset_id.
-        dataset_url = self.fetcher.get_download_url().split("?")[0]
-        return standardise_df(glider_df, dataset_url)
+        glider_df = _to_pandas(self, query=query)
+        # We need to reset to avoid fetching a single dataset_id when
+        # making multiple requests.
+        self.fetcher.dataset_id = None
+
+        return glider_df
 
     def query(  # noqa: PLR0913
         self: "GliderDataFetcher",
@@ -196,8 +211,10 @@ class DatasetList:
 
     def __init__(
         self: "DatasetList",
+        *,
         server: OptionalStr = _server,
         search_for: OptionalStr = None,
+        delayed: OptionalBool = False,
     ) -> None:
         """Instantiate main class attributes.
 
@@ -216,6 +233,7 @@ def __init__(
             protocol="tabledap",
         )
         self.search_for = search_for
+        self.delayed = delayed
 
     def get_ids(self: "DatasetList") -> list:
         """Return the allDatasets list for the glider server."""
@@ -229,5 +247,10 @@ def get_ids(self: "DatasetList") -> list:
             self.e.dataset_id = "allDatasets"
             dataset_ids = self.e.to_pandas()["datasetID"].to_list()
             dataset_ids.remove("allDatasets")
-        self.dataset_ids = dataset_ids
+        if not self.delayed:
+            self.dataset_ids = [
+                dataset_id
+                for dataset_id in dataset_ids
+                if not dataset_id.endswith("-delayed")
+            ]
         return self.dataset_ids

notebooks/00-quick_intro.ipynb

@@ -71,8 +71,9 @@
     "\n",
     "glider_grab = GliderDataFetcher()\n",
     "\n",
-    "glider_grab.fetcher.dataset_id = \"whoi_406-20160902T1700\"\n",
-    "df = glider_grab.to_pandas()\n",
+    "glider_grab.dataset_ids = [\"whoi_406-20160902T1700\"]\n",
+    "dfs = glider_grab.to_pandas()\n",
+    "df = dfs[\"whoi_406-20160902T1700\"]\n",
     "df.head()"
    ]
   },
@@ -174,7 +175,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.13"
+   "version": "3.13.3"
   }
  },
  "nbformat": 4,

notebooks/01-plotting_intro.ipynb

@@ -20,8 +20,8 @@
     "\n",
     "glider_grab = GliderDataFetcher()\n",
     "\n",
-    "glider_grab.fetcher.dataset_id = \"whoi_406-20160902T1700\"\n",
-    "df = glider_grab.to_pandas()"
+    "glider_grab.dataset_ids = [\"whoi_406-20160902T1700\"]\n",
+    "dfs = glider_grab.to_pandas()"
    ]
   },
   {
@@ -39,6 +39,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "df = dfs[\"whoi_406-20160902T1700\"]\n",
+    "\n",
     "fig, ax = df.plot_track()"
    ]
   },
@@ -141,7 +143,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.4"
+   "version": "3.13.3"
   }
  },
  "nbformat": 4,

pyproject.toml

@@ -28,7 +28,7 @@ dynamic = [
   "version",
 ]
 
-dependencies = [ "erddapy", "httpx", "pandas", "pandas-flavor", "xarray" ]
+dependencies = [ "erddapy", "httpx", "pandas", "pandas-flavor", "stamina", "xarray" ]
 
 optional-dependencies.docs = [ "jupyter", "nbconvert", "nbsphinx", "palettable", "sphinx" ]
 optional-dependencies.plotting = [ "cartopy", "gsw", "matplotlib" ]