Merge pull request #595 from cmu-delphi/dev

Release minor workflow and docs fixes

Author: krivard

.github/workflows/create-delphi-epidata-release.yml

@@ -20,7 +20,7 @@ jobs:
       - name: Reset dev branch
         run: |
           git fetch origin dev:dev
-          git reset --hard dev
+          git reset --hard origin/dev
       - name: Set up Python 3.8
         uses: actions/setup-python@v2
         with:

.github/workflows/deploy-docs.yaml

@@ -25,7 +25,7 @@ General topics not specific to any particular data source are discussed in the
 
 This data source provides various measures of COVID-19 burden on patients and healthcare in the US.
 - Data source: US Department of Health & Human Services (HHS) [COVID-19 Reported Patient Impact and
-Hospital Capacity by State Timeseries](https://healthdata.gov/Hospital/COVID-19-Reported-Patient-Impact-and-Hospital-Capa/g62h-syeh) 
+Hospital Capacity by State Timeseries](https://healthdata.gov/Hospital/COVID-19-Reported-Patient-Impact-and-Hospital-Capa/g62h-syeh)
   and [COVID-19 Reported Patient Impact and Hospital Capacity by State](https://healthdata.gov/dataset/COVID-19-Reported-Patient-Impact-and-Hospital-Capa/6xf2-c3ie)
 - Temporal Resolution: Daily, starting 2020-01-01
 - Spatial Resolution: US States plus DC, PR, and VI
@@ -36,7 +36,7 @@ weekly.
 
 # The API
 
-The base URL is: https://delphi.cmu.edu/epidata/api.php
+The base URL is: https://delphi.cmu.edu/epidata/covid_hosp_state_timeseries/
 
 See [this documentation](README.md) for details on specifying locations and dates.
 
@@ -72,7 +72,7 @@ If `issues` is not specified, then the most recent issue is used by default.
 # Example URLs
 
 ### MA on 2020-05-10 (per most recent issue)
-https://delphi.cmu.edu/epidata/covid_hosp/?states=MA&dates=20200510
+https://delphi.cmu.edu/epidata/covid_hosp_state_timeseries/?states=MA&dates=20200510
 
 ```json
 {

docs/api/covid_hosp.md

@@ -1,19 +1,21 @@
 """Integration tests for acquisition of COVID hospitalization."""
 
 # standard library
+from datetime import date
 import unittest
-from unittest.mock import MagicMock
 from unittest.mock import patch
 
 # third party
 from freezegun import freeze_time
+import pandas as pd
 
 # first party
-from delphi.epidata.acquisition.covid_hosp.common.database import Database
+from delphi.epidata.acquisition.covid_hosp.state_daily.database import Database
 from delphi.epidata.acquisition.covid_hosp.common.test_utils import UnitTestUtils
 from delphi.epidata.client.delphi_epidata import Epidata
 from delphi.epidata.acquisition.covid_hosp.state_daily.update import Update
 from delphi.epidata.acquisition.covid_hosp.state_daily.network import Network
+from delphi.epidata.acquisition.covid_hosp.common.utils import Utils
 import delphi.operations.secrets as secrets
 
 # py3tester coverage target (equivalent to `import *`)
@@ -97,3 +99,31 @@ def test_acquire_dataset(self):
       response = Epidata.covid_hosp('WY', Epidata.range(20200101, 20210101))
       self.assertEqual(response['result'], 1)
       self.assertEqual(len(response['epidata']), 1)
+
+
+  @freeze_time("2021-03-16")
+  def test_acquire_specific_issue(self):
+    """Acquire a new dataset."""
+
+    # make sure the data does not yet exist
+    with self.subTest(name='no data yet'):
+      response = Epidata.covid_hosp('MA', Epidata.range(20200101, 20210101))
+      self.assertEqual(response['result'], -2)
+
+    # acquire sample data into local database
+    # mock out network calls to external hosts
+    with Database.connect() as db:
+      pre_max_issue = db.get_max_issue()
+    self.assertEqual(pre_max_issue, pd.Timestamp('1900-01-01 00:00:00'))
+    with self.subTest(name='first acquisition'), \
+         patch.object(Network, 'fetch_metadata', return_value=self.test_utils.load_sample_metadata()) as mock_fetch_meta, \
+         patch.object(Network, 'fetch_dataset', side_effect=[self.test_utils.load_sample_dataset("dataset0.csv")]
+                      ) as mock_fetch:
+      acquired = Utils.update_dataset(Database,
+                                      Network,
+                                      date(2021, 3, 12),
+                                      date(2021, 3, 14))
+      with Database.connect() as db:
+        post_max_issue = db.get_max_issue()
+      self.assertEqual(post_max_issue, pd.Timestamp('2021-03-13 00:00:00'))
+      self.assertTrue(acquired)

integrations/acquisition/covid_hosp/state_daily/test_scenarios.py

@@ -82,7 +82,8 @@ def issues_to_fetch(metadata, newer_than, older_than):
       Upper bound (exclusive) of days to get issues for
     Returns
     -------
-      Dictionary of {issue day: list of download urls} for issues after newer_than and before older_than
+      Dictionary of {issue day: list of (download urls, index)}
+      for issues after newer_than and before older_than
     """
     daily_issues = {}
     for index in sorted(set(metadata.index)):
@@ -127,7 +128,7 @@ def merge_by_key_cols(dfs, key_cols):
     return result.reset_index(level=key_cols)
 
   @staticmethod
-  def update_dataset(database, network):
+  def update_dataset(database, network, newer_than=None, older_than=None):
     """Acquire the most recent dataset, unless it was previously acquired.
 
     Parameters
@@ -136,18 +137,22 @@ def update_dataset(database, network):
       A `Database` subclass for a particular dataset.
     network : delphi.epidata.acquisition.covid_hosp.common.network.Network
       A `Network` subclass for a particular dataset.
+    newer_than : date
+      Lower bound (exclusive) of days to get issues for.
+    older_than : date
+      Upper bound (exclusive) of days to get issues for
 
     Returns
     -------
     bool
       Whether a new dataset was acquired.
     """
-    today = datetime.datetime.today().date()
     metadata = network.fetch_metadata()
-
     with database.connect() as db:
       max_issue = db.get_max_issue()
-      daily_issues = Utils.issues_to_fetch(metadata, max_issue, today)
+      older_than = datetime.datetime.today().date() if newer_than is None else older_than
+      newer_than = max_issue if newer_than is None else newer_than
+      daily_issues = Utils.issues_to_fetch(metadata, newer_than, older_than)
       if not daily_issues:
         print("no new issues, nothing to do")
         return False

src/acquisition/covid_hosp/common/utils.py

@@ -1,4 +1,4 @@
 from .delphi_epidata import Epidata
 
 name = 'delphi_epidata'
-__version__ = '0.0.12'
+__version__ = '0.1.0'