Add integration tests to state_daily

Author: chinandrew

integrations/acquisition/covid_hosp/state_daily/__init__.py

@@ -0,0 +1,4 @@
+import sys
+import os
+
+sys.path.append(os.getcwd())

integrations/acquisition/covid_hosp/state_daily/test_scenarios.py

@@ -0,0 +1,87 @@
+"""Integration tests for acquisition of COVID hospitalization."""
+
+# standard library
+import unittest
+from unittest.mock import MagicMock
+
+# first party
+from delphi.epidata.acquisition.covid_hosp.common.database import Database
+from delphi.epidata.acquisition.covid_hosp.common.test_utils import TestUtils
+from delphi.epidata.client.delphi_epidata import Epidata
+from delphi.epidata.acquisition.covid_hosp.state_daily.update import Update
+import delphi.operations.secrets as secrets
+
+# py3tester coverage target (equivalent to `import *`)
+__test_target__ = \
+    'delphi.epidata.acquisition.covid_hosp.state_daily.update'
+
+
+class AcquisitionTests(unittest.TestCase):
+
+  def setUp(self):
+    """Perform per-test setup."""
+
+    # configure test data
+    self.test_utils = TestUtils(__file__)
+
+    # use the local instance of the Epidata API
+    Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
+
+    # use the local instance of the epidata database
+    secrets.db.host = 'delphi_database_epidata'
+    secrets.db.epi = ('user', 'pass')
+
+    # clear relevant tables
+    with Database.connect() as db:
+      with db.new_cursor() as cur:
+        cur.execute('truncate table covid_hosp_state_timeseries')
+        cur.execute('truncate table covid_hosp_meta')
+
+  def test_acquire_dataset(self):
+    """Acquire a new dataset."""
+
+    # only mock out network calls to external hosts
+    mock_network = MagicMock()
+    mock_network.fetch_metadata.return_value = \
+        self.test_utils.load_sample_metadata()
+    mock_network.fetch_dataset.return_value = \
+        self.test_utils.load_sample_dataset()
+
+    # make sure the data does not yet exist
+    with self.subTest(name='no data yet'):
+      response = Epidata.covid_hosp('MA', Epidata.range(20200101, 20210101))
+      self.assertEqual(response['result'], -2)
+
+    # acquire sample data into local database
+    with self.subTest(name='first acquisition'):
+      acquired = Update.run(network=mock_network)
+      self.assertTrue(acquired)
+
+    # make sure the data now exists
+    with self.subTest(name='initial data checks'):
+      response = Epidata.covid_hosp('WY', Epidata.range(20200101, 20210101))
+      self.assertEqual(response['result'], 1)
+      self.assertEqual(len(response['epidata']), 1)
+      row = response['epidata'][0]
+      self.assertEqual(row['state'], 'WY')
+      self.assertEqual(row['date'], 20201209)
+      self.assertEqual(row['issue'], 20201213)
+      self.assertEqual(row['critical_staffing_shortage_today_yes'], 8)
+      actual = row['inpatient_bed_covid_utilization']
+      expected = 0.11729857819905214
+      self.assertAlmostEqual(actual, expected)
+      self.assertIsNone(row['critical_staffing_shortage_today_no'])
+
+      # expect 61 fields per row (63 database columns, except `id` and `record_type`)
+      self.assertEqual(len(row), 61)
+
+    # re-acquisition of the same dataset should be a no-op
+    with self.subTest(name='second acquisition'):
+      acquired = Update.run(network=mock_network)
+      self.assertFalse(acquired)
+
+    # make sure the data still exists
+    with self.subTest(name='final data checks'):
+      response = Epidata.covid_hosp('WY', Epidata.range(20200101, 20210101))
+      self.assertEqual(response['result'], 1)
+      self.assertEqual(len(response['epidata']), 1)

src/acquisition/covid_hosp/state_daily/database.py

@@ -18,7 +18,7 @@ class Database(BaseDatabase):
   # to update a column name, do it in both places.
   ORDERED_CSV_COLUMNS = [
     ('state', 'state', str),
-    ('reporting_cutoff_start', 'reporting_cutoff_start', Utils.int_from_date),
+    ('reporting_cutoff_start', 'date', Utils.int_from_date),
     ('critical_staffing_shortage_today_yes', 'critical_staffing_shortage_today_yes', int),
     ('critical_staffing_shortage_today_no', 'critical_staffing_shortage_today_no', int),
     ('critical_staffing_shortage_today_not_reported',

testdata/acquisition/covid_hosp/state_daily/dataset.csv

@@ -51,4 +51,4 @@ MI,30,129,4,32,127,4,41,159,23598,163,18003,163,3812,159,376,163,162,159,9,159,9
 MN,21,116,2,26,111,2,63,138,10358,139,7558,139,1516,138,182,139,70,138,3,138,2,138,806,139,346,138,355,139,1490,138,1358,139,26,138,21,138,1019,139,0.7296775439273991,139,7558,10358,0.20824175824175825,138,1516,7280,0.15163032606521304,138,1516,9998,0.36536430834213307,138,346,947,0.7909715407262021,139,806,1019,2020-12-09
 MO,47,78,16,62,63,16,22,137,17433,141,13521,141,2611,137,315,141,239,137,5,137,18,137,1615,141,645,137,604,141,2546,137,2307,141,65,137,26,137,1931,141,0.7755980037859233,141,13521,17433,0.19644872470092545,137,2611,13291,0.1523959610109146,137,2611,17133,0.34565916398713825,137,645,1866,0.8363542206110823,141,1615,1931,2020-12-09
 MS,21,85,2,30,76,2,12,106,8799,108,5637,108,1254,106,142,108,30,106,3,106,5,106,718,108,338,106,263,108,1234,106,1066,108,20,106,5,106,881,108,0.6406409819297647,108,5637,8799,0.22501345774268797,106,1254,5573,0.14392287386663605,106,1254,8713,0.3953216374269006,106,338,855,0.8149829738933031,108,718,881,2020-12-09
-WY,8,21,2,7,22,2,5,29,1729,31,856,31,198,29,26,31,15,29,0,29,0,29,58,31,32,29,32,31,196,29,189,31,2,29,2,29,137,31,0.4950838635049161,31,856,1729,0.23627684964200477,29,198,838,0.11729857819905214,29,198,1688,0.25196850393700787,29,32,127,0.4233576642335766,31,58,137,2020-12-09
+WY,8,,2,7,22,2,5,29,1729,31,856,31,198,29,26,31,15,29,0,29,0,29,58,31,32,29,32,31,196,29,189,31,2,29,2,29,137,31,0.4950838635049161,31,856,1729,0.23627684964200477,29,198,838,0.11729857819905214,29,198,1688,0.25196850393700787,29,32,127,0.4233576642335766,31,58,137,2020-12-09

tests/acquisition/covid_hosp/state_daily/test_database.py

@@ -40,7 +40,7 @@ def test_insert_dataset(self):
 
     last_query_values = mock_cursor.execute.call_args[0][-1]
     expected_query_values = (
-        0, sentinel.issue, 'WY',  20201209, 8, 21, 2, 7, 22, 2, 5, 29, 1729,
+        0, sentinel.issue, 'WY',  20201209, 8, None, 2, 7, 22, 2, 5, 29, 1729,
         31, 856, 31, 198, 29, 26, 31, 15, 29, 0, 29, 0, 29, 58, 31, 32, 29,
         32, 31, 196, 29, 189, 31, 2, 29, 2, 29, 137, 31, 0.4950838635049161, 31,
         856, 1729, 0.23627684964200477, 29, 198, 838, 0.11729857819905214,