Merge pull request #862 from cmu-delphi/ds/archive-differ-fix

Allow NAs in missing columns

Author: krivard

src/acquisition/covidcast/csv_importer.py

@@ -43,9 +43,9 @@ class CsvImporter:
     "val": float,
     "se": float,
     "sample_size": float,
-    "missing_val": int,
-    "missing_se": int,
-    "missing_sample_size": int
+    "missing_val": "Int64",
+    "missing_se": "Int64",
+    "missing_sample_size": "Int64"
   }
 
   # NOTE: this should be a Python 3.7+ `dataclass`, but the server is on 3.4

tests/acquisition/covidcast/test_csv_importer.py

@@ -278,13 +278,13 @@ def test_load_csv_with_valid_header(self):
 
     # now with missing values!
     data = {
-      'geo_id': ['ca', 'tx', 'fl', 'ak'],
-      'val': [np.nan, '1.2', '1.3', '1.4'],
-      'se': ['2.1', "na", '2.3', '2.4'],
-      'sample_size': ['301', '302', None, '304'],
-      'missing_value': [Nans.NOT_APPLICABLE] + [Nans.NOT_MISSING] * 3,
-      'missing_stderr': [Nans.NOT_MISSING, Nans.REGION_EXCEPTION, Nans.NOT_MISSING, Nans.NOT_MISSING],
-      'missing_sample_size': [Nans.NOT_MISSING] * 2 + [Nans.REGION_EXCEPTION] * 2
+      'geo_id': ['ca', 'tx', 'fl', 'ak', 'wa'],
+      'val': [np.nan, '1.2', '1.3', '1.4', '1.5'],
+      'se': ['2.1', "na", '2.3', '2.4', '2.5'],
+      'sample_size': ['301', '302', None, '304', None],
+      'missing_value': [Nans.NOT_APPLICABLE] + [Nans.NOT_MISSING] * 3 + [None],
+      'missing_stderr': [Nans.NOT_MISSING, Nans.REGION_EXCEPTION, Nans.NOT_MISSING, Nans.NOT_MISSING] + [None],
+      'missing_sample_size': [Nans.NOT_MISSING] * 2 + [Nans.REGION_EXCEPTION] * 2 + [None]
     }
     mock_pandas = MagicMock()
     mock_pandas.read_csv.return_value = pandas.DataFrame(data=data)
@@ -295,7 +295,7 @@ def test_load_csv_with_valid_header(self):
 
     self.assertTrue(mock_pandas.read_csv.called)
     self.assertTrue(mock_pandas.read_csv.call_args[0][0], filepath)
-    self.assertEqual(len(rows), 4)
+    self.assertEqual(len(rows), 5)
 
     self.assertEqual(rows[0].geo_value, 'ca')
     self.assertIsNone(rows[0].value)
@@ -328,3 +328,11 @@ def test_load_csv_with_valid_header(self):
     self.assertEqual(rows[3].missing_value, Nans.NOT_MISSING)
     self.assertEqual(rows[3].missing_stderr, Nans.NOT_MISSING)
     self.assertEqual(rows[3].missing_sample_size, Nans.NOT_MISSING)
+
+    self.assertEqual(rows[4].geo_value, 'wa')
+    self.assertEqual(rows[4].value, 1.5)
+    self.assertEqual(rows[4].stderr, 2.5)
+    self.assertEqual(rows[4].sample_size, None)
+    self.assertEqual(rows[4].missing_value, Nans.NOT_MISSING)
+    self.assertEqual(rows[4].missing_stderr, Nans.NOT_MISSING)
+    self.assertEqual(rows[4].missing_sample_size, Nans.OTHER)