Merge pull request #395 from chinandrew/update-lag-issue

Have nowcasting acquisition use issue date from file

Author: krivard

integrations/acquisition/covidcast_nowcast/test_csv_uploading.py

@@ -68,9 +68,9 @@ def test_uploading(self):
     # print full diff if something unexpected comes out
     self.maxDiff=None
 
-    receiving_dir = '/common/covidcast_nowcast/receiving/src/'
-    success_dir = '/common/covidcast_nowcast/archive/successful/src/'
-    failed_dir = '/common/covidcast_nowcast/archive/failed/src/'
+    receiving_dir = '/common/covidcast_nowcast/receiving/issue_20200421/src/'
+    success_dir = '/common/covidcast_nowcast/archive/successful/issue_20200421/src/'
+    failed_dir = '/common/covidcast_nowcast/archive/failed/issue_20200421/src/'
     os.makedirs(receiving_dir, exist_ok=True)
 
     # valid
@@ -125,7 +125,7 @@ def test_duplicate_row(self):
     # print full diff if something unexpected comes out
     self.maxDiff=None
 
-    receiving_dir = '/common/covidcast_nowcast/receiving/src/'
+    receiving_dir = '/common/covidcast_nowcast/receiving/issue_20200425/src/'
     os.makedirs(receiving_dir, exist_ok=True)
 
     with open(receiving_dir + '20200419_state_sig.csv', 'w') as f:
@@ -146,8 +146,8 @@ def test_duplicate_row(self):
         'time_value': 20200419,
         'geo_value': 'ca',
         'value': 2,
-        'issue': 20200421,
-        'lag': 2,
+        'issue': 20200425,
+        'lag': 6,
         'signal': 'sig',
       }],
       'message': 'success',

src/acquisition/covidcast_nowcast/load_sensors.py

@@ -1,4 +1,5 @@
 from shutil import move
+from datetime import datetime
 import os
 import time
 
@@ -36,7 +37,7 @@ def main(csv_path: str = SENSOR_CSV_PATH) -> None:
     """
     user, pw = secrets.db.epi
     engine = sqlalchemy.create_engine(f"mysql+pymysql://{user}:{pw}@{secrets.db.host}/{DB_NAME}")
-    for filepath, attribute in CsvImporter.find_csv_files(csv_path):
+    for filepath, attribute in CsvImporter.find_issue_specific_csv_files(csv_path):
         if attribute is None:
             _move_after_processing(filepath, success=False)
             continue

tests/acquisition/covidcast_nowcast/test_load_sensors.py

@@ -24,9 +24,9 @@ def test_load_and_prepare_file(self):
                        "test_signal",
                        "test_time_type",
                        "test_geo_type",
-                       "test_time_value",
-                       "test_issue_value",
-                       "test_lag_value")
+                       20201231,
+                       20210102,
+                       3)
 
     test_df = load_and_prepare_file(StringIO("sensor_name,geo_value,value\ntestname,01001,1.5"), test_attributes)
     pd.testing.assert_frame_equal(test_df,
@@ -37,8 +37,8 @@ def test_load_and_prepare_file(self):
                                                 "signal": ["test_signal"],
                                                 "time_type": ["test_time_type"],
                                                 "geo_type": ["test_geo_type"],
-                                                "time_value": ["test_time_value"],
-                                                "issue": ["test_issue_value"],
-                                                "lag": ["test_lag_value"],
+                                                "time_value": [20201231],
+                                                "issue": [20210102],
+                                                "lag": [3],
                                                 "value_updated_timestamp": [12345]})
                                   )