Change CHNG and claims_hosp to use timestamp as the date_col

Author: chinandrew

changehc/delphi_changehc/config.py

@@ -29,7 +29,7 @@ class Config:
     FLU_LIKE_COL = "Flu-like"
     COVID_LIKE_COL = "Covid-like"
     COUNT_COLS = [COVID_COL,DENOM_COL,FLU_COL,MIXED_COL,FLU_LIKE_COL,COVID_LIKE_COL]
-    DATE_COL = "date"
+    DATE_COL = "timestamp"
     GEO_COL = "fips"
     ID_COLS = [DATE_COL] + [GEO_COL]
     FILT_COLS = ID_COLS + COUNT_COLS

changehc/tests/test_load_data.py

@@ -45,7 +45,7 @@ def test_base_unit(self):
 
     def test_denom_columns(self):
         assert "fips" in self.denom_data.index.names
-        assert "date" in self.denom_data.index.names
+        assert "timestamp" in self.denom_data.index.names
 
         expected_denom_columns = ["Denominator"]
         for col in expected_denom_columns:
@@ -54,7 +54,7 @@ def test_denom_columns(self):
 
     def test_claims_columns(self):
         assert "fips" in self.covid_data.index.names
-        assert "date" in self.covid_data.index.names
+        assert "timestamp" in self.covid_data.index.names
 
         expected_covid_columns = ["COVID"]
         for col in expected_covid_columns:
@@ -63,7 +63,7 @@ def test_claims_columns(self):
 
     def test_combined_columns(self):
         assert "fips" in self.combined_data.index.names
-        assert "date" in self.combined_data.index.names
+        assert "timestamp" in self.combined_data.index.names
 
         expected_combined_columns = ["num", "den"]
         for col in expected_combined_columns:
@@ -75,16 +75,16 @@ def test_edge_values(self):
         for data in [self.denom_data,
                      self.covid_data,
                      self.combined_data]:
-            assert data.index.get_level_values('date').max() >= Config.FIRST_DATA_DATE
-            assert data.index.get_level_values('date').min() < DROP_DATE
+            assert data.index.get_level_values("timestamp").max() >= Config.FIRST_DATA_DATE
+            assert data.index.get_level_values("timestamp").min() < DROP_DATE
 
     def test_fips_values(self):
         for data in [self.denom_data,
                      self.covid_data,
                      self.combined_data]:
             assert (
                     len(data.index.get_level_values(
-                        'fips').unique()) <= len(self.gmpr.get_geo_values("fips"))
+                        "fips").unique()) <= len(self.gmpr.get_geo_values("fips"))
             )
 
     def test_combined_fips_values(self):

changehc/tests/test_update_sensor.py

@@ -40,7 +40,7 @@ class TestCHCSensorUpdator:
         "num": [0, 100, 200, 300, 400, 500, 600, 100, 200, 300, 400, 500, 600],
         "fips": ['01001'] * 7 + ['04007'] * 6,
         "den": [1000] * 7 + [2000] * 6,
-        "date": [pd.Timestamp(f'03-{i}-2020') for i in range(1, 14)]}).set_index(["fips","date"])
+        "timestamp": [pd.Timestamp(f'03-{i}-2020') for i in range(1, 14)]}).set_index(["fips","timestamp"])
 
     def test_shift_dates(self):
         """Tests that dates in the data are shifted according to the burn-in and lag."""
@@ -84,7 +84,7 @@ def test_geo_reindex(self):
                 "num": [0, 100, 200, 300, 400, 500, 600, 100, 200, 300, 400, 500, 600],
                 "fips": ['01001'] * 7 + ['04007'] * 6,
                 "den": [1000] * 7 + [2000] * 6,
-                "date": [pd.Timestamp(f'03-{i}-2020') for i in range(1, 14)]})
+                "timestamp": [pd.Timestamp(f'03-{i}-2020') for i in range(1, 14)]})
             data_frame = su_inst.geo_reindex(test_data)
             assert data_frame.shape[0] == multiple*len(su_inst.fit_dates)
             assert (data_frame.sum() == (4200,19000)).all()
@@ -113,8 +113,8 @@ def test_update_sensor(self):
                 "num": [0, 100, 200, 300, 400, 500, 600, 100, 200, 300, 400, 500, 600] * 2,
                 "fips": ["01001"] * 13 + ["42003"] * 13,
                 "den": [30, 50, 50, 10, 1, 5, 5, 50, 50, 50, 0, 0, 0] * 2,
-                "date": list(pd.date_range("20200301", "20200313")) * 2
-            }).set_index(["fips", "date"])
+                "timestamp": list(pd.date_range("20200301", "20200313")) * 2
+            }).set_index(["fips", "timestamp"])
             su_inst.update_sensor(small_test_data,  td.name)
             for f in os.listdir(td.name):
                 outputs[f] = pd.read_csv(os.path.join(td.name, f))

claims_hosp/delphi_claims_hosp/config.py

@@ -32,8 +32,13 @@ class Config:
     # data columns
     CLAIMS_COUNT_COLS = ["Denominator", "Covid_like"]
     CLAIMS_DATE_COL = "ServiceDate"
-    CLAIMS_RENAME_COLS = {"Pat HRR ID": "hrr", "ServiceDate": "date",
-                          "PatCountyFIPS": "fips", "PatAgeGroup": "age_group"}
+    FIPS_COL = "fips"
+    DATE_COL = "timestamp"
+    AGE_COL = "age_group"
+    HRR_COL = "hrr"
+
+    CLAIMS_RENAME_COLS = {"Pat HRR ID": HRR_COL, "ServiceDate":  DATE_COL,
+                          "PatCountyFIPS": FIPS_COL, "PatAgeGroup": AGE_COL}
     CLAIMS_DTYPES = {
         "ServiceDate": str,
         "PatCountyFIPS": str,
@@ -43,10 +48,7 @@ class Config:
         "Pat HRR ID": str,
     }
 
-    FIPS_COL = "fips"
-    DATE_COL = "date"
-    AGE_COL = "age_group"
-    HRR_COL = "hrr"
+
 
     SMOOTHER_BANDWIDTH = 100  # bandwidth for the linear left Gaussian filter
     MIN_DEN = 100  # number of total visits needed to produce a sensor

claims_hosp/delphi_claims_hosp/load_data.py

@@ -48,7 +48,7 @@ def load_claims_data(claims_filepath, dropdate, base_geo):
     ), "Claims counts must be nonnegative"
 
     # aggregate age groups (so data is unique by date and base geography)
-    claims_data = claims_data.groupby([base_geo, "date"]).sum()
+    claims_data = claims_data.groupby([base_geo, Config.DATE_COL]).sum()
     claims_data.dropna(inplace=True)  # drop rows with any missing entries
 
     return claims_data

claims_hosp/delphi_claims_hosp/update_indicator.py

@@ -120,11 +120,11 @@ def geo_reindex(self, data):
             return False
 
         unique_geo_ids = pd.unique(data_frame[self.geo])
-        data_frame.set_index([self.geo, 'date'], inplace=True)
+        data_frame.set_index([self.geo, "timestamp"], inplace=True)
 
         # for each location, fill in all missing dates with 0 values
         multiindex = pd.MultiIndex.from_product((unique_geo_ids, self.fit_dates),
-                                                names=[self.geo, "date"])
+                                                names=[self.geo, Config.DATE_COL])
         assert (
                 len(multiindex) <= (GeoConstants.MAX_GEO[self.geo] * len(self.fit_dates))
         ), "more loc-date pairs than maximum number of geographies x number of dates"

claims_hosp/tests/test_indicator.py

@@ -56,7 +56,7 @@ def test_backwards_pad(self):
     def test_fit_fips(self):
         date_range = pd.date_range("2020-05-01", "2020-05-20")
         all_fips = self.fips_data.fips.unique()
-        loc_index_fips_data = self.fips_data.set_index(["fips", "date"])
+        loc_index_fips_data = self.fips_data.set_index(["fips", "timestamp"])
         sample_fips = nr.choice(all_fips, 10)
 
         for fips in sample_fips:
@@ -79,7 +79,7 @@ def test_fit_fips(self):
     def test_fit_hrrs(self):
         date_range = pd.date_range("2020-05-01", "2020-05-20")
         all_hrrs = self.hrr_data.hrr.unique()
-        loc_index_hrr_data = self.hrr_data.set_index(["hrr", "date"])
+        loc_index_hrr_data = self.hrr_data.set_index(["hrr", "timestamp"])
         sample_hrrs = nr.choice(all_hrrs, 10)
 
         for hrr in sample_hrrs:

claims_hosp/tests/test_load_data.py

@@ -34,8 +34,8 @@ def test_base_unit(self):
     def test_claims_columns(self):
         assert "hrr" in self.hrr_claims_data.index.names
         assert "fips" in self.fips_claims_data.index.names
-        assert "date" in self.hrr_claims_data.index.names
-        assert "date" in self.fips_claims_data.index.names
+        assert "timestamp" in self.hrr_claims_data.index.names
+        assert "timestamp" in self.fips_claims_data.index.names
 
         expected_claims_columns = ["Denominator", "Covid_like"]
         for col in expected_claims_columns:
@@ -47,8 +47,8 @@ def test_claims_columns(self):
     def test_data_columns(self):
         assert "hrr" in self.hrr_data.columns
         assert "fips" in self.fips_data.columns
-        assert "date" in self.hrr_data.columns
-        assert "date" in self.fips_data.columns
+        assert "timestamp" in self.hrr_data.columns
+        assert "timestamp" in self.fips_data.columns
 
         expected_columns = ["num", "den"]
         for col in expected_columns:
@@ -57,12 +57,12 @@ def test_data_columns(self):
 
     def test_edge_values(self):
         for data in [self.hrr_claims_data, self.fips_claims_data]:
-            assert data.index.get_level_values('date').max() >= Config.FIRST_DATA_DATE
-            assert data.index.get_level_values('date').min() < DROP_DATE
+            assert data.index.get_level_values("timestamp").max() >= Config.FIRST_DATA_DATE
+            assert data.index.get_level_values("timestamp").min() < DROP_DATE
 
         for data in [self.hrr_data, self.fips_data]:
-            assert data.date.max() >= Config.FIRST_DATA_DATE
-            assert data.date.min() < DROP_DATE
+            assert data["timestamp"].max() >= Config.FIRST_DATA_DATE
+            assert data["timestamp"].min() < DROP_DATE
 
     def test_hrrs_values(self):
         assert len(self.hrr_data.hrr.unique()) <= CONSTANTS.NUM_HRRS

claims_hosp/tests/test_update_indicator.py

@@ -37,8 +37,8 @@ class TestClaimsHospIndicatorUpdater:
         "num": [0, 100, 200, 300, 400, 500, 600, 100, 200, 300, 400, 500, 600],
         "hrr": [1.0] * 7 + [2.0] * 6,
         "den": [1000] * 7 + [2000] * 6,
-        "date": [pd.Timestamp(f'03-{i}-2020') for i in range(1, 14)]}).set_index(
-        ["hrr", "date"])
+        "timestamp": [pd.Timestamp(f'03-{i}-2020') for i in range(1, 14)]}).set_index(
+        ["hrr", "timestamp"])
 
     def test_shift_dates(self):
         updater = ClaimsHospIndicatorUpdater(