Merge pull request #1266 from cmu-delphi/clean-up

Minor clean up of changehc sensor updating

Author: krivard

_delphi_utils_python/tests/validator/test_dynamic.py

@@ -49,7 +49,7 @@ def test_half_padding(self):
         assert new_ref_df.time_value.max() == datetime.strptime("2021-01-11",
             "%Y-%m-%d").date()
         assert new_ref_df.shape[0] == 11
-        assert new_ref_df.loc[:, "val"].iloc[5] == 2
+        assert new_ref_df["val"].iloc[5] == 2
 
     def test_full_padding(self):
         validator = DynamicValidator(self.params)
@@ -71,7 +71,7 @@ def test_full_padding(self):
         assert new_ref_df.time_value.max() == datetime.strptime("2021-01-15",
             "%Y-%m-%d").date()
         assert new_ref_df.shape[0] == 15
-        assert new_ref_df.loc[:, "val"].iloc[5] == 2
+        assert new_ref_df["val"].iloc[5] == 2
 
 class TestCheckRapidChange:
     params = {

cdc_covidnet/delphi_cdc_covidnet/update_sensor.py

@@ -77,7 +77,7 @@ def update_sensor(
                             right_on=["year", "weeknumber"])
 
     # Select relevant columns and standardize naming
-    hosp_df = hosp_df.loc[:, APIConfig.HOSP_RENAME_COLS.keys()]\
+    hosp_df = hosp_df[APIConfig.HOSP_RENAME_COLS.keys()]\
         .rename(columns=APIConfig.HOSP_RENAME_COLS)
 
     # Restrict to start and end date

changehc/README.md

@@ -73,7 +73,7 @@ should not include critical sub-routines.
 
 ## Code tour
 
-- update_sensor.py: CHCSensorUpdator: reads the data, makes transformations, writes results to file
+- update_sensor.py: CHCSensorUpdater: reads the data, makes transformations, writes results to file
 - sensor.py: CHCSensor: methods for transforming data, including backfill and smoothing
 - smooth.py: implements local linear left Gaussian filter
 - load_data.py: methods for loading denominator and covid data

changehc/delphi_changehc/run.py

@@ -16,7 +16,7 @@
 # first party
 from .download_ftp_files import download_covid, download_cli
 from .load_data import load_combined_data, load_cli_data
-from .update_sensor import CHCSensorUpdator
+from .update_sensor import CHCSensorUpdater
 
 
 def retrieve_files(params, filedate, logger):
@@ -164,7 +164,7 @@ def run_module(params: Dict[str, Dict[str, Any]]):
                     logger.info("starting weekday adj", geo = geo, numtype = numtype)
                 else:
                     logger.info("starting no adj", geo = geo, numtype = numtype)
-                su_inst = CHCSensorUpdator(
+                su_inst = CHCSensorUpdater(
                     startdate,
                     enddate,
                     dropdate,

changehc/delphi_changehc/update_sensor.py

@@ -37,10 +37,6 @@ def write_to_csv(df, geo_level, write_se, day_shift, out_name, logger, output_pa
 
     # shift dates forward for labeling
     df["timestamp"] += day_shift
-    if start_date is None:
-        start_date = min(df["timestamp"])
-    if end_date is None:
-        end_date = max(df["timestamp"])
 
     # suspicious value warnings
     suspicious_se_mask = df["se"].gt(5)
@@ -49,7 +45,7 @@ def write_to_csv(df, geo_level, write_se, day_shift, out_name, logger, output_pa
     if write_se:
         logger.info("========= WARNING: WRITING SEs TO {0} =========".format(out_name))
     else:
-        df.loc[:, "se"] = np.nan
+        df["se"] = np.nan
 
     assert not df["val"].isna().any(), " val contains nan values"
     suspicious_val_mask = df["val"].gt(90)
@@ -75,7 +71,7 @@ def write_to_csv(df, geo_level, write_se, day_shift, out_name, logger, output_pa
     return dates
 
 
-class CHCSensorUpdator:  # pylint: disable=too-many-instance-attributes
+class CHCSensorUpdater:  # pylint: disable=too-many-instance-attributes
     """Contains methods to update sensor and write results to csv."""
 
     def __init__(self,
@@ -89,7 +85,7 @@ def __init__(self,
                  se,
                  wip_signal,
                  logger):
-        """Init Sensor Updator.
+        """Init Sensor Updater.
 
         Args:
             startdate: first sensor date (YYYY-mm-dd)
@@ -116,13 +112,13 @@ def __init__(self,
 
         # output file naming
         if self.numtype == "covid":
-            signals = [SMOOTHED_ADJ if self.weekday else SMOOTHED]
+            signal_name = SMOOTHED_ADJ if self.weekday else SMOOTHED
         elif self.numtype == "cli":
-            signals = [SMOOTHED_ADJ_CLI if self.weekday else SMOOTHED_CLI]
-        signal_names = add_prefix(
-            signals,
-            wip_signal=wip_signal)
-        self.updated_signal_names = signal_names
+            signal_name = SMOOTHED_ADJ_CLI if self.weekday else SMOOTHED_CLI
+        else:
+            raise ValueError(f'Unsupported numtype received "{numtype}",'
+                             f' must be one of ["covid", "cli"]')
+        self.signal_name = add_prefix([signal_name], wip_signal=wip_signal)[0]
 
         # initialize members set in shift_dates().
         self.burnindate = None
@@ -234,22 +230,21 @@ def update_sensor(self,
         # conform to naming expected by create_export_csv()
         df = df.reset_index().rename(columns={"rate": "val"})
         # df.loc[~df['incl'], ["val", "se"]] = np.nan  # update to this line after nancodes get merged in
-        df = df[df['incl']]
+        df = df[df["incl"]]
 
         # write out results
+        dates = write_to_csv(
+            df,
+            geo_level=self.geo,
+            start_date=min(self.sensor_dates),
+            end_date=max(self.sensor_dates),
+            write_se=self.se,
+            day_shift=Config.DAY_SHIFT,
+            out_name=self.signal_name,
+            output_path=output_path,
+            logger=self.logger
+        )
         stats = []
-        for signal in self.updated_signal_names:
-            dates = write_to_csv(
-                df,
-                geo_level=self.geo,
-                start_date=min(self.sensor_dates),
-                end_date=max(self.sensor_dates),
-                write_se=self.se,
-                day_shift=Config.DAY_SHIFT,
-                out_name=signal,
-                output_path=output_path,
-                logger=self.logger
-            )
-            if len(dates) > 0:
-                stats.append((max(dates), len(dates)))
+        if len(dates) > 0:
+            stats = [(max(dates), len(dates))]
         return stats

changehc/tests/test_update_sensor.py

@@ -14,7 +14,7 @@
 
 # first party
 from delphi_changehc.config import Config
-from delphi_changehc.update_sensor import write_to_csv, CHCSensorUpdator
+from delphi_changehc.update_sensor import write_to_csv, CHCSensorUpdater
 
 CONFIG = Config()
 PARAMS = {
@@ -30,7 +30,7 @@
 OUTPATH="test_data/"
 TEST_LOGGER = logging.getLogger()
 
-class TestCHCSensorUpdator:
+class TestCHCSensorUpdater:
     """Tests for updating the sensors."""
     geo = "county"
     parallel = False
@@ -46,7 +46,7 @@ class TestCHCSensorUpdator:
 
     def test_shift_dates(self):
         """Tests that dates in the data are shifted according to the burn-in and lag."""
-        su_inst = CHCSensorUpdator(
+        su_inst = CHCSensorUpdater(
             "02-01-2020",
             "06-01-2020",
             "06-12-2020",
@@ -71,7 +71,7 @@ def test_shift_dates(self):
     def test_geo_reindex(self):
         """Tests that the geo reindexer changes the geographic resolution."""
         for geo, multiple in [("nation", 1), ("county", 2), ("hhs", 2)]:
-            su_inst = CHCSensorUpdator(
+            su_inst = CHCSensorUpdater(
                 "02-01-2020",
                 "06-01-2020",
                 "06-12-2020",
@@ -98,7 +98,7 @@ def test_update_sensor(self):
         outputs = {}
         for geo in ["county", "state", "hhs", "nation"]:
             td = TemporaryDirectory()
-            su_inst = CHCSensorUpdator(
+            su_inst = CHCSensorUpdater(
                 "03-01-2020",
                 "03-22-2020",
                 "03-27-2020",

nchs_mortality/delphi_nchs_mortality/pull.py

@@ -98,7 +98,7 @@ def pull_nchs_mortality_data(token: str, test_file: Optional[str]=None):
 
     # Drop rows for locations outside US
     df = df[df["state"] != "United States"]
-    df = df.loc[:, keep_columns + ["timestamp", "state"]].set_index("timestamp")
+    df = df[keep_columns + ["timestamp", "state"]].set_index("timestamp")
 
     # NCHS considers NYC as an individual state, however, we want it included
     # in NY. If values are nan for both NYC and NY, the aggreagtion should