Merge pull request #1006 from dshemetov/nans_hhs_facilities

Add NAN code support to HHS Facilities

Author: krivard

hhs_facilities/delphi_hhs_facilities/constants.py

@@ -1,7 +1,12 @@
 """Registry for signals and geographies to process."""
+from numpy import nan
 from .generate_signals import sum_cols
 
-NAN_VALUES = [None, -999999, -999999.0]
+NAN_VALUES = {
+    None: nan,
+    -999999: 1.5, # -999,999 represents the data range [0-3], so we use the range mean
+    -999999.0: 1.5
+}
 
 CONFIRMED_ADMISSIONS = "confirmed_admissions_7d"
 CONFIRMED_SUSPECTED_ADMISSIONS = "sum_confirmed_suspected_admissions_7d"

hhs_facilities/delphi_hhs_facilities/generate_signals.py

@@ -5,6 +5,20 @@
 import pandas as pd
 import numpy as np
 
+from delphi_utils import Nans
+
+
+def add_nancodes(df):
+    """Add nancodes to a signal dataframe."""
+    # Default missingness codes
+    df["missing_val"] = Nans.NOT_MISSING
+    df["missing_se"] = Nans.NOT_APPLICABLE
+    df["missing_sample_size"] = Nans.NOT_APPLICABLE
+
+    # Mark any remaining nans with unknown
+    remaining_nans_mask = df["val"].isnull()
+    df.loc[remaining_nans_mask, "missing_val"] = Nans.OTHER
+    return df
 
 def generate_signal(df: pd.DataFrame,
                     input_cols: list,
@@ -34,10 +48,13 @@ def generate_signal(df: pd.DataFrame,
     df_cols = [df[i] for i in input_cols]
     df["val"] = signal_func(df_cols)
     df["timestamp"] = df["timestamp"] + pd.Timedelta(days=date_offset)
-    df.dropna(subset=["val"], inplace=True)
-    df = df.groupby(["timestamp", "geo_id"], as_index=False).sum()
+    df = df.groupby(["timestamp", "geo_id"], as_index=False).sum(min_count=1)
     df["se"] = df["sample_size"] = np.nan
-    return df[["timestamp", "geo_id", "val", "se", "sample_size"]]
+    df = add_nancodes(df)
+    export_columns = [
+        "timestamp", "geo_id", "val", "se", "sample_size",
+        "missing_val", "missing_se", "missing_sample_size"]
+    return df[export_columns]
 
 
 def sum_cols(cols: list) -> pd.Series:

hhs_facilities/delphi_hhs_facilities/pull.py

@@ -3,7 +3,6 @@
 from datetime import date
 
 import pandas as pd
-import numpy as np
 from delphi_utils.geomap import GeoMapper
 from delphi_epidata import Epidata
 
@@ -54,6 +53,6 @@ def pull_data() -> pd.DataFrame:
     past_reference_day = int(date(2020, 1, 1).strftime("%Y%m%d"))  # first available date in DB
     all_states = GeoMapper().get_geo_values("state_id")
     responses = pull_data_iteratively(all_states, Epidata.range(past_reference_day, today))
-    all_columns = pd.DataFrame(responses).replace(NAN_VALUES, np.nan)
+    all_columns = pd.DataFrame(responses).replace(NAN_VALUES)
     all_columns["timestamp"] = pd.to_datetime(all_columns["collection_week"], format="%Y%m%d")
     return all_columns

hhs_facilities/tests/expected/20200131_county_confirmed_admissions_7d.csv

@@ -1,4 +1,4 @@
-geo_id,val,se,sample_size
-25013,33.0,NA,NA
-72001,76.56462035541196,NA,NA
-72141,0.4353796445880453,NA,NA
+geo_id,val,se,sample_size,missing_val,missing_se,missing_sample_size
+25013,33.00000000,NA,NA,0,1,1
+72001,76.56462040,NA,NA,0,1,1
+72141,0.43537960,NA,NA,0,1,1

hhs_facilities/tests/expected/20200131_county_sum_confirmed_suspected_admissions_7d.csv

@@ -1,4 +1,4 @@
-geo_id,val,se,sample_size
-25013,98.0,NA,NA
-72001,161.08400646203557,NA,NA
-72141,0.9159935379644588,NA,NA
+geo_id,val,se,sample_size,missing_val,missing_se,missing_sample_size
+25013,98.00000000,NA,NA,0,1,1
+72001,161.08400650,NA,NA,0,1,1
+72141,0.91599350,NA,NA,0,1,1

hhs_facilities/tests/expected/20200131_hrr_confirmed_admissions_7d.csv

@@ -1,2 +1,2 @@
-geo_id,val,se,sample_size
-230,33.0,NA,NA
+geo_id,val,se,sample_size,missing_val,missing_se,missing_sample_size
+230,33.00000000,NA,NA,0,1,1

hhs_facilities/tests/expected/20200131_hrr_sum_confirmed_suspected_admissions_7d.csv

@@ -1,2 +1,2 @@
-geo_id,val,se,sample_size
-230,98.0,NA,NA
+geo_id,val,se,sample_size,missing_val,missing_se,missing_sample_size
+230,98.00000000,NA,NA,0,1,1

hhs_facilities/tests/expected/20200131_msa_confirmed_admissions_7d.csv

@@ -1,4 +1,4 @@
-geo_id,val,se,sample_size
-10380,0.4353796445880453,NA,NA
-38660,76.56462035541196,NA,NA
-44140,33.0,NA,NA
+geo_id,val,se,sample_size,missing_val,missing_se,missing_sample_size
+10380,0.43537960,NA,NA,0,1,1
+38660,76.56462040,NA,NA,0,1,1
+44140,33.00000000,NA,NA,0,1,1

hhs_facilities/tests/expected/20200131_msa_sum_confirmed_suspected_admissions_7d.csv

@@ -1,4 +1,4 @@
-geo_id,val,se,sample_size
-10380,0.9159935379644588,NA,NA
-38660,161.08400646203557,NA,NA
-44140,98.0,NA,NA
+geo_id,val,se,sample_size,missing_val,missing_se,missing_sample_size
+10380,0.91599350,NA,NA,0,1,1
+38660,161.08400650,NA,NA,0,1,1
+44140,98.00000000,NA,NA,0,1,1

hhs_facilities/tests/expected/20200131_state_confirmed_admissions_7d.csv

@@ -1,3 +1,3 @@
-geo_id,val,se,sample_size
-AL,33.0,NA,NA
-PR,33.0,NA,NA
+geo_id,val,se,sample_size,missing_val,missing_se,missing_sample_size
+AL,33.00000000,NA,NA,0,1,1
+PR,33.00000000,NA,NA,0,1,1