Merge pull request #1386 from cmu-delphi/release/indicators_v0.2.8_utils_v0.2.5

Release covidcast-indicators 0.2.8

Author: krivard

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.2.7
+current_version = 0.2.8
 commit = True
 message = chore: bump covidcast-indicators to {new_version}
 tag = False

Jenkinsfile

@@ -9,7 +9,7 @@
    - Keep in sync with '.github/workflows/python-ci.yml'.
    - TODO: #527 Get this list automatically from python-ci.yml at runtime.
  */
-def indicator_list = ["cdc_covidnet", "changehc", "claims_hosp", "combo_cases_and_deaths", "covid_act_now", "facebook", "google_symptoms", "hhs_hosp", "jhu", "nchs_mortality", "quidel", "quidel_covidtest", "safegraph_patterns", "sir_complainsalot", "usafacts"]
+def indicator_list = ["changehc", "claims_hosp", "covid_act_now", "facebook", "google_symptoms", "hhs_hosp", "jhu", "nchs_mortality", "quidel", "quidel_covidtest", "safegraph_patterns", "sir_complainsalot", "usafacts"]
 def build_package = [:]
 def deploy_staging = [:]
 def deploy_production = [:]

ansible/templates/sir_complainsalot-params-prod.json.j2

@@ -89,11 +89,6 @@
         ["smoothed_dontneed_reason_precautions", "hrr"], ["smoothed_wdontneed_reason_precautions", "hrr"]
       ]
     },
-    "indicator-combination": {
-      "max_age": 4,
-      "maintainers": ["U01AP8GSWG3","U01069KCRS7"],
-      "retired-signals": ["nmf_day_doc_fbs_ght", "nmf_day_doc_fbc_fbs_ght", "confirmed_7dav_cumulative_num", "confirmed_7dav_cumulative_prop", "deaths_7dav_cumulative_num", "deaths_7dav_cumulative_prop"]
-    },
     "quidel": {
       "max_age":6,
       "maintainers": ["U01AP8GSWG3","U01069KCRS7"],

hhs_facilities/delphi_hhs_facilities/constants.py

@@ -1,7 +1,12 @@
 """Registry for signals and geographies to process."""
+from numpy import nan
 from .generate_signals import sum_cols
 
-NAN_VALUES = [None, -999999, -999999.0]
+NAN_VALUES = {
+    None: nan,
+    -999999: 1.5, # -999,999 represents the data range [0-3], so we use the range mean
+    -999999.0: 1.5
+}
 
 CONFIRMED_ADMISSIONS = "confirmed_admissions_7d"
 CONFIRMED_SUSPECTED_ADMISSIONS = "sum_confirmed_suspected_admissions_7d"

hhs_facilities/delphi_hhs_facilities/generate_signals.py

@@ -5,6 +5,20 @@
 import pandas as pd
 import numpy as np
 
+from delphi_utils import Nans
+
+
+def add_nancodes(df):
+    """Add nancodes to a signal dataframe."""
+    # Default missingness codes
+    df["missing_val"] = Nans.NOT_MISSING
+    df["missing_se"] = Nans.NOT_APPLICABLE
+    df["missing_sample_size"] = Nans.NOT_APPLICABLE
+
+    # Mark any remaining nans with unknown
+    remaining_nans_mask = df["val"].isnull()
+    df.loc[remaining_nans_mask, "missing_val"] = Nans.OTHER
+    return df
 
 def generate_signal(df: pd.DataFrame,
                     input_cols: list,
@@ -34,10 +48,13 @@ def generate_signal(df: pd.DataFrame,
     df_cols = [df[i] for i in input_cols]
     df["val"] = signal_func(df_cols)
     df["timestamp"] = df["timestamp"] + pd.Timedelta(days=date_offset)
-    df.dropna(subset=["val"], inplace=True)
-    df = df.groupby(["timestamp", "geo_id"], as_index=False).sum()
+    df = df.groupby(["timestamp", "geo_id"], as_index=False).sum(min_count=1)
     df["se"] = df["sample_size"] = np.nan
-    return df[["timestamp", "geo_id", "val", "se", "sample_size"]]
+    df = add_nancodes(df)
+    export_columns = [
+        "timestamp", "geo_id", "val", "se", "sample_size",
+        "missing_val", "missing_se", "missing_sample_size"]
+    return df[export_columns]
 
 
 def sum_cols(cols: list) -> pd.Series:

hhs_facilities/delphi_hhs_facilities/pull.py

@@ -3,7 +3,6 @@
 from datetime import date
 
 import pandas as pd
-import numpy as np
 from delphi_utils.geomap import GeoMapper
 from delphi_epidata import Epidata
 
@@ -54,6 +53,6 @@ def pull_data() -> pd.DataFrame:
     past_reference_day = int(date(2020, 1, 1).strftime("%Y%m%d"))  # first available date in DB
     all_states = GeoMapper().get_geo_values("state_id")
     responses = pull_data_iteratively(all_states, Epidata.range(past_reference_day, today))
-    all_columns = pd.DataFrame(responses).replace(NAN_VALUES, np.nan)
+    all_columns = pd.DataFrame(responses).replace(NAN_VALUES)
     all_columns["timestamp"] = pd.to_datetime(all_columns["collection_week"], format="%Y%m%d")
     return all_columns

hhs_facilities/tests/expected/20200131_county_confirmed_admissions_7d.csv

@@ -1,4 +1,4 @@
-geo_id,val,se,sample_size
-25013,33.0,NA,NA
-72001,76.56462035541196,NA,NA
-72141,0.4353796445880453,NA,NA
+geo_id,val,se,sample_size,missing_val,missing_se,missing_sample_size
+25013,33.00000000,NA,NA,0,1,1
+72001,76.56462040,NA,NA,0,1,1
+72141,0.43537960,NA,NA,0,1,1

hhs_facilities/tests/expected/20200131_county_sum_confirmed_suspected_admissions_7d.csv

@@ -1,4 +1,4 @@
-geo_id,val,se,sample_size
-25013,98.0,NA,NA
-72001,161.08400646203557,NA,NA
-72141,0.9159935379644588,NA,NA
+geo_id,val,se,sample_size,missing_val,missing_se,missing_sample_size
+25013,98.00000000,NA,NA,0,1,1
+72001,161.08400650,NA,NA,0,1,1
+72141,0.91599350,NA,NA,0,1,1

hhs_facilities/tests/expected/20200131_hrr_confirmed_admissions_7d.csv

@@ -1,2 +1,2 @@
-geo_id,val,se,sample_size
-230,33.0,NA,NA
+geo_id,val,se,sample_size,missing_val,missing_se,missing_sample_size
+230,33.00000000,NA,NA,0,1,1

hhs_facilities/tests/expected/20200131_hrr_sum_confirmed_suspected_admissions_7d.csv

@@ -1,2 +1,2 @@
-geo_id,val,se,sample_size
-230,98.0,NA,NA
+geo_id,val,se,sample_size,missing_val,missing_se,missing_sample_size
+230,98.00000000,NA,NA,0,1,1