fixed errors in filling missing values with 0s in raw

Author: Jingjing Tang

google_symptoms/delphi_google_symptoms/pull.py

@@ -1,15 +1,12 @@
 # -*- coding: utf-8 -*-
 import re
+from datetime import timedelta
 
 import numpy as np
 import pandas as pd
 
 from .constants import STATE_TO_ABBREV
 
-REPLACE_FIPS = [
-    ("46102", "46113"),
-]
-
 def get_geo_id(region_code):
     """
     There are region code in the format of "US-state" and "US-state-fips". In
@@ -95,17 +92,16 @@ def pull_gs_data(base_url, metrics, level):
             "schema may have changed.  Please investigate."
         )
 
-    # Let each FIPS/state has same number of rows
+    # Make sure each FIPS/state has same number of rows
     geo_list = df["geo_id"].unique()
-    date_list = pd.date_range(start=df["date"].min(),
+    date_list = pd.date_range(start=df["date"].min()-timedelta(days=7),
                               end=df["date"].max(),
                               freq='D')
     index_df = pd.MultiIndex.from_product(
         [geo_list, date_list], names=['geo_id', 'date']
     )
-    df = df.groupby(
-            ["geo_id", "date"]
-        ).sum().reindex(
+    df = df.set_index(["geo_id", "date"]
+        ).reindex(
             index_df
         ).reset_index(
         ).rename({"date": "timestamp"}, axis = 1)

google_symptoms/delphi_google_symptoms/run.py

@@ -22,9 +22,12 @@ def run_module():
     base_url = params["base_url"]
 
     for geo_res in GEO_RESOLUTIONS:
-        df = pull_gs_data(base_url, METRICS, geo_res)
+        df_pull = pull_gs_data(base_url, METRICS, geo_res)
         for metric, smoother in product(METRICS, SMOOTHERS):
             print(geo_res, metric, smoother)
+            df = df_pull.copy()
+            if smoother == "smoothed":
+                df = df.fillna(0)
             df["val"] = SMOOTHERS_MAP[smoother][0](df["symptom:"+metric].values)
             df["se"] = np.nan
             df["sample_size"] = np.nan