rewrite code for smoothing(groupby geo_id)

Author: Jingjing Tang

google_symptoms/delphi_google_symptoms/pull.py

@@ -1,6 +1,5 @@
 # -*- coding: utf-8 -*-
 import re
-from datetime import timedelta
 
 import numpy as np
 import pandas as pd
@@ -94,7 +93,7 @@ def pull_gs_data(base_url, metrics, level):
 
     # Make sure each FIPS/state has same number of rows
     geo_list = df["geo_id"].unique()
-    date_list = pd.date_range(start=df["date"].min()-timedelta(days=7),
+    date_list = pd.date_range(start=df["date"].min(),
                               end=df["date"].max(),
                               freq='D')
     index_df = pd.MultiIndex.from_product(

google_symptoms/delphi_google_symptoms/run.py

@@ -25,14 +25,17 @@ def run_module():
         df_pull = pull_gs_data(base_url, METRICS, geo_res)
         for metric, smoother in product(METRICS, SMOOTHERS):
             print(geo_res, metric, smoother)
-            df = df_pull.copy()
-            if smoother == "smoothed":
-                df = df.fillna(0)
-            df["val"] = SMOOTHERS_MAP[smoother][0](df["symptom:"+metric].values)
+#            df = df_pull.copy()
+#            if smoother == "smoothed":
+#                df = df.fillna(0)  
+            df = df_pull.set_index(["timestamp", "geo_id"])
+            df["val"] = df["symptom:"+metric].groupby(level=1
+                         ).transform(SMOOTHERS_MAP[smoother][0])
             df["se"] = np.nan
             df["sample_size"] = np.nan
             # Drop early entries where data insufficient for smoothing
             df = df.loc[~df["val"].isnull(), :]
+            df = df.reset_index()
             sensor_name = "_".join(["wip", smoother, "search"])
             create_export_csv(
                 df,