switch to streaming approach to save memory

Author: Jingjing Tang

google_symptoms/delphi_google_symptoms/geo.py

@@ -60,7 +60,7 @@ def geo_map(df, geo_res):
         return df
 
     map_df = generate_transition_matrix(geo_res)
-    dfList = []
+    converted_df = pd.DataFrame(columns = df.columns)
     for _date in df["timestamp"].unique():
         newdf = pd.DataFrame({
                               "timestamp": _date,
@@ -77,6 +77,6 @@ def geo_map(df, geo_res):
         newdf["geo_id"] = list(map_df.keys())[1:]
         mask = (newdf[METRICS].sum(axis=1) == 0)
         newdf.loc[mask, METRICS + ["combined_symptoms"]] = np.nan
-        dfList.append(newdf)
-    return pd.concat(dfList)
+        converted_df = converted_df.append(newdf)
+    return converted_df
 

google_symptoms/delphi_google_symptoms/pull.py

@@ -123,15 +123,14 @@ def pull_gs_data(base_url):
     dfs["state"] = preprocess(df, "state")
 
     # For county level data
-    dfList = []
+    dfs["county"] = pd.DataFrame(columns = dfs["state"].columns)
     for state in list(STATE_TO_ABBREV.keys()):
         sub_url = "/subregions/" + "%20".join(state.split("_")) + "/"
-        dfList.append(pd.read_csv(base_url.format(sub_url=sub_url,
-                                                  state=state+"_"),
-                                  parse_dates = ["date"]))
-    df = pd.concat(dfList)
-    df["geo_id"] = df["open_covid_region_code"].apply(get_geo_id)
-    dfs["county"] = preprocess(df, "county")
+        df = pd.read_csv(base_url.format(sub_url=sub_url,
+                                         state=state+"_"),
+                         parse_dates = ["date"])
+        df["geo_id"] = df["open_covid_region_code"].apply(get_geo_id)
+        dfs["county"] = dfs["county"].append(preprocess(df, "county"))
 
     # Add District of Columbia County
     try: