Merge pull request #677 from cmu-delphi/release/delphi-epidata-0.2.7

Release Delphi Epidata 0.2.7

Author: sgratzl

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.2.6
+current_version = 0.2.7
 commit = False
 tag = False
 

docs/symptom-survey/problems.md

@@ -17,6 +17,50 @@ interpretation of results. All problems will be logged here.
 1. TOC
 {:toc}
 
+## Incorrect Wave Sampling (June-August, 2021)
+
+A sampling error caused a portion of our respondents to be accidentally sent the
+Wave 10 survey instrument instead of the newer Wave 11 survey instrument. The
+error was fixed on August 2, 2021, but as a result of this error:
+
+* A random 1% of the sample was accidentally sent the W10 instrument between
+  June 15 and July 4, 2021.
+* A random 3% of the sample was accidentally sent the W10 instrument between
+  July 5 and July 11, 2021.
+* A random 10% of the sample was accidentally sent the W10 instrument between
+  July 12 and July 21, 2021.
+* A random 50% of the sample was accidentally sent the W10 instrument between
+  July 22 and July 30, 2021.
+* A random 91-98% of the sample was accidentally sent the W10 instrument between
+  July 31 and August 1, 2021.
+
+Despite differences between the two instruments, we have not detected trend
+discontinuities as a result of the error. We will continue to investigate.
+
+## Response Volume Drop (July 20-22, 2021)
+
+On July 20, 2021, a technical problem in the process that samples Facebook users
+for invitation to the survey caused lower-than-normal sample sizes. The problem
+was resolved on July 22 and users that had been incorrectly not invited were
+invited over the following several days. This caused sample sizes to temporarily
+increase, compensating for the initial drop.
+
+## Android Sampling Problem (June 17-24, 2021)
+
+There was a bug in the in-app web browser in the Android version of the Facebook
+app released on June 17. It affected all links within the Android version of the
+Facebook app, not just the Carnegie Mellon University (CMU) and University of
+Maryland (UMD) COVID-19 Trends and Impact Surveys, which are conducted in
+partnership with Facebook. It did not affect links within other versions of
+Facebook, including iOS. The issue has been fixed in the version of the Facebook
+app released on June 24.
+
+The issue caused a decrease in total response volume across the CMU and Maryland
+surveys, since Android users who were invited could not take the surveys. As of
+June 26, 2021, 89.5% of the initial decrease in total response volume has been
+recovered. The response volume continued to recover as Android users updated to
+the fixed version of the Facebook app.
+
 ## Incorrect Coding in Documentation
 
 We found a Qualtrics bug that affects the exported text of the survey (but not

requirements.txt

@@ -7,3 +7,4 @@ pandas==1.2.3
 scipy==1.6.2
 tenacity==7.0.0
 newrelic
+epiweeks==2.1.2

src/acquisition/covidcast/signal_dash_data_generator.py

@@ -9,6 +9,7 @@
 import pandas as pd
 
 from dataclasses import dataclass
+from epiweeks import Week
 from typing import List
 
 # first party
@@ -17,7 +18,9 @@
 from delphi.epidata.acquisition.covidcast.logger import get_structured_logger
 
 
-LOOKBACK_DAYS_FOR_COVERAGE = 28
+LOOKBACK_DAYS_FOR_COVERAGE = 56
+BASE_COVIDCAST = covidcast.covidcast.Epidata.BASE_URL[:-len("api.php")] + "covidcast"
+COVERAGE_URL = f"{BASE_COVIDCAST}/coverage?format=csv&signal={{source}}:{{signal}}&days={LOOKBACK_DAYS_FOR_COVERAGE}"
 
 @dataclass
 class DashboardSignal:
@@ -195,27 +198,23 @@ def get_latest_time_value_from_metadata(dashboard_signal, metadata):
 def get_coverage(dashboard_signal: DashboardSignal,
                  metadata) -> List[DashboardSignalCoverage]:
     """Get the most recent coverage for the signal."""
-    latest_time_value = get_latest_time_value_from_metadata(
-        dashboard_signal, metadata)
-    start_day = latest_time_value - datetime.timedelta(days = LOOKBACK_DAYS_FOR_COVERAGE)
-    latest_data = covidcast.signal(
-        dashboard_signal.source,
-        dashboard_signal.covidcast_signal,
-        end_day = latest_time_value,
-        start_day = start_day)
-    latest_data_without_megacounties = latest_data[~latest_data['geo_value'].str.endswith(
-        '000')]
-    count_by_geo_type_df = latest_data_without_megacounties.groupby(
-        ['geo_type', 'data_source', 'time_value', 'signal']).size().to_frame(
-        'count').reset_index()
+    count_by_geo_type_df = pd.read_csv(
+        COVERAGE_URL.format(source=dashboard_signal.source,
+                            signal=dashboard_signal.covidcast_signal))
+    try:
+        count_by_geo_type_df["time_value"] = count_by_geo_type_df["time_value"].apply(
+            lambda x: pd.to_datetime(str(x), format="%Y%m%d"))
+    except:
+        count_by_geo_type_df["time_value"] = count_by_geo_type_df["time_value"].apply(
+            lambda x: pd.to_datetime(Week(x // 100, x % 100).startdate()))
 
     signal_coverage_list = []
 
     for _, row in count_by_geo_type_df.iterrows():
         signal_coverage = DashboardSignalCoverage(
             signal_id=dashboard_signal.db_id,
             date=row['time_value'].date(),
-            geo_type=row['geo_type'],
+            geo_type='county',
             count=row['count'])
         signal_coverage_list.append(signal_coverage)
 

src/client/delphi_epidata.R

@@ -15,7 +15,7 @@ Epidata <- (function() {
   # API base url
   BASE_URL <- 'https://delphi.cmu.edu/epidata/api.php'
 
-  client_version <- '0.2.6'
+  client_version <- '0.2.7'
 
   # Helper function to cast values and/or ranges to strings
   .listitem <- function(value) {

src/client/delphi_epidata.js

@@ -22,7 +22,7 @@
   }
 })(this, function (exports, fetchImpl, jQuery) {
   const BASE_URL = "https://delphi.cmu.edu/epidata/";
-  const client_version = "0.2.6";
+  const client_version = "0.2.7";
 
   // Helper function to cast values and/or ranges to strings
   function _listitem(value) {

src/client/packaging/npm/package.json

@@ -2,7 +2,7 @@
   "name": "delphi_epidata",
   "description": "Delphi Epidata API Client",
   "authors": "Delphi Group",
-  "version": "0.2.6",
+  "version": "0.2.7",
   "license": "MIT",
   "homepage": "https://github.com/cmu-delphi/delphi-epidata",
   "bugs": {

src/client/packaging/pypi/delphi_epidata/__init__.py

@@ -1,4 +1,4 @@
 from .delphi_epidata import Epidata
 
 name = 'delphi_epidata'
-__version__ = '0.2.6'
+__version__ = '0.2.7'

src/client/packaging/pypi/setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name="delphi_epidata",
-    version="0.2.6",
+    version="0.2.7",
     author="David Farrow",
     author_email="dfarrow0@gmail.com",
     description="A programmatic interface to Delphi's Epidata API.",

src/server/_config.py

@@ -5,7 +5,7 @@
 
 load_dotenv()
 
-VERSION = "0.2.6"
+VERSION = "0.2.7"
 
 MAX_RESULTS = int(10e6)
 MAX_COMPATIBILITY_RESULTS = int(3650)