Merge pull request #1300 from cmu-delphi/release/indicators_v0.1.22_utils_v0.1.16

Release covidcast-indicators 0.1.22

Author: krivard

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.1.21
+current_version = 0.1.22
 commit = True
 message = chore: bump covidcast-indicators to {new_version}
 tag = False

_delphi_utils_python/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.1.15
+current_version = 0.1.16
 commit = True
 message = chore: bump delphi_utils to {new_version}
 tag = False

_delphi_utils_python/delphi_utils/__init__.py

@@ -14,4 +14,4 @@
 from .signal import add_prefix
 from .nancodes import Nans
 
-__version__ = "0.1.15"
+__version__ = "0.1.16"

_delphi_utils_python/delphi_utils/export.py

@@ -70,6 +70,9 @@ def create_export_csv(
         even if there is no data for the day. If false, only the days present are written.
     logger: Optional[logging.Logger]
         Pass a logger object here to log information about contradictory missing codes.
+    weekly_dates: Optional[bool]
+        Whether the output data are weekly or not. If True, will prefix files with
+        "weekly_YYYYWW" where WW is the epiweek instead of the usual YYYYMMDD for daily files.
 
     Returns
     ---------

_delphi_utils_python/setup.py

@@ -25,7 +25,7 @@
 
 setup(
     name="delphi_utils",
-    version="0.1.15",
+    version="0.1.16",
     description="Shared Utility Functions for Indicators",
     long_description=long_description,
     long_description_content_type="text/markdown",

ansible/templates/sir_complainsalot-params-prod.json.j2

@@ -72,8 +72,8 @@
         ["smoothed_vaccine_barrier_language_tried", "msa"], ["smoothed_wvaccine_barrier_language_tried", "msa"],
         ["smoothed_vaccine_barrier_no_appointments_tried", "msa"], ["smoothed_wvaccine_barrier_no_appointments_tried", "msa"],
         ["smoothed_vaccine_barrier_none_tried", "msa"], ["smoothed_wvaccine_barrier_none_tried", "msa"],
-        ["smoothed_wvaccine_barrier_technical_difficulties_tried", "msa"], ["smoothed_vaccine_barrier_technical_difficulties_tried", "msa"],
-        ["smoothed_wvaccine_barrier_technology_access_tried", "msa"], ["smoothed_wvaccine_barrier_technology_access_tried", "msa"],
+        ["smoothed_vaccine_barrier_technical_difficulties_tried", "msa"], ["smoothed_wvaccine_barrier_technical_difficulties_tried", "msa"],
+        ["smoothed_vaccine_barrier_technology_access_tried", "msa"], ["smoothed_wvaccine_barrier_technology_access_tried", "msa"],
         ["smoothed_vaccine_barrier_time_tried", "msa"], ["smoothed_wvaccine_barrier_time_tried", "msa"],
         ["smoothed_vaccine_barrier_travel_tried", "msa"], ["smoothed_wvaccine_barrier_travel_tried", "msa"],
         ["smoothed_vaccine_barrier_type_tried", "msa"], ["smoothed_wvaccine_barrier_type_tried", "msa"]

facebook/delphiFacebook/R/responses.R

@@ -170,6 +170,7 @@ load_response_one <- function(input_filename, params, contingency_run) {
   input_data <- bodge_v4_translation(input_data, wave)
   input_data <- bodge_C6_C8(input_data, wave)
   input_data <- bodge_B13(input_data, wave)
+  input_data <- bodge_E1(input_data, wave)
 
   input_data <- code_symptoms(input_data, wave)
   input_data <- code_hh_size(input_data, wave)
@@ -486,6 +487,34 @@ bodge_B13 <- function(input_data, wave) {
   return(input_data)
 }
 
+#' Fix E1_* names in Wave 11 data after ~June 16, 2021.
+#' 
+#' Items E1_1 through E1_4 are part of a matrix. Qualtrics, for unknown reasons,
+#' switched to naming these E1_4 through E1_7 in June. Convert back to the
+#' intended names.
+#'
+#' @param input_data data frame of responses, before subsetting to select
+#'   variables
+#' @param wave integer indicating survey version
+#'   
+#' @return corrected data frame
+#' @importFrom dplyr rename
+bodge_E1 <- function(input_data, wave) {
+  E14_present <- all(c("E1_1", "E1_2", "E1_3", "E1_4") %in% names(input_data))
+  E47_present <- all(c("E1_4", "E1_5", "E1_6", "E1_7") %in% names(input_data))
+  assert(!(E14_present && E47_present), "fields E1_1-E1_4 should not be present at the same time as fields E1_4-E1_7")
+  
+  if ( E47_present ) {
+    input_data <- rename(input_data,
+                         E1_1 = "E1_4",
+                         E1_2 = "E1_5",
+                         E1_3 = "E1_6",
+                         E1_4 = "E1_7"
+    )
+  }
+  return(input_data)
+}
+
 #' Process module assignment column.
 #' 
 #' Rename `module` and recode to A/B/`NA`. Note: module assignment column name

facebook/delphiFacebook/man/bodge_E1.Rd

@@ -349,3 +349,43 @@ test_that("B13 bodge works correctly", {
                input)
 })
 
+test_that("E1 bodge works correctly", {
+  input <- tibble(
+    E1_1 = c(1, 2, 3, 1),
+    E1_2 = c(1, 2, 3, 2),
+    E1_3 = c(1, 2, 3, 3),
+    E1_4 = c(1, 2, 3, 4)
+  )
+  expect_equal(bodge_E1(input, wave = NA), input)
+  
+  input_4_to_7 <- tibble(
+    E1_4 = c(1, 2, 3, 1),
+    E1_5 = c(1, 2, 3, 2),
+    E1_6 = c(1, 2, 3, 3),
+    E1_7 = c(1, 2, 3, 4)
+  )
+
+  expect_equal(bodge_E1(input_4_to_7, wave = NA), input)
+  
+  input_mixed <- tibble(
+    E1_4 = c(1, 2, 3, 1),
+    E1_2 = c(1, 2, 3, 2),
+    E1_3 = c(1, 2, 3, 3),
+    E1_7 = c(1, 2, 3, 4)
+  )
+  expect_equal(bodge_E1(input_mixed, wave = NA), input_mixed)
+  
+  input_both <- tibble(
+    E1_1 = c(1, 2, 3, 1),
+    E1_2 = c(1, 2, 3, 2),
+    E1_3 = c(1, 2, 3, 3),
+    E1_4 = c(1, 2, 3, 1),
+    E1_5 = c(1, 2, 3, 2),
+    E1_6 = c(1, 2, 3, 3),
+    E1_7 = c(1, 2, 3, 4)
+  )
+  expect_error(bodge_E1(input_both, wave = NA),
+               "fields E1_1-E1_4 should not be present at the same time as fields E1_4-E1_7"
+  )
+})
+

facebook/delphiFacebook/unit-tests/testthat/test-responses.R

@@ -326,7 +326,7 @@ process_qsf <- function(path_to_qsf,
 
   if (any(is.na(qdf$description))) {
     nonlabelled_items <- qdf$variable[is.na(qdf$description)]
-    stop(sprintf("items %s do not have a short name assigned",
+    stop(sprintf("items %s do not have a description provided",
                  paste(nonlabelled_items, collapse=", "))
     )
   }
@@ -396,7 +396,14 @@ add_static_fields <- function(codebook,
                               path_to_static_fields="./static/static_microdata_fields.csv") {
   static_fields <- get_static_fields(wave, path_to_static_fields)
 
-  return(bind_rows(codebook, static_fields))
+  codebook <- bind_rows(codebook, static_fields) %>% 
+    filter(!(variable == "module" & wave < 11), # module field is only available for wave >= 11
+           !(variable %in% c("wave", "UserLanguage", "fips") & wave < 4), # wave, UserLangauge, and fips fields are only available for wave >= 4
+           !(variable == "w12_treatment" & wave == 12.5), # experimental arm field is only available for wave == 12.5
+           variable != "Random_Number"
+    )
+
+  return(codebook)
 }
 
 #' Load dataframe of non-Qualtrics data fields
@@ -417,8 +424,7 @@ get_static_fields <- function(wave,
                                              response_option_randomization = col_character()
                             )) %>%
     mutate(wave = wave) %>% 
-    select(wave, everything()) %>% 
-    filter(!(variable == "module" & wave < 11))
+    select(wave, everything())
 
   return(static_fields)
 }