fix: hub data Julia script, notebook

Author: dsweber2

scripts/covid_hosp_explore.R

@@ -203,7 +203,7 @@ get_partially_applied_forecaster <- function(id) {
 # ================================ TARGETS =================================
 # ================================ PARAMETERS TARGETS ======================
 parameter_targets <- list2(
-  tar_target(name = aheads, command = c(0, 7, 14, 21)),
+  tar_target(name = aheads, command = c(0, 7, 14, 21, 28)),
   tar_target(name = ref_time_values, command = forecast_dates),
   # This is used for parameter lookup.
   tar_target(name = forecaster_parameter_grid, command = forecaster_parameter_combinations),
@@ -532,8 +532,10 @@ external_forecasts_and_scores <- rlang::list2(
   tar_target(
     external_forecasts_file,
     command = {
-      s3load("covid19_forecast_hub_2023.rds", bucket = "forecasting-team-data", verbose = FALSE)
-      full_results
+      s3load("covid19_forecast_hub_2023_full_summed.rds", bucket = "forecasting-team-data", verbose = FALSE)
+      full_results %>%
+        mutate(target_end_date = as.Date(forecast_date) + 7 * as.numeric(week_ahead)) %>%
+        rename(ahead = week_ahead)
     }
   ),
   tar_target(

scripts/one_offs/read_covid_forecast_hub_data.jl

@@ -2,33 +2,32 @@
 # specifically in the data-processed folder
 # to get the rds, run
 #
-# full_results <- readr::read_csv("../covid19-forecast-hub/data-processed/covid19-2023season-results.csv")
-# aws.s3::s3save(full_results, object = "covid19_forecast_hub_2023.rds", bucket = "forecasting-team-data")
+# full_results <- readr::read_csv("../OLDcovid19-forecast-hub/data-processed/covid19-2023season-results.csv")
+# aws.s3::s3save(full_results, object = "covid19_forecast_hub_2023_full_summed.rds", bucket = "forecasting-team-data")
 #
+using Base: floatrange
 using CSV
 using DataFrames
 using DataFramesMeta
 using Dates
 using RData
+import Base.lowercase
 pwd()
-res = CSV.read("COVIDhub-ensemble/2023-10-02-COVIDhub-ensemble.csv", DataFrame)
-pathname = "COVIDhub-ensemble/"
-filename = "2023-10-02-COVIDhub-ensemble.csv"
+res = CSV.read("COVIDhub_CDC-ensemble/2023-10-02-COVIDhub_CDC-ensemble.csv", DataFrame)
+pathname = "COVIDhub_CDC-ensemble/"
+filename = "2023-10-02-COVIDhub_CDC-ensemble.csv"
 state_names = CSV.read("../data-locations/locations.csv", DataFrame)
 lowercase(m::Missing) = m
 @rtransform! state_names @passmissing :abbreviation = lowercase(:abbreviation)
 @select! state_names :abbreviation :location
-
 function format_file(pathname, filename, state_names)
     if length(filename) < 10 ||
        match(r"[0-9]{4}-[0-9]{2}-[0-9]{2}", filename[1:10]) == nothing ||
        Date(filename[1:10]) < Date(2023, 1, 1)
         return DataFrame()
     end
     println(joinpath(pathname, filename))
-
-    res = CSV.read(joinpath(pathname, filename), DataFrame, missingstring="NA")
-
+    res = CSV.read(joinpath(pathname, filename), DataFrame, missingstring="NA", types=Dict("value" => Float64))
     if !("forecast_date" in names(res)) ||
        res[!, :forecast_date] |> minimum < Date(2023, 1, 1)
         return DataFrame()
@@ -41,7 +40,11 @@ function format_file(pathname, filename, state_names)
     end
     res = leftjoin(res, state_names, on=:location)
     @select! res :forecaster :geo_value = :abbreviation :forecast_date :target_end_date :ahead = :target :quantile :value
-    res
+    @chain res begin
+        @rtransform :week_ahead = div(:ahead, 7)
+        @groupby :forecaster :geo_value :forecast_date :week_ahead :quantile
+        @combine :value = sum(:value)
+    end
 end
 results = DataFrame[]
 for (root, dirs, files) in walkdir(".")
@@ -50,11 +53,4 @@ for (root, dirs, files) in walkdir(".")
     end
 end
 full_results = vcat(results...)
-CSV.write("covid19-2023season-results.csv", full_results)
-full_results[!, :forecaster] |> unique
-@rsubset! full_results :ahead % 7 == 0
-@rtransform! full_results :forecaster = :forecaster[3:end]
-"./fqfae"[3:end]
-3 % 7
-@rsubset full_results !ismissing(:geo_value) :forecast_date == Date(2023,11,13)
-@rsubset res :forecast_date == Date(2023,11,0)
+CSV.write("covid19-2023season-results.csv", full_results)

scripts/reports/comparison-notebook.Rmd

@@ -177,21 +177,11 @@ subtitle <- sprintf(
   format(max(forecast_dates), "%B %d, %Y")
 )
 p <- ggplot(
-  normalized_df %>% filter(forecaster != ensemble_forecaster_name),
+  normalized_df,
   aes(x = forecast_date, y = !!sym(var))
 ) +
   geom_line(aes(color = forecaster, group = forecaster)) +
   geom_point(aes(color = forecaster, group = forecaster)) +
-  geom_line(
-    data = normalized_df %>% filter(forecaster == ensemble_forecaster_name),
-    aes(x = forecast_date, y = !!sym(var)),
-    color = "black", linetype = 2
-  ) +
-  geom_point(
-    data = normalized_df %>% filter(forecaster == ensemble_forecaster_name),
-    aes(x = forecast_date, y = !!sym(var)),
-    color = "black", shape = 21, fill = "white"
-  ) +
   geom_hline(yintercept = 1, linetype = 1, color = "black") +
   facet_grid(rows = vars(ahead)) +
   facet_wrap(~ahead, nrow = 4, labeller = labeller(ahead = facets.label)) +
@@ -602,7 +592,7 @@ Fan plots showing the 90% prediction intervals for the forecasts made by the CMU
 if (params$disease == "flu") {
   plot_dates <- seq.Date(as.Date("2023-10-07"), by = "4 weeks", length.out = 8)
 } else {
-  plot_dates <- seq.Date(as.Date("2023-07-03"), by = "4 weeks", length.out = 7)
+  plot_dates <- seq.Date(as.Date("2023-10-07"), by = "4 weeks", length.out = 7)
 }
 ```
 
@@ -611,6 +601,7 @@ if (params$disease == "flu") {
 ```{r}
 # We plot a subset of the dates and geos for the fan plot
 geo_vals <- c("ca", "fl", "pa", "tx", "ny")
+# geo_vals <- c("ca")
 forecast_subset <- params$forecasts %>%
   filter(
     geo_value %in% geo_vals,