Merge pull request #1353 from cmu-delphi/report-file-names-load-error

[survey] Stop with useful message if any input files fail on load/processing

Author: krivard

facebook/delphiFacebook/R/responses.R

@@ -18,13 +18,25 @@
 #' @export
 load_responses_all <- function(params, contingency_run = FALSE) {
   msg_plain(paste0("Loading ", length(params$input), " CSVs"))
-  
+
   map_fn <- if (params$parallel) { mclapply } else { lapply }
   input_data <- map_fn(seq_along(params$input), function(i) {
     load_response_one(params$input[i], params, contingency_run)
   })
 
   msg_plain(paste0("Finished loading CSVs"))
+  
+  which_errors <- unlist(lapply(input_data, inherits, "try-error"))
+  if (any( which_errors )) {
+    errored_filenames <- paste(params$input[which_errors], collapse=", ")
+    stop(
+      "ingestion and field creation failed for at least one of input data file(s) ",
+      errored_filenames,
+      " with error(s)\n",
+      unique(input_data[which_errors])
+    )
+  }
+  
   input_data <- bind_rows(input_data)
   msg_plain(paste0("Finished combining CSVs"))
   return(input_data)

facebook/delphiFacebook/integration-tests/testthat/test-integration.R

@@ -385,3 +385,10 @@ test_that("testing national aggregation", {
   }
 
 })
+
+test_that("testing load_responses behavior for missing input", {
+  params <- relativize_params(read_params(test_path("params-test.json")))
+  params$input <- c(params$input, "file-does-not-exist.csv")
+  params$parallel <- TRUE
+  expect_error(load_responses_all(params), regexp="ingestion and field creation failed")
+})