stop if any input files fail on load/processing

Author: nmdefries

facebook/delphiFacebook/R/responses.R

@@ -18,13 +18,29 @@
 #' @export
 load_responses_all <- function(params, contingency_run = FALSE) {
   msg_plain(paste0("Loading ", length(params$input), " CSVs"))
-  
-  map_fn <- if (params$parallel) { mclapply } else { lapply }
+
+  # To prevent errors from one file from "contaminating" output for other files
+  # that happen to be in the same `mclapply` group, turn `mc.preschedule` off.
+  # This makes debugging problematic input files easier.
+  # See https://stackoverflow.com/questions/18330274/an-error-in-one-job-contaminates-others-with-mclapply
+  # for details.
+  mclapply_no_preschedule <- function(...) {
+    mclapply(..., mc.preschedule = FALSE)
+  }
+
+  map_fn <- if (params$parallel) { mclapply_no_preschedule } else { lapply }
   input_data <- map_fn(seq_along(params$input), function(i) {
     load_response_one(params$input[i], params, contingency_run)
   })
 
   msg_plain(paste0("Finished loading CSVs"))
+  
+  which_errors <- unlist(lapply(input_data, inherits, "try-error"))
+  if (any( which_errors )) {
+    errored_filenames <- paste(params$input[which_errors], collapse=", ")
+    stop(paste("ingestion and field creation failed for input data file(s)", errored_filenames))
+  }
+  
   input_data <- bind_rows(input_data)
   msg_plain(paste0("Finished combining CSVs"))
   return(input_data)