Automate gdx creation for new calibration names

scripts/start/prepare.R

@@ -182,17 +182,16 @@ prepare <- function() {
   replace_in_file('core/sets.gms',content,"MODULES",comment="***")
   ### ADD MODULE INFO IN SETS  ############# END #########
 
-  # copy right gdx file to the output folder
-  gdx_name <- paste0("config/gdx-files/", cfg$gms$cm_CES_configuration, ".gdx")
-  if (!file.copy(gdx_name, file.path(cfg$results_folder, 'input.gdx'))) {
-    errmsg <- 'Could not copy gdx file:\n   '
-    if (cfg$gms$CES_parameters == 'calibrate') {
-      errmsg <- paste0(errmsg,
-        'Calibration requires a start point, so copy the gdx file with the closest configuration and paste it to:\n   ')
-    }
-    stop(errmsg, gdx_name, '\n\n')
-  } else {
-    message('Copied ', gdx_name, ' to input.gdx')
+  # Retrieve appropriate gdx file
+  gdxConfig <- paste0("config/gdx-files/", cfg$gms$cm_CES_configuration, ".gdx")
+  gdxInput <- file.path(cfg$results_folder, "input.gdx")
+  if (file.copy(gdxConfig, gdxInput)) {
+    message("Copied: ", gdxConfig, "\n    to: ", gdxInput)
+  } else  {
+    stop(ifelse (cfg$gms$CES_parameters == "calibrate",
+        "Calibration requires a starting gdx; please copy the gdx file with the closest configuration and paste it to:",
+        "Could not find gdx file:"),
+      "\n    ", gdxConfig, "\n\n")
   }
 
   # choose which conopt files to copy

scripts/utils/set-local-calibration.sh

@@ -12,7 +12,7 @@ grep -q "^$caldir/$" .gitignore || echo "$caldir/" >> .gitignore
 
 cd "$caldir"
 
-git init > /dev/null
+git init -b main > /dev/null
 cp ../scripts/utils/set-local-calibration/collect_calibration ./
 cp ../scripts/utils/set-local-calibration/gitignore .gitignore
 chmod u+x collect_calibration
@@ -24,10 +24,10 @@ cp ../scripts/utils/set-local-calibration/pre-commit .git/hooks
 cp ../scripts/utils/set-local-calibration/post-commit .git/hooks
 chmod u+x .git/hooks/pre-commit .git/hooks/post-commit
 
-cd "$OLDPWD"
-
 # create additional .Rprofile (sourced through default .Rprofile)
 echo -e "options(remind_repos = c(\n" \
 	"    getOption(\"remind_repos\"),\n" \
-	"    stats::setNames(list(x = NULL), \"$PWD/$caldir/\")))" \
-	> calibration_results/.Rprofile_calibration_results
+	"    stats::setNames(list(x = NULL), \"$PWD\")))" \
+	> .Rprofile_calibration_results
+
+cd "$OLDPWD"

start.R

@@ -123,6 +123,11 @@ startedRuns <- 0
 waitingRuns <- 0
 modeltestRunsUsed <- 0
 
+nonStoppingError <- function(...) { # error display that increments the global errorsfound counter
+  message(red, "Error", NC, ": ", ...)
+  errorsfound <<- errorsfound + 1 # operator <<- ensures that the global variable errorsfound is modified
+}
+
 # Returns TRUE if 'fullname' ends with 'extension' (eg. if "C_SSP2-Base/fulldata.gdx" ends with "fulldata.gdx")
 # AND if the file given in 'fullname' exists.
 .isFileAndAvailable <- function(fullname, extension) {
@@ -309,7 +314,7 @@ if (any(c("--reprepare", "--restart") %in% flags)) {
   }
 
   ###################### Loop over scenarios ###############################
-  
+
   # Modify and save cfg for all runs
   for (scen in rownames(scenarios)) {
 
@@ -327,8 +332,7 @@ if (any(c("--reprepare", "--restart") %in% flags)) {
       cfg$gms$optimization <- "testOneRegi"
       cfg$output           <- NA
       cfg$results_folder   <- paste0("output/", cfg$title)
-      # delete existing Results directory
-      cfg$force_replace    <- TRUE
+      cfg$force_replace    <- TRUE # delete existing Results directory
       if (testOneRegi_region != "") cfg$gms$c_testOneRegi_region <- testOneRegi_region
     }
     if ("--quick" %in% flags) {
@@ -346,6 +350,7 @@ if (any(c("--reprepare", "--restart") %in% flags)) {
                           verboseGamsCompile = ! "--gamscompile" %in% flags || "--interactive" %in% flags)
       errorsfound <- sum(errorsfound, cfg$errorsfoundInConfigureCfg)
       cfg$errorsfoundInConfigureCfg <- NULL
+
       # set optimization mode to testOneRegi, if specified as command line argument
       if (any(c("--quick", "--testOneRegi") %in% flags)) {
         cfg$description      <- paste("testOneRegi:", cfg$description)
@@ -378,6 +383,50 @@ if (any(c("--reprepare", "--restart") %in% flags)) {
     # abort on too long paths ----
     cfg$gms$cm_CES_configuration <- calculate_CES_configuration(cfg, check = TRUE)
 
+    # offer to copy existing gdx when missing required calibration gdx (interactive mode only) ----
+    if ("--interactive" %in% flags && cfg$gms$CES_parameters == "calibrate") {
+      gdxFolder <- "./config/gdx-files"
+      gdxConfig <- file.path(gdxFolder, paste0(cfg$gms$cm_CES_configuration, ".gdx"))
+      if (!file.exists(gdxConfig)) {
+        message("\nCalibration requires a starting gdx that does not exist:\n    ", gdxConfig)
+        abortText <- paste0("Please copy the gdx file with the closest configuration and paste it to:\n    ", gdxConfig, "\n")
+
+        # List available gdx files
+        gdxFiles <- list.files(gdxFolder, pattern = "\\.gdx$", full.names = TRUE)
+        if (length(gdxFiles) > 0) { # length is zero when input data has not been collected yet and copying gdx is impossible
+          # Prompt user to choose an existing gdx file
+          gdxClosest <- gdxFiles[which.min(adist(basename(gdxConfig), basename(gdxFiles)))] # existing file with the closest name
+          abortOption <- paste0(crayon::red("ABORT"), ": you will then need to copy the gdx of your choice manually")
+          gdxFiles <- c(abortOption, gdxFiles)
+          gdxSelection <- gdxFiles[gms::chooseFromList(
+            ifelse(gdxFiles == gdxClosest, crayon::cyan(gdxFiles), gdxFiles),
+            type = "an existing gdx file that you would like to use",
+            userinfo = paste0("Leave empty to select existing gdx with ", crayon::cyan("most similar name")),
+            returnBoolean = TRUE,
+            multiple = FALSE
+          )]
+
+          if (length(gdxSelection) == 0) { gdxSelection <- gdxClosest } # default option
+          if (gdxSelection == abortOption || !file.copy(gdxSelection, gdxConfig)) { # abort option or copy failure
+            nonStoppingError(abortText)
+          } else {
+            message("Copied: ", gdxSelection, "\n    to: ", gdxConfig, "\n")
+
+            # Add the .gdx and .inc to list of possible names
+            addLine <- function(line, path = "files") {
+              if (!file.exists(path)) message(path, " does not exist, you may have to manually add ", line)
+              else if (!(line %in% readLines(path))) {
+                write(line, path, append = TRUE)
+                message("Added in ", path, " the line ", line)
+              }
+            }
+            addLine(paste0(cfg$gms$cm_CES_configuration, ".gdx"), path = file.path(gdxFolder, "files"))
+            addLine(paste0(cfg$gms$cm_CES_configuration, ".inc"), path = file.path("./modules/29_CES_parameters/load/input", "files"))
+          }
+        }
+      }
+    }
+
     # =================== MAgPIE coupling ===================
 
     if (exists("scenarios_magpie")) {
@@ -413,8 +462,7 @@ if (any(c("--reprepare", "--restart") %in% flags)) {
           cfg$files2export$start["input.gdx"] <- scenarios_magpie[scen, "continueFromHere"]
           message("   Continuing MAgPIE coupling from REMIND gdx ", scenarios_magpie[scen, "continueFromHere"])
         } else {
-          message(red, "Error", NC, ": Could not find what is given in 'scenarios_magpie[scen, continueFromHere]': ", scenarios_magpie[scen, "continueFromHere"])
-          errorsfound <- errorsfound + 1
+          nonStoppingError("Could not find what is given in 'scenarios_magpie[scen, continueFromHere]': ", scenarios_magpie[scen, "continueFromHere"])
         }
       }
 
@@ -481,14 +529,12 @@ if (any(c("--reprepare", "--restart") %in% flags)) {
               # if no real file is given but a reference to another scenario (that has to run first) create path to the reference scenario
               #ghgprice_remindrun <- paste0(prefix_runname, scenarios_magpie[scen, "path_mif_ghgprice_land"], "-rem-", i)
               #path_mif_ghgprice_land <- file.path(path_remind, "output", ghgprice_remindrun, paste0("REMIND_generic_", ghgprice_remindrun, ".mif"))
-              message(red, "Error", NC, ": path_mif_ghgprice_land must be a path to an existing file and cannot reference another scenario by name currently: ",
+              nonStoppingError("path_mif_ghgprice_land must be a path to an existing file and cannot reference another scenario by name currently: ",
                       scenarios_magpie[scen, "path_mif_ghgprice_land"])
-              errorsfound <- errorsfound + 1
               path_mif_ghgprice_land <- FALSE
           } else {
-            message(red, "Error", NC, ": path_mif_ghgprice_land neither an existing file nor a scenario that will be started: ",
+            nonStoppingError("path_mif_ghgprice_land is neither an existing file nor a scenario that will be started: ",
                     scenarios_magpie[scen, "path_mif_ghgprice_land"])
-            errorsfound <- errorsfound + 1
             path_mif_ghgprice_land <- FALSE
           }
           cfg_mag$path_to_report_ghgprices <- path_mif_ghgprice_land
@@ -530,6 +576,16 @@ if (any(c("--reprepare", "--restart") %in% flags)) {
       errorsfound <- errorsfound + ! gcresult
     } else if (start_now) {
       if (errorsfound == 0) {
+        caldir <- "calibration_results/"
+        if (cfg$gms$CES_parameters == "calibrate" && !dir.exists(caldir)) {
+          if (0 == system("./scripts/utils/set-local-calibration.sh")) {
+            message("   Folder ", caldir, " has been automatically set up.")
+            cfg$repositories <- append(cfg$repositories, setNames(list(NULL), normalizePath(caldir)))
+            source(file.path(caldir, ".Rprofile_calibration_results"))
+          } else {
+            warning("   Could not set up ", caldir, " automatically. Please run 'make set-local-calibration' manually.")
+          }
+        }
         submit(cfg)
       } else {
         message("   Not started, as errors were found.")

tutorials/12_Calibrating_CES_Parameters.md

@@ -1,6 +1,7 @@
 # Calibrating CES Parameters
 
-## CES Production Function Basics
+## Theory
+### CES Production Function Basics
 
 REMIND uses a nested CES production function of the form
 
@@ -27,7 +28,9 @@ Therefore,
 > **Anytime either the REMIND model or the input data change in a way that affects
 the results of a baseline scenario, that baseline scenario needs to be calibrated.**
 
-## Iterative Calibration
+
+
+### Iterative Calibration
 
 As it is not possible to calculate $n$ parameters $\alpha_i$ from a single
 equation, we use an iterative approach.  The
@@ -61,37 +64,34 @@ $\pi_i^{(j)} = \alpha_i^{(j)} {V_i^{(j)}}^{\rho_o - 1} {V_o^{(j)}}^{1 - \rho_o}$
 $\alpha_i^{(j+1)} = \pi_i^{(j)} \left(\frac{V_i^\ast}{V_o^\ast}\right)^{1 - \rho_o}$
 
 
-## Requirements
 
-For the calibration process to work, we need 
+## Practice
+### Requirements and settings
 
-1. trajectories for all primary production factors (`ppf`, final energy and
-   energy service demands, labour, capital) and the output (GDP), and
-2. the previous iterations' `ppf` prices.
+For the calibration process to work, we need both:
 
-Trajectories under (1) come from the input files `./core/input/f_gdp.cs3r`,
-`./core/input/f_pop.cs3r`,
-`./modules/29_CES_parameters/calibrate/input/f29_capitalQuantity.cs4r`, and
-`./core/input/f_fedemand.cs4r` which are generated automatically as part of the
-input data generation and always present.
+1. Trajectories for all primary production factors (`ppf`: final energy and energy service demands,
+   labour, capital) and the output (GDP). They come from input files that are automatically
+   generated as part of the input data generation:
+  - `./core/input/f_gdp.cs3r`
+  - `./core/input/f_pop.cs3r`
+  - `./modules/29_CES_parameters/calibrate/input/f29_capitalQuantity.cs4r`
+  - `./core/input/f_fedemand.cs4r`
 
-Prices under (2) are calculated using the `input.gdx` provided to the
-calibration run.  User intervention is only required when prices cannot be
-derived from the `.gdx` file in case of a change in the CES structure:
-when nodes have been added or removed from the CES tree 
-(technically when the set `cesOut2cesIn` differs between `input.gdx` and the
-current calibration run), or seldom in cases of convergence problems.
+2. Prices of `ppf` at the previous iteration. They are calculated using the `input.gdx` provided to
+   the calibration run. User intervention is only required when prices cannot be derived from the `.gdx`
+   file in case of a change in the CES structure: when nodes have been added or removed from the CES tree 
+   (technically when the set `cesOut2cesIn` differs between `input.gdx` and the current calibration run),
+   or seldom in cases of convergence problems.
 
 
-## Settings
-
-To set up a CES calibration run, simply set module 29 `CES_parameters` to the
-`calibration` realisation.  All data relevant to the calibration is configured
-according to the selected scenario configuration.  Keep them identical to the
-baseline scenario you want to calibrate.
-If your calibration depends on new input data, you need to update the configuration.
-Use the command `lastrev` to find the latest input data revisions on the cluster.
-If your choice is _rev1.23abc_, update the following line of the configuration file
+To set up a CES calibration run, simply set module 29 `CES_parameters` to the `calibrate` realisation.
+All data relevant to the calibration is configured according to the selected scenario configuration;
+keep them identical to the baseline scenario you want to calibrate.
+If your calibration depends on new input data (for instance an update in `mrremind`),
+you need to update the configuration.
+Use the command `lastrev` to find the latest input data revisions on the cluster;
+if your choice is _rev1.23abc_, update the following line of the configuration file
 `./config/default.cfg` (without _rev_ and always with quotation marks):
 
 ```R
@@ -122,46 +122,50 @@ The calibration can further be adjusted using the following switches:
   Can help finding a suitable value for `cm_CES_calibration_default_price`.
 
 
-## Results
 
-The CES calibration outputs a `.gdx` file and an `.inc` file with all
-the CES parameters.  Their long name, for instance
-`indu_subsectors-buil_simple-tran_edge_esm-GDPpop_SSP2-En_SSP2-Kap_debt_limit-Reg_62eff8f7`
-indicates the CES configuration, the GDP/population scenarios, the capital market
-module realisation and the [region configuration](17_Regions.md)
-(_62eff8f7_ for H12, _2b1450bc_ for EU21).
-You don't need to change these names, they are matched automatically using the
-switch `cm_CES_configuration`.  The parameter files also include a counter for
+### Results
+
+The CES calibration outputs a `.gdx` file and an `.inc` file containing all the CES parameters.
+The long name of these files indicates the CES configuration, the GDP/population scenarios,
+the capital market module realisation, and the [region configuration](17_Regions.md)
+(_62eff8f7_ for H12, _2b1450bc_ for EU21), for instance:
+`indu_subsectors-buil_simple-tran_edge_esm-GDPpop_SSP2-En_SSP2-Kap_debt_limit-Reg_62eff8f7`.
+Do not change these names, as they are matched automatically with the
+switch `cm_CES_configuration`. The parameter files also include a counter for
 the calibration iteration they resulted from (e.g. `_ITERATION_10.inc`).
 
-Calibration results can be included the PIK calibration repository (for use by
-all REMIND users), or used in a local directory (e.g. for project work).
+Calibration results can be included in the PIK calibration repository (for use by
+all REMIND users) or used in a local directory (for project work):
 
 1. Prepare the calibration directory:
   - To include calibration results in the PIK calibration repository, navigate to
-  `/p/projects/remind/inputdata/CESparametersAndGDX/`.
-  - For use in a local directory, go to your local REMIND folder and type
-    `make set-local-calibration`. Navigate to the newly created folder `calibration_results/`.
-2. Use the `collect_calibration` script with one ore more paths to the completed
-   calibration run directories as a parameter, for instance:
-   ```sh
-   ./collect_calibration /p/tmp/username/Remind/output/SSP2-calibrate_2024-12-31_23.59.59/
-   ```
-    Note that an absolute or relative path may be used. 
-    The script copies the necessary `.inc` and `.gdx` files to the repository
-    (adjusting the file names as needed), stages and commits them; you may review and
-    modify the commit message before committing (or abort with `:cq` in vim).
-    The script then generates a `.tgz` archive, which is what REMIND will be looking for
-    in order to run.  Finally it displays the commit hash and offers to include it as the
-    `CESandGDXrevision` in the REMIND configuration.
-3. If the specific calibration settings (`cm_CES_configuration`) have never been 
-calibrated and used in REMIND before, add the name of the `.gdx` file to `./config/gdx-files/files`
-and add the name of the  `.inc` file to `./modules/29_CES_parameters/load/input/files`, so that the
-new calibration results are copied into these directories during run setup.
-
-
-
-## Diagnostic and validity 
+    `/p/projects/remind/inputdata/CESparametersAndGDX/`.
+  - For use in a local directory, navigate to the folder `calibration_results/`. This folder
+    is automatically created during CES calibration runs, or you can set it up manually
+    with `make set-local-calibration`.
+
+2. Use the `collect_calibration` script with one or more paths to the completed
+  calibration run directories as a parameter. For instance:
+  ```sh
+  ./collect_calibration /p/tmp/username/Remind/output/SSP2-calibrate_2024-12-31_23.59.59/
+  ```
+  Note that both absolute and relative paths may be used. 
+  The script copies the necessary `.inc` and `.gdx` files to `calibration_results/`,
+  adjusting the file names as needed, stages and commits them (you may review and
+  modify the commit message, then commit with `:wq` or abort with `:cq`).
+  The script then displays the commit hash and offers to include it as the
+  `CESandGDXrevision` in the REMIND configuration, so that the next REMIND runs are 
+  able to find the `.tgz` archive containing the calibration files.
+
+3. New calibration files are automatically copied from the `.tgz` archive to the relevant directories
+   during the run setup. This requires the calibration name `cm_CES_configuration` to be listed in
+   the following files (which happens automatically if you ran the calibration yourself):
+  - `./config/gdx-files/files` (for the `.gdx` file)
+  - `./modules/29_CES_parameters/load/input/files` (for the `.inc` file)
+
+
+
+### Diagnostic and validity 
 
 To diagnose the outputs, you may use the `full.log` and `full.lst` files for each
 calibration iteration (`full_01.log` …), the file `CES_calibration.csv`