diff --git a/scripts/start/prepare.R b/scripts/start/prepare.R index 3ef6df060a..559d91fdd3 100644 --- a/scripts/start/prepare.R +++ b/scripts/start/prepare.R @@ -182,17 +182,16 @@ prepare <- function() { replace_in_file('core/sets.gms',content,"MODULES",comment="***") ### ADD MODULE INFO IN SETS ############# END ######### - # copy right gdx file to the output folder - gdx_name <- paste0("config/gdx-files/", cfg$gms$cm_CES_configuration, ".gdx") - if (!file.copy(gdx_name, file.path(cfg$results_folder, 'input.gdx'))) { - errmsg <- 'Could not copy gdx file:\n ' - if (cfg$gms$CES_parameters == 'calibrate') { - errmsg <- paste0(errmsg, - 'Calibration requires a start point, so copy the gdx file with the closest configuration and paste it to:\n ') - } - stop(errmsg, gdx_name, '\n\n') - } else { - message('Copied ', gdx_name, ' to input.gdx') + # Retrieve appropriate gdx file + gdxConfig <- paste0("config/gdx-files/", cfg$gms$cm_CES_configuration, ".gdx") + gdxInput <- file.path(cfg$results_folder, "input.gdx") + if (file.copy(gdxConfig, gdxInput)) { + message("Copied: ", gdxConfig, "\n to: ", gdxInput) + } else { + stop(ifelse (cfg$gms$CES_parameters == "calibrate", + "Calibration requires a starting gdx; please copy the gdx file with the closest configuration and paste it to:", + "Could not find gdx file:"), + "\n ", gdxConfig, "\n\n") } # choose which conopt files to copy diff --git a/scripts/utils/set-local-calibration.sh b/scripts/utils/set-local-calibration.sh index 051bc91199..a5c256c760 100755 --- a/scripts/utils/set-local-calibration.sh +++ b/scripts/utils/set-local-calibration.sh @@ -12,7 +12,7 @@ grep -q "^$caldir/$" .gitignore || echo "$caldir/" >> .gitignore cd "$caldir" -git init > /dev/null +git init -b main > /dev/null cp ../scripts/utils/set-local-calibration/collect_calibration ./ cp ../scripts/utils/set-local-calibration/gitignore .gitignore chmod u+x collect_calibration @@ -24,10 +24,10 @@ cp ../scripts/utils/set-local-calibration/pre-commit .git/hooks cp ../scripts/utils/set-local-calibration/post-commit .git/hooks chmod u+x .git/hooks/pre-commit .git/hooks/post-commit -cd "$OLDPWD" - # create additional .Rprofile (sourced through default .Rprofile) echo -e "options(remind_repos = c(\n" \ " getOption(\"remind_repos\"),\n" \ - " stats::setNames(list(x = NULL), \"$PWD/$caldir/\")))" \ - > calibration_results/.Rprofile_calibration_results + " stats::setNames(list(x = NULL), \"$PWD\")))" \ + > .Rprofile_calibration_results + +cd "$OLDPWD" diff --git a/start.R b/start.R index 24a7c42517..ff2eb5416d 100755 --- a/start.R +++ b/start.R @@ -123,6 +123,11 @@ startedRuns <- 0 waitingRuns <- 0 modeltestRunsUsed <- 0 +nonStoppingError <- function(...) { # error display that increments the global errorsfound counter + message(red, "Error", NC, ": ", ...) + errorsfound <<- errorsfound + 1 # operator <<- ensures that the global variable errorsfound is modified +} + # Returns TRUE if 'fullname' ends with 'extension' (eg. if "C_SSP2-Base/fulldata.gdx" ends with "fulldata.gdx") # AND if the file given in 'fullname' exists. .isFileAndAvailable <- function(fullname, extension) { @@ -309,7 +314,7 @@ if (any(c("--reprepare", "--restart") %in% flags)) { } ###################### Loop over scenarios ############################### - + # Modify and save cfg for all runs for (scen in rownames(scenarios)) { @@ -327,8 +332,7 @@ if (any(c("--reprepare", "--restart") %in% flags)) { cfg$gms$optimization <- "testOneRegi" cfg$output <- NA cfg$results_folder <- paste0("output/", cfg$title) - # delete existing Results directory - cfg$force_replace <- TRUE + cfg$force_replace <- TRUE # delete existing Results directory if (testOneRegi_region != "") cfg$gms$c_testOneRegi_region <- testOneRegi_region } if ("--quick" %in% flags) { @@ -346,6 +350,7 @@ if (any(c("--reprepare", "--restart") %in% flags)) { verboseGamsCompile = ! "--gamscompile" %in% flags || "--interactive" %in% flags) errorsfound <- sum(errorsfound, cfg$errorsfoundInConfigureCfg) cfg$errorsfoundInConfigureCfg <- NULL + # set optimization mode to testOneRegi, if specified as command line argument if (any(c("--quick", "--testOneRegi") %in% flags)) { cfg$description <- paste("testOneRegi:", cfg$description) @@ -378,6 +383,50 @@ if (any(c("--reprepare", "--restart") %in% flags)) { # abort on too long paths ---- cfg$gms$cm_CES_configuration <- calculate_CES_configuration(cfg, check = TRUE) + # offer to copy existing gdx when missing required calibration gdx (interactive mode only) ---- + if ("--interactive" %in% flags && cfg$gms$CES_parameters == "calibrate") { + gdxFolder <- "./config/gdx-files" + gdxConfig <- file.path(gdxFolder, paste0(cfg$gms$cm_CES_configuration, ".gdx")) + if (!file.exists(gdxConfig)) { + message("\nCalibration requires a starting gdx that does not exist:\n ", gdxConfig) + abortText <- paste0("Please copy the gdx file with the closest configuration and paste it to:\n ", gdxConfig, "\n") + + # List available gdx files + gdxFiles <- list.files(gdxFolder, pattern = "\\.gdx$", full.names = TRUE) + if (length(gdxFiles) > 0) { # length is zero when input data has not been collected yet and copying gdx is impossible + # Prompt user to choose an existing gdx file + gdxClosest <- gdxFiles[which.min(adist(basename(gdxConfig), basename(gdxFiles)))] # existing file with the closest name + abortOption <- paste0(crayon::red("ABORT"), ": you will then need to copy the gdx of your choice manually") + gdxFiles <- c(abortOption, gdxFiles) + gdxSelection <- gdxFiles[gms::chooseFromList( + ifelse(gdxFiles == gdxClosest, crayon::cyan(gdxFiles), gdxFiles), + type = "an existing gdx file that you would like to use", + userinfo = paste0("Leave empty to select existing gdx with ", crayon::cyan("most similar name")), + returnBoolean = TRUE, + multiple = FALSE + )] + + if (length(gdxSelection) == 0) { gdxSelection <- gdxClosest } # default option + if (gdxSelection == abortOption || !file.copy(gdxSelection, gdxConfig)) { # abort option or copy failure + nonStoppingError(abortText) + } else { + message("Copied: ", gdxSelection, "\n to: ", gdxConfig, "\n") + + # Add the .gdx and .inc to list of possible names + addLine <- function(line, path = "files") { + if (!file.exists(path)) message(path, " does not exist, you may have to manually add ", line) + else if (!(line %in% readLines(path))) { + write(line, path, append = TRUE) + message("Added in ", path, " the line ", line) + } + } + addLine(paste0(cfg$gms$cm_CES_configuration, ".gdx"), path = file.path(gdxFolder, "files")) + addLine(paste0(cfg$gms$cm_CES_configuration, ".inc"), path = file.path("./modules/29_CES_parameters/load/input", "files")) + } + } + } + } + # =================== MAgPIE coupling =================== if (exists("scenarios_magpie")) { @@ -413,8 +462,7 @@ if (any(c("--reprepare", "--restart") %in% flags)) { cfg$files2export$start["input.gdx"] <- scenarios_magpie[scen, "continueFromHere"] message(" Continuing MAgPIE coupling from REMIND gdx ", scenarios_magpie[scen, "continueFromHere"]) } else { - message(red, "Error", NC, ": Could not find what is given in 'scenarios_magpie[scen, continueFromHere]': ", scenarios_magpie[scen, "continueFromHere"]) - errorsfound <- errorsfound + 1 + nonStoppingError("Could not find what is given in 'scenarios_magpie[scen, continueFromHere]': ", scenarios_magpie[scen, "continueFromHere"]) } } @@ -481,14 +529,12 @@ if (any(c("--reprepare", "--restart") %in% flags)) { # if no real file is given but a reference to another scenario (that has to run first) create path to the reference scenario #ghgprice_remindrun <- paste0(prefix_runname, scenarios_magpie[scen, "path_mif_ghgprice_land"], "-rem-", i) #path_mif_ghgprice_land <- file.path(path_remind, "output", ghgprice_remindrun, paste0("REMIND_generic_", ghgprice_remindrun, ".mif")) - message(red, "Error", NC, ": path_mif_ghgprice_land must be a path to an existing file and cannot reference another scenario by name currently: ", + nonStoppingError("path_mif_ghgprice_land must be a path to an existing file and cannot reference another scenario by name currently: ", scenarios_magpie[scen, "path_mif_ghgprice_land"]) - errorsfound <- errorsfound + 1 path_mif_ghgprice_land <- FALSE } else { - message(red, "Error", NC, ": path_mif_ghgprice_land neither an existing file nor a scenario that will be started: ", + nonStoppingError("path_mif_ghgprice_land is neither an existing file nor a scenario that will be started: ", scenarios_magpie[scen, "path_mif_ghgprice_land"]) - errorsfound <- errorsfound + 1 path_mif_ghgprice_land <- FALSE } cfg_mag$path_to_report_ghgprices <- path_mif_ghgprice_land @@ -530,6 +576,16 @@ if (any(c("--reprepare", "--restart") %in% flags)) { errorsfound <- errorsfound + ! gcresult } else if (start_now) { if (errorsfound == 0) { + caldir <- "calibration_results/" + if (cfg$gms$CES_parameters == "calibrate" && !dir.exists(caldir)) { + if (0 == system("./scripts/utils/set-local-calibration.sh")) { + message(" Folder ", caldir, " has been automatically set up.") + cfg$repositories <- append(cfg$repositories, setNames(list(NULL), normalizePath(caldir))) + source(file.path(caldir, ".Rprofile_calibration_results")) + } else { + warning(" Could not set up ", caldir, " automatically. Please run 'make set-local-calibration' manually.") + } + } submit(cfg) } else { message(" Not started, as errors were found.") diff --git a/tutorials/12_Calibrating_CES_Parameters.md b/tutorials/12_Calibrating_CES_Parameters.md index 124462ea1f..6b1da20070 100644 --- a/tutorials/12_Calibrating_CES_Parameters.md +++ b/tutorials/12_Calibrating_CES_Parameters.md @@ -1,6 +1,7 @@ # Calibrating CES Parameters -## CES Production Function Basics +## Theory +### CES Production Function Basics REMIND uses a nested CES production function of the form @@ -27,7 +28,9 @@ Therefore, > **Anytime either the REMIND model or the input data change in a way that affects the results of a baseline scenario, that baseline scenario needs to be calibrated.** -## Iterative Calibration + + +### Iterative Calibration As it is not possible to calculate $n$ parameters $\alpha_i$ from a single equation, we use an iterative approach. The @@ -61,37 +64,34 @@ $\pi_i^{(j)} = \alpha_i^{(j)} {V_i^{(j)}}^{\rho_o - 1} {V_o^{(j)}}^{1 - \rho_o}$ $\alpha_i^{(j+1)} = \pi_i^{(j)} \left(\frac{V_i^\ast}{V_o^\ast}\right)^{1 - \rho_o}$ -## Requirements -For the calibration process to work, we need +## Practice +### Requirements and settings -1. trajectories for all primary production factors (`ppf`, final energy and - energy service demands, labour, capital) and the output (GDP), and -2. the previous iterations' `ppf` prices. +For the calibration process to work, we need both: -Trajectories under (1) come from the input files `./core/input/f_gdp.cs3r`, -`./core/input/f_pop.cs3r`, -`./modules/29_CES_parameters/calibrate/input/f29_capitalQuantity.cs4r`, and -`./core/input/f_fedemand.cs4r` which are generated automatically as part of the -input data generation and always present. +1. Trajectories for all primary production factors (`ppf`: final energy and energy service demands, + labour, capital) and the output (GDP). They come from input files that are automatically + generated as part of the input data generation: + - `./core/input/f_gdp.cs3r` + - `./core/input/f_pop.cs3r` + - `./modules/29_CES_parameters/calibrate/input/f29_capitalQuantity.cs4r` + - `./core/input/f_fedemand.cs4r` -Prices under (2) are calculated using the `input.gdx` provided to the -calibration run. User intervention is only required when prices cannot be -derived from the `.gdx` file in case of a change in the CES structure: -when nodes have been added or removed from the CES tree -(technically when the set `cesOut2cesIn` differs between `input.gdx` and the -current calibration run), or seldom in cases of convergence problems. +2. Prices of `ppf` at the previous iteration. They are calculated using the `input.gdx` provided to + the calibration run. User intervention is only required when prices cannot be derived from the `.gdx` + file in case of a change in the CES structure: when nodes have been added or removed from the CES tree + (technically when the set `cesOut2cesIn` differs between `input.gdx` and the current calibration run), + or seldom in cases of convergence problems. -## Settings - -To set up a CES calibration run, simply set module 29 `CES_parameters` to the -`calibration` realisation. All data relevant to the calibration is configured -according to the selected scenario configuration. Keep them identical to the -baseline scenario you want to calibrate. -If your calibration depends on new input data, you need to update the configuration. -Use the command `lastrev` to find the latest input data revisions on the cluster. -If your choice is _rev1.23abc_, update the following line of the configuration file +To set up a CES calibration run, simply set module 29 `CES_parameters` to the `calibrate` realisation. +All data relevant to the calibration is configured according to the selected scenario configuration; +keep them identical to the baseline scenario you want to calibrate. +If your calibration depends on new input data (for instance an update in `mrremind`), +you need to update the configuration. +Use the command `lastrev` to find the latest input data revisions on the cluster; +if your choice is _rev1.23abc_, update the following line of the configuration file `./config/default.cfg` (without _rev_ and always with quotation marks): ```R @@ -122,46 +122,50 @@ The calibration can further be adjusted using the following switches: Can help finding a suitable value for `cm_CES_calibration_default_price`. -## Results -The CES calibration outputs a `.gdx` file and an `.inc` file with all -the CES parameters. Their long name, for instance -`indu_subsectors-buil_simple-tran_edge_esm-GDPpop_SSP2-En_SSP2-Kap_debt_limit-Reg_62eff8f7` -indicates the CES configuration, the GDP/population scenarios, the capital market -module realisation and the [region configuration](17_Regions.md) -(_62eff8f7_ for H12, _2b1450bc_ for EU21). -You don't need to change these names, they are matched automatically using the -switch `cm_CES_configuration`. The parameter files also include a counter for +### Results + +The CES calibration outputs a `.gdx` file and an `.inc` file containing all the CES parameters. +The long name of these files indicates the CES configuration, the GDP/population scenarios, +the capital market module realisation, and the [region configuration](17_Regions.md) +(_62eff8f7_ for H12, _2b1450bc_ for EU21), for instance: +`indu_subsectors-buil_simple-tran_edge_esm-GDPpop_SSP2-En_SSP2-Kap_debt_limit-Reg_62eff8f7`. +Do not change these names, as they are matched automatically with the +switch `cm_CES_configuration`. The parameter files also include a counter for the calibration iteration they resulted from (e.g. `_ITERATION_10.inc`). -Calibration results can be included the PIK calibration repository (for use by -all REMIND users), or used in a local directory (e.g. for project work). +Calibration results can be included in the PIK calibration repository (for use by +all REMIND users) or used in a local directory (for project work): 1. Prepare the calibration directory: - To include calibration results in the PIK calibration repository, navigate to - `/p/projects/remind/inputdata/CESparametersAndGDX/`. - - For use in a local directory, go to your local REMIND folder and type - `make set-local-calibration`. Navigate to the newly created folder `calibration_results/`. -2. Use the `collect_calibration` script with one ore more paths to the completed - calibration run directories as a parameter, for instance: - ```sh - ./collect_calibration /p/tmp/username/Remind/output/SSP2-calibrate_2024-12-31_23.59.59/ - ``` - Note that an absolute or relative path may be used. - The script copies the necessary `.inc` and `.gdx` files to the repository - (adjusting the file names as needed), stages and commits them; you may review and - modify the commit message before committing (or abort with `:cq` in vim). - The script then generates a `.tgz` archive, which is what REMIND will be looking for - in order to run. Finally it displays the commit hash and offers to include it as the - `CESandGDXrevision` in the REMIND configuration. -3. If the specific calibration settings (`cm_CES_configuration`) have never been -calibrated and used in REMIND before, add the name of the `.gdx` file to `./config/gdx-files/files` -and add the name of the `.inc` file to `./modules/29_CES_parameters/load/input/files`, so that the -new calibration results are copied into these directories during run setup. - - - -## Diagnostic and validity + `/p/projects/remind/inputdata/CESparametersAndGDX/`. + - For use in a local directory, navigate to the folder `calibration_results/`. This folder + is automatically created during CES calibration runs, or you can set it up manually + with `make set-local-calibration`. + +2. Use the `collect_calibration` script with one or more paths to the completed + calibration run directories as a parameter. For instance: + ```sh + ./collect_calibration /p/tmp/username/Remind/output/SSP2-calibrate_2024-12-31_23.59.59/ + ``` + Note that both absolute and relative paths may be used. + The script copies the necessary `.inc` and `.gdx` files to `calibration_results/`, + adjusting the file names as needed, stages and commits them (you may review and + modify the commit message, then commit with `:wq` or abort with `:cq`). + The script then displays the commit hash and offers to include it as the + `CESandGDXrevision` in the REMIND configuration, so that the next REMIND runs are + able to find the `.tgz` archive containing the calibration files. + +3. New calibration files are automatically copied from the `.tgz` archive to the relevant directories + during the run setup. This requires the calibration name `cm_CES_configuration` to be listed in + the following files (which happens automatically if you ran the calibration yourself): + - `./config/gdx-files/files` (for the `.gdx` file) + - `./modules/29_CES_parameters/load/input/files` (for the `.inc` file) + + + +### Diagnostic and validity To diagnose the outputs, you may use the `full.log` and `full.lst` files for each calibration iteration (`full_01.log` …), the file `CES_calibration.csv`