From 447bb2d0da7f6b41a149344eb055ed16ca188663 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Tue, 24 Jan 2023 21:59:41 +0100
Subject: [PATCH 01/58] added a function to obtain full path to a tool

---
 utils.sh | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/utils.sh b/utils.sh
index 5d8455bb68..cd7f758a0d 100644
--- a/utils.sh
+++ b/utils.sh
@@ -31,3 +31,35 @@ function check_exit_code {
         fatal_error "${fail_msg}"
     fi
 }
+
+function get_path_for_tool {
+    tool_name=$1
+    tool_envvar_name=$2
+
+    which_out=$(which ${tool_name} 2>&1)
+    exit_code=$?
+    if [[ ${exit_code} -eq 0 ]]; then
+        echo "INFO: found tool ${tool_name} in PATH (${which_out})" >&2
+        echo "${which_out}"
+        return 0
+    fi
+    if [[ -z "${tool_envvar_name}" ]]; then
+        msg="no env var holding the full path to tool '${tool_name}' provided"
+        echo "${msg}" >&2
+        return 1
+    else
+        tool_envvar_value=${!tool_envvar_name}
+        if [[ -x "${tool_envvar_value}" ]]; then
+            msg="INFO: found tool ${tool_envvar_value} via env var ${tool_envvar_name}"
+            echo "${msg}" >&2
+            echo "${tool_envvar_value}"
+            return 0
+        else
+            msg="ERROR: tool '${tool_name}' not in PATH\n"
+            msg+="ERROR: tool '${tool_envvar_value}' via '${tool_envvar_name}' not in PATH"
+            echo "${msg}" >&2
+            echo ""
+            return 2
+        fi
+    fi
+}

From e2c24cf757b912aef167364aa34d0abf723b7b03 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Tue, 24 Jan 2023 22:01:13 +0100
Subject: [PATCH 02/58] partial bot/build.sh for eessi_container.sh

---
 bot/build.sh | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 78 insertions(+)
 create mode 100755 bot/build.sh

diff --git a/bot/build.sh b/bot/build.sh
new file mode 100755
index 0000000000..90753db8a8
--- /dev/null
+++ b/bot/build.sh
@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+#
+# script to build the EESSI software layer. Intended use is that it is called
+# by a (batch) job running on a compute node.
+#
+# This script is part of the EESSI software layer, see
+# https://github.com/EESSI/software-layer.git
+#
+# author: Thomas Roeblitz (@trz42)
+#
+# license: GPLv2
+#
+
+# ASSUMPTIONs:
+#  - working directory has been prepared by the bot with a checkout of a
+#    pull request (OR by some other means)
+#  - the working directory contains a directory 'cfg' where the main config
+#    file 'job.cfg' has been deposited
+#  - the directory may contain any additional files references in job.cfg
+#  - the tool 'yq' for working with json files is available via the PATH or
+#    the environment variable BOT_YQ (see https://github.com/mikefarah/yq)
+
+# defaults
+export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}"
+
+# source utils.sh
+source utils.sh
+
+# check setup / define key variables
+# get path for 'yq' (if not found, an empty string is returned)
+YQ=$(get_path_for_tool "yq" "BOT_YQ")
+exit_code=$?
+if [[ ${exit_code} -ne 0 ]]; then
+    fatal_error "could not find path to 'yq'; exiting"
+else
+    echo_green "found yq (${YQ})"
+fi
+
+# check if './cfg/job.cfg' exists
+if [[ ! -r "${JOB_CFG_FILE}" ]]; then
+    fatal_error "job config file (JOB_CFG_FILE=${JOB_CFG_FILE}) does not exist or not readable"
+fi
+echo "obtaining configuration settings from '${JOB_CFG_FILE}'"
+
+LOCAL_TMP=$(${YQ} '.site_config.local_tmp // ""' < ${JOB_CFG_FILE})
+echo "LOCAL_TMP='${LOCAL_TMP}'"
+echo -n "setting \$storage by replacing any var in '${LOCAL_TMP}' -> "
+# replace any env variable in ${LOCAL_TMP} with its
+#   current value (e.g., a value that is local to the job)
+storage=$(envsubst <<< ${LOCAL_TMP})
+echo "'${storage}'"
+
+
+# singularity/apptainer settings: load_modules, HOME, TMPDIR, BIND 
+LOAD_MODULES=$(${YQ} '.site_config.load_modules // ""' < ${JOB_CFG_FILE})
+echo "LOAD_MODULES='${LOAD_MODULES}'"
+
+export SINGULARITY_HOME="$(pwd):/eessi_bot_job"
+export SINGULARITY_TMPDIR="$(pwd)/singularity_tmpdir"
+mkdir -p ${SINGULARITY_TMPDIR}
+
+if [[ ${storage} != /tmp* ]] ;
+then
+    export SINGULARITY_BIND="${storage}:/tmp"
+fi
+echo "SINGULARITY_BIND='${SINGULARITY_BIND}'"
+
+# load modules LOAD_MODULES is not empty
+if [[ ! -z ${LOAD_MODULES} ]]; then
+    for mod in $(echo ${LOAD_MODULES} | tr ',' '\n')
+    do
+        echo "bot/build.sh: loading module '${mod}'"
+        module load ${mod}
+    done
+else
+    echo "bot/build.sh: no modules to be loaded"
+fi
+

From a71f8e4f79d53f5be848d5f7064ce3569c4e0477 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= <thomas.roblitz@uib.no>
Date: Wed, 25 Jan 2023 19:37:55 +0100
Subject: [PATCH 03/58] added function to determine IPv4 address

---
 utils.sh | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/utils.sh b/utils.sh
index cd7f758a0d..db533adc12 100644
--- a/utils.sh
+++ b/utils.sh
@@ -63,3 +63,12 @@ function get_path_for_tool {
         fi
     fi
 }
+
+function get_ipv4_address {
+    hname=$1
+    hipv4=$(grep ${hname} /etc/hosts | grep -v '^[[:space:]]*#' | cut -d ' ' -f 1)
+    # TODO try other methods if the one above does not work --> tool that verifies
+    #      what method can be used?
+    echo "${hipv4}"
+    return 0
+}

From b5bf008e4136b7877e06ca0602b6fd6aa49128c0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= <thomas.roblitz@uib.no>
Date: Wed, 25 Jan 2023 19:39:53 +0100
Subject: [PATCH 04/58] various updates for bot/build.sh and eessi_container.sh

---
 bot/build.sh       |  69 ++++++++++++++++++++++----
 eessi_container.sh | 117 ++++++++++++++++++++++++---------------------
 2 files changed, 121 insertions(+), 65 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index 90753db8a8..e7cc7983bb 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -42,30 +42,37 @@ if [[ ! -r "${JOB_CFG_FILE}" ]]; then
 fi
 echo "obtaining configuration settings from '${JOB_CFG_FILE}'"
 
-LOCAL_TMP=$(${YQ} '.site_config.local_tmp // ""' < ${JOB_CFG_FILE})
+# if http_proxy is in cfg/job.cfg use it, if not use env var $http_proxy
+HTTP_PROXY=$(${YQ} '.site_config.http_proxy // "${http_proxy}"' ${JOB_CFG_FILE})
+echo "HTTP_PROXY='${HTTP_PROXY}'"
+
+LOCAL_TMP=$(${YQ} '.site_config.local_tmp // ""' ${JOB_CFG_FILE})
 echo "LOCAL_TMP='${LOCAL_TMP}'"
-echo -n "setting \$storage by replacing any var in '${LOCAL_TMP}' -> "
+# TODO should local_tmp be mandatory? --> then we check here and exit if it is not provided
+
+echo -n "setting \$STORAGE by replacing any var in '${LOCAL_TMP}' -> "
 # replace any env variable in ${LOCAL_TMP} with its
 #   current value (e.g., a value that is local to the job)
-storage=$(envsubst <<< ${LOCAL_TMP})
-echo "'${storage}'"
-
+STORAGE=$(envsubst <<< ${LOCAL_TMP})
+echo "'${STORAGE}'"
 
-# singularity/apptainer settings: load_modules, HOME, TMPDIR, BIND 
-LOAD_MODULES=$(${YQ} '.site_config.load_modules // ""' < ${JOB_CFG_FILE})
+# obtain list of modules to be loaded
+LOAD_MODULES=$(${YQ} '.site_config.load_modules // ""' ${JOB_CFG_FILE})
 echo "LOAD_MODULES='${LOAD_MODULES}'"
 
+# singularity/apptainer settings: CONTAINER, HOME, TMPDIR, BIND
+CONTAINER=$(${YQ} '.site_config.container // ""' ${JOB_CFG_FILE})
 export SINGULARITY_HOME="$(pwd):/eessi_bot_job"
 export SINGULARITY_TMPDIR="$(pwd)/singularity_tmpdir"
 mkdir -p ${SINGULARITY_TMPDIR}
 
-if [[ ${storage} != /tmp* ]] ;
+if [[ ${STORAGE} != /tmp* ]] ;
 then
-    export SINGULARITY_BIND="${storage}:/tmp"
+    export SINGULARITY_BIND="${STORAGE}:/tmp"
 fi
 echo "SINGULARITY_BIND='${SINGULARITY_BIND}'"
 
-# load modules LOAD_MODULES is not empty
+# load modules if LOAD_MODULES is not empty
 if [[ ! -z ${LOAD_MODULES} ]]; then
     for mod in $(echo ${LOAD_MODULES} | tr ',' '\n')
     do
@@ -76,3 +83,45 @@ else
     echo "bot/build.sh: no modules to be loaded"
 fi
 
+# determine repository to be used from entry .repository in cfg/job.cfg
+REPOSITORY=$(${YQ} '.repository.repo_id // ""' ${JOB_CFG_FILE})
+EESSI_REPOS_CFG_FILE_OVERRIDE=$(${YQ} '.repository.repos_cfg_file // "cfg/repos.cfg"' ${JOB_CFG_FILE})
+
+# determine architecture to be used from entry .architecture in cfg/job.cfg
+# default: leave empty to let downstream script(s) determine subdir to be used
+EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(${YQ} '.architecture.software_subdir // ""' ${JOB_CFG_FILE})
+
+source init/minimal_eessi_env
+
+# TODO
+#   - CODED add handling of EESSI_SOFTWARE_SUBDIR_OVERRIDE to eessi_container.sh
+#   - add handling of http(s)_proxy to eessi_container.sh, in there needs the
+#     CVMFS_HTTP_PROXY added to /etc/cvmfs/default.local (this needs a robust
+#     way to determine the IP address of a proxy)
+#   - bot needs to make repos.cfg and cfg_bundle available to job (likely, by copying
+#     files into './cfg/.' and defining '.repository.repos_cfg_file' in './cfg/job.cfg')
+
+# prepare options and directories for calling eessi_container.sh
+mkdir -p previous_tmp
+run_outerr=$(mktemp eessi_container.outerr.XXXXXXXXXX)
+CONTAINER_OPT=
+if [[ ! -z ${CONTAINER} ]]; then
+    CONTAINER_OPT="--container ${CONTAINER}"
+fi
+HTTP_PROXY_OPT=
+if [[ ! -z ${HTTP_PROXY} ]]; then
+    HTTP_PROXY_OPT="--container ${HTTP_PROXY}"
+fi
+REPOSITORY_OPT=
+if [[ ! -z ${REPOSITORY} ]]; then
+    REPOSITORY_OPT="--repository ${REPOSITORY}"
+fi
+./eessi_container.sh --access rw \
+                     ${CONTAINER_OPT} \
+                     ${HTTP_PROXY_OPT} \
+                     --info \
+                     --mode run \
+                     ${REPOSITORY_OPT} \
+                     --save $(pwd)/previous_tmp \
+                     --storage ${STORAGE} \
+                     ./install_software_layer.sh TODO installopts TODO commonopts TODO remainingopts "$@" 2>&1 | tee -a $(run_outerr)
diff --git a/eessi_container.sh b/eessi_container.sh
index 57b003368b..0016c8c9f0 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -38,8 +38,8 @@ MODE_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 5))
 REPOSITORY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 6))
 RESUME_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 7))
 SAVE_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 8))
-#HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9))
-#HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10))
+HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9))
+HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10))
 RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11))
 
 # CernVM-FS settings
@@ -59,43 +59,43 @@ export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_FILE_OVERRIDE:=repos.cfg}"
 display_help() {
   echo "usage: $0 [OPTIONS] [SCRIPT]"
   echo " OPTIONS:"
-  echo "  -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]"
-  echo "  -c | --container IMG  - image file or URL defining the container to use"
-  echo "                          [default: docker://ghcr.io/eessi/build-node:debian10]"
-  echo "  -h | --help           - display this usage information [default: false]"
-  echo "  -g | --storage DIR    - directory space on host machine (used for"
-  echo "                          temporary data) [default: 1. TMPDIR, 2. /tmp]"
-  echo "  -m | --mode MODE      - with MODE==shell (launch interactive shell) or"
-  echo "                          MODE==run (run a script) [default: shell]"
-  echo "  -r | --repository CFG - configuration file or identifier defining the"
-  echo "                          repository to use [default: EESSI-pilot]"
-  echo "  -u | --resume DIR/TGZ - resume a previous run from a directory or tarball,"
-  echo "                          where DIR points to a previously used tmp directory"
-  echo "                          (check for output 'Using DIR as tmp ...' of a previous"
-  echo "                          run) and TGZ is the path to a tarball which is"
-  echo "                          unpacked the tmp dir stored on the local storage space"
-  echo "                          (see option --storage above) [default: not set]"
-  echo "  -s | --save DIR/TGZ   - save contents of tmp directory to a tarball in"
-  echo "                          directory DIR or provided with the fixed full path TGZ"
-  echo "                          when a directory is provided, the format of the"
-  echo "                          tarball's name will be {REPO_ID}-{TIMESTAMP}.tgz"
-  echo "                          [default: not set]"
+  echo "  -a | --access {ro,rw}  - ro (read-only), rw (read & write) [default: ro]"
+  echo "  -c | --container IMG   - image file or URL defining the container to use"
+  echo "                           [default: docker://ghcr.io/eessi/build-node:debian11]"
+  echo "  -h | --help            - display this usage information [default: false]"
+  echo "  -i | --info            - display configuration information [default: false]"
+  echo "  -g | --storage DIR     - directory space on host machine (used for"
+  echo "                           temporary data) [default: 1. TMPDIR, 2. /tmp]"
+  echo "  -m | --mode MODE       - with MODE==shell (launch interactive shell) or"
+  echo "                           MODE==run (run a script) [default: shell]"
+  echo "  -r | --repository CFG  - configuration file or identifier defining the"
+  echo "                           repository to use [default: EESSI-pilot]"
+  echo "  -u | --resume DIR/TGZ  - resume a previous run from a directory or tarball,"
+  echo "                           where DIR points to a previously used tmp directory"
+  echo "                           (check for output 'Using DIR as tmp ...' of a previous"
+  echo "                           run) and TGZ is the path to a tarball which is"
+  echo "                           unpacked the tmp dir stored on the local storage space"
+  echo "                           (see option --storage above) [default: not set]"
+  echo "  -s | --save DIR/TGZ    - save contents of tmp directory to a tarball in"
+  echo "                           directory DIR or provided with the fixed full path TGZ"
+  echo "                           when a directory is provided, the format of the"
+  echo "                           tarball's name will be {REPO_ID}-{TIMESTAMP}.tgz"
+  echo "                           [default: not set]"
+  echo "  -x | --http-proxy URL  - provides URL for the env variable http_proxy"
+  echo "                           [default: not set]; uses env var \$http_proxy if set"
+  echo "  -y | --https-proxy URL - provides URL for the env variable https_proxy"
+  echo "                           [default: not set]; uses env var \$https_proxy if set"
   echo
   echo " If value for --mode is 'run', the SCRIPT provided is executed."
   echo
   echo " FEATURES/OPTIONS to be implemented:"
-  echo "  -d | --dry-run           -  run script except for executing the container,"
-  echo "                              print information about setup [default: false]"
-  echo "  -i | --info              -  display configured repositories [default: false]"
-  #echo "  -x | --http-proxy URL    -  provides URL for the env variable http_proxy"
-  #echo "                              [default: not set]"
-  #echo "  -y | --https-proxy URL   -  provides URL for the env variable https_proxy"
-  #echo "                              [default: not set]"
+  echo "  -d | --dry-run         -  run script except for executing the container,"
+  echo "                            print information about setup [default: false]"
 }
 
 # set defaults for command line arguments
 ACCESS="ro"
-CONTAINER="docker://ghcr.io/eessi/build-node:debian10"
+CONTAINER="docker://ghcr.io/eessi/build-node:debian11"
 DRY_RUN=0
 INFO=0
 STORAGE=
@@ -103,8 +103,8 @@ MODE="shell"
 REPOSITORY="EESSI-pilot"
 RESUME=
 SAVE=
-#HTTP_PROXY=
-#HTTPS_PROXY=
+HTTP_PROXY=${http_proxy:-}
+HTTPS_PROXY=${https_proxy:-}
 RUN_SCRIPT_AND_ARGS=
 
 POSITIONAL_ARGS=()
@@ -151,16 +151,16 @@ while [[ $# -gt 0 ]]; do
       RESUME="$2"
       shift 2
       ;;
-#    -x|--http-proxy)
-#      HTTP_PROXY="$2"
-#      export http_proxy=${HTTP_PROXY}
-#      shift 2
-#      ;;
-#    -y|--https-proxy)
-#      HTTPS_PROXY="$2"
-#      export https_proxy=${HTTPS_PROXY}
-#      shift 2
-#      ;;
+    -x|--http-proxy)
+      HTTP_PROXY="$2"
+      export http_proxy=${HTTP_PROXY}
+      shift 2
+      ;;
+    -y|--https-proxy)
+      HTTPS_PROXY="$2"
+      export https_proxy=${HTTPS_PROXY}
+      shift 2
+      ;;
     -*|--*)
       fatal_error "Unknown option: $1" "${CMDLINE_ARG_UNKNOWN_EXITCODE}"
       ;;
@@ -281,26 +281,26 @@ fi
 # tmp dir for EESSI
 EESSI_TMPDIR=${EESSI_HOST_STORAGE}
 mkdir -p ${EESSI_TMPDIR}
-[[ ${INFO} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}"
+[[ ${INFO} -eq 1 ]] && echo "EESSI_TMPDIR='${EESSI_TMPDIR}'"
 
 # configure Singularity
 export SINGULARITY_CACHEDIR=${EESSI_TMPDIR}/singularity_cache
 mkdir -p ${SINGULARITY_CACHEDIR}
-[[ ${INFO} -eq 1 ]] && echo "SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}"
+[[ ${INFO} -eq 1 ]] && echo "SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'"
 
 # set env vars and create directories for CernVM-FS
 EESSI_CVMFS_VAR_LIB=${EESSI_TMPDIR}/${CVMFS_VAR_LIB}
 EESSI_CVMFS_VAR_RUN=${EESSI_TMPDIR}/${CVMFS_VAR_RUN}
 mkdir -p ${EESSI_CVMFS_VAR_LIB}
 mkdir -p ${EESSI_CVMFS_VAR_RUN}
-[[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_VAR_LIB=${EESSI_CVMFS_VAR_LIB}"
-[[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_VAR_RUN=${EESSI_CVMFS_VAR_RUN}"
+[[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_VAR_LIB='${EESSI_CVMFS_VAR_LIB}'"
+[[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_VAR_RUN='${EESSI_CVMFS_VAR_RUN}'"
 
 # allow that SINGULARITY_HOME is defined before script is run
 if [[ -z ${SINGULARITY_HOME} ]]; then
   export SINGULARITY_HOME="${EESSI_TMPDIR}/home:/home/${USER}"
   mkdir -p ${EESSI_TMPDIR}/home
-  [[ ${INFO} -eq 1 ]] && echo "SINGULARITY_HOME=${SINGULARITY_HOME}"
+  [[ ${INFO} -eq 1 ]] && echo "SINGULARITY_HOME='${SINGULARITY_HOME}'"
 fi
 
 # define paths to add to SINGULARITY_BIND (added later when all BIND mounts are defined)
@@ -308,9 +308,9 @@ BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/ru
 BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp"
 [[ ${INFO} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}"
 
-# set up repository config (always create cfg dir and populate it with info when
+# set up repository config (always create directory repos_cfg and populate it with info when
 # arg -r|--repository is used)
-mkdir -p ${EESSI_TMPDIR}/cfg
+mkdir -p ${EESSI_TMPDIR}/repos_cfg
 if [[ "${REPOSITORY}" == "EESSI-pilot" ]]; then
   # need to source defaults as late as possible (see other sourcing below)
   source ${base_dir}/init/eessi_defaults
@@ -367,13 +367,13 @@ else
 
   # only unpack config_bundle if we're not resuming from a previous run
   if [[ -z ${RESUME} ]]; then
-    tar xf ${config_bundle} -C ${EESSI_TMPDIR}/cfg
+    tar xf ${config_bundle} -C ${EESSI_TMPDIR}/repos_cfg
   fi
 
   for src in "${!cfg_file_map[@]}"
   do
     target=${cfg_file_map[${src}]}
-    BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/cfg/${src}:${target}"
+    BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/${src}:${target}"
   done
   export EESSI_PILOT_VERSION_OVERRIDE=${repo_version}
   export EESSI_CVMFS_REPO_OVERRIDE="/cvmfs/${repo_name}"
@@ -397,8 +397,8 @@ if [[ "${ACCESS}" == "rw" ]]; then
   EESSI_CVMFS_OVERLAY_WORK=/tmp/overlay-work
   mkdir -p ${EESSI_TMPDIR}/overlay-upper
   mkdir -p ${EESSI_TMPDIR}/overlay-work
-  [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_UPPER=${EESSI_CVMFS_OVERLAY_UPPER}"
-  [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_WORK=${EESSI_CVMFS_OVERLAY_WORK}"
+  [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_UPPER='${EESSI_CVMFS_OVERLAY_UPPER}'"
+  [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_WORK='${EESSI_CVMFS_OVERLAY_WORK}'"
 
   # set environment variables for fuse mounts in Singularity container
   export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}"
@@ -427,7 +427,14 @@ if [[ -z ${SINGULARITY_BIND} ]]; then
 else
     export SINGULARITY_BIND="${SINGULARITY_BIND},${BIND_PATHS}"
 fi
-[[ ${INFO} -eq 1 ]] && echo "SINGULARITY_BIND=${SINGULARITY_BIND}"
+[[ ${INFO} -eq 1 ]] && echo "SINGULARITY_BIND='${SINGULARITY_BIND}'"
+
+# pass $EESSI_SOFTWARE_SUBDIR_OVERRIDE into build container (if set)
+if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
+    export SINGULARITYENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
+    # also specify via $APPTAINERENV_* (future proof, cfr. https://apptainer.org/docs/user/latest/singularity_compatibility.html#singularity-environment-variable-compatibility)
+    export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
+fi
 
 echo "Launching container with command (next line):"
 echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}"

From b5c07eeb99e77191d818e64788b0d419031cf526 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Fri, 27 Jan 2023 20:30:58 +0100
Subject: [PATCH 05/58] provide CVMFS_HTTP_PROXY if necessary

---
 bot/build.sh       |  3 ++-
 eessi_container.sh | 39 ++++++++++++++++++++++++++++-----------
 2 files changed, 30 insertions(+), 12 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index e7cc7983bb..a4f54cba3c 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -95,7 +95,8 @@ source init/minimal_eessi_env
 
 # TODO
 #   - CODED add handling of EESSI_SOFTWARE_SUBDIR_OVERRIDE to eessi_container.sh
-#   - add handling of http(s)_proxy to eessi_container.sh, in there needs the
+#     TODO ensure that the bot makes use of that.
+#   - CODED add handling of http(s)_proxy to eessi_container.sh, in there needs the
 #     CVMFS_HTTP_PROXY added to /etc/cvmfs/default.local (this needs a robust
 #     way to determine the IP address of a proxy)
 #   - bot needs to make repos.cfg and cfg_bundle available to job (likely, by copying
diff --git a/eessi_container.sh b/eessi_container.sh
index 0016c8c9f0..c2b4988162 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -19,8 +19,8 @@
 # 2. set up host storage/tmp
 # 3. set up common vars and directories
 # 4. set up vars specific to a scenario
-# 5. initialize host storage/tmp from previous run if provided
-# 6. run container
+# 5. run container
+# 6. save tmp (if requested)
 
 # -. initial settings & exit codes
 base_dir=$(dirname $(realpath $0))
@@ -276,7 +276,7 @@ fi
 #      |-overlay-upper
 #      |-overlay-work
 #      |-home
-#      |-cfg
+#      |-repos_cfg
 
 # tmp dir for EESSI
 EESSI_TMPDIR=${EESSI_HOST_STORAGE}
@@ -358,9 +358,9 @@ else
   cfg_init_file_map "${config_map}"
   [[ ${INFO} -eq 1 ]] && cfg_print_map
 
-  # TODO use information to set up dir ${EESSI_TMPDIR}/cfg,
-  #      define BIND mounts and override repo name and version
-  # check if config_bundle exists, if so, unpack it into ${EESSI_TMPDIR}/cfg
+  # use information to set up dir ${EESSI_TMPDIR}/repos_cfg,
+  #     define BIND mounts and override repo name and version
+  # check if config_bundle exists, if so, unpack it into ${EESSI_TMPDIR}/repos_cfg
   if [[ ! -r ${config_bundle} ]]; then
     fatal_error "config bundle '${config_bundle}' is not readable" ${REPOSITORY_ERROR_EXITCODE}
   fi
@@ -381,6 +381,26 @@ else
   source ${base_dir}/init/eessi_defaults
 fi
 
+# if http_proxy is not empty, we assume that the machine accesses internet
+# via a proxy. then we need to add CVMFS_HTTP_PROXY to
+# ${EESSI_TMPDIR}/repos_cfg/default.local on host (and possibly add a BIND
+# MOUNT if it was not yet in BIND_PATHS)
+if [[ ! -z ${http_proxy} ]]; then
+    # TODO tolerate other formats for proxy URLs, for now assume format is
+    # http://SOME_HOSTNAME:SOME_PORT
+    PROXY_HOST_AND_PORT=${http_proxy#http:\/\//} # strip http://
+    PROXY_PORT=${PROXY_HOST_AND_PORT#.*:/} # remove hostname: to get port
+    HTTP_PROXY_HOSTNAME=${PROXY_HOST_AND_PORT%:${PROX_PORT}/}
+    HTTP_PROXY_IPV4=$(get_ipv4_address ${HTTP_PROXY_HOSTNAME}) 
+    echo "CVMFS_HTTP_PROXY=\"${http_proxy}|" \
+         "http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \
+       >> ${EESSI_TMPDIR}/repos_cfg/default.local
+    cat ${EESSI_TMPDIR}/repos_cfg/default.local
+    # if default.local is not BIND mounted into container, add it to BIND_PATHS
+    if [[ ${BIND_PATHS} !~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then
+        export BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local"
+    fi
+fi
 
 # 4. set up vars and dirs specific to a scenario
 
@@ -417,10 +437,7 @@ if [[ "${ACCESS}" == "rw" ]]; then
 fi
 
 
-# 5. initialize host storage/tmp from previous run if provided
-
-
-# 6. run container
+# 5. run container
 # final settings
 if [[ -z ${SINGULARITY_BIND} ]]; then
     export SINGULARITY_BIND="${BIND_PATHS}"
@@ -442,7 +459,7 @@ echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ${RUN_SCRIPT_AND_
 #      provided to the script
 singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}
 
-# 7. save tmp if requested (arg -s|--save)
+# 6. save tmp if requested (arg -s|--save)
 if [[ ! -z ${SAVE} ]]; then
   # Note, for now we don't try to be smart and record in any way the OS and
   #   ARCH which might have been used internally, eg, when software packages

From 4ff37218793559a6e3a8ea9cef58198e33b7dcdf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= <thomas.roblitz@uib.no>
Date: Fri, 27 Jan 2023 22:41:36 +0100
Subject: [PATCH 06/58] add functions to derive hostname and port number from
 proxy URL

---
 utils.sh | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/utils.sh b/utils.sh
index db533adc12..fb4eff1e3f 100644
--- a/utils.sh
+++ b/utils.sh
@@ -64,6 +64,30 @@ function get_path_for_tool {
     fi
 }
 
+function get_host_from_url {
+    url=$1
+    re="(http|https)://([^/:]+)"
+    if [[ $url =~ $re ]]; then
+        echo ${BASH_REMATCH[2]}
+        return 0
+    else
+        echo ""
+        return 1
+    fi 
+}
+
+function get_port_from_url {
+    url=$1
+    re="(http|https)://[^:]+:([0-9]+)"
+    if [[ $url =~ $re ]]; then
+        echo ${BASH_REMATCH[2]}
+        return 0
+    else
+        echo ""
+        return 1
+    fi 
+}
+
 function get_ipv4_address {
     hname=$1
     hipv4=$(grep ${hname} /etc/hosts | grep -v '^[[:space:]]*#' | cut -d ' ' -f 1)

From 7a58fbff63712d81dc7dd3206ae2d35535c58b24 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= <thomas.roblitz@uib.no>
Date: Fri, 27 Jan 2023 22:43:27 +0100
Subject: [PATCH 07/58] add function to determine config sections

---
 cfg_files.sh | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/cfg_files.sh b/cfg_files.sh
index 885ebd0877..ffd29db50a 100644
--- a/cfg_files.sh
+++ b/cfg_files.sh
@@ -47,9 +47,7 @@ function cfg_load {
     else
       val=$(cfg_get_key_value $line)
       # trim leading and trailing spaces as well
-      #cur_key=$(echo $val | cut -f1 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//')
       cur_key=$(echo $val | cut -f1 -d'=' | cfg_trim_spaces)
-      #cur_val=$(echo $val | cut -f2 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//')
       cur_val=$(echo $val | cut -f2 -d'=' | cfg_trim_spaces)
       if [[ -n "$cur_key" ]]; then
         # section + key is the associative in bash array, the field separator is space
@@ -69,6 +67,20 @@ function cfg_print {
   done
 }
 
+function cfg_sections {
+  declare -A sections
+  for key in "${!cfg_repos[@]}"
+  do
+    # extract section from the associative key
+    section=$(echo $key | cut -f1 -d ' ')
+    sections[${section}]=1
+  done
+  for repo in "${!sections[@]}"
+  do
+      echo "${repo}"
+  done
+}
+
 function cfg_get_value {
   section=$1
   key=$2

From a5ea7d8aea2c92380839ecbf290c28d278c43835 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= <thomas.roblitz@uib.no>
Date: Fri, 27 Jan 2023 22:44:06 +0100
Subject: [PATCH 08/58] add parameter to list repositories, fix bugs

fixed bugs

- determining IP address for proxy host was wrong
- handling of positional parameters for executing commands with arguments was wrong
---
 eessi_container.sh | 43 ++++++++++++++++++++++++++++++-------------
 1 file changed, 30 insertions(+), 13 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index c2b4988162..d8d7cb2bdb 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -66,6 +66,7 @@ display_help() {
   echo "  -i | --info            - display configuration information [default: false]"
   echo "  -g | --storage DIR     - directory space on host machine (used for"
   echo "                           temporary data) [default: 1. TMPDIR, 2. /tmp]"
+  echo "  -l | --list-repos      - list available repository identifiers [default: false]"
   echo "  -m | --mode MODE       - with MODE==shell (launch interactive shell) or"
   echo "                           MODE==run (run a script) [default: shell]"
   echo "  -r | --repository CFG  - configuration file or identifier defining the"
@@ -99,13 +100,13 @@ CONTAINER="docker://ghcr.io/eessi/build-node:debian11"
 DRY_RUN=0
 INFO=0
 STORAGE=
+LIST_REPOS=0
 MODE="shell"
 REPOSITORY="EESSI-pilot"
 RESUME=
 SAVE=
 HTTP_PROXY=${http_proxy:-}
 HTTPS_PROXY=${https_proxy:-}
-RUN_SCRIPT_AND_ARGS=
 
 POSITIONAL_ARGS=()
 
@@ -135,6 +136,10 @@ while [[ $# -gt 0 ]]; do
       INFO=1
       shift 1
       ;;
+    -l|--list-repos)
+      LIST_REPOS=1
+      shift 1
+      ;;
     -m|--mode)
       MODE="$2"
       shift 2
@@ -173,6 +178,17 @@ done
 
 set -- "${POSITIONAL_ARGS[@]}"
 
+if [[ ${LIST_REPOS} -eq 1 ]]; then
+    echo "Repositories defined in the config file '${EESSI_REPOS_CFG_FILE}':"
+    echo "    EESSI-pilot [default]"
+    cfg_load ${EESSI_REPOS_CFG_FILE}
+    sections=$(cfg_sections)
+    while IFS= read -r repo_id
+    do
+        echo "    ${repo_id}"
+    done <<< "${sections}"
+    exit 0
+fi
 
 # 1. check if argument values are valid
 # (arg -a|--access) check if ACCESS is supported
@@ -214,8 +230,6 @@ fi
 if [[ "${MODE}" == "run" ]]; then
   if [[ $# -eq 0 ]]; then
     fatal_error "no command specified to run?!" "${RUN_SCRIPT_MISSING_EXITCODE}"
-  else
-    RUN_SCRIPT_AND_ARGS=$@
   fi
 fi
 
@@ -387,17 +401,20 @@ fi
 # MOUNT if it was not yet in BIND_PATHS)
 if [[ ! -z ${http_proxy} ]]; then
     # TODO tolerate other formats for proxy URLs, for now assume format is
-    # http://SOME_HOSTNAME:SOME_PORT
-    PROXY_HOST_AND_PORT=${http_proxy#http:\/\//} # strip http://
-    PROXY_PORT=${PROXY_HOST_AND_PORT#.*:/} # remove hostname: to get port
-    HTTP_PROXY_HOSTNAME=${PROXY_HOST_AND_PORT%:${PROX_PORT}/}
-    HTTP_PROXY_IPV4=$(get_ipv4_address ${HTTP_PROXY_HOSTNAME}) 
-    echo "CVMFS_HTTP_PROXY=\"${http_proxy}|" \
-         "http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \
+    # http://SOME_HOSTNAME:SOME_PORT/
+    [[ ${INFO} -eq 1 ]] && echo "http_proxy='${http_proxy}'"
+    PROXY_HOST=$(get_host_from_url ${http_proxy})
+    [[ ${INFO} -eq 1 ]] && echo "PROXY_HOST='${PROXY_HOST}'"
+    PROXY_PORT=$(get_port_from_url ${http_proxy})
+    [[ ${INFO} -eq 1 ]] && echo "PROXY_PORT='${PROXY_PORT}'"
+    HTTP_PROXY_IPV4=$(get_ipv4_address ${PROXY_HOST}) 
+    [[ ${INFO} -eq 1 ]] && echo "HTTP_PROXY_IPV4='${HTTP_PROXY_IPV4}'"
+    echo "CVMFS_HTTP_PROXY=\"${http_proxy}|http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \
        >> ${EESSI_TMPDIR}/repos_cfg/default.local
     cat ${EESSI_TMPDIR}/repos_cfg/default.local
+
     # if default.local is not BIND mounted into container, add it to BIND_PATHS
-    if [[ ${BIND_PATHS} !~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then
+    if [[ ! ${BIND_PATHS} =~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then
         export BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local"
     fi
 fi
@@ -454,10 +471,10 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
 fi
 
 echo "Launching container with command (next line):"
-echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}"
+echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@"
 # TODO for now we run singularity with '-q' (quiet), later adjust this to the log level
 #      provided to the script
-singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}
+singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@"
 
 # 6. save tmp if requested (arg -s|--save)
 if [[ ! -z ${SAVE} ]]; then

From 6ed3d85ff5f11bca37f838c533320733c3ddaf08 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sun, 29 Jan 2023 14:27:26 +0100
Subject: [PATCH 09/58] fix bash pattern matching operator

---
 eessi_container.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index c2b4988162..0cab47e8fb 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -397,7 +397,7 @@ if [[ ! -z ${http_proxy} ]]; then
        >> ${EESSI_TMPDIR}/repos_cfg/default.local
     cat ${EESSI_TMPDIR}/repos_cfg/default.local
     # if default.local is not BIND mounted into container, add it to BIND_PATHS
-    if [[ ${BIND_PATHS} !~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then
+    if [[ ! ${BIND_PATHS} =~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then
         export BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local"
     fi
 fi

From fd5cc3d339bffd7db248c43be909d195db8a64bd Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sun, 29 Jan 2023 14:29:25 +0100
Subject: [PATCH 10/58] various updates to bot/build.sh

- added handling of HTTPS_PROXY
- moved CONTAINER setting from site_config to repository section (different
  repositories could require different containers)
- added handling of CPU_TARGET and EESSI_SOFTWARE_SUBDIR_OVERRIDE
- updated TODOs
- added setting of EESSI_REPOS_CFG_FILE_OVERRIDE
- bug fixes
---
 bot/build.sh | 27 ++++++++++++++++++++++-----
 1 file changed, 22 insertions(+), 5 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index a4f54cba3c..3d06477a15 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -46,6 +46,10 @@ echo "obtaining configuration settings from '${JOB_CFG_FILE}'"
 HTTP_PROXY=$(${YQ} '.site_config.http_proxy // "${http_proxy}"' ${JOB_CFG_FILE})
 echo "HTTP_PROXY='${HTTP_PROXY}'"
 
+# if https_proxy is in cfg/job.cfg use it, if not use env var $https_proxy
+HTTPS_PROXY=$(${YQ} '.site_config.https_proxy // "${https_proxy}"' ${JOB_CFG_FILE})
+echo "HTTPS_PROXY='${HTTPS_PROXY}'"
+
 LOCAL_TMP=$(${YQ} '.site_config.local_tmp // ""' ${JOB_CFG_FILE})
 echo "LOCAL_TMP='${LOCAL_TMP}'"
 # TODO should local_tmp be mandatory? --> then we check here and exit if it is not provided
@@ -61,7 +65,7 @@ LOAD_MODULES=$(${YQ} '.site_config.load_modules // ""' ${JOB_CFG_FILE})
 echo "LOAD_MODULES='${LOAD_MODULES}'"
 
 # singularity/apptainer settings: CONTAINER, HOME, TMPDIR, BIND
-CONTAINER=$(${YQ} '.site_config.container // ""' ${JOB_CFG_FILE})
+CONTAINER=$(${YQ} '.repository.container // ""' ${JOB_CFG_FILE})
 export SINGULARITY_HOME="$(pwd):/eessi_bot_job"
 export SINGULARITY_TMPDIR="$(pwd)/singularity_tmpdir"
 mkdir -p ${SINGULARITY_TMPDIR}
@@ -89,13 +93,19 @@ EESSI_REPOS_CFG_FILE_OVERRIDE=$(${YQ} '.repository.repos_cfg_file // "cfg/repos.
 
 # determine architecture to be used from entry .architecture in cfg/job.cfg
 # default: leave empty to let downstream script(s) determine subdir to be used
-EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(${YQ} '.architecture.software_subdir // ""' ${JOB_CFG_FILE})
+if [[ ! -z "${CPU_TARGET}" ]]; then
+    EESSI_SOFTWARE_SUBDIR_OVERRIDE=${CPU_TARGET}
+else
+    EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(${YQ} '.architecture.software_subdir // ""' ${JOB_CFG_FILE})
+fi
 
 source init/minimal_eessi_env
 
 # TODO
 #   - CODED add handling of EESSI_SOFTWARE_SUBDIR_OVERRIDE to eessi_container.sh
-#     TODO ensure that the bot makes use of that.
+#     TODO ensure that the bot makes use of that. (currently sets env var
+#     CPU_TARGET & adds --export=ALL,CPU_TARGET=val to sbatch command ... also
+#     add it to cfg/job.cfg - .architecture.software_subdir)
 #   - CODED add handling of http(s)_proxy to eessi_container.sh, in there needs the
 #     CVMFS_HTTP_PROXY added to /etc/cvmfs/default.local (this needs a robust
 #     way to determine the IP address of a proxy)
@@ -111,18 +121,25 @@ if [[ ! -z ${CONTAINER} ]]; then
 fi
 HTTP_PROXY_OPT=
 if [[ ! -z ${HTTP_PROXY} ]]; then
-    HTTP_PROXY_OPT="--container ${HTTP_PROXY}"
+    HTTP_PROXY_OPT="--http-proxy ${HTTP_PROXY}"
+fi
+HTTPS_PROXY_OPT=
+if [[ ! -z ${HTTPS_PROXY} ]]; then
+    HTTPS_PROXY_OPT="--https-proxy ${HTTPS_PROXY}"
 fi
 REPOSITORY_OPT=
 if [[ ! -z ${REPOSITORY} ]]; then
     REPOSITORY_OPT="--repository ${REPOSITORY}"
 fi
+# set EESSI_REPOS_CFG_FILE_OVERRIDE to ./cfg/repos.cfg
+export EESSI_REPOS_CFG_FILE_OVERRIDE=$(pwd)/cfg/repos.cfg
 ./eessi_container.sh --access rw \
                      ${CONTAINER_OPT} \
                      ${HTTP_PROXY_OPT} \
+                     ${HTTPS_PROXY_OPT} \
                      --info \
                      --mode run \
                      ${REPOSITORY_OPT} \
                      --save $(pwd)/previous_tmp \
                      --storage ${STORAGE} \
-                     ./install_software_layer.sh TODO installopts TODO commonopts TODO remainingopts "$@" 2>&1 | tee -a $(run_outerr)
+                     ./install_software_layer.sh TODO installopts TODO commonopts TODO remainingopts "$@" 2>&1 | tee -a ${run_outerr}

From b2eab080cba313c72e2c72f91429242b1bf04990 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sun, 29 Jan 2023 19:10:40 +0100
Subject: [PATCH 11/58] replace FILE with DIR in EESSI_REPOS_CFG_FILE_OVERRIDE

---
 bot/build.sh       | 8 ++++----
 eessi_container.sh | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index 3d06477a15..7de3594d41 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -89,7 +89,7 @@ fi
 
 # determine repository to be used from entry .repository in cfg/job.cfg
 REPOSITORY=$(${YQ} '.repository.repo_id // ""' ${JOB_CFG_FILE})
-EESSI_REPOS_CFG_FILE_OVERRIDE=$(${YQ} '.repository.repos_cfg_file // "cfg/repos.cfg"' ${JOB_CFG_FILE})
+EESSI_REPOS_CFG_DIR_OVERRIDE=$(${YQ} '.repository.repos_cfg_dir // "cfg"' ${JOB_CFG_FILE})
 
 # determine architecture to be used from entry .architecture in cfg/job.cfg
 # default: leave empty to let downstream script(s) determine subdir to be used
@@ -131,8 +131,8 @@ REPOSITORY_OPT=
 if [[ ! -z ${REPOSITORY} ]]; then
     REPOSITORY_OPT="--repository ${REPOSITORY}"
 fi
-# set EESSI_REPOS_CFG_FILE_OVERRIDE to ./cfg/repos.cfg
-export EESSI_REPOS_CFG_FILE_OVERRIDE=$(pwd)/cfg/repos.cfg
+# set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg
+export EESSI_REPOS_CFG_DIR_OVERRIDE=$(pwd)/cfg
 ./eessi_container.sh --access rw \
                      ${CONTAINER_OPT} \
                      ${HTTP_PROXY_OPT} \
@@ -142,4 +142,4 @@ export EESSI_REPOS_CFG_FILE_OVERRIDE=$(pwd)/cfg/repos.cfg
                      ${REPOSITORY_OPT} \
                      --save $(pwd)/previous_tmp \
                      --storage ${STORAGE} \
-                     ./install_software_layer.sh TODO installopts TODO commonopts TODO remainingopts "$@" 2>&1 | tee -a ${run_outerr}
+                     ./install_software_layer.sh "$@" 2>&1 | tee -a ${run_outerr}
diff --git a/eessi_container.sh b/eessi_container.sh
index d8d7cb2bdb..4ca494e136 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -47,8 +47,8 @@ CVMFS_VAR_LIB="var-lib-cvmfs"
 CVMFS_VAR_RUN="var-run-cvmfs"
 
 # repository cfg file, default name (default location: $PWD)
-#   can be overwritten by setting env var EESSI_REPOS_CFG_FILE_OVERRIDE
-export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_FILE_OVERRIDE:=repos.cfg}"
+#   can be overwritten by setting env var EESSI_REPOS_CFG_DIR_OVERRIDE
+export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR_OVERRIDE:=.}/repos.cfg"
 
 
 # 0. parse args

From 7e200a185782727ad61f7c1e78f4b52228300a4d Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sun, 29 Jan 2023 19:56:41 +0100
Subject: [PATCH 12/58] fix whitespace issues

---
 cfg_files.sh        | 1 -
 eessi_container.sh  | 2 +-
 init/eessi_defaults | 1 -
 utils.sh            | 4 ++--
 4 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/cfg_files.sh b/cfg_files.sh
index ffd29db50a..57ea2f7c03 100644
--- a/cfg_files.sh
+++ b/cfg_files.sh
@@ -165,4 +165,3 @@ function cfg_print_map {
     echo "${index} --> ${cfg_file_map[${index}]}"
   done
 }
-
diff --git a/eessi_container.sh b/eessi_container.sh
index 4ca494e136..3a9fac2955 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -407,7 +407,7 @@ if [[ ! -z ${http_proxy} ]]; then
     [[ ${INFO} -eq 1 ]] && echo "PROXY_HOST='${PROXY_HOST}'"
     PROXY_PORT=$(get_port_from_url ${http_proxy})
     [[ ${INFO} -eq 1 ]] && echo "PROXY_PORT='${PROXY_PORT}'"
-    HTTP_PROXY_IPV4=$(get_ipv4_address ${PROXY_HOST}) 
+    HTTP_PROXY_IPV4=$(get_ipv4_address ${PROXY_HOST})
     [[ ${INFO} -eq 1 ]] && echo "HTTP_PROXY_IPV4='${HTTP_PROXY_IPV4}'"
     echo "CVMFS_HTTP_PROXY=\"${http_proxy}|http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \
        >> ${EESSI_TMPDIR}/repos_cfg/default.local
diff --git a/init/eessi_defaults b/init/eessi_defaults
index 1b5ce07fb1..f482cbc269 100644
--- a/init/eessi_defaults
+++ b/init/eessi_defaults
@@ -10,4 +10,3 @@
 
 export EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO_OVERRIDE:=/cvmfs/pilot.eessi-hpc.org}"
 export EESSI_PILOT_VERSION="${EESSI_PILOT_VERSION_OVERRIDE:=2021.12}"
-
diff --git a/utils.sh b/utils.sh
index fb4eff1e3f..d0da95e87f 100644
--- a/utils.sh
+++ b/utils.sh
@@ -73,7 +73,7 @@ function get_host_from_url {
     else
         echo ""
         return 1
-    fi 
+    fi
 }
 
 function get_port_from_url {
@@ -85,7 +85,7 @@ function get_port_from_url {
     else
         echo ""
         return 1
-    fi 
+    fi
 }
 
 function get_ipv4_address {

From 57b6f27191ffce26c05065b1ce2a7e0841309e4d Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Mon, 30 Jan 2023 17:48:52 +0100
Subject: [PATCH 13/58] fix handling of HTTP(S)_PROXY settings

---
 bot/build.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index 7de3594d41..d82048e86f 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -43,11 +43,13 @@ fi
 echo "obtaining configuration settings from '${JOB_CFG_FILE}'"
 
 # if http_proxy is in cfg/job.cfg use it, if not use env var $http_proxy
-HTTP_PROXY=$(${YQ} '.site_config.http_proxy // "${http_proxy}"' ${JOB_CFG_FILE})
+HTTP_PROXY=$(${YQ} '.site_config.http_proxy // ""' ${JOB_CFG_FILE})
+HTTP_PROXY=${HTTP_PROXY:-${http_proxy}}
 echo "HTTP_PROXY='${HTTP_PROXY}'"
 
 # if https_proxy is in cfg/job.cfg use it, if not use env var $https_proxy
-HTTPS_PROXY=$(${YQ} '.site_config.https_proxy // "${https_proxy}"' ${JOB_CFG_FILE})
+HTTPS_PROXY=$(${YQ} '.site_config.https_proxy // ""' ${JOB_CFG_FILE})
+HTTPS_PROXY=${HTTPS_PROXY:-${https_proxy}}
 echo "HTTPS_PROXY='${HTTPS_PROXY}'"
 
 LOCAL_TMP=$(${YQ} '.site_config.local_tmp // ""' ${JOB_CFG_FILE})

From fce504fb466be005f1d417e96e47f442c8f29c52 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Fri, 3 Feb 2023 11:54:47 +0100
Subject: [PATCH 14/58] include changes made to test PR for building with the
 bot

---
 bot/build.sh               | 87 ++++++++++++++++++++++++++++++++------
 eessi_container.sh         | 50 +++++++++++++++-------
 run_in_compat_layer_env.sh | 15 ++++++-
 3 files changed, 121 insertions(+), 31 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index d82048e86f..8565511bfa 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -72,12 +72,6 @@ export SINGULARITY_HOME="$(pwd):/eessi_bot_job"
 export SINGULARITY_TMPDIR="$(pwd)/singularity_tmpdir"
 mkdir -p ${SINGULARITY_TMPDIR}
 
-if [[ ${STORAGE} != /tmp* ]] ;
-then
-    export SINGULARITY_BIND="${STORAGE}:/tmp"
-fi
-echo "SINGULARITY_BIND='${SINGULARITY_BIND}'"
-
 # load modules if LOAD_MODULES is not empty
 if [[ ! -z ${LOAD_MODULES} ]]; then
     for mod in $(echo ${LOAD_MODULES} | tr ',' '\n')
@@ -91,7 +85,20 @@ fi
 
 # determine repository to be used from entry .repository in cfg/job.cfg
 REPOSITORY=$(${YQ} '.repository.repo_id // ""' ${JOB_CFG_FILE})
-EESSI_REPOS_CFG_DIR_OVERRIDE=$(${YQ} '.repository.repos_cfg_dir // "cfg"' ${JOB_CFG_FILE})
+EESSI_REPOS_CFG_DIR_OVERRIDE=$(${YQ} '.repository.repos_cfg_dir // ""' ${JOB_CFG_FILE})
+export EESSI_REPOS_CFG_DIR_OVERRIDE=${EESSI_REPOS_CFG_DIR_OVERRIDE:-${PWD}/cfg}
+
+# determine pilot version to be used from .repository.repo_version in cfg/job.cfg
+# here, just set & export EESSI_PILOT_VERSION_OVERRIDE
+# next script (eessi_container.sh) makes use of it via sourcing init scripts
+# (e.g., init/eessi_defaults or init/minimal_eessi_env)
+export EESSI_PILOT_VERSION_OVERRIDE=$(${YQ} '.repository.repo_version // ""' ${JOB_CFG_FILE})
+
+# determine CVMFS repo to be used from .repository.repo_name in cfg/job.cfg
+# here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down
+# "source init/eessi_defaults" via sourcing init/minimal_eessi_env
+export EESSI_CVMFS_REPO_OVERRIDE=$(${YQ} '.repository.repo_name // ""' ${JOB_CFG_FILE})
+
 
 # determine architecture to be used from entry .architecture in cfg/job.cfg
 # default: leave empty to let downstream script(s) determine subdir to be used
@@ -100,8 +107,11 @@ if [[ ! -z "${CPU_TARGET}" ]]; then
 else
     EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(${YQ} '.architecture.software_subdir // ""' ${JOB_CFG_FILE})
 fi
+export EESSI_SOFTWARE_SUBDIR_OVERRIDE
 
-source init/minimal_eessi_env
+# get EESSI_OS_TYPE from .architecture.os_type in cfg/job.cfg (default: linux)
+EESSI_OS_TYPE=$(${YQ} '.architecture.os_type // ""' ${JOB_CFG_FILE})
+export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux}
 
 # TODO
 #   - CODED add handling of EESSI_SOFTWARE_SUBDIR_OVERRIDE to eessi_container.sh
@@ -112,11 +122,9 @@ source init/minimal_eessi_env
 #     CVMFS_HTTP_PROXY added to /etc/cvmfs/default.local (this needs a robust
 #     way to determine the IP address of a proxy)
 #   - bot needs to make repos.cfg and cfg_bundle available to job (likely, by copying
-#     files into './cfg/.' and defining '.repository.repos_cfg_file' in './cfg/job.cfg')
+#     files into './cfg/.' and defining '.repository.repos_cfg_dir' in './cfg/job.cfg')
 
 # prepare options and directories for calling eessi_container.sh
-mkdir -p previous_tmp
-run_outerr=$(mktemp eessi_container.outerr.XXXXXXXXXX)
 CONTAINER_OPT=
 if [[ ! -z ${CONTAINER} ]]; then
     CONTAINER_OPT="--container ${CONTAINER}"
@@ -133,8 +141,21 @@ REPOSITORY_OPT=
 if [[ ! -z ${REPOSITORY} ]]; then
     REPOSITORY_OPT="--repository ${REPOSITORY}"
 fi
+mkdir -p previous_tmp
+build_outerr=$(mktemp build.outerr.XXXX)
+echo "Executing command to build software:"
+echo "./eessi_container.sh --access rw"
+echo "                     ${CONTAINER_OPT}"
+echo "                     ${HTTP_PROXY_OPT}"
+echo "                     ${HTTPS_PROXY_OPT}"
+echo "                     --info"
+echo "                     --mode run"
+echo "                     ${REPOSITORY_OPT}"
+echo "                     --save ${PWD}/previous_tmp"
+echo "                     --storage ${STORAGE}"
+echo "                     ./install_software_layer.sh \"$@\" 2>&1 | tee -a ${build_outerr}"
 # set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg
-export EESSI_REPOS_CFG_DIR_OVERRIDE=$(pwd)/cfg
+export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
 ./eessi_container.sh --access rw \
                      ${CONTAINER_OPT} \
                      ${HTTP_PROXY_OPT} \
@@ -142,6 +163,44 @@ export EESSI_REPOS_CFG_DIR_OVERRIDE=$(pwd)/cfg
                      --info \
                      --mode run \
                      ${REPOSITORY_OPT} \
-                     --save $(pwd)/previous_tmp \
+                     --save ${PWD}/previous_tmp \
                      --storage ${STORAGE} \
-                     ./install_software_layer.sh "$@" 2>&1 | tee -a ${run_outerr}
+                     ./install_software_layer.sh "$@" 2>&1 | tee -a ${build_outerr}
+
+# determine temporary directory to resume from
+BUILD_TMPDIR=$(grep 'RESUME_FROM_DIR' ${build_outerr} | sed -e "s/^RESUME_FROM_DIR //")
+
+tar_outerr=$(mktemp tar.outerr.XXXX)
+timestamp=$(date +%s)
+# to set EESSI_PILOT_VERSION we need to source init/eessi_defaults now
+source init/eessi_defaults
+export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_PILOT_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${timestamp})
+
+# value of first parameter to create_tarball.sh - TMP_IN_CONTAINER - needs to be
+# synchronised with setting of TMP_IN_CONTAINER in eessi_container.sh
+# TODO should we make this a configurable parameter of eessi_container.sh using
+# /tmp as default?
+TMP_IN_CONTAINER=/tmp
+echo "Executing command to create tarball:"
+echo "./eessi_container.sh --access rw"
+echo "                     ${CONTAINER_OPT}"
+echo "                     ${HTTP_PROXY_OPT}"
+echo "                     ${HTTPS_PROXY_OPT}"
+echo "                     --info"
+echo "                     --mode run"
+echo "                     ${REPOSITORY_OPT}"
+echo "                     --resume ${BUILD_TMPDIR}"
+echo "                     --save ${PWD}/previous_tmp"
+echo "                     ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}"
+./eessi_container.sh --access rw \
+                     ${CONTAINER_OPT} \
+                     ${HTTP_PROXY_OPT} \
+                     ${HTTPS_PROXY_OPT} \
+                     --info \
+                     --mode run \
+                     ${REPOSITORY_OPT} \
+                     --resume ${BUILD_TMPDIR} \
+                     --save ${PWD}/previous_tmp \
+                     ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}
+
+exit 0
diff --git a/eessi_container.sh b/eessi_container.sh
index 3a9fac2955..1eb630a096 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -46,9 +46,15 @@ RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11))
 CVMFS_VAR_LIB="var-lib-cvmfs"
 CVMFS_VAR_RUN="var-run-cvmfs"
 
+# directory for tmp used inside container
+export TMP_IN_CONTAINER=/tmp
+
 # repository cfg file, default name (default location: $PWD)
 #   can be overwritten by setting env var EESSI_REPOS_CFG_DIR_OVERRIDE
-export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR_OVERRIDE:=.}/repos.cfg"
+export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR_OVERRIDE:=${PWD}}/repos.cfg"
+# other repository cfg files in directory, default location: $PWD
+#   can be overwritten by setting env var EESSI_REPOS_CFG_DIR_OVERRIDE
+export EESSI_REPOS_CFG_DIR="${EESSI_REPOS_CFG_DIR_OVERRIDE:=${PWD}}"
 
 
 # 0. parse args
@@ -193,7 +199,7 @@ fi
 # 1. check if argument values are valid
 # (arg -a|--access) check if ACCESS is supported
 if [[ "${ACCESS}" != "ro" && "${ACCESS}" != "rw" ]]; then
-  fatal_error "unknown access method '${ACCESS}'" "${ACCESS_UNKNOWN_EXITCODE}"
+    fatal_error "unknown access method '${ACCESS}'" "${ACCESS_UNKNOWN_EXITCODE}"
 fi
 
 # TODO (arg -c|--container) check container (is it a file or URL & access those)
@@ -205,11 +211,14 @@ fi
 
 # (arg -m|--mode) check if MODE is known
 if [[ "${MODE}" != "shell" && "${MODE}" != "run" ]]; then
-  fatal_error "unknown execution mode '${MODE}'" "${MODE_UNKNOWN_EXITCODE}"
+    fatal_error "unknown execution mode '${MODE}'" "${MODE_UNKNOWN_EXITCODE}"
 fi
 
 # TODO (arg -r|--repository) check if repository is known
 # REPOSITORY_ERROR_EXITCODE
+if [[ ! -z "${REPOSITORY}" && "${REPOSITORY}" != "EESSI-pilot" && ! -r ${EESSI_REPOS_CFG_FILE} ]]; then
+    fatal_error "arg '--repository ${REPOSITORY}' requires a cfg file at '${EESSI_REPOS_CFG_FILE}'" "${REPOSITORY_ERROR_EXITCODE}"
+fi
 
 # TODO (arg -u|--resume) check if it exists, if user has read permission,
 #      if it contains data from a previous run
@@ -243,6 +252,7 @@ if [[ ! -z ${RESUME} && -d ${RESUME} ]]; then
   #   skip creating a new tmp directory, just set environment variables
   echo "Resuming from previous run using temporary storage at ${RESUME}"
   EESSI_HOST_STORAGE=${RESUME}
+  echo "RESUME_FROM_DIR ${EESSI_HOST_STORAGE}"
 else
   # we need a tmp location (and possibly init it with ${RESUME} if it was not
   #   a directory
@@ -319,7 +329,9 @@ fi
 
 # define paths to add to SINGULARITY_BIND (added later when all BIND mounts are defined)
 BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs"
-BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp"
+# provide a '/tmp' inside the container
+BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:${TMP_IN_CONTAINER}"
+
 [[ ${INFO} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}"
 
 # set up repository config (always create directory repos_cfg and populate it with info when
@@ -375,13 +387,22 @@ else
   # use information to set up dir ${EESSI_TMPDIR}/repos_cfg,
   #     define BIND mounts and override repo name and version
   # check if config_bundle exists, if so, unpack it into ${EESSI_TMPDIR}/repos_cfg
-  if [[ ! -r ${config_bundle} ]]; then
-    fatal_error "config bundle '${config_bundle}' is not readable" ${REPOSITORY_ERROR_EXITCODE}
+  # if config_bundle is relative path (no '/' at start) prepend it with
+  # EESSI_REPOS_CFG_DIR
+  config_bundle_path=
+  if [[ ! "${config_bundle}" =~ ^/ ]]; then
+      config_bundle_path=${EESSI_REPOS_CFG_DIR}/${config_bundle}
+  else
+      config_bundle_path=${config_bundle}
+  fi
+
+  if [[ ! -r ${config_bundle_path} ]]; then
+    fatal_error "config bundle '${config_bundle_path}' is not readable" ${REPOSITORY_ERROR_EXITCODE}
   fi
 
   # only unpack config_bundle if we're not resuming from a previous run
   if [[ -z ${RESUME} ]]; then
-    tar xf ${config_bundle} -C ${EESSI_TMPDIR}/repos_cfg
+    tar xf ${config_bundle_path} -C ${EESSI_TMPDIR}/repos_cfg
   fi
 
   for src in "${!cfg_file_map[@]}"
@@ -430,12 +451,8 @@ if [[ "${ACCESS}" == "ro" ]]; then
 fi
 
 if [[ "${ACCESS}" == "rw" ]]; then
-  EESSI_CVMFS_OVERLAY_UPPER=/tmp/overlay-upper
-  EESSI_CVMFS_OVERLAY_WORK=/tmp/overlay-work
   mkdir -p ${EESSI_TMPDIR}/overlay-upper
   mkdir -p ${EESSI_TMPDIR}/overlay-work
-  [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_UPPER='${EESSI_CVMFS_OVERLAY_UPPER}'"
-  [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_WORK='${EESSI_CVMFS_OVERLAY_WORK}'"
 
   # set environment variables for fuse mounts in Singularity container
   export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}"
@@ -444,8 +461,8 @@ if [[ "${ACCESS}" == "rw" ]]; then
 
   EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs"
   EESSI_PILOT_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${repo_name}"
-  EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=/tmp/overlay-upper"
-  EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=/tmp/overlay-work"
+  EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/overlay-upper"
+  EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/overlay-work"
   EESSI_PILOT_WRITABLE_OVERLAY+=" ${EESSI_CVMFS_REPO}"
   export EESSI_PILOT_WRITABLE_OVERLAY
 
@@ -470,11 +487,13 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
     export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
 fi
 
+# if INFO is set to 1 (arg --info), add argument '-q'
+RUN_QUIET=${INFO:--q}
 echo "Launching container with command (next line):"
-echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@"
+echo "singularity ${RUN_QUIET} ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@"
 # TODO for now we run singularity with '-q' (quiet), later adjust this to the log level
 #      provided to the script
-singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@"
+singularity ${RUN_QUIET} ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@"
 
 # 6. save tmp if requested (arg -s|--save)
 if [[ ! -z ${SAVE} ]]; then
@@ -493,6 +512,7 @@ if [[ ! -z ${SAVE} ]]; then
   fi
   tar cf ${TGZ} -C ${EESSI_TMPDIR} .
   echo "Saved contents of '${EESSI_TMPDIR}' to '${TGZ}' (to resume, add '--resume ${TGZ}')"
+  echo "RESUME_FROM_TGZ ${TGZ}"
 fi
 
 # TODO clean up tmp by default? only retain if another option provided (--retain-tmp)
diff --git a/run_in_compat_layer_env.sh b/run_in_compat_layer_env.sh
index e4850d677d..c70077bf15 100755
--- a/run_in_compat_layer_env.sh
+++ b/run_in_compat_layer_env.sh
@@ -3,7 +3,6 @@
 base_dir=$(dirname $(realpath $0))
 source ${base_dir}/init/eessi_defaults
 
-BUILD_CONTAINER="docker://ghcr.io/eessi/build-node:debian10"
 if [ -z $EESSI_PILOT_VERSION ]; then
     echo "ERROR: \$EESSI_PILOT_VERSION must be set!" >&2
     exit 1
@@ -18,6 +17,18 @@ INPUT=$(echo "$@")
 if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
     INPUT="export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}; ${INPUT}"
 fi
+if [ ! -z ${EESSI_CVMFS_REPO_OVERRIDE} ]; then
+    INPUT="export EESSI_CVMFS_REPO_OVERRIDE=${EESSI_CVMFS_REPO_OVERRIDE}; ${INPUT}"
+fi
+if [ ! -z ${EESSI_PILOT_VERSION_OVERRIDE} ]; then
+    INPUT="export EESSI_PILOT_VERSION_OVERRIDE=${EESSI_PILOT_VERSION_OVERRIDE}; ${INPUT}"
+fi
+if [ ! -z ${http_proxy} ]; then
+    INPUT="export http_proxy=${http_proxy}; ${INPUT}"
+fi
+if [ ! -z ${https_proxy} ]; then
+    INPUT="export https_proxy=${https_proxy}; ${INPUT}"
+fi
 
-echo "Running '${INPUT}' in EESSI ${EESSI_PILOT_VERSION} compatibility layer environment..."
+echo "Running '${INPUT}' in EESSI (${EESSI_CVMFS_REPO}) ${EESSI_PILOT_VERSION} compatibility layer environment..."
 ${EESSI_COMPAT_LAYER_DIR}/startprefix <<< "${INPUT}"

From b1a36355de66fa9a9557edbef79508f88c763cbb Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Tue, 24 Jan 2023 21:59:41 +0100
Subject: [PATCH 15/58] added a function to obtain full path to a tool

---
 scripts/utils.sh | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/scripts/utils.sh b/scripts/utils.sh
index 5d8455bb68..cd7f758a0d 100644
--- a/scripts/utils.sh
+++ b/scripts/utils.sh
@@ -31,3 +31,35 @@ function check_exit_code {
         fatal_error "${fail_msg}"
     fi
 }
+
+function get_path_for_tool {
+    tool_name=$1
+    tool_envvar_name=$2
+
+    which_out=$(which ${tool_name} 2>&1)
+    exit_code=$?
+    if [[ ${exit_code} -eq 0 ]]; then
+        echo "INFO: found tool ${tool_name} in PATH (${which_out})" >&2
+        echo "${which_out}"
+        return 0
+    fi
+    if [[ -z "${tool_envvar_name}" ]]; then
+        msg="no env var holding the full path to tool '${tool_name}' provided"
+        echo "${msg}" >&2
+        return 1
+    else
+        tool_envvar_value=${!tool_envvar_name}
+        if [[ -x "${tool_envvar_value}" ]]; then
+            msg="INFO: found tool ${tool_envvar_value} via env var ${tool_envvar_name}"
+            echo "${msg}" >&2
+            echo "${tool_envvar_value}"
+            return 0
+        else
+            msg="ERROR: tool '${tool_name}' not in PATH\n"
+            msg+="ERROR: tool '${tool_envvar_value}' via '${tool_envvar_name}' not in PATH"
+            echo "${msg}" >&2
+            echo ""
+            return 2
+        fi
+    fi
+}

From 27bc2035b00f59442fd46bcdbd6e104c09944f18 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= <thomas.roblitz@uib.no>
Date: Wed, 25 Jan 2023 19:37:55 +0100
Subject: [PATCH 16/58] added function to determine IPv4 address

---
 scripts/utils.sh | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/scripts/utils.sh b/scripts/utils.sh
index cd7f758a0d..db533adc12 100644
--- a/scripts/utils.sh
+++ b/scripts/utils.sh
@@ -63,3 +63,12 @@ function get_path_for_tool {
         fi
     fi
 }
+
+function get_ipv4_address {
+    hname=$1
+    hipv4=$(grep ${hname} /etc/hosts | grep -v '^[[:space:]]*#' | cut -d ' ' -f 1)
+    # TODO try other methods if the one above does not work --> tool that verifies
+    #      what method can be used?
+    echo "${hipv4}"
+    return 0
+}

From bfe6d36bf056c9e4e9719fa3a8110899fb125dc2 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sat, 18 Feb 2023 20:44:16 +0100
Subject: [PATCH 17/58] various updates for bot/build.sh and eessi_container.sh

cherry-picked via commit b5bf008e4136b7877e06ca0602b6fd6aa49128c0
ONLY picked chages to eessi_container.sh
---
 eessi_container.sh | 103 ++++++++++++++++++++++++---------------------
 1 file changed, 55 insertions(+), 48 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 96a9900a5e..3045fd5a89 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -43,8 +43,8 @@ MODE_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 5))
 REPOSITORY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 6))
 RESUME_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 7))
 SAVE_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 8))
-#HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9))
-#HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10))
+HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9))
+HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10))
 RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11))
 
 # CernVM-FS settings
@@ -64,39 +64,39 @@ export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_FILE_OVERRIDE:=repos.cfg}"
 display_help() {
   echo "usage: $0 [OPTIONS] [SCRIPT]"
   echo " OPTIONS:"
-  echo "  -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]"
-  echo "  -c | --container IMG  - image file or URL defining the container to use"
-  echo "                          [default: docker://ghcr.io/eessi/build-node:debian11]"
-  echo "  -h | --help           - display this usage information [default: false]"
-  echo "  -g | --storage DIR    - directory space on host machine (used for"
-  echo "                          temporary data) [default: 1. TMPDIR, 2. /tmp]"
-  echo "  -m | --mode MODE      - with MODE==shell (launch interactive shell) or"
-  echo "                          MODE==run (run a script) [default: shell]"
-  echo "  -r | --repository CFG - configuration file or identifier defining the"
-  echo "                          repository to use [default: EESSI-pilot via"
+  echo "  -a | --access {ro,rw}  - ro (read-only), rw (read & write) [default: ro]"
+  echo "  -c | --container IMG   - image file or URL defining the container to use"
+  echo "                           [default: docker://ghcr.io/eessi/build-node:debian11]"
+  echo "  -h | --help            - display this usage information [default: false]"
+  echo "  -g | --storage DIR     - directory space on host machine (used for"
+  echo "                           temporary data) [default: 1. TMPDIR, 2. /tmp]"
+  echo "  -m | --mode MODE       - with MODE==shell (launch interactive shell) or"
+  echo "                           MODE==run (run a script) [default: shell]"
+  echo "  -r | --repository CFG  - configuration file or identifier defining the"
+  echo "                           repository to use [default: EESSI-pilot via"
   echo "                          container configuration]"
-  echo "  -u | --resume DIR/TGZ - resume a previous run from a directory or tarball,"
-  echo "                          where DIR points to a previously used tmp directory"
-  echo "                          (check for output 'Using DIR as tmp ...' of a previous"
-  echo "                          run) and TGZ is the path to a tarball which is"
-  echo "                          unpacked the tmp dir stored on the local storage space"
-  echo "                          (see option --storage above) [default: not set]"
-  echo "  -s | --save DIR/TGZ   - save contents of tmp directory to a tarball in"
-  echo "                          directory DIR or provided with the fixed full path TGZ"
-  echo "                          when a directory is provided, the format of the"
-  echo "                          tarball's name will be {REPO_ID}-{TIMESTAMP}.tgz"
-  echo "                          [default: not set]"
-  echo "  -v | --verbose        - display more information [default: false]"
+  echo "  -u | --resume DIR/TGZ  - resume a previous run from a directory or tarball,"
+  echo "                           where DIR points to a previously used tmp directory"
+  echo "                           (check for output 'Using DIR as tmp ...' of a previous"
+  echo "                           run) and TGZ is the path to a tarball which is"
+  echo "                           unpacked the tmp dir stored on the local storage space"
+  echo "                           (see option --storage above) [default: not set]"
+  echo "  -s | --save DIR/TGZ    - save contents of tmp directory to a tarball in"
+  echo "                           directory DIR or provided with the fixed full path TGZ"
+  echo "                           when a directory is provided, the format of the"
+  echo "                           tarball's name will be {REPO_ID}-{TIMESTAMP}.tgz"
+  echo "                           [default: not set]"
+  echo "  -v | --verbose         - display more information [default: false]"
+  echo "  -x | --http-proxy URL  - provides URL for the env variable http_proxy"
+  echo "                           [default: not set]; uses env var \$http_proxy if set"
+  echo "  -y | --https-proxy URL - provides URL for the env variable https_proxy"
+  echo "                           [default: not set]; uses env var \$https_proxy if set"
   echo
   echo " If value for --mode is 'run', the SCRIPT provided is executed."
-  #echo
-  #echo " FEATURES/OPTIONS to be implemented:"
-  #echo "  -d | --dry-run           -  run script except for executing the container,"
-  #echo "                              print information about setup [default: false]"
-  #echo "  -x | --http-proxy URL    -  provides URL for the env variable http_proxy"
-  #echo "                              [default: not set]"
-  #echo "  -y | --https-proxy URL   -  provides URL for the env variable https_proxy"
-  #echo "                              [default: not set]"
+  echo
+  echo " FEATURES/OPTIONS to be implemented:"
+  echo "  -d | --dry-run         -  run script except for executing the container,"
+  echo "                            print information about setup [default: false]"
 }
 
 # set defaults for command line arguments
@@ -109,8 +109,8 @@ MODE="shell"
 REPOSITORY="EESSI-pilot"
 RESUME=
 SAVE=
-#HTTP_PROXY=
-#HTTPS_PROXY=
+HTTP_PROXY=${http_proxy:-}
+HTTPS_PROXY=${https_proxy:-}
 
 POSITIONAL_ARGS=()
 
@@ -156,16 +156,16 @@ while [[ $# -gt 0 ]]; do
       VERBOSE=1
       shift 1
       ;;
-#    -x|--http-proxy)
-#      HTTP_PROXY="$2"
-#      export http_proxy=${HTTP_PROXY}
-#      shift 2
-#      ;;
-#    -y|--https-proxy)
-#      HTTPS_PROXY="$2"
-#      export https_proxy=${HTTPS_PROXY}
-#      shift 2
-#      ;;
+    -x|--http-proxy)
+      HTTP_PROXY="$2"
+      export http_proxy=${HTTP_PROXY}
+      shift 2
+      ;;
+    -y|--https-proxy)
+      HTTPS_PROXY="$2"
+      export https_proxy=${HTTPS_PROXY}
+      shift 2
+      ;;
     -*|--*)
       fatal_error "Unknown option: $1" "${CMDLINE_ARG_UNKNOWN_EXITCODE}"
       ;;
@@ -311,9 +311,9 @@ BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/ru
 BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp"
 [[ ${VERBOSE} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}"
 
-# set up repository config (always create cfg dir and populate it with info when
+# set up repository config (always create directory repos_cfg and populate it with info when
 # arg -r|--repository is used)
-mkdir -p ${EESSI_TMPDIR}/cfg
+mkdir -p ${EESSI_TMPDIR}/repos_cfg
 if [[ "${REPOSITORY}" == "EESSI-pilot" ]]; then
   # need to source defaults as late as possible (see other sourcing below)
   source ${TOPDIR}/init/eessi_defaults
@@ -370,13 +370,13 @@ else
 
   # only unpack config_bundle if we're not resuming from a previous run
   if [[ -z ${RESUME} ]]; then
-    tar xf ${config_bundle} -C ${EESSI_TMPDIR}/cfg
+    tar xf ${config_bundle} -C ${EESSI_TMPDIR}/repos_cfg
   fi
 
   for src in "${!cfg_file_map[@]}"
   do
     target=${cfg_file_map[${src}]}
-    BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/cfg/${src}:${target}"
+    BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/${src}:${target}"
   done
   export EESSI_PILOT_VERSION_OVERRIDE=${repo_version}
   export EESSI_CVMFS_REPO_OVERRIDE="/cvmfs/${repo_name}"
@@ -432,6 +432,13 @@ else
 fi
 [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_BIND=${SINGULARITY_BIND}"
 
+# pass $EESSI_SOFTWARE_SUBDIR_OVERRIDE into build container (if set)
+if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
+    export SINGULARITYENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
+    # also specify via $APPTAINERENV_* (future proof, cfr. https://apptainer.org/docs/user/latest/singularity_compatibility.html#singularity-environment-variable-compatibility)
+    export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
+fi
+
 echo "Launching container with command (next line):"
 echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@"
 # TODO for now we run singularity with '-q' (quiet), later adjust this to the log level

From 837234107d28cd9b2d759a66af8f7e2ec5244923 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sat, 18 Feb 2023 20:55:30 +0100
Subject: [PATCH 18/58] provide CVMFS_HTTP_PROXY if necessary

cherry-picked via commit b5c07eeb99e77191d818e64788b0d419031cf526
NOTE, only applied changes to eessi_container.sh
---
 eessi_container.sh | 39 ++++++++++++++++++++++++++++-----------
 1 file changed, 28 insertions(+), 11 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 3045fd5a89..9238d3c793 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -24,8 +24,8 @@
 # 2. set up host storage/tmp
 # 3. set up common vars and directories
 # 4. set up vars specific to a scenario
-# 5. initialize host storage/tmp from previous run if provided
-# 6. run container
+# 5. run container
+# 6. save tmp (if requested)
 
 # -. initial settings & exit codes
 TOPDIR=$(dirname $(realpath $0))
@@ -279,7 +279,7 @@ fi
 #      |-overlay-upper
 #      |-overlay-work
 #      |-home
-#      |-cfg
+#      |-repos_cfg
 
 # tmp dir for EESSI
 EESSI_TMPDIR=${EESSI_HOST_STORAGE}
@@ -361,9 +361,9 @@ else
   cfg_init_file_map "${config_map}"
   [[ ${VERBOSE} -eq 1 ]] && cfg_print_map
 
-  # TODO use information to set up dir ${EESSI_TMPDIR}/cfg,
-  #      define BIND mounts and override repo name and version
-  # check if config_bundle exists, if so, unpack it into ${EESSI_TMPDIR}/cfg
+  # use information to set up dir ${EESSI_TMPDIR}/repos_cfg,
+  #     define BIND mounts and override repo name and version
+  # check if config_bundle exists, if so, unpack it into ${EESSI_TMPDIR}/repos_cfg
   if [[ ! -r ${config_bundle} ]]; then
     fatal_error "config bundle '${config_bundle}' is not readable" ${REPOSITORY_ERROR_EXITCODE}
   fi
@@ -384,6 +384,26 @@ else
   source ${TOPDIR}/init/eessi_defaults
 fi
 
+# if http_proxy is not empty, we assume that the machine accesses internet
+# via a proxy. then we need to add CVMFS_HTTP_PROXY to
+# ${EESSI_TMPDIR}/repos_cfg/default.local on host (and possibly add a BIND
+# MOUNT if it was not yet in BIND_PATHS)
+if [[ ! -z ${http_proxy} ]]; then
+    # TODO tolerate other formats for proxy URLs, for now assume format is
+    # http://SOME_HOSTNAME:SOME_PORT
+    PROXY_HOST_AND_PORT=${http_proxy#http:\/\//} # strip http://
+    PROXY_PORT=${PROXY_HOST_AND_PORT#.*:/} # remove hostname: to get port
+    HTTP_PROXY_HOSTNAME=${PROXY_HOST_AND_PORT%:${PROX_PORT}/}
+    HTTP_PROXY_IPV4=$(get_ipv4_address ${HTTP_PROXY_HOSTNAME}) 
+    echo "CVMFS_HTTP_PROXY=\"${http_proxy}|" \
+         "http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \
+       >> ${EESSI_TMPDIR}/repos_cfg/default.local
+    cat ${EESSI_TMPDIR}/repos_cfg/default.local
+    # if default.local is not BIND mounted into container, add it to BIND_PATHS
+    if [[ ${BIND_PATHS} !~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then
+        export BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local"
+    fi
+fi
 
 # 4. set up vars and dirs specific to a scenario
 
@@ -420,10 +440,7 @@ if [[ "${ACCESS}" == "rw" ]]; then
 fi
 
 
-# 5. initialize host storage/tmp from previous run if provided
-
-
-# 6. run container
+# 5. run container
 # final settings
 if [[ -z ${SINGULARITY_BIND} ]]; then
     export SINGULARITY_BIND="${BIND_PATHS}"
@@ -446,7 +463,7 @@ echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@"
 singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@"
 exit_code=$?
 
-# 7. save tmp if requested (arg -s|--save)
+# 6. save tmp if requested (arg -s|--save)
 if [[ ! -z ${SAVE} ]]; then
   # Note, for now we don't try to be smart and record in any way the OS and
   #   ARCH which might have been used internally, eg, when software packages

From 3f1aed9a2b5da7fa7e017e415d24a412dbfa18fc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= <thomas.roblitz@uib.no>
Date: Fri, 27 Jan 2023 22:41:36 +0100
Subject: [PATCH 19/58] add functions to derive hostname and port number from
 proxy URL

---
 scripts/utils.sh | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/scripts/utils.sh b/scripts/utils.sh
index db533adc12..fb4eff1e3f 100644
--- a/scripts/utils.sh
+++ b/scripts/utils.sh
@@ -64,6 +64,30 @@ function get_path_for_tool {
     fi
 }
 
+function get_host_from_url {
+    url=$1
+    re="(http|https)://([^/:]+)"
+    if [[ $url =~ $re ]]; then
+        echo ${BASH_REMATCH[2]}
+        return 0
+    else
+        echo ""
+        return 1
+    fi 
+}
+
+function get_port_from_url {
+    url=$1
+    re="(http|https)://[^:]+:([0-9]+)"
+    if [[ $url =~ $re ]]; then
+        echo ${BASH_REMATCH[2]}
+        return 0
+    else
+        echo ""
+        return 1
+    fi 
+}
+
 function get_ipv4_address {
     hname=$1
     hipv4=$(grep ${hname} /etc/hosts | grep -v '^[[:space:]]*#' | cut -d ' ' -f 1)

From b7abfc6e7de85e2b46e3241d8946a1211dcde568 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= <thomas.roblitz@uib.no>
Date: Fri, 27 Jan 2023 22:43:27 +0100
Subject: [PATCH 20/58] add function to determine config sections

---
 cfg_files.sh | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/cfg_files.sh b/cfg_files.sh
index 885ebd0877..ffd29db50a 100644
--- a/cfg_files.sh
+++ b/cfg_files.sh
@@ -47,9 +47,7 @@ function cfg_load {
     else
       val=$(cfg_get_key_value $line)
       # trim leading and trailing spaces as well
-      #cur_key=$(echo $val | cut -f1 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//')
       cur_key=$(echo $val | cut -f1 -d'=' | cfg_trim_spaces)
-      #cur_val=$(echo $val | cut -f2 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//')
       cur_val=$(echo $val | cut -f2 -d'=' | cfg_trim_spaces)
       if [[ -n "$cur_key" ]]; then
         # section + key is the associative in bash array, the field separator is space
@@ -69,6 +67,20 @@ function cfg_print {
   done
 }
 
+function cfg_sections {
+  declare -A sections
+  for key in "${!cfg_repos[@]}"
+  do
+    # extract section from the associative key
+    section=$(echo $key | cut -f1 -d ' ')
+    sections[${section}]=1
+  done
+  for repo in "${!sections[@]}"
+  do
+      echo "${repo}"
+  done
+}
+
 function cfg_get_value {
   section=$1
   key=$2

From 9b7b8a4db00261b0fe2bd40152d7adacde11ffe3 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sat, 18 Feb 2023 21:17:11 +0100
Subject: [PATCH 21/58] add parameter to list repositories, fix bugs

fixed bugs

- determining IP address for proxy host was wrong
- handling of positional parameters for executing commands with arguments was wrong
---
 eessi_container.sh | 36 ++++++++++++++++++++++++++++--------
 1 file changed, 28 insertions(+), 8 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 9238d3c793..06b77a5acf 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -70,6 +70,7 @@ display_help() {
   echo "  -h | --help            - display this usage information [default: false]"
   echo "  -g | --storage DIR     - directory space on host machine (used for"
   echo "                           temporary data) [default: 1. TMPDIR, 2. /tmp]"
+  echo "  -l | --list-repos      - list available repository identifiers [default: false]"
   echo "  -m | --mode MODE       - with MODE==shell (launch interactive shell) or"
   echo "                           MODE==run (run a script) [default: shell]"
   echo "  -r | --repository CFG  - configuration file or identifier defining the"
@@ -105,6 +106,7 @@ CONTAINER="docker://ghcr.io/eessi/build-node:debian11"
 #DRY_RUN=0
 VERBOSE=0
 STORAGE=
+LIST_REPOS=0
 MODE="shell"
 REPOSITORY="EESSI-pilot"
 RESUME=
@@ -136,6 +138,10 @@ while [[ $# -gt 0 ]]; do
       display_help
       exit 0
       ;;
+    -l|--list-repos)
+      LIST_REPOS=1
+      shift 1
+      ;;
     -m|--mode)
       MODE="$2"
       shift 2
@@ -178,6 +184,17 @@ done
 
 set -- "${POSITIONAL_ARGS[@]}"
 
+if [[ ${LIST_REPOS} -eq 1 ]]; then
+    echo "Repositories defined in the config file '${EESSI_REPOS_CFG_FILE}':"
+    echo "    EESSI-pilot [default]"
+    cfg_load ${EESSI_REPOS_CFG_FILE}
+    sections=$(cfg_sections)
+    while IFS= read -r repo_id
+    do
+        echo "    ${repo_id}"
+    done <<< "${sections}"
+    exit 0
+fi
 
 # 1. check if argument values are valid
 # (arg -a|--access) check if ACCESS is supported
@@ -390,17 +407,20 @@ fi
 # MOUNT if it was not yet in BIND_PATHS)
 if [[ ! -z ${http_proxy} ]]; then
     # TODO tolerate other formats for proxy URLs, for now assume format is
-    # http://SOME_HOSTNAME:SOME_PORT
-    PROXY_HOST_AND_PORT=${http_proxy#http:\/\//} # strip http://
-    PROXY_PORT=${PROXY_HOST_AND_PORT#.*:/} # remove hostname: to get port
-    HTTP_PROXY_HOSTNAME=${PROXY_HOST_AND_PORT%:${PROX_PORT}/}
-    HTTP_PROXY_IPV4=$(get_ipv4_address ${HTTP_PROXY_HOSTNAME}) 
-    echo "CVMFS_HTTP_PROXY=\"${http_proxy}|" \
-         "http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \
+    # http://SOME_HOSTNAME:SOME_PORT/
+    [[ ${VERBOSE} -eq 1 ]] && echo "http_proxy='${http_proxy}'"
+    PROXY_HOST=$(get_host_from_url ${http_proxy})
+    [[ ${VERBOSE} -eq 1 ]] && echo "PROXY_HOST='${PROXY_HOST}'"
+    PROXY_PORT=$(get_port_from_url ${http_proxy})
+    [[ ${VERBOSE} -eq 1 ]] && echo "PROXY_PORT='${PROXY_PORT}'"
+    HTTP_PROXY_IPV4=$(get_ipv4_address ${PROXY_HOST}) 
+    [[ ${VERBOSE} -eq 1 ]] && echo "HTTP_PROXY_IPV4='${HTTP_PROXY_IPV4}'"
+    echo "CVMFS_HTTP_PROXY=\"${http_proxy}|http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \
        >> ${EESSI_TMPDIR}/repos_cfg/default.local
     cat ${EESSI_TMPDIR}/repos_cfg/default.local
+
     # if default.local is not BIND mounted into container, add it to BIND_PATHS
-    if [[ ${BIND_PATHS} !~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then
+    if [[ ! ${BIND_PATHS} =~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then
         export BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local"
     fi
 fi

From 4a414f593d6ff02629d1374e67006dcd1a85f71c Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sat, 18 Feb 2023 21:26:19 +0100
Subject: [PATCH 22/58] replace FILE with DIR in EESSI_REPOS_CFG_FILE_OVERRIDE

NOTE only applied changes to eessi_container.sh
---
 eessi_container.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 06b77a5acf..d59bb38687 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -52,8 +52,8 @@ CVMFS_VAR_LIB="var-lib-cvmfs"
 CVMFS_VAR_RUN="var-run-cvmfs"
 
 # repository cfg file, default name (default location: $PWD)
-#   can be overwritten by setting env var EESSI_REPOS_CFG_FILE_OVERRIDE
-export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_FILE_OVERRIDE:=repos.cfg}"
+#   can be overwritten by setting env var EESSI_REPOS_CFG_DIR_OVERRIDE
+export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR_OVERRIDE:=.}/repos.cfg"
 
 
 # 0. parse args

From 37abf4f8aab9d138ad763cd457d2476bef16a6aa Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sun, 29 Jan 2023 19:56:41 +0100
Subject: [PATCH 23/58] fix whitespace issues

---
 cfg_files.sh        | 1 -
 eessi_container.sh  | 2 +-
 init/eessi_defaults | 1 -
 scripts/utils.sh    | 4 ++--
 4 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/cfg_files.sh b/cfg_files.sh
index ffd29db50a..57ea2f7c03 100644
--- a/cfg_files.sh
+++ b/cfg_files.sh
@@ -165,4 +165,3 @@ function cfg_print_map {
     echo "${index} --> ${cfg_file_map[${index}]}"
   done
 }
-
diff --git a/eessi_container.sh b/eessi_container.sh
index d59bb38687..1a2e61227b 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -413,7 +413,7 @@ if [[ ! -z ${http_proxy} ]]; then
     [[ ${VERBOSE} -eq 1 ]] && echo "PROXY_HOST='${PROXY_HOST}'"
     PROXY_PORT=$(get_port_from_url ${http_proxy})
     [[ ${VERBOSE} -eq 1 ]] && echo "PROXY_PORT='${PROXY_PORT}'"
-    HTTP_PROXY_IPV4=$(get_ipv4_address ${PROXY_HOST}) 
+    HTTP_PROXY_IPV4=$(get_ipv4_address ${PROXY_HOST})
     [[ ${VERBOSE} -eq 1 ]] && echo "HTTP_PROXY_IPV4='${HTTP_PROXY_IPV4}'"
     echo "CVMFS_HTTP_PROXY=\"${http_proxy}|http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \
        >> ${EESSI_TMPDIR}/repos_cfg/default.local
diff --git a/init/eessi_defaults b/init/eessi_defaults
index 1b5ce07fb1..f482cbc269 100644
--- a/init/eessi_defaults
+++ b/init/eessi_defaults
@@ -10,4 +10,3 @@
 
 export EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO_OVERRIDE:=/cvmfs/pilot.eessi-hpc.org}"
 export EESSI_PILOT_VERSION="${EESSI_PILOT_VERSION_OVERRIDE:=2021.12}"
-
diff --git a/scripts/utils.sh b/scripts/utils.sh
index fb4eff1e3f..d0da95e87f 100644
--- a/scripts/utils.sh
+++ b/scripts/utils.sh
@@ -73,7 +73,7 @@ function get_host_from_url {
     else
         echo ""
         return 1
-    fi 
+    fi
 }
 
 function get_port_from_url {
@@ -85,7 +85,7 @@ function get_port_from_url {
     else
         echo ""
         return 1
-    fi 
+    fi
 }
 
 function get_ipv4_address {

From d468a1ababa8072c4069917f4c16529cd362011f Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sat, 18 Feb 2023 21:42:34 +0100
Subject: [PATCH 24/58] include changes made to test PR for building with the
 bot

cherry-picked via commit fce504fb466be005f1d417e96e47f442c8f29c52
NOTE, only changed eessi_container.sh
---
 eessi_container.sh | 50 ++++++++++++++++++++++++++++++++--------------
 1 file changed, 35 insertions(+), 15 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 1a2e61227b..14fb716abb 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -51,9 +51,15 @@ RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11))
 CVMFS_VAR_LIB="var-lib-cvmfs"
 CVMFS_VAR_RUN="var-run-cvmfs"
 
+# directory for tmp used inside container
+export TMP_IN_CONTAINER=/tmp
+
 # repository cfg file, default name (default location: $PWD)
 #   can be overwritten by setting env var EESSI_REPOS_CFG_DIR_OVERRIDE
-export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR_OVERRIDE:=.}/repos.cfg"
+export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR_OVERRIDE:=${PWD}}/repos.cfg"
+# other repository cfg files in directory, default location: $PWD
+#   can be overwritten by setting env var EESSI_REPOS_CFG_DIR_OVERRIDE
+export EESSI_REPOS_CFG_DIR="${EESSI_REPOS_CFG_DIR_OVERRIDE:=${PWD}}"
 
 
 # 0. parse args
@@ -199,7 +205,7 @@ fi
 # 1. check if argument values are valid
 # (arg -a|--access) check if ACCESS is supported
 if [[ "${ACCESS}" != "ro" && "${ACCESS}" != "rw" ]]; then
-  fatal_error "unknown access method '${ACCESS}'" "${ACCESS_UNKNOWN_EXITCODE}"
+    fatal_error "unknown access method '${ACCESS}'" "${ACCESS_UNKNOWN_EXITCODE}"
 fi
 
 # TODO (arg -c|--container) check container (is it a file or URL & access those)
@@ -211,11 +217,14 @@ fi
 
 # (arg -m|--mode) check if MODE is known
 if [[ "${MODE}" != "shell" && "${MODE}" != "run" ]]; then
-  fatal_error "unknown execution mode '${MODE}'" "${MODE_UNKNOWN_EXITCODE}"
+    fatal_error "unknown execution mode '${MODE}'" "${MODE_UNKNOWN_EXITCODE}"
 fi
 
 # TODO (arg -r|--repository) check if repository is known
 # REPOSITORY_ERROR_EXITCODE
+if [[ ! -z "${REPOSITORY}" && "${REPOSITORY}" != "EESSI-pilot" && ! -r ${EESSI_REPOS_CFG_FILE} ]]; then
+    fatal_error "arg '--repository ${REPOSITORY}' requires a cfg file at '${EESSI_REPOS_CFG_FILE}'" "${REPOSITORY_ERROR_EXITCODE}"
+fi
 
 # TODO (arg -u|--resume) check if it exists, if user has read permission,
 #      if it contains data from a previous run
@@ -249,6 +258,7 @@ if [[ ! -z ${RESUME} && -d ${RESUME} ]]; then
   #   skip creating a new tmp directory, just set environment variables
   echo "Resuming from previous run using temporary storage at ${RESUME}"
   EESSI_HOST_STORAGE=${RESUME}
+  echo "RESUME_FROM_DIR ${EESSI_HOST_STORAGE}"
 else
   # we need a tmp location (and possibly init it with ${RESUME} if it was not
   #   a directory
@@ -325,7 +335,9 @@ fi
 
 # define paths to add to SINGULARITY_BIND (added later when all BIND mounts are defined)
 BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs"
-BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp"
+# provide a '/tmp' inside the container
+BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:${TMP_IN_CONTAINER}"
+
 [[ ${VERBOSE} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}"
 
 # set up repository config (always create directory repos_cfg and populate it with info when
@@ -381,13 +393,22 @@ else
   # use information to set up dir ${EESSI_TMPDIR}/repos_cfg,
   #     define BIND mounts and override repo name and version
   # check if config_bundle exists, if so, unpack it into ${EESSI_TMPDIR}/repos_cfg
-  if [[ ! -r ${config_bundle} ]]; then
-    fatal_error "config bundle '${config_bundle}' is not readable" ${REPOSITORY_ERROR_EXITCODE}
+  # if config_bundle is relative path (no '/' at start) prepend it with
+  # EESSI_REPOS_CFG_DIR
+  config_bundle_path=
+  if [[ ! "${config_bundle}" =~ ^/ ]]; then
+      config_bundle_path=${EESSI_REPOS_CFG_DIR}/${config_bundle}
+  else
+      config_bundle_path=${config_bundle}
+  fi
+
+  if [[ ! -r ${config_bundle_path} ]]; then
+    fatal_error "config bundle '${config_bundle_path}' is not readable" ${REPOSITORY_ERROR_EXITCODE}
   fi
 
   # only unpack config_bundle if we're not resuming from a previous run
   if [[ -z ${RESUME} ]]; then
-    tar xf ${config_bundle} -C ${EESSI_TMPDIR}/repos_cfg
+    tar xf ${config_bundle_path} -C ${EESSI_TMPDIR}/repos_cfg
   fi
 
   for src in "${!cfg_file_map[@]}"
@@ -436,12 +457,8 @@ if [[ "${ACCESS}" == "ro" ]]; then
 fi
 
 if [[ "${ACCESS}" == "rw" ]]; then
-  EESSI_CVMFS_OVERLAY_UPPER=/tmp/overlay-upper
-  EESSI_CVMFS_OVERLAY_WORK=/tmp/overlay-work
   mkdir -p ${EESSI_TMPDIR}/overlay-upper
   mkdir -p ${EESSI_TMPDIR}/overlay-work
-  [[ ${VERBOSE} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_UPPER=${EESSI_CVMFS_OVERLAY_UPPER}"
-  [[ ${VERBOSE} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_WORK=${EESSI_CVMFS_OVERLAY_WORK}"
 
   # set environment variables for fuse mounts in Singularity container
   export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}"
@@ -450,8 +467,8 @@ if [[ "${ACCESS}" == "rw" ]]; then
 
   EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs"
   EESSI_PILOT_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${repo_name}"
-  EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=/tmp/overlay-upper"
-  EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=/tmp/overlay-work"
+  EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/overlay-upper"
+  EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/overlay-work"
   EESSI_PILOT_WRITABLE_OVERLAY+=" ${EESSI_CVMFS_REPO}"
   export EESSI_PILOT_WRITABLE_OVERLAY
 
@@ -476,11 +493,13 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
     export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
 fi
 
+# if INFO is set to 1 (arg --info), add argument '-q'
+RUN_QUIET=${INFO:--q}
 echo "Launching container with command (next line):"
-echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@"
+echo "singularity ${RUN_QUIET} ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@"
 # TODO for now we run singularity with '-q' (quiet), later adjust this to the log level
 #      provided to the script
-singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@"
+singularity ${RUN_QUIET} ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@"
 exit_code=$?
 
 # 6. save tmp if requested (arg -s|--save)
@@ -500,6 +519,7 @@ if [[ ! -z ${SAVE} ]]; then
   fi
   tar cf ${TGZ} -C ${EESSI_TMPDIR} .
   echo "Saved contents of '${EESSI_TMPDIR}' to '${TGZ}' (to resume, add '--resume ${TGZ}')"
+  echo "RESUME_FROM_TGZ ${TGZ}"
 fi
 
 # TODO clean up tmp by default? only retain if another option provided (--retain-tmp)

From af658ee9229f36eac972bef87a36bbb6b883ef48 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Fri, 3 Feb 2023 20:58:01 +0100
Subject: [PATCH 25/58] fix issues in eessi_container.sh

---
 eessi_container.sh | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 14fb716abb..23118d0d72 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -258,7 +258,6 @@ if [[ ! -z ${RESUME} && -d ${RESUME} ]]; then
   #   skip creating a new tmp directory, just set environment variables
   echo "Resuming from previous run using temporary storage at ${RESUME}"
   EESSI_HOST_STORAGE=${RESUME}
-  echo "RESUME_FROM_DIR ${EESSI_HOST_STORAGE}"
 else
   # we need a tmp location (and possibly init it with ${RESUME} if it was not
   #   a directory
@@ -290,6 +289,7 @@ else
   EESSI_HOST_STORAGE=$(mktemp -d --tmpdir eessi.XXXXXXXXXX)
   echo "Using ${EESSI_HOST_STORAGE} as tmp storage (add '--resume ${EESSI_HOST_STORAGE}' to resume where this session ended)."
 fi
+echo "RESUME_FROM_DIR ${EESSI_HOST_STORAGE}"
 
 # if ${RESUME} is a file (assume a tgz), unpack it into ${EESSI_HOST_STORAGE}
 if [[ ! -z ${RESUME} && -f ${RESUME} ]]; then
@@ -494,7 +494,12 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
 fi
 
 # if INFO is set to 1 (arg --info), add argument '-q'
-RUN_QUIET=${INFO:--q}
+if [[ -z ${INFO} ]]; then
+    RUN_QUIET='-q'
+else
+    RUN_QUIET=''
+fi
+
 echo "Launching container with command (next line):"
 echo "singularity ${RUN_QUIET} ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@"
 # TODO for now we run singularity with '-q' (quiet), later adjust this to the log level

From eb66e1c4361f458114d2186a846fa9fe56b1df16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= <thomas.roblitz@uib.no>
Date: Sat, 4 Feb 2023 15:07:23 +0100
Subject: [PATCH 26/58] fix RUN_QUIET setting

---
 eessi_container.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 23118d0d72..89d4bd98f6 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -493,8 +493,8 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
     export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
 fi
 
-# if INFO is set to 1 (arg --info), add argument '-q'
-if [[ -z ${INFO} ]]; then
+# if INFO is set to 0 (no arg --info), add argument '-q'
+if [[ ${INFO} -eq 0 ]]; then
     RUN_QUIET='-q'
 else
     RUN_QUIET=''

From 1cb4674978bee259253330d769ca220ad795202a Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sat, 18 Feb 2023 21:59:39 +0100
Subject: [PATCH 27/58] improvements to resuming job environment

cherry-picked via commit 23e773ce0d8c6504e1440629a4178318265b73d2
NOTE only applied changes to eessi_container.sh
---
 eessi_container.sh | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 89d4bd98f6..e714f8e2d0 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -360,6 +360,9 @@ else
   #    standard EESSI repositories)
   cfg_load ${EESSI_REPOS_CFG_FILE}
 
+  # copy repos.cfg to job directory --> makes it easier to inspect the job
+  cp ${EESSI_REPOS_CFG_FILE} ${EESSI_TMPDIR}/repos_cfg/.
+
   # cfg file should include: repo_name, repo_version, config_bundle,
   #   map { local_filepath -> container_filepath }
   #
@@ -493,8 +496,8 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
     export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
 fi
 
-# if INFO is set to 0 (no arg --info), add argument '-q'
-if [[ ${INFO} -eq 0 ]]; then
+# if INFO is set to 1 (arg --info), add argument '-q'
+if [[ -z ${INFO} ]]; then
     RUN_QUIET='-q'
 else
     RUN_QUIET=''

From 0c465da3de12ed1bb7494651031d64d7f17928e4 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Tue, 7 Feb 2023 19:27:07 +0100
Subject: [PATCH 28/58] address requested change

---
 eessi_container.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index e714f8e2d0..e8698cd719 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -496,8 +496,8 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
     export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
 fi
 
-# if INFO is set to 1 (arg --info), add argument '-q'
-if [[ -z ${INFO} ]]; then
+# if INFO is set to 0 (no arg --info), add argument '-q'
+if [[ ${INFO} -eq 0 ]]; then
     RUN_QUIET='-q'
 else
     RUN_QUIET=''

From 26ab98c1f1cc901d6905424c9fdb2f5c9395ed79 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sat, 18 Feb 2023 22:16:36 +0100
Subject: [PATCH 29/58] cleaning up leftovers after adding updates from NESSI

---
 eessi_container.sh | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index e8698cd719..e7ad625a15 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -100,10 +100,6 @@ display_help() {
   echo "                           [default: not set]; uses env var \$https_proxy if set"
   echo
   echo " If value for --mode is 'run', the SCRIPT provided is executed."
-  echo
-  echo " FEATURES/OPTIONS to be implemented:"
-  echo "  -d | --dry-run         -  run script except for executing the container,"
-  echo "                            print information about setup [default: false]"
 }
 
 # set defaults for command line arguments
@@ -496,8 +492,8 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
     export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
 fi
 
-# if INFO is set to 0 (no arg --info), add argument '-q'
-if [[ ${INFO} -eq 0 ]]; then
+# if VERBOSE is set to 0 (no arg --verbose), add argument '-q'
+if [[ ${VERBOSE} -eq 0 ]]; then
     RUN_QUIET='-q'
 else
     RUN_QUIET=''
@@ -505,8 +501,6 @@ fi
 
 echo "Launching container with command (next line):"
 echo "singularity ${RUN_QUIET} ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@"
-# TODO for now we run singularity with '-q' (quiet), later adjust this to the log level
-#      provided to the script
 singularity ${RUN_QUIET} ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@"
 exit_code=$?
 

From a44088e4c33694c0047a6e5874df780e3c2ff15e Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sat, 18 Feb 2023 22:21:42 +0100
Subject: [PATCH 30/58] moving cfg_files.sh to scripts dir

---
 eessi_container.sh                   | 2 +-
 cfg_files.sh => scripts/cfg_files.sh | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename cfg_files.sh => scripts/cfg_files.sh (100%)

diff --git a/eessi_container.sh b/eessi_container.sh
index e7ad625a15..6d3fbc0adc 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -31,7 +31,7 @@
 TOPDIR=$(dirname $(realpath $0))
 
 source ${TOPDIR}/scripts/utils.sh
-source ${TOPDIR}/cfg_files.sh
+source ${TOPDIR}/scripts/cfg_files.sh
 
 # exit codes: bitwise shift codes to allow for combination of exit codes
 # ANY_ERROR_EXITCODE is sourced from ${TOPDIR}/scripts/utils.sh
diff --git a/cfg_files.sh b/scripts/cfg_files.sh
similarity index 100%
rename from cfg_files.sh
rename to scripts/cfg_files.sh

From bbbad0dce469299e4168a54a939facc4c1be7c60 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Mon, 20 Feb 2023 10:07:03 +0100
Subject: [PATCH 31/58] improved handling of container cache + explicit pull of
 image

cherry-picked via commit bfb1b29103dc309a12573f3ec9c247d432f08f73
Note, only part for eessi_container.sh included here
---
 eessi_container.sh | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 6d3fbc0adc..cdd2c017fb 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -310,10 +310,23 @@ mkdir -p ${EESSI_TMPDIR}
 [[ ${VERBOSE} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}"
 
 # configure Singularity
-export SINGULARITY_CACHEDIR=${EESSI_TMPDIR}/singularity_cache
-mkdir -p ${SINGULARITY_CACHEDIR}
+if [[ -z ${SINGULARITY_CACHEDIR} ]]; then
+    export SINGULARITY_CACHEDIR=${EESSI_TMPDIR}/singularity_cache
+    mkdir -p ${SINGULARITY_CACHEDIR}
+fi
 [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}"
 
+# pull & convert image and reset CONTAINER
+CONTAINER_URL_FMT=".*://(.*)"
+if [[ ${CONTAINER} == ${CONTAINER_URL_FMT} ]]; then
+    CONTAINER_IMG=${BASH_REMATCH[1]//[:-]/_}.sif
+    singularity pull ${CONTAINER_IMG} ${CONTAINER}
+    if [[ -x ${CONTAINER_IMG} ]]; then
+        CONTAINER="${PWD}/${CONTAINER_IMG}"
+    fi
+fi
+[[ ${INFO} -eq 1 ]] && echo "CONTAINER='${CONTAINER}'"
+
 # set env vars and create directories for CernVM-FS
 EESSI_CVMFS_VAR_LIB=${EESSI_TMPDIR}/${CVMFS_VAR_LIB}
 EESSI_CVMFS_VAR_RUN=${EESSI_TMPDIR}/${CVMFS_VAR_RUN}
@@ -326,8 +339,8 @@ mkdir -p ${EESSI_CVMFS_VAR_RUN}
 if [[ -z ${SINGULARITY_HOME} ]]; then
   export SINGULARITY_HOME="${EESSI_TMPDIR}/home:/home/${USER}"
   mkdir -p ${EESSI_TMPDIR}/home
-  [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_HOME=${SINGULARITY_HOME}"
 fi
+[[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_HOME=${SINGULARITY_HOME}"
 
 # define paths to add to SINGULARITY_BIND (added later when all BIND mounts are defined)
 BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs"

From 15ae58df240b1b1396611ee6fb5b5dbd50d4e793 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Mon, 20 Feb 2023 10:30:17 +0100
Subject: [PATCH 32/58] use VERBOSE instead of INFO

---
 .../workflows/test_eessi_container_script.yml | 22 +++++++++++++++++++
 eessi_container.sh                            |  2 +-
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml
index 7678aedc64..5a44bae155 100644
--- a/.github/workflows/test_eessi_container_script.yml
+++ b/.github/workflows/test_eessi_container_script.yml
@@ -11,6 +11,8 @@ jobs:
       matrix:
         SCRIPT_TEST:
         - help
+        - listrepos_default
+        - listrepos_custom
         - run
         - shell
         - container
@@ -39,6 +41,26 @@ jobs:
               if [[ ${{matrix.SCRIPT_TEST}} == 'help' ]]; then
                 ./eessi_container.sh --help
 
+              # test use of --list-repos without custom repos.cfg
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_default' ]]; then
+                outfile=out_listrepos.txt
+                ./eessi_container.sh --verbose --list-repos | tee ${outfile}
+                grep "EESSI-pilot" ${outfile}
+
+              # test use of --list-repos with custom repos.cfg
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_custom' ]]; then
+                outfile=out_listrepos.txt
+                outfile2=out_listrepos_2.txt
+                mkdir -p ${PWD}/cfg
+                echo "[EESSI/2021.12]" > cfg/repos.cfg
+                echo "[EESSI/2023.02]" >> cfg/repos.cfg
+                ./eessi_container.sh --verbose --list-repos | tee ${outfile}
+                grep "EESSI-pilot" ${outfile}
+
+                export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
+                ./eessi_container.sh --verbose --list-repos | tee ${outfile2}
+                grep "[EESSI/2023.02]" ${outfile2}
+
               # test use of --mode run
               elif [[ ${{matrix.SCRIPT_TEST}} == 'run' ]]; then
                 outfile=out_run.txt
diff --git a/eessi_container.sh b/eessi_container.sh
index cdd2c017fb..76a4d34143 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -325,7 +325,7 @@ if [[ ${CONTAINER} == ${CONTAINER_URL_FMT} ]]; then
         CONTAINER="${PWD}/${CONTAINER_IMG}"
     fi
 fi
-[[ ${INFO} -eq 1 ]] && echo "CONTAINER='${CONTAINER}'"
+[[ ${VERBOSE} -eq 1 ]] && echo "CONTAINER=${CONTAINER}"
 
 # set env vars and create directories for CernVM-FS
 EESSI_CVMFS_VAR_LIB=${EESSI_TMPDIR}/${CVMFS_VAR_LIB}

From 764e7138b0f8dfd82c81c1647bf78d04da1aec68 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Mon, 20 Feb 2023 10:48:38 +0100
Subject: [PATCH 33/58] fix test for --list-repos; improve output for
 --list-repos

---
 .../workflows/test_eessi_container_script.yml   |  2 ++
 eessi_container.sh                              | 17 ++++++++++-------
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml
index 5a44bae155..74cfd1fb10 100644
--- a/.github/workflows/test_eessi_container_script.yml
+++ b/.github/workflows/test_eessi_container_script.yml
@@ -53,7 +53,9 @@ jobs:
                 outfile2=out_listrepos_2.txt
                 mkdir -p ${PWD}/cfg
                 echo "[EESSI/2021.12]" > cfg/repos.cfg
+                echo "repo_version = 2021.12" >> cfg/repos.cfg
                 echo "[EESSI/2023.02]" >> cfg/repos.cfg
+                echo "repo_version = 2023.02" >> cfg/repos.cfg
                 ./eessi_container.sh --verbose --list-repos | tee ${outfile}
                 grep "EESSI-pilot" ${outfile}
 
diff --git a/eessi_container.sh b/eessi_container.sh
index 76a4d34143..dec082ce04 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -186,15 +186,18 @@ done
 
 set -- "${POSITIONAL_ARGS[@]}"
 
+
 if [[ ${LIST_REPOS} -eq 1 ]]; then
-    echo "Repositories defined in the config file '${EESSI_REPOS_CFG_FILE}':"
+    echo "Listing available repositories with format 'name [source]':"
     echo "    EESSI-pilot [default]"
-    cfg_load ${EESSI_REPOS_CFG_FILE}
-    sections=$(cfg_sections)
-    while IFS= read -r repo_id
-    do
-        echo "    ${repo_id}"
-    done <<< "${sections}"
+    if [[ -r ${EESSI_REPOS_CFG_FILE} ]]; then
+        cfg_load ${EESSI_REPOS_CFG_FILE}
+        sections=$(cfg_sections)
+        while IFS= read -r repo_id
+        do
+            echo "    ${repo_id} [${EESSI_REPOS_CFG_FILE}]"
+        done <<< "${sections}"
+    fi
     exit 0
 fi
 

From 47cc1c6be44896675949567d88f5ae85a998b64f Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Tue, 21 Feb 2023 21:27:56 +0100
Subject: [PATCH 34/58] implemented requested changes

---
 .../workflows/test_eessi_container_script.yml |  8 +--
 eessi_container.sh                            | 59 ++++++++++++++-----
 2 files changed, 48 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml
index 74cfd1fb10..d4a9dfd53e 100644
--- a/.github/workflows/test_eessi_container_script.yml
+++ b/.github/workflows/test_eessi_container_script.yml
@@ -52,10 +52,10 @@ jobs:
                 outfile=out_listrepos.txt
                 outfile2=out_listrepos_2.txt
                 mkdir -p ${PWD}/cfg
-                echo "[EESSI/2021.12]" > cfg/repos.cfg
-                echo "repo_version = 2021.12" >> cfg/repos.cfg
-                echo "[EESSI/2023.02]" >> cfg/repos.cfg
-                echo "repo_version = 2023.02" >> cfg/repos.cfg
+                echo "[EESSI/20AB.CD]" > cfg/repos.cfg
+                echo "repo_version = 20AB.CD" >> cfg/repos.cfg
+                echo "[EESSI/20HT.TP]" >> cfg/repos.cfg
+                echo "repo_version = 20HT.TP" >> cfg/repos.cfg
                 ./eessi_container.sh --verbose --list-repos | tee ${outfile}
                 grep "EESSI-pilot" ${outfile}
 
diff --git a/eessi_container.sh b/eessi_container.sh
index dec082ce04..86a510d56e 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -54,12 +54,11 @@ CVMFS_VAR_RUN="var-run-cvmfs"
 # directory for tmp used inside container
 export TMP_IN_CONTAINER=/tmp
 
-# repository cfg file, default name (default location: $PWD)
-#   can be overwritten by setting env var EESSI_REPOS_CFG_DIR_OVERRIDE
-export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR_OVERRIDE:=${PWD}}/repos.cfg"
-# other repository cfg files in directory, default location: $PWD
-#   can be overwritten by setting env var EESSI_REPOS_CFG_DIR_OVERRIDE
+# repository cfg directory and file
+#   directory: default $PWD or EESSI_REPOS_CFG_DIR_OVERRIDE if set
+#   file: directory + '/repos.cfg'
 export EESSI_REPOS_CFG_DIR="${EESSI_REPOS_CFG_DIR_OVERRIDE:=${PWD}}"
+export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR}/repos.cfg"
 
 
 # 0. parse args
@@ -288,7 +287,6 @@ else
   EESSI_HOST_STORAGE=$(mktemp -d --tmpdir eessi.XXXXXXXXXX)
   echo "Using ${EESSI_HOST_STORAGE} as tmp storage (add '--resume ${EESSI_HOST_STORAGE}' to resume where this session ended)."
 fi
-echo "RESUME_FROM_DIR ${EESSI_HOST_STORAGE}"
 
 # if ${RESUME} is a file (assume a tgz), unpack it into ${EESSI_HOST_STORAGE}
 if [[ ! -z ${RESUME} && -f ${RESUME} ]]; then
@@ -312,21 +310,52 @@ EESSI_TMPDIR=${EESSI_HOST_STORAGE}
 mkdir -p ${EESSI_TMPDIR}
 [[ ${VERBOSE} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}"
 
-# configure Singularity
+# configure Singularity: if SINGULARITY_CACHEDIR is already defined, use that
+#   a global SINGULARITY_CACHEDIR would ensure that we don't consume
+#   storage space again and again for the container & also speed-up
+#   launch times across different sessions
 if [[ -z ${SINGULARITY_CACHEDIR} ]]; then
     export SINGULARITY_CACHEDIR=${EESSI_TMPDIR}/singularity_cache
     mkdir -p ${SINGULARITY_CACHEDIR}
 fi
 [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}"
 
-# pull & convert image and reset CONTAINER
+# we try our best to make sure that we retain access to the container image in
+# a subsequent session ("best effort" only because pulling or copying operations
+# can fail ... in those cases the script may still succeed, but it is not
+# guaranteed that we have access to the same container when resuming later on)
+# - if CONTAINER references an image in a registry, pull & convert image
+#   and store it in ${EESSI_TMPDIR}
+#   + however, only pull image if there is no matching image in ${EESSI_TMPDIR} yet
+# - if CONTAINER references an image file, copy it to ${EESSI_TMPDIR}
+#   + however, only copy it if its base name does not yet exist in ${EESSI_TMPDIR}
+# - if the image file created (pulled or copied) or resumed exists in
+#   ${EESSI_TMPDIR}, let CONTAINER point to it
+#   + thus subsequent singularity commands in this script would just use the
+#     image file in EESSI_TMPDIR or the originally given source (some URL or
+#     path to an image file)
+CONTAINER_IMG=
 CONTAINER_URL_FMT=".*://(.*)"
-if [[ ${CONTAINER} == ${CONTAINER_URL_FMT} ]]; then
+if [[ ${CONTAINER} =~ ${CONTAINER_URL_FMT} ]]; then
+    # replace : and - with _ in match (everything after ://) and append .sif
     CONTAINER_IMG=${BASH_REMATCH[1]//[:-]/_}.sif
-    singularity pull ${CONTAINER_IMG} ${CONTAINER}
-    if [[ -x ${CONTAINER_IMG} ]]; then
-        CONTAINER="${PWD}/${CONTAINER_IMG}"
+    # pull container to ${EESSI_TMPDIR} if it is not there yet (i.e. when
+    # resuming from a previous session)
+    if [[ ! -x ${EESSI_TMPDIR}/${CONTAINER_IMG} ]]; then
+        singularity pull ${EESSI_TMPDIR}/${CONTAINER_IMG} ${CONTAINER}
     fi
+else
+    # determine file name as basename of CONTAINER
+    CONTAINER_IMG=$(basename ${CONTAINER})
+    # copy image file to ${EESSI_TMPDIR} if it is not there yet (i.e. when
+    # resuming from a previous session)
+    if [[ ! -x ${EESSI_TMPDIR}/${CONTAINER_IMG} ]]; then
+        cp -a ${CONTAINER} ${EESSI_TMPDIR}/.
+    fi
+fi
+# let CONTAINER point to the pulled, copied or resumed image file
+if [[ -x ${EESSI_TMPDIR}/${CONTAINER_IMG} ]]; then
+    CONTAINER="${EESSI_TMPDIR}/${CONTAINER_IMG}"
 fi
 [[ ${VERBOSE} -eq 1 ]] && echo "CONTAINER=${CONTAINER}"
 
@@ -373,7 +402,7 @@ else
   cfg_load ${EESSI_REPOS_CFG_FILE}
 
   # copy repos.cfg to job directory --> makes it easier to inspect the job
-  cp ${EESSI_REPOS_CFG_FILE} ${EESSI_TMPDIR}/repos_cfg/.
+  cp -a ${EESSI_REPOS_CFG_FILE} ${EESSI_TMPDIR}/repos_cfg/.
 
   # cfg file should include: repo_name, repo_version, config_bundle,
   #   map { local_filepath -> container_filepath }
@@ -453,7 +482,8 @@ if [[ ! -z ${http_proxy} ]]; then
     [[ ${VERBOSE} -eq 1 ]] && echo "HTTP_PROXY_IPV4='${HTTP_PROXY_IPV4}'"
     echo "CVMFS_HTTP_PROXY=\"${http_proxy}|http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \
        >> ${EESSI_TMPDIR}/repos_cfg/default.local
-    cat ${EESSI_TMPDIR}/repos_cfg/default.local
+    [[ ${VERBOSE} -eq 1 ]] && echo "contents of default.local"
+    [[ ${VERBOSE} -eq 1 ]] && cat ${EESSI_TMPDIR}/repos_cfg/default.local
 
     # if default.local is not BIND mounted into container, add it to BIND_PATHS
     if [[ ! ${BIND_PATHS} =~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then
@@ -537,7 +567,6 @@ if [[ ! -z ${SAVE} ]]; then
   fi
   tar cf ${TGZ} -C ${EESSI_TMPDIR} .
   echo "Saved contents of '${EESSI_TMPDIR}' to '${TGZ}' (to resume, add '--resume ${TGZ}')"
-  echo "RESUME_FROM_TGZ ${TGZ}"
 fi
 
 # TODO clean up tmp by default? only retain if another option provided (--retain-tmp)

From 712c40e567842da1c50b8c196fad0c9c5d30a69a Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Wed, 22 Feb 2023 22:54:43 +0100
Subject: [PATCH 35/58] improvements to resuming job environment

cherry-picked via 23e773ce0d8c6504e1440629a4178318265b73d2
Note only applied part for bot/build.sh
---
 bot/build.sh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index 8565511bfa..2c15b904f3 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -141,7 +141,7 @@ REPOSITORY_OPT=
 if [[ ! -z ${REPOSITORY} ]]; then
     REPOSITORY_OPT="--repository ${REPOSITORY}"
 fi
-mkdir -p previous_tmp
+mkdir -p previous_tmp/{build_step,tarball_step}
 build_outerr=$(mktemp build.outerr.XXXX)
 echo "Executing command to build software:"
 echo "./eessi_container.sh --access rw"
@@ -151,7 +151,7 @@ echo "                     ${HTTPS_PROXY_OPT}"
 echo "                     --info"
 echo "                     --mode run"
 echo "                     ${REPOSITORY_OPT}"
-echo "                     --save ${PWD}/previous_tmp"
+echo "                     --save ${PWD}/previous_tmp/build_step"
 echo "                     --storage ${STORAGE}"
 echo "                     ./install_software_layer.sh \"$@\" 2>&1 | tee -a ${build_outerr}"
 # set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg
@@ -163,7 +163,7 @@ export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
                      --info \
                      --mode run \
                      ${REPOSITORY_OPT} \
-                     --save ${PWD}/previous_tmp \
+                     --save ${PWD}/previous_tmp/build_step \
                      --storage ${STORAGE} \
                      ./install_software_layer.sh "$@" 2>&1 | tee -a ${build_outerr}
 
@@ -190,7 +190,7 @@ echo "                     --info"
 echo "                     --mode run"
 echo "                     ${REPOSITORY_OPT}"
 echo "                     --resume ${BUILD_TMPDIR}"
-echo "                     --save ${PWD}/previous_tmp"
+echo "                     --save ${PWD}/previous_tmp/tarball_step"
 echo "                     ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}"
 ./eessi_container.sh --access rw \
                      ${CONTAINER_OPT} \
@@ -200,7 +200,7 @@ echo "                     ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT
                      --mode run \
                      ${REPOSITORY_OPT} \
                      --resume ${BUILD_TMPDIR} \
-                     --save ${PWD}/previous_tmp \
+                     --save ${PWD}/previous_tmp/tarball_step \
                      ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}
 
 exit 0

From 16d98572d4148049a3a9f6a4258467eaf785d9f4 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Wed, 22 Feb 2023 23:00:42 +0100
Subject: [PATCH 36/58] improved handling of container cache + explicit pull of
 image

cherry-picked via bfb1b29103dc309a12573f3ec9c247d432f08f73
Note, only applied part for bot/build.sh
---
 bot/build.sh | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/bot/build.sh b/bot/build.sh
index 2c15b904f3..06503e59a0 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -56,6 +56,12 @@ LOCAL_TMP=$(${YQ} '.site_config.local_tmp // ""' ${JOB_CFG_FILE})
 echo "LOCAL_TMP='${LOCAL_TMP}'"
 # TODO should local_tmp be mandatory? --> then we check here and exit if it is not provided
 
+SINGULARITY_CACHEDIR=$(${YQ} '.site_config.container_cachedir // ""' ${JOB_CFG_FILE})
+echo "SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'"
+if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then
+    export SINGULARITY_CACHEDIR
+fi
+
 echo -n "setting \$STORAGE by replacing any var in '${LOCAL_TMP}' -> "
 # replace any env variable in ${LOCAL_TMP} with its
 #   current value (e.g., a value that is local to the job)

From 9e8ca62cab90b013c6fb1b55180e37592aea02b5 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Wed, 22 Feb 2023 23:02:35 +0100
Subject: [PATCH 37/58] changed --info -> --verbose

---
 bot/build.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index 06503e59a0..77aa2d9077 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -154,7 +154,7 @@ echo "./eessi_container.sh --access rw"
 echo "                     ${CONTAINER_OPT}"
 echo "                     ${HTTP_PROXY_OPT}"
 echo "                     ${HTTPS_PROXY_OPT}"
-echo "                     --info"
+echo "                     --verbose"
 echo "                     --mode run"
 echo "                     ${REPOSITORY_OPT}"
 echo "                     --save ${PWD}/previous_tmp/build_step"
@@ -166,7 +166,7 @@ export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
                      ${CONTAINER_OPT} \
                      ${HTTP_PROXY_OPT} \
                      ${HTTPS_PROXY_OPT} \
-                     --info \
+                     --verbose \
                      --mode run \
                      ${REPOSITORY_OPT} \
                      --save ${PWD}/previous_tmp/build_step \
@@ -192,7 +192,7 @@ echo "./eessi_container.sh --access rw"
 echo "                     ${CONTAINER_OPT}"
 echo "                     ${HTTP_PROXY_OPT}"
 echo "                     ${HTTPS_PROXY_OPT}"
-echo "                     --info"
+echo "                     --verbose"
 echo "                     --mode run"
 echo "                     ${REPOSITORY_OPT}"
 echo "                     --resume ${BUILD_TMPDIR}"
@@ -202,7 +202,7 @@ echo "                     ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT
                      ${CONTAINER_OPT} \
                      ${HTTP_PROXY_OPT} \
                      ${HTTPS_PROXY_OPT} \
-                     --info \
+                     --verbose \
                      --mode run \
                      ${REPOSITORY_OPT} \
                      --resume ${BUILD_TMPDIR} \

From c0350a9da92f6632c9ba52a57a5a28e03cb9c040 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Wed, 22 Feb 2023 23:14:38 +0100
Subject: [PATCH 38/58] improved messages for resume info and adjusted parsing
 in bot/build.sh

---
 bot/build.sh       | 2 +-
 eessi_container.sh | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index 77aa2d9077..f9d8c9cede 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -174,7 +174,7 @@ export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
                      ./install_software_layer.sh "$@" 2>&1 | tee -a ${build_outerr}
 
 # determine temporary directory to resume from
-BUILD_TMPDIR=$(grep 'RESUME_FROM_DIR' ${build_outerr} | sed -e "s/^RESUME_FROM_DIR //")
+BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2)
 
 tar_outerr=$(mktemp tar.outerr.XXXX)
 timestamp=$(date +%s)
diff --git a/eessi_container.sh b/eessi_container.sh
index f1c755938c..060b003f33 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -284,7 +284,7 @@ else
     [[ ${VERBOSE} -eq 1 ]] && echo "skipping sanity checks for /tmp"
   fi
   EESSI_HOST_STORAGE=$(mktemp -d --tmpdir eessi.XXXXXXXXXX)
-  echo "Using ${EESSI_HOST_STORAGE} as tmp storage (add '--resume ${EESSI_HOST_STORAGE}' to resume where this session ended)."
+  echo "Using ${EESSI_HOST_STORAGE} as tmp directory (to resume session add '--resume ${EESSI_HOST_STORAGE}')."
 fi
 
 # if ${RESUME} is a file (assume a tgz), unpack it into ${EESSI_HOST_STORAGE}
@@ -565,7 +565,7 @@ if [[ ! -z ${SAVE} ]]; then
     TGZ=${SAVE}
   fi
   tar cf ${TGZ} -C ${EESSI_TMPDIR} .
-  echo "Saved contents of '${EESSI_TMPDIR}' to '${TGZ}' (to resume, add '--resume ${TGZ}')"
+  echo "Saved contents of tmp directory '${EESSI_TMPDIR}' to tarball '${TGZ}' (to resume session add '--resume ${TGZ}')"
 fi
 
 # TODO clean up tmp by default? only retain if another option provided (--retain-tmp)

From ef9a552aa1b0a96a7b13c4319b501e0a79459766 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Wed, 22 Feb 2023 23:25:27 +0100
Subject: [PATCH 39/58] delete one of two tmp storage tarballs created by
 bot/build.sh

---
 bot/build.sh | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/bot/build.sh b/bot/build.sh
index f9d8c9cede..5926f005d5 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -209,4 +209,10 @@ echo "                     ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT
                      --save ${PWD}/previous_tmp/tarball_step \
                      ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}
 
+# if two tarballs have been generated, only keep the one from tarball step
+NUM_TARBALLS=$(find ${PWD}/previous_tmp -type f -name "*tgz" | wc -l)
+if [[ ${NUM_TARBALLS} -eq 2 ]]; then
+    rm -f previous_tmp/build_step/*.tgz
+fi
+
 exit 0

From 6974e4b80e3a59643d4eac0f229f50f8f547d430 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 24 Feb 2023 13:33:54 +0100
Subject: [PATCH 40/58] add verbose messages on pulling/copying/reusing of
 container image

---
 eessi_container.sh | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 86a510d56e..01c69bdd92 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -320,6 +320,13 @@ if [[ -z ${SINGULARITY_CACHEDIR} ]]; then
 fi
 [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}"
 
+# if VERBOSE is set to 0 (no arg --verbose), add argument '-q'
+if [[ ${VERBOSE} -eq 0 ]]; then
+    RUN_QUIET='-q'
+else
+    RUN_QUIET=''
+fi
+
 # we try our best to make sure that we retain access to the container image in
 # a subsequent session ("best effort" only because pulling or copying operations
 # can fail ... in those cases the script may still succeed, but it is not
@@ -342,7 +349,10 @@ if [[ ${CONTAINER} =~ ${CONTAINER_URL_FMT} ]]; then
     # pull container to ${EESSI_TMPDIR} if it is not there yet (i.e. when
     # resuming from a previous session)
     if [[ ! -x ${EESSI_TMPDIR}/${CONTAINER_IMG} ]]; then
-        singularity pull ${EESSI_TMPDIR}/${CONTAINER_IMG} ${CONTAINER}
+        [[ ${VERBOSE} -eq 1 ]] && echo "Pulling container image from ${CONTAINER} to ${EESSI_TMPDIR}/${CONTAINER_IMG}"
+        singularity ${RUN_QUIET} pull ${EESSI_TMPDIR}/${CONTAINER_IMG} ${CONTAINER}
+    else
+        [[ ${VERBOSE} -eq 1 ]] && echo "Reusing existing container image ${EESSI_TMPDIR}/${CONTAINER_IMG}"
     fi
 else
     # determine file name as basename of CONTAINER
@@ -350,7 +360,10 @@ else
     # copy image file to ${EESSI_TMPDIR} if it is not there yet (i.e. when
     # resuming from a previous session)
     if [[ ! -x ${EESSI_TMPDIR}/${CONTAINER_IMG} ]]; then
+        [[ ${VERBOSE} -eq 1 ]] && echo "Copying container image from ${CONTAINER} to ${EESSI_TMPDIR}/${CONTAINER_IMG}"
         cp -a ${CONTAINER} ${EESSI_TMPDIR}/.
+    else
+        [[ ${VERBOSE} -eq 1 ]] && echo "Reusing existing container image ${EESSI_TMPDIR}/${CONTAINER_IMG}"
     fi
 fi
 # let CONTAINER point to the pulled, copied or resumed image file
@@ -538,13 +551,6 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
     export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
 fi
 
-# if VERBOSE is set to 0 (no arg --verbose), add argument '-q'
-if [[ ${VERBOSE} -eq 0 ]]; then
-    RUN_QUIET='-q'
-else
-    RUN_QUIET=''
-fi
-
 echo "Launching container with command (next line):"
 echo "singularity ${RUN_QUIET} ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@"
 singularity ${RUN_QUIET} ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@"

From 54bdea52e86d853a923d4884c4b62682440c5b09 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 24 Feb 2023 13:37:47 +0100
Subject: [PATCH 41/58] don't require --verbose for info message on
 pulling/copying/reusing container image

---
 eessi_container.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 01c69bdd92..e98cc53493 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -349,10 +349,10 @@ if [[ ${CONTAINER} =~ ${CONTAINER_URL_FMT} ]]; then
     # pull container to ${EESSI_TMPDIR} if it is not there yet (i.e. when
     # resuming from a previous session)
     if [[ ! -x ${EESSI_TMPDIR}/${CONTAINER_IMG} ]]; then
-        [[ ${VERBOSE} -eq 1 ]] && echo "Pulling container image from ${CONTAINER} to ${EESSI_TMPDIR}/${CONTAINER_IMG}"
+        echo "Pulling container image from ${CONTAINER} to ${EESSI_TMPDIR}/${CONTAINER_IMG}"
         singularity ${RUN_QUIET} pull ${EESSI_TMPDIR}/${CONTAINER_IMG} ${CONTAINER}
     else
-        [[ ${VERBOSE} -eq 1 ]] && echo "Reusing existing container image ${EESSI_TMPDIR}/${CONTAINER_IMG}"
+        echo "Reusing existing container image ${EESSI_TMPDIR}/${CONTAINER_IMG}"
     fi
 else
     # determine file name as basename of CONTAINER
@@ -360,10 +360,10 @@ else
     # copy image file to ${EESSI_TMPDIR} if it is not there yet (i.e. when
     # resuming from a previous session)
     if [[ ! -x ${EESSI_TMPDIR}/${CONTAINER_IMG} ]]; then
-        [[ ${VERBOSE} -eq 1 ]] && echo "Copying container image from ${CONTAINER} to ${EESSI_TMPDIR}/${CONTAINER_IMG}"
+        echo "Copying container image from ${CONTAINER} to ${EESSI_TMPDIR}/${CONTAINER_IMG}"
         cp -a ${CONTAINER} ${EESSI_TMPDIR}/.
     else
-        [[ ${VERBOSE} -eq 1 ]] && echo "Reusing existing container image ${EESSI_TMPDIR}/${CONTAINER_IMG}"
+        echo "Reusing existing container image ${EESSI_TMPDIR}/${CONTAINER_IMG}"
     fi
 fi
 # let CONTAINER point to the pulled, copied or resumed image file

From a4cea9afb12242340360c9ad922e13c8492c9d57 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 24 Feb 2023 13:53:06 +0100
Subject: [PATCH 42/58] add check to make sure that container image exists in
 tmpdir

---
 .github/workflows/test_eessi_container_script.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml
index d4a9dfd53e..929fb22cec 100644
--- a/.github/workflows/test_eessi_container_script.yml
+++ b/.github/workflows/test_eessi_container_script.yml
@@ -108,6 +108,9 @@ jobs:
                 tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g")
                 rm -f ${outfile}
 
+                # make sure that container image exists
+                test -f ${tmpdir}/ghcr.io_eessi_build_node_debian11.sif || (echo "Container image not found in ${tmpdir}" >&2 && ls ${tmpdir} && exit 1)
+
                 ./eessi_container.sh --verbose --resume ${tmpdir} --mode shell <<< "${test_cmd}" > ${outfile}
                 cat ${outfile}
                 grep "Resuming from previous run using temporary storage at ${tmpdir}" ${outfile}

From c9758326512d824ae11e790f836c4db36dc3c527 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 24 Feb 2023 15:46:52 +0100
Subject: [PATCH 43/58] fix determining filename from container URL

---
 eessi_container.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index e98cc53493..1d76360735 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -344,8 +344,8 @@ fi
 CONTAINER_IMG=
 CONTAINER_URL_FMT=".*://(.*)"
 if [[ ${CONTAINER} =~ ${CONTAINER_URL_FMT} ]]; then
-    # replace : and - with _ in match (everything after ://) and append .sif
-    CONTAINER_IMG=${BASH_REMATCH[1]//[:-]/_}.sif
+    # replace ':', '-', '/' with '_' in match (everything after ://) and append .sif
+    CONTAINER_IMG="$(echo ${BASH_REMATCH[1]} | sed 's/[:\/-]/_/g').sif"
     # pull container to ${EESSI_TMPDIR} if it is not there yet (i.e. when
     # resuming from a previous session)
     if [[ ! -x ${EESSI_TMPDIR}/${CONTAINER_IMG} ]]; then

From 7cc65750c50bae0a2ea3f0cb9d96137fb21aa452 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Sat, 25 Feb 2023 02:55:37 +0100
Subject: [PATCH 44/58] support cmd line arg flag terminator

---
 eessi_container.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/eessi_container.sh b/eessi_container.sh
index b1f8922d61..b9553a7ea1 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -173,6 +173,11 @@ while [[ $# -gt 0 ]]; do
       export https_proxy=${HTTPS_PROXY}
       shift 2
       ;;
+    --)
+      shift
+      POSITIONAL_ARGS+=("$@") # save positional args
+      break
+      ;;
     -*|--*)
       fatal_error "Unknown option: $1" "${CMDLINE_ARG_UNKNOWN_EXITCODE}"
       ;;

From 0def4e91ce9d52281e0e8fc3d30c96f095c75bd5 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Fri, 24 Feb 2023 22:07:02 +0100
Subject: [PATCH 45/58] just print contents of cfg/job.cfg

---
 bot/build.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/bot/build.sh b/bot/build.sh
index 5926f005d5..906f1bfd7c 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -23,6 +23,10 @@
 # defaults
 export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}"
 
+echo "bot/build.sh: Showing job.cfg from software-layer side"
+cat cfg/job.cfg
+exit 0
+
 # source utils.sh
 source utils.sh
 

From ef608d15f20bea94bae116b7ece71a21586c7e3c Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Fri, 24 Feb 2023 23:55:44 +0100
Subject: [PATCH 46/58] switching from json (YQ) to ini (cfg_files.sh)

---
 bot/build.sh | 43 ++++++++++++++++---------------------------
 1 file changed, 16 insertions(+), 27 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index 906f1bfd7c..c621b9524f 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -17,50 +17,39 @@
 #  - the working directory contains a directory 'cfg' where the main config
 #    file 'job.cfg' has been deposited
 #  - the directory may contain any additional files references in job.cfg
-#  - the tool 'yq' for working with json files is available via the PATH or
-#    the environment variable BOT_YQ (see https://github.com/mikefarah/yq)
 
 # defaults
 export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}"
 
 echo "bot/build.sh: Showing job.cfg from software-layer side"
 cat cfg/job.cfg
-exit 0
-
-# source utils.sh
-source utils.sh
 
-# check setup / define key variables
-# get path for 'yq' (if not found, an empty string is returned)
-YQ=$(get_path_for_tool "yq" "BOT_YQ")
-exit_code=$?
-if [[ ${exit_code} -ne 0 ]]; then
-    fatal_error "could not find path to 'yq'; exiting"
-else
-    echo_green "found yq (${YQ})"
-fi
+# source utils.sh and cfg_files.sh
+source scripts/utils.sh
+source scripts/cfg_files.sh
 
 # check if './cfg/job.cfg' exists
 if [[ ! -r "${JOB_CFG_FILE}" ]]; then
     fatal_error "job config file (JOB_CFG_FILE=${JOB_CFG_FILE}) does not exist or not readable"
 fi
 echo "obtaining configuration settings from '${JOB_CFG_FILE}'"
+cfg_load ${JOB_CFG_FILE}
 
 # if http_proxy is in cfg/job.cfg use it, if not use env var $http_proxy
-HTTP_PROXY=$(${YQ} '.site_config.http_proxy // ""' ${JOB_CFG_FILE})
+HTTP_PROXY=$(cfg_get_value "site_config" "http_proxy")
 HTTP_PROXY=${HTTP_PROXY:-${http_proxy}}
 echo "HTTP_PROXY='${HTTP_PROXY}'"
 
 # if https_proxy is in cfg/job.cfg use it, if not use env var $https_proxy
-HTTPS_PROXY=$(${YQ} '.site_config.https_proxy // ""' ${JOB_CFG_FILE})
+HTTPS_PROXY=$(cfg_get_value "site_config" "https_proxy")
 HTTPS_PROXY=${HTTPS_PROXY:-${https_proxy}}
 echo "HTTPS_PROXY='${HTTPS_PROXY}'"
 
-LOCAL_TMP=$(${YQ} '.site_config.local_tmp // ""' ${JOB_CFG_FILE})
+LOCAL_TMP=$(cfg_get_value "site_config" "local_tmp")
 echo "LOCAL_TMP='${LOCAL_TMP}'"
 # TODO should local_tmp be mandatory? --> then we check here and exit if it is not provided
 
-SINGULARITY_CACHEDIR=$(${YQ} '.site_config.container_cachedir // ""' ${JOB_CFG_FILE})
+SINGULARITY_CACHEDIR=$(cfg_get_value "site_config" "container_cachedir")
 echo "SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'"
 if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then
     export SINGULARITY_CACHEDIR
@@ -73,11 +62,11 @@ STORAGE=$(envsubst <<< ${LOCAL_TMP})
 echo "'${STORAGE}'"
 
 # obtain list of modules to be loaded
-LOAD_MODULES=$(${YQ} '.site_config.load_modules // ""' ${JOB_CFG_FILE})
+LOAD_MODULES=$(cfg_get_value "site_config" "load_modules")
 echo "LOAD_MODULES='${LOAD_MODULES}'"
 
 # singularity/apptainer settings: CONTAINER, HOME, TMPDIR, BIND
-CONTAINER=$(${YQ} '.repository.container // ""' ${JOB_CFG_FILE})
+CONTAINER=$(cfg_get_value "repository" "container")
 export SINGULARITY_HOME="$(pwd):/eessi_bot_job"
 export SINGULARITY_TMPDIR="$(pwd)/singularity_tmpdir"
 mkdir -p ${SINGULARITY_TMPDIR}
@@ -94,20 +83,20 @@ else
 fi
 
 # determine repository to be used from entry .repository in cfg/job.cfg
-REPOSITORY=$(${YQ} '.repository.repo_id // ""' ${JOB_CFG_FILE})
-EESSI_REPOS_CFG_DIR_OVERRIDE=$(${YQ} '.repository.repos_cfg_dir // ""' ${JOB_CFG_FILE})
+REPOSITORY=$(cfg_get_value "repository" "repo_id")
+EESSI_REPOS_CFG_DIR_OVERRIDE=$(cfg_get_value "repository" "repos_cfg_dir")
 export EESSI_REPOS_CFG_DIR_OVERRIDE=${EESSI_REPOS_CFG_DIR_OVERRIDE:-${PWD}/cfg}
 
 # determine pilot version to be used from .repository.repo_version in cfg/job.cfg
 # here, just set & export EESSI_PILOT_VERSION_OVERRIDE
 # next script (eessi_container.sh) makes use of it via sourcing init scripts
 # (e.g., init/eessi_defaults or init/minimal_eessi_env)
-export EESSI_PILOT_VERSION_OVERRIDE=$(${YQ} '.repository.repo_version // ""' ${JOB_CFG_FILE})
+export EESSI_PILOT_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version")
 
 # determine CVMFS repo to be used from .repository.repo_name in cfg/job.cfg
 # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down
 # "source init/eessi_defaults" via sourcing init/minimal_eessi_env
-export EESSI_CVMFS_REPO_OVERRIDE=$(${YQ} '.repository.repo_name // ""' ${JOB_CFG_FILE})
+export EESSI_CVMFS_REPO_OVERRIDE=$(cfg_get_value "repository" "repo_name")
 
 
 # determine architecture to be used from entry .architecture in cfg/job.cfg
@@ -115,12 +104,12 @@ export EESSI_CVMFS_REPO_OVERRIDE=$(${YQ} '.repository.repo_name // ""' ${JOB_CFG
 if [[ ! -z "${CPU_TARGET}" ]]; then
     EESSI_SOFTWARE_SUBDIR_OVERRIDE=${CPU_TARGET}
 else
-    EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(${YQ} '.architecture.software_subdir // ""' ${JOB_CFG_FILE})
+    EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(cfg_get_value "architecture" "software_subdir")
 fi
 export EESSI_SOFTWARE_SUBDIR_OVERRIDE
 
 # get EESSI_OS_TYPE from .architecture.os_type in cfg/job.cfg (default: linux)
-EESSI_OS_TYPE=$(${YQ} '.architecture.os_type // ""' ${JOB_CFG_FILE})
+EESSI_OS_TYPE=$(cfg_get_value "architecture" "os_type")
 export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux}
 
 # TODO

From 429d5a03ad0da75e8e8c52fb8897b341d1c70511 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Thu, 23 Feb 2023 23:56:28 +0100
Subject: [PATCH 47/58] add --generic arg when running install script

---
 bot/build.sh | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index c621b9524f..387a9c95c9 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -140,6 +140,11 @@ REPOSITORY_OPT=
 if [[ ! -z ${REPOSITORY} ]]; then
     REPOSITORY_OPT="--repository ${REPOSITORY}"
 fi
+GENERIC_OPT=
+if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ ".*/generic" ]]; then
+    GENERIC_OPT="--generic"
+fi
+
 mkdir -p previous_tmp/{build_step,tarball_step}
 build_outerr=$(mktemp build.outerr.XXXX)
 echo "Executing command to build software:"
@@ -152,7 +157,7 @@ echo "                     --mode run"
 echo "                     ${REPOSITORY_OPT}"
 echo "                     --save ${PWD}/previous_tmp/build_step"
 echo "                     --storage ${STORAGE}"
-echo "                     ./install_software_layer.sh \"$@\" 2>&1 | tee -a ${build_outerr}"
+echo "                     ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}"
 # set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg
 export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
 ./eessi_container.sh --access rw \
@@ -164,7 +169,7 @@ export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
                      ${REPOSITORY_OPT} \
                      --save ${PWD}/previous_tmp/build_step \
                      --storage ${STORAGE} \
-                     ./install_software_layer.sh "$@" 2>&1 | tee -a ${build_outerr}
+                     ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr}
 
 # determine temporary directory to resume from
 BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2)

From bd31faa6b6e4e3945447118b6121667d01b4af7d Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Fri, 24 Feb 2023 00:17:57 +0100
Subject: [PATCH 48/58] fix error in regex

---
 bot/build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bot/build.sh b/bot/build.sh
index 387a9c95c9..860f888a46 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -141,7 +141,7 @@ if [[ ! -z ${REPOSITORY} ]]; then
     REPOSITORY_OPT="--repository ${REPOSITORY}"
 fi
 GENERIC_OPT=
-if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ ".*/generic" ]]; then
+if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then
     GENERIC_OPT="--generic"
 fi
 

From 1bafb7889fa8f49b183e048d63a8fbdabee041f1 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Fri, 24 Feb 2023 00:43:23 +0100
Subject: [PATCH 49/58] use command separator --

---
 bot/build.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index 860f888a46..cdd59d6081 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -157,7 +157,7 @@ echo "                     --mode run"
 echo "                     ${REPOSITORY_OPT}"
 echo "                     --save ${PWD}/previous_tmp/build_step"
 echo "                     --storage ${STORAGE}"
-echo "                     ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}"
+echo "                     -- ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}"
 # set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg
 export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
 ./eessi_container.sh --access rw \
@@ -169,7 +169,7 @@ export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
                      ${REPOSITORY_OPT} \
                      --save ${PWD}/previous_tmp/build_step \
                      --storage ${STORAGE} \
-                     ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr}
+                     -- ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr}
 
 # determine temporary directory to resume from
 BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2)

From c9041e34604ec881e6a5aeb96fbf31da91bf63a1 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Sat, 25 Feb 2023 00:29:25 +0100
Subject: [PATCH 50/58] try fixing arg issue

---
 bot/build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bot/build.sh b/bot/build.sh
index cdd59d6081..f56f456dcf 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -161,7 +161,7 @@ echo "                     -- ./install_software_layer.sh ${GENERIC_OPT} \"$@\"
 # set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg
 export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
 ./eessi_container.sh --access rw \
-                     ${CONTAINER_OPT} \
+                     "${CONTAINER_OPT}" \
                      ${HTTP_PROXY_OPT} \
                      ${HTTPS_PROXY_OPT} \
                      --verbose \

From 9d46a22892a4462f1b87c52c679d90cfed6256bb Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Sat, 25 Feb 2023 00:41:36 +0100
Subject: [PATCH 51/58] put build args into an array

---
 bot/build.sh | 27 +++++++++++----------------
 1 file changed, 11 insertions(+), 16 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index f56f456dcf..8720c137a5 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -124,21 +124,30 @@ export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux}
 #     files into './cfg/.' and defining '.repository.repos_cfg_dir' in './cfg/job.cfg')
 
 # prepare options and directories for calling eessi_container.sh
+declare -a BUILD_STEP_ARGS=()
+BUILD_STEP_ARGS+=("--access" "rw")
+BUILD_STEP_ARGS+=("--mode" "run")
+BUILD_STEP_ARGS+=("--save" "${PWD}/previous_tmp/build_step")
+BUILD_STEP_ARGS+=("--storage" "${STORAGE}")
 CONTAINER_OPT=
 if [[ ! -z ${CONTAINER} ]]; then
     CONTAINER_OPT="--container ${CONTAINER}"
+    BUILD_STEP_ARGS+=("--container" "${CONTAINER}")
 fi
 HTTP_PROXY_OPT=
 if [[ ! -z ${HTTP_PROXY} ]]; then
     HTTP_PROXY_OPT="--http-proxy ${HTTP_PROXY}"
+    BUILD_STEP_ARGS+=("--http-proxy" "${HTTP_PROXY}")
 fi
 HTTPS_PROXY_OPT=
 if [[ ! -z ${HTTPS_PROXY} ]]; then
     HTTPS_PROXY_OPT="--https-proxy ${HTTPS_PROXY}"
+    BUILD_STEP_ARGS+=("--https-proxy" "${HTTPS_PROXY}")
 fi
 REPOSITORY_OPT=
 if [[ ! -z ${REPOSITORY} ]]; then
     REPOSITORY_OPT="--repository ${REPOSITORY}"
+    BUILD_STEP_ARGS+=("--repository" "${REPOSITORY}")
 fi
 GENERIC_OPT=
 if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then
@@ -148,27 +157,13 @@ fi
 mkdir -p previous_tmp/{build_step,tarball_step}
 build_outerr=$(mktemp build.outerr.XXXX)
 echo "Executing command to build software:"
-echo "./eessi_container.sh --access rw"
-echo "                     ${CONTAINER_OPT}"
-echo "                     ${HTTP_PROXY_OPT}"
-echo "                     ${HTTPS_PROXY_OPT}"
+echo "./eessi_container.sh ${BUILD_STEP_ARGS[@]}"
 echo "                     --verbose"
-echo "                     --mode run"
-echo "                     ${REPOSITORY_OPT}"
-echo "                     --save ${PWD}/previous_tmp/build_step"
-echo "                     --storage ${STORAGE}"
 echo "                     -- ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}"
 # set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg
 export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
-./eessi_container.sh --access rw \
-                     "${CONTAINER_OPT}" \
-                     ${HTTP_PROXY_OPT} \
-                     ${HTTPS_PROXY_OPT} \
+./eessi_container.sh "${BUILD_STEP_ARGS[@]}" \
                      --verbose \
-                     --mode run \
-                     ${REPOSITORY_OPT} \
-                     --save ${PWD}/previous_tmp/build_step \
-                     --storage ${STORAGE} \
                      -- ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr}
 
 # determine temporary directory to resume from

From ff5045f0940730951c25cfb5260148655fbf933d Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Sat, 25 Feb 2023 01:19:25 +0100
Subject: [PATCH 52/58] use array to define args for build and tarball step

---
 bot/build.sh | 37 +++++++++++++++----------------------
 1 file changed, 15 insertions(+), 22 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index 8720c137a5..bb1f4319a1 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -125,29 +125,39 @@ export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux}
 
 # prepare options and directories for calling eessi_container.sh
 declare -a BUILD_STEP_ARGS=()
+BUILD_STEP_ARGS+=("--verbose")
 BUILD_STEP_ARGS+=("--access" "rw")
 BUILD_STEP_ARGS+=("--mode" "run")
 BUILD_STEP_ARGS+=("--save" "${PWD}/previous_tmp/build_step")
 BUILD_STEP_ARGS+=("--storage" "${STORAGE}")
+declare -a TARBALL_STEP_ARGS=()
+TARBALL_STEP_ARGS+=("--verbose")
+TARBALL_STEP_ARGS+=("--access" "rw")
+TARBALL_STEP_ARGS+=("--mode" "run")
+TARBALL_STEP_ARGS+=("--save" "${PWD}/previous_tmp/tarball_step")
 CONTAINER_OPT=
 if [[ ! -z ${CONTAINER} ]]; then
     CONTAINER_OPT="--container ${CONTAINER}"
     BUILD_STEP_ARGS+=("--container" "${CONTAINER}")
+    TARBALL_STEP_ARGS+=("--container" "${CONTAINER}")
 fi
 HTTP_PROXY_OPT=
 if [[ ! -z ${HTTP_PROXY} ]]; then
     HTTP_PROXY_OPT="--http-proxy ${HTTP_PROXY}"
     BUILD_STEP_ARGS+=("--http-proxy" "${HTTP_PROXY}")
+    TARBALL_STEP_ARGS+=("--http-proxy" "${HTTP_PROXY}")
 fi
 HTTPS_PROXY_OPT=
 if [[ ! -z ${HTTPS_PROXY} ]]; then
     HTTPS_PROXY_OPT="--https-proxy ${HTTPS_PROXY}"
     BUILD_STEP_ARGS+=("--https-proxy" "${HTTPS_PROXY}")
+    TARBALL_STEP_ARGS+=("--https-proxy" "${HTTPS_PROXY}")
 fi
 REPOSITORY_OPT=
 if [[ ! -z ${REPOSITORY} ]]; then
     REPOSITORY_OPT="--repository ${REPOSITORY}"
     BUILD_STEP_ARGS+=("--repository" "${REPOSITORY}")
+    TARBALL_STEP_ARGS+=("--repository" "${REPOSITORY}")
 fi
 GENERIC_OPT=
 if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then
@@ -158,16 +168,15 @@ mkdir -p previous_tmp/{build_step,tarball_step}
 build_outerr=$(mktemp build.outerr.XXXX)
 echo "Executing command to build software:"
 echo "./eessi_container.sh ${BUILD_STEP_ARGS[@]}"
-echo "                     --verbose"
 echo "                     -- ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}"
 # set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg
 export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
 ./eessi_container.sh "${BUILD_STEP_ARGS[@]}" \
-                     --verbose \
                      -- ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr}
 
 # determine temporary directory to resume from
 BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2)
+TARBALL_STEP_ARGS+=("--resume" "${BUILD_TMPDIR}")
 
 tar_outerr=$(mktemp tar.outerr.XXXX)
 timestamp=$(date +%s)
@@ -181,26 +190,10 @@ export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_PILOT_VERSION} $
 # /tmp as default?
 TMP_IN_CONTAINER=/tmp
 echo "Executing command to create tarball:"
-echo "./eessi_container.sh --access rw"
-echo "                     ${CONTAINER_OPT}"
-echo "                     ${HTTP_PROXY_OPT}"
-echo "                     ${HTTPS_PROXY_OPT}"
-echo "                     --verbose"
-echo "                     --mode run"
-echo "                     ${REPOSITORY_OPT}"
-echo "                     --resume ${BUILD_TMPDIR}"
-echo "                     --save ${PWD}/previous_tmp/tarball_step"
-echo "                     ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}"
-./eessi_container.sh --access rw \
-                     ${CONTAINER_OPT} \
-                     ${HTTP_PROXY_OPT} \
-                     ${HTTPS_PROXY_OPT} \
-                     --verbose \
-                     --mode run \
-                     ${REPOSITORY_OPT} \
-                     --resume ${BUILD_TMPDIR} \
-                     --save ${PWD}/previous_tmp/tarball_step \
-                     ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}
+echo "./eessi_container.sh ${TARBALL_STEP_ARGS[@]}"
+echo "                     -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}"
+./eessi_container.sh "${TARBALL_STEP_ARGS[@]}" \
+                     -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}
 
 # if two tarballs have been generated, only keep the one from tarball step
 NUM_TARBALLS=$(find ${PWD}/previous_tmp -type f -name "*tgz" | wc -l)

From d08bd211ed6c23c5b917b01bd01ef75a5cb7b1b9 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Tue, 7 Mar 2023 14:43:52 +0100
Subject: [PATCH 53/58] addressed comments and changes requested by first
 review

---
 bot/build.sh | 172 ++++++++++++++++++++++++---------------------------
 1 file changed, 82 insertions(+), 90 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index bb1f4319a1..004e2d4881 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -16,41 +16,44 @@
 #    pull request (OR by some other means)
 #  - the working directory contains a directory 'cfg' where the main config
 #    file 'job.cfg' has been deposited
-#  - the directory may contain any additional files references in job.cfg
+#  - the directory may contain any additional files referenced in job.cfg
 
-# defaults
-export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}"
-
-echo "bot/build.sh: Showing job.cfg from software-layer side"
-cat cfg/job.cfg
+# stop as soon as something fails
+set -e
 
 # source utils.sh and cfg_files.sh
 source scripts/utils.sh
 source scripts/cfg_files.sh
 
-# check if './cfg/job.cfg' exists
+# defaults
+export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}"
+
+# check if ${JOB_CFG_FILE} exists
 if [[ ! -r "${JOB_CFG_FILE}" ]]; then
     fatal_error "job config file (JOB_CFG_FILE=${JOB_CFG_FILE}) does not exist or not readable"
 fi
-echo "obtaining configuration settings from '${JOB_CFG_FILE}'"
+echo "bot/build.sh: showing ${JOB_CFG_FILE} from software-layer side"
+cat ${JOB_CFG_FILE}
+
+echo "bot/build.sh: obtaining configuration settings from '${JOB_CFG_FILE}'"
 cfg_load ${JOB_CFG_FILE}
 
-# if http_proxy is in cfg/job.cfg use it, if not use env var $http_proxy
+# if http_proxy is defined in ${JOB_CFG_FILE} use it, if not use env var $http_proxy
 HTTP_PROXY=$(cfg_get_value "site_config" "http_proxy")
 HTTP_PROXY=${HTTP_PROXY:-${http_proxy}}
-echo "HTTP_PROXY='${HTTP_PROXY}'"
+echo "bot/build.sh: HTTP_PROXY='${HTTP_PROXY}'"
 
-# if https_proxy is in cfg/job.cfg use it, if not use env var $https_proxy
+# if https_proxy is defined in ${JOB_CFG_FILE} use it, if not use env var $https_proxy
 HTTPS_PROXY=$(cfg_get_value "site_config" "https_proxy")
 HTTPS_PROXY=${HTTPS_PROXY:-${https_proxy}}
-echo "HTTPS_PROXY='${HTTPS_PROXY}'"
+echo "bot/build.sh: HTTPS_PROXY='${HTTPS_PROXY}'"
 
 LOCAL_TMP=$(cfg_get_value "site_config" "local_tmp")
-echo "LOCAL_TMP='${LOCAL_TMP}'"
+echo "bot/build.sh: LOCAL_TMP='${LOCAL_TMP}'"
 # TODO should local_tmp be mandatory? --> then we check here and exit if it is not provided
 
 SINGULARITY_CACHEDIR=$(cfg_get_value "site_config" "container_cachedir")
-echo "SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'"
+echo "bot/build.sh: SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'"
 if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then
     export SINGULARITY_CACHEDIR
 fi
@@ -61,14 +64,21 @@ echo -n "setting \$STORAGE by replacing any var in '${LOCAL_TMP}' -> "
 STORAGE=$(envsubst <<< ${LOCAL_TMP})
 echo "'${STORAGE}'"
 
+# make sure ${STORAGE} exists
+mkdir -p ${STORAGE}
+
+# make sure the base tmp storage is unique
+JOB_STORAGE=$(mktemp --directory --tmpdir=${STORAGE} bot_job_tmp_XXX)
+echo "bot/build.sh: created unique base tmp storage directory at ${JOB_STORAGE}"
+
 # obtain list of modules to be loaded
 LOAD_MODULES=$(cfg_get_value "site_config" "load_modules")
-echo "LOAD_MODULES='${LOAD_MODULES}'"
+echo "bot/build.sh: LOAD_MODULES='${LOAD_MODULES}'"
 
 # singularity/apptainer settings: CONTAINER, HOME, TMPDIR, BIND
 CONTAINER=$(cfg_get_value "repository" "container")
-export SINGULARITY_HOME="$(pwd):/eessi_bot_job"
-export SINGULARITY_TMPDIR="$(pwd)/singularity_tmpdir"
+export SINGULARITY_HOME="${pwd}:/eessi_bot_job"
+export SINGULARITY_TMPDIR="${pwd}/singularity_tmpdir"
 mkdir -p ${SINGULARITY_TMPDIR}
 
 # load modules if LOAD_MODULES is not empty
@@ -82,103 +92,91 @@ else
     echo "bot/build.sh: no modules to be loaded"
 fi
 
-# determine repository to be used from entry .repository in cfg/job.cfg
+# determine repository to be used from entry .repository in ${JOB_CFG_FILE}
 REPOSITORY=$(cfg_get_value "repository" "repo_id")
 EESSI_REPOS_CFG_DIR_OVERRIDE=$(cfg_get_value "repository" "repos_cfg_dir")
 export EESSI_REPOS_CFG_DIR_OVERRIDE=${EESSI_REPOS_CFG_DIR_OVERRIDE:-${PWD}/cfg}
+echo "bot/build.sh: EESSI_REPOS_CFG_DIR_OVERRIDE='${EESSI_REPOS_CFG_DIR_OVERRIDE}'"
 
-# determine pilot version to be used from .repository.repo_version in cfg/job.cfg
+# determine pilot version to be used from .repository.repo_version in ${JOB_CFG_FILE}
 # here, just set & export EESSI_PILOT_VERSION_OVERRIDE
 # next script (eessi_container.sh) makes use of it via sourcing init scripts
 # (e.g., init/eessi_defaults or init/minimal_eessi_env)
 export EESSI_PILOT_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version")
+echo "bot/build.sh: EESSI_PILOT_VERSION_OVERRIDE='${EESSI_PILOT_VERSION_OVERRIDE}'"
 
-# determine CVMFS repo to be used from .repository.repo_name in cfg/job.cfg
+# determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE}
 # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down
 # "source init/eessi_defaults" via sourcing init/minimal_eessi_env
 export EESSI_CVMFS_REPO_OVERRIDE=$(cfg_get_value "repository" "repo_name")
-
-
-# determine architecture to be used from entry .architecture in cfg/job.cfg
-# default: leave empty to let downstream script(s) determine subdir to be used
-if [[ ! -z "${CPU_TARGET}" ]]; then
-    EESSI_SOFTWARE_SUBDIR_OVERRIDE=${CPU_TARGET}
-else
-    EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(cfg_get_value "architecture" "software_subdir")
-fi
+echo "bot/build.sh: EESSI_CVMFS_REPO_OVERRIDE='${EESSI_CVMFS_REPO_OVERRIDE}'"
+
+# determine architecture to be used from entry .architecture in ${JOB_CFG_FILE}
+# fallbacks:
+#  - ${CPU_TARGET} handed over from bot
+#  - left empty to let downstream script(s) determine subdir to be used
+EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(cfg_get_value "architecture" "software_subdir")
+EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE:-${CPU_TARGET}}
 export EESSI_SOFTWARE_SUBDIR_OVERRIDE
+echo "bot/build.sh: EESSI_SOFTWARE_SUBDIR_OVERRIDE='${EESSI_SOFTWARE_SUBDIR_OVERRIDE}'"
 
-# get EESSI_OS_TYPE from .architecture.os_type in cfg/job.cfg (default: linux)
+# get EESSI_OS_TYPE from .architecture.os_type in ${JOB_CFG_FILE} (default: linux)
 EESSI_OS_TYPE=$(cfg_get_value "architecture" "os_type")
 export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux}
-
-# TODO
-#   - CODED add handling of EESSI_SOFTWARE_SUBDIR_OVERRIDE to eessi_container.sh
-#     TODO ensure that the bot makes use of that. (currently sets env var
-#     CPU_TARGET & adds --export=ALL,CPU_TARGET=val to sbatch command ... also
-#     add it to cfg/job.cfg - .architecture.software_subdir)
-#   - CODED add handling of http(s)_proxy to eessi_container.sh, in there needs the
-#     CVMFS_HTTP_PROXY added to /etc/cvmfs/default.local (this needs a robust
-#     way to determine the IP address of a proxy)
-#   - bot needs to make repos.cfg and cfg_bundle available to job (likely, by copying
-#     files into './cfg/.' and defining '.repository.repos_cfg_dir' in './cfg/job.cfg')
-
-# prepare options and directories for calling eessi_container.sh
+echo "bot/build.sh: EESSI_OS_TYPE='${EESSI_OS_TYPE}'"
+
+# prepare arguments to eessi_container.sh common to build and tarball steps
+declare -a COMMON_ARGS=()
+COMMON_ARGS+=("--verbose")
+COMMON_ARGS+=("--access" "rw")
+COMMON_ARGS+=("--mode" "run")
+[[ ! -z ${CONTAINER} ]] && COMMON_ARGS+=("--container" "${CONTAINER}")
+[[ ! -z ${HTTP_PROXY} ]] && COMMON_ARGS+=("--http-proxy" "${HTTP_PROXY}")
+[[ ! -z ${HTTPS_PROXY} ]] && COMMON_ARGS+=("--https-proxy" "${HTTPS_PROXY}")
+[[ ! -z ${REPOSITORY} ]] && COMMON_ARGS+=("--repository" "${REPOSITORY}")
+
+# make sure to use the same parent dir for storing tarballs of tmp
+PREVIOUS_TMP_DIR=${PWD}/previous_tmp
+
+# prepare directory to store tarball of tmp for build step
+TARBALL_TMP_BUILD_STEP_DIR=${PREVIOUS_TMP_DIR}/build_step
+mkdir -p ${TARBALL_TMP_BUILD_STEP_DIR}
+
+# prepare arguments to eessi_container.sh specific to build step
 declare -a BUILD_STEP_ARGS=()
-BUILD_STEP_ARGS+=("--verbose")
-BUILD_STEP_ARGS+=("--access" "rw")
-BUILD_STEP_ARGS+=("--mode" "run")
-BUILD_STEP_ARGS+=("--save" "${PWD}/previous_tmp/build_step")
+BUILD_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}")
 BUILD_STEP_ARGS+=("--storage" "${STORAGE}")
-declare -a TARBALL_STEP_ARGS=()
-TARBALL_STEP_ARGS+=("--verbose")
-TARBALL_STEP_ARGS+=("--access" "rw")
-TARBALL_STEP_ARGS+=("--mode" "run")
-TARBALL_STEP_ARGS+=("--save" "${PWD}/previous_tmp/tarball_step")
-CONTAINER_OPT=
-if [[ ! -z ${CONTAINER} ]]; then
-    CONTAINER_OPT="--container ${CONTAINER}"
-    BUILD_STEP_ARGS+=("--container" "${CONTAINER}")
-    TARBALL_STEP_ARGS+=("--container" "${CONTAINER}")
-fi
-HTTP_PROXY_OPT=
-if [[ ! -z ${HTTP_PROXY} ]]; then
-    HTTP_PROXY_OPT="--http-proxy ${HTTP_PROXY}"
-    BUILD_STEP_ARGS+=("--http-proxy" "${HTTP_PROXY}")
-    TARBALL_STEP_ARGS+=("--http-proxy" "${HTTP_PROXY}")
-fi
-HTTPS_PROXY_OPT=
-if [[ ! -z ${HTTPS_PROXY} ]]; then
-    HTTPS_PROXY_OPT="--https-proxy ${HTTPS_PROXY}"
-    BUILD_STEP_ARGS+=("--https-proxy" "${HTTPS_PROXY}")
-    TARBALL_STEP_ARGS+=("--https-proxy" "${HTTPS_PROXY}")
-fi
-REPOSITORY_OPT=
-if [[ ! -z ${REPOSITORY} ]]; then
-    REPOSITORY_OPT="--repository ${REPOSITORY}"
-    BUILD_STEP_ARGS+=("--repository" "${REPOSITORY}")
-    TARBALL_STEP_ARGS+=("--repository" "${REPOSITORY}")
-fi
+
+# prepare arguments to install_software_layer.sh (specific to build step)
 GENERIC_OPT=
 if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then
     GENERIC_OPT="--generic"
 fi
 
-mkdir -p previous_tmp/{build_step,tarball_step}
+# create tmp file for output of build step
 build_outerr=$(mktemp build.outerr.XXXX)
+
 echo "Executing command to build software:"
-echo "./eessi_container.sh ${BUILD_STEP_ARGS[@]}"
+echo "./eessi_container.sh ${COMMON_ARGS[@]} ${BUILD_STEP_ARGS[@]}"
 echo "                     -- ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}"
-# set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg
-export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
-./eessi_container.sh "${BUILD_STEP_ARGS[@]}" \
+./eessi_container.sh "${COMMON_ARGS[@]}" "${BUILD_STEP_ARGS[@]}" \
                      -- ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr}
 
+# prepare directory to store tarball of tmp for tarball step
+TARBALL_TMP_TARBALL_STEP_DIR=${PREVIOUS_TMP_DIR}/tarball_step
+mkdir -p ${TARBALL_TMP_TARBALL_STEP_DIR}
+
+# create tmp file for output of tarball step
+tar_outerr=$(mktemp tar.outerr.XXXX)
+
+# prepare arguments to eessi_container.sh specific to tarball step
+declare -a TARBALL_STEP_ARGS=()
+TARBALL_STEP_ARGS+=("--save" "${TARBALL_TMP_TARBALL_STEP_DIR}")
+
 # determine temporary directory to resume from
 BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2)
 TARBALL_STEP_ARGS+=("--resume" "${BUILD_TMPDIR}")
 
-tar_outerr=$(mktemp tar.outerr.XXXX)
 timestamp=$(date +%s)
 # to set EESSI_PILOT_VERSION we need to source init/eessi_defaults now
 source init/eessi_defaults
@@ -190,15 +188,9 @@ export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_PILOT_VERSION} $
 # /tmp as default?
 TMP_IN_CONTAINER=/tmp
 echo "Executing command to create tarball:"
-echo "./eessi_container.sh ${TARBALL_STEP_ARGS[@]}"
+echo "./eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}"
 echo "                     -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}"
-./eessi_container.sh "${TARBALL_STEP_ARGS[@]}" \
+./eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \
                      -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}
 
-# if two tarballs have been generated, only keep the one from tarball step
-NUM_TARBALLS=$(find ${PWD}/previous_tmp -type f -name "*tgz" | wc -l)
-if [[ ${NUM_TARBALLS} -eq 2 ]]; then
-    rm -f previous_tmp/build_step/*.tgz
-fi
-
 exit 0

From 54d2a21f04f7e10ee8577c674be7b2d411da5026 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Wed, 8 Mar 2023 13:31:25 +0100
Subject: [PATCH 54/58] improved usage information

---
 eessi_container.sh | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index b9553a7ea1..48c4653ba9 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -67,7 +67,7 @@ export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR}/repos.cfg"
 #    https://stackoverflow.com/questions/192249/how-do-i-parse-command-line-arguments-in-bash
 
 display_help() {
-  echo "usage: $0 [OPTIONS] [SCRIPT]"
+  echo "usage: $0 [OPTIONS] [[--] SCRIPT or COMMAND]"
   echo " OPTIONS:"
   echo "  -a | --access {ro,rw}  - ro (read-only), rw (read & write) [default: ro]"
   echo "  -c | --container IMG   - image file or URL defining the container to use"
@@ -77,7 +77,7 @@ display_help() {
   echo "                           temporary data) [default: 1. TMPDIR, 2. /tmp]"
   echo "  -l | --list-repos      - list available repository identifiers [default: false]"
   echo "  -m | --mode MODE       - with MODE==shell (launch interactive shell) or"
-  echo "                           MODE==run (run a script) [default: shell]"
+  echo "                           MODE==run (run a script or command) [default: shell]"
   echo "  -r | --repository CFG  - configuration file or identifier defining the"
   echo "                           repository to use [default: EESSI-pilot via"
   echo "                           default container, see --container]"
@@ -98,7 +98,9 @@ display_help() {
   echo "  -y | --https-proxy URL - provides URL for the env variable https_proxy"
   echo "                           [default: not set]; uses env var \$https_proxy if set"
   echo
-  echo " If value for --mode is 'run', the SCRIPT provided is executed."
+  echo " If value for --mode is 'run', the SCRIPT/COMMAND provided is executed. If"
+  echo " arguments to the script/command start with '-' or '--', use the flag terminator"
+  echo " '--' to let eessi_container.sh stop parsing arguments."
 }
 
 # set defaults for command line arguments

From 64d1866638288e315de91a41ad37a40fdaf53736 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Sat, 11 Mar 2023 01:33:50 +0100
Subject: [PATCH 55/58] fix bug, env var PWD is uppercase

---
 bot/build.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index 004e2d4881..20334501ed 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -77,8 +77,8 @@ echo "bot/build.sh: LOAD_MODULES='${LOAD_MODULES}'"
 
 # singularity/apptainer settings: CONTAINER, HOME, TMPDIR, BIND
 CONTAINER=$(cfg_get_value "repository" "container")
-export SINGULARITY_HOME="${pwd}:/eessi_bot_job"
-export SINGULARITY_TMPDIR="${pwd}/singularity_tmpdir"
+export SINGULARITY_HOME="${PWD}:/eessi_bot_job"
+export SINGULARITY_TMPDIR="${PWD}/singularity_tmpdir"
 mkdir -p ${SINGULARITY_TMPDIR}
 
 # load modules if LOAD_MODULES is not empty

From ff78c62df541075ecfb58c37e8a14bc4a0e0d80f Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Wed, 15 Mar 2023 07:04:02 +0100
Subject: [PATCH 56/58] make sure that CPU arch specific directories are used
 as container cache dir

---
 bot/build.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/bot/build.sh b/bot/build.sh
index 20334501ed..c8def2cdd3 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -27,6 +27,7 @@ source scripts/cfg_files.sh
 
 # defaults
 export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}"
+HOST_ARCH=$(uname -m)
 
 # check if ${JOB_CFG_FILE} exists
 if [[ ! -r "${JOB_CFG_FILE}" ]]; then
@@ -55,6 +56,8 @@ echo "bot/build.sh: LOCAL_TMP='${LOCAL_TMP}'"
 SINGULARITY_CACHEDIR=$(cfg_get_value "site_config" "container_cachedir")
 echo "bot/build.sh: SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'"
 if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then
+    # make sure that separate directories are used for different CPU families
+    SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}/${HOST_ARCH}
     export SINGULARITY_CACHEDIR
 fi
 

From 6299c9b4a84a5ced2afde69e9c668c4de9752331 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 17 Mar 2023 19:36:00 +0100
Subject: [PATCH 57/58] add configuration file for production build-and-deploy
 bot in AWS CitC cluster

---
 bot/bot-eessi-aws-citc.cfg | 151 +++++++++++++++++++++++++++++++++++++
 1 file changed, 151 insertions(+)
 create mode 100644 bot/bot-eessi-aws-citc.cfg

diff --git a/bot/bot-eessi-aws-citc.cfg b/bot/bot-eessi-aws-citc.cfg
new file mode 100644
index 0000000000..5b3ad34612
--- /dev/null
+++ b/bot/bot-eessi-aws-citc.cfg
@@ -0,0 +1,151 @@
+# Also see documentation at https://github.com/EESSI/eessi-bot-software-layer/blob/main/README.md#step5.5
+
+[github]
+# replace '123456' with the ID of your GitHub App
+app_id = 281041
+
+# a short (!) name for your app instance that can be used for example
+#   when adding/updating a comment to a PR
+# (!) a short yet descriptive name is preferred because it appears in
+#   comments to the PR
+# for example, the name could include the name of the cluster the bot
+#   runs on and the username which runs the bot
+# NOTE avoid putting an actual username here as it will be visible on
+#      potentially publicly accessible GitHub pages.
+app_name = eessi-bot-citc-aws
+
+# replace '12345678' with the ID of the installation of your GitHub App
+#   (can be derived by creating an event and then checking for the list
+#   of sent events and its payload either via the Smee channel's web page
+#   or via the Advanced section of your GitHub App on github.com)
+installation_id = 33078935
+
+# path to the private key that was generated when the GitHub App was registered
+private_key = /mnt/shared/home/bot/eessi-bot-software-layer/eessi-bot-citc-aws-private-key.pem
+
+
+[buildenv]
+# name of the job script used for building an EESSI stack
+build_job_script = /mnt/shared/home/bot/eessi-bot-software-layer/scripts/bot-build.slurm
+
+# The container_cachedir may be used to reuse downloaded container image files
+# across jobs. Thus, jobs can more quickly launch containers.
+container_cachedir = /mnt/shared/home/bot/eessi-bot-software-layer/containers-cache-dir
+
+# it may happen that we need to customize some CVMFS configuration
+#   the value of cvmfs_customizations is a dictionary which maps a file
+#   name to an entry that needs to be added to that file
+cvmfs_customizations = {}
+
+# if compute nodes have no internet connection, we need to set http(s)_proxy
+#   or commands such as pip3 cannot download software from package repositories
+#   for example, the temporary EasyBuild is installed via pip3 first
+# http_proxy = http://PROXY_DNS:3128/
+# https_proxy = http://PROXY_DNS:3128/
+
+# directory under which the bot prepares directories per job
+#   structure created is as follows: YYYY.MM/pr_PR_NUMBER/event_EVENT_ID/run_RUN_NUMBER/OS+SUBDIR
+jobs_base_dir = /mnt/shared/home/bot/eessi-bot-software-layer/jobs
+
+# configure environment
+#   list of comma-separated modules to be loaded by build_job_script
+#   useful/needed if some tool is not provided as system-wide package
+#   (read by bot and handed over to build_job_script via parameter
+#   --load-modules)
+load_modules =
+
+# PATH to temporary directory on build node ... ends up being used for
+#     for example, EESSI_TMPDIR --> /tmp/$USER/EESSI
+#   escaping variables with '\' delays expansion to the start of the
+#     build_job_script; this can be used for referencing environment
+#     variables that are only set inside a Slurm job
+local_tmp = /tmp/$USER/EESSI
+
+# parameters to be added to all job submissions
+# NOTE do not quote parameter string. Quotes are retained when reading in config and
+#      then the whole 'string' is recognised as a single parameter.
+# NOTE 2 '--get-user-env' may be needed on systems where the job's environment needs
+#        to be initialised as if it is for a login shell.
+# note: hardcoded 24h time limit until https://github.com/EESSI/eessi-bot-software-layer/issues/146 is fixed
+slurm_params = --hold --time=24:0:0
+
+# full path to the job submission command
+submit_command = /usr/bin/sbatch
+
+# which GH account has the permission to trigger the build (by setting
+# the label 'bot:build' (apparently this cannot be restricted on GitHub)
+# if value is left/empty everyone can trigger the build
+# value can be a space delimited list of GH accounts
+build_permission = boegel trz42 bedroge
+
+[architecturetargets]
+# defines both for which architectures the bot will build
+#   and what submission parameters shall be used
+# medium instances (8 cores, 16GB RAM)
+#arch_target_map = { "linux/x86_64/generic" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/haswell" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/skylake_avx512" : "--constraint shape=c5.4xlarge", "linux/x86_64/amd/zen2": "--constraint shape=c5a.4xlarge", "linux/x86_64/amd/zen3" : "--constraint shape=c6a.4xlarge", "linux/aarch64/generic" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton2" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton3" : "--constraint shape=c7g.4xlarge"}
+# larger instances (16 cores, 32GB RAM)
+arch_target_map = { "linux/x86_64/generic" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/haswell" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/skylake_avx512" : "--constraint shape=c5.4xlarge", "linux/x86_64/amd/zen2": "--constraint shape=c5a.4xlarge", "linux/x86_64/amd/zen3" : "--constraint shape=c6a.4xlarge", "linux/aarch64/generic" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton2" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton3" : "--constraint shape=c7g.4xlarge"}
+
+[repo_targets]
+# defines for which repository a arch_target should be build for
+#
+# only building for repository EESSI-pilot
+repo_target_map = { "linux/x86_64/generic" : ["EESSI-pilot"], "linux/x86_64/intel/haswell" : ["EESSI-pilot"], "linux/x86_64/intel/skylake_avx512" : ["EESSI-pilot"], "linux/x86_64/amd/zen2": ["EESSI-pilot"], "linux/x86_64/amd/zen3" : ["EESSI-pilot"], "linux/aarch64/generic" : ["EESSI-pilot"], "linux/aarch64/graviton2" : ["EESSI-pilot"], "linux/aarch64/graviton3" : ["EESSI-pilot"]}
+
+# points to definition of repositories (default EESSI-pilot defined by build container)
+repos_cfg_dir = /mnt/shared/home/bot/eessi-bot-software-layer/cfg-bundles
+
+# configuration for event handler which receives events from a GitHub repository.
+[event_handler]
+# path to the log file to log messages for event handler
+log_path = /mnt/shared/home/bot/eessi-bot-software-layer/eessi_bot_event_handler.log
+
+
+[job_manager]
+# path to the log file to log messages for job manager
+log_path = /mnt/shared/home/bot/eessi-bot-software-layer/eessi_bot_job_manager.log
+
+# directory where job manager stores information about jobs to be tracked
+#   e.g. as symbolic link JOBID -> directory to job
+job_ids_dir = /mnt/shared/home/bot/eessi-bot-software-layer/jobs
+
+# full path to the job status checking command
+poll_command = /usr/bin/squeue
+
+# polling interval in seconds
+poll_interval = 60
+
+# full path to the command for manipulating existing jobs
+scontrol_command = /usr/bin/scontrol
+
+[deploycfg]
+# script for uploading built software packages
+tarball_upload_script = /mnt/shared/home/bot/eessi-bot-software-layer/scripts/eessi-upload-to-staging
+
+# URL to S3/minio bucket
+#   if attribute is set, bucket_base will be constructed as follows
+#     bucket_base=${endpoint_url}/${bucket_name}
+#   otherwise, bucket_base will be constructed as follows
+#     bucket_base=https://${bucket_name}.s3.amazonaws.com
+# - The former variant is used for non AWS S3 services, eg, minio, or when
+#   the bucket name is not provided in the hostname (see latter case).
+# - The latter variant is used for AWS S3 services.
+#endpoint_url = URL_TO_S3_SERVER
+
+# bucket name
+bucket_name = eessi-staging
+
+# upload policy: defines what policy is used for uploading built artefacts
+#                to an S3 bucket
+# 'all' ..: upload all artefacts (mulitple uploads of the same artefact possible)
+# 'latest': for each build target (eessi-VERSION-{software,init,compat}-OS-ARCH)
+#           only upload the latest built artefact
+# 'once'  : only once upload any built artefact for the build target
+# 'none'  : do not upload any built artefacts
+upload_policy = once
+
+# which GH account has the permission to trigger the deployment (by setting
+# the label 'bot:deploy' (apparently this cannot be restricted on GitHub)
+# if value is left/empty everyone can trigger the deployment
+# value can be a space delimited list of GH accounts
+deploy_permission = boegel trz42 bedroge

From 0353ed047b89ccc8bc0ab75b88e07ef84caf4ebb Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Thu, 6 Apr 2023 11:59:03 +0200
Subject: [PATCH 58/58] add CaDiCaL to test bot with branch main of
 EESSI/software-layer

---
 EESSI-pilot-install-software.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh
index 2830754b29..96adadc5fd 100755
--- a/EESSI-pilot-install-software.sh
+++ b/EESSI-pilot-install-software.sh
@@ -430,6 +430,7 @@ $EB SciPy-bundle-2021.05-foss-2021a.eb --robot
 check_exit_code $? "${ok_msg}" "${fail_msg}"
 
 ### add packages here
+$EB CaDiCaL-1.3.0-GCC-9.3.0.eb --robot
 
 echo ">> Creating/updating Lmod cache..."
 export LMOD_RC="${EASYBUILD_INSTALLPATH}/.lmod/lmodrc.lua"