diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml index 7678aedc64..929fb22cec 100644 --- a/.github/workflows/test_eessi_container_script.yml +++ b/.github/workflows/test_eessi_container_script.yml @@ -11,6 +11,8 @@ jobs: matrix: SCRIPT_TEST: - help + - listrepos_default + - listrepos_custom - run - shell - container @@ -39,6 +41,28 @@ jobs: if [[ ${{matrix.SCRIPT_TEST}} == 'help' ]]; then ./eessi_container.sh --help + # test use of --list-repos without custom repos.cfg + elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_default' ]]; then + outfile=out_listrepos.txt + ./eessi_container.sh --verbose --list-repos | tee ${outfile} + grep "EESSI-pilot" ${outfile} + + # test use of --list-repos with custom repos.cfg + elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_custom' ]]; then + outfile=out_listrepos.txt + outfile2=out_listrepos_2.txt + mkdir -p ${PWD}/cfg + echo "[EESSI/20AB.CD]" > cfg/repos.cfg + echo "repo_version = 20AB.CD" >> cfg/repos.cfg + echo "[EESSI/20HT.TP]" >> cfg/repos.cfg + echo "repo_version = 20HT.TP" >> cfg/repos.cfg + ./eessi_container.sh --verbose --list-repos | tee ${outfile} + grep "EESSI-pilot" ${outfile} + + export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg + ./eessi_container.sh --verbose --list-repos | tee ${outfile2} + grep "[EESSI/2023.02]" ${outfile2} + # test use of --mode run elif [[ ${{matrix.SCRIPT_TEST}} == 'run' ]]; then outfile=out_run.txt @@ -84,6 +108,9 @@ jobs: tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g") rm -f ${outfile} + # make sure that container image exists + test -f ${tmpdir}/ghcr.io_eessi_build_node_debian11.sif || (echo "Container image not found in ${tmpdir}" >&2 && ls ${tmpdir} && exit 1) + ./eessi_container.sh --verbose --resume ${tmpdir} --mode shell <<< "${test_cmd}" > ${outfile} cat ${outfile} grep "Resuming from previous run using temporary storage at ${tmpdir}" ${outfile} diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 2d6514f5c7..c9e6c84704 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -431,6 +431,7 @@ check_exit_code $? "${ok_msg}" "${fail_msg}" ### add packages here echo "a devel PR for EESSI" +$EB CaDiCaL-1.3.0-GCC-9.3.0.eb --robot echo ">> Creating/updating Lmod cache..." export LMOD_RC="${EASYBUILD_INSTALLPATH}/.lmod/lmodrc.lua" diff --git a/bot/bot-eessi-aws-citc.cfg b/bot/bot-eessi-aws-citc.cfg new file mode 100644 index 0000000000..5b3ad34612 --- /dev/null +++ b/bot/bot-eessi-aws-citc.cfg @@ -0,0 +1,151 @@ +# Also see documentation at https://github.com/EESSI/eessi-bot-software-layer/blob/main/README.md#step5.5 + +[github] +# replace '123456' with the ID of your GitHub App +app_id = 281041 + +# a short (!) name for your app instance that can be used for example +# when adding/updating a comment to a PR +# (!) a short yet descriptive name is preferred because it appears in +# comments to the PR +# for example, the name could include the name of the cluster the bot +# runs on and the username which runs the bot +# NOTE avoid putting an actual username here as it will be visible on +# potentially publicly accessible GitHub pages. +app_name = eessi-bot-citc-aws + +# replace '12345678' with the ID of the installation of your GitHub App +# (can be derived by creating an event and then checking for the list +# of sent events and its payload either via the Smee channel's web page +# or via the Advanced section of your GitHub App on github.com) +installation_id = 33078935 + +# path to the private key that was generated when the GitHub App was registered +private_key = /mnt/shared/home/bot/eessi-bot-software-layer/eessi-bot-citc-aws-private-key.pem + + +[buildenv] +# name of the job script used for building an EESSI stack +build_job_script = /mnt/shared/home/bot/eessi-bot-software-layer/scripts/bot-build.slurm + +# The container_cachedir may be used to reuse downloaded container image files +# across jobs. Thus, jobs can more quickly launch containers. +container_cachedir = /mnt/shared/home/bot/eessi-bot-software-layer/containers-cache-dir + +# it may happen that we need to customize some CVMFS configuration +# the value of cvmfs_customizations is a dictionary which maps a file +# name to an entry that needs to be added to that file +cvmfs_customizations = {} + +# if compute nodes have no internet connection, we need to set http(s)_proxy +# or commands such as pip3 cannot download software from package repositories +# for example, the temporary EasyBuild is installed via pip3 first +# http_proxy = http://PROXY_DNS:3128/ +# https_proxy = http://PROXY_DNS:3128/ + +# directory under which the bot prepares directories per job +# structure created is as follows: YYYY.MM/pr_PR_NUMBER/event_EVENT_ID/run_RUN_NUMBER/OS+SUBDIR +jobs_base_dir = /mnt/shared/home/bot/eessi-bot-software-layer/jobs + +# configure environment +# list of comma-separated modules to be loaded by build_job_script +# useful/needed if some tool is not provided as system-wide package +# (read by bot and handed over to build_job_script via parameter +# --load-modules) +load_modules = + +# PATH to temporary directory on build node ... ends up being used for +# for example, EESSI_TMPDIR --> /tmp/$USER/EESSI +# escaping variables with '\' delays expansion to the start of the +# build_job_script; this can be used for referencing environment +# variables that are only set inside a Slurm job +local_tmp = /tmp/$USER/EESSI + +# parameters to be added to all job submissions +# NOTE do not quote parameter string. Quotes are retained when reading in config and +# then the whole 'string' is recognised as a single parameter. +# NOTE 2 '--get-user-env' may be needed on systems where the job's environment needs +# to be initialised as if it is for a login shell. +# note: hardcoded 24h time limit until https://github.com/EESSI/eessi-bot-software-layer/issues/146 is fixed +slurm_params = --hold --time=24:0:0 + +# full path to the job submission command +submit_command = /usr/bin/sbatch + +# which GH account has the permission to trigger the build (by setting +# the label 'bot:build' (apparently this cannot be restricted on GitHub) +# if value is left/empty everyone can trigger the build +# value can be a space delimited list of GH accounts +build_permission = boegel trz42 bedroge + +[architecturetargets] +# defines both for which architectures the bot will build +# and what submission parameters shall be used +# medium instances (8 cores, 16GB RAM) +#arch_target_map = { "linux/x86_64/generic" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/haswell" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/skylake_avx512" : "--constraint shape=c5.4xlarge", "linux/x86_64/amd/zen2": "--constraint shape=c5a.4xlarge", "linux/x86_64/amd/zen3" : "--constraint shape=c6a.4xlarge", "linux/aarch64/generic" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton2" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton3" : "--constraint shape=c7g.4xlarge"} +# larger instances (16 cores, 32GB RAM) +arch_target_map = { "linux/x86_64/generic" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/haswell" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/skylake_avx512" : "--constraint shape=c5.4xlarge", "linux/x86_64/amd/zen2": "--constraint shape=c5a.4xlarge", "linux/x86_64/amd/zen3" : "--constraint shape=c6a.4xlarge", "linux/aarch64/generic" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton2" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton3" : "--constraint shape=c7g.4xlarge"} + +[repo_targets] +# defines for which repository a arch_target should be build for +# +# only building for repository EESSI-pilot +repo_target_map = { "linux/x86_64/generic" : ["EESSI-pilot"], "linux/x86_64/intel/haswell" : ["EESSI-pilot"], "linux/x86_64/intel/skylake_avx512" : ["EESSI-pilot"], "linux/x86_64/amd/zen2": ["EESSI-pilot"], "linux/x86_64/amd/zen3" : ["EESSI-pilot"], "linux/aarch64/generic" : ["EESSI-pilot"], "linux/aarch64/graviton2" : ["EESSI-pilot"], "linux/aarch64/graviton3" : ["EESSI-pilot"]} + +# points to definition of repositories (default EESSI-pilot defined by build container) +repos_cfg_dir = /mnt/shared/home/bot/eessi-bot-software-layer/cfg-bundles + +# configuration for event handler which receives events from a GitHub repository. +[event_handler] +# path to the log file to log messages for event handler +log_path = /mnt/shared/home/bot/eessi-bot-software-layer/eessi_bot_event_handler.log + + +[job_manager] +# path to the log file to log messages for job manager +log_path = /mnt/shared/home/bot/eessi-bot-software-layer/eessi_bot_job_manager.log + +# directory where job manager stores information about jobs to be tracked +# e.g. as symbolic link JOBID -> directory to job +job_ids_dir = /mnt/shared/home/bot/eessi-bot-software-layer/jobs + +# full path to the job status checking command +poll_command = /usr/bin/squeue + +# polling interval in seconds +poll_interval = 60 + +# full path to the command for manipulating existing jobs +scontrol_command = /usr/bin/scontrol + +[deploycfg] +# script for uploading built software packages +tarball_upload_script = /mnt/shared/home/bot/eessi-bot-software-layer/scripts/eessi-upload-to-staging + +# URL to S3/minio bucket +# if attribute is set, bucket_base will be constructed as follows +# bucket_base=${endpoint_url}/${bucket_name} +# otherwise, bucket_base will be constructed as follows +# bucket_base=https://${bucket_name}.s3.amazonaws.com +# - The former variant is used for non AWS S3 services, eg, minio, or when +# the bucket name is not provided in the hostname (see latter case). +# - The latter variant is used for AWS S3 services. +#endpoint_url = URL_TO_S3_SERVER + +# bucket name +bucket_name = eessi-staging + +# upload policy: defines what policy is used for uploading built artefacts +# to an S3 bucket +# 'all' ..: upload all artefacts (mulitple uploads of the same artefact possible) +# 'latest': for each build target (eessi-VERSION-{software,init,compat}-OS-ARCH) +# only upload the latest built artefact +# 'once' : only once upload any built artefact for the build target +# 'none' : do not upload any built artefacts +upload_policy = once + +# which GH account has the permission to trigger the deployment (by setting +# the label 'bot:deploy' (apparently this cannot be restricted on GitHub) +# if value is left/empty everyone can trigger the deployment +# value can be a space delimited list of GH accounts +deploy_permission = boegel trz42 bedroge diff --git a/bot/build.sh b/bot/build.sh new file mode 100755 index 0000000000..c8def2cdd3 --- /dev/null +++ b/bot/build.sh @@ -0,0 +1,199 @@ +#!/usr/bin/env bash +# +# script to build the EESSI software layer. Intended use is that it is called +# by a (batch) job running on a compute node. +# +# This script is part of the EESSI software layer, see +# https://github.com/EESSI/software-layer.git +# +# author: Thomas Roeblitz (@trz42) +# +# license: GPLv2 +# + +# ASSUMPTIONs: +# - working directory has been prepared by the bot with a checkout of a +# pull request (OR by some other means) +# - the working directory contains a directory 'cfg' where the main config +# file 'job.cfg' has been deposited +# - the directory may contain any additional files referenced in job.cfg + +# stop as soon as something fails +set -e + +# source utils.sh and cfg_files.sh +source scripts/utils.sh +source scripts/cfg_files.sh + +# defaults +export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}" +HOST_ARCH=$(uname -m) + +# check if ${JOB_CFG_FILE} exists +if [[ ! -r "${JOB_CFG_FILE}" ]]; then + fatal_error "job config file (JOB_CFG_FILE=${JOB_CFG_FILE}) does not exist or not readable" +fi +echo "bot/build.sh: showing ${JOB_CFG_FILE} from software-layer side" +cat ${JOB_CFG_FILE} + +echo "bot/build.sh: obtaining configuration settings from '${JOB_CFG_FILE}'" +cfg_load ${JOB_CFG_FILE} + +# if http_proxy is defined in ${JOB_CFG_FILE} use it, if not use env var $http_proxy +HTTP_PROXY=$(cfg_get_value "site_config" "http_proxy") +HTTP_PROXY=${HTTP_PROXY:-${http_proxy}} +echo "bot/build.sh: HTTP_PROXY='${HTTP_PROXY}'" + +# if https_proxy is defined in ${JOB_CFG_FILE} use it, if not use env var $https_proxy +HTTPS_PROXY=$(cfg_get_value "site_config" "https_proxy") +HTTPS_PROXY=${HTTPS_PROXY:-${https_proxy}} +echo "bot/build.sh: HTTPS_PROXY='${HTTPS_PROXY}'" + +LOCAL_TMP=$(cfg_get_value "site_config" "local_tmp") +echo "bot/build.sh: LOCAL_TMP='${LOCAL_TMP}'" +# TODO should local_tmp be mandatory? --> then we check here and exit if it is not provided + +SINGULARITY_CACHEDIR=$(cfg_get_value "site_config" "container_cachedir") +echo "bot/build.sh: SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'" +if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then + # make sure that separate directories are used for different CPU families + SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}/${HOST_ARCH} + export SINGULARITY_CACHEDIR +fi + +echo -n "setting \$STORAGE by replacing any var in '${LOCAL_TMP}' -> " +# replace any env variable in ${LOCAL_TMP} with its +# current value (e.g., a value that is local to the job) +STORAGE=$(envsubst <<< ${LOCAL_TMP}) +echo "'${STORAGE}'" + +# make sure ${STORAGE} exists +mkdir -p ${STORAGE} + +# make sure the base tmp storage is unique +JOB_STORAGE=$(mktemp --directory --tmpdir=${STORAGE} bot_job_tmp_XXX) +echo "bot/build.sh: created unique base tmp storage directory at ${JOB_STORAGE}" + +# obtain list of modules to be loaded +LOAD_MODULES=$(cfg_get_value "site_config" "load_modules") +echo "bot/build.sh: LOAD_MODULES='${LOAD_MODULES}'" + +# singularity/apptainer settings: CONTAINER, HOME, TMPDIR, BIND +CONTAINER=$(cfg_get_value "repository" "container") +export SINGULARITY_HOME="${PWD}:/eessi_bot_job" +export SINGULARITY_TMPDIR="${PWD}/singularity_tmpdir" +mkdir -p ${SINGULARITY_TMPDIR} + +# load modules if LOAD_MODULES is not empty +if [[ ! -z ${LOAD_MODULES} ]]; then + for mod in $(echo ${LOAD_MODULES} | tr ',' '\n') + do + echo "bot/build.sh: loading module '${mod}'" + module load ${mod} + done +else + echo "bot/build.sh: no modules to be loaded" +fi + +# determine repository to be used from entry .repository in ${JOB_CFG_FILE} +REPOSITORY=$(cfg_get_value "repository" "repo_id") +EESSI_REPOS_CFG_DIR_OVERRIDE=$(cfg_get_value "repository" "repos_cfg_dir") +export EESSI_REPOS_CFG_DIR_OVERRIDE=${EESSI_REPOS_CFG_DIR_OVERRIDE:-${PWD}/cfg} +echo "bot/build.sh: EESSI_REPOS_CFG_DIR_OVERRIDE='${EESSI_REPOS_CFG_DIR_OVERRIDE}'" + +# determine pilot version to be used from .repository.repo_version in ${JOB_CFG_FILE} +# here, just set & export EESSI_PILOT_VERSION_OVERRIDE +# next script (eessi_container.sh) makes use of it via sourcing init scripts +# (e.g., init/eessi_defaults or init/minimal_eessi_env) +export EESSI_PILOT_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version") +echo "bot/build.sh: EESSI_PILOT_VERSION_OVERRIDE='${EESSI_PILOT_VERSION_OVERRIDE}'" + +# determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE} +# here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down +# "source init/eessi_defaults" via sourcing init/minimal_eessi_env +export EESSI_CVMFS_REPO_OVERRIDE=$(cfg_get_value "repository" "repo_name") +echo "bot/build.sh: EESSI_CVMFS_REPO_OVERRIDE='${EESSI_CVMFS_REPO_OVERRIDE}'" + +# determine architecture to be used from entry .architecture in ${JOB_CFG_FILE} +# fallbacks: +# - ${CPU_TARGET} handed over from bot +# - left empty to let downstream script(s) determine subdir to be used +EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(cfg_get_value "architecture" "software_subdir") +EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE:-${CPU_TARGET}} +export EESSI_SOFTWARE_SUBDIR_OVERRIDE +echo "bot/build.sh: EESSI_SOFTWARE_SUBDIR_OVERRIDE='${EESSI_SOFTWARE_SUBDIR_OVERRIDE}'" + +# get EESSI_OS_TYPE from .architecture.os_type in ${JOB_CFG_FILE} (default: linux) +EESSI_OS_TYPE=$(cfg_get_value "architecture" "os_type") +export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux} +echo "bot/build.sh: EESSI_OS_TYPE='${EESSI_OS_TYPE}'" + +# prepare arguments to eessi_container.sh common to build and tarball steps +declare -a COMMON_ARGS=() +COMMON_ARGS+=("--verbose") +COMMON_ARGS+=("--access" "rw") +COMMON_ARGS+=("--mode" "run") +[[ ! -z ${CONTAINER} ]] && COMMON_ARGS+=("--container" "${CONTAINER}") +[[ ! -z ${HTTP_PROXY} ]] && COMMON_ARGS+=("--http-proxy" "${HTTP_PROXY}") +[[ ! -z ${HTTPS_PROXY} ]] && COMMON_ARGS+=("--https-proxy" "${HTTPS_PROXY}") +[[ ! -z ${REPOSITORY} ]] && COMMON_ARGS+=("--repository" "${REPOSITORY}") + +# make sure to use the same parent dir for storing tarballs of tmp +PREVIOUS_TMP_DIR=${PWD}/previous_tmp + +# prepare directory to store tarball of tmp for build step +TARBALL_TMP_BUILD_STEP_DIR=${PREVIOUS_TMP_DIR}/build_step +mkdir -p ${TARBALL_TMP_BUILD_STEP_DIR} + +# prepare arguments to eessi_container.sh specific to build step +declare -a BUILD_STEP_ARGS=() +BUILD_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}") +BUILD_STEP_ARGS+=("--storage" "${STORAGE}") + +# prepare arguments to install_software_layer.sh (specific to build step) +GENERIC_OPT= +if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then + GENERIC_OPT="--generic" +fi + +# create tmp file for output of build step +build_outerr=$(mktemp build.outerr.XXXX) + +echo "Executing command to build software:" +echo "./eessi_container.sh ${COMMON_ARGS[@]} ${BUILD_STEP_ARGS[@]}" +echo " -- ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}" +./eessi_container.sh "${COMMON_ARGS[@]}" "${BUILD_STEP_ARGS[@]}" \ + -- ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr} + +# prepare directory to store tarball of tmp for tarball step +TARBALL_TMP_TARBALL_STEP_DIR=${PREVIOUS_TMP_DIR}/tarball_step +mkdir -p ${TARBALL_TMP_TARBALL_STEP_DIR} + +# create tmp file for output of tarball step +tar_outerr=$(mktemp tar.outerr.XXXX) + +# prepare arguments to eessi_container.sh specific to tarball step +declare -a TARBALL_STEP_ARGS=() +TARBALL_STEP_ARGS+=("--save" "${TARBALL_TMP_TARBALL_STEP_DIR}") + +# determine temporary directory to resume from +BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2) +TARBALL_STEP_ARGS+=("--resume" "${BUILD_TMPDIR}") + +timestamp=$(date +%s) +# to set EESSI_PILOT_VERSION we need to source init/eessi_defaults now +source init/eessi_defaults +export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_PILOT_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${timestamp}) + +# value of first parameter to create_tarball.sh - TMP_IN_CONTAINER - needs to be +# synchronised with setting of TMP_IN_CONTAINER in eessi_container.sh +# TODO should we make this a configurable parameter of eessi_container.sh using +# /tmp as default? +TMP_IN_CONTAINER=/tmp +echo "Executing command to create tarball:" +echo "./eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}" +echo " -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" +./eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \ + -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} + +exit 0 diff --git a/eessi_container.sh b/eessi_container.sh index 96a9900a5e..48c4653ba9 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -24,14 +24,14 @@ # 2. set up host storage/tmp # 3. set up common vars and directories # 4. set up vars specific to a scenario -# 5. initialize host storage/tmp from previous run if provided -# 6. run container +# 5. run container +# 6. save tmp (if requested) # -. initial settings & exit codes TOPDIR=$(dirname $(realpath $0)) source ${TOPDIR}/scripts/utils.sh -source ${TOPDIR}/cfg_files.sh +source ${TOPDIR}/scripts/cfg_files.sh # exit codes: bitwise shift codes to allow for combination of exit codes # ANY_ERROR_EXITCODE is sourced from ${TOPDIR}/scripts/utils.sh @@ -43,17 +43,22 @@ MODE_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 5)) REPOSITORY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 6)) RESUME_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 7)) SAVE_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 8)) -#HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9)) -#HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10)) +HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9)) +HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10)) RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11)) # CernVM-FS settings CVMFS_VAR_LIB="var-lib-cvmfs" CVMFS_VAR_RUN="var-run-cvmfs" -# repository cfg file, default name (default location: $PWD) -# can be overwritten by setting env var EESSI_REPOS_CFG_FILE_OVERRIDE -export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_FILE_OVERRIDE:=repos.cfg}" +# directory for tmp used inside container +export TMP_IN_CONTAINER=/tmp + +# repository cfg directory and file +# directory: default $PWD or EESSI_REPOS_CFG_DIR_OVERRIDE if set +# file: directory + '/repos.cfg' +export EESSI_REPOS_CFG_DIR="${EESSI_REPOS_CFG_DIR_OVERRIDE:=${PWD}}" +export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR}/repos.cfg" # 0. parse args @@ -62,41 +67,40 @@ export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_FILE_OVERRIDE:=repos.cfg}" # https://stackoverflow.com/questions/192249/how-do-i-parse-command-line-arguments-in-bash display_help() { - echo "usage: $0 [OPTIONS] [SCRIPT]" + echo "usage: $0 [OPTIONS] [[--] SCRIPT or COMMAND]" echo " OPTIONS:" - echo " -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]" - echo " -c | --container IMG - image file or URL defining the container to use" - echo " [default: docker://ghcr.io/eessi/build-node:debian11]" - echo " -h | --help - display this usage information [default: false]" - echo " -g | --storage DIR - directory space on host machine (used for" - echo " temporary data) [default: 1. TMPDIR, 2. /tmp]" - echo " -m | --mode MODE - with MODE==shell (launch interactive shell) or" - echo " MODE==run (run a script) [default: shell]" - echo " -r | --repository CFG - configuration file or identifier defining the" - echo " repository to use [default: EESSI-pilot via" - echo " container configuration]" - echo " -u | --resume DIR/TGZ - resume a previous run from a directory or tarball," - echo " where DIR points to a previously used tmp directory" - echo " (check for output 'Using DIR as tmp ...' of a previous" - echo " run) and TGZ is the path to a tarball which is" - echo " unpacked the tmp dir stored on the local storage space" - echo " (see option --storage above) [default: not set]" - echo " -s | --save DIR/TGZ - save contents of tmp directory to a tarball in" - echo " directory DIR or provided with the fixed full path TGZ" - echo " when a directory is provided, the format of the" - echo " tarball's name will be {REPO_ID}-{TIMESTAMP}.tgz" - echo " [default: not set]" - echo " -v | --verbose - display more information [default: false]" + echo " -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]" + echo " -c | --container IMG - image file or URL defining the container to use" + echo " [default: docker://ghcr.io/eessi/build-node:debian11]" + echo " -h | --help - display this usage information [default: false]" + echo " -g | --storage DIR - directory space on host machine (used for" + echo " temporary data) [default: 1. TMPDIR, 2. /tmp]" + echo " -l | --list-repos - list available repository identifiers [default: false]" + echo " -m | --mode MODE - with MODE==shell (launch interactive shell) or" + echo " MODE==run (run a script or command) [default: shell]" + echo " -r | --repository CFG - configuration file or identifier defining the" + echo " repository to use [default: EESSI-pilot via" + echo " default container, see --container]" + echo " -u | --resume DIR/TGZ - resume a previous run from a directory or tarball," + echo " where DIR points to a previously used tmp directory" + echo " (check for output 'Using DIR as tmp ...' of a previous" + echo " run) and TGZ is the path to a tarball which is" + echo " unpacked the tmp dir stored on the local storage space" + echo " (see option --storage above) [default: not set]" + echo " -s | --save DIR/TGZ - save contents of tmp directory to a tarball in" + echo " directory DIR or provided with the fixed full path TGZ" + echo " when a directory is provided, the format of the" + echo " tarball's name will be {REPO_ID}-{TIMESTAMP}.tgz" + echo " [default: not set]" + echo " -v | --verbose - display more information [default: false]" + echo " -x | --http-proxy URL - provides URL for the env variable http_proxy" + echo " [default: not set]; uses env var \$http_proxy if set" + echo " -y | --https-proxy URL - provides URL for the env variable https_proxy" + echo " [default: not set]; uses env var \$https_proxy if set" echo - echo " If value for --mode is 'run', the SCRIPT provided is executed." - #echo - #echo " FEATURES/OPTIONS to be implemented:" - #echo " -d | --dry-run - run script except for executing the container," - #echo " print information about setup [default: false]" - #echo " -x | --http-proxy URL - provides URL for the env variable http_proxy" - #echo " [default: not set]" - #echo " -y | --https-proxy URL - provides URL for the env variable https_proxy" - #echo " [default: not set]" + echo " If value for --mode is 'run', the SCRIPT/COMMAND provided is executed. If" + echo " arguments to the script/command start with '-' or '--', use the flag terminator" + echo " '--' to let eessi_container.sh stop parsing arguments." } # set defaults for command line arguments @@ -105,12 +109,13 @@ CONTAINER="docker://ghcr.io/eessi/build-node:debian11" #DRY_RUN=0 VERBOSE=0 STORAGE= +LIST_REPOS=0 MODE="shell" REPOSITORY="EESSI-pilot" RESUME= SAVE= -#HTTP_PROXY= -#HTTPS_PROXY= +HTTP_PROXY=${http_proxy:-} +HTTPS_PROXY=${https_proxy:-} POSITIONAL_ARGS=() @@ -136,6 +141,10 @@ while [[ $# -gt 0 ]]; do display_help exit 0 ;; + -l|--list-repos) + LIST_REPOS=1 + shift 1 + ;; -m|--mode) MODE="$2" shift 2 @@ -156,16 +165,21 @@ while [[ $# -gt 0 ]]; do VERBOSE=1 shift 1 ;; -# -x|--http-proxy) -# HTTP_PROXY="$2" -# export http_proxy=${HTTP_PROXY} -# shift 2 -# ;; -# -y|--https-proxy) -# HTTPS_PROXY="$2" -# export https_proxy=${HTTPS_PROXY} -# shift 2 -# ;; + -x|--http-proxy) + HTTP_PROXY="$2" + export http_proxy=${HTTP_PROXY} + shift 2 + ;; + -y|--https-proxy) + HTTPS_PROXY="$2" + export https_proxy=${HTTPS_PROXY} + shift 2 + ;; + --) + shift + POSITIONAL_ARGS+=("$@") # save positional args + break + ;; -*|--*) fatal_error "Unknown option: $1" "${CMDLINE_ARG_UNKNOWN_EXITCODE}" ;; @@ -178,11 +192,24 @@ done set -- "${POSITIONAL_ARGS[@]}" +if [[ ${LIST_REPOS} -eq 1 ]]; then + echo "Listing available repositories with format 'name [source]':" + echo " EESSI-pilot [default]" + if [[ -r ${EESSI_REPOS_CFG_FILE} ]]; then + cfg_load ${EESSI_REPOS_CFG_FILE} + sections=$(cfg_sections) + while IFS= read -r repo_id + do + echo " ${repo_id} [${EESSI_REPOS_CFG_FILE}]" + done <<< "${sections}" + fi + exit 0 +fi # 1. check if argument values are valid # (arg -a|--access) check if ACCESS is supported if [[ "${ACCESS}" != "ro" && "${ACCESS}" != "rw" ]]; then - fatal_error "unknown access method '${ACCESS}'" "${ACCESS_UNKNOWN_EXITCODE}" + fatal_error "unknown access method '${ACCESS}'" "${ACCESS_UNKNOWN_EXITCODE}" fi # TODO (arg -c|--container) check container (is it a file or URL & access those) @@ -194,11 +221,14 @@ fi # (arg -m|--mode) check if MODE is known if [[ "${MODE}" != "shell" && "${MODE}" != "run" ]]; then - fatal_error "unknown execution mode '${MODE}'" "${MODE_UNKNOWN_EXITCODE}" + fatal_error "unknown execution mode '${MODE}'" "${MODE_UNKNOWN_EXITCODE}" fi # TODO (arg -r|--repository) check if repository is known # REPOSITORY_ERROR_EXITCODE +if [[ ! -z "${REPOSITORY}" && "${REPOSITORY}" != "EESSI-pilot" && ! -r ${EESSI_REPOS_CFG_FILE} ]]; then + fatal_error "arg '--repository ${REPOSITORY}' requires a cfg file at '${EESSI_REPOS_CFG_FILE}'" "${REPOSITORY_ERROR_EXITCODE}" +fi # TODO (arg -u|--resume) check if it exists, if user has read permission, # if it contains data from a previous run @@ -261,7 +291,7 @@ else [[ ${VERBOSE} -eq 1 ]] && echo "skipping sanity checks for /tmp" fi EESSI_HOST_STORAGE=$(mktemp -d --tmpdir eessi.XXXXXXXXXX) - echo "Using ${EESSI_HOST_STORAGE} as tmp storage (add '--resume ${EESSI_HOST_STORAGE}' to resume where this session ended)." + echo "Using ${EESSI_HOST_STORAGE} as tmp directory (to resume session add '--resume ${EESSI_HOST_STORAGE}')." fi # if ${RESUME} is a file (assume a tgz), unpack it into ${EESSI_HOST_STORAGE} @@ -279,18 +309,75 @@ fi # |-overlay-upper # |-overlay-work # |-home -# |-cfg +# |-repos_cfg # tmp dir for EESSI EESSI_TMPDIR=${EESSI_HOST_STORAGE} mkdir -p ${EESSI_TMPDIR} [[ ${VERBOSE} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}" -# configure Singularity -export SINGULARITY_CACHEDIR=${EESSI_TMPDIR}/singularity_cache -mkdir -p ${SINGULARITY_CACHEDIR} +# configure Singularity: if SINGULARITY_CACHEDIR is already defined, use that +# a global SINGULARITY_CACHEDIR would ensure that we don't consume +# storage space again and again for the container & also speed-up +# launch times across different sessions +if [[ -z ${SINGULARITY_CACHEDIR} ]]; then + export SINGULARITY_CACHEDIR=${EESSI_TMPDIR}/singularity_cache + mkdir -p ${SINGULARITY_CACHEDIR} +fi [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}" +# if VERBOSE is set to 0 (no arg --verbose), add argument '-q' +if [[ ${VERBOSE} -eq 0 ]]; then + RUN_QUIET='-q' +else + RUN_QUIET='' +fi + +# we try our best to make sure that we retain access to the container image in +# a subsequent session ("best effort" only because pulling or copying operations +# can fail ... in those cases the script may still succeed, but it is not +# guaranteed that we have access to the same container when resuming later on) +# - if CONTAINER references an image in a registry, pull & convert image +# and store it in ${EESSI_TMPDIR} +# + however, only pull image if there is no matching image in ${EESSI_TMPDIR} yet +# - if CONTAINER references an image file, copy it to ${EESSI_TMPDIR} +# + however, only copy it if its base name does not yet exist in ${EESSI_TMPDIR} +# - if the image file created (pulled or copied) or resumed exists in +# ${EESSI_TMPDIR}, let CONTAINER point to it +# + thus subsequent singularity commands in this script would just use the +# image file in EESSI_TMPDIR or the originally given source (some URL or +# path to an image file) +CONTAINER_IMG= +CONTAINER_URL_FMT=".*://(.*)" +if [[ ${CONTAINER} =~ ${CONTAINER_URL_FMT} ]]; then + # replace ':', '-', '/' with '_' in match (everything after ://) and append .sif + CONTAINER_IMG="$(echo ${BASH_REMATCH[1]} | sed 's/[:\/-]/_/g').sif" + # pull container to ${EESSI_TMPDIR} if it is not there yet (i.e. when + # resuming from a previous session) + if [[ ! -x ${EESSI_TMPDIR}/${CONTAINER_IMG} ]]; then + echo "Pulling container image from ${CONTAINER} to ${EESSI_TMPDIR}/${CONTAINER_IMG}" + singularity ${RUN_QUIET} pull ${EESSI_TMPDIR}/${CONTAINER_IMG} ${CONTAINER} + else + echo "Reusing existing container image ${EESSI_TMPDIR}/${CONTAINER_IMG}" + fi +else + # determine file name as basename of CONTAINER + CONTAINER_IMG=$(basename ${CONTAINER}) + # copy image file to ${EESSI_TMPDIR} if it is not there yet (i.e. when + # resuming from a previous session) + if [[ ! -x ${EESSI_TMPDIR}/${CONTAINER_IMG} ]]; then + echo "Copying container image from ${CONTAINER} to ${EESSI_TMPDIR}/${CONTAINER_IMG}" + cp -a ${CONTAINER} ${EESSI_TMPDIR}/. + else + echo "Reusing existing container image ${EESSI_TMPDIR}/${CONTAINER_IMG}" + fi +fi +# let CONTAINER point to the pulled, copied or resumed image file +if [[ -x ${EESSI_TMPDIR}/${CONTAINER_IMG} ]]; then + CONTAINER="${EESSI_TMPDIR}/${CONTAINER_IMG}" +fi +[[ ${VERBOSE} -eq 1 ]] && echo "CONTAINER=${CONTAINER}" + # set env vars and create directories for CernVM-FS EESSI_CVMFS_VAR_LIB=${EESSI_TMPDIR}/${CVMFS_VAR_LIB} EESSI_CVMFS_VAR_RUN=${EESSI_TMPDIR}/${CVMFS_VAR_RUN} @@ -303,17 +390,19 @@ mkdir -p ${EESSI_CVMFS_VAR_RUN} if [[ -z ${SINGULARITY_HOME} ]]; then export SINGULARITY_HOME="${EESSI_TMPDIR}/home:/home/${USER}" mkdir -p ${EESSI_TMPDIR}/home - [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_HOME=${SINGULARITY_HOME}" fi +[[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_HOME=${SINGULARITY_HOME}" # define paths to add to SINGULARITY_BIND (added later when all BIND mounts are defined) BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs" -BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp" +# provide a '/tmp' inside the container +BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:${TMP_IN_CONTAINER}" + [[ ${VERBOSE} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}" -# set up repository config (always create cfg dir and populate it with info when +# set up repository config (always create directory repos_cfg and populate it with info when # arg -r|--repository is used) -mkdir -p ${EESSI_TMPDIR}/cfg +mkdir -p ${EESSI_TMPDIR}/repos_cfg if [[ "${REPOSITORY}" == "EESSI-pilot" ]]; then # need to source defaults as late as possible (see other sourcing below) source ${TOPDIR}/init/eessi_defaults @@ -331,6 +420,9 @@ else # standard EESSI repositories) cfg_load ${EESSI_REPOS_CFG_FILE} + # copy repos.cfg to job directory --> makes it easier to inspect the job + cp -a ${EESSI_REPOS_CFG_FILE} ${EESSI_TMPDIR}/repos_cfg/. + # cfg file should include: repo_name, repo_version, config_bundle, # map { local_filepath -> container_filepath } # @@ -361,22 +453,31 @@ else cfg_init_file_map "${config_map}" [[ ${VERBOSE} -eq 1 ]] && cfg_print_map - # TODO use information to set up dir ${EESSI_TMPDIR}/cfg, - # define BIND mounts and override repo name and version - # check if config_bundle exists, if so, unpack it into ${EESSI_TMPDIR}/cfg - if [[ ! -r ${config_bundle} ]]; then - fatal_error "config bundle '${config_bundle}' is not readable" ${REPOSITORY_ERROR_EXITCODE} + # use information to set up dir ${EESSI_TMPDIR}/repos_cfg, + # define BIND mounts and override repo name and version + # check if config_bundle exists, if so, unpack it into ${EESSI_TMPDIR}/repos_cfg + # if config_bundle is relative path (no '/' at start) prepend it with + # EESSI_REPOS_CFG_DIR + config_bundle_path= + if [[ ! "${config_bundle}" =~ ^/ ]]; then + config_bundle_path=${EESSI_REPOS_CFG_DIR}/${config_bundle} + else + config_bundle_path=${config_bundle} + fi + + if [[ ! -r ${config_bundle_path} ]]; then + fatal_error "config bundle '${config_bundle_path}' is not readable" ${REPOSITORY_ERROR_EXITCODE} fi # only unpack config_bundle if we're not resuming from a previous run if [[ -z ${RESUME} ]]; then - tar xf ${config_bundle} -C ${EESSI_TMPDIR}/cfg + tar xf ${config_bundle_path} -C ${EESSI_TMPDIR}/repos_cfg fi for src in "${!cfg_file_map[@]}" do target=${cfg_file_map[${src}]} - BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/cfg/${src}:${target}" + BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/${src}:${target}" done export EESSI_PILOT_VERSION_OVERRIDE=${repo_version} export EESSI_CVMFS_REPO_OVERRIDE="/cvmfs/${repo_name}" @@ -384,6 +485,30 @@ else source ${TOPDIR}/init/eessi_defaults fi +# if http_proxy is not empty, we assume that the machine accesses internet +# via a proxy. then we need to add CVMFS_HTTP_PROXY to +# ${EESSI_TMPDIR}/repos_cfg/default.local on host (and possibly add a BIND +# MOUNT if it was not yet in BIND_PATHS) +if [[ ! -z ${http_proxy} ]]; then + # TODO tolerate other formats for proxy URLs, for now assume format is + # http://SOME_HOSTNAME:SOME_PORT/ + [[ ${VERBOSE} -eq 1 ]] && echo "http_proxy='${http_proxy}'" + PROXY_HOST=$(get_host_from_url ${http_proxy}) + [[ ${VERBOSE} -eq 1 ]] && echo "PROXY_HOST='${PROXY_HOST}'" + PROXY_PORT=$(get_port_from_url ${http_proxy}) + [[ ${VERBOSE} -eq 1 ]] && echo "PROXY_PORT='${PROXY_PORT}'" + HTTP_PROXY_IPV4=$(get_ipv4_address ${PROXY_HOST}) + [[ ${VERBOSE} -eq 1 ]] && echo "HTTP_PROXY_IPV4='${HTTP_PROXY_IPV4}'" + echo "CVMFS_HTTP_PROXY=\"${http_proxy}|http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \ + >> ${EESSI_TMPDIR}/repos_cfg/default.local + [[ ${VERBOSE} -eq 1 ]] && echo "contents of default.local" + [[ ${VERBOSE} -eq 1 ]] && cat ${EESSI_TMPDIR}/repos_cfg/default.local + + # if default.local is not BIND mounted into container, add it to BIND_PATHS + if [[ ! ${BIND_PATHS} =~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then + export BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" + fi +fi # 4. set up vars and dirs specific to a scenario @@ -396,12 +521,8 @@ if [[ "${ACCESS}" == "ro" ]]; then fi if [[ "${ACCESS}" == "rw" ]]; then - EESSI_CVMFS_OVERLAY_UPPER=/tmp/overlay-upper - EESSI_CVMFS_OVERLAY_WORK=/tmp/overlay-work mkdir -p ${EESSI_TMPDIR}/overlay-upper mkdir -p ${EESSI_TMPDIR}/overlay-work - [[ ${VERBOSE} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_UPPER=${EESSI_CVMFS_OVERLAY_UPPER}" - [[ ${VERBOSE} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_WORK=${EESSI_CVMFS_OVERLAY_WORK}" # set environment variables for fuse mounts in Singularity container export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}" @@ -410,8 +531,8 @@ if [[ "${ACCESS}" == "rw" ]]; then EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs" EESSI_PILOT_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${repo_name}" - EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=/tmp/overlay-upper" - EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=/tmp/overlay-work" + EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/overlay-upper" + EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/overlay-work" EESSI_PILOT_WRITABLE_OVERLAY+=" ${EESSI_CVMFS_REPO}" export EESSI_PILOT_WRITABLE_OVERLAY @@ -420,10 +541,7 @@ if [[ "${ACCESS}" == "rw" ]]; then fi -# 5. initialize host storage/tmp from previous run if provided - - -# 6. run container +# 5. run container # final settings if [[ -z ${SINGULARITY_BIND} ]]; then export SINGULARITY_BIND="${BIND_PATHS}" @@ -432,14 +550,19 @@ else fi [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_BIND=${SINGULARITY_BIND}" +# pass $EESSI_SOFTWARE_SUBDIR_OVERRIDE into build container (if set) +if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then + export SINGULARITYENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE} + # also specify via $APPTAINERENV_* (future proof, cfr. https://apptainer.org/docs/user/latest/singularity_compatibility.html#singularity-environment-variable-compatibility) + export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE} +fi + echo "Launching container with command (next line):" -echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@" -# TODO for now we run singularity with '-q' (quiet), later adjust this to the log level -# provided to the script -singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@" +echo "singularity ${RUN_QUIET} ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@" +singularity ${RUN_QUIET} ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@" exit_code=$? -# 7. save tmp if requested (arg -s|--save) +# 6. save tmp if requested (arg -s|--save) if [[ ! -z ${SAVE} ]]; then # Note, for now we don't try to be smart and record in any way the OS and # ARCH which might have been used internally, eg, when software packages @@ -455,7 +578,7 @@ if [[ ! -z ${SAVE} ]]; then TGZ=${SAVE} fi tar cf ${TGZ} -C ${EESSI_TMPDIR} . - echo "Saved contents of '${EESSI_TMPDIR}' to '${TGZ}' (to resume, add '--resume ${TGZ}')" + echo "Saved contents of tmp directory '${EESSI_TMPDIR}' to tarball '${TGZ}' (to resume session add '--resume ${TGZ}')" fi # TODO clean up tmp by default? only retain if another option provided (--retain-tmp) diff --git a/init/eessi_defaults b/init/eessi_defaults index 1b5ce07fb1..f482cbc269 100644 --- a/init/eessi_defaults +++ b/init/eessi_defaults @@ -10,4 +10,3 @@ export EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO_OVERRIDE:=/cvmfs/pilot.eessi-hpc.org}" export EESSI_PILOT_VERSION="${EESSI_PILOT_VERSION_OVERRIDE:=2021.12}" - diff --git a/run_in_compat_layer_env.sh b/run_in_compat_layer_env.sh index be01bcd201..c70077bf15 100755 --- a/run_in_compat_layer_env.sh +++ b/run_in_compat_layer_env.sh @@ -17,6 +17,18 @@ INPUT=$(echo "$@") if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then INPUT="export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}; ${INPUT}" fi +if [ ! -z ${EESSI_CVMFS_REPO_OVERRIDE} ]; then + INPUT="export EESSI_CVMFS_REPO_OVERRIDE=${EESSI_CVMFS_REPO_OVERRIDE}; ${INPUT}" +fi +if [ ! -z ${EESSI_PILOT_VERSION_OVERRIDE} ]; then + INPUT="export EESSI_PILOT_VERSION_OVERRIDE=${EESSI_PILOT_VERSION_OVERRIDE}; ${INPUT}" +fi +if [ ! -z ${http_proxy} ]; then + INPUT="export http_proxy=${http_proxy}; ${INPUT}" +fi +if [ ! -z ${https_proxy} ]; then + INPUT="export https_proxy=${https_proxy}; ${INPUT}" +fi -echo "Running '${INPUT}' in EESSI ${EESSI_PILOT_VERSION} compatibility layer environment..." +echo "Running '${INPUT}' in EESSI (${EESSI_CVMFS_REPO}) ${EESSI_PILOT_VERSION} compatibility layer environment..." ${EESSI_COMPAT_LAYER_DIR}/startprefix <<< "${INPUT}" diff --git a/cfg_files.sh b/scripts/cfg_files.sh similarity index 93% rename from cfg_files.sh rename to scripts/cfg_files.sh index 885ebd0877..57ea2f7c03 100644 --- a/cfg_files.sh +++ b/scripts/cfg_files.sh @@ -47,9 +47,7 @@ function cfg_load { else val=$(cfg_get_key_value $line) # trim leading and trailing spaces as well - #cur_key=$(echo $val | cut -f1 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//') cur_key=$(echo $val | cut -f1 -d'=' | cfg_trim_spaces) - #cur_val=$(echo $val | cut -f2 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//') cur_val=$(echo $val | cut -f2 -d'=' | cfg_trim_spaces) if [[ -n "$cur_key" ]]; then # section + key is the associative in bash array, the field separator is space @@ -69,6 +67,20 @@ function cfg_print { done } +function cfg_sections { + declare -A sections + for key in "${!cfg_repos[@]}" + do + # extract section from the associative key + section=$(echo $key | cut -f1 -d ' ') + sections[${section}]=1 + done + for repo in "${!sections[@]}" + do + echo "${repo}" + done +} + function cfg_get_value { section=$1 key=$2 @@ -153,4 +165,3 @@ function cfg_print_map { echo "${index} --> ${cfg_file_map[${index}]}" done } - diff --git a/scripts/utils.sh b/scripts/utils.sh index 5d8455bb68..d0da95e87f 100644 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -31,3 +31,68 @@ function check_exit_code { fatal_error "${fail_msg}" fi } + +function get_path_for_tool { + tool_name=$1 + tool_envvar_name=$2 + + which_out=$(which ${tool_name} 2>&1) + exit_code=$? + if [[ ${exit_code} -eq 0 ]]; then + echo "INFO: found tool ${tool_name} in PATH (${which_out})" >&2 + echo "${which_out}" + return 0 + fi + if [[ -z "${tool_envvar_name}" ]]; then + msg="no env var holding the full path to tool '${tool_name}' provided" + echo "${msg}" >&2 + return 1 + else + tool_envvar_value=${!tool_envvar_name} + if [[ -x "${tool_envvar_value}" ]]; then + msg="INFO: found tool ${tool_envvar_value} via env var ${tool_envvar_name}" + echo "${msg}" >&2 + echo "${tool_envvar_value}" + return 0 + else + msg="ERROR: tool '${tool_name}' not in PATH\n" + msg+="ERROR: tool '${tool_envvar_value}' via '${tool_envvar_name}' not in PATH" + echo "${msg}" >&2 + echo "" + return 2 + fi + fi +} + +function get_host_from_url { + url=$1 + re="(http|https)://([^/:]+)" + if [[ $url =~ $re ]]; then + echo ${BASH_REMATCH[2]} + return 0 + else + echo "" + return 1 + fi +} + +function get_port_from_url { + url=$1 + re="(http|https)://[^:]+:([0-9]+)" + if [[ $url =~ $re ]]; then + echo ${BASH_REMATCH[2]} + return 0 + else + echo "" + return 1 + fi +} + +function get_ipv4_address { + hname=$1 + hipv4=$(grep ${hname} /etc/hosts | grep -v '^[[:space:]]*#' | cut -d ' ' -f 1) + # TODO try other methods if the one above does not work --> tool that verifies + # what method can be used? + echo "${hipv4}" + return 0 +}