Create utils function for nvidia-smi check

.github/workflows/tests_scripts.yml

@@ -51,7 +51,7 @@ jobs:
         # can't test with EasyBuild versions older than v4.5.2 when using EESSI 2023.06,
         # since Python in compat layer is Python 3.11.x;
         # testing with a single EasyBuild version takes a while in GitHub Actions, so stick to a single sensible version
-        for EB_VERSION in '4.6.0'; do
+        for EB_VERSION in '5.1.0'; do
           # Create script that uses load_easybuild_module.sh which we can run in compat layer environment
           # note: Be careful with single vs double quotes below!
           #       ${EB_VERSION} should be expanded, so use double quotes;
@@ -113,7 +113,7 @@ jobs:
 
           # scripts need to be copied to /tmp,
           # since create_directory_tarballs.sh must be accessible from within build container
-          ./eessi_container.sh --mode run --verbose /software-layer-scripts/create_directory_tarballs.sh 2023.06
+          ./eessi_container.sh --mode run --verbose /software-layer-scripts/create_directory_tarballs.sh "${{matrix.EESSI_VERSION}}"
           # check if tarballs have been produced
           ls -l *.tar.gz
 

EESSI-install-software.sh

@@ -17,11 +17,6 @@ display_help() {
   echo "  --skip-cuda-install    -  disable installing a full CUDA SDK in the host_injections prefix (e.g. in CI)"
 }
 
-# Function to check if a command exists
-function command_exists() {
-    command -v "$1" >/dev/null 2>&1
-}
-
 function copy_build_log() {
     # copy specified build log to specified directory, with some context added
     build_log=${1}
@@ -315,18 +310,9 @@ else
 fi
 
 # Install NVIDIA drivers in host_injections (if they exist)
-if command_exists "nvidia-smi"; then
-    export LD_LIBRARY_PATH="/.singularity.d/libs:${LD_LIBRARY_PATH}"
-    nvidia-smi --version
-    ec=$?
-    if [ ${ec} -eq 0 ]; then 
-        echo "Command 'nvidia-smi' found. Installing NVIDIA drivers for use in prefix shell..."
-        ${EESSI_PREFIX}/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh
-    else
-        echo "Warning: command 'nvidia-smi' found, but 'nvidia-smi --version' did not run succesfully."
-        echo "This script now assumes this is NOT a GPU node."
-        echo "If, and only if, the current node actually does contain Nvidia GPUs, this should be considered an error."
-    fi
+if nvidia_gpu_available; then
+    echo "Installing NVIDIA drivers for use in prefix shell..."
+    ${EESSI_PREFIX}/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh
 fi
 
 if [ ! -z "${shared_fs_path}" ]; then

bot/build.sh

@@ -220,23 +220,9 @@ BUILD_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}")
 BUILD_STEP_ARGS+=("--storage" "${STORAGE}")
 
 # add options required to handle NVIDIA support
-if command_exists "nvidia-smi"; then
-    # Accept that this may fail
-    set +e
-    nvidia-smi --version
-    ec=$?
-    set -e
-    if [ ${ec} -eq 0 ]; then
-        echo "Command 'nvidia-smi' found, using available GPU"
-        BUILD_STEP_ARGS+=("--nvidia" "all")
-    else
-        echo "Warning: command 'nvidia-smi' found, but 'nvidia-smi --version' did not run succesfully."
-        echo "This script now assumes this is NOT a GPU node."
-        echo "If, and only if, the current node actually does contain Nvidia GPUs, this should be considered an error."
-        BUILD_STEP_ARGS+=("--nvidia" "install")
-    fi
+if nvidia_gpu_available; then
+    BUILD_STEP_ARGS+=("--nvidia" "all")
 else
-    echo "No 'nvidia-smi' found, no available GPU but allowing overriding this check"
     BUILD_STEP_ARGS+=("--nvidia" "install")
 fi
 

bot/test.sh

@@ -225,20 +225,8 @@ fi
 TEST_STEP_ARGS+=("--extra-bind-paths" "/sys/fs/cgroup:/hostsys/fs/cgroup:ro")
 
 # add options required to handle NVIDIA support
-if command_exists "nvidia-smi"; then
-    # Accept that this may fail
-    set +e
-    nvidia-smi --version
-    ec=$?
-    set -e
-    if [ ${ec} -eq 0 ]; then
-        echo "Command 'nvidia-smi' found, using available GPU"
-        TEST_STEP_ARGS+=("--nvidia" "run")
-    else
-        echo "Warning: command 'nvidia-smi' found, but 'nvidia-smi --version' did not run succesfully."
-        echo "This script now assumes this is NOT a GPU node."
-        echo "If, and only if, the current node actually does contain Nvidia GPUs, this should be considered an error."
-    fi
+if nvidia_gpu_available; then
+    TEST_STEP_ARGS+=("--nvidia" "run")
 fi
 
 # prepare arguments to test_suite.sh (specific to test step)

scripts/utils.sh

@@ -147,3 +147,22 @@ function get_ipv4_address {
     echo "${hipv4}"
     return 0
 }
+
+function nvidia_gpu_available {
+    if command_exists "nvidia-smi"; then
+        # We are careful here in case we are running in a container and LD_LIBARY_PATH has been wiped
+        LD_LIBRARY_PATH="/.singularity.d/libs:${LD_LIBRARY_PATH}" nvidia-smi --version
+        ec=$?
+        if [ ${ec} -eq 0 ]; then
+            echo "Command 'nvidia-smi' found."
+	    return 0
+        else
+            echo "Warning: command 'nvidia-smi' found, but 'nvidia-smi --version' did not run succesfully."
+            echo "This script now assumes this is NOT a GPU node."
+	    return 1
+	fi
+    else
+        echo "No 'nvidia-smi' found, no available GPU." 
+	return 2
+    fi
+}