nrl_batch_install.sh improvements with NRL site config updates, Nautilus oneapi@2024.2.1 support, re-enable parallel-netcdf in NEPTUNE templates

configs/common/packages.yaml

@@ -286,6 +286,7 @@ packages:
     parallelio:
       require:
       - '@2.6.2'
+      - +pnetcdf
     # DH* CAN WE UPDATE? NEED A NEW SHARED-INTEL PATCH?
     parallel-netcdf:
       require:

configs/sites/tier1/atlantis/packages_clang-21.1.0.yaml

@@ -1,4 +1,8 @@
 packages:
+  all:
+    require:
+    - any_of: ['%llvm@=21.1.0']
+      when: '%llvm'
   mpi:
     buildable: False
     require:

configs/sites/tier1/atlantis/packages_gcc-13.4.0.yaml

@@ -1,4 +1,8 @@
 packages:
+  all:
+    require:
+    - any_of: ['%gcc@=13.4.0']
+      when: '%gcc'
   mpi:
     buildable: False
     require:

configs/sites/tier1/atlantis/packages_oneapi-2024.2.1.yaml

@@ -1,4 +1,10 @@
 packages:
+  all:
+    require:
+    - any_of: ['%intel-oneapi-compilers@=2024.2.1']
+      when: '%intel-oneapi-compilers'
+    - any_of: ['%gcc@=11.2.0']
+      when: '%gcc'
   mpi:
     buildable: false
     require:

configs/sites/tier1/atlantis/packages_oneapi-2025.3.0.yaml

@@ -1,4 +1,10 @@
 packages:
+  all:
+    require:
+    - any_of: ['%intel-oneapi-compilers@=2025.3.0']
+      when: '%intel-oneapi-compilers'
+    - any_of: ['%gcc@=11.2.0']
+      when: '%gcc'
   mpi:
     buildable: false
     require:

configs/sites/tier1/blueback/packages_gcc-13.3.0.yaml

@@ -1,4 +1,8 @@
 packages:
+  all:
+    require:
+    - any_of: ['%gcc@=13.3.0']
+      when: '%gcc'
   mpi:
     buildable: false
     require:

configs/sites/tier1/blueback/packages_oneapi-2025.0.4.yaml

@@ -1,4 +1,10 @@
 packages:
+  all:
+    require:
+    - any_of: ['%intel-oneapi-compilers@=2025.0.4']
+      when: '%intel-oneapi-compilers'
+    - any_of: ['%gcc@=13.3.0']
+      when: '%gcc'
   mpi:
     buildable: false
     require:

configs/sites/tier1/narwhal/packages_gcc-12.2.0.yaml

@@ -1,4 +1,8 @@
 packages:
+  all:
+    require:
+    - any_of: ['%gcc@=12.2.0']
+      when: '%gcc'
   mpi:
     buildable: false
     require:

configs/sites/tier1/narwhal/packages_oneapi-2024.2.0.yaml

@@ -1,4 +1,10 @@
 packages:
+  all:
+    require:
+    - any_of: ['%intel-oneapi-compilers@=2024.2.0']
+      when: '%intel-oneapi-compilers'
+    - any_of: ['%gcc@=12.2.0']
+      when: '%gcc'
   mpi:
     buildable: false
     require:

configs/sites/tier1/nautilus/packages_gcc-13.3.1.yaml

@@ -1,4 +1,8 @@
 packages:
+  all:
+    require:
+    - any_of: ['%gcc@=13.3.1']
+      when: '%gcc'
   mpi:
     buildable: False
     require:

configs/sites/tier1/nautilus/packages_oneapi-2024.2.1.yaml

@@ -0,0 +1,57 @@
+packages:
+  all:
+    require:
+    - any_of: ['%intel-oneapi-compilers@=2024.2.1']
+      when: '%intel-oneapi-compilers'
+    - any_of: ['%gcc@=11.2.1']
+      when: '%gcc'
+  mpi:
+    buildable: false
+    require:
+    - intel-oneapi-mpi@2021.13
+  intel-oneapi-compilers:
+    buildable: false
+    externals:
+    - spec: intel-oneapi-compilers@2024.2.1
+      prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1
+      modules:
+      - tbb/2021.13
+      - compiler-rt/2024.2.1
+      - compiler/2024.2.1
+      extra_attributes:
+        compilers:
+          c: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/icx
+          cxx: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/icpx
+          fortran: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/ifort
+        environment:
+          prepend_path:
+            PATH: /opt/rh/gcc-toolset-11/root/usr/bin
+            CPATH: /opt/rh/gcc-toolset-11/root/usr/include
+        extra_rpaths:
+        - /opt/rh/gcc-toolset-11/root/usr/lib64
+        - /opt/rh/gcc-toolset-11/root/usr/lib
+  gcc:
+    buildable: false
+    externals:
+    - spec: gcc@11.2.1 languages:='c,c++'
+      prefix: /opt/rh/gcc-toolset-11/root/usr
+      modules:
+      - scl/gcc-toolset-11
+      extra_attributes:
+        compilers:
+          c: /opt/rh/gcc-toolset-11/root/usr/bin/gcc
+          cxx: /opt/rh/gcc-toolset-11/root/usr/bin/g++
+          fortran: /opt/rh/gcc-toolset-11/root/usr/bin/gfortran
+  intel-oneapi-mpi:
+    buildable: false
+    externals:
+    - spec: intel-oneapi-mpi@2021.13
+      prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1
+      modules:
+      - mpi/2021.13
+      - slurm
+  intel-oneapi-mkl:
+    buildable: false
+    externals:
+    - spec: intel-oneapi-mkl@2024.2
+      prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1

configs/sites/tier1/nautilus/packages_oneapi-2025.3.0.yaml

@@ -1,4 +1,10 @@
 packages:
+  all:
+    require:
+    - any_of: ['%intel-oneapi-compilers@=2025.3.0']
+      when: '%intel-oneapi-compilers'
+    - any_of: ['%gcc@=11.2.1']
+      when: '%gcc'
   mpi:
     buildable: false
     require:

configs/sites/tier2/blackpearl/packages_gcc-13.2.1.yaml

@@ -1,4 +1,8 @@
 packages:
+  all:
+    require:
+    - any_of: ['%gcc@=13.2.1']
+      when: '%gcc'
   mpi:
     require:
     - "openmpi@=5.0.8"

configs/sites/tier2/blackpearl/packages_oneapi-2024.2.1.yaml

@@ -1,4 +1,10 @@
 packages:
+  all:
+    require:
+    - any_of: ['%intel-oneapi-compilers@=2024.2.1']
+      when: '%intel-oneapi-compilers'
+    - any_of: ['%gcc@=11.4.1']
+      when: '%gcc'
   mpi:
     buildable: False
     require:

configs/sites/tier2/bounty/packages_clang-21.1.1.yaml

@@ -1,4 +1,8 @@
 packages:
+  all:
+    require:
+    - any_of: ['%llvm@=21.1.1']
+      when: '%llvm'
   mpi:
     require:
     - "openmpi@=5.0.8"

configs/sites/tier2/bounty/packages_gcc-13.3.1.yaml

@@ -1,4 +1,8 @@
 packages:
+  all:
+    require:
+    - any_of: ['%gcc@=13.3.1']
+      when: '%gcc'
   mpi:
     require:
     - "openmpi@=5.0.8"

configs/sites/tier2/bounty/packages_oneapi-2025.3.0.yaml

@@ -1,4 +1,10 @@
 packages:
+  all:
+    require:
+    - any_of: ['%intel-oneapi-compilers@=2025.3.0']
+      when: '%intel-oneapi-compilers'
+    - any_of: ['%gcc@=11.5.0']
+      when: '%gcc'
   mpi:
     buildable: False
     require:

configs/templates/neptune-dev-llvm/spack.yaml

@@ -29,7 +29,3 @@ spack:
     esmf:
       require:
       - +python
-    # Avoid duplicate builds of parallelio +pnetcdf and ~pnetcdf
-    parallelio:
-      require:
-      - ~pnetcdf

configs/templates/neptune-dev/spack.yaml

@@ -22,7 +22,3 @@ spack:
     esmf:
       require:
       - +python
-    # Avoid duplicate builds of parallelio +pnetcdf and ~pnetcdf
-    parallelio:
-      require:
-      - ~pnetcdf

configs/templates/neptune-ops/spack.yaml

@@ -15,7 +15,3 @@ spack:
     esmf:
       require:
       - ~python
-    # Avoid duplicate builds of parallelio +pnetcdf and ~pnetcdf
-    parallelio:
-      require:
-      - ~pnetcdf

util/nrl/batch_install.sh

@@ -138,7 +138,7 @@ case ${SPACK_STACK_BATCH_HOST} in
     SPACK_STACK_CARGO_MIRROR="/p/cwfs/projects/NEPTUNE/spack-stack/cargo-mirror"
     ;;
   nautilus)
-    SPACK_STACK_BATCH_COMPILERS=("oneapi@=2025.3.0" "gcc@=13.3.1")
+    SPACK_STACK_BATCH_COMPILERS=("oneapi@=2024.2.1" "oneapi@=2025.3.0" "gcc@=13.3.1")
     SPACK_STACK_BATCH_TEMPLATES=("neptune-dev" "unified-dev" "cylc-dev")
     SPACK_STACK_MODULE_CHOICE="tcl"
     SPACK_STACK_BOOTSTRAP_MIRROR="/p/cwfs/projects/NEPTUNE/spack-stack/bootstrap-mirror"
@@ -333,6 +333,7 @@ fi
 ignore_env_exist=${SPACK_STACK_IGNORE_ENV_EXIST:-false}
 
 # Loop through all compilers and templates for this host
+first_pass="true"
 for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do
 
   if [[ ! ${compiler} == *"@="* ]]; then
@@ -445,6 +446,14 @@ for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do
       nautilus)
         umask 0022
         module purge
+        case ${compiler} in
+          oneapi@=2024.2.1)
+            module use /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/modulefiles
+            ;;
+          oneapi@=2025.3.0)
+            module use /p/app/projects/NEPTUNE/spack-stack/oneapi-2025.3.0/modulefiles
+            ;;
+        esac
         ;;
       navy-aws)
         umask 0022
@@ -479,7 +488,28 @@ for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do
     module li
 
     source setup.sh
-    spack clean -a
+    if [[ "${first_pass}" == "true" ]]; then
+      spack clean -a
+    else
+      # Don't remove software and configuration needed to bootstrap Spack
+      spack clean -d -f -m -p -s
+    fi
+
+    # Update bootstrap mirror if requested before creating any
+    # environments. It is sufficient to do this one time only.
+    if [[ "${update_bootstrap_mirror}" == "true"*  ]]; then
+      tmp_bootstrap_mirror_path=${PWD}/tmp-bootstrap-mirror
+      echo "Creating bootstrap mirror ${tmp_bootstrap_mirror_path} ..."
+      rm -fr ${tmp_bootstrap_mirror_path}
+      spack bootstrap mirror --binary-packages ${tmp_bootstrap_mirror_path} 2>&1 | tee log.bootstrap-mirror.001
+      rsync -a ${tmp_bootstrap_mirror_path}/ ${bootstrap_mirror_path}/
+      rm -fr ${tmp_bootstrap_mirror_path}
+      # Update buildcache index
+      spack buildcache update-index ${bootstrap_mirror_path}/bootstrap_cache
+      # Fix permissions for the bootstrap mirror
+      fix_permissions ${host} ${bootstrap_mirror_path} 0
+      update_bootstrap_mirror="false"
+    fi
 
     if [[ ! ${env_exists} == "true" ]]; then
       spack stack create env --name=${env_name} \
@@ -495,33 +525,15 @@ for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do
     # Workaround for ParallelWorks (no NRL Enterprise GitHub access yet)
     sed -i 's/+adp/~adp/g' ${env_dir}/spack.yaml
 
-    # Update bootstrap mirror if requested
-    if [[ "${update_bootstrap_mirror}" == "true"*  ]]; then
-      tmp_bootstrap_mirror_path=${PWD}/tmp-bootstrap-mirror-${env_name}
-      echo "Creating bootstrap mirror ${tmp_bootstrap_mirror_path} ..."
-      rm -fr ${tmp_bootstrap_mirror_path}
-      spack bootstrap mirror --binary-packages ${tmp_bootstrap_mirror_path} 2>&1 | tee log.bootstrap-mirror.${env_name}.001
-      rsync -a ${tmp_bootstrap_mirror_path}/ ${bootstrap_mirror_path}/
-      rm -fr ${tmp_bootstrap_mirror_path}
-      # Update buildcache index
-      spack buildcache update-index ${bootstrap_mirror_path}/bootstrap_cache
-    fi
-
     echo "Registering bootstrap mirror ${bootstrap_mirror_path} ..."
     if [[ ! -d ${bootstrap_mirror_path} ]]; then
       echo "ERROR, directory ${bootstrap_mirror_path} not found"
       exit 1
     fi
-    # If the environment already existed, then it is possible that the bootstrap
-    # sources were already added. In this case, ignore errors from these commands.
-    if [[ ${env_exists} == "true" ]]; then
-      set +e
-    fi
-    spack bootstrap add --trust local-sources ${bootstrap_mirror_path}/metadata/sources
-    spack bootstrap add --trust local-binaries ${bootstrap_mirror_path}/metadata/binaries
-    if [[ ${env_exists} == "true" ]]; then
-      set -e
-    fi
+    spack bootstrap list | grep local-sources || \
+        spack bootstrap add --trust local-sources ${bootstrap_mirror_path}/metadata/sources
+    spack bootstrap list | grep local-binaries || \
+        spack bootstrap add --trust local-binaries ${bootstrap_mirror_path}/metadata/binaries
 
     # Check that the site has mirrors configured for local source and build caches,
     # and extract the local path on disk. Need to strip leading "file://" from path
@@ -610,9 +622,6 @@ for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do
 
     # When creating or updating buildcaches, fix permissions for mirrors.
     # Mirrors do not contain executables, therefore skip looking for them.
-    if [[ "${update_bootstrap_mirror}" == "true" ]]; then
-      fix_permissions ${host} ${bootstrap_mirror_path} 0
-    fi
     if [[ "${update_source_cache}" == "true" ]]; then
       fix_permissions ${host} ${source_mirror_path} 0
     fi
@@ -623,9 +632,10 @@ for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do
       fix_permissions ${host} ${cargo_mirror_path} 0
     fi
 
-    # Clean up
-    spack clean -a
+    # Clean up (don't remove software and configuration needed to bootstrap Spack)
+    spack clean -d -f -m -p -s
     spack env deactivate
+    first_pass="false"
 
   done