diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 85ed7e4cb..ea53088b1 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -286,6 +286,7 @@ packages: parallelio: require: - '@2.6.2' + - +pnetcdf # DH* CAN WE UPDATE? NEED A NEW SHARED-INTEL PATCH? parallel-netcdf: require: diff --git a/configs/sites/tier1/atlantis/packages_clang-21.1.0.yaml b/configs/sites/tier1/atlantis/packages_clang-21.1.0.yaml index 2ee2afaf6..221d2f1d0 100644 --- a/configs/sites/tier1/atlantis/packages_clang-21.1.0.yaml +++ b/configs/sites/tier1/atlantis/packages_clang-21.1.0.yaml @@ -1,4 +1,8 @@ packages: + all: + require: + - any_of: ['%llvm@=21.1.0'] + when: '%llvm' mpi: buildable: False require: diff --git a/configs/sites/tier1/atlantis/packages_gcc-13.4.0.yaml b/configs/sites/tier1/atlantis/packages_gcc-13.4.0.yaml index 3adfb1dcc..f679ce2bf 100644 --- a/configs/sites/tier1/atlantis/packages_gcc-13.4.0.yaml +++ b/configs/sites/tier1/atlantis/packages_gcc-13.4.0.yaml @@ -1,4 +1,8 @@ packages: + all: + require: + - any_of: ['%gcc@=13.4.0'] + when: '%gcc' mpi: buildable: False require: diff --git a/configs/sites/tier1/atlantis/packages_oneapi-2024.2.1.yaml b/configs/sites/tier1/atlantis/packages_oneapi-2024.2.1.yaml index 25e83ccf6..bf6bd097f 100644 --- a/configs/sites/tier1/atlantis/packages_oneapi-2024.2.1.yaml +++ b/configs/sites/tier1/atlantis/packages_oneapi-2024.2.1.yaml @@ -1,4 +1,10 @@ packages: + all: + require: + - any_of: ['%intel-oneapi-compilers@=2024.2.1'] + when: '%intel-oneapi-compilers' + - any_of: ['%gcc@=11.2.0'] + when: '%gcc' mpi: buildable: false require: diff --git a/configs/sites/tier1/atlantis/packages_oneapi-2025.3.0.yaml b/configs/sites/tier1/atlantis/packages_oneapi-2025.3.0.yaml index ca673fce0..40749709b 100644 --- a/configs/sites/tier1/atlantis/packages_oneapi-2025.3.0.yaml +++ b/configs/sites/tier1/atlantis/packages_oneapi-2025.3.0.yaml @@ -1,4 +1,10 @@ packages: + all: + require: + - any_of: ['%intel-oneapi-compilers@=2025.3.0'] + when: '%intel-oneapi-compilers' + - any_of: ['%gcc@=11.2.0'] + when: '%gcc' mpi: buildable: false require: diff --git a/configs/sites/tier1/blueback/packages_gcc-13.3.0.yaml b/configs/sites/tier1/blueback/packages_gcc-13.3.0.yaml index 78fb81ee6..1c49a5fa4 100644 --- a/configs/sites/tier1/blueback/packages_gcc-13.3.0.yaml +++ b/configs/sites/tier1/blueback/packages_gcc-13.3.0.yaml @@ -1,4 +1,8 @@ packages: + all: + require: + - any_of: ['%gcc@=13.3.0'] + when: '%gcc' mpi: buildable: false require: diff --git a/configs/sites/tier1/blueback/packages_oneapi-2025.0.4.yaml b/configs/sites/tier1/blueback/packages_oneapi-2025.0.4.yaml index 0924a7055..13fe28a26 100644 --- a/configs/sites/tier1/blueback/packages_oneapi-2025.0.4.yaml +++ b/configs/sites/tier1/blueback/packages_oneapi-2025.0.4.yaml @@ -1,4 +1,10 @@ packages: + all: + require: + - any_of: ['%intel-oneapi-compilers@=2025.0.4'] + when: '%intel-oneapi-compilers' + - any_of: ['%gcc@=13.3.0'] + when: '%gcc' mpi: buildable: false require: diff --git a/configs/sites/tier1/narwhal/packages_gcc-12.2.0.yaml b/configs/sites/tier1/narwhal/packages_gcc-12.2.0.yaml index eddc18aff..6aee77338 100644 --- a/configs/sites/tier1/narwhal/packages_gcc-12.2.0.yaml +++ b/configs/sites/tier1/narwhal/packages_gcc-12.2.0.yaml @@ -1,4 +1,8 @@ packages: + all: + require: + - any_of: ['%gcc@=12.2.0'] + when: '%gcc' mpi: buildable: false require: diff --git a/configs/sites/tier1/narwhal/packages_oneapi-2024.2.0.yaml b/configs/sites/tier1/narwhal/packages_oneapi-2024.2.0.yaml index f4484548c..a290e967a 100644 --- a/configs/sites/tier1/narwhal/packages_oneapi-2024.2.0.yaml +++ b/configs/sites/tier1/narwhal/packages_oneapi-2024.2.0.yaml @@ -1,4 +1,10 @@ packages: + all: + require: + - any_of: ['%intel-oneapi-compilers@=2024.2.0'] + when: '%intel-oneapi-compilers' + - any_of: ['%gcc@=12.2.0'] + when: '%gcc' mpi: buildable: false require: diff --git a/configs/sites/tier1/nautilus/packages_gcc-13.3.1.yaml b/configs/sites/tier1/nautilus/packages_gcc-13.3.1.yaml index 3a4a8adee..faf514071 100644 --- a/configs/sites/tier1/nautilus/packages_gcc-13.3.1.yaml +++ b/configs/sites/tier1/nautilus/packages_gcc-13.3.1.yaml @@ -1,4 +1,8 @@ packages: + all: + require: + - any_of: ['%gcc@=13.3.1'] + when: '%gcc' mpi: buildable: False require: diff --git a/configs/sites/tier1/nautilus/packages_oneapi-2024.2.1.yaml b/configs/sites/tier1/nautilus/packages_oneapi-2024.2.1.yaml new file mode 100644 index 000000000..95166c31d --- /dev/null +++ b/configs/sites/tier1/nautilus/packages_oneapi-2024.2.1.yaml @@ -0,0 +1,57 @@ +packages: + all: + require: + - any_of: ['%intel-oneapi-compilers@=2024.2.1'] + when: '%intel-oneapi-compilers' + - any_of: ['%gcc@=11.2.1'] + when: '%gcc' + mpi: + buildable: false + require: + - intel-oneapi-mpi@2021.13 + intel-oneapi-compilers: + buildable: false + externals: + - spec: intel-oneapi-compilers@2024.2.1 + prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1 + modules: + - tbb/2021.13 + - compiler-rt/2024.2.1 + - compiler/2024.2.1 + extra_attributes: + compilers: + c: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/icx + cxx: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/icpx + fortran: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/ifort + environment: + prepend_path: + PATH: /opt/rh/gcc-toolset-11/root/usr/bin + CPATH: /opt/rh/gcc-toolset-11/root/usr/include + extra_rpaths: + - /opt/rh/gcc-toolset-11/root/usr/lib64 + - /opt/rh/gcc-toolset-11/root/usr/lib + gcc: + buildable: false + externals: + - spec: gcc@11.2.1 languages:='c,c++' + prefix: /opt/rh/gcc-toolset-11/root/usr + modules: + - scl/gcc-toolset-11 + extra_attributes: + compilers: + c: /opt/rh/gcc-toolset-11/root/usr/bin/gcc + cxx: /opt/rh/gcc-toolset-11/root/usr/bin/g++ + fortran: /opt/rh/gcc-toolset-11/root/usr/bin/gfortran + intel-oneapi-mpi: + buildable: false + externals: + - spec: intel-oneapi-mpi@2021.13 + prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1 + modules: + - mpi/2021.13 + - slurm + intel-oneapi-mkl: + buildable: false + externals: + - spec: intel-oneapi-mkl@2024.2 + prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1 diff --git a/configs/sites/tier1/nautilus/packages_oneapi-2025.3.0.yaml b/configs/sites/tier1/nautilus/packages_oneapi-2025.3.0.yaml index 3cc64c1ac..1698fe453 100644 --- a/configs/sites/tier1/nautilus/packages_oneapi-2025.3.0.yaml +++ b/configs/sites/tier1/nautilus/packages_oneapi-2025.3.0.yaml @@ -1,4 +1,10 @@ packages: + all: + require: + - any_of: ['%intel-oneapi-compilers@=2025.3.0'] + when: '%intel-oneapi-compilers' + - any_of: ['%gcc@=11.2.1'] + when: '%gcc' mpi: buildable: false require: diff --git a/configs/sites/tier2/blackpearl/packages_gcc-13.2.1.yaml b/configs/sites/tier2/blackpearl/packages_gcc-13.2.1.yaml index 695941343..c9cb6577b 100644 --- a/configs/sites/tier2/blackpearl/packages_gcc-13.2.1.yaml +++ b/configs/sites/tier2/blackpearl/packages_gcc-13.2.1.yaml @@ -1,4 +1,8 @@ packages: + all: + require: + - any_of: ['%gcc@=13.2.1'] + when: '%gcc' mpi: require: - "openmpi@=5.0.8" diff --git a/configs/sites/tier2/blackpearl/packages_oneapi-2024.2.1.yaml b/configs/sites/tier2/blackpearl/packages_oneapi-2024.2.1.yaml index ff2b795b0..19c6c965b 100644 --- a/configs/sites/tier2/blackpearl/packages_oneapi-2024.2.1.yaml +++ b/configs/sites/tier2/blackpearl/packages_oneapi-2024.2.1.yaml @@ -1,4 +1,10 @@ packages: + all: + require: + - any_of: ['%intel-oneapi-compilers@=2024.2.1'] + when: '%intel-oneapi-compilers' + - any_of: ['%gcc@=11.4.1'] + when: '%gcc' mpi: buildable: False require: diff --git a/configs/sites/tier2/bounty/packages_clang-21.1.1.yaml b/configs/sites/tier2/bounty/packages_clang-21.1.1.yaml index 567b19442..74f856b4c 100644 --- a/configs/sites/tier2/bounty/packages_clang-21.1.1.yaml +++ b/configs/sites/tier2/bounty/packages_clang-21.1.1.yaml @@ -1,4 +1,8 @@ packages: + all: + require: + - any_of: ['%llvm@=21.1.1'] + when: '%llvm' mpi: require: - "openmpi@=5.0.8" diff --git a/configs/sites/tier2/bounty/packages_gcc-13.3.1.yaml b/configs/sites/tier2/bounty/packages_gcc-13.3.1.yaml index f7e40520c..7d7a68dc9 100644 --- a/configs/sites/tier2/bounty/packages_gcc-13.3.1.yaml +++ b/configs/sites/tier2/bounty/packages_gcc-13.3.1.yaml @@ -1,4 +1,8 @@ packages: + all: + require: + - any_of: ['%gcc@=13.3.1'] + when: '%gcc' mpi: require: - "openmpi@=5.0.8" diff --git a/configs/sites/tier2/bounty/packages_oneapi-2025.3.0.yaml b/configs/sites/tier2/bounty/packages_oneapi-2025.3.0.yaml index 398fa1c5a..86fd28a86 100644 --- a/configs/sites/tier2/bounty/packages_oneapi-2025.3.0.yaml +++ b/configs/sites/tier2/bounty/packages_oneapi-2025.3.0.yaml @@ -1,4 +1,10 @@ packages: + all: + require: + - any_of: ['%intel-oneapi-compilers@=2025.3.0'] + when: '%intel-oneapi-compilers' + - any_of: ['%gcc@=11.5.0'] + when: '%gcc' mpi: buildable: False require: diff --git a/configs/templates/neptune-dev-llvm/spack.yaml b/configs/templates/neptune-dev-llvm/spack.yaml index a01ce9c4c..a305e9639 100644 --- a/configs/templates/neptune-dev-llvm/spack.yaml +++ b/configs/templates/neptune-dev-llvm/spack.yaml @@ -29,7 +29,3 @@ spack: esmf: require: - +python - # Avoid duplicate builds of parallelio +pnetcdf and ~pnetcdf - parallelio: - require: - - ~pnetcdf diff --git a/configs/templates/neptune-dev/spack.yaml b/configs/templates/neptune-dev/spack.yaml index fb7d251eb..105c9f9d4 100644 --- a/configs/templates/neptune-dev/spack.yaml +++ b/configs/templates/neptune-dev/spack.yaml @@ -22,7 +22,3 @@ spack: esmf: require: - +python - # Avoid duplicate builds of parallelio +pnetcdf and ~pnetcdf - parallelio: - require: - - ~pnetcdf diff --git a/configs/templates/neptune-ops/spack.yaml b/configs/templates/neptune-ops/spack.yaml index 36ac735b4..51b7264b2 100644 --- a/configs/templates/neptune-ops/spack.yaml +++ b/configs/templates/neptune-ops/spack.yaml @@ -15,7 +15,3 @@ spack: esmf: require: - ~python - # Avoid duplicate builds of parallelio +pnetcdf and ~pnetcdf - parallelio: - require: - - ~pnetcdf diff --git a/spack b/spack index 459bb8abc..45341d881 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 459bb8abcfbfd33b5b9f097f88ca594b92232d2f +Subproject commit 45341d881fe198be63a8bc7e89929355b2b3bba0 diff --git a/util/nrl/batch_install.sh b/util/nrl/batch_install.sh index 604861e6b..61365b5b2 100755 --- a/util/nrl/batch_install.sh +++ b/util/nrl/batch_install.sh @@ -138,7 +138,7 @@ case ${SPACK_STACK_BATCH_HOST} in SPACK_STACK_CARGO_MIRROR="/p/cwfs/projects/NEPTUNE/spack-stack/cargo-mirror" ;; nautilus) - SPACK_STACK_BATCH_COMPILERS=("oneapi@=2025.3.0" "gcc@=13.3.1") + SPACK_STACK_BATCH_COMPILERS=("oneapi@=2024.2.1" "oneapi@=2025.3.0" "gcc@=13.3.1") SPACK_STACK_BATCH_TEMPLATES=("neptune-dev" "unified-dev" "cylc-dev") SPACK_STACK_MODULE_CHOICE="tcl" SPACK_STACK_BOOTSTRAP_MIRROR="/p/cwfs/projects/NEPTUNE/spack-stack/bootstrap-mirror" @@ -333,6 +333,7 @@ fi ignore_env_exist=${SPACK_STACK_IGNORE_ENV_EXIST:-false} # Loop through all compilers and templates for this host +first_pass="true" for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do if [[ ! ${compiler} == *"@="* ]]; then @@ -445,6 +446,14 @@ for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do nautilus) umask 0022 module purge + case ${compiler} in + oneapi@=2024.2.1) + module use /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/modulefiles + ;; + oneapi@=2025.3.0) + module use /p/app/projects/NEPTUNE/spack-stack/oneapi-2025.3.0/modulefiles + ;; + esac ;; navy-aws) umask 0022 @@ -479,7 +488,28 @@ for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do module li source setup.sh - spack clean -a + if [[ "${first_pass}" == "true" ]]; then + spack clean -a + else + # Don't remove software and configuration needed to bootstrap Spack + spack clean -d -f -m -p -s + fi + + # Update bootstrap mirror if requested before creating any + # environments. It is sufficient to do this one time only. + if [[ "${update_bootstrap_mirror}" == "true"* ]]; then + tmp_bootstrap_mirror_path=${PWD}/tmp-bootstrap-mirror + echo "Creating bootstrap mirror ${tmp_bootstrap_mirror_path} ..." + rm -fr ${tmp_bootstrap_mirror_path} + spack bootstrap mirror --binary-packages ${tmp_bootstrap_mirror_path} 2>&1 | tee log.bootstrap-mirror.001 + rsync -a ${tmp_bootstrap_mirror_path}/ ${bootstrap_mirror_path}/ + rm -fr ${tmp_bootstrap_mirror_path} + # Update buildcache index + spack buildcache update-index ${bootstrap_mirror_path}/bootstrap_cache + # Fix permissions for the bootstrap mirror + fix_permissions ${host} ${bootstrap_mirror_path} 0 + update_bootstrap_mirror="false" + fi if [[ ! ${env_exists} == "true" ]]; then spack stack create env --name=${env_name} \ @@ -495,33 +525,15 @@ for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do # Workaround for ParallelWorks (no NRL Enterprise GitHub access yet) sed -i 's/+adp/~adp/g' ${env_dir}/spack.yaml - # Update bootstrap mirror if requested - if [[ "${update_bootstrap_mirror}" == "true"* ]]; then - tmp_bootstrap_mirror_path=${PWD}/tmp-bootstrap-mirror-${env_name} - echo "Creating bootstrap mirror ${tmp_bootstrap_mirror_path} ..." - rm -fr ${tmp_bootstrap_mirror_path} - spack bootstrap mirror --binary-packages ${tmp_bootstrap_mirror_path} 2>&1 | tee log.bootstrap-mirror.${env_name}.001 - rsync -a ${tmp_bootstrap_mirror_path}/ ${bootstrap_mirror_path}/ - rm -fr ${tmp_bootstrap_mirror_path} - # Update buildcache index - spack buildcache update-index ${bootstrap_mirror_path}/bootstrap_cache - fi - echo "Registering bootstrap mirror ${bootstrap_mirror_path} ..." if [[ ! -d ${bootstrap_mirror_path} ]]; then echo "ERROR, directory ${bootstrap_mirror_path} not found" exit 1 fi - # If the environment already existed, then it is possible that the bootstrap - # sources were already added. In this case, ignore errors from these commands. - if [[ ${env_exists} == "true" ]]; then - set +e - fi - spack bootstrap add --trust local-sources ${bootstrap_mirror_path}/metadata/sources - spack bootstrap add --trust local-binaries ${bootstrap_mirror_path}/metadata/binaries - if [[ ${env_exists} == "true" ]]; then - set -e - fi + spack bootstrap list | grep local-sources || \ + spack bootstrap add --trust local-sources ${bootstrap_mirror_path}/metadata/sources + spack bootstrap list | grep local-binaries || \ + spack bootstrap add --trust local-binaries ${bootstrap_mirror_path}/metadata/binaries # Check that the site has mirrors configured for local source and build caches, # and extract the local path on disk. Need to strip leading "file://" from path @@ -610,9 +622,6 @@ for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do # When creating or updating buildcaches, fix permissions for mirrors. # Mirrors do not contain executables, therefore skip looking for them. - if [[ "${update_bootstrap_mirror}" == "true" ]]; then - fix_permissions ${host} ${bootstrap_mirror_path} 0 - fi if [[ "${update_source_cache}" == "true" ]]; then fix_permissions ${host} ${source_mirror_path} 0 fi @@ -623,9 +632,10 @@ for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do fix_permissions ${host} ${cargo_mirror_path} 0 fi - # Clean up - spack clean -a + # Clean up (don't remove software and configuration needed to bootstrap Spack) + spack clean -d -f -m -p -s spack env deactivate + first_pass="false" done