diff --git a/dev/parm/config/gfs/config.resources.GAEAC6 b/dev/parm/config/gfs/config.resources.GAEAC6 index 3e1442b676..82d9ae1382 100644 --- a/dev/parm/config/gfs/config.resources.GAEAC6 +++ b/dev/parm/config/gfs/config.resources.GAEAC6 @@ -5,7 +5,6 @@ unset memory unset "memory_${RUN}" -export FI_CXI_RX_MATCH_MODE=hybrid case ${step} in "fcst" | "efcs") @@ -48,7 +47,7 @@ case ${step} in "C1152") export threads_per_task=16 ;; - *) + *) ;; esac if [[ "${CASE}" == "C768" || "${CASE}" == "C1152" ]]; then @@ -69,7 +68,7 @@ case ${step} in "marinebmat") case ${OCNRES} in "025") - tasks_per_node=$(( max_tasks_per_node / 4 )) + tasks_per_node=$(( max_tasks_per_node / 4 )) ;; *) ;; @@ -117,6 +116,6 @@ case ${step} in NTASKS=${ntasks} ;; - *) + *) ;; esac diff --git a/env/GAEAC6.env b/env/GAEAC6.env index 9a5c51d3a3..31182f89da 100755 --- a/env/GAEAC6.env +++ b/env/GAEAC6.env @@ -14,6 +14,9 @@ export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out" export OMP_STACKSIZE=2048000 export NTHSTACK=1024000000 +# Use the hybrid mode to avoid RXM issues with large messages +# Note that this seems to have issues with larger jobs. These will need to use the software mode. +export FI_CXI_RX_MATCH_MODE=hybrid ulimit -s unlimited ulimit -a @@ -134,6 +137,7 @@ case ${step} in export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export APRUN_MARINEANLVAR="${APRUN_default}" + export FI_CXI_RX_MATCH_MODE=software ;; "marineanlecen") @@ -173,6 +177,7 @@ case ${step} in export MKL_NUM_THREADS=4 export MKL_CBWR=AUTO + export FI_CXI_RX_MATCH_MODE=software export CFP_MP=${CFP_MP:-"YES"} export USE_CFP=${USE_CFP:-"YES"} @@ -258,7 +263,7 @@ case ${step} in export MPICH_COLL_SYNC=MPI_Bcast export FI_VERBS_PREFER_XRC=0 - export FI_CXI_RX_MATCH_MODE=hybrid + export FI_CXI_RX_MATCH_MODE=software export COMEX_EAGER_THRESHOLD=65536 export FI_CXI_RDZV_THRESHOLD=65536 export FI_CXI_DEFAULT_CQ_SIZE=1048576 diff --git a/modulefiles/gw_setup.gaeac6.lua b/modulefiles/gw_setup.gaeac6.lua index d3c7be0aa0..2967cd1399 100644 --- a/modulefiles/gw_setup.gaeac6.lua +++ b/modulefiles/gw_setup.gaeac6.lua @@ -9,7 +9,6 @@ prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/c6/spack-stack-1.9.2/env local stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0" local python_ver=os.getenv("python_ver") or "3.11.7" -local cmake_ver=os.getenv("cmake_ver") or "3.27.9" load(pathJoin("stack-intel", stack_intel_ver)) load(pathJoin("python", python_ver)) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index fe812db8d8..811d86b57a 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit fe812db8d8aa9a91f97ae1c97c212aad4b095c4f +Subproject commit 811d86b57a34aff3452f7a2869ab1b4cf0c2daa4