Update Tribol and MFEM

scripts/spack/configs/versions.yaml

@@ -21,7 +21,7 @@ packages:
     - spec: "@2.33.0"
   mfem:
     require:
-    - spec: "@4.9.0.3"
+    - spec: "@4.9.0.4"
   petsc:
     require:
     - spec: "@3.21.6"
@@ -33,7 +33,7 @@ packages:
     - spec: "@7.5.0"
   tribol:
     require:
-    - spec: "@0.1.0.24"
+    - spec: "@0.1.0.25"
   umpire:
     require:
     - spec: "@2025.09.0"

scripts/spack/spack_repo/smith/packages/enzyme/package.py

@@ -5,6 +5,17 @@
 from spack.package import *
 from spack_repo.builtin.packages.enzyme.package import Enzyme as BuiltinEnzyme
 
+import os
+
 class Enzyme(BuiltinEnzyme):
     # Add newer enzyme versions not added to Spack package repo
-    version("0.0.234", commit="8acdab5e870bf05e3b85f6d95d72877cb7dec470")
+    version("0.0.249", commit="c3c973213c604028762dfbb30cf9f8ec9c83fc38")
+
+    @property
+    def llvm_prefix(self):
+        spec = self.spec
+        if spec.satisfies("%libllvm=llvm"):
+            return os.path.join(spec["llvm"].prefix, "llvm")
+        if spec.satisfies("%libllvm=llvm-amdgpu"):
+            return os.path.join(spec["llvm-amdgpu"].prefix, "llvm")
+        raise InstallError("Unknown 'libllvm' provider!")

scripts/spack/spack_repo/smith/packages/mfem/fe_base_omp.patch

@@ -0,0 +1,96 @@
+diff --git a/fem/fe/fe_base.cpp b/fem/fe/fe_base.cpp
+index 645968b708..3afe7f2f65 100644
+--- a/fem/fe/fe_base.cpp
++++ b/fem/fe/fe_base.cpp
+@@ -665,56 +665,56 @@ const
+    {
+       // If the new Dof2Quad is already present, e.g. added in a previous call
+       // or added by another omp thread, return.
+-      if (DofToQuad::SearchArray(dof2quad_array, ir,
+-                                 DofToQuad::LEXICOGRAPHIC_FULL))
+-      { return; }
+-
+-      // Undo the native ordering which is what FiniteElement::GetDofToQuad
+-      // returns.
+-      auto *d2q_new = new DofToQuad(d2q);
+-      d2q_new->mode = DofToQuad::LEXICOGRAPHIC_FULL;
+-      const int nqpt = ir.GetNPoints();
++      if (DofToQuad::SearchArray(dof2quad_array, ir, DofToQuad::LEXICOGRAPHIC_FULL) == nullptr)
++      {
++         // Undo the native ordering which is what FiniteElement::GetDofToQuad
++         // returns.
++         auto *d2q_new = new DofToQuad(d2q);
++         d2q_new->mode = DofToQuad::LEXICOGRAPHIC_FULL;
++         const int nqpt = ir.GetNPoints();
+
+-      const int b_dim = (range_type == VECTOR) ? dim : 1;
++         const int b_dim = (range_type == VECTOR) ? dim : 1;
+
+-      for (int i = 0; i < nqpt; i++)
+-      {
+-         for (int d = 0; d < b_dim; d++)
++         for (int i = 0; i < nqpt; i++)
+          {
+-            for (int j = 0; j < dof; j++)
++            for (int d = 0; d < b_dim; d++)
+             {
+-               const double val = d2q.B[i + nqpt*(d+b_dim*lex_ordering[j])];
+-               d2q_new->B[i+nqpt*(d+b_dim*j)] = val;
+-               d2q_new->Bt[j+dof*(i+nqpt*d)] = val;
++               for (int j = 0; j < dof; j++)
++               {
++                  const double val = d2q.B[i + nqpt*(d+b_dim*lex_ordering[j])];
++                  d2q_new->B[i+nqpt*(d+b_dim*j)] = val;
++                  d2q_new->Bt[j+dof*(i+nqpt*d)] = val;
++               }
+             }
+          }
+-      }
+
+-      const int g_dim = [this]()
+-      {
+-         switch (deriv_type)
++         const int g_dim = [this]()
+          {
+-            case GRAD: return dim;
+-            case DIV: return 1;
+-            case CURL: return cdim;
+-            default: return 0;
+-         }
+-      }();
++            switch (deriv_type)
++            {
++               case GRAD: return dim;
++               case DIV: return 1;
++               case CURL: return cdim;
++               default: return 0;
++            }
++         }();
+
+-      for (int i = 0; i < nqpt; i++)
+-      {
+-         for (int d = 0; d < g_dim; d++)
++         for (int i = 0; i < nqpt; i++)
+          {
+-            for (int j = 0; j < dof; j++)
++            for (int d = 0; d < g_dim; d++)
+             {
+-               const double val = d2q.G[i + nqpt*(d+g_dim*lex_ordering[j])];
+-               d2q_new->G[i+nqpt*(d+g_dim*j)] = val;
+-               d2q_new->Gt[j+dof*(i+nqpt*d)] = val;
++               for (int j = 0; j < dof; j++)
++               {
++                  const double val = d2q.G[i + nqpt*(d+g_dim*lex_ordering[j])];
++                  d2q_new->G[i+nqpt*(d+g_dim*j)] = val;
++                  d2q_new->Gt[j+dof*(i+nqpt*d)] = val;
++               }
+             }
+          }
++
++         dof2quad_array.Append(d2q_new);
+       }
+
+-      dof2quad_array.Append(d2q_new);
+    }
+ }
+

scripts/spack/spack_repo/smith/packages/mfem/package.py

@@ -11,7 +11,9 @@ class Mfem(BuiltinMfem):
     # Note: Make sure this sha coincides with the git submodule
     # Note: We add a number to the end of the real version number to indicate that we have
     # moved forward past the release. Increment the last number when updating the commit sha.
-    version("4.9.0.3", commit="232853214d99a7ef7a8fb85bcf7a28ee92131115")
+    version("4.9.0.4", commit="8efb9483bc8353cfcda3cc2531427f048f606e01")
+
+    patch("fe_base_omp.patch", when="+openmp+threadsafe")
 
     variant('asan', default=False, description='Add Address Sanitizer flags')
 

scripts/spack/spack_repo/smith/packages/tribol/package.py

@@ -52,7 +52,7 @@ class Tribol(CachedCMakePackage, CudaPackage, ROCmPackage):
     # SMITH EDIT START
     # Note: We add a number to the end of the real version number to indicate that we have
     #  moved forward past the release. Increment the last number when updating the commit sha.
-    version("0.1.0.24", commit="9f9d4d6fc8735106e9118992ed11a9188249962a", submodules=True, preferred=True)
+    version("0.1.0.25", commit="65101cdad68c2b4b8b17e582ce167c93ad912c9c", submodules=True, preferred=True)
     # SMITH EDIT END
 
     # -----------------------------------------------------------------------

scripts/spack/specs.json

@@ -18,7 +18,7 @@
       "+devtools+adiak+caliper~enzyme %gcc_13" ],
 
     "toss_4_x86_64_ib_cray":
-    [ "~devtools+adiak+caliper~openmp+rocm~enzyme~petsc~slepc amdgpu_target=gfx942,gfx90a %rocm_6_4_2"],
+    [ "~devtools+adiak+caliper~openmp+rocm+enzyme~petsc~slepc amdgpu_target=gfx942,gfx90a %rocm_6_4_2"],
 
     "blueos_3_ppc64le_ib_p9":
     [ "+devtools+cuda+adiak+caliper~strumpack %clang_gfortran cuda_arch=70" ],

src/smith/physics/tests/CMakeLists.txt

@@ -59,14 +59,6 @@ smith_add_tests(SOURCES       ${physics_serial_test_sources}
                 DEPENDS_ON    ${physics_test_depends}
                 NUM_MPI_TASKS 1)
 
-if(SMITH_ENABLE_HIP AND TRIBOL_FOUND)
-    foreach(filename ${physics_serial_tribol_test_sources})
-        get_filename_component(test_name ${filename} NAME_WE)
-        blt_add_target_compile_flags(TO ${test_name} FLAGS "-fgpu-rdc")
-        blt_add_target_link_flags(TO ${test_name} FLAGS "-fgpu-rdc")
-    endforeach()
-endif()
-
 set(physics_parallel_test_sources
     lce_Brighenti_tensile.cpp
     lce_Bertoldi_lattice.cpp
@@ -95,11 +87,3 @@ blt_list_append(TO       physics_parallel_test_sources
 smith_add_tests(SOURCES       ${physics_parallel_test_sources}
                 DEPENDS_ON    ${physics_test_depends}
                 NUM_MPI_TASKS 2)
-
-if(SMITH_ENABLE_HIP AND TRIBOL_FOUND)
-    foreach(filename ${physics_parallel_tribol_test_sources})
-        get_filename_component(test_name ${filename} NAME_WE)
-        blt_add_target_compile_flags(TO ${test_name} FLAGS "-fgpu-rdc")
-        blt_add_target_link_flags(TO ${test_name} FLAGS "-fgpu-rdc")
-    endforeach()
-endif()