From c870d364067157404393401d6bb3a9b7fec5bb29 Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Mon, 1 Dec 2025 13:57:55 -0800 Subject: [PATCH 01/58] Add centered pressure gradient implimentation --- components/omega/src/ocn/Eos.cpp | 84 ++++++++--- components/omega/src/ocn/Eos.h | 5 + components/omega/src/ocn/PGrad.cpp | 187 ++++++++++++++++++++++++ components/omega/src/ocn/PGrad.h | 169 +++++++++++++++++++++ components/omega/src/ocn/Tendencies.cpp | 2 + 5 files changed, 429 insertions(+), 18 deletions(-) create mode 100644 components/omega/src/ocn/PGrad.cpp create mode 100644 components/omega/src/ocn/PGrad.h diff --git a/components/omega/src/ocn/Eos.cpp b/components/omega/src/ocn/Eos.cpp index 1b49fe400eea..4ee381b0e751 100644 --- a/components/omega/src/ocn/Eos.cpp +++ b/components/omega/src/ocn/Eos.cpp @@ -45,24 +45,6 @@ Eos::Eos(const std::string &Name, ///< [in] Name for eos object BruntVaisalaFreqSq = Array2DReal("BruntVaisalaFreqSq", Mesh->NCellsAll, VCoord->NVertLayers); - defineFields(); -} - -/// Destructor for Eos -Eos::~Eos() {} - -/// Instance management -Eos *Eos::Instance = nullptr; - -/// Get instance of Eos -Eos *Eos::getInstance() { return Instance; } - -/// Destroy instance of Eos -void Eos::destroyInstance() { - delete Instance; - Instance = nullptr; -} - /// Initializes the Eos (Equation of State) class and its options. /// it ASSUMES that HorzMesh was initialized and initializes the Eos class by /// using the default mesh, reading the config file, and setting parameters @@ -168,6 +150,72 @@ void Eos::computeSpecVol(const Array2DReal &ConservTemp, } } +/// Compute specific volume for all layer interfaces(no displacement) +void Eos::computeSpecVolInterface(const Array2DReal &ConservTemp, + const Array2DReal &AbsSalinity, + const Array2DReal &PressureInterface) { + OMEGA_SCOPE(LocSpecVolInterface, + SpecVolInterface); /// Create a local view for computation + OMEGA_SCOPE(LocComputeSpecVolLinear, + ComputeSpecVolLinear); /// Local view for linear EOS computation + OMEGA_SCOPE(LocComputeSpecVolTeos10, + ComputeSpecVolTeos10); /// Local view for TEOS-10 computation + deepCopy(LocSpecVolInterface, + 0); /// Initialize local specific volume to zero + + I4 KDisp = 0; /// No displacement in this case + + Array2DReal ConservTempInterface("ConservTempInterface", NCellsAll, + NVertLayers + 1); + Array2DReal AbsSalinityInterface("AbsSalinityInterface", NCellsAll, + NVertLayers + 1); + + // Compute interface values by averaging adjacent layer values + parallelFor( + "compute-interface-values", {NCellsAll, NVertLayers + 1}, + KOKKOS_LAMBDA(I4 ICell, I4 KInterface) { + if (KInterface == 0) { + ConservTempInterface(ICell, KInterface) = + ConservTemp(ICell, KInterface); + AbsSalinityInterface(ICell, KInterface) = + AbsSalinity(ICell, KInterface); + } else if (KInterface == NVertLayers) { + ConservTempInterface(ICell, KInterface) = + ConservTemp(ICell, KInterface - 1); + AbsSalinityInterface(ICell, KInterface) = + AbsSalinity(ICell, KInterface - 1); + } else { + ConservTempInterface(ICell, KInterface) = + 0.5_Real * (ConservTemp(ICell, KInterface - 1) + + ConservTemp(ICell, KInterface)); + AbsSalinityInterface(ICell, KInterface) = + 0.5_Real * (AbsSalinity(ICell, KInterface - 1) + + AbsSalinity(ICell, KInterface)); + } + }); + + int NChunksP1 = (NVertLayers + 1) / VecLength; + + /// Dispatch to the correct EOS calculation + if (EosChoice == EosType::LinearEos) { + parallelFor( + "eos-linear", {NCellsAll, NChunksP1}, + KOKKOS_LAMBDA(I4 ICell, I4 KChunk) { + LocComputeSpecVolLinear(LocSpecVolInterface, ICell, KChunk, + ConservTempInterface, + AbsSalinityInterface); + }); + } else if (EosChoice == EosType::Teos10Eos) { + parallelFor( + "eos-teos10", {NCellsAll, NChunksP1}, + KOKKOS_LAMBDA(I4 ICell, I4 KChunk) { + LocComputeSpecVolTeos10(LocSpecVolInterface, ICell, KChunk, + ConservTempInterface, AbsSalinityInterface, + PressureInterface, KDisp); + }); + } +} + /// Compute displaced specific volume (for vertical displacement) void Eos::computeSpecVolDisp(const Array2DReal &ConservTemp, const Array2DReal &AbsSalinity, diff --git a/components/omega/src/ocn/Eos.h b/components/omega/src/ocn/Eos.h index 707297f01113..0eb111741d58 100644 --- a/components/omega/src/ocn/Eos.h +++ b/components/omega/src/ocn/Eos.h @@ -542,6 +542,7 @@ class Eos { EosType EosChoice; ///< Current EOS type in use Array2DReal SpecVol; ///< Specific volume field at level centers + Array2DReal SpecVolInterface; ///< Specific volume field at interfaces Array2DReal SpecVolDisplaced; ///< Displaced specific volume field Array2DReal BruntVaisalaFreqSq; ///< Squared Brunt-Vaisala frequency field @@ -568,6 +569,10 @@ class Eos { const Array2DReal &AbsSalinity, const Array2DReal &Pressure, const Array2DReal &SpecVol); + /// Compute specific volume at interfaces for all cells/layers+1 + void computeSpecVolInterface(const Array2DReal &ConservTemp, + const Array2DReal &AbsSalinity, + const Array2DReal &PressureInterface); /// Initialize EOS from config and mesh static void init(); diff --git a/components/omega/src/ocn/PGrad.cpp b/components/omega/src/ocn/PGrad.cpp new file mode 100644 index 000000000000..5bf2909667b2 --- /dev/null +++ b/components/omega/src/ocn/PGrad.cpp @@ -0,0 +1,187 @@ +//===-- ocn/PGrad.cpp - Pressure Gradient Term -----------------*- C++ -*-===// +// +// Implements the PGrad manager and two discretizations: Centered and +// HighOrder. +// +//===----------------------------------------------------------------------===// + +#include "PGrad.h" +#include "Error.h" +#include "Field.h" +#include "HorzMesh.h" +#include "VertCoord.h" + +namespace OMEGA { + +PressureGrad *PressureGrad::DefaultPGrad = nullptr; +std::map> PressureGrad::AllPGrads; + +//------------------------------------------------------------------------------ +// Initialize the PressureGrad. Assumes that HorzMesh and VertCoord have already +// been initialized. +void PressureGrad::init() { + + // Retrieve default mesh and vertical coordinate + HorzMesh *DefMesh = HorzMesh::getDefault(); + VertCoord *DefVCoord = VertCoord::getDefault(); + + // Retrieve PressureGrad config group + Config *OmegaConfig = Config::getOmegaConfig(); + Config PGradConfig("PressureGrad"); + Error Err; + Err += OmegaConfig->get(PGradConfig); + CHECK_ERROR_ABORT(Err, "PressureGrad: PressureGrad group not found in Config"); + + // Create the default PressureGrad and set pointer to it + PressureGrad::DefaultPGrad = PressureGrad::create( + "Default", DefMesh, DefVCoord, &PGradConfig); + +} // end init + +//------------------------------------------------------------------------------ +// Create a new PressureGrad object and add it to the map +PressureGrad *PressureGrad::create(const std::string &Name, /// [in] Name for PressureGrad + const HorzMesh *Mesh, ///< [in] Horizontal mesh + const VertCoord *VCoord, ///< [in] Vertical coordinate + Config *Options) { ///< [in] Configuration options + + // Check to see if a PressureGrad of the same name already exists and + // if so, exit with an error + if (AllPGrads.find(Name) != AllPGrads.end()) { + LOG_ERROR("Attempted to create a PressureGrad with name {} but a " + "PressureGrad of that name already exists", + Name); + return nullptr; + } + + // create a new PressureGrad on the heap and put it in a map of + // unique_ptrs, which will manage its lifetime + auto *NewPGrad = new PressureGrad(Mesh, VCoord, Options); + AllPGrads.emplace(Name, NewPGrad); + + return NewPGrad; + +} // end create + +//------------------------------------------------------------------------------ +// Get the default pressure gradient instance +PressureGrad *PressureGrad::getDefault() { + + return DefaultPGrad; + +} // end get default + +//------------------------------------------------------------------------------ +// Constructor for PressureGrad +PressureGrad::PressureGrad(const HorzMesh *Mesh, ///< [in] Horizontal mesh + const VertCoord *VCoord, ///< [in] Vertical coordinate + Config *Options) ///< [in] Configuration options + : CellsOnEdge(Mesh->CellsOnEdge), + DvEdge(Mesh->DvEdge), + EdgeSignOnCell(Mesh->EdgeSignOnCell), + CenteredPGrad(Mesh), HighOrderPGrad(Mesh) { + + // store mesh sizes + NEdgesAll = Mesh->NEdgesAll; + NVertLayers = VCoord->NVertLayers; + NChunks = NVertLayers / VecLength; + + // Read config options for PressureGrad type + // and enable the appropriate functor + Config PGradConfig("PressureGrad"); + Error Err; + Err += Options->get(PGradConfig); + CHECK_ERROR_ABORT(Err, "PressureGrad: PressureGrad group not found in Config"); + std::string PGradTypeStr; + Err += PGradConfig.get("PressureGradType", PGradTypeStr); + + if (PGradTypeStr == "centered" || PGradTypeStr == "Centered") { + PressureGradChoice = PressureGradType::Centered; + this->CenteredPGrad.Enabled = true; + } else if (PGradTypeStr == "HighOrder1") { + PressureGradChoice = PressureGradType::HighOrder1; + this->HighOrderPGrad.Enabled = true; + } else { + LOG_INFO("PGrad: Unknown PressureGradType in config, defaulting to centered"); + } + +} // end constructor + +//------------------------------------------------------------------------------ +// Destructor for PressureGrad +PressureGrad::~PressureGrad() { + + // No operations needed, Kokkos arrays removed when no longer in scope + +} // end destructor + +//------------------------------------------------------------------------------ +// Remove PressureGrad instances before exit +void PressureGrad::clear() { + + AllPGrads.clear(); + + } // end clear + +//------------------------------------------------------------------------------ +// Remove PressureGrad from list by name +void PressureGrad::erase(const std::string &Name) { + + AllPGrads.erase(Name); + +} // end erase + +//------------------------------------------------------------------------------ +// Get pressure gradient instance by name +PressureGrad *PressureGrad::get(const std::string &Name ///< [in] Name of +) { + + auto it = AllPGrads.find(Name); + + if (it != AllPGrads.end()) { + return it->second.get(); + } else { + LOG_ERROR("PressureGrad::get: Attempt to retrieve non-existent " + "PressureGrad:"); + LOG_ERROR("{} has not been defined or has been removed", Name); + return nullptr; + } + +} // end get pressure gradient + +//------------------------------------------------------------------------------ +// Compute pressure gradient tendencies and add into Tend array +void PressureGrad::computePressureGrad(Array2DReal Tend, + const OceanState *State, + const VertCoord *VCoord, + const Eos *EqState, + const int TimeLevel) { + + OMEGA_SCOPE(LocCenteredPGrad, CenteredPGrad); + OMEGA_SCOPE(LocHighOrderPGrad, HighOrderPGrad); + + const Array2DReal &PressureMid = VCoord->PressureMid; + const Array2DReal &PressureInterface = VCoord->PressureInterface; + const Array2DReal &Geopotential = VCoord->GeopotentialMid; + const Array2DReal &SpecVol = EqState->SpecVol; + const Array2DReal &SpecVolInterface = EqState->SpecVolInterface; + const Array2DReal &ZInterface = VCoord->ZInterface; + Array2DReal LayerThick; + State->getLayerThickness(LayerThick, TimeLevel); + + if (PressureGradChoice == PressureGradType::Centered) { + parallelFor("pgrad-centered", {NEdgesAll, NChunks}, + KOKKOS_LAMBDA(I4 IEdge, I4 KChunk) { + LocCenteredPGrad(Tend, IEdge, KChunk, PressureMid, PressureInterface, Geopotential, + LayerThick, ZInterface, SpecVol, SpecVolInterface); + }); + } else { + parallelFor("pgrad-highorder", {NEdgesAll, NChunks}, + KOKKOS_LAMBDA(I4 IEdge, I4 KChunk) { + LocHighOrderPGrad(Tend, IEdge, KChunk, PressureMid, Geopotential, + SpecVol); + }); + } +} // end compute pressure gradient + +} // namespace OMEGA \ No newline at end of file diff --git a/components/omega/src/ocn/PGrad.h b/components/omega/src/ocn/PGrad.h new file mode 100644 index 000000000000..1d8ea87793c7 --- /dev/null +++ b/components/omega/src/ocn/PGrad.h @@ -0,0 +1,169 @@ +#ifndef OMEGA_PGRAD_H +#define OMEGA_PGRAD_H +//===-- ocn/PGrad.h - Pressure Gradient -----------------*- C++ -*-===// +/// +/// Implements the PressureGrad class which provides a centered and +/// high-order pressure gradient option and dispatches computations to +/// functor objects. This follows the patterns used in Eos.h/Eos.cpp. +// +//===----------------------------------------------------------------------===// + +#include "Config.h" +#include "HorzMesh.h" +#include "OceanState.h" +#include "Eos.h" +#include "OmegaKokkos.h" +#include "VertCoord.h" +#include + +namespace OMEGA { + +enum class PressureGradType { Centered, HighOrder1, HighOrder2 }; + +/// Centered pressure gradient functor +class PressureGradCentered { + public: + bool Enabled; + + /// constructor declaration + PressureGradCentered(const HorzMesh *Mesh); + + // Compute centered pressure gradient contribution for given edge and + // vertical chunk. This appends results into the Tend array (in-place). + KOKKOS_FUNCTION void operator()(const Array2DReal &Tend, I4 IEdge, I4 KChunk, + const Array2DReal &PressureMid, + const Array2DReal &PressureInterface, + const Array2DReal &Geopotential, + const Array2DReal &LayerThick, + const Array2DReal &ZInterface, + const Array2DReal &SpecVol, + const Array2DReal &SpecVolInterface) const { + const I4 KStart = KChunk * VecLength; + + const I4 ICell0 = CellsOnEdge(IEdge, 0); + const I4 ICell1 = CellsOnEdge(IEdge, 1); + const Real InvDcEdge = 1.0_Real / DcEdge(IEdge); + + for (int KVec = 0; KVec < VecLength; ++KVec) { + const I4 K = KStart + KVec; + + // Average quantities to edge + const Real AlphaEdgeK = 0.5*(SpecVolInterface(ICell1, K) + SpecVolInterface(ICell0, K)); + const Real PEdgeK = 0.5*(PressureInterface(ICell1, K) + PressureInterface(ICell0, K)); + const Real AlphaEdgeKP1 = 0.5*(SpecVolInterface(ICell1, K+1) + SpecVolInterface(ICell0, K+1)); + const Real PEdgeKP1 = 0.5*(PressureInterface(ICell1, K+1) + PressureInterface(ICell0, K+1)); + const Real InvLayerThickEdge = 2.0_Real / (LayerThick(ICell1, K) + LayerThick(ICell0, K)); + + Real GeoTerm = (Geopotential(ICell1, K) - Geopotential(ICell0, K)) * InvDcEdge; + Real PresTerm = (LayerThick(ICell1, K)*SpecVol(ICell1, K)*PressureMid(ICell1, K) + - LayerThick(ICell0, K)*SpecVol(ICell0, K)*PressureMid(ICell0, K)) * InvDcEdge * InvLayerThickEdge; + Real InterfaceTerm = (AlphaEdgeK * PEdgeK * (ZInterface(ICell1, K) - ZInterface(ICell0, K)) + - AlphaEdgeKP1 * PEdgeKP1 * (ZInterface(ICell1, K+1) - ZInterface(ICell0, K+1))) * InvDcEdge * InvLayerThickEdge; + + Tend(IEdge, K) += EdgeMask(IEdge, K) * (-GeoTerm - PresTerm + InterfaceTerm); + } + } + + private: + Array2DI4 CellsOnEdge; + Array1DReal DcEdge; + Array2DReal EdgeMask; +}; + +/// High-order pressure gradient functor (placeholder) +class PressureGradHighOrder { + public: + bool Enabled; + + /// constructor declaration + PressureGradHighOrder(const HorzMesh *Mesh); + + KOKKOS_FUNCTION void operator()(const Array2DReal &Tend, I4 IEdge, I4 KChunk, + const Array2DReal &Pressure, + const Array2DReal &Geopotential, + const Array2DReal &SpecVol) const { + // Placeholder: for now, no-op (future high-order implementation) + const I4 KStart = KChunk * VecLength; + for (int KVec = 0; KVec < VecLength; ++KVec) { + const I4 K = KStart + KVec; + Tend(IEdge, K) += 0.0_Real; + } + } + private: + Array2DI4 CellsOnEdge; + Array1DReal DcEdge; + Array2DReal EdgeMask; +}; + +/// Pressure gradient manager class +class PressureGrad { + public: + /// Initialize the default instance + static void init(); + + /// Deallocates arrays and deletes instance + static void clear(); + + /// Remove pressure gradient object by name + static void erase(const std::string &Name ///< [in] + ); + + /// Get the default instance + static PressureGrad *getDefault(); + + /// Get instance by name + static PressureGrad *get(const std::string &Name ///< [in] + ); + + // Destructor + ~PressureGrad(); + + /// Compute pressure gradient tendencies and add into Tend array + void computePressureGrad(Array2DReal Tend, + const OceanState *State, + const VertCoord *VCoord, + const Eos *EqState, + const int TimeLevel); + + static PressureGrad *create(const std::string &Name, + const HorzMesh *Mesh, + const VertCoord *VCoord, + Config *Options); + + private: + // Construct a new pressure gradient object + PressureGrad(const HorzMesh *Mesh, + const VertCoord *VCoord, + Config *Options); + + // forbid copy and move construction + PressureGrad(const PressureGrad &) = delete; + PressureGrad(PressureGrad &&) = delete; + + // Pointer to default pressure gradient object + static PressureGrad *DefaultPGrad; + + // Mesh-related sizes + I4 NEdgesAll = 0; + I4 NChunks = 0; + I4 NVertLayers = 0; + + // Data required for computation (stored copies of mesh/VCoord arrays) + Array2DI4 CellsOnEdge; ///< cells surrounding each edge + Array1DReal DvEdge; ///< distance between cell centers across edge + Array2DReal EdgeSignOnCell; ///< orientation of edge relative to cell + + // Instances of functors + PressureGradCentered CenteredPGrad; + PressureGradHighOrder HighOrderPGrad; + + // Choice from config + PressureGradType PressureGradChoice = PressureGradType::Centered; + + // Map of all pressure gradient objects by name + static std::map> AllPGrads; + +}; // end class PressureGrad + +} // namespace OMEGA +#endif diff --git a/components/omega/src/ocn/Tendencies.cpp b/components/omega/src/ocn/Tendencies.cpp index b228c829082b..8714f56514ea 100644 --- a/components/omega/src/ocn/Tendencies.cpp +++ b/components/omega/src/ocn/Tendencies.cpp @@ -10,7 +10,9 @@ #include "Tendencies.h" #include "CustomTendencyTerms.h" +#include "Eos.h" #include "Error.h" +#include "PGrad.h" #include "Pacer.h" #include "TimeStepper.h" #include "Tracers.h" From 58069635142bfec8790e18442b1417dbdbacb393 Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Tue, 4 Nov 2025 10:27:23 -0800 Subject: [PATCH 02/58] Udpate pressure gradient - Average specific volume to interfaces - Average product of spcific volume and pressure to edge - Add centered and high order functor constructors - Format linting --- components/omega/src/ocn/Eos.cpp | 85 ++++++++------- components/omega/src/ocn/PGrad.cpp | 161 ++++++++++++++++------------- components/omega/src/ocn/PGrad.h | 79 +++++++------- 3 files changed, 178 insertions(+), 147 deletions(-) diff --git a/components/omega/src/ocn/Eos.cpp b/components/omega/src/ocn/Eos.cpp index 4ee381b0e751..5926a77318bf 100644 --- a/components/omega/src/ocn/Eos.cpp +++ b/components/omega/src/ocn/Eos.cpp @@ -165,55 +165,62 @@ void Eos::computeSpecVolInterface(const Array2DReal &ConservTemp, I4 KDisp = 0; /// No displacement in this case - Array2DReal ConservTempInterface("ConservTempInterface", NCellsAll, - NVertLayers + 1); - Array2DReal AbsSalinityInterface("AbsSalinityInterface", NCellsAll, - NVertLayers + 1); + // Array2DReal ConservTempInterface("ConservTempInterface", NCellsAll, + // NVertLayers + 1); + // Array2DReal AbsSalinityInterface("AbsSalinityInterface", NCellsAll, + // NVertLayers + 1); // Compute interface values by averaging adjacent layer values parallelFor( "compute-interface-values", {NCellsAll, NVertLayers + 1}, KOKKOS_LAMBDA(I4 ICell, I4 KInterface) { if (KInterface == 0) { - ConservTempInterface(ICell, KInterface) = - ConservTemp(ICell, KInterface); - AbsSalinityInterface(ICell, KInterface) = - AbsSalinity(ICell, KInterface); - } else if (KInterface == NVertLayers) { - ConservTempInterface(ICell, KInterface) = - ConservTemp(ICell, KInterface - 1); - AbsSalinityInterface(ICell, KInterface) = - AbsSalinity(ICell, KInterface - 1); + // ConservTempInterface(ICell, KInterface) = + // ConservTemp(ICell, KInterface); + // AbsSalinityInterface(ICell, KInterface) = + // AbsSalinity(ICell, KInterface); + SpecVolInterface(ICell, KInterface) = SpecVol(ICell, KInterface); + } else if (KInterface == NVertLayers + 1) { + // ConservTempInterface(ICell, KInterface) = + // ConservTemp(ICell, KInterface - 1); + // AbsSalinityInterface(ICell, KInterface) = + // AbsSalinity(ICell, KInterface - 1); + SpecVolInterface(ICell, KInterface) = + SpecVol(ICell, KInterface - 1); } else { - ConservTempInterface(ICell, KInterface) = - 0.5_Real * (ConservTemp(ICell, KInterface - 1) + - ConservTemp(ICell, KInterface)); - AbsSalinityInterface(ICell, KInterface) = - 0.5_Real * (AbsSalinity(ICell, KInterface - 1) + - AbsSalinity(ICell, KInterface)); + // ConservTempInterface(ICell, KInterface) = + // 0.5_Real * (ConservTemp(ICell, KInterface - 1) + + // ConservTemp(ICell, KInterface)); + // AbsSalinityInterface(ICell, KInterface) = + // 0.5_Real * (AbsSalinity(ICell, KInterface - 1) + + // AbsSalinity(ICell, KInterface)); + SpecVolInterface(ICell, KInterface) = + 0.5_Real * + (SpecVol(ICell, KInterface - 1) + SpecVol(ICell, KInterface)); } }); - int NChunksP1 = (NVertLayers + 1) / VecLength; - - /// Dispatch to the correct EOS calculation - if (EosChoice == EosType::LinearEos) { - parallelFor( - "eos-linear", {NCellsAll, NChunksP1}, - KOKKOS_LAMBDA(I4 ICell, I4 KChunk) { - LocComputeSpecVolLinear(LocSpecVolInterface, ICell, KChunk, - ConservTempInterface, - AbsSalinityInterface); - }); - } else if (EosChoice == EosType::Teos10Eos) { - parallelFor( - "eos-teos10", {NCellsAll, NChunksP1}, - KOKKOS_LAMBDA(I4 ICell, I4 KChunk) { - LocComputeSpecVolTeos10(LocSpecVolInterface, ICell, KChunk, - ConservTempInterface, AbsSalinityInterface, - PressureInterface, KDisp); - }); - } + // int NChunksP1 = (NVertLayers + 1) / VecLength; + + ///// Dispatch to the correct EOS calculation + // if (EosChoice == EosType::LinearEos) { + // parallelFor( + // "eos-linear", {NCellsAll, NChunksP1}, + // KOKKOS_LAMBDA(I4 ICell, I4 KChunk) { + // LocComputeSpecVolLinear(LocSpecVolInterface, ICell, KChunk, + // ConservTempInterface, + // AbsSalinityInterface); + // }); + // } else if (EosChoice == EosType::Teos10Eos) { + // parallelFor( + // "eos-teos10", {NCellsAll, NChunksP1}, + // KOKKOS_LAMBDA(I4 ICell, I4 KChunk) { + // LocComputeSpecVolTeos10(LocSpecVolInterface, ICell, KChunk, + // ConservTempInterface, + // AbsSalinityInterface, PressureInterface, + // KDisp); + // }); + // } } /// Compute displaced specific volume (for vertical displacement) diff --git a/components/omega/src/ocn/PGrad.cpp b/components/omega/src/ocn/PGrad.cpp index 5bf2909667b2..e478b392e195 100644 --- a/components/omega/src/ocn/PGrad.cpp +++ b/components/omega/src/ocn/PGrad.cpp @@ -1,7 +1,7 @@ //===-- ocn/PGrad.cpp - Pressure Gradient Term -----------------*- C++ -*-===// // // Implements the PGrad manager and two discretizations: Centered and -// HighOrder. +// HighOrder. // //===----------------------------------------------------------------------===// @@ -22,7 +22,7 @@ std::map> PressureGrad::AllPGrads; void PressureGrad::init() { // Retrieve default mesh and vertical coordinate - HorzMesh *DefMesh = HorzMesh::getDefault(); + HorzMesh *DefMesh = HorzMesh::getDefault(); VertCoord *DefVCoord = VertCoord::getDefault(); // Retrieve PressureGrad config group @@ -30,20 +30,22 @@ void PressureGrad::init() { Config PGradConfig("PressureGrad"); Error Err; Err += OmegaConfig->get(PGradConfig); - CHECK_ERROR_ABORT(Err, "PressureGrad: PressureGrad group not found in Config"); + CHECK_ERROR_ABORT(Err, + "PressureGrad: PressureGrad group not found in Config"); // Create the default PressureGrad and set pointer to it - PressureGrad::DefaultPGrad = PressureGrad::create( - "Default", DefMesh, DefVCoord, &PGradConfig); + PressureGrad::DefaultPGrad = + PressureGrad::create("Default", DefMesh, DefVCoord, &PGradConfig); -} // end init +} // end init //------------------------------------------------------------------------------ // Create a new PressureGrad object and add it to the map -PressureGrad *PressureGrad::create(const std::string &Name, /// [in] Name for PressureGrad - const HorzMesh *Mesh, ///< [in] Horizontal mesh - const VertCoord *VCoord, ///< [in] Vertical coordinate - Config *Options) { ///< [in] Configuration options +PressureGrad * +PressureGrad::create(const std::string &Name, /// [in] Name for PressureGrad + const HorzMesh *Mesh, ///< [in] Horizontal mesh + const VertCoord *VCoord, ///< [in] Vertical coordinate + Config *Options) { ///< [in] Configuration options // Check to see if a PressureGrad of the same name already exists and // if so, exit with an error @@ -64,7 +66,7 @@ PressureGrad *PressureGrad::create(const std::string &Name, /// [in] Name for P } // end create //------------------------------------------------------------------------------ -// Get the default pressure gradient instance +// Get the default pressure gradient instance PressureGrad *PressureGrad::getDefault() { return DefaultPGrad; @@ -73,37 +75,39 @@ PressureGrad *PressureGrad::getDefault() { //------------------------------------------------------------------------------ // Constructor for PressureGrad -PressureGrad::PressureGrad(const HorzMesh *Mesh, ///< [in] Horizontal mesh - const VertCoord *VCoord, ///< [in] Vertical coordinate - Config *Options) ///< [in] Configuration options - : CellsOnEdge(Mesh->CellsOnEdge), - DvEdge(Mesh->DvEdge), - EdgeSignOnCell(Mesh->EdgeSignOnCell), - CenteredPGrad(Mesh), HighOrderPGrad(Mesh) { - - // store mesh sizes - NEdgesAll = Mesh->NEdgesAll; - NVertLayers = VCoord->NVertLayers; - NChunks = NVertLayers / VecLength; - - // Read config options for PressureGrad type - // and enable the appropriate functor - Config PGradConfig("PressureGrad"); - Error Err; - Err += Options->get(PGradConfig); - CHECK_ERROR_ABORT(Err, "PressureGrad: PressureGrad group not found in Config"); - std::string PGradTypeStr; - Err += PGradConfig.get("PressureGradType", PGradTypeStr); - - if (PGradTypeStr == "centered" || PGradTypeStr == "Centered") { - PressureGradChoice = PressureGradType::Centered; - this->CenteredPGrad.Enabled = true; - } else if (PGradTypeStr == "HighOrder1") { - PressureGradChoice = PressureGradType::HighOrder1; - this->HighOrderPGrad.Enabled = true; - } else { - LOG_INFO("PGrad: Unknown PressureGradType in config, defaulting to centered"); - } +PressureGrad::PressureGrad( + const HorzMesh *Mesh, ///< [in] Horizontal mesh + const VertCoord *VCoord, ///< [in] Vertical coordinate + Config *Options) ///< [in] Configuration options + : CellsOnEdge(Mesh->CellsOnEdge), DvEdge(Mesh->DvEdge), + EdgeSignOnCell(Mesh->EdgeSignOnCell), CenteredPGrad(Mesh), + HighOrderPGrad(Mesh) { + + // store mesh sizes + NEdgesAll = Mesh->NEdgesAll; + NVertLayers = VCoord->NVertLayers; + NChunks = NVertLayers / VecLength; + + // Read config options for PressureGrad type + // and enable the appropriate functor + Config PGradConfig("PressureGrad"); + Error Err; + Err += Options->get(PGradConfig); + CHECK_ERROR_ABORT(Err, + "PressureGrad: PressureGrad group not found in Config"); + std::string PGradTypeStr; + Err += PGradConfig.get("PressureGradType", PGradTypeStr); + + if (PGradTypeStr == "centered" || PGradTypeStr == "Centered") { + PressureGradChoice = PressureGradType::Centered; + this->CenteredPGrad.Enabled = true; + } else if (PGradTypeStr == "HighOrder1") { + PressureGradChoice = PressureGradType::HighOrder1; + this->HighOrderPGrad.Enabled = true; + } else { + LOG_INFO( + "PGrad: Unknown PressureGradType in config, defaulting to centered"); + } } // end constructor @@ -117,11 +121,7 @@ PressureGrad::~PressureGrad() { //------------------------------------------------------------------------------ // Remove PressureGrad instances before exit -void PressureGrad::clear() { - - AllPGrads.clear(); - - } // end clear +void PressureGrad::clear() { AllPGrads.clear(); } // end clear //------------------------------------------------------------------------------ // Remove PressureGrad from list by name @@ -157,31 +157,46 @@ void PressureGrad::computePressureGrad(Array2DReal Tend, const Eos *EqState, const int TimeLevel) { - OMEGA_SCOPE(LocCenteredPGrad, CenteredPGrad); - OMEGA_SCOPE(LocHighOrderPGrad, HighOrderPGrad); - - const Array2DReal &PressureMid = VCoord->PressureMid; - const Array2DReal &PressureInterface = VCoord->PressureInterface; - const Array2DReal &Geopotential = VCoord->GeopotentialMid; - const Array2DReal &SpecVol = EqState->SpecVol; - const Array2DReal &SpecVolInterface = EqState->SpecVolInterface; - const Array2DReal &ZInterface = VCoord->ZInterface; - Array2DReal LayerThick; - State->getLayerThickness(LayerThick, TimeLevel); - - if (PressureGradChoice == PressureGradType::Centered) { - parallelFor("pgrad-centered", {NEdgesAll, NChunks}, - KOKKOS_LAMBDA(I4 IEdge, I4 KChunk) { - LocCenteredPGrad(Tend, IEdge, KChunk, PressureMid, PressureInterface, Geopotential, - LayerThick, ZInterface, SpecVol, SpecVolInterface); - }); - } else { - parallelFor("pgrad-highorder", {NEdgesAll, NChunks}, - KOKKOS_LAMBDA(I4 IEdge, I4 KChunk) { - LocHighOrderPGrad(Tend, IEdge, KChunk, PressureMid, Geopotential, - SpecVol); - }); - } + OMEGA_SCOPE(LocCenteredPGrad, CenteredPGrad); + OMEGA_SCOPE(LocHighOrderPGrad, HighOrderPGrad); + + const Array2DReal &PressureMid = VCoord->PressureMid; + const Array2DReal &PressureInterface = VCoord->PressureInterface; + const Array2DReal &Geopotential = VCoord->GeopotentialMid; + const Array2DReal &SpecVol = EqState->SpecVol; + const Array2DReal &SpecVolInterface = EqState->SpecVolInterface; + const Array2DReal &ZInterface = VCoord->ZInterface; + Array2DReal LayerThick; + State->getLayerThickness(LayerThick, TimeLevel); + + if (PressureGradChoice == PressureGradType::Centered) { + parallelFor( + "pgrad-centered", {NEdgesAll, NChunks}, + KOKKOS_LAMBDA(I4 IEdge, I4 KChunk) { + LocCenteredPGrad(Tend, IEdge, KChunk, PressureMid, + PressureInterface, Geopotential, LayerThick, + ZInterface, SpecVol, SpecVolInterface); + }); + } else { + parallelFor( + "pgrad-highorder", {NEdgesAll, NChunks}, + KOKKOS_LAMBDA(I4 IEdge, I4 KChunk) { + LocHighOrderPGrad(Tend, IEdge, KChunk, PressureMid, Geopotential, + SpecVol); + }); + } } // end compute pressure gradient -} // namespace OMEGA \ No newline at end of file +//------------------------------------------------------------------------------ +// Constructor for centered pressure gradient functor +PressureGradCentered::PressureGradCentered(const HorzMesh *Mesh) + : CellsOnEdge(Mesh->CellsOnEdge), DcEdge(Mesh->DcEdge), + EdgeMask(Mesh->EdgeMask) {} + +//------------------------------------------------------------------------------ +// Constructor for high order pressure gradient functor +PressureGradHighOrder::PressureGradHighOrder(const HorzMesh *Mesh) + : CellsOnEdge(Mesh->CellsOnEdge), DcEdge(Mesh->DcEdge), + EdgeMask(Mesh->EdgeMask) {} + +} // namespace OMEGA diff --git a/components/omega/src/ocn/PGrad.h b/components/omega/src/ocn/PGrad.h index 1d8ea87793c7..bb8b3262d785 100644 --- a/components/omega/src/ocn/PGrad.h +++ b/components/omega/src/ocn/PGrad.h @@ -9,9 +9,9 @@ //===----------------------------------------------------------------------===// #include "Config.h" +#include "Eos.h" #include "HorzMesh.h" #include "OceanState.h" -#include "Eos.h" #include "OmegaKokkos.h" #include "VertCoord.h" #include @@ -24,7 +24,7 @@ enum class PressureGradType { Centered, HighOrder1, HighOrder2 }; class PressureGradCentered { public: bool Enabled; - + /// constructor declaration PressureGradCentered(const HorzMesh *Mesh); @@ -40,27 +40,41 @@ class PressureGradCentered { const Array2DReal &SpecVolInterface) const { const I4 KStart = KChunk * VecLength; - const I4 ICell0 = CellsOnEdge(IEdge, 0); - const I4 ICell1 = CellsOnEdge(IEdge, 1); + const I4 ICell0 = CellsOnEdge(IEdge, 0); + const I4 ICell1 = CellsOnEdge(IEdge, 1); const Real InvDcEdge = 1.0_Real / DcEdge(IEdge); for (int KVec = 0; KVec < VecLength; ++KVec) { const I4 K = KStart + KVec; // Average quantities to edge - const Real AlphaEdgeK = 0.5*(SpecVolInterface(ICell1, K) + SpecVolInterface(ICell0, K)); - const Real PEdgeK = 0.5*(PressureInterface(ICell1, K) + PressureInterface(ICell0, K)); - const Real AlphaEdgeKP1 = 0.5*(SpecVolInterface(ICell1, K+1) + SpecVolInterface(ICell0, K+1)); - const Real PEdgeKP1 = 0.5*(PressureInterface(ICell1, K+1) + PressureInterface(ICell0, K+1)); - const Real InvLayerThickEdge = 2.0_Real / (LayerThick(ICell1, K) + LayerThick(ICell0, K)); - - Real GeoTerm = (Geopotential(ICell1, K) - Geopotential(ICell0, K)) * InvDcEdge; - Real PresTerm = (LayerThick(ICell1, K)*SpecVol(ICell1, K)*PressureMid(ICell1, K) - - LayerThick(ICell0, K)*SpecVol(ICell0, K)*PressureMid(ICell0, K)) * InvDcEdge * InvLayerThickEdge; - Real InterfaceTerm = (AlphaEdgeK * PEdgeK * (ZInterface(ICell1, K) - ZInterface(ICell0, K)) - - AlphaEdgeKP1 * PEdgeKP1 * (ZInterface(ICell1, K+1) - ZInterface(ICell0, K+1))) * InvDcEdge * InvLayerThickEdge; - - Tend(IEdge, K) += EdgeMask(IEdge, K) * (-GeoTerm - PresTerm + InterfaceTerm); + const Real PAlphaEdgeK = + 0.5_Real * + (PressureInterface(ICell1, K) * SpecVolInterface(ICell1, K) + + PressureInterface(ICell0, K) * SpecVolInterface(ICell0, K)); + const Real PAlphaEdgeKP1 = + 0.5_Real * (PressureInterface(ICell1, K + 1) * + SpecVolInterface(ICell1, K + 1) + + PressureInterface(ICell0, K + 1) * + SpecVolInterface(ICell0, K + 1)); + const Real InvLayerThickEdge = + 2.0_Real / (LayerThick(ICell1, K) + LayerThick(ICell0, K)); + + Real GeoTerm = + (Geopotential(ICell1, K) - Geopotential(ICell0, K)) * InvDcEdge; + Real PresTerm = (LayerThick(ICell1, K) * SpecVol(ICell1, K) * + PressureMid(ICell1, K) - + LayerThick(ICell0, K) * SpecVol(ICell0, K) * + PressureMid(ICell0, K)) * + InvDcEdge * InvLayerThickEdge; + Real InterfaceTerm = + (PAlphaEdgeK * (ZInterface(ICell1, K) - ZInterface(ICell0, K)) - + PAlphaEdgeKP1 * + (ZInterface(ICell1, K + 1) - ZInterface(ICell0, K + 1))) * + InvDcEdge * InvLayerThickEdge; + + Tend(IEdge, K) += + EdgeMask(IEdge, K) * (-GeoTerm - PresTerm + InterfaceTerm); } } @@ -89,19 +103,20 @@ class PressureGradHighOrder { Tend(IEdge, K) += 0.0_Real; } } + private: Array2DI4 CellsOnEdge; Array1DReal DcEdge; Array2DReal EdgeMask; }; -/// Pressure gradient manager class +/// Pressure gradient manager class class PressureGrad { public: /// Initialize the default instance static void init(); - /// Deallocates arrays and deletes instance + /// Deallocates arrays and deletes instance static void clear(); /// Remove pressure gradient object by name @@ -119,38 +134,32 @@ class PressureGrad { ~PressureGrad(); /// Compute pressure gradient tendencies and add into Tend array - void computePressureGrad(Array2DReal Tend, - const OceanState *State, - const VertCoord *VCoord, - const Eos *EqState, + void computePressureGrad(Array2DReal Tend, const OceanState *State, + const VertCoord *VCoord, const Eos *EqState, const int TimeLevel); - static PressureGrad *create(const std::string &Name, - const HorzMesh *Mesh, - const VertCoord *VCoord, - Config *Options); + static PressureGrad *create(const std::string &Name, const HorzMesh *Mesh, + const VertCoord *VCoord, Config *Options); private: // Construct a new pressure gradient object - PressureGrad(const HorzMesh *Mesh, - const VertCoord *VCoord, - Config *Options); + PressureGrad(const HorzMesh *Mesh, const VertCoord *VCoord, Config *Options); // forbid copy and move construction PressureGrad(const PressureGrad &) = delete; - PressureGrad(PressureGrad &&) = delete; + PressureGrad(PressureGrad &&) = delete; // Pointer to default pressure gradient object static PressureGrad *DefaultPGrad; // Mesh-related sizes - I4 NEdgesAll = 0; - I4 NChunks = 0; + I4 NEdgesAll = 0; + I4 NChunks = 0; I4 NVertLayers = 0; // Data required for computation (stored copies of mesh/VCoord arrays) - Array2DI4 CellsOnEdge; ///< cells surrounding each edge - Array1DReal DvEdge; ///< distance between cell centers across edge + Array2DI4 CellsOnEdge; ///< cells surrounding each edge + Array1DReal DvEdge; ///< distance between cell centers across edge Array2DReal EdgeSignOnCell; ///< orientation of edge relative to cell // Instances of functors From eae3812a5197f4e8c862b15f030f3fbbdd4df244 Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Tue, 4 Nov 2025 10:31:47 -0800 Subject: [PATCH 03/58] Add test driver --- components/omega/test/CMakeLists.txt | 11 ++ components/omega/test/ocn/PGradTest.cpp | 177 ++++++++++++++++++++++++ 2 files changed, 188 insertions(+) create mode 100644 components/omega/test/ocn/PGradTest.cpp diff --git a/components/omega/test/CMakeLists.txt b/components/omega/test/CMakeLists.txt index 271af1c40d46..c2dceb09c5a1 100644 --- a/components/omega/test/CMakeLists.txt +++ b/components/omega/test/CMakeLists.txt @@ -346,6 +346,17 @@ add_omega_test( "-n;8" ) +################## +# Pressure gradient test +################## + +add_omega_test( + PGRAD_TEST + testPGrad.exe + ocn/PGradTest.cpp + "-n;8" +) + ################## # State test ################## diff --git a/components/omega/test/ocn/PGradTest.cpp b/components/omega/test/ocn/PGradTest.cpp new file mode 100644 index 000000000000..fb479cccee3e --- /dev/null +++ b/components/omega/test/ocn/PGradTest.cpp @@ -0,0 +1,177 @@ +//===-- Test driver for OMEGA Pressure Gradient (PGrad) --------------*- C++-*-===/ +// +/// \file +/// \brief Test driver for PressureGrad module +// +//===----------------------------------------------------------------------===/ + +#include "PGrad.h" + +#include "DataTypes.h" +#include "Decomp.h" +#include "Dimension.h" +#include "Error.h" +#include "Eos.h" +#include "Field.h" +#include "Halo.h" +#include "HorzMesh.h" +#include "IO.h" +#include "Logging.h" +#include "MachEnv.h" +#include "OceanState.h" +#include "OmegaKokkos.h" +#include "Pacer.h" +#include "PGrad.h" +#include "TimeStepper.h" +#include "VertCoord.h" +#include "mpi.h" + +using namespace OMEGA; + +void initPGradTest() { + + Error Err; + int Err1; + + MachEnv::init(MPI_COMM_WORLD); + MachEnv *DefEnv = MachEnv::getDefault(); + MPI_Comm DefComm = DefEnv->getComm(); + + // Initialize the Logging system + initLogging(DefEnv); + + // Read default config if present + Config("Omega"); + Config::readAll("omega.yml"); + + // First step of time stepper initialization needed for IOstream + TimeStepper::init1(); + + // Initialize the IO system + IO::init(DefComm); + + // Create the default decomposition (initializes the decomposition) + Decomp::init(); + + // Initialize the default halo + Err1 = Halo::init(); + if (Err1 != 0) { + LOG_ERROR("PGrad: error initializing default halo"); + Err += Error(ErrorCode::Fail, + "PGrad: error initializing default halo"); + } + + // Begin initialization of the default vertical coordinate + VertCoord::init1(); + + // Initialize the default mesh + HorzMesh::init(); + + // Complete initialization of the default vertical coordinate + VertCoord::init2(); + + // Initialize the equation of state + Eos::init(); + + // Initialize ocean state + OceanState::init(); + + + CHECK_ERROR_ABORT(Err, "PGrad: error during initialization"); +} + +int main(int argc, char *argv[]) { + int RetVal = 0; + int Err; + + MPI_Init(&argc, &argv); + Kokkos::initialize(); + Pacer::initialize(MPI_COMM_WORLD); + Pacer::setPrefix("Omega:"); + { + initPGradTest(); + + // Initialize PressureGrad manager + PressureGrad::init(); + + MachEnv *DefEnv = MachEnv::getDefault(); + HorzMesh *DefMesh = HorzMesh::getDefault(); + VertCoord *DefVCoord = VertCoord::getDefault(); + OceanState *DefState = OceanState::getDefault(); + Eos *DefEos = Eos::getInstance(); + Config *Options = Config::getOmegaConfig(); + + I4 NEdgesOwned = DefMesh->NEdgesOwned; + I4 NVertLayers = VertCoord::getDefault()->NVertLayers; + I4 NChunks = NVertLayers / VecLength; + + // create arrays: Tend on edges, Pressure/Geopotential/SpecVol on cells + Array2DReal Tend("Tend", DefMesh->NEdgesAll, NVertLayers); + Array2DReal Pressure("Pressure", DefMesh->NCellsAll, NVertLayers); + Array2DReal Geopotential("Geopotential", DefMesh->NCellsAll, NVertLayers); + Array2DReal SpecVol("SpecVol", DefMesh->NCellsAll, NVertLayers); + + // initialize arrays to zero + parallelFor({DefMesh->NEdgesAll, NVertLayers}, KOKKOS_LAMBDA(int i, int k) { + Tend(i,k) = 0.0_Real; + }); + parallelFor({DefMesh->NCellsAll, NVertLayers}, KOKKOS_LAMBDA(int i, int k) { + Pressure(i,k) = 0.0_Real; + Geopotential(i,k) = 0.0_Real; + SpecVol(i,k) = 1.0_Real; // unit specific volume + }); + + // set a simple pressure difference between two neighboring cells + if (DefMesh->NCellsAll >= 2) { + Pressure(0,0) = 1.0_Real; + Pressure(1,0) = 2.0_Real; + } + + // call computePressureGrad + PressureGrad *Pg = PressureGrad::getDefault(); + if (!Pg) { + LOG_INFO("PGrad: default instance not present, creating via create"); + Pg = PressureGrad::create("default", DefMesh, DefVCoord, Options); + } + + int TimeLevel = 0; + if (Pg) { + Pg->computePressureGrad(Tend, DefState, DefVCoord, DefEos, TimeLevel); + + // simple check: ensure Tend contains some finite values (not all zero) + I4 CountNonZero = 0; + parallelReduce("countNonZero", {NEdgesOwned, NVertLayers}, + KOKKOS_LAMBDA(int e, int k, I4 &acc) { + if (Tend(e,k) != 0.0_Real) acc++; + }, CountNonZero); + + if (CountNonZero > 0) { + LOG_INFO("PGrad: computePressureGrad produced non-zero tendencies PASS"); + } else { + RetVal += 1; + LOG_ERROR("PGrad: computePressureGrad produced all-zero tendencies FAIL"); + } + } else { + RetVal += 1; + LOG_ERROR("PGrad: failed to obtain PressureGrad instance FAIL"); + } + + // cleanup + PressureGrad::clear(); + VertCoord::clear(); + OceanState::clear(); + HorzMesh::clear(); + Halo::clear(); + Decomp::clear(); + MachEnv::removeAll(); + FieldGroup::clear(); + Field::clear(); + Dimension::clear(); + } + Pacer::finalize(); + Kokkos::finalize(); + MPI_Finalize(); + + if (RetVal >= 256) RetVal = 255; + return RetVal; +} From 43dd27a015596cfa69275123102e196fb4467779 Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Thu, 20 Nov 2025 14:47:07 -0800 Subject: [PATCH 04/58] Add PressureGrad section to Default.yml --- components/omega/configs/Default.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/omega/configs/Default.yml b/components/omega/configs/Default.yml index d46de8c24e95..2964147e33ec 100644 --- a/components/omega/configs/Default.yml +++ b/components/omega/configs/Default.yml @@ -34,6 +34,8 @@ Omega: VertCoord: Density0: 1026.0 MovementWeightType: Uniform + PressureGrad: + PressureGradType: Centered Tendencies: ThicknessFluxTendencyEnable: true PVTendencyEnable: true From acaf474d51588747476d992a39e598d73bac975d Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Mon, 1 Dec 2025 14:01:52 -0800 Subject: [PATCH 05/58] Convert Pa to dbar in Teos10Eos functor --- components/omega/src/ocn/Eos.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/components/omega/src/ocn/Eos.h b/components/omega/src/ocn/Eos.h index 0eb111741d58..0807b8a1b484 100644 --- a/components/omega/src/ocn/Eos.h +++ b/components/omega/src/ocn/Eos.h @@ -44,7 +44,7 @@ class Teos10Eos { I4 KDisp) const { Real SpecVolPCoeffs[6 * VecLength]; - + Real PaToDBar = 1.0e-4_Real; // Conversion factor from Pa to dbar const I4 KStart = chunkStart(KChunk, MinLayerCell(ICell)); const I4 KLen = chunkLength(KChunk, KStart, MaxLayerCell(ICell)); @@ -63,15 +63,15 @@ class Teos10Eos { if (KDisp == 0) { // No displacement SpecVol(ICell, K) = - calcRefProfile(Pressure(ICell, K)) + - calcDelta(SpecVolPCoeffs, KVec, Pressure(ICell, K)); + calcRefProfile(Pressure(ICell, K) * PaToDBar) + + calcDelta(SpecVolPCoeffs, KVec, Pressure(ICell, K) * PaToDBar); } else { // Displacement, use the displaced pressure I4 KTmp = Kokkos::min(K + KDisp, MaxLayerCell(ICell)); KTmp = Kokkos::max(MinLayerCell(ICell), KTmp); SpecVol(ICell, K) = - calcRefProfile(Pressure(ICell, KTmp)) + - calcDelta(SpecVolPCoeffs, KVec, Pressure(ICell, KTmp)); + calcRefProfile(Pressure(ICell, KTmp) * PaToDBar) + + calcDelta(SpecVolPCoeffs, KVec, Pressure(ICell, KTmp) * PaToDBar); } } } From 5197b968a4613dbb071da99cc46b4a8b6ba4b93b Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Thu, 20 Nov 2025 14:56:09 -0800 Subject: [PATCH 06/58] Updates to PGradTest --- components/omega/src/ocn/PGrad.cpp | 11 +- components/omega/src/ocn/PGrad.h | 2 +- components/omega/test/ocn/PGradTest.cpp | 264 +++++++++++++++++++----- 3 files changed, 221 insertions(+), 56 deletions(-) diff --git a/components/omega/src/ocn/PGrad.cpp b/components/omega/src/ocn/PGrad.cpp index e478b392e195..effdccca8a72 100644 --- a/components/omega/src/ocn/PGrad.cpp +++ b/components/omega/src/ocn/PGrad.cpp @@ -25,17 +25,12 @@ void PressureGrad::init() { HorzMesh *DefMesh = HorzMesh::getDefault(); VertCoord *DefVCoord = VertCoord::getDefault(); - // Retrieve PressureGrad config group + // Retrieve omega config Config *OmegaConfig = Config::getOmegaConfig(); - Config PGradConfig("PressureGrad"); - Error Err; - Err += OmegaConfig->get(PGradConfig); - CHECK_ERROR_ABORT(Err, - "PressureGrad: PressureGrad group not found in Config"); // Create the default PressureGrad and set pointer to it PressureGrad::DefaultPGrad = - PressureGrad::create("Default", DefMesh, DefVCoord, &PGradConfig); + PressureGrad::create("Default", DefMesh, DefVCoord, OmegaConfig); } // end init @@ -151,7 +146,7 @@ PressureGrad *PressureGrad::get(const std::string &Name ///< [in] Name of //------------------------------------------------------------------------------ // Compute pressure gradient tendencies and add into Tend array -void PressureGrad::computePressureGrad(Array2DReal Tend, +void PressureGrad::computePressureGrad(Array2DReal &Tend, const OceanState *State, const VertCoord *VCoord, const Eos *EqState, diff --git a/components/omega/src/ocn/PGrad.h b/components/omega/src/ocn/PGrad.h index bb8b3262d785..e104db110321 100644 --- a/components/omega/src/ocn/PGrad.h +++ b/components/omega/src/ocn/PGrad.h @@ -134,7 +134,7 @@ class PressureGrad { ~PressureGrad(); /// Compute pressure gradient tendencies and add into Tend array - void computePressureGrad(Array2DReal Tend, const OceanState *State, + void computePressureGrad(Array2DReal &Tend, const OceanState *State, const VertCoord *VCoord, const Eos *EqState, const int TimeLevel); diff --git a/components/omega/test/ocn/PGradTest.cpp b/components/omega/test/ocn/PGradTest.cpp index fb479cccee3e..14c838b8d2a4 100644 --- a/components/omega/test/ocn/PGradTest.cpp +++ b/components/omega/test/ocn/PGradTest.cpp @@ -16,12 +16,14 @@ #include "Halo.h" #include "HorzMesh.h" #include "IO.h" +#include "IOStream.h" #include "Logging.h" #include "MachEnv.h" #include "OceanState.h" #include "OmegaKokkos.h" #include "Pacer.h" #include "PGrad.h" +#include "Tracers.h" #include "TimeStepper.h" #include "VertCoord.h" #include "mpi.h" @@ -53,6 +55,9 @@ void initPGradTest() { // Create the default decomposition (initializes the decomposition) Decomp::init(); + // Initialize streams + IOStream::init(); + // Initialize the default halo Err1 = Halo::init(); if (Err1 != 0) { @@ -76,6 +81,8 @@ void initPGradTest() { // Initialize ocean state OceanState::init(); + // Initialize tracers + Tracers::init(); CHECK_ERROR_ABORT(Err, "PGrad: error during initialization"); } @@ -88,85 +95,248 @@ int main(int argc, char *argv[]) { Kokkos::initialize(); Pacer::initialize(MPI_COMM_WORLD); Pacer::setPrefix("Omega:"); + { initPGradTest(); - // Initialize PressureGrad manager PressureGrad::init(); + MachEnv *DefEnv = MachEnv::getDefault(); HorzMesh *DefMesh = HorzMesh::getDefault(); - VertCoord *DefVCoord = VertCoord::getDefault(); + //VertCoord *DefVCoord = VertCoord::getDefault(); + VertCoord *VCoord = VertCoord::getDefault(); OceanState *DefState = OceanState::getDefault(); + Decomp *DefDecomp = Decomp::getDefault(); Eos *DefEos = Eos::getInstance(); Config *Options = Config::getOmegaConfig(); - I4 NEdgesOwned = DefMesh->NEdgesOwned; - I4 NVertLayers = VertCoord::getDefault()->NVertLayers; + + I4 NVertLayers =VCoord->NVertLayers; I4 NChunks = NVertLayers / VecLength; - // create arrays: Tend on edges, Pressure/Geopotential/SpecVol on cells - Array2DReal Tend("Tend", DefMesh->NEdgesAll, NVertLayers); - Array2DReal Pressure("Pressure", DefMesh->NCellsAll, NVertLayers); - Array2DReal Geopotential("Geopotential", DefMesh->NCellsAll, NVertLayers); - Array2DReal SpecVol("SpecVol", DefMesh->NCellsAll, NVertLayers); - - // initialize arrays to zero - parallelFor({DefMesh->NEdgesAll, NVertLayers}, KOKKOS_LAMBDA(int i, int k) { - Tend(i,k) = 0.0_Real; + // // set two column mesh + //DefMesh->NCellsAll = 2; + //DefMesh->NCellsOwned = 2; + //DefMesh->NEdgesAll = 1; + //DefMesh->NEdgesOwned = 1; + //DefDecomp->NCellsAll = 2; + //DefDecomp->NCellsSize = 2; + //DefDecomp->NCellsOwned = 2; + //DefDecomp->NEdgesAll = 1; + //DefDecomp->NEdgesOwned = 1; + + auto &MinLayerCell = VCoord->MinLayerCell; + auto &MaxLayerCell = VCoord->MaxLayerCell; + parallelFor({DefMesh->NCellsAll}, KOKKOS_LAMBDA(int i) { + MinLayerCell(i) = 0; + MaxLayerCell(i) = NVertLayers - 1; }); - parallelFor({DefMesh->NCellsAll, NVertLayers}, KOKKOS_LAMBDA(int i, int k) { - Pressure(i,k) = 0.0_Real; - Geopotential(i,k) = 0.0_Real; - SpecVol(i,k) = 1.0_Real; // unit specific volume + + auto &CellsOnEdge = DefMesh->CellsOnEdge; + auto &DcEdge = DefMesh->DcEdge; + auto &EdgeMask = DefMesh->EdgeMask; + parallelFor({DefMesh->NEdgesAll}, KOKKOS_LAMBDA(int e) { + CellsOnEdge(e, 0)= 0; + CellsOnEdge(e, 1)= 1; + DcEdge(e) = 100.0_Real; + for (int k = 0; k < NVertLayers; ++k) { + EdgeMask(e, k) = 1.0_Real; + } + }); + + // Fetch reference desnity from Config + Real Density0; + Error ErrorCode; + Config TendConfig("Tendencies"); + ErrorCode.reset(); + ErrorCode += Options->get(TendConfig); + CHECK_ERROR_ABORT(ErrorCode, "VertCoord: Tendencies group not found in Config"); + ErrorCode += TendConfig.get("Density0", Density0); + CHECK_ERROR_ABORT(ErrorCode, "VertCoord: Density0 not found in TendConfig"); + + I4 TimeLevel = 0; + + // create arrays: Tend on edges, Pressure/Geopotential/SpecVol on cells + Array2DReal Tend("Tend", DefMesh->NEdgesSize, NVertLayers); + Array2DReal SpecVolOld("SpecVolOld", DefMesh->NCellsSize, NVertLayers); + Array2DReal PressureMidOld("PressureMidOld", DefMesh->NCellsSize, NVertLayers); + Array2DReal LayerThick; + DefState->getLayerThickness(LayerThick, TimeLevel); + Array1DReal SurfacePressure("SurfacePressure", DefMesh->NCellsSize); + Array2DReal Temp; + Array2DReal Salinity; + Err = Tracers::getByName(Temp, TimeLevel, "Temperature"); + Err = Tracers::getByName(Salinity, TimeLevel, "Salinity"); + + + + // set Z interface and mid-point locations + Real ZBottom = -1000.0_Real; + Real dZ = -ZBottom / (NVertLayers + NVertLayers / 2); + auto &BottomDepth = VCoord->BottomDepth; + auto &ZInterface = VCoord->ZInterface; + auto &ZMid = VCoord->ZMid; + parallelFor({DefMesh->NCellsAll}, KOKKOS_LAMBDA(int i) { + ZInterface(i, NVertLayers) = ZBottom; + SurfacePressure(i) = 0.0_Real; + BottomDepth(i) = 0.0_Real; + for (int k = NVertLayers - 1; k >= 0; --k) { + Real dz = (((k + i) % 2) + 1.0_Real) * dZ; // staggered layer thickness + ZInterface(i, k) = ZInterface(i, k + 1) + dz; + LayerThick(i, k) = ZInterface(i, k) - ZInterface(i, k + 1); + ZMid(i, k) = 0.5_Real * (ZInterface(i, k) + ZInterface(i, k + 1)); + BottomDepth(i) += dz; + } }); - // set a simple pressure difference between two neighboring cells - if (DefMesh->NCellsAll >= 2) { - Pressure(0,0) = 1.0_Real; - Pressure(1,0) = 2.0_Real; + LOG_INFO("NVertLayers = {}", NVertLayers); + for (int i = 0; i < 2; ++i) { + for (int k = 0; k <= NVertLayers; ++k) { + LOG_INFO("ZInterface({}, {}) = {}", i, k, ZInterface(i, k)); + } } - - // call computePressureGrad - PressureGrad *Pg = PressureGrad::getDefault(); - if (!Pg) { - LOG_INFO("PGrad: default instance not present, creating via create"); - Pg = PressureGrad::create("default", DefMesh, DefVCoord, Options); + LOG_INFO("NVertLayers = {}", NVertLayers); + for (int i = 0; i < 2; ++i) { + for (int k = 0; k <= NVertLayers; ++k) { + LOG_INFO("LayerThick({}, {}) = {}", i, k, LayerThick(i, k)); + } } + LOG_INFO("NVertLayers = {}", NVertLayers); + for (int i = 0; i < 2; ++i) { + for (int k = 0; k <= NVertLayers; ++k) { + LOG_INFO("ZMid({}, {}) = {}", i, k, ZMid(i, k)); + } + } + + // set simple temperature and salinity profiles + auto &SpecVol = DefEos->SpecVol; + parallelFor({DefMesh->NCellsAll, NVertLayers}, KOKKOS_LAMBDA(int i, int k) { + Real T0 = 30.0; + Real TB = 1.0; + Real S0 = 30.0; + Real SB = 40.0; - int TimeLevel = 0; - if (Pg) { - Pg->computePressureGrad(Tend, DefState, DefVCoord, DefEos, TimeLevel); - // simple check: ensure Tend contains some finite values (not all zero) - I4 CountNonZero = 0; - parallelReduce("countNonZero", {NEdgesOwned, NVertLayers}, - KOKKOS_LAMBDA(int e, int k, I4 &acc) { - if (Tend(e,k) != 0.0_Real) acc++; - }, CountNonZero); + Real phi0 = (ZMid(i, k) - ZBottom) / (-ZBottom); + Real phiB = 1.0_Real - phi0; - if (CountNonZero > 0) { - LOG_INFO("PGrad: computePressureGrad produced non-zero tendencies PASS"); + Temp(i, k) = T0 * phi0 + TB * phiB; + Salinity(i, k) = S0 * phi0 + SB * phiB; + SpecVol(i, k) = 1.0_Real / Density0; + SpecVolOld(i, k) = SpecVol(i, k); + }); + + //for (int i = 0; i < 2; ++i) { + // for (int k = 0; k < NVertLayers; ++k) { + // LOG_INFO("Temp({}, {}) = {}", i, k, Temp(i, k)); + // } + //} + //for (int i = 0; i < 2; ++i) { + // for (int k = 0; k < NVertLayers; ++k) { + // LOG_INFO("Salinity({}, {}) = {}", i, k, Salinity(i, k)); + // } + //} + + // iterate to converge SpecVol + auto &PressureMid = VCoord->PressureMid; + VCoord->computePressure(LayerThick, SurfacePressure); + deepCopy(PressureMidOld, PressureMid); + //for (int i = 0; i < 2; ++i) { + // for (int k = 0; k < NVertLayers; ++k) { + // LOG_INFO("PressureMid({}, {}) = {}, PressureMidOld({}, {}) = {}", i, k, PressureMid(i, k), i, k, PressureMidOld(i, k)); + // } + //} + for (int iteration = 0; iteration < 5; ++iteration) { + + // compute specific volume from EOS + VCoord->computePressure(LayerThick, SurfacePressure); + DefEos->computeSpecVol(Temp, Salinity, PressureMid); + + for (int i = 0; i < 2; ++i) { + for (int k = 0; k < NVertLayers; ++k) { + LOG_INFO("PressureMid({}, {}) = {}, PressureMidOld({}, {}) = {}", i, k, PressureMid(i, k), i, k, PressureMidOld(i, k)); + } + } + for (int i = 0; i < 2; ++i) { + for (int k = 0; k < NVertLayers; ++k) { + LOG_INFO(" Density({}, {}) = {}", i, k, 1.0_Real / SpecVol(i, k)); + } + } + + // Compute psuedo thickness + parallelFor({DefMesh->NCellsAll, NVertLayers}, KOKKOS_LAMBDA(int i, int k) { + LayerThick(i, k) = 1.0_Real / (SpecVol(i, k) * Density0) * (ZInterface(i, k) - ZInterface(i, k + 1)); + }); + + // check for convergence + Real max_value = 0.0_Real; + parallelReduce({DefMesh->NCellsAll, NVertLayers}, + KOKKOS_LAMBDA(int i, int k, Real &max) { + Real diff = Kokkos::abs(SpecVol(i, k) - SpecVolOld(i, k)); + if (diff > max) max = diff; + }, Kokkos::Max(max_value) ); + + if (max_value < 1e-12_Real) { + LOG_INFO("converged: max diff = {}", max_value); + break; } else { - RetVal += 1; - LOG_ERROR("PGrad: computePressureGrad produced all-zero tendencies FAIL"); + LOG_INFO("max diff = {}", max_value); + parallelFor({DefMesh->NCellsAll, NVertLayers}, KOKKOS_LAMBDA(int i, int k) { + SpecVolOld(i, k) = SpecVol(i, k); + }); + } + + } + //for (int i = 0; i < 2; ++i) { + // for (int k = 0; k < NVertLayers; ++k) { + // LOG_INFO("SpecVol({}, {}) = {}", i, k, SpecVol(i, k)); + // } + //} + + + VCoord->computeZHeight(LayerThick, SpecVol); + Array1DReal SelfAttractionLoading("SelfAttractionLoading", DefMesh->NCellsAll); + Array1DReal TidalPotential("TidalPotential", DefMesh->NCellsAll); + deepCopy(TidalPotential, 0.0_Real); + deepCopy(SelfAttractionLoading, 0.0_Real); + VCoord->computeGeopotential(TidalPotential, SelfAttractionLoading); + + // create PressureGrad instance + PressureGrad *DefPGrad = PressureGrad::getDefault(); + if (!DefPGrad) { + LOG_INFO("PGrad: default instance not created by init"); + } + auto PGradTest = PressureGrad::create("Test", DefMesh, VCoord, Options); + + for (int i = 0; i < 2; ++i) { + for (int k = 0; k < NVertLayers; ++k) { + Tend(i, k) = 0.0_Real; + LOG_INFO("Tend({}, {}) = {}", i, k, Tend(i, k)); + } + } + PGradTest->computePressureGrad(Tend, DefState, VCoord, DefEos, TimeLevel); + for (int i = 0; i < 2; ++i) { + for (int k = 0; k < NVertLayers; ++k) { + LOG_INFO("Tend({}, {}) = {}", i, k, Tend(i, k)); } - } else { - RetVal += 1; - LOG_ERROR("PGrad: failed to obtain PressureGrad instance FAIL"); } + // cleanup PressureGrad::clear(); - VertCoord::clear(); + IOStream::finalize(); + TimeStepper::clear(); + Tracers::clear(); + Eos::destroyInstance(); OceanState::clear(); + VertCoord::clear(); + Field::clear(); + Dimension::clear(); HorzMesh::clear(); Halo::clear(); Decomp::clear(); MachEnv::removeAll(); - FieldGroup::clear(); - Field::clear(); - Dimension::clear(); } Pacer::finalize(); Kokkos::finalize(); From 177c917447621194cf347244c78cde7bea867d77 Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Fri, 21 Nov 2025 08:59:50 -0800 Subject: [PATCH 07/58] Add tilt factor to PGradTest --- components/omega/test/ocn/PGradTest.cpp | 101 ++++++++++-------------- 1 file changed, 42 insertions(+), 59 deletions(-) diff --git a/components/omega/test/ocn/PGradTest.cpp b/components/omega/test/ocn/PGradTest.cpp index 14c838b8d2a4..02968e6bf4cf 100644 --- a/components/omega/test/ocn/PGradTest.cpp +++ b/components/omega/test/ocn/PGradTest.cpp @@ -104,30 +104,19 @@ int main(int argc, char *argv[]) { MachEnv *DefEnv = MachEnv::getDefault(); HorzMesh *DefMesh = HorzMesh::getDefault(); - //VertCoord *DefVCoord = VertCoord::getDefault(); - VertCoord *VCoord = VertCoord::getDefault(); + //VertCoord *DefDefVCoord = VertCoord::getDefault(); + VertCoord *DefVCoord = VertCoord::getDefault(); OceanState *DefState = OceanState::getDefault(); Decomp *DefDecomp = Decomp::getDefault(); Eos *DefEos = Eos::getInstance(); Config *Options = Config::getOmegaConfig(); - I4 NVertLayers =VCoord->NVertLayers; + I4 NVertLayers =DefVCoord->NVertLayers; I4 NChunks = NVertLayers / VecLength; - // // set two column mesh - //DefMesh->NCellsAll = 2; - //DefMesh->NCellsOwned = 2; - //DefMesh->NEdgesAll = 1; - //DefMesh->NEdgesOwned = 1; - //DefDecomp->NCellsAll = 2; - //DefDecomp->NCellsSize = 2; - //DefDecomp->NCellsOwned = 2; - //DefDecomp->NEdgesAll = 1; - //DefDecomp->NEdgesOwned = 1; - - auto &MinLayerCell = VCoord->MinLayerCell; - auto &MaxLayerCell = VCoord->MaxLayerCell; + auto &MinLayerCell = DefVCoord->MinLayerCell; + auto &MaxLayerCell = DefVCoord->MaxLayerCell; parallelFor({DefMesh->NCellsAll}, KOKKOS_LAMBDA(int i) { MinLayerCell(i) = 0; MaxLayerCell(i) = NVertLayers - 1; @@ -139,10 +128,10 @@ int main(int argc, char *argv[]) { parallelFor({DefMesh->NEdgesAll}, KOKKOS_LAMBDA(int e) { CellsOnEdge(e, 0)= 0; CellsOnEdge(e, 1)= 1; - DcEdge(e) = 100.0_Real; - for (int k = 0; k < NVertLayers; ++k) { - EdgeMask(e, k) = 1.0_Real; - } + //DcEdge(e) = 100.0_Real; + //for (int k = 0; k < NVertLayers; ++k) { + // EdgeMask(e, k) = 1.0_Real; + //} }); // Fetch reference desnity from Config @@ -173,16 +162,18 @@ int main(int argc, char *argv[]) { // set Z interface and mid-point locations Real ZBottom = -1000.0_Real; - Real dZ = -ZBottom / (NVertLayers + NVertLayers / 2); - auto &BottomDepth = VCoord->BottomDepth; - auto &ZInterface = VCoord->ZInterface; - auto &ZMid = VCoord->ZMid; + Real dZ = 2.0_Real * (-ZBottom / NVertLayers); + auto &BottomDepth = DefVCoord->BottomDepth; + auto &ZInterface = DefVCoord->ZInterface; + auto &ZMid = DefVCoord->ZMid; + Real tilt_factor = 0.495_Real; parallelFor({DefMesh->NCellsAll}, KOKKOS_LAMBDA(int i) { ZInterface(i, NVertLayers) = ZBottom; SurfacePressure(i) = 0.0_Real; BottomDepth(i) = 0.0_Real; for (int k = NVertLayers - 1; k >= 0; --k) { - Real dz = (((k + i) % 2) + 1.0_Real) * dZ; // staggered layer thickness + Real x = (k + i) % 2; + Real dz = ( 2.0_Real * tilt_factor - 1.0_Real ) * x * dZ + (1.0_Real - tilt_factor) * dZ; // staggered layer thickness ZInterface(i, k) = ZInterface(i, k + 1) + dz; LayerThick(i, k) = ZInterface(i, k) - ZInterface(i, k + 1); ZMid(i, k) = 0.5_Real * (ZInterface(i, k) + ZInterface(i, k + 1)); @@ -202,12 +193,12 @@ int main(int argc, char *argv[]) { LOG_INFO("LayerThick({}, {}) = {}", i, k, LayerThick(i, k)); } } - LOG_INFO("NVertLayers = {}", NVertLayers); - for (int i = 0; i < 2; ++i) { - for (int k = 0; k <= NVertLayers; ++k) { - LOG_INFO("ZMid({}, {}) = {}", i, k, ZMid(i, k)); - } - } + //LOG_INFO("NVertLayers = {}", NVertLayers); + //for (int i = 0; i < 2; ++i) { + // for (int k = 0; k <= NVertLayers; ++k) { + // LOG_INFO("ZMid({}, {}) = {}", i, k, ZMid(i, k)); + // } + //} // set simple temperature and salinity profiles auto &SpecVol = DefEos->SpecVol; @@ -239,37 +230,26 @@ int main(int argc, char *argv[]) { //} // iterate to converge SpecVol - auto &PressureMid = VCoord->PressureMid; - VCoord->computePressure(LayerThick, SurfacePressure); + auto &PressureMid = DefVCoord->PressureMid; + DefVCoord->computePressure(LayerThick, SurfacePressure); deepCopy(PressureMidOld, PressureMid); //for (int i = 0; i < 2; ++i) { // for (int k = 0; k < NVertLayers; ++k) { // LOG_INFO("PressureMid({}, {}) = {}, PressureMidOld({}, {}) = {}", i, k, PressureMid(i, k), i, k, PressureMidOld(i, k)); // } //} - for (int iteration = 0; iteration < 5; ++iteration) { + for (int iteration = 0; iteration < 15; ++iteration) { // compute specific volume from EOS - VCoord->computePressure(LayerThick, SurfacePressure); + DefVCoord->computePressure(LayerThick, SurfacePressure); DefEos->computeSpecVol(Temp, Salinity, PressureMid); - for (int i = 0; i < 2; ++i) { - for (int k = 0; k < NVertLayers; ++k) { - LOG_INFO("PressureMid({}, {}) = {}, PressureMidOld({}, {}) = {}", i, k, PressureMid(i, k), i, k, PressureMidOld(i, k)); - } - } - for (int i = 0; i < 2; ++i) { - for (int k = 0; k < NVertLayers; ++k) { - LOG_INFO(" Density({}, {}) = {}", i, k, 1.0_Real / SpecVol(i, k)); - } - } - - // Compute psuedo thickness + // compute psuedo thickness from specific volume parallelFor({DefMesh->NCellsAll, NVertLayers}, KOKKOS_LAMBDA(int i, int k) { LayerThick(i, k) = 1.0_Real / (SpecVol(i, k) * Density0) * (ZInterface(i, k) - ZInterface(i, k + 1)); }); - // check for convergence + // compute difference from previous iteration Real max_value = 0.0_Real; parallelReduce({DefMesh->NCellsAll, NVertLayers}, KOKKOS_LAMBDA(int i, int k, Real &max) { @@ -277,6 +257,7 @@ int main(int argc, char *argv[]) { if (diff > max) max = diff; }, Kokkos::Max(max_value) ); + // check convergence if (max_value < 1e-12_Real) { LOG_INFO("converged: max diff = {}", max_value); break; @@ -294,28 +275,30 @@ int main(int argc, char *argv[]) { // } //} - - VCoord->computeZHeight(LayerThick, SpecVol); + // compute geopotential + //DefVCoord->computeZHeight(LayerThick, SpecVol); Array1DReal SelfAttractionLoading("SelfAttractionLoading", DefMesh->NCellsAll); Array1DReal TidalPotential("TidalPotential", DefMesh->NCellsAll); deepCopy(TidalPotential, 0.0_Real); deepCopy(SelfAttractionLoading, 0.0_Real); - VCoord->computeGeopotential(TidalPotential, SelfAttractionLoading); + DefVCoord->computeGeopotential(TidalPotential, SelfAttractionLoading); // create PressureGrad instance PressureGrad *DefPGrad = PressureGrad::getDefault(); if (!DefPGrad) { LOG_INFO("PGrad: default instance not created by init"); } - auto PGradTest = PressureGrad::create("Test", DefMesh, VCoord, Options); + auto PGradTest = PressureGrad::create("Test", DefMesh, DefVCoord, Options); - for (int i = 0; i < 2; ++i) { - for (int k = 0; k < NVertLayers; ++k) { - Tend(i, k) = 0.0_Real; - LOG_INFO("Tend({}, {}) = {}", i, k, Tend(i, k)); - } - } - PGradTest->computePressureGrad(Tend, DefState, VCoord, DefEos, TimeLevel); + //for (int i = 0; i < 2; ++i) { + // for (int k = 0; k < NVertLayers; ++k) { + // Tend(i, k) = 0.0_Real; + // LOG_INFO("Tend({}, {}) = {}", i, k, Tend(i, k)); + // } + //} + + // compute pressure gradient + PGradTest->computePressureGrad(Tend, DefState, DefVCoord, DefEos, TimeLevel); for (int i = 0; i < 2; ++i) { for (int k = 0; k < NVertLayers; ++k) { LOG_INFO("Tend({}, {}) = {}", i, k, Tend(i, k)); From 582d618e0c2f162b007d5200ac4fc1c2ce982033 Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Mon, 24 Nov 2025 13:53:57 -0800 Subject: [PATCH 08/58] Test for pressure gradient convergence in PGradTest --- components/omega/test/ocn/PGradTest.cpp | 382 +++++++++++++----------- 1 file changed, 204 insertions(+), 178 deletions(-) diff --git a/components/omega/test/ocn/PGradTest.cpp b/components/omega/test/ocn/PGradTest.cpp index 02968e6bf4cf..ac66e6ee9b3f 100644 --- a/components/omega/test/ocn/PGradTest.cpp +++ b/components/omega/test/ocn/PGradTest.cpp @@ -104,207 +104,233 @@ int main(int argc, char *argv[]) { MachEnv *DefEnv = MachEnv::getDefault(); HorzMesh *DefMesh = HorzMesh::getDefault(); - //VertCoord *DefDefVCoord = VertCoord::getDefault(); - VertCoord *DefVCoord = VertCoord::getDefault(); + //VertCoord *DefVCoord = VertCoord::getDefault(); + VertCoord *VCoord = VertCoord::getDefault(); OceanState *DefState = OceanState::getDefault(); Decomp *DefDecomp = Decomp::getDefault(); Eos *DefEos = Eos::getInstance(); Config *Options = Config::getOmegaConfig(); - - I4 NVertLayers =DefVCoord->NVertLayers; - I4 NChunks = NVertLayers / VecLength; - - auto &MinLayerCell = DefVCoord->MinLayerCell; - auto &MaxLayerCell = DefVCoord->MaxLayerCell; - parallelFor({DefMesh->NCellsAll}, KOKKOS_LAMBDA(int i) { - MinLayerCell(i) = 0; - MaxLayerCell(i) = NVertLayers - 1; - }); - - auto &CellsOnEdge = DefMesh->CellsOnEdge; - auto &DcEdge = DefMesh->DcEdge; - auto &EdgeMask = DefMesh->EdgeMask; - parallelFor({DefMesh->NEdgesAll}, KOKKOS_LAMBDA(int e) { - CellsOnEdge(e, 0)= 0; - CellsOnEdge(e, 1)= 1; - //DcEdge(e) = 100.0_Real; - //for (int k = 0; k < NVertLayers; ++k) { - // EdgeMask(e, k) = 1.0_Real; - //} - }); - - // Fetch reference desnity from Config - Real Density0; - Error ErrorCode; - Config TendConfig("Tendencies"); - ErrorCode.reset(); - ErrorCode += Options->get(TendConfig); - CHECK_ERROR_ABORT(ErrorCode, "VertCoord: Tendencies group not found in Config"); - ErrorCode += TendConfig.get("Density0", Density0); - CHECK_ERROR_ABORT(ErrorCode, "VertCoord: Density0 not found in TendConfig"); - - I4 TimeLevel = 0; + I4 NVertLayers = 60; + Real dC = 30000.0_Real; // create arrays: Tend on edges, Pressure/Geopotential/SpecVol on cells Array2DReal Tend("Tend", DefMesh->NEdgesSize, NVertLayers); Array2DReal SpecVolOld("SpecVolOld", DefMesh->NCellsSize, NVertLayers); Array2DReal PressureMidOld("PressureMidOld", DefMesh->NCellsSize, NVertLayers); - Array2DReal LayerThick; - DefState->getLayerThickness(LayerThick, TimeLevel); Array1DReal SurfacePressure("SurfacePressure", DefMesh->NCellsSize); - Array2DReal Temp; - Array2DReal Salinity; - Err = Tracers::getByName(Temp, TimeLevel, "Temperature"); - Err = Tracers::getByName(Salinity, TimeLevel, "Salinity"); - - - - // set Z interface and mid-point locations - Real ZBottom = -1000.0_Real; - Real dZ = 2.0_Real * (-ZBottom / NVertLayers); - auto &BottomDepth = DefVCoord->BottomDepth; - auto &ZInterface = DefVCoord->ZInterface; - auto &ZMid = DefVCoord->ZMid; - Real tilt_factor = 0.495_Real; - parallelFor({DefMesh->NCellsAll}, KOKKOS_LAMBDA(int i) { - ZInterface(i, NVertLayers) = ZBottom; - SurfacePressure(i) = 0.0_Real; - BottomDepth(i) = 0.0_Real; - for (int k = NVertLayers - 1; k >= 0; --k) { - Real x = (k + i) % 2; - Real dz = ( 2.0_Real * tilt_factor - 1.0_Real ) * x * dZ + (1.0_Real - tilt_factor) * dZ; // staggered layer thickness - ZInterface(i, k) = ZInterface(i, k + 1) + dz; - LayerThick(i, k) = ZInterface(i, k) - ZInterface(i, k + 1); - ZMid(i, k) = 0.5_Real * (ZInterface(i, k) + ZInterface(i, k + 1)); - BottomDepth(i) += dz; - } - }); - LOG_INFO("NVertLayers = {}", NVertLayers); - for (int i = 0; i < 2; ++i) { - for (int k = 0; k <= NVertLayers; ++k) { - LOG_INFO("ZInterface({}, {}) = {}", i, k, ZInterface(i, k)); - } - } - LOG_INFO("NVertLayers = {}", NVertLayers); - for (int i = 0; i < 2; ++i) { - for (int k = 0; k <= NVertLayers; ++k) { - LOG_INFO("LayerThick({}, {}) = {}", i, k, LayerThick(i, k)); + for (int refinement = 0; refinement < 3; ++refinement) { + + LOG_INFO("PGradTest: Starting refinement level {}", refinement); + VCoord->NVertLayers = NVertLayers; + VCoord->NVertLayersP1 = NVertLayers + 1; + + auto &MinLayerCell = VCoord->MinLayerCell; + auto &MaxLayerCell = VCoord->MaxLayerCell; + parallelFor({DefMesh->NCellsAll}, KOKKOS_LAMBDA(int i) { + MinLayerCell(i) = 0; + MaxLayerCell(i) = NVertLayers - 1; + }); + + auto &CellsOnEdge = DefMesh->CellsOnEdge; + auto &DcEdge = DefMesh->DcEdge; + auto &EdgeMask = DefMesh->EdgeMask; + parallelFor({DefMesh->NEdgesAll}, KOKKOS_LAMBDA(int e) { + CellsOnEdge(e, 0)= 0; + CellsOnEdge(e, 1)= 1; + DcEdge(e) = dC; + //for (int k = 0; k < NVertLayers; ++k) { + // EdgeMask(e, k) = 1.0_Real; + //} + }); + + // Fetch reference desnity from Config + Real Density0; + Error ErrorCode; + Config TendConfig("Tendencies"); + ErrorCode.reset(); + ErrorCode += Options->get(TendConfig); + CHECK_ERROR_ABORT(ErrorCode, "VertCoord: Tendencies group not found in Config"); + ErrorCode += TendConfig.get("Density0", Density0); + CHECK_ERROR_ABORT(ErrorCode, "VertCoord: Density0 not found in TendConfig"); + + I4 TimeLevel = 0; + + // get state and tracer arrays + Array2DReal LayerThick; + DefState->getLayerThickness(LayerThick, TimeLevel); + Array2DReal Temp; + Array2DReal Salinity; + Err = Tracers::getByName(Temp, TimeLevel, "Temperature"); + Err = Tracers::getByName(Salinity, TimeLevel, "Salinity"); + + + + // set Z interface and mid-point locations + Real ZBottom = -1000.0_Real; + Real dZ = 2.0_Real * (-ZBottom / NVertLayers); + auto &BottomDepth = VCoord->BottomDepth; + auto &ZInterface = VCoord->ZInterface; + auto &ZMid = VCoord->ZMid; + Real tilt_factor = 0.495_Real; + parallelFor({DefMesh->NCellsAll}, KOKKOS_LAMBDA(int i) { + ZInterface(i, NVertLayers) = ZBottom; + SurfacePressure(i) = 0.0_Real; + BottomDepth(i) = 0.0_Real; + for (int k = NVertLayers - 1; k >= 0; --k) { + Real x = (k + i) % 2; + Real dz = ( 2.0_Real * tilt_factor - 1.0_Real ) * x * dZ + (1.0_Real - tilt_factor) * dZ; // staggered layer thickness + ZInterface(i, k) = ZInterface(i, k + 1) + dz; + LayerThick(i, k) = ZInterface(i, k) - ZInterface(i, k + 1); + ZMid(i, k) = 0.5_Real * (ZInterface(i, k) + ZInterface(i, k + 1)); + BottomDepth(i) += dz; + } + }); + + LOG_INFO("NVertLayers = {}", NVertLayers); + LOG_INFO("dC = {}", dC); + //for (int i = 0; i < 2; ++i) { + // for (int k = 0; k <= NVertLayers; ++k) { + // LOG_INFO("ZInterface({}, {}) = {}", i, k, ZInterface(i, k)); + // } + //} + //LOG_INFO("NVertLayers = {}", NVertLayers); + for (int i = 0; i < 2; ++i) { + //for (int k = 0; k <= NVertLayers; ++k) { + for (int k = 0; k < 2; ++k) { + LOG_INFO("LayerThick({}, {}) = {}", i, k, LayerThick(i, k)); + } } - } - //LOG_INFO("NVertLayers = {}", NVertLayers); - //for (int i = 0; i < 2; ++i) { - // for (int k = 0; k <= NVertLayers; ++k) { - // LOG_INFO("ZMid({}, {}) = {}", i, k, ZMid(i, k)); - // } - //} - - // set simple temperature and salinity profiles - auto &SpecVol = DefEos->SpecVol; - parallelFor({DefMesh->NCellsAll, NVertLayers}, KOKKOS_LAMBDA(int i, int k) { - Real T0 = 30.0; - Real TB = 1.0; - Real S0 = 30.0; - Real SB = 40.0; - - - Real phi0 = (ZMid(i, k) - ZBottom) / (-ZBottom); - Real phiB = 1.0_Real - phi0; - - Temp(i, k) = T0 * phi0 + TB * phiB; - Salinity(i, k) = S0 * phi0 + SB * phiB; - SpecVol(i, k) = 1.0_Real / Density0; - SpecVolOld(i, k) = SpecVol(i, k); - }); - - //for (int i = 0; i < 2; ++i) { - // for (int k = 0; k < NVertLayers; ++k) { - // LOG_INFO("Temp({}, {}) = {}", i, k, Temp(i, k)); - // } - //} - //for (int i = 0; i < 2; ++i) { - // for (int k = 0; k < NVertLayers; ++k) { - // LOG_INFO("Salinity({}, {}) = {}", i, k, Salinity(i, k)); - // } - //} - - // iterate to converge SpecVol - auto &PressureMid = DefVCoord->PressureMid; - DefVCoord->computePressure(LayerThick, SurfacePressure); - deepCopy(PressureMidOld, PressureMid); - //for (int i = 0; i < 2; ++i) { - // for (int k = 0; k < NVertLayers; ++k) { - // LOG_INFO("PressureMid({}, {}) = {}, PressureMidOld({}, {}) = {}", i, k, PressureMid(i, k), i, k, PressureMidOld(i, k)); - // } - //} - for (int iteration = 0; iteration < 15; ++iteration) { - - // compute specific volume from EOS - DefVCoord->computePressure(LayerThick, SurfacePressure); - DefEos->computeSpecVol(Temp, Salinity, PressureMid); - - // compute psuedo thickness from specific volume + //LOG_INFO("NVertLayers = {}", NVertLayers); + //for (int i = 0; i < 2; ++i) { + // for (int k = 0; k <= NVertLayers; ++k) { + // LOG_INFO("ZMid({}, {}) = {}", i, k, ZMid(i, k)); + // } + //} + + // set simple temperature and salinity profiles + auto &SpecVol = DefEos->SpecVol; parallelFor({DefMesh->NCellsAll, NVertLayers}, KOKKOS_LAMBDA(int i, int k) { - LayerThick(i, k) = 1.0_Real / (SpecVol(i, k) * Density0) * (ZInterface(i, k) - ZInterface(i, k + 1)); + Real T0 = 30.0; + Real TB = 5.0; + Real S0 = 30.0; + Real SB = 40.0; + + + Real phi0 = (ZMid(i, k) - ZBottom) / (-ZBottom); + Real phiB = 1.0_Real - phi0; + + Temp(i, k) = T0 * phi0 + TB * phiB; + Salinity(i, k) = S0 * phi0 + SB * phiB; + SpecVol(i, k) = 1.0_Real / Density0; + SpecVolOld(i, k) = SpecVol(i, k); }); - // compute difference from previous iteration - Real max_value = 0.0_Real; - parallelReduce({DefMesh->NCellsAll, NVertLayers}, - KOKKOS_LAMBDA(int i, int k, Real &max) { - Real diff = Kokkos::abs(SpecVol(i, k) - SpecVolOld(i, k)); - if (diff > max) max = diff; - }, Kokkos::Max(max_value) ); + //for (int i = 0; i < 2; ++i) { + // for (int k = 0; k < NVertLayers; ++k) { + // LOG_INFO("Temp({}, {}) = {}", i, k, Temp(i, k)); + // } + //} + //for (int i = 0; i < 2; ++i) { + // for (int k = 0; k < NVertLayers; ++k) { + // LOG_INFO("Salinity({}, {}) = {}", i, k, Salinity(i, k)); + // } + //} + + // iterate to converge SpecVol + auto &PressureMid = VCoord->PressureMid; + VCoord->computePressure(LayerThick, SurfacePressure); + deepCopy(PressureMidOld, PressureMid); + //for (int i = 0; i < 2; ++i) { + // for (int k = 0; k < NVertLayers; ++k) { + // LOG_INFO("PressureMid({}, {}) = {}, PressureMidOld({}, {}) = {}", i, k, PressureMid(i, k), i, k, PressureMidOld(i, k)); + // } + //} + for (int iteration = 0; iteration < 15; ++iteration) { + + // compute specific volume from EOS + VCoord->computePressure(LayerThick, SurfacePressure); + DefEos->computeSpecVol(Temp, Salinity, PressureMid); - // check convergence - if (max_value < 1e-12_Real) { - LOG_INFO("converged: max diff = {}", max_value); - break; - } else { - LOG_INFO("max diff = {}", max_value); + // compute psuedo thickness from specific volume parallelFor({DefMesh->NCellsAll, NVertLayers}, KOKKOS_LAMBDA(int i, int k) { - SpecVolOld(i, k) = SpecVol(i, k); + LayerThick(i, k) = 1.0_Real / (SpecVol(i, k) * Density0) * (ZInterface(i, k) - ZInterface(i, k + 1)); }); + + // compute difference from previous iteration + Real max_value = 0.0_Real; + parallelReduce({DefMesh->NCellsAll, NVertLayers}, + KOKKOS_LAMBDA(int i, int k, Real &max) { + Real diff = Kokkos::abs(SpecVol(i, k) - SpecVolOld(i, k)); + if (diff > max) max = diff; + }, Kokkos::Max(max_value) ); + + // check convergence + if (max_value < 1e-12_Real) { + LOG_INFO("converged: max diff = {}", max_value); + break; + } else { + //LOG_INFO("max diff = {}", max_value); + parallelFor({DefMesh->NCellsAll, NVertLayers}, KOKKOS_LAMBDA(int i, int k) { + SpecVolOld(i, k) = SpecVol(i, k); + }); + } + } + //for (int i = 0; i < 2; ++i) { + // for (int k = 0; k < NVertLayers; ++k) { + // LOG_INFO("SpecVol({}, {}) = {}", i, k, SpecVol(i, k)); + // } + //} - } - //for (int i = 0; i < 2; ++i) { - // for (int k = 0; k < NVertLayers; ++k) { - // LOG_INFO("SpecVol({}, {}) = {}", i, k, SpecVol(i, k)); - // } - //} - - // compute geopotential - //DefVCoord->computeZHeight(LayerThick, SpecVol); - Array1DReal SelfAttractionLoading("SelfAttractionLoading", DefMesh->NCellsAll); - Array1DReal TidalPotential("TidalPotential", DefMesh->NCellsAll); - deepCopy(TidalPotential, 0.0_Real); - deepCopy(SelfAttractionLoading, 0.0_Real); - DefVCoord->computeGeopotential(TidalPotential, SelfAttractionLoading); - - // create PressureGrad instance - PressureGrad *DefPGrad = PressureGrad::getDefault(); - if (!DefPGrad) { - LOG_INFO("PGrad: default instance not created by init"); - } - auto PGradTest = PressureGrad::create("Test", DefMesh, DefVCoord, Options); - - //for (int i = 0; i < 2; ++i) { - // for (int k = 0; k < NVertLayers; ++k) { - // Tend(i, k) = 0.0_Real; - // LOG_INFO("Tend({}, {}) = {}", i, k, Tend(i, k)); - // } - //} - - // compute pressure gradient - PGradTest->computePressureGrad(Tend, DefState, DefVCoord, DefEos, TimeLevel); - for (int i = 0; i < 2; ++i) { - for (int k = 0; k < NVertLayers; ++k) { - LOG_INFO("Tend({}, {}) = {}", i, k, Tend(i, k)); + // compute geopotential + //VCoord->computeZHeight(LayerThick, SpecVol); + Array1DReal SelfAttractionLoading("SelfAttractionLoading", DefMesh->NCellsAll); + Array1DReal TidalPotential("TidalPotential", DefMesh->NCellsAll); + deepCopy(TidalPotential, 0.0_Real); + deepCopy(SelfAttractionLoading, 0.0_Real); + VCoord->computeGeopotential(TidalPotential, SelfAttractionLoading); + + // create PressureGrad instance + PressureGrad *DefPGrad = PressureGrad::getDefault(); + if (!DefPGrad) { + LOG_INFO("PGrad: default instance not created by init"); } - } + //auto PGradTest = PressureGrad::create("Test", DefMesh, VCoord, Options); + + //for (int i = 0; i < 2; ++i) { + // for (int k = 0; k < NVertLayers; ++k) { + // Tend(i, k) = 0.0_Real; + // LOG_INFO("Tend({}, {}) = {}", i, k, Tend(i, k)); + // } + //} + + // compute pressure gradient + //PGradTest->computePressureGrad(Tend, DefState, VCoord, DefEos, TimeLevel); + DefPGrad->computePressureGrad(Tend, DefState, VCoord, DefEos, TimeLevel); + //for (int i = 0; i < 2; ++i) { + // for (int k = 0; k < NVertLayers; ++k) { + // LOG_INFO("Tend({}, {}) = {}", i, k, Tend(i, k)); + // } + //} + Real max_value = 0.0_Real; + parallelReduce({DefMesh->NEdgesAll, NVertLayers}, + KOKKOS_LAMBDA(int i, int k, Real &max) { + Real val = Kokkos::abs(Tend(i, k)); + if (val > max) max = val; + }, Kokkos::Max(max_value) ); + Real sum_value = 0.0_Real; + parallelReduce({DefMesh->NEdgesAll, NVertLayers}, + KOKKOS_LAMBDA(int i, int k, Real &lsum) { + lsum += Tend(i,k) * Tend(i, k); + }, Kokkos::Sum(sum_value) ); + LOG_INFO("refinement level {}: max |Tend| = {}, average Tend = {}", refinement, max_value, std::sqrt(sum_value) / (DefMesh->NCellsAll * NVertLayers)); + + dC = dC * 2.0_Real; + NVertLayers = NVertLayers / 2; + } // refinement loop // cleanup PressureGrad::clear(); From 0127a95085f4c94c4e54994a81de1b476143c334 Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Tue, 25 Nov 2025 14:38:38 -0800 Subject: [PATCH 09/58] Update interface product averaging --- components/omega/src/ocn/PGrad.cpp | 58 +++++++++++++++++++++++-- components/omega/src/ocn/PGrad.h | 35 ++++++--------- components/omega/test/ocn/PGradTest.cpp | 22 +++++----- 3 files changed, 81 insertions(+), 34 deletions(-) diff --git a/components/omega/src/ocn/PGrad.cpp b/components/omega/src/ocn/PGrad.cpp index effdccca8a72..5fe768e5214b 100644 --- a/components/omega/src/ocn/PGrad.cpp +++ b/components/omega/src/ocn/PGrad.cpp @@ -74,15 +74,18 @@ PressureGrad::PressureGrad( const HorzMesh *Mesh, ///< [in] Horizontal mesh const VertCoord *VCoord, ///< [in] Vertical coordinate Config *Options) ///< [in] Configuration options - : CellsOnEdge(Mesh->CellsOnEdge), DvEdge(Mesh->DvEdge), + : CellsOnEdge(Mesh->CellsOnEdge), DcEdge(Mesh->DcEdge), EdgeSignOnCell(Mesh->EdgeSignOnCell), CenteredPGrad(Mesh), HighOrderPGrad(Mesh) { // store mesh sizes NEdgesAll = Mesh->NEdgesAll; NVertLayers = VCoord->NVertLayers; + NVertLayersP1 = NVertLayers + 1; NChunks = NVertLayers / VecLength; + InterfaceProduct = Array2DReal("InterfaceProduct", NEdgesAll, NVertLayersP1); + // Read config options for PressureGrad type // and enable the appropriate functor Config PGradConfig("PressureGrad"); @@ -154,6 +157,7 @@ void PressureGrad::computePressureGrad(Array2DReal &Tend, OMEGA_SCOPE(LocCenteredPGrad, CenteredPGrad); OMEGA_SCOPE(LocHighOrderPGrad, HighOrderPGrad); + OMEGA_SCOPE(LocInterfaceProduct, InterfaceProduct); const Array2DReal &PressureMid = VCoord->PressureMid; const Array2DReal &PressureInterface = VCoord->PressureInterface; @@ -165,12 +169,16 @@ void PressureGrad::computePressureGrad(Array2DReal &Tend, State->getLayerThickness(LayerThick, TimeLevel); if (PressureGradChoice == PressureGradType::Centered) { + + computeInterfaceProduct(PressureMid, SpecVol, + LayerThick, ZInterface); + parallelFor( "pgrad-centered", {NEdgesAll, NChunks}, KOKKOS_LAMBDA(I4 IEdge, I4 KChunk) { LocCenteredPGrad(Tend, IEdge, KChunk, PressureMid, - PressureInterface, Geopotential, LayerThick, - ZInterface, SpecVol, SpecVolInterface); + Geopotential, LayerThick, + LocInterfaceProduct, SpecVol); }); } else { parallelFor( @@ -182,6 +190,50 @@ void PressureGrad::computePressureGrad(Array2DReal &Tend, } } // end compute pressure gradient +//------------------------------------------------------------------------------ +// Compute interface product needed for centered pressure gradient +void PressureGrad::computeInterfaceProduct( + const Array2DReal &PressureMid, + const Array2DReal &SpecVol, + const Array2DReal &LayerThick, + const Array2DReal &ZInterface) { + + + OMEGA_SCOPE(LocCellsOnEdge, CellsOnEdge); + OMEGA_SCOPE(LocDcEdge, DcEdge); + OMEGA_SCOPE(LocInterfaceProduct, InterfaceProduct); + parallelFor( + "compute-interface-product", {NEdgesAll, NVertLayersP1}, + KOKKOS_LAMBDA(I4 IEdge, I4 K) { + const I4 ICell0 = LocCellsOnEdge(IEdge, 0); + const I4 ICell1 = LocCellsOnEdge(IEdge, 1); + + if (K == 0) { + LocInterfaceProduct(IEdge, K) = + PressureMid(ICell1, K) * SpecVol(ICell1, K) * LayerThick(ICell1, K) + + PressureMid(ICell0, K) * SpecVol(ICell0, K) * LayerThick(ICell0, K); + LocInterfaceProduct(IEdge, K) /= (LayerThick(ICell1, K) + LayerThick(ICell0, K)); + + } else if (K == NVertLayers) { + LocInterfaceProduct(IEdge, K) = + PressureMid(ICell1, K-1) * SpecVol(ICell1, K-1) * LayerThick(ICell1, K-1) + + PressureMid(ICell0, K-1) * SpecVol(ICell0, K-1) * LayerThick(ICell0, K-1); + LocInterfaceProduct(IEdge, K) /= (LayerThick(ICell1, K-1) + LayerThick(ICell0, K-1)); + + } else { + LocInterfaceProduct(IEdge, K) = + PressureMid(ICell1, K) * SpecVol(ICell1, K) * LayerThick(ICell1, K) + + PressureMid(ICell0, K) * SpecVol(ICell0, K) * LayerThick(ICell0, K) + + PressureMid(ICell1, K-1) * SpecVol(ICell1, K-1) * LayerThick(ICell1, K-1) + + PressureMid(ICell0, K-1) * SpecVol(ICell0, K-1) * LayerThick(ICell0, K-1); + LocInterfaceProduct(IEdge, K) /= (LayerThick(ICell1, K) + LayerThick(ICell0, K) + + LayerThick(ICell1, K-1) + LayerThick(ICell0, K-1));} + + LocInterfaceProduct(IEdge, K) *= (ZInterface(ICell1, K) - ZInterface(ICell0, K)) / LocDcEdge(IEdge); + }); + +} // end compute interface product + //------------------------------------------------------------------------------ // Constructor for centered pressure gradient functor PressureGradCentered::PressureGradCentered(const HorzMesh *Mesh) diff --git a/components/omega/src/ocn/PGrad.h b/components/omega/src/ocn/PGrad.h index e104db110321..f17813f3fc4e 100644 --- a/components/omega/src/ocn/PGrad.h +++ b/components/omega/src/ocn/PGrad.h @@ -32,12 +32,10 @@ class PressureGradCentered { // vertical chunk. This appends results into the Tend array (in-place). KOKKOS_FUNCTION void operator()(const Array2DReal &Tend, I4 IEdge, I4 KChunk, const Array2DReal &PressureMid, - const Array2DReal &PressureInterface, const Array2DReal &Geopotential, const Array2DReal &LayerThick, - const Array2DReal &ZInterface, - const Array2DReal &SpecVol, - const Array2DReal &SpecVolInterface) const { + const Array2DReal &InterfaceProduct, + const Array2DReal &SpecVol) const { const I4 KStart = KChunk * VecLength; const I4 ICell0 = CellsOnEdge(IEdge, 0); @@ -46,17 +44,6 @@ class PressureGradCentered { for (int KVec = 0; KVec < VecLength; ++KVec) { const I4 K = KStart + KVec; - - // Average quantities to edge - const Real PAlphaEdgeK = - 0.5_Real * - (PressureInterface(ICell1, K) * SpecVolInterface(ICell1, K) + - PressureInterface(ICell0, K) * SpecVolInterface(ICell0, K)); - const Real PAlphaEdgeKP1 = - 0.5_Real * (PressureInterface(ICell1, K + 1) * - SpecVolInterface(ICell1, K + 1) + - PressureInterface(ICell0, K + 1) * - SpecVolInterface(ICell0, K + 1)); const Real InvLayerThickEdge = 2.0_Real / (LayerThick(ICell1, K) + LayerThick(ICell0, K)); @@ -67,11 +54,8 @@ class PressureGradCentered { LayerThick(ICell0, K) * SpecVol(ICell0, K) * PressureMid(ICell0, K)) * InvDcEdge * InvLayerThickEdge; - Real InterfaceTerm = - (PAlphaEdgeK * (ZInterface(ICell1, K) - ZInterface(ICell0, K)) - - PAlphaEdgeKP1 * - (ZInterface(ICell1, K + 1) - ZInterface(ICell0, K + 1))) * - InvDcEdge * InvLayerThickEdge; + Real InterfaceTerm = (InterfaceProduct(IEdge, K) - InterfaceProduct(IEdge, K+1)) * + InvLayerThickEdge; Tend(IEdge, K) += EdgeMask(IEdge, K) * (-GeoTerm - PresTerm + InterfaceTerm); @@ -133,11 +117,19 @@ class PressureGrad { // Destructor ~PressureGrad(); + Array2DReal InterfaceProduct; + /// Compute pressure gradient tendencies and add into Tend array void computePressureGrad(Array2DReal &Tend, const OceanState *State, const VertCoord *VCoord, const Eos *EqState, const int TimeLevel); + void computeInterfaceProduct(const Array2DReal &PressureMid, + const Array2DReal &SpecVol, + const Array2DReal &LayerThick, + const Array2DReal &ZInterface); + + /// Create a new pressure gradient object and add to map static PressureGrad *create(const std::string &Name, const HorzMesh *Mesh, const VertCoord *VCoord, Config *Options); @@ -156,10 +148,11 @@ class PressureGrad { I4 NEdgesAll = 0; I4 NChunks = 0; I4 NVertLayers = 0; + I4 NVertLayersP1 = 0; // Data required for computation (stored copies of mesh/VCoord arrays) Array2DI4 CellsOnEdge; ///< cells surrounding each edge - Array1DReal DvEdge; ///< distance between cell centers across edge + Array1DReal DcEdge; ///< distance between cell centers across edge Array2DReal EdgeSignOnCell; ///< orientation of edge relative to cell // Instances of functors diff --git a/components/omega/test/ocn/PGradTest.cpp b/components/omega/test/ocn/PGradTest.cpp index ac66e6ee9b3f..5c9e9e84827e 100644 --- a/components/omega/test/ocn/PGradTest.cpp +++ b/components/omega/test/ocn/PGradTest.cpp @@ -111,15 +111,15 @@ int main(int argc, char *argv[]) { Eos *DefEos = Eos::getInstance(); Config *Options = Config::getOmegaConfig(); - I4 NVertLayers = 60; - Real dC = 30000.0_Real; // create arrays: Tend on edges, Pressure/Geopotential/SpecVol on cells - Array2DReal Tend("Tend", DefMesh->NEdgesSize, NVertLayers); - Array2DReal SpecVolOld("SpecVolOld", DefMesh->NCellsSize, NVertLayers); - Array2DReal PressureMidOld("PressureMidOld", DefMesh->NCellsSize, NVertLayers); + Array2DReal Tend("Tend", DefMesh->NEdgesSize, VCoord->NVertLayers); + Array2DReal SpecVolOld("SpecVolOld", DefMesh->NCellsSize, VCoord->NVertLayers); + Array2DReal PressureMidOld("PressureMidOld", DefMesh->NCellsSize, VCoord->NVertLayers); Array1DReal SurfacePressure("SurfacePressure", DefMesh->NCellsSize); + I4 NVertLayers = 60; + Real dC = 30000.0_Real; for (int refinement = 0; refinement < 3; ++refinement) { LOG_INFO("PGradTest: Starting refinement level {}", refinement); @@ -174,6 +174,7 @@ int main(int argc, char *argv[]) { auto &ZInterface = VCoord->ZInterface; auto &ZMid = VCoord->ZMid; Real tilt_factor = 0.495_Real; + //Real tilt_factor = 0.45_Real; parallelFor({DefMesh->NCellsAll}, KOKKOS_LAMBDA(int i) { ZInterface(i, NVertLayers) = ZBottom; SurfacePressure(i) = 0.0_Real; @@ -308,6 +309,7 @@ int main(int argc, char *argv[]) { // compute pressure gradient //PGradTest->computePressureGrad(Tend, DefState, VCoord, DefEos, TimeLevel); + deepCopy(Tend, 0.0_Real); DefPGrad->computePressureGrad(Tend, DefState, VCoord, DefEos, TimeLevel); //for (int i = 0; i < 2; ++i) { // for (int k = 0; k < NVertLayers; ++k) { @@ -315,17 +317,17 @@ int main(int argc, char *argv[]) { // } //} Real max_value = 0.0_Real; - parallelReduce({DefMesh->NEdgesAll, NVertLayers}, + parallelReduce({DefMesh->NEdgesAll, NVertLayers - 1}, KOKKOS_LAMBDA(int i, int k, Real &max) { - Real val = Kokkos::abs(Tend(i, k)); + Real val = Kokkos::abs(Tend(i + 1, k)); if (val > max) max = val; }, Kokkos::Max(max_value) ); Real sum_value = 0.0_Real; - parallelReduce({DefMesh->NEdgesAll, NVertLayers}, + parallelReduce({DefMesh->NEdgesAll, NVertLayers - 1}, KOKKOS_LAMBDA(int i, int k, Real &lsum) { - lsum += Tend(i,k) * Tend(i, k); + lsum += Tend(i + 1, k) * Tend(i + 1, k); }, Kokkos::Sum(sum_value) ); - LOG_INFO("refinement level {}: max |Tend| = {}, average Tend = {}", refinement, max_value, std::sqrt(sum_value) / (DefMesh->NCellsAll * NVertLayers)); + LOG_INFO("refinement level {}: max |Tend| = {}, average Tend = {}", refinement, max_value, std::sqrt(sum_value) / (DefMesh->NEdgesAll * (NVertLayers - 1))); dC = dC * 2.0_Real; NVertLayers = NVertLayers / 2; From 6f1b9df3d5504c6959801bb1e70001752d340966 Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Mon, 1 Dec 2025 14:07:36 -0800 Subject: [PATCH 10/58] Remove SpecVolInterface calculation --- components/omega/src/ocn/Eos.cpp | 91 ++++++------------------------ components/omega/src/ocn/Eos.h | 6 -- components/omega/src/ocn/PGrad.cpp | 1 - 3 files changed, 18 insertions(+), 80 deletions(-) diff --git a/components/omega/src/ocn/Eos.cpp b/components/omega/src/ocn/Eos.cpp index 5926a77318bf..1b49fe400eea 100644 --- a/components/omega/src/ocn/Eos.cpp +++ b/components/omega/src/ocn/Eos.cpp @@ -45,6 +45,24 @@ Eos::Eos(const std::string &Name, ///< [in] Name for eos object BruntVaisalaFreqSq = Array2DReal("BruntVaisalaFreqSq", Mesh->NCellsAll, VCoord->NVertLayers); + defineFields(); +} + +/// Destructor for Eos +Eos::~Eos() {} + +/// Instance management +Eos *Eos::Instance = nullptr; + +/// Get instance of Eos +Eos *Eos::getInstance() { return Instance; } + +/// Destroy instance of Eos +void Eos::destroyInstance() { + delete Instance; + Instance = nullptr; +} + /// Initializes the Eos (Equation of State) class and its options. /// it ASSUMES that HorzMesh was initialized and initializes the Eos class by /// using the default mesh, reading the config file, and setting parameters @@ -150,79 +168,6 @@ void Eos::computeSpecVol(const Array2DReal &ConservTemp, } } -/// Compute specific volume for all layer interfaces(no displacement) -void Eos::computeSpecVolInterface(const Array2DReal &ConservTemp, - const Array2DReal &AbsSalinity, - const Array2DReal &PressureInterface) { - OMEGA_SCOPE(LocSpecVolInterface, - SpecVolInterface); /// Create a local view for computation - OMEGA_SCOPE(LocComputeSpecVolLinear, - ComputeSpecVolLinear); /// Local view for linear EOS computation - OMEGA_SCOPE(LocComputeSpecVolTeos10, - ComputeSpecVolTeos10); /// Local view for TEOS-10 computation - deepCopy(LocSpecVolInterface, - 0); /// Initialize local specific volume to zero - - I4 KDisp = 0; /// No displacement in this case - - // Array2DReal ConservTempInterface("ConservTempInterface", NCellsAll, - // NVertLayers + 1); - // Array2DReal AbsSalinityInterface("AbsSalinityInterface", NCellsAll, - // NVertLayers + 1); - - // Compute interface values by averaging adjacent layer values - parallelFor( - "compute-interface-values", {NCellsAll, NVertLayers + 1}, - KOKKOS_LAMBDA(I4 ICell, I4 KInterface) { - if (KInterface == 0) { - // ConservTempInterface(ICell, KInterface) = - // ConservTemp(ICell, KInterface); - // AbsSalinityInterface(ICell, KInterface) = - // AbsSalinity(ICell, KInterface); - SpecVolInterface(ICell, KInterface) = SpecVol(ICell, KInterface); - } else if (KInterface == NVertLayers + 1) { - // ConservTempInterface(ICell, KInterface) = - // ConservTemp(ICell, KInterface - 1); - // AbsSalinityInterface(ICell, KInterface) = - // AbsSalinity(ICell, KInterface - 1); - SpecVolInterface(ICell, KInterface) = - SpecVol(ICell, KInterface - 1); - } else { - // ConservTempInterface(ICell, KInterface) = - // 0.5_Real * (ConservTemp(ICell, KInterface - 1) + - // ConservTemp(ICell, KInterface)); - // AbsSalinityInterface(ICell, KInterface) = - // 0.5_Real * (AbsSalinity(ICell, KInterface - 1) + - // AbsSalinity(ICell, KInterface)); - SpecVolInterface(ICell, KInterface) = - 0.5_Real * - (SpecVol(ICell, KInterface - 1) + SpecVol(ICell, KInterface)); - } - }); - - // int NChunksP1 = (NVertLayers + 1) / VecLength; - - ///// Dispatch to the correct EOS calculation - // if (EosChoice == EosType::LinearEos) { - // parallelFor( - // "eos-linear", {NCellsAll, NChunksP1}, - // KOKKOS_LAMBDA(I4 ICell, I4 KChunk) { - // LocComputeSpecVolLinear(LocSpecVolInterface, ICell, KChunk, - // ConservTempInterface, - // AbsSalinityInterface); - // }); - // } else if (EosChoice == EosType::Teos10Eos) { - // parallelFor( - // "eos-teos10", {NCellsAll, NChunksP1}, - // KOKKOS_LAMBDA(I4 ICell, I4 KChunk) { - // LocComputeSpecVolTeos10(LocSpecVolInterface, ICell, KChunk, - // ConservTempInterface, - // AbsSalinityInterface, PressureInterface, - // KDisp); - // }); - // } -} - /// Compute displaced specific volume (for vertical displacement) void Eos::computeSpecVolDisp(const Array2DReal &ConservTemp, const Array2DReal &AbsSalinity, diff --git a/components/omega/src/ocn/Eos.h b/components/omega/src/ocn/Eos.h index 0807b8a1b484..4f10d75c8fec 100644 --- a/components/omega/src/ocn/Eos.h +++ b/components/omega/src/ocn/Eos.h @@ -542,7 +542,6 @@ class Eos { EosType EosChoice; ///< Current EOS type in use Array2DReal SpecVol; ///< Specific volume field at level centers - Array2DReal SpecVolInterface; ///< Specific volume field at interfaces Array2DReal SpecVolDisplaced; ///< Displaced specific volume field Array2DReal BruntVaisalaFreqSq; ///< Squared Brunt-Vaisala frequency field @@ -569,11 +568,6 @@ class Eos { const Array2DReal &AbsSalinity, const Array2DReal &Pressure, const Array2DReal &SpecVol); - /// Compute specific volume at interfaces for all cells/layers+1 - void computeSpecVolInterface(const Array2DReal &ConservTemp, - const Array2DReal &AbsSalinity, - const Array2DReal &PressureInterface); - /// Initialize EOS from config and mesh static void init(); diff --git a/components/omega/src/ocn/PGrad.cpp b/components/omega/src/ocn/PGrad.cpp index 5fe768e5214b..ddad9d5b878f 100644 --- a/components/omega/src/ocn/PGrad.cpp +++ b/components/omega/src/ocn/PGrad.cpp @@ -163,7 +163,6 @@ void PressureGrad::computePressureGrad(Array2DReal &Tend, const Array2DReal &PressureInterface = VCoord->PressureInterface; const Array2DReal &Geopotential = VCoord->GeopotentialMid; const Array2DReal &SpecVol = EqState->SpecVol; - const Array2DReal &SpecVolInterface = EqState->SpecVolInterface; const Array2DReal &ZInterface = VCoord->ZInterface; Array2DReal LayerThick; State->getLayerThickness(LayerThick, TimeLevel); From 3dcad106aa01ad6f6e04ee26cd69facd0bafbbf3 Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Mon, 1 Dec 2025 13:18:20 -0800 Subject: [PATCH 11/58] Increase number of refinements in convergence test loop --- components/omega/test/ocn/PGradTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/omega/test/ocn/PGradTest.cpp b/components/omega/test/ocn/PGradTest.cpp index 5c9e9e84827e..c59a50af73b5 100644 --- a/components/omega/test/ocn/PGradTest.cpp +++ b/components/omega/test/ocn/PGradTest.cpp @@ -120,7 +120,7 @@ int main(int argc, char *argv[]) { I4 NVertLayers = 60; Real dC = 30000.0_Real; - for (int refinement = 0; refinement < 3; ++refinement) { + for (int refinement = 0; refinement < 4; ++refinement) { LOG_INFO("PGradTest: Starting refinement level {}", refinement); VCoord->NVertLayers = NVertLayers; From 55c69de4f3757491ba7a44a89a1c44424b5ced96 Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Mon, 1 Dec 2025 13:19:03 -0800 Subject: [PATCH 12/58] Remove debug comments --- components/omega/test/ocn/PGradTest.cpp | 54 ------------------------- 1 file changed, 54 deletions(-) diff --git a/components/omega/test/ocn/PGradTest.cpp b/components/omega/test/ocn/PGradTest.cpp index c59a50af73b5..da15fd2ba9f8 100644 --- a/components/omega/test/ocn/PGradTest.cpp +++ b/components/omega/test/ocn/PGradTest.cpp @@ -104,7 +104,6 @@ int main(int argc, char *argv[]) { MachEnv *DefEnv = MachEnv::getDefault(); HorzMesh *DefMesh = HorzMesh::getDefault(); - //VertCoord *DefVCoord = VertCoord::getDefault(); VertCoord *VCoord = VertCoord::getDefault(); OceanState *DefState = OceanState::getDefault(); Decomp *DefDecomp = Decomp::getDefault(); @@ -140,9 +139,6 @@ int main(int argc, char *argv[]) { CellsOnEdge(e, 0)= 0; CellsOnEdge(e, 1)= 1; DcEdge(e) = dC; - //for (int k = 0; k < NVertLayers; ++k) { - // EdgeMask(e, k) = 1.0_Real; - //} }); // Fetch reference desnity from Config @@ -165,8 +161,6 @@ int main(int argc, char *argv[]) { Err = Tracers::getByName(Temp, TimeLevel, "Temperature"); Err = Tracers::getByName(Salinity, TimeLevel, "Salinity"); - - // set Z interface and mid-point locations Real ZBottom = -1000.0_Real; Real dZ = 2.0_Real * (-ZBottom / NVertLayers); @@ -191,24 +185,12 @@ int main(int argc, char *argv[]) { LOG_INFO("NVertLayers = {}", NVertLayers); LOG_INFO("dC = {}", dC); - //for (int i = 0; i < 2; ++i) { - // for (int k = 0; k <= NVertLayers; ++k) { - // LOG_INFO("ZInterface({}, {}) = {}", i, k, ZInterface(i, k)); - // } - //} - //LOG_INFO("NVertLayers = {}", NVertLayers); for (int i = 0; i < 2; ++i) { //for (int k = 0; k <= NVertLayers; ++k) { for (int k = 0; k < 2; ++k) { LOG_INFO("LayerThick({}, {}) = {}", i, k, LayerThick(i, k)); } } - //LOG_INFO("NVertLayers = {}", NVertLayers); - //for (int i = 0; i < 2; ++i) { - // for (int k = 0; k <= NVertLayers; ++k) { - // LOG_INFO("ZMid({}, {}) = {}", i, k, ZMid(i, k)); - // } - //} // set simple temperature and salinity profiles auto &SpecVol = DefEos->SpecVol; @@ -228,26 +210,10 @@ int main(int argc, char *argv[]) { SpecVolOld(i, k) = SpecVol(i, k); }); - //for (int i = 0; i < 2; ++i) { - // for (int k = 0; k < NVertLayers; ++k) { - // LOG_INFO("Temp({}, {}) = {}", i, k, Temp(i, k)); - // } - //} - //for (int i = 0; i < 2; ++i) { - // for (int k = 0; k < NVertLayers; ++k) { - // LOG_INFO("Salinity({}, {}) = {}", i, k, Salinity(i, k)); - // } - //} - // iterate to converge SpecVol auto &PressureMid = VCoord->PressureMid; VCoord->computePressure(LayerThick, SurfacePressure); deepCopy(PressureMidOld, PressureMid); - //for (int i = 0; i < 2; ++i) { - // for (int k = 0; k < NVertLayers; ++k) { - // LOG_INFO("PressureMid({}, {}) = {}, PressureMidOld({}, {}) = {}", i, k, PressureMid(i, k), i, k, PressureMidOld(i, k)); - // } - //} for (int iteration = 0; iteration < 15; ++iteration) { // compute specific volume from EOS @@ -272,18 +238,12 @@ int main(int argc, char *argv[]) { LOG_INFO("converged: max diff = {}", max_value); break; } else { - //LOG_INFO("max diff = {}", max_value); parallelFor({DefMesh->NCellsAll, NVertLayers}, KOKKOS_LAMBDA(int i, int k) { SpecVolOld(i, k) = SpecVol(i, k); }); } } - //for (int i = 0; i < 2; ++i) { - // for (int k = 0; k < NVertLayers; ++k) { - // LOG_INFO("SpecVol({}, {}) = {}", i, k, SpecVol(i, k)); - // } - //} // compute geopotential //VCoord->computeZHeight(LayerThick, SpecVol); @@ -298,24 +258,10 @@ int main(int argc, char *argv[]) { if (!DefPGrad) { LOG_INFO("PGrad: default instance not created by init"); } - //auto PGradTest = PressureGrad::create("Test", DefMesh, VCoord, Options); - - //for (int i = 0; i < 2; ++i) { - // for (int k = 0; k < NVertLayers; ++k) { - // Tend(i, k) = 0.0_Real; - // LOG_INFO("Tend({}, {}) = {}", i, k, Tend(i, k)); - // } - //} // compute pressure gradient - //PGradTest->computePressureGrad(Tend, DefState, VCoord, DefEos, TimeLevel); deepCopy(Tend, 0.0_Real); DefPGrad->computePressureGrad(Tend, DefState, VCoord, DefEos, TimeLevel); - //for (int i = 0; i < 2; ++i) { - // for (int k = 0; k < NVertLayers; ++k) { - // LOG_INFO("Tend({}, {}) = {}", i, k, Tend(i, k)); - // } - //} Real max_value = 0.0_Real; parallelReduce({DefMesh->NEdgesAll, NVertLayers - 1}, KOKKOS_LAMBDA(int i, int k, Real &max) { From 19698f2c0df8ecf97d2b264e9dadcc95c2431974 Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Mon, 1 Dec 2025 13:47:38 -0800 Subject: [PATCH 13/58] Run PGrad test in serial --- components/omega/test/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/omega/test/CMakeLists.txt b/components/omega/test/CMakeLists.txt index c2dceb09c5a1..301a04ede49c 100644 --- a/components/omega/test/CMakeLists.txt +++ b/components/omega/test/CMakeLists.txt @@ -354,7 +354,7 @@ add_omega_test( PGRAD_TEST testPGrad.exe ocn/PGradTest.cpp - "-n;8" + "-n;1" ) ################## From 96574767bf71d2b77fd310b3eb5ebb5876424035 Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Tue, 2 Dec 2025 10:26:29 -0800 Subject: [PATCH 14/58] Fix error calculation to exclude top and bottom levels --- components/omega/test/ocn/PGradTest.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/components/omega/test/ocn/PGradTest.cpp b/components/omega/test/ocn/PGradTest.cpp index da15fd2ba9f8..55b7d25bccfb 100644 --- a/components/omega/test/ocn/PGradTest.cpp +++ b/components/omega/test/ocn/PGradTest.cpp @@ -186,7 +186,6 @@ int main(int argc, char *argv[]) { LOG_INFO("NVertLayers = {}", NVertLayers); LOG_INFO("dC = {}", dC); for (int i = 0; i < 2; ++i) { - //for (int k = 0; k <= NVertLayers; ++k) { for (int k = 0; k < 2; ++k) { LOG_INFO("LayerThick({}, {}) = {}", i, k, LayerThick(i, k)); } @@ -263,17 +262,17 @@ int main(int argc, char *argv[]) { deepCopy(Tend, 0.0_Real); DefPGrad->computePressureGrad(Tend, DefState, VCoord, DefEos, TimeLevel); Real max_value = 0.0_Real; - parallelReduce({DefMesh->NEdgesAll, NVertLayers - 1}, + parallelReduce({DefMesh->NEdgesAll, NVertLayers - 2}, KOKKOS_LAMBDA(int i, int k, Real &max) { Real val = Kokkos::abs(Tend(i + 1, k)); if (val > max) max = val; }, Kokkos::Max(max_value) ); Real sum_value = 0.0_Real; - parallelReduce({DefMesh->NEdgesAll, NVertLayers - 1}, + parallelReduce({DefMesh->NEdgesAll, NVertLayers - 2}, KOKKOS_LAMBDA(int i, int k, Real &lsum) { lsum += Tend(i + 1, k) * Tend(i + 1, k); }, Kokkos::Sum(sum_value) ); - LOG_INFO("refinement level {}: max |Tend| = {}, average Tend = {}", refinement, max_value, std::sqrt(sum_value) / (DefMesh->NEdgesAll * (NVertLayers - 1))); + LOG_INFO("refinement level {}: max |Tend| = {}, average Tend = {}", refinement, max_value, std::sqrt(sum_value) / (DefMesh->NEdgesAll * (NVertLayers - 2))); dC = dC * 2.0_Real; NVertLayers = NVertLayers / 2; From 7bebc868b55afbe7d7846aba36445e351b647c42 Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Tue, 2 Dec 2025 10:32:41 -0800 Subject: [PATCH 15/58] Convert pressure to Pa in EosTest --- components/omega/test/ocn/EosTest.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/components/omega/test/ocn/EosTest.cpp b/components/omega/test/ocn/EosTest.cpp index 5202594abedb..1b507781e8e9 100644 --- a/components/omega/test/ocn/EosTest.cpp +++ b/components/omega/test/ocn/EosTest.cpp @@ -57,6 +57,7 @@ const Real P = 1000.0; // Pressure in dbar const I4 KDisp = 1; // Displate parcel to K=1 for TEOS-10 eos const Real RTol = 1e-10; // Relative tolerance for isApprox checks +double PaToDBar = 1e-4; // Conversion factor from Pa to dbar /// The initialization routine for Eos testing. It calls various /// init routines, including the creation of the default decomposition. @@ -119,7 +120,7 @@ void testEosLinear() { /// Use Kokkos::deep_copy to fill the entire view with the ref value deepCopy(SArray, Sa); deepCopy(TArray, Ct); - deepCopy(PArray, P); + deepCopy(PArray, P / PaToDBar); deepCopy(TestEos->SpecVol, 0.0); /// Compute specific volume @@ -189,7 +190,7 @@ void testEosLinearDisplaced() { /// Use Kokkos::deep_copy to fill the entire view with the ref value deepCopy(SArray, Sa); deepCopy(TArray, Ct); - deepCopy(PArray, P); + deepCopy(PArray, P / PaToDBar); deepCopy(TestEos->SpecVolDisplaced, 0.0); /// Compute displaced specific volume @@ -384,7 +385,7 @@ void testEosTeos10() { /// Use Kokkos::deep_copy to fill the entire view with the ref value deepCopy(SArray, Sa); deepCopy(TArray, Ct); - deepCopy(PArray, P); + deepCopy(PArray, P / PaToDBar); deepCopy(TestEos->SpecVol, 0.0); /// Compute specific volume @@ -454,7 +455,7 @@ void testEosTeos10Displaced() { /// Use Kokkos::deep_copy to fill the entire view with the ref value deepCopy(SArray, Sa); deepCopy(TArray, Ct); - deepCopy(PArray, P); + deepCopy(PArray, P / PaToDBar); deepCopy(TestEos->SpecVolDisplaced, 0.0); /// Compute displaced specific volume From 730be5bf6eb0683fd81c7c1f262099202cbaa9c3 Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Tue, 2 Dec 2025 13:30:27 -0800 Subject: [PATCH 16/58] Switch to hierarchical parallelism --- components/omega/src/ocn/PGrad.cpp | 193 ++++++++++++++++++------ components/omega/src/ocn/PGrad.h | 28 +++- components/omega/test/ocn/PGradTest.cpp | 4 + 3 files changed, 175 insertions(+), 50 deletions(-) diff --git a/components/omega/src/ocn/PGrad.cpp b/components/omega/src/ocn/PGrad.cpp index ddad9d5b878f..cfbe2b85b6b1 100644 --- a/components/omega/src/ocn/PGrad.cpp +++ b/components/omega/src/ocn/PGrad.cpp @@ -10,6 +10,7 @@ #include "Field.h" #include "HorzMesh.h" #include "VertCoord.h" +#include "OmegaKokkos.h" namespace OMEGA { @@ -75,8 +76,9 @@ PressureGrad::PressureGrad( const VertCoord *VCoord, ///< [in] Vertical coordinate Config *Options) ///< [in] Configuration options : CellsOnEdge(Mesh->CellsOnEdge), DcEdge(Mesh->DcEdge), - EdgeSignOnCell(Mesh->EdgeSignOnCell), CenteredPGrad(Mesh), - HighOrderPGrad(Mesh) { + EdgeSignOnCell(Mesh->EdgeSignOnCell), MinLayerEdgeBot(VCoord->MinLayerEdgeBot), + MaxLayerEdgeTop(VCoord->MaxLayerEdgeTop), CenteredPGrad(Mesh, VCoord), + HighOrderPGrad(Mesh, VCoord) { // store mesh sizes NEdgesAll = Mesh->NEdgesAll; @@ -158,6 +160,8 @@ void PressureGrad::computePressureGrad(Array2DReal &Tend, OMEGA_SCOPE(LocCenteredPGrad, CenteredPGrad); OMEGA_SCOPE(LocHighOrderPGrad, HighOrderPGrad); OMEGA_SCOPE(LocInterfaceProduct, InterfaceProduct); + OMEGA_SCOPE(LocMinLayerEdgeBot, MinLayerEdgeBot); + OMEGA_SCOPE(LocMaxLayerEdgeTop, MaxLayerEdgeTop); const Array2DReal &PressureMid = VCoord->PressureMid; const Array2DReal &PressureInterface = VCoord->PressureInterface; @@ -172,20 +176,51 @@ void PressureGrad::computePressureGrad(Array2DReal &Tend, computeInterfaceProduct(PressureMid, SpecVol, LayerThick, ZInterface); - parallelFor( - "pgrad-centered", {NEdgesAll, NChunks}, - KOKKOS_LAMBDA(I4 IEdge, I4 KChunk) { - LocCenteredPGrad(Tend, IEdge, KChunk, PressureMid, - Geopotential, LayerThick, - LocInterfaceProduct, SpecVol); + parallelForOuter( + "pgrad-centered", {NEdgesAll}, + KOKKOS_LAMBDA(I4 IEdge, const TeamMember &Team) { + const int KMin = LocMinLayerEdgeBot(IEdge); + const int KMax = LocMaxLayerEdgeTop(IEdge); + const int KRange = vertRange(KMin, KMax); + + parallelForInner( + Team, KRange, + INNER_LAMBDA(int KChunk) { + LocCenteredPGrad(Tend, IEdge, KChunk, PressureMid, + Geopotential, LayerThick, + LocInterfaceProduct, SpecVol); + }); }); + + //parallelFor( + // "pgrad-centered", {NEdgesAll, NChunks}, + // KOKKOS_LAMBDA(I4 IEdge, I4 KChunk) { + // LocCenteredPGrad(Tend, IEdge, KChunk, PressureMid, + // Geopotential, LayerThick, + // LocInterfaceProduct, SpecVol); + // }); } else { - parallelFor( - "pgrad-highorder", {NEdgesAll, NChunks}, - KOKKOS_LAMBDA(I4 IEdge, I4 KChunk) { - LocHighOrderPGrad(Tend, IEdge, KChunk, PressureMid, Geopotential, - SpecVol); + parallelForOuter( + "pgrad-highorder", {NEdgesAll}, + KOKKOS_LAMBDA(I4 IEdge, const TeamMember &Team) { + const int KMin = MinLayerEdgeBot(IEdge); + const int KMax = MaxLayerEdgeTop(IEdge); + const int KRange = vertRange(KMin, KMax); + + parallelForInner( + Team, KRange, + INNER_LAMBDA(int KChunk) { + LocHighOrderPGrad(Tend, IEdge, KChunk, PressureMid, + Geopotential, SpecVol); + }); }); + + //parallelFor( + // "pgrad-highorder", {NEdgesAll, NChunks}, + // KOKKOS_LAMBDA(I4 IEdge, I4 KChunk) { + // LocHighOrderPGrad(Tend, IEdge, KChunk, PressureMid, Geopotential, + // SpecVol); + // }); } } // end compute pressure gradient @@ -201,48 +236,120 @@ void PressureGrad::computeInterfaceProduct( OMEGA_SCOPE(LocCellsOnEdge, CellsOnEdge); OMEGA_SCOPE(LocDcEdge, DcEdge); OMEGA_SCOPE(LocInterfaceProduct, InterfaceProduct); - parallelFor( - "compute-interface-product", {NEdgesAll, NVertLayersP1}, - KOKKOS_LAMBDA(I4 IEdge, I4 K) { + OMEGA_SCOPE(LocMinLayerEdgeBot, MinLayerEdgeBot); + OMEGA_SCOPE(LocMaxLayerEdgeTop, MaxLayerEdgeTop); + parallelForOuter( + "compute-interface-product", {NEdgesAll}, + KOKKOS_LAMBDA(I4 IEdge, const TeamMember &Team) { + const int KMin = LocMinLayerEdgeBot(IEdge); + const int KMax = LocMaxLayerEdgeTop(IEdge); + const I4 ICell0 = LocCellsOnEdge(IEdge, 0); const I4 ICell1 = LocCellsOnEdge(IEdge, 1); - if (K == 0) { - LocInterfaceProduct(IEdge, K) = - PressureMid(ICell1, K) * SpecVol(ICell1, K) * LayerThick(ICell1, K) + - PressureMid(ICell0, K) * SpecVol(ICell0, K) * LayerThick(ICell0, K); - LocInterfaceProduct(IEdge, K) /= (LayerThick(ICell1, K) + LayerThick(ICell0, K)); - - } else if (K == NVertLayers) { - LocInterfaceProduct(IEdge, K) = - PressureMid(ICell1, K-1) * SpecVol(ICell1, K-1) * LayerThick(ICell1, K-1) + - PressureMid(ICell0, K-1) * SpecVol(ICell0, K-1) * LayerThick(ICell0, K-1); - LocInterfaceProduct(IEdge, K) /= (LayerThick(ICell1, K-1) + LayerThick(ICell0, K-1)); - - } else { - LocInterfaceProduct(IEdge, K) = - PressureMid(ICell1, K) * SpecVol(ICell1, K) * LayerThick(ICell1, K) + - PressureMid(ICell0, K) * SpecVol(ICell0, K) * LayerThick(ICell0, K) + - PressureMid(ICell1, K-1) * SpecVol(ICell1, K-1) * LayerThick(ICell1, K-1) + - PressureMid(ICell0, K-1) * SpecVol(ICell0, K-1) * LayerThick(ICell0, K-1); - LocInterfaceProduct(IEdge, K) /= (LayerThick(ICell1, K) + LayerThick(ICell0, K) + - LayerThick(ICell1, K-1) + LayerThick(ICell0, K-1));} - - LocInterfaceProduct(IEdge, K) *= (ZInterface(ICell1, K) - ZInterface(ICell0, K)) / LocDcEdge(IEdge); - }); + LocInterfaceProduct(IEdge, KMin) = + PressureMid(ICell1, KMin) * SpecVol(ICell1, KMin) * + LayerThick(ICell1, KMin) + + PressureMid(ICell0, KMin) * SpecVol(ICell0, KMin) * + LayerThick(ICell0, KMin); + LocInterfaceProduct(IEdge, KMin) /= + (LayerThick(ICell1, KMin) + LayerThick(ICell0, KMin)); + + LocInterfaceProduct(IEdge, KMin) *= + (ZInterface(ICell1, KMin) - ZInterface(ICell0, KMin)) / + LocDcEdge(IEdge); + + const int KRange = vertRange(KMin + 1, KMax - 1); + parallelForInner( + Team, KRange, + INNER_LAMBDA(int KChunk) { + const I4 KStart = chunkStart(KChunk, KMin + 1); + const I4 KLen = chunkLength(KChunk, KStart, KMax - 1); + + + for (int KVec = 0; KVec < KLen; ++KVec) { + const I4 K = KStart + KVec; + + LocInterfaceProduct(IEdge, K) = + PressureMid(ICell1, K) * SpecVol(ICell1, K) * + LayerThick(ICell1, K) + + PressureMid(ICell0, K) * SpecVol(ICell0, K) * + LayerThick(ICell0, K) + + PressureMid(ICell1, K - 1) * SpecVol(ICell1, K - 1) * + LayerThick(ICell1, K - 1) + + PressureMid(ICell0, K - 1) * SpecVol(ICell0, K - 1) * + LayerThick(ICell0, K - 1); + LocInterfaceProduct(IEdge, K) /= + (LayerThick(ICell1, K) + LayerThick(ICell0, K) + + LayerThick(ICell1, K - 1) + + LayerThick(ICell0, K - 1)); + + LocInterfaceProduct(IEdge, K) *= + (ZInterface(ICell1, K) - ZInterface(ICell0, K)) / + LocDcEdge(IEdge); + } + }); + + LocInterfaceProduct(IEdge, KMax) = + PressureMid(ICell1, KMax - 1) * SpecVol(ICell1, KMax - 1) * + LayerThick(ICell1, KMax - 1) + + PressureMid(ICell0, KMax - 1) * SpecVol(ICell0, KMax - 1) * + LayerThick(ICell0, KMax - 1); + LocInterfaceProduct(IEdge, KMax) /= + (LayerThick(ICell1, KMax - 1) + + LayerThick(ICell0, KMax - 1)); + LocInterfaceProduct(IEdge, KMax) *= + (ZInterface(ICell1, KMax) - ZInterface(ICell0, KMax)) / + LocDcEdge(IEdge); + }); + //parallelFor( + // "compute-interface-product", {NEdgesAll, NVertLayersP1}, + // KOKKOS_LAMBDA(I4 IEdge, I4 K) { + // const I4 ICell0 = LocCellsOnEdge(IEdge, 0); + // const I4 ICell1 = LocCellsOnEdge(IEdge, 1); + + // if (K == 0) { + // LocInterfaceProduct(IEdge, K) = + // PressureMid(ICell1, K) * SpecVol(ICell1, K) * LayerThick(ICell1, K) + + // PressureMid(ICell0, K) * SpecVol(ICell0, K) * LayerThick(ICell0, K); + // LocInterfaceProduct(IEdge, K) /= (LayerThick(ICell1, K) + LayerThick(ICell0, K)); + + // } else if (K == NVertLayers) { + // LocInterfaceProduct(IEdge, K) = + // PressureMid(ICell1, K-1) * SpecVol(ICell1, K-1) * LayerThick(ICell1, K-1) + + // PressureMid(ICell0, K-1) * SpecVol(ICell0, K-1) * LayerThick(ICell0, K-1); + // LocInterfaceProduct(IEdge, K) /= (LayerThick(ICell1, K-1) + LayerThick(ICell0, K-1)); + + // } else { + // LocInterfaceProduct(IEdge, K) = + // PressureMid(ICell1, K) * SpecVol(ICell1, K) * LayerThick(ICell1, K) + + // PressureMid(ICell0, K) * SpecVol(ICell0, K) * LayerThick(ICell0, K) + + // PressureMid(ICell1, K-1) * SpecVol(ICell1, K-1) * LayerThick(ICell1, K-1) + + // PressureMid(ICell0, K-1) * SpecVol(ICell0, K-1) * LayerThick(ICell0, K-1); + // LocInterfaceProduct(IEdge, K) /= (LayerThick(ICell1, K) + LayerThick(ICell0, K) + + // LayerThick(ICell1, K-1) + LayerThick(ICell0, K-1));} + + // LocInterfaceProduct(IEdge, K) *= (ZInterface(ICell1, K) - ZInterface(ICell0, K)) / LocDcEdge(IEdge); + // }); } // end compute interface product //------------------------------------------------------------------------------ // Constructor for centered pressure gradient functor -PressureGradCentered::PressureGradCentered(const HorzMesh *Mesh) +PressureGradCentered::PressureGradCentered(const HorzMesh *Mesh, ///< [in] Horizontal mesh + const VertCoord *VCoord ///< [in] Vertical coordinate + ) : CellsOnEdge(Mesh->CellsOnEdge), DcEdge(Mesh->DcEdge), - EdgeMask(Mesh->EdgeMask) {} + EdgeMask(Mesh->EdgeMask), MinLayerEdgeBot(VCoord->MinLayerEdgeBot), + MaxLayerEdgeTop(VCoord->MaxLayerEdgeTop) {} //------------------------------------------------------------------------------ // Constructor for high order pressure gradient functor -PressureGradHighOrder::PressureGradHighOrder(const HorzMesh *Mesh) +PressureGradHighOrder::PressureGradHighOrder(const HorzMesh *Mesh, ///< [in] Horizontal mesh + const VertCoord *VCoord ///< [in] Vertical coordinate + ) : CellsOnEdge(Mesh->CellsOnEdge), DcEdge(Mesh->DcEdge), - EdgeMask(Mesh->EdgeMask) {} + EdgeMask(Mesh->EdgeMask), MinLayerEdgeBot(VCoord->MinLayerEdgeBot), + MaxLayerEdgeTop(VCoord->MaxLayerEdgeTop) {} } // namespace OMEGA diff --git a/components/omega/src/ocn/PGrad.h b/components/omega/src/ocn/PGrad.h index f17813f3fc4e..04d6254f754e 100644 --- a/components/omega/src/ocn/PGrad.h +++ b/components/omega/src/ocn/PGrad.h @@ -26,7 +26,9 @@ class PressureGradCentered { bool Enabled; /// constructor declaration - PressureGradCentered(const HorzMesh *Mesh); + PressureGradCentered(const HorzMesh *Mesh, ///< [in] Horizontal mesh + const VertCoord *VCoord ///< [in] Vertical coordinate + ); // Compute centered pressure gradient contribution for given edge and // vertical chunk. This appends results into the Tend array (in-place). @@ -36,13 +38,14 @@ class PressureGradCentered { const Array2DReal &LayerThick, const Array2DReal &InterfaceProduct, const Array2DReal &SpecVol) const { - const I4 KStart = KChunk * VecLength; - + const I4 KStart = chunkStart(KChunk, MinLayerEdgeBot(IEdge)); + const I4 KLen = chunkLength(KChunk, KStart, MaxLayerEdgeTop(IEdge)); + const I4 ICell0 = CellsOnEdge(IEdge, 0); const I4 ICell1 = CellsOnEdge(IEdge, 1); const Real InvDcEdge = 1.0_Real / DcEdge(IEdge); - for (int KVec = 0; KVec < VecLength; ++KVec) { + for (int KVec = 0; KVec < KLen; ++KVec) { const I4 K = KStart + KVec; const Real InvLayerThickEdge = 2.0_Real / (LayerThick(ICell1, K) + LayerThick(ICell0, K)); @@ -66,6 +69,8 @@ class PressureGradCentered { Array2DI4 CellsOnEdge; Array1DReal DcEdge; Array2DReal EdgeMask; + Array1DI4 MinLayerEdgeBot; + Array1DI4 MaxLayerEdgeTop; }; /// High-order pressure gradient functor (placeholder) @@ -74,15 +79,20 @@ class PressureGradHighOrder { bool Enabled; /// constructor declaration - PressureGradHighOrder(const HorzMesh *Mesh); + PressureGradHighOrder(const HorzMesh *Mesh, ///< [in] Horizontal mesh + const VertCoord *VCoord ///< [in] Vertical coordinate + ); KOKKOS_FUNCTION void operator()(const Array2DReal &Tend, I4 IEdge, I4 KChunk, const Array2DReal &Pressure, const Array2DReal &Geopotential, const Array2DReal &SpecVol) const { + // Placeholder: for now, no-op (future high-order implementation) - const I4 KStart = KChunk * VecLength; - for (int KVec = 0; KVec < VecLength; ++KVec) { + const I4 KStart = chunkStart(KChunk, MinLayerEdgeBot(IEdge)); + const I4 KLen = chunkLength(KChunk, KStart, MaxLayerEdgeTop(IEdge)); + + for (int KVec = 0; KVec < KLen; ++KVec) { const I4 K = KStart + KVec; Tend(IEdge, K) += 0.0_Real; } @@ -92,6 +102,8 @@ class PressureGradHighOrder { Array2DI4 CellsOnEdge; Array1DReal DcEdge; Array2DReal EdgeMask; + Array1DI4 MinLayerEdgeBot; + Array1DI4 MaxLayerEdgeTop; }; /// Pressure gradient manager class @@ -154,6 +166,8 @@ class PressureGrad { Array2DI4 CellsOnEdge; ///< cells surrounding each edge Array1DReal DcEdge; ///< distance between cell centers across edge Array2DReal EdgeSignOnCell; ///< orientation of edge relative to cell + Array1DI4 MinLayerEdgeBot; ///< min vertical layer on each edge + Array1DI4 MaxLayerEdgeTop; ///< max vertical layer on each edge // Instances of functors PressureGradCentered CenteredPGrad; diff --git a/components/omega/test/ocn/PGradTest.cpp b/components/omega/test/ocn/PGradTest.cpp index 55b7d25bccfb..23f39b03216e 100644 --- a/components/omega/test/ocn/PGradTest.cpp +++ b/components/omega/test/ocn/PGradTest.cpp @@ -127,9 +127,13 @@ int main(int argc, char *argv[]) { auto &MinLayerCell = VCoord->MinLayerCell; auto &MaxLayerCell = VCoord->MaxLayerCell; + auto &MinLayerEdgeBot = VCoord->MinLayerEdgeBot; + auto &MaxLayerEdgeTop = VCoord->MaxLayerEdgeTop; parallelFor({DefMesh->NCellsAll}, KOKKOS_LAMBDA(int i) { MinLayerCell(i) = 0; MaxLayerCell(i) = NVertLayers - 1; + MinLayerEdgeBot(i) = 0; + MaxLayerEdgeTop(i) = NVertLayers - 1; }); auto &CellsOnEdge = DefMesh->CellsOnEdge; From 75bea527f393c344eaaa5648cbbb951918ffc631 Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Thu, 4 Dec 2025 14:23:11 -0800 Subject: [PATCH 17/58] Fix bounds for bottom interface product --- components/omega/src/ocn/PGrad.cpp | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/components/omega/src/ocn/PGrad.cpp b/components/omega/src/ocn/PGrad.cpp index cfbe2b85b6b1..0a01de95d72a 100644 --- a/components/omega/src/ocn/PGrad.cpp +++ b/components/omega/src/ocn/PGrad.cpp @@ -259,13 +259,13 @@ void PressureGrad::computeInterfaceProduct( (ZInterface(ICell1, KMin) - ZInterface(ICell0, KMin)) / LocDcEdge(IEdge); - const int KRange = vertRange(KMin + 1, KMax - 1); + // Four-point average for interior interface products + const int KRange = vertRange(KMin + 1, KMax); parallelForInner( Team, KRange, INNER_LAMBDA(int KChunk) { const I4 KStart = chunkStart(KChunk, KMin + 1); - const I4 KLen = chunkLength(KChunk, KStart, KMax - 1); - + const I4 KLen = chunkLength(KChunk, KStart, KMax); for (int KVec = 0; KVec < KLen; ++KVec) { const I4 K = KStart + KVec; @@ -290,16 +290,17 @@ void PressureGrad::computeInterfaceProduct( } }); - LocInterfaceProduct(IEdge, KMax) = - PressureMid(ICell1, KMax - 1) * SpecVol(ICell1, KMax - 1) * - LayerThick(ICell1, KMax - 1) + - PressureMid(ICell0, KMax - 1) * SpecVol(ICell0, KMax - 1) * - LayerThick(ICell0, KMax - 1); - LocInterfaceProduct(IEdge, KMax) /= - (LayerThick(ICell1, KMax - 1) + - LayerThick(ICell0, KMax - 1)); - LocInterfaceProduct(IEdge, KMax) *= - (ZInterface(ICell1, KMax) - ZInterface(ICell0, KMax)) / + // Two-point average for bottom interface product + LocInterfaceProduct(IEdge, KMax + 1) = + PressureMid(ICell1, KMax) * SpecVol(ICell1, KMax) * + LayerThick(ICell1, KMax) + + PressureMid(ICell0, KMax) * SpecVol(ICell0, KMax) * + LayerThick(ICell0, KMax); + LocInterfaceProduct(IEdge, KMax + 1) /= + (LayerThick(ICell1, KMax) + + LayerThick(ICell0, KMax)); + LocInterfaceProduct(IEdge, KMax + 1) *= + (ZInterface(ICell1, KMax + 1) - ZInterface(ICell0, KMax + 1)) / LocDcEdge(IEdge); }); //parallelFor( From cb8ae86a66f6a447253fa385e2739e8c56f20ce7 Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Thu, 4 Dec 2025 14:25:18 -0800 Subject: [PATCH 18/58] Remove debug comments and cleanup --- components/omega/src/ocn/PGrad.cpp | 52 ++++--------------------- components/omega/test/ocn/PGradTest.cpp | 49 ++++++++++++++--------- 2 files changed, 37 insertions(+), 64 deletions(-) diff --git a/components/omega/src/ocn/PGrad.cpp b/components/omega/src/ocn/PGrad.cpp index 0a01de95d72a..8ed4cec67a4a 100644 --- a/components/omega/src/ocn/PGrad.cpp +++ b/components/omega/src/ocn/PGrad.cpp @@ -173,9 +173,11 @@ void PressureGrad::computePressureGrad(Array2DReal &Tend, if (PressureGradChoice == PressureGradType::Centered) { + // computes alpha*p*grad(z) term at edge interfaces computeInterfaceProduct(PressureMid, SpecVol, LayerThick, ZInterface); + // computes centered geopotential and pressure gradient tendency parallelForOuter( "pgrad-centered", {NEdgesAll}, KOKKOS_LAMBDA(I4 IEdge, const TeamMember &Team) { @@ -192,13 +194,6 @@ void PressureGrad::computePressureGrad(Array2DReal &Tend, }); }); - //parallelFor( - // "pgrad-centered", {NEdgesAll, NChunks}, - // KOKKOS_LAMBDA(I4 IEdge, I4 KChunk) { - // LocCenteredPGrad(Tend, IEdge, KChunk, PressureMid, - // Geopotential, LayerThick, - // LocInterfaceProduct, SpecVol); - // }); } else { parallelForOuter( "pgrad-highorder", {NEdgesAll}, @@ -215,29 +210,24 @@ void PressureGrad::computePressureGrad(Array2DReal &Tend, }); }); - //parallelFor( - // "pgrad-highorder", {NEdgesAll, NChunks}, - // KOKKOS_LAMBDA(I4 IEdge, I4 KChunk) { - // LocHighOrderPGrad(Tend, IEdge, KChunk, PressureMid, Geopotential, - // SpecVol); - // }); } } // end compute pressure gradient //------------------------------------------------------------------------------ -// Compute interface product needed for centered pressure gradient +// Compute the alpha*p*grad(z) product at edge interfaces that is +// needed for the centered pressure gradient void PressureGrad::computeInterfaceProduct( const Array2DReal &PressureMid, const Array2DReal &SpecVol, const Array2DReal &LayerThick, const Array2DReal &ZInterface) { - OMEGA_SCOPE(LocCellsOnEdge, CellsOnEdge); OMEGA_SCOPE(LocDcEdge, DcEdge); OMEGA_SCOPE(LocInterfaceProduct, InterfaceProduct); OMEGA_SCOPE(LocMinLayerEdgeBot, MinLayerEdgeBot); OMEGA_SCOPE(LocMaxLayerEdgeTop, MaxLayerEdgeTop); + parallelForOuter( "compute-interface-product", {NEdgesAll}, KOKKOS_LAMBDA(I4 IEdge, const TeamMember &Team) { @@ -247,6 +237,7 @@ void PressureGrad::computeInterfaceProduct( const I4 ICell0 = LocCellsOnEdge(IEdge, 0); const I4 ICell1 = LocCellsOnEdge(IEdge, 1); + // Two-point average for surface interface product LocInterfaceProduct(IEdge, KMin) = PressureMid(ICell1, KMin) * SpecVol(ICell1, KMin) * LayerThick(ICell1, KMin) + @@ -302,36 +293,7 @@ void PressureGrad::computeInterfaceProduct( LocInterfaceProduct(IEdge, KMax + 1) *= (ZInterface(ICell1, KMax + 1) - ZInterface(ICell0, KMax + 1)) / LocDcEdge(IEdge); - }); - //parallelFor( - // "compute-interface-product", {NEdgesAll, NVertLayersP1}, - // KOKKOS_LAMBDA(I4 IEdge, I4 K) { - // const I4 ICell0 = LocCellsOnEdge(IEdge, 0); - // const I4 ICell1 = LocCellsOnEdge(IEdge, 1); - - // if (K == 0) { - // LocInterfaceProduct(IEdge, K) = - // PressureMid(ICell1, K) * SpecVol(ICell1, K) * LayerThick(ICell1, K) + - // PressureMid(ICell0, K) * SpecVol(ICell0, K) * LayerThick(ICell0, K); - // LocInterfaceProduct(IEdge, K) /= (LayerThick(ICell1, K) + LayerThick(ICell0, K)); - - // } else if (K == NVertLayers) { - // LocInterfaceProduct(IEdge, K) = - // PressureMid(ICell1, K-1) * SpecVol(ICell1, K-1) * LayerThick(ICell1, K-1) + - // PressureMid(ICell0, K-1) * SpecVol(ICell0, K-1) * LayerThick(ICell0, K-1); - // LocInterfaceProduct(IEdge, K) /= (LayerThick(ICell1, K-1) + LayerThick(ICell0, K-1)); - - // } else { - // LocInterfaceProduct(IEdge, K) = - // PressureMid(ICell1, K) * SpecVol(ICell1, K) * LayerThick(ICell1, K) + - // PressureMid(ICell0, K) * SpecVol(ICell0, K) * LayerThick(ICell0, K) + - // PressureMid(ICell1, K-1) * SpecVol(ICell1, K-1) * LayerThick(ICell1, K-1) + - // PressureMid(ICell0, K-1) * SpecVol(ICell0, K-1) * LayerThick(ICell0, K-1); - // LocInterfaceProduct(IEdge, K) /= (LayerThick(ICell1, K) + LayerThick(ICell0, K) + - // LayerThick(ICell1, K-1) + LayerThick(ICell0, K-1));} - - // LocInterfaceProduct(IEdge, K) *= (ZInterface(ICell1, K) - ZInterface(ICell0, K)) / LocDcEdge(IEdge); - // }); + }); } // end compute interface product diff --git a/components/omega/test/ocn/PGradTest.cpp b/components/omega/test/ocn/PGradTest.cpp index 23f39b03216e..abd7cca99d89 100644 --- a/components/omega/test/ocn/PGradTest.cpp +++ b/components/omega/test/ocn/PGradTest.cpp @@ -117,6 +117,9 @@ int main(int argc, char *argv[]) { Array2DReal PressureMidOld("PressureMidOld", DefMesh->NCellsSize, VCoord->NVertLayers); Array1DReal SurfacePressure("SurfacePressure", DefMesh->NCellsSize); + I4 NEdgesAll = DefMesh->NEdgesAll; + I4 NCellsAll = DefMesh->NCellsAll; + I4 NVertLayers = 60; Real dC = 30000.0_Real; for (int refinement = 0; refinement < 4; ++refinement) { @@ -127,22 +130,26 @@ int main(int argc, char *argv[]) { auto &MinLayerCell = VCoord->MinLayerCell; auto &MaxLayerCell = VCoord->MaxLayerCell; - auto &MinLayerEdgeBot = VCoord->MinLayerEdgeBot; - auto &MaxLayerEdgeTop = VCoord->MaxLayerEdgeTop; - parallelFor({DefMesh->NCellsAll}, KOKKOS_LAMBDA(int i) { + parallelFor({NCellsAll}, KOKKOS_LAMBDA(int i) { MinLayerCell(i) = 0; MaxLayerCell(i) = NVertLayers - 1; + }); + + auto &MinLayerEdgeBot = VCoord->MinLayerEdgeBot; + auto &MaxLayerEdgeTop = VCoord->MaxLayerEdgeTop; + parallelFor({NEdgesAll}, KOKKOS_LAMBDA(int i) { MinLayerEdgeBot(i) = 0; - MaxLayerEdgeTop(i) = NVertLayers - 1; + MaxLayerEdgeTop(i) = NVertLayers - 1; + //MaxLayerEdgeTop(i) = 4; }); auto &CellsOnEdge = DefMesh->CellsOnEdge; auto &DcEdge = DefMesh->DcEdge; auto &EdgeMask = DefMesh->EdgeMask; - parallelFor({DefMesh->NEdgesAll}, KOKKOS_LAMBDA(int e) { - CellsOnEdge(e, 0)= 0; - CellsOnEdge(e, 1)= 1; - DcEdge(e) = dC; + parallelFor({NEdgesAll}, KOKKOS_LAMBDA(int i) { + CellsOnEdge(i, 0)= 0; + CellsOnEdge(i, 1)= 1; + DcEdge(i) = dC; }); // Fetch reference desnity from Config @@ -173,7 +180,7 @@ int main(int argc, char *argv[]) { auto &ZMid = VCoord->ZMid; Real tilt_factor = 0.495_Real; //Real tilt_factor = 0.45_Real; - parallelFor({DefMesh->NCellsAll}, KOKKOS_LAMBDA(int i) { + parallelFor({NCellsAll}, KOKKOS_LAMBDA(int i) { ZInterface(i, NVertLayers) = ZBottom; SurfacePressure(i) = 0.0_Real; BottomDepth(i) = 0.0_Real; @@ -197,7 +204,7 @@ int main(int argc, char *argv[]) { // set simple temperature and salinity profiles auto &SpecVol = DefEos->SpecVol; - parallelFor({DefMesh->NCellsAll, NVertLayers}, KOKKOS_LAMBDA(int i, int k) { + parallelFor({NCellsAll, NVertLayers}, KOKKOS_LAMBDA(int i, int k) { Real T0 = 30.0; Real TB = 5.0; Real S0 = 30.0; @@ -213,7 +220,7 @@ int main(int argc, char *argv[]) { SpecVolOld(i, k) = SpecVol(i, k); }); - // iterate to converge SpecVol + // Iterate to converge LayerThick, SpecVol, PressureMid auto &PressureMid = VCoord->PressureMid; VCoord->computePressure(LayerThick, SurfacePressure); deepCopy(PressureMidOld, PressureMid); @@ -224,13 +231,13 @@ int main(int argc, char *argv[]) { DefEos->computeSpecVol(Temp, Salinity, PressureMid); // compute psuedo thickness from specific volume - parallelFor({DefMesh->NCellsAll, NVertLayers}, KOKKOS_LAMBDA(int i, int k) { + parallelFor({NCellsAll, NVertLayers}, KOKKOS_LAMBDA(int i, int k) { LayerThick(i, k) = 1.0_Real / (SpecVol(i, k) * Density0) * (ZInterface(i, k) - ZInterface(i, k + 1)); }); // compute difference from previous iteration Real max_value = 0.0_Real; - parallelReduce({DefMesh->NCellsAll, NVertLayers}, + parallelReduce({NCellsAll, NVertLayers}, KOKKOS_LAMBDA(int i, int k, Real &max) { Real diff = Kokkos::abs(SpecVol(i, k) - SpecVolOld(i, k)); if (diff > max) max = diff; @@ -241,17 +248,20 @@ int main(int argc, char *argv[]) { LOG_INFO("converged: max diff = {}", max_value); break; } else { - parallelFor({DefMesh->NCellsAll, NVertLayers}, KOKKOS_LAMBDA(int i, int k) { + parallelFor({NCellsAll, NVertLayers}, KOKKOS_LAMBDA(int i, int k) { SpecVolOld(i, k) = SpecVol(i, k); }); } } + + // compute pressure once more with converged LayerThick + VCoord->computePressure(LayerThick, SurfacePressure); // compute geopotential //VCoord->computeZHeight(LayerThick, SpecVol); - Array1DReal SelfAttractionLoading("SelfAttractionLoading", DefMesh->NCellsAll); - Array1DReal TidalPotential("TidalPotential", DefMesh->NCellsAll); + Array1DReal SelfAttractionLoading("SelfAttractionLoading", NCellsAll); + Array1DReal TidalPotential("TidalPotential", NCellsAll); deepCopy(TidalPotential, 0.0_Real); deepCopy(SelfAttractionLoading, 0.0_Real); VCoord->computeGeopotential(TidalPotential, SelfAttractionLoading); @@ -266,18 +276,19 @@ int main(int argc, char *argv[]) { deepCopy(Tend, 0.0_Real); DefPGrad->computePressureGrad(Tend, DefState, VCoord, DefEos, TimeLevel); Real max_value = 0.0_Real; - parallelReduce({DefMesh->NEdgesAll, NVertLayers - 2}, + parallelReduce({NEdgesAll, NVertLayers - 2}, KOKKOS_LAMBDA(int i, int k, Real &max) { Real val = Kokkos::abs(Tend(i + 1, k)); if (val > max) max = val; }, Kokkos::Max(max_value) ); Real sum_value = 0.0_Real; - parallelReduce({DefMesh->NEdgesAll, NVertLayers - 2}, + parallelReduce({NEdgesAll, NVertLayers - 2}, KOKKOS_LAMBDA(int i, int k, Real &lsum) { lsum += Tend(i + 1, k) * Tend(i + 1, k); }, Kokkos::Sum(sum_value) ); - LOG_INFO("refinement level {}: max |Tend| = {}, average Tend = {}", refinement, max_value, std::sqrt(sum_value) / (DefMesh->NEdgesAll * (NVertLayers - 2))); + LOG_INFO("refinement level {}: max |Tend| = {}, average Tend = {}", refinement, max_value, std::sqrt(sum_value) / (NEdgesAll * (NVertLayers - 2))); + // coarsen for next iteration dC = dC * 2.0_Real; NVertLayers = NVertLayers / 2; From 315e9aa1036e8c8c7813c2b9c65fba0266bdd26e Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Fri, 5 Dec 2025 13:09:24 -0800 Subject: [PATCH 19/58] Clean up --- components/omega/src/ocn/PGrad.cpp | 2 ++ components/omega/src/ocn/PGrad.h | 53 ++++++++++++++++-------------- 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/components/omega/src/ocn/PGrad.cpp b/components/omega/src/ocn/PGrad.cpp index 8ed4cec67a4a..61d7183da66e 100644 --- a/components/omega/src/ocn/PGrad.cpp +++ b/components/omega/src/ocn/PGrad.cpp @@ -195,6 +195,8 @@ void PressureGrad::computePressureGrad(Array2DReal &Tend, }); } else { + + // computes high-order geopotential and pressure gradient tendency parallelForOuter( "pgrad-highorder", {NEdgesAll}, KOKKOS_LAMBDA(I4 IEdge, const TeamMember &Team) { diff --git a/components/omega/src/ocn/PGrad.h b/components/omega/src/ocn/PGrad.h index 04d6254f754e..bb02388204c9 100644 --- a/components/omega/src/ocn/PGrad.h +++ b/components/omega/src/ocn/PGrad.h @@ -20,12 +20,12 @@ namespace OMEGA { enum class PressureGradType { Centered, HighOrder1, HighOrder2 }; -/// Centered pressure gradient functor +// Centered pressure gradient functor class PressureGradCentered { public: bool Enabled; - /// constructor declaration + // constructor declaration PressureGradCentered(const HorzMesh *Mesh, ///< [in] Horizontal mesh const VertCoord *VCoord ///< [in] Vertical coordinate ); @@ -73,12 +73,12 @@ class PressureGradCentered { Array1DI4 MaxLayerEdgeTop; }; -/// High-order pressure gradient functor (placeholder) +// High-order pressure gradient functor (placeholder) class PressureGradHighOrder { public: bool Enabled; - /// constructor declaration + // constructor declaration PressureGradHighOrder(const HorzMesh *Mesh, ///< [in] Horizontal mesh const VertCoord *VCoord ///< [in] Vertical coordinate ); @@ -106,44 +106,38 @@ class PressureGradHighOrder { Array1DI4 MaxLayerEdgeTop; }; -/// Pressure gradient manager class +// Pressure gradient manager class class PressureGrad { public: - /// Initialize the default instance + // Initialize the default instance static void init(); - /// Deallocates arrays and deletes instance - static void clear(); - - /// Remove pressure gradient object by name - static void erase(const std::string &Name ///< [in] - ); + // Create a new pressure gradient object and add to map + static PressureGrad *create(const std::string &Name, const HorzMesh *Mesh, + const VertCoord *VCoord, Config *Options); - /// Get the default instance + // Get the default instance static PressureGrad *getDefault(); - /// Get instance by name + // Get instance by name static PressureGrad *get(const std::string &Name ///< [in] ); + // Deallocates arrays and deletes instance + static void clear(); + + // Remove pressure gradient object by name + static void erase(const std::string &Name ///< [in] + ); + // Destructor ~PressureGrad(); - Array2DReal InterfaceProduct; - - /// Compute pressure gradient tendencies and add into Tend array + // Compute pressure gradient tendencies and add into Tend array void computePressureGrad(Array2DReal &Tend, const OceanState *State, const VertCoord *VCoord, const Eos *EqState, const int TimeLevel); - void computeInterfaceProduct(const Array2DReal &PressureMid, - const Array2DReal &SpecVol, - const Array2DReal &LayerThick, - const Array2DReal &ZInterface); - - /// Create a new pressure gradient object and add to map - static PressureGrad *create(const std::string &Name, const HorzMesh *Mesh, - const VertCoord *VCoord, Config *Options); private: // Construct a new pressure gradient object @@ -173,12 +167,21 @@ class PressureGrad { PressureGradCentered CenteredPGrad; PressureGradHighOrder HighOrderPGrad; + // Array for interface product needed in centered pressure gradient + Array2DReal InterfaceProduct; + // Choice from config PressureGradType PressureGradChoice = PressureGradType::Centered; // Map of all pressure gradient objects by name static std::map> AllPGrads; + // Compute interface product needed for centered pressure gradient + void computeInterfaceProduct(const Array2DReal &PressureMid, + const Array2DReal &SpecVol, + const Array2DReal &LayerThick, + const Array2DReal &ZInterface); + }; // end class PressureGrad } // namespace OMEGA From 11114f7f9e7706f8502dbeca29808626a8d83da0 Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Mon, 8 Dec 2025 14:03:57 -0800 Subject: [PATCH 20/58] Fix for gradient in interface term --- components/omega/src/ocn/PGrad.cpp | 22 ++++++++++++---------- components/omega/src/ocn/PGrad.h | 4 +++- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/components/omega/src/ocn/PGrad.cpp b/components/omega/src/ocn/PGrad.cpp index 61d7183da66e..162fd14249db 100644 --- a/components/omega/src/ocn/PGrad.cpp +++ b/components/omega/src/ocn/PGrad.cpp @@ -167,7 +167,6 @@ void PressureGrad::computePressureGrad(Array2DReal &Tend, const Array2DReal &PressureInterface = VCoord->PressureInterface; const Array2DReal &Geopotential = VCoord->GeopotentialMid; const Array2DReal &SpecVol = EqState->SpecVol; - const Array2DReal &ZInterface = VCoord->ZInterface; Array2DReal LayerThick; State->getLayerThickness(LayerThick, TimeLevel); @@ -175,7 +174,7 @@ void PressureGrad::computePressureGrad(Array2DReal &Tend, // computes alpha*p*grad(z) term at edge interfaces computeInterfaceProduct(PressureMid, SpecVol, - LayerThick, ZInterface); + LayerThick, PressureInterface); // computes centered geopotential and pressure gradient tendency parallelForOuter( @@ -195,7 +194,7 @@ void PressureGrad::computePressureGrad(Array2DReal &Tend, }); } else { - + // computes high-order geopotential and pressure gradient tendency parallelForOuter( "pgrad-highorder", {NEdgesAll}, @@ -222,7 +221,7 @@ void PressureGrad::computeInterfaceProduct( const Array2DReal &PressureMid, const Array2DReal &SpecVol, const Array2DReal &LayerThick, - const Array2DReal &ZInterface) { + const Array2DReal &PInterface) { OMEGA_SCOPE(LocCellsOnEdge, CellsOnEdge); OMEGA_SCOPE(LocDcEdge, DcEdge); @@ -230,6 +229,9 @@ void PressureGrad::computeInterfaceProduct( OMEGA_SCOPE(LocMinLayerEdgeBot, MinLayerEdgeBot); OMEGA_SCOPE(LocMaxLayerEdgeTop, MaxLayerEdgeTop); + Real Gravity = 9.80616_Real; + Real Rho0 = 1026.0_Real; + parallelForOuter( "compute-interface-product", {NEdgesAll}, KOKKOS_LAMBDA(I4 IEdge, const TeamMember &Team) { @@ -249,8 +251,8 @@ void PressureGrad::computeInterfaceProduct( (LayerThick(ICell1, KMin) + LayerThick(ICell0, KMin)); LocInterfaceProduct(IEdge, KMin) *= - (ZInterface(ICell1, KMin) - ZInterface(ICell0, KMin)) / - LocDcEdge(IEdge); + -(PInterface(ICell1, KMin) - PInterface(ICell0, KMin)) / + (LocDcEdge(IEdge) * Gravity * Rho0); // Four-point average for interior interface products const int KRange = vertRange(KMin + 1, KMax); @@ -278,8 +280,8 @@ void PressureGrad::computeInterfaceProduct( LayerThick(ICell0, K - 1)); LocInterfaceProduct(IEdge, K) *= - (ZInterface(ICell1, K) - ZInterface(ICell0, K)) / - LocDcEdge(IEdge); + -(PInterface(ICell1, K) - PInterface(ICell0, K)) / + (LocDcEdge(IEdge) * Gravity * Rho0); } }); @@ -293,8 +295,8 @@ void PressureGrad::computeInterfaceProduct( (LayerThick(ICell1, KMax) + LayerThick(ICell0, KMax)); LocInterfaceProduct(IEdge, KMax + 1) *= - (ZInterface(ICell1, KMax + 1) - ZInterface(ICell0, KMax + 1)) / - LocDcEdge(IEdge); + -(PInterface(ICell1, KMax + 1) - PInterface(ICell0, KMax + 1)) / + (LocDcEdge(IEdge) * Gravity * Rho0); }); } // end compute interface product diff --git a/components/omega/src/ocn/PGrad.h b/components/omega/src/ocn/PGrad.h index bb02388204c9..623ffb17d9eb 100644 --- a/components/omega/src/ocn/PGrad.h +++ b/components/omega/src/ocn/PGrad.h @@ -61,7 +61,9 @@ class PressureGradCentered { InvLayerThickEdge; Tend(IEdge, K) += - EdgeMask(IEdge, K) * (-GeoTerm - PresTerm + InterfaceTerm); + EdgeMask(IEdge, K) * (GeoTerm + PresTerm - InterfaceTerm); + if (IEdge == 0) + LOG_INFO("IEdge {}, K {}: GeoTerm {}, PresTerm {}, InterfaceTerm {}, Tend {}", IEdge, K, GeoTerm, PresTerm, InterfaceTerm, Tend(IEdge, K)); } } From de7a550df6f431eb317f8a3367678c854bcb588c Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Mon, 15 Dec 2025 13:32:49 -0800 Subject: [PATCH 21/58] Add Liebnez terms for geopotential gradient --- components/omega/src/ocn/PGrad.cpp | 22 ++++++++++++++++++---- components/omega/src/ocn/PGrad.h | 21 ++++++++++++++++----- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/components/omega/src/ocn/PGrad.cpp b/components/omega/src/ocn/PGrad.cpp index 162fd14249db..55e23f3556a5 100644 --- a/components/omega/src/ocn/PGrad.cpp +++ b/components/omega/src/ocn/PGrad.cpp @@ -167,6 +167,8 @@ void PressureGrad::computePressureGrad(Array2DReal &Tend, const Array2DReal &PressureInterface = VCoord->PressureInterface; const Array2DReal &Geopotential = VCoord->GeopotentialMid; const Array2DReal &SpecVol = EqState->SpecVol; + const Array2DReal &ZInterface = VCoord->ZInterface; + const Array2DReal &ZMid = VCoord->ZMid; Array2DReal LayerThick; State->getLayerThickness(LayerThick, TimeLevel); @@ -174,7 +176,8 @@ void PressureGrad::computePressureGrad(Array2DReal &Tend, // computes alpha*p*grad(z) term at edge interfaces computeInterfaceProduct(PressureMid, SpecVol, - LayerThick, PressureInterface); + LayerThick, PressureInterface, + ZMid, Geopotential); // computes centered geopotential and pressure gradient tendency parallelForOuter( @@ -189,7 +192,7 @@ void PressureGrad::computePressureGrad(Array2DReal &Tend, INNER_LAMBDA(int KChunk) { LocCenteredPGrad(Tend, IEdge, KChunk, PressureMid, Geopotential, LayerThick, - LocInterfaceProduct, SpecVol); + LocInterfaceProduct, SpecVol, ZMid); }); }); @@ -221,7 +224,10 @@ void PressureGrad::computeInterfaceProduct( const Array2DReal &PressureMid, const Array2DReal &SpecVol, const Array2DReal &LayerThick, - const Array2DReal &PInterface) { + const Array2DReal &PInterface, + const Array2DReal &ZMid, + const Array2DReal &Geopotential + ) { OMEGA_SCOPE(LocCellsOnEdge, CellsOnEdge); OMEGA_SCOPE(LocDcEdge, DcEdge); @@ -246,7 +252,9 @@ void PressureGrad::computeInterfaceProduct( PressureMid(ICell1, KMin) * SpecVol(ICell1, KMin) * LayerThick(ICell1, KMin) + PressureMid(ICell0, KMin) * SpecVol(ICell0, KMin) * - LayerThick(ICell0, KMin); + LayerThick(ICell0, KMin) + + Geopotential(ICell1, KMin) + Geopotential(ICell0, KMin) - + Gravity * (ZMid(ICell1, KMin) + ZMid(ICell0, KMin)); LocInterfaceProduct(IEdge, KMin) /= (LayerThick(ICell1, KMin) + LayerThick(ICell0, KMin)); @@ -274,6 +282,10 @@ void PressureGrad::computeInterfaceProduct( LayerThick(ICell1, K - 1) + PressureMid(ICell0, K - 1) * SpecVol(ICell0, K - 1) * LayerThick(ICell0, K - 1); + Geopotential(ICell1, K) + Geopotential(ICell0, K) + + Geopotential(ICell1, K - 1) + Geopotential(ICell0, K - 1) - + Gravity * (ZMid(ICell1, K) + ZMid(ICell0, K) + + ZMid(ICell1, K - 1) + ZMid(ICell0, K - 1)); LocInterfaceProduct(IEdge, K) /= (LayerThick(ICell1, K) + LayerThick(ICell0, K) + LayerThick(ICell1, K - 1) + @@ -291,6 +303,8 @@ void PressureGrad::computeInterfaceProduct( LayerThick(ICell1, KMax) + PressureMid(ICell0, KMax) * SpecVol(ICell0, KMax) * LayerThick(ICell0, KMax); + Geopotential(ICell1, KMax) + Geopotential(ICell0, KMax) - + Gravity * (ZMid(ICell1, KMax) + ZMid(ICell0, KMax)); LocInterfaceProduct(IEdge, KMax + 1) /= (LayerThick(ICell1, KMax) + LayerThick(ICell0, KMax)); diff --git a/components/omega/src/ocn/PGrad.h b/components/omega/src/ocn/PGrad.h index 623ffb17d9eb..e28e0cff4c6f 100644 --- a/components/omega/src/ocn/PGrad.h +++ b/components/omega/src/ocn/PGrad.h @@ -37,21 +37,30 @@ class PressureGradCentered { const Array2DReal &Geopotential, const Array2DReal &LayerThick, const Array2DReal &InterfaceProduct, - const Array2DReal &SpecVol) const { + const Array2DReal &SpecVol, + const Array2DReal &ZMid) const { + const I4 KStart = chunkStart(KChunk, MinLayerEdgeBot(IEdge)); const I4 KLen = chunkLength(KChunk, KStart, MaxLayerEdgeTop(IEdge)); const I4 ICell0 = CellsOnEdge(IEdge, 0); const I4 ICell1 = CellsOnEdge(IEdge, 1); const Real InvDcEdge = 1.0_Real / DcEdge(IEdge); + Real Rho0 = 1026.0_Real; + Real Gravity = 9.80616_Real; for (int KVec = 0; KVec < KLen; ++KVec) { const I4 K = KStart + KVec; const Real InvLayerThickEdge = 2.0_Real / (LayerThick(ICell1, K) + LayerThick(ICell0, K)); + //Real RhoEdge = 1.0_Real / (0.5_Real*(SpecVol(ICell1, K) + SpecVol(ICell0, K))); + //Real GeoEdgeK = (ZInterface(ICell1, K) + ZInterface(ICell0, K)) * 0.5_Real; + //Real GeoEdgeKp1 = (ZInterface(ICell1, K+1) + ZInterface(ICell0, K+1)) * 0.5_Real; Real GeoTerm = - (Geopotential(ICell1, K) - Geopotential(ICell0, K)) * InvDcEdge; + (LayerThick(ICell1, K) * (Geopotential(ICell1, K) - Gravity * ZMid(ICell1, K)) - + LayerThick(ICell0, K) * (Geopotential(ICell0, K) - Gravity * ZMid(ICell0, K))) * InvDcEdge; + //GeoTerm *= (1.0_Real - (RhoEdge / Rho0)*(GeoEdgeK - GeoEdgeKp1) * InvLayerThickEdge); Real PresTerm = (LayerThick(ICell1, K) * SpecVol(ICell1, K) * PressureMid(ICell1, K) - LayerThick(ICell0, K) * SpecVol(ICell0, K) * @@ -180,9 +189,11 @@ class PressureGrad { // Compute interface product needed for centered pressure gradient void computeInterfaceProduct(const Array2DReal &PressureMid, - const Array2DReal &SpecVol, - const Array2DReal &LayerThick, - const Array2DReal &ZInterface); + const Array2DReal &SpecVol, + const Array2DReal &LayerThick, + const Array2DReal &PressureInterface, + const Array2DReal &ZMid, + const Array2DReal &Geopotential); }; // end class PressureGrad From 98dcd61dbc80c4c1add5ae80e9944cb793fa4ef3 Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Mon, 15 Dec 2025 15:28:42 -0800 Subject: [PATCH 22/58] Montgomery potential implementation --- components/omega/src/ocn/PGrad.cpp | 118 ++--------------------------- components/omega/src/ocn/PGrad.h | 49 ++++-------- 2 files changed, 21 insertions(+), 146 deletions(-) diff --git a/components/omega/src/ocn/PGrad.cpp b/components/omega/src/ocn/PGrad.cpp index 55e23f3556a5..096c90d1f262 100644 --- a/components/omega/src/ocn/PGrad.cpp +++ b/components/omega/src/ocn/PGrad.cpp @@ -75,10 +75,9 @@ PressureGrad::PressureGrad( const HorzMesh *Mesh, ///< [in] Horizontal mesh const VertCoord *VCoord, ///< [in] Vertical coordinate Config *Options) ///< [in] Configuration options - : CellsOnEdge(Mesh->CellsOnEdge), DcEdge(Mesh->DcEdge), - EdgeSignOnCell(Mesh->EdgeSignOnCell), MinLayerEdgeBot(VCoord->MinLayerEdgeBot), - MaxLayerEdgeTop(VCoord->MaxLayerEdgeTop), CenteredPGrad(Mesh, VCoord), - HighOrderPGrad(Mesh, VCoord) { + : CellsOnEdge(Mesh->CellsOnEdge), DcEdge(Mesh->DcEdge), EdgeSignOnCell(Mesh->EdgeSignOnCell), + MinLayerEdgeBot(VCoord->MinLayerEdgeBot), MaxLayerEdgeTop(VCoord->MaxLayerEdgeTop), + CenteredPGrad(Mesh, VCoord), HighOrderPGrad(Mesh, VCoord) { // store mesh sizes NEdgesAll = Mesh->NEdgesAll; @@ -86,8 +85,6 @@ PressureGrad::PressureGrad( NVertLayersP1 = NVertLayers + 1; NChunks = NVertLayers / VecLength; - InterfaceProduct = Array2DReal("InterfaceProduct", NEdgesAll, NVertLayersP1); - // Read config options for PressureGrad type // and enable the appropriate functor Config PGradConfig("PressureGrad"); @@ -159,7 +156,6 @@ void PressureGrad::computePressureGrad(Array2DReal &Tend, OMEGA_SCOPE(LocCenteredPGrad, CenteredPGrad); OMEGA_SCOPE(LocHighOrderPGrad, HighOrderPGrad); - OMEGA_SCOPE(LocInterfaceProduct, InterfaceProduct); OMEGA_SCOPE(LocMinLayerEdgeBot, MinLayerEdgeBot); OMEGA_SCOPE(LocMaxLayerEdgeTop, MaxLayerEdgeTop); @@ -174,11 +170,6 @@ void PressureGrad::computePressureGrad(Array2DReal &Tend, if (PressureGradChoice == PressureGradType::Centered) { - // computes alpha*p*grad(z) term at edge interfaces - computeInterfaceProduct(PressureMid, SpecVol, - LayerThick, PressureInterface, - ZMid, Geopotential); - // computes centered geopotential and pressure gradient tendency parallelForOuter( "pgrad-centered", {NEdgesAll}, @@ -191,8 +182,9 @@ void PressureGrad::computePressureGrad(Array2DReal &Tend, Team, KRange, INNER_LAMBDA(int KChunk) { LocCenteredPGrad(Tend, IEdge, KChunk, PressureMid, - Geopotential, LayerThick, - LocInterfaceProduct, SpecVol, ZMid); + PressureInterface, ZInterface, + LayerThick, + SpecVol); }); }); @@ -217,104 +209,6 @@ void PressureGrad::computePressureGrad(Array2DReal &Tend, } } // end compute pressure gradient -//------------------------------------------------------------------------------ -// Compute the alpha*p*grad(z) product at edge interfaces that is -// needed for the centered pressure gradient -void PressureGrad::computeInterfaceProduct( - const Array2DReal &PressureMid, - const Array2DReal &SpecVol, - const Array2DReal &LayerThick, - const Array2DReal &PInterface, - const Array2DReal &ZMid, - const Array2DReal &Geopotential - ) { - - OMEGA_SCOPE(LocCellsOnEdge, CellsOnEdge); - OMEGA_SCOPE(LocDcEdge, DcEdge); - OMEGA_SCOPE(LocInterfaceProduct, InterfaceProduct); - OMEGA_SCOPE(LocMinLayerEdgeBot, MinLayerEdgeBot); - OMEGA_SCOPE(LocMaxLayerEdgeTop, MaxLayerEdgeTop); - - Real Gravity = 9.80616_Real; - Real Rho0 = 1026.0_Real; - - parallelForOuter( - "compute-interface-product", {NEdgesAll}, - KOKKOS_LAMBDA(I4 IEdge, const TeamMember &Team) { - const int KMin = LocMinLayerEdgeBot(IEdge); - const int KMax = LocMaxLayerEdgeTop(IEdge); - - const I4 ICell0 = LocCellsOnEdge(IEdge, 0); - const I4 ICell1 = LocCellsOnEdge(IEdge, 1); - - // Two-point average for surface interface product - LocInterfaceProduct(IEdge, KMin) = - PressureMid(ICell1, KMin) * SpecVol(ICell1, KMin) * - LayerThick(ICell1, KMin) + - PressureMid(ICell0, KMin) * SpecVol(ICell0, KMin) * - LayerThick(ICell0, KMin) + - Geopotential(ICell1, KMin) + Geopotential(ICell0, KMin) - - Gravity * (ZMid(ICell1, KMin) + ZMid(ICell0, KMin)); - LocInterfaceProduct(IEdge, KMin) /= - (LayerThick(ICell1, KMin) + LayerThick(ICell0, KMin)); - - LocInterfaceProduct(IEdge, KMin) *= - -(PInterface(ICell1, KMin) - PInterface(ICell0, KMin)) / - (LocDcEdge(IEdge) * Gravity * Rho0); - - // Four-point average for interior interface products - const int KRange = vertRange(KMin + 1, KMax); - parallelForInner( - Team, KRange, - INNER_LAMBDA(int KChunk) { - const I4 KStart = chunkStart(KChunk, KMin + 1); - const I4 KLen = chunkLength(KChunk, KStart, KMax); - - for (int KVec = 0; KVec < KLen; ++KVec) { - const I4 K = KStart + KVec; - - LocInterfaceProduct(IEdge, K) = - PressureMid(ICell1, K) * SpecVol(ICell1, K) * - LayerThick(ICell1, K) + - PressureMid(ICell0, K) * SpecVol(ICell0, K) * - LayerThick(ICell0, K) + - PressureMid(ICell1, K - 1) * SpecVol(ICell1, K - 1) * - LayerThick(ICell1, K - 1) + - PressureMid(ICell0, K - 1) * SpecVol(ICell0, K - 1) * - LayerThick(ICell0, K - 1); - Geopotential(ICell1, K) + Geopotential(ICell0, K) + - Geopotential(ICell1, K - 1) + Geopotential(ICell0, K - 1) - - Gravity * (ZMid(ICell1, K) + ZMid(ICell0, K) + - ZMid(ICell1, K - 1) + ZMid(ICell0, K - 1)); - LocInterfaceProduct(IEdge, K) /= - (LayerThick(ICell1, K) + LayerThick(ICell0, K) + - LayerThick(ICell1, K - 1) + - LayerThick(ICell0, K - 1)); - - LocInterfaceProduct(IEdge, K) *= - -(PInterface(ICell1, K) - PInterface(ICell0, K)) / - (LocDcEdge(IEdge) * Gravity * Rho0); - } - }); - - // Two-point average for bottom interface product - LocInterfaceProduct(IEdge, KMax + 1) = - PressureMid(ICell1, KMax) * SpecVol(ICell1, KMax) * - LayerThick(ICell1, KMax) + - PressureMid(ICell0, KMax) * SpecVol(ICell0, KMax) * - LayerThick(ICell0, KMax); - Geopotential(ICell1, KMax) + Geopotential(ICell0, KMax) - - Gravity * (ZMid(ICell1, KMax) + ZMid(ICell0, KMax)); - LocInterfaceProduct(IEdge, KMax + 1) /= - (LayerThick(ICell1, KMax) + - LayerThick(ICell0, KMax)); - LocInterfaceProduct(IEdge, KMax + 1) *= - -(PInterface(ICell1, KMax + 1) - PInterface(ICell0, KMax + 1)) / - (LocDcEdge(IEdge) * Gravity * Rho0); - }); - -} // end compute interface product - //------------------------------------------------------------------------------ // Constructor for centered pressure gradient functor PressureGradCentered::PressureGradCentered(const HorzMesh *Mesh, ///< [in] Horizontal mesh diff --git a/components/omega/src/ocn/PGrad.h b/components/omega/src/ocn/PGrad.h index e28e0cff4c6f..11004978ccf0 100644 --- a/components/omega/src/ocn/PGrad.h +++ b/components/omega/src/ocn/PGrad.h @@ -34,11 +34,11 @@ class PressureGradCentered { // vertical chunk. This appends results into the Tend array (in-place). KOKKOS_FUNCTION void operator()(const Array2DReal &Tend, I4 IEdge, I4 KChunk, const Array2DReal &PressureMid, - const Array2DReal &Geopotential, + const Array2DReal &PressureInterface, + const Array2DReal &ZInterface, const Array2DReal &LayerThick, - const Array2DReal &InterfaceProduct, - const Array2DReal &SpecVol, - const Array2DReal &ZMid) const { + const Array2DReal &SpecVol + ) const { const I4 KStart = chunkStart(KChunk, MinLayerEdgeBot(IEdge)); const I4 KLen = chunkLength(KChunk, KStart, MaxLayerEdgeTop(IEdge)); @@ -51,28 +51,20 @@ class PressureGradCentered { for (int KVec = 0; KVec < KLen; ++KVec) { const I4 K = KStart + KVec; - const Real InvLayerThickEdge = - 2.0_Real / (LayerThick(ICell1, K) + LayerThick(ICell0, K)); - //Real RhoEdge = 1.0_Real / (0.5_Real*(SpecVol(ICell1, K) + SpecVol(ICell0, K))); - //Real GeoEdgeK = (ZInterface(ICell1, K) + ZInterface(ICell0, K)) * 0.5_Real; - //Real GeoEdgeKp1 = (ZInterface(ICell1, K+1) + ZInterface(ICell0, K+1)) * 0.5_Real; - - Real GeoTerm = - (LayerThick(ICell1, K) * (Geopotential(ICell1, K) - Gravity * ZMid(ICell1, K)) - - LayerThick(ICell0, K) * (Geopotential(ICell0, K) - Gravity * ZMid(ICell0, K))) * InvDcEdge; - //GeoTerm *= (1.0_Real - (RhoEdge / Rho0)*(GeoEdgeK - GeoEdgeKp1) * InvLayerThickEdge); - Real PresTerm = (LayerThick(ICell1, K) * SpecVol(ICell1, K) * - PressureMid(ICell1, K) - - LayerThick(ICell0, K) * SpecVol(ICell0, K) * - PressureMid(ICell0, K)) * - InvDcEdge * InvLayerThickEdge; - Real InterfaceTerm = (InterfaceProduct(IEdge, K) - InterfaceProduct(IEdge, K+1)) * - InvLayerThickEdge; + Real MontPotCell0K = PressureInterface(ICell0, K) * SpecVol(ICell0, K) + Gravity * ZInterface(ICell0, K); + Real MontPotCell1K = PressureInterface(ICell1, K) * SpecVol(ICell1, K) + Gravity * ZInterface(ICell1, K); + Real GradMontPotK = (MontPotCell1K - MontPotCell0K) * InvDcEdge; + + Real MontPotCell0Kp1 = PressureInterface(ICell0, K+1) * SpecVol(ICell0, K) + Gravity * ZInterface(ICell0, K+1); + Real MontPotCell1Kp1 = PressureInterface(ICell1, K+1) * SpecVol(ICell1, K) + Gravity * ZInterface(ICell1, K+1); + Real GradMontPotKp1 = (MontPotCell1Kp1 - MontPotCell0Kp1) * InvDcEdge; + Real GradMontPot = 0.5_Real * (GradMontPotK + GradMontPotKp1); + Real PGradAlpha = 0.5 * (PressureMid(ICell1, K) + PressureMid(ICell0, K)) * (SpecVol(ICell1, K) - SpecVol(ICell0, K)) * InvDcEdge; Tend(IEdge, K) += - EdgeMask(IEdge, K) * (GeoTerm + PresTerm - InterfaceTerm); + EdgeMask(IEdge, K) * (GradMontPot - PGradAlpha); if (IEdge == 0) - LOG_INFO("IEdge {}, K {}: GeoTerm {}, PresTerm {}, InterfaceTerm {}, Tend {}", IEdge, K, GeoTerm, PresTerm, InterfaceTerm, Tend(IEdge, K)); + LOG_INFO("IEdge {}, K {}: GradMontPot {}, PGradAlpha {}, Tend {}", IEdge, K, GradMontPot, PGradAlpha, Tend(IEdge, K)); } } @@ -178,23 +170,12 @@ class PressureGrad { PressureGradCentered CenteredPGrad; PressureGradHighOrder HighOrderPGrad; - // Array for interface product needed in centered pressure gradient - Array2DReal InterfaceProduct; - // Choice from config PressureGradType PressureGradChoice = PressureGradType::Centered; // Map of all pressure gradient objects by name static std::map> AllPGrads; - // Compute interface product needed for centered pressure gradient - void computeInterfaceProduct(const Array2DReal &PressureMid, - const Array2DReal &SpecVol, - const Array2DReal &LayerThick, - const Array2DReal &PressureInterface, - const Array2DReal &ZMid, - const Array2DReal &Geopotential); - }; // end class PressureGrad } // namespace OMEGA From 757298e0ca04ccb354a0b35281458042fcc1a33f Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Tue, 23 Dec 2025 15:24:28 -0800 Subject: [PATCH 23/58] Work toward outputing PGrad tendencies (need to fix unit tests) --- components/omega/configs/Default.yml | 1 + components/omega/src/ocn/PGrad.cpp | 2 +- components/omega/src/ocn/PGrad.h | 8 +- components/omega/src/ocn/Tendencies.cpp | 82 ++++++++++++++++++- components/omega/src/ocn/Tendencies.h | 12 ++- components/omega/test/ocn/TendenciesTest.cpp | 19 ++++- .../test/timeStepping/TimeStepperTest.cpp | 11 ++- 7 files changed, 122 insertions(+), 13 deletions(-) diff --git a/components/omega/configs/Default.yml b/components/omega/configs/Default.yml index 2964147e33ec..d71f8538d4b8 100644 --- a/components/omega/configs/Default.yml +++ b/components/omega/configs/Default.yml @@ -59,6 +59,7 @@ Omega: ThicknessVertAdvTendencyEnable: true VelocityVertAdvTendencyEnable: true TracerVertAdvTendencyEnable: true + PressureGradTendencyEnable: true Tracers: Base: [Temperature, Salinity] Debug: [Debug1, Debug2, Debug3] diff --git a/components/omega/src/ocn/PGrad.cpp b/components/omega/src/ocn/PGrad.cpp index 096c90d1f262..0fc1a4b022cc 100644 --- a/components/omega/src/ocn/PGrad.cpp +++ b/components/omega/src/ocn/PGrad.cpp @@ -152,7 +152,7 @@ void PressureGrad::computePressureGrad(Array2DReal &Tend, const OceanState *State, const VertCoord *VCoord, const Eos *EqState, - const int TimeLevel) { + const int TimeLevel) const { OMEGA_SCOPE(LocCenteredPGrad, CenteredPGrad); OMEGA_SCOPE(LocHighOrderPGrad, HighOrderPGrad); diff --git a/components/omega/src/ocn/PGrad.h b/components/omega/src/ocn/PGrad.h index 11004978ccf0..31e9386fc36c 100644 --- a/components/omega/src/ocn/PGrad.h +++ b/components/omega/src/ocn/PGrad.h @@ -62,7 +62,7 @@ class PressureGradCentered { Real PGradAlpha = 0.5 * (PressureMid(ICell1, K) + PressureMid(ICell0, K)) * (SpecVol(ICell1, K) - SpecVol(ICell0, K)) * InvDcEdge; Tend(IEdge, K) += - EdgeMask(IEdge, K) * (GradMontPot - PGradAlpha); + EdgeMask(IEdge, K) * (-GradMontPot + PGradAlpha); if (IEdge == 0) LOG_INFO("IEdge {}, K {}: GradMontPot {}, PGradAlpha {}, Tend {}", IEdge, K, GradMontPot, PGradAlpha, Tend(IEdge, K)); } @@ -112,6 +112,10 @@ class PressureGradHighOrder { // Pressure gradient manager class class PressureGrad { public: + + // Flag to indicate if pressure gradient term is enabled + bool Enabled; + // Initialize the default instance static void init(); @@ -139,7 +143,7 @@ class PressureGrad { // Compute pressure gradient tendencies and add into Tend array void computePressureGrad(Array2DReal &Tend, const OceanState *State, const VertCoord *VCoord, const Eos *EqState, - const int TimeLevel); + const int TimeLevel) const; private: diff --git a/components/omega/src/ocn/Tendencies.cpp b/components/omega/src/ocn/Tendencies.cpp index 8714f56514ea..4aa68b2ea998 100644 --- a/components/omega/src/ocn/Tendencies.cpp +++ b/components/omega/src/ocn/Tendencies.cpp @@ -12,6 +12,7 @@ #include "CustomTendencyTerms.h" #include "Eos.h" #include "Error.h" +#include "Field.h" #include "PGrad.h" #include "Pacer.h" #include "TimeStepper.h" @@ -34,6 +35,8 @@ void Tendencies::init() { VertCoord *DefVertCoord = VertCoord::getDefault(); VertAdv *DefVertAdv = VertAdv::getDefault(); TimeStepper *DefTimeStepper = TimeStepper::getDefault(); + Eos *DefEos = Eos::getInstance(); + PressureGrad *DefPGrad = PressureGrad::getDefault(); I4 NTracers = Tracers::getNumTracers(); @@ -74,7 +77,7 @@ void Tendencies::init() { // Ceate default tendencies Tendencies::DefaultTendencies = - create("Default", DefHorzMesh, DefVertCoord, DefVertAdv, NTracers, + create("Default", DefHorzMesh, DefVertCoord, DefVertAdv, DefPGrad, DefEos, NTracers, TimeStep, &TendConfig, CustomThickTend, CustomVelTend); DefaultTendencies->readConfig(OmegaConfig); @@ -242,12 +245,75 @@ void Tendencies::readConfig(Config *OmegaConfig ///< [in] Omega config } } +//------------------------------------------------------------------------------ +// Define fields associated with tendencies +void Tendencies::defineFields() { + auto LayerThicknessTendFieldName = "LayerThicknessTend"; + auto NormalVelocityTendFieldName = "NormalVelocityTend"; + auto TracerTendFieldName = "TracerTend"; + + int NDims = 2; + std::vector DimNamesThickness(NDims); + DimNamesThickness[0] = "NCells"; + DimNamesThickness[1] = "NVertLayers"; + auto LayerThicknessTendField = Field::create(LayerThicknessTendFieldName, + "Layer thickness tendency", + "m/s", + "cell_thickness_tendency", + -9.99E+10, + 9.99E+10, + -9.99E+30, + NDims, + DimNamesThickness); + NDims = 3; + std::vector DimNamesTracer(NDims); + DimNamesTracer[0] = "NTracers"; + DimNamesTracer[1] = "NCells"; + DimNamesTracer[2] = "NVertLayers"; + auto TracerTendField = Field::create(TracerTendFieldName, + "Tracer tendency", + "kg/m^3/s", + "tracer_tendency", + -9.99E+10, + 9.99E+10, + -9.99E+30, + NDims, + DimNamesTracer); + NDims = 2; + std::vector DimNamesVelocity(NDims); + DimNamesVelocity[0] = "NEdges"; + DimNamesVelocity[1] = "NVertLayers"; + auto NormalVelocityTendField = Field::create(NormalVelocityTendFieldName, + "Normal velocity tendency", + "m/s^2", + "sea_water_velocity_tendency", + -9.99E+10, + 9.99E+10, + -9.99E+30, + NDims, + DimNamesVelocity); + + auto TendGroupName = "Tendencies"; + auto TendGroup = FieldGroup::create(TendGroupName); + + TendGroup->addField(LayerThicknessTendFieldName); + TendGroup->addField(NormalVelocityTendFieldName); + TendGroup->addField(TracerTendFieldName); + + LayerThicknessTendField->attachData(LayerThicknessTend); + NormalVelocityTendField->attachData(NormalVelocityTend); + TracerTendField->attachData(TracerTend); + +} // end defineFields + //------------------------------------------------------------------------------ // Construct a new group of tendencies Tendencies::Tendencies(const std::string &Name, ///< [in] Name for tendencies const HorzMesh *Mesh, ///< [in] Horizontal mesh const VertCoord *VCoord, ///< [in] Vertical coordinate VertAdv *VAdv, ///< [in] Vertical advection + const PressureGrad *PGrad, ///< [in] Pressure gradient + const Eos *EqState, ///< [in] Equation of state int NTracersIn, ///< [in] Number of tracers TimeInterval TimeStepIn, ///< [in] Time step Config *Options, ///< [in] Configuration options @@ -260,7 +326,8 @@ Tendencies::Tendencies(const std::string &Name, ///< [in] Name for tendencies BottomDrag(Mesh, VCoord), TracerDiffusion(Mesh, VCoord), TracerHyperDiff(Mesh, VCoord), TracerHorzAdv(Mesh, VCoord), CustomThicknessTend(InCustomThicknessTend), - CustomVelocityTend(InCustomVelocityTend) { + CustomVelocityTend(InCustomVelocityTend), + EqState(EqState), PGrad(PGrad) { // Tendency arrays LayerThicknessTend = @@ -279,10 +346,12 @@ Tendencies::Tendencies(const std::string &Name, ///< [in] Name for tendencies const HorzMesh *Mesh, ///< [in] Horizontal mesh const VertCoord *VCoord, ///< [in] Vertical coordinate VertAdv *VAdv, ///< [in] Vertical advection + const PressureGrad *PGrad, ///< [in] Pressure gradient + const Eos *EqState, ///< [in] Equation of state int NTracersIn, ///< [in] Number of tracers TimeInterval TimeStepIn, ///< [in] Time step Config *Options) ///< [in] Configuration options - : Tendencies(Name, Mesh, VCoord, VAdv, NTracersIn, TimeStepIn, Options, + : Tendencies(Name, Mesh, VCoord, VAdv, PGrad, EqState, NTracersIn, TimeStepIn, Options, CustomTendencyType{}, CustomTendencyType{}) {} //------------------------------------------------------------------------------ @@ -531,6 +600,13 @@ void Tendencies::computeVelocityTendenciesOnly( Pacer::stop("Tend:customVelocityTend", 2); } + if (PGrad->Enabled) { + Pacer::start("Tend:pressureGradTerm", 2); + PGrad->computePressureGrad( + LocNormalVelocityTend, State, VCoord, EqState, VelTimeLevel); + Pacer::stop("Tend:pressureGradTerm", 2); + } + Pacer::stop("Tend:computeVelocityTendenciesOnly", 1); } // end velocity tendency compute diff --git a/components/omega/src/ocn/Tendencies.h b/components/omega/src/ocn/Tendencies.h index 9bc4f0584b5a..06e91144d8d4 100644 --- a/components/omega/src/ocn/Tendencies.h +++ b/components/omega/src/ocn/Tendencies.h @@ -37,6 +37,8 @@ #include "OceanState.h" #include "TendencyTerms.h" #include "TimeMgr.h" +#include "Eos.h" +#include "PGrad.h" #include "VertAdv.h" #include @@ -152,6 +154,8 @@ class Tendencies { const HorzMesh *Mesh, ///< [in] Horizontal mesh const VertCoord *VCoord, ///< [in] Vertical coordinate VertAdv *VAdv, ///< [in] Vertical advection + const PressureGrad *PGrad, ///< [in] Pressure gradient + const Eos *EqState, ///< [in] Equation of state int NTracersIn, ///< [in] Number of tracers TimeInterval TimeStep, ///< [in] Time step Config *Options, ///< [in] Configuration options @@ -162,18 +166,24 @@ class Tendencies { const HorzMesh *Mesh, ///< [in] Horizontal mesh const VertCoord *VCoord, ///< [in] Vertical coordinate VertAdv *VAdv, ///< [in] Vertical advection + const PressureGrad *PGrad, ///< [in] Pressure gradient + const Eos *EqState, ///< [in] Equation of state int NTracersIn, ///< [in] Number of tracers TimeInterval TimeStep, ///< [in] Time step Config *Options ///< [in] Configuration options ); + void defineFields(); + // forbid copy and move construction Tendencies(const Tendencies &) = delete; Tendencies(Tendencies &&) = delete; const HorzMesh *Mesh; ///< Pointer to horizontal mesh const VertCoord *VCoord; ///< Pointer to vertical coordinate - VertAdv *VAdv; ///< Pointer to vertical advection + VertAdv *VAdv; ///< Pointer to vertical advection + const PressureGrad *PGrad; ///< Pointer to pressure gradient + const Eos *EqState; ///< Pointer to equation of state I4 NTracers; ///< Number of tracers TimeInterval TimeStep; ///< Time step diff --git a/components/omega/test/ocn/TendenciesTest.cpp b/components/omega/test/ocn/TendenciesTest.cpp index 40afa13e531f..35a78b6888fb 100644 --- a/components/omega/test/ocn/TendenciesTest.cpp +++ b/components/omega/test/ocn/TendenciesTest.cpp @@ -5,6 +5,7 @@ #include "Decomp.h" #include "Dimension.h" #include "Error.h" +#include "Eos.h" #include "Field.h" #include "GlobalConstants.h" #include "Halo.h" @@ -16,6 +17,7 @@ #include "OceanTestCommon.h" #include "OmegaKokkos.h" #include "Pacer.h" +#include "PGrad.h" #include "TimeStepper.h" #include "VertCoord.h" #include "mpi.h" @@ -57,8 +59,15 @@ int initState() { auto *VCoord = VertCoord::getDefault(); auto *State = OceanState::getDefault(); - Array2DReal LayerThickCell = State->getLayerThickness(0); - Array2DReal NormalVelEdge = State->getNormalVelocity(0); + // Define tendency fields + int NDims = 2; + std::vector DimNamesThickness(NDims); + DimNamesThickness[0] = "NCells"; + + Array2DReal LayerThickCell; + Array2DReal NormalVelEdge; + Err += State->getLayerThickness(LayerThickCell, 0); + Err += State->getNormalVelocity(NormalVelEdge, 0); Array3DReal TracersArray = Tracers::getAll(0); const auto &TracersCell = TracersArray; @@ -156,13 +165,15 @@ int testTendencies() { const auto Mesh = HorzMesh::getDefault(); const auto VCoord = VertCoord::getDefault(); const auto VAdv = VertAdv::getDefault(); + auto *PGrad = PressureGrad::getDefault(); + auto *EqState = Eos::getInstance(); VCoord->NVertLayers = 12; // test creation of another tendencies TimeInterval ZeroTimeStep; // Zero-length time step placeholder Config *Options = Config::getOmegaConfig(); - Tendencies::create("TestTendencies", Mesh, VCoord, VAdv, 3, ZeroTimeStep, - Options); + Tendencies::create("TestTendencies", Mesh, VCoord, VAdv, PGrad, EqState, 3, + ZeroTimeStep, Options); // test retrievel of another tendencies if (Tendencies::get("TestTendencies")) { diff --git a/components/omega/test/timeStepping/TimeStepperTest.cpp b/components/omega/test/timeStepping/TimeStepperTest.cpp index 95aca280fa42..dc9965139909 100644 --- a/components/omega/test/timeStepping/TimeStepperTest.cpp +++ b/components/omega/test/timeStepping/TimeStepperTest.cpp @@ -19,6 +19,7 @@ #include "DataTypes.h" #include "Decomp.h" #include "Dimension.h" +#include "Eos.h" #include "Error.h" #include "Field.h" #include "Halo.h" @@ -29,6 +30,7 @@ #include "OceanState.h" #include "OmegaKokkos.h" #include "Pacer.h" +#include "PGrad.h" #include "TendencyTerms.h" #include "TimeMgr.h" #include "Tracers.h" @@ -173,6 +175,8 @@ int initTimeStepperTest(const std::string &mesh) { auto *DefVAdv = VertAdv::getDefault(); AuxiliaryState::init(); + Eos::init(); + PressureGrad::init(); Tendencies::init(); // finish initializing default time stepper @@ -189,6 +193,8 @@ int initTimeStepperTest(const std::string &mesh) { auto *DefMesh = HorzMesh::getDefault(); auto *DefHalo = Halo::getDefault(); + auto *DefEos = Eos::getInstance(); + auto *DefPGrad = PressureGrad::getDefault(); int NTracers = Tracers::getNumTracers(); const int NTimeLevels = 2; @@ -216,8 +222,9 @@ int initTimeStepperTest(const std::string &mesh) { // Creating non-default tendencies with custom velocity tendencies auto *TestTendencies = Tendencies::create( - "TestTendencies", DefMesh, DefVertCoord, DefVAdv, NTracers, ZeroTimeStep, - &Options, Tendencies::CustomTendencyType{}, DecayVelocityTendency{}); + "TestTendencies", DefMesh, DefVertCoord, DefVAdv, DefPGrad, DefEos, NTracers, + ZeroTimeStep, &Options, Tendencies::CustomTendencyType{}, + DecayVelocityTendency{}); if (!TestTendencies) { Err++; LOG_ERROR("TimeStepperTest: error creating test tendencies"); From 73966d0c8f732e32d5f905dfaccb13ae4cf48e81 Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Tue, 6 Jan 2026 11:13:24 -0800 Subject: [PATCH 24/58] Fix Driver and Timestepper ctests --- components/omega/src/ocn/OceanFinal.cpp | 4 ++++ components/omega/src/ocn/OceanInit.cpp | 4 ++++ components/omega/test/timeStepping/TimeStepperTest.cpp | 2 ++ 3 files changed, 10 insertions(+) diff --git a/components/omega/src/ocn/OceanFinal.cpp b/components/omega/src/ocn/OceanFinal.cpp index 3715e95baef6..a518db52a175 100644 --- a/components/omega/src/ocn/OceanFinal.cpp +++ b/components/omega/src/ocn/OceanFinal.cpp @@ -7,6 +7,7 @@ #include "AuxiliaryState.h" #include "Decomp.h" +#include "Eos.h" #include "Field.h" #include "Halo.h" #include "HorzMesh.h" @@ -14,6 +15,7 @@ #include "MachEnv.h" #include "OceanDriver.h" #include "OceanState.h" +#include "PGrad.h" #include "Tendencies.h" #include "TimeMgr.h" #include "TimeStepper.h" @@ -33,6 +35,8 @@ int ocnFinalize(const TimeInstant &CurrTime ///< [in] current sim time // clean up all objects Tracers::clear(); TimeStepper::clear(); + PressureGrad::clear(); + Eos::destroyInstance(); Tendencies::clear(); AuxiliaryState::clear(); OceanState::clear(); diff --git a/components/omega/src/ocn/OceanInit.cpp b/components/omega/src/ocn/OceanInit.cpp index b5c21aa1508f..022b18d5b498 100644 --- a/components/omega/src/ocn/OceanInit.cpp +++ b/components/omega/src/ocn/OceanInit.cpp @@ -11,6 +11,7 @@ #include "Config.h" #include "DataTypes.h" #include "Decomp.h" +#include "Eos.h" #include "Error.h" #include "Field.h" #include "Halo.h" @@ -22,6 +23,7 @@ #include "OceanDriver.h" #include "OceanState.h" #include "Pacer.h" +#include "PGrad.h" #include "Tendencies.h" #include "TimeMgr.h" #include "TimeStepper.h" @@ -134,6 +136,8 @@ int initOmegaModules(MPI_Comm Comm) { Tracers::init(); VertAdv::init(); AuxiliaryState::init(); + PressureGrad::init(); + Eos::init(); Tendencies::init(); TimeStepper::init2(); diff --git a/components/omega/test/timeStepping/TimeStepperTest.cpp b/components/omega/test/timeStepping/TimeStepperTest.cpp index dc9965139909..1d3ce044b1d2 100644 --- a/components/omega/test/timeStepping/TimeStepperTest.cpp +++ b/components/omega/test/timeStepping/TimeStepperTest.cpp @@ -282,6 +282,8 @@ void timeLoop(TimeInstant TimeStart, Real TimeEnd) { void finalizeTimeStepperTest() { Tracers::clear(); TimeStepper::clear(); + PressureGrad::clear(); + Eos::destroyInstance(); Tendencies::clear(); AuxiliaryState::clear(); OceanState::clear(); From eb07a5571d640ffe154144d6875aaad916bb912c Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Thu, 8 Jan 2026 09:04:36 -0800 Subject: [PATCH 25/58] Fix Tendencies ctest --- components/omega/test/ocn/TendenciesTest.cpp | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/components/omega/test/ocn/TendenciesTest.cpp b/components/omega/test/ocn/TendenciesTest.cpp index 35a78b6888fb..6a0b82f687cf 100644 --- a/components/omega/test/ocn/TendenciesTest.cpp +++ b/components/omega/test/ocn/TendenciesTest.cpp @@ -1,6 +1,7 @@ #include "Tendencies.h" #include "AuxiliaryState.h" #include "Config.h" +#include "CustomTendencyTerms.h" #include "DataTypes.h" #include "Decomp.h" #include "Dimension.h" @@ -147,6 +148,7 @@ int initTendenciesTest(const std::string &mesh) { int testTendencies() { int Err = 0; + Error Err1; // test initialization Tendencies::init(); @@ -165,15 +167,20 @@ int testTendencies() { const auto Mesh = HorzMesh::getDefault(); const auto VCoord = VertCoord::getDefault(); const auto VAdv = VertAdv::getDefault(); - auto *PGrad = PressureGrad::getDefault(); - auto *EqState = Eos::getInstance(); + const auto PGrad = PressureGrad::getDefault(); + const auto EqState = Eos::getInstance(); VCoord->NVertLayers = 12; + // test creation of another tendencies TimeInterval ZeroTimeStep; // Zero-length time step placeholder Config *Options = Config::getOmegaConfig(); - Tendencies::create("TestTendencies", Mesh, VCoord, VAdv, PGrad, EqState, 3, - ZeroTimeStep, Options); + Config TendConfig("Tendencies"); + Err1 = Options->get(TendConfig); + int NTracersTest = 3; + + Tendencies::create("TestTendencies", Mesh, VCoord, VAdv, PGrad, EqState, + NTracersTest, ZeroTimeStep, &TendConfig); // test retrievel of another tendencies if (Tendencies::get("TestTendencies")) { @@ -246,6 +253,8 @@ int testTendencies() { void finalizeTendenciesTest() { Tracers::clear(); + PressureGrad::clear(); + Eos::destroyInstance(); AuxiliaryState::clear(); OceanState::clear(); VertAdv::clear(); From 7b4fad6e19cec9d9085d2be9a8d8d9d0ecc289de Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Thu, 8 Jan 2026 09:06:15 -0800 Subject: [PATCH 26/58] Fix some rebase issues - EdgeMask moved to VertCoord = Density0 no longer in omega.yaml --- components/omega/src/ocn/PGrad.cpp | 4 ++-- components/omega/test/ocn/PGradTest.cpp | 18 +++++------------- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/components/omega/src/ocn/PGrad.cpp b/components/omega/src/ocn/PGrad.cpp index 0fc1a4b022cc..286b13bfe75e 100644 --- a/components/omega/src/ocn/PGrad.cpp +++ b/components/omega/src/ocn/PGrad.cpp @@ -215,7 +215,7 @@ PressureGradCentered::PressureGradCentered(const HorzMesh *Mesh, ///< [in] Horiz const VertCoord *VCoord ///< [in] Vertical coordinate ) : CellsOnEdge(Mesh->CellsOnEdge), DcEdge(Mesh->DcEdge), - EdgeMask(Mesh->EdgeMask), MinLayerEdgeBot(VCoord->MinLayerEdgeBot), + EdgeMask(VCoord->EdgeMask), MinLayerEdgeBot(VCoord->MinLayerEdgeBot), MaxLayerEdgeTop(VCoord->MaxLayerEdgeTop) {} //------------------------------------------------------------------------------ @@ -224,7 +224,7 @@ PressureGradHighOrder::PressureGradHighOrder(const HorzMesh *Mesh, ///< [in] Hor const VertCoord *VCoord ///< [in] Vertical coordinate ) : CellsOnEdge(Mesh->CellsOnEdge), DcEdge(Mesh->DcEdge), - EdgeMask(Mesh->EdgeMask), MinLayerEdgeBot(VCoord->MinLayerEdgeBot), + EdgeMask(VCoord->EdgeMask), MinLayerEdgeBot(VCoord->MinLayerEdgeBot), MaxLayerEdgeTop(VCoord->MaxLayerEdgeTop) {} } // namespace OMEGA diff --git a/components/omega/test/ocn/PGradTest.cpp b/components/omega/test/ocn/PGradTest.cpp index abd7cca99d89..fb5e34f6af26 100644 --- a/components/omega/test/ocn/PGradTest.cpp +++ b/components/omega/test/ocn/PGradTest.cpp @@ -13,6 +13,7 @@ #include "Error.h" #include "Eos.h" #include "Field.h" +#include "GlobalConstants.h" #include "Halo.h" #include "HorzMesh.h" #include "IO.h" @@ -66,14 +67,11 @@ void initPGradTest() { "PGrad: error initializing default halo"); } - // Begin initialization of the default vertical coordinate - VertCoord::init1(); - // Initialize the default mesh HorzMesh::init(); - // Complete initialization of the default vertical coordinate - VertCoord::init2(); + // Initialize the default vertical coordinate + VertCoord::init(); // Initialize the equation of state Eos::init(); @@ -145,7 +143,7 @@ int main(int argc, char *argv[]) { auto &CellsOnEdge = DefMesh->CellsOnEdge; auto &DcEdge = DefMesh->DcEdge; - auto &EdgeMask = DefMesh->EdgeMask; + auto &EdgeMask = VCoord->EdgeMask; parallelFor({NEdgesAll}, KOKKOS_LAMBDA(int i) { CellsOnEdge(i, 0)= 0; CellsOnEdge(i, 1)= 1; @@ -154,13 +152,7 @@ int main(int argc, char *argv[]) { // Fetch reference desnity from Config Real Density0; - Error ErrorCode; - Config TendConfig("Tendencies"); - ErrorCode.reset(); - ErrorCode += Options->get(TendConfig); - CHECK_ERROR_ABORT(ErrorCode, "VertCoord: Tendencies group not found in Config"); - ErrorCode += TendConfig.get("Density0", Density0); - CHECK_ERROR_ABORT(ErrorCode, "VertCoord: Density0 not found in TendConfig"); + Density0 = RhoSw; I4 TimeLevel = 0; From 8860b31f859779d88bfa29df1af72340b98a68ab Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Thu, 8 Jan 2026 09:08:53 -0800 Subject: [PATCH 27/58] Compute BruntVaisalaFreqSq with input pressure in Pa --- components/omega/src/ocn/Eos.h | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/components/omega/src/ocn/Eos.h b/components/omega/src/ocn/Eos.h index 4f10d75c8fec..06c9695f7267 100644 --- a/components/omega/src/ocn/Eos.h +++ b/components/omega/src/ocn/Eos.h @@ -44,7 +44,6 @@ class Teos10Eos { I4 KDisp) const { Real SpecVolPCoeffs[6 * VecLength]; - Real PaToDBar = 1.0e-4_Real; // Conversion factor from Pa to dbar const I4 KStart = chunkStart(KChunk, MinLayerCell(ICell)); const I4 KLen = chunkLength(KChunk, KStart, MaxLayerCell(ICell)); @@ -63,15 +62,15 @@ class Teos10Eos { if (KDisp == 0) { // No displacement SpecVol(ICell, K) = - calcRefProfile(Pressure(ICell, K) * PaToDBar) + - calcDelta(SpecVolPCoeffs, KVec, Pressure(ICell, K) * PaToDBar); + calcRefProfile(Pressure(ICell, K) * Pa2Db) + + calcDelta(SpecVolPCoeffs, KVec, Pressure(ICell, K) * Pa2Db); } else { // Displacement, use the displaced pressure I4 KTmp = Kokkos::min(K + KDisp, MaxLayerCell(ICell)); KTmp = Kokkos::max(MinLayerCell(ICell), KTmp); SpecVol(ICell, K) = - calcRefProfile(Pressure(ICell, KTmp) * PaToDBar) + - calcDelta(SpecVolPCoeffs, KVec, Pressure(ICell, KTmp) * PaToDBar); + calcRefProfile(Pressure(ICell, KTmp) * Pa2Db) + + calcDelta(SpecVolPCoeffs, KVec, Pressure(ICell, KTmp) * Pa2Db); } } } @@ -312,15 +311,15 @@ class Teos10BruntVaisalaFreqSq { Real PInt = 0.5_Real * (Pressure(ICell, K) + Pressure(ICell, K - 1)); Real SpInt = 0.5_Real * (SpecVol(ICell, K) + SpecVol(ICell, K - 1)); - Real AlphaInt = calcAlpha(SaInt, CtInt, PInt, SpInt); - Real BetaInt = calcBeta(SaInt, CtInt, PInt, SpInt); + Real AlphaInt = calcAlpha(SaInt, CtInt, PInt * Pa2Db, SpInt); + Real BetaInt = calcBeta(SaInt, CtInt, PInt * Pa2Db, SpInt); Real DSa = AbsSalinity(ICell, K) - AbsSalinity(ICell, K - 1); Real DCt = ConservTemp(ICell, K) - ConservTemp(ICell, K - 1); Real DP = Pressure(ICell, K) - Pressure(ICell, K - 1); BruntVaisalaFreqSq(ICell, K) = Gravity * Gravity * (BetaInt * DSa - AlphaInt * DCt) / - (SpInt * Db2Pa * DP); + (SpInt * DP); } } } From ba644920021054afda6cc73934193aff33adfbb1 Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Thu, 8 Jan 2026 09:09:45 -0800 Subject: [PATCH 28/58] Fix EosTest, pressure dbar -> Pa conversion --- components/omega/test/ocn/EosTest.cpp | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/components/omega/test/ocn/EosTest.cpp b/components/omega/test/ocn/EosTest.cpp index 1b507781e8e9..5797b77798d2 100644 --- a/components/omega/test/ocn/EosTest.cpp +++ b/components/omega/test/ocn/EosTest.cpp @@ -53,11 +53,10 @@ const Real GswBVFExpValue = /// Test input values const Real Sa = 30.0; // Absolute Salinity in g/kg const Real Ct = 10.0; // Conservative Temperature in degC -const Real P = 1000.0; // Pressure in dbar +const Real P = 1000.0 * Db2Pa; // Pressure in Pa const I4 KDisp = 1; // Displate parcel to K=1 for TEOS-10 eos const Real RTol = 1e-10; // Relative tolerance for isApprox checks -double PaToDBar = 1e-4; // Conversion factor from Pa to dbar /// The initialization routine for Eos testing. It calls various /// init routines, including the creation of the default decomposition. @@ -120,7 +119,7 @@ void testEosLinear() { /// Use Kokkos::deep_copy to fill the entire view with the ref value deepCopy(SArray, Sa); deepCopy(TArray, Ct); - deepCopy(PArray, P / PaToDBar); + deepCopy(PArray, P); deepCopy(TestEos->SpecVol, 0.0); /// Compute specific volume @@ -190,7 +189,7 @@ void testEosLinearDisplaced() { /// Use Kokkos::deep_copy to fill the entire view with the ref value deepCopy(SArray, Sa); deepCopy(TArray, Ct); - deepCopy(PArray, P / PaToDBar); + deepCopy(PArray, P); deepCopy(TestEos->SpecVolDisplaced, 0.0); /// Compute displaced specific volume @@ -385,7 +384,7 @@ void testEosTeos10() { /// Use Kokkos::deep_copy to fill the entire view with the ref value deepCopy(SArray, Sa); deepCopy(TArray, Ct); - deepCopy(PArray, P / PaToDBar); + deepCopy(PArray, P); deepCopy(TestEos->SpecVol, 0.0); /// Compute specific volume @@ -455,7 +454,7 @@ void testEosTeos10Displaced() { /// Use Kokkos::deep_copy to fill the entire view with the ref value deepCopy(SArray, Sa); deepCopy(TArray, Ct); - deepCopy(PArray, P / PaToDBar); + deepCopy(PArray, P); deepCopy(TestEos->SpecVolDisplaced, 0.0); /// Compute displaced specific volume @@ -541,17 +540,17 @@ void testBruntVaisalaFreqSqTeos10() { ZMid(ICell, 1) = -993.1071379053125_Real; SArray(ICell, 1) = Sa; TArray(ICell, 1) = Ct + 10.0_Real; - PArray(ICell, 1) = P + 1.0_Real; + PArray(ICell, 1) = (P * Pa2Db + 1.0_Real) * Db2Pa; } else if (K == 2) { ZMid(ICell, 2) = -994.0968821072275_Real; SArray(ICell, 2) = Sa + 1.0_Real; TArray(ICell, 2) = Ct + 5.0_Real; - PArray(ICell, K) = P + 2.0_Real; + PArray(ICell, K) = (P * Pa2Db + 2.0_Real) * Db2Pa; } else { // fill rest with valid junk to avoid Nans and Inf ZMid(ICell, K) = -994.0968821072275_Real - 0.1_Real * K; SArray(ICell, K) = Sa + 1.0_Real + 0.1_Real * K; TArray(ICell, K) = Ct + 5.0_Real - 0.01_Real * K; - PArray(ICell, K) = P + 2.0_Real + 0.1_Real * K; + PArray(ICell, K) = (P * Pa2Db + 2.0_Real + 0.1_Real * K) * Db2Pa; } }); @@ -648,7 +647,7 @@ void checkValueGswcSpecVol() { const Real RTol = 1e-10; /// Get specific volume from GSW-C library - double SpecVol = gsw_specvol(Sa, Ct, P); + double SpecVol = gsw_specvol(Sa, Ct, P * Pa2Db); /// Check the value against the expected TEOS-10 value bool Check = isApprox(SpecVol, TeosSVExpValue, RTol); if (!Check) { @@ -669,7 +668,7 @@ void checkValueGswcN2() { double Salt[4] = {Sa - 1.0, Sa, Sa + 1.0}; // Absolute Salinity (g/kg) double Temp[4] = {Ct + 15.0, Ct + 10.0, Ct + 5.0}; // Conservative Temperature (deg C) - double Press[4] = {P, P + 1.0, P + 2.0}; // Pressure (dbar) + double Press[4] = {P * Pa2Db, P * Pa2Db + 1.0, P * Pa2Db + 2.0}; // Pressure (dbar) // Latitude (degrees north) double Latitude[4] = {0.0, 0.0, 0.0}; From 2ab7c8a7a2b3f68461b84d70371569d81a138a46 Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Thu, 8 Jan 2026 14:09:01 -0800 Subject: [PATCH 29/58] Fix Tendencies ctest --- components/omega/test/ocn/TendenciesTest.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/omega/test/ocn/TendenciesTest.cpp b/components/omega/test/ocn/TendenciesTest.cpp index 6a0b82f687cf..60591ef7331b 100644 --- a/components/omega/test/ocn/TendenciesTest.cpp +++ b/components/omega/test/ocn/TendenciesTest.cpp @@ -134,6 +134,8 @@ int initTendenciesTest(const std::string &mesh) { VertCoord::init(); Tracers::init(); VertAdv::init(); + PressureGrad::init(); + Eos::init(); int StateErr = OceanState::init(); if (StateErr != 0) { From e07dade600320b454747ed7ef0e94c8e1e785f72 Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Tue, 10 Feb 2026 14:31:28 -0800 Subject: [PATCH 30/58] Add computation of SpecVol to Tendencies - Requires compptation of pressure --- components/omega/src/ocn/PGrad.cpp | 1 + components/omega/src/ocn/Tendencies.cpp | 27 +++++++++++++++++++++---- components/omega/src/ocn/Tendencies.h | 12 +++++------ 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/components/omega/src/ocn/PGrad.cpp b/components/omega/src/ocn/PGrad.cpp index 286b13bfe75e..f2afcdf36557 100644 --- a/components/omega/src/ocn/PGrad.cpp +++ b/components/omega/src/ocn/PGrad.cpp @@ -6,6 +6,7 @@ //===----------------------------------------------------------------------===// #include "PGrad.h" +#include "Eos.h" #include "Error.h" #include "Field.h" #include "HorzMesh.h" diff --git a/components/omega/src/ocn/Tendencies.cpp b/components/omega/src/ocn/Tendencies.cpp index 4aa68b2ea998..2d6bf11f01a2 100644 --- a/components/omega/src/ocn/Tendencies.cpp +++ b/components/omega/src/ocn/Tendencies.cpp @@ -310,10 +310,10 @@ void Tendencies::defineFields() { // Construct a new group of tendencies Tendencies::Tendencies(const std::string &Name, ///< [in] Name for tendencies const HorzMesh *Mesh, ///< [in] Horizontal mesh - const VertCoord *VCoord, ///< [in] Vertical coordinate + VertCoord *VCoord, ///< [in] Vertical coordinate VertAdv *VAdv, ///< [in] Vertical advection const PressureGrad *PGrad, ///< [in] Pressure gradient - const Eos *EqState, ///< [in] Equation of state + Eos *EqState, ///< [in] Equation of state int NTracersIn, ///< [in] Number of tracers TimeInterval TimeStepIn, ///< [in] Time step Config *Options, ///< [in] Configuration options @@ -340,14 +340,16 @@ Tendencies::Tendencies(const std::string &Name, ///< [in] Name for tendencies NTracers = NTracersIn; TimeStep = TimeStepIn; + defineFields(); + } // end constructor Tendencies::Tendencies(const std::string &Name, ///< [in] Name for tendencies const HorzMesh *Mesh, ///< [in] Horizontal mesh - const VertCoord *VCoord, ///< [in] Vertical coordinate + VertCoord *VCoord, ///< [in] Vertical coordinate VertAdv *VAdv, ///< [in] Vertical advection const PressureGrad *PGrad, ///< [in] Pressure gradient - const Eos *EqState, ///< [in] Equation of state + Eos *EqState, ///< [in] Equation of state int NTracersIn, ///< [in] Number of tracers TimeInterval TimeStepIn, ///< [in] Time step Config *Options) ///< [in] Configuration options @@ -600,8 +602,25 @@ void Tendencies::computeVelocityTendenciesOnly( Pacer::stop("Tend:customVelocityTend", 2); } + // Compute pressure gradient if (PGrad->Enabled) { + I4 Err; + Array2DReal Temp; + Array2DReal Salinity; + Array2DReal LayerThick; + + // Temporary handling of surface pressure + Array1DReal SurfacePressure("SurfacePressure", Mesh->NCellsSize); + deepCopy(SurfacePressure, 0.0_Real); + Pacer::start("Tend:pressureGradTerm", 2); + State->getLayerThickness(LayerThick, VelTimeLevel); + VCoord->computePressure(LayerThick, SurfacePressure); + OMEGA_SCOPE(LocPressureMid, VCoord->PressureMid); + Err = Tracers::getByName(Temp, VelTimeLevel, "Temperature"); + Err = Tracers::getByName(Salinity, VelTimeLevel, "Salinity"); + + EqState->computeSpecVol(Temp, Salinity, LocPressureMid); PGrad->computePressureGrad( LocNormalVelocityTend, State, VCoord, EqState, VelTimeLevel); Pacer::stop("Tend:pressureGradTerm", 2); diff --git a/components/omega/src/ocn/Tendencies.h b/components/omega/src/ocn/Tendencies.h index 06e91144d8d4..5cbd565c6f6f 100644 --- a/components/omega/src/ocn/Tendencies.h +++ b/components/omega/src/ocn/Tendencies.h @@ -152,10 +152,10 @@ class Tendencies { // Construct a new tendency object Tendencies(const std::string &Name, ///< [in] Name for tendencies const HorzMesh *Mesh, ///< [in] Horizontal mesh - const VertCoord *VCoord, ///< [in] Vertical coordinate + VertCoord *VCoord, ///< [in] Vertical coordinate VertAdv *VAdv, ///< [in] Vertical advection const PressureGrad *PGrad, ///< [in] Pressure gradient - const Eos *EqState, ///< [in] Equation of state + Eos *EqState, ///< [in] Equation of state int NTracersIn, ///< [in] Number of tracers TimeInterval TimeStep, ///< [in] Time step Config *Options, ///< [in] Configuration options @@ -164,10 +164,10 @@ class Tendencies { Tendencies(const std::string &Name, ///< [in] Name for tendencies const HorzMesh *Mesh, ///< [in] Horizontal mesh - const VertCoord *VCoord, ///< [in] Vertical coordinate + VertCoord *VCoord, ///< [in] Vertical coordinate VertAdv *VAdv, ///< [in] Vertical advection const PressureGrad *PGrad, ///< [in] Pressure gradient - const Eos *EqState, ///< [in] Equation of state + Eos *EqState, ///< [in] Equation of state int NTracersIn, ///< [in] Number of tracers TimeInterval TimeStep, ///< [in] Time step Config *Options ///< [in] Configuration options @@ -180,10 +180,10 @@ class Tendencies { Tendencies(Tendencies &&) = delete; const HorzMesh *Mesh; ///< Pointer to horizontal mesh - const VertCoord *VCoord; ///< Pointer to vertical coordinate + VertCoord *VCoord; ///< Pointer to vertical coordinate VertAdv *VAdv; ///< Pointer to vertical advection const PressureGrad *PGrad; ///< Pointer to pressure gradient - const Eos *EqState; ///< Pointer to equation of state + Eos *EqState; ///< Pointer to equation of state I4 NTracers; ///< Number of tracers TimeInterval TimeStep; ///< Time step From ee875b6a56636d2103860dc103f92cfe40057854 Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Tue, 10 Feb 2026 15:04:14 -0800 Subject: [PATCH 31/58] Read PressureGradTendencyEnable and set PGrad enable flag --- components/omega/src/ocn/Tendencies.cpp | 19 ++++++++++++------- components/omega/src/ocn/Tendencies.h | 20 ++++++++++---------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/components/omega/src/ocn/Tendencies.cpp b/components/omega/src/ocn/Tendencies.cpp index 2d6bf11f01a2..0f3be0956644 100644 --- a/components/omega/src/ocn/Tendencies.cpp +++ b/components/omega/src/ocn/Tendencies.cpp @@ -243,6 +243,11 @@ void Tendencies::readConfig(Config *OmegaConfig ///< [in] Omega config Err += TendConfig.get("EddyDiff4", this->TracerHyperDiff.EddyDiff4); CHECK_ERROR_ABORT(Err, "Tendencies: EddyDiff4 not found in TendConfig"); } + + Err += TendConfig->get("PressureGradTendencyEnable", + this->PGrad->Enabled); + CHECK_ERROR_ABORT( + Err, "Tendencies: PressureGradTendencyEnable not found in TendConfig"); } //------------------------------------------------------------------------------ @@ -310,9 +315,9 @@ void Tendencies::defineFields() { // Construct a new group of tendencies Tendencies::Tendencies(const std::string &Name, ///< [in] Name for tendencies const HorzMesh *Mesh, ///< [in] Horizontal mesh - VertCoord *VCoord, ///< [in] Vertical coordinate - VertAdv *VAdv, ///< [in] Vertical advection - const PressureGrad *PGrad, ///< [in] Pressure gradient + VertCoord *VCoord, ///< [in] Vertical coordinate + VertAdv *VAdv, ///< [in] Vertical advection + PressureGrad *PGrad, ///< [in] Pressure gradient Eos *EqState, ///< [in] Equation of state int NTracersIn, ///< [in] Number of tracers TimeInterval TimeStepIn, ///< [in] Time step @@ -346,9 +351,9 @@ Tendencies::Tendencies(const std::string &Name, ///< [in] Name for tendencies Tendencies::Tendencies(const std::string &Name, ///< [in] Name for tendencies const HorzMesh *Mesh, ///< [in] Horizontal mesh - VertCoord *VCoord, ///< [in] Vertical coordinate - VertAdv *VAdv, ///< [in] Vertical advection - const PressureGrad *PGrad, ///< [in] Pressure gradient + VertCoord *VCoord, ///< [in] Vertical coordinate + VertAdv *VAdv, ///< [in] Vertical advection + PressureGrad *PGrad, ///< [in] Pressure gradient Eos *EqState, ///< [in] Equation of state int NTracersIn, ///< [in] Number of tracers TimeInterval TimeStepIn, ///< [in] Time step @@ -608,7 +613,7 @@ void Tendencies::computeVelocityTendenciesOnly( Array2DReal Temp; Array2DReal Salinity; Array2DReal LayerThick; - + // Temporary handling of surface pressure Array1DReal SurfacePressure("SurfacePressure", Mesh->NCellsSize); deepCopy(SurfacePressure, 0.0_Real); diff --git a/components/omega/src/ocn/Tendencies.h b/components/omega/src/ocn/Tendencies.h index 5cbd565c6f6f..68ebefca3af4 100644 --- a/components/omega/src/ocn/Tendencies.h +++ b/components/omega/src/ocn/Tendencies.h @@ -152,9 +152,9 @@ class Tendencies { // Construct a new tendency object Tendencies(const std::string &Name, ///< [in] Name for tendencies const HorzMesh *Mesh, ///< [in] Horizontal mesh - VertCoord *VCoord, ///< [in] Vertical coordinate - VertAdv *VAdv, ///< [in] Vertical advection - const PressureGrad *PGrad, ///< [in] Pressure gradient + VertCoord *VCoord, ///< [in] Vertical coordinate + VertAdv *VAdv, ///< [in] Vertical advection + PressureGrad *PGrad, ///< [in] Pressure gradient Eos *EqState, ///< [in] Equation of state int NTracersIn, ///< [in] Number of tracers TimeInterval TimeStep, ///< [in] Time step @@ -164,9 +164,9 @@ class Tendencies { Tendencies(const std::string &Name, ///< [in] Name for tendencies const HorzMesh *Mesh, ///< [in] Horizontal mesh - VertCoord *VCoord, ///< [in] Vertical coordinate - VertAdv *VAdv, ///< [in] Vertical advection - const PressureGrad *PGrad, ///< [in] Pressure gradient + VertCoord *VCoord, ///< [in] Vertical coordinate + VertAdv *VAdv, ///< [in] Vertical advection + PressureGrad *PGrad, ///< [in] Pressure gradient Eos *EqState, ///< [in] Equation of state int NTracersIn, ///< [in] Number of tracers TimeInterval TimeStep, ///< [in] Time step @@ -180,10 +180,10 @@ class Tendencies { Tendencies(Tendencies &&) = delete; const HorzMesh *Mesh; ///< Pointer to horizontal mesh - VertCoord *VCoord; ///< Pointer to vertical coordinate - VertAdv *VAdv; ///< Pointer to vertical advection - const PressureGrad *PGrad; ///< Pointer to pressure gradient - Eos *EqState; ///< Pointer to equation of state + VertCoord *VCoord; ///< Pointer to vertical coordinate + VertAdv *VAdv; ///< Pointer to vertical advection + PressureGrad *PGrad; ///< Pointer to pressure gradient + Eos *EqState; ///< Pointer to equation of state I4 NTracers; ///< Number of tracers TimeInterval TimeStep; ///< Time step From 91c6a74858db186b7ddc87cca91fd6b2bfdadef8 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Tue, 17 Feb 2026 07:57:06 -0600 Subject: [PATCH 32/58] Lintng of Tendencies.cpp --- components/omega/src/ocn/Tendencies.cpp | 132 ++++++++++-------------- 1 file changed, 57 insertions(+), 75 deletions(-) diff --git a/components/omega/src/ocn/Tendencies.cpp b/components/omega/src/ocn/Tendencies.cpp index 0f3be0956644..216e68227fad 100644 --- a/components/omega/src/ocn/Tendencies.cpp +++ b/components/omega/src/ocn/Tendencies.cpp @@ -77,8 +77,8 @@ void Tendencies::init() { // Ceate default tendencies Tendencies::DefaultTendencies = - create("Default", DefHorzMesh, DefVertCoord, DefVertAdv, DefPGrad, DefEos, NTracers, - TimeStep, &TendConfig, CustomThickTend, CustomVelTend); + create("Default", DefHorzMesh, DefVertCoord, DefVertAdv, DefPGrad, DefEos, + NTracers, TimeStep, &TendConfig, CustomThickTend, CustomVelTend); DefaultTendencies->readConfig(OmegaConfig); @@ -244,8 +244,7 @@ void Tendencies::readConfig(Config *OmegaConfig ///< [in] Omega config CHECK_ERROR_ABORT(Err, "Tendencies: EddyDiff4 not found in TendConfig"); } - Err += TendConfig->get("PressureGradTendencyEnable", - this->PGrad->Enabled); + Err += TendConfig->get("PressureGradTendencyEnable", this->PGrad->Enabled); CHECK_ERROR_ABORT( Err, "Tendencies: PressureGradTendencyEnable not found in TendConfig"); } @@ -253,72 +252,56 @@ void Tendencies::readConfig(Config *OmegaConfig ///< [in] Omega config //------------------------------------------------------------------------------ // Define fields associated with tendencies void Tendencies::defineFields() { - auto LayerThicknessTendFieldName = "LayerThicknessTend"; - auto NormalVelocityTendFieldName = "NormalVelocityTend"; - auto TracerTendFieldName = "TracerTend"; - - int NDims = 2; - std::vector DimNamesThickness(NDims); - DimNamesThickness[0] = "NCells"; - DimNamesThickness[1] = "NVertLayers"; - auto LayerThicknessTendField = Field::create(LayerThicknessTendFieldName, - "Layer thickness tendency", - "m/s", - "cell_thickness_tendency", - -9.99E+10, - 9.99E+10, - -9.99E+30, - NDims, - DimNamesThickness); - NDims = 3; - std::vector DimNamesTracer(NDims); - DimNamesTracer[0] = "NTracers"; - DimNamesTracer[1] = "NCells"; - DimNamesTracer[2] = "NVertLayers"; - auto TracerTendField = Field::create(TracerTendFieldName, - "Tracer tendency", - "kg/m^3/s", - "tracer_tendency", - -9.99E+10, - 9.99E+10, - -9.99E+30, - NDims, - DimNamesTracer); - NDims = 2; - std::vector DimNamesVelocity(NDims); - DimNamesVelocity[0] = "NEdges"; - DimNamesVelocity[1] = "NVertLayers"; - auto NormalVelocityTendField = Field::create(NormalVelocityTendFieldName, - "Normal velocity tendency", - "m/s^2", - "sea_water_velocity_tendency", - -9.99E+10, - 9.99E+10, - -9.99E+30, - NDims, - DimNamesVelocity); - - auto TendGroupName = "Tendencies"; - auto TendGroup = FieldGroup::create(TendGroupName); - - TendGroup->addField(LayerThicknessTendFieldName); - TendGroup->addField(NormalVelocityTendFieldName); - TendGroup->addField(TracerTendFieldName); - - LayerThicknessTendField->attachData(LayerThicknessTend); - NormalVelocityTendField->attachData(NormalVelocityTend); - TracerTendField->attachData(TracerTend); - -} // end defineFields + auto LayerThicknessTendFieldName = "LayerThicknessTend"; + auto NormalVelocityTendFieldName = "NormalVelocityTend"; + auto TracerTendFieldName = "TracerTend"; + + int NDims = 2; + std::vector DimNamesThickness(NDims); + DimNamesThickness[0] = "NCells"; + DimNamesThickness[1] = "NVertLayers"; + auto LayerThicknessTendField = + Field::create(LayerThicknessTendFieldName, "Layer thickness tendency", + "m/s", "cell_thickness_tendency", -9.99E+10, 9.99E+10, + -9.99E+30, NDims, DimNamesThickness); + NDims = 3; + std::vector DimNamesTracer(NDims); + DimNamesTracer[0] = "NTracers"; + DimNamesTracer[1] = "NCells"; + DimNamesTracer[2] = "NVertLayers"; + auto TracerTendField = Field::create( + TracerTendFieldName, "Tracer tendency", "kg/m^3/s", "tracer_tendency", + -9.99E+10, 9.99E+10, -9.99E+30, NDims, DimNamesTracer); + NDims = 2; + std::vector DimNamesVelocity(NDims); + DimNamesVelocity[0] = "NEdges"; + DimNamesVelocity[1] = "NVertLayers"; + auto NormalVelocityTendField = + Field::create(NormalVelocityTendFieldName, "Normal velocity tendency", + "m/s^2", "sea_water_velocity_tendency", -9.99E+10, + 9.99E+10, -9.99E+30, NDims, DimNamesVelocity); + + auto TendGroupName = "Tendencies"; + auto TendGroup = FieldGroup::create(TendGroupName); + + TendGroup->addField(LayerThicknessTendFieldName); + TendGroup->addField(NormalVelocityTendFieldName); + TendGroup->addField(TracerTendFieldName); + + LayerThicknessTendField->attachData(LayerThicknessTend); + NormalVelocityTendField->attachData(NormalVelocityTend); + TracerTendField->attachData(TracerTend); + +} // end defineFields //------------------------------------------------------------------------------ // Construct a new group of tendencies Tendencies::Tendencies(const std::string &Name, ///< [in] Name for tendencies const HorzMesh *Mesh, ///< [in] Horizontal mesh - VertCoord *VCoord, ///< [in] Vertical coordinate - VertAdv *VAdv, ///< [in] Vertical advection - PressureGrad *PGrad, ///< [in] Pressure gradient - Eos *EqState, ///< [in] Equation of state + VertCoord *VCoord, ///< [in] Vertical coordinate + VertAdv *VAdv, ///< [in] Vertical advection + PressureGrad *PGrad, ///< [in] Pressure gradient + Eos *EqState, ///< [in] Equation of state int NTracersIn, ///< [in] Number of tracers TimeInterval TimeStepIn, ///< [in] Time step Config *Options, ///< [in] Configuration options @@ -331,8 +314,7 @@ Tendencies::Tendencies(const std::string &Name, ///< [in] Name for tendencies BottomDrag(Mesh, VCoord), TracerDiffusion(Mesh, VCoord), TracerHyperDiff(Mesh, VCoord), TracerHorzAdv(Mesh, VCoord), CustomThicknessTend(InCustomThicknessTend), - CustomVelocityTend(InCustomVelocityTend), - EqState(EqState), PGrad(PGrad) { + CustomVelocityTend(InCustomVelocityTend), EqState(EqState), PGrad(PGrad) { // Tendency arrays LayerThicknessTend = @@ -351,10 +333,10 @@ Tendencies::Tendencies(const std::string &Name, ///< [in] Name for tendencies Tendencies::Tendencies(const std::string &Name, ///< [in] Name for tendencies const HorzMesh *Mesh, ///< [in] Horizontal mesh - VertCoord *VCoord, ///< [in] Vertical coordinate - VertAdv *VAdv, ///< [in] Vertical advection - PressureGrad *PGrad, ///< [in] Pressure gradient - Eos *EqState, ///< [in] Equation of state + VertCoord *VCoord, ///< [in] Vertical coordinate + VertAdv *VAdv, ///< [in] Vertical advection + PressureGrad *PGrad, ///< [in] Pressure gradient + Eos *EqState, ///< [in] Equation of state int NTracersIn, ///< [in] Number of tracers TimeInterval TimeStepIn, ///< [in] Time step Config *Options) ///< [in] Configuration options @@ -613,7 +595,7 @@ void Tendencies::computeVelocityTendenciesOnly( Array2DReal Temp; Array2DReal Salinity; Array2DReal LayerThick; - + // Temporary handling of surface pressure Array1DReal SurfacePressure("SurfacePressure", Mesh->NCellsSize); deepCopy(SurfacePressure, 0.0_Real); @@ -624,10 +606,10 @@ void Tendencies::computeVelocityTendenciesOnly( OMEGA_SCOPE(LocPressureMid, VCoord->PressureMid); Err = Tracers::getByName(Temp, VelTimeLevel, "Temperature"); Err = Tracers::getByName(Salinity, VelTimeLevel, "Salinity"); - + EqState->computeSpecVol(Temp, Salinity, LocPressureMid); - PGrad->computePressureGrad( - LocNormalVelocityTend, State, VCoord, EqState, VelTimeLevel); + PGrad->computePressureGrad(LocNormalVelocityTend, State, VCoord, EqState, + VelTimeLevel); Pacer::stop("Tend:pressureGradTerm", 2); } From 2f9bb86c72e7ea471b6be045556aa2500090b3d9 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Tue, 17 Feb 2026 07:57:56 -0600 Subject: [PATCH 33/58] Add temporary call to computeZHeight() in pgrad tendency --- components/omega/src/ocn/Tendencies.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/components/omega/src/ocn/Tendencies.cpp b/components/omega/src/ocn/Tendencies.cpp index 216e68227fad..9625c6853c37 100644 --- a/components/omega/src/ocn/Tendencies.cpp +++ b/components/omega/src/ocn/Tendencies.cpp @@ -607,6 +607,10 @@ void Tendencies::computeVelocityTendenciesOnly( Err = Tracers::getByName(Temp, VelTimeLevel, "Temperature"); Err = Tracers::getByName(Salinity, VelTimeLevel, "Salinity"); + // Temporary: ensure vertical geometric/geopotential fields are updated + // for pressure-gradient tendency calculations. + VCoord->computeZHeight(LayerThick, EqState->SpecVol); + EqState->computeSpecVol(Temp, Salinity, LocPressureMid); PGrad->computePressureGrad(LocNormalVelocityTend, State, VCoord, EqState, VelTimeLevel); From ad892fa36ff4d3e62a2bc84f75ffc65993cefc5e Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Tue, 17 Feb 2026 11:46:12 -0600 Subject: [PATCH 34/58] Compute geometric z after specific volume --- components/omega/src/ocn/Tendencies.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/omega/src/ocn/Tendencies.cpp b/components/omega/src/ocn/Tendencies.cpp index 9625c6853c37..809c80c98a4b 100644 --- a/components/omega/src/ocn/Tendencies.cpp +++ b/components/omega/src/ocn/Tendencies.cpp @@ -607,11 +607,12 @@ void Tendencies::computeVelocityTendenciesOnly( Err = Tracers::getByName(Temp, VelTimeLevel, "Temperature"); Err = Tracers::getByName(Salinity, VelTimeLevel, "Salinity"); + EqState->computeSpecVol(Temp, Salinity, LocPressureMid); + // Temporary: ensure vertical geometric/geopotential fields are updated // for pressure-gradient tendency calculations. VCoord->computeZHeight(LayerThick, EqState->SpecVol); - EqState->computeSpecVol(Temp, Salinity, LocPressureMid); PGrad->computePressureGrad(LocNormalVelocityTend, State, VCoord, EqState, VelTimeLevel); Pacer::stop("Tend:pressureGradTerm", 2); From f541d5de6e305a3fd47ec1a2d5e42020c7e820ca Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Tue, 17 Feb 2026 11:50:31 -0600 Subject: [PATCH 35/58] Lint pressure-gradient code --- components/omega/src/ocn/PGrad.cpp | 48 +++++++++++----------- components/omega/src/ocn/PGrad.h | 65 +++++++++++++++++------------- 2 files changed, 60 insertions(+), 53 deletions(-) diff --git a/components/omega/src/ocn/PGrad.cpp b/components/omega/src/ocn/PGrad.cpp index f2afcdf36557..5d9c2056bd37 100644 --- a/components/omega/src/ocn/PGrad.cpp +++ b/components/omega/src/ocn/PGrad.cpp @@ -10,8 +10,8 @@ #include "Error.h" #include "Field.h" #include "HorzMesh.h" -#include "VertCoord.h" #include "OmegaKokkos.h" +#include "VertCoord.h" namespace OMEGA { @@ -76,15 +76,17 @@ PressureGrad::PressureGrad( const HorzMesh *Mesh, ///< [in] Horizontal mesh const VertCoord *VCoord, ///< [in] Vertical coordinate Config *Options) ///< [in] Configuration options - : CellsOnEdge(Mesh->CellsOnEdge), DcEdge(Mesh->DcEdge), EdgeSignOnCell(Mesh->EdgeSignOnCell), - MinLayerEdgeBot(VCoord->MinLayerEdgeBot), MaxLayerEdgeTop(VCoord->MaxLayerEdgeTop), - CenteredPGrad(Mesh, VCoord), HighOrderPGrad(Mesh, VCoord) { + : CellsOnEdge(Mesh->CellsOnEdge), DcEdge(Mesh->DcEdge), + EdgeSignOnCell(Mesh->EdgeSignOnCell), + MinLayerEdgeBot(VCoord->MinLayerEdgeBot), + MaxLayerEdgeTop(VCoord->MaxLayerEdgeTop), CenteredPGrad(Mesh, VCoord), + HighOrderPGrad(Mesh, VCoord) { // store mesh sizes - NEdgesAll = Mesh->NEdgesAll; - NVertLayers = VCoord->NVertLayers; + NEdgesAll = Mesh->NEdgesAll; + NVertLayers = VCoord->NVertLayers; NVertLayersP1 = NVertLayers + 1; - NChunks = NVertLayers / VecLength; + NChunks = NVertLayers / VecLength; // Read config options for PressureGrad type // and enable the appropriate functor @@ -175,16 +177,14 @@ void PressureGrad::computePressureGrad(Array2DReal &Tend, parallelForOuter( "pgrad-centered", {NEdgesAll}, KOKKOS_LAMBDA(I4 IEdge, const TeamMember &Team) { - const int KMin = LocMinLayerEdgeBot(IEdge); - const int KMax = LocMaxLayerEdgeTop(IEdge); + const int KMin = LocMinLayerEdgeBot(IEdge); + const int KMax = LocMaxLayerEdgeTop(IEdge); const int KRange = vertRange(KMin, KMax); parallelForInner( - Team, KRange, - INNER_LAMBDA(int KChunk) { + Team, KRange, INNER_LAMBDA(int KChunk) { LocCenteredPGrad(Tend, IEdge, KChunk, PressureMid, - PressureInterface, ZInterface, - LayerThick, + PressureInterface, ZInterface, LayerThick, SpecVol); }); }); @@ -195,35 +195,35 @@ void PressureGrad::computePressureGrad(Array2DReal &Tend, parallelForOuter( "pgrad-highorder", {NEdgesAll}, KOKKOS_LAMBDA(I4 IEdge, const TeamMember &Team) { - const int KMin = MinLayerEdgeBot(IEdge); - const int KMax = MaxLayerEdgeTop(IEdge); + const int KMin = MinLayerEdgeBot(IEdge); + const int KMax = MaxLayerEdgeTop(IEdge); const int KRange = vertRange(KMin, KMax); parallelForInner( - Team, KRange, - INNER_LAMBDA(int KChunk) { + Team, KRange, INNER_LAMBDA(int KChunk) { LocHighOrderPGrad(Tend, IEdge, KChunk, PressureMid, Geopotential, SpecVol); }); }); - } } // end compute pressure gradient //------------------------------------------------------------------------------ // Constructor for centered pressure gradient functor -PressureGradCentered::PressureGradCentered(const HorzMesh *Mesh, ///< [in] Horizontal mesh - const VertCoord *VCoord ///< [in] Vertical coordinate - ) +PressureGradCentered::PressureGradCentered( + const HorzMesh *Mesh, ///< [in] Horizontal mesh + const VertCoord *VCoord ///< [in] Vertical coordinate + ) : CellsOnEdge(Mesh->CellsOnEdge), DcEdge(Mesh->DcEdge), EdgeMask(VCoord->EdgeMask), MinLayerEdgeBot(VCoord->MinLayerEdgeBot), MaxLayerEdgeTop(VCoord->MaxLayerEdgeTop) {} //------------------------------------------------------------------------------ // Constructor for high order pressure gradient functor -PressureGradHighOrder::PressureGradHighOrder(const HorzMesh *Mesh, ///< [in] Horizontal mesh - const VertCoord *VCoord ///< [in] Vertical coordinate - ) +PressureGradHighOrder::PressureGradHighOrder( + const HorzMesh *Mesh, ///< [in] Horizontal mesh + const VertCoord *VCoord ///< [in] Vertical coordinate + ) : CellsOnEdge(Mesh->CellsOnEdge), DcEdge(Mesh->DcEdge), EdgeMask(VCoord->EdgeMask), MinLayerEdgeBot(VCoord->MinLayerEdgeBot), MaxLayerEdgeTop(VCoord->MaxLayerEdgeTop) {} diff --git a/components/omega/src/ocn/PGrad.h b/components/omega/src/ocn/PGrad.h index 31e9386fc36c..629059c9cfec 100644 --- a/components/omega/src/ocn/PGrad.h +++ b/components/omega/src/ocn/PGrad.h @@ -26,9 +26,9 @@ class PressureGradCentered { bool Enabled; // constructor declaration - PressureGradCentered(const HorzMesh *Mesh, ///< [in] Horizontal mesh - const VertCoord *VCoord ///< [in] Vertical coordinate - ); + PressureGradCentered(const HorzMesh *Mesh, ///< [in] Horizontal mesh + const VertCoord *VCoord ///< [in] Vertical coordinate + ); // Compute centered pressure gradient contribution for given edge and // vertical chunk. This appends results into the Tend array (in-place). @@ -37,34 +37,43 @@ class PressureGradCentered { const Array2DReal &PressureInterface, const Array2DReal &ZInterface, const Array2DReal &LayerThick, - const Array2DReal &SpecVol - ) const { + const Array2DReal &SpecVol) const { const I4 KStart = chunkStart(KChunk, MinLayerEdgeBot(IEdge)); const I4 KLen = chunkLength(KChunk, KStart, MaxLayerEdgeTop(IEdge)); - + const I4 ICell0 = CellsOnEdge(IEdge, 0); const I4 ICell1 = CellsOnEdge(IEdge, 1); const Real InvDcEdge = 1.0_Real / DcEdge(IEdge); - Real Rho0 = 1026.0_Real; - Real Gravity = 9.80616_Real; + Real Rho0 = 1026.0_Real; + Real Gravity = 9.80616_Real; for (int KVec = 0; KVec < KLen; ++KVec) { const I4 K = KStart + KVec; - Real MontPotCell0K = PressureInterface(ICell0, K) * SpecVol(ICell0, K) + Gravity * ZInterface(ICell0, K); - Real MontPotCell1K = PressureInterface(ICell1, K) * SpecVol(ICell1, K) + Gravity * ZInterface(ICell1, K); + Real MontPotCell0K = + PressureInterface(ICell0, K) * SpecVol(ICell0, K) + + Gravity * ZInterface(ICell0, K); + Real MontPotCell1K = + PressureInterface(ICell1, K) * SpecVol(ICell1, K) + + Gravity * ZInterface(ICell1, K); Real GradMontPotK = (MontPotCell1K - MontPotCell0K) * InvDcEdge; - Real MontPotCell0Kp1 = PressureInterface(ICell0, K+1) * SpecVol(ICell0, K) + Gravity * ZInterface(ICell0, K+1); - Real MontPotCell1Kp1 = PressureInterface(ICell1, K+1) * SpecVol(ICell1, K) + Gravity * ZInterface(ICell1, K+1); + Real MontPotCell0Kp1 = + PressureInterface(ICell0, K + 1) * SpecVol(ICell0, K) + + Gravity * ZInterface(ICell0, K + 1); + Real MontPotCell1Kp1 = + PressureInterface(ICell1, K + 1) * SpecVol(ICell1, K) + + Gravity * ZInterface(ICell1, K + 1); Real GradMontPotKp1 = (MontPotCell1Kp1 - MontPotCell0Kp1) * InvDcEdge; - Real GradMontPot = 0.5_Real * (GradMontPotK + GradMontPotKp1); - - Real PGradAlpha = 0.5 * (PressureMid(ICell1, K) + PressureMid(ICell0, K)) * (SpecVol(ICell1, K) - SpecVol(ICell0, K)) * InvDcEdge; - Tend(IEdge, K) += - EdgeMask(IEdge, K) * (-GradMontPot + PGradAlpha); - if (IEdge == 0) - LOG_INFO("IEdge {}, K {}: GradMontPot {}, PGradAlpha {}, Tend {}", IEdge, K, GradMontPot, PGradAlpha, Tend(IEdge, K)); + Real GradMontPot = 0.5_Real * (GradMontPotK + GradMontPotKp1); + + Real PGradAlpha = + 0.5 * (PressureMid(ICell1, K) + PressureMid(ICell0, K)) * + (SpecVol(ICell1, K) - SpecVol(ICell0, K)) * InvDcEdge; + Tend(IEdge, K) += EdgeMask(IEdge, K) * (-GradMontPot + PGradAlpha); + if (IEdge == 0) + LOG_INFO("IEdge {}, K {}: GradMontPot {}, PGradAlpha {}, Tend {}", + IEdge, K, GradMontPot, PGradAlpha, Tend(IEdge, K)); } } @@ -82,9 +91,9 @@ class PressureGradHighOrder { bool Enabled; // constructor declaration - PressureGradHighOrder(const HorzMesh *Mesh, ///< [in] Horizontal mesh - const VertCoord *VCoord ///< [in] Vertical coordinate - ); + PressureGradHighOrder(const HorzMesh *Mesh, ///< [in] Horizontal mesh + const VertCoord *VCoord ///< [in] Vertical coordinate + ); KOKKOS_FUNCTION void operator()(const Array2DReal &Tend, I4 IEdge, I4 KChunk, const Array2DReal &Pressure, @@ -112,7 +121,6 @@ class PressureGradHighOrder { // Pressure gradient manager class class PressureGrad { public: - // Flag to indicate if pressure gradient term is enabled bool Enabled; @@ -145,7 +153,6 @@ class PressureGrad { const VertCoord *VCoord, const Eos *EqState, const int TimeLevel) const; - private: // Construct a new pressure gradient object PressureGrad(const HorzMesh *Mesh, const VertCoord *VCoord, Config *Options); @@ -158,17 +165,17 @@ class PressureGrad { static PressureGrad *DefaultPGrad; // Mesh-related sizes - I4 NEdgesAll = 0; - I4 NChunks = 0; - I4 NVertLayers = 0; + I4 NEdgesAll = 0; + I4 NChunks = 0; + I4 NVertLayers = 0; I4 NVertLayersP1 = 0; // Data required for computation (stored copies of mesh/VCoord arrays) Array2DI4 CellsOnEdge; ///< cells surrounding each edge Array1DReal DcEdge; ///< distance between cell centers across edge Array2DReal EdgeSignOnCell; ///< orientation of edge relative to cell - Array1DI4 MinLayerEdgeBot; ///< min vertical layer on each edge - Array1DI4 MaxLayerEdgeTop; ///< max vertical layer on each edge + Array1DI4 MinLayerEdgeBot; ///< min vertical layer on each edge + Array1DI4 MaxLayerEdgeTop; ///< max vertical layer on each edge // Instances of functors PressureGradCentered CenteredPGrad; From 52d7f765da37d25ff9e40795dd4a5b78862be3ff Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Tue, 17 Feb 2026 11:52:34 -0600 Subject: [PATCH 36/58] Use Gravity from GlobalConstants in PGrad Drop unused Rho0 --- components/omega/src/ocn/PGrad.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/components/omega/src/ocn/PGrad.h b/components/omega/src/ocn/PGrad.h index 629059c9cfec..21fb500ca20c 100644 --- a/components/omega/src/ocn/PGrad.h +++ b/components/omega/src/ocn/PGrad.h @@ -10,6 +10,7 @@ #include "Config.h" #include "Eos.h" +#include "GlobalConstants.h" #include "HorzMesh.h" #include "OceanState.h" #include "OmegaKokkos.h" @@ -45,8 +46,6 @@ class PressureGradCentered { const I4 ICell0 = CellsOnEdge(IEdge, 0); const I4 ICell1 = CellsOnEdge(IEdge, 1); const Real InvDcEdge = 1.0_Real / DcEdge(IEdge); - Real Rho0 = 1026.0_Real; - Real Gravity = 9.80616_Real; for (int KVec = 0; KVec < KLen; ++KVec) { const I4 K = KStart + KVec; From 4dbbc50be19639f46555a76838c8b6c60da6c3d1 Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Thu, 19 Feb 2026 12:51:51 -0800 Subject: [PATCH 37/58] Initialize/clear Tendencies in IOStreamTest --- components/omega/test/infra/IOStreamTest.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/components/omega/test/infra/IOStreamTest.cpp b/components/omega/test/infra/IOStreamTest.cpp index ff89d97f71b5..9e3c1823344e 100644 --- a/components/omega/test/infra/IOStreamTest.cpp +++ b/components/omega/test/infra/IOStreamTest.cpp @@ -26,6 +26,7 @@ #include "Pacer.h" #include "TimeMgr.h" #include "TimeStepper.h" +#include "Tendencies.h" #include "Tracers.h" #include "VertCoord.h" #include "mpi.h" @@ -103,6 +104,9 @@ void initIOStreamTest(Clock *&ModelClock // Model clock if (TmpErr != 0) ABORT_ERROR("IOStreamTest: Error initializing OceanState"); + // Intialize Tendencies + Tendencies::init(); + // Initialize Tracers Tracers::init(); @@ -220,6 +224,7 @@ int main(int argc, char **argv) { // Clean up environments TimeStepper::clear(); + Tendencies::clear(); Tracers::clear(); OceanState::clear(); AuxiliaryState::clear(); From 188ee3508d40c53852fae791f67a95a95fdcaf92 Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Mon, 23 Feb 2026 07:47:37 -0800 Subject: [PATCH 38/58] Add gradient of tidal potential and SAL --- components/omega/src/ocn/OceanState.cpp | 3 +++ components/omega/src/ocn/PGrad.cpp | 17 ++++++++++--- components/omega/src/ocn/PGrad.h | 34 +++++++++++++++++-------- 3 files changed, 39 insertions(+), 15 deletions(-) diff --git a/components/omega/src/ocn/OceanState.cpp b/components/omega/src/ocn/OceanState.cpp index aef68a66ef51..4d3e7ce8919a 100644 --- a/components/omega/src/ocn/OceanState.cpp +++ b/components/omega/src/ocn/OceanState.cpp @@ -44,6 +44,9 @@ int OceanState::init() { LOG_ERROR("TimeStepper needs to be initialized before OceanState"); } int NTimeLevels = DefTimeStepper->getNTimeLevels(); + LOG_INFO("OceanState: Initializing default state with {} vertical layers " + "and {} time levels", + NVertLayers, NTimeLevels); if (NTimeLevels < 2) { LOG_ERROR("OceanState: the number of time level is lower than 2"); diff --git a/components/omega/src/ocn/PGrad.cpp b/components/omega/src/ocn/PGrad.cpp index 5d9c2056bd37..66b50923f438 100644 --- a/components/omega/src/ocn/PGrad.cpp +++ b/components/omega/src/ocn/PGrad.cpp @@ -109,6 +109,10 @@ PressureGrad::PressureGrad( "PGrad: Unknown PressureGradType in config, defaulting to centered"); } + // Temporary: initialization of tidal potential and SAL + TidalPotential = Array1DReal("TidalPotential", Mesh->NCellsSize); + SelfAttractionLoading = Array1DReal("SelfAttractionLoading", Mesh->NCellsSize); + } // end constructor //------------------------------------------------------------------------------ @@ -161,6 +165,8 @@ void PressureGrad::computePressureGrad(Array2DReal &Tend, OMEGA_SCOPE(LocHighOrderPGrad, HighOrderPGrad); OMEGA_SCOPE(LocMinLayerEdgeBot, MinLayerEdgeBot); OMEGA_SCOPE(LocMaxLayerEdgeTop, MaxLayerEdgeTop); + OMEGA_SCOPE(LocTidalPotential, TidalPotential); + OMEGA_SCOPE(LocSelfAttractionLoading, SelfAttractionLoading); const Array2DReal &PressureMid = VCoord->PressureMid; const Array2DReal &PressureInterface = VCoord->PressureInterface; @@ -184,7 +190,8 @@ void PressureGrad::computePressureGrad(Array2DReal &Tend, parallelForInner( Team, KRange, INNER_LAMBDA(int KChunk) { LocCenteredPGrad(Tend, IEdge, KChunk, PressureMid, - PressureInterface, ZInterface, LayerThick, + PressureInterface, ZInterface, + LocTidalPotential, LocSelfAttractionLoading, SpecVol); }); }); @@ -195,14 +202,16 @@ void PressureGrad::computePressureGrad(Array2DReal &Tend, parallelForOuter( "pgrad-highorder", {NEdgesAll}, KOKKOS_LAMBDA(I4 IEdge, const TeamMember &Team) { - const int KMin = MinLayerEdgeBot(IEdge); - const int KMax = MaxLayerEdgeTop(IEdge); + const int KMin = LocMinLayerEdgeBot(IEdge); + const int KMax = LocMaxLayerEdgeTop(IEdge); const int KRange = vertRange(KMin, KMax); parallelForInner( Team, KRange, INNER_LAMBDA(int KChunk) { LocHighOrderPGrad(Tend, IEdge, KChunk, PressureMid, - Geopotential, SpecVol); + PressureInterface, ZInterface, + LocTidalPotential, LocSelfAttractionLoading, + SpecVol); }); }); } diff --git a/components/omega/src/ocn/PGrad.h b/components/omega/src/ocn/PGrad.h index 21fb500ca20c..7d20b5927614 100644 --- a/components/omega/src/ocn/PGrad.h +++ b/components/omega/src/ocn/PGrad.h @@ -37,8 +37,10 @@ class PressureGradCentered { const Array2DReal &PressureMid, const Array2DReal &PressureInterface, const Array2DReal &ZInterface, - const Array2DReal &LayerThick, - const Array2DReal &SpecVol) const { + const Array1DReal &TidalPotential, + const Array1DReal &SelfAttractionLoading, + const Array2DReal &SpecVol + ) const { const I4 KStart = chunkStart(KChunk, MinLayerEdgeBot(IEdge)); const I4 KLen = chunkLength(KChunk, KStart, MaxLayerEdgeTop(IEdge)); @@ -47,6 +49,9 @@ class PressureGradCentered { const I4 ICell1 = CellsOnEdge(IEdge, 1); const Real InvDcEdge = 1.0_Real / DcEdge(IEdge); + Real GradGeoPot = (TidalPotential(ICell1) - TidalPotential(ICell0)) * InvDcEdge + + (SelfAttractionLoading(ICell1) - SelfAttractionLoading(ICell0)) * InvDcEdge; + for (int KVec = 0; KVec < KLen; ++KVec) { const I4 K = KStart + KVec; Real MontPotCell0K = @@ -69,10 +74,10 @@ class PressureGradCentered { Real PGradAlpha = 0.5 * (PressureMid(ICell1, K) + PressureMid(ICell0, K)) * (SpecVol(ICell1, K) - SpecVol(ICell0, K)) * InvDcEdge; - Tend(IEdge, K) += EdgeMask(IEdge, K) * (-GradMontPot + PGradAlpha); - if (IEdge == 0) - LOG_INFO("IEdge {}, K {}: GradMontPot {}, PGradAlpha {}, Tend {}", - IEdge, K, GradMontPot, PGradAlpha, Tend(IEdge, K)); + Tend(IEdge, K) += + EdgeMask(IEdge, K) * (-GradMontPot + PGradAlpha - GradGeoPot); + //if (IEdge == 0) + //LOG_INFO("IEdge {}, K {}: GradMontPot {}, PGradAlpha {}, Tend {}", IEdge, K, GradMontPot, PGradAlpha, Tend(IEdge, K)); } } @@ -95,8 +100,11 @@ class PressureGradHighOrder { ); KOKKOS_FUNCTION void operator()(const Array2DReal &Tend, I4 IEdge, I4 KChunk, - const Array2DReal &Pressure, - const Array2DReal &Geopotential, + const Array2DReal &PressureMid, + const Array2DReal &PressureInterface, + const Array2DReal &ZInterface, + const Array1DReal &TidalPotential, + const Array1DReal &SelfAttractionLoading, const Array2DReal &SpecVol) const { // Placeholder: for now, no-op (future high-order implementation) @@ -134,21 +142,21 @@ class PressureGrad { static PressureGrad *getDefault(); // Get instance by name - static PressureGrad *get(const std::string &Name ///< [in] + static PressureGrad *get(const std::string &Name ///< [in] Name of PressureGrad ); // Deallocates arrays and deletes instance static void clear(); // Remove pressure gradient object by name - static void erase(const std::string &Name ///< [in] + static void erase(const std::string &Name ///< [in] Name of PressureGrad ); // Destructor ~PressureGrad(); // Compute pressure gradient tendencies and add into Tend array - void computePressureGrad(Array2DReal &Tend, const OceanState *State, + void computePressureGrad(Array2DReal &Tend, const OceanState *State, const VertCoord *VCoord, const Eos *EqState, const int TimeLevel) const; @@ -176,6 +184,10 @@ class PressureGrad { Array1DI4 MinLayerEdgeBot; ///< min vertical layer on each edge Array1DI4 MaxLayerEdgeTop; ///< max vertical layer on each edge + // Temporary: to be moveed to tidal forcing module in future + Array1DReal TidalPotential; ///< Tidal potential for tidal forcing + Array1DReal SelfAttractionLoading; ///< Self attraction and loading for tidal forcing + // Instances of functors PressureGradCentered CenteredPGrad; PressureGradHighOrder HighOrderPGrad; From 2fb9994609b14b2c66f0115d8770d7a4d7e38ce8 Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Mon, 23 Feb 2026 10:04:24 -0800 Subject: [PATCH 39/58] Fix creation of tendency fields with same name --- components/omega/src/ocn/Tendencies.cpp | 56 +++++++++++++++---------- components/omega/src/ocn/Tendencies.h | 2 + 2 files changed, 35 insertions(+), 23 deletions(-) diff --git a/components/omega/src/ocn/Tendencies.cpp b/components/omega/src/ocn/Tendencies.cpp index 809c80c98a4b..eec5ec015b1b 100644 --- a/components/omega/src/ocn/Tendencies.cpp +++ b/components/omega/src/ocn/Tendencies.cpp @@ -252,9 +252,14 @@ void Tendencies::readConfig(Config *OmegaConfig ///< [in] Omega config //------------------------------------------------------------------------------ // Define fields associated with tendencies void Tendencies::defineFields() { - auto LayerThicknessTendFieldName = "LayerThicknessTend"; - auto NormalVelocityTendFieldName = "NormalVelocityTend"; - auto TracerTendFieldName = "TracerTend"; + std::string LayerThicknessTendFieldName = "LayerThicknessTend"; + std::string NormalVelocityTendFieldName = "NormalVelocityTend"; + std::string TracerTendFieldName = "TracerTend"; + if (Name != "Default") { + LayerThicknessTendFieldName.append(Name); + NormalVelocityTendFieldName.append(Name); + TracerTendFieldName.append(Name); + } int NDims = 2; std::vector DimNamesThickness(NDims); @@ -281,7 +286,10 @@ void Tendencies::defineFields() { "m/s^2", "sea_water_velocity_tendency", -9.99E+10, 9.99E+10, -9.99E+30, NDims, DimNamesVelocity); - auto TendGroupName = "Tendencies"; + std::string TendGroupName = "Tendencies"; + if (Name != "Default") { + TendGroupName.append(Name); + } auto TendGroup = FieldGroup::create(TendGroupName); TendGroup->addField(LayerThicknessTendFieldName); @@ -296,15 +304,15 @@ void Tendencies::defineFields() { //------------------------------------------------------------------------------ // Construct a new group of tendencies -Tendencies::Tendencies(const std::string &Name, ///< [in] Name for tendencies - const HorzMesh *Mesh, ///< [in] Horizontal mesh - VertCoord *VCoord, ///< [in] Vertical coordinate - VertAdv *VAdv, ///< [in] Vertical advection - PressureGrad *PGrad, ///< [in] Pressure gradient - Eos *EqState, ///< [in] Equation of state - int NTracersIn, ///< [in] Number of tracers - TimeInterval TimeStepIn, ///< [in] Time step - Config *Options, ///< [in] Configuration options +Tendencies::Tendencies(const std::string &Name_, ///< [in] Name for tendencies + const HorzMesh *Mesh, ///< [in] Horizontal mesh + VertCoord *VCoord, ///< [in] Vertical coordinate + VertAdv *VAdv, ///< [in] Vertical advection + PressureGrad *PGrad, ///< [in] Pressure gradient + Eos *EqState, ///< [in] Equation of state + int NTracersIn, ///< [in] Number of tracers + TimeInterval TimeStepIn, ///< [in] Time step + Config *Options, ///< [in] Configuration options CustomTendencyType InCustomThicknessTend, CustomTendencyType InCustomVelocityTend) : Mesh(Mesh), VCoord(VCoord), VAdv(VAdv), ThicknessFluxDiv(Mesh, VCoord), @@ -324,6 +332,8 @@ Tendencies::Tendencies(const std::string &Name, ///< [in] Name for tendencies TracerTend = Array3DReal("TracerTend", NTracersIn, Mesh->NCellsSize, VCoord->NVertLayers); + Name = Name_; + NTracers = NTracersIn; TimeStep = TimeStepIn; @@ -331,16 +341,16 @@ Tendencies::Tendencies(const std::string &Name, ///< [in] Name for tendencies } // end constructor -Tendencies::Tendencies(const std::string &Name, ///< [in] Name for tendencies - const HorzMesh *Mesh, ///< [in] Horizontal mesh - VertCoord *VCoord, ///< [in] Vertical coordinate - VertAdv *VAdv, ///< [in] Vertical advection - PressureGrad *PGrad, ///< [in] Pressure gradient - Eos *EqState, ///< [in] Equation of state - int NTracersIn, ///< [in] Number of tracers - TimeInterval TimeStepIn, ///< [in] Time step - Config *Options) ///< [in] Configuration options - : Tendencies(Name, Mesh, VCoord, VAdv, PGrad, EqState, NTracersIn, TimeStepIn, Options, +Tendencies::Tendencies(const std::string &Name_, ///< [in] Name for tendencies + const HorzMesh *Mesh, ///< [in] Horizontal mesh + VertCoord *VCoord, ///< [in] Vertical coordinate + VertAdv *VAdv, ///< [in] Vertical advection + PressureGrad *PGrad, ///< [in] Pressure gradient + Eos *EqState, ///< [in] Equation of state + int NTracersIn, ///< [in] Number of tracers + TimeInterval TimeStepIn, ///< [in] Time step + Config *Options) ///< [in] Configuration options + : Tendencies(Name_, Mesh, VCoord, VAdv, PGrad, EqState, NTracersIn, TimeStepIn, Options, CustomTendencyType{}, CustomTendencyType{}) {} //------------------------------------------------------------------------------ diff --git a/components/omega/src/ocn/Tendencies.h b/components/omega/src/ocn/Tendencies.h index 68ebefca3af4..882f04c61ea4 100644 --- a/components/omega/src/ocn/Tendencies.h +++ b/components/omega/src/ocn/Tendencies.h @@ -71,6 +71,8 @@ class Tendencies { TracerDiffOnCell TracerDiffusion; TracerHyperDiffOnCell TracerHyperDiff; + std::string Name; + // Methods to compute tendency groups void computeThicknessTendencies(const OceanState *State, const AuxiliaryState *AuxState, From b9b31792dbd4be973595a1cf6536f16147cdf93c Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Mon, 23 Feb 2026 10:05:33 -0800 Subject: [PATCH 40/58] Initialize pressure gradient in state ctest --- components/omega/test/ocn/StateTest.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/components/omega/test/ocn/StateTest.cpp b/components/omega/test/ocn/StateTest.cpp index 0c2f631db277..15ce8d3e3169 100644 --- a/components/omega/test/ocn/StateTest.cpp +++ b/components/omega/test/ocn/StateTest.cpp @@ -24,6 +24,8 @@ #include "OceanState.h" #include "OmegaKokkos.h" #include "Pacer.h" +#include "PGrad.h" +#include "Tendencies.h" #include "TimeStepper.h" #include "VertCoord.h" #include "mpi.h" @@ -90,6 +92,9 @@ void initStateTest() { // Initialize Aux State variables AuxiliaryState::init(); + // Initialize pressure gradient + PressureGrad::init(); + // Create tendencies Tendencies::init(); @@ -399,6 +404,7 @@ int main(int argc, char *argv[]) { OceanState::clear(); Tracers::clear(); AuxiliaryState::clear(); + PressureGrad::clear(); Tendencies::clear(); TimeStepper::clear(); HorzMesh::clear(); From bf37ee5b367e5ad037ac5389eccf80714d75f4c4 Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Mon, 23 Feb 2026 11:19:16 -0800 Subject: [PATCH 41/58] Fix IOStreamTest --- components/omega/test/infra/IOStreamTest.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/components/omega/test/infra/IOStreamTest.cpp b/components/omega/test/infra/IOStreamTest.cpp index 9e3c1823344e..ee0cd4608a41 100644 --- a/components/omega/test/infra/IOStreamTest.cpp +++ b/components/omega/test/infra/IOStreamTest.cpp @@ -24,6 +24,7 @@ #include "OceanState.h" #include "OmegaKokkos.h" #include "Pacer.h" +#include "PGrad.h" #include "TimeMgr.h" #include "TimeStepper.h" #include "Tendencies.h" @@ -104,6 +105,8 @@ void initIOStreamTest(Clock *&ModelClock // Model clock if (TmpErr != 0) ABORT_ERROR("IOStreamTest: Error initializing OceanState"); + PressureGrad::init(); + // Intialize Tendencies Tendencies::init(); @@ -224,6 +227,7 @@ int main(int argc, char **argv) { // Clean up environments TimeStepper::clear(); + PressureGrad::clear(); Tendencies::clear(); Tracers::clear(); OceanState::clear(); From ebf9999920a71519ad714e7dac015af124228fcc Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Mon, 23 Feb 2026 13:13:56 -0800 Subject: [PATCH 42/58] Fix PGradTest for GPU --- components/omega/test/ocn/PGradTest.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/components/omega/test/ocn/PGradTest.cpp b/components/omega/test/ocn/PGradTest.cpp index fb5e34f6af26..eb1033448c03 100644 --- a/components/omega/test/ocn/PGradTest.cpp +++ b/components/omega/test/ocn/PGradTest.cpp @@ -188,9 +188,12 @@ int main(int argc, char *argv[]) { LOG_INFO("NVertLayers = {}", NVertLayers); LOG_INFO("dC = {}", dC); + DefState->copyToHost(0); + HostArray2DReal LayerThickH; + DefState->getLayerThicknessH(LayerThickH,0); for (int i = 0; i < 2; ++i) { for (int k = 0; k < 2; ++k) { - LOG_INFO("LayerThick({}, {}) = {}", i, k, LayerThick(i, k)); + LOG_INFO("LayerThick({}, {}) = {}", i, k, LayerThickH(i, k)); } } From 1421f7ae982d14381bbcbd364b3c81c05091d93c Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Mon, 23 Feb 2026 15:27:43 -0800 Subject: [PATCH 43/58] Add pass/fail criteria to PGradTest --- components/omega/test/ocn/PGradTest.cpp | 35 ++++++++++++++++--------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/components/omega/test/ocn/PGradTest.cpp b/components/omega/test/ocn/PGradTest.cpp index eb1033448c03..8045c88669e0 100644 --- a/components/omega/test/ocn/PGradTest.cpp +++ b/components/omega/test/ocn/PGradTest.cpp @@ -96,7 +96,7 @@ int main(int argc, char *argv[]) { { initPGradTest(); - // Initialize PressureGrad manager + // Initialize default PressureGrad PressureGrad::init(); @@ -120,7 +120,9 @@ int main(int argc, char *argv[]) { I4 NVertLayers = 60; Real dC = 30000.0_Real; - for (int refinement = 0; refinement < 4; ++refinement) { + I4 NRefinements = 4; + HostArray1DReal RMSE("RMSE", NRefinements); + for (int refinement = 0; refinement < NRefinements; ++refinement) { LOG_INFO("PGradTest: Starting refinement level {}", refinement); VCoord->NVertLayers = NVertLayers; @@ -253,15 +255,10 @@ int main(int argc, char *argv[]) { // compute pressure once more with converged LayerThick VCoord->computePressure(LayerThick, SurfacePressure); - // compute geopotential - //VCoord->computeZHeight(LayerThick, SpecVol); - Array1DReal SelfAttractionLoading("SelfAttractionLoading", NCellsAll); - Array1DReal TidalPotential("TidalPotential", NCellsAll); - deepCopy(TidalPotential, 0.0_Real); - deepCopy(SelfAttractionLoading, 0.0_Real); - VCoord->computeGeopotential(TidalPotential, SelfAttractionLoading); + // compute z levels + VCoord->computeZHeight(LayerThick, SpecVol); - // create PressureGrad instance + // get PressureGrad instance PressureGrad *DefPGrad = PressureGrad::getDefault(); if (!DefPGrad) { LOG_INFO("PGrad: default instance not created by init"); @@ -270,18 +267,23 @@ int main(int argc, char *argv[]) { // compute pressure gradient deepCopy(Tend, 0.0_Real); DefPGrad->computePressureGrad(Tend, DefState, VCoord, DefEos, TimeLevel); + + //compute errors Real max_value = 0.0_Real; parallelReduce({NEdgesAll, NVertLayers - 2}, KOKKOS_LAMBDA(int i, int k, Real &max) { - Real val = Kokkos::abs(Tend(i + 1, k)); + Real val = Kokkos::abs(Tend(i, k + 1)); if (val > max) max = val; }, Kokkos::Max(max_value) ); Real sum_value = 0.0_Real; parallelReduce({NEdgesAll, NVertLayers - 2}, KOKKOS_LAMBDA(int i, int k, Real &lsum) { - lsum += Tend(i + 1, k) * Tend(i + 1, k); + lsum += Tend(i, k + 1) * Tend(i, k + 1); }, Kokkos::Sum(sum_value) ); - LOG_INFO("refinement level {}: max |Tend| = {}, average Tend = {}", refinement, max_value, std::sqrt(sum_value) / (NEdgesAll * (NVertLayers - 2))); + Real rmse = std::sqrt(sum_value / (NEdgesAll * (NVertLayers - 2))); + RMSE(refinement) = rmse; + + LOG_INFO("refinement level {}: max |Tend| = {}, average Tend = {}", refinement, max_value, rmse); // coarsen for next iteration dC = dC * 2.0_Real; @@ -289,6 +291,13 @@ int main(int argc, char *argv[]) { } // refinement loop + // Test for second order convergence + if (RMSE(NRefinements) < RMSE(0) / pow(4.0_Real, NRefinements - 1)) { + RetVal = 0; + } else { + RetVal = 1; + } + // cleanup PressureGrad::clear(); IOStream::finalize(); From e5acfee3a66b7587afef050c6faf6bdb7123a6c2 Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Tue, 24 Feb 2026 08:33:54 -0800 Subject: [PATCH 44/58] Lint files --- components/omega/src/ocn/OceanInit.cpp | 2 +- components/omega/src/ocn/PGrad.cpp | 15 +- components/omega/src/ocn/PGrad.h | 22 +- components/omega/src/ocn/Tendencies.cpp | 4 +- components/omega/src/ocn/Tendencies.h | 27 ++- components/omega/test/infra/IOStreamTest.cpp | 4 +- components/omega/test/ocn/EosTest.cpp | 9 +- components/omega/test/ocn/PGradTest.cpp | 226 ++++++++++-------- components/omega/test/ocn/StateTest.cpp | 4 +- components/omega/test/ocn/TendenciesTest.cpp | 6 +- .../test/timeStepping/TimeStepperTest.cpp | 8 +- 11 files changed, 177 insertions(+), 150 deletions(-) diff --git a/components/omega/src/ocn/OceanInit.cpp b/components/omega/src/ocn/OceanInit.cpp index 022b18d5b498..a22e3b3b0667 100644 --- a/components/omega/src/ocn/OceanInit.cpp +++ b/components/omega/src/ocn/OceanInit.cpp @@ -22,8 +22,8 @@ #include "MachEnv.h" #include "OceanDriver.h" #include "OceanState.h" -#include "Pacer.h" #include "PGrad.h" +#include "Pacer.h" #include "Tendencies.h" #include "TimeMgr.h" #include "TimeStepper.h" diff --git a/components/omega/src/ocn/PGrad.cpp b/components/omega/src/ocn/PGrad.cpp index 66b50923f438..84f25086916c 100644 --- a/components/omega/src/ocn/PGrad.cpp +++ b/components/omega/src/ocn/PGrad.cpp @@ -111,7 +111,8 @@ PressureGrad::PressureGrad( // Temporary: initialization of tidal potential and SAL TidalPotential = Array1DReal("TidalPotential", Mesh->NCellsSize); - SelfAttractionLoading = Array1DReal("SelfAttractionLoading", Mesh->NCellsSize); + SelfAttractionLoading = + Array1DReal("SelfAttractionLoading", Mesh->NCellsSize); } // end constructor @@ -191,8 +192,8 @@ void PressureGrad::computePressureGrad(Array2DReal &Tend, Team, KRange, INNER_LAMBDA(int KChunk) { LocCenteredPGrad(Tend, IEdge, KChunk, PressureMid, PressureInterface, ZInterface, - LocTidalPotential, LocSelfAttractionLoading, - SpecVol); + LocTidalPotential, + LocSelfAttractionLoading, SpecVol); }); }); @@ -202,16 +203,16 @@ void PressureGrad::computePressureGrad(Array2DReal &Tend, parallelForOuter( "pgrad-highorder", {NEdgesAll}, KOKKOS_LAMBDA(I4 IEdge, const TeamMember &Team) { - const int KMin = LocMinLayerEdgeBot(IEdge); - const int KMax = LocMaxLayerEdgeTop(IEdge); + const int KMin = LocMinLayerEdgeBot(IEdge); + const int KMax = LocMaxLayerEdgeTop(IEdge); const int KRange = vertRange(KMin, KMax); parallelForInner( Team, KRange, INNER_LAMBDA(int KChunk) { LocHighOrderPGrad(Tend, IEdge, KChunk, PressureMid, PressureInterface, ZInterface, - LocTidalPotential, LocSelfAttractionLoading, - SpecVol); + LocTidalPotential, + LocSelfAttractionLoading, SpecVol); }); }); } diff --git a/components/omega/src/ocn/PGrad.h b/components/omega/src/ocn/PGrad.h index 7d20b5927614..f879e6f333b8 100644 --- a/components/omega/src/ocn/PGrad.h +++ b/components/omega/src/ocn/PGrad.h @@ -39,8 +39,7 @@ class PressureGradCentered { const Array2DReal &ZInterface, const Array1DReal &TidalPotential, const Array1DReal &SelfAttractionLoading, - const Array2DReal &SpecVol - ) const { + const Array2DReal &SpecVol) const { const I4 KStart = chunkStart(KChunk, MinLayerEdgeBot(IEdge)); const I4 KLen = chunkLength(KChunk, KStart, MaxLayerEdgeTop(IEdge)); @@ -49,8 +48,10 @@ class PressureGradCentered { const I4 ICell1 = CellsOnEdge(IEdge, 1); const Real InvDcEdge = 1.0_Real / DcEdge(IEdge); - Real GradGeoPot = (TidalPotential(ICell1) - TidalPotential(ICell0)) * InvDcEdge + - (SelfAttractionLoading(ICell1) - SelfAttractionLoading(ICell0)) * InvDcEdge; + Real GradGeoPot = + (TidalPotential(ICell1) - TidalPotential(ICell0)) * InvDcEdge + + (SelfAttractionLoading(ICell1) - SelfAttractionLoading(ICell0)) * + InvDcEdge; for (int KVec = 0; KVec < KLen; ++KVec) { const I4 K = KStart + KVec; @@ -76,8 +77,9 @@ class PressureGradCentered { (SpecVol(ICell1, K) - SpecVol(ICell0, K)) * InvDcEdge; Tend(IEdge, K) += EdgeMask(IEdge, K) * (-GradMontPot + PGradAlpha - GradGeoPot); - //if (IEdge == 0) - //LOG_INFO("IEdge {}, K {}: GradMontPot {}, PGradAlpha {}, Tend {}", IEdge, K, GradMontPot, PGradAlpha, Tend(IEdge, K)); + // if (IEdge == 0) + // LOG_INFO("IEdge {}, K {}: GradMontPot {}, PGradAlpha {}, Tend {}", + // IEdge, K, GradMontPot, PGradAlpha, Tend(IEdge, K)); } } @@ -142,7 +144,8 @@ class PressureGrad { static PressureGrad *getDefault(); // Get instance by name - static PressureGrad *get(const std::string &Name ///< [in] Name of PressureGrad + static PressureGrad * + get(const std::string &Name ///< [in] Name of PressureGrad ); // Deallocates arrays and deletes instance @@ -156,7 +159,7 @@ class PressureGrad { ~PressureGrad(); // Compute pressure gradient tendencies and add into Tend array - void computePressureGrad(Array2DReal &Tend, const OceanState *State, + void computePressureGrad(Array2DReal &Tend, const OceanState *State, const VertCoord *VCoord, const Eos *EqState, const int TimeLevel) const; @@ -186,7 +189,8 @@ class PressureGrad { // Temporary: to be moveed to tidal forcing module in future Array1DReal TidalPotential; ///< Tidal potential for tidal forcing - Array1DReal SelfAttractionLoading; ///< Self attraction and loading for tidal forcing + Array1DReal + SelfAttractionLoading; ///< Self attraction and loading for tidal forcing // Instances of functors PressureGradCentered CenteredPGrad; diff --git a/components/omega/src/ocn/Tendencies.cpp b/components/omega/src/ocn/Tendencies.cpp index eec5ec015b1b..092b7b74d281 100644 --- a/components/omega/src/ocn/Tendencies.cpp +++ b/components/omega/src/ocn/Tendencies.cpp @@ -259,7 +259,7 @@ void Tendencies::defineFields() { LayerThicknessTendFieldName.append(Name); NormalVelocityTendFieldName.append(Name); TracerTendFieldName.append(Name); - } + } int NDims = 2; std::vector DimNamesThickness(NDims); @@ -290,7 +290,7 @@ void Tendencies::defineFields() { if (Name != "Default") { TendGroupName.append(Name); } - auto TendGroup = FieldGroup::create(TendGroupName); + auto TendGroup = FieldGroup::create(TendGroupName); TendGroup->addField(LayerThicknessTendFieldName); TendGroup->addField(NormalVelocityTendFieldName); diff --git a/components/omega/src/ocn/Tendencies.h b/components/omega/src/ocn/Tendencies.h index 882f04c61ea4..ad5bd8b4f3ee 100644 --- a/components/omega/src/ocn/Tendencies.h +++ b/components/omega/src/ocn/Tendencies.h @@ -33,13 +33,14 @@ #include "AuxiliaryState.h" #include "Config.h" +#include "Eos.h" #include "HorzMesh.h" #include "OceanState.h" +#include "PGrad.h" #include "TendencyTerms.h" #include "TimeMgr.h" -#include "Eos.h" -#include "PGrad.h" #include "VertAdv.h" +#include "VertCoord.h" #include #include @@ -154,10 +155,10 @@ class Tendencies { // Construct a new tendency object Tendencies(const std::string &Name, ///< [in] Name for tendencies const HorzMesh *Mesh, ///< [in] Horizontal mesh - VertCoord *VCoord, ///< [in] Vertical coordinate - VertAdv *VAdv, ///< [in] Vertical advection - PressureGrad *PGrad, ///< [in] Pressure gradient - Eos *EqState, ///< [in] Equation of state + VertCoord *VCoord, ///< [in] Vertical coordinate + VertAdv *VAdv, ///< [in] Vertical advection + PressureGrad *PGrad, ///< [in] Pressure gradient + Eos *EqState, ///< [in] Equation of state int NTracersIn, ///< [in] Number of tracers TimeInterval TimeStep, ///< [in] Time step Config *Options, ///< [in] Configuration options @@ -166,10 +167,10 @@ class Tendencies { Tendencies(const std::string &Name, ///< [in] Name for tendencies const HorzMesh *Mesh, ///< [in] Horizontal mesh - VertCoord *VCoord, ///< [in] Vertical coordinate - VertAdv *VAdv, ///< [in] Vertical advection - PressureGrad *PGrad, ///< [in] Pressure gradient - Eos *EqState, ///< [in] Equation of state + VertCoord *VCoord, ///< [in] Vertical coordinate + VertAdv *VAdv, ///< [in] Vertical advection + PressureGrad *PGrad, ///< [in] Pressure gradient + Eos *EqState, ///< [in] Equation of state int NTracersIn, ///< [in] Number of tracers TimeInterval TimeStep, ///< [in] Time step Config *Options ///< [in] Configuration options @@ -181,13 +182,13 @@ class Tendencies { Tendencies(const Tendencies &) = delete; Tendencies(Tendencies &&) = delete; - const HorzMesh *Mesh; ///< Pointer to horizontal mesh + const HorzMesh *Mesh; ///< Pointer to horizontal mesh VertCoord *VCoord; ///< Pointer to vertical coordinate VertAdv *VAdv; ///< Pointer to vertical advection PressureGrad *PGrad; ///< Pointer to pressure gradient Eos *EqState; ///< Pointer to equation of state - I4 NTracers; ///< Number of tracers - TimeInterval TimeStep; ///< Time step + I4 NTracers; ///< Number of tracers + TimeInterval TimeStep; ///< Time step // Pointer to default tendencies static Tendencies *DefaultTendencies; diff --git a/components/omega/test/infra/IOStreamTest.cpp b/components/omega/test/infra/IOStreamTest.cpp index ee0cd4608a41..451ca4293edf 100644 --- a/components/omega/test/infra/IOStreamTest.cpp +++ b/components/omega/test/infra/IOStreamTest.cpp @@ -23,11 +23,11 @@ #include "MachEnv.h" #include "OceanState.h" #include "OmegaKokkos.h" -#include "Pacer.h" #include "PGrad.h" +#include "Pacer.h" +#include "Tendencies.h" #include "TimeMgr.h" #include "TimeStepper.h" -#include "Tendencies.h" #include "Tracers.h" #include "VertCoord.h" #include "mpi.h" diff --git a/components/omega/test/ocn/EosTest.cpp b/components/omega/test/ocn/EosTest.cpp index 5797b77798d2..aaf7b1d96e80 100644 --- a/components/omega/test/ocn/EosTest.cpp +++ b/components/omega/test/ocn/EosTest.cpp @@ -51,8 +51,8 @@ const Real GswBVFExpValue = 0.02081197958166906; // Expected value from GSW-C library /// Test input values -const Real Sa = 30.0; // Absolute Salinity in g/kg -const Real Ct = 10.0; // Conservative Temperature in degC +const Real Sa = 30.0; // Absolute Salinity in g/kg +const Real Ct = 10.0; // Conservative Temperature in degC const Real P = 1000.0 * Db2Pa; // Pressure in Pa const I4 KDisp = 1; // Displate parcel to K=1 for TEOS-10 eos @@ -667,8 +667,9 @@ void checkValueGswcN2() { // Input arrays: length nz+1 double Salt[4] = {Sa - 1.0, Sa, Sa + 1.0}; // Absolute Salinity (g/kg) double Temp[4] = {Ct + 15.0, Ct + 10.0, - Ct + 5.0}; // Conservative Temperature (deg C) - double Press[4] = {P * Pa2Db, P * Pa2Db + 1.0, P * Pa2Db + 2.0}; // Pressure (dbar) + Ct + 5.0}; // Conservative Temperature (deg C) + double Press[4] = {P * Pa2Db, P * Pa2Db + 1.0, + P * Pa2Db + 2.0}; // Pressure (dbar) // Latitude (degrees north) double Latitude[4] = {0.0, 0.0, 0.0}; diff --git a/components/omega/test/ocn/PGradTest.cpp b/components/omega/test/ocn/PGradTest.cpp index 8045c88669e0..84b40842b5ae 100644 --- a/components/omega/test/ocn/PGradTest.cpp +++ b/components/omega/test/ocn/PGradTest.cpp @@ -1,4 +1,5 @@ -//===-- Test driver for OMEGA Pressure Gradient (PGrad) --------------*- C++-*-===/ +//===-- Test driver for OMEGA Pressure Gradient (PGrad) --------------*- +// C++-*-===/ // /// \file /// \brief Test driver for PressureGrad module @@ -10,8 +11,8 @@ #include "DataTypes.h" #include "Decomp.h" #include "Dimension.h" -#include "Error.h" #include "Eos.h" +#include "Error.h" #include "Field.h" #include "GlobalConstants.h" #include "Halo.h" @@ -22,10 +23,10 @@ #include "MachEnv.h" #include "OceanState.h" #include "OmegaKokkos.h" -#include "Pacer.h" #include "PGrad.h" -#include "Tracers.h" +#include "Pacer.h" #include "TimeStepper.h" +#include "Tracers.h" #include "VertCoord.h" #include "mpi.h" @@ -37,7 +38,7 @@ void initPGradTest() { int Err1; MachEnv::init(MPI_COMM_WORLD); - MachEnv *DefEnv = MachEnv::getDefault(); + MachEnv *DefEnv = MachEnv::getDefault(); MPI_Comm DefComm = DefEnv->getComm(); // Initialize the Logging system @@ -63,8 +64,7 @@ void initPGradTest() { Err1 = Halo::init(); if (Err1 != 0) { LOG_ERROR("PGrad: error initializing default halo"); - Err += Error(ErrorCode::Fail, - "PGrad: error initializing default halo"); + Err += Error(ErrorCode::Fail, "PGrad: error initializing default halo"); } // Initialize the default mesh @@ -93,69 +93,72 @@ int main(int argc, char *argv[]) { Kokkos::initialize(); Pacer::initialize(MPI_COMM_WORLD); Pacer::setPrefix("Omega:"); - + { initPGradTest(); // Initialize default PressureGrad PressureGrad::init(); - - MachEnv *DefEnv = MachEnv::getDefault(); - HorzMesh *DefMesh = HorzMesh::getDefault(); - VertCoord *VCoord = VertCoord::getDefault(); + MachEnv *DefEnv = MachEnv::getDefault(); + HorzMesh *DefMesh = HorzMesh::getDefault(); + VertCoord *VCoord = VertCoord::getDefault(); OceanState *DefState = OceanState::getDefault(); - Decomp *DefDecomp = Decomp::getDefault(); - Eos *DefEos = Eos::getInstance(); - Config *Options = Config::getOmegaConfig(); - + Decomp *DefDecomp = Decomp::getDefault(); + Eos *DefEos = Eos::getInstance(); + Config *Options = Config::getOmegaConfig(); // create arrays: Tend on edges, Pressure/Geopotential/SpecVol on cells Array2DReal Tend("Tend", DefMesh->NEdgesSize, VCoord->NVertLayers); - Array2DReal SpecVolOld("SpecVolOld", DefMesh->NCellsSize, VCoord->NVertLayers); - Array2DReal PressureMidOld("PressureMidOld", DefMesh->NCellsSize, VCoord->NVertLayers); + Array2DReal SpecVolOld("SpecVolOld", DefMesh->NCellsSize, + VCoord->NVertLayers); + Array2DReal PressureMidOld("PressureMidOld", DefMesh->NCellsSize, + VCoord->NVertLayers); Array1DReal SurfacePressure("SurfacePressure", DefMesh->NCellsSize); I4 NEdgesAll = DefMesh->NEdgesAll; I4 NCellsAll = DefMesh->NCellsAll; - I4 NVertLayers = 60; - Real dC = 30000.0_Real; + I4 NVertLayers = 60; + Real dC = 30000.0_Real; I4 NRefinements = 4; HostArray1DReal RMSE("RMSE", NRefinements); for (int refinement = 0; refinement < NRefinements; ++refinement) { LOG_INFO("PGradTest: Starting refinement level {}", refinement); - VCoord->NVertLayers = NVertLayers; + VCoord->NVertLayers = NVertLayers; VCoord->NVertLayersP1 = NVertLayers + 1; auto &MinLayerCell = VCoord->MinLayerCell; auto &MaxLayerCell = VCoord->MaxLayerCell; - parallelFor({NCellsAll}, KOKKOS_LAMBDA(int i) { - MinLayerCell(i) = 0; - MaxLayerCell(i) = NVertLayers - 1; - }); + parallelFor( + {NCellsAll}, KOKKOS_LAMBDA(int i) { + MinLayerCell(i) = 0; + MaxLayerCell(i) = NVertLayers - 1; + }); auto &MinLayerEdgeBot = VCoord->MinLayerEdgeBot; auto &MaxLayerEdgeTop = VCoord->MaxLayerEdgeTop; - parallelFor({NEdgesAll}, KOKKOS_LAMBDA(int i) { - MinLayerEdgeBot(i) = 0; - MaxLayerEdgeTop(i) = NVertLayers - 1; - //MaxLayerEdgeTop(i) = 4; - }); + parallelFor( + {NEdgesAll}, KOKKOS_LAMBDA(int i) { + MinLayerEdgeBot(i) = 0; + MaxLayerEdgeTop(i) = NVertLayers - 1; + // MaxLayerEdgeTop(i) = 4; + }); auto &CellsOnEdge = DefMesh->CellsOnEdge; - auto &DcEdge = DefMesh->DcEdge; - auto &EdgeMask = VCoord->EdgeMask; - parallelFor({NEdgesAll}, KOKKOS_LAMBDA(int i) { - CellsOnEdge(i, 0)= 0; - CellsOnEdge(i, 1)= 1; - DcEdge(i) = dC; - }); + auto &DcEdge = DefMesh->DcEdge; + auto &EdgeMask = VCoord->EdgeMask; + parallelFor( + {NEdgesAll}, KOKKOS_LAMBDA(int i) { + CellsOnEdge(i, 0) = 0; + CellsOnEdge(i, 1) = 1; + DcEdge(i) = dC; + }); // Fetch reference desnity from Config Real Density0; Density0 = RhoSw; - + I4 TimeLevel = 0; // get state and tracer arrays @@ -167,32 +170,36 @@ int main(int argc, char *argv[]) { Err = Tracers::getByName(Salinity, TimeLevel, "Salinity"); // set Z interface and mid-point locations - Real ZBottom = -1000.0_Real; - Real dZ = 2.0_Real * (-ZBottom / NVertLayers); + Real ZBottom = -1000.0_Real; + Real dZ = 2.0_Real * (-ZBottom / NVertLayers); auto &BottomDepth = VCoord->BottomDepth; - auto &ZInterface = VCoord->ZInterface; - auto &ZMid = VCoord->ZMid; - Real tilt_factor = 0.495_Real; - //Real tilt_factor = 0.45_Real; - parallelFor({NCellsAll}, KOKKOS_LAMBDA(int i) { - ZInterface(i, NVertLayers) = ZBottom; - SurfacePressure(i) = 0.0_Real; - BottomDepth(i) = 0.0_Real; - for (int k = NVertLayers - 1; k >= 0; --k) { - Real x = (k + i) % 2; - Real dz = ( 2.0_Real * tilt_factor - 1.0_Real ) * x * dZ + (1.0_Real - tilt_factor) * dZ; // staggered layer thickness - ZInterface(i, k) = ZInterface(i, k + 1) + dz; - LayerThick(i, k) = ZInterface(i, k) - ZInterface(i, k + 1); - ZMid(i, k) = 0.5_Real * (ZInterface(i, k) + ZInterface(i, k + 1)); - BottomDepth(i) += dz; - } - }); + auto &ZInterface = VCoord->ZInterface; + auto &ZMid = VCoord->ZMid; + Real tilt_factor = 0.495_Real; + // Real tilt_factor = 0.45_Real; + parallelFor( + {NCellsAll}, KOKKOS_LAMBDA(int i) { + ZInterface(i, NVertLayers) = ZBottom; + SurfacePressure(i) = 0.0_Real; + BottomDepth(i) = 0.0_Real; + for (int k = NVertLayers - 1; k >= 0; --k) { + Real x = (k + i) % 2; + Real dz = (2.0_Real * tilt_factor - 1.0_Real) * x * dZ + + (1.0_Real - tilt_factor) * + dZ; // staggered layer thickness + ZInterface(i, k) = ZInterface(i, k + 1) + dz; + LayerThick(i, k) = ZInterface(i, k) - ZInterface(i, k + 1); + ZMid(i, k) = + 0.5_Real * (ZInterface(i, k) + ZInterface(i, k + 1)); + BottomDepth(i) += dz; + } + }); LOG_INFO("NVertLayers = {}", NVertLayers); LOG_INFO("dC = {}", dC); DefState->copyToHost(0); HostArray2DReal LayerThickH; - DefState->getLayerThicknessH(LayerThickH,0); + DefState->getLayerThicknessH(LayerThickH, 0); for (int i = 0; i < 2; ++i) { for (int k = 0; k < 2; ++k) { LOG_INFO("LayerThick({}, {}) = {}", i, k, LayerThickH(i, k)); @@ -201,21 +208,21 @@ int main(int argc, char *argv[]) { // set simple temperature and salinity profiles auto &SpecVol = DefEos->SpecVol; - parallelFor({NCellsAll, NVertLayers}, KOKKOS_LAMBDA(int i, int k) { - Real T0 = 30.0; - Real TB = 5.0; - Real S0 = 30.0; - Real SB = 40.0; - - - Real phi0 = (ZMid(i, k) - ZBottom) / (-ZBottom); - Real phiB = 1.0_Real - phi0; - - Temp(i, k) = T0 * phi0 + TB * phiB; - Salinity(i, k) = S0 * phi0 + SB * phiB; - SpecVol(i, k) = 1.0_Real / Density0; - SpecVolOld(i, k) = SpecVol(i, k); - }); + parallelFor( + {NCellsAll, NVertLayers}, KOKKOS_LAMBDA(int i, int k) { + Real T0 = 30.0; + Real TB = 5.0; + Real S0 = 30.0; + Real SB = 40.0; + + Real phi0 = (ZMid(i, k) - ZBottom) / (-ZBottom); + Real phiB = 1.0_Real - phi0; + + Temp(i, k) = T0 * phi0 + TB * phiB; + Salinity(i, k) = S0 * phi0 + SB * phiB; + SpecVol(i, k) = 1.0_Real / Density0; + SpecVolOld(i, k) = SpecVol(i, k); + }); // Iterate to converge LayerThick, SpecVol, PressureMid auto &PressureMid = VCoord->PressureMid; @@ -228,37 +235,42 @@ int main(int argc, char *argv[]) { DefEos->computeSpecVol(Temp, Salinity, PressureMid); // compute psuedo thickness from specific volume - parallelFor({NCellsAll, NVertLayers}, KOKKOS_LAMBDA(int i, int k) { - LayerThick(i, k) = 1.0_Real / (SpecVol(i, k) * Density0) * (ZInterface(i, k) - ZInterface(i, k + 1)); - }); + parallelFor( + {NCellsAll, NVertLayers}, KOKKOS_LAMBDA(int i, int k) { + LayerThick(i, k) = 1.0_Real / (SpecVol(i, k) * Density0) * + (ZInterface(i, k) - ZInterface(i, k + 1)); + }); // compute difference from previous iteration Real max_value = 0.0_Real; - parallelReduce({NCellsAll, NVertLayers}, - KOKKOS_LAMBDA(int i, int k, Real &max) { - Real diff = Kokkos::abs(SpecVol(i, k) - SpecVolOld(i, k)); - if (diff > max) max = diff; - }, Kokkos::Max(max_value) ); - - // check convergence + parallelReduce( + {NCellsAll, NVertLayers}, + KOKKOS_LAMBDA(int i, int k, Real &max) { + Real diff = Kokkos::abs(SpecVol(i, k) - SpecVolOld(i, k)); + if (diff > max) + max = diff; + }, + Kokkos::Max(max_value)); + + // check convergence if (max_value < 1e-12_Real) { LOG_INFO("converged: max diff = {}", max_value); break; } else { - parallelFor({NCellsAll, NVertLayers}, KOKKOS_LAMBDA(int i, int k) { - SpecVolOld(i, k) = SpecVol(i, k); - }); + parallelFor( + {NCellsAll, NVertLayers}, KOKKOS_LAMBDA(int i, int k) { + SpecVolOld(i, k) = SpecVol(i, k); + }); } - } - + // compute pressure once more with converged LayerThick VCoord->computePressure(LayerThick, SurfacePressure); // compute z levels VCoord->computeZHeight(LayerThick, SpecVol); - // get PressureGrad instance + // get PressureGrad instance PressureGrad *DefPGrad = PressureGrad::getDefault(); if (!DefPGrad) { LOG_INFO("PGrad: default instance not created by init"); @@ -266,27 +278,34 @@ int main(int argc, char *argv[]) { // compute pressure gradient deepCopy(Tend, 0.0_Real); - DefPGrad->computePressureGrad(Tend, DefState, VCoord, DefEos, TimeLevel); + DefPGrad->computePressureGrad(Tend, DefState, VCoord, DefEos, + TimeLevel); - //compute errors + // compute errors Real max_value = 0.0_Real; - parallelReduce({NEdgesAll, NVertLayers - 2}, - KOKKOS_LAMBDA(int i, int k, Real &max) { - Real val = Kokkos::abs(Tend(i, k + 1)); - if (val > max) max = val; - }, Kokkos::Max(max_value) ); + parallelReduce( + {NEdgesAll, NVertLayers - 2}, + KOKKOS_LAMBDA(int i, int k, Real &max) { + Real val = Kokkos::abs(Tend(i, k + 1)); + if (val > max) + max = val; + }, + Kokkos::Max(max_value)); Real sum_value = 0.0_Real; - parallelReduce({NEdgesAll, NVertLayers - 2}, - KOKKOS_LAMBDA(int i, int k, Real &lsum) { - lsum += Tend(i, k + 1) * Tend(i, k + 1); - }, Kokkos::Sum(sum_value) ); + parallelReduce( + {NEdgesAll, NVertLayers - 2}, + KOKKOS_LAMBDA(int i, int k, Real &lsum) { + lsum += Tend(i, k + 1) * Tend(i, k + 1); + }, + Kokkos::Sum(sum_value)); Real rmse = std::sqrt(sum_value / (NEdgesAll * (NVertLayers - 2))); RMSE(refinement) = rmse; - LOG_INFO("refinement level {}: max |Tend| = {}, average Tend = {}", refinement, max_value, rmse); + LOG_INFO("refinement level {}: max |Tend| = {}, average Tend = {}", + refinement, max_value, rmse); // coarsen for next iteration - dC = dC * 2.0_Real; + dC = dC * 2.0_Real; NVertLayers = NVertLayers / 2; } // refinement loop @@ -317,6 +336,7 @@ int main(int argc, char *argv[]) { Kokkos::finalize(); MPI_Finalize(); - if (RetVal >= 256) RetVal = 255; + if (RetVal >= 256) + RetVal = 255; return RetVal; } diff --git a/components/omega/test/ocn/StateTest.cpp b/components/omega/test/ocn/StateTest.cpp index 15ce8d3e3169..e93a32ad2030 100644 --- a/components/omega/test/ocn/StateTest.cpp +++ b/components/omega/test/ocn/StateTest.cpp @@ -23,8 +23,8 @@ #include "MachEnv.h" #include "OceanState.h" #include "OmegaKokkos.h" -#include "Pacer.h" #include "PGrad.h" +#include "Pacer.h" #include "Tendencies.h" #include "TimeStepper.h" #include "VertCoord.h" @@ -93,7 +93,7 @@ void initStateTest() { AuxiliaryState::init(); // Initialize pressure gradient - PressureGrad::init(); + PressureGrad::init(); // Create tendencies Tendencies::init(); diff --git a/components/omega/test/ocn/TendenciesTest.cpp b/components/omega/test/ocn/TendenciesTest.cpp index 60591ef7331b..cef94e9e2144 100644 --- a/components/omega/test/ocn/TendenciesTest.cpp +++ b/components/omega/test/ocn/TendenciesTest.cpp @@ -5,8 +5,8 @@ #include "DataTypes.h" #include "Decomp.h" #include "Dimension.h" -#include "Error.h" #include "Eos.h" +#include "Error.h" #include "Field.h" #include "GlobalConstants.h" #include "Halo.h" @@ -17,8 +17,8 @@ #include "MachEnv.h" #include "OceanTestCommon.h" #include "OmegaKokkos.h" -#include "Pacer.h" #include "PGrad.h" +#include "Pacer.h" #include "TimeStepper.h" #include "VertCoord.h" #include "mpi.h" @@ -178,7 +178,7 @@ int testTendencies() { TimeInterval ZeroTimeStep; // Zero-length time step placeholder Config *Options = Config::getOmegaConfig(); Config TendConfig("Tendencies"); - Err1 = Options->get(TendConfig); + Err1 = Options->get(TendConfig); int NTracersTest = 3; Tendencies::create("TestTendencies", Mesh, VCoord, VAdv, PGrad, EqState, diff --git a/components/omega/test/timeStepping/TimeStepperTest.cpp b/components/omega/test/timeStepping/TimeStepperTest.cpp index 1d3ce044b1d2..10f802617e5a 100644 --- a/components/omega/test/timeStepping/TimeStepperTest.cpp +++ b/components/omega/test/timeStepping/TimeStepperTest.cpp @@ -29,8 +29,8 @@ #include "MachEnv.h" #include "OceanState.h" #include "OmegaKokkos.h" -#include "Pacer.h" #include "PGrad.h" +#include "Pacer.h" #include "TendencyTerms.h" #include "TimeMgr.h" #include "Tracers.h" @@ -191,9 +191,9 @@ int initTimeStepperTest(const std::string &mesh) { // Creating non-default state and auxiliary state to use only one vertical // layer - auto *DefMesh = HorzMesh::getDefault(); - auto *DefHalo = Halo::getDefault(); - auto *DefEos = Eos::getInstance(); + auto *DefMesh = HorzMesh::getDefault(); + auto *DefHalo = Halo::getDefault(); + auto *DefEos = Eos::getInstance(); auto *DefPGrad = PressureGrad::getDefault(); int NTracers = Tracers::getNumTracers(); From 25586561d23222a92e7a15d7b96f277047a335ef Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Thu, 26 Feb 2026 13:14:40 -0800 Subject: [PATCH 45/58] Remove commented out code for debugging --- components/omega/src/ocn/PGrad.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/components/omega/src/ocn/PGrad.h b/components/omega/src/ocn/PGrad.h index f879e6f333b8..0d029b163973 100644 --- a/components/omega/src/ocn/PGrad.h +++ b/components/omega/src/ocn/PGrad.h @@ -77,9 +77,6 @@ class PressureGradCentered { (SpecVol(ICell1, K) - SpecVol(ICell0, K)) * InvDcEdge; Tend(IEdge, K) += EdgeMask(IEdge, K) * (-GradMontPot + PGradAlpha - GradGeoPot); - // if (IEdge == 0) - // LOG_INFO("IEdge {}, K {}: GradMontPot {}, PGradAlpha {}, Tend {}", - // IEdge, K, GradMontPot, PGradAlpha, Tend(IEdge, K)); } } From 70ae706d5e3c4a5268b79b746d87d5bc126216be Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Thu, 26 Feb 2026 13:15:20 -0800 Subject: [PATCH 46/58] Remove unsed member variables --- components/omega/src/ocn/PGrad.cpp | 4 +--- components/omega/src/ocn/PGrad.h | 9 +++------ 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/components/omega/src/ocn/PGrad.cpp b/components/omega/src/ocn/PGrad.cpp index 84f25086916c..7af9baeed21d 100644 --- a/components/omega/src/ocn/PGrad.cpp +++ b/components/omega/src/ocn/PGrad.cpp @@ -76,9 +76,7 @@ PressureGrad::PressureGrad( const HorzMesh *Mesh, ///< [in] Horizontal mesh const VertCoord *VCoord, ///< [in] Vertical coordinate Config *Options) ///< [in] Configuration options - : CellsOnEdge(Mesh->CellsOnEdge), DcEdge(Mesh->DcEdge), - EdgeSignOnCell(Mesh->EdgeSignOnCell), - MinLayerEdgeBot(VCoord->MinLayerEdgeBot), + : MinLayerEdgeBot(VCoord->MinLayerEdgeBot), MaxLayerEdgeTop(VCoord->MaxLayerEdgeTop), CenteredPGrad(Mesh, VCoord), HighOrderPGrad(Mesh, VCoord) { diff --git a/components/omega/src/ocn/PGrad.h b/components/omega/src/ocn/PGrad.h index 0d029b163973..e0fa06e8c2fc 100644 --- a/components/omega/src/ocn/PGrad.h +++ b/components/omega/src/ocn/PGrad.h @@ -177,12 +177,9 @@ class PressureGrad { I4 NVertLayers = 0; I4 NVertLayersP1 = 0; - // Data required for computation (stored copies of mesh/VCoord arrays) - Array2DI4 CellsOnEdge; ///< cells surrounding each edge - Array1DReal DcEdge; ///< distance between cell centers across edge - Array2DReal EdgeSignOnCell; ///< orientation of edge relative to cell - Array1DI4 MinLayerEdgeBot; ///< min vertical layer on each edge - Array1DI4 MaxLayerEdgeTop; ///< max vertical layer on each edge + // Data required for computation (stored copies of VCoord arrays) + Array1DI4 MinLayerEdgeBot; ///< min vertical layer on each edge + Array1DI4 MaxLayerEdgeTop; ///< max vertical layer on each edge // Temporary: to be moveed to tidal forcing module in future Array1DReal TidalPotential; ///< Tidal potential for tidal forcing From 5864ca31240e45d69411d4c0269d298fca5b517a Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Thu, 26 Feb 2026 14:24:35 -0800 Subject: [PATCH 47/58] Fix for updated accessor state/tracer functions following rebase --- components/omega/src/ocn/PGrad.cpp | 3 +-- components/omega/src/ocn/Tendencies.cpp | 11 ++++------- components/omega/test/ocn/PGradTest.cpp | 12 ++++-------- components/omega/test/ocn/TendenciesTest.cpp | 6 ++---- 4 files changed, 11 insertions(+), 21 deletions(-) diff --git a/components/omega/src/ocn/PGrad.cpp b/components/omega/src/ocn/PGrad.cpp index 7af9baeed21d..82438be6d30c 100644 --- a/components/omega/src/ocn/PGrad.cpp +++ b/components/omega/src/ocn/PGrad.cpp @@ -173,8 +173,7 @@ void PressureGrad::computePressureGrad(Array2DReal &Tend, const Array2DReal &SpecVol = EqState->SpecVol; const Array2DReal &ZInterface = VCoord->ZInterface; const Array2DReal &ZMid = VCoord->ZMid; - Array2DReal LayerThick; - State->getLayerThickness(LayerThick, TimeLevel); + Array2DReal LayerThick = State->getLayerThickness(TimeLevel); if (PressureGradChoice == PressureGradType::Centered) { diff --git a/components/omega/src/ocn/Tendencies.cpp b/components/omega/src/ocn/Tendencies.cpp index 092b7b74d281..e5441b0383cc 100644 --- a/components/omega/src/ocn/Tendencies.cpp +++ b/components/omega/src/ocn/Tendencies.cpp @@ -13,6 +13,7 @@ #include "Eos.h" #include "Error.h" #include "Field.h" +#include "OceanState.h" #include "PGrad.h" #include "Pacer.h" #include "TimeStepper.h" @@ -601,21 +602,17 @@ void Tendencies::computeVelocityTendenciesOnly( // Compute pressure gradient if (PGrad->Enabled) { - I4 Err; - Array2DReal Temp; - Array2DReal Salinity; - Array2DReal LayerThick; // Temporary handling of surface pressure Array1DReal SurfacePressure("SurfacePressure", Mesh->NCellsSize); deepCopy(SurfacePressure, 0.0_Real); Pacer::start("Tend:pressureGradTerm", 2); - State->getLayerThickness(LayerThick, VelTimeLevel); + Array2DReal LayerThick = State->getLayerThickness(VelTimeLevel); VCoord->computePressure(LayerThick, SurfacePressure); OMEGA_SCOPE(LocPressureMid, VCoord->PressureMid); - Err = Tracers::getByName(Temp, VelTimeLevel, "Temperature"); - Err = Tracers::getByName(Salinity, VelTimeLevel, "Salinity"); + Array2DReal Temp = Tracers::getByName(VelTimeLevel, "Temperature"); + Array2DReal Salinity = Tracers::getByName(VelTimeLevel, "Salinity"); EqState->computeSpecVol(Temp, Salinity, LocPressureMid); diff --git a/components/omega/test/ocn/PGradTest.cpp b/components/omega/test/ocn/PGradTest.cpp index 84b40842b5ae..8c1e63100c00 100644 --- a/components/omega/test/ocn/PGradTest.cpp +++ b/components/omega/test/ocn/PGradTest.cpp @@ -162,12 +162,9 @@ int main(int argc, char *argv[]) { I4 TimeLevel = 0; // get state and tracer arrays - Array2DReal LayerThick; - DefState->getLayerThickness(LayerThick, TimeLevel); - Array2DReal Temp; - Array2DReal Salinity; - Err = Tracers::getByName(Temp, TimeLevel, "Temperature"); - Err = Tracers::getByName(Salinity, TimeLevel, "Salinity"); + Array2DReal LayerThick = DefState->getLayerThickness(TimeLevel); + Array2DReal Temp = Tracers::getByName(TimeLevel, "Temperature"); + Array2DReal Salinity = Tracers::getByName(TimeLevel, "Salinity"); // set Z interface and mid-point locations Real ZBottom = -1000.0_Real; @@ -198,8 +195,7 @@ int main(int argc, char *argv[]) { LOG_INFO("NVertLayers = {}", NVertLayers); LOG_INFO("dC = {}", dC); DefState->copyToHost(0); - HostArray2DReal LayerThickH; - DefState->getLayerThicknessH(LayerThickH, 0); + HostArray2DReal LayerThickH = DefState->getLayerThicknessH(TimeLevel); for (int i = 0; i < 2; ++i) { for (int k = 0; k < 2; ++k) { LOG_INFO("LayerThick({}, {}) = {}", i, k, LayerThickH(i, k)); diff --git a/components/omega/test/ocn/TendenciesTest.cpp b/components/omega/test/ocn/TendenciesTest.cpp index cef94e9e2144..09a42e15ff74 100644 --- a/components/omega/test/ocn/TendenciesTest.cpp +++ b/components/omega/test/ocn/TendenciesTest.cpp @@ -65,10 +65,8 @@ int initState() { std::vector DimNamesThickness(NDims); DimNamesThickness[0] = "NCells"; - Array2DReal LayerThickCell; - Array2DReal NormalVelEdge; - Err += State->getLayerThickness(LayerThickCell, 0); - Err += State->getNormalVelocity(NormalVelEdge, 0); + Array2DReal LayerThickCell = State->getLayerThickness(0); + Array2DReal NormalVelEdge = State->getNormalVelocity(0); Array3DReal TracersArray = Tracers::getAll(0); const auto &TracersCell = TracersArray; From e29cf6f880bd5dce958c18e7df589aeb57a64e58 Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Thu, 26 Feb 2026 14:57:34 -0800 Subject: [PATCH 48/58] Add user and dev documentation --- components/omega/doc/devGuide/PGrad.md | 185 ++++++++++++++++++++++++ components/omega/doc/index.md | 2 + components/omega/doc/userGuide/PGrad.md | 68 +++++++++ 3 files changed, 255 insertions(+) create mode 100644 components/omega/doc/devGuide/PGrad.md create mode 100644 components/omega/doc/userGuide/PGrad.md diff --git a/components/omega/doc/devGuide/PGrad.md b/components/omega/doc/devGuide/PGrad.md new file mode 100644 index 000000000000..cfbe45fcabd8 --- /dev/null +++ b/components/omega/doc/devGuide/PGrad.md @@ -0,0 +1,185 @@ +(omega-dev-pgrad)= + +# Pressure Gradient (PGrad) + +Omega includes a `PressureGrad` class that computes horizontal pressure gradient +tendencies for the non-Boussinesq momentum equation. The implementation supports a +centered difference scheme as the default, with a placeholder for future high-order +methods. The class follows the same factory pattern used by other Omega modules. + +## PressureGradType enum + +An enumeration of the available pressure gradient schemes is defined in `PGrad.h`: + +```c++ +enum class PressureGradType { Centered, HighOrder1, HighOrder2 }; +``` + +This is used to select which pressure gradient method is applied at runtime. + +## Initialization + +An instance of `PressureGrad` requires both a [`HorzMesh`](#omega-dev-horz-mesh) and +a [`VertCoord`](#omega-dev-vert-coord), so these classes and all of their dependencies +must be initialized before `PressureGrad` can be initialized. The static method: + +```c++ +OMEGA::PressureGrad::init(); +``` + +initializes the default `PressureGrad` instance using the default `HorzMesh` and +`VertCoord` instances and the global Omega configuration. A pointer to the default +instance can be retrieved at any time using: + +```c++ +OMEGA::PressureGrad* DefPGrad = OMEGA::PressureGrad::getDefault(); +``` + +## Creating additional instances + +Additional named instances can be created using: + +```c++ +OMEGA::PressureGrad* MyPGrad = + OMEGA::PressureGrad::create("MyPGrad", Mesh, VCoord, Options); +``` + +where `Mesh` is a pointer to a `HorzMesh`, `VCoord` is a pointer to a `VertCoord`, +and `Options` is a pointer to the `Config`. A named instance can be retrieved later +using: + +```c++ +OMEGA::PressureGrad* MyPGrad = OMEGA::PressureGrad::get("MyPGrad"); +``` + +## Constructor behavior + +The constructor reads the `PressureGrad` section from the configuration, stores +references to mesh and vertical coordinate data needed for computation, and enables +the appropriate functor based on the configured `PressureGradType`. It also allocates +placeholder arrays for tidal potential and self-attraction/loading, which are intended +to be populated by a future tidal forcing module. Currently these arrays are initialized +to zero. + +## Computing the pressure gradient + +To compute pressure gradient tendencies and accumulate them into a tendency array: + +```c++ +PGrad->computePressureGrad(Tend, State, VCoord, EqState, TimeLevel); +``` + +where: +- `Tend` is a 2D array `(NEdgesAll × NVertLayers)` that the pressure gradient + tendency is accumulated into +- `State` is the current `OceanState`, from which layer thickness is extracted + at the given `TimeLevel` +- `VCoord` provides pressure, interface height, and geopotential fields +- `EqState` provides the specific volume field +- `TimeLevel` selects which time level of the state to use + +The method uses hierarchical Kokkos parallelism: an outer `parallelForOuter` loop +iterates over edges, and an inner `parallelForInner` loop iterates over vertical +chunks. The appropriate functor is dispatched based on `PressureGradChoice`. + +## Functors + +### PressureGradCentered + +This functor implements a centered difference approximation of the pressure gradient +tendency. For each edge, it first computes the layer-invariant tidal and +self-attraction/loading contribution: + +``` +GradGeoPot = grad(TidalPotential) + grad(SelfAttractionLoading) +``` + +Then, for each vertical layer `K`, it computes three terms: + +1. **Montgomery potential gradient**: The average of the horizontal gradients of the + Montgomery potential ($\alpha p + g z$) at the top (interface `K`) and bottom + (interface `K+1`) of the layer. This compactly represents the combined effect + of the pressure gradient and the geopotential contribution from tilted coordinate + surfaces. + +2. **Specific volume correction**: A correction term equal to the edge-averaged + pressure at mid-layer multiplied by the horizontal gradient of specific volume. + This accounts for horizontal density variations that are not captured by the + Montgomery potential form. + +3. **Tidal and geopotential forcing** (`GradGeoPot`): The external geopotential + contribution from tidal forcing and self-attraction/loading, applied uniformly + across all layers at an edge. + +The tendency update for each layer is: + +``` +Tend(IEdge, K) += EdgeMask(IEdge, K) * (-GradMontPot + PGradAlpha - GradGeoPot) +``` + +where `EdgeMask` is applied to enforce land boundary conditions. The functor operator +signature is: + +```c++ +KOKKOS_FUNCTION void operator()(const Array2DReal &Tend, I4 IEdge, I4 KChunk, + const Array2DReal &PressureMid, + const Array2DReal &PressureInterface, + const Array2DReal &ZInterface, + const Array1DReal &TidalPotential, + const Array1DReal &SelfAttractionLoading, + const Array2DReal &SpecVol) const; +``` + +### PressureGradHighOrder + +This functor is a placeholder for a future high-order pressure gradient implementation +suitable for ice shelf cavities and complex bathymetry. Currently it performs no +computation (a no-op). + +## Configuration + +The pressure gradient type is selected in the input YAML file: + +```yaml +PressureGrad: + PressureGradType: 'centered' +``` + +Valid options for `PressureGradType` are: +- `'centered'` or `'Centered'`: centered difference approximation (default) +- `'HighOrder1'`: first high-order method (placeholder, future implementation) + +If an unrecognized value is provided, the implementation falls back to the centered +scheme and logs an informational message. + +## Data members + +The `PressureGrad` class stores the following key data: + +| Member | Type | Description | +| ------ | ---- | ----------- | +| `NEdgesAll` | `I4` | Total number of edges including halo | +| `NChunks` | `I4` | Number of vertical chunks for vectorization | +| `NVertLayers` | `I4` | Number of vertical layers | +| `NVertLayersP1` | `I4` | Number of vertical layers plus one | +| `MinLayerEdgeBot` | `Array1DI4` | Minimum active layer index for each edge | +| `MaxLayerEdgeTop` | `Array1DI4` | Maximum active layer index for each edge | +| `TidalPotential` | `Array1DReal` | Tidal potential (placeholder, currently zero) | +| `SelfAttractionLoading` | `Array1DReal` | Self-attraction and loading term (placeholder, currently zero) | +| `CenteredPGrad` | `PressureGradCentered` | Centered pressure gradient functor | +| `HighOrderPGrad` | `PressureGradHighOrder` | High-order pressure gradient functor | +| `PressureGradChoice` | `PressureGradType` | Selected pressure gradient method | + +## Removal + +To remove all `PressureGrad` instances: + +```c++ +OMEGA::PressureGrad::clear(); +``` + +To remove a specific named instance: + +```c++ +OMEGA::PressureGrad::erase("MyPGrad"); +``` diff --git a/components/omega/doc/index.md b/components/omega/doc/index.md index 1dd11e400a43..41bd611140b3 100644 --- a/components/omega/doc/index.md +++ b/components/omega/doc/index.md @@ -49,6 +49,7 @@ userGuide/Reductions userGuide/Tracers userGuide/TridiagonalSolvers userGuide/VertCoord +userGuide/PGrad userGuide/Timing userGuide/VerticalMixingCoeff userGuide/VertAdv @@ -93,6 +94,7 @@ devGuide/Reductions devGuide/Tracers devGuide/TridiagonalSolvers devGuide/VertCoord +devGuide/PGrad devGuide/Timing devGuide/VerticalMixingCoeff devGuide/VertAdv diff --git a/components/omega/doc/userGuide/PGrad.md b/components/omega/doc/userGuide/PGrad.md new file mode 100644 index 000000000000..0fc7bb5968a2 --- /dev/null +++ b/components/omega/doc/userGuide/PGrad.md @@ -0,0 +1,68 @@ +(omega-user-pgrad)= + +# Pressure Gradient + +The pressure gradient term in the momentum equation represents the force per unit +mass due to horizontal variations in pressure and geopotential. This term is +essential for capturing the dynamics of +ocean circulation, including both barotropic and baroclinic motions. + +## Physical Background + +In the layered non-Boussinesq momentum equation solved in Omega, the pressure +gradient tendency for each edge and layer includes three contributions: + +1. **Montgomery potential gradient**: The horizontal gradient of the Montgomery + potential ($\alpha p + g z$), averaged across the top and bottom interfaces of + each layer. The Montgomery potential combines the pressure gradient and the + geopotential, and its gradient along coordinate surfaces accounts for both the + direct pressure force and the effect of tilted layer interfaces that arise when + using a general vertical coordinate. + +2. **Specific volume correction**: A correction term proportional to the gradient + of specific volume (inverse density) at each edge. This term ensures that + horizontal density variations between the two cells sharing an edge are properly + represented in the pressure gradient force. + +3. **External geopotential forcing**: Contributions from the tidal potential and + the self-attraction and loading (SAL) terms. These represent gravitational + forcing from astronomical tides and the deformation of the solid Earth and ocean + surface in response to the ocean mass distribution. These terms are currently set + to zero and will be provided by a future tidal forcing module. + +## Configuration Options + +The pressure gradient method is configured in the input YAML file under the +`PressureGrad` section: + +```yaml +PressureGrad: + PressureGradType: 'centered' +``` + +### Available Methods + +**Centered Difference** (`'centered'` or `'Centered'`) +- Computes the pressure gradient using a centered finite-difference approximation + of the Montgomery potential gradient and specific volume correction +- Suitable for global ocean simulations without ice shelf cavities +- Default and currently the only fully implemented option + +**High-Order** (`'HighOrder1'`) +- Placeholder for a future high-order pressure gradient method based on volume + integral formulations +- Intended for simulations with ice shelf cavities and steep bathymetry where the + centered scheme may be inaccurate +- Not yet implemented; selecting this option produces zero pressure gradient tendency + +## Dependencies + +The pressure gradient calculation requires the following Omega components to be +initialized first: + +- [**Horizontal Mesh**](omega-user-horz-mesh): provides mesh geometry including + distances between cell centers and edge connectivity +- [**Vertical Coordinate**](omega-user-vert-coord): provides pressure at layer + mid-points and interfaces, interface heights ($z$), and geopotential +- [**Equation of State**](omega-user-eos): provides the specific volume field +- [**Ocean State**](omega-user-state): provides the current layer thicknesses From 491b75ce8c0c49821ec9fa6d08b2cd010572f6c3 Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Thu, 5 Mar 2026 15:52:32 -0800 Subject: [PATCH 49/58] Fix compiler warnings from unsed variables --- components/omega/src/ocn/PGrad.cpp | 2 -- components/omega/src/ocn/Tendencies.h | 6 +++--- components/omega/test/ocn/PGradTest.cpp | 5 ----- 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/components/omega/src/ocn/PGrad.cpp b/components/omega/src/ocn/PGrad.cpp index 82438be6d30c..bbd11a889fb4 100644 --- a/components/omega/src/ocn/PGrad.cpp +++ b/components/omega/src/ocn/PGrad.cpp @@ -169,10 +169,8 @@ void PressureGrad::computePressureGrad(Array2DReal &Tend, const Array2DReal &PressureMid = VCoord->PressureMid; const Array2DReal &PressureInterface = VCoord->PressureInterface; - const Array2DReal &Geopotential = VCoord->GeopotentialMid; const Array2DReal &SpecVol = EqState->SpecVol; const Array2DReal &ZInterface = VCoord->ZInterface; - const Array2DReal &ZMid = VCoord->ZMid; Array2DReal LayerThick = State->getLayerThickness(TimeLevel); if (PressureGradChoice == PressureGradType::Centered) { diff --git a/components/omega/src/ocn/Tendencies.h b/components/omega/src/ocn/Tendencies.h index ad5bd8b4f3ee..1296e8fd80ce 100644 --- a/components/omega/src/ocn/Tendencies.h +++ b/components/omega/src/ocn/Tendencies.h @@ -185,8 +185,10 @@ class Tendencies { const HorzMesh *Mesh; ///< Pointer to horizontal mesh VertCoord *VCoord; ///< Pointer to vertical coordinate VertAdv *VAdv; ///< Pointer to vertical advection - PressureGrad *PGrad; ///< Pointer to pressure gradient + CustomTendencyType CustomThicknessTend; + CustomTendencyType CustomVelocityTend; Eos *EqState; ///< Pointer to equation of state + PressureGrad *PGrad; ///< Pointer to pressure gradient I4 NTracers; ///< Number of tracers TimeInterval TimeStep; ///< Time step @@ -196,8 +198,6 @@ class Tendencies { // Map of all tendency objects static std::map> AllTendencies; - CustomTendencyType CustomThicknessTend; - CustomTendencyType CustomVelocityTend; }; // end class Tendencies diff --git a/components/omega/test/ocn/PGradTest.cpp b/components/omega/test/ocn/PGradTest.cpp index 8c1e63100c00..6d70640f10b6 100644 --- a/components/omega/test/ocn/PGradTest.cpp +++ b/components/omega/test/ocn/PGradTest.cpp @@ -87,7 +87,6 @@ void initPGradTest() { int main(int argc, char *argv[]) { int RetVal = 0; - int Err; MPI_Init(&argc, &argv); Kokkos::initialize(); @@ -99,13 +98,10 @@ int main(int argc, char *argv[]) { // Initialize default PressureGrad PressureGrad::init(); - MachEnv *DefEnv = MachEnv::getDefault(); HorzMesh *DefMesh = HorzMesh::getDefault(); VertCoord *VCoord = VertCoord::getDefault(); OceanState *DefState = OceanState::getDefault(); - Decomp *DefDecomp = Decomp::getDefault(); Eos *DefEos = Eos::getInstance(); - Config *Options = Config::getOmegaConfig(); // create arrays: Tend on edges, Pressure/Geopotential/SpecVol on cells Array2DReal Tend("Tend", DefMesh->NEdgesSize, VCoord->NVertLayers); @@ -147,7 +143,6 @@ int main(int argc, char *argv[]) { auto &CellsOnEdge = DefMesh->CellsOnEdge; auto &DcEdge = DefMesh->DcEdge; - auto &EdgeMask = VCoord->EdgeMask; parallelFor( {NEdgesAll}, KOKKOS_LAMBDA(int i) { CellsOnEdge(i, 0) = 0; From 9f02c7e6d97318934e4e269ecdb711a448ce8dc6 Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Thu, 5 Mar 2026 15:54:45 -0800 Subject: [PATCH 50/58] Fix linting issues --- components/omega/src/ocn/Tendencies.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/components/omega/src/ocn/Tendencies.h b/components/omega/src/ocn/Tendencies.h index 1296e8fd80ce..e6b98bdfd0b1 100644 --- a/components/omega/src/ocn/Tendencies.h +++ b/components/omega/src/ocn/Tendencies.h @@ -187,9 +187,9 @@ class Tendencies { VertAdv *VAdv; ///< Pointer to vertical advection CustomTendencyType CustomThicknessTend; CustomTendencyType CustomVelocityTend; - Eos *EqState; ///< Pointer to equation of state - PressureGrad *PGrad; ///< Pointer to pressure gradient - I4 NTracers; ///< Number of tracers + Eos *EqState; ///< Pointer to equation of state + PressureGrad *PGrad; ///< Pointer to pressure gradient + I4 NTracers; ///< Number of tracers TimeInterval TimeStep; ///< Time step // Pointer to default tendencies @@ -198,7 +198,6 @@ class Tendencies { // Map of all tendency objects static std::map> AllTendencies; - }; // end class Tendencies } // namespace OMEGA From ee4be1a92dfb38bb29dfed7da967c7a5dcb779e7 Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Mon, 16 Mar 2026 18:15:26 -0700 Subject: [PATCH 51/58] Fix linting issues after rebase --- components/omega/src/ocn/Tendencies.cpp | 13 +++++++------ components/omega/src/ocn/Tendencies.h | 6 +++--- .../omega/test/timeStepping/TimeStepperTest.cpp | 4 ++-- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/components/omega/src/ocn/Tendencies.cpp b/components/omega/src/ocn/Tendencies.cpp index e5441b0383cc..8d46f598e1f3 100644 --- a/components/omega/src/ocn/Tendencies.cpp +++ b/components/omega/src/ocn/Tendencies.cpp @@ -77,9 +77,9 @@ void Tendencies::init() { TimeInterval TimeStep = DefTimeStepper->getTimeStep(); // Ceate default tendencies - Tendencies::DefaultTendencies = - create("Default", DefHorzMesh, DefVertCoord, DefVertAdv, DefPGrad, DefEos, - NTracers, TimeStep, &TendConfig, CustomThickTend, CustomVelTend); + Tendencies::DefaultTendencies = create( + "Default", DefHorzMesh, DefVertCoord, DefVertAdv, DefPGrad, DefEos, + NTracers, TimeStep, &TendConfig, CustomThickTend, CustomVelTend); DefaultTendencies->readConfig(OmegaConfig); @@ -245,7 +245,7 @@ void Tendencies::readConfig(Config *OmegaConfig ///< [in] Omega config CHECK_ERROR_ABORT(Err, "Tendencies: EddyDiff4 not found in TendConfig"); } - Err += TendConfig->get("PressureGradTendencyEnable", this->PGrad->Enabled); + Err += TendConfig.get("PressureGradTendencyEnable", this->PGrad->Enabled); CHECK_ERROR_ABORT( Err, "Tendencies: PressureGradTendencyEnable not found in TendConfig"); } @@ -351,8 +351,9 @@ Tendencies::Tendencies(const std::string &Name_, ///< [in] Name for tendencies int NTracersIn, ///< [in] Number of tracers TimeInterval TimeStepIn, ///< [in] Time step Config *Options) ///< [in] Configuration options - : Tendencies(Name_, Mesh, VCoord, VAdv, PGrad, EqState, NTracersIn, TimeStepIn, Options, - CustomTendencyType{}, CustomTendencyType{}) {} + : Tendencies(Name_, Mesh, VCoord, VAdv, PGrad, EqState, NTracersIn, + TimeStepIn, Options, CustomTendencyType{}, + CustomTendencyType{}) {} //------------------------------------------------------------------------------ // Compute tendencies for layer thickness equation diff --git a/components/omega/src/ocn/Tendencies.h b/components/omega/src/ocn/Tendencies.h index e6b98bdfd0b1..757abc168d45 100644 --- a/components/omega/src/ocn/Tendencies.h +++ b/components/omega/src/ocn/Tendencies.h @@ -187,9 +187,9 @@ class Tendencies { VertAdv *VAdv; ///< Pointer to vertical advection CustomTendencyType CustomThicknessTend; CustomTendencyType CustomVelocityTend; - Eos *EqState; ///< Pointer to equation of state - PressureGrad *PGrad; ///< Pointer to pressure gradient - I4 NTracers; ///< Number of tracers + Eos *EqState; ///< Pointer to equation of state + PressureGrad *PGrad; ///< Pointer to pressure gradient + I4 NTracers; ///< Number of tracers TimeInterval TimeStep; ///< Time step // Pointer to default tendencies diff --git a/components/omega/test/timeStepping/TimeStepperTest.cpp b/components/omega/test/timeStepping/TimeStepperTest.cpp index 10f802617e5a..0de465c17fe4 100644 --- a/components/omega/test/timeStepping/TimeStepperTest.cpp +++ b/components/omega/test/timeStepping/TimeStepperTest.cpp @@ -222,8 +222,8 @@ int initTimeStepperTest(const std::string &mesh) { // Creating non-default tendencies with custom velocity tendencies auto *TestTendencies = Tendencies::create( - "TestTendencies", DefMesh, DefVertCoord, DefVAdv, DefPGrad, DefEos, NTracers, - ZeroTimeStep, &Options, Tendencies::CustomTendencyType{}, + "TestTendencies", DefMesh, DefVertCoord, DefVAdv, DefPGrad, DefEos, + NTracers, ZeroTimeStep, &Options, Tendencies::CustomTendencyType{}, DecayVelocityTendency{}); if (!TestTendencies) { Err++; From e58f59680157f2c5ca02d88cf6b5c3c2e9039bd0 Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Thu, 19 Mar 2026 14:46:54 -0700 Subject: [PATCH 52/58] Fix vertRange -> vertRangeChunked --- components/omega/src/ocn/PGrad.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/omega/src/ocn/PGrad.cpp b/components/omega/src/ocn/PGrad.cpp index bbd11a889fb4..20257fc6ccdd 100644 --- a/components/omega/src/ocn/PGrad.cpp +++ b/components/omega/src/ocn/PGrad.cpp @@ -181,7 +181,7 @@ void PressureGrad::computePressureGrad(Array2DReal &Tend, KOKKOS_LAMBDA(I4 IEdge, const TeamMember &Team) { const int KMin = LocMinLayerEdgeBot(IEdge); const int KMax = LocMaxLayerEdgeTop(IEdge); - const int KRange = vertRange(KMin, KMax); + const int KRange = vertRangeChunked(KMin, KMax); parallelForInner( Team, KRange, INNER_LAMBDA(int KChunk) { @@ -200,7 +200,7 @@ void PressureGrad::computePressureGrad(Array2DReal &Tend, KOKKOS_LAMBDA(I4 IEdge, const TeamMember &Team) { const int KMin = LocMinLayerEdgeBot(IEdge); const int KMax = LocMaxLayerEdgeTop(IEdge); - const int KRange = vertRange(KMin, KMax); + const int KRange = vertRangeChunked(KMin, KMax); parallelForInner( Team, KRange, INNER_LAMBDA(int KChunk) { From 45b90e5be1da98b631cc8b3218fb3a4fd5fbceb0 Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Thu, 19 Mar 2026 15:30:03 -0700 Subject: [PATCH 53/58] Fix converence criteria in CTest --- components/omega/test/ocn/PGradTest.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/omega/test/ocn/PGradTest.cpp b/components/omega/test/ocn/PGradTest.cpp index 6d70640f10b6..4259c300a3d5 100644 --- a/components/omega/test/ocn/PGradTest.cpp +++ b/components/omega/test/ocn/PGradTest.cpp @@ -302,7 +302,8 @@ int main(int argc, char *argv[]) { } // refinement loop // Test for second order convergence - if (RMSE(NRefinements) < RMSE(0) / pow(4.0_Real, NRefinements - 1)) { + // resolution (dC) increases in refimenent loop + if (RMSE(0) < RMSE(NRefinements - 1) / pow(4.0_Real, NRefinements - 1)) { RetVal = 0; } else { RetVal = 1; From f016d0f5fffa3b4ef56d5b2f89b54927e46f214e Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Thu, 26 Mar 2026 14:20:08 -0700 Subject: [PATCH 54/58] Fix variable naming convention in PGradTest.cpp --- components/omega/test/ocn/PGradTest.cpp | 82 ++++++++++++------------- 1 file changed, 40 insertions(+), 42 deletions(-) diff --git a/components/omega/test/ocn/PGradTest.cpp b/components/omega/test/ocn/PGradTest.cpp index 4259c300a3d5..d2a58a8dd365 100644 --- a/components/omega/test/ocn/PGradTest.cpp +++ b/components/omega/test/ocn/PGradTest.cpp @@ -115,12 +115,12 @@ int main(int argc, char *argv[]) { I4 NCellsAll = DefMesh->NCellsAll; I4 NVertLayers = 60; - Real dC = 30000.0_Real; + Real DC = 30000.0_Real; I4 NRefinements = 4; - HostArray1DReal RMSE("RMSE", NRefinements); - for (int refinement = 0; refinement < NRefinements; ++refinement) { + HostArray1DReal Rmse("Rmse", NRefinements); + for (int Refinement = 0; Refinement < NRefinements; ++Refinement) { - LOG_INFO("PGradTest: Starting refinement level {}", refinement); + LOG_INFO("PGradTest: Starting refinement level {}", Refinement); VCoord->NVertLayers = NVertLayers; VCoord->NVertLayersP1 = NVertLayers + 1; @@ -138,7 +138,6 @@ int main(int argc, char *argv[]) { {NEdgesAll}, KOKKOS_LAMBDA(int i) { MinLayerEdgeBot(i) = 0; MaxLayerEdgeTop(i) = NVertLayers - 1; - // MaxLayerEdgeTop(i) = 4; }); auto &CellsOnEdge = DefMesh->CellsOnEdge; @@ -147,7 +146,7 @@ int main(int argc, char *argv[]) { {NEdgesAll}, KOKKOS_LAMBDA(int i) { CellsOnEdge(i, 0) = 0; CellsOnEdge(i, 1) = 1; - DcEdge(i) = dC; + DcEdge(i) = DC; }); // Fetch reference desnity from Config @@ -163,32 +162,31 @@ int main(int argc, char *argv[]) { // set Z interface and mid-point locations Real ZBottom = -1000.0_Real; - Real dZ = 2.0_Real * (-ZBottom / NVertLayers); + Real DZ = 2.0_Real * (-ZBottom / NVertLayers); auto &BottomDepth = VCoord->BottomDepth; auto &ZInterface = VCoord->ZInterface; auto &ZMid = VCoord->ZMid; - Real tilt_factor = 0.495_Real; - // Real tilt_factor = 0.45_Real; + Real TiltFactor = 0.495_Real; parallelFor( {NCellsAll}, KOKKOS_LAMBDA(int i) { ZInterface(i, NVertLayers) = ZBottom; SurfacePressure(i) = 0.0_Real; BottomDepth(i) = 0.0_Real; for (int k = NVertLayers - 1; k >= 0; --k) { - Real x = (k + i) % 2; - Real dz = (2.0_Real * tilt_factor - 1.0_Real) * x * dZ + - (1.0_Real - tilt_factor) * - dZ; // staggered layer thickness - ZInterface(i, k) = ZInterface(i, k + 1) + dz; + Real X = (k + i) % 2; + Real Dz = (2.0_Real * TiltFactor - 1.0_Real) * X * DZ + + (1.0_Real - TiltFactor) * + DZ; // staggered layer thickness + ZInterface(i, k) = ZInterface(i, k + 1) + Dz; LayerThick(i, k) = ZInterface(i, k) - ZInterface(i, k + 1); ZMid(i, k) = 0.5_Real * (ZInterface(i, k) + ZInterface(i, k + 1)); - BottomDepth(i) += dz; + BottomDepth(i) += Dz; } }); LOG_INFO("NVertLayers = {}", NVertLayers); - LOG_INFO("dC = {}", dC); + LOG_INFO("dC = {}", DC); DefState->copyToHost(0); HostArray2DReal LayerThickH = DefState->getLayerThicknessH(TimeLevel); for (int i = 0; i < 2; ++i) { @@ -206,11 +204,11 @@ int main(int argc, char *argv[]) { Real S0 = 30.0; Real SB = 40.0; - Real phi0 = (ZMid(i, k) - ZBottom) / (-ZBottom); - Real phiB = 1.0_Real - phi0; + Real Phi0 = (ZMid(i, k) - ZBottom) / (-ZBottom); + Real PhiB = 1.0_Real - Phi0; - Temp(i, k) = T0 * phi0 + TB * phiB; - Salinity(i, k) = S0 * phi0 + SB * phiB; + Temp(i, k) = T0 * Phi0 + TB * PhiB; + Salinity(i, k) = S0 * Phi0 + SB * PhiB; SpecVol(i, k) = 1.0_Real / Density0; SpecVolOld(i, k) = SpecVol(i, k); }); @@ -219,7 +217,7 @@ int main(int argc, char *argv[]) { auto &PressureMid = VCoord->PressureMid; VCoord->computePressure(LayerThick, SurfacePressure); deepCopy(PressureMidOld, PressureMid); - for (int iteration = 0; iteration < 15; ++iteration) { + for (int Iteration = 0; Iteration < 15; ++Iteration) { // compute specific volume from EOS VCoord->computePressure(LayerThick, SurfacePressure); @@ -233,19 +231,19 @@ int main(int argc, char *argv[]) { }); // compute difference from previous iteration - Real max_value = 0.0_Real; + Real MaxValue = 0.0_Real; parallelReduce( {NCellsAll, NVertLayers}, KOKKOS_LAMBDA(int i, int k, Real &max) { - Real diff = Kokkos::abs(SpecVol(i, k) - SpecVolOld(i, k)); - if (diff > max) - max = diff; + Real Diff = Kokkos::abs(SpecVol(i, k) - SpecVolOld(i, k)); + if (Diff > max) + max = Diff; }, - Kokkos::Max(max_value)); + Kokkos::Max(MaxValue)); // check convergence - if (max_value < 1e-12_Real) { - LOG_INFO("converged: max diff = {}", max_value); + if (MaxValue < 1e-12_Real) { + LOG_INFO("converged: max diff = {}", MaxValue); break; } else { parallelFor( @@ -273,37 +271,37 @@ int main(int argc, char *argv[]) { TimeLevel); // compute errors - Real max_value = 0.0_Real; + Real MaxValue = 0.0_Real; parallelReduce( {NEdgesAll, NVertLayers - 2}, KOKKOS_LAMBDA(int i, int k, Real &max) { - Real val = Kokkos::abs(Tend(i, k + 1)); - if (val > max) - max = val; + Real Val = Kokkos::abs(Tend(i, k + 1)); + if (Val > max) + max = Val; }, - Kokkos::Max(max_value)); - Real sum_value = 0.0_Real; + Kokkos::Max(MaxValue)); + Real SumValue = 0.0_Real; parallelReduce( {NEdgesAll, NVertLayers - 2}, - KOKKOS_LAMBDA(int i, int k, Real &lsum) { - lsum += Tend(i, k + 1) * Tend(i, k + 1); + KOKKOS_LAMBDA(int i, int k, Real &LSum) { + LSum += Tend(i, k + 1) * Tend(i, k + 1); }, - Kokkos::Sum(sum_value)); - Real rmse = std::sqrt(sum_value / (NEdgesAll * (NVertLayers - 2))); - RMSE(refinement) = rmse; + Kokkos::Sum(SumValue)); + Real RmseVal = std::sqrt(SumValue / (NEdgesAll * (NVertLayers - 2))); + Rmse(Refinement) = RmseVal; LOG_INFO("refinement level {}: max |Tend| = {}, average Tend = {}", - refinement, max_value, rmse); + Refinement, MaxValue, RmseVal); // coarsen for next iteration - dC = dC * 2.0_Real; + DC = DC * 2.0_Real; NVertLayers = NVertLayers / 2; } // refinement loop // Test for second order convergence // resolution (dC) increases in refimenent loop - if (RMSE(0) < RMSE(NRefinements - 1) / pow(4.0_Real, NRefinements - 1)) { + if (Rmse(0) < Rmse(NRefinements - 1) / pow(4.0_Real, NRefinements - 1)) { RetVal = 0; } else { RetVal = 1; From bb0e1be95ae165ff7d66d12a63c2abc5537ea643 Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Thu, 26 Mar 2026 14:50:14 -0700 Subject: [PATCH 55/58] Incorporate review feedback --- components/omega/src/ocn/PGrad.cpp | 25 +++++++++-------- components/omega/src/ocn/PGrad.h | 13 +++++---- components/omega/src/ocn/Tendencies.cpp | 28 ++++++++++++------- components/omega/src/ocn/Tendencies.h | 7 +++-- .../timeStepping/ForwardBackwardStepper.cpp | 2 +- .../src/timeStepping/RungeKutta2Stepper.cpp | 4 +-- .../src/timeStepping/RungeKutta4Stepper.cpp | 4 +-- components/omega/test/ocn/PGradTest.cpp | 7 +++-- components/omega/test/ocn/TendenciesTest.cpp | 4 ++- 9 files changed, 56 insertions(+), 38 deletions(-) diff --git a/components/omega/src/ocn/PGrad.cpp b/components/omega/src/ocn/PGrad.cpp index 20257fc6ccdd..e91d03b28695 100644 --- a/components/omega/src/ocn/PGrad.cpp +++ b/components/omega/src/ocn/PGrad.cpp @@ -82,9 +82,9 @@ PressureGrad::PressureGrad( // store mesh sizes NEdgesAll = Mesh->NEdgesAll; + NEdgesOwned = Mesh->NEdgesOwned; NVertLayers = VCoord->NVertLayers; NVertLayersP1 = NVertLayers + 1; - NChunks = NVertLayers / VecLength; // Read config options for PressureGrad type // and enable the appropriate functor @@ -124,7 +124,13 @@ PressureGrad::~PressureGrad() { //------------------------------------------------------------------------------ // Remove PressureGrad instances before exit -void PressureGrad::clear() { AllPGrads.clear(); } // end clear +void PressureGrad::clear() { + + AllPGrads.clear(); + + DefaultPGrad = nullptr; // prevent dangling pointe'r + + } // end clear //------------------------------------------------------------------------------ // Remove PressureGrad from list by name @@ -155,10 +161,11 @@ PressureGrad *PressureGrad::get(const std::string &Name ///< [in] Name of //------------------------------------------------------------------------------ // Compute pressure gradient tendencies and add into Tend array void PressureGrad::computePressureGrad(Array2DReal &Tend, - const OceanState *State, - const VertCoord *VCoord, - const Eos *EqState, - const int TimeLevel) const { + const Array2DReal &PressureMid, + const Array2DReal &PressureInterface, + const Array2DReal &SpecVol, + const Array2DReal &ZInterface, + const Array2DReal &LayerThick) const { OMEGA_SCOPE(LocCenteredPGrad, CenteredPGrad); OMEGA_SCOPE(LocHighOrderPGrad, HighOrderPGrad); @@ -167,12 +174,6 @@ void PressureGrad::computePressureGrad(Array2DReal &Tend, OMEGA_SCOPE(LocTidalPotential, TidalPotential); OMEGA_SCOPE(LocSelfAttractionLoading, SelfAttractionLoading); - const Array2DReal &PressureMid = VCoord->PressureMid; - const Array2DReal &PressureInterface = VCoord->PressureInterface; - const Array2DReal &SpecVol = EqState->SpecVol; - const Array2DReal &ZInterface = VCoord->ZInterface; - Array2DReal LayerThick = State->getLayerThickness(TimeLevel); - if (PressureGradChoice == PressureGradType::Centered) { // computes centered geopotential and pressure gradient tendency diff --git a/components/omega/src/ocn/PGrad.h b/components/omega/src/ocn/PGrad.h index e0fa06e8c2fc..50941762ef76 100644 --- a/components/omega/src/ocn/PGrad.h +++ b/components/omega/src/ocn/PGrad.h @@ -73,7 +73,7 @@ class PressureGradCentered { Real GradMontPot = 0.5_Real * (GradMontPotK + GradMontPotKp1); Real PGradAlpha = - 0.5 * (PressureMid(ICell1, K) + PressureMid(ICell0, K)) * + 0.5_Real * (PressureMid(ICell1, K) + PressureMid(ICell0, K)) * (SpecVol(ICell1, K) - SpecVol(ICell0, K)) * InvDcEdge; Tend(IEdge, K) += EdgeMask(IEdge, K) * (-GradMontPot + PGradAlpha - GradGeoPot); @@ -156,9 +156,12 @@ class PressureGrad { ~PressureGrad(); // Compute pressure gradient tendencies and add into Tend array - void computePressureGrad(Array2DReal &Tend, const OceanState *State, - const VertCoord *VCoord, const Eos *EqState, - const int TimeLevel) const; + void computePressureGrad(Array2DReal &Tend, + const Array2DReal &PressureMid, + const Array2DReal &PressureInterface, + const Array2DReal &SpecVol, + const Array2DReal &ZInterface, + const Array2DReal &LayerThick) const; private: // Construct a new pressure gradient object @@ -173,7 +176,7 @@ class PressureGrad { // Mesh-related sizes I4 NEdgesAll = 0; - I4 NChunks = 0; + I4 NEdgesOwned = 0; I4 NVertLayers = 0; I4 NVertLayersP1 = 0; diff --git a/components/omega/src/ocn/Tendencies.cpp b/components/omega/src/ocn/Tendencies.cpp index 8d46f598e1f3..c42b9fa3ed76 100644 --- a/components/omega/src/ocn/Tendencies.cpp +++ b/components/omega/src/ocn/Tendencies.cpp @@ -431,6 +431,7 @@ void Tendencies::computeVelocityTendenciesOnly( const AuxiliaryState *AuxState, ///< [in] Auxilary state variables int ThickTimeLevel, ///< [in] Time level int VelTimeLevel, ///< [in] Time level + int TracerTimeLevel, ///< [in] Time level TimeInstant Time ///< [in] Time ) { @@ -609,20 +610,24 @@ void Tendencies::computeVelocityTendenciesOnly( deepCopy(SurfacePressure, 0.0_Real); Pacer::start("Tend:pressureGradTerm", 2); - Array2DReal LayerThick = State->getLayerThickness(VelTimeLevel); + Array2DReal LayerThick = State->getLayerThickness(ThickTimeLevel); VCoord->computePressure(LayerThick, SurfacePressure); - OMEGA_SCOPE(LocPressureMid, VCoord->PressureMid); - Array2DReal Temp = Tracers::getByName(VelTimeLevel, "Temperature"); - Array2DReal Salinity = Tracers::getByName(VelTimeLevel, "Salinity"); - EqState->computeSpecVol(Temp, Salinity, LocPressureMid); + const auto &PressureMid = VCoord->PressureMid; + const auto &PressureInterface = VCoord->PressureInterface; + Array2DReal Temp = Tracers::getByName(TracerTimeLevel, "Temperature"); + Array2DReal Salinity = Tracers::getByName(TracerTimeLevel, "Salinity"); + EqState->computeSpecVol(Temp, Salinity, PressureMid); // Temporary: ensure vertical geometric/geopotential fields are updated // for pressure-gradient tendency calculations. - VCoord->computeZHeight(LayerThick, EqState->SpecVol); + const auto &SpecVol = EqState->SpecVol; + VCoord->computeZHeight(LayerThick, SpecVol); - PGrad->computePressureGrad(LocNormalVelocityTend, State, VCoord, EqState, - VelTimeLevel); + const auto &ZInterface = VCoord->ZInterface; + PGrad->computePressureGrad(LocNormalVelocityTend, PressureMid, + PressureInterface, SpecVol, ZInterface, + LayerThick); Pacer::stop("Tend:pressureGradTerm", 2); } @@ -795,13 +800,14 @@ void Tendencies::computeVelocityTendencies( const AuxiliaryState *AuxState, ///< [in] Auxilary state variables int ThickTimeLevel, ///< [in] Time level int VelTimeLevel, ///< [in] Time level + int TracerTimeLevel, ///< [in] Time level TimeInstant Time ///< [in] Time ) { Pacer::start("Tend:computeVelocityTendencies", 1); AuxState->computeMomAux(State, ThickTimeLevel, VelTimeLevel); computeVelocityTendenciesOnly(State, AuxState, ThickTimeLevel, VelTimeLevel, - Time); + TracerTimeLevel, Time); Pacer::stop("Tend:computeVelocityTendencies", 1); } @@ -856,13 +862,15 @@ void Tendencies::computeAllTendencies( const Array3DReal &TracerArray, ///< [in] Tracer array int ThickTimeLevel, ///< [in] Time level int VelTimeLevel, ///< [in] Time level + int TracerTimeLevel, ///< [in] Time level TimeInstant Time ///< [in] Time ) { AuxState->computeAll(State, TracerArray, ThickTimeLevel, VelTimeLevel); + computeThicknessTendenciesOnly(State, AuxState, ThickTimeLevel, VelTimeLevel, Time); computeVelocityTendenciesOnly(State, AuxState, ThickTimeLevel, VelTimeLevel, - Time); + TracerTimeLevel, Time); computeTracerTendenciesOnly(State, AuxState, TracerArray, ThickTimeLevel, VelTimeLevel, Time); } // end all tendency compute diff --git a/components/omega/src/ocn/Tendencies.h b/components/omega/src/ocn/Tendencies.h index 757abc168d45..1955fa7c130e 100644 --- a/components/omega/src/ocn/Tendencies.h +++ b/components/omega/src/ocn/Tendencies.h @@ -82,7 +82,7 @@ class Tendencies { void computeVelocityTendencies(const OceanState *State, const AuxiliaryState *AuxState, int ThickTimeLevel, int VelTimeLevel, - TimeInstant Time); + int TracerTimeLevel, TimeInstant Time); void computeTracerTendencies(const OceanState *State, const AuxiliaryState *AuxState, const Array3DReal &TracerArray, @@ -91,7 +91,8 @@ class Tendencies { void computeAllTendencies(const OceanState *State, const AuxiliaryState *AuxState, const Array3DReal &TracerArray, int ThickTimeLevel, - int VelTimeLevel, TimeInstant Time); + int VelTimeLevel, int TracerTimeLevel, + TimeInstant Time); void computeThicknessTendenciesOnly(const OceanState *State, const AuxiliaryState *AuxState, int ThickTimeLevel, int VelTimeLevel, @@ -99,7 +100,7 @@ class Tendencies { void computeVelocityTendenciesOnly(const OceanState *State, const AuxiliaryState *AuxState, int ThickTimeLevel, int VelTimeLevel, - TimeInstant Time); + int TracerTimeLevel, TimeInstant Time); void computeTracerTendenciesOnly(const OceanState *State, const AuxiliaryState *AuxState, const Array3DReal &TracerArray, diff --git a/components/omega/src/timeStepping/ForwardBackwardStepper.cpp b/components/omega/src/timeStepping/ForwardBackwardStepper.cpp index 12c53a59a436..a686dcfae0d6 100644 --- a/components/omega/src/timeStepping/ForwardBackwardStepper.cpp +++ b/components/omega/src/timeStepping/ForwardBackwardStepper.cpp @@ -57,7 +57,7 @@ void ForwardBackwardStepper::doStep( // R_u^{n+1} = RHS_u(u^{n}, h^{n+1}, t^{n+1}) Tend->computeVelocityTendencies(State, AuxState, NextLevel, CurLevel, - SimTime + TimeStep); + NextLevel, SimTime + TimeStep); // u^{n+1} = u^{n} + R_u^{n+1} updateVelocityByTend(State, NextLevel, State, CurLevel, TimeStep); diff --git a/components/omega/src/timeStepping/RungeKutta2Stepper.cpp b/components/omega/src/timeStepping/RungeKutta2Stepper.cpp index f4b697529db0..47ffc54b71fe 100644 --- a/components/omega/src/timeStepping/RungeKutta2Stepper.cpp +++ b/components/omega/src/timeStepping/RungeKutta2Stepper.cpp @@ -37,7 +37,7 @@ void RungeKutta2Stepper::doStep(OceanState *State, // model state // q = (h,u,phi) // R_q^{n} = RHS_q(u^{n}, h^{n}, phi^{n}, t^{n}) Tend->computeAllTendencies(State, AuxState, CurTracerArray, CurLevel, - CurLevel, SimTime); + CurLevel, CurLevel, SimTime); // q^{n+0.5} = q^{n} + 0.5*dt*R_q^{n} updateStateByTend(State, NextLevel, State, CurLevel, 0.5 * TimeStep); @@ -49,7 +49,7 @@ void RungeKutta2Stepper::doStep(OceanState *State, // model state // R_q^{n+0.5} = RHS_q(u^{n+0.5}, h^{n+0.5}, phi^{n+0.5}, t^{n+0.5}) Tend->computeAllTendencies(State, AuxState, NextTracerArray, NextLevel, - NextLevel, SimTime + 0.5 * TimeStep); + NextLevel, NextLevel, SimTime + 0.5 * TimeStep); // q^{n+1} = q^{n} + dt*R_q^{n+0.5} updateStateByTend(State, NextLevel, State, CurLevel, TimeStep); diff --git a/components/omega/src/timeStepping/RungeKutta4Stepper.cpp b/components/omega/src/timeStepping/RungeKutta4Stepper.cpp index 1b882e7631b9..1160a4ab7213 100644 --- a/components/omega/src/timeStepping/RungeKutta4Stepper.cpp +++ b/components/omega/src/timeStepping/RungeKutta4Stepper.cpp @@ -85,7 +85,7 @@ void RungeKutta4Stepper::doStep(OceanState *State, // model state if (Stage == 0) { weightTracers(NextTracerArray, CurTracerArray, State, CurLevel); Tend->computeAllTendencies(State, AuxState, CurTracerArray, CurLevel, - CurLevel, StageTime); + CurLevel, CurLevel, StageTime); updateStateByTend(State, NextLevel, State, CurLevel, RKB[Stage] * TimeStep); accumulateTracersUpdate(NextTracerArray, RKB[Stage] * TimeStep); @@ -111,7 +111,7 @@ void RungeKutta4Stepper::doStep(OceanState *State, // model state Pacer::stop("RK4:haloExchProvis", 3); Tend->computeAllTendencies(ProvisState, AuxState, ProvisTracers, - CurLevel, CurLevel, StageTime); + CurLevel, CurLevel, CurLevel, StageTime); updateStateByTend(State, NextLevel, State, NextLevel, RKB[Stage] * TimeStep); accumulateTracersUpdate(NextTracerArray, RKB[Stage] * TimeStep); diff --git a/components/omega/test/ocn/PGradTest.cpp b/components/omega/test/ocn/PGradTest.cpp index d2a58a8dd365..8fabb3a0bab2 100644 --- a/components/omega/test/ocn/PGradTest.cpp +++ b/components/omega/test/ocn/PGradTest.cpp @@ -267,8 +267,11 @@ int main(int argc, char *argv[]) { // compute pressure gradient deepCopy(Tend, 0.0_Real); - DefPGrad->computePressureGrad(Tend, DefState, VCoord, DefEos, - TimeLevel); + + const auto &PressureInterface = VCoord->PressureInterface; + DefPGrad->computePressureGrad(Tend, PressureMid, PressureInterface, + SpecVol, ZInterface, + LayerThick); // compute errors Real MaxValue = 0.0_Real; diff --git a/components/omega/test/ocn/TendenciesTest.cpp b/components/omega/test/ocn/TendenciesTest.cpp index 09a42e15ff74..72c680b0d303 100644 --- a/components/omega/test/ocn/TendenciesTest.cpp +++ b/components/omega/test/ocn/TendenciesTest.cpp @@ -213,9 +213,11 @@ int testTendencies() { Array3DReal TracerArray = Tracers::getAll(0); int ThickTimeLevel = 0; int VelTimeLevel = 0; + int TracerTimeLevel = 0; TimeInstant Time; DefTendencies->computeAllTendencies(State, AuxState, TracerArray, - ThickTimeLevel, VelTimeLevel, Time); + ThickTimeLevel, VelTimeLevel, + TracerTimeLevel, Time); // check that everything got computed correctly int NCellsOwned = Mesh->NCellsOwned; From 6f2b1a523496eee4340ae707730d17a625c6cd2e Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Thu, 26 Mar 2026 15:14:36 -0700 Subject: [PATCH 56/58] Fix linting issues --- components/omega/src/ocn/PGrad.cpp | 2 +- components/omega/src/ocn/PGrad.h | 3 +-- components/omega/src/ocn/Tendencies.cpp | 2 +- components/omega/src/ocn/Tendencies.h | 2 +- components/omega/test/ocn/PGradTest.cpp | 5 ++--- components/omega/test/ocn/TendenciesTest.cpp | 4 ++-- 6 files changed, 8 insertions(+), 10 deletions(-) diff --git a/components/omega/src/ocn/PGrad.cpp b/components/omega/src/ocn/PGrad.cpp index e91d03b28695..33f3320c7606 100644 --- a/components/omega/src/ocn/PGrad.cpp +++ b/components/omega/src/ocn/PGrad.cpp @@ -130,7 +130,7 @@ void PressureGrad::clear() { DefaultPGrad = nullptr; // prevent dangling pointe'r - } // end clear +} // end clear //------------------------------------------------------------------------------ // Remove PressureGrad from list by name diff --git a/components/omega/src/ocn/PGrad.h b/components/omega/src/ocn/PGrad.h index 50941762ef76..9c64b4b86e82 100644 --- a/components/omega/src/ocn/PGrad.h +++ b/components/omega/src/ocn/PGrad.h @@ -156,8 +156,7 @@ class PressureGrad { ~PressureGrad(); // Compute pressure gradient tendencies and add into Tend array - void computePressureGrad(Array2DReal &Tend, - const Array2DReal &PressureMid, + void computePressureGrad(Array2DReal &Tend, const Array2DReal &PressureMid, const Array2DReal &PressureInterface, const Array2DReal &SpecVol, const Array2DReal &ZInterface, diff --git a/components/omega/src/ocn/Tendencies.cpp b/components/omega/src/ocn/Tendencies.cpp index c42b9fa3ed76..39611b85bc35 100644 --- a/components/omega/src/ocn/Tendencies.cpp +++ b/components/omega/src/ocn/Tendencies.cpp @@ -613,7 +613,7 @@ void Tendencies::computeVelocityTendenciesOnly( Array2DReal LayerThick = State->getLayerThickness(ThickTimeLevel); VCoord->computePressure(LayerThick, SurfacePressure); - const auto &PressureMid = VCoord->PressureMid; + const auto &PressureMid = VCoord->PressureMid; const auto &PressureInterface = VCoord->PressureInterface; Array2DReal Temp = Tracers::getByName(TracerTimeLevel, "Temperature"); Array2DReal Salinity = Tracers::getByName(TracerTimeLevel, "Salinity"); diff --git a/components/omega/src/ocn/Tendencies.h b/components/omega/src/ocn/Tendencies.h index 1955fa7c130e..605517034a79 100644 --- a/components/omega/src/ocn/Tendencies.h +++ b/components/omega/src/ocn/Tendencies.h @@ -91,7 +91,7 @@ class Tendencies { void computeAllTendencies(const OceanState *State, const AuxiliaryState *AuxState, const Array3DReal &TracerArray, int ThickTimeLevel, - int VelTimeLevel, int TracerTimeLevel, + int VelTimeLevel, int TracerTimeLevel, TimeInstant Time); void computeThicknessTendenciesOnly(const OceanState *State, const AuxiliaryState *AuxState, diff --git a/components/omega/test/ocn/PGradTest.cpp b/components/omega/test/ocn/PGradTest.cpp index 8fabb3a0bab2..09887a6d9ce6 100644 --- a/components/omega/test/ocn/PGradTest.cpp +++ b/components/omega/test/ocn/PGradTest.cpp @@ -267,11 +267,10 @@ int main(int argc, char *argv[]) { // compute pressure gradient deepCopy(Tend, 0.0_Real); - + const auto &PressureInterface = VCoord->PressureInterface; DefPGrad->computePressureGrad(Tend, PressureMid, PressureInterface, - SpecVol, ZInterface, - LayerThick); + SpecVol, ZInterface, LayerThick); // compute errors Real MaxValue = 0.0_Real; diff --git a/components/omega/test/ocn/TendenciesTest.cpp b/components/omega/test/ocn/TendenciesTest.cpp index 72c680b0d303..c87828f6bccf 100644 --- a/components/omega/test/ocn/TendenciesTest.cpp +++ b/components/omega/test/ocn/TendenciesTest.cpp @@ -213,10 +213,10 @@ int testTendencies() { Array3DReal TracerArray = Tracers::getAll(0); int ThickTimeLevel = 0; int VelTimeLevel = 0; - int TracerTimeLevel = 0; + int TracerTimeLevel = 0; TimeInstant Time; DefTendencies->computeAllTendencies(State, AuxState, TracerArray, - ThickTimeLevel, VelTimeLevel, + ThickTimeLevel, VelTimeLevel, TracerTimeLevel, Time); // check that everything got computed correctly From afd6126d53c9598f53cc4218c0a9697ecd167bdd Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Mon, 30 Mar 2026 19:08:16 -0700 Subject: [PATCH 57/58] Use provis tracers for RK4 --- components/omega/src/ocn/Tendencies.cpp | 15 ++++++++++----- components/omega/src/ocn/Tendencies.h | 2 ++ .../src/timeStepping/ForwardBackwardStepper.cpp | 2 +- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/components/omega/src/ocn/Tendencies.cpp b/components/omega/src/ocn/Tendencies.cpp index 39611b85bc35..677e7bd9a42e 100644 --- a/components/omega/src/ocn/Tendencies.cpp +++ b/components/omega/src/ocn/Tendencies.cpp @@ -429,6 +429,7 @@ void Tendencies::computeThicknessTendenciesOnly( void Tendencies::computeVelocityTendenciesOnly( const OceanState *State, ///< [in] State variables const AuxiliaryState *AuxState, ///< [in] Auxilary state variables + const Array3DReal &TracerArray, ///< [in] Tracer array int ThickTimeLevel, ///< [in] Time level int VelTimeLevel, ///< [in] Time level int TracerTimeLevel, ///< [in] Time level @@ -615,8 +616,10 @@ void Tendencies::computeVelocityTendenciesOnly( const auto &PressureMid = VCoord->PressureMid; const auto &PressureInterface = VCoord->PressureInterface; - Array2DReal Temp = Tracers::getByName(TracerTimeLevel, "Temperature"); - Array2DReal Salinity = Tracers::getByName(TracerTimeLevel, "Salinity"); + Array2DReal Temp = Kokkos::subview(TracerArray, Tracers::IndxTemp, + Kokkos::ALL, Kokkos::ALL); + Array2DReal Salinity = Kokkos::subview(TracerArray, Tracers::IndxSalt, + Kokkos::ALL, Kokkos::ALL); EqState->computeSpecVol(Temp, Salinity, PressureMid); // Temporary: ensure vertical geometric/geopotential fields are updated @@ -798,6 +801,7 @@ void Tendencies::computeThicknessTendencies( void Tendencies::computeVelocityTendencies( const OceanState *State, ///< [in] State variables const AuxiliaryState *AuxState, ///< [in] Auxilary state variables + const Array3DReal &TracerArray, ///< [in] Tracer array int ThickTimeLevel, ///< [in] Time level int VelTimeLevel, ///< [in] Time level int TracerTimeLevel, ///< [in] Time level @@ -806,7 +810,8 @@ void Tendencies::computeVelocityTendencies( Pacer::start("Tend:computeVelocityTendencies", 1); AuxState->computeMomAux(State, ThickTimeLevel, VelTimeLevel); - computeVelocityTendenciesOnly(State, AuxState, ThickTimeLevel, VelTimeLevel, + computeVelocityTendenciesOnly(State, AuxState, TracerArray, + ThickTimeLevel, VelTimeLevel, TracerTimeLevel, Time); Pacer::stop("Tend:computeVelocityTendencies", 1); @@ -869,8 +874,8 @@ void Tendencies::computeAllTendencies( computeThicknessTendenciesOnly(State, AuxState, ThickTimeLevel, VelTimeLevel, Time); - computeVelocityTendenciesOnly(State, AuxState, ThickTimeLevel, VelTimeLevel, - TracerTimeLevel, Time); + computeVelocityTendenciesOnly(State, AuxState, TracerArray, ThickTimeLevel, + VelTimeLevel, TracerTimeLevel, Time); computeTracerTendenciesOnly(State, AuxState, TracerArray, ThickTimeLevel, VelTimeLevel, Time); } // end all tendency compute diff --git a/components/omega/src/ocn/Tendencies.h b/components/omega/src/ocn/Tendencies.h index 605517034a79..4ba53a313e82 100644 --- a/components/omega/src/ocn/Tendencies.h +++ b/components/omega/src/ocn/Tendencies.h @@ -81,6 +81,7 @@ class Tendencies { TimeInstant Time); void computeVelocityTendencies(const OceanState *State, const AuxiliaryState *AuxState, + const Array3DReal &TracerArray, int ThickTimeLevel, int VelTimeLevel, int TracerTimeLevel, TimeInstant Time); void computeTracerTendencies(const OceanState *State, @@ -99,6 +100,7 @@ class Tendencies { TimeInstant Time); void computeVelocityTendenciesOnly(const OceanState *State, const AuxiliaryState *AuxState, + const Array3DReal &TracerArray, int ThickTimeLevel, int VelTimeLevel, int TracerTimeLevel, TimeInstant Time); void computeTracerTendenciesOnly(const OceanState *State, diff --git a/components/omega/src/timeStepping/ForwardBackwardStepper.cpp b/components/omega/src/timeStepping/ForwardBackwardStepper.cpp index a686dcfae0d6..418a01f928ad 100644 --- a/components/omega/src/timeStepping/ForwardBackwardStepper.cpp +++ b/components/omega/src/timeStepping/ForwardBackwardStepper.cpp @@ -56,7 +56,7 @@ void ForwardBackwardStepper::doStep( CurLevel, TimeStep); // R_u^{n+1} = RHS_u(u^{n}, h^{n+1}, t^{n+1}) - Tend->computeVelocityTendencies(State, AuxState, NextLevel, CurLevel, + Tend->computeVelocityTendencies(State, AuxState, NextTracerArray, NextLevel, CurLevel, NextLevel, SimTime + TimeStep); // u^{n+1} = u^{n} + R_u^{n+1} From bd19daae91788c82e0d7639cc379867e979a65f8 Mon Sep 17 00:00:00 2001 From: Steven Brus Date: Mon, 30 Mar 2026 20:02:41 -0700 Subject: [PATCH 58/58] Fix clang-format issues --- components/omega/src/ocn/Tendencies.cpp | 9 ++++----- .../omega/src/timeStepping/ForwardBackwardStepper.cpp | 4 ++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/components/omega/src/ocn/Tendencies.cpp b/components/omega/src/ocn/Tendencies.cpp index 677e7bd9a42e..23259f36c379 100644 --- a/components/omega/src/ocn/Tendencies.cpp +++ b/components/omega/src/ocn/Tendencies.cpp @@ -616,8 +616,8 @@ void Tendencies::computeVelocityTendenciesOnly( const auto &PressureMid = VCoord->PressureMid; const auto &PressureInterface = VCoord->PressureInterface; - Array2DReal Temp = Kokkos::subview(TracerArray, Tracers::IndxTemp, - Kokkos::ALL, Kokkos::ALL); + Array2DReal Temp = Kokkos::subview(TracerArray, Tracers::IndxTemp, + Kokkos::ALL, Kokkos::ALL); Array2DReal Salinity = Kokkos::subview(TracerArray, Tracers::IndxSalt, Kokkos::ALL, Kokkos::ALL); EqState->computeSpecVol(Temp, Salinity, PressureMid); @@ -810,9 +810,8 @@ void Tendencies::computeVelocityTendencies( Pacer::start("Tend:computeVelocityTendencies", 1); AuxState->computeMomAux(State, ThickTimeLevel, VelTimeLevel); - computeVelocityTendenciesOnly(State, AuxState, TracerArray, - ThickTimeLevel, VelTimeLevel, - TracerTimeLevel, Time); + computeVelocityTendenciesOnly(State, AuxState, TracerArray, ThickTimeLevel, + VelTimeLevel, TracerTimeLevel, Time); Pacer::stop("Tend:computeVelocityTendencies", 1); } diff --git a/components/omega/src/timeStepping/ForwardBackwardStepper.cpp b/components/omega/src/timeStepping/ForwardBackwardStepper.cpp index 418a01f928ad..e72719a3b2f4 100644 --- a/components/omega/src/timeStepping/ForwardBackwardStepper.cpp +++ b/components/omega/src/timeStepping/ForwardBackwardStepper.cpp @@ -56,8 +56,8 @@ void ForwardBackwardStepper::doStep( CurLevel, TimeStep); // R_u^{n+1} = RHS_u(u^{n}, h^{n+1}, t^{n+1}) - Tend->computeVelocityTendencies(State, AuxState, NextTracerArray, NextLevel, CurLevel, - NextLevel, SimTime + TimeStep); + Tend->computeVelocityTendencies(State, AuxState, NextTracerArray, NextLevel, + CurLevel, NextLevel, SimTime + TimeStep); // u^{n+1} = u^{n} + R_u^{n+1} updateVelocityByTend(State, NextLevel, State, CurLevel, TimeStep);