From 8818025a72e550f1c425828c08ae2780e32b0551 Mon Sep 17 00:00:00 2001 From: Azamat Mametjanov Date: Wed, 8 Apr 2026 07:40:07 +0000 Subject: [PATCH] Add state validation checks for NaNs and OOBounds Initial validateOceanState function to check for NaNs and bounds - LayerThickness: 1e-10 to 1000 - KineticEnergyCell: 0 to 10 - Temperature: -10 to 50 - Salinity: -2 to 60 --- components/omega/src/ocn/StateValidation.cpp | 219 +++++++++++++++++ components/omega/src/ocn/StateValidation.h | 43 ++++ components/omega/test/CMakeLists.txt | 11 + .../omega/test/ocn/StateValidationTest.cpp | 221 ++++++++++++++++++ 4 files changed, 494 insertions(+) create mode 100644 components/omega/src/ocn/StateValidation.cpp create mode 100644 components/omega/src/ocn/StateValidation.h create mode 100644 components/omega/test/ocn/StateValidationTest.cpp diff --git a/components/omega/src/ocn/StateValidation.cpp b/components/omega/src/ocn/StateValidation.cpp new file mode 100644 index 000000000000..a47832c1cc2d --- /dev/null +++ b/components/omega/src/ocn/StateValidation.cpp @@ -0,0 +1,219 @@ +//===-- ocn/StateValidation.cpp - ocean state validation --------*- C++ -*-===// +// +// Validates ocean state fields by checking for NaN values and +// out-of-bounds conditions. Any failure triggers a critical error log with +// backtrace and MPI_Abort on the local communicator. +// +//===----------------------------------------------------------------------===// + +#include "StateValidation.h" + +#include "AuxiliaryState.h" +#include "DataTypes.h" +#include "Error.h" +#include "Logging.h" +#include "MachEnv.h" +#include "OceanState.h" +#include "OmegaKokkos.h" +#include "Tracers.h" +#include "mpi.h" + +#include +#include +#include +#include + +namespace OMEGA { + +//------------------------------------------------------------------------------ +// Helper: abort on the local Omega communicator with a message and backtrace +static void abortWithMessage(const std::string &Msg) { + LOG_CRITICAL("{}", Msg); + cpptrace::generate_trace().print(); + MPI_Comm Comm = MachEnv::getDefault()->getComm(); + MPI_Abort(Comm, static_cast(ErrorCode::Critical)); +} + +//------------------------------------------------------------------------------ +// Helper: count NaN entries and out-of-range entries in a 2-D Real device +// array over the first NCells/NEdges rows and NVert columns. +// Returns {NaNCount, OutOfRangeCount}. +static std::pair checkArray2D(const Array2DReal &Arr, I4 NRows, + I4 NCols, Real MinVal, Real MaxVal, + bool CheckMin) { + I4 NaNCount = 0; + I4 OutOfRangeCount = 0; + + parallelReduce( + "CheckNaN", {NRows, NCols}, + KOKKOS_LAMBDA(int Row, int Col, int &Accum) { + Real Val = Arr(Row, Col); + if (Kokkos::isnan(Val)) { + ++Accum; + } + }, + NaNCount); + + parallelReduce( + "CheckBounds", {NRows, NCols}, + KOKKOS_LAMBDA(int Row, int Col, int &Accum) { + Real Val = Arr(Row, Col); + if (!Kokkos::isnan(Val)) { + if (Val > MaxVal) { + ++Accum; + } else if (CheckMin && Val < MinVal) { + ++Accum; + } + } + }, + OutOfRangeCount); + + return {NaNCount, OutOfRangeCount}; +} + +//------------------------------------------------------------------------------ +// Helper: count NaN and out-of-range entries for a single tracer (row = cell, +// col = vert) extracted from the 3-D tracer array at the given tracer index. +static std::pair checkTracerArray(const Array3DReal &Tracers3D, + I4 TracerIdx, I4 NCells, I4 NVert, + Real MinVal, Real MaxVal) { + I4 NaNCount = 0; + I4 OutOfRangeCount = 0; + + parallelReduce( + "CheckTracerNaN", {NCells, NVert}, + KOKKOS_LAMBDA(int Cell, int K, int &Accum) { + Real Val = Tracers3D(TracerIdx, Cell, K); + if (Kokkos::isnan(Val)) { + ++Accum; + } + }, + NaNCount); + + parallelReduce( + "CheckTracerBounds", {NCells, NVert}, + KOKKOS_LAMBDA(int Cell, int K, int &Accum) { + Real Val = Tracers3D(TracerIdx, Cell, K); + if (!Kokkos::isnan(Val)) { + if (Val < MinVal || Val > MaxVal) { + ++Accum; + } + } + }, + OutOfRangeCount); + + return {NaNCount, OutOfRangeCount}; +} + +//------------------------------------------------------------------------------ +/// Validate ocean state fields for NaN and out-of-bounds conditions. +/// Aborts via MPI_Abort on failure. +void validateOceanState(const OceanState *State, const AuxiliaryState *AuxState, + I4 TimeLevel) { + + bool AnyFailure = false; + + // ------------------------------------------------------------------------- + // LayerThickness: valid range [1e-10, 1000] + // ------------------------------------------------------------------------- + { + Array2DReal LayerThick = State->getLayerThickness(TimeLevel); + auto [NaNs, OOB] = + checkArray2D(LayerThick, State->NCellsOwned, State->NVertLayers, + static_cast(1e-10), static_cast(1000.0), + /*CheckMin=*/true); + + if (NaNs > 0) { + LOG_CRITICAL( + "StateValidation: LayerThickness contains {} NaN value(s)", NaNs); + AnyFailure = true; + } + if (OOB > 0) { + LOG_CRITICAL("StateValidation: LayerThickness has {} value(s) outside " + "valid range [1e-10, 1000]", + OOB); + AnyFailure = true; + } + } + + // ------------------------------------------------------------------------- + // KineticEnergyCell: valid range [0, 10] + // ------------------------------------------------------------------------- + { + const Array2DReal &KE = AuxState->KineticAux.KineticEnergyCell; + auto [NaNs, OOB] = + checkArray2D(KE, State->NCellsOwned, State->NVertLayers, + static_cast(0.0), static_cast(10.0), + /*CheckMin=*/true); + + if (NaNs > 0) { + LOG_CRITICAL( + "StateValidation: KineticEnergyCell contains {} NaN value(s)", + NaNs); + AnyFailure = true; + } + if (OOB > 0) { + LOG_CRITICAL( + "StateValidation: KineticEnergyCell has {} value(s) outside " + "valid range [0, 10]", + OOB); + AnyFailure = true; + } + } + + // ------------------------------------------------------------------------- + // Temperature tracer: valid range [-10, 50] + // ------------------------------------------------------------------------- + if (Tracers::IndxTemp != Tracers::IndxInvalid) { + Array3DReal AllTracers = Tracers::getAll(TimeLevel); + auto [NaNs, OOB] = checkTracerArray( + AllTracers, Tracers::IndxTemp, State->NCellsOwned, State->NVertLayers, + static_cast(-10.0), static_cast(50.0)); + + if (NaNs > 0) { + LOG_CRITICAL("StateValidation: Temperature contains {} NaN value(s)", + NaNs); + AnyFailure = true; + } + if (OOB > 0) { + LOG_CRITICAL("StateValidation: Temperature has {} value(s) outside " + "valid range [-10, 50]", + OOB); + AnyFailure = true; + } + } + + // ------------------------------------------------------------------------- + // Salinity tracer: valid range [-2, 60] + // ------------------------------------------------------------------------- + if (Tracers::IndxSalt != Tracers::IndxInvalid) { + Array3DReal AllTracers = Tracers::getAll(TimeLevel); + auto [NaNs, OOB] = checkTracerArray( + AllTracers, Tracers::IndxSalt, State->NCellsOwned, State->NVertLayers, + static_cast(-2.0), static_cast(60.0)); + + if (NaNs > 0) { + LOG_CRITICAL("StateValidation: Salinity contains {} NaN value(s)", + NaNs); + AnyFailure = true; + } + if (OOB > 0) { + LOG_CRITICAL("StateValidation: Salinity has {} value(s) outside " + "valid range [-2, 60]", + OOB); + AnyFailure = true; + } + } + + // ------------------------------------------------------------------------- + // Abort if any check failed + // ------------------------------------------------------------------------- + if (AnyFailure) { + abortWithMessage("StateValidation: Ocean state validation failed. " + "See critical messages above for details."); + } +} + +} // namespace OMEGA + +//===----------------------------------------------------------------------===// diff --git a/components/omega/src/ocn/StateValidation.h b/components/omega/src/ocn/StateValidation.h new file mode 100644 index 000000000000..2395aaa76286 --- /dev/null +++ b/components/omega/src/ocn/StateValidation.h @@ -0,0 +1,43 @@ +#ifndef OMEGA_STATEVALIDATION_H +#define OMEGA_STATEVALIDATION_H +//===-- ocn/StateValidation.h - ocean state validation ----------*- C++ -*-===// +// +/// \file +/// \brief Declares the validateOceanState function for ocean state validation +/// +/// Provides a function that validates the ocean prognostic state and selected +/// auxiliary/tracer fields by checking for NaN values and out-of-bounds +/// conditions. If any check fails the function logs a critical error with a +/// backtrace and aborts via MPI_Abort on the local MPI communicator. +// +//===----------------------------------------------------------------------===// + +#include "AuxiliaryState.h" +#include "OceanState.h" +#include "Tracers.h" + +namespace OMEGA { + +/// Check ocean state fields for NaN values and out-of-bounds conditions. +/// +/// The following fields are validated: +/// - LayerThickness : [1e-10, 1000] (from OceanState) +/// - KineticEnergyCell : [0, 10] +/// (from AuxiliaryState::KineticAux) +/// - Temperature tracer : [-10, 50] (from Tracers) +/// - Salinity tracer : [-2, 60] (from Tracers) +/// +/// If any check fails a critical error is logged with an informative message +/// and a stack backtrace, and the run is aborted via MPI_Abort on the +/// communicator obtained from the default MachEnv. +/// +/// \param[in] State Ocean state to validate +/// \param[in] AuxState Auxiliary state containing KineticEnergyCell +/// \param[in] TimeLevel Time level index to validate (typically 0 = current) +void validateOceanState(const OceanState *State, const AuxiliaryState *AuxState, + I4 TimeLevel); + +} // namespace OMEGA + +//===----------------------------------------------------------------------===// +#endif // defined OMEGA_STATEVALIDATION_H diff --git a/components/omega/test/CMakeLists.txt b/components/omega/test/CMakeLists.txt index 301a04ede49c..4d3830103691 100644 --- a/components/omega/test/CMakeLists.txt +++ b/components/omega/test/CMakeLists.txt @@ -516,3 +516,14 @@ add_omega_test( ocn/VertAdvTest.cpp "-n;1" ) + +########################## +# State Validation test +########################## + +add_omega_test( + STATE_VALIDATION_TEST + testStateValidation.exe + ocn/StateValidationTest.cpp + "-n;8" +) diff --git a/components/omega/test/ocn/StateValidationTest.cpp b/components/omega/test/ocn/StateValidationTest.cpp new file mode 100644 index 000000000000..3c1baf9221f8 --- /dev/null +++ b/components/omega/test/ocn/StateValidationTest.cpp @@ -0,0 +1,221 @@ +//===-- Test driver for OMEGA StateValidation -----------------------*- C++ +//-*-===/ +// +/// \file +/// \brief Test driver for ocean state validation +/// +/// Tests the validateOceanState function by verifying that it passes on valid +/// state data. Checks cover LayerThickness, KineticEnergyCell, Temperature, +/// and Salinity fields. +// +//===-----------------------------------------------------------------------===/ + +#include "StateValidation.h" +#include "AuxiliaryState.h" +#include "Config.h" +#include "DataTypes.h" +#include "Decomp.h" +#include "Dimension.h" +#include "Error.h" +#include "Field.h" +#include "Halo.h" +#include "HorzMesh.h" +#include "IO.h" +#include "IOStream.h" +#include "Logging.h" +#include "MachEnv.h" +#include "OceanState.h" +#include "OmegaKokkos.h" +#include "Pacer.h" +#include "TimeStepper.h" +#include "Tracers.h" +#include "VertAdv.h" +#include "VertCoord.h" +#include "mpi.h" + +#include + +using namespace OMEGA; + +//------------------------------------------------------------------------------ +// Initialize the Omega subsystems required for state validation testing + +int initStateValidationTest(const std::string &MeshFile) { + int Err = 0; + + MachEnv::init(MPI_COMM_WORLD); + MachEnv *DefEnv = MachEnv::getDefault(); + MPI_Comm DefComm = DefEnv->getComm(); + + initLogging(DefEnv); + LOG_INFO("------ StateValidation unit tests ------"); + + Config("Omega"); + Config::readAll("omega.yml"); + + TimeStepper::init1(); + + IO::init(DefComm); + Decomp::init(MeshFile); + + IOStream::init(); + + int HaloErr = Halo::init(); + if (HaloErr != 0) { + Err++; + LOG_ERROR("StateValidationTest: error initializing default halo"); + } + + HorzMesh::init(); + VertCoord::init(); + Tracers::init(); + + int StateErr = OceanState::init(); + if (StateErr != 0) { + Err++; + LOG_ERROR("StateValidationTest: error initializing default state"); + } + + VertAdv::init(); + + return Err; +} + +//------------------------------------------------------------------------------ +// Fill state and auxiliary/tracer arrays with known-valid values + +static int fillValidState() { + int Err = 0; + + auto *State = OceanState::getDefault(); + const int NCells = State->NCellsAll; + const int NVert = State->NVertLayers; + const int NEdges = State->NEdgesAll; + + // LayerThickness: fill with 100 m (valid range [1e-10, 1000]) + Array2DReal LayerThick = State->getLayerThickness(0); + parallelFor( + "FillLayerThick", {NCells, NVert}, + KOKKOS_LAMBDA(int ICell, int K) { LayerThick(ICell, K) = 100.0; }); + + // NormalVelocity: fill with 0 (not checked, but needed for AuxState) + Array2DReal NormalVel = State->getNormalVelocity(0); + parallelFor( + "FillNormalVel", {NEdges, NVert}, + KOKKOS_LAMBDA(int IEdge, int K) { NormalVel(IEdge, K) = 0.0; }); + + // Exchange halos so auxiliary state computations are consistent + State->exchangeHalo(0); + + // Tracers: fill Temperature with 10 C and Salinity with 35 g/kg + // Use deepCopy with individual tracer subviews + if (Tracers::getNumTracers() > 0) { + // Temperature = 10.0 (valid: -10 to 50) + if (Tracers::IndxTemp != Tracers::IndxInvalid) { + Array2DReal TempArr = Tracers::getByIndex(0, Tracers::IndxTemp); + deepCopy(TempArr, static_cast(10.0)); + } + + // Salinity = 35.0 (valid: -2 to 60) + if (Tracers::IndxSalt != Tracers::IndxInvalid) { + Array2DReal SaltArr = Tracers::getByIndex(0, Tracers::IndxSalt); + deepCopy(SaltArr, static_cast(35.0)); + } + } + + return Err; +} + +//------------------------------------------------------------------------------ +// Run state validation tests + +int testStateValidation() { + int Err = 0; + + // Initialize the auxiliary state (needed for KineticEnergyCell) + AuxiliaryState::init(); + auto *DefAuxState = AuxiliaryState::getDefault(); + + if (!DefAuxState) { + Err++; + LOG_ERROR("StateValidationTest: Default AuxiliaryState not found"); + return Err; + } + + auto *DefState = OceanState::getDefault(); + if (!DefState) { + Err++; + LOG_ERROR("StateValidationTest: Default OceanState not found"); + return Err; + } + + // Fill state arrays with valid values + Err += fillValidState(); + + // Compute auxiliary variables so KineticEnergyCell is populated + { + Array3DReal AllTracers = Tracers::getAll(0); + DefAuxState->computeAll(DefState, AllTracers, 0); + } + + // Test: validation should pass on valid state (no abort expected) + LOG_INFO("StateValidationTest: Testing validation on valid state"); + validateOceanState(DefState, DefAuxState, 0); + LOG_INFO("StateValidationTest: Valid state validation PASS"); + + AuxiliaryState::clear(); + return Err; +} + +//------------------------------------------------------------------------------ +// Finalize Omega objects + +void finalizeStateValidationTest() { + Tracers::clear(); + OceanState::clear(); + VertAdv::clear(); + VertCoord::clear(); + HorzMesh::clear(); + Field::clear(); + Dimension::clear(); + TimeStepper::clear(); + Halo::clear(); + Decomp::clear(); + MachEnv::removeAll(); +} + +//------------------------------------------------------------------------------ +// Main entry point + +int main(int argc, char *argv[]) { + int RetVal = 0; + + MPI_Init(&argc, &argv); + Kokkos::initialize(argc, argv); + Pacer::initialize(MPI_COMM_WORLD); + Pacer::setPrefix("Omega:"); + + { + int Err = initStateValidationTest("OmegaMesh.nc"); + if (Err != 0) { + LOG_CRITICAL("StateValidationTest: Error during initialization"); + } else { + RetVal += testStateValidation(); + } + finalizeStateValidationTest(); + } + + if (RetVal == 0) + LOG_INFO("------ StateValidation unit tests successful ------"); + + Pacer::finalize(); + Kokkos::finalize(); + MPI_Finalize(); + + if (RetVal >= 256) + RetVal = 255; + + return RetVal; + +} // end of main +//===-----------------------------------------------------------------------===/