Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions components/omega/configs/Default.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ Omega:
IODefaultFormat: pnetcdf
State:
NTimeLevels: 2
PrescribeThicknessType: None
PrescribeVelocityType: None
Advection:
Coef3rdOrder: 0.25
FluxThicknessType: Center
Expand Down
4 changes: 4 additions & 0 deletions components/omega/src/timeStepping/ForwardBackwardStepper.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,10 @@ void ForwardBackwardStepper::doStep(
// u^{n+1} = u^{n} + R_u^{n+1}
updateVelocityByTend(State, NextLevel, State, CurLevel, TimeStep);

prescribeThickness(State, NextLevel, State, CurLevel);
prescribeVelocity(State, NextLevel, State, CurLevel,
SimTime + TimeStep);

// Update time levels (New -> Old) of prognostic variables with halo
// exchanges
const MPI_Comm Comm = MeshHalo->getComm();
Expand Down
2 changes: 2 additions & 0 deletions components/omega/src/timeStepping/RungeKutta2Stepper.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,8 @@ void RungeKutta2Stepper::doStep(OceanState *State, // model state
updateTracersByTend(NextTracerArray, CurTracerArray, State, NextLevel, State,
CurLevel, TimeStep);

prescribeState(State, NextLevel, State, CurLevel, SimTime + TimeStep);

// Update time levels (New -> Old) of prognostic variables with halo
// exchanges
const MPI_Comm Comm = MeshHalo->getComm();
Expand Down
3 changes: 3 additions & 0 deletions components/omega/src/timeStepping/RungeKutta4Stepper.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ void RungeKutta4Stepper::doStep(OceanState *State, // model state
CurLevel, CurLevel, StageTime);
updateStateByTend(State, NextLevel, State, CurLevel,
RKB[Stage] * TimeStep);
prescribeState(State, NextLevel, State, CurLevel, StageTime);
accumulateTracersUpdate(NextTracerArray, RKB[Stage] * TimeStep);
} else {
// every other stage does:
Expand All @@ -96,6 +97,7 @@ void RungeKutta4Stepper::doStep(OceanState *State, // model state
// q^{n+1} += RKB[stage] * dt * R^{(s)}
updateStateByTend(ProvisState, CurLevel, State, CurLevel,
RKA[Stage] * TimeStep);
prescribeState(State, NextLevel, State, CurLevel, StageTime);
updateTracersByTend(ProvisTracers, CurTracerArray, ProvisState,
CurLevel, State, CurLevel, RKA[Stage] * TimeStep);

Expand All @@ -114,6 +116,7 @@ void RungeKutta4Stepper::doStep(OceanState *State, // model state
CurLevel, CurLevel, CurLevel, StageTime);
updateStateByTend(State, NextLevel, State, NextLevel,
RKB[Stage] * TimeStep);
prescribeState(State, NextLevel, State, CurLevel, StageTime);
accumulateTracersUpdate(NextTracerArray, RKB[Stage] * TimeStep);
}
}
Expand Down
225 changes: 224 additions & 1 deletion components/omega/src/timeStepping/TimeStepper.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,20 @@
#include "ForwardBackwardStepper.h"
#include "RungeKutta2Stepper.h"
#include "RungeKutta4Stepper.h"
#include "Logging.h"

namespace OMEGA {

//------------------------------------------------------------------------------
// create the static class members
// Default model time stepper
TimeStepper *TimeStepper::DefaultTimeStepper = nullptr;
// All defined time steppers
std::map<std::string, std::unique_ptr<TimeStepper>>
TimeStepper::AllTimeSteppers;
PrescribeStateType TimeStepper::DefaultPrescribeThicknessMode =
PrescribeStateType::None;
PrescribeStateType TimeStepper::DefaultPrescribeVelocityMode =
PrescribeStateType::None;

//------------------------------------------------------------------------------
// utility functions
Expand All @@ -44,6 +48,36 @@ TimeStepperType getTimeStepperFromStr(const std::string &InString) {
return TimeStepperChoice;
}

PrescribeStateType getPrescribeThicknessTypeFromStr(const std::string &InString) {

if (InString == "None") {
return PrescribeStateType::None;
}
if (InString == "Init") {
return PrescribeStateType::Init;
}

ABORT_ERROR("PrescribeStateType should be 'None' or 'Init' for thickness but got {}",
InString);
return PrescribeStateType::Invalid;
}
PrescribeStateType getPrescribeVelocityTypeFromStr(const std::string &InString) {

if (InString == "None") {
return PrescribeStateType::None;
}else if (InString == "Init") {
return PrescribeStateType::Init;
}else if (InString == "NonDivergent") {
return PrescribeStateType::NonDivergent;
}else if (InString == "Divergent") {
return PrescribeStateType::Divergent;
}

ABORT_ERROR("PrescribeStateType should be 'None', 'Init', 'NonDivergent' or 'Divergent' for velocity but got {}",
InString);
return PrescribeStateType::Invalid;
}

//------------------------------------------------------------------------------
// Constructors and creation methods.

Expand Down Expand Up @@ -119,6 +153,11 @@ TimeStepper *TimeStepper::create(
ABORT_ERROR("Unknown time stepping method");
}

NewTimeStepper->PrescribeThicknessMode =
DefaultPrescribeThicknessMode;
NewTimeStepper->PrescribeVelocityMode =
DefaultPrescribeVelocityMode;

// Attach data pointers
NewTimeStepper->attachData(InTend, InAuxState, InMesh, InVCoord, InMeshHalo);

Expand Down Expand Up @@ -170,6 +209,11 @@ TimeStepper *TimeStepper::create(
ABORT_ERROR("Unknown time stepping method");
}

NewTimeStepper->PrescribeThicknessMode =
DefaultPrescribeThicknessMode;
NewTimeStepper->PrescribeVelocityMode =
DefaultPrescribeVelocityMode;

// Store instance
AllTimeSteppers.emplace(InName, NewTimeStepper);

Expand Down Expand Up @@ -298,6 +342,22 @@ void TimeStepper::init1() {
StopTime = StopTime2;
}

Config StateConfig("State");
Error StateErr = OmegaConfig->get(StateConfig);
if (StateErr.isSuccess()) {
std::string ThicknessMode;
if (StateConfig.get("PrescribeThicknessType", ThicknessMode).isSuccess()) {
TimeStepper::DefaultPrescribeThicknessMode =
getPrescribeThicknessTypeFromStr(ThicknessMode);
}

std::string VelocityMode;
if (StateConfig.get("PrescribeVelocityType", VelocityMode).isSuccess()) {
TimeStepper::DefaultPrescribeVelocityMode =
getPrescribeVelocityTypeFromStr(VelocityMode);
}
}

// Now that all the inputs are defined, create the default time stepper
// Use the partial creation function for only the time info. Data
// pointers will be attached in phase 2 initialization
Expand Down Expand Up @@ -460,6 +520,169 @@ void TimeStepper::updateStateByTend(OceanState *State1, int TimeLevel1,
updateVelocityByTend(State1, TimeLevel1, State2, TimeLevel2, Coeff);
}

//------------------------------------------------------------------------------
// Reset state variables to their initial values
void TimeStepper::prescribeThickness(OceanState *State1, int TimeLevel1,
OceanState *State2, int TimeLevel2) const {

if (PrescribeThicknessMode == PrescribeStateType::None) {
return;
}

if (PrescribeThicknessMode == PrescribeStateType::Init) {
Array2DReal LayerThick1 = State1->getLayerThickness(TimeLevel1);
Array2DReal LayerThick2 = State2->getLayerThickness(TimeLevel2);

OMEGA_SCOPE(MinLayerCell, VCoord->MinLayerCell);
OMEGA_SCOPE(MaxLayerCell, VCoord->MaxLayerCell);

parallelForOuter(
"prescribeThickness", {Mesh->NCellsAll},
KOKKOS_LAMBDA(int ICell, const TeamMember &Team) {
const int KMin = MinLayerCell(ICell);
const int KMax = MaxLayerCell(ICell);
const int KRange = vertRange(KMin, KMax);

parallelForInner(
Team, KRange, INNER_LAMBDA(int KChunk) {
const int K = KMin + KChunk;
LayerThick1(ICell, K) = LayerThick2(ICell, K);
});
});
return;
}
}

//------------------------------------------------------------------------------
void TimeStepper::prescribeVelocity(OceanState *State1, int TimeLevel1,
OceanState *State2, int TimeLevel2,
const TimeInstant &SimTime) const {

if (PrescribeVelocityMode == PrescribeStateType::None) {
return;
}

if (PrescribeVelocityMode == PrescribeStateType::Init) {
Array2DReal NormalVel1 = State1->getNormalVelocity(TimeLevel1);
Array2DReal NormalVel2 = State2->getNormalVelocity(TimeLevel2);

OMEGA_SCOPE(MinLayerEdgeBot, VCoord->MinLayerEdgeBot);
OMEGA_SCOPE(MaxLayerEdgeTop, VCoord->MaxLayerEdgeTop);

parallelForOuter(
"prescribeVelocity", {Mesh->NEdgesAll},
KOKKOS_LAMBDA(int IEdge, const TeamMember &Team) {
const int KMin = MinLayerEdgeBot(IEdge);
const int KMax = MaxLayerEdgeTop(IEdge);
const int KRange = vertRange(KMin, KMax);

parallelForInner(
Team, KRange, INNER_LAMBDA(int KChunk) {
const int K = KMin + KChunk;
NormalVel2(IEdge, K) = NormalVel1(IEdge, K);
});
});
return;
} else if (PrescribeVelocityMode == PrescribeStateType::NonDivergent) {
Array2DReal NormalVel2 = State2->getNormalVelocity(TimeLevel2);

OMEGA_SCOPE(LatEdge, Mesh->LatEdgeH);
OMEGA_SCOPE(LonEdge, Mesh->LonEdgeH);
OMEGA_SCOPE(AngleEdge, Mesh->AngleEdgeH);
OMEGA_SCOPE(MinLayerEdgeBot, VCoord->MinLayerEdgeBotH);
OMEGA_SCOPE(MaxLayerEdgeTop, VCoord->MaxLayerEdgeTopH);

const Clock *ModelClock = StepClock.get();
R8 ElapsedTimeSec;
TimeInterval ElapsedTimeInterval = SimTime - ModelClock->getCurrentTime();
ElapsedTimeInterval.get(ElapsedTimeSec, TimeUnits::Seconds);

const R8 Tau = 12. * Day2Sec; // 12 days in seconds
const R8 TSim = ElapsedTimeSec;

parallelForOuter(
"prescribeVelocityNonDivergent", {Mesh->NEdgesAll},
KOKKOS_LAMBDA(int IEdge, const TeamMember &Team) {
const int KMin = MinLayerEdgeBot(IEdge);
const int KMax = MaxLayerEdgeTop(IEdge);
const int KRange = vertRange(KMin, KMax);

const R8 lon_p =
LonEdge(IEdge) - 2.0 * Pi * TSim / Tau;
const R8 u = (1 / Tau) *
(10.0 * Kokkos::pow(sin(lon_p), 2) *
sin(2.0 * LatEdge(IEdge)) *
cos(Pi * TSim / Tau) +
2.0 * Pi * cos(LatEdge(IEdge)));
Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@cbegeman
I see in MPAS-Ocean:
u = (1/twelve_days)*(10*(sin(lon_p)**2)*sin(2*lat)*cost + two_pi*coslat)
I think the factor of 10 was moved slightly. Might make a difference in time step stability if the change resulted in a larger velocity.

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@overfelt Thanks for pointing that out! I'll fix and retest

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@overfelt That fix seems to have resolved at least the stability issues with the divergent velocity case (top) but not the non-divergent case (bottom). I tried reducing the time step to 1s per km horizontal resolution. Only the rotation_2d has convergence with space and time refinement. Can I delegate the debugging to you?

image image

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@cbegeman I'll take a look. Maybe I'll get lucky and find a difference with mpas-ocean. Maybe......

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thank you @overfelt!

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

After this fix and reducing the timestep, the velocity fields for MPAS-O and Omega match.

There is a significant increase in the error levels for Omega (convergence plots pending)

These are the results for the tracer fields with MPAS-O:
image
image
image

And these are the results with Omega:
image
image
image

const R8 v = (10.0 / Tau) * sin(2.0 * lon_p) *
cos(LatEdge(IEdge)) * cos(Pi * TSim / Tau);
const R8 normalVel = REarth * (
u * cos(AngleEdge(IEdge)) + v * sin(AngleEdge(IEdge)));

parallelForInner(
Team, KRange, INNER_LAMBDA(int KChunk) {
const int K = KMin + KChunk;
NormalVel2(IEdge, K) = normalVel;
});
});
return;
} else if (PrescribeVelocityMode == PrescribeStateType::Divergent) {
Array2DReal NormalVel2 = State2->getNormalVelocity(TimeLevel2);

OMEGA_SCOPE(LatEdge, Mesh->LatEdgeH);
OMEGA_SCOPE(LonEdge, Mesh->LonEdgeH);
OMEGA_SCOPE(AngleEdge, Mesh->AngleEdgeH);
OMEGA_SCOPE(MinLayerEdgeBot, VCoord->MinLayerEdgeBotH);
OMEGA_SCOPE(MaxLayerEdgeTop, VCoord->MaxLayerEdgeTopH);

const Clock *ModelClock = StepClock.get();
R8 ElapsedTimeSec;
TimeInterval ElapsedTimeInterval = SimTime - ModelClock->getCurrentTime();
ElapsedTimeInterval.get(ElapsedTimeSec, TimeUnits::Seconds);

const R8 Tau = 12. * Day2Sec; // 14 days in seconds
const R8 TSim = ElapsedTimeSec;

parallelForOuter(
"prescribeVelocityDivergent", {Mesh->NEdgesAll},
KOKKOS_LAMBDA(int IEdge, const TeamMember &Team) {
const int KMin = MinLayerEdgeBot(IEdge);
const int KMax = MaxLayerEdgeTop(IEdge);
const int KRange = vertRange(KMin, KMax);

const R8 lon_p =
LonEdge(IEdge) - 2.0 * Pi * TSim / Tau;
const R8 u = (1.0 / Tau) *
(-5.0 * Kokkos::pow(sin(lon_p / 2), 2) *
sin(2.0 * LatEdge(IEdge)) *
Kokkos::pow(cos(LatEdge(IEdge)), 2) *
cos(Pi * TSim / Tau) +
2.0 * Pi * cos(LatEdge(IEdge)));
const R8 v = ((2.5 / Tau) *
sin(lon_p) *
Kokkos::pow(cos(LatEdge(IEdge)), 3) *
cos(Pi * TSim / Tau));
const R8 normalVel = REarth * (
u * cos(AngleEdge(IEdge)) + v * sin(AngleEdge(IEdge)));
Comment on lines +655 to +666
Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I haven't been able to track down the reason for the discrepancy between MPAS-O's velocity field (top) and Omega's velocity field (bottom) for the divergent case. I need to pivot to other Omega work so @overfelt any time you have to look into this would be much appreciated!
image
image
image
image

It should be possible to reproduce these results using E3SM-Project/polaris#500. Let me know if you have any trouble


parallelForInner(
Team, KRange, INNER_LAMBDA(int KChunk) {
const int K = KMin + KChunk;
NormalVel2(IEdge, K) = normalVel;
});
});
return;
}
}

//------------------------------------------------------------------------------
void TimeStepper::prescribeState(OceanState *State1, int TimeLevel1,
OceanState *State2, int TimeLevel2,
const TimeInstant &SimTime) const {
prescribeThickness(State1, TimeLevel1, State2, TimeLevel2);
prescribeVelocity(State1, TimeLevel1, State2, TimeLevel2, SimTime);
}

//------------------------------------------------------------------------------
// Updates tracers
// NextTracers = (CurTracers * LayerThickness2(TimeLevel2)) +
Expand Down
Loading
Loading