diff --git a/CarpetX/src/interpolate.cxx b/CarpetX/src/interpolate.cxx index eee39a9a8..40ec7779c 100644 --- a/CarpetX/src/interpolate.cxx +++ b/CarpetX/src/interpolate.cxx @@ -664,6 +664,9 @@ extern "C" void CarpetX_Interpolate(const CCTK_POINTER_TO_CONST cctkGH_, givis.at(v) = {gi, vi}; } + rat64 min_level_iteration_used = -1; + rat64 max_level_iteration_used = -1; + bool requires_interpolation_in_time = false; for (const auto &patchdata : ghext->patchdata) { const int patch = patchdata.patch; for (const auto &leveldata : patchdata.leveldata) { @@ -676,6 +679,15 @@ extern "C" void CarpetX_Interpolate(const CCTK_POINTER_TO_CONST cctkGH_, const GridDesc grid(leveldata, mfp); // const int component = mfp.index(); + min_level_iteration_used = + min_level_iteration_used != -1 + ? std::min(min_level_iteration_used, leveldata.iteration) + : leveldata.iteration; + max_level_iteration_used = + max_level_iteration_used != -1 + ? std::max(max_level_iteration_used, leveldata.iteration) + : leveldata.iteration; + const int np = pti.numParticles(); const auto &particles = pti.GetArrayOfStructs(); @@ -823,8 +835,32 @@ extern "C" void CarpetX_Interpolate(const CCTK_POINTER_TO_CONST cctkGH_, } } - // Collect particles back const MPI_Comm comm = amrex::ParallelDescriptor::Communicator(); + + // check if interpolation in time (unsupported) would be needed + if (use_subcycling) { + const double local_iteration_used[2] = { + min_level_iteration_used != -1 + ? -double(min_level_iteration_used) + : -std::numeric_limits::infinity(), + max_level_iteration_used != -1 + ? +double(max_level_iteration_used) + : -std::numeric_limits::infinity()}; + double global_iteration_used[2] = {0.}; + MPI_Allreduce(local_iteration_used, global_iteration_used, 2, MPI_DOUBLE, + MPI_MAX, comm); + const double min_iteration_used = -global_iteration_used[0]; + const double max_iteration_used = +global_iteration_used[1]; + // Did two (or one) ranks use different iterations? + if (max_iteration_used != min_iteration_used) { + CCTK_VERROR("Interpolation in time required when interpolating %s at " + "iteration %d", + nvars > 0 ? CCTK_FullVarName(varinds[0]) : "no variable", + cctkGH->cctk_iteration); + } + } + + // Collect particles back const MPI_Datatype datatype = mpi_datatype::value; std::vector sendcounts(nprocs); diff --git a/CarpetX/src/io.cxx b/CarpetX/src/io.cxx index e6fdff93a..1a920c05d 100644 --- a/CarpetX/src/io.cxx +++ b/CarpetX/src/io.cxx @@ -609,6 +609,24 @@ void Checkpoint(const cGH *const restrict cctkGH) { return; } + if (use_subcycling) { // currently checkpoint and recovery only works if all + // levels are aligned in time + rat64 level_iteration = -1; + for (const auto &patchdata : ghext->patchdata) { + for (const auto &leveldata : patchdata.leveldata) { + if (level_iteration == -1) { + level_iteration = leveldata.iteration; + } else { + if (leveldata.iteration != level_iteration) { + CCTK_VERROR("Checkpointing is only allowed if all levels are " + "aligned in time, not at %d", + cctkGH->cctk_iteration); + } + } + } + } + } + if (cctkGH->cctk_iteration <= last_checkpoint_iteration) { CCTK_VINFO( "Already wrote checkpoint at iteration %d; skipping checkpointing",