From 99b69c57652e62bebbd26bb48f1c4b3dfd80580a Mon Sep 17 00:00:00 2001 From: Allen Wen Date: Thu, 12 Feb 2026 15:49:43 -0600 Subject: [PATCH 1/7] nuX_M1: calculate fiducial velocity everywhere --- nuX_M1/schedule.ccl | 47 ++++++++++--------------- nuX_M1/src/nuX_M1_fiducial_velocity.cxx | 6 ++-- 2 files changed, 22 insertions(+), 31 deletions(-) diff --git a/nuX_M1/schedule.ccl b/nuX_M1/schedule.ccl index aa463b5..f63be90 100644 --- a/nuX_M1/schedule.ccl +++ b/nuX_M1/schedule.ccl @@ -72,13 +72,11 @@ SCHEDULE nuX_M1_FiducialVelocity IN nuX_M1_InitialGroup AFTER (nuX_M1_Reset) { LANG: C - READS: ADMBaseX::metric(interior) - READS: AsterX::dens(interior) - READS: HydroBaseX::vel(interior) - WRITES: fidu_velx(interior) fidu_vely(interior) fidu_velz(interior) - WRITES: fidu_w_lorentz(interior) - SYNC: fidu_vel - SYNC: fidu_w_lorentz + READS: ADMBaseX::metric(everywhere) + READS: AsterX::dens(everywhere) + READS: HydroBaseX::vel(everywhere) + WRITES: fidu_velx(everywhere) fidu_vely(everywhere) fidu_velz(everywhere) + WRITES: fidu_w_lorentz(everywhere) } "Compute the fiducial velocity" @@ -132,14 +130,12 @@ if (reset_to_equilibrium) { AFTER nuX_M1_Reset { LANG: C - READS: ADMBaseX::metric(interior) - READS: AsterX::dens(interior) - READS: HydroBaseX::vel(interior) + READS: ADMBaseX::metric(everywhere) + READS: AsterX::dens(everywhere) + READS: HydroBaseX::vel(everywhere) WRITES: fidu_velx(everywhere) fidu_vely(everywhere) fidu_velz(everywhere) WRITES: fidu_w_lorentz(everywhere) - SYNC: fidu_vel - SYNC: fidu_w_lorentz } "Compute the fiducial velocity" # SCHEDULE nuX_M1_SetToEquilibrium AT CCTK_POST_RECOVER_VARIABLES \ @@ -229,14 +225,11 @@ SCHEDULE nuX_M1_SyncConserved IN nuX_M1_TimeStep AFTER nuX_M1_InitFluxesRHS SCHEDULE nuX_M1_FiducialVelocity IN nuX_M1_TimeStep { LANG: C - READS: ADMBaseX::metric(interior) - READS: AsterX::dens(interior) - READS: HydroBaseX::vel(interior) - WRITES: fidu_velx(interior) fidu_vely(interior) fidu_velz(interior) - WRITES: fidu_w_lorentz(interior) - SYNC: fidu_vel - SYNC: fidu_w_lorentz - + READS: ADMBaseX::metric(everywhere) + READS: AsterX::dens(everywhere) + READS: HydroBaseX::vel(everywhere) + WRITES: fidu_velx(everywhere) fidu_vely(everywhere) fidu_velz(everywhere) + WRITES: fidu_w_lorentz(everywhere) } "Compute the fiducial velocity" SCHEDULE nuX_M1_CalcClosure IN nuX_M1_TimeStep AFTER nuX_M1_FiducialVelocity @@ -459,14 +452,12 @@ if (add_to_Tmunu) { SCHEDULE nuX_M1_FiducialVelocity IN nuX_M1_Tmunu BEFORE nuX_M1_AddToTmunu { LANG: C - READS: ADMBaseX::metric(interior) - READS: AsterX::dens(interior) - READS: HydroBaseX::vel(interior) - READS: nuX_m1_mask(interior) - WRITES: fidu_velx(interior) fidu_vely(interior) fidu_velz(interior) - WRITES: fidu_w_lorentz(interior) - SYNC: fidu_vel - SYNC: fidu_w_lorentz + READS: ADMBaseX::metric(everywhere) + READS: AsterX::dens(everywhere) + READS: HydroBaseX::vel(everywhere) + READS: nuX_m1_mask(everywhere) + WRITES: fidu_velx(everywhere) fidu_vely(everywhere) fidu_velz(everywhere) + WRITES: fidu_w_lorentz(everywhere) } "Compute the fiducial velocity" SCHEDULE nuX_M1_AddToTmunu IN nuX_M1_Tmunu diff --git a/nuX_M1/src/nuX_M1_fiducial_velocity.cxx b/nuX_M1/src/nuX_M1_fiducial_velocity.cxx index ca2977c..3510b4f 100644 --- a/nuX_M1/src/nuX_M1_fiducial_velocity.cxx +++ b/nuX_M1/src/nuX_M1_fiducial_velocity.cxx @@ -37,7 +37,7 @@ extern "C" void nuX_M1_FiducialVelocity(CCTK_ARGUMENTS) { if (CCTK_Equals(fiducial_velocity, "fluid")) { - grid.loop_int_device<1, 1, 1>( + grid.loop_all_device<1, 1, 1>( grid.nghostzones, [=] CCTK_DEVICE(const PointDesc &p) CCTK_ATTRIBUTE_ALWAYS_INLINE { const int ijk = layout_cc.linear(p.i, p.j, p.k); @@ -66,7 +66,7 @@ extern "C" void nuX_M1_FiducialVelocity(CCTK_ARGUMENTS) { } else if (CCTK_Equals(fiducial_velocity, "mixed")) { - grid.loop_int_device<1, 1, 1>( + grid.loop_all_device<1, 1, 1>( grid.nghostzones, [=] CCTK_DEVICE(const PointDesc &p) CCTK_ATTRIBUTE_ALWAYS_INLINE { const int ijk = layout_cc.linear(p.i, p.j, p.k); @@ -97,7 +97,7 @@ extern "C" void nuX_M1_FiducialVelocity(CCTK_ARGUMENTS) { } else { - grid.loop_int_device<1, 1, 1>( + grid.loop_all_device<1, 1, 1>( grid.nghostzones, [=] CCTK_DEVICE(const PointDesc &p) CCTK_ATTRIBUTE_ALWAYS_INLINE { const int ijk = layout_cc.linear(p.i, p.j, p.k); From a6221a7ec2ed5bfd66a588cff49db52da24dbf56 Mon Sep 17 00:00:00 2001 From: Allen Wen Date: Fri, 13 Feb 2026 16:55:11 -0600 Subject: [PATCH 2/7] nuX_M1: remove some extra syncs --- nuX_M1/schedule.ccl | 61 +++++++++++++++++++------------------ nuX_M1/src/nuX_M1_sync.cxx | 62 +++++++++++++++++++------------------- 2 files changed, 62 insertions(+), 61 deletions(-) diff --git a/nuX_M1/schedule.ccl b/nuX_M1/schedule.ccl index f63be90..b9268be 100644 --- a/nuX_M1/schedule.ccl +++ b/nuX_M1/schedule.ccl @@ -213,14 +213,14 @@ SCHEDULE nuX_M1_InitFluxesRHS IN nuX_M1_TimeStep } "Initialize explicit RHSs to zero" # Ensure conserved vars have valid ghost zones before closure/fluxes -SCHEDULE nuX_M1_SyncConserved IN nuX_M1_TimeStep AFTER nuX_M1_InitFluxesRHS -{ - LANG: C - OPTIONS: SINGLEMAP - SYNC: rN - SYNC: rE - SYNC: rF -} "Sync rN/rE/rF into ghost zones prior to closure/flux computations" +# SCHEDULE nuX_M1_SyncConserved IN nuX_M1_TimeStep AFTER nuX_M1_InitFluxesRHS +# { +# LANG: C +# OPTIONS: SINGLEMAP +# SYNC: rN +# SYNC: rE +# SYNC: rF +# } "Sync rN/rE/rF into ghost zones prior to closure/flux computations" SCHEDULE nuX_M1_FiducialVelocity IN nuX_M1_TimeStep { @@ -272,19 +272,19 @@ else { } # Sync closure/opacity outputs so 4-pt stencil sees valid ghost data -SCHEDULE nuX_M1_SyncDerived IN nuX_M1_TimeStep \ - AFTER (nuX_M1_CalcOpacity nuX_M1_CalcFakeOpacity nuX_M1_CalcClosure) - BEFORE nuX_M1_CalcFluxes -{ - LANG: C - OPTIONS: SINGLEMAP - SYNC: rP - SYNC: rJ - SYNC: rHt - SYNC: rH - SYNC: chi - SYNC: opacs -} "Sync closure/opacities into ghost zones before flux divergence" +# SCHEDULE nuX_M1_SyncDerived IN nuX_M1_TimeStep \ +# AFTER (nuX_M1_CalcOpacity nuX_M1_CalcFakeOpacity nuX_M1_CalcClosure) +# BEFORE nuX_M1_CalcFluxes +# { +# LANG: C +# OPTIONS: SINGLEMAP +# SYNC: rP +# SYNC: rJ +# SYNC: rHt +# SYNC: rH +# SYNC: chi +# SYNC: opacs +# } "Sync closure/opacities into ghost zones before flux divergence" if (include_fluxes) { @@ -389,7 +389,8 @@ SCHEDULE nuX_M1_CalcUpdate IN nuX_M1_TimeStep \ WRITES: AsterX::mom(interior) AsterX::tau(interior) AsterX::DYe(interior) WRITES: rN(interior) rE(interior) rF(interior) - SYNC: chi rN rE rF fluidsources AsterX::mom AsterX::tau AsterX::DYe + #SYNC: chi rN rE rF fluidsources AsterX::mom AsterX::tau AsterX::DYe + SYNC: rN rE rF fluidsources AsterX::mom AsterX::tau AsterX::DYe } "Semi-implicit update of radiation and matter fields" @@ -430,14 +431,14 @@ SCHEDULE GROUP nuX_M1_BoundaryConditions IN nuX_M1_TimeStep \ } "Apply boundary conditions during the evolution" # ----------------------------------------------------------------------------- -SCHEDULE nuX_M1_Sync IN nuX_M1_BoundaryConditions -{ - LANG: C - OPTIONS: SINGLEMAP - SYNC: rN - SYNC: rE - SYNC: rF -} "Select boundary conditions" +# SCHEDULE nuX_M1_Sync IN nuX_M1_BoundaryConditions +# { +# LANG: C +# OPTIONS: SINGLEMAP +# SYNC: rN +# SYNC: rE +# SYNC: rF +# } "Select boundary conditions" SCHEDULE GROUP ApplyBCs AS nuX_M1_ApplyBCs IN nuX_M1_BoundaryConditions \ AFTER nuX_M1_Sync diff --git a/nuX_M1/src/nuX_M1_sync.cxx b/nuX_M1/src/nuX_M1_sync.cxx index d65da3c..1a5a9e6 100644 --- a/nuX_M1/src/nuX_M1_sync.cxx +++ b/nuX_M1/src/nuX_M1_sync.cxx @@ -8,38 +8,38 @@ namespace nuX_M1 { // Sync conserved radiation fields (rN, rE, rF) before computing // closures/fluxes. -extern "C" void nuX_M1_SyncConserved(CCTK_ARGUMENTS) { - DECLARE_CCTK_ARGUMENTS_nuX_M1_SyncConserved; - DECLARE_CCTK_PARAMETERS; - - if (verbose) { - CCTK_INFO("nuX_M1_SyncConserved"); - } - - // Do nothing, let SYNC handle BCs. -} +// extern "C" void nuX_M1_SyncConserved(CCTK_ARGUMENTS) { +// DECLARE_CCTK_ARGUMENTS_nuX_M1_SyncConserved; +// DECLARE_CCTK_PARAMETERS; +// +// if (verbose) { +// CCTK_INFO("nuX_M1_SyncConserved"); +// } +// +// // Do nothing, let SYNC handle BCs. +// } // Sync derived radiation fields (closure outputs/opacities) before fluxes. -extern "C" void nuX_M1_SyncDerived(CCTK_ARGUMENTS) { - DECLARE_CCTK_ARGUMENTS_nuX_M1_SyncDerived; - DECLARE_CCTK_PARAMETERS; - - if (verbose) { - CCTK_INFO("nuX_M1_SyncDerived"); - } - - // Do nothing, let SYNC handle BCs. -} - -extern "C" void nuX_M1_Sync(CCTK_ARGUMENTS) { - DECLARE_CCTK_ARGUMENTS_nuX_M1_Sync; - DECLARE_CCTK_PARAMETERS; - - if (verbose) { - CCTK_INFO("nuX_M1_Sync"); - } - - // Do nothing, let SYNC handle BCs. -} +// extern "C" void nuX_M1_SyncDerived(CCTK_ARGUMENTS) { +// DECLARE_CCTK_ARGUMENTS_nuX_M1_SyncDerived; +// DECLARE_CCTK_PARAMETERS; +// +// if (verbose) { +// CCTK_INFO("nuX_M1_SyncDerived"); +// } +// +// // Do nothing, let SYNC handle BCs. +// } + +// extern "C" void nuX_M1_Sync(CCTK_ARGUMENTS) { +// DECLARE_CCTK_ARGUMENTS_nuX_M1_Sync; +// DECLARE_CCTK_PARAMETERS; +// +// if (verbose) { +// CCTK_INFO("nuX_M1_Sync"); +// } +// +// // Do nothing, let SYNC handle BCs. +// } } // namespace nuX_M1 From 6781356bb18add197e7466dd9d29c83e9633c684 Mon Sep 17 00:00:00 2001 From: Allen Wen Date: Thu, 12 Feb 2026 15:49:43 -0600 Subject: [PATCH 3/7] nuX_M1: calculate fiducial velocity everywhere --- nuX_M1/schedule.ccl | 47 ++++++++++--------------- nuX_M1/src/nuX_M1_fiducial_velocity.cxx | 6 ++-- 2 files changed, 22 insertions(+), 31 deletions(-) diff --git a/nuX_M1/schedule.ccl b/nuX_M1/schedule.ccl index aa463b5..f63be90 100644 --- a/nuX_M1/schedule.ccl +++ b/nuX_M1/schedule.ccl @@ -72,13 +72,11 @@ SCHEDULE nuX_M1_FiducialVelocity IN nuX_M1_InitialGroup AFTER (nuX_M1_Reset) { LANG: C - READS: ADMBaseX::metric(interior) - READS: AsterX::dens(interior) - READS: HydroBaseX::vel(interior) - WRITES: fidu_velx(interior) fidu_vely(interior) fidu_velz(interior) - WRITES: fidu_w_lorentz(interior) - SYNC: fidu_vel - SYNC: fidu_w_lorentz + READS: ADMBaseX::metric(everywhere) + READS: AsterX::dens(everywhere) + READS: HydroBaseX::vel(everywhere) + WRITES: fidu_velx(everywhere) fidu_vely(everywhere) fidu_velz(everywhere) + WRITES: fidu_w_lorentz(everywhere) } "Compute the fiducial velocity" @@ -132,14 +130,12 @@ if (reset_to_equilibrium) { AFTER nuX_M1_Reset { LANG: C - READS: ADMBaseX::metric(interior) - READS: AsterX::dens(interior) - READS: HydroBaseX::vel(interior) + READS: ADMBaseX::metric(everywhere) + READS: AsterX::dens(everywhere) + READS: HydroBaseX::vel(everywhere) WRITES: fidu_velx(everywhere) fidu_vely(everywhere) fidu_velz(everywhere) WRITES: fidu_w_lorentz(everywhere) - SYNC: fidu_vel - SYNC: fidu_w_lorentz } "Compute the fiducial velocity" # SCHEDULE nuX_M1_SetToEquilibrium AT CCTK_POST_RECOVER_VARIABLES \ @@ -229,14 +225,11 @@ SCHEDULE nuX_M1_SyncConserved IN nuX_M1_TimeStep AFTER nuX_M1_InitFluxesRHS SCHEDULE nuX_M1_FiducialVelocity IN nuX_M1_TimeStep { LANG: C - READS: ADMBaseX::metric(interior) - READS: AsterX::dens(interior) - READS: HydroBaseX::vel(interior) - WRITES: fidu_velx(interior) fidu_vely(interior) fidu_velz(interior) - WRITES: fidu_w_lorentz(interior) - SYNC: fidu_vel - SYNC: fidu_w_lorentz - + READS: ADMBaseX::metric(everywhere) + READS: AsterX::dens(everywhere) + READS: HydroBaseX::vel(everywhere) + WRITES: fidu_velx(everywhere) fidu_vely(everywhere) fidu_velz(everywhere) + WRITES: fidu_w_lorentz(everywhere) } "Compute the fiducial velocity" SCHEDULE nuX_M1_CalcClosure IN nuX_M1_TimeStep AFTER nuX_M1_FiducialVelocity @@ -459,14 +452,12 @@ if (add_to_Tmunu) { SCHEDULE nuX_M1_FiducialVelocity IN nuX_M1_Tmunu BEFORE nuX_M1_AddToTmunu { LANG: C - READS: ADMBaseX::metric(interior) - READS: AsterX::dens(interior) - READS: HydroBaseX::vel(interior) - READS: nuX_m1_mask(interior) - WRITES: fidu_velx(interior) fidu_vely(interior) fidu_velz(interior) - WRITES: fidu_w_lorentz(interior) - SYNC: fidu_vel - SYNC: fidu_w_lorentz + READS: ADMBaseX::metric(everywhere) + READS: AsterX::dens(everywhere) + READS: HydroBaseX::vel(everywhere) + READS: nuX_m1_mask(everywhere) + WRITES: fidu_velx(everywhere) fidu_vely(everywhere) fidu_velz(everywhere) + WRITES: fidu_w_lorentz(everywhere) } "Compute the fiducial velocity" SCHEDULE nuX_M1_AddToTmunu IN nuX_M1_Tmunu diff --git a/nuX_M1/src/nuX_M1_fiducial_velocity.cxx b/nuX_M1/src/nuX_M1_fiducial_velocity.cxx index d76db13..68acb71 100644 --- a/nuX_M1/src/nuX_M1_fiducial_velocity.cxx +++ b/nuX_M1/src/nuX_M1_fiducial_velocity.cxx @@ -38,7 +38,7 @@ extern "C" void nuX_M1_FiducialVelocity(CCTK_ARGUMENTS) { if (CCTK_Equals(fiducial_velocity, "fluid")) { - grid.loop_int_device<1, 1, 1>( + grid.loop_all_device<1, 1, 1>( grid.nghostzones, [=] CCTK_DEVICE(const PointDesc &p) CCTK_ATTRIBUTE_ALWAYS_INLINE { const int ijk = layout_cc.linear(p.i, p.j, p.k); @@ -67,7 +67,7 @@ extern "C" void nuX_M1_FiducialVelocity(CCTK_ARGUMENTS) { } else if (CCTK_Equals(fiducial_velocity, "mixed")) { - grid.loop_int_device<1, 1, 1>( + grid.loop_all_device<1, 1, 1>( grid.nghostzones, [=] CCTK_DEVICE(const PointDesc &p) CCTK_ATTRIBUTE_ALWAYS_INLINE { const int ijk = layout_cc.linear(p.i, p.j, p.k); @@ -98,7 +98,7 @@ extern "C" void nuX_M1_FiducialVelocity(CCTK_ARGUMENTS) { } else { - grid.loop_int_device<1, 1, 1>( + grid.loop_all_device<1, 1, 1>( grid.nghostzones, [=] CCTK_DEVICE(const PointDesc &p) CCTK_ATTRIBUTE_ALWAYS_INLINE { const int ijk = layout_cc.linear(p.i, p.j, p.k); From 9f9a89ce45d798d9cb1e00b330848983808b5570 Mon Sep 17 00:00:00 2001 From: Allen Wen Date: Fri, 13 Feb 2026 16:55:11 -0600 Subject: [PATCH 4/7] nuX_M1: remove some extra syncs --- nuX_M1/schedule.ccl | 61 +++++++++++++++++++------------------ nuX_M1/src/nuX_M1_sync.cxx | 62 +++++++++++++++++++------------------- 2 files changed, 62 insertions(+), 61 deletions(-) diff --git a/nuX_M1/schedule.ccl b/nuX_M1/schedule.ccl index f63be90..b9268be 100644 --- a/nuX_M1/schedule.ccl +++ b/nuX_M1/schedule.ccl @@ -213,14 +213,14 @@ SCHEDULE nuX_M1_InitFluxesRHS IN nuX_M1_TimeStep } "Initialize explicit RHSs to zero" # Ensure conserved vars have valid ghost zones before closure/fluxes -SCHEDULE nuX_M1_SyncConserved IN nuX_M1_TimeStep AFTER nuX_M1_InitFluxesRHS -{ - LANG: C - OPTIONS: SINGLEMAP - SYNC: rN - SYNC: rE - SYNC: rF -} "Sync rN/rE/rF into ghost zones prior to closure/flux computations" +# SCHEDULE nuX_M1_SyncConserved IN nuX_M1_TimeStep AFTER nuX_M1_InitFluxesRHS +# { +# LANG: C +# OPTIONS: SINGLEMAP +# SYNC: rN +# SYNC: rE +# SYNC: rF +# } "Sync rN/rE/rF into ghost zones prior to closure/flux computations" SCHEDULE nuX_M1_FiducialVelocity IN nuX_M1_TimeStep { @@ -272,19 +272,19 @@ else { } # Sync closure/opacity outputs so 4-pt stencil sees valid ghost data -SCHEDULE nuX_M1_SyncDerived IN nuX_M1_TimeStep \ - AFTER (nuX_M1_CalcOpacity nuX_M1_CalcFakeOpacity nuX_M1_CalcClosure) - BEFORE nuX_M1_CalcFluxes -{ - LANG: C - OPTIONS: SINGLEMAP - SYNC: rP - SYNC: rJ - SYNC: rHt - SYNC: rH - SYNC: chi - SYNC: opacs -} "Sync closure/opacities into ghost zones before flux divergence" +# SCHEDULE nuX_M1_SyncDerived IN nuX_M1_TimeStep \ +# AFTER (nuX_M1_CalcOpacity nuX_M1_CalcFakeOpacity nuX_M1_CalcClosure) +# BEFORE nuX_M1_CalcFluxes +# { +# LANG: C +# OPTIONS: SINGLEMAP +# SYNC: rP +# SYNC: rJ +# SYNC: rHt +# SYNC: rH +# SYNC: chi +# SYNC: opacs +# } "Sync closure/opacities into ghost zones before flux divergence" if (include_fluxes) { @@ -389,7 +389,8 @@ SCHEDULE nuX_M1_CalcUpdate IN nuX_M1_TimeStep \ WRITES: AsterX::mom(interior) AsterX::tau(interior) AsterX::DYe(interior) WRITES: rN(interior) rE(interior) rF(interior) - SYNC: chi rN rE rF fluidsources AsterX::mom AsterX::tau AsterX::DYe + #SYNC: chi rN rE rF fluidsources AsterX::mom AsterX::tau AsterX::DYe + SYNC: rN rE rF fluidsources AsterX::mom AsterX::tau AsterX::DYe } "Semi-implicit update of radiation and matter fields" @@ -430,14 +431,14 @@ SCHEDULE GROUP nuX_M1_BoundaryConditions IN nuX_M1_TimeStep \ } "Apply boundary conditions during the evolution" # ----------------------------------------------------------------------------- -SCHEDULE nuX_M1_Sync IN nuX_M1_BoundaryConditions -{ - LANG: C - OPTIONS: SINGLEMAP - SYNC: rN - SYNC: rE - SYNC: rF -} "Select boundary conditions" +# SCHEDULE nuX_M1_Sync IN nuX_M1_BoundaryConditions +# { +# LANG: C +# OPTIONS: SINGLEMAP +# SYNC: rN +# SYNC: rE +# SYNC: rF +# } "Select boundary conditions" SCHEDULE GROUP ApplyBCs AS nuX_M1_ApplyBCs IN nuX_M1_BoundaryConditions \ AFTER nuX_M1_Sync diff --git a/nuX_M1/src/nuX_M1_sync.cxx b/nuX_M1/src/nuX_M1_sync.cxx index d65da3c..1a5a9e6 100644 --- a/nuX_M1/src/nuX_M1_sync.cxx +++ b/nuX_M1/src/nuX_M1_sync.cxx @@ -8,38 +8,38 @@ namespace nuX_M1 { // Sync conserved radiation fields (rN, rE, rF) before computing // closures/fluxes. -extern "C" void nuX_M1_SyncConserved(CCTK_ARGUMENTS) { - DECLARE_CCTK_ARGUMENTS_nuX_M1_SyncConserved; - DECLARE_CCTK_PARAMETERS; - - if (verbose) { - CCTK_INFO("nuX_M1_SyncConserved"); - } - - // Do nothing, let SYNC handle BCs. -} +// extern "C" void nuX_M1_SyncConserved(CCTK_ARGUMENTS) { +// DECLARE_CCTK_ARGUMENTS_nuX_M1_SyncConserved; +// DECLARE_CCTK_PARAMETERS; +// +// if (verbose) { +// CCTK_INFO("nuX_M1_SyncConserved"); +// } +// +// // Do nothing, let SYNC handle BCs. +// } // Sync derived radiation fields (closure outputs/opacities) before fluxes. -extern "C" void nuX_M1_SyncDerived(CCTK_ARGUMENTS) { - DECLARE_CCTK_ARGUMENTS_nuX_M1_SyncDerived; - DECLARE_CCTK_PARAMETERS; - - if (verbose) { - CCTK_INFO("nuX_M1_SyncDerived"); - } - - // Do nothing, let SYNC handle BCs. -} - -extern "C" void nuX_M1_Sync(CCTK_ARGUMENTS) { - DECLARE_CCTK_ARGUMENTS_nuX_M1_Sync; - DECLARE_CCTK_PARAMETERS; - - if (verbose) { - CCTK_INFO("nuX_M1_Sync"); - } - - // Do nothing, let SYNC handle BCs. -} +// extern "C" void nuX_M1_SyncDerived(CCTK_ARGUMENTS) { +// DECLARE_CCTK_ARGUMENTS_nuX_M1_SyncDerived; +// DECLARE_CCTK_PARAMETERS; +// +// if (verbose) { +// CCTK_INFO("nuX_M1_SyncDerived"); +// } +// +// // Do nothing, let SYNC handle BCs. +// } + +// extern "C" void nuX_M1_Sync(CCTK_ARGUMENTS) { +// DECLARE_CCTK_ARGUMENTS_nuX_M1_Sync; +// DECLARE_CCTK_PARAMETERS; +// +// if (verbose) { +// CCTK_INFO("nuX_M1_Sync"); +// } +// +// // Do nothing, let SYNC handle BCs. +// } } // namespace nuX_M1 From 8e8c3b6efb9d29ba00386233548a57693f11e358 Mon Sep 17 00:00:00 2001 From: Allen Wen Date: Mon, 9 Mar 2026 15:13:21 -0500 Subject: [PATCH 5/7] nuX_M1: only leave sync of fluxes and state vector --- nuX_M1/schedule.ccl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nuX_M1/schedule.ccl b/nuX_M1/schedule.ccl index b9268be..75c58b6 100644 --- a/nuX_M1/schedule.ccl +++ b/nuX_M1/schedule.ccl @@ -344,7 +344,7 @@ if (include_fluxes) { READS: rN_rhs(interior) rE_rhs(interior) rF_rhs(interior) WRITES: rN_rhs(interior) rE_rhs(interior) rF_rhs(interior) - SYNC: rN_rhs rE_rhs rF_rhs + # SYNC: rN_rhs rE_rhs rF_rhs } "Add limited flux divergence to the neutrino RHS" } @@ -390,7 +390,7 @@ SCHEDULE nuX_M1_CalcUpdate IN nuX_M1_TimeStep \ WRITES: rN(interior) rE(interior) rF(interior) #SYNC: chi rN rE rF fluidsources AsterX::mom AsterX::tau AsterX::DYe - SYNC: rN rE rF fluidsources AsterX::mom AsterX::tau AsterX::DYe + SYNC: rN rE rF AsterX::mom AsterX::tau AsterX::DYe } "Semi-implicit update of radiation and matter fields" @@ -470,7 +470,7 @@ if (add_to_Tmunu) { READS: rE(everywhere) rF(everywhere) READS: TmunuBaseX::eTtt(everywhere) TmunuBaseX::eTti(everywhere) TmunuBaseX::eTij(everywhere) WRITES: TmunuBaseX::eTtt(everywhere) TmunuBaseX::eTti(everywhere) TmunuBaseX::eTij(everywhere) - SYNC: TmunuBaseX::eTtt TmunuBaseX::eTti TmunuBaseX::eTij + # SYNC: TmunuBaseX::eTtt TmunuBaseX::eTti TmunuBaseX::eTij } "Adds the contribution of radiation to the global stress-energy tensor" } From 781113af6dc6fc7b0592aa79348cf26a165de964 Mon Sep 17 00:00:00 2001 From: Allen Wen Date: Mon, 9 Mar 2026 22:59:47 -0500 Subject: [PATCH 6/7] nuX_M1: use new loop macro to remove sync of fluxes --- nuX_M1/schedule.ccl | 29 ----------------------------- nuX_M1/src/nuX_M1_calc_fluxes.cxx | 19 ++++++++++++++----- 2 files changed, 14 insertions(+), 34 deletions(-) diff --git a/nuX_M1/schedule.ccl b/nuX_M1/schedule.ccl index 75c58b6..13dce18 100644 --- a/nuX_M1/schedule.ccl +++ b/nuX_M1/schedule.ccl @@ -212,16 +212,6 @@ SCHEDULE nuX_M1_InitFluxesRHS IN nuX_M1_TimeStep WRITES: nu_flux_x(everywhere) nu_flux_y(everywhere) nu_flux_z(everywhere) } "Initialize explicit RHSs to zero" -# Ensure conserved vars have valid ghost zones before closure/fluxes -# SCHEDULE nuX_M1_SyncConserved IN nuX_M1_TimeStep AFTER nuX_M1_InitFluxesRHS -# { -# LANG: C -# OPTIONS: SINGLEMAP -# SYNC: rN -# SYNC: rE -# SYNC: rF -# } "Sync rN/rE/rF into ghost zones prior to closure/flux computations" - SCHEDULE nuX_M1_FiducialVelocity IN nuX_M1_TimeStep { LANG: C @@ -271,21 +261,6 @@ else { } "Compute the opacities" } -# Sync closure/opacity outputs so 4-pt stencil sees valid ghost data -# SCHEDULE nuX_M1_SyncDerived IN nuX_M1_TimeStep \ -# AFTER (nuX_M1_CalcOpacity nuX_M1_CalcFakeOpacity nuX_M1_CalcClosure) -# BEFORE nuX_M1_CalcFluxes -# { -# LANG: C -# OPTIONS: SINGLEMAP -# SYNC: rP -# SYNC: rJ -# SYNC: rHt -# SYNC: rH -# SYNC: chi -# SYNC: opacs -# } "Sync closure/opacities into ghost zones before flux divergence" - if (include_fluxes) { SCHEDULE nuX_M1_CalcFluxes IN nuX_M1_TimeStep \ @@ -314,7 +289,6 @@ if (include_fluxes) { WRITES: nu_flux_x(everywhere) nu_flux_y(everywhere) nu_flux_z(everywhere) - SYNC: nu_flux_x nu_flux_y nu_flux_z } "Compute neutrino transport fluxes at cell centres" SCHEDULE nuX_M1_UpdateRHSFromFluxes IN nuX_M1_TimeStep \ @@ -344,7 +318,6 @@ if (include_fluxes) { READS: rN_rhs(interior) rE_rhs(interior) rF_rhs(interior) WRITES: rN_rhs(interior) rE_rhs(interior) rF_rhs(interior) - # SYNC: rN_rhs rE_rhs rF_rhs } "Add limited flux divergence to the neutrino RHS" } @@ -389,7 +362,6 @@ SCHEDULE nuX_M1_CalcUpdate IN nuX_M1_TimeStep \ WRITES: AsterX::mom(interior) AsterX::tau(interior) AsterX::DYe(interior) WRITES: rN(interior) rE(interior) rF(interior) - #SYNC: chi rN rE rF fluidsources AsterX::mom AsterX::tau AsterX::DYe SYNC: rN rE rF AsterX::mom AsterX::tau AsterX::DYe } "Semi-implicit update of radiation and matter fields" @@ -470,7 +442,6 @@ if (add_to_Tmunu) { READS: rE(everywhere) rF(everywhere) READS: TmunuBaseX::eTtt(everywhere) TmunuBaseX::eTti(everywhere) TmunuBaseX::eTij(everywhere) WRITES: TmunuBaseX::eTtt(everywhere) TmunuBaseX::eTti(everywhere) TmunuBaseX::eTij(everywhere) - # SYNC: TmunuBaseX::eTtt TmunuBaseX::eTti TmunuBaseX::eTij } "Adds the contribution of radiation to the global stress-energy tensor" } diff --git a/nuX_M1/src/nuX_M1_calc_fluxes.cxx b/nuX_M1/src/nuX_M1_calc_fluxes.cxx index f6dd87c..bd1a1ff 100644 --- a/nuX_M1/src/nuX_M1_calc_fluxes.cxx +++ b/nuX_M1/src/nuX_M1_calc_fluxes.cxx @@ -78,6 +78,9 @@ CCTK_DEVICE CCTK_HOST void M1_ComputePhysicalFluxes(CCTK_ARGUMENTS) { CCTK_REAL *restrict nu_flux_dir = (dir == 0 ? nu_flux_x : (dir == 1 ? nu_flux_y : nu_flux_z)); + constexpr vect face_centered = {!(dir == 0), !(dir == 1), + !(dir == 2)}; + tensor::slicing_geometry_const geom(layout_vc, layout_cc, alp, betax, betay, betaz, gxx, gxy, gxz, gyy, gyz, gzz, kxx, kxy, kxz, kyy, kyz, kzz); @@ -89,14 +92,15 @@ CCTK_DEVICE CCTK_HOST void M1_ComputePhysicalFluxes(CCTK_ARGUMENTS) { const int groupspec = ngroups * nspecies; //-------------------------------------------------------------------- - // GPU loop over all cells (INCLUDING ghost zones) + // GPU loop over all cells (INCLUDING ghost zones) in flux direction, + // interior in other directions. // // This matches THC's "1st pass compute the fluxes", which computes // fluxes over the full 1D line including ghosts so that the subsequent // RHS update can safely reference fluxes near the interior boundary. //-------------------------------------------------------------------- - grid.loop_all_device<1, 1, 1>( - grid.nghostzones, [=] CCTK_DEVICE(const PointDesc &p) { + grid.loop_mixcc_device<1, 1, 1>( + grid.nghostzones, face_centered, grid.nghostzones[dir], [=] CCTK_DEVICE(const PointDesc &p) { const int i = p.i; const int j = p.j; const int k = p.k; @@ -205,6 +209,10 @@ template void M1_UpdateRHSFromFluxes(CCTK_ARGUMENTS) { const CCTK_REAL *nu_flux_dir = (dir == 0 ? nu_flux_x : (dir == 1 ? nu_flux_y : nu_flux_z)); + constexpr vect face_centered = {!(dir == 0), !(dir == 1), + !(dir == 2)}; + + tensor::slicing_geometry_const geom(layout_vc, layout_cc, alp, betax, betay, betaz, gxx, gxy, gxz, gyy, gyz, gzz, kxx, kxy, kxz, kyy, kyz, kzz); @@ -212,8 +220,9 @@ template void M1_UpdateRHSFromFluxes(CCTK_ARGUMENTS) { // THC needs at least 2 ghost zones along each direction assert(cctk_nghostzones[dir] >= 2); - grid.loop_int_device<1, 1, 1>( - grid.nghostzones, + // Loop over interior + 1, as flux gridfunctions are physically face-centered but defined as cell-centered + grid.loop_mixcc_device<1, 1, 1>( + grid.nghostzones, face_centered, 1, [=] CCTK_DEVICE(const PointDesc &p) CCTK_ATTRIBUTE_ALWAYS_INLINE { const int i = p.i; const int j = p.j; From 92c3185ba36e3f17ecc559d10957805122365a27 Mon Sep 17 00:00:00 2001 From: Allen Wen Date: Mon, 9 Mar 2026 23:04:11 -0500 Subject: [PATCH 7/7] nuX_M1: remove file with sync functions as they are no longer needed --- nuX_M1/src/make.code.defn | 1 - nuX_M1/src/nuX_M1_calc_fluxes.cxx | 1 - nuX_M1/src/nuX_M1_sync.cxx | 45 ------------------------------- 3 files changed, 47 deletions(-) delete mode 100644 nuX_M1/src/nuX_M1_sync.cxx diff --git a/nuX_M1/src/make.code.defn b/nuX_M1/src/make.code.defn index cbc6219..2023619 100644 --- a/nuX_M1/src/make.code.defn +++ b/nuX_M1/src/make.code.defn @@ -18,7 +18,6 @@ nuX_M1_init_fluxes_rhs.cxx \ nuX_M1_param_check.cxx \ nuX_M1_reset.cxx \ nuX_M1_setmask.cxx \ -nuX_M1_sync.cxx \ nuX_M1_time_integrator.cxx \ nuX_M1_tmunu.cxx \ nuX_M1_calc_update.cxx \ diff --git a/nuX_M1/src/nuX_M1_calc_fluxes.cxx b/nuX_M1/src/nuX_M1_calc_fluxes.cxx index bd1a1ff..49c5881 100644 --- a/nuX_M1/src/nuX_M1_calc_fluxes.cxx +++ b/nuX_M1/src/nuX_M1_calc_fluxes.cxx @@ -212,7 +212,6 @@ template void M1_UpdateRHSFromFluxes(CCTK_ARGUMENTS) { constexpr vect face_centered = {!(dir == 0), !(dir == 1), !(dir == 2)}; - tensor::slicing_geometry_const geom(layout_vc, layout_cc, alp, betax, betay, betaz, gxx, gxy, gxz, gyy, gyz, gzz, kxx, kxy, kxz, kyy, kyz, kzz); diff --git a/nuX_M1/src/nuX_M1_sync.cxx b/nuX_M1/src/nuX_M1_sync.cxx deleted file mode 100644 index 1a5a9e6..0000000 --- a/nuX_M1/src/nuX_M1_sync.cxx +++ /dev/null @@ -1,45 +0,0 @@ -#include - -#include "cctk.h" -#include "cctk_Arguments.h" -#include "cctk_Parameters.h" - -namespace nuX_M1 { - -// Sync conserved radiation fields (rN, rE, rF) before computing -// closures/fluxes. -// extern "C" void nuX_M1_SyncConserved(CCTK_ARGUMENTS) { -// DECLARE_CCTK_ARGUMENTS_nuX_M1_SyncConserved; -// DECLARE_CCTK_PARAMETERS; -// -// if (verbose) { -// CCTK_INFO("nuX_M1_SyncConserved"); -// } -// -// // Do nothing, let SYNC handle BCs. -// } - -// Sync derived radiation fields (closure outputs/opacities) before fluxes. -// extern "C" void nuX_M1_SyncDerived(CCTK_ARGUMENTS) { -// DECLARE_CCTK_ARGUMENTS_nuX_M1_SyncDerived; -// DECLARE_CCTK_PARAMETERS; -// -// if (verbose) { -// CCTK_INFO("nuX_M1_SyncDerived"); -// } -// -// // Do nothing, let SYNC handle BCs. -// } - -// extern "C" void nuX_M1_Sync(CCTK_ARGUMENTS) { -// DECLARE_CCTK_ARGUMENTS_nuX_M1_Sync; -// DECLARE_CCTK_PARAMETERS; -// -// if (verbose) { -// CCTK_INFO("nuX_M1_Sync"); -// } -// -// // Do nothing, let SYNC handle BCs. -// } - -} // namespace nuX_M1