Simulation gets frozen during load balancing

[milin333martin]

I am developing a solver and I wanted to integrate this wonderful library. For the most part its working as I intended and even got it to work with OFv2406. The only issue I have is, sometimes after a couple of iterations (quite a significant number of iterations) the code gets stuck at:

`
Max Courant Number = 0.299909210528
deltaT = 7.8125e-09
Time = 2.28125e-06 deltaT = 7.8125e-09

Maximum imbalance found = 4.59569517161 %
Stage 1/5 (λ=0, γ=0.465388589249)
Stage 2/5 (λ=0, γ=0.465388589249)
Stage 3/5 (λ=0.682342861037, γ=0.124745797314)
Stage 4/5 (λ=0, γ=0.465388589249)
Stage 5/5 (λ=0.0452309744824, γ=0.154263303749)
Final state:
min/max(rho) = 0.0848992519574 / 1.11989817295
min/max(p) = 101325 / 141201.92445
min/max(T) = 300 / 4180.66393636
min/max(Y) = 9.97878073365e-06 / 1
min/max(AMRError) = 0 / 1.30764887612
Diagnostics:
Heat release: 133.339506173 W
ExecutionTime = 51.7 s ClockTime = 75 s

Max Courant Number = 0.299891954152
deltaT = 7.8125e-09
Time = 2.28906e-06 deltaT = 7.8125e-09

Maximum imbalance found = 4.59569517161 %
Stage 1/5 (λ=0, γ=0.465388589249)
Stage 2/5 (λ=0, γ=0.465388589249)
Stage 3/5 (λ=0.682342861037, γ=0.124745797314)
Stage 4/5 (λ=0, γ=0.465388589249)
Stage 5/5 (λ=0.0452309744824, γ=0.154263303749)
Final state:
min/max(rho) = 0.0849030149856 / 1.1208709764
min/max(p) = 101325 / 141201.956747
min/max(T) = 300 / 4180.66489257
min/max(Y) = 9.5738399691e-06 / 1
min/max(AMRError) = 0 / 1.29931546337
Diagnostics:
Heat release: 135.53278561 W
ExecutionTime = 51.72 s ClockTime = 75 s

Max Courant Number = 0.299860685004
deltaT = 7.8125e-09
Time = 2.29688e-06 deltaT = 7.8125e-09

Protecting 0 cells next to requested boundary patches.
`

I am unsure as to why this happens, because I have not seen this problem for a non blastAMR based dynamic meshing. Any guidance is greatly appreciated.

Thanks in advance

[FoamScience]

Ok; so this is a little bit strange; but it will require some debugging on my end to solve. I'll try to take a look when i get time but it would really help if you could share the setup that lead to this; or at least the dynamicMeshDict.

[milin333martin]

Thank you so much for the prompt reply. Here is the dynamicMeshDict for your reference. I did try using polyRefiner and that led to the freeze, while hexRefiner worked good, it doesn't seem to unrefine the previously refined cells.

`

FoamFile
{
    version     2.0;
    format      ascii;
    class       dictionary;
    object      dynamicMeshDict;
}

libs
(
    "libROUNDSchemes.so"
    "libamrDynamicMesh.so"
    "libamrDynamicFvMesh.so"
    "libamrIndicators.so"
    "libamrLoadPolicies.so"
);

dynamicFvMesh   adaptiveFvMesh;
refiner         polyRefiner;

errorEstimator  fieldValue;
fieldName       AMRError;

lowerRefineLevel    0.1;
upperRefineLevel    GREAT;
unrefineLevel       0.05;
upperUnrefineLevel  GREAT;

maxRefinement       4;

refineInterval      10;
unrefineInterval    10;

nBufferLayers               3;
nRefinementBufferLayers     3;
nUnrefinementBufferLayers   6;

balance             yes;
allowableBalance    0.1;
loadPolicy          cellCount;

nRefinementDimensions   2;

// -----------------------------------------------------------------------------
// Flux Correction
// -----------------------------------------------------------------------------
correctFluxes
(
    (phi none)
    (rhoPhi none)
    (rhoUPhi none)
    (rhoEPhi none)
    (rhoYPhi none)
    (phiM none)
    (phiCorr none)
    (phiT none)
    (neg none)
    (pos none)
    (e_neg none)
    (e_pos none)
    (aphiv_pos none)
    (aphiv_neg none)
    (a_pos none)
    (a_neg none)
    (rPsi_pos none)
    (rPsi_neg none)
    (interpolate(rho) none)
    (p_pos none)
    (p_neg none)
    (HLL::SNei none)
    (HLL::SOwn none)
    (HLL::SStar none)
    (HLL::pStar none)
    (HLLC::SNei none)
    (HLLC::SOwn none)
    (HLLC::SStar none)
    (HLLC::pStar none)
    (HLLCP::SNei none)
    (HLLCP::SOwn none)
    (HLLCP::SStar none)
    (HLLCP::pStar none)
    (fluxScheme::own none)
    (amaxSf none)
    (HLL::UvNei none)
    (HLL::UvOwn none)
    (HLLC::UvNei none)
    (HLLC::UvOwn none)
    (HLLCP::UvNei none)
    (HLLCP::UvOwn none)
    (am_ref none)
    (rho_pos none)
    (cSf_neg none)
    (cSf_pos none)
    (HLLC::phip none)
    (HLLCP::phip none)
    (ap_ref none)
    (aSf_ref none)
    (phiv_pos none)
    (phiv_neg none)
    (rho_neg none)
    (fluxScheme::nei none)
    (HLL::pStarNei none)
    (HLL::pStarOwn none)
    (HLLC::pStarNei none)
    (HLLC::pStarOwn none)
    (HLLCP::pStarNei none)
    (HLLCP::pStarOwn none)
    (Kurganov::aNei none)
    (Kurganov::aOwn none)
    (Kurganov::aSf none)
    (Kurganov::aPhivOwn none)
    (Kurganov::aPhivNei none)
    (AUSMPlus::phi none)
    (AUSMPlusUp::phi none)
    (HLLC_LM::SOwn none)
    (HLLC_LM::SNei none)
    (HLLC_LM::SStar none)
    (HLLC_LM::pStarOwn none)
    (HLLC_LM::pStarNei none)
    (HLLC_LM::UvOwn none)
    (HLLC_LM::UvNei none)
    (interpolate(rho_q1) none)
    (interpolate(rhoU_q1) none)
    (interpolate(rhoE_q1) none)
    (interpolate(rhoY_q1) none)
    (interpolate(U_work) none)
    (interpolate(e_work) none)
    (interpolate(p_work) none)
    (interpolate(Y_work) none)
    (interpolate(c_work) none)
);

dumpLevel           true;

`

`

/*--------------------------------*- C++ -*----------------------------------*\
| =========                 |                                                 |
| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
|  \\    /   O peration     | Version:  v2406                                 |
|   \\  /    A nd           | Website:  www.openfoam.com                      |
|    \\/     M anipulation  |                                                 |
\*---------------------------------------------------------------------------*/
FoamFile
{
    version     2.0;
    format      ascii;
    class       dictionary;
    object      dynamicMeshDict;
}
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //

// Load blastAMR libraries
libs
(
    "libROUNDSchemes.so"
    "libamrDynamicMesh.so"
    "libamrDynamicFvMesh.so"
    "libamrIndicators.so"
    "libamrLoadPolicies.so"
);

// Use blastAMR's adaptive mesh
dynamicFvMesh   adaptiveFvMesh;

// Refiner type: hexRefiner for hex meshes, polyRefiner for polyhedral
refiner         hexRefiner;

// -----------------------------------------------------------------------------
// Error Estimator Configuration
// -----------------------------------------------------------------------------
// Use the AMRError field computed by your solver
errorEstimator  fieldValue;
fieldName       normGradRho; 

// -----------------------------------------------------------------------------
// Refinement Criteria
// -----------------------------------------------------------------------------
// Refine cells where AMRError > lowerRefineLevel
lowerRefineLevel    0.025;
upperRefineLevel    1.0;

// Unrefine cells where AMRError < unrefineLevel
unrefineLevel       0.01;
upperUnrefineLevel  1.05;

// Maximum refinement levels
maxRefinement       4;

// Maximum number of cells (safety limit)
maxCells            3000000;

// -----------------------------------------------------------------------------
// Refinement Timing
// -----------------------------------------------------------------------------
// Check for refinement every N timesteps
refineInterval      10;

// Check for unrefinement every N timesteps
unrefineInterval    25;

// -----------------------------------------------------------------------------
// Buffer Layers (smoothing)
// -----------------------------------------------------------------------------
nBufferLayers       7;
nRefinementBufferLayers  4;
nUnrefinementBufferLayers 4;

// -----------------------------------------------------------------------------
// Load Balancing (KEY FEATURE)
// -----------------------------------------------------------------------------
balance             yes;
allowableBalance    0.15;
//balanceInterval     500;

// -----------------------------------------------------------------------------
// 2D Settings - protect empty patches from refinement
// -----------------------------------------------------------------------------
//protectedPatches    (frontAndBack);
nRefinementDimensions   2;

// -----------------------------------------------------------------------------
// Flux Correction
// -----------------------------------------------------------------------------
correctFluxes
(
    (phi none)
    (rhoPhi none)
    (rhoUPhi none)
    (rhoEPhi none)
    (rhoYPhi none)
    (phiM none)
    (phiCorr none)
    (phiT none)
    (neg none)
    (pos none)
    (e_neg none)
    (e_pos none)
    (aphiv_pos none)
    (aphiv_neg none)
    (a_pos none)
    (a_neg none)
    (rPsi_pos none)
    (rPsi_neg none)
    (interpolate(rho) none)
    (p_pos none)
    (p_neg none)
    (HLL::SNei none)
    (HLL::SOwn none)
    (HLL::SStar none)
    (HLL::pStar none)
    (HLLC::SNei none)
    (HLLC::SOwn none)
    (HLLC::SStar none)
    (HLLC::pStar none)
    (HLLCP::SNei none)
    (HLLCP::SOwn none)
    (HLLCP::SStar none)
    (HLLCP::pStar none)
    (fluxScheme::own none)
    (amaxSf none)
    (HLL::UvNei none)
    (HLL::UvOwn none)
    (HLLC::UvNei none)
    (HLLC::UvOwn none)
    (HLLCP::UvNei none)
    (HLLCP::UvOwn none)
    (am_ref none)
    (rho_pos none)
    (cSf_neg none)
    (cSf_pos none)
    (HLLC::phip none)
    (HLLCP::phip none)
    (ap_ref none)
    (aSf_ref none)
    (phiv_pos none)
    (phiv_neg none)
    (rho_neg none)
    (fluxScheme::nei none)
    (HLL::pStarNei none)
    (HLL::pStarOwn none)
    (HLLC::pStarNei none)
    (HLLC::pStarOwn none)
    (HLLCP::pStarNei none)
    (HLLCP::pStarOwn none)
    (Kurganov::aNei none)
    (Kurganov::aOwn none)
    (Kurganov::aSf none)
    (Kurganov::aPhivOwn none)
    (Kurganov::aPhivNei none)
    (AUSMPlus::phi none)
    (AUSMPlusUp::phi none)
    (HLLC_LM::SOwn none)
    (HLLC_LM::SNei none)
    (HLLC_LM::SStar none)
    (HLLC_LM::pStarOwn none)
    (HLLC_LM::pStarNei none)
    (HLLC_LM::UvOwn none)
    (HLLC_LM::UvNei none)
    (interpolate(rho_q1) none)
    (interpolate(rhoU_q1) none)
    (interpolate(rhoE_q1) none)
    (interpolate(rhoY_q1) none)
    (interpolate(U_work) none)
    (interpolate(e_work) none)
    (interpolate(p_work) none)
    (interpolate(Y_work) none)
    (interpolate(c_work) none)
);

// Write refinement level field
dumpLevel           true;

// ************************************************************************* //

`

[FoamScience]

It seems it is a bit challenging to reproduce this really help if you could provide a little bit more info on the initial mesh so I can help.
Also does this happen on first load-balancing call?

[MahadevP2001]

I too am experiencing this problem, but with the hexRefiner and not the polyRefiner. My log file has been stuck on this for several hours :

Max Courant Numbers = 0.03999956212
deltaT = 5.091578232e-10
Maximum imbalance found = 50.12726077 %
Decomposition method scotch [240]
Mapping the fields ...
Distributing the mesh ...
Successfully distributed mesh
Distributing ...

This happened during the first load balancing call. Please tell me if I can provide any information that can help you to diagnose the problem. I will be truly grateful.

Here is my dynamicMeshDict :

  /*--------------------------------*- C++ -*----------------------------------*\
  | =========                 |                                                 |
  | \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
  |  \\    /   O peration     | Version:  2012                                  |
  |   \\  /    A nd           | Website:  www.openfoam.com                      |
  |    \\/     M anipulation  |                                                 |
  \*---------------------------------------------------------------------------*/
  // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
  FoamFile
  {
      version         2;
      format          ascii;
      class           dictionary;
      object          dynamicMeshDict;
  }
  
  dynamicFvMesh   adaptiveFvMesh;
  
  refiner         hexRefiner;
  
  errorEstimator  coded;
  
  name emulateDelta;
  
  
  code
  #{
      Info<< "---->! custom error estimator !<----" << endl;
      error_ = 0.0; // <== set default behavior to not touch the cells
      /*
          End result must be (error_ is a volScalarField here):
          - error_ == 1  if the cell needs to be refined
          - error_ == 0  if the cell is to be left alone
          - error_ == -1 if the vertices of the cell are to be unrefined
      */
  
      // get references to few useful things
      const auto& volumeFraction1 = mesh_.lookupObject<volScalarField>("volumeFraction1");
      
      const label nCells = mesh_.nCells();
      
      // Loop through interal faces and store difference between owner and neighbour
      // note that we must take of the current error value as a single cell will have
      // multiple faces, hence, will be processed multiple times
      for (label celli = 0; celli < nCells; celli++)
      {
      
            const scalar alpha = volumeFraction1[celli];
            //const scalar gradP = normGradPgas[celli];
            if ((alpha > 0.001 && alpha < 0.999) )
          {
              error_[celli] = 1;
          }
          // unrefine condition
          else if (alpha < 0.0003)
          {
              error_[celli] = -1;
          }
          
      }
  
      
      Info<< "---->! end    error estimator !<----" << endl;
  #};
  
  balance         yes;
  
  allowableImbalance 0.5;
  
  refineInterval  10;
  
  unrefineInterval 50;
  
  maxRefinement   3;
  
  nBufferLayers   4;
  
  nRefinementBufferLayers 4;
  
  nUnrefinementBufferLayers 4;
  
  lowerRefineLevel 0.001;
  
  upperRefineLevel 0.999;
  
  unrefineLevel   0.001;
  
  upperUnrefineLevel 0.999;
  
  
  dumpLevel       yes;
  
  refineProbes    no;
  
  balanceInterval  30;
  
  // ************************************************************************* //