Conditional auxiliary halo exchange when turbulence is not used

source/mus_auxField_module.f90

@@ -223,7 +223,8 @@ end subroutine mus_init_auxFieldArrays
   !! PDF initialized by IC
   subroutine mus_initAuxFieldFluidAndExchange(auxField, state, neigh, nElems,  &
     &                                         nSize, nFields, stencil, varSys, &
-    &                                         derVarPos, iLevel, general, quantities)
+    &                                         derVarPos, iLevel, general,      &
+    &                                         quantities, needAuxHaloComm)
     !---------------------------------------------------------------------------
     !> auxilary field array
     type(mus_auxFieldVar_type), intent(inout) :: auxField
@@ -249,6 +250,8 @@ subroutine mus_initAuxFieldFluidAndExchange(auxField, state, neigh, nElems,  &
     type(tem_general_type), intent(in) :: general
     !> Class that contains pointers to the proper derived quantities functions
     type(mus_scheme_derived_quantities_type), intent(in) :: quantities
+    !> Whether auxField halo communication is required
+    logical, intent(in) :: needAuxHaloComm
     !---------------------------------------------------------------------------
     integer :: iField
     !---------------------------------------------------------------------------
@@ -269,12 +272,14 @@ subroutine mus_initAuxFieldFluidAndExchange(auxField, state, neigh, nElems,  &
     ! communicate velocity field. Requires for tubulence to compute ShearRate
     ! from velocity gradient.
     ! exchange velocity halo on current level
-    call general%commpattern%exchange_real(   &
-      &  send         = auxField%sendBuffer,  &
-      &  recv         = auxField%recvBuffer, &
-      &  state        = auxField%val(:),      &
-      &  message_flag = iLevel+100,           &
-      &  comm         = general%proc%comm     )
+    if (needAuxHaloComm) then
+      call general%commpattern%exchange_real(  &
+        &  send         = auxField%sendBuffer, &
+        &  recv         = auxField%recvBuffer, &
+        &  state        = auxField%val(:),     &
+        &  message_flag = iLevel+100,          &
+        &  comm         = general%proc%comm    )
+    end if
 
   end subroutine mus_initAuxFieldFluidAndExchange
   ! ************************************************************************** !
@@ -285,7 +290,7 @@ end subroutine mus_initAuxFieldFluidAndExchange
   !! halos
   subroutine mus_calcAuxFieldAndExchange(auxField, calcAuxField, state, &
     &  pdfData, nFields, field, globSrc, stencil, varSys, derVarPos,    &
-    &  phyConvFac, general, iLevel, minLevel, schemeHeader, quantities)
+    &  phyConvFac, general, iLevel, minLevel, schemeHeader, quantities  )
     ! -------------------------------------------------------------------- !
     !> auxilary field array
     type(mus_auxFieldVar_type), intent(inout) :: auxField
@@ -378,15 +383,17 @@ subroutine mus_calcAuxFieldAndExchange(auxField, calcAuxField, state, &
     ! communicate velocity field. Requires for tubulence to compute ShearRate
     ! from velocity gradient.
     ! exchange velocity halo on current level
-    call general%commpattern%exchange_real(   &
-      &  send         = auxField%sendBuffer,  &
-      &  recv         = auxField%recvBuffer , &
-      &  state        = auxField%val(:),      &
-      &  message_flag = iLevel+100,           &
-      &  comm         = general%proc%comm     )
+    if (schemeHeader%needAuxHaloComm) then
+        call general%commpattern%exchange_real(   &
+          &  send         = auxField%sendBuffer,  &
+          &  recv         = auxField%recvBuffer , &
+          &  state        = auxField%val(:),      &
+          &  message_flag = iLevel+100,           &
+          &  comm         = general%proc%comm     )
+    end if
 
     ! communicate ghost halos from coarser
-    if (iLevel > minLevel) then
+    if (schemeHeader%needAuxHaloComm .and. iLevel > minLevel) then
        call general%commpattern%exchange_real(             &
         &  send         = auxField%sendBufferFromCoarser,  &
         &  recv         = auxField%recvBufferFromCoarser , &
@@ -403,7 +410,8 @@ end subroutine mus_calcAuxFieldAndExchange
   !! halos
   subroutine mus_intpAuxFieldCoarserAndExchange(intp, tAuxField, sAuxField,  &
     &                                           tLevelDesc, stencil, iLevel, &
-    &                                           nAuxScalars,  general)
+    &                                           nAuxScalars, general,        &
+    &                                           needAuxHaloComm)
     ! -------------------------------------------------------------------- !
     !> Interpolation type
     type(mus_interpolation_type), intent(inout) :: intp
@@ -421,6 +429,8 @@ subroutine mus_intpAuxFieldCoarserAndExchange(intp, tAuxField, sAuxField,  &
     integer, intent(in) :: nAuxScalars
     !> contains commPattern, MPI communicator and simControl
     type(tem_general_type), intent(in) :: general
+    !> Whether auxField halo communication is required
+    logical, intent(in) :: needAuxHaloComm
     ! -------------------------------------------------------------------- !
     call intp%fillMineFromFiner%do_intpArbiVal(      &
       & tLevelDesc = tLevelDesc,                     &
@@ -434,12 +444,14 @@ subroutine mus_intpAuxFieldCoarserAndExchange(intp, tAuxField, sAuxField,  &
 
     ! exchange velocity halo fromFiner, required to compute velocity
     ! gradient
-    call general%commPattern%exchange_real(            &
-      &  send         = tAuxField%sendBufferFromFiner, &
-      &  recv         = tAuxField%recvBufferFromFiner, &
-      &  state        = tAuxField%val(:),              &
-      &  message_flag = iLevel+300,                    &
-      &  comm         = general%proc%comm              )
+    if (needAuxHaloComm) then
+      call general%commPattern%exchange_real(            &
+        &  send         = tAuxField%sendBufferFromFiner, &
+        &  recv         = tAuxField%recvBufferFromFiner, &
+        &  state        = tAuxField%val(:),              &
+        &  message_flag = iLevel+300,                    &
+        &  comm         = general%proc%comm              )
+    end if
 
   end subroutine mus_intpAuxFieldCoarserAndExchange
   ! ************************************************************************* !
@@ -449,7 +461,8 @@ end subroutine mus_intpAuxFieldCoarserAndExchange
   !! halos
   subroutine mus_intpAuxFieldFinerAndExchange(intp, tAuxField, sAuxField,  &
     &                                         tLevelDesc, stencil, iLevel, &
-    &                                         nAuxScalars, general)
+    &                                         nAuxScalars, general,        &
+    &                                         needAuxHaloComm)
     ! -------------------------------------------------------------------- !
     !> Interpolation type
     type(mus_interpolation_type), intent(inout) :: intp
@@ -467,6 +480,8 @@ subroutine mus_intpAuxFieldFinerAndExchange(intp, tAuxField, sAuxField,  &
     integer, intent(in) :: nAuxScalars
     !> contains commPattern, MPI communicator and simControl
     type(tem_general_type), intent(in) :: general
+    !> Whether auxField halo communication is required
+    logical, intent(in) :: needAuxHaloComm
     ! -------------------------------------------------------------------- !
     integer :: iOrder
     ! -------------------------------------------------------------------- !
@@ -484,14 +499,15 @@ subroutine mus_intpAuxFieldFinerAndExchange(intp, tAuxField, sAuxField,  &
 
     ! exchange velocity halo fromFiner, required to compute velocity
     ! gradient
-    call general%commPattern%exchange_real(              &
-      &  send         = tAuxField%sendBufferFromCoarser, &
-      &  recv         = tAuxField%recvBufferFromCoarser, &
-      &  state        = tAuxField%val(:),                &
-      &  message_flag = iLevel+200,                      &
-      &  comm         = general%proc%comm                )
+    if (needAuxHaloComm) then
+      call general%commPattern%exchange_real(              &
+        &  send         = tAuxField%sendBufferFromCoarser, &
+        &  recv         = tAuxField%recvBufferFromCoarser, &
+        &  state        = tAuxField%val(:),                &
+        &  message_flag = iLevel+200,                      &
+        &  comm         = general%proc%comm                )
+    end if
 
   end subroutine mus_intpAuxFieldFinerAndExchange
   ! ************************************************************************* !
 end module mus_auxField_module
-

source/mus_control_module.f90

@@ -319,23 +319,23 @@ recursive subroutine do_recursive_multiLevel(me, iLevel)
     ! and exchange them if turbulence is active
     call tem_startTimer( timerHandle =  mus_timerHandles%aux(iLevel) )
     write(logUnit(10), "(A)") 'Calculate auxField'
-    call mus_calcAuxFieldAndExchange(                                &
-      & auxField          = me%scheme%auxField(iLevel),              &
-      & calcAuxField      = me%scheme%calcAuxField,                  &
-      & state             = me%scheme%state(iLevel)%val(:, now),     &
-      & pdfData           = me%scheme%pdf(iLevel),                   &
-      & nFields           = me%scheme%nFields,                       &
-      & field             = me%scheme%field(:),                      &
-      & globSrc           = me%scheme%globSrc,                       &
-      & stencil           = me%scheme%layout%fStencil,               &
-      & varSys            = me%scheme%varSys,                        &
-      & derVarPos         = me%scheme%derVarPos,                     &
-      & general           = me%params%general,                       &
-      & phyConvFac        = me%params%physics%fac(iLevel),           &
-      & iLevel            = iLevel,                                  &
-      & minLevel          = me%geometry%tree%global%minLevel,        &
-      & schemeHeader      = me%scheme%header,                        &
-      & quantities        = me%scheme%layout%quantities              )
+    call mus_calcAuxFieldAndExchange(                            &
+      & auxField          = me%scheme%auxField(iLevel),          &
+      & calcAuxField      = me%scheme%calcAuxField,              &
+      & state             = me%scheme%state(iLevel)%val(:, now), &
+      & pdfData           = me%scheme%pdf(iLevel),               &
+      & nFields           = me%scheme%nFields,                   &
+      & field             = me%scheme%field(:),                  &
+      & globSrc           = me%scheme%globSrc,                   &
+      & stencil           = me%scheme%layout%fStencil,           &
+      & varSys            = me%scheme%varSys,                    &
+      & derVarPos         = me%scheme%derVarPos,                 &
+      & general           = me%params%general,                   &
+      & phyConvFac        = me%params%physics%fac(iLevel),       &
+      & iLevel            = iLevel,                              &
+      & minLevel          = me%geometry%tree%global%minLevel,    &
+      & schemeHeader      = me%scheme%header,                    &
+      & quantities        = me%scheme%layout%quantities          )
 
     if (iLevel < me%geometry%tree%global%maxLevel) then
       write(logUnit(10), "(A)") 'Interpolate and exchange auxField in ' &
@@ -348,7 +348,8 @@ recursive subroutine do_recursive_multiLevel(me, iLevel)
         & stencil     = me%scheme%layout%fStencil,    &
         & iLevel      = iLevel,                       &
         & nAuxScalars = me%scheme%varSys%nAuxScalars, &
-        & general     = me%params%general             )
+        & general     = me%params%general,            &
+        & needAuxHaloComm = me%scheme%header%needAuxHaloComm )
     end if
 
     call tem_stopTimer( timerHandle =  mus_timerHandles%aux(iLevel) )

source/mus_dynLoadBal_module.f90

@@ -461,7 +461,8 @@ subroutine mus_reset_aux( scheme, params, geometry)
           & stencil     = scheme%layout%fStencil,    &
           & iLevel      = iLevel,                    &
           & nAuxScalars = scheme%varSys%nAuxScalars, &
-          & general     = params%general             )
+          & general     = params%general,            &
+          & needAuxHaloComm = scheme%header%needAuxHaloComm )
        end if
     end do
 

source/mus_flow_module.fpp

@@ -1704,7 +1704,8 @@ write(dbgUnit(5), *) ''
         & varSys     = scheme%varSys,                    &
         & derVarPos  = scheme%derVarPos,                 &
         & general    = general,                          &
-        & quantities = scheme%layout%quantities          )
+        & quantities = scheme%layout%quantities,         &
+        & needAuxHaloComm = scheme%header%needAuxHaloComm )
     end do
 
     ! Initilialize auxField ghostFromFiner and ghostFromCoarser with
@@ -1718,7 +1719,8 @@ write(dbgUnit(5), *) ''
         & stencil     = scheme%layout%fStencil,    &
         & iLevel      = iLevel,                    &
         & nAuxScalars = scheme%varSys%nAuxScalars, &
-        & general     = general                    )
+        & general     = general,                   &
+        & needAuxHaloComm = scheme%header%needAuxHaloComm )
     end do
 
     do iLevel = minLevel+1, maxLevel
@@ -1730,7 +1732,8 @@ write(dbgUnit(5), *) ''
         & stencil     = scheme%layout%fStencil,     &
         & iLevel      = iLevel,                     &
         & nAuxScalars = scheme%varSys%nAuxScalars,  &
-        & general     = general                     )
+        & general     = general,                    &
+        & needAuxHaloComm = scheme%header%needAuxHaloComm  )
     end do
 
   end subroutine mus_initAuxField

source/mus_tracking_module.f90

@@ -72,6 +72,11 @@ subroutine mus_init_tracker( scheme, geometry, params )
     integer :: iTrack, iConfig
     ! --------------------------------------------------------------------------
 
+    select case (trim(scheme%header%kind))
+    case ('fluid', 'fluid_incompressible')
+      scheme%header%needAuxHaloComm = scheme%field(1)%fieldProp%fluid%turbulence%active
+    end select
+
     write(dbgUnit(1),*) 'Enter mus_init_tracker'
     write(dbgUnit(1),*) 'Tracking control active is: ', &
       &                 scheme%track%control%active
@@ -113,8 +118,59 @@ subroutine mus_init_tracker( scheme, geometry, params )
       &                    solver   = params%general%solver,  &
       &                    varSys   = scheme%varSys           )
 
+    scheme%header%needAuxHaloComm = scheme%header%needAuxHaloComm      &
+      &                     .or. mus_tracking_needs_velocity_gradient( &
+      &                           scheme = scheme                      )
+
+    if (scheme%header%needAuxHaloComm) then
+      write(logUnit(1),*) 'Auxiliary field halo communication activated.'
+    else
+      write(logUnit(1),*) 'Auxiliary field halo communication deactivated.'
+    end if
+
   end subroutine mus_init_tracker
 ! **************************************************************************** !
 
+
+! **************************************************************************** !
+  !> Check whether any active tracking object requests velocity-gradient based
+  !! variables that require auxField halo values.
+  !!
+  !! TODO rather than setting this flag according to the tracking variables,
+  !!      change the computation in the tracking variables based on this
+  !!      flag and compute the values on the fly if they are not available
+  !!      in the aux field.
+  logical function mus_tracking_needs_velocity_gradient(scheme)
+    ! --------------------------------------------------------------------------
+    type(mus_scheme_type), intent(in) :: scheme
+    ! --------------------------------------------------------------------------
+    integer :: iTrack, iConfig, iVar
+    ! --------------------------------------------------------------------------
+
+    mus_tracking_needs_velocity_gradient = .false.
+
+    if (.not. scheme%track%control%active) return
+
+    select case (trim(scheme%header%kind))
+    case ('fluid', 'fluid_incompressible')
+      continue
+    case default
+      return
+    end select
+
+    do iTrack = 1, scheme%track%control%nActive
+      iConfig = scheme%track%instance(iTrack)%pntConfig
+      do iVar = 1, size(scheme%track%config(iConfig)%varName)
+        select case(trim(scheme%track%config(iConfig)%varName(iVar)))
+        case ('grad_velocity', 'vorticity', 'q_criterion',             &
+              & 'grad_velocity_phy', 'vorticity_phy', 'q_criterion_phy')
+          mus_tracking_needs_velocity_gradient = .true.
+          return
+        end select
+      end do
+    end do
+  end function mus_tracking_needs_velocity_gradient
+! **************************************************************************** !
+
 end module mus_tracking_module
 ! **************************************************************************** !

source/scheme/mus_scheme_header_module.f90

@@ -127,6 +127,11 @@ module mus_scheme_header_module
     character(len=labelLen) :: relaxation
     !> Variant and additional options for a relaxation
     type(mus_relaxation_header_type) :: relaxHeader
+    !> Controls auxField halo communication.
+    !! Set during tracking initialization:
+    !! fluid/incompressible -> turbulence OR tracked grad-vars
+    !! otherwise remains false.
+    logical :: needAuxHaloComm = .false.
   end type mus_scheme_header_type
 
 contains