diff --git a/source/mus_auxField_module.f90 b/source/mus_auxField_module.f90 index 49d8791..2ba29f9 100644 --- a/source/mus_auxField_module.f90 +++ b/source/mus_auxField_module.f90 @@ -223,7 +223,8 @@ end subroutine mus_init_auxFieldArrays !! PDF initialized by IC subroutine mus_initAuxFieldFluidAndExchange(auxField, state, neigh, nElems, & & nSize, nFields, stencil, varSys, & - & derVarPos, iLevel, general, quantities) + & derVarPos, iLevel, general, & + & quantities, needAuxHaloComm) !--------------------------------------------------------------------------- !> auxilary field array type(mus_auxFieldVar_type), intent(inout) :: auxField @@ -249,6 +250,8 @@ subroutine mus_initAuxFieldFluidAndExchange(auxField, state, neigh, nElems, & type(tem_general_type), intent(in) :: general !> Class that contains pointers to the proper derived quantities functions type(mus_scheme_derived_quantities_type), intent(in) :: quantities + !> Whether auxField halo communication is required + logical, intent(in) :: needAuxHaloComm !--------------------------------------------------------------------------- integer :: iField !--------------------------------------------------------------------------- @@ -269,12 +272,14 @@ subroutine mus_initAuxFieldFluidAndExchange(auxField, state, neigh, nElems, & ! communicate velocity field. Requires for tubulence to compute ShearRate ! from velocity gradient. ! exchange velocity halo on current level - call general%commpattern%exchange_real( & - & send = auxField%sendBuffer, & - & recv = auxField%recvBuffer, & - & state = auxField%val(:), & - & message_flag = iLevel+100, & - & comm = general%proc%comm ) + if (needAuxHaloComm) then + call general%commpattern%exchange_real( & + & send = auxField%sendBuffer, & + & recv = auxField%recvBuffer, & + & state = auxField%val(:), & + & message_flag = iLevel+100, & + & comm = general%proc%comm ) + end if end subroutine mus_initAuxFieldFluidAndExchange ! ************************************************************************** ! @@ -285,7 +290,7 @@ end subroutine mus_initAuxFieldFluidAndExchange !! halos subroutine mus_calcAuxFieldAndExchange(auxField, calcAuxField, state, & & pdfData, nFields, field, globSrc, stencil, varSys, derVarPos, & - & phyConvFac, general, iLevel, minLevel, schemeHeader, quantities) + & phyConvFac, general, iLevel, minLevel, schemeHeader, quantities ) ! -------------------------------------------------------------------- ! !> auxilary field array type(mus_auxFieldVar_type), intent(inout) :: auxField @@ -378,15 +383,17 @@ subroutine mus_calcAuxFieldAndExchange(auxField, calcAuxField, state, & ! communicate velocity field. Requires for tubulence to compute ShearRate ! from velocity gradient. ! exchange velocity halo on current level - call general%commpattern%exchange_real( & - & send = auxField%sendBuffer, & - & recv = auxField%recvBuffer , & - & state = auxField%val(:), & - & message_flag = iLevel+100, & - & comm = general%proc%comm ) + if (schemeHeader%needAuxHaloComm) then + call general%commpattern%exchange_real( & + & send = auxField%sendBuffer, & + & recv = auxField%recvBuffer , & + & state = auxField%val(:), & + & message_flag = iLevel+100, & + & comm = general%proc%comm ) + end if ! communicate ghost halos from coarser - if (iLevel > minLevel) then + if (schemeHeader%needAuxHaloComm .and. iLevel > minLevel) then call general%commpattern%exchange_real( & & send = auxField%sendBufferFromCoarser, & & recv = auxField%recvBufferFromCoarser , & @@ -403,7 +410,8 @@ end subroutine mus_calcAuxFieldAndExchange !! halos subroutine mus_intpAuxFieldCoarserAndExchange(intp, tAuxField, sAuxField, & & tLevelDesc, stencil, iLevel, & - & nAuxScalars, general) + & nAuxScalars, general, & + & needAuxHaloComm) ! -------------------------------------------------------------------- ! !> Interpolation type type(mus_interpolation_type), intent(inout) :: intp @@ -421,6 +429,8 @@ subroutine mus_intpAuxFieldCoarserAndExchange(intp, tAuxField, sAuxField, & integer, intent(in) :: nAuxScalars !> contains commPattern, MPI communicator and simControl type(tem_general_type), intent(in) :: general + !> Whether auxField halo communication is required + logical, intent(in) :: needAuxHaloComm ! -------------------------------------------------------------------- ! call intp%fillMineFromFiner%do_intpArbiVal( & & tLevelDesc = tLevelDesc, & @@ -434,12 +444,14 @@ subroutine mus_intpAuxFieldCoarserAndExchange(intp, tAuxField, sAuxField, & ! exchange velocity halo fromFiner, required to compute velocity ! gradient - call general%commPattern%exchange_real( & - & send = tAuxField%sendBufferFromFiner, & - & recv = tAuxField%recvBufferFromFiner, & - & state = tAuxField%val(:), & - & message_flag = iLevel+300, & - & comm = general%proc%comm ) + if (needAuxHaloComm) then + call general%commPattern%exchange_real( & + & send = tAuxField%sendBufferFromFiner, & + & recv = tAuxField%recvBufferFromFiner, & + & state = tAuxField%val(:), & + & message_flag = iLevel+300, & + & comm = general%proc%comm ) + end if end subroutine mus_intpAuxFieldCoarserAndExchange ! ************************************************************************* ! @@ -449,7 +461,8 @@ end subroutine mus_intpAuxFieldCoarserAndExchange !! halos subroutine mus_intpAuxFieldFinerAndExchange(intp, tAuxField, sAuxField, & & tLevelDesc, stencil, iLevel, & - & nAuxScalars, general) + & nAuxScalars, general, & + & needAuxHaloComm) ! -------------------------------------------------------------------- ! !> Interpolation type type(mus_interpolation_type), intent(inout) :: intp @@ -467,6 +480,8 @@ subroutine mus_intpAuxFieldFinerAndExchange(intp, tAuxField, sAuxField, & integer, intent(in) :: nAuxScalars !> contains commPattern, MPI communicator and simControl type(tem_general_type), intent(in) :: general + !> Whether auxField halo communication is required + logical, intent(in) :: needAuxHaloComm ! -------------------------------------------------------------------- ! integer :: iOrder ! -------------------------------------------------------------------- ! @@ -484,14 +499,15 @@ subroutine mus_intpAuxFieldFinerAndExchange(intp, tAuxField, sAuxField, & ! exchange velocity halo fromFiner, required to compute velocity ! gradient - call general%commPattern%exchange_real( & - & send = tAuxField%sendBufferFromCoarser, & - & recv = tAuxField%recvBufferFromCoarser, & - & state = tAuxField%val(:), & - & message_flag = iLevel+200, & - & comm = general%proc%comm ) + if (needAuxHaloComm) then + call general%commPattern%exchange_real( & + & send = tAuxField%sendBufferFromCoarser, & + & recv = tAuxField%recvBufferFromCoarser, & + & state = tAuxField%val(:), & + & message_flag = iLevel+200, & + & comm = general%proc%comm ) + end if end subroutine mus_intpAuxFieldFinerAndExchange ! ************************************************************************* ! end module mus_auxField_module - diff --git a/source/mus_control_module.f90 b/source/mus_control_module.f90 index 9698933..cde131c 100644 --- a/source/mus_control_module.f90 +++ b/source/mus_control_module.f90 @@ -319,23 +319,23 @@ recursive subroutine do_recursive_multiLevel(me, iLevel) ! and exchange them if turbulence is active call tem_startTimer( timerHandle = mus_timerHandles%aux(iLevel) ) write(logUnit(10), "(A)") 'Calculate auxField' - call mus_calcAuxFieldAndExchange( & - & auxField = me%scheme%auxField(iLevel), & - & calcAuxField = me%scheme%calcAuxField, & - & state = me%scheme%state(iLevel)%val(:, now), & - & pdfData = me%scheme%pdf(iLevel), & - & nFields = me%scheme%nFields, & - & field = me%scheme%field(:), & - & globSrc = me%scheme%globSrc, & - & stencil = me%scheme%layout%fStencil, & - & varSys = me%scheme%varSys, & - & derVarPos = me%scheme%derVarPos, & - & general = me%params%general, & - & phyConvFac = me%params%physics%fac(iLevel), & - & iLevel = iLevel, & - & minLevel = me%geometry%tree%global%minLevel, & - & schemeHeader = me%scheme%header, & - & quantities = me%scheme%layout%quantities ) + call mus_calcAuxFieldAndExchange( & + & auxField = me%scheme%auxField(iLevel), & + & calcAuxField = me%scheme%calcAuxField, & + & state = me%scheme%state(iLevel)%val(:, now), & + & pdfData = me%scheme%pdf(iLevel), & + & nFields = me%scheme%nFields, & + & field = me%scheme%field(:), & + & globSrc = me%scheme%globSrc, & + & stencil = me%scheme%layout%fStencil, & + & varSys = me%scheme%varSys, & + & derVarPos = me%scheme%derVarPos, & + & general = me%params%general, & + & phyConvFac = me%params%physics%fac(iLevel), & + & iLevel = iLevel, & + & minLevel = me%geometry%tree%global%minLevel, & + & schemeHeader = me%scheme%header, & + & quantities = me%scheme%layout%quantities ) if (iLevel < me%geometry%tree%global%maxLevel) then write(logUnit(10), "(A)") 'Interpolate and exchange auxField in ' & @@ -348,7 +348,8 @@ recursive subroutine do_recursive_multiLevel(me, iLevel) & stencil = me%scheme%layout%fStencil, & & iLevel = iLevel, & & nAuxScalars = me%scheme%varSys%nAuxScalars, & - & general = me%params%general ) + & general = me%params%general, & + & needAuxHaloComm = me%scheme%header%needAuxHaloComm ) end if call tem_stopTimer( timerHandle = mus_timerHandles%aux(iLevel) ) diff --git a/source/mus_dynLoadBal_module.f90 b/source/mus_dynLoadBal_module.f90 index f983956..e587fe3 100644 --- a/source/mus_dynLoadBal_module.f90 +++ b/source/mus_dynLoadBal_module.f90 @@ -461,7 +461,8 @@ subroutine mus_reset_aux( scheme, params, geometry) & stencil = scheme%layout%fStencil, & & iLevel = iLevel, & & nAuxScalars = scheme%varSys%nAuxScalars, & - & general = params%general ) + & general = params%general, & + & needAuxHaloComm = scheme%header%needAuxHaloComm ) end if end do diff --git a/source/mus_flow_module.fpp b/source/mus_flow_module.fpp index ff17e49..37ace4f 100644 --- a/source/mus_flow_module.fpp +++ b/source/mus_flow_module.fpp @@ -1704,7 +1704,8 @@ write(dbgUnit(5), *) '' & varSys = scheme%varSys, & & derVarPos = scheme%derVarPos, & & general = general, & - & quantities = scheme%layout%quantities ) + & quantities = scheme%layout%quantities, & + & needAuxHaloComm = scheme%header%needAuxHaloComm ) end do ! Initilialize auxField ghostFromFiner and ghostFromCoarser with @@ -1718,7 +1719,8 @@ write(dbgUnit(5), *) '' & stencil = scheme%layout%fStencil, & & iLevel = iLevel, & & nAuxScalars = scheme%varSys%nAuxScalars, & - & general = general ) + & general = general, & + & needAuxHaloComm = scheme%header%needAuxHaloComm ) end do do iLevel = minLevel+1, maxLevel @@ -1730,7 +1732,8 @@ write(dbgUnit(5), *) '' & stencil = scheme%layout%fStencil, & & iLevel = iLevel, & & nAuxScalars = scheme%varSys%nAuxScalars, & - & general = general ) + & general = general, & + & needAuxHaloComm = scheme%header%needAuxHaloComm ) end do end subroutine mus_initAuxField diff --git a/source/mus_tracking_module.f90 b/source/mus_tracking_module.f90 index 5d86e4d..ac03b47 100644 --- a/source/mus_tracking_module.f90 +++ b/source/mus_tracking_module.f90 @@ -72,6 +72,11 @@ subroutine mus_init_tracker( scheme, geometry, params ) integer :: iTrack, iConfig ! -------------------------------------------------------------------------- + select case (trim(scheme%header%kind)) + case ('fluid', 'fluid_incompressible') + scheme%header%needAuxHaloComm = scheme%field(1)%fieldProp%fluid%turbulence%active + end select + write(dbgUnit(1),*) 'Enter mus_init_tracker' write(dbgUnit(1),*) 'Tracking control active is: ', & & scheme%track%control%active @@ -113,8 +118,59 @@ subroutine mus_init_tracker( scheme, geometry, params ) & solver = params%general%solver, & & varSys = scheme%varSys ) + scheme%header%needAuxHaloComm = scheme%header%needAuxHaloComm & + & .or. mus_tracking_needs_velocity_gradient( & + & scheme = scheme ) + + if (scheme%header%needAuxHaloComm) then + write(logUnit(1),*) 'Auxiliary field halo communication activated.' + else + write(logUnit(1),*) 'Auxiliary field halo communication deactivated.' + end if + end subroutine mus_init_tracker ! **************************************************************************** ! + +! **************************************************************************** ! + !> Check whether any active tracking object requests velocity-gradient based + !! variables that require auxField halo values. + !! + !! TODO rather than setting this flag according to the tracking variables, + !! change the computation in the tracking variables based on this + !! flag and compute the values on the fly if they are not available + !! in the aux field. + logical function mus_tracking_needs_velocity_gradient(scheme) + ! -------------------------------------------------------------------------- + type(mus_scheme_type), intent(in) :: scheme + ! -------------------------------------------------------------------------- + integer :: iTrack, iConfig, iVar + ! -------------------------------------------------------------------------- + + mus_tracking_needs_velocity_gradient = .false. + + if (.not. scheme%track%control%active) return + + select case (trim(scheme%header%kind)) + case ('fluid', 'fluid_incompressible') + continue + case default + return + end select + + do iTrack = 1, scheme%track%control%nActive + iConfig = scheme%track%instance(iTrack)%pntConfig + do iVar = 1, size(scheme%track%config(iConfig)%varName) + select case(trim(scheme%track%config(iConfig)%varName(iVar))) + case ('grad_velocity', 'vorticity', 'q_criterion', & + & 'grad_velocity_phy', 'vorticity_phy', 'q_criterion_phy') + mus_tracking_needs_velocity_gradient = .true. + return + end select + end do + end do + end function mus_tracking_needs_velocity_gradient +! **************************************************************************** ! + end module mus_tracking_module ! **************************************************************************** ! diff --git a/source/scheme/mus_scheme_header_module.f90 b/source/scheme/mus_scheme_header_module.f90 index 30566b3..8f09185 100644 --- a/source/scheme/mus_scheme_header_module.f90 +++ b/source/scheme/mus_scheme_header_module.f90 @@ -127,6 +127,11 @@ module mus_scheme_header_module character(len=labelLen) :: relaxation !> Variant and additional options for a relaxation type(mus_relaxation_header_type) :: relaxHeader + !> Controls auxField halo communication. + !! Set during tracking initialization: + !! fluid/incompressible -> turbulence OR tracked grad-vars + !! otherwise remains false. + logical :: needAuxHaloComm = .false. end type mus_scheme_header_type contains