Skip to content

Conditional aux halo exchange via aux-field flag #55

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
haraldkl merged 7 commits into from
Feb 28, 2026
Merged

Conditional aux halo exchange via aux-field flag #55

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
6247990
Condition auxiliary halo exchange when turbulence enabled
MikeW097 Feb 16, 2026
9e66e00
Add scheme-level needAuxHaloComm flag for aux halo exchanges.
MikeW097 Feb 24, 2026
b695905
flag aux to avoid interface change (corrections based on old base)
MikeW097 Feb 25, 2026
3309ee5
function auxfield_configure_from_tracking for needHaloComm flag setup
MikeW097 Feb 27, 2026
93be170
Some formatting
haraldkl Feb 28, 2026
5599eab
Merge branch 'main' into aux-exchange-conditional-oldbase
haraldkl Feb 28, 2026
6e87f5e
add copyright lines on top of files
MikeW097 Feb 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
161 changes: 129 additions & 32 deletions source/mus_auxField_module.f90
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
! Copyright (c) 2019-2020, 2022 Kannan Masilamani <kannan.masilamani@uni-siegen.de>
! Copyright (c) 2025 Mengyu Wang <m.wang-2@utwente.nl>
!
! Redistribution and use in source and binary forms, with or without
! modification, are permitted provided that the following conditions are met:
Expand Down Expand Up @@ -39,6 +40,8 @@ module mus_auxField_module
& tem_comm_init
use tem_construction_module, only: tem_levelDesc_type
use tem_general_module, only: tem_general_type
use tem_tracking_module, only: tem_tracking_type
use tem_logging_module, only: logUnit

use mus_derVarPos_module, only: mus_derVarPos_type
use mus_scheme_header_module, only: mus_scheme_header_type
Expand All @@ -60,6 +63,7 @@ module mus_auxField_module
public :: mus_calcAuxFieldAndExchange
public :: mus_intpAuxFieldCoarserAndExchange
public :: mus_intpAuxFieldFinerAndExchange
public :: mus_auxField_configure_from_tracking

!> Contains auxiliary field variable values per level and communication
!! buffers
Expand All @@ -85,6 +89,8 @@ module mus_auxField_module
type( tem_communication_type ) :: recvBufferFromCoarser
!> My halos which are ghostFromFiner on remote processes
type( tem_communication_type ) :: recvBufferFromFiner
!> Controls whether auxField halo exchange is required on this level.
logical :: needHaloComm = .false.
end type mus_auxFieldVar_type

abstract interface
Expand Down Expand Up @@ -127,12 +133,13 @@ end subroutine mus_proc_calcAuxField
end interface



contains


! ************************************************************************* !
!> This routine initialize auxField var val array and communication buffers
subroutine mus_init_auxFieldArrays(me, levelDesc, pattern, nSize, nAuxScalars)
subroutine mus_init_auxFieldArrays(me, levelDesc, pattern, nSize, &
& nAuxScalars, needHaloComm )
! --------------------------------------------------------------------- !
!> Auxiliary field variable
type(mus_auxFieldVar_type), intent(out) :: me
Expand All @@ -144,8 +151,14 @@ subroutine mus_init_auxFieldArrays(me, levelDesc, pattern, nSize, nAuxScalars)
integer, intent(in) :: nSize
!> Number of scalars in auxiliary variables
integer, intent(in) :: nAuxScalars
!> Default for halo communication flag
logical, intent(in), optional :: needHaloComm
! --------------------------------------------------------------------- !
! --------------------------------------------------------------------- !
if (present(needHaloComm)) then
me%needHaloComm = needHaloComm
end if

allocate(me%val(nSize * nAuxScalars))
me%val(:) = -1000000.0_rk

Expand Down Expand Up @@ -223,7 +236,8 @@ end subroutine mus_init_auxFieldArrays
!! PDF initialized by IC
subroutine mus_initAuxFieldFluidAndExchange(auxField, state, neigh, nElems, &
& nSize, nFields, stencil, varSys, &
& derVarPos, iLevel, general, quantities)
& derVarPos, iLevel, general, &
& quantities)
!---------------------------------------------------------------------------
!> auxilary field array
type(mus_auxFieldVar_type), intent(inout) :: auxField
Expand Down Expand Up @@ -269,12 +283,14 @@ subroutine mus_initAuxFieldFluidAndExchange(auxField, state, neigh, nElems, &
! communicate velocity field. Requires for tubulence to compute ShearRate
! from velocity gradient.
! exchange velocity halo on current level
call general%commpattern%exchange_real( &
& send = auxField%sendBuffer, &
& recv = auxField%recvBuffer, &
& state = auxField%val(:), &
& message_flag = iLevel+100, &
& comm = general%proc%comm )
if (auxField%needHaloComm) then
call general%commpattern%exchange_real( &
& send = auxField%sendBuffer, &
& recv = auxField%recvBuffer, &
& state = auxField%val(:), &
& message_flag = iLevel+100, &
& comm = general%proc%comm )
end if

end subroutine mus_initAuxFieldFluidAndExchange
! ************************************************************************** !
Expand All @@ -285,7 +301,7 @@ end subroutine mus_initAuxFieldFluidAndExchange
!! halos
subroutine mus_calcAuxFieldAndExchange(auxField, calcAuxField, state, &
& pdfData, nFields, field, globSrc, stencil, varSys, derVarPos, &
& phyConvFac, general, iLevel, minLevel, schemeHeader, quantities)
& phyConvFac, general, iLevel, minLevel, schemeHeader, quantities )
! -------------------------------------------------------------------- !
!> auxilary field array
type(mus_auxFieldVar_type), intent(inout) :: auxField
Expand Down Expand Up @@ -378,15 +394,17 @@ subroutine mus_calcAuxFieldAndExchange(auxField, calcAuxField, state, &
! communicate velocity field. Requires for tubulence to compute ShearRate
! from velocity gradient.
! exchange velocity halo on current level
call general%commpattern%exchange_real( &
& send = auxField%sendBuffer, &
& recv = auxField%recvBuffer , &
& state = auxField%val(:), &
& message_flag = iLevel+100, &
& comm = general%proc%comm )
if (auxField%needHaloComm) then
call general%commpattern%exchange_real( &
& send = auxField%sendBuffer, &
& recv = auxField%recvBuffer , &
& state = auxField%val(:), &
& message_flag = iLevel+100, &
& comm = general%proc%comm )
end if

! communicate ghost halos from coarser
if (iLevel > minLevel) then
if (auxField%needHaloComm .and. iLevel > minLevel) then
call general%commpattern%exchange_real( &
& send = auxField%sendBufferFromCoarser, &
& recv = auxField%recvBufferFromCoarser , &
Expand All @@ -403,7 +421,7 @@ end subroutine mus_calcAuxFieldAndExchange
!! halos
subroutine mus_intpAuxFieldCoarserAndExchange(intp, tAuxField, sAuxField, &
& tLevelDesc, stencil, iLevel, &
& nAuxScalars, general)
& nAuxScalars, general )
! -------------------------------------------------------------------- !
!> Interpolation type
type(mus_interpolation_type), intent(inout) :: intp
Expand Down Expand Up @@ -434,12 +452,14 @@ subroutine mus_intpAuxFieldCoarserAndExchange(intp, tAuxField, sAuxField, &

! exchange velocity halo fromFiner, required to compute velocity
! gradient
call general%commPattern%exchange_real( &
& send = tAuxField%sendBufferFromFiner, &
& recv = tAuxField%recvBufferFromFiner, &
& state = tAuxField%val(:), &
& message_flag = iLevel+300, &
& comm = general%proc%comm )
if (tAuxField%needHaloComm) then
call general%commPattern%exchange_real( &
& send = tAuxField%sendBufferFromFiner, &
& recv = tAuxField%recvBufferFromFiner, &
& state = tAuxField%val(:), &
& message_flag = iLevel+300, &
& comm = general%proc%comm )
end if

end subroutine mus_intpAuxFieldCoarserAndExchange
! ************************************************************************* !
Expand All @@ -449,7 +469,7 @@ end subroutine mus_intpAuxFieldCoarserAndExchange
!! halos
subroutine mus_intpAuxFieldFinerAndExchange(intp, tAuxField, sAuxField, &
& tLevelDesc, stencil, iLevel, &
& nAuxScalars, general)
& nAuxScalars, general )
! -------------------------------------------------------------------- !
!> Interpolation type
type(mus_interpolation_type), intent(inout) :: intp
Expand Down Expand Up @@ -484,14 +504,91 @@ subroutine mus_intpAuxFieldFinerAndExchange(intp, tAuxField, sAuxField, &

! exchange velocity halo fromFiner, required to compute velocity
! gradient
call general%commPattern%exchange_real( &
& send = tAuxField%sendBufferFromCoarser, &
& recv = tAuxField%recvBufferFromCoarser, &
& state = tAuxField%val(:), &
& message_flag = iLevel+200, &
& comm = general%proc%comm )
if (tAuxField%needHaloComm) then
call general%commPattern%exchange_real( &
& send = tAuxField%sendBufferFromCoarser, &
& recv = tAuxField%recvBufferFromCoarser, &
& state = tAuxField%val(:), &
& message_flag = iLevel+200, &
& comm = general%proc%comm )
end if

end subroutine mus_intpAuxFieldFinerAndExchange
! ************************************************************************* !
end module mus_auxField_module


! ************************************************************************* !
!> Configure auxField halo communication once tracking information is known.
!! Uses the initialization default as baseline and only enables additional
!! communication when velocity-gradient tracking variables are requested.
subroutine mus_auxField_configure_from_tracking(auxField, track, schemeKind)
! -------------------------------------------------------------------- !
type(mus_auxFieldVar_type), intent(inout) :: auxField(:)
type(tem_tracking_type), intent(in) :: track
character(len=*), intent(in) :: schemeKind
! -------------------------------------------------------------------- !
integer :: iLevel
logical :: needHaloComm
! -------------------------------------------------------------------- !

if (size(auxField) == 0) return

needHaloComm = auxField(lbound(auxField, 1))%needHaloComm
needHaloComm = needHaloComm &
& .or. aux_tracking_needs_velocity_gradient( &
& track = track, &
& schemeKind = schemeKind )

do iLevel = lbound(auxField, 1), ubound(auxField, 1)
auxField(iLevel)%needHaloComm = needHaloComm
end do

if (needHaloComm) then
write(logUnit(1),*) 'Auxiliary field halo communication activated.'
else
write(logUnit(1),*) 'Auxiliary field halo communication DEACTIVATED.'
end if

end subroutine mus_auxField_configure_from_tracking
! ************************************************************************* !


! ************************************************************************* !
!> Check if active tracking requests velocity-gradient based quantities.
function aux_tracking_needs_velocity_gradient(track, schemeKind) &
& result(needed)
! -------------------------------------------------------------------- !
type(tem_tracking_type), intent(in) :: track
character(len=*), intent(in) :: schemeKind
logical :: needed
! -------------------------------------------------------------------- !
integer :: iTrack, iConfig, iVar
! -------------------------------------------------------------------- !

needed = .false.

if (.not. track%control%active) return

select case (trim(schemeKind))
case ('fluid', 'fluid_incompressible')
continue
case default
return
end select

do iTrack = 1, track%control%nActive
iConfig = track%instance(iTrack)%pntConfig
do iVar = 1, size(track%config(iConfig)%varName)
select case(trim(track%config(iConfig)%varName(iVar)))
case ('grad_velocity', 'vorticity', 'q_criterion', &
& 'grad_velocity_phy', 'vorticity_phy', 'q_criterion_phy')
needed = .true.
return
end select
end do
end do

end function aux_tracking_needs_velocity_gradient
! ************************************************************************* !

end module mus_auxField_module
7 changes: 7 additions & 0 deletions source/mus_aux_module.f90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
! Copyright (c) 2016 Peter Vitt <peter.vitt2@uni-siegen.de>
! Copyright (c) 2016-2017 Raphael Haupt <raphael.haupt@uni-siegen.de>
! Copyright (c) 2022 Gregorio Gerardo Spinelli <gregoriogerardo.spinelli@dlr.de>
! Copyright (c) 2025 Mengyu Wang <m.wang-2@utwente.nl>
!
! Redistribution and use in source and binary forms, with or without
! modification, are permitted provided that the following conditions are met:
Expand Down Expand Up @@ -63,6 +64,7 @@ module mus_aux_module
use mus_turbulence_module, only: mus_turb_updateViscOfTurbWall
use mus_field_module, only: setParameters_multispecies
use mus_tracking_module, only: mus_init_tracker
use mus_auxField_module, only: mus_auxField_configure_from_tracking
use mus_restart_module, only: mus_writeRestart
use mus_timer_module, only: mus_timerHandles
use mus_physics_module, only: mus_physics_type
Expand Down Expand Up @@ -376,6 +378,11 @@ subroutine mus_init_aux( scheme, geometry, params)
& geometry = geometry, &
& params = params )

call mus_auxField_configure_from_tracking( &
& auxField = scheme%auxField, &
& track = scheme%track, &
& schemeKind = scheme%header%kind )

! convergence objects
if ( params%general%simControl%abortCriteria%steady_state ) then
write(logUnit(1),*) 'Initializing convergence...'
Expand Down
Loading