From dfa143c1530ce5b8231e0404d91253b67953ae2c Mon Sep 17 00:00:00 2001 From: Daniel Languiller Date: Tue, 16 Sep 2025 16:36:48 +1000 Subject: [PATCH 01/25] temp: add diag statement --- .azure-pipelines/PipelineSteps/test-steps.ps1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/PipelineSteps/test-steps.ps1 b/.azure-pipelines/PipelineSteps/test-steps.ps1 index 1540fcb8618f..70ab037629e3 100644 --- a/.azure-pipelines/PipelineSteps/test-steps.ps1 +++ b/.azure-pipelines/PipelineSteps/test-steps.ps1 @@ -39,9 +39,10 @@ if ($IsLinux) { Write-Host -ForegroundColor Yellow "Detected Linux agent - applying memory tuning for tests" $env:DOTNET_gcServer = "0" $env:DOTNET_gcHeapCount = "2" + $buildArgs += ";RunConfiguration.MaxCpuCount=2" } -dotnet msbuild $buildProjPath /t:Test $buildArgs +dotnet msbuild $buildProjPath /t:Test $buildArgs /v:diag Write-Host -ForegroundColor DarkGreen "-------------------- End testing ... --------------------`n`n`n`n`n" From 956b2d6fd85f3cb2d96bf7db236dfdb540d473ec Mon Sep 17 00:00:00 2001 From: Daniel Languiller Date: Tue, 16 Sep 2025 17:05:59 +1000 Subject: [PATCH 02/25] temp: try with -m flag --- .azure-pipelines/PipelineSteps/test-steps.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/PipelineSteps/test-steps.ps1 b/.azure-pipelines/PipelineSteps/test-steps.ps1 index 70ab037629e3..dac948a85d38 100644 --- a/.azure-pipelines/PipelineSteps/test-steps.ps1 +++ b/.azure-pipelines/PipelineSteps/test-steps.ps1 @@ -39,10 +39,10 @@ if ($IsLinux) { Write-Host -ForegroundColor Yellow "Detected Linux agent - applying memory tuning for tests" $env:DOTNET_gcServer = "0" $env:DOTNET_gcHeapCount = "2" - $buildArgs += ";RunConfiguration.MaxCpuCount=2" + $env:DOTNET_MSBUILD_CLI_OPTIONS = "-m:2" } -dotnet msbuild $buildProjPath /t:Test $buildArgs /v:diag +dotnet msbuild $buildProjPath /t:Test $buildArgs Write-Host -ForegroundColor DarkGreen "-------------------- End testing ... --------------------`n`n`n`n`n" From a700efa46051d477ddd8503680efadcd32b7c77e Mon Sep 17 00:00:00 2001 From: Daniel Languiller Date: Tue, 16 Sep 2025 17:44:12 +1000 Subject: [PATCH 03/25] temp: reduce m flag to 1 --- .azure-pipelines/PipelineSteps/test-steps.ps1 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/PipelineSteps/test-steps.ps1 b/.azure-pipelines/PipelineSteps/test-steps.ps1 index dac948a85d38..26d96e937836 100644 --- a/.azure-pipelines/PipelineSteps/test-steps.ps1 +++ b/.azure-pipelines/PipelineSteps/test-steps.ps1 @@ -36,10 +36,11 @@ $buildArgs = "/p:Configuration=$Configuration;TestFramework=$TestFramework" if ($IsLinux) { # Applying various parallelism/heap restriction behaviors to prevent OOM issues on Ubuntu 24.04 - Write-Host -ForegroundColor Yellow "Detected Linux agent - applying memory tuning for tests" + Write-Host -ForegroundColor Yellow "Detected Linux agent. Akpplying memory tuning for tests" $env:DOTNET_gcServer = "0" $env:DOTNET_gcHeapCount = "2" - $env:DOTNET_MSBUILD_CLI_OPTIONS = "-m:2" + $env:DOTNET_MSBUILD_CLI_OPTIONS = "-m:1" + # $buildArgs += ";RunConfiguration.MaxCpuCount=2" } dotnet msbuild $buildProjPath /t:Test $buildArgs From 0182528fab18961e05c26c67df93dd09d6de3001 Mon Sep 17 00:00:00 2001 From: Daniel Languiller Date: Tue, 16 Sep 2025 18:30:48 +1000 Subject: [PATCH 04/25] fix: change typo --- .azure-pipelines/PipelineSteps/test-steps.ps1 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.azure-pipelines/PipelineSteps/test-steps.ps1 b/.azure-pipelines/PipelineSteps/test-steps.ps1 index 26d96e937836..e506a7774421 100644 --- a/.azure-pipelines/PipelineSteps/test-steps.ps1 +++ b/.azure-pipelines/PipelineSteps/test-steps.ps1 @@ -36,11 +36,10 @@ $buildArgs = "/p:Configuration=$Configuration;TestFramework=$TestFramework" if ($IsLinux) { # Applying various parallelism/heap restriction behaviors to prevent OOM issues on Ubuntu 24.04 - Write-Host -ForegroundColor Yellow "Detected Linux agent. Akpplying memory tuning for tests" + Write-Host -ForegroundColor Yellow "Detected Linux agent. Applying memory tuning for tests" $env:DOTNET_gcServer = "0" $env:DOTNET_gcHeapCount = "2" $env:DOTNET_MSBUILD_CLI_OPTIONS = "-m:1" - # $buildArgs += ";RunConfiguration.MaxCpuCount=2" } dotnet msbuild $buildProjPath /t:Test $buildArgs From d99330d06e4abad7de39bddba398da330de23684 Mon Sep 17 00:00:00 2001 From: Daniel Languiller Date: Tue, 16 Sep 2025 18:36:27 +1000 Subject: [PATCH 05/25] fix: change arg name per copilot suggestion --- .azure-pipelines/PipelineSteps/test-steps.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/PipelineSteps/test-steps.ps1 b/.azure-pipelines/PipelineSteps/test-steps.ps1 index e506a7774421..6337536cfcfb 100644 --- a/.azure-pipelines/PipelineSteps/test-steps.ps1 +++ b/.azure-pipelines/PipelineSteps/test-steps.ps1 @@ -39,7 +39,7 @@ if ($IsLinux) { Write-Host -ForegroundColor Yellow "Detected Linux agent. Applying memory tuning for tests" $env:DOTNET_gcServer = "0" $env:DOTNET_gcHeapCount = "2" - $env:DOTNET_MSBUILD_CLI_OPTIONS = "-m:1" + $env:MSBUILDCLIOPTIONSOVERRIDE = "-m:1" } dotnet msbuild $buildProjPath /t:Test $buildArgs From 9dbcbf4672d4e28f1a23cf3c270e0d849ccce177 Mon Sep 17 00:00:00 2001 From: Daniel Languiller Date: Wed, 17 Sep 2025 08:27:59 +1000 Subject: [PATCH 06/25] revert: using DOTNET_MSBUILD_CLI_OPTIONS --- .azure-pipelines/PipelineSteps/test-steps.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/PipelineSteps/test-steps.ps1 b/.azure-pipelines/PipelineSteps/test-steps.ps1 index 6337536cfcfb..e506a7774421 100644 --- a/.azure-pipelines/PipelineSteps/test-steps.ps1 +++ b/.azure-pipelines/PipelineSteps/test-steps.ps1 @@ -39,7 +39,7 @@ if ($IsLinux) { Write-Host -ForegroundColor Yellow "Detected Linux agent. Applying memory tuning for tests" $env:DOTNET_gcServer = "0" $env:DOTNET_gcHeapCount = "2" - $env:MSBUILDCLIOPTIONSOVERRIDE = "-m:1" + $env:DOTNET_MSBUILD_CLI_OPTIONS = "-m:1" } dotnet msbuild $buildProjPath /t:Test $buildArgs From 7ff3fac227619c6cb9da5ca9f775a75ea8ea5fa3 Mon Sep 17 00:00:00 2001 From: Daniel Languiller Date: Wed, 17 Sep 2025 09:48:02 +1000 Subject: [PATCH 07/25] fix: trying with GCheaphardlimitpercent --- .azure-pipelines/PipelineSteps/test-steps.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/.azure-pipelines/PipelineSteps/test-steps.ps1 b/.azure-pipelines/PipelineSteps/test-steps.ps1 index e506a7774421..d8f4207080e1 100644 --- a/.azure-pipelines/PipelineSteps/test-steps.ps1 +++ b/.azure-pipelines/PipelineSteps/test-steps.ps1 @@ -40,6 +40,7 @@ if ($IsLinux) { $env:DOTNET_gcServer = "0" $env:DOTNET_gcHeapCount = "2" $env:DOTNET_MSBUILD_CLI_OPTIONS = "-m:1" + $env:DOTNET_GCHeapHardLimitPercent = "75" } dotnet msbuild $buildProjPath /t:Test $buildArgs From e64c12d7d7c33dd5e86fba7da6a3fbdfb6466b4c Mon Sep 17 00:00:00 2001 From: Daniel Languiller Date: Wed, 17 Sep 2025 10:05:59 +1000 Subject: [PATCH 08/25] fix: add more memory tuning params --- .azure-pipelines/PipelineSteps/test-steps.ps1 | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/.azure-pipelines/PipelineSteps/test-steps.ps1 b/.azure-pipelines/PipelineSteps/test-steps.ps1 index d8f4207080e1..323c3eda12bd 100644 --- a/.azure-pipelines/PipelineSteps/test-steps.ps1 +++ b/.azure-pipelines/PipelineSteps/test-steps.ps1 @@ -35,15 +35,20 @@ $buildProjPath = Join-Path $RepoRoot 'build.proj' $buildArgs = "/p:Configuration=$Configuration;TestFramework=$TestFramework" if ($IsLinux) { - # Applying various parallelism/heap restriction behaviors to prevent OOM issues on Ubuntu 24.04 Write-Host -ForegroundColor Yellow "Detected Linux agent. Applying memory tuning for tests" - $env:DOTNET_gcServer = "0" - $env:DOTNET_gcHeapCount = "2" - $env:DOTNET_MSBUILD_CLI_OPTIONS = "-m:1" - $env:DOTNET_GCHeapHardLimitPercent = "75" + + # GC and MSBuild tuning + $env:DOTNET_gcServer = "0" # Use workstation GC + $env:DOTNET_gcHeapCount = "2" # Limit GC heap count + $env:DOTNET_MSBUILD_CLI_OPTIONS = "-m:1" # Disable MSBuild parallelism + $env:MSBUILDDISABLENODEREUSE = "1" # Prevent node reuse + $env:DOTNET_GCHeapHardLimitPercent = "75" # Heap limit as percent + $env:DOTNET_GCHeapHardLimit = "1610612736" # ~1.5 GB hard limit + $env:DOTNET_GCHeapAffinitizeMask = "0x3" # Bind heaps to CPUs } -dotnet msbuild $buildProjPath /t:Test $buildArgs +# @TODO: remove /v:diag from final PR +dotnet msbuild $buildProjPath /t:Test $buildArgs /v:diag Write-Host -ForegroundColor DarkGreen "-------------------- End testing ... --------------------`n`n`n`n`n" From f791c045eab7043a18a6961fc7e79a19248f5b46 Mon Sep 17 00:00:00 2001 From: Daniel Languiller Date: Wed, 17 Sep 2025 11:02:37 +1000 Subject: [PATCH 09/25] fix: try with hardcap memory limit --- .azure-pipelines/PipelineSteps/test-steps.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/.azure-pipelines/PipelineSteps/test-steps.ps1 b/.azure-pipelines/PipelineSteps/test-steps.ps1 index 323c3eda12bd..def48f036879 100644 --- a/.azure-pipelines/PipelineSteps/test-steps.ps1 +++ b/.azure-pipelines/PipelineSteps/test-steps.ps1 @@ -37,6 +37,7 @@ $buildArgs = "/p:Configuration=$Configuration;TestFramework=$TestFramework" if ($IsLinux) { Write-Host -ForegroundColor Yellow "Detected Linux agent. Applying memory tuning for tests" + bash -c "ulimit -v 2097152" # GC and MSBuild tuning $env:DOTNET_gcServer = "0" # Use workstation GC $env:DOTNET_gcHeapCount = "2" # Limit GC heap count From 1af9ea706131b1b5f0da2c0232103c6465cd025c Mon Sep 17 00:00:00 2001 From: Daniel Languiller Date: Wed, 17 Sep 2025 11:03:32 +1000 Subject: [PATCH 10/25] temp: add GC logging --- .azure-pipelines/PipelineSteps/test-steps.ps1 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.azure-pipelines/PipelineSteps/test-steps.ps1 b/.azure-pipelines/PipelineSteps/test-steps.ps1 index def48f036879..3096c9c762c0 100644 --- a/.azure-pipelines/PipelineSteps/test-steps.ps1 +++ b/.azure-pipelines/PipelineSteps/test-steps.ps1 @@ -46,6 +46,12 @@ if ($IsLinux) { $env:DOTNET_GCHeapHardLimitPercent = "75" # Heap limit as percent $env:DOTNET_GCHeapHardLimit = "1610612736" # ~1.5 GB hard limit $env:DOTNET_GCHeapAffinitizeMask = "0x3" # Bind heaps to CPUs + + # GC logging + $env:COMPlus_LogEnable = "1" + $env:COMPlus_LogLevel = "6" + $env:COMPlus_LogFacility = "0x0001" + } # @TODO: remove /v:diag from final PR From ef275926cd426b2b7ec4ac9f330e1ef0bb572cd5 Mon Sep 17 00:00:00 2001 From: Daniel Languiller Date: Wed, 17 Sep 2025 12:01:07 +1000 Subject: [PATCH 11/25] temp: add bash memory logs --- .azure-pipelines/PipelineSteps/test-steps.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/PipelineSteps/test-steps.ps1 b/.azure-pipelines/PipelineSteps/test-steps.ps1 index 3096c9c762c0..beffeb798c3c 100644 --- a/.azure-pipelines/PipelineSteps/test-steps.ps1 +++ b/.azure-pipelines/PipelineSteps/test-steps.ps1 @@ -51,7 +51,7 @@ if ($IsLinux) { $env:COMPlus_LogEnable = "1" $env:COMPlus_LogLevel = "6" $env:COMPlus_LogFacility = "0x0001" - + bash -c "while true; do date; free -h; sleep 10; done &" } # @TODO: remove /v:diag from final PR From bf80b0b9c5fea2eddb5602662a445d60546826e9 Mon Sep 17 00:00:00 2001 From: Daniel Languiller Date: Wed, 17 Sep 2025 12:44:30 +1000 Subject: [PATCH 12/25] temp: attempting to limit bash shell memory --- .azure-pipelines/PipelineSteps/test-steps.ps1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.azure-pipelines/PipelineSteps/test-steps.ps1 b/.azure-pipelines/PipelineSteps/test-steps.ps1 index beffeb798c3c..05d6b2b2c657 100644 --- a/.azure-pipelines/PipelineSteps/test-steps.ps1 +++ b/.azure-pipelines/PipelineSteps/test-steps.ps1 @@ -37,7 +37,6 @@ $buildArgs = "/p:Configuration=$Configuration;TestFramework=$TestFramework" if ($IsLinux) { Write-Host -ForegroundColor Yellow "Detected Linux agent. Applying memory tuning for tests" - bash -c "ulimit -v 2097152" # GC and MSBuild tuning $env:DOTNET_gcServer = "0" # Use workstation GC $env:DOTNET_gcHeapCount = "2" # Limit GC heap count @@ -46,16 +45,17 @@ if ($IsLinux) { $env:DOTNET_GCHeapHardLimitPercent = "75" # Heap limit as percent $env:DOTNET_GCHeapHardLimit = "1610612736" # ~1.5 GB hard limit $env:DOTNET_GCHeapAffinitizeMask = "0x3" # Bind heaps to CPUs - + # GC logging $env:COMPlus_LogEnable = "1" $env:COMPlus_LogLevel = "6" $env:COMPlus_LogFacility = "0x0001" bash -c "while true; do date; free -h; sleep 10; done &" + bash -c "ulimit -v 2097152; dotnet msbuild $buildProjPath /t:Test $buildArgs" +} else { + dotnet msbuild $buildProjPath /t:Test $buildArgs } -# @TODO: remove /v:diag from final PR -dotnet msbuild $buildProjPath /t:Test $buildArgs /v:diag Write-Host -ForegroundColor DarkGreen "-------------------- End testing ... --------------------`n`n`n`n`n" From 784589ecbe8b9389d93324898b4f86df89bb5a48 Mon Sep 17 00:00:00 2001 From: Daniel Languiller Date: Wed, 17 Sep 2025 15:08:22 +1000 Subject: [PATCH 13/25] temp: trying with 2 processes and higher virtual memory --- .azure-pipelines/PipelineSteps/test-steps.ps1 | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.azure-pipelines/PipelineSteps/test-steps.ps1 b/.azure-pipelines/PipelineSteps/test-steps.ps1 index 05d6b2b2c657..c995535d0b4d 100644 --- a/.azure-pipelines/PipelineSteps/test-steps.ps1 +++ b/.azure-pipelines/PipelineSteps/test-steps.ps1 @@ -40,18 +40,17 @@ if ($IsLinux) { # GC and MSBuild tuning $env:DOTNET_gcServer = "0" # Use workstation GC $env:DOTNET_gcHeapCount = "2" # Limit GC heap count - $env:DOTNET_MSBUILD_CLI_OPTIONS = "-m:1" # Disable MSBuild parallelism + $env:DOTNET_MSBUILD_CLI_OPTIONS = "-m:2" # Disable MSBuild parallelism $env:MSBUILDDISABLENODEREUSE = "1" # Prevent node reuse - $env:DOTNET_GCHeapHardLimitPercent = "75" # Heap limit as percent - $env:DOTNET_GCHeapHardLimit = "1610612736" # ~1.5 GB hard limit - $env:DOTNET_GCHeapAffinitizeMask = "0x3" # Bind heaps to CPUs + $env:DOTNET_GCHeapHardLimit = "5368709120" + $env:DOTNET_GCHeapAffinitizeMask = "0x3" - # GC logging + # @TODO: remove before merging: GC logging $env:COMPlus_LogEnable = "1" $env:COMPlus_LogLevel = "6" $env:COMPlus_LogFacility = "0x0001" bash -c "while true; do date; free -h; sleep 10; done &" - bash -c "ulimit -v 2097152; dotnet msbuild $buildProjPath /t:Test $buildArgs" + bash -c "ulimit -v 6291456; dotnet msbuild $buildProjPath /t:Test $buildArgs" } else { dotnet msbuild $buildProjPath /t:Test $buildArgs } From 4b0b751f7da8b70d2095b814e3cd3bc0f93bd4b0 Mon Sep 17 00:00:00 2001 From: Daniel Languiller Date: Wed, 17 Sep 2025 16:01:17 +1000 Subject: [PATCH 14/25] temp: remove mem profile bash command as was causing the pipeline step to hang --- .azure-pipelines/PipelineSteps/test-steps.ps1 | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/PipelineSteps/test-steps.ps1 b/.azure-pipelines/PipelineSteps/test-steps.ps1 index c995535d0b4d..3fe4b9353d92 100644 --- a/.azure-pipelines/PipelineSteps/test-steps.ps1 +++ b/.azure-pipelines/PipelineSteps/test-steps.ps1 @@ -45,11 +45,14 @@ if ($IsLinux) { $env:DOTNET_GCHeapHardLimit = "5368709120" $env:DOTNET_GCHeapAffinitizeMask = "0x3" - # @TODO: remove before merging: GC logging + # @TODO: remove before merging: GC logging /// $env:COMPlus_LogEnable = "1" $env:COMPlus_LogLevel = "6" $env:COMPlus_LogFacility = "0x0001" - bash -c "while true; do date; free -h; sleep 10; done &" + # /// + # @TODO: remove before merging: memory output: /// + # bash -c "while true; do date; free -h; sleep 10; done &" + # /// bash -c "ulimit -v 6291456; dotnet msbuild $buildProjPath /t:Test $buildArgs" } else { dotnet msbuild $buildProjPath /t:Test $buildArgs From 83ef9aa5255fa140106b3b40daef69eff9c2a6dc Mon Sep 17 00:00:00 2001 From: Daniel Languiller Date: Wed, 17 Sep 2025 16:33:08 +1000 Subject: [PATCH 15/25] temp: reverting to m:1 as GC heap fails to init process --- .azure-pipelines/PipelineSteps/test-steps.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/PipelineSteps/test-steps.ps1 b/.azure-pipelines/PipelineSteps/test-steps.ps1 index 3fe4b9353d92..6afa768880cd 100644 --- a/.azure-pipelines/PipelineSteps/test-steps.ps1 +++ b/.azure-pipelines/PipelineSteps/test-steps.ps1 @@ -40,7 +40,7 @@ if ($IsLinux) { # GC and MSBuild tuning $env:DOTNET_gcServer = "0" # Use workstation GC $env:DOTNET_gcHeapCount = "2" # Limit GC heap count - $env:DOTNET_MSBUILD_CLI_OPTIONS = "-m:2" # Disable MSBuild parallelism + $env:DOTNET_MSBUILD_CLI_OPTIONS = "-m:1" # Disable MSBuild parallelism $env:MSBUILDDISABLENODEREUSE = "1" # Prevent node reuse $env:DOTNET_GCHeapHardLimit = "5368709120" $env:DOTNET_GCHeapAffinitizeMask = "0x3" From e285d581a4d3bbe12b3034abf409ce24b34aadea Mon Sep 17 00:00:00 2001 From: Daniel Languiller Date: Wed, 17 Sep 2025 19:05:45 +1000 Subject: [PATCH 16/25] limit heapcount to 1 and reduce heap hard limit --- .azure-pipelines/PipelineSteps/test-steps.ps1 | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.azure-pipelines/PipelineSteps/test-steps.ps1 b/.azure-pipelines/PipelineSteps/test-steps.ps1 index 6afa768880cd..b9f0a1910652 100644 --- a/.azure-pipelines/PipelineSteps/test-steps.ps1 +++ b/.azure-pipelines/PipelineSteps/test-steps.ps1 @@ -35,25 +35,25 @@ $buildProjPath = Join-Path $RepoRoot 'build.proj' $buildArgs = "/p:Configuration=$Configuration;TestFramework=$TestFramework" if ($IsLinux) { - Write-Host -ForegroundColor Yellow "Detected Linux agent. Applying memory tuning for tests" + Write-Host "Detected Linux agent. Applying memory tuning for tests" # GC and MSBuild tuning - $env:DOTNET_gcServer = "0" # Use workstation GC - $env:DOTNET_gcHeapCount = "2" # Limit GC heap count - $env:DOTNET_MSBUILD_CLI_OPTIONS = "-m:1" # Disable MSBuild parallelism - $env:MSBUILDDISABLENODEREUSE = "1" # Prevent node reuse - $env:DOTNET_GCHeapHardLimit = "5368709120" - $env:DOTNET_GCHeapAffinitizeMask = "0x3" - - # @TODO: remove before merging: GC logging /// + $env:DOTNET_gcServer = "0" # Workstation GC + $env:DOTNET_gcHeapCount = "1" # Only 1 heap + $env:DOTNET_MSBUILD_CLI_OPTIONS = "-m:1" # Single MSBuild node + $env:MSBUILDDISABLENODEREUSE = "1" # Prevent node reuse + + # GC heap: give enough room for initialization (~3 GB) + $env:DOTNET_GCHeapHardLimit = "3221225472" # 3 GB + $env:DOTNET_GCHeapAffinitizeMask = "0x1" # Only 1 CPU + + # Optional GC logging $env:COMPlus_LogEnable = "1" $env:COMPlus_LogLevel = "6" $env:COMPlus_LogFacility = "0x0001" - # /// - # @TODO: remove before merging: memory output: /// - # bash -c "while true; do date; free -h; sleep 10; done &" - # /// - bash -c "ulimit -v 6291456; dotnet msbuild $buildProjPath /t:Test $buildArgs" + + # Limit process memory to slightly more than heap (~3.5–4 GB) + bash -c "ulimit -v 4194304; dotnet msbuild $buildProjPath /t:Test $buildArgs" } else { dotnet msbuild $buildProjPath /t:Test $buildArgs } From 4861d5ecdfb0a474108c0052fa9a01164ba4abdd Mon Sep 17 00:00:00 2001 From: Daniel Languiller Date: Wed, 17 Sep 2025 19:25:11 +1000 Subject: [PATCH 17/25] remove gcheap params --- .azure-pipelines/PipelineSteps/test-steps.ps1 | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.azure-pipelines/PipelineSteps/test-steps.ps1 b/.azure-pipelines/PipelineSteps/test-steps.ps1 index b9f0a1910652..61b74f9ce211 100644 --- a/.azure-pipelines/PipelineSteps/test-steps.ps1 +++ b/.azure-pipelines/PipelineSteps/test-steps.ps1 @@ -43,15 +43,6 @@ if ($IsLinux) { $env:DOTNET_MSBUILD_CLI_OPTIONS = "-m:1" # Single MSBuild node $env:MSBUILDDISABLENODEREUSE = "1" # Prevent node reuse - # GC heap: give enough room for initialization (~3 GB) - $env:DOTNET_GCHeapHardLimit = "3221225472" # 3 GB - $env:DOTNET_GCHeapAffinitizeMask = "0x1" # Only 1 CPU - - # Optional GC logging - $env:COMPlus_LogEnable = "1" - $env:COMPlus_LogLevel = "6" - $env:COMPlus_LogFacility = "0x0001" - # Limit process memory to slightly more than heap (~3.5–4 GB) bash -c "ulimit -v 4194304; dotnet msbuild $buildProjPath /t:Test $buildArgs" } else { From f49bca2db13e5a521efb83504051153a48a7ac52 Mon Sep 17 00:00:00 2001 From: Daniel Languiller Date: Wed, 17 Sep 2025 19:46:59 +1000 Subject: [PATCH 18/25] increase ulimit to 5gb --- .azure-pipelines/PipelineSteps/test-steps.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/PipelineSteps/test-steps.ps1 b/.azure-pipelines/PipelineSteps/test-steps.ps1 index 61b74f9ce211..1ea3f104f010 100644 --- a/.azure-pipelines/PipelineSteps/test-steps.ps1 +++ b/.azure-pipelines/PipelineSteps/test-steps.ps1 @@ -43,8 +43,8 @@ if ($IsLinux) { $env:DOTNET_MSBUILD_CLI_OPTIONS = "-m:1" # Single MSBuild node $env:MSBUILDDISABLENODEREUSE = "1" # Prevent node reuse - # Limit process memory to slightly more than heap (~3.5–4 GB) - bash -c "ulimit -v 4194304; dotnet msbuild $buildProjPath /t:Test $buildArgs" + # If this run fails, try to limit GCheap: $env:DOTNET_GCHeapHardLimit = "4026531840" # ~3.75 GB heap limit + bash -c "ulimit -v 5505024; dotnet msbuild $buildProjPath /t:Test $buildArgs" } else { dotnet msbuild $buildProjPath /t:Test $buildArgs } From 5ed780ee26e4150516339ddcb16c6ca61348fb17 Mon Sep 17 00:00:00 2001 From: Daniel Languiller Date: Wed, 17 Sep 2025 20:13:08 +1000 Subject: [PATCH 19/25] add gcheap limit to be below ulimit --- .azure-pipelines/PipelineSteps/test-steps.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/PipelineSteps/test-steps.ps1 b/.azure-pipelines/PipelineSteps/test-steps.ps1 index 1ea3f104f010..fcb115ac1315 100644 --- a/.azure-pipelines/PipelineSteps/test-steps.ps1 +++ b/.azure-pipelines/PipelineSteps/test-steps.ps1 @@ -42,8 +42,8 @@ if ($IsLinux) { $env:DOTNET_gcHeapCount = "1" # Only 1 heap $env:DOTNET_MSBUILD_CLI_OPTIONS = "-m:1" # Single MSBuild node $env:MSBUILDDISABLENODEREUSE = "1" # Prevent node reuse + $env:DOTNET_GCHeapHardLimit = "4026531840" # ~3.75 GB - # If this run fails, try to limit GCheap: $env:DOTNET_GCHeapHardLimit = "4026531840" # ~3.75 GB heap limit bash -c "ulimit -v 5505024; dotnet msbuild $buildProjPath /t:Test $buildArgs" } else { dotnet msbuild $buildProjPath /t:Test $buildArgs From cf0ba52d37e76ac6949a21af5d5bafa2aab5cd92 Mon Sep 17 00:00:00 2001 From: Daniel Languiller Date: Wed, 17 Sep 2025 20:34:55 +1000 Subject: [PATCH 20/25] increase ulimit, reduce gc --- .azure-pipelines/PipelineSteps/test-steps.ps1 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/PipelineSteps/test-steps.ps1 b/.azure-pipelines/PipelineSteps/test-steps.ps1 index fcb115ac1315..d15b8a0431fc 100644 --- a/.azure-pipelines/PipelineSteps/test-steps.ps1 +++ b/.azure-pipelines/PipelineSteps/test-steps.ps1 @@ -42,9 +42,10 @@ if ($IsLinux) { $env:DOTNET_gcHeapCount = "1" # Only 1 heap $env:DOTNET_MSBUILD_CLI_OPTIONS = "-m:1" # Single MSBuild node $env:MSBUILDDISABLENODEREUSE = "1" # Prevent node reuse - $env:DOTNET_GCHeapHardLimit = "4026531840" # ~3.75 GB + $env:DOTNET_GCHeapHardLimit = "3489660928" - bash -c "ulimit -v 5505024; dotnet msbuild $buildProjPath /t:Test $buildArgs" + Write-Host "GCHeapHardLimit = $env:DOTNET_GCHeapHardLimit" + bash -c "ulimit -v 6291456; dotnet msbuild $buildProjPath /t:Test $buildArgs" } else { dotnet msbuild $buildProjPath /t:Test $buildArgs } From abe403452e45b6055a24194719f7e0dcc2184616 Mon Sep 17 00:00:00 2001 From: Daniel Languiller Date: Wed, 17 Sep 2025 21:02:40 +1000 Subject: [PATCH 21/25] remove gcheap and other params --- .azure-pipelines/PipelineSteps/test-steps.ps1 | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.azure-pipelines/PipelineSteps/test-steps.ps1 b/.azure-pipelines/PipelineSteps/test-steps.ps1 index d15b8a0431fc..798d24618e3e 100644 --- a/.azure-pipelines/PipelineSteps/test-steps.ps1 +++ b/.azure-pipelines/PipelineSteps/test-steps.ps1 @@ -37,14 +37,11 @@ $buildArgs = "/p:Configuration=$Configuration;TestFramework=$TestFramework" if ($IsLinux) { Write-Host "Detected Linux agent. Applying memory tuning for tests" - # GC and MSBuild tuning $env:DOTNET_gcServer = "0" # Workstation GC - $env:DOTNET_gcHeapCount = "1" # Only 1 heap $env:DOTNET_MSBUILD_CLI_OPTIONS = "-m:1" # Single MSBuild node - $env:MSBUILDDISABLENODEREUSE = "1" # Prevent node reuse - $env:DOTNET_GCHeapHardLimit = "3489660928" + $env:MSBUILDDISABLENODEREUSE = "1" # Prevent node reuse - Write-Host "GCHeapHardLimit = $env:DOTNET_GCHeapHardLimit" + # Limit total process memory via ulimit (6 GB here) bash -c "ulimit -v 6291456; dotnet msbuild $buildProjPath /t:Test $buildArgs" } else { dotnet msbuild $buildProjPath /t:Test $buildArgs From f3c0e3154ac5240f9af4c48ad2c11c3e7f061067 Mon Sep 17 00:00:00 2001 From: Daniel Languiller Date: Wed, 17 Sep 2025 21:20:48 +1000 Subject: [PATCH 22/25] with diagnostic steps --- .azure-pipelines/PipelineSteps/test-steps.ps1 | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.azure-pipelines/PipelineSteps/test-steps.ps1 b/.azure-pipelines/PipelineSteps/test-steps.ps1 index 798d24618e3e..64f3ee4969cc 100644 --- a/.azure-pipelines/PipelineSteps/test-steps.ps1 +++ b/.azure-pipelines/PipelineSteps/test-steps.ps1 @@ -41,6 +41,24 @@ if ($IsLinux) { $env:DOTNET_MSBUILD_CLI_OPTIONS = "-m:1" # Single MSBuild node $env:MSBUILDDISABLENODEREUSE = "1" # Prevent node reuse + # ---- DIAGNOSTIC LOGGING ---- + Write-Host "---- DIAGNOSTIC: Testhost / CoreCLR Memory Check ----" + + Write-Host "`n[Top 10 memory-consuming processes]" + bash -c "ps -eo pid,ppid,cmd,%mem,%cpu --sort=-%mem | head -10" + + Write-Host "`n[All dotnet testhost processes]" + bash -c "ps -ef | grep -i testhost | grep -v grep" + + Write-Host "`n[Free / total memory snapshot]" + bash -c "free -h" + + Write-Host "`n[Virtual memory limits]" + bash -c "ulimit -a" + + Write-Host "---- END DIAGNOSTIC ----`n" + # ---------------------------- + # Limit total process memory via ulimit (6 GB here) bash -c "ulimit -v 6291456; dotnet msbuild $buildProjPath /t:Test $buildArgs" } else { From 700bca9be1fea20724d7f27e8d8286aacd270e7e Mon Sep 17 00:00:00 2001 From: Daniel Languiller Date: Thu, 18 Sep 2025 08:51:37 +1000 Subject: [PATCH 23/25] test: remove gcserver settings and raise ulimit --- .azure-pipelines/PipelineSteps/test-steps.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.azure-pipelines/PipelineSteps/test-steps.ps1 b/.azure-pipelines/PipelineSteps/test-steps.ps1 index 64f3ee4969cc..e76a8e010c7a 100644 --- a/.azure-pipelines/PipelineSteps/test-steps.ps1 +++ b/.azure-pipelines/PipelineSteps/test-steps.ps1 @@ -37,9 +37,9 @@ $buildArgs = "/p:Configuration=$Configuration;TestFramework=$TestFramework" if ($IsLinux) { Write-Host "Detected Linux agent. Applying memory tuning for tests" - $env:DOTNET_gcServer = "0" # Workstation GC + # $env:DOTNET_gcServer = "0" # Workstation GC $env:DOTNET_MSBUILD_CLI_OPTIONS = "-m:1" # Single MSBuild node - $env:MSBUILDDISABLENODEREUSE = "1" # Prevent node reuse + # $env:MSBUILDDISABLENODEREUSE = "1" # Prevent node reuse # ---- DIAGNOSTIC LOGGING ---- Write-Host "---- DIAGNOSTIC: Testhost / CoreCLR Memory Check ----" @@ -60,7 +60,7 @@ if ($IsLinux) { # ---------------------------- # Limit total process memory via ulimit (6 GB here) - bash -c "ulimit -v 6291456; dotnet msbuild $buildProjPath /t:Test $buildArgs" + bash -c "ulimit -v 7340032; dotnet msbuild $buildProjPath /t:Test $buildArgs" } else { dotnet msbuild $buildProjPath /t:Test $buildArgs } From 867c84bb989fd2aeaf959b8a09c9b8dfdb3b2d5f Mon Sep 17 00:00:00 2001 From: Daniel Languiller Date: Thu, 18 Sep 2025 09:18:59 +1000 Subject: [PATCH 24/25] trying with only -m:1 flag --- .azure-pipelines/PipelineSteps/test-steps.ps1 | 21 ++----------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/.azure-pipelines/PipelineSteps/test-steps.ps1 b/.azure-pipelines/PipelineSteps/test-steps.ps1 index e76a8e010c7a..f81ce349120b 100644 --- a/.azure-pipelines/PipelineSteps/test-steps.ps1 +++ b/.azure-pipelines/PipelineSteps/test-steps.ps1 @@ -41,26 +41,9 @@ if ($IsLinux) { $env:DOTNET_MSBUILD_CLI_OPTIONS = "-m:1" # Single MSBuild node # $env:MSBUILDDISABLENODEREUSE = "1" # Prevent node reuse - # ---- DIAGNOSTIC LOGGING ---- - Write-Host "---- DIAGNOSTIC: Testhost / CoreCLR Memory Check ----" - - Write-Host "`n[Top 10 memory-consuming processes]" - bash -c "ps -eo pid,ppid,cmd,%mem,%cpu --sort=-%mem | head -10" - - Write-Host "`n[All dotnet testhost processes]" - bash -c "ps -ef | grep -i testhost | grep -v grep" - - Write-Host "`n[Free / total memory snapshot]" - bash -c "free -h" - - Write-Host "`n[Virtual memory limits]" - bash -c "ulimit -a" - - Write-Host "---- END DIAGNOSTIC ----`n" - # ---------------------------- - # Limit total process memory via ulimit (6 GB here) - bash -c "ulimit -v 7340032; dotnet msbuild $buildProjPath /t:Test $buildArgs" + # bash -c "ulimit -v 7340032; dotnet msbuild $buildProjPath /t:Test $buildArgs" + dotnet msbuild $buildProjPath /t:Test $buildArgs } else { dotnet msbuild $buildProjPath /t:Test $buildArgs } From d6e268077e0f5e376df8a910ea5a98399ada673d Mon Sep 17 00:00:00 2001 From: Daniel Languiller Date: Thu, 18 Sep 2025 13:45:18 +1000 Subject: [PATCH 25/25] temp: add diagnostic logging to see free disk space --- .azure-pipelines/PipelineSteps/test-steps.ps1 | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.azure-pipelines/PipelineSteps/test-steps.ps1 b/.azure-pipelines/PipelineSteps/test-steps.ps1 index f81ce349120b..89aed888c49f 100644 --- a/.azure-pipelines/PipelineSteps/test-steps.ps1 +++ b/.azure-pipelines/PipelineSteps/test-steps.ps1 @@ -37,12 +37,17 @@ $buildArgs = "/p:Configuration=$Configuration;TestFramework=$TestFramework" if ($IsLinux) { Write-Host "Detected Linux agent. Applying memory tuning for tests" - # $env:DOTNET_gcServer = "0" # Workstation GC - $env:DOTNET_MSBUILD_CLI_OPTIONS = "-m:1" # Single MSBuild node - # $env:MSBUILDDISABLENODEREUSE = "1" # Prevent node reuse + # ---- DIAGNOSTIC LOGGING ---- + Write-Host "=== Disk layout (lsblk) ===" + bash -c "lsblk -o NAME,SIZE,TYPE,MOUNTPOINT" + + Write-Host "`n=== Filesystem usage (df -h) ===" + bash -c "df -h" + + Write-Host "`n=== Memory snapshot (free -h) ===" + bash -c "free -h" + # ---------------------------- - # Limit total process memory via ulimit (6 GB here) - # bash -c "ulimit -v 7340032; dotnet msbuild $buildProjPath /t:Test $buildArgs" dotnet msbuild $buildProjPath /t:Test $buildArgs } else { dotnet msbuild $buildProjPath /t:Test $buildArgs