[build] Add windows-11-arm and ubuntu-24.04-arm to CI run-tests.yaml

.github/workflows/run-tests.yaml

@@ -12,7 +12,10 @@ concurrency:
 jobs:
 
   test-windows-core:
-    runs-on: windows-latest
+    strategy:
+      matrix:
+        os: [windows-latest, windows-11-arm]
+    runs-on: ${{ matrix.os }}
     steps:
       - name: Disable Windows Defender
         run: Set-MpPreference -DisableRealtimeMonitoring $true
@@ -30,11 +33,14 @@ jobs:
         uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: test-windows-core-trx-${{ github.run_id }}
+          name: test-windows-core-trx-${{ github.run_id }}-${{ matrix.os }}
           path: "**/*.trx"
 
   test-windows-full:
-    runs-on: windows-latest
+    strategy:
+      matrix:
+        os: [windows-latest, windows-11-arm]
+    runs-on: ${{ matrix.os }}
     steps:
       - name: Disable Windows Defender
         run: Set-MpPreference -DisableRealtimeMonitoring $true
@@ -52,11 +58,14 @@ jobs:
         uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: test-windows-full-trx-${{ github.run_id }}
+          name: test-windows-full-trx-${{ github.run_id }}-${{ matrix.os }}
           path: "**/*.trx"
 
   test-linux:
-    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        os: [ubuntu-latest, ubuntu-24.04-arm]
+    runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
       # Set up the environment
@@ -87,7 +96,7 @@ jobs:
         uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: test-linux-trx-${{ github.run_id }}
+          name: test-linux-trx-${{ github.run_id }}-${{ matrix.os }}
           path: "**/*.trx"
 
   test-macos:

tests/BenchmarkDotNet.IntegrationTests/FailingProcessSpawnTests.cs

@@ -23,6 +23,12 @@ public void NoHangs()
                 _ => Platform.X64
             };
 
+            if (wrongPlatform == Platform.X64 && RuntimeInformation.IsFullFramework)
+            {
+                // It seems full Framework on Arm ignores the platform and simply runs the native platform, causing this test to fail.
+                return;
+            }
+
             var invalidPlatformJob = Job.Dry.WithPlatform(wrongPlatform);
             var config = CreateSimpleConfig(job: invalidPlatformJob);
 

tests/BenchmarkDotNet.IntegrationTests/LargeAddressAwareTest.cs

@@ -5,6 +5,7 @@
 using BenchmarkDotNet.Configs;
 using BenchmarkDotNet.Environments;
 using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Portability;
 using BenchmarkDotNet.Running;
 using BenchmarkDotNet.Tests.Loggers;
 using BenchmarkDotNet.Tests.XUnit;
@@ -19,35 +20,57 @@ public class LargeAddressAwareTest
 
         public LargeAddressAwareTest(ITestOutputHelper outputHelper) => output = outputHelper;
 
-        [FactEnvSpecific("CLR is a valid job only on Windows", EnvRequirement.WindowsOnly)]
-        public void BenchmarkCanAllocateMoreThan2Gb()
+        [Fact]
+        public void BenchmarkCanAllocateMoreThan2Gb_Core()
         {
-            var summary = BenchmarkRunner
-                .Run<NeedsMoreThan2GB>(
-                    ManualConfig.CreateEmpty()
-                        .AddJob(Job.Dry.WithRuntime(CoreRuntime.Core80).WithPlatform(Platform.X64).WithId("Core"))
-                        .AddJob(Job.Dry.WithRuntime(ClrRuntime.Net462).WithPlatform(Platform.X86).WithGcServer(false).WithLargeAddressAware().WithId("Framework"))
-                        .AddColumnProvider(DefaultColumnProviders.Instance)
-                        .AddLogger(new OutputLogger(output)));
+            var platform = RuntimeInformation.GetCurrentPlatform();
+            var config = ManualConfig.CreateEmpty();
+            // Running 32-bit benchmarks with .Net Core requires passing the path to 32-bit SDK,
+            // which makes this test more complex than it's worth in CI, so we only test 64-bit.
+            config.AddJob(Job.Dry.WithRuntime(CoreRuntime.Core80).WithPlatform(platform).WithId(platform.ToString()));
+            config.AddColumnProvider(DefaultColumnProviders.Instance)
+                  .AddLogger(new OutputLogger(output));
+
+            var summary = BenchmarkRunner.Run<NeedsMoreThan2GB>(config);
 
             Assert.True(summary.Reports
                 .All(report => report.ExecuteResults
                 .All(executeResult => executeResult.FoundExecutable)));
 
             Assert.True(summary.Reports.All(report => report.AllMeasurements.Any()));
+            Assert.True(summary.Reports.All(report => report.ExecuteResults.Any()));
+            Assert.Equal(1, summary.Reports.Count(report => report.BenchmarkCase.Job.Environment.Runtime is CoreRuntime));
 
-            Assert.True(summary.Reports
-                .Single(report => report.BenchmarkCase.Job.Environment.Runtime is ClrRuntime)
-                .ExecuteResults
-                .Any());
+            Assert.Contains(".NET 8.0", summary.AllRuntimes);
+        }
+
+        [FactEnvSpecific("Framework is only on Windows", EnvRequirement.WindowsOnly)]
+        public void BenchmarkCanAllocateMoreThan2Gb_Framework()
+        {
+            var platform = RuntimeInformation.GetCurrentPlatform();
+            var config = ManualConfig.CreateEmpty();
+            // Net481 officially only supports x86, x64, and Arm64.
+            config.AddJob(Job.Dry.WithRuntime(ClrRuntime.Net481).WithPlatform(platform).WithGcServer(false).WithLargeAddressAware().WithId(platform.ToString()));
+            int jobCount = 1;
+            if (platform == Platform.X64)
+            {
+                ++jobCount;
+                config.AddJob(Job.Dry.WithRuntime(ClrRuntime.Net462).WithPlatform(Platform.X86).WithGcServer(false).WithLargeAddressAware().WithId("X86"));
+            }
+            config.AddColumnProvider(DefaultColumnProviders.Instance)
+                  .AddLogger(new OutputLogger(output));
+
+            var summary = BenchmarkRunner.Run<NeedsMoreThan2GB>(config);
 
             Assert.True(summary.Reports
-                .Single(report => report.BenchmarkCase.Job.Environment.Runtime is CoreRuntime)
-                .ExecuteResults
-                .Any());
+                .All(report => report.ExecuteResults
+                .All(executeResult => executeResult.FoundExecutable)));
+
+            Assert.True(summary.Reports.All(report => report.AllMeasurements.Any()));
+            Assert.True(summary.Reports.All(report => report.ExecuteResults.Any()));
+            Assert.Equal(jobCount, summary.Reports.Count(report => report.BenchmarkCase.Job.Environment.Runtime is ClrRuntime));
 
             Assert.Contains(".NET Framework", summary.AllRuntimes);
-            Assert.Contains(".NET 8.0", summary.AllRuntimes);
         }
     }
 

tests/BenchmarkDotNet.IntegrationTests/MultipleRuntimesTest.cs

@@ -29,7 +29,7 @@ public void SingleBenchmarkCanBeExecutedForMultipleRuntimes()
             var summary = BenchmarkRunner
                 .Run<C>(
                     ManualConfig.CreateEmpty()
-                        .AddJob(Job.Dry.WithRuntime(CoreRuntime.Core80).WithPlatform(Platform.X64).WithId("Core"))
+                        .AddJob(Job.Dry.WithRuntime(CoreRuntime.Core80).WithId("Core"))
                         .AddJob(Job.Dry.WithRuntime(ClrRuntime.Net462).WithId("Framework"))
                         .AddColumnProvider(DefaultColumnProviders.Instance)
                         .AddLogger(new OutputLogger(output)));

tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs

@@ -2,6 +2,7 @@
 using System.IO;
 using BenchmarkDotNet.Attributes;
 using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Detectors;
 using BenchmarkDotNet.Environments;
 using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Portability;
@@ -26,6 +27,12 @@ public class WasmTests(ITestOutputHelper output) : BenchmarkTestExecutor(output)
         [FactEnvSpecific("WASM is only supported on Unix", EnvRequirement.NonWindows)]
         public void WasmIsSupported()
         {
+            // Test fails on Linux non-x64.
+            if (OsDetector.IsLinux() && RuntimeInformation.GetCurrentPlatform() != Platform.X64)
+            {
+                return;
+            }
+
             var dotnetVersion = "net8.0";
             var logger = new OutputLogger(Output);
             var netCoreAppSettings = new NetCoreAppSettings(dotnetVersion, null, "Wasm");