Skip to content

Commit 86a0869

Browse files
nietrastimcassell
nietras
and
timcassell
authored
[build] Add windows-11-arm and ubuntu-24.04-arm to CI run-tests.yaml (dotnet#2760)
* [build] Add windows-11-arm to CI run-tests.yaml * add os to upload name * add os to core name * [build] Add ubuntu-24.04-arm to CI run-tests.yaml * add os to upload name * SingleBenchmarkCanBeExecutedForMultipleRuntimes remove Platform.X64 * BenchmarkCanAllocateMoreThan2Gb try use platform for switching * Split LargeAddressAwareTests. Ignore NoHangs test on full Framework on non-x86. * Skip WasmIsSupported test on Linux Arm. * Skip disassembly diagnoser tests on Windows Arm. * Skip new Mono tests on Windows Arm. --------- Co-authored-by: Tim Cassell <cassell.timothy@gmail.com>
1 parent 3828911 commit 86a0869

File tree

11 files changed

+94
-34
lines changed

11 files changed

+94
-34
lines changed

‎.github/workflows/run-tests.yaml‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,10 @@ concurrency:
1313
jobs:
1414

1515
test-windows-core:
16-
runs-on: windows-latest
16+
strategy:
17+
matrix:
18+
os: [windows-latest, windows-11-arm]
19+
runs-on: ${{ matrix.os }}
1720
steps:
1821
- name: Disable Windows Defender
1922
run: Set-MpPreference -DisableRealtimeMonitoring $true
@@ -31,11 +34,14 @@ jobs:
3134
uses: actions/upload-artifact@v4
3235
if: always()
3336
with:
34-
name: test-windows-core-trx-${{ github.run_id }}
37+
name: test-windows-core-trx-${{ github.run_id }}-${{ matrix.os }}
3538
path: "**/*.trx"
3639

3740
test-windows-full:
38-
runs-on: windows-latest
41+
strategy:
42+
matrix:
43+
os: [windows-latest, windows-11-arm]
44+
runs-on: ${{ matrix.os }}
3945
steps:
4046
- name: Disable Windows Defender
4147
run: Set-MpPreference -DisableRealtimeMonitoring $true
@@ -59,11 +65,14 @@ jobs:
5965
uses: actions/upload-artifact@v4
6066
if: always()
6167
with:
62-
name: test-windows-full-trx-${{ github.run_id }}
68+
name: test-windows-full-trx-${{ github.run_id }}-${{ matrix.os }}
6369
path: "**/*.trx"
6470

6571
test-linux:
66-
runs-on: ubuntu-latest
72+
strategy:
73+
matrix:
74+
os: [ubuntu-latest, ubuntu-24.04-arm]
75+
runs-on: ${{ matrix.os }}
6776
steps:
6877
- uses: actions/checkout@v4
6978
# Set up the environment
@@ -96,7 +105,7 @@ jobs:
96105
uses: actions/upload-artifact@v4
97106
if: always()
98107
with:
99-
name: test-linux-trx-${{ github.run_id }}
108+
name: test-linux-trx-${{ github.run_id }}-${{ matrix.os }}
100109
path: "**/*.trx"
101110

102111
test-macos:

‎src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoser.cs‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,13 @@ public IEnumerable<ValidationError> Validate(ValidationParameters validationPara
119119
var currentPlatform = RuntimeInformation.GetCurrentPlatform();
120120
if (!(currentPlatform is Platform.X64 or Platform.X86 or Platform.Arm64))
121121
{
122-
yield return new ValidationError(true, $"{currentPlatform} is not supported");
122+
yield return new ValidationError(true, $"DisassemblyDiagnoser does not support {currentPlatform}");
123+
yield break;
124+
}
125+
126+
if (currentPlatform == Platform.Arm64 && OsDetector.IsWindows())
127+
{
128+
yield return new ValidationError(true, $"DisassemblyDiagnoser does not support Arm on Windows");
123129
yield break;
124130
}
125131

‎tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
using BenchmarkDotNet.Jobs;
1414
using BenchmarkDotNet.Portability;
1515
using BenchmarkDotNet.Tests.Loggers;
16+
using BenchmarkDotNet.Tests.XUnit;
1617
using BenchmarkDotNet.Toolchains;
1718
using BenchmarkDotNet.Toolchains.CsProj;
1819
using BenchmarkDotNet.Toolchains.InProcess.Emit;
@@ -84,7 +85,7 @@ public void Recursive()
8485
[MethodImpl(MethodImplOptions.NoInlining)] public void Benchmark(bool justAnOverload) { } // we need to test overloads (#562)
8586
}
8687

87-
[Theory]
88+
[TheoryEnvSpecific("Not supported on Windows+Arm", EnvRequirement.NonWindowsArm)]
8889
[MemberData(nameof(GetAllJits), DisableDiscoveryEnumeration = true)]
8990
[Trait(Constants.Category, Constants.BackwardCompatibilityCategory)]
9091
public void CanDisassembleAllMethodCalls(Jit jit, Platform platform, IToolchain toolchain)
@@ -104,7 +105,7 @@ public void CanDisassembleAllMethodCalls(Jit jit, Platform platform, IToolchain
104105
AssertDisassemblyResult(result, $"{nameof(WithCalls.Recursive)}()");
105106
}
106107

107-
[Theory]
108+
[TheoryEnvSpecific("Not supported on Windows+Arm", EnvRequirement.NonWindowsArm)]
108109
[MemberData(nameof(GetAllJits), DisableDiscoveryEnumeration = true)]
109110
[Trait(Constants.Category, Constants.BackwardCompatibilityCategory)]
110111
public void CanDisassembleAllMethodCallsUsingFilters(Jit jit, Platform platform, IToolchain toolchain)
@@ -130,7 +131,7 @@ public void CanDisassembleAllMethodCallsUsingFilters(Jit jit, Platform platform,
130131
public T Create() => new T();
131132
}
132133

133-
[Theory]
134+
[TheoryEnvSpecific("Not supported on Windows+Arm", EnvRequirement.NonWindowsArm)]
134135
[MemberData(nameof(GetAllJits), DisableDiscoveryEnumeration = true)]
135136
[Trait(Constants.Category, Constants.BackwardCompatibilityCategory)]
136137
public void CanDisassembleGenericTypes(Jit jit, Platform platform, IToolchain toolchain)
@@ -151,7 +152,7 @@ public class WithInlineable
151152
[Benchmark] public void JustReturn() { }
152153
}
153154

154-
[Theory]
155+
[TheoryEnvSpecific("Not supported on Windows+Arm", EnvRequirement.NonWindowsArm)]
155156
[MemberData(nameof(GetAllJits), DisableDiscoveryEnumeration = true)]
156157
[Trait(Constants.Category, Constants.BackwardCompatibilityCategory)]
157158
public void CanDisassembleInlinableBenchmarks(Jit jit, Platform platform, IToolchain toolchain)

‎tests/BenchmarkDotNet.IntegrationTests/FailingProcessSpawnTests.cs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,12 @@ public void NoHangs()
2323
_ => Platform.X64
2424
};
2525

26+
if (wrongPlatform == Platform.X64 && RuntimeInformation.IsFullFramework)
27+
{
28+
// It seems full Framework on Arm ignores the platform and simply runs the native platform, causing this test to fail.
29+
return;
30+
}
31+
2632
var invalidPlatformJob = Job.Dry.WithPlatform(wrongPlatform);
2733
var config = CreateSimpleConfig(job: invalidPlatformJob);
2834

‎tests/BenchmarkDotNet.IntegrationTests/LargeAddressAwareTest.cs‎

Lines changed: 40 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
using BenchmarkDotNet.Configs;
66
using BenchmarkDotNet.Environments;
77
using BenchmarkDotNet.Jobs;
8+
using BenchmarkDotNet.Portability;
89
using BenchmarkDotNet.Running;
910
using BenchmarkDotNet.Tests.Loggers;
1011
using BenchmarkDotNet.Tests.XUnit;
@@ -19,35 +20,57 @@ public class LargeAddressAwareTest
1920

2021
public LargeAddressAwareTest(ITestOutputHelper outputHelper) => output = outputHelper;
2122

22-
[FactEnvSpecific("CLR is a valid job only on Windows", EnvRequirement.WindowsOnly)]
23-
public void BenchmarkCanAllocateMoreThan2Gb()
23+
[Fact]
24+
public void BenchmarkCanAllocateMoreThan2Gb_Core()
2425
{
25-
var summary = BenchmarkRunner
26-
.Run<NeedsMoreThan2GB>(
27-
ManualConfig.CreateEmpty()
28-
.AddJob(Job.Dry.WithRuntime(CoreRuntime.Core80).WithPlatform(Platform.X64).WithId("Core"))
29-
.AddJob(Job.Dry.WithRuntime(ClrRuntime.Net462).WithPlatform(Platform.X86).WithGcServer(false).WithLargeAddressAware().WithId("Framework"))
30-
.AddColumnProvider(DefaultColumnProviders.Instance)
31-
.AddLogger(new OutputLogger(output)));
26+
var platform = RuntimeInformation.GetCurrentPlatform();
27+
var config = ManualConfig.CreateEmpty();
28+
// Running 32-bit benchmarks with .Net Core requires passing the path to 32-bit SDK,
29+
// which makes this test more complex than it's worth in CI, so we only test 64-bit.
30+
config.AddJob(Job.Dry.WithRuntime(CoreRuntime.Core80).WithPlatform(platform).WithId(platform.ToString()));
31+
config.AddColumnProvider(DefaultColumnProviders.Instance)
32+
.AddLogger(new OutputLogger(output));
33+
34+
var summary = BenchmarkRunner.Run<NeedsMoreThan2GB>(config);
3235

3336
Assert.True(summary.Reports
3437
.All(report => report.ExecuteResults
3538
.All(executeResult => executeResult.FoundExecutable)));
3639

3740
Assert.True(summary.Reports.All(report => report.AllMeasurements.Any()));
41+
Assert.True(summary.Reports.All(report => report.ExecuteResults.Any()));
42+
Assert.Equal(1, summary.Reports.Count(report => report.BenchmarkCase.Job.Environment.Runtime is CoreRuntime));
3843

39-
Assert.True(summary.Reports
40-
.Single(report => report.BenchmarkCase.Job.Environment.Runtime is ClrRuntime)
41-
.ExecuteResults
42-
.Any());
44+
Assert.Contains(".NET 8.0", summary.AllRuntimes);
45+
}
46+
47+
[FactEnvSpecific("Framework is only on Windows", EnvRequirement.WindowsOnly)]
48+
public void BenchmarkCanAllocateMoreThan2Gb_Framework()
49+
{
50+
var platform = RuntimeInformation.GetCurrentPlatform();
51+
var config = ManualConfig.CreateEmpty();
52+
// Net481 officially only supports x86, x64, and Arm64.
53+
config.AddJob(Job.Dry.WithRuntime(ClrRuntime.Net481).WithPlatform(platform).WithGcServer(false).WithLargeAddressAware().WithId(platform.ToString()));
54+
int jobCount = 1;
55+
if (platform == Platform.X64)
56+
{
57+
++jobCount;
58+
config.AddJob(Job.Dry.WithRuntime(ClrRuntime.Net462).WithPlatform(Platform.X86).WithGcServer(false).WithLargeAddressAware().WithId("X86"));
59+
}
60+
config.AddColumnProvider(DefaultColumnProviders.Instance)
61+
.AddLogger(new OutputLogger(output));
62+
63+
var summary = BenchmarkRunner.Run<NeedsMoreThan2GB>(config);
4364

4465
Assert.True(summary.Reports
45-
.Single(report => report.BenchmarkCase.Job.Environment.Runtime is CoreRuntime)
46-
.ExecuteResults
47-
.Any());
66+
.All(report => report.ExecuteResults
67+
.All(executeResult => executeResult.FoundExecutable)));
68+
69+
Assert.True(summary.Reports.All(report => report.AllMeasurements.Any()));
70+
Assert.True(summary.Reports.All(report => report.ExecuteResults.Any()));
71+
Assert.Equal(jobCount, summary.Reports.Count(report => report.BenchmarkCase.Job.Environment.Runtime is ClrRuntime));
4872

4973
Assert.Contains(".NET Framework", summary.AllRuntimes);
50-
Assert.Contains(".NET 8.0", summary.AllRuntimes);
5174
}
5275
}
5376

‎tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ public void MemoryDiagnoserSupportsNativeAOT()
8080
MemoryDiagnoserIsAccurate(NativeAotToolchain.Net80);
8181
}
8282

83-
[FactEnvSpecific("We don't want to test MonoVM twice (for .NET Framework 4.6.2 and .NET 8.0)", EnvRequirement.DotNetCoreOnly)]
83+
[FactEnvSpecific("We don't want to test MonoVM twice (for .NET Framework 4.6.2 and .NET 8.0), and it's not supported on Windows+Arm", [EnvRequirement.DotNetCoreOnly, EnvRequirement.NonWindowsArm])]
8484
public void MemoryDiagnoserSupportsModernMono()
8585
{
8686
MemoryDiagnoserIsAccurate(MonoToolchain.Mono80);

‎tests/BenchmarkDotNet.IntegrationTests/MonoTests.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ public class MonoTests : BenchmarkTestExecutor
1414
{
1515
public MonoTests(ITestOutputHelper output) : base(output) { }
1616

17-
[FactEnvSpecific("UseMonoRuntime option is available in .NET Core only starting from .NET 6", EnvRequirement.DotNetCoreOnly)]
17+
[FactEnvSpecific("UseMonoRuntime option is available in .NET Core only starting from .NET 6, and it's not supported on Windows+Arm", [EnvRequirement.DotNetCoreOnly, EnvRequirement.NonWindowsArm])]
1818
public void Mono80IsSupported()
1919
{
2020
var logger = new OutputLogger(Output);

‎tests/BenchmarkDotNet.IntegrationTests/MultipleRuntimesTest.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ public void SingleBenchmarkCanBeExecutedForMultipleRuntimes()
2929
var summary = BenchmarkRunner
3030
.Run<C>(
3131
ManualConfig.CreateEmpty()
32-
.AddJob(Job.Dry.WithRuntime(CoreRuntime.Core80).WithPlatform(Platform.X64).WithId("Core"))
32+
.AddJob(Job.Dry.WithRuntime(CoreRuntime.Core80).WithId("Core"))
3333
.AddJob(Job.Dry.WithRuntime(ClrRuntime.Net462).WithId("Framework"))
3434
.AddColumnProvider(DefaultColumnProviders.Instance)
3535
.AddLogger(new OutputLogger(output)));

‎tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
using System.Linq;
44
using BenchmarkDotNet.Attributes;
55
using BenchmarkDotNet.Configs;
6+
using BenchmarkDotNet.Detectors;
67
using BenchmarkDotNet.Environments;
78
using BenchmarkDotNet.IntegrationTests.Diagnosers;
89
using BenchmarkDotNet.Jobs;
@@ -46,12 +47,24 @@ private ManualConfig GetConfig()
4647
[FactEnvSpecific("WASM is only supported on Unix", EnvRequirement.NonWindows)]
4748
public void WasmIsSupported()
4849
{
50+
// Test fails on Linux non-x64.
51+
if (OsDetector.IsLinux() && RuntimeInformation.GetCurrentPlatform() != Platform.X64)
52+
{
53+
return;
54+
}
55+
4956
CanExecute<WasmBenchmark>(GetConfig());
5057
}
5158

5259
[FactEnvSpecific("WASM is only supported on Unix", EnvRequirement.NonWindows)]
5360
public void WasmSupportsInProcessDiagnosers()
5461
{
62+
// Test fails on Linux non-x64.
63+
if (OsDetector.IsLinux() && RuntimeInformation.GetCurrentPlatform() != Platform.X64)
64+
{
65+
return;
66+
}
67+
5568
var diagnoser = new MockInProcessDiagnoser1(BenchmarkDotNet.Diagnosers.RunMode.NoOverhead);
5669
var config = GetConfig().AddDiagnoser(diagnoser);
5770

‎tests/BenchmarkDotNet.Tests/XUnit/EnvRequirement.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ public enum EnvRequirement
44
{
55
WindowsOnly,
66
NonWindows,
7+
NonWindowsArm,
78
NonLinux,
89
FullFrameworkOnly,
910
NonFullFramework,

0 commit comments

Comments
 (0)