From a9d8ab0be670dbde416b4f580e6a6b275604ecd9 Mon Sep 17 00:00:00 2001 From: Youssef Victor Date: Fri, 21 Nov 2025 18:04:11 +0100 Subject: [PATCH 1/6] [dotnet test][MTP] Avoid surfacing `MissingTestSessionEnd` when the exit code is already non-success --- .../dotnet/Commands/Test/MTP/TestApplication.cs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/Cli/dotnet/Commands/Test/MTP/TestApplication.cs b/src/Cli/dotnet/Commands/Test/MTP/TestApplication.cs index 16ca1bb218e7..ee68aefb0c5e 100644 --- a/src/Cli/dotnet/Commands/Test/MTP/TestApplication.cs +++ b/src/Cli/dotnet/Commands/Test/MTP/TestApplication.cs @@ -61,14 +61,20 @@ public async Task RunAsync() var outputAndError = await Task.WhenAll(stdOutTask, stdErrTask); await process.WaitForExitAsync(); - _handler.OnTestProcessExited(process.ExitCode, outputAndError[0], outputAndError[1]); - - if (_handler.HasMismatchingTestSessionEventCount()) + var exitCode = process.ExitCode; + _handler.OnTestProcessExited(exitCode, outputAndError[0], outputAndError[1]); + + // This condition is to prevent considering the test app as successful when we didn't receive test session end. + // We don't produce the exception if the exit code is already non-zero to avoid surfacing this exception when there is already a known failure. + // For example, if hangdump timeout is reached, the process will be killed and we will have mismatching count. + // Or if there is a crash (e.g, Environment.FailFast), etc. + // So this is only a safe guard to avoid passing the test run if Environment.Exit(0) is called in one of the tests for example. + if (exitCode != 0 && _handler.HasMismatchingTestSessionEventCount()) { throw new InvalidOperationException(CliCommandStrings.MissingTestSessionEnd); } - return process.ExitCode; + return exitCode; } finally { From 5c2775f17d4ba2c82c2c8a36d458ea1e302dbb94 Mon Sep 17 00:00:00 2001 From: Youssef Victor Date: Mon, 24 Nov 2025 18:12:07 +0100 Subject: [PATCH 2/6] Fix typo --- src/Cli/dotnet/Commands/Test/MTP/TestApplication.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Cli/dotnet/Commands/Test/MTP/TestApplication.cs b/src/Cli/dotnet/Commands/Test/MTP/TestApplication.cs index ee68aefb0c5e..9708c9246017 100644 --- a/src/Cli/dotnet/Commands/Test/MTP/TestApplication.cs +++ b/src/Cli/dotnet/Commands/Test/MTP/TestApplication.cs @@ -69,7 +69,7 @@ public async Task RunAsync() // For example, if hangdump timeout is reached, the process will be killed and we will have mismatching count. // Or if there is a crash (e.g, Environment.FailFast), etc. // So this is only a safe guard to avoid passing the test run if Environment.Exit(0) is called in one of the tests for example. - if (exitCode != 0 && _handler.HasMismatchingTestSessionEventCount()) + if (exitCode == 0 && _handler.HasMismatchingTestSessionEventCount()) { throw new InvalidOperationException(CliCommandStrings.MissingTestSessionEnd); } From fa63dfc92e42f7b349041b2e8c2827d3ad8cea32 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Mon, 24 Nov 2025 19:06:45 +0100 Subject: [PATCH 3/6] Add test --- .../TestProject1/Program.cs | 47 +++++++++++++++++++ .../TestProject1/TestProject1.csproj | 17 +++++++ .../TestProjectMTPCrashNonZero.sln | 34 ++++++++++++++ .../TestProjectMTPCrashNonZero/global.json | 5 ++ .../Test/GivenDotnetTestBuildsAndRunsTests.cs | 29 +++++++++++- 5 files changed, 131 insertions(+), 1 deletion(-) create mode 100644 test/TestAssets/TestProjects/TestProjectMTPCrashNonZero/TestProject1/Program.cs create mode 100644 test/TestAssets/TestProjects/TestProjectMTPCrashNonZero/TestProject1/TestProject1.csproj create mode 100644 test/TestAssets/TestProjects/TestProjectMTPCrashNonZero/TestProjectMTPCrashNonZero.sln create mode 100644 test/TestAssets/TestProjects/TestProjectMTPCrashNonZero/global.json diff --git a/test/TestAssets/TestProjects/TestProjectMTPCrashNonZero/TestProject1/Program.cs b/test/TestAssets/TestProjects/TestProjectMTPCrashNonZero/TestProject1/Program.cs new file mode 100644 index 000000000000..e46a91171e49 --- /dev/null +++ b/test/TestAssets/TestProjects/TestProjectMTPCrashNonZero/TestProject1/Program.cs @@ -0,0 +1,47 @@ +using Microsoft.Testing.Platform.Builder; +using Microsoft.Testing.Platform.Capabilities.TestFramework; +using Microsoft.Testing.Platform.Extensions.Messages; +using Microsoft.Testing.Platform.Extensions.TestFramework; + +var testApplicationBuilder = await TestApplication.CreateBuilderAsync(args); + +testApplicationBuilder.RegisterTestFramework(_ => new TestFrameworkCapabilities(), (_, __) => new DummyTestAdapter()); + +using var testApplication = await testApplicationBuilder.BuildAsync(); +return await testApplication.RunAsync(); + +public class DummyTestAdapter : ITestFramework, IDataProducer +{ + public string Uid => nameof(DummyTestAdapter); + + public string Version => "2.0.0"; + + public string DisplayName => nameof(DummyTestAdapter); + + public string Description => nameof(DummyTestAdapter); + + public Task IsEnabledAsync() => Task.FromResult(true); + + public Type[] DataTypesProduced => [typeof(TestNodeUpdateMessage)]; + public Task CreateTestSessionAsync(CreateTestSessionContext context) + => Task.FromResult(new CreateTestSessionResult() { IsSuccess = true }); + + public Task CloseTestSessionAsync(CloseTestSessionContext context) + => Task.FromResult(new CloseTestSessionResult() { IsSuccess = true }); + + public async Task ExecuteRequestAsync(ExecuteRequestContext context) + { + await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage(context.Request.Session.SessionUid, new TestNode() + { + Uid = $"Test1", + DisplayName = $"Test1", + Properties = new PropertyBag(PassedTestNodeStateProperty.CachedInstance), + })); + + await Task.Delay(1000); + + Environment.Exit(1313); + + context.Complete(); + } +} \ No newline at end of file diff --git a/test/TestAssets/TestProjects/TestProjectMTPCrashNonZero/TestProject1/TestProject1.csproj b/test/TestAssets/TestProjects/TestProjectMTPCrashNonZero/TestProject1/TestProject1.csproj new file mode 100644 index 000000000000..9857918867b1 --- /dev/null +++ b/test/TestAssets/TestProjects/TestProjectMTPCrashNonZero/TestProject1/TestProject1.csproj @@ -0,0 +1,17 @@ + + + + + $(CurrentTargetFramework) + Exe + + enable + enable + + false + + + + + + diff --git a/test/TestAssets/TestProjects/TestProjectMTPCrashNonZero/TestProjectMTPCrashNonZero.sln b/test/TestAssets/TestProjects/TestProjectMTPCrashNonZero/TestProjectMTPCrashNonZero.sln new file mode 100644 index 000000000000..aaee5bd7dc36 --- /dev/null +++ b/test/TestAssets/TestProjects/TestProjectMTPCrashNonZero/TestProjectMTPCrashNonZero.sln @@ -0,0 +1,34 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.0.31903.59 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestProject1", "TestProject1\TestProject1.csproj", "{3E834ED1-92D9-4454-BBB4-B1A2463E5726}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Debug|x64.ActiveCfg = Debug|Any CPU + {3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Debug|x64.Build.0 = Debug|Any CPU + {3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Debug|x86.ActiveCfg = Debug|Any CPU + {3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Debug|x86.Build.0 = Debug|Any CPU + {3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Release|Any CPU.Build.0 = Release|Any CPU + {3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Release|x64.ActiveCfg = Release|Any CPU + {3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Release|x64.Build.0 = Release|Any CPU + {3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Release|x86.ActiveCfg = Release|Any CPU + {3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Release|x86.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/test/TestAssets/TestProjects/TestProjectMTPCrashNonZero/global.json b/test/TestAssets/TestProjects/TestProjectMTPCrashNonZero/global.json new file mode 100644 index 000000000000..9009caf0ba8f --- /dev/null +++ b/test/TestAssets/TestProjects/TestProjectMTPCrashNonZero/global.json @@ -0,0 +1,5 @@ +{ + "test": { + "runner": "Microsoft.Testing.Platform" + } +} diff --git a/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestBuildsAndRunsTests.cs b/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestBuildsAndRunsTests.cs index 23855c3cb442..40f3668b4671 100644 --- a/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestBuildsAndRunsTests.cs +++ b/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestBuildsAndRunsTests.cs @@ -442,7 +442,6 @@ public void RunMTPSolutionWithMinimumExpectedTests(string value, int expectedExi [Fact] public void RunMTPProjectThatCrashesWithExitCodeZero_ShouldFail() { - // The solution has two test projects. Each reports 5 tests. So, total 10 tests. TestAsset testInstance = _testAssetsManager.CopyTestAsset("TestProjectMTPCrash", Guid.NewGuid().ToString()) .WithSource(); @@ -477,6 +476,34 @@ at Microsoft.DotNet.Cli.Commands.Test.TestApplicationActionQueue.Read(BuildOptio } } + [Fact] + public void RunMTPProjectThatCrashesWithExitCodeNonZero_ShouldFail_WithSameExitCode() + { + TestAsset testInstance = _testAssetsManager.CopyTestAsset("TestProjectMTPCrashNonZero", Guid.NewGuid().ToString()) + .WithSource(); + + CommandResult result = new DotnetTestCommand(Log, disableNewOutput: false) + .WithWorkingDirectory(testInstance.Path) + .Execute(); + + // The test asset exits with Environment.Exit(1313); + result.ExitCode.Should().Be(1313); + if (!TestContext.IsLocalized()) + { + result.StdErr.Should().NotContain("A test session start event was received without a corresponding test session end"); + + // TODO: It's much better to introduce a new kind of "summary" indicating + // that the test app exited with zero exit code before sending test session end event + result.StdOut.Should().Contain("Test run summary: Passed!") + .And.Contain("total: 1") + .And.Contain("succeeded: 1") + .And.Contain("failed: 0") + .And.Contain("skipped: 0"); + + result.StdOut.Contains("Test run completed with non-success exit code: 1313 (see: https://aka.ms/testingplatform/exitcodes)"); + } + } + [Theory] [InlineData(TestingConstants.Debug)] [InlineData(TestingConstants.Release)] From b678d5e5e4454f7b6afde194ee6d3242ce43f911 Mon Sep 17 00:00:00 2001 From: Youssef Victor Date: Tue, 25 Nov 2025 09:02:37 +0100 Subject: [PATCH 4/6] Fix test --- .../CommandTests/Test/GivenDotnetTestBuildsAndRunsTests.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestBuildsAndRunsTests.cs b/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestBuildsAndRunsTests.cs index 40f3668b4671..59ad59866305 100644 --- a/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestBuildsAndRunsTests.cs +++ b/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestBuildsAndRunsTests.cs @@ -494,7 +494,8 @@ public void RunMTPProjectThatCrashesWithExitCodeNonZero_ShouldFail_WithSameExitC // TODO: It's much better to introduce a new kind of "summary" indicating // that the test app exited with zero exit code before sending test session end event - result.StdOut.Should().Contain("Test run summary: Passed!") + result.StdOut.Should().Contain("Test run summary: Failed!") + .And.Contain("error: 1") .And.Contain("total: 1") .And.Contain("succeeded: 1") .And.Contain("failed: 0") From d221446dd3ef2687ff99bfa1e926307c560cd493 Mon Sep 17 00:00:00 2001 From: Youssef Victor Date: Wed, 26 Nov 2025 22:36:14 +0100 Subject: [PATCH 5/6] Fix --- .../TestProjectMTPCrashNonZero/TestProject1/Program.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/TestAssets/TestProjects/TestProjectMTPCrashNonZero/TestProject1/Program.cs b/test/TestAssets/TestProjects/TestProjectMTPCrashNonZero/TestProject1/Program.cs index e46a91171e49..29e5063ab469 100644 --- a/test/TestAssets/TestProjects/TestProjectMTPCrashNonZero/TestProject1/Program.cs +++ b/test/TestAssets/TestProjects/TestProjectMTPCrashNonZero/TestProject1/Program.cs @@ -40,8 +40,8 @@ public async Task ExecuteRequestAsync(ExecuteRequestContext context) await Task.Delay(1000); - Environment.Exit(1313); + Environment.Exit(47); context.Complete(); } -} \ No newline at end of file +} From d9d1633e1f91ed4bbe5a63af1cb709760831b445 Mon Sep 17 00:00:00 2001 From: Youssef Victor Date: Wed, 26 Nov 2025 22:36:34 +0100 Subject: [PATCH 6/6] Fix test --- .../CommandTests/Test/GivenDotnetTestBuildsAndRunsTests.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestBuildsAndRunsTests.cs b/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestBuildsAndRunsTests.cs index 59ad59866305..41d9ae0fbe10 100644 --- a/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestBuildsAndRunsTests.cs +++ b/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestBuildsAndRunsTests.cs @@ -486,8 +486,8 @@ public void RunMTPProjectThatCrashesWithExitCodeNonZero_ShouldFail_WithSameExitC .WithWorkingDirectory(testInstance.Path) .Execute(); - // The test asset exits with Environment.Exit(1313); - result.ExitCode.Should().Be(1313); + // The test asset exits with Environment.Exit(47); + result.ExitCode.Should().Be(47); if (!TestContext.IsLocalized()) { result.StdErr.Should().NotContain("A test session start event was received without a corresponding test session end"); @@ -501,7 +501,7 @@ public void RunMTPProjectThatCrashesWithExitCodeNonZero_ShouldFail_WithSameExitC .And.Contain("failed: 0") .And.Contain("skipped: 0"); - result.StdOut.Contains("Test run completed with non-success exit code: 1313 (see: https://aka.ms/testingplatform/exitcodes)"); + result.StdOut.Contains("Test run completed with non-success exit code: 47 (see: https://aka.ms/testingplatform/exitcodes)"); } }