diff --git a/src/Cli/dotnet/Commands/Test/MTP/TestApplication.cs b/src/Cli/dotnet/Commands/Test/MTP/TestApplication.cs index 16ca1bb218e7..9708c9246017 100644 --- a/src/Cli/dotnet/Commands/Test/MTP/TestApplication.cs +++ b/src/Cli/dotnet/Commands/Test/MTP/TestApplication.cs @@ -61,14 +61,20 @@ public async Task RunAsync() var outputAndError = await Task.WhenAll(stdOutTask, stdErrTask); await process.WaitForExitAsync(); - _handler.OnTestProcessExited(process.ExitCode, outputAndError[0], outputAndError[1]); - - if (_handler.HasMismatchingTestSessionEventCount()) + var exitCode = process.ExitCode; + _handler.OnTestProcessExited(exitCode, outputAndError[0], outputAndError[1]); + + // This condition is to prevent considering the test app as successful when we didn't receive test session end. + // We don't produce the exception if the exit code is already non-zero to avoid surfacing this exception when there is already a known failure. + // For example, if hangdump timeout is reached, the process will be killed and we will have mismatching count. + // Or if there is a crash (e.g, Environment.FailFast), etc. + // So this is only a safe guard to avoid passing the test run if Environment.Exit(0) is called in one of the tests for example. + if (exitCode == 0 && _handler.HasMismatchingTestSessionEventCount()) { throw new InvalidOperationException(CliCommandStrings.MissingTestSessionEnd); } - return process.ExitCode; + return exitCode; } finally { diff --git a/test/TestAssets/TestProjects/TestProjectMTPCrashNonZero/TestProject1/Program.cs b/test/TestAssets/TestProjects/TestProjectMTPCrashNonZero/TestProject1/Program.cs new file mode 100644 index 000000000000..29e5063ab469 --- /dev/null +++ b/test/TestAssets/TestProjects/TestProjectMTPCrashNonZero/TestProject1/Program.cs @@ -0,0 +1,47 @@ +using Microsoft.Testing.Platform.Builder; +using Microsoft.Testing.Platform.Capabilities.TestFramework; +using Microsoft.Testing.Platform.Extensions.Messages; +using Microsoft.Testing.Platform.Extensions.TestFramework; + +var testApplicationBuilder = await TestApplication.CreateBuilderAsync(args); + +testApplicationBuilder.RegisterTestFramework(_ => new TestFrameworkCapabilities(), (_, __) => new DummyTestAdapter()); + +using var testApplication = await testApplicationBuilder.BuildAsync(); +return await testApplication.RunAsync(); + +public class DummyTestAdapter : ITestFramework, IDataProducer +{ + public string Uid => nameof(DummyTestAdapter); + + public string Version => "2.0.0"; + + public string DisplayName => nameof(DummyTestAdapter); + + public string Description => nameof(DummyTestAdapter); + + public Task IsEnabledAsync() => Task.FromResult(true); + + public Type[] DataTypesProduced => [typeof(TestNodeUpdateMessage)]; + public Task CreateTestSessionAsync(CreateTestSessionContext context) + => Task.FromResult(new CreateTestSessionResult() { IsSuccess = true }); + + public Task CloseTestSessionAsync(CloseTestSessionContext context) + => Task.FromResult(new CloseTestSessionResult() { IsSuccess = true }); + + public async Task ExecuteRequestAsync(ExecuteRequestContext context) + { + await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage(context.Request.Session.SessionUid, new TestNode() + { + Uid = $"Test1", + DisplayName = $"Test1", + Properties = new PropertyBag(PassedTestNodeStateProperty.CachedInstance), + })); + + await Task.Delay(1000); + + Environment.Exit(47); + + context.Complete(); + } +} diff --git a/test/TestAssets/TestProjects/TestProjectMTPCrashNonZero/TestProject1/TestProject1.csproj b/test/TestAssets/TestProjects/TestProjectMTPCrashNonZero/TestProject1/TestProject1.csproj new file mode 100644 index 000000000000..9857918867b1 --- /dev/null +++ b/test/TestAssets/TestProjects/TestProjectMTPCrashNonZero/TestProject1/TestProject1.csproj @@ -0,0 +1,17 @@ + + + + + $(CurrentTargetFramework) + Exe + + enable + enable + + false + + + + + + diff --git a/test/TestAssets/TestProjects/TestProjectMTPCrashNonZero/TestProjectMTPCrashNonZero.sln b/test/TestAssets/TestProjects/TestProjectMTPCrashNonZero/TestProjectMTPCrashNonZero.sln new file mode 100644 index 000000000000..aaee5bd7dc36 --- /dev/null +++ b/test/TestAssets/TestProjects/TestProjectMTPCrashNonZero/TestProjectMTPCrashNonZero.sln @@ -0,0 +1,34 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.0.31903.59 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestProject1", "TestProject1\TestProject1.csproj", "{3E834ED1-92D9-4454-BBB4-B1A2463E5726}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Debug|x64.ActiveCfg = Debug|Any CPU + {3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Debug|x64.Build.0 = Debug|Any CPU + {3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Debug|x86.ActiveCfg = Debug|Any CPU + {3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Debug|x86.Build.0 = Debug|Any CPU + {3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Release|Any CPU.Build.0 = Release|Any CPU + {3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Release|x64.ActiveCfg = Release|Any CPU + {3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Release|x64.Build.0 = Release|Any CPU + {3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Release|x86.ActiveCfg = Release|Any CPU + {3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Release|x86.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/test/TestAssets/TestProjects/TestProjectMTPCrashNonZero/global.json b/test/TestAssets/TestProjects/TestProjectMTPCrashNonZero/global.json new file mode 100644 index 000000000000..9009caf0ba8f --- /dev/null +++ b/test/TestAssets/TestProjects/TestProjectMTPCrashNonZero/global.json @@ -0,0 +1,5 @@ +{ + "test": { + "runner": "Microsoft.Testing.Platform" + } +} diff --git a/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestBuildsAndRunsTests.cs b/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestBuildsAndRunsTests.cs index 23855c3cb442..41d9ae0fbe10 100644 --- a/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestBuildsAndRunsTests.cs +++ b/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestBuildsAndRunsTests.cs @@ -442,7 +442,6 @@ public void RunMTPSolutionWithMinimumExpectedTests(string value, int expectedExi [Fact] public void RunMTPProjectThatCrashesWithExitCodeZero_ShouldFail() { - // The solution has two test projects. Each reports 5 tests. So, total 10 tests. TestAsset testInstance = _testAssetsManager.CopyTestAsset("TestProjectMTPCrash", Guid.NewGuid().ToString()) .WithSource(); @@ -477,6 +476,35 @@ at Microsoft.DotNet.Cli.Commands.Test.TestApplicationActionQueue.Read(BuildOptio } } + [Fact] + public void RunMTPProjectThatCrashesWithExitCodeNonZero_ShouldFail_WithSameExitCode() + { + TestAsset testInstance = _testAssetsManager.CopyTestAsset("TestProjectMTPCrashNonZero", Guid.NewGuid().ToString()) + .WithSource(); + + CommandResult result = new DotnetTestCommand(Log, disableNewOutput: false) + .WithWorkingDirectory(testInstance.Path) + .Execute(); + + // The test asset exits with Environment.Exit(47); + result.ExitCode.Should().Be(47); + if (!TestContext.IsLocalized()) + { + result.StdErr.Should().NotContain("A test session start event was received without a corresponding test session end"); + + // TODO: It's much better to introduce a new kind of "summary" indicating + // that the test app exited with zero exit code before sending test session end event + result.StdOut.Should().Contain("Test run summary: Failed!") + .And.Contain("error: 1") + .And.Contain("total: 1") + .And.Contain("succeeded: 1") + .And.Contain("failed: 0") + .And.Contain("skipped: 0"); + + result.StdOut.Contains("Test run completed with non-success exit code: 47 (see: https://aka.ms/testingplatform/exitcodes)"); + } + } + [Theory] [InlineData(TestingConstants.Debug)] [InlineData(TestingConstants.Release)]