[dotnet test][MTP] Avoid surfacing MissingTestSessionEnd when the exit code is already non-success

src/Cli/dotnet/Commands/Test/MTP/TestApplication.cs

@@ -61,14 +61,20 @@ public async Task<int> RunAsync()
             var outputAndError = await Task.WhenAll(stdOutTask, stdErrTask);
             await process.WaitForExitAsync();
 
-            _handler.OnTestProcessExited(process.ExitCode, outputAndError[0], outputAndError[1]);
-
-            if (_handler.HasMismatchingTestSessionEventCount())
+            var exitCode = process.ExitCode;
+            _handler.OnTestProcessExited(exitCode, outputAndError[0], outputAndError[1]);
+
+            // This condition is to prevent considering the test app as successful when we didn't receive test session end.
+            // We don't produce the exception if the exit code is already non-zero to avoid surfacing this exception when there is already a known failure.
+            // For example, if hangdump timeout is reached, the process will be killed and we will have mismatching count.
+            // Or if there is a crash (e.g, Environment.FailFast), etc.
+            // So this is only a safe guard to avoid passing the test run if Environment.Exit(0) is called in one of the tests for example.
+            if (exitCode == 0 && _handler.HasMismatchingTestSessionEventCount())
             {
                 throw new InvalidOperationException(CliCommandStrings.MissingTestSessionEnd);
             }
 
-            return process.ExitCode;
+            return exitCode;
         }
         finally
         {

test/TestAssets/TestProjects/TestProjectMTPCrashNonZero/TestProject1/Program.cs

@@ -0,0 +1,47 @@
+using Microsoft.Testing.Platform.Builder;
+using Microsoft.Testing.Platform.Capabilities.TestFramework;
+using Microsoft.Testing.Platform.Extensions.Messages;
+using Microsoft.Testing.Platform.Extensions.TestFramework;
+
+var testApplicationBuilder = await TestApplication.CreateBuilderAsync(args);
+
+testApplicationBuilder.RegisterTestFramework(_ => new TestFrameworkCapabilities(), (_, __) => new DummyTestAdapter());
+
+using var testApplication = await testApplicationBuilder.BuildAsync();
+return await testApplication.RunAsync();
+
+public class DummyTestAdapter : ITestFramework, IDataProducer
+{
+	public string Uid => nameof(DummyTestAdapter);
+
+	public string Version => "2.0.0";
+
+	public string DisplayName => nameof(DummyTestAdapter);
+
+	public string Description => nameof(DummyTestAdapter);
+
+	public Task<bool> IsEnabledAsync() => Task.FromResult(true);
+
+	public Type[] DataTypesProduced => [typeof(TestNodeUpdateMessage)];
+	public Task<CreateTestSessionResult> CreateTestSessionAsync(CreateTestSessionContext context)
+		=> Task.FromResult(new CreateTestSessionResult() { IsSuccess = true });
+
+	public Task<CloseTestSessionResult> CloseTestSessionAsync(CloseTestSessionContext context)
+		=> Task.FromResult(new CloseTestSessionResult() { IsSuccess = true });
+
+	public async Task ExecuteRequestAsync(ExecuteRequestContext context)
+	{
+		await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage(context.Request.Session.SessionUid, new TestNode()
+		{
+			Uid = $"Test1",
+			DisplayName = $"Test1",
+			Properties = new PropertyBag(PassedTestNodeStateProperty.CachedInstance),
+		}));
+
+		await Task.Delay(1000);
+
+		Environment.Exit(47);
+
+		context.Complete();
+	}
+}

test/TestAssets/TestProjects/TestProjectMTPCrashNonZero/TestProject1/TestProject1.csproj

@@ -0,0 +1,17 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), testAsset.props))\testAsset.props" />
+
+	<PropertyGroup>
+		<TargetFramework>$(CurrentTargetFramework)</TargetFramework>
+		<OutputType>Exe</OutputType>
+
+		<ImplicitUsings>enable</ImplicitUsings>
+		<Nullable>enable</Nullable>
+
+		<ManagePackageVersionsCentrally>false</ManagePackageVersionsCentrally>
+	</PropertyGroup>
+
+	<ItemGroup>
+		<PackageReference Include="Microsoft.Testing.Platform" Version="$(MicrosoftTestingPlatformVersion)" />
+	</ItemGroup>
+</Project>

test/TestAssets/TestProjects/TestProjectMTPCrashNonZero/TestProjectMTPCrashNonZero.sln

@@ -0,0 +1,34 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.0.31903.59
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestProject1", "TestProject1\TestProject1.csproj", "{3E834ED1-92D9-4454-BBB4-B1A2463E5726}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Debug|x64 = Debug|x64
+		Debug|x86 = Debug|x86
+		Release|Any CPU = Release|Any CPU
+		Release|x64 = Release|x64
+		Release|x86 = Release|x86
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Debug|x64.Build.0 = Debug|Any CPU
+		{3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Debug|x86.Build.0 = Debug|Any CPU
+		{3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Release|Any CPU.Build.0 = Release|Any CPU
+		{3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Release|x64.ActiveCfg = Release|Any CPU
+		{3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Release|x64.Build.0 = Release|Any CPU
+		{3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Release|x86.ActiveCfg = Release|Any CPU
+		{3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Release|x86.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal

test/TestAssets/TestProjects/TestProjectMTPCrashNonZero/global.json

@@ -0,0 +1,5 @@
+{
+    "test": {
+        "runner": "Microsoft.Testing.Platform"
+    }
+}

test/dotnet.Tests/CommandTests/Test/GivenDotnetTestBuildsAndRunsTests.cs

@@ -442,7 +442,6 @@ public void RunMTPSolutionWithMinimumExpectedTests(string value, int expectedExi
         [Fact]
         public void RunMTPProjectThatCrashesWithExitCodeZero_ShouldFail()
         {
-            // The solution has two test projects. Each reports 5 tests. So, total 10 tests.
             TestAsset testInstance = _testAssetsManager.CopyTestAsset("TestProjectMTPCrash", Guid.NewGuid().ToString())
                 .WithSource();
 
@@ -477,6 +476,35 @@ at Microsoft.DotNet.Cli.Commands.Test.TestApplicationActionQueue.Read(BuildOptio
             }
         }
 
+        [Fact]
+        public void RunMTPProjectThatCrashesWithExitCodeNonZero_ShouldFail_WithSameExitCode()
+        {
+            TestAsset testInstance = _testAssetsManager.CopyTestAsset("TestProjectMTPCrashNonZero", Guid.NewGuid().ToString())
+                .WithSource();
+
+            CommandResult result = new DotnetTestCommand(Log, disableNewOutput: false)
+                .WithWorkingDirectory(testInstance.Path)
+                .Execute();
+
+            // The test asset exits with Environment.Exit(47);
+            result.ExitCode.Should().Be(47);
+            if (!TestContext.IsLocalized())
+            {
+                result.StdErr.Should().NotContain("A test session start event was received without a corresponding test session end");
+
+                // TODO: It's much better to introduce a new kind of "summary" indicating
+                // that the test app exited with zero exit code before sending test session end event
+                result.StdOut.Should().Contain("Test run summary: Failed!")
+                    .And.Contain("error: 1")
+                    .And.Contain("total: 1")
+                    .And.Contain("succeeded: 1")
+                    .And.Contain("failed: 0")
+                    .And.Contain("skipped: 0");
+
+                result.StdOut.Contains("Test run completed with non-success exit code: 47 (see: https://aka.ms/testingplatform/exitcodes)");
+            }
+        }
+
         [Theory]
         [InlineData(TestingConstants.Debug)]
         [InlineData(TestingConstants.Release)]