Mark XCTest suites as passed/failed immediately on suite completion (#1031)

Add back some code that was removed that marked XCTest suites as
passed/failed as soon as all their tests completed. Currently the
TestExplorer shows the test suites as pending until all tests in the
whole test run have completed. This is a regression from how the Test
Explorer used to work, where suites were marked as passed/failed
immediately upon suite completion.

Issue: #1029

Author: plemarquand

src/TestExplorer/TestRunner.ts

@@ -1193,12 +1193,40 @@ export class TestRunnerTestRunState implements ITestRunState {
         // Nothing to do here
     }
     // passed suite
-    passedSuite() {
-        // Nothing to do here
+    passedSuite(name: string) {
+        // Regular runs don't provide the full suite name (Target.Suite)
+        // in the output, so reference the last passing/failing test item
+        // and derive the suite from that.
+
+        // However, when running a parallel test run the XUnit XML output
+        // provides the full suite name, and the `lastTestItem` set is not
+        // guarenteed to be in this suite due to the parallel nature of the run.
+
+        // If we can look the suite up by name then we're doing a parallel run
+        // and can mark it as passed, otherwise derive the suite from the last
+        // completed test item.
+        const suiteIndex = this.testRun.getTestIndex(name);
+        if (suiteIndex !== -1) {
+            this.testRun.passed(this.testRun.testItems[suiteIndex]);
+        } else {
+            const lastClassTestItem = this.lastTestItem?.parent;
+            if (lastClassTestItem && lastClassTestItem.id.endsWith(`.${name}`)) {
+                this.testRun.passed(lastClassTestItem);
+            }
+        }
     }
     // failed suite
-    failedSuite() {
-        // Nothing to do here
+    failedSuite(name: string) {
+        // See comment in `passedSuite` for more context.
+        const suiteIndex = this.testRun.getTestIndex(name);
+        if (suiteIndex !== -1) {
+            this.testRun.failed(this.testRun.testItems[suiteIndex], []);
+        } else {
+            const lastClassTestItem = this.lastTestItem?.parent;
+            if (lastClassTestItem && lastClassTestItem.id.endsWith(`.${name}`)) {
+                this.testRun.failed(lastClassTestItem, []);
+            }
+        }
     }
 
     recordOutput(index: number | undefined, output: string): void {

src/TestExplorer/TestXUnitParser.ts

@@ -66,11 +66,15 @@ export class TestXUnitParser {
         let failures = 0;
         let errors = 0;
         xUnit.testsuites.testsuite.forEach(testsuite => {
+            const suiteFailures = parseInt(testsuite.$.failures);
+            failures += suiteFailures;
             tests = tests + parseInt(testsuite.$.tests);
-            failures += parseInt(testsuite.$.failures);
             errors += parseInt(testsuite.$.errors);
+
+            let className: string | undefined;
             testsuite.testcase.forEach(testcase => {
-                const id = `${testcase.$.classname}/${testcase.$.name}`;
+                className = testcase.$.classname;
+                const id = `${className}/${testcase.$.name}`;
                 const index = runState.getTestItemIndex(id);
 
                 // From 5.7 to 5.10 running with the --parallel option dumps the test results out
@@ -85,6 +89,14 @@ export class TestXUnitParser {
                 }
                 runState.completed(index, { duration: testcase.$.time });
             });
+
+            if (className !== undefined) {
+                if (className && suiteFailures === 0) {
+                    runState.passedSuite(className);
+                } else if (className) {
+                    runState.failedSuite(className);
+                }
+            }
         });
         return { tests: tests, failures: failures, errors: errors };
     }

test/suite/testexplorer/MockTestRunState.ts

@@ -130,11 +130,13 @@ export class TestRunState implements ITestRunState {
         //
     }
     // passed suite
-    passedSuite() {
-        //
+    passedSuite(name: string) {
+        const index = this.testItemFinder.getIndex(name);
+        this.testItemFinder.tests[index].status = TestStatus.passed;
     }
     // failed suite
-    failedSuite() {
-        //
+    failedSuite(name: string) {
+        const index = this.testItemFinder.getIndex(name);
+        this.testItemFinder.tests[index].status = TestStatus.failed;
     }
 }

test/suite/testexplorer/TestExplorerIntegration.test.ts

@@ -278,7 +278,10 @@ suite("Test Explorer Suite", function () {
                 await eventPromise(testRun.onTestRunComplete);
 
                 assertTestResults(testRun, {
-                    passed: ["PackageTests.MixedXCTestSuite/testPassing"],
+                    passed: [
+                        "PackageTests.MixedXCTestSuite",
+                        "PackageTests.MixedXCTestSuite/testPassing",
+                    ],
                 });
             });
         });
@@ -293,7 +296,10 @@ suite("Test Explorer Suite", function () {
             );
 
             assertTestResults(testRun, {
-                passed: ["PackageTests.DebugReleaseTestSuite/testDebug"],
+                passed: [
+                    "PackageTests.DebugReleaseTestSuite",
+                    "PackageTests.DebugReleaseTestSuite/testDebug",
+                ],
             });
         });
 
@@ -308,7 +314,10 @@ suite("Test Explorer Suite", function () {
             );
 
             assertTestResults(passingRun, {
-                passed: ["PackageTests.DebugReleaseTestSuite/testRelease"],
+                passed: [
+                    "PackageTests.DebugReleaseTestSuite",
+                    "PackageTests.DebugReleaseTestSuite/testRelease",
+                ],
             });
         });
 
@@ -516,6 +525,10 @@ suite("Test Explorer Suite", function () {
                                 test: "PackageTests.MixedXCTestSuite/testFailing",
                                 issues: [xcTestFailureMessage],
                             },
+                            {
+                                issues: [],
+                                test: "PackageTests.MixedXCTestSuite",
+                            },
                         ],
                     });
                 });
@@ -530,7 +543,10 @@ suite("Test Explorer Suite", function () {
                     );
 
                     assertTestResults(testRun, {
-                        passed: ["PackageTests.PassingXCTestSuite/testPassing"],
+                        passed: [
+                            "PackageTests.PassingXCTestSuite",
+                            "PackageTests.PassingXCTestSuite/testPassing",
+                        ],
                     });
                 });
 
@@ -547,6 +563,10 @@ suite("Test Explorer Suite", function () {
                                 test: "PackageTests.FailingXCTestSuite/testFailing",
                                 issues: [xcTestFailureMessage],
                             },
+                            {
+                                issues: [],
+                                test: "PackageTests.FailingXCTestSuite",
+                            },
                         ],
                     });
                 });
@@ -565,6 +585,10 @@ suite("Test Explorer Suite", function () {
                                 test: "PackageTests.MixedXCTestSuite/testFailing",
                                 issues: [xcTestFailureMessage],
                             },
+                            {
+                                issues: [],
+                                test: "PackageTests.MixedXCTestSuite",
+                            },
                         ],
                     });
                 });

test/suite/testexplorer/XCTestOutputParser.test.ts

@@ -263,6 +263,31 @@ Test Case '-[MyTests.MyTests`;
             });
         });
 
+        test("Suite", () => {
+            const testRunState = new TestRunState(["MyTests", "MyTests.MyTests/testPass"], true);
+            const input = `Test Suite 'MyTests' started at 2024-08-26 13:19:25.325.
+Test Case '-[MyTests.MyTests testPass]' started.
+Test Case '-[MyTests.MyTests testPass]' passed (0.001 seconds).
+Test Suite 'MyTests' passed at 2024-08-26 13:19:25.328.
+         Executed 1 test, with 0 failures (0 unexpected) in 0.001 (0.001) seconds
+`;
+            outputParser.parseResult(input, testRunState);
+
+            assert.deepEqual(testRunState.tests, [
+                {
+                    name: "MyTests",
+                    output: [],
+                    status: TestStatus.passed,
+                },
+                {
+                    name: "MyTests.MyTests/testPass",
+                    status: TestStatus.passed,
+                    timing: { duration: 0.001 },
+                    output: inputToTestOutput(input).slice(1, -2), // trim the suite text
+                },
+            ]);
+        });
+
         suite("Diffs", () => {
             const testRun = (message: string, expected?: string, actual?: string) => {
                 const testRunState = new TestRunState(["MyTests.MyTests/testFail"], true);