Add swift.testBuildArguments Setting (#1020)

* Add swift.testBuildArguments Setting

Adds a new setting that allows users to specify extra arguments when
running `swift test` and `swift build --build-tests` during a test run
from inside VS Code.

For example users could set `swift.testBuildArguments: ["-Xswiftc",
"-DTEST"]` to enable code guarded by an `#if TEST` compiler directive.

This patch required reworking how we do a build all before starting a
debug session since using the premade Build All tasks do not allow for
dynamically customizing arguments.

Author: plemarquand

assets/test/.vscode/settings.json

@@ -1,3 +1,7 @@
 {
-    "swift.disableAutoResolve": true
+    "swift.disableAutoResolve": true,
+    "swift.additionalTestArguments": [
+        "-Xswiftc",
+        "-DTEST_ARGUMENT_SET_VIA_TEST_BUILD_ARGUMENTS_SETTING"
+    ]
 }

assets/test/defaultPackage/Tests/PackageTests/PackageTests.swift

@@ -2,7 +2,11 @@ import PackageLib
 import XCTest
 
 final class PassingXCTestSuite: XCTestCase {
-  func testPassing() throws {}
+  func testPassing() throws {
+    #if !TEST_ARGUMENT_SET_VIA_TEST_BUILD_ARGUMENTS_SETTING
+    XCTFail("Expected TEST_ARGUMENT_SET_VIA_TEST_BUILD_ARGUMENTS_SETTING to be set at compilation")
+    #endif
+  }
 }
 
 // Should not run when PassingXCTestSuite is run.
@@ -43,7 +47,11 @@ import Testing
 
 @Test func topLevelTestPassing() {
   print("A print statement in a test.")
+  #if !TEST_ARGUMENT_SET_VIA_TEST_BUILD_ARGUMENTS_SETTING
+    Issue.record("Expected TEST_ARGUMENT_SET_VIA_TEST_BUILD_ARGUMENTS_SETTING to be set at compilation")
+  #endif
 }
+
 @Test func topLevelTestFailing() {
   #expect(1 == 2)
 }

package.json

@@ -226,17 +226,24 @@
           "swift.path": {
             "type": "string",
             "default": "",
-            "markdownDescription": "Override the default path of the folder containing the Swift executables. The default is to look in the `PATH` environment variable. This path is also used to search for other executables used by the extension like `sourcekit-lsp` and `lldb`.",
-            "order": 1
+            "markdownDescription": "Override the default path of the folder containing the Swift executables. The default is to look in the `PATH` environment variable. This path is also used to search for other executables used by the extension like `sourcekit-lsp` and `lldb`."
           },
           "swift.buildArguments": {
             "type": "array",
             "default": [],
             "items": {
               "type": "string"
             },
-            "markdownDescription": "Additional arguments to pass to `swift build`. Keys and values should be provided as individual entries in the list. If you have created a copy of the build task in `tasks.json` then these build arguments will not be propogated to that task.",
-            "order": 2
+            "markdownDescription": "Additional arguments to pass to `swift` commands such as `swift build`, `swift package`, `swift test`, etc... Keys and values should be provided as individual entries in the list. If you have created a copy of the build task in `tasks.json` then these build arguments will not be propogated to that task."
+          },
+          "swift.additionalTestArguments": {
+            "type": "array",
+            "default": [],
+            "items": {
+              "type": "string"
+            },
+            "markdownDescription": "Additional arguments to pass to the `swift test` or `swift build` commands used when building and running tests from within VS Code.",
+            "scope": "machine-overridable"
           },
           "swift.testEnvironmentVariables": {
             "type": "object",
@@ -247,8 +254,7 @@
             },
             "default": {},
             "markdownDescription": "Environment variables to set when running tests. To set environment variables when debugging an application you should edit the `env` field in the relevant `launch.json` configuration.",
-            "scope": "machine-overridable",
-            "order": 3
+            "scope": "machine-overridable"
           },
           "swift.sanitizer": {
             "type": "string",
@@ -259,29 +265,25 @@
               "address"
             ],
             "markdownDescription": "Runtime [sanitizer instrumentation](https://www.swift.org/documentation/server/guides/llvm-sanitizers.html).",
-            "scope": "machine-overridable",
-            "order": 4
+            "scope": "machine-overridable"
           },
           "swift.searchSubfoldersForPackages": {
             "type": "boolean",
             "default": false,
             "markdownDescription": "Search sub-folders of workspace folder for Swift Packages at start up.",
-            "scope": "machine-overridable",
-            "order": 5
+            "scope": "machine-overridable"
           },
           "swift.autoGenerateLaunchConfigurations": {
             "type": "boolean",
             "default": true,
             "markdownDescription": "When loading a `Package.swift`, auto-generate `launch.json` configurations for running any executables.",
-            "scope": "machine-overridable",
-            "order": 6
+            "scope": "machine-overridable"
           },
           "swift.disableAutoResolve": {
             "type": "boolean",
             "default": false,
             "markdownDescription": "Disable automatic running of `swift package resolve` whenever the `Package.swift` or `Package.resolve` files are updated. This will also disable searching for command plugins and the initial test discovery process.",
-            "scope": "machine-overridable",
-            "order": 7
+            "scope": "machine-overridable"
           },
           "swift.diagnosticsCollection": {
             "type": "string",
@@ -300,8 +302,7 @@
               "When merging diagnostics, give precedence to diagnostics from `swiftc`.",
               "When merging diagnostics, give precedence to diagnostics from `SourceKit`.",
               "Keep diagnostics from all providers."
-            ],
-            "order": 8
+            ]
           },
           "swift.diagnosticsStyle": {
             "type": "string",
@@ -316,14 +317,12 @@
               "Use whichever diagnostics style `swiftc` produces by default.",
               "Use the `llvm` diagnostic style. This allows the parsing of \"notes\".",
               "Use the `swift` diagnostic style. This means that \"notes\" will not be parsed. This option has no effect in Swift versions prior to 5.10."
-            ],
-            "order": 9
+            ]
           },
           "swift.backgroundCompilation": {
             "type": "boolean",
             "default": false,
-            "markdownDescription": "**Experimental**: Run `swift build` in the background whenever a file is saved. It is possible the background compilation will already be running when you attempt a compile yourself, so this is disabled by default.",
-            "order": 10
+            "markdownDescription": "**Experimental**: Run `swift build` in the background whenever a file is saved. It is possible the background compilation will already be running when you attempt a compile yourself, so this is disabled by default."
           },
           "swift.actionAfterBuildError": {
             "type": "string",
@@ -337,33 +336,28 @@
               "Focus on Problems View",
               "Focus on Build Task Terminal"
             ],
-            "markdownDescription": "Action after a Build task generates errors.",
-            "order": 11
+            "markdownDescription": "Action after a Build task generates errors."
           },
           "swift.buildPath": {
             "type": "string",
             "default": "",
-            "markdownDescription": "The path to a directory that will be used for build artifacts. This path will be added to all swift package manager commands that are executed by vscode-swift extension via `--scratch-path` option. When no value provided - nothing gets passed to swift package manager and it will use its default value of `.build` folder in the workspace.\n\nYou can use absolute path for directory or the relative path, which will use the workspace path as a base. Note that VS Code does not respect tildes (`~`) in paths which represents user home folder under *nix systems.",
-            "order": 12
+            "markdownDescription": "The path to a directory that will be used for build artifacts. This path will be added to all swift package manager commands that are executed by vscode-swift extension via `--scratch-path` option. When no value provided - nothing gets passed to swift package manager and it will use its default value of `.build` folder in the workspace.\n\nYou can use absolute path for directory or the relative path, which will use the workspace path as a base. Note that VS Code does not respect tildes (`~`) in paths which represents user home folder under *nix systems."
           },
           "swift.disableSwiftPackageManagerIntegration": {
             "type": "boolean",
             "default": false,
-            "markdownDescription": "Disables automated Build Tasks, Package Dependency view, Launch configuration generation and TestExplorer.",
-            "order": 13
+            "markdownDescription": "Disables automated Build Tasks, Package Dependency view, Launch configuration generation and TestExplorer."
           },
           "swift.warnAboutSymlinkCreation": {
             "type": "boolean",
             "default": true,
             "markdownDescription": "Controls whether or not the extension will warn about being unable to create symlinks. (Windows only)",
-            "scope": "application",
-            "order": 14
+            "scope": "application"
           },
           "swift.enableTerminalEnvironment": {
             "type": "boolean",
             "default": true,
-            "markdownDescription": "Controls whether or not the extension will contribute environment variables defined in `Swift: Environment Variables` to the integrated terminal. If this is set to `true` and a custom `Swift: Path` is also set then the swift path is appended to the terminal's `PATH`.",
-            "order": 15
+            "markdownDescription": "Controls whether or not the extension will contribute environment variables defined in `Swift: Environment Variables` to the integrated terminal. If this is set to `true` and a custom `Swift: Path` is also set then the swift path is appended to the terminal's `PATH`."
           }
         }
       },

src/TestExplorer/TestRunner.ts

@@ -19,7 +19,7 @@ import * as os from "os";
 import * as asyncfs from "fs/promises";
 import { FolderContext } from "../FolderContext";
 import { execFile, getErrorDescription } from "../utilities/utilities";
-import { createSwiftTask, getBuildAllTask } from "../tasks/SwiftTaskProvider";
+import { createSwiftTask } from "../tasks/SwiftTaskProvider";
 import configuration from "../configuration";
 import { WorkspaceContext } from "../WorkspaceContext";
 import {
@@ -36,8 +36,7 @@ import { TestRunArguments } from "./TestRunArguments";
 import { TemporaryFolder } from "../utilities/tempFolder";
 import { TestClass, runnableTag, upsertTestItem } from "./TestDiscovery";
 import { TestCoverage } from "../coverage/LcovResults";
-import { TestingDebugConfigurationFactory } from "../debugger/buildConfig";
-import { SwiftExecution } from "../tasks/SwiftExecution";
+import { BuildConfigurationFactory, TestingConfigurationFactory } from "../debugger/buildConfig";
 import { TestKind, isDebugging, isRelease } from "./TestKind";
 import { reduceTestItemChildren } from "./TestUtils";
 
@@ -202,6 +201,11 @@ export class TestRunProxy {
     }
 
     public async end() {
+        // If the test run never started (typically due to a build error)
+        // start it to flush any queued output, and then immediately end it.
+        if (!this.runStarted) {
+            this.testRunStarted();
+        }
         this.testRun?.end();
         this.testRunCompleteEmitter.fire();
     }
@@ -473,7 +477,7 @@ export class TestRunner {
                     await execFile("mkfifo", [fifoPipePath], undefined, this.folderContext);
                 }
 
-                const testBuildConfig = await TestingDebugConfigurationFactory.swiftTestingConfig(
+                const testBuildConfig = await TestingConfigurationFactory.swiftTestingConfig(
                     this.folderContext,
                     fifoPipePath,
                     this.testKind,
@@ -507,7 +511,7 @@ export class TestRunner {
         }
 
         if (this.testArgs.hasXCTests) {
-            const testBuildConfig = await TestingDebugConfigurationFactory.xcTestConfig(
+            const testBuildConfig = await TestingConfigurationFactory.xcTestConfig(
                 this.folderContext,
                 this.testKind,
                 this.testArgs.xcTestArgs,
@@ -714,34 +718,31 @@ export class TestRunner {
 
     /** Run test session inside debugger */
     async debugSession(token: vscode.CancellationToken, runState: TestRunnerTestRunState) {
-        const buildAllTask = await getBuildAllTask(this.folderContext, isRelease(this.testKind));
-        if (!buildAllTask) {
-            return;
+        // Perform a build all first to produce the binaries we'll run later.
+        let buildOutput = "";
+        try {
+            await this.runStandardSession(
+                token,
+                // discard the output as we dont want to associate it with the test run.
+                new stream.Writable({
+                    write: (chunk, encoding, next) => {
+                        buildOutput += chunk.toString();
+                        next();
+                    },
+                }),
+                BuildConfigurationFactory.buildAll(
+                    this.folderContext,
+                    true,
+                    isRelease(this.testKind)
+                ),
+                this.testKind
+            );
+        } catch (buildExitCode) {
+            runState.recordOutput(undefined, buildOutput);
+            throw new Error(`Build failed with exit code ${buildExitCode}`);
         }
 
         const subscriptions: vscode.Disposable[] = [];
-        let buildExitCode = 0;
-        const buildTask = vscode.tasks.onDidStartTask(e => {
-            if (e.execution.task.name === buildAllTask.name) {
-                const exec = e.execution.task.execution as SwiftExecution;
-                const didCloseBuildTask = exec.onDidClose(exitCode => {
-                    buildExitCode = exitCode ?? 0;
-                });
-                subscriptions.push(didCloseBuildTask);
-            }
-        });
-        subscriptions.push(buildTask);
-
-        // Perform a build all before the tests are run. We want to avoid the "Debug Anyway" dialog
-        // since choosing that with no prior build, when debugging swift-testing tests, will cause
-        // the extension to start listening to the fifo pipe when no results will be produced,
-        // hanging the test run.
-        await this.folderContext.taskQueue.queueOperation(new TaskOperation(buildAllTask));
-
-        if (buildExitCode !== 0) {
-            throw new Error(`${buildAllTask.name} failed with exit code ${buildExitCode}`);
-        }
-
         const buildConfigs: Array<vscode.DebugConfiguration | undefined> = [];
         const fifoPipePath = this.generateFifoPipePath();
 
@@ -753,14 +754,13 @@ export class TestRunner {
                     await execFile("mkfifo", [fifoPipePath], undefined, this.folderContext);
                 }
 
-                const swiftTestBuildConfig =
-                    await TestingDebugConfigurationFactory.swiftTestingConfig(
-                        this.folderContext,
-                        fifoPipePath,
-                        this.testKind,
-                        this.testArgs.swiftTestArgs,
-                        true
-                    );
+                const swiftTestBuildConfig = await TestingConfigurationFactory.swiftTestingConfig(
+                    this.folderContext,
+                    fifoPipePath,
+                    this.testKind,
+                    this.testArgs.swiftTestArgs,
+                    true
+                );
 
                 if (swiftTestBuildConfig !== null) {
                     swiftTestBuildConfig.testType = TestLibrary.swiftTesting;
@@ -788,7 +788,7 @@ export class TestRunner {
 
             // create launch config for testing
             if (this.testArgs.hasXCTests) {
-                const xcTestBuildConfig = await TestingDebugConfigurationFactory.xcTestConfig(
+                const xcTestBuildConfig = await TestingConfigurationFactory.xcTestConfig(
                     this.folderContext,
                     this.testKind,
                     this.testArgs.xcTestArgs,

src/configuration.ts

@@ -57,6 +57,8 @@ export interface DebuggerConfiguration {
 export interface FolderConfiguration {
     /** Environment variables to set when running tests */
     readonly testEnvironmentVariables: { [key: string]: string };
+    /** Extra arguments to set when building tests */
+    readonly additionalTestArguments: string[];
     /** search sub-folder of workspace folder for Swift Packages */
     readonly searchSubfoldersForPackages: boolean;
     /** auto-generate launch.json configurations */
@@ -119,6 +121,12 @@ const configuration = {
                     .getConfiguration("swift", workspaceFolder)
                     .get<{ [key: string]: string }>("testEnvironmentVariables", {});
             },
+            /** Extra arguments to pass to swift test and swift build when running and debugging tests. */
+            get additionalTestArguments(): string[] {
+                return vscode.workspace
+                    .getConfiguration("swift", workspaceFolder)
+                    .get<string[]>("additionalTestArguments", []);
+            },
             /** auto-generate launch.json configurations */
             get autoGenerateLaunchConfigurations(): boolean {
                 return vscode.workspace

src/coverage/LcovResults.ts

@@ -25,7 +25,7 @@ import { BuildFlags } from "../toolchain/BuildFlags";
 import { TestLibrary } from "../TestExplorer/TestRunner";
 import { DisposableFileCollection } from "../utilities/tempFolder";
 import { TargetType } from "../SwiftPackage";
-import { TestingDebugConfigurationFactory } from "../debugger/buildConfig";
+import { TestingConfigurationFactory } from "../debugger/buildConfig";
 import { TestKind } from "../TestExplorer/TestKind";
 
 interface CodeCovFile {
@@ -144,7 +144,7 @@ export class TestCoverage {
     private async exportProfdata(types: TestLibrary[], mergedProfileFile: string): Promise<Buffer> {
         const coveredBinaries = new Set<string>();
         if (types.includes(TestLibrary.xctest)) {
-            let xcTestBinary = await TestingDebugConfigurationFactory.testExecutableOutputPath(
+            let xcTestBinary = await TestingConfigurationFactory.testExecutableOutputPath(
                 this.folderContext,
                 TestKind.coverage,
                 TestLibrary.xctest
@@ -156,7 +156,7 @@ export class TestCoverage {
         }
 
         if (types.includes(TestLibrary.swiftTesting)) {
-            const swiftTestBinary = await TestingDebugConfigurationFactory.testExecutableOutputPath(
+            const swiftTestBinary = await TestingConfigurationFactory.testExecutableOutputPath(
                 this.folderContext,
                 TestKind.coverage,
                 TestLibrary.swiftTesting