[Dependency Scanning] Add support for dependency-scan-specific serialized diagnostic output

Author: artemcm

Sources/SwiftDriver/Driver/Driver.swift

@@ -374,6 +374,9 @@ public struct Driver {
   /// Path to the serialized diagnostics file of the emit-module task.
   let emitModuleSerializedDiagnosticsFilePath: VirtualPath.Handle?
 
+  /// Path to the serialized diagnostics file of the dependency scanning task.
+  let dependencyScanSerializedDiagnosticsPath: VirtualPath.Handle?
+
   /// Path to the discovered dependencies file of the emit-module task.
   let emitModuleDependenciesFilePath: VirtualPath.Handle?
 
@@ -1241,6 +1244,14 @@ public struct Driver {
         emitModuleSeparately: emitModuleSeparately,
         outputFileMap: self.outputFileMap,
         moduleName: moduleOutputInfo.name)
+    self.dependencyScanSerializedDiagnosticsPath = try Self.computeSupplementaryOutputPath(
+        &parsedOptions, type: .dependencyScanDiagnostics, isOutputOptions: [.serializeDiagnostics],
+        outputPath: .dependencyScanSerializeDiagnosticsPath,
+        compilerOutputType: compilerOutputType,
+        compilerMode: compilerMode,
+        emitModuleSeparately: emitModuleSeparately,
+        outputFileMap: self.outputFileMap,
+        moduleName: moduleOutputInfo.name)
     self.emitModuleDependenciesFilePath = try Self.computeSupplementaryOutputPath(
         &parsedOptions, type: .emitModuleDependencies, isOutputOptions: [.emitDependencies],
         outputPath: .emitModuleDependenciesPath,

Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift

@@ -193,6 +193,11 @@ public extension Driver {
       commandLine.appendFlag(.resolvedPluginVerification)
     }
 
+    if let depScanSerializedDiagnosticsPath = dependencyScanSerializedDiagnosticsPath {
+      commandLine.appendFlag("-serialize-diagnostics-path")
+      commandLine.appendPath(VirtualPath.lookup(depScanSerializedDiagnosticsPath))
+    }
+
     // Pass on the input files
     commandLine.append(contentsOf: inputFiles.filter { $0.type == .swift }.map { .path($0.file) })
     return (inputs, commandLine)

Sources/SwiftDriver/Jobs/CompileJob.swift

@@ -97,7 +97,7 @@ extension Driver {
          .moduleTrace, .yamlOptimizationRecord, .bitstreamOptimizationRecord, .pcm, .pch,
          .clangModuleMap, .jsonCompilerFeatures, .jsonTargetInfo, .jsonSwiftArtifacts,
          .indexUnitOutputPath, .modDepCache, .jsonAPIBaseline, .jsonABIBaseline,
-         .swiftConstValues, .jsonAPIDescriptor, .moduleSummary, .moduleSemanticInfo,
+         .swiftConstValues, .jsonAPIDescriptor, .moduleSummary, .moduleSemanticInfo, .dependencyScanDiagnostics,
          .cachedDiagnostics, .jsonSupportedFeatures, nil:
       return false
     }
@@ -500,7 +500,7 @@ extension FileType {
          .swiftSourceInfoFile, .clangModuleMap, .jsonSwiftArtifacts,
          .indexUnitOutputPath, .modDepCache, .jsonAPIBaseline, .jsonABIBaseline,
          .swiftConstValues, .jsonAPIDescriptor, .moduleSummary, .moduleSemanticInfo,
-         .cachedDiagnostics:
+         .cachedDiagnostics, .dependencyScanDiagnostics:
       fatalError("Output type can never be a primary output")
     }
   }

Sources/SwiftDriver/Utilities/FileType.swift

@@ -87,6 +87,9 @@ public enum FileType: String, Hashable, CaseIterable, Codable {
   /// Serialized diagnostics produced by module-generation
   case emitModuleDiagnostics = "emit-module.dia"
 
+  /// Serialized diagnostics produced by dependency scanning
+  case dependencyScanDiagnostics = "dependency-scan.dia"
+
   /// Serialized diagnostics produced by module-generation
   case emitModuleDependencies = "emit-module.d"
 
@@ -254,6 +257,9 @@ extension FileType: CustomStringConvertible {
     case .emitModuleDiagnostics:
       return "emit-module-diagnostics"
 
+    case .dependencyScanDiagnostics:
+      return "dependency-scan-diagnostics"
+
     case .jsonAPIBaseline:
       return "api-baseline-json"
 
@@ -291,7 +297,7 @@ extension FileType {
     case .object, .pch, .ast, .llvmIR, .llvmBitcode, .assembly, .swiftModule,
          .importedModules, .indexData, .remap, .dSYM, .autolink, .dependencies,
          .emitModuleDependencies, .swiftDocumentation, .pcm, .diagnostics,
-         .emitModuleDiagnostics, .objcHeader, .image, .swiftDeps, .moduleTrace,
+         .emitModuleDiagnostics, .dependencyScanDiagnostics, .objcHeader, .image, .swiftDeps, .moduleTrace,
          .tbd, .yamlOptimizationRecord, .bitstreamOptimizationRecord,
          .swiftInterface, .privateSwiftInterface, .packageSwiftInterface, .swiftSourceInfoFile,
          .jsonDependencies, .clangModuleMap, .jsonTargetInfo, .jsonCompilerFeatures,
@@ -399,6 +405,8 @@ extension FileType {
       return "diagnostics"
     case .emitModuleDiagnostics:
       return "emit-module-diagnostics"
+    case .dependencyScanDiagnostics:
+      return "dependency-scan-diagnostics"
     case .indexUnitOutputPath:
       return "index-unit-output-path"
     case .jsonAPIBaseline:
@@ -435,7 +443,7 @@ extension FileType {
     case .image, .object, .dSYM, .pch, .sib, .raw_sib, .swiftModule,
          .swiftDocumentation, .swiftSourceInfoFile, .llvmBitcode, .diagnostics,
          .pcm, .swiftDeps, .remap, .indexData, .bitstreamOptimizationRecord,
-         .indexUnitOutputPath, .modDepCache, .emitModuleDiagnostics:
+         .indexUnitOutputPath, .modDepCache, .emitModuleDiagnostics, .dependencyScanDiagnostics:
       return false
     }
   }
@@ -455,7 +463,7 @@ extension FileType {
          .clangModuleMap, .jsonCompilerFeatures, .jsonTargetInfo, .jsonSwiftArtifacts,
          .indexUnitOutputPath, .jsonAPIBaseline, .jsonABIBaseline, .swiftConstValues,
          .jsonAPIDescriptor, .moduleSummary, .moduleSemanticInfo, .cachedDiagnostics,
-         .raw_llvmIr, .jsonSupportedFeatures:
+         .raw_llvmIr, .jsonSupportedFeatures, .dependencyScanDiagnostics:
       return false
     }
   }
@@ -465,7 +473,7 @@ extension FileType {
     switch self {
     case .swift, .ast, .indexData, .indexUnitOutputPath, .jsonCompilerFeatures, .jsonTargetInfo, .jsonSupportedFeatures:
       return false
-    case .sil, .sib, .image, .object, .dSYM, .dependencies, .autolink, .swiftModule, .swiftDocumentation, .swiftInterface, .privateSwiftInterface, .packageSwiftInterface, .swiftSourceInfoFile, .swiftConstValues, .assembly, .raw_sil, .raw_sib, .llvmIR, .llvmBitcode, .diagnostics, .emitModuleDiagnostics, .emitModuleDependencies, .objcHeader, .swiftDeps, .modDepCache, .remap, .importedModules, .tbd, .jsonDependencies, .jsonSwiftArtifacts, .moduleTrace, .yamlOptimizationRecord, .bitstreamOptimizationRecord, .pcm, .pch, .clangModuleMap, .jsonAPIBaseline, .jsonABIBaseline, .jsonAPIDescriptor, .moduleSummary, .moduleSemanticInfo, .cachedDiagnostics, .raw_llvmIr:
+    case .sil, .sib, .image, .object, .dSYM, .dependencies, .autolink, .swiftModule, .swiftDocumentation, .swiftInterface, .privateSwiftInterface, .packageSwiftInterface, .swiftSourceInfoFile, .swiftConstValues, .assembly, .raw_sil, .raw_sib, .llvmIR, .llvmBitcode, .diagnostics, .emitModuleDiagnostics, .emitModuleDependencies, .objcHeader, .swiftDeps, .modDepCache, .remap, .importedModules, .tbd, .jsonDependencies, .jsonSwiftArtifacts, .moduleTrace, .yamlOptimizationRecord, .bitstreamOptimizationRecord, .pcm, .pch, .clangModuleMap, .jsonAPIBaseline, .jsonABIBaseline, .jsonAPIDescriptor, .moduleSummary, .moduleSemanticInfo, .cachedDiagnostics, .raw_llvmIr, .dependencyScanDiagnostics:
       return true
     }
   }
@@ -481,7 +489,7 @@ extension FileType {
          .jsonCompilerFeatures, .jsonTargetInfo, .autolink, .jsonSupportedFeatures:
       return false
     case .assembly, .llvmIR, .llvmBitcode, .object, .sil, .sib, .ast,
-         .dependencies, .emitModuleDependencies, .swiftModule,
+         .dependencies, .emitModuleDependencies, .swiftModule, .dependencyScanDiagnostics,
          .swiftDocumentation, .swiftInterface, .privateSwiftInterface, .packageSwiftInterface,
          .swiftSourceInfoFile, .raw_sil, .raw_sib, .objcHeader, .swiftDeps, .tbd,
          .moduleTrace, .indexData, .yamlOptimizationRecord,

Sources/SwiftOptions/Options.swift

@@ -372,6 +372,7 @@ extension Option {
   public static let emitModuleSemanticInfoPath: Option = Option("-emit-module-semantic-info-path", .separate, attributes: [.frontend, .noDriver, .cacheInvariant], metaVar: "<path>", helpText: "Output semantic info of current module to <path>")
   public static let emitModuleSeparatelyWMO: Option = Option("-emit-module-separately-wmo", .flag, attributes: [.helpHidden], helpText: "Emit module files as a distinct job in wmo builds")
   public static let emitModuleSerializeDiagnosticsPath: Option = Option("-emit-module-serialize-diagnostics-path", .separate, attributes: [.argumentIsPath, .supplementaryOutput], metaVar: "<path>", helpText: "Emit a serialized diagnostics file for the emit-module task to <path>")
+  public static let dependencyScanSerializeDiagnosticsPath: Option = Option("-dependency-scan-serialize-diagnostics-path", .separate, attributes: [.argumentIsPath, .supplementaryOutput], metaVar: "<path>", helpText: "Emit a serialized diagnostics file for the dependency scanning task to <path>")
   public static let emitModuleSourceInfoPath: Option = Option("-emit-module-source-info-path", .separate, attributes: [.frontend, .noInteractive, .argumentIsPath, .supplementaryOutput, .cacheInvariant], metaVar: "<path>", helpText: "Output module source info file to <path>")
   public static let emitModuleSourceInfo: Option = Option("-emit-module-source-info", .flag, attributes: [.frontend, .noDriver], helpText: "Output module source info file")
   public static let emitModuleSummaryPath: Option = Option("-emit-module-summary-path", .separate, attributes: [.frontend, .noInteractive, .argumentIsPath, .supplementaryOutput, .cacheInvariant], metaVar: "<path>", helpText: "Output module summary file to <path>")
@@ -1342,6 +1343,7 @@ extension Option {
       Option.emitModuleSemanticInfoPath,
       Option.emitModuleSeparatelyWMO,
       Option.emitModuleSerializeDiagnosticsPath,
+      Option.dependencyScanSerializeDiagnosticsPath,
       Option.emitModuleSourceInfoPath,
       Option.emitModuleSourceInfo,
       Option.emitModuleSummaryPath,

Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift

@@ -1652,6 +1652,55 @@ final class ExplicitModuleBuildTests: XCTestCase {
     }
   }
 
+  func testInMemoryScanWithSerializedDiagnostics() throws {
+    try withTemporaryDirectory { path in
+      let (stdLibPath, shimsPath, _, hostTriple) = try getDriverArtifactsForScanning()
+      let scannerCachePath: AbsolutePath = path.appending(component: "ClangScannerCache")
+      let moduleCachePath = path.appending(component: "ModuleCache")
+      let serializedDiagnosticsOutputPath = path.appending(component: "ScanDiags.dia")
+
+      // Setup our main test module
+      let mainSourcePath = path.appending(component: "Foo.swift")
+      try localFileSystem.writeFileContents(mainSourcePath, bytes: "import Swift")
+
+      // Setup the build plan
+      let sdkArgumentsForTesting = (try? Driver.sdkArgumentsForTesting()) ?? []
+      var driver = try Driver(args: ["swiftc",
+                                     "-I", stdLibPath.nativePathString(escaped: true),
+                                     "-I", shimsPath.nativePathString(escaped: true),
+                                     "-explicit-module-build",
+                                     "-dependency-scan-serialize-diagnostics-path",
+                                     serializedDiagnosticsOutputPath.nativePathString(escaped: false),
+                                     "-module-name", "main",
+                                     "-target", hostTriple.triple,
+                                     "-working-directory", path.nativePathString(escaped: true),
+                                     "-clang-scanner-module-cache-path",
+                                     scannerCachePath.nativePathString(escaped: false),
+                                     "-module-cache-path",
+                                     moduleCachePath.nativePathString(escaped: true),
+                                     mainSourcePath.nativePathString(escaped: true)] + sdkArgumentsForTesting)
+
+      // Set up the in-memory dependency scan using the dependency oracle
+      let dependencyOracle = driver.interModuleDependencyOracle
+      let scanLibPath = try XCTUnwrap(driver.toolchain.lookupSwiftScanLib())
+      try dependencyOracle.verifyOrCreateScannerInstance(swiftScanLibPath: scanLibPath)
+      let resolver = try ArgsResolver(fileSystem: localFileSystem)
+      let scannerCommand = try driver.dependencyScannerInvocationCommand().1.map { try resolver.resolve($0) }
+      XCTAssertTrue(scannerCommand.contains(subsequence: ["-serialize-diagnostics-path", serializedDiagnosticsOutputPath.pathString]))
+
+      // Perform the scan
+      var scanDiagnostics: [ScannerDiagnosticPayload] = []
+      let _ = try dependencyOracle.getDependencies(workingDirectory: path,
+                                                   commandLine: scannerCommand,
+                                                   diagnostics: &scanDiagnostics)
+
+      // TODO: Ensure the serialized diagnostics output got written out
+      // This requires an ability to confirm first whether the compiler we're using
+      // has this capability.
+      // XCTAssertTrue(localFileSystem.exists(serializedDiagnosticsOutputPath))
+    }
+  }
+
   func testBinaryFrameworkDependencyScan() throws {
     try withTemporaryDirectory { path in
       let (stdLibPath, shimsPath, toolchain, hostTriple) = try getDriverArtifactsForScanning()