Skip to content

Adopt uniquely-named driver response files #809

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Sep 25, 2025
Merged

Adopt uniquely-named driver response files #809

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 27 additions & 10 deletions Sources/SWBCore/LibSwiftDriver/PlannedBuild.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,8 @@ public struct SwiftDriverJob: Serializable, CustomDebugStringConvertible {
public let outputs: [Path]
/// The command line to execute for this job
public let commandLine: [SWBUtil.ByteString]
/// The signature uniquely identifying the command line, looking through any indirection through response files.
public let commandLineSignature: SWBUtil.ByteString
/// Cache keys for the swift-frontend invocation (one key per output producing input)
public let cacheKeys: [String]

Expand All @@ -91,27 +93,43 @@ public struct SwiftDriverJob: Serializable, CustomDebugStringConvertible {
self.displayInputs = try job.displayInputs.map { try Path(resolver.resolve(.path($0.file))) }
self.outputs = try job.outputs.map { try Path(resolver.resolve(.path($0.file))) }
self.descriptionForLifecycle = job.descriptionForLifecycle
if categorizer.isExplicitDependencyBuild {
self.commandLine = try explicitModulesResolver.resolveArgumentList(for: job, useResponseFiles: .heuristic).map { ByteString(encodingAsUTF8: $0) }
self.kind = .explicitModule(uniqueID: commandLine.hashValue)
} else {
self.commandLine = try resolver.resolveArgumentList(for: job, useResponseFiles: .heuristic).map { ByteString(encodingAsUTF8: $0) }
self.kind = .target
let chosenResolver = categorizer.isExplicitDependencyBuild ? explicitModulesResolver : resolver
let args: ResolvedCommandLine = try chosenResolver.resolveArgumentList(for: job, useResponseFiles: .heuristic)
switch args {
case .plain(let args):
self.commandLine = args.map { ByteString(encodingAsUTF8: $0) }
let ctx = InsecureHashContext()
for arg in args {
ctx.add(string: arg)
}
self.commandLineSignature = ctx.signature
self.kind = categorizer.isExplicitDependencyBuild ? .explicitModule(uniqueID: args.hashValue) : .target
case .usingResponseFile(resolved: let args, responseFileContents: let responseFileContents):
// When using a response file, jobs should be uniqued based on the contents of the response file
self.commandLine = args.map { ByteString(encodingAsUTF8: $0) }
let ctx = InsecureHashContext()
for arg in responseFileContents {
ctx.add(string: arg)
}
self.commandLineSignature = ctx.signature
self.kind = categorizer.isExplicitDependencyBuild ? .explicitModule(uniqueID: responseFileContents.hashValue) : .target
}

self.cacheKeys = job.outputCacheKeys.reduce(into: [String]()) { result, key in
result.append(key.value)
}.sorted()
}

public func serialize<T>(to serializer: T) where T : Serializer {
serializer.serializeAggregate(9) {
serializer.serializeAggregate(10) {
serializer.serialize(kind)
serializer.serialize(ruleInfoType)
serializer.serialize(moduleName)
serializer.serialize(inputs)
serializer.serialize(displayInputs)
serializer.serialize(outputs)
serializer.serialize(commandLine)
serializer.serialize(commandLineSignature)
serializer.serialize(descriptionForLifecycle)
serializer.serialize(cacheKeys)
}
Expand All @@ -126,6 +144,7 @@ public struct SwiftDriverJob: Serializable, CustomDebugStringConvertible {
try self.displayInputs = deserializer.deserialize()
try self.outputs = deserializer.deserialize()
try self.commandLine = deserializer.deserialize()
try self.commandLineSignature = deserializer.deserialize()
try self.descriptionForLifecycle = deserializer.deserialize()
try self.cacheKeys = deserializer.deserialize()
}
Expand Down Expand Up @@ -173,9 +192,7 @@ extension LibSwiftDriver {
self.dependencies = dependencies
self.workingDirectory = workingDirectory
let md5 = InsecureHashContext()
for arg in driverJob.commandLine {
md5.add(bytes: arg)
}
md5.add(bytes: driverJob.commandLineSignature)
md5.add(string: workingDirectory.str)
md5.add(number: dependencies.hashValue)
self.signature = md5.signature
Expand Down
38 changes: 29 additions & 9 deletions Tests/SWBBuildSystemTests/SwiftDriverTests.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1441,6 +1441,7 @@ fileprivate struct SwiftDriverTests: CoreBasedTests {
children: [
TestFile("file1.swift"),
TestFile("file2.swift"),
TestFile("file3.swift"),
]),
buildConfigurations: [
TestBuildConfiguration(
Expand All @@ -1451,11 +1452,23 @@ fileprivate struct SwiftDriverTests: CoreBasedTests {
"BUILD_VARIANTS": "normal",
"SWIFT_USE_INTEGRATED_DRIVER": "YES",
"SWIFT_ENABLE_EXPLICIT_MODULES": "YES",
"BUILD_LIBRARY_FOR_DISTRIBUTION": "YES",
// Force use of a response file
"GCC_PREPROCESSOR_DEFINITIONS": Array(repeating: "ABCD=1", count: 10000).joined(separator: " ")
"GCC_PREPROCESSOR_DEFINITIONS": Array(repeating: "ABCD=1", count: 10000).joined(separator: " "),
"OTHER_SWIFT_FLAGS": "-Xfrontend -module-load-mode -Xfrontend only-interface"
])
],
targets: [
TestStandardTarget("TargetC",
type: .framework,
buildConfigurations: [
TestBuildConfiguration("Debug"),
],
buildPhases: [
TestSourcesBuildPhase([
"file3.swift",
]),
]),
TestStandardTarget("TargetA",
type: .framework,
buildConfigurations: [
Expand All @@ -1465,7 +1478,7 @@ fileprivate struct SwiftDriverTests: CoreBasedTests {
TestSourcesBuildPhase([
"file1.swift",
]),
], dependencies: ["TargetB"]),
], dependencies: ["TargetB", "TargetC"]),
TestStandardTarget("TargetB",
type: .framework,
buildConfigurations: [
Expand All @@ -1475,12 +1488,11 @@ fileprivate struct SwiftDriverTests: CoreBasedTests {
TestSourcesBuildPhase([
"file2.swift"
]),
])
], dependencies: ["TargetC"])
])
])

let tester = try await BuildOperationTester(getCore(), testWorkspace, simulated: false)
tester.userInfo = tester.userInfo.withAdditionalEnvironment(environment: ["SWIFT_FORCE_MODULE_LOADING": "only-interface"])
try tester.fs.createDirectory(moduleCacheDir)
let parameters = BuildParameters(configuration: "Debug")
let targetsToBuild = tester.workspace.projects.flatMap { project in project.targets.map({ BuildRequest.BuildTargetInfo(parameters: parameters, target: $0) }) }
Expand All @@ -1491,21 +1503,29 @@ fileprivate struct SwiftDriverTests: CoreBasedTests {
try await tester.fs.writeFileContents(SRCROOTA.join("Sources/file1.swift")) { file in
file <<<
"""
import Foundation
import TargetC
func foo() { baz() }
"""
}
try await tester.fs.writeFileContents(SRCROOTA.join("Sources/file2.swift")) { file in
file <<<
"""
import Foundation
import TargetC
func bar() { baz() }
"""
}
try await tester.fs.writeFileContents(SRCROOTA.join("Sources/file3.swift")) { file in
file <<<
"""
public func baz() {}
"""
}

try await tester.checkBuild(runDestination: .macOS, buildRequest: buildRequest, persistent: true) { results in
results.checkNoErrors()
results.checkTasks(.matchRulePattern(["SwiftExplicitDependencyCompileModuleFromInterface", "normal", .any, .contains("SwiftExplicitPrecompiledModules/Foundation-")])) { compileFoundationTasks in
// We only expect to need one variant of Foundation
#expect(compileFoundationTasks.count == 1)
results.checkTasks(.matchRulePattern(["SwiftExplicitDependencyCompileModuleFromInterface", .any, .contains("SwiftExplicitPrecompiledModules/TargetC-")])) { compileTasks in
// We only expect to need one variant of TargetC
#expect(compileTasks.count == 1)
}
}
}
Expand Down
Loading