From 3d914da26804afa844ab17efee2401c9be147a32 Mon Sep 17 00:00:00 2001 From: Pete Bentley Date: Thu, 11 Sep 2025 14:11:43 +0100 Subject: [PATCH 1/4] Move native build logic to a Kotlin class. No functional change but removes some cruft from the openjdk build.gradle. BoringSSL build directory is still a bit of a hack but I'll address that in a later, smaller PR. --- buildSrc/build.gradle.kts | 13 ++ .../src/main/kotlin/NativeBuildResolver.kt | 73 +++++++++ .../test/kotlin/NativeBuildResolverTest.kt | 61 +++++++ openjdk/build.gradle | 151 ++++-------------- 4 files changed, 175 insertions(+), 123 deletions(-) create mode 100644 buildSrc/build.gradle.kts create mode 100644 buildSrc/src/main/kotlin/NativeBuildResolver.kt create mode 100644 buildSrc/src/test/kotlin/NativeBuildResolverTest.kt diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts new file mode 100644 index 000000000..5de27ba73 --- /dev/null +++ b/buildSrc/build.gradle.kts @@ -0,0 +1,13 @@ +plugins { + `kotlin-dsl` +} + +repositories { + mavenCentral() + google() +} + +dependencies { + testImplementation(kotlin("test")) + testImplementation(gradleTestKit()) +} diff --git a/buildSrc/src/main/kotlin/NativeBuildResolver.kt b/buildSrc/src/main/kotlin/NativeBuildResolver.kt new file mode 100644 index 000000000..ec7e7959e --- /dev/null +++ b/buildSrc/src/main/kotlin/NativeBuildResolver.kt @@ -0,0 +1,73 @@ +import org.gradle.api.provider.Provider +import org.gradle.nativeplatform.platform.NativePlatform +import java.io.File +import org.gradle.api.file.Directory + +/** + * Gradle mostly uses Java os.arch names for architectures which feeds into default + * targetPlatform names. Notable exception Gradle 6.9.x reports MacOS/ARM as arm-v8. + * + * The Maven osdetector plugin (which we recommend to developers) uses different + * arch names, so that's what we need for artifacts. + * + * This class encapsulates both naming schemes as well as other per-platform information + * about native builds, more of which will migrate in here over time. + */ +enum class NativeBuildVariant( + val os: String, + val mavenArch: String, // osdetector / Maven architecture name + val gradleArch: String, // Gradle architecture, used for things like NDK or toolchain selection + val boringBuildDir: String = "build64", // Where to find prebuilt libcrypto + ) { + WINDOWS_X64("windows", "x86_64", "x86-64"), + LINUX_X64("linux", "x86_64", "x86-64"), + OSX_X64("osx", "x86_64", "x86-64", "build.x86"), + OSX_ARM64("osx", "aarch_64", "aarch64", "build.arm"); + + override fun toString(): String + = "" + + companion object { + fun find(os: String, arch: String) + = values().find { it.os == os && it.mavenArch == arch } + fun findForGradle(os: String, arch: String) + = values().find { it.os == os && it.gradleArch == arch } + fun findAll(os: String) = values().filter { it.os == os } + } +} + +data class NativeBuildInfo( + val buildDir: Provider, + private val variant: NativeBuildVariant +) { + val mavenClassifier: String = "${variant.os}-${variant.mavenArch}" + val targetPlatform: String = "${variant.os}_${variant.gradleArch}" + + val nativeResourcesDir: String + get() = File(buildDir.get().asFile, "$mavenClassifier/native-resources").absolutePath + + val jarNativeResourcesDir: String + get() = File(nativeResourcesDir, "META-INF/native").absolutePath + + val boringBuildDir + get() = variant.boringBuildDir + + override fun toString(): String + = "NativeBuildInfo" +} + +class NativeBuildResolver(private val buildDir: Provider) { + private fun wrap(variant: NativeBuildVariant?) = variant?.let { + NativeBuildInfo(buildDir, it) + } + + fun find(os: String, arch: String) = wrap(NativeBuildVariant.find(os, arch)) + + fun find(nativePlatform: NativePlatform) = wrap(NativeBuildVariant.findForGradle( + nativePlatform.operatingSystem.name, + nativePlatform.architecture.name)) + + fun findAll(os: String): List = NativeBuildVariant.findAll(os). map { + NativeBuildInfo(buildDir, it) + } +} diff --git a/buildSrc/src/test/kotlin/NativeBuildResolverTest.kt b/buildSrc/src/test/kotlin/NativeBuildResolverTest.kt new file mode 100644 index 000000000..93a3ff8ce --- /dev/null +++ b/buildSrc/src/test/kotlin/NativeBuildResolverTest.kt @@ -0,0 +1,61 @@ +import org.gradle.testfixtures.ProjectBuilder +import kotlin.test.* + +class NativeBuildResolverTest { + @Test + fun findByOsdetectorExact() { + assertEquals(NativeBuildVariant.OSX_ARM64, + NativeBuildVariant.find("osx", "aarch_64")) + assertEquals(NativeBuildVariant.LINUX_X64, + NativeBuildVariant.find("linux", "x86_64")) + } + + @Test + fun findByGradleExact() { + assertEquals(NativeBuildVariant.OSX_X64, + NativeBuildVariant.findForGradle("osx", "x86-64")) + assertEquals(NativeBuildVariant.OSX_ARM64, + NativeBuildVariant.findForGradle("osx", "aarch64")) + } + + @Test + fun findUnknownReturnsNull() { + assertNull(NativeBuildVariant.find("linux", "armv7")) + assertNull(NativeBuildVariant.findForGradle("windows", "aarch64")) + } + + @Test + fun findAllByOs() { + val osx = NativeBuildVariant.findAll("osx").toSet() + assertEquals(setOf(NativeBuildVariant.OSX_X64, NativeBuildVariant.OSX_ARM64), osx) + } + + @Test + fun computedStringsAreStable() { + assertEquals("osx-aarch_64", NativeBuildVariant.OSX_ARM64.let { "${it.os}-${it.mavenArch}" }) + assertEquals("osx_aarch64", NativeBuildVariant.OSX_ARM64.let { "${it.os}_${it.gradleArch}" }) + assertEquals("build.arm", NativeBuildVariant.OSX_ARM64.boringBuildDir) + assertEquals("build64", NativeBuildVariant.LINUX_X64.boringBuildDir) + } + + @Test + fun directoriesAreDerivedCorrectlyFromBuilddir() { + val tmp = createTempDir().apply { deleteOnExit() } + val project = ProjectBuilder.builder().withProjectDir(tmp).build() + val info = NativeBuildInfo(project.layout.buildDirectory, NativeBuildVariant.OSX_X64) + + assertTrue(info.nativeResourcesDir.endsWith("osx-x86_64/native-resources")) + assertTrue(info.jarNativeResourcesDir.endsWith("osx-x86_64/native-resources/META-INF/native")) + assertEquals("osx-x86_64", info.mavenClassifier) + assertEquals("osx_x86-64", info.targetPlatform) + } + + @Test fun resolver_wraps_variants() { + val project = ProjectBuilder.builder().build() + val resolver = NativeBuildResolver(project.layout.buildDirectory) + + val info = resolver.findAll("linux").single() // Only one for now + assertEquals("linux-x86_64", info.mavenClassifier) + assertEquals(project.layout.buildDirectory.get().asFile, info.buildDir.get().asFile) + } +} diff --git a/openjdk/build.gradle b/openjdk/build.gradle index b63b4c8de..f6659f708 100644 --- a/openjdk/build.gradle +++ b/openjdk/build.gradle @@ -14,123 +14,27 @@ import org.codehaus.groovy.runtime.InvokerHelper description = 'Conscrypt: OpenJdk' -// Gradle mostly uses Java os.arch names for architectures which feeds into default -// targetPlatform names. Notable exception Gradle 6.9.x reports MacOS/ARM as -// arm-v8. -// -// The Maven osdetector plugin (which we recommend to developers) uses different -// arch names, so that's what we need for artifacts. -// -// This class encapsulates both naming schemes as well as other per-platform information -// about native builds, more of which will migrate in here over time. -enum NativeBuildInfo { - WINDOWS_X86_64("windows", "x86_64"), - LINUX_X86_64("linux", "x86_64"), - MAC_X86_64("osx", "x86_64") { - String libDir() { - "build.x86" - } - }, - MAC_AARCH64("osx", "aarch_64") { - String libDir() { - "build.arm" - } - }; - - static String buildDir = "FIXME" // See below - - public final String os - public final String arch - - // Maps osdetector arch to Gradle equivalent. - private static final gradleArchMap = [ - "aarch_64": "aarch64", - "x86_64" : "x86-64", - ] - - NativeBuildInfo(String os, String arch) { - this.os = os - this.arch = arch - } - - // Classifier as generated by Maven osdetector. - String mavenClassifier() { - "${os}-${arch}" - } - - // Gradle equivalent to Maven arch - String gradleArch() { - gradleArch(arch) - } - - // Output directory for native resources - String nativeResourcesDir() { - "$buildDir/${mavenClassifier()}/native-resources" - } - - // Directory for native resources inside final jar. - String jarNativeResourcesDir() { - nativeResourcesDir() + '/META-INF/native' - } - - // Target platform identifier as used by Gradle - String targetPlatform() { - "${os}_${gradleArch()}" - } - - String libDir() { - "build64" - } - - static String gradleArch(String arch) { - gradleArchMap.get(arch) - } - - static NativeBuildInfo findForGradle(String os, String arch) { - values().find { - it.os == os && it.gradleArch() == arch - } - } - - static NativeBuildInfo find(String os, String arch) { - values().find { - it.os == os && it.arch == arch - } - } - - static NativeBuildInfo find(NativePlatform targetPlatform) { - String targetOS = targetPlatform.operatingSystem.name - String targetArch = targetPlatform.architecture.name - def result = findForGradle(targetOS, targetArch) - assert result != null : "Unknown target platform: ${targetOS}-${targetArch}" - result - } - - static findAll(String os) { - values().findAll { - it.os == os - } - } -} - -// TODO: There has to be a better way of accessing Gradle properties from Groovy code than this -NativeBuildInfo.buildDir = "$buildDir" - ext { jniSourceDir = "$rootDir/common/src/jni" assert file("$jniSourceDir").exists() // Decide which targets we should build and test - nativeBuilds = NativeBuildInfo.findAll("${osdetector.os}") - buildToTest = NativeBuildInfo.find("${osdetector.os}", "${osdetector.arch}") + nativeResolver = new NativeBuildResolver(project.layout.buildDirectory) + nativeBuilds = nativeResolver.findAll("${osdetector.os}") + buildToTest = nativeResolver.find("${osdetector.os}", "${osdetector.arch}") assert !nativeBuilds.isEmpty() : "No native builds selected." assert buildToTest != null : "No test build selected for os.arch = ${osdetector.arch}" // Compatibility with other sub-projects - preferredSourceSet = buildToTest.mavenClassifier() - preferredNativeFileDir = buildToTest.nativeResourcesDir() + preferredSourceSet = buildToTest.mavenClassifier + preferredNativeFileDir = buildToTest.nativeResourcesDir +} + +nativeBuilds.each { build -> + logger.warn("Building native JNI for $build") } +logger.warn("Testing against $buildToTest") // Since we're not taking a direct dependency on the constants module, we need to add an // explicit task dependency to make sure the code is generated. @@ -162,14 +66,14 @@ sourceSets { srcDirs += "${rootDir}/common/src/test/resources" // This shouldn't be needed but seems to help IntelliJ locate the native artifact. // srcDirs += preferredNativeFileDir - srcDirs += buildToTest.nativeResourcesDir() + srcDirs += buildToTest.nativeResourcesDir } } // Add the source sets for each of the native builds - nativeBuilds.each { nativeBuild -> - String sourceSetName = nativeBuild.mavenClassifier() - String nativeDir = nativeBuild.nativeResourcesDir() + nativeBuilds.each { nativeBuildInfo -> + String sourceSetName = nativeBuildInfo.mavenClassifier + String nativeDir = nativeBuildInfo.nativeResourcesDir // Main sources for the native build "$sourceSetName" { @@ -293,23 +197,23 @@ dependencies { platformCompileOnly sourceSets.main.output } -nativeBuilds.each { nativeBuild -> +nativeBuilds.each { nativeBuildInfo -> // Create the JAR task and add it's output to the published archives for this project - addNativeJar(nativeBuild) + addNativeJar(nativeBuildInfo) // Build the classes as part of the standard build. - classes.dependsOn sourceSets[nativeBuild.mavenClassifier()].classesTaskName + classes.dependsOn sourceSets[nativeBuildInfo.mavenClassifier].classesTaskName } // Adds a JAR task for the native library. -def addNativeJar(NativeBuildInfo nativeBuild) { +def addNativeJar(NativeBuildInfo nativeBuildInfo) { // Create a JAR for this configuration and add it to the output archives. - SourceSet sourceSet = sourceSets[nativeBuild.mavenClassifier()] + SourceSet sourceSet = sourceSets[nativeBuildInfo.mavenClassifier] def jarTask = tasks.register(sourceSet.jarTaskName, Jar) { Jar t -> // Depend on the regular classes task dependsOn classes manifest = jar.manifest - archiveClassifier = nativeBuild.mavenClassifier() + archiveClassifier = nativeBuildInfo.mavenClassifier from sourceSet.output + sourceSets.main.output @@ -392,8 +296,8 @@ model { components { // Builds the JNI library. conscrypt_openjdk_jni(NativeLibrarySpec) { - nativeBuilds.each { nativeBuild -> - targetPlatform nativeBuild.targetPlatform() + nativeBuilds.each { nativeBuildInfo -> + targetPlatform nativeBuildInfo.targetPlatform } sources { @@ -410,8 +314,9 @@ model { withType (SharedLibraryBinarySpec) { cppCompiler.define "CONSCRYPT_OPENJDK" def jdkIncludeDir = jniIncludeDir() - def nativeBuild = NativeBuildInfo.find(targetPlatform) - String libPath = "$boringsslHome/${nativeBuild.libDir()}" + def copy = targetPlatform + def nativeBuildInfo = nativeResolver.find(targetPlatform) + String libPath = boringsslHome + '/' + nativeBuildInfo.boringBuildDir if (toolChain in Clang || toolChain in Gcc) { cppCompiler.args "-Wall", @@ -503,8 +408,8 @@ model { tasks { t -> $.binaries.withType(SharedLibraryBinarySpec).each { binary -> - def nativeBuild = NativeBuildInfo.find(binary.targetPlatform) - def classifier = nativeBuild.mavenClassifier() + def nativeBuildInfo = nativeResolver.find(binary.targetPlatform) + def classifier = nativeBuildInfo.mavenClassifier def source = binary.sharedLibraryFile // Copies the native library to a resource location that will be included in the jar. @@ -514,7 +419,7 @@ model { // Rename the artifact to include the generated classifier rename '(.+)(\\.[^\\.]+)', "\$1-$classifier\$2" // Everything under will be included in the native jar. - into nativeBuild.jarNativeResourcesDir() + into nativeBuildInfo.jarNativeResourcesDir } processResources { dependsOn copyTask From dba4a75c0f92aa3439e4fad5db5930f769b74a03 Mon Sep 17 00:00:00 2001 From: Pete Bentley Date: Wed, 1 Oct 2025 13:38:03 +0100 Subject: [PATCH 2/4] Fix Windows paths...... --- buildSrc/src/test/kotlin/NativeBuildResolverTest.kt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/buildSrc/src/test/kotlin/NativeBuildResolverTest.kt b/buildSrc/src/test/kotlin/NativeBuildResolverTest.kt index 93a3ff8ce..6839e61f9 100644 --- a/buildSrc/src/test/kotlin/NativeBuildResolverTest.kt +++ b/buildSrc/src/test/kotlin/NativeBuildResolverTest.kt @@ -44,13 +44,15 @@ class NativeBuildResolverTest { val project = ProjectBuilder.builder().withProjectDir(tmp).build() val info = NativeBuildInfo(project.layout.buildDirectory, NativeBuildVariant.OSX_X64) - assertTrue(info.nativeResourcesDir.endsWith("osx-x86_64/native-resources")) - assertTrue(info.jarNativeResourcesDir.endsWith("osx-x86_64/native-resources/META-INF/native")) + assertTrue(info.nativeResourcesDir.replace('\\', '/') + .endsWith("osx-x86_64/native-resources")) + assertTrue(info.jarNativeResourcesDir.replace('\\', '/') + .endsWith("osx-x86_64/native-resources/META-INF/native")) assertEquals("osx-x86_64", info.mavenClassifier) assertEquals("osx_x86-64", info.targetPlatform) } - @Test fun resolver_wraps_variants() { + @Test fun resolverWrapsVariants() { val project = ProjectBuilder.builder().build() val resolver = NativeBuildResolver(project.layout.buildDirectory) From f191357fb7baa9d0bec249dc1e3acf598e1f6d86 Mon Sep 17 00:00:00 2001 From: Pete Bentley Date: Thu, 2 Oct 2025 09:25:19 +0100 Subject: [PATCH 3/4] Remove unused variable. --- openjdk/build.gradle | 1 - 1 file changed, 1 deletion(-) diff --git a/openjdk/build.gradle b/openjdk/build.gradle index f6659f708..028e48f4d 100644 --- a/openjdk/build.gradle +++ b/openjdk/build.gradle @@ -314,7 +314,6 @@ model { withType (SharedLibraryBinarySpec) { cppCompiler.define "CONSCRYPT_OPENJDK" def jdkIncludeDir = jniIncludeDir() - def copy = targetPlatform def nativeBuildInfo = nativeResolver.find(targetPlatform) String libPath = boringsslHome + '/' + nativeBuildInfo.boringBuildDir From 8d9cc21469330283195eba0a5a3d3b2579bb94ad Mon Sep 17 00:00:00 2001 From: Pete Bentley Date: Fri, 31 Oct 2025 15:19:37 +0000 Subject: [PATCH 4/4] Add a cross-compile flag to NativeBuildVariants. Will make things like ARM Linux simpler. --- buildSrc/src/main/kotlin/NativeBuildResolver.kt | 16 ++++++++++------ .../src/test/kotlin/NativeBuildResolverTest.kt | 5 +++-- openjdk/build.gradle | 2 +- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/buildSrc/src/main/kotlin/NativeBuildResolver.kt b/buildSrc/src/main/kotlin/NativeBuildResolver.kt index ec7e7959e..12cb63754 100644 --- a/buildSrc/src/main/kotlin/NativeBuildResolver.kt +++ b/buildSrc/src/main/kotlin/NativeBuildResolver.kt @@ -18,11 +18,12 @@ enum class NativeBuildVariant( val mavenArch: String, // osdetector / Maven architecture name val gradleArch: String, // Gradle architecture, used for things like NDK or toolchain selection val boringBuildDir: String = "build64", // Where to find prebuilt libcrypto + val crossCompile: Boolean = false // Whether to cross-compile on other archs for this OS ) { WINDOWS_X64("windows", "x86_64", "x86-64"), LINUX_X64("linux", "x86_64", "x86-64"), - OSX_X64("osx", "x86_64", "x86-64", "build.x86"), - OSX_ARM64("osx", "aarch_64", "aarch64", "build.arm"); + OSX_X64("osx", "x86_64", "x86-64", "build.x86", true), + OSX_ARM64("osx", "aarch_64", "aarch64", "build.arm", true); override fun toString(): String = "" @@ -32,7 +33,9 @@ enum class NativeBuildVariant( = values().find { it.os == os && it.mavenArch == arch } fun findForGradle(os: String, arch: String) = values().find { it.os == os && it.gradleArch == arch } - fun findAll(os: String) = values().filter { it.os == os } + fun findAll(os: String, arch: String) = values().filter { + it.os == os && (it.mavenArch == arch || it.crossCompile) + } } } @@ -67,7 +70,8 @@ class NativeBuildResolver(private val buildDir: Provider) { nativePlatform.operatingSystem.name, nativePlatform.architecture.name)) - fun findAll(os: String): List = NativeBuildVariant.findAll(os). map { - NativeBuildInfo(buildDir, it) - } + fun findAll(os: String, arch: String): List = + NativeBuildVariant.findAll(os, arch). map { + NativeBuildInfo(buildDir, it) + } } diff --git a/buildSrc/src/test/kotlin/NativeBuildResolverTest.kt b/buildSrc/src/test/kotlin/NativeBuildResolverTest.kt index 6839e61f9..80c8a8ea3 100644 --- a/buildSrc/src/test/kotlin/NativeBuildResolverTest.kt +++ b/buildSrc/src/test/kotlin/NativeBuildResolverTest.kt @@ -26,7 +26,7 @@ class NativeBuildResolverTest { @Test fun findAllByOs() { - val osx = NativeBuildVariant.findAll("osx").toSet() + val osx = NativeBuildVariant.findAll("osx", "aarch_64").toSet() assertEquals(setOf(NativeBuildVariant.OSX_X64, NativeBuildVariant.OSX_ARM64), osx) } @@ -56,7 +56,8 @@ class NativeBuildResolverTest { val project = ProjectBuilder.builder().build() val resolver = NativeBuildResolver(project.layout.buildDirectory) - val info = resolver.findAll("linux").single() // Only one for now + // There should only be a single Linux variant for now. + val info = resolver.findAll("linux", "x86_64").single() assertEquals("linux-x86_64", info.mavenClassifier) assertEquals(project.layout.buildDirectory.get().asFile, info.buildDir.get().asFile) } diff --git a/openjdk/build.gradle b/openjdk/build.gradle index 028e48f4d..32e24f4dc 100644 --- a/openjdk/build.gradle +++ b/openjdk/build.gradle @@ -20,7 +20,7 @@ ext { // Decide which targets we should build and test nativeResolver = new NativeBuildResolver(project.layout.buildDirectory) - nativeBuilds = nativeResolver.findAll("${osdetector.os}") + nativeBuilds = nativeResolver.findAll("${osdetector.os}", "${osdetector.arch}") buildToTest = nativeResolver.find("${osdetector.os}", "${osdetector.arch}") assert !nativeBuilds.isEmpty() : "No native builds selected."