KAFKA-19664 Support building with Java 25 (LTS release)

.github/actions/setup-gradle/action.yml

@@ -42,7 +42,7 @@ runs:
         distribution: temurin
         java-version: ${{ inputs.java-version }}
     - name: Setup Gradle
-      uses: gradle/actions/setup-gradle@94baf225fe0a508e581a564467443d0e2379123b # v4.3.0
+      uses: gradle/actions/setup-gradle@ed408507eac070d1f99cc633dbcf757c94c7933a # v4.4.3
       env:
         GRADLE_BUILD_ACTION_CACHE_DEBUG_ENABLED: true
       with:

.github/workflows/build.yml

@@ -127,7 +127,7 @@ jobs:
       - name: Setup Gradle
         uses: ./.github/actions/setup-gradle
         with:
-          java-version: 24
+          java-version: 17
           gradle-cache-read-only: ${{ !inputs.is-trunk }}
           gradle-cache-write-only: ${{ inputs.is-trunk }}
           develocity-access-key: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
@@ -181,7 +181,7 @@ jobs:
       fail-fast: false
       matrix:
         # If we change these, make sure to adjust ci-complete.yml
-        java: [ 24, 17 ]
+        java: [ 25, 17 ]
         run-flaky: [ true, false ]
         run-new: [ true, false ]
         exclude:
@@ -270,7 +270,7 @@ jobs:
           python .github/scripts/junit.py \
            --path build/junit-xml >> $GITHUB_STEP_SUMMARY
 
-  # This job downloads all the JUnit XML files and thread dumps from the JDK 24 test runs.
+  # This job downloads all the JUnit XML files and thread dumps from the JDK 25 test runs.
   # If any test job fails, we will not run this job. Also, if any thread dump artifacts
   # are present, this means there was a timeout in the tests and so we will not proceed
   # with catalog creation.
@@ -288,7 +288,7 @@ jobs:
       - name: Download Thread Dumps
         uses: actions/download-artifact@v4
         with:
-          pattern: junit-thread-dumps-24-*
+          pattern: junit-thread-dumps-25-*
           path: thread-dumps
           merge-multiple: true
       - name: Check For Thread Dump
@@ -302,7 +302,7 @@ jobs:
       - name: Download JUnit XMLs
         uses: actions/download-artifact@v4
         with:
-          pattern: junit-xml-24-*  # Only look at JDK 24 tests for the test catalog
+          pattern: junit-xml-25-*  # Only look at JDK 25 tests for the test catalog
           path: junit-xml
           merge-multiple: true
       - name: Collate Test Catalog

.github/workflows/ci-complete.yml

@@ -44,7 +44,7 @@ jobs:
       fail-fast: false
       matrix:
         # Make sure these match build.yml
-        java: [ 24, 17 ]
+        java: [ 25, 17 ]
         run-flaky: [ true, false ]
         run-new: [ true, false ]
         exclude:

README.md

@@ -13,7 +13,7 @@
 
 You need to have [Java](http://www.oracle.com/technetwork/java/javase/downloads/index.html) installed.
 
-We build and test Apache Kafka with 17 and 24. The `release` parameter in javac is set to `11` for the clients 
+We build and test Apache Kafka with 17 and 25. The `release` parameter in javac is set to `11` for the clients 
 and streams modules, and `17` for the rest, ensuring compatibility with their respective
 minimum Java versions. Similarly, the `release` parameter in scalac is set to `11` for the streams modules and `17`
 for the rest.

build.gradle

@@ -29,22 +29,21 @@ buildscript {
 }
 
 plugins {
-  id 'com.github.ben-manes.versions' version '0.48.0'
+  id 'com.github.ben-manes.versions' version '0.52.0'
   id 'idea'
   id 'jacoco'
   id 'java-library'
-  id 'org.owasp.dependencycheck' version '8.2.1'
+  id 'org.owasp.dependencycheck' version '12.1.3'
   id 'org.nosphere.apache.rat' version "0.8.1"
   id "io.swagger.core.v3.swagger-gradle-plugin" version "${swaggerVersion}"
 
-  id "com.github.spotbugs" version '6.2.3' apply false
+  id "com.github.spotbugs" version '6.2.5' apply false
   id 'org.scoverage' version '8.0.3' apply false
-  id 'com.gradleup.shadow' version '8.3.6' apply false
-  id 'com.diffplug.spotless' version "6.25.0"
+  id 'com.gradleup.shadow' version '9.0.2' apply false
+  id 'com.diffplug.spotless' version "7.2.1"
 }
 
 ext {
-  gradleVersion = versions.gradle
   minClientJavaVersion = 11
   minNonClientJavaVersion = 17
   modulesNeedingJava11 = [":clients", ":generator", ":streams", ":streams:test-utils", ":streams:examples", ":streams-scala", ":test-common:test-common-util"]
@@ -72,6 +71,13 @@ ext {
       "--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED"
     )
 
+  if (JavaVersion.current().isCompatibleWith(JavaVersion.VERSION_25)) {
+    // Spotbugs is not compatible with Java 25+ so Gradle related tasks are disabled
+    // until version can be upgraded: https://github.com/spotbugs/spotbugs/issues/3564
+    project.gradle.startParameter.excludedTaskNames.add("spotbugsMain")
+    project.gradle.startParameter.excludedTaskNames.add("spotbugsTest")
+  }    
+
   maxTestForks = project.hasProperty('maxParallelForks') ? maxParallelForks.toInteger() : Runtime.runtime.availableProcessors()
   maxScalacThreads = project.hasProperty('maxScalacThreads') ? maxScalacThreads.toInteger() :
       Math.min(Runtime.runtime.availableProcessors(), 8)
@@ -275,6 +281,7 @@ if (repo != null) {
     // And some of the files that we have checked in should also be excluded from this check
     excludes.addAll([
         '**/.git/**',
+        '**/.gitkeep',
         '**/build/**',
         '.github/pull_request_template.md',
         'CONTRIBUTING.md',
@@ -299,7 +306,7 @@ if (repo != null) {
 } else {
   rat.enabled = false
 }
-println("Starting build with version $version (commit id ${commitId == null ? "null" : commitId.take(8)}) using Gradle $gradleVersion, Java ${JavaVersion.current()} and Scala ${versions.scala}")
+println("Starting build with version $version (commit id ${commitId == null ? "null" : commitId.take(8)}) using Gradle $versions.gradle, Java ${JavaVersion.current()} and Scala ${versions.scala}")
 println("Build properties: ignoreFailures=$userIgnoreFailures, maxParallelForks=$maxTestForks, maxScalacThreads=$maxScalacThreads, maxTestRetries=$userMaxTestRetries")
 
 subprojects {
@@ -330,6 +337,23 @@ subprojects {
     tasks.register('uploadArchives').configure { dependsOn(publish) }
   }
 
+    def  submodulesToIncludeInDistribution = ['core', 'tools', 'trogdor', 'shell', 'api', 'runtime', 'transforms',
+                                              'json', 'file', 'basic-auth-extension', 'mirror', 'mirror-client',
+                                              'streams', 'streams-scala', 'test-utils', 'examples', 'tools-api']
+
+    def shouldModuleResolveRuntimeClasspath = (submodulesToIncludeInDistribution.contains(project.name))
+
+    if (shouldModuleResolveRuntimeClasspath) {
+        configurations.create('resolvedRuntimeClasspath') {
+            extendsFrom(configurations.runtimeClasspath)
+            canBeResolved = true
+        }
+    }
+
+    tasks.withType(AbstractTestTask).configureEach {
+      failOnNoDiscoveredTests = false
+    }
+
   // apply the eclipse plugin only to subprojects that hold code. 'connect' is just a folder.
   if (!project.name.equals('connect')) {
     apply plugin: 'eclipse'
@@ -1170,71 +1194,6 @@ project(':core') {
     standardOutput = new File(generatedDocsDir, "producer_metrics.html").newOutputStream()
   }
 
-  task siteDocsTar(dependsOn: ['genProtocolErrorDocs', 'genProtocolTypesDocs', 'genProtocolApiKeyDocs', 'genProtocolMessageDocs',
-                               'genAdminClientConfigDocs', 'genProducerConfigDocs', 'genConsumerConfigDocs',
-                               'genKafkaConfigDocs', 'genTopicConfigDocs', 'genGroupConfigDocs',
-                               ':connect:runtime:genConnectConfigDocs', ':connect:runtime:genConnectTransformationDocs',
-                               ':connect:runtime:genConnectPredicateDocs',
-                               ':connect:runtime:genSinkConnectorConfigDocs', ':connect:runtime:genSourceConnectorConfigDocs',
-                               ':streams:genStreamsConfigDocs', 'genConsumerMetricsDocs', 'genProducerMetricsDocs',
-                               ':connect:runtime:genConnectMetricsDocs', ':connect:runtime:genConnectOpenAPIDocs',
-                               ':connect:mirror:genMirrorSourceConfigDocs', ':connect:mirror:genMirrorCheckpointConfigDocs',
-                               ':connect:mirror:genMirrorHeartbeatConfigDocs', ':connect:mirror:genMirrorConnectorConfigDocs',
-                               ':storage:genRemoteLogManagerConfigDoc', ':storage:genRemoteLogMetadataManagerConfigDoc'], type: Tar) {
-    archiveClassifier = 'site-docs'
-    compression = Compression.GZIP
-    from project.file("$rootDir/docs")
-    into 'site-docs'
-    duplicatesStrategy 'exclude'
-  }
-
-  tasks.create(name: "releaseTarGz", dependsOn: configurations.archives.artifacts, type: Tar) {
-    into "kafka_${versions.baseScala}-${archiveVersion.get()}"
-    compression = Compression.GZIP
-    from(project.file("$rootDir/bin")) { into "bin/" }
-    from(project.file("$rootDir/config")) { into "config/" }
-    from(project.file("$rootDir/licenses")) { into "licenses/" }
-    from "$rootDir/LICENSE-binary" rename {String filename -> filename.replace("-binary", "")}
-    from "$rootDir/NOTICE-binary" rename {String filename -> filename.replace("-binary", "")}
-    from(configurations.runtimeClasspath) { into("libs/") }
-    from(configurations.archives.artifacts.files) { into("libs/") }
-    from(configurations.releaseOnly) { into("libs/") }
-    from(project.siteDocsTar) { into("site-docs/") }
-    from(project(':tools').jar) { into("libs/") }
-    from(project(':tools').configurations.runtimeClasspath) { into("libs/") }
-    from(project(':trogdor').jar) { into("libs/") }
-    from(project(':trogdor').configurations.runtimeClasspath) { into("libs/") }
-    from(project(':shell').jar) { into("libs/") }
-    from(project(':shell').configurations.runtimeClasspath) { into("libs/") }
-    from(project(':connect:api').jar) { into("libs/") }
-    from(project(':connect:api').configurations.runtimeClasspath) { into("libs/") }
-    from(project(':connect:runtime').jar) { into("libs/") }
-    from(project(':connect:runtime').configurations.runtimeClasspath) { into("libs/") }
-    from(project(':connect:transforms').jar) { into("libs/") }
-    from(project(':connect:transforms').configurations.runtimeClasspath) { into("libs/") }
-    from(project(':connect:json').jar) { into("libs/") }
-    from(project(':connect:json').configurations.runtimeClasspath) { into("libs/") }
-    from(project(':connect:file').jar) { into("libs/") }
-    from(project(':connect:file').configurations.runtimeClasspath) { into("libs/") }
-    from(project(':connect:basic-auth-extension').jar) { into("libs/") }
-    from(project(':connect:basic-auth-extension').configurations.runtimeClasspath) { into("libs/") }
-    from(project(':connect:mirror').jar) { into("libs/") }
-    from(project(':connect:mirror').configurations.runtimeClasspath) { into("libs/") }
-    from(project(':connect:mirror-client').jar) { into("libs/") }
-    from(project(':connect:mirror-client').configurations.runtimeClasspath) { into("libs/") }
-    from(project(':streams').jar) { into("libs/") }
-    from(project(':streams').configurations.runtimeClasspath) { into("libs/") }
-    from(project(':streams:streams-scala').jar) { into("libs/") }
-    from(project(':streams:streams-scala').configurations.runtimeClasspath) { into("libs/") }
-    from(project(':streams:test-utils').jar) { into("libs/") }
-    from(project(':streams:test-utils').configurations.runtimeClasspath) { into("libs/") }
-    from(project(':streams:examples').jar) { into("libs/") }
-    from(project(':streams:examples').configurations.runtimeClasspath) { into("libs/") }
-    from(project(':tools:tools-api').jar) { into("libs/") }
-    from(project(':tools:tools-api').configurations.runtimeClasspath) { into("libs/") }
-    duplicatesStrategy 'exclude'
-  }
-
   jar {
     dependsOn copyDependantLibs
   }
@@ -3904,6 +3863,109 @@ project(':connect:test-plugins') {
   }
 }
 
+project(':distribution') {
+  base {
+    archivesName = "kafka_${versions.baseScala}"
+  }
+
+  configurations {
+    includeIntoDistribution
+  }
+
+  dependencies {
+    includeIntoDistribution project(path: ':core', configuration: 'releaseOnly')
+    includeIntoDistribution project(path: ':core', configuration: 'archives')
+    includeIntoDistribution project(path: ':core', configuration: 'resolvedRuntimeClasspath')
+    includeIntoDistribution project(path: ':tools', configuration: 'archives')
+    includeIntoDistribution project(path: ':tools', configuration: 'resolvedRuntimeClasspath')
+    includeIntoDistribution project(path: ':trogdor', configuration: 'archives')
+    includeIntoDistribution project(path: ':trogdor', configuration: 'resolvedRuntimeClasspath')
+    includeIntoDistribution project(path: ':shell', configuration: 'archives')
+    includeIntoDistribution project(path: ':shell', configuration: 'resolvedRuntimeClasspath')
+    includeIntoDistribution project(path: ':connect:api', configuration: 'archives')
+    includeIntoDistribution project(path: ':connect:api', configuration: 'resolvedRuntimeClasspath')
+    includeIntoDistribution project(path: ':connect:runtime', configuration: 'archives')
+    includeIntoDistribution project(path: ':connect:runtime', configuration: 'resolvedRuntimeClasspath')
+    includeIntoDistribution project(path: ':connect:transforms', configuration: 'archives')
+    includeIntoDistribution project(path: ':connect:transforms', configuration: 'resolvedRuntimeClasspath')
+    includeIntoDistribution project(path: ':connect:json', configuration: 'archives')
+    includeIntoDistribution project(path: ':connect:json', configuration: 'resolvedRuntimeClasspath')
+    includeIntoDistribution project(path: ':connect:file', configuration: 'archives')
+    includeIntoDistribution project(path: ':connect:file', configuration: 'resolvedRuntimeClasspath')
+    includeIntoDistribution project(path: ':connect:basic-auth-extension', configuration: 'archives')
+    includeIntoDistribution project(path: ':connect:basic-auth-extension', configuration: 'resolvedRuntimeClasspath')
+    includeIntoDistribution project(path: ':connect:mirror', configuration: 'archives')
+    includeIntoDistribution project(path: ':connect:mirror', configuration: 'resolvedRuntimeClasspath')
+    includeIntoDistribution project(path: ':connect:mirror-client', configuration: 'archives')
+    includeIntoDistribution project(path: ':connect:mirror-client', configuration: 'resolvedRuntimeClasspath')
+    includeIntoDistribution project(path: ':streams', configuration: 'archives')
+    includeIntoDistribution project(path: ':streams', configuration: 'resolvedRuntimeClasspath')
+    includeIntoDistribution project(path: ':streams:streams-scala', configuration: 'archives')
+    includeIntoDistribution project(path: ':streams:streams-scala', configuration: 'resolvedRuntimeClasspath')
+    includeIntoDistribution project(path: ':streams:test-utils', configuration: 'archives')
+    includeIntoDistribution project(path: ':streams:test-utils', configuration: 'resolvedRuntimeClasspath')
+    includeIntoDistribution project(path: ':streams:examples', configuration: 'archives')
+    includeIntoDistribution project(path: ':streams:examples', configuration: 'resolvedRuntimeClasspath')
+    includeIntoDistribution project(path: ':tools:tools-api', configuration: 'archives')
+    includeIntoDistribution project(path: ':tools:tools-api', configuration: 'resolvedRuntimeClasspath')     
+  }
+
+  task siteDocsTar(dependsOn: [':core:genProtocolErrorDocs', ':core:genProtocolTypesDocs', ':core:genProtocolApiKeyDocs', ':core:genProtocolMessageDocs',
+                               ':core:genAdminClientConfigDocs', ':core:genProducerConfigDocs', ':core:genConsumerConfigDocs',
+                               ':core:genKafkaConfigDocs', ':core:genTopicConfigDocs', ':core:genGroupConfigDocs',
+                               ':connect:runtime:genConnectConfigDocs', ':connect:runtime:genConnectTransformationDocs',
+                               ':connect:runtime:genConnectPredicateDocs',
+                               ':connect:runtime:genSinkConnectorConfigDocs', ':connect:runtime:genSourceConnectorConfigDocs',
+                               ':streams:genStreamsConfigDocs', ':core:genConsumerMetricsDocs', ':core:genProducerMetricsDocs',
+                               ':connect:runtime:genConnectMetricsDocs', ':connect:runtime:genConnectOpenAPIDocs',
+                               ':connect:mirror:genMirrorSourceConfigDocs', ':connect:mirror:genMirrorCheckpointConfigDocs',
+                               ':connect:mirror:genMirrorHeartbeatConfigDocs', ':connect:mirror:genMirrorConnectorConfigDocs',
+                               ':storage:genRemoteLogManagerConfigDoc', ':storage:genRemoteLogMetadataManagerConfigDoc'], type: Tar) {
+    archiveClassifier = 'site-docs'
+    compression = Compression.GZIP
+    from project.file("$rootDir/docs")
+    into 'site-docs'
+    destinationDirectory = project(':core').layout.buildDirectory.dir('distributions')
+    duplicatesStrategy 'exclude'
+    preserveFileTimestamps = true
+  }
+
+  tasks.create(name: "releaseTarGz", dependsOn: project(':core').configurations.archives.artifacts, type: Tar) {
+    into "kafka_${versions.baseScala}-${archiveVersion.get()}"
+    compression = Compression.GZIP
+    from(project.file("$rootDir/bin")) { into "bin/" }
+    from(project.file("$rootDir/config")) { into "config/" }
+    from(project.file("$rootDir/licenses")) { into "licenses/" }
+    from "$rootDir/LICENSE-binary" rename {String filename -> filename.replace("-binary", "")}
+    from "$rootDir/NOTICE-binary" rename {String filename -> filename.replace("-binary", "")}
+    from(project.siteDocsTar) { into("site-docs/") }
+    from(configurations.includeIntoDistribution) { into("libs/") }
+    destinationDirectory = project(':core').layout.buildDirectory.dir('distributions')
+    duplicatesStrategy 'exclude'
+    preserveFileTimestamps = true
+    filePermissions {
+        user {
+            read = true
+            write = true
+        }
+        group.read = true
+        other.read = true
+    }
+    dirPermissions {
+        unix('rwxr-xr-x')
+    }
+    eachFile {
+        if (name.endsWith( ".sh")) {
+            permissions {
+                user.execute = true
+                group.execute = true
+                other.execute = true
+            }
+        }
+    }
+  }
+}
+
 task aggregatedJavadoc(type: Javadoc, dependsOn: compileJava) {
   def projectsWithJavadoc = subprojects.findAll { it.javadoc.enabled }
   source = projectsWithJavadoc.collect { it.sourceSets.main.allJava }

gradle/dependencies.gradle

@@ -61,8 +61,8 @@ versions += [
   commonsLang: "3.18.0",
   commonsValidator: "1.9.0",
   classgraph: "4.8.179",
-  gradle: "8.14.3",
-  grgit: "4.1.1",
+  gradle: "9.1.0",
+  grgit: "5.3.0",
   httpclient: "4.5.14",
   jackson: "2.19.0",
   jacoco: "0.8.13",
@@ -126,7 +126,7 @@ versions += [
   snappy: "1.1.10.7",
   spotbugs: "4.9.4",
   mockOAuth2Server: "2.2.1",
-  zinc: "1.9.2",
+  zinc: "1.10.8",
   // When updating the zstd version, please do as well in docker/native/native-image-configs/resource-config.json
   // Also make sure the compression levels in org.apache.kafka.common.record.CompressionType are still valid
   zstd: "1.5.6-10",

gradle/wrapper/gradle-wrapper.properties

@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=bd71102213493060956ec229d946beee57158dbd89d0e62b91bca0fa2c5f3531
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-bin.zip
+distributionSha256Sum=a17ddd85a26b6a7f5ddb71ff8b05fc5104c0202c6e64782429790c933686c806
+distributionUrl=https\://services.gradle.org/distributions/gradle-9.1.0-bin.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME