From 9305e0b336f977c31730014f2032b5eff3f49c6c Mon Sep 17 00:00:00 2001 From: dejan2609 Date: Thu, 7 Aug 2025 23:27:36 +0200 Subject: [PATCH 01/10] KAFKA-19591 solving issues with `gradlew` content: an old template version (i.e. file `unixStartScript.txt`) was being referenced --- gradlew | 38 ++++++++++++++++++++------------------ wrapper.gradle | 4 +++- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/gradlew b/gradlew index f4bb3360e17ee..39803762879c4 100755 --- a/gradlew +++ b/gradlew @@ -15,6 +15,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# SPDX-License-Identifier: Apache-2.0 +# ############################################################################## # @@ -55,7 +57,7 @@ # Darwin, MinGW, and NonStop. # # (3) This script is generated from the Groovy template -# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt # within the Gradle project. # # You can find Gradle at https://github.com/gradle/gradle/. @@ -84,7 +86,7 @@ done # shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} # Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) -APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum @@ -112,20 +114,7 @@ case "$( uname )" in #( NONSTOP* ) nonstop=true ;; esac - -# Loop in case we encounter an error. -for attempt in 1 2 3; do - if [ ! -e "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" ]; then - if ! curl -s -S --retry 3 -L -o "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" "https://raw.githubusercontent.com/gradle/gradle/v8.14.3/gradle/wrapper/gradle-wrapper.jar"; then - rm -f "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" - # Pause for a bit before looping in case the server throttled us. - sleep 5 - continue - fi - fi -done - -CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar +CLASSPATH="\\\"\\\"" # Determine the Java command to use to start the JVM. @@ -212,11 +201,24 @@ if "$cygwin" || "$msys" ; then fi + +# Loop in case we encounter an error. +for attempt in 1 2 3; do + if [ ! -e "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" ]; then + if ! curl -s -S --retry 3 -L -o "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" "https://raw.githubusercontent.com/gradle/gradle/v8.14.1/gradle/wrapper/gradle-wrapper.jar"; then + rm -f "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" + # Pause for a bit before looping in case the server throttled us. + sleep 5 + continue + fi + fi +done + # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' # Collect all arguments for the java command: -# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, # and any embedded shellness will be escaped. # * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be # treated as '${Hostname}' itself on the command line. @@ -224,7 +226,7 @@ DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' set -- \ "-Dorg.gradle.appname=$APP_BASE_NAME" \ -classpath "$CLASSPATH" \ - org.gradle.wrapper.GradleWrapperMain \ + -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \ "$@" # Stop when "xargs" is not available. diff --git a/wrapper.gradle b/wrapper.gradle index d64aeab4de942..0e0d6da201123 100644 --- a/wrapper.gradle +++ b/wrapper.gradle @@ -59,13 +59,15 @@ task bootstrapWrapper() { done """.stripIndent() + String putBootstrapStringAbove = "# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script." + def wrapperScript = wrapper.scriptFile def wrapperLines = wrapperScript.readLines() wrapperScript.withPrintWriter { out -> def bootstrapWritten = false wrapperLines.each { line -> // Print the wrapper bootstrap before the first usage of the wrapper jar. - if (!bootstrapWritten && line.contains("gradle-wrapper.jar")) { + if (!bootstrapWritten && line.contains(putBootstrapStringAbove)) { out.println(bootstrapString) bootstrapWritten = true } From 14203aa526f6bd84d6e30c4bf40de48958c8fbee Mon Sep 17 00:00:00 2001 From: dejan2609 Date: Thu, 21 Aug 2025 21:46:47 +0200 Subject: [PATCH 02/10] KAFKA-19174 Gradle version upgrade (8 -->> 9); build logic is refactored details: - gradle: 8.14.3 -->> 9.0.0 - https://github.com/gradle/gradle/releases/tag/v9.0.0 - https://gradle.org/whats-new/gradle-9 - https://docs.gradle.org/9.0.0/userguide/upgrading_major_version_9.html#changes_major_9 - shadow plugin: 8.3.6 -->> 9.0.2 - https://github.com/GradleUp/shadow/releases/tag/9.0.0 - https://github.com/GradleUp/shadow/releases/tag/9.0.1 - https://github.com/GradleUp/shadow/releases/tag/9.0.2 refactoring (performed in order to accomodate Gradle 9 breaking changes): - tasks `siteDocsTar` and `releaseTarGz` are moved out of `core` (and into a new dummy module: `distribution`); note that distribution destination folder stays as-is (:core) - `preserveFileTimestamps` Tar taks property value explicitly set to `true` (default values: Gradle 8: `true`, Gradle 9: 'false') - files and folders permissions are introduced - git placeholder file (.gitkeep) is added in order to preserve `distribution` folder (Apache Rat check for that file is skipped) - `wrapper.gradle` changes: no need to check number of dots for Gradle versions >=9 anymore - redundant 'gradleVersion' ext property is removed (in favor of 'versions.gradle' usage) --- build.gradle | 187 ++++++++++++++--------- distribution/.gitkeep | 0 gradle/dependencies.gradle | 2 +- gradle/wrapper/gradle-wrapper.properties | 4 +- gradlew | 2 +- settings.gradle | 1 + wrapper.gradle | 6 +- 7 files changed, 126 insertions(+), 76 deletions(-) create mode 100644 distribution/.gitkeep diff --git a/build.gradle b/build.gradle index 26ca93d0738e8..60b8b1f72569d 100644 --- a/build.gradle +++ b/build.gradle @@ -39,12 +39,11 @@ plugins { id "com.github.spotbugs" version '6.2.3' apply false id 'org.scoverage' version '8.0.3' apply false - id 'com.gradleup.shadow' version '8.3.6' apply false + id 'com.gradleup.shadow' version '9.0.2' apply false id 'com.diffplug.spotless' version "6.25.0" } ext { - gradleVersion = versions.gradle minClientJavaVersion = 11 minNonClientJavaVersion = 17 modulesNeedingJava11 = [":clients", ":generator", ":streams", ":streams:test-utils", ":streams:examples", ":streams-scala", ":test-common:test-common-util"] @@ -275,6 +274,7 @@ if (repo != null) { // And some of the files that we have checked in should also be excluded from this check excludes.addAll([ '**/.git/**', + '**/.gitkeep', '**/build/**', '.github/pull_request_template.md', 'CONTRIBUTING.md', @@ -299,7 +299,7 @@ if (repo != null) { } else { rat.enabled = false } -println("Starting build with version $version (commit id ${commitId == null ? "null" : commitId.take(8)}) using Gradle $gradleVersion, Java ${JavaVersion.current()} and Scala ${versions.scala}") +println("Starting build with version $version (commit id ${commitId == null ? "null" : commitId.take(8)}) using Gradle $versions.gradle, Java ${JavaVersion.current()} and Scala ${versions.scala}") println("Build properties: ignoreFailures=$userIgnoreFailures, maxParallelForks=$maxTestForks, maxScalacThreads=$maxScalacThreads, maxTestRetries=$userMaxTestRetries") subprojects { @@ -330,6 +330,19 @@ subprojects { tasks.register('uploadArchives').configure { dependsOn(publish) } } + def submodulesToIncludeInDistribution = ['core', 'tools', 'trogdor', 'shell', 'api', 'runtime', 'transforms', + 'json', 'file', 'basic-auth-extension', 'mirror', 'mirror-client', + 'streams', 'streams-scala', 'test-utils', 'examples', 'tools-api'] + + def shouldModuleResolveRuntimeClasspath = (submodulesToIncludeInDistribution.contains(project.name)) + + if (shouldModuleResolveRuntimeClasspath) { + configurations.create('resolvedRuntimeClasspath') { + extendsFrom(configurations.runtimeClasspath) + canBeResolved = true + } + } + // apply the eclipse plugin only to subprojects that hold code. 'connect' is just a folder. if (!project.name.equals('connect')) { apply plugin: 'eclipse' @@ -1170,71 +1183,6 @@ project(':core') { standardOutput = new File(generatedDocsDir, "producer_metrics.html").newOutputStream() } - task siteDocsTar(dependsOn: ['genProtocolErrorDocs', 'genProtocolTypesDocs', 'genProtocolApiKeyDocs', 'genProtocolMessageDocs', - 'genAdminClientConfigDocs', 'genProducerConfigDocs', 'genConsumerConfigDocs', - 'genKafkaConfigDocs', 'genTopicConfigDocs', 'genGroupConfigDocs', - ':connect:runtime:genConnectConfigDocs', ':connect:runtime:genConnectTransformationDocs', - ':connect:runtime:genConnectPredicateDocs', - ':connect:runtime:genSinkConnectorConfigDocs', ':connect:runtime:genSourceConnectorConfigDocs', - ':streams:genStreamsConfigDocs', 'genConsumerMetricsDocs', 'genProducerMetricsDocs', - ':connect:runtime:genConnectMetricsDocs', ':connect:runtime:genConnectOpenAPIDocs', - ':connect:mirror:genMirrorSourceConfigDocs', ':connect:mirror:genMirrorCheckpointConfigDocs', - ':connect:mirror:genMirrorHeartbeatConfigDocs', ':connect:mirror:genMirrorConnectorConfigDocs', - ':storage:genRemoteLogManagerConfigDoc', ':storage:genRemoteLogMetadataManagerConfigDoc'], type: Tar) { - archiveClassifier = 'site-docs' - compression = Compression.GZIP - from project.file("$rootDir/docs") - into 'site-docs' - duplicatesStrategy 'exclude' - } - - tasks.create(name: "releaseTarGz", dependsOn: configurations.archives.artifacts, type: Tar) { - into "kafka_${versions.baseScala}-${archiveVersion.get()}" - compression = Compression.GZIP - from(project.file("$rootDir/bin")) { into "bin/" } - from(project.file("$rootDir/config")) { into "config/" } - from(project.file("$rootDir/licenses")) { into "licenses/" } - from "$rootDir/LICENSE-binary" rename {String filename -> filename.replace("-binary", "")} - from "$rootDir/NOTICE-binary" rename {String filename -> filename.replace("-binary", "")} - from(configurations.runtimeClasspath) { into("libs/") } - from(configurations.archives.artifacts.files) { into("libs/") } - from(configurations.releaseOnly) { into("libs/") } - from(project.siteDocsTar) { into("site-docs/") } - from(project(':tools').jar) { into("libs/") } - from(project(':tools').configurations.runtimeClasspath) { into("libs/") } - from(project(':trogdor').jar) { into("libs/") } - from(project(':trogdor').configurations.runtimeClasspath) { into("libs/") } - from(project(':shell').jar) { into("libs/") } - from(project(':shell').configurations.runtimeClasspath) { into("libs/") } - from(project(':connect:api').jar) { into("libs/") } - from(project(':connect:api').configurations.runtimeClasspath) { into("libs/") } - from(project(':connect:runtime').jar) { into("libs/") } - from(project(':connect:runtime').configurations.runtimeClasspath) { into("libs/") } - from(project(':connect:transforms').jar) { into("libs/") } - from(project(':connect:transforms').configurations.runtimeClasspath) { into("libs/") } - from(project(':connect:json').jar) { into("libs/") } - from(project(':connect:json').configurations.runtimeClasspath) { into("libs/") } - from(project(':connect:file').jar) { into("libs/") } - from(project(':connect:file').configurations.runtimeClasspath) { into("libs/") } - from(project(':connect:basic-auth-extension').jar) { into("libs/") } - from(project(':connect:basic-auth-extension').configurations.runtimeClasspath) { into("libs/") } - from(project(':connect:mirror').jar) { into("libs/") } - from(project(':connect:mirror').configurations.runtimeClasspath) { into("libs/") } - from(project(':connect:mirror-client').jar) { into("libs/") } - from(project(':connect:mirror-client').configurations.runtimeClasspath) { into("libs/") } - from(project(':streams').jar) { into("libs/") } - from(project(':streams').configurations.runtimeClasspath) { into("libs/") } - from(project(':streams:streams-scala').jar) { into("libs/") } - from(project(':streams:streams-scala').configurations.runtimeClasspath) { into("libs/") } - from(project(':streams:test-utils').jar) { into("libs/") } - from(project(':streams:test-utils').configurations.runtimeClasspath) { into("libs/") } - from(project(':streams:examples').jar) { into("libs/") } - from(project(':streams:examples').configurations.runtimeClasspath) { into("libs/") } - from(project(':tools:tools-api').jar) { into("libs/") } - from(project(':tools:tools-api').configurations.runtimeClasspath) { into("libs/") } - duplicatesStrategy 'exclude' - } - jar { dependsOn copyDependantLibs } @@ -3904,6 +3852,109 @@ project(':connect:test-plugins') { } } +project(':distribution') { + base { + archivesName = "kafka_${versions.baseScala}" + } + + configurations { + includeIntoDistribution + } + + dependencies { + includeIntoDistribution project(path: ':core', configuration: 'releaseOnly') + includeIntoDistribution project(path: ':core', configuration: 'archives') + includeIntoDistribution project(path: ':core', configuration: 'resolvedRuntimeClasspath') + includeIntoDistribution project(path: ':tools', configuration: 'archives') + includeIntoDistribution project(path: ':tools', configuration: 'resolvedRuntimeClasspath') + includeIntoDistribution project(path: ':trogdor', configuration: 'archives') + includeIntoDistribution project(path: ':trogdor', configuration: 'resolvedRuntimeClasspath') + includeIntoDistribution project(path: ':shell', configuration: 'archives') + includeIntoDistribution project(path: ':shell', configuration: 'resolvedRuntimeClasspath') + includeIntoDistribution project(path: ':connect:api', configuration: 'archives') + includeIntoDistribution project(path: ':connect:api', configuration: 'resolvedRuntimeClasspath') + includeIntoDistribution project(path: ':connect:runtime', configuration: 'archives') + includeIntoDistribution project(path: ':connect:runtime', configuration: 'resolvedRuntimeClasspath') + includeIntoDistribution project(path: ':connect:transforms', configuration: 'archives') + includeIntoDistribution project(path: ':connect:transforms', configuration: 'resolvedRuntimeClasspath') + includeIntoDistribution project(path: ':connect:json', configuration: 'archives') + includeIntoDistribution project(path: ':connect:json', configuration: 'resolvedRuntimeClasspath') + includeIntoDistribution project(path: ':connect:file', configuration: 'archives') + includeIntoDistribution project(path: ':connect:file', configuration: 'resolvedRuntimeClasspath') + includeIntoDistribution project(path: ':connect:basic-auth-extension', configuration: 'archives') + includeIntoDistribution project(path: ':connect:basic-auth-extension', configuration: 'resolvedRuntimeClasspath') + includeIntoDistribution project(path: ':connect:mirror', configuration: 'archives') + includeIntoDistribution project(path: ':connect:mirror', configuration: 'resolvedRuntimeClasspath') + includeIntoDistribution project(path: ':connect:mirror-client', configuration: 'archives') + includeIntoDistribution project(path: ':connect:mirror-client', configuration: 'resolvedRuntimeClasspath') + includeIntoDistribution project(path: ':streams', configuration: 'archives') + includeIntoDistribution project(path: ':streams', configuration: 'resolvedRuntimeClasspath') + includeIntoDistribution project(path: ':streams:streams-scala', configuration: 'archives') + includeIntoDistribution project(path: ':streams:streams-scala', configuration: 'resolvedRuntimeClasspath') + includeIntoDistribution project(path: ':streams:test-utils', configuration: 'archives') + includeIntoDistribution project(path: ':streams:test-utils', configuration: 'resolvedRuntimeClasspath') + includeIntoDistribution project(path: ':streams:examples', configuration: 'archives') + includeIntoDistribution project(path: ':streams:examples', configuration: 'resolvedRuntimeClasspath') + includeIntoDistribution project(path: ':tools:tools-api', configuration: 'archives') + includeIntoDistribution project(path: ':tools:tools-api', configuration: 'resolvedRuntimeClasspath') + } + + task siteDocsTar(dependsOn: [':core:genProtocolErrorDocs', ':core:genProtocolTypesDocs', ':core:genProtocolApiKeyDocs', ':core:genProtocolMessageDocs', + ':core:genAdminClientConfigDocs', ':core:genProducerConfigDocs', ':core:genConsumerConfigDocs', + ':core:genKafkaConfigDocs', ':core:genTopicConfigDocs', ':core:genGroupConfigDocs', + ':connect:runtime:genConnectConfigDocs', ':connect:runtime:genConnectTransformationDocs', + ':connect:runtime:genConnectPredicateDocs', + ':connect:runtime:genSinkConnectorConfigDocs', ':connect:runtime:genSourceConnectorConfigDocs', + ':streams:genStreamsConfigDocs', ':core:genConsumerMetricsDocs', ':core:genProducerMetricsDocs', + ':connect:runtime:genConnectMetricsDocs', ':connect:runtime:genConnectOpenAPIDocs', + ':connect:mirror:genMirrorSourceConfigDocs', ':connect:mirror:genMirrorCheckpointConfigDocs', + ':connect:mirror:genMirrorHeartbeatConfigDocs', ':connect:mirror:genMirrorConnectorConfigDocs', + ':storage:genRemoteLogManagerConfigDoc', ':storage:genRemoteLogMetadataManagerConfigDoc'], type: Tar) { + archiveClassifier = 'site-docs' + compression = Compression.GZIP + from project.file("$rootDir/docs") + into 'site-docs' + destinationDirectory = project(':core').layout.buildDirectory.dir('distributions') + duplicatesStrategy 'exclude' + preserveFileTimestamps = true + } + + tasks.create(name: "releaseTarGz", dependsOn: project(':core').configurations.archives.artifacts, type: Tar) { + into "kafka_${versions.baseScala}-${archiveVersion.get()}" + compression = Compression.GZIP + from(project.file("$rootDir/bin")) { into "bin/" } + from(project.file("$rootDir/config")) { into "config/" } + from(project.file("$rootDir/licenses")) { into "licenses/" } + from "$rootDir/LICENSE-binary" rename {String filename -> filename.replace("-binary", "")} + from "$rootDir/NOTICE-binary" rename {String filename -> filename.replace("-binary", "")} + from(project.siteDocsTar) { into("site-docs/") } + from(configurations.includeIntoDistribution) { into("libs/") } + destinationDirectory = project(':core').layout.buildDirectory.dir('distributions') + duplicatesStrategy 'exclude' + preserveFileTimestamps = true + filePermissions { + user { + read = true + write = true + } + group.read = true + other.read = true + } + dirPermissions { + unix('rwxr-xr-x') + } + eachFile { + if (name.endsWith( ".sh")) { + permissions { + user.execute = true + group.execute = true + other.execute = true + } + } + } + } +} + task aggregatedJavadoc(type: Javadoc, dependsOn: compileJava) { def projectsWithJavadoc = subprojects.findAll { it.javadoc.enabled } source = projectsWithJavadoc.collect { it.sourceSets.main.allJava } diff --git a/distribution/.gitkeep b/distribution/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 390c7e830c80d..ee594b7c569e2 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -61,7 +61,7 @@ versions += [ commonsLang: "3.18.0", commonsValidator: "1.9.0", classgraph: "4.8.179", - gradle: "8.14.3", + gradle: "9.0.0", grgit: "4.1.1", httpclient: "4.5.14", jackson: "2.19.0", diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 78cb6e16a49f9..3e781fbad9c7f 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionSha256Sum=bd71102213493060956ec229d946beee57158dbd89d0e62b91bca0fa2c5f3531 -distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-bin.zip +distributionSha256Sum=8fad3d78296ca518113f3d29016617c7f9367dc005f932bd9d93bf45ba46072b +distributionUrl=https\://services.gradle.org/distributions/gradle-9.0.0-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/gradlew b/gradlew index 39803762879c4..795b507ea2329 100755 --- a/gradlew +++ b/gradlew @@ -205,7 +205,7 @@ fi # Loop in case we encounter an error. for attempt in 1 2 3; do if [ ! -e "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" ]; then - if ! curl -s -S --retry 3 -L -o "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" "https://raw.githubusercontent.com/gradle/gradle/v8.14.1/gradle/wrapper/gradle-wrapper.jar"; then + if ! curl -s -S --retry 3 -L -o "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" "https://raw.githubusercontent.com/gradle/gradle/v9.0.0/gradle/wrapper/gradle-wrapper.jar"; then rm -f "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" # Pause for a bit before looping in case the server throttled us. sleep 5 diff --git a/settings.gradle b/settings.gradle index 7c37a046838d8..2e273faabaed7 100644 --- a/settings.gradle +++ b/settings.gradle @@ -76,6 +76,7 @@ include 'clients', 'connect:transforms', 'coordinator-common', 'core', + 'distribution', 'examples', 'generator', 'group-coordinator', diff --git a/wrapper.gradle b/wrapper.gradle index 0e0d6da201123..4cd9fc65f9d45 100644 --- a/wrapper.gradle +++ b/wrapper.gradle @@ -23,7 +23,7 @@ // and not the version installed on the machine running the task. // Read more about the wrapper here: https://docs.gradle.org/current/userguide/gradle_wrapper.html wrapper { - gradleVersion = project.gradleVersion + gradleVersion = versions.gradle } // Custom task to inject support for downloading the gradle wrapper jar if it doesn't exist. @@ -35,14 +35,12 @@ task bootstrapWrapper() { def wrapperBasePath = "\$APP_HOME/gradle/wrapper" def wrapperJarPath = wrapperBasePath + "/gradle-wrapper.jar" - // Add a trailing zero to the version if needed. - def fullVersion = project.gradleVersion.count(".") == 1 ? "${project.gradleVersion}.0" : versions.gradle // Leverages the wrapper jar checked into the gradle project on github because the jar isn't // available elsewhere. Using raw.githubusercontent.com instead of github.com because // github.com servers deprecated TLSv1/TLSv1.1 support some time ago, so older versions // of curl (built against OpenSSL library that doesn't support TLSv1.2) would fail to // fetch the jar. - def wrapperBaseUrl = "https://raw.githubusercontent.com/gradle/gradle/v$fullVersion/gradle/wrapper" + def wrapperBaseUrl = "https://raw.githubusercontent.com/gradle/gradle/v$versions.gradle/gradle/wrapper" def wrapperJarUrl = wrapperBaseUrl + "/gradle-wrapper.jar" def bootstrapString = """ From d01ace0f9765f51a109743eec9c5c268107007a8 Mon Sep 17 00:00:00 2001 From: dejan2609 Date: Fri, 22 Aug 2025 17:57:02 +0200 Subject: [PATCH 03/10] KAFKA-19174 (note: optional commit) version upgrades for Gradle build dependencies/plugins and GitHub Actions details: - `setup-gradle` GitHub Action: 4.3.0 -->> 4.4.3 (note: this version was tested against Gradle 9) - grgit: 4.1.1 -->> 5.3.0 - gradle versions plugin: 0.48.0 -->> 0.52.0 - owasp dependency check : 8.2.1 -->> 12.1.3 - spotbugs: 6.2.3 -->> 6.2.5 - spotless: 6.25.0 -->> 7.2.1 - Zinc Scala compiler: 1.9.2 -->> 1.10.8 --- .github/actions/setup-gradle/action.yml | 2 +- build.gradle | 8 ++++---- gradle/dependencies.gradle | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/actions/setup-gradle/action.yml b/.github/actions/setup-gradle/action.yml index 1a5b0902703ab..af153ad9bcbab 100644 --- a/.github/actions/setup-gradle/action.yml +++ b/.github/actions/setup-gradle/action.yml @@ -42,7 +42,7 @@ runs: distribution: temurin java-version: ${{ inputs.java-version }} - name: Setup Gradle - uses: gradle/actions/setup-gradle@94baf225fe0a508e581a564467443d0e2379123b # v4.3.0 + uses: gradle/actions/setup-gradle@ed408507eac070d1f99cc633dbcf757c94c7933a # v4.4.3 env: GRADLE_BUILD_ACTION_CACHE_DEBUG_ENABLED: true with: diff --git a/build.gradle b/build.gradle index 60b8b1f72569d..d407282533fbe 100644 --- a/build.gradle +++ b/build.gradle @@ -29,18 +29,18 @@ buildscript { } plugins { - id 'com.github.ben-manes.versions' version '0.48.0' + id 'com.github.ben-manes.versions' version '0.52.0' id 'idea' id 'jacoco' id 'java-library' - id 'org.owasp.dependencycheck' version '8.2.1' + id 'org.owasp.dependencycheck' version '12.1.3' id 'org.nosphere.apache.rat' version "0.8.1" id "io.swagger.core.v3.swagger-gradle-plugin" version "${swaggerVersion}" - id "com.github.spotbugs" version '6.2.3' apply false + id "com.github.spotbugs" version '6.2.5' apply false id 'org.scoverage' version '8.0.3' apply false id 'com.gradleup.shadow' version '9.0.2' apply false - id 'com.diffplug.spotless' version "6.25.0" + id 'com.diffplug.spotless' version "7.2.1" } ext { diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index ee594b7c569e2..fbcec1730e6fb 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -62,7 +62,7 @@ versions += [ commonsValidator: "1.9.0", classgraph: "4.8.179", gradle: "9.0.0", - grgit: "4.1.1", + grgit: "5.3.0", httpclient: "4.5.14", jackson: "2.19.0", jacoco: "0.8.13", @@ -126,7 +126,7 @@ versions += [ snappy: "1.1.10.7", spotbugs: "4.9.4", mockOAuth2Server: "2.2.1", - zinc: "1.9.2", + zinc: "1.10.8", // When updating the zstd version, please do as well in docker/native/native-image-configs/resource-config.json // Also make sure the compression levels in org.apache.kafka.common.record.CompressionType are still valid zstd: "1.5.6-10", From a0646ff4ae44853bfd7f6714cbf87e7b29994a82 Mon Sep 17 00:00:00 2001 From: dejan2609 Date: Thu, 28 Aug 2025 23:43:53 +0200 Subject: [PATCH 04/10] KAFKA-16801 do not fail Gradle 9 build when no test are discovered in a `upgrade-system-tests*` submodules related link: https://docs.gradle.org/9.0.0/userguide/upgrading_major_version_9.html#test_task_fails_when_no_tests_are_discovered --- build.gradle | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/build.gradle b/build.gradle index d407282533fbe..fa4cdc31ce121 100644 --- a/build.gradle +++ b/build.gradle @@ -343,6 +343,12 @@ subprojects { } } + if (project.name.startsWith("upgrade-system-tests")) { + tasks.withType(AbstractTestTask).configureEach { + failOnNoDiscoveredTests = false + } + } + // apply the eclipse plugin only to subprojects that hold code. 'connect' is just a folder. if (!project.name.equals('connect')) { apply plugin: 'eclipse' From f0946dbc6a5714d0ba09d70a5676bec1696fefd7 Mon Sep 17 00:00:00 2001 From: dejan2609 Date: Thu, 11 Sep 2025 22:09:54 +0200 Subject: [PATCH 05/10] KAFKA-19654 do not fail Github Actions flaky/new build matrix build when no test are discovered (i.e. solution for KAFKA-16801 is expanded) --- build.gradle | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/build.gradle b/build.gradle index fa4cdc31ce121..16a133929e7dd 100644 --- a/build.gradle +++ b/build.gradle @@ -343,10 +343,8 @@ subprojects { } } - if (project.name.startsWith("upgrade-system-tests")) { - tasks.withType(AbstractTestTask).configureEach { - failOnNoDiscoveredTests = false - } + tasks.withType(AbstractTestTask).configureEach { + failOnNoDiscoveredTests = false } // apply the eclipse plugin only to subprojects that hold code. 'connect' is just a folder. From 218f5a941a84c4df80d9a6678f13bb44da924c52 Mon Sep 17 00:00:00 2001 From: dejan2609 Date: Thu, 18 Sep 2025 16:13:53 +0200 Subject: [PATCH 06/10] KAFKA-19174 Gradle version upgrade: 9.0.0 -->> 9.1.0 details: support for Java 25: https://docs.gradle.org/9.1.0/release-notes.html#support-for-java-25 --- gradle/dependencies.gradle | 2 +- gradle/wrapper/gradle-wrapper.properties | 4 ++-- gradlew | 7 ++----- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index fbcec1730e6fb..ef2827d110226 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -61,7 +61,7 @@ versions += [ commonsLang: "3.18.0", commonsValidator: "1.9.0", classgraph: "4.8.179", - gradle: "9.0.0", + gradle: "9.1.0", grgit: "5.3.0", httpclient: "4.5.14", jackson: "2.19.0", diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 3e781fbad9c7f..6a38a8cea6117 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionSha256Sum=8fad3d78296ca518113f3d29016617c7f9367dc005f932bd9d93bf45ba46072b -distributionUrl=https\://services.gradle.org/distributions/gradle-9.0.0-bin.zip +distributionSha256Sum=a17ddd85a26b6a7f5ddb71ff8b05fc5104c0202c6e64782429790c933686c806 +distributionUrl=https\://services.gradle.org/distributions/gradle-9.1.0-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/gradlew b/gradlew index 795b507ea2329..98a05d787ee04 100755 --- a/gradlew +++ b/gradlew @@ -1,7 +1,7 @@ #!/bin/sh # -# Copyright © 2015-2021 the original authors. +# Copyright © 2015 the original authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -114,7 +114,6 @@ case "$( uname )" in #( NONSTOP* ) nonstop=true ;; esac -CLASSPATH="\\\"\\\"" # Determine the Java command to use to start the JVM. @@ -172,7 +171,6 @@ fi # For Cygwin or MSYS, switch paths to Windows format before running java if "$cygwin" || "$msys" ; then APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) - CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) JAVACMD=$( cygpath --unix "$JAVACMD" ) @@ -205,7 +203,7 @@ fi # Loop in case we encounter an error. for attempt in 1 2 3; do if [ ! -e "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" ]; then - if ! curl -s -S --retry 3 -L -o "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" "https://raw.githubusercontent.com/gradle/gradle/v9.0.0/gradle/wrapper/gradle-wrapper.jar"; then + if ! curl -s -S --retry 3 -L -o "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" "https://raw.githubusercontent.com/gradle/gradle/v9.1.0/gradle/wrapper/gradle-wrapper.jar"; then rm -f "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" # Pause for a bit before looping in case the server throttled us. sleep 5 @@ -225,7 +223,6 @@ DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' set -- \ "-Dorg.gradle.appname=$APP_BASE_NAME" \ - -classpath "$CLASSPATH" \ -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \ "$@" From 6955aa260ca484d3b51f88282a0fac369abc9b93 Mon Sep 17 00:00:00 2001 From: dejan2609 Date: Thu, 18 Sep 2025 19:16:53 +0200 Subject: [PATCH 07/10] KAFKA-19664 Support building with Java 25 (LTS release) note: spotbugs related Gradle tasks temporarily are disabled for Java 25 --- .github/workflows/build.yml | 10 +++++----- .github/workflows/ci-complete.yml | 2 +- README.md | 2 +- build.gradle | 7 +++++++ 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5137f8bf37292..f1e913a6a5e3a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -127,7 +127,7 @@ jobs: - name: Setup Gradle uses: ./.github/actions/setup-gradle with: - java-version: 24 + java-version: 17 gradle-cache-read-only: ${{ !inputs.is-trunk }} gradle-cache-write-only: ${{ inputs.is-trunk }} develocity-access-key: ${{ secrets.DEVELOCITY_ACCESS_KEY }} @@ -181,7 +181,7 @@ jobs: fail-fast: false matrix: # If we change these, make sure to adjust ci-complete.yml - java: [ 24, 17 ] + java: [ 25-ea, 17 ] run-flaky: [ true, false ] run-new: [ true, false ] exclude: @@ -270,7 +270,7 @@ jobs: python .github/scripts/junit.py \ --path build/junit-xml >> $GITHUB_STEP_SUMMARY - # This job downloads all the JUnit XML files and thread dumps from the JDK 24 test runs. + # This job downloads all the JUnit XML files and thread dumps from the JDK 25 test runs. # If any test job fails, we will not run this job. Also, if any thread dump artifacts # are present, this means there was a timeout in the tests and so we will not proceed # with catalog creation. @@ -288,7 +288,7 @@ jobs: - name: Download Thread Dumps uses: actions/download-artifact@v4 with: - pattern: junit-thread-dumps-24-* + pattern: junit-thread-dumps-25-* path: thread-dumps merge-multiple: true - name: Check For Thread Dump @@ -302,7 +302,7 @@ jobs: - name: Download JUnit XMLs uses: actions/download-artifact@v4 with: - pattern: junit-xml-24-* # Only look at JDK 24 tests for the test catalog + pattern: junit-xml-25-* # Only look at JDK 25 tests for the test catalog path: junit-xml merge-multiple: true - name: Collate Test Catalog diff --git a/.github/workflows/ci-complete.yml b/.github/workflows/ci-complete.yml index 44d4f5a9c1d62..2cc8731aa0552 100644 --- a/.github/workflows/ci-complete.yml +++ b/.github/workflows/ci-complete.yml @@ -44,7 +44,7 @@ jobs: fail-fast: false matrix: # Make sure these match build.yml - java: [ 24, 17 ] + java: [ 25-ea, 17 ] run-flaky: [ true, false ] run-new: [ true, false ] exclude: diff --git a/README.md b/README.md index 06c0e3921ebc1..95386750a9315 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ You need to have [Java](http://www.oracle.com/technetwork/java/javase/downloads/index.html) installed. -We build and test Apache Kafka with 17 and 24. The `release` parameter in javac is set to `11` for the clients +We build and test Apache Kafka with 17 and 25. The `release` parameter in javac is set to `11` for the clients and streams modules, and `17` for the rest, ensuring compatibility with their respective minimum Java versions. Similarly, the `release` parameter in scalac is set to `11` for the streams modules and `17` for the rest. diff --git a/build.gradle b/build.gradle index 16a133929e7dd..db852c57025a3 100644 --- a/build.gradle +++ b/build.gradle @@ -71,6 +71,13 @@ ext { "--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED" ) + if (JavaVersion.current().isCompatibleWith(JavaVersion.VERSION_25)) { + // Spotbugs is not compatible with Java 25+ so Gradle related tasks are disabled + // until version can be upgraded: https://github.com/spotbugs/spotbugs/issues/3564 + project.gradle.startParameter.excludedTaskNames.add("spotbugsMain") + project.gradle.startParameter.excludedTaskNames.add("spotbugsTest") + } + maxTestForks = project.hasProperty('maxParallelForks') ? maxParallelForks.toInteger() : Runtime.runtime.availableProcessors() maxScalacThreads = project.hasProperty('maxScalacThreads') ? maxScalacThreads.toInteger() : Math.min(Runtime.runtime.availableProcessors(), 8) From ff1add59b5d6b65f36f3b5a5aeaf4db3039ce431 Mon Sep 17 00:00:00 2001 From: dejan2609 Date: Fri, 19 Sep 2025 13:30:41 +0200 Subject: [PATCH 08/10] KAFKA-19664 fix ? (it could be that `-ea` label should be added on two more places) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit note: `build / JUnit tests Java 25-ea` Github Action results with a previous commit: https://github.com/apache/kafka/actions/runs/17853628508/job/50769215024?pr=20561 ``` Gradle task had a failure exit code. Failing this script. 25373 tests cases run in 1h17m30s. 17471 PASSED ✅, 5676 FAILED ❌, 0 FLAKY ⚠️ , 0 SKIPPED 🙈, 0 QUARANTINED 😷, and 0 errors. Error: Process completed with exit code 1. Error: $GITHUB_STEP_SUMMARY upload aborted, supports content up to a size of 1024k, got 6145k. For more information see: https://docs.github.com/actions/using-workflows/workflow-commands-for-github-actions#adding-a-markdown-summary ``` --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f1e913a6a5e3a..e5ab30d7f322c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -288,7 +288,7 @@ jobs: - name: Download Thread Dumps uses: actions/download-artifact@v4 with: - pattern: junit-thread-dumps-25-* + pattern: junit-thread-dumps-25-ea-* path: thread-dumps merge-multiple: true - name: Check For Thread Dump @@ -302,7 +302,7 @@ jobs: - name: Download JUnit XMLs uses: actions/download-artifact@v4 with: - pattern: junit-xml-25-* # Only look at JDK 25 tests for the test catalog + pattern: junit-xml-25-ea-* # Only look at JDK 25 tests for the test catalog path: junit-xml merge-multiple: true - name: Collate Test Catalog From 71ff59305897ca2d82cc7355aa65461e451f964e Mon Sep 17 00:00:00 2001 From: dejan2609 Date: Wed, 24 Sep 2025 12:24:41 +0200 Subject: [PATCH 09/10] Revert "KAFKA-19664 fix ? (it could be that `-ea` label should be added on two more places)" This reverts commit ff1add59b5d6b65f36f3b5a5aeaf4db3039ce431. --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e5ab30d7f322c..f1e913a6a5e3a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -288,7 +288,7 @@ jobs: - name: Download Thread Dumps uses: actions/download-artifact@v4 with: - pattern: junit-thread-dumps-25-ea-* + pattern: junit-thread-dumps-25-* path: thread-dumps merge-multiple: true - name: Check For Thread Dump @@ -302,7 +302,7 @@ jobs: - name: Download JUnit XMLs uses: actions/download-artifact@v4 with: - pattern: junit-xml-25-ea-* # Only look at JDK 25 tests for the test catalog + pattern: junit-xml-25-* # Only look at JDK 25 tests for the test catalog path: junit-xml merge-multiple: true - name: Collate Test Catalog From 72b09470c28acb89a71a57d08b600ebd739e7a48 Mon Sep 17 00:00:00 2001 From: dejan2609 Date: Wed, 24 Sep 2025 12:28:06 +0200 Subject: [PATCH 10/10] KAFKA-19664 it seems that GitHub Action now supports Java 25 --- .github/workflows/build.yml | 2 +- .github/workflows/ci-complete.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f1e913a6a5e3a..be4a839b1726d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -181,7 +181,7 @@ jobs: fail-fast: false matrix: # If we change these, make sure to adjust ci-complete.yml - java: [ 25-ea, 17 ] + java: [ 25, 17 ] run-flaky: [ true, false ] run-new: [ true, false ] exclude: diff --git a/.github/workflows/ci-complete.yml b/.github/workflows/ci-complete.yml index 2cc8731aa0552..bbef323d95a88 100644 --- a/.github/workflows/ci-complete.yml +++ b/.github/workflows/ci-complete.yml @@ -44,7 +44,7 @@ jobs: fail-fast: false matrix: # Make sure these match build.yml - java: [ 25-ea, 17 ] + java: [ 25, 17 ] run-flaky: [ true, false ] run-new: [ true, false ] exclude: