From 0df7a4f88b696618b1700d1628bcaf43f5bab32c Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Mon, 26 Jan 2026 13:00:04 -0800 Subject: [PATCH 01/45] test emulation --- .semaphore/cp_dockerfile_build.yml | 217 ++++++++++++++++++++++++++++- .semaphore/semaphore.yml | 40 ++++++ base-java-micro/pom.xml | 5 + base-java/pom.xml | 5 + base-lite/pom.xml | 5 + base/pom.xml | 5 + pom.xml | 2 + 7 files changed, 274 insertions(+), 5 deletions(-) diff --git a/.semaphore/cp_dockerfile_build.yml b/.semaphore/cp_dockerfile_build.yml index 40c15a834e..2ed8c40a37 100644 --- a/.semaphore/cp_dockerfile_build.yml +++ b/.semaphore/cp_dockerfile_build.yml @@ -106,6 +106,7 @@ global_job_config: - export DOCKER_DEV_TAG="dev-$BRANCH_TAG-$BUILD_NUMBER" - export AMD_ARCH=.amd64 - export ARM_ARCH=.arm64 + - export S390X_ARCH=.s390x blocks: - name: Validation dependencies: [] @@ -491,9 +492,214 @@ blocks: - export LATEST_PUSH_TAG=$LATEST_TAG$OS_TAG$ARM_ARCH - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$LATEST_PUSH_TAG - docker push $PROD_IMAGE_NAME:$LATEST_PUSH_TAG + - name: Build & Test S390X + dependencies: ["Validation"] + run: + # don't run the tests on non-functional changes... + when: "change_in('/', {exclude: ['/.deployed-versions/', '.github/', 'service.yml', 'README.md'], default_branch: 'master'})" + task: + agent: + machine: + type: s1-prod-ubuntu24-04-amd64-1 + jobs: + - name: Build & Test ubi9 + commands: + # Setup QEMU for s390x emulation + - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + - docker buildx create --name s390x-builder --use || docker buildx use s390x-builder + - docker buildx inspect --bootstrap + # Make buildx the default builder so 'docker build' uses buildx + - docker buildx install + # Set default platform for all docker commands + - export DOCKER_DEFAULT_PLATFORM=linux/s390x + - export OS_TAG="-ubi9" + - export DOCKER_DEV_FULL_IMAGES=$DOCKER_DEV_REGISTRY${DOCKER_REPOS// /:$DOCKER_DEV_TAG$OS_TAG $DOCKER_DEV_REGISTRY}:$DOCKER_DEV_TAG$OS_TAG + - export S390X_DOCKER_DEV_FULL_IMAGES=${DOCKER_DEV_FULL_IMAGES// /$S390X_ARCH }$S390X_ARCH + - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") + - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" + - ci-tools ci-update-version + - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY + -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$S390X_ARCH + -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$S390X_ARCH -Ddocker.os_type=ubi9 -Ddocker.platform=linux/s390x $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS + - . cache-maven store + # Verify images are built for s390x architecture + - for image in $S390X_DOCKER_DEV_FULL_IMAGES; do echo "Checking architecture of $image:" && docker inspect --format='{{.Architecture}}' $image; done + - for image in $S390X_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done + epilogue: + always: + commands: + - . publish-test-results + - artifact push workflow target/test-results + - artifact push workflow target --destination target-S390X + - name: Deploy S390X confluentinc/cp-base-java + dependencies: ["Build & Test S390X"] + run: + when: "branch = 'master' or branch =~ '^[0-9]+\\.[0-9]+\\.x$' or branch =~ '^[0-9]+\\.[0-9]+\\.[0-9]+(-cp[0-9]+)?(-rc[0-9]+)?$'" + task: + agent: + machine: + type: s1-prod-ubuntu24-04-amd64-1 + jobs: + - name: Deploy S390X confluentinc/cp-base-java ubi9 + env_vars: + - name: DOCKER_IMAGE + value: confluentinc/cp-base-java + commands: + - export OS_TAG="-ubi9" + - export PROD_IMAGE_NAME=${DOCKER_PROD_REGISTRY}confluentinc/cp-base-java + - export GIT_COMMIT_TAG=$GIT_COMMIT$OS_TAG$S390X_ARCH + - export BRANCH_BUILD_TAG=$BRANCH_TAG-$BUILD_NUMBER$OS_TAG$S390X_ARCH + - export DEV_IMAGE_FULL=${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:$DOCKER_DEV_TAG$OS_TAG$S390X_ARCH + - docker pull $DEV_IMAGE_FULL + - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$GIT_COMMIT_TAG + - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG + - docker push $PROD_IMAGE_NAME:$GIT_COMMIT_TAG + - docker push $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG + - sign-images $PROD_IMAGE_NAME:$GIT_COMMIT_TAG + - sign-images $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG + - export PACKAGE_TAG=$BRANCH_TAG-$PACKAGING_BUILD_NUMBER$OS_TAG$S390X_ARCH + - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$PACKAGE_TAG + - docker push $PROD_IMAGE_NAME:$PACKAGE_TAG + - sign-images $PROD_IMAGE_NAME:$PACKAGE_TAG + - export LATEST_PUSH_TAG=$LATEST_TAG$OS_TAG$S390X_ARCH + - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$LATEST_PUSH_TAG + - docker push $PROD_IMAGE_NAME:$LATEST_PUSH_TAG + - name: Deploy S390X confluentinc/cp-base-java-micro + dependencies: ["Build & Test S390X"] + run: + when: "branch = 'master' or branch =~ '^[0-9]+\\.[0-9]+\\.x$' or branch =~ '^[0-9]+\\.[0-9]+\\.[0-9]+(-cp[0-9]+)?(-rc[0-9]+)?$'" + task: + agent: + machine: + type: s1-prod-ubuntu24-04-amd64-1 + jobs: + - name: Deploy S390X confluentinc/cp-base-java-micro ubi9 + env_vars: + - name: DOCKER_IMAGE + value: confluentinc/cp-base-java-micro + commands: + - export OS_TAG="-ubi9" + - export PROD_IMAGE_NAME=${DOCKER_PROD_REGISTRY}confluentinc/cp-base-java-micro + - export GIT_COMMIT_TAG=$GIT_COMMIT$OS_TAG$S390X_ARCH + - export BRANCH_BUILD_TAG=$BRANCH_TAG-$BUILD_NUMBER$OS_TAG$S390X_ARCH + - export DEV_IMAGE_FULL=${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:$DOCKER_DEV_TAG$OS_TAG$S390X_ARCH + - docker pull $DEV_IMAGE_FULL + - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$GIT_COMMIT_TAG + - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG + - docker push $PROD_IMAGE_NAME:$GIT_COMMIT_TAG + - docker push $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG + - sign-images $PROD_IMAGE_NAME:$GIT_COMMIT_TAG + - sign-images $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG + - export PACKAGE_TAG=$BRANCH_TAG-$PACKAGING_BUILD_NUMBER$OS_TAG$S390X_ARCH + - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$PACKAGE_TAG + - docker push $PROD_IMAGE_NAME:$PACKAGE_TAG + - sign-images $PROD_IMAGE_NAME:$PACKAGE_TAG + - export LATEST_PUSH_TAG=$LATEST_TAG$OS_TAG$S390X_ARCH + - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$LATEST_PUSH_TAG + - docker push $PROD_IMAGE_NAME:$LATEST_PUSH_TAG + - name: Deploy S390X confluentinc/cp-base-new + dependencies: ["Build & Test S390X"] + run: + when: "branch = 'master' or branch =~ '^[0-9]+\\.[0-9]+\\.x$' or branch =~ '^[0-9]+\\.[0-9]+\\.[0-9]+(-cp[0-9]+)?(-rc[0-9]+)?$'" + task: + agent: + machine: + type: s1-prod-ubuntu24-04-amd64-1 + jobs: + - name: Deploy S390X confluentinc/cp-base-new ubi9 + env_vars: + - name: DOCKER_IMAGE + value: confluentinc/cp-base-new + commands: + - export OS_TAG="-ubi9" + - export PROD_IMAGE_NAME=${DOCKER_PROD_REGISTRY}confluentinc/cp-base-new + - export GIT_COMMIT_TAG=$GIT_COMMIT$OS_TAG$S390X_ARCH + - export BRANCH_BUILD_TAG=$BRANCH_TAG-$BUILD_NUMBER$OS_TAG$S390X_ARCH + - export DEV_IMAGE_FULL=${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:$DOCKER_DEV_TAG$OS_TAG$S390X_ARCH + - docker pull $DEV_IMAGE_FULL + - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$GIT_COMMIT_TAG + - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG + - docker push $PROD_IMAGE_NAME:$GIT_COMMIT_TAG + - docker push $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG + - sign-images $PROD_IMAGE_NAME:$GIT_COMMIT_TAG + - sign-images $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG + - export PACKAGE_TAG=$BRANCH_TAG-$PACKAGING_BUILD_NUMBER$OS_TAG$S390X_ARCH + - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$PACKAGE_TAG + - docker push $PROD_IMAGE_NAME:$PACKAGE_TAG + - sign-images $PROD_IMAGE_NAME:$PACKAGE_TAG + - export LATEST_PUSH_TAG=$LATEST_TAG$OS_TAG$S390X_ARCH + - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$LATEST_PUSH_TAG + - docker push $PROD_IMAGE_NAME:$LATEST_PUSH_TAG + - name: Deploy S390X confluentinc/cp-base-lite + dependencies: ["Build & Test S390X"] + run: + when: "branch = 'master' or branch =~ '^[0-9]+\\.[0-9]+\\.x$' or branch =~ '^[0-9]+\\.[0-9]+\\.[0-9]+(-cp[0-9]+)?(-rc[0-9]+)?$'" + task: + agent: + machine: + type: s1-prod-ubuntu24-04-amd64-1 + jobs: + - name: Deploy S390X confluentinc/cp-base-lite ubi9 + env_vars: + - name: DOCKER_IMAGE + value: confluentinc/cp-base-lite + commands: + - export OS_TAG="-ubi9" + - export PROD_IMAGE_NAME=${DOCKER_PROD_REGISTRY}confluentinc/cp-base-lite + - export GIT_COMMIT_TAG=$GIT_COMMIT$OS_TAG$S390X_ARCH + - export BRANCH_BUILD_TAG=$BRANCH_TAG-$BUILD_NUMBER$OS_TAG$S390X_ARCH + - export DEV_IMAGE_FULL=${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:$DOCKER_DEV_TAG$OS_TAG$S390X_ARCH + - docker pull $DEV_IMAGE_FULL + - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$GIT_COMMIT_TAG + - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG + - docker push $PROD_IMAGE_NAME:$GIT_COMMIT_TAG + - docker push $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG + - sign-images $PROD_IMAGE_NAME:$GIT_COMMIT_TAG + - sign-images $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG + - export PACKAGE_TAG=$BRANCH_TAG-$PACKAGING_BUILD_NUMBER$OS_TAG$S390X_ARCH + - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$PACKAGE_TAG + - docker push $PROD_IMAGE_NAME:$PACKAGE_TAG + - sign-images $PROD_IMAGE_NAME:$PACKAGE_TAG + - export LATEST_PUSH_TAG=$LATEST_TAG$OS_TAG$S390X_ARCH + - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$LATEST_PUSH_TAG + - docker push $PROD_IMAGE_NAME:$LATEST_PUSH_TAG + - name: Deploy S390X confluentinc/cp-jmxterm + dependencies: ["Build & Test S390X"] + run: + when: "branch = 'master' or branch =~ '^[0-9]+\\.[0-9]+\\.x$' or branch =~ '^[0-9]+\\.[0-9]+\\.[0-9]+(-cp[0-9]+)?(-rc[0-9]+)?$'" + task: + agent: + machine: + type: s1-prod-ubuntu24-04-amd64-1 + jobs: + - name: Deploy S390X confluentinc/cp-jmxterm ubi9 + env_vars: + - name: DOCKER_IMAGE + value: confluentinc/cp-jmxterm + commands: + - export OS_TAG="-ubi9" + - export PROD_IMAGE_NAME=${DOCKER_PROD_REGISTRY}confluentinc/cp-jmxterm + - export GIT_COMMIT_TAG=$GIT_COMMIT$OS_TAG$S390X_ARCH + - export BRANCH_BUILD_TAG=$BRANCH_TAG-$BUILD_NUMBER$OS_TAG$S390X_ARCH + - export DEV_IMAGE_FULL=${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:$DOCKER_DEV_TAG$OS_TAG$S390X_ARCH + - docker pull $DEV_IMAGE_FULL + - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$GIT_COMMIT_TAG + - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG + - docker push $PROD_IMAGE_NAME:$GIT_COMMIT_TAG + - docker push $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG + - sign-images $PROD_IMAGE_NAME:$GIT_COMMIT_TAG + - sign-images $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG + - export PACKAGE_TAG=$BRANCH_TAG-$PACKAGING_BUILD_NUMBER$OS_TAG$S390X_ARCH + - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$PACKAGE_TAG + - docker push $PROD_IMAGE_NAME:$PACKAGE_TAG + - sign-images $PROD_IMAGE_NAME:$PACKAGE_TAG + - export LATEST_PUSH_TAG=$LATEST_TAG$OS_TAG$S390X_ARCH + - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$LATEST_PUSH_TAG + - docker push $PROD_IMAGE_NAME:$LATEST_PUSH_TAG - name: Create Manifest and Maven Deploy dependencies: ["Deploy AMD confluentinc/cp-base-java", "Deploy AMD confluentinc/cp-base-java-micro", "Deploy AMD confluentinc/cp-base-new", "Deploy AMD confluentinc/cp-base-lite", "Deploy AMD confluentinc/cp-jmxterm", - "Deploy ARM confluentinc/cp-base-java", "Deploy ARM confluentinc/cp-base-java-micro", "Deploy ARM confluentinc/cp-base-new", "Deploy ARM confluentinc/cp-base-lite", "Deploy ARM confluentinc/cp-jmxterm"] + "Deploy ARM confluentinc/cp-base-java", "Deploy ARM confluentinc/cp-base-java-micro", "Deploy ARM confluentinc/cp-base-new", "Deploy ARM confluentinc/cp-base-lite", "Deploy ARM confluentinc/cp-jmxterm", + "Deploy S390X confluentinc/cp-base-java", "Deploy S390X confluentinc/cp-base-java-micro", "Deploy S390X confluentinc/cp-base-new", "Deploy S390X confluentinc/cp-base-lite", "Deploy S390X confluentinc/cp-jmxterm"] run: when: "branch = 'master' or branch =~ '^[0-9]+\\.[0-9]+\\.x$' or branch =~ '^[0-9]+\\.[0-9]+\\.[0-9]+(-cp[0-9]+)?(-rc[0-9]+)?$'" task: @@ -513,20 +719,20 @@ blocks: do export OS_TAG="-ubi9" export GIT_TAG=$GIT_COMMIT$OS_TAG - docker manifest create $image:$GIT_TAG $image:$GIT_TAG$AMD_ARCH $image:$GIT_TAG$ARM_ARCH + docker manifest create $image:$GIT_TAG $image:$GIT_TAG$AMD_ARCH $image:$GIT_TAG$ARM_ARCH $image:$GIT_TAG$S390X_ARCH docker manifest push $image:$GIT_TAG docker pull $image:$GIT_TAG sign-images $image:$GIT_TAG export BRANCH_BUILD_TAG=$BRANCH_TAG-$BUILD_NUMBER$OS_TAG - docker manifest create $image:$BRANCH_BUILD_TAG $image:$BRANCH_BUILD_TAG$AMD_ARCH $image:$BRANCH_BUILD_TAG$ARM_ARCH + docker manifest create $image:$BRANCH_BUILD_TAG $image:$BRANCH_BUILD_TAG$AMD_ARCH $image:$BRANCH_BUILD_TAG$ARM_ARCH $image:$BRANCH_BUILD_TAG$S390X_ARCH docker manifest push $image:$BRANCH_BUILD_TAG docker pull $image:$BRANCH_BUILD_TAG sign-images $image:$BRANCH_BUILD_TAG export PACKAGE_TAG=$BRANCH_TAG-$PACKAGING_BUILD_NUMBER$OS_TAG - docker manifest create $image:$PACKAGE_TAG $image:$PACKAGE_TAG$AMD_ARCH $image:$PACKAGE_TAG$ARM_ARCH + docker manifest create $image:$PACKAGE_TAG $image:$PACKAGE_TAG$AMD_ARCH $image:$PACKAGE_TAG$ARM_ARCH $image:$PACKAGE_TAG$S390X_ARCH docker manifest push $image:$PACKAGE_TAG export LATEST_MANIFEST_TAG=$LATEST_TAG$OS_TAG - docker manifest create $image:$LATEST_MANIFEST_TAG $image:$LATEST_MANIFEST_TAG$AMD_ARCH $image:$LATEST_MANIFEST_TAG$ARM_ARCH + docker manifest create $image:$LATEST_MANIFEST_TAG $image:$LATEST_MANIFEST_TAG$AMD_ARCH $image:$LATEST_MANIFEST_TAG$ARM_ARCH $image:$LATEST_MANIFEST_TAG$S390X_ARCH docker manifest push $image:$LATEST_MANIFEST_TAG done after_pipeline: @@ -546,4 +752,5 @@ after_pipeline: - checkout - artifact pull workflow target-AMD - artifact pull workflow target-ARM + - artifact pull workflow target-S390X - emit-sonarqube-data --run_only_sonar_scan diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index cd1b0cb1c5..f09962d4bf 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -99,6 +99,7 @@ global_job_config: - export DOCKER_DEV_TAG="dev-$BRANCH_TAG-$BUILD_NUMBER" - export AMD_ARCH=.amd64 - export ARM_ARCH=.arm64 + - export S390X_ARCH=.s390x blocks: - name: Validation dependencies: [] @@ -167,6 +168,44 @@ blocks: - . publish-test-results - artifact push workflow target/test-results - artifact push workflow target --destination target-ARM + - name: Build & Test S390X + dependencies: ["Validation"] + run: + when: "pull_request =~ '.*'" + task: + agent: + machine: + type: s1-prod-ubuntu24-04-amd64-1 + jobs: + - name: Build & Test ubi9 + commands: + # Setup QEMU for s390x emulation + - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + - docker buildx create --name s390x-builder --use || docker buildx use s390x-builder + - docker buildx inspect --bootstrap + # Make buildx the default builder so 'docker build' uses buildx + - docker buildx install + # Set default platform for all docker commands + - export DOCKER_DEFAULT_PLATFORM=linux/s390x + - export OS_TAG="-ubi9" + - export DOCKER_DEV_FULL_IMAGES=$DOCKER_DEV_REGISTRY${DOCKER_REPOS// /:$DOCKER_DEV_TAG$OS_TAG $DOCKER_DEV_REGISTRY}:$DOCKER_DEV_TAG$OS_TAG + - export S390X_DOCKER_DEV_FULL_IMAGES=${DOCKER_DEV_FULL_IMAGES// /$S390X_ARCH }$S390X_ARCH + - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") + - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" + - ci-tools ci-update-version + - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY + -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$S390X_ARCH + -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$S390X_ARCH -Ddocker.os_type=ubi9 -Ddocker.platform=linux/s390x $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS + - . cache-maven store + # Verify images are built for s390x architecture + - for image in $S390X_DOCKER_DEV_FULL_IMAGES; do echo "Checking architecture of $image:" && docker inspect --format='{{.Architecture}}' $image; done + - for image in $S390X_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done + epilogue: + always: + commands: + - . publish-test-results + - artifact push workflow target/test-results + - artifact push workflow target --destination target-S390X after_pipeline: task: agent: @@ -184,4 +223,5 @@ after_pipeline: - checkout - artifact pull workflow target-AMD - artifact pull workflow target-ARM + - artifact pull workflow target-S390X - emit-sonarqube-data --run_only_sonar_scan diff --git a/base-java-micro/pom.xml b/base-java-micro/pom.xml index 820970d590..acb49ac50d 100644 --- a/base-java-micro/pom.xml +++ b/base-java-micro/pom.xml @@ -149,6 +149,11 @@ + + + ${docker.platform} + + ${app.uid} ${app.gid} diff --git a/base-java/pom.xml b/base-java/pom.xml index a563cde830..d51d4aa5eb 100644 --- a/base-java/pom.xml +++ b/base-java/pom.xml @@ -145,6 +145,11 @@ + + + ${docker.platform} + + ${ubi9-minimal.image.version} -${ubi9-minimal.temurin-21-jdk.version} diff --git a/base-lite/pom.xml b/base-lite/pom.xml index 81c8051414..7719168c75 100644 --- a/base-lite/pom.xml +++ b/base-lite/pom.xml @@ -100,6 +100,11 @@ + + + ${docker.platform} + + ${ubi9-minimal.image.version} -${ubi9-minimal.temurin-21-jdk.version} diff --git a/base/pom.xml b/base/pom.xml index 6ce60eeebb..a7b23f8765 100644 --- a/base/pom.xml +++ b/base/pom.xml @@ -158,6 +158,11 @@ + + + ${docker.platform} + + ${ubi9-minimal.image.version} -${ubi9-minimal.openssl.version} diff --git a/pom.xml b/pom.xml index dc20e8aa6c..018457d1fd 100644 --- a/pom.xml +++ b/pom.xml @@ -35,6 +35,8 @@ Dockerfile.${docker.ubi9.os_type} ${io.confluent.common-docker.version}-${docker.ubi9.os_type} 8.3.0-0 + + From 1dbd76ee6bfe0b40b7fff49a9176ff8550b8454f Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Mon, 26 Jan 2026 13:04:32 -0800 Subject: [PATCH 02/45] override to use fabric8 instead of spotify --- pom.xml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/pom.xml b/pom.xml index 018457d1fd..b9dc665ecb 100644 --- a/pom.xml +++ b/pom.xml @@ -139,4 +139,31 @@ `-Ddocker.skip-security-update-check=true` --> false + + + + + docker + + + env.DOCKER_HOST + + + + + + + com.spotify + dockerfile-maven-plugin + + + package + none + + + + + + + From 081a9c701be9fa433b31a7b991fbdfdc48f9718b Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Mon, 26 Jan 2026 19:18:11 -0800 Subject: [PATCH 03/45] fix --- .semaphore/cp_dockerfile_build.yml | 2 +- .semaphore/semaphore.yml | 2 +- pom.xml | 75 +++++++++++++++++++++++------- 3 files changed, 61 insertions(+), 18 deletions(-) diff --git a/.semaphore/cp_dockerfile_build.yml b/.semaphore/cp_dockerfile_build.yml index 2ed8c40a37..34875725f5 100644 --- a/.semaphore/cp_dockerfile_build.yml +++ b/.semaphore/cp_dockerfile_build.yml @@ -520,7 +520,7 @@ blocks: - ci-tools ci-update-version - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$S390X_ARCH - -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$S390X_ARCH -Ddocker.os_type=ubi9 -Ddocker.platform=linux/s390x $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS + -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$S390X_ARCH -Ddocker.os_type=ubi9 -Ddocker.platform=linux/s390x -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store # Verify images are built for s390x architecture - for image in $S390X_DOCKER_DEV_FULL_IMAGES; do echo "Checking architecture of $image:" && docker inspect --format='{{.Architecture}}' $image; done diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index f09962d4bf..d7a9704828 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -195,7 +195,7 @@ blocks: - ci-tools ci-update-version - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$S390X_ARCH - -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$S390X_ARCH -Ddocker.os_type=ubi9 -Ddocker.platform=linux/s390x $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS + -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$S390X_ARCH -Ddocker.os_type=ubi9 -Ddocker.platform=linux/s390x -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store # Verify images are built for s390x architecture - for image in $S390X_DOCKER_DEV_FULL_IMAGES; do echo "Checking architecture of $image:" && docker inspect --format='{{.Architecture}}' $image; done diff --git a/pom.xml b/pom.xml index b9dc665ecb..8e504422fc 100644 --- a/pom.xml +++ b/pom.xml @@ -141,28 +141,71 @@ - + + - docker + buildx - env.DOCKER_HOST + docker.platform - - - - com.spotify - dockerfile-maven-plugin - - - package - none - - - - + + + + + com.spotify + dockerfile-maven-plugin + + + default + none + + + package + none + + + tag + none + + + + + + io.fabric8 + docker-maven-plugin + 0.43.4 + + + + ${docker.registry}confluentinc/${project.artifactId}:${docker.tag} + + ${docker.file} + ${project.basedir} + + + ${docker.platform} + + + + + + true + + + + build-image + package + + build + + + + + + From ee714848ce92e15ae8786a12c8947e76ef25761e Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Mon, 26 Jan 2026 19:28:33 -0800 Subject: [PATCH 04/45] fix --- base-java-micro/pom.xml | 3 +++ base-java/pom.xml | 3 +++ base-lite/pom.xml | 3 +++ base/pom.xml | 3 +++ jmxterm/pom.xml | 29 +++++++++++++++++++++++++++++ pom.xml | 17 ----------------- 6 files changed, 41 insertions(+), 17 deletions(-) diff --git a/base-java-micro/pom.xml b/base-java-micro/pom.xml index acb49ac50d..8cb29e17bb 100644 --- a/base-java-micro/pom.xml +++ b/base-java-micro/pom.xml @@ -148,7 +148,10 @@ + ${docker.registry}confluentinc/${project.artifactId}:${docker.tag} + ${docker.file} + ${project.basedir} ${docker.platform} diff --git a/base-java/pom.xml b/base-java/pom.xml index d51d4aa5eb..817dd46f2c 100644 --- a/base-java/pom.xml +++ b/base-java/pom.xml @@ -144,7 +144,10 @@ + ${docker.registry}confluentinc/${project.artifactId}:${docker.tag} + ${docker.file} + ${project.basedir} ${docker.platform} diff --git a/base-lite/pom.xml b/base-lite/pom.xml index 7719168c75..877093f659 100644 --- a/base-lite/pom.xml +++ b/base-lite/pom.xml @@ -99,7 +99,10 @@ + ${docker.registry}confluentinc/${project.artifactId}:${docker.tag} + ${docker.file} + ${project.basedir} ${docker.platform} diff --git a/base/pom.xml b/base/pom.xml index a7b23f8765..50b3689504 100644 --- a/base/pom.xml +++ b/base/pom.xml @@ -157,7 +157,10 @@ + ${docker.registry}confluentinc/${project.artifactId}:${docker.tag} + ${docker.file} + ${project.basedir} ${docker.platform} diff --git a/jmxterm/pom.xml b/jmxterm/pom.xml index d3ef9a36ea..e8ad4a5221 100644 --- a/jmxterm/pom.xml +++ b/jmxterm/pom.xml @@ -47,4 +47,33 @@ + + + + io.fabric8 + docker-maven-plugin + 0.43.4 + + + + ${docker.registry}confluentinc/${project.artifactId}:${docker.tag} + + ${docker.file} + ${project.basedir} + + + ${docker.platform} + + + + ${docker.registry} + ${docker.upstream-tag} + + + + + + + + diff --git a/pom.xml b/pom.xml index 8e504422fc..ea88c7a04d 100644 --- a/pom.xml +++ b/pom.xml @@ -177,23 +177,6 @@ io.fabric8 docker-maven-plugin 0.43.4 - - - - ${docker.registry}confluentinc/${project.artifactId}:${docker.tag} - - ${docker.file} - ${project.basedir} - - - ${docker.platform} - - - - - - true - build-image From 53944851726cbb2da7d4fd98d2883da20090114c Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Mon, 26 Jan 2026 19:38:00 -0800 Subject: [PATCH 05/45] fix --- base-java-micro/pom.xml | 8 +++++--- base-java/pom.xml | 8 +++++--- base-lite/pom.xml | 8 +++++--- base/pom.xml | 17 ++++++++--------- jmxterm/pom.xml | 4 ++++ 5 files changed, 27 insertions(+), 18 deletions(-) diff --git a/base-java-micro/pom.xml b/base-java-micro/pom.xml index 8cb29e17bb..8f1dd43680 100644 --- a/base-java-micro/pom.xml +++ b/base-java-micro/pom.xml @@ -158,15 +158,17 @@ + ${project.artifactId} + ${project.version} + ${GIT_COMMIT} + ${BUILD_NUMBER} ${app.uid} ${app.gid} ${ubi9-micro.image.version} ${ubi9.image.version} -${ubi9.temurin-21-jdk.version} -${ubi9.procps-ng.version} - - ${docker.skip-security-update-check} - + ${docker.skip-security-update-check} ${golang.image.version} -${ubi9.crypto-policies-scripts.version} -${ubi9.findutils.version} diff --git a/base-java/pom.xml b/base-java/pom.xml index 817dd46f2c..10e25002ac 100644 --- a/base-java/pom.xml +++ b/base-java/pom.xml @@ -154,12 +154,14 @@ + ${project.artifactId} + ${project.version} + ${GIT_COMMIT} + ${BUILD_NUMBER} ${ubi9-minimal.image.version} -${ubi9-minimal.temurin-21-jdk.version} -${ubi9-minimal.procps-ng.version} - - ${docker.skip-security-update-check} - + ${docker.skip-security-update-check} ${golang.image.version} -${ubi9-minimal.crypto-policies-scripts.version} -${ubi9-minimal.findutils.version} diff --git a/base-lite/pom.xml b/base-lite/pom.xml index 877093f659..e75dd92efe 100644 --- a/base-lite/pom.xml +++ b/base-lite/pom.xml @@ -109,11 +109,13 @@ + ${project.artifactId} + ${project.version} + ${GIT_COMMIT} + ${BUILD_NUMBER} ${ubi9-minimal.image.version} -${ubi9-minimal.temurin-21-jdk.version} - - ${docker.skip-security-update-check} - + ${docker.skip-security-update-check} ${golang.image.version} diff --git a/base/pom.xml b/base/pom.xml index 50b3689504..8d1f6b20da 100644 --- a/base/pom.xml +++ b/base/pom.xml @@ -167,6 +167,10 @@ + ${project.artifactId} + ${project.version} + ${GIT_COMMIT} + ${BUILD_NUMBER} ${ubi9-minimal.image.version} -${ubi9-minimal.openssl.version} -${ubi9-minimal.wget.version} @@ -174,8 +178,7 @@ -${ubi9-minimal.python3.version} -${ubi9-minimal.tar.version} -${ubi9-minimal.procps-ng.version} - -${ubi9-minimal.krb5-workstation.version} - + -${ubi9-minimal.krb5-workstation.version} -${ubi9-minimal.iputils.version} -${ubi9-minimal.hostname.version} -${ubi9-minimal.xz-libs.version} @@ -184,13 +187,9 @@ -${ubi9-minimal.crypto-policies-scripts.version} -${ubi9-minimal.temurin-21-jdk.version} -${ubi9-minimal.python3-pip.version} - ==${python.setuptools.version} - - - ${git-repo.confluent-docker-utils.tag} - - ${docker.skip-security-update-check} - + ==${python.setuptools.version} + ${git-repo.confluent-docker-utils.tag} + ${docker.skip-security-update-check} diff --git a/jmxterm/pom.xml b/jmxterm/pom.xml index e8ad4a5221..2b815fb3ff 100644 --- a/jmxterm/pom.xml +++ b/jmxterm/pom.xml @@ -66,6 +66,10 @@ + ${project.artifactId} + ${project.version} + ${GIT_COMMIT} + ${BUILD_NUMBER} ${docker.registry} ${docker.upstream-tag} From 1a687d91a5d485a523175f6626de632eef3aabca Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Mon, 2 Feb 2026 12:18:04 -0800 Subject: [PATCH 06/45] revert pom override and use docker directly --- .semaphore/cp_dockerfile_build.yml | 76 ++++++++++++++++++++++++++++-- .semaphore/semaphore.yml | 76 ++++++++++++++++++++++++++++-- base-java-micro/pom.xml | 40 ---------------- base-java/pom.xml | 36 -------------- base-lite/pom.xml | 31 ------------ base/pom.xml | 46 ------------------ jmxterm/pom.xml | 34 +------------ pom.xml | 55 --------------------- 8 files changed, 143 insertions(+), 251 deletions(-) diff --git a/.semaphore/cp_dockerfile_build.yml b/.semaphore/cp_dockerfile_build.yml index 34875725f5..983a6b715c 100644 --- a/.semaphore/cp_dockerfile_build.yml +++ b/.semaphore/cp_dockerfile_build.yml @@ -508,20 +508,86 @@ blocks: - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - docker buildx create --name s390x-builder --use || docker buildx use s390x-builder - docker buildx inspect --bootstrap - # Make buildx the default builder so 'docker build' uses buildx - - docker buildx install - # Set default platform for all docker commands - - export DOCKER_DEFAULT_PLATFORM=linux/s390x - export OS_TAG="-ubi9" - export DOCKER_DEV_FULL_IMAGES=$DOCKER_DEV_REGISTRY${DOCKER_REPOS// /:$DOCKER_DEV_TAG$OS_TAG $DOCKER_DEV_REGISTRY}:$DOCKER_DEV_TAG$OS_TAG - export S390X_DOCKER_DEV_FULL_IMAGES=${DOCKER_DEV_FULL_IMAGES// /$S390X_ARCH }$S390X_ARCH - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - ci-tools ci-update-version + # Build artifacts only with maven (skip docker image build) - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$S390X_ARCH - -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$S390X_ARCH -Ddocker.os_type=ubi9 -Ddocker.platform=linux/s390x -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS + -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$S390X_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store + # Extract essential version properties from pom.xml for docker build args + - export PROJECT_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) + - export UBI9_MINIMAL_VERSION=$(mvn help:evaluate -Dexpression=ubi9-minimal.image.version -q -DforceStdout) + - export UBI9_MICRO_VERSION=$(mvn help:evaluate -Dexpression=ubi9-micro.image.version -q -DforceStdout) + - export UBI9_VERSION=$(mvn help:evaluate -Dexpression=ubi9.image.version -q -DforceStdout) + - export GOLANG_VERSION=$(mvn help:evaluate -Dexpression=golang.image.version -q -DforceStdout) + - export CP_DOCKER_UTILS_VERSION=$(mvn help:evaluate -Dexpression=git-repo.cp-docker-utils.tag -q -DforceStdout) + - export CONFLUENT_DOCKER_UTILS_VERSION=$(mvn help:evaluate -Dexpression=git-repo.confluent-docker-utils.tag -q -DforceStdout) + # Build Docker images with docker buildx for s390x platform + # Note: Package versions (openssl, wget, etc.) use defaults - latest available in UBI repos + # Build cp-base-new (base image for others) + - | + docker buildx build --platform linux/s390x --load \ + -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \ + -f base/Dockerfile.ubi9 \ + --build-arg ARTIFACT_ID=cp-base-new \ + --build-arg PROJECT_VERSION=${PROJECT_VERSION} \ + --build-arg GIT_COMMIT=${GIT_COMMIT} \ + --build-arg BUILD_NUMBER=${BUILD_NUMBER} \ + --build-arg UBI_MINIMAL_VERSION=${UBI9_MINIMAL_VERSION} \ + --build-arg PYTHON_CONFLUENT_DOCKER_UTILS_VERSION=${CONFLUENT_DOCKER_UTILS_VERSION} \ + base/ + # Build cp-base-java + - | + docker buildx build --platform linux/s390x --load \ + -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \ + -f base-java/Dockerfile.ubi9 \ + --build-arg ARTIFACT_ID=cp-base-java \ + --build-arg PROJECT_VERSION=${PROJECT_VERSION} \ + --build-arg GIT_COMMIT=${GIT_COMMIT} \ + --build-arg BUILD_NUMBER=${BUILD_NUMBER} \ + --build-arg UBI_MINIMAL_VERSION=${UBI9_MINIMAL_VERSION} \ + --build-arg GOLANG_VERSION=${GOLANG_VERSION} \ + --build-arg CP_DOCKER_UTILS_VERSION=${CP_DOCKER_UTILS_VERSION} \ + base-java/ + # Build cp-base-java-micro + - | + docker buildx build --platform linux/s390x --load \ + -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \ + -f base-java-micro/Dockerfile.ubi9 \ + --build-arg ARTIFACT_ID=cp-base-java-micro \ + --build-arg PROJECT_VERSION=${PROJECT_VERSION} \ + --build-arg GIT_COMMIT=${GIT_COMMIT} \ + --build-arg BUILD_NUMBER=${BUILD_NUMBER} \ + --build-arg UBI_MICRO_VERSION=${UBI9_MICRO_VERSION} \ + --build-arg UBI9_VERSION=${UBI9_VERSION} \ + --build-arg GOLANG_VERSION=${GOLANG_VERSION} \ + --build-arg CP_DOCKER_UTILS_VERSION=${CP_DOCKER_UTILS_VERSION} \ + base-java-micro/ + # Build cp-base-lite + - | + docker buildx build --platform linux/s390x --load \ + -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \ + -f base-lite/Dockerfile.ubi9 \ + --build-arg ARTIFACT_ID=cp-base-lite \ + --build-arg PROJECT_VERSION=${PROJECT_VERSION} \ + --build-arg GIT_COMMIT=${GIT_COMMIT} \ + --build-arg BUILD_NUMBER=${BUILD_NUMBER} \ + --build-arg UBI_MINIMAL_VERSION=${UBI9_MINIMAL_VERSION} \ + --build-arg GOLANG_VERSION=${GOLANG_VERSION} \ + base-lite/ + # Build cp-jmxterm (depends on cp-base-new) + - | + docker buildx build --platform linux/s390x --load \ + -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \ + -f jmxterm/Dockerfile.ubi9 \ + --build-arg DOCKER_REGISTRY=${DOCKER_DEV_REGISTRY} \ + --build-arg DOCKER_TAG=${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \ + jmxterm/ # Verify images are built for s390x architecture - for image in $S390X_DOCKER_DEV_FULL_IMAGES; do echo "Checking architecture of $image:" && docker inspect --format='{{.Architecture}}' $image; done - for image in $S390X_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index d7a9704828..739e56c2e8 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -183,20 +183,86 @@ blocks: - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - docker buildx create --name s390x-builder --use || docker buildx use s390x-builder - docker buildx inspect --bootstrap - # Make buildx the default builder so 'docker build' uses buildx - - docker buildx install - # Set default platform for all docker commands - - export DOCKER_DEFAULT_PLATFORM=linux/s390x - export OS_TAG="-ubi9" - export DOCKER_DEV_FULL_IMAGES=$DOCKER_DEV_REGISTRY${DOCKER_REPOS// /:$DOCKER_DEV_TAG$OS_TAG $DOCKER_DEV_REGISTRY}:$DOCKER_DEV_TAG$OS_TAG - export S390X_DOCKER_DEV_FULL_IMAGES=${DOCKER_DEV_FULL_IMAGES// /$S390X_ARCH }$S390X_ARCH - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - ci-tools ci-update-version + # Build artifacts only with maven (skip docker image build) - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$S390X_ARCH - -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$S390X_ARCH -Ddocker.os_type=ubi9 -Ddocker.platform=linux/s390x -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS + -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$S390X_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store + # Extract essential version properties from pom.xml for docker build args + - export PROJECT_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) + - export UBI9_MINIMAL_VERSION=$(mvn help:evaluate -Dexpression=ubi9-minimal.image.version -q -DforceStdout) + - export UBI9_MICRO_VERSION=$(mvn help:evaluate -Dexpression=ubi9-micro.image.version -q -DforceStdout) + - export UBI9_VERSION=$(mvn help:evaluate -Dexpression=ubi9.image.version -q -DforceStdout) + - export GOLANG_VERSION=$(mvn help:evaluate -Dexpression=golang.image.version -q -DforceStdout) + - export CP_DOCKER_UTILS_VERSION=$(mvn help:evaluate -Dexpression=git-repo.cp-docker-utils.tag -q -DforceStdout) + - export CONFLUENT_DOCKER_UTILS_VERSION=$(mvn help:evaluate -Dexpression=git-repo.confluent-docker-utils.tag -q -DforceStdout) + # Build Docker images with docker buildx for s390x platform + # Note: Package versions (openssl, wget, etc.) use defaults - latest available in UBI repos + # Build cp-base-new (base image for others) + - | + docker buildx build --platform linux/s390x --load \ + -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \ + -f base/Dockerfile.ubi9 \ + --build-arg ARTIFACT_ID=cp-base-new \ + --build-arg PROJECT_VERSION=${PROJECT_VERSION} \ + --build-arg GIT_COMMIT=${GIT_COMMIT} \ + --build-arg BUILD_NUMBER=${BUILD_NUMBER} \ + --build-arg UBI_MINIMAL_VERSION=${UBI9_MINIMAL_VERSION} \ + --build-arg PYTHON_CONFLUENT_DOCKER_UTILS_VERSION=${CONFLUENT_DOCKER_UTILS_VERSION} \ + base/ + # Build cp-base-java + - | + docker buildx build --platform linux/s390x --load \ + -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \ + -f base-java/Dockerfile.ubi9 \ + --build-arg ARTIFACT_ID=cp-base-java \ + --build-arg PROJECT_VERSION=${PROJECT_VERSION} \ + --build-arg GIT_COMMIT=${GIT_COMMIT} \ + --build-arg BUILD_NUMBER=${BUILD_NUMBER} \ + --build-arg UBI_MINIMAL_VERSION=${UBI9_MINIMAL_VERSION} \ + --build-arg GOLANG_VERSION=${GOLANG_VERSION} \ + --build-arg CP_DOCKER_UTILS_VERSION=${CP_DOCKER_UTILS_VERSION} \ + base-java/ + # Build cp-base-java-micro + - | + docker buildx build --platform linux/s390x --load \ + -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \ + -f base-java-micro/Dockerfile.ubi9 \ + --build-arg ARTIFACT_ID=cp-base-java-micro \ + --build-arg PROJECT_VERSION=${PROJECT_VERSION} \ + --build-arg GIT_COMMIT=${GIT_COMMIT} \ + --build-arg BUILD_NUMBER=${BUILD_NUMBER} \ + --build-arg UBI_MICRO_VERSION=${UBI9_MICRO_VERSION} \ + --build-arg UBI9_VERSION=${UBI9_VERSION} \ + --build-arg GOLANG_VERSION=${GOLANG_VERSION} \ + --build-arg CP_DOCKER_UTILS_VERSION=${CP_DOCKER_UTILS_VERSION} \ + base-java-micro/ + # Build cp-base-lite + - | + docker buildx build --platform linux/s390x --load \ + -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \ + -f base-lite/Dockerfile.ubi9 \ + --build-arg ARTIFACT_ID=cp-base-lite \ + --build-arg PROJECT_VERSION=${PROJECT_VERSION} \ + --build-arg GIT_COMMIT=${GIT_COMMIT} \ + --build-arg BUILD_NUMBER=${BUILD_NUMBER} \ + --build-arg UBI_MINIMAL_VERSION=${UBI9_MINIMAL_VERSION} \ + --build-arg GOLANG_VERSION=${GOLANG_VERSION} \ + base-lite/ + # Build cp-jmxterm (depends on cp-base-new) + - | + docker buildx build --platform linux/s390x --load \ + -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \ + -f jmxterm/Dockerfile.ubi9 \ + --build-arg DOCKER_REGISTRY=${DOCKER_DEV_REGISTRY} \ + --build-arg DOCKER_TAG=${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \ + jmxterm/ # Verify images are built for s390x architecture - for image in $S390X_DOCKER_DEV_FULL_IMAGES; do echo "Checking architecture of $image:" && docker inspect --format='{{.Architecture}}' $image; done - for image in $S390X_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done diff --git a/base-java-micro/pom.xml b/base-java-micro/pom.xml index 8f1dd43680..6cc87b386f 100644 --- a/base-java-micro/pom.xml +++ b/base-java-micro/pom.xml @@ -141,46 +141,6 @@ - - io.fabric8 - docker-maven-plugin - 0.43.4 - - - - ${docker.registry}confluentinc/${project.artifactId}:${docker.tag} - - ${docker.file} - ${project.basedir} - - - ${docker.platform} - - - - ${project.artifactId} - ${project.version} - ${GIT_COMMIT} - ${BUILD_NUMBER} - ${app.uid} - ${app.gid} - ${ubi9-micro.image.version} - ${ubi9.image.version} - -${ubi9.temurin-21-jdk.version} - -${ubi9.procps-ng.version} - ${docker.skip-security-update-check} - ${golang.image.version} - -${ubi9.crypto-policies-scripts.version} - -${ubi9.findutils.version} - -${ubi9.hostname.version} - -${ubi9.shadow-utils.version} - ${git-repo.cp-docker-utils.tag} - - - - - - diff --git a/base-java/pom.xml b/base-java/pom.xml index 10e25002ac..310c3e97ed 100644 --- a/base-java/pom.xml +++ b/base-java/pom.xml @@ -137,42 +137,6 @@ - - io.fabric8 - docker-maven-plugin - 0.43.4 - - - - ${docker.registry}confluentinc/${project.artifactId}:${docker.tag} - - ${docker.file} - ${project.basedir} - - - ${docker.platform} - - - - ${project.artifactId} - ${project.version} - ${GIT_COMMIT} - ${BUILD_NUMBER} - ${ubi9-minimal.image.version} - -${ubi9-minimal.temurin-21-jdk.version} - -${ubi9-minimal.procps-ng.version} - ${docker.skip-security-update-check} - ${golang.image.version} - -${ubi9-minimal.crypto-policies-scripts.version} - -${ubi9-minimal.findutils.version} - -${ubi9-minimal.hostname.version} - ${git-repo.cp-docker-utils.tag} - - - - - - diff --git a/base-lite/pom.xml b/base-lite/pom.xml index e75dd92efe..060a58a46d 100644 --- a/base-lite/pom.xml +++ b/base-lite/pom.xml @@ -92,37 +92,6 @@ - - io.fabric8 - docker-maven-plugin - 0.43.4 - - - - ${docker.registry}confluentinc/${project.artifactId}:${docker.tag} - - ${docker.file} - ${project.basedir} - - - ${docker.platform} - - - - ${project.artifactId} - ${project.version} - ${GIT_COMMIT} - ${BUILD_NUMBER} - ${ubi9-minimal.image.version} - -${ubi9-minimal.temurin-21-jdk.version} - ${docker.skip-security-update-check} - ${golang.image.version} - - - - - - diff --git a/base/pom.xml b/base/pom.xml index 8d1f6b20da..de46c5d149 100644 --- a/base/pom.xml +++ b/base/pom.xml @@ -150,52 +150,6 @@ - - io.fabric8 - docker-maven-plugin - 0.43.4 - - - - ${docker.registry}confluentinc/${project.artifactId}:${docker.tag} - - ${docker.file} - ${project.basedir} - - - ${docker.platform} - - - - ${project.artifactId} - ${project.version} - ${GIT_COMMIT} - ${BUILD_NUMBER} - ${ubi9-minimal.image.version} - -${ubi9-minimal.openssl.version} - -${ubi9-minimal.wget.version} - -${ubi9-minimal.nmap-ncat.version} - -${ubi9-minimal.python3.version} - -${ubi9-minimal.tar.version} - -${ubi9-minimal.procps-ng.version} - -${ubi9-minimal.krb5-workstation.version} - -${ubi9-minimal.iputils.version} - -${ubi9-minimal.hostname.version} - -${ubi9-minimal.xz-libs.version} - -${ubi9-minimal.glibc.version} - -${ubi9-minimal.findutils.version} - -${ubi9-minimal.crypto-policies-scripts.version} - -${ubi9-minimal.temurin-21-jdk.version} - -${ubi9-minimal.python3-pip.version} - ==${python.setuptools.version} - ${git-repo.confluent-docker-utils.tag} - ${docker.skip-security-update-check} - - - - - - diff --git a/jmxterm/pom.xml b/jmxterm/pom.xml index 2b815fb3ff..d1916a1ae2 100644 --- a/jmxterm/pom.xml +++ b/jmxterm/pom.xml @@ -47,37 +47,5 @@ - - - - io.fabric8 - docker-maven-plugin - 0.43.4 - - - - ${docker.registry}confluentinc/${project.artifactId}:${docker.tag} - - ${docker.file} - ${project.basedir} - - - ${docker.platform} - - - - ${project.artifactId} - ${project.version} - ${GIT_COMMIT} - ${BUILD_NUMBER} - ${docker.registry} - ${docker.upstream-tag} - - - - - - - - + diff --git a/pom.xml b/pom.xml index ea88c7a04d..dc20e8aa6c 100644 --- a/pom.xml +++ b/pom.xml @@ -35,8 +35,6 @@ Dockerfile.${docker.ubi9.os_type} ${io.confluent.common-docker.version}-${docker.ubi9.os_type} 8.3.0-0 - - @@ -139,57 +137,4 @@ `-Ddocker.skip-security-update-check=true` --> false - - - - - - buildx - - - docker.platform - - - - - - - - com.spotify - dockerfile-maven-plugin - - - default - none - - - package - none - - - tag - none - - - - - - io.fabric8 - docker-maven-plugin - 0.43.4 - - - build-image - package - - build - - - - - - - - - From 9f023114152d34384584a3af33518cff8265419f Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Mon, 2 Feb 2026 12:33:25 -0800 Subject: [PATCH 07/45] update ubi package versions --- base/pom.xml | 2 +- pom.xml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/base/pom.xml b/base/pom.xml index de46c5d149..b96d0d48d5 100644 --- a/base/pom.xml +++ b/base/pom.xml @@ -141,7 +141,7 @@ -${ubi9-minimal.xz-libs.version} -${ubi9-minimal.glibc.version} -${ubi9-minimal.findutils.version} - -${ubi8-minimal.crypto-policies-scripts.version} + -${ubi9-minimal.crypto-policies-scripts.version} -${ubi9-minimal.temurin-21-jdk.version} -${ubi9-minimal.python3-pip.version} ==${python.setuptools.version} diff --git a/pom.xml b/pom.xml index dc20e8aa6c..e7595fe9b6 100644 --- a/pom.xml +++ b/pom.xml @@ -79,10 +79,10 @@ 3.23-6.el9 4.9-15.el9 - 3.5.1-5.el9_7 + 3.5.1-7.el9_7 1.21.1-8.el9_4 7.92-3.el9 - 3.9.25-2.el9_7 + 3.9.25-3.el9_7 1.34-9.el9_7 3.3.17-14.el9 1.21.1-8.el9_6 From 6c3c24dee634406b7aa50009295c9eaa9a68f9c0 Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Mon, 2 Feb 2026 12:38:41 -0800 Subject: [PATCH 08/45] add some build tools for python packages --- base/Dockerfile.ubi9 | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/base/Dockerfile.ubi9 b/base/Dockerfile.ubi9 index 7eeb662384..6649dedf6f 100644 --- a/base/Dockerfile.ubi9 +++ b/base/Dockerfile.ubi9 @@ -89,11 +89,14 @@ RUN microdnf --nodocs -y install yum \ && yum --nodocs update -y \ && yum --nodocs install -y --setopt=install_weak_deps=False \ git \ + gcc \ + make \ "openssl${OPENSSL_VERSION}" \ "wget${WGET_VERSION}" \ "nmap-ncat${NETCAT_VERSION}" \ "python3${PYTHON39_VERSION}" \ "python3-pip${PYTHON_PIP_VERSION}" \ + "python3-devel" \ "tar${TAR_VERSION}" \ "procps-ng${PROCPS_VERSION}" \ "krb5-workstation${KRB5_WORKSTATION_VERSION}" \ @@ -106,13 +109,20 @@ RUN microdnf --nodocs -y install yum \ "findutils${FINDUTILS_VERSION}" \ "crypto-policies-scripts${CRYPTO_POLICIES_SCRIPTS_VERSION}" \ "temurin-21-jdk${TEMURIN_JDK_VERSION}" \ + "libffi-devel" \ + "openssl-devel" \ + "libsodium-devel" \ && alternatives --install /usr/bin/python python /usr/bin/python3 2000 \ && alternatives --set python /usr/bin/python3 \ && python3 -m pip install --upgrade "setuptools${PYTHON_SETUPTOOLS_VERSION}" \ + # Install Rust for building bcrypt/cryptography on s390x (removed after pip install) + && wget -qO - https://sh.rustup.rs | sh -s -- -y --profile minimal \ + && . $HOME/.cargo/env \ && python3 -m pip install --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}" \ - && yum remove -y git \ + && rustup self uninstall -y \ + && yum remove -y git gcc make python3-devel libffi-devel openssl-devel libsodium-devel \ && yum clean all \ - && rm -rf /tmp/* \ + && rm -rf /tmp/* /root/.cargo /root/.rustup \ && mkdir -p /etc/confluent/docker /usr/logs \ && useradd --no-log-init --create-home --shell /bin/bash appuser \ && chown appuser:appuser -R /etc/confluent/ /usr/logs \ From 29c01de434b8fb1bd03b86249d19b303c2865ca1 Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Mon, 2 Feb 2026 12:51:22 -0800 Subject: [PATCH 09/45] fix --- base/Dockerfile.ubi9 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/base/Dockerfile.ubi9 b/base/Dockerfile.ubi9 index 6649dedf6f..4f8ce4c42f 100644 --- a/base/Dockerfile.ubi9 +++ b/base/Dockerfile.ubi9 @@ -111,7 +111,6 @@ RUN microdnf --nodocs -y install yum \ "temurin-21-jdk${TEMURIN_JDK_VERSION}" \ "libffi-devel" \ "openssl-devel" \ - "libsodium-devel" \ && alternatives --install /usr/bin/python python /usr/bin/python3 2000 \ && alternatives --set python /usr/bin/python3 \ && python3 -m pip install --upgrade "setuptools${PYTHON_SETUPTOOLS_VERSION}" \ @@ -120,7 +119,7 @@ RUN microdnf --nodocs -y install yum \ && . $HOME/.cargo/env \ && python3 -m pip install --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}" \ && rustup self uninstall -y \ - && yum remove -y git gcc make python3-devel libffi-devel openssl-devel libsodium-devel \ + && yum remove -y git gcc make python3-devel libffi-devel openssl-devel \ && yum clean all \ && rm -rf /tmp/* /root/.cargo /root/.rustup \ && mkdir -p /etc/confluent/docker /usr/logs \ From 8cdbbcfa8ff9f059480c61d39ec3560ece7ff45d Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Mon, 2 Feb 2026 13:07:49 -0800 Subject: [PATCH 10/45] separate maven and docker for amd/arm --- .semaphore/cp_dockerfile_build.yml | 93 +++++---------------- .semaphore/semaphore.yml | 93 +++++---------------- base/Dockerfile.ubi9 | 128 +++++++++++++---------------- 3 files changed, 100 insertions(+), 214 deletions(-) diff --git a/.semaphore/cp_dockerfile_build.yml b/.semaphore/cp_dockerfile_build.yml index 983a6b715c..d04cfca293 100644 --- a/.semaphore/cp_dockerfile_build.yml +++ b/.semaphore/cp_dockerfile_build.yml @@ -132,10 +132,17 @@ blocks: - ci-tools ci-update-version - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" + # Build artifacts only with maven (skip docker image build) - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH - -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi9 $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS + -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store + # Build Docker images (ARG defaults are in Dockerfiles) + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} -f base/Dockerfile.ubi9 base/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} --build-arg ARTIFACT_ID=cp-base-java -f base-java/Dockerfile.ubi9 base-java/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} --build-arg ARTIFACT_ID=cp-base-java-micro -f base-java-micro/Dockerfile.ubi9 base-java-micro/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} --build-arg ARTIFACT_ID=cp-base-lite -f base-lite/Dockerfile.ubi9 base-lite/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} --build-arg DOCKER_REGISTRY=${DOCKER_DEV_REGISTRY} --build-arg DOCKER_TAG=${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f jmxterm/Dockerfile.ubi9 jmxterm/ - >- for dev_image in $AMD_DOCKER_DEV_FULL_IMAGES; do @@ -316,10 +323,17 @@ blocks: - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - ci-tools ci-update-version + # Build artifacts only with maven (skip docker image build) - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH - -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi9 $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS + -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store + # Build Docker images (ARG defaults are in Dockerfiles) + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} -f base/Dockerfile.ubi9 base/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} --build-arg ARTIFACT_ID=cp-base-java -f base-java/Dockerfile.ubi9 base-java/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} --build-arg ARTIFACT_ID=cp-base-java-micro -f base-java-micro/Dockerfile.ubi9 base-java-micro/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} --build-arg ARTIFACT_ID=cp-base-lite -f base-lite/Dockerfile.ubi9 base-lite/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} --build-arg DOCKER_REGISTRY=${DOCKER_DEV_REGISTRY} --build-arg DOCKER_TAG=${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f jmxterm/Dockerfile.ubi9 jmxterm/ - for image in $ARM_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done epilogue: always: @@ -519,75 +533,12 @@ blocks: -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$S390X_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$S390X_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store - # Extract essential version properties from pom.xml for docker build args - - export PROJECT_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) - - export UBI9_MINIMAL_VERSION=$(mvn help:evaluate -Dexpression=ubi9-minimal.image.version -q -DforceStdout) - - export UBI9_MICRO_VERSION=$(mvn help:evaluate -Dexpression=ubi9-micro.image.version -q -DforceStdout) - - export UBI9_VERSION=$(mvn help:evaluate -Dexpression=ubi9.image.version -q -DforceStdout) - - export GOLANG_VERSION=$(mvn help:evaluate -Dexpression=golang.image.version -q -DforceStdout) - - export CP_DOCKER_UTILS_VERSION=$(mvn help:evaluate -Dexpression=git-repo.cp-docker-utils.tag -q -DforceStdout) - - export CONFLUENT_DOCKER_UTILS_VERSION=$(mvn help:evaluate -Dexpression=git-repo.confluent-docker-utils.tag -q -DforceStdout) - # Build Docker images with docker buildx for s390x platform - # Note: Package versions (openssl, wget, etc.) use defaults - latest available in UBI repos - # Build cp-base-new (base image for others) - - | - docker buildx build --platform linux/s390x --load \ - -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \ - -f base/Dockerfile.ubi9 \ - --build-arg ARTIFACT_ID=cp-base-new \ - --build-arg PROJECT_VERSION=${PROJECT_VERSION} \ - --build-arg GIT_COMMIT=${GIT_COMMIT} \ - --build-arg BUILD_NUMBER=${BUILD_NUMBER} \ - --build-arg UBI_MINIMAL_VERSION=${UBI9_MINIMAL_VERSION} \ - --build-arg PYTHON_CONFLUENT_DOCKER_UTILS_VERSION=${CONFLUENT_DOCKER_UTILS_VERSION} \ - base/ - # Build cp-base-java - - | - docker buildx build --platform linux/s390x --load \ - -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \ - -f base-java/Dockerfile.ubi9 \ - --build-arg ARTIFACT_ID=cp-base-java \ - --build-arg PROJECT_VERSION=${PROJECT_VERSION} \ - --build-arg GIT_COMMIT=${GIT_COMMIT} \ - --build-arg BUILD_NUMBER=${BUILD_NUMBER} \ - --build-arg UBI_MINIMAL_VERSION=${UBI9_MINIMAL_VERSION} \ - --build-arg GOLANG_VERSION=${GOLANG_VERSION} \ - --build-arg CP_DOCKER_UTILS_VERSION=${CP_DOCKER_UTILS_VERSION} \ - base-java/ - # Build cp-base-java-micro - - | - docker buildx build --platform linux/s390x --load \ - -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \ - -f base-java-micro/Dockerfile.ubi9 \ - --build-arg ARTIFACT_ID=cp-base-java-micro \ - --build-arg PROJECT_VERSION=${PROJECT_VERSION} \ - --build-arg GIT_COMMIT=${GIT_COMMIT} \ - --build-arg BUILD_NUMBER=${BUILD_NUMBER} \ - --build-arg UBI_MICRO_VERSION=${UBI9_MICRO_VERSION} \ - --build-arg UBI9_VERSION=${UBI9_VERSION} \ - --build-arg GOLANG_VERSION=${GOLANG_VERSION} \ - --build-arg CP_DOCKER_UTILS_VERSION=${CP_DOCKER_UTILS_VERSION} \ - base-java-micro/ - # Build cp-base-lite - - | - docker buildx build --platform linux/s390x --load \ - -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \ - -f base-lite/Dockerfile.ubi9 \ - --build-arg ARTIFACT_ID=cp-base-lite \ - --build-arg PROJECT_VERSION=${PROJECT_VERSION} \ - --build-arg GIT_COMMIT=${GIT_COMMIT} \ - --build-arg BUILD_NUMBER=${BUILD_NUMBER} \ - --build-arg UBI_MINIMAL_VERSION=${UBI9_MINIMAL_VERSION} \ - --build-arg GOLANG_VERSION=${GOLANG_VERSION} \ - base-lite/ - # Build cp-jmxterm (depends on cp-base-new) - - | - docker buildx build --platform linux/s390x --load \ - -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \ - -f jmxterm/Dockerfile.ubi9 \ - --build-arg DOCKER_REGISTRY=${DOCKER_DEV_REGISTRY} \ - --build-arg DOCKER_TAG=${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \ - jmxterm/ + # Build Docker images with buildx for s390x (ARG defaults are in Dockerfiles) + - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} -f base/Dockerfile.ubi9 base/ + - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} --build-arg ARTIFACT_ID=cp-base-java -f base-java/Dockerfile.ubi9 base-java/ + - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} --build-arg ARTIFACT_ID=cp-base-java-micro -f base-java-micro/Dockerfile.ubi9 base-java-micro/ + - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} --build-arg ARTIFACT_ID=cp-base-lite -f base-lite/Dockerfile.ubi9 base-lite/ + - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} --build-arg DOCKER_REGISTRY=${DOCKER_DEV_REGISTRY} --build-arg DOCKER_TAG=${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f jmxterm/Dockerfile.ubi9 jmxterm/ # Verify images are built for s390x architecture - for image in $S390X_DOCKER_DEV_FULL_IMAGES; do echo "Checking architecture of $image:" && docker inspect --format='{{.Architecture}}' $image; done - for image in $S390X_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 739e56c2e8..173a1f9579 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -124,10 +124,17 @@ blocks: - ci-tools ci-update-version - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" + # Build artifacts only with maven (skip docker image build) - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH - -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi9 $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS + -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store + # Build Docker images (ARG defaults are in Dockerfiles) + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} -f base/Dockerfile.ubi9 base/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} --build-arg ARTIFACT_ID=cp-base-java -f base-java/Dockerfile.ubi9 base-java/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} --build-arg ARTIFACT_ID=cp-base-java-micro -f base-java-micro/Dockerfile.ubi9 base-java-micro/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} --build-arg ARTIFACT_ID=cp-base-lite -f base-lite/Dockerfile.ubi9 base-lite/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} --build-arg DOCKER_REGISTRY=${DOCKER_DEV_REGISTRY} --build-arg DOCKER_TAG=${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f jmxterm/Dockerfile.ubi9 jmxterm/ - >- for dev_image in $AMD_DOCKER_DEV_FULL_IMAGES; do @@ -157,10 +164,17 @@ blocks: - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - ci-tools ci-update-version + # Build artifacts only with maven (skip docker image build) - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH - -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi9 $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS + -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store + # Build Docker images (ARG defaults are in Dockerfiles) + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} -f base/Dockerfile.ubi9 base/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} --build-arg ARTIFACT_ID=cp-base-java -f base-java/Dockerfile.ubi9 base-java/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} --build-arg ARTIFACT_ID=cp-base-java-micro -f base-java-micro/Dockerfile.ubi9 base-java-micro/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} --build-arg ARTIFACT_ID=cp-base-lite -f base-lite/Dockerfile.ubi9 base-lite/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} --build-arg DOCKER_REGISTRY=${DOCKER_DEV_REGISTRY} --build-arg DOCKER_TAG=${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f jmxterm/Dockerfile.ubi9 jmxterm/ - for image in $ARM_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done epilogue: always: @@ -194,75 +208,12 @@ blocks: -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$S390X_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$S390X_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store - # Extract essential version properties from pom.xml for docker build args - - export PROJECT_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) - - export UBI9_MINIMAL_VERSION=$(mvn help:evaluate -Dexpression=ubi9-minimal.image.version -q -DforceStdout) - - export UBI9_MICRO_VERSION=$(mvn help:evaluate -Dexpression=ubi9-micro.image.version -q -DforceStdout) - - export UBI9_VERSION=$(mvn help:evaluate -Dexpression=ubi9.image.version -q -DforceStdout) - - export GOLANG_VERSION=$(mvn help:evaluate -Dexpression=golang.image.version -q -DforceStdout) - - export CP_DOCKER_UTILS_VERSION=$(mvn help:evaluate -Dexpression=git-repo.cp-docker-utils.tag -q -DforceStdout) - - export CONFLUENT_DOCKER_UTILS_VERSION=$(mvn help:evaluate -Dexpression=git-repo.confluent-docker-utils.tag -q -DforceStdout) - # Build Docker images with docker buildx for s390x platform - # Note: Package versions (openssl, wget, etc.) use defaults - latest available in UBI repos - # Build cp-base-new (base image for others) - - | - docker buildx build --platform linux/s390x --load \ - -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \ - -f base/Dockerfile.ubi9 \ - --build-arg ARTIFACT_ID=cp-base-new \ - --build-arg PROJECT_VERSION=${PROJECT_VERSION} \ - --build-arg GIT_COMMIT=${GIT_COMMIT} \ - --build-arg BUILD_NUMBER=${BUILD_NUMBER} \ - --build-arg UBI_MINIMAL_VERSION=${UBI9_MINIMAL_VERSION} \ - --build-arg PYTHON_CONFLUENT_DOCKER_UTILS_VERSION=${CONFLUENT_DOCKER_UTILS_VERSION} \ - base/ - # Build cp-base-java - - | - docker buildx build --platform linux/s390x --load \ - -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \ - -f base-java/Dockerfile.ubi9 \ - --build-arg ARTIFACT_ID=cp-base-java \ - --build-arg PROJECT_VERSION=${PROJECT_VERSION} \ - --build-arg GIT_COMMIT=${GIT_COMMIT} \ - --build-arg BUILD_NUMBER=${BUILD_NUMBER} \ - --build-arg UBI_MINIMAL_VERSION=${UBI9_MINIMAL_VERSION} \ - --build-arg GOLANG_VERSION=${GOLANG_VERSION} \ - --build-arg CP_DOCKER_UTILS_VERSION=${CP_DOCKER_UTILS_VERSION} \ - base-java/ - # Build cp-base-java-micro - - | - docker buildx build --platform linux/s390x --load \ - -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \ - -f base-java-micro/Dockerfile.ubi9 \ - --build-arg ARTIFACT_ID=cp-base-java-micro \ - --build-arg PROJECT_VERSION=${PROJECT_VERSION} \ - --build-arg GIT_COMMIT=${GIT_COMMIT} \ - --build-arg BUILD_NUMBER=${BUILD_NUMBER} \ - --build-arg UBI_MICRO_VERSION=${UBI9_MICRO_VERSION} \ - --build-arg UBI9_VERSION=${UBI9_VERSION} \ - --build-arg GOLANG_VERSION=${GOLANG_VERSION} \ - --build-arg CP_DOCKER_UTILS_VERSION=${CP_DOCKER_UTILS_VERSION} \ - base-java-micro/ - # Build cp-base-lite - - | - docker buildx build --platform linux/s390x --load \ - -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \ - -f base-lite/Dockerfile.ubi9 \ - --build-arg ARTIFACT_ID=cp-base-lite \ - --build-arg PROJECT_VERSION=${PROJECT_VERSION} \ - --build-arg GIT_COMMIT=${GIT_COMMIT} \ - --build-arg BUILD_NUMBER=${BUILD_NUMBER} \ - --build-arg UBI_MINIMAL_VERSION=${UBI9_MINIMAL_VERSION} \ - --build-arg GOLANG_VERSION=${GOLANG_VERSION} \ - base-lite/ - # Build cp-jmxterm (depends on cp-base-new) - - | - docker buildx build --platform linux/s390x --load \ - -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \ - -f jmxterm/Dockerfile.ubi9 \ - --build-arg DOCKER_REGISTRY=${DOCKER_DEV_REGISTRY} \ - --build-arg DOCKER_TAG=${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \ - jmxterm/ + # Build Docker images with buildx for s390x (ARG defaults are in Dockerfiles) + - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} -f base/Dockerfile.ubi9 base/ + - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} --build-arg ARTIFACT_ID=cp-base-java -f base-java/Dockerfile.ubi9 base-java/ + - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} --build-arg ARTIFACT_ID=cp-base-java-micro -f base-java-micro/Dockerfile.ubi9 base-java-micro/ + - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} --build-arg ARTIFACT_ID=cp-base-lite -f base-lite/Dockerfile.ubi9 base-lite/ + - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} --build-arg DOCKER_REGISTRY=${DOCKER_DEV_REGISTRY} --build-arg DOCKER_TAG=${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f jmxterm/Dockerfile.ubi9 jmxterm/ # Verify images are built for s390x architecture - for image in $S390X_DOCKER_DEV_FULL_IMAGES; do echo "Checking architecture of $image:" && docker inspect --format='{{.Architecture}}' $image; done - for image in $S390X_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done diff --git a/base/Dockerfile.ubi9 b/base/Dockerfile.ubi9 index 4f8ce4c42f..725337a723 100644 --- a/base/Dockerfile.ubi9 +++ b/base/Dockerfile.ubi9 @@ -12,21 +12,22 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -ARG UBI_MINIMAL_VERSION="latest" +# Default versions from pom.xml - can be overridden at build time +ARG UBI_MINIMAL_VERSION="9.7-1768783948" FROM registry.access.redhat.com/ubi9/ubi-minimal:${UBI_MINIMAL_VERSION} -ARG PROJECT_VERSION -ARG ARTIFACT_ID +# Build args with defaults +ARG PROJECT_VERSION="8.3.0-0" +ARG ARTIFACT_ID="cp-base-new" +ARG GIT_COMMIT="unknown" +ARG BUILD_NUMBER=-1 +# TARGETARCH is automatically set by Docker buildx (amd64, arm64, s390x) +ARG TARGETARCH -# Remember where we came from +# Labels LABEL io.confluent.docker.git.repo="confluentinc/common-docker" - -ARG GIT_COMMIT LABEL io.confluent.docker.git.id=$GIT_COMMIT - -ARG BUILD_NUMBER=-1 LABEL io.confluent.docker.build.number=$BUILD_NUMBER - LABEL maintainer="tools@confluent.io" LABEL vendor="Confluent" LABEL version=$GIT_COMMIT @@ -44,37 +45,10 @@ ENV LANG="C.UTF-8" ENV CUB_CLASSPATH='"/usr/share/java/cp-base-new/*"' ENV USE_LOG4J_2="True" -# These ARGs are left blank indicating to the Dnf package manager to install the latest package -# version that happens to be availible at this time. For reproducible builds, versions should be specified -# as '-1.2.3-4.el8' on the command line. Or more preferibly the 'dockerfile-maven-plugin' is used -# where these arguments are set in base/pom.xml under the elements based on the commit you're -# building from. - -# Redhat Package Versions -ARG OPENSSL_VERSION="" -ARG WGET_VERSION="" -ARG NETCAT_VERSION="" -ARG PYTHON39_VERSION="" -ARG TAR_VERSION="" -ARG PROCPS_VERSION="" -ARG KRB5_WORKSTATION_VERSION="" -ARG IPUTILS_VERSION="" -ARG HOSTNAME_VERSION="" -ARG XZ_LIBS_VERSION="" -ARG GLIBC_VERSION="" -ARG CURL_VERSION="" - -# Temurin JDK version -ARG TEMURIN_JDK_VERSION="" - -# Python Module Versions -ARG PYTHON_PIP_VERSION="" -ARG PYTHON_SETUPTOOLS_VERSION="" - -# Confluent Docker Utils Version (Namely the tag or branch to grab from git to install) -ARG PYTHON_CONFLUENT_DOCKER_UTILS_VERSION="master" - -# This can be overriden for an offline/air-gapped builds +# Confluent Docker Utils Version - default from pom.xml +ARG PYTHON_CONFLUENT_DOCKER_UTILS_VERSION="v0.0.164" + +# This can be overriden for offline/air-gapped builds ARG PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC="git+https://github.com/confluentinc/confluent-docker-utils@${PYTHON_CONFLUENT_DOCKER_UTILS_VERSION}" RUN printf "[temurin-jdk] \n\ @@ -85,47 +59,57 @@ gpgcheck=1 \n\ gpgkey=https://adoptium.jfrog.io/artifactory/api/gpg/key/public \n\ " > /etc/yum.repos.d/adoptium.repo +# Install base packages RUN microdnf --nodocs -y install yum \ && yum --nodocs update -y \ && yum --nodocs install -y --setopt=install_weak_deps=False \ git \ - gcc \ - make \ - "openssl${OPENSSL_VERSION}" \ - "wget${WGET_VERSION}" \ - "nmap-ncat${NETCAT_VERSION}" \ - "python3${PYTHON39_VERSION}" \ - "python3-pip${PYTHON_PIP_VERSION}" \ - "python3-devel" \ - "tar${TAR_VERSION}" \ - "procps-ng${PROCPS_VERSION}" \ - "krb5-workstation${KRB5_WORKSTATION_VERSION}" \ - "iputils${IPUTILS_VERSION}" \ - "hostname${HOSTNAME_VERSION}" \ - "xz-libs${XZ_LIBS_VERSION}" \ - "glibc${GLIBC_VERSION}" \ - "glibc-common${GLIBC_VERSION}" \ - "glibc-minimal-langpack${GLIBC_VERSION}" \ - "findutils${FINDUTILS_VERSION}" \ - "crypto-policies-scripts${CRYPTO_POLICIES_SCRIPTS_VERSION}" \ - "temurin-21-jdk${TEMURIN_JDK_VERSION}" \ - "libffi-devel" \ - "openssl-devel" \ + openssl \ + wget \ + nmap-ncat \ + python3 \ + python3-pip \ + tar \ + procps-ng \ + krb5-workstation \ + iputils \ + hostname \ + xz-libs \ + glibc \ + glibc-common \ + glibc-minimal-langpack \ + findutils \ + crypto-policies-scripts \ + temurin-21-jdk \ && alternatives --install /usr/bin/python python /usr/bin/python3 2000 \ - && alternatives --set python /usr/bin/python3 \ - && python3 -m pip install --upgrade "setuptools${PYTHON_SETUPTOOLS_VERSION}" \ - # Install Rust for building bcrypt/cryptography on s390x (removed after pip install) - && wget -qO - https://sh.rustup.rs | sh -s -- -y --profile minimal \ - && . $HOME/.cargo/env \ - && python3 -m pip install --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}" \ - && rustup self uninstall -y \ - && yum remove -y git gcc make python3-devel libffi-devel openssl-devel \ + && alternatives --set python /usr/bin/python3 + +# For s390x: Install build tools needed to compile Python packages from source +# (bcrypt, cryptography, pynacl don't have pre-built wheels for s390x) +# For amd64/arm64: Skip this step as pre-built wheels are available +RUN if [ "$TARGETARCH" = "s390x" ]; then \ + yum --nodocs install -y --setopt=install_weak_deps=False \ + gcc make python3-devel libffi-devel openssl-devel \ + && wget -qO - https://sh.rustup.rs | sh -s -- -y --profile minimal \ + && . $HOME/.cargo/env \ + && python3 -m pip install --upgrade setuptools \ + && python3 -m pip install --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}" \ + && rustup self uninstall -y \ + && yum remove -y gcc make python3-devel libffi-devel openssl-devel \ + && rm -rf /root/.cargo /root/.rustup; \ + else \ + python3 -m pip install --upgrade setuptools \ + && python3 -m pip install --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}"; \ + fi + +# Cleanup +RUN yum remove -y git \ && yum clean all \ - && rm -rf /tmp/* /root/.cargo /root/.rustup \ + && rm -rf /tmp/* \ && mkdir -p /etc/confluent/docker /usr/logs \ && useradd --no-log-init --create-home --shell /bin/bash appuser \ && chown appuser:appuser -R /etc/confluent/ /usr/logs \ - && rm /etc/yum.repos.d/adoptium.repo # Remove temurin-jdk repo to reduce intermittent build failures + && rm /etc/yum.repos.d/adoptium.repo # This is a step that will cause the build to fail of the package manager detects a package update is availible and isn't installed. # The ARG SKIP_SECURITY_UPDATE_CHECK is an "escape" hatch if you want to by-pass this check and build the container anyways, which From f381b4d96ce18937f69b3aabfa658882632c691a Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Mon, 2 Feb 2026 13:23:46 -0800 Subject: [PATCH 11/45] fix --- .semaphore/cp_dockerfile_build.yml | 48 +++++++++++++++--------------- .semaphore/semaphore.yml | 48 +++++++++++++++--------------- 2 files changed, 48 insertions(+), 48 deletions(-) diff --git a/.semaphore/cp_dockerfile_build.yml b/.semaphore/cp_dockerfile_build.yml index d04cfca293..f69957e07c 100644 --- a/.semaphore/cp_dockerfile_build.yml +++ b/.semaphore/cp_dockerfile_build.yml @@ -132,17 +132,17 @@ blocks: - ci-tools ci-update-version - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - # Build artifacts only with maven (skip docker image build) + # Build artifacts with Maven (skip Docker image build, use docker CLI instead) - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH - -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS + -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi9 -Ddockerfile.skip=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store - # Build Docker images (ARG defaults are in Dockerfiles) - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} -f base/Dockerfile.ubi9 base/ - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} --build-arg ARTIFACT_ID=cp-base-java -f base-java/Dockerfile.ubi9 base-java/ - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} --build-arg ARTIFACT_ID=cp-base-java-micro -f base-java-micro/Dockerfile.ubi9 base-java-micro/ - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} --build-arg ARTIFACT_ID=cp-base-lite -f base-lite/Dockerfile.ubi9 base-lite/ - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} --build-arg DOCKER_REGISTRY=${DOCKER_DEV_REGISTRY} --build-arg DOCKER_TAG=${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f jmxterm/Dockerfile.ubi9 jmxterm/ + # Build Docker images directly + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base/Dockerfile.ubi9 base/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base-java/Dockerfile.ubi9 base-java/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base-java-micro/Dockerfile.ubi9 base-java-micro/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base-lite/Dockerfile.ubi9 base-lite/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f jmxterm/Dockerfile.ubi9 jmxterm/ - >- for dev_image in $AMD_DOCKER_DEV_FULL_IMAGES; do @@ -323,17 +323,17 @@ blocks: - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - ci-tools ci-update-version - # Build artifacts only with maven (skip docker image build) + # Build artifacts with Maven (skip Docker image build, use docker CLI instead) - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH - -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS + -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi9 -Ddockerfile.skip=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store - # Build Docker images (ARG defaults are in Dockerfiles) - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} -f base/Dockerfile.ubi9 base/ - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} --build-arg ARTIFACT_ID=cp-base-java -f base-java/Dockerfile.ubi9 base-java/ - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} --build-arg ARTIFACT_ID=cp-base-java-micro -f base-java-micro/Dockerfile.ubi9 base-java-micro/ - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} --build-arg ARTIFACT_ID=cp-base-lite -f base-lite/Dockerfile.ubi9 base-lite/ - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} --build-arg DOCKER_REGISTRY=${DOCKER_DEV_REGISTRY} --build-arg DOCKER_TAG=${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f jmxterm/Dockerfile.ubi9 jmxterm/ + # Build Docker images directly + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base/Dockerfile.ubi9 base/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base-java/Dockerfile.ubi9 base-java/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base-java-micro/Dockerfile.ubi9 base-java-micro/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base-lite/Dockerfile.ubi9 base-lite/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f jmxterm/Dockerfile.ubi9 jmxterm/ - for image in $ARM_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done epilogue: always: @@ -528,17 +528,17 @@ blocks: - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - ci-tools ci-update-version - # Build artifacts only with maven (skip docker image build) + # Build artifacts with Maven (skip Docker image build, use docker buildx instead) - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$S390X_ARCH - -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$S390X_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS + -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$S390X_ARCH -Ddocker.os_type=ubi9 -Ddockerfile.skip=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store - # Build Docker images with buildx for s390x (ARG defaults are in Dockerfiles) - - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} -f base/Dockerfile.ubi9 base/ - - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} --build-arg ARTIFACT_ID=cp-base-java -f base-java/Dockerfile.ubi9 base-java/ - - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} --build-arg ARTIFACT_ID=cp-base-java-micro -f base-java-micro/Dockerfile.ubi9 base-java-micro/ - - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} --build-arg ARTIFACT_ID=cp-base-lite -f base-lite/Dockerfile.ubi9 base-lite/ - - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} --build-arg DOCKER_REGISTRY=${DOCKER_DEV_REGISTRY} --build-arg DOCKER_TAG=${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f jmxterm/Dockerfile.ubi9 jmxterm/ + # Build Docker images with buildx for s390x platform + - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base/Dockerfile.ubi9 base/ + - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-java/Dockerfile.ubi9 base-java/ + - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-java-micro/Dockerfile.ubi9 base-java-micro/ + - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-lite/Dockerfile.ubi9 base-lite/ + - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f jmxterm/Dockerfile.ubi9 jmxterm/ # Verify images are built for s390x architecture - for image in $S390X_DOCKER_DEV_FULL_IMAGES; do echo "Checking architecture of $image:" && docker inspect --format='{{.Architecture}}' $image; done - for image in $S390X_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 173a1f9579..36d7584069 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -124,17 +124,17 @@ blocks: - ci-tools ci-update-version - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - # Build artifacts only with maven (skip docker image build) + # Build artifacts with Maven (skip Docker image build, use docker CLI instead) - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH - -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS + -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi9 -Ddockerfile.skip=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store - # Build Docker images (ARG defaults are in Dockerfiles) - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} -f base/Dockerfile.ubi9 base/ - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} --build-arg ARTIFACT_ID=cp-base-java -f base-java/Dockerfile.ubi9 base-java/ - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} --build-arg ARTIFACT_ID=cp-base-java-micro -f base-java-micro/Dockerfile.ubi9 base-java-micro/ - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} --build-arg ARTIFACT_ID=cp-base-lite -f base-lite/Dockerfile.ubi9 base-lite/ - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} --build-arg DOCKER_REGISTRY=${DOCKER_DEV_REGISTRY} --build-arg DOCKER_TAG=${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f jmxterm/Dockerfile.ubi9 jmxterm/ + # Build Docker images directly + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base/Dockerfile.ubi9 base/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base-java/Dockerfile.ubi9 base-java/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base-java-micro/Dockerfile.ubi9 base-java-micro/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base-lite/Dockerfile.ubi9 base-lite/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f jmxterm/Dockerfile.ubi9 jmxterm/ - >- for dev_image in $AMD_DOCKER_DEV_FULL_IMAGES; do @@ -164,17 +164,17 @@ blocks: - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - ci-tools ci-update-version - # Build artifacts only with maven (skip docker image build) + # Build artifacts with Maven (skip Docker image build, use docker CLI instead) - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH - -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS + -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi9 -Ddockerfile.skip=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store - # Build Docker images (ARG defaults are in Dockerfiles) - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} -f base/Dockerfile.ubi9 base/ - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} --build-arg ARTIFACT_ID=cp-base-java -f base-java/Dockerfile.ubi9 base-java/ - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} --build-arg ARTIFACT_ID=cp-base-java-micro -f base-java-micro/Dockerfile.ubi9 base-java-micro/ - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} --build-arg ARTIFACT_ID=cp-base-lite -f base-lite/Dockerfile.ubi9 base-lite/ - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} --build-arg DOCKER_REGISTRY=${DOCKER_DEV_REGISTRY} --build-arg DOCKER_TAG=${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f jmxterm/Dockerfile.ubi9 jmxterm/ + # Build Docker images directly + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base/Dockerfile.ubi9 base/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base-java/Dockerfile.ubi9 base-java/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base-java-micro/Dockerfile.ubi9 base-java-micro/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base-lite/Dockerfile.ubi9 base-lite/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f jmxterm/Dockerfile.ubi9 jmxterm/ - for image in $ARM_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done epilogue: always: @@ -203,17 +203,17 @@ blocks: - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - ci-tools ci-update-version - # Build artifacts only with maven (skip docker image build) + # Build artifacts with Maven (skip Docker image build, use docker buildx instead) - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$S390X_ARCH - -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$S390X_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS + -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$S390X_ARCH -Ddocker.os_type=ubi9 -Ddockerfile.skip=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store - # Build Docker images with buildx for s390x (ARG defaults are in Dockerfiles) - - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} -f base/Dockerfile.ubi9 base/ - - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} --build-arg ARTIFACT_ID=cp-base-java -f base-java/Dockerfile.ubi9 base-java/ - - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} --build-arg ARTIFACT_ID=cp-base-java-micro -f base-java-micro/Dockerfile.ubi9 base-java-micro/ - - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} --build-arg GIT_COMMIT=${GIT_COMMIT} --build-arg BUILD_NUMBER=${BUILD_NUMBER} --build-arg ARTIFACT_ID=cp-base-lite -f base-lite/Dockerfile.ubi9 base-lite/ - - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} --build-arg DOCKER_REGISTRY=${DOCKER_DEV_REGISTRY} --build-arg DOCKER_TAG=${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f jmxterm/Dockerfile.ubi9 jmxterm/ + # Build Docker images with buildx for s390x platform + - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base/Dockerfile.ubi9 base/ + - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-java/Dockerfile.ubi9 base-java/ + - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-java-micro/Dockerfile.ubi9 base-java-micro/ + - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-lite/Dockerfile.ubi9 base-lite/ + - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f jmxterm/Dockerfile.ubi9 jmxterm/ # Verify images are built for s390x architecture - for image in $S390X_DOCKER_DEV_FULL_IMAGES; do echo "Checking architecture of $image:" && docker inspect --format='{{.Architecture}}' $image; done - for image in $S390X_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done From fc63df605c2afb90369f08223fd52284982d5f25 Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Mon, 2 Feb 2026 15:33:05 -0800 Subject: [PATCH 12/45] fix: correct Docker build context to access Maven artifacts Changed Docker build context from subdirectories (base/, base-java/, etc.) to project root (.) to allow Dockerfiles to access the Maven-built artifacts in the target/ directory. Also updated all COPY commands in Dockerfiles to use correct paths relative to the new build context (e.g., base/include/ instead of include/). This fixes the build failure where Docker couldn't find the package artifacts: "lstat .../target/cp-base-new-8.3.0-0-package/share/doc: no such file or directory" Co-Authored-By: Claude Sonnet 4.5 --- .semaphore/semaphore.yml | 30 +++++++++++++++--------------- base-java-micro/Dockerfile.ubi9 | 6 +++--- base-java/Dockerfile.ubi9 | 6 +++--- base-lite/Dockerfile.ubi9 | 8 ++++---- base/Dockerfile.ubi9 | 6 +++--- 5 files changed, 28 insertions(+), 28 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 36d7584069..46a1a4e074 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -130,11 +130,11 @@ blocks: -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi9 -Ddockerfile.skip=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store # Build Docker images directly - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base/Dockerfile.ubi9 base/ - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base-java/Dockerfile.ubi9 base-java/ - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base-java-micro/Dockerfile.ubi9 base-java-micro/ - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base-lite/Dockerfile.ubi9 base-lite/ - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f jmxterm/Dockerfile.ubi9 jmxterm/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base/Dockerfile.ubi9 . + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base-java/Dockerfile.ubi9 . + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base-java-micro/Dockerfile.ubi9 . + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base-lite/Dockerfile.ubi9 . + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f jmxterm/Dockerfile.ubi9 . - >- for dev_image in $AMD_DOCKER_DEV_FULL_IMAGES; do @@ -170,11 +170,11 @@ blocks: -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi9 -Ddockerfile.skip=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store # Build Docker images directly - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base/Dockerfile.ubi9 base/ - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base-java/Dockerfile.ubi9 base-java/ - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base-java-micro/Dockerfile.ubi9 base-java-micro/ - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base-lite/Dockerfile.ubi9 base-lite/ - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f jmxterm/Dockerfile.ubi9 jmxterm/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base/Dockerfile.ubi9 . + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base-java/Dockerfile.ubi9 . + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base-java-micro/Dockerfile.ubi9 . + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base-lite/Dockerfile.ubi9 . + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f jmxterm/Dockerfile.ubi9 . - for image in $ARM_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done epilogue: always: @@ -209,11 +209,11 @@ blocks: -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$S390X_ARCH -Ddocker.os_type=ubi9 -Ddockerfile.skip=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store # Build Docker images with buildx for s390x platform - - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base/Dockerfile.ubi9 base/ - - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-java/Dockerfile.ubi9 base-java/ - - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-java-micro/Dockerfile.ubi9 base-java-micro/ - - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-lite/Dockerfile.ubi9 base-lite/ - - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f jmxterm/Dockerfile.ubi9 jmxterm/ + - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base/Dockerfile.ubi9 . + - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-java/Dockerfile.ubi9 . + - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-java-micro/Dockerfile.ubi9 . + - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-lite/Dockerfile.ubi9 . + - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f jmxterm/Dockerfile.ubi9 . # Verify images are built for s390x architecture - for image in $S390X_DOCKER_DEV_FULL_IMAGES; do echo "Checking architecture of $image:" && docker inspect --format='{{.Architecture}}' $image; done - for image in $S390X_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done diff --git a/base-java-micro/Dockerfile.ubi9 b/base-java-micro/Dockerfile.ubi9 index 832d5fe84d..9df844e8ef 100644 --- a/base-java-micro/Dockerfile.ubi9 +++ b/base-java-micro/Dockerfile.ubi9 @@ -87,15 +87,15 @@ RUN update-crypto-policies --set FIPS && \ mkdir -p /etc/confluent/docker /usr/logs /licenses && \ chown ${APP_UID}:${APP_GID} -R /etc/confluent/ /usr/logs -COPY license.txt /licenses +COPY base-java-micro/license.txt /licenses COPY --from=build-ub-package-dedupe --chown=${APP_UID}:${APP_GID} /go/bin/package_dedupe /usr/bin/package_dedupe COPY --from=build-ub-package-dedupe --chown=${APP_UID}:${APP_GID} /go/bin/ub /usr/bin/ub COPY --chown=${APP_UID}:${APP_GID} target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/ COPY --chown=${APP_UID}:${APP_GID} target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${ARTIFACT_ID}/ -COPY --chown=${APP_UID}:${APP_GID} include/etc/confluent/docker /etc/confluent/docker -COPY --chown=${APP_UID}:${APP_GID} include/etc/cp-base-java-micro /etc/cp-base-java-micro +COPY --chown=${APP_UID}:${APP_GID} base-java-micro/include/etc/confluent/docker /etc/confluent/docker +COPY --chown=${APP_UID}:${APP_GID} base-java-micro/include/etc/cp-base-java-micro /etc/cp-base-java-micro # Some components have hardcoded paths to /usr/share/java/cp-base-new, so to keep backward compatibility a symlink is created RUN ln -s /usr/share/java/${ARTIFACT_ID} /usr/share/java/cp-base-new diff --git a/base-java/Dockerfile.ubi9 b/base-java/Dockerfile.ubi9 index 068beb0317..a1efd72143 100644 --- a/base-java/Dockerfile.ubi9 +++ b/base-java/Dockerfile.ubi9 @@ -71,14 +71,14 @@ RUN echo "installing temurin-21-jre:${TEMURIN_JDK_VERSION}" \ # enable FIPS in docker image, this will only work if underlying OS has FIPS enabled as well else is a NO OP. RUN update-crypto-policies --set FIPS -COPY license.txt /licenses +COPY base-java/license.txt /licenses COPY --from=build-ub-package-dedupe /go/bin/package_dedupe /usr/bin/package_dedupe COPY --from=build-ub-package-dedupe /go/bin/ub /usr/bin/ub COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/ COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${ARTIFACT_ID}/ -COPY --chown=appuser:appuser include/etc/confluent/docker /etc/confluent/docker -COPY --chown=appuser:appuser include/etc/cp-base-java /etc/cp-base-java +COPY --chown=appuser:appuser base-java/include/etc/confluent/docker /etc/confluent/docker +COPY --chown=appuser:appuser base-java/include/etc/cp-base-java /etc/cp-base-java # Some components have hardcoded paths to /usr/share/java/cp-base-new, so to keep backward compatibility a symlink is created RUN ln -s /usr/share/java/${ARTIFACT_ID} /usr/share/java/cp-base-new diff --git a/base-lite/Dockerfile.ubi9 b/base-lite/Dockerfile.ubi9 index 8117900c3a..e241ba4d8f 100644 --- a/base-lite/Dockerfile.ubi9 +++ b/base-lite/Dockerfile.ubi9 @@ -20,7 +20,7 @@ ARG CURL_VERSION FROM golang:${GOLANG_VERSION} AS build-ub WORKDIR /build RUN useradd --no-log-init --create-home --shell /bin/bash appuser -COPY --chown=appuser:appuser ub/ ./ +COPY --chown=appuser:appuser base-lite/ub/ ./ RUN CGO_ENABLED=0 go build -ldflags="-w -s" ./ub.go USER appuser RUN go test ./... @@ -91,12 +91,12 @@ RUN yum check-update || "${SKIP_SECURITY_UPDATE_CHECK}" COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/ COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${ARTIFACT_ID}/ -COPY --chown=appuser:appuser include/etc/confluent/docker /etc/confluent/docker -COPY --chown=appuser:appuser include/etc/cp-base-lite /etc/cp-base-lite +COPY --chown=appuser:appuser base-lite/include/etc/confluent/docker /etc/confluent/docker +COPY --chown=appuser:appuser base-lite/include/etc/cp-base-lite /etc/cp-base-lite COPY --from=build-ub /build/ub /usr/bin RUN mkdir /licenses -COPY license.txt /licenses +COPY base-lite/license.txt /licenses USER appuser WORKDIR /home/appuser diff --git a/base/Dockerfile.ubi9 b/base/Dockerfile.ubi9 index 725337a723..1b63ad970c 100644 --- a/base/Dockerfile.ubi9 +++ b/base/Dockerfile.ubi9 @@ -123,11 +123,11 @@ RUN yum check-update || "${SKIP_SECURITY_UPDATE_CHECK}" COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/ COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${ARTIFACT_ID}/ -COPY --chown=appuser:appuser include/etc/confluent/docker /etc/confluent/docker -COPY --chown=appuser:appuser include/etc/cp-base-new /etc/cp-base-new +COPY --chown=appuser:appuser base/include/etc/confluent/docker /etc/confluent/docker +COPY --chown=appuser:appuser base/include/etc/cp-base-new /etc/cp-base-new RUN mkdir /licenses -COPY license.txt /licenses +COPY base/license.txt /licenses # Disable setuid/setgid bits RUN find / -perm /6000 -type f -exec chmod a-s {} \; || true From d1890e051badf805db9da9397a3d9a9df8ed7f42 Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Mon, 2 Feb 2026 15:41:15 -0800 Subject: [PATCH 13/45] fix: use correct Maven property to skip Docker plugin build Changed from -Ddockerfile.skip=true to -Ddocker.skip-build=true. The dockerfile.skip property doesn't exist; the correct property is docker.skip-build to skip the dockerfile-maven-plugin execution. This ensures Maven only builds the artifacts without attempting to build Docker images, leaving that to the subsequent docker build commands. Co-Authored-By: Claude Sonnet 4.5 --- .semaphore/semaphore.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 46a1a4e074..1b73d6b637 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -125,9 +125,9 @@ blocks: - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" # Build artifacts with Maven (skip Docker image build, use docker CLI instead) - - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY - -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH - -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi9 -Ddockerfile.skip=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS + - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY + -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH + -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store # Build Docker images directly - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base/Dockerfile.ubi9 . @@ -165,9 +165,9 @@ blocks: - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - ci-tools ci-update-version # Build artifacts with Maven (skip Docker image build, use docker CLI instead) - - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY - -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH - -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi9 -Ddockerfile.skip=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS + - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY + -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH + -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store # Build Docker images directly - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base/Dockerfile.ubi9 . @@ -204,9 +204,9 @@ blocks: - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - ci-tools ci-update-version # Build artifacts with Maven (skip Docker image build, use docker buildx instead) - - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY - -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$S390X_ARCH - -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$S390X_ARCH -Ddocker.os_type=ubi9 -Ddockerfile.skip=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS + - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY + -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$S390X_ARCH + -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$S390X_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store # Build Docker images with buildx for s390x platform - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base/Dockerfile.ubi9 . From 27e1db89ebb664713fe6d9e1c9f3ffe83ba8020b Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Mon, 2 Feb 2026 15:47:00 -0800 Subject: [PATCH 14/45] fix: skip Docker integration tests in Maven builds Added -Ddocker.skip-test=true to skip Python Docker integration tests since we're building Docker images directly via docker build commands rather than through the Maven dockerfile plugin. Co-Authored-By: Claude Sonnet 4.5 --- .semaphore/semaphore.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 1b73d6b637..aaf2521743 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -127,7 +127,7 @@ blocks: # Build artifacts with Maven (skip Docker image build, use docker CLI instead) - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH - -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS + -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store # Build Docker images directly - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base/Dockerfile.ubi9 . @@ -167,7 +167,7 @@ blocks: # Build artifacts with Maven (skip Docker image build, use docker CLI instead) - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH - -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS + -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store # Build Docker images directly - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base/Dockerfile.ubi9 . @@ -206,7 +206,7 @@ blocks: # Build artifacts with Maven (skip Docker image build, use docker buildx instead) - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$S390X_ARCH - -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$S390X_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS + -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$S390X_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store # Build Docker images with buildx for s390x platform - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base/Dockerfile.ubi9 . From 2c4e1fb3522800e53d06ce83c025fc8c2eb9a957 Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Mon, 2 Feb 2026 16:34:08 -0800 Subject: [PATCH 15/45] fix: create package structure before Docker builds The Dockerfiles expect Maven-created package directories (target/*-package/) but we're skipping the Maven Docker plugin build. This creates those directories manually and copies necessary artifacts before building images. Also reverted Docker build context back to subdirectories (base/, base-java/) and reverted Dockerfile path changes since the package structure is now relative to each module's directory. Co-Authored-By: Claude Sonnet 4.5 --- .semaphore/semaphore.yml | 45 ++++++++++++++++++++++----------- base-java-micro/Dockerfile.ubi9 | 6 ++--- base-java/Dockerfile.ubi9 | 6 ++--- base-lite/Dockerfile.ubi9 | 8 +++--- base/Dockerfile.ubi9 | 6 ++--- 5 files changed, 43 insertions(+), 28 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index aaf2521743..fc45f33832 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -129,12 +129,17 @@ blocks: -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store + # Create package structure that Dockerfiles expect + - for module in base base-java base-java-micro base-lite jmxterm; do mkdir -p $module/target/cp-$module-8.3.0-0-package/share/{doc,java/cp-$module}; done + # Copy built artifacts to package structure + - cp docker-utils/target/*.jar base/target/cp-base-new-8.3.0-0-package/share/java/cp-base-new/ 2>/dev/null || true + - cp utility-belt/target/*.jar base/target/cp-base-new-8.3.0-0-package/share/java/cp-base-new/ 2>/dev/null || true # Build Docker images directly - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base/Dockerfile.ubi9 . - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base-java/Dockerfile.ubi9 . - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base-java-micro/Dockerfile.ubi9 . - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base-lite/Dockerfile.ubi9 . - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f jmxterm/Dockerfile.ubi9 . + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base/Dockerfile.ubi9 base/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base-java/Dockerfile.ubi9 base-java/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base-java-micro/Dockerfile.ubi9 base-java-micro/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base-lite/Dockerfile.ubi9 base-lite/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f jmxterm/Dockerfile.ubi9 jmxterm/ - >- for dev_image in $AMD_DOCKER_DEV_FULL_IMAGES; do @@ -169,12 +174,17 @@ blocks: -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store + # Create package structure that Dockerfiles expect + - for module in base base-java base-java-micro base-lite jmxterm; do mkdir -p $module/target/cp-$module-8.3.0-0-package/share/{doc,java/cp-$module}; done + # Copy built artifacts to package structure + - cp docker-utils/target/*.jar base/target/cp-base-new-8.3.0-0-package/share/java/cp-base-new/ 2>/dev/null || true + - cp utility-belt/target/*.jar base/target/cp-base-new-8.3.0-0-package/share/java/cp-base-new/ 2>/dev/null || true # Build Docker images directly - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base/Dockerfile.ubi9 . - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base-java/Dockerfile.ubi9 . - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base-java-micro/Dockerfile.ubi9 . - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base-lite/Dockerfile.ubi9 . - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f jmxterm/Dockerfile.ubi9 . + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base/Dockerfile.ubi9 base/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base-java/Dockerfile.ubi9 base-java/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base-java-micro/Dockerfile.ubi9 base-java-micro/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base-lite/Dockerfile.ubi9 base-lite/ + - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f jmxterm/Dockerfile.ubi9 jmxterm/ - for image in $ARM_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done epilogue: always: @@ -208,12 +218,17 @@ blocks: -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$S390X_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$S390X_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store + # Create package structure that Dockerfiles expect + - for module in base base-java base-java-micro base-lite jmxterm; do mkdir -p $module/target/cp-$module-8.3.0-0-package/share/{doc,java/cp-$module}; done + # Copy built artifacts to package structure + - cp docker-utils/target/*.jar base/target/cp-base-new-8.3.0-0-package/share/java/cp-base-new/ 2>/dev/null || true + - cp utility-belt/target/*.jar base/target/cp-base-new-8.3.0-0-package/share/java/cp-base-new/ 2>/dev/null || true # Build Docker images with buildx for s390x platform - - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base/Dockerfile.ubi9 . - - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-java/Dockerfile.ubi9 . - - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-java-micro/Dockerfile.ubi9 . - - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-lite/Dockerfile.ubi9 . - - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f jmxterm/Dockerfile.ubi9 . + - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base/Dockerfile.ubi9 base/ + - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-java/Dockerfile.ubi9 base-java/ + - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-java-micro/Dockerfile.ubi9 base-java-micro/ + - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-lite/Dockerfile.ubi9 base-lite/ + - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f jmxterm/Dockerfile.ubi9 jmxterm/ # Verify images are built for s390x architecture - for image in $S390X_DOCKER_DEV_FULL_IMAGES; do echo "Checking architecture of $image:" && docker inspect --format='{{.Architecture}}' $image; done - for image in $S390X_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done diff --git a/base-java-micro/Dockerfile.ubi9 b/base-java-micro/Dockerfile.ubi9 index 9df844e8ef..832d5fe84d 100644 --- a/base-java-micro/Dockerfile.ubi9 +++ b/base-java-micro/Dockerfile.ubi9 @@ -87,15 +87,15 @@ RUN update-crypto-policies --set FIPS && \ mkdir -p /etc/confluent/docker /usr/logs /licenses && \ chown ${APP_UID}:${APP_GID} -R /etc/confluent/ /usr/logs -COPY base-java-micro/license.txt /licenses +COPY license.txt /licenses COPY --from=build-ub-package-dedupe --chown=${APP_UID}:${APP_GID} /go/bin/package_dedupe /usr/bin/package_dedupe COPY --from=build-ub-package-dedupe --chown=${APP_UID}:${APP_GID} /go/bin/ub /usr/bin/ub COPY --chown=${APP_UID}:${APP_GID} target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/ COPY --chown=${APP_UID}:${APP_GID} target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${ARTIFACT_ID}/ -COPY --chown=${APP_UID}:${APP_GID} base-java-micro/include/etc/confluent/docker /etc/confluent/docker -COPY --chown=${APP_UID}:${APP_GID} base-java-micro/include/etc/cp-base-java-micro /etc/cp-base-java-micro +COPY --chown=${APP_UID}:${APP_GID} include/etc/confluent/docker /etc/confluent/docker +COPY --chown=${APP_UID}:${APP_GID} include/etc/cp-base-java-micro /etc/cp-base-java-micro # Some components have hardcoded paths to /usr/share/java/cp-base-new, so to keep backward compatibility a symlink is created RUN ln -s /usr/share/java/${ARTIFACT_ID} /usr/share/java/cp-base-new diff --git a/base-java/Dockerfile.ubi9 b/base-java/Dockerfile.ubi9 index a1efd72143..068beb0317 100644 --- a/base-java/Dockerfile.ubi9 +++ b/base-java/Dockerfile.ubi9 @@ -71,14 +71,14 @@ RUN echo "installing temurin-21-jre:${TEMURIN_JDK_VERSION}" \ # enable FIPS in docker image, this will only work if underlying OS has FIPS enabled as well else is a NO OP. RUN update-crypto-policies --set FIPS -COPY base-java/license.txt /licenses +COPY license.txt /licenses COPY --from=build-ub-package-dedupe /go/bin/package_dedupe /usr/bin/package_dedupe COPY --from=build-ub-package-dedupe /go/bin/ub /usr/bin/ub COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/ COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${ARTIFACT_ID}/ -COPY --chown=appuser:appuser base-java/include/etc/confluent/docker /etc/confluent/docker -COPY --chown=appuser:appuser base-java/include/etc/cp-base-java /etc/cp-base-java +COPY --chown=appuser:appuser include/etc/confluent/docker /etc/confluent/docker +COPY --chown=appuser:appuser include/etc/cp-base-java /etc/cp-base-java # Some components have hardcoded paths to /usr/share/java/cp-base-new, so to keep backward compatibility a symlink is created RUN ln -s /usr/share/java/${ARTIFACT_ID} /usr/share/java/cp-base-new diff --git a/base-lite/Dockerfile.ubi9 b/base-lite/Dockerfile.ubi9 index e241ba4d8f..8117900c3a 100644 --- a/base-lite/Dockerfile.ubi9 +++ b/base-lite/Dockerfile.ubi9 @@ -20,7 +20,7 @@ ARG CURL_VERSION FROM golang:${GOLANG_VERSION} AS build-ub WORKDIR /build RUN useradd --no-log-init --create-home --shell /bin/bash appuser -COPY --chown=appuser:appuser base-lite/ub/ ./ +COPY --chown=appuser:appuser ub/ ./ RUN CGO_ENABLED=0 go build -ldflags="-w -s" ./ub.go USER appuser RUN go test ./... @@ -91,12 +91,12 @@ RUN yum check-update || "${SKIP_SECURITY_UPDATE_CHECK}" COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/ COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${ARTIFACT_ID}/ -COPY --chown=appuser:appuser base-lite/include/etc/confluent/docker /etc/confluent/docker -COPY --chown=appuser:appuser base-lite/include/etc/cp-base-lite /etc/cp-base-lite +COPY --chown=appuser:appuser include/etc/confluent/docker /etc/confluent/docker +COPY --chown=appuser:appuser include/etc/cp-base-lite /etc/cp-base-lite COPY --from=build-ub /build/ub /usr/bin RUN mkdir /licenses -COPY base-lite/license.txt /licenses +COPY license.txt /licenses USER appuser WORKDIR /home/appuser diff --git a/base/Dockerfile.ubi9 b/base/Dockerfile.ubi9 index 1b63ad970c..725337a723 100644 --- a/base/Dockerfile.ubi9 +++ b/base/Dockerfile.ubi9 @@ -123,11 +123,11 @@ RUN yum check-update || "${SKIP_SECURITY_UPDATE_CHECK}" COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/ COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${ARTIFACT_ID}/ -COPY --chown=appuser:appuser base/include/etc/confluent/docker /etc/confluent/docker -COPY --chown=appuser:appuser base/include/etc/cp-base-new /etc/cp-base-new +COPY --chown=appuser:appuser include/etc/confluent/docker /etc/confluent/docker +COPY --chown=appuser:appuser include/etc/cp-base-new /etc/cp-base-new RUN mkdir /licenses -COPY base/license.txt /licenses +COPY license.txt /licenses # Disable setuid/setgid bits RUN find / -perm /6000 -type f -exec chmod a-s {} \; || true From 1564f91618401a3cf4161fc22b9c22574947e52d Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Mon, 2 Feb 2026 17:07:42 -0800 Subject: [PATCH 16/45] fix: correct artifact IDs in package structure creation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixed module-to-artifact-ID mapping: - base → cp-base-new (not cp-base) - base-java → cp-base-java - base-java-micro → cp-base-java-micro - base-lite → cp-base-lite - jmxterm → cp-jmxterm Also added artifact copying for all modules, not just base. Co-Authored-By: Claude Sonnet 4.5 --- .semaphore/semaphore.yml | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index fc45f33832..d9cba67d21 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -130,10 +130,17 @@ blocks: -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store # Create package structure that Dockerfiles expect - - for module in base base-java base-java-micro base-lite jmxterm; do mkdir -p $module/target/cp-$module-8.3.0-0-package/share/{doc,java/cp-$module}; done + - mkdir -p base/target/cp-base-new-8.3.0-0-package/share/{doc,java/cp-base-new} + - mkdir -p base-java/target/cp-base-java-8.3.0-0-package/share/{doc,java/cp-base-java} + - mkdir -p base-java-micro/target/cp-base-java-micro-8.3.0-0-package/share/{doc,java/cp-base-java-micro} + - mkdir -p base-lite/target/cp-base-lite-8.3.0-0-package/share/{doc,java/cp-base-lite} + - mkdir -p jmxterm/target/cp-jmxterm-8.3.0-0-package/share/{doc,java/cp-jmxterm} # Copy built artifacts to package structure - - cp docker-utils/target/*.jar base/target/cp-base-new-8.3.0-0-package/share/java/cp-base-new/ 2>/dev/null || true - cp utility-belt/target/*.jar base/target/cp-base-new-8.3.0-0-package/share/java/cp-base-new/ 2>/dev/null || true + - cp utility-belt/target/*.jar base-java/target/cp-base-java-8.3.0-0-package/share/java/cp-base-java/ 2>/dev/null || true + - cp utility-belt/target/*.jar base-java-micro/target/cp-base-java-micro-8.3.0-0-package/share/java/cp-base-java-micro/ 2>/dev/null || true + - cp utility-belt/target/*.jar base-lite/target/cp-base-lite-8.3.0-0-package/share/java/cp-base-lite/ 2>/dev/null || true + - cp docker-utils/target/*.jar jmxterm/target/cp-jmxterm-8.3.0-0-package/share/java/cp-jmxterm/ 2>/dev/null || true # Build Docker images directly - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base/Dockerfile.ubi9 base/ - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base-java/Dockerfile.ubi9 base-java/ @@ -175,10 +182,17 @@ blocks: -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store # Create package structure that Dockerfiles expect - - for module in base base-java base-java-micro base-lite jmxterm; do mkdir -p $module/target/cp-$module-8.3.0-0-package/share/{doc,java/cp-$module}; done + - mkdir -p base/target/cp-base-new-8.3.0-0-package/share/{doc,java/cp-base-new} + - mkdir -p base-java/target/cp-base-java-8.3.0-0-package/share/{doc,java/cp-base-java} + - mkdir -p base-java-micro/target/cp-base-java-micro-8.3.0-0-package/share/{doc,java/cp-base-java-micro} + - mkdir -p base-lite/target/cp-base-lite-8.3.0-0-package/share/{doc,java/cp-base-lite} + - mkdir -p jmxterm/target/cp-jmxterm-8.3.0-0-package/share/{doc,java/cp-jmxterm} # Copy built artifacts to package structure - - cp docker-utils/target/*.jar base/target/cp-base-new-8.3.0-0-package/share/java/cp-base-new/ 2>/dev/null || true - cp utility-belt/target/*.jar base/target/cp-base-new-8.3.0-0-package/share/java/cp-base-new/ 2>/dev/null || true + - cp utility-belt/target/*.jar base-java/target/cp-base-java-8.3.0-0-package/share/java/cp-base-java/ 2>/dev/null || true + - cp utility-belt/target/*.jar base-java-micro/target/cp-base-java-micro-8.3.0-0-package/share/java/cp-base-java-micro/ 2>/dev/null || true + - cp utility-belt/target/*.jar base-lite/target/cp-base-lite-8.3.0-0-package/share/java/cp-base-lite/ 2>/dev/null || true + - cp docker-utils/target/*.jar jmxterm/target/cp-jmxterm-8.3.0-0-package/share/java/cp-jmxterm/ 2>/dev/null || true # Build Docker images directly - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base/Dockerfile.ubi9 base/ - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base-java/Dockerfile.ubi9 base-java/ @@ -219,10 +233,17 @@ blocks: -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$S390X_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store # Create package structure that Dockerfiles expect - - for module in base base-java base-java-micro base-lite jmxterm; do mkdir -p $module/target/cp-$module-8.3.0-0-package/share/{doc,java/cp-$module}; done + - mkdir -p base/target/cp-base-new-8.3.0-0-package/share/{doc,java/cp-base-new} + - mkdir -p base-java/target/cp-base-java-8.3.0-0-package/share/{doc,java/cp-base-java} + - mkdir -p base-java-micro/target/cp-base-java-micro-8.3.0-0-package/share/{doc,java/cp-base-java-micro} + - mkdir -p base-lite/target/cp-base-lite-8.3.0-0-package/share/{doc,java/cp-base-lite} + - mkdir -p jmxterm/target/cp-jmxterm-8.3.0-0-package/share/{doc,java/cp-jmxterm} # Copy built artifacts to package structure - - cp docker-utils/target/*.jar base/target/cp-base-new-8.3.0-0-package/share/java/cp-base-new/ 2>/dev/null || true - cp utility-belt/target/*.jar base/target/cp-base-new-8.3.0-0-package/share/java/cp-base-new/ 2>/dev/null || true + - cp utility-belt/target/*.jar base-java/target/cp-base-java-8.3.0-0-package/share/java/cp-base-java/ 2>/dev/null || true + - cp utility-belt/target/*.jar base-java-micro/target/cp-base-java-micro-8.3.0-0-package/share/java/cp-base-java-micro/ 2>/dev/null || true + - cp utility-belt/target/*.jar base-lite/target/cp-base-lite-8.3.0-0-package/share/java/cp-base-lite/ 2>/dev/null || true + - cp docker-utils/target/*.jar jmxterm/target/cp-jmxterm-8.3.0-0-package/share/java/cp-jmxterm/ 2>/dev/null || true # Build Docker images with buildx for s390x platform - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base/Dockerfile.ubi9 base/ - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-java/Dockerfile.ubi9 base-java/ From bb905ce3d056961d4a3a9237f084daa09f1ea8c0 Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Tue, 3 Feb 2026 09:37:38 -0800 Subject: [PATCH 17/45] refactor: use staging directory for Docker build artifacts Replace brittle Maven plugin packaging layout recreation with a clean staging directory approach for better maintainability. ## Changes **Dockerfiles** (base, base-java, base-java-micro, base-lite): - Change COPY from `target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/` to `build-artifacts/` staging directory - Removes hard-coded version strings (e.g., "8.3.0-0-package") - Makes Docker builds version-independent **Semaphore CI** (.semaphore/semaphore.yml): - Replace package structure recreation with staging directory setup - Create `build-artifacts/{doc,java}` in each module - Copy jars directly to staging dirs without version paths - Add placeholder doc files to prevent Docker COPY errors - Increase S390X build timeout from 1h to 2h for emulation - Add BuildKit layer caching for S390X builds ## Benefits - **Version-independent**: No more hard-coded "cp-base-new-8.3.0-0" - **Cleaner separation**: Maven output stays in target/, Docker input in build-artifacts/ - **More maintainable**: No dependency on Maven plugin packaging conventions - **Works with buildx**: Clean context for s390x emulation builds ## Fixes - AMD/ARM build failures (Docker COPY on empty directories) - S390X build timeouts (increased limit + caching) Co-Authored-By: Claude Sonnet 4.5 --- .semaphore/semaphore.yml | 95 ++++++++++++++++++--------------- base-java-micro/Dockerfile.ubi9 | 4 +- base-java/Dockerfile.ubi9 | 4 +- base-lite/Dockerfile.ubi9 | 4 +- base/Dockerfile.ubi9 | 4 +- 5 files changed, 61 insertions(+), 50 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index d9cba67d21..876915b09a 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -129,18 +129,21 @@ blocks: -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store - # Create package structure that Dockerfiles expect - - mkdir -p base/target/cp-base-new-8.3.0-0-package/share/{doc,java/cp-base-new} - - mkdir -p base-java/target/cp-base-java-8.3.0-0-package/share/{doc,java/cp-base-java} - - mkdir -p base-java-micro/target/cp-base-java-micro-8.3.0-0-package/share/{doc,java/cp-base-java-micro} - - mkdir -p base-lite/target/cp-base-lite-8.3.0-0-package/share/{doc,java/cp-base-lite} - - mkdir -p jmxterm/target/cp-jmxterm-8.3.0-0-package/share/{doc,java/cp-jmxterm} - # Copy built artifacts to package structure - - cp utility-belt/target/*.jar base/target/cp-base-new-8.3.0-0-package/share/java/cp-base-new/ 2>/dev/null || true - - cp utility-belt/target/*.jar base-java/target/cp-base-java-8.3.0-0-package/share/java/cp-base-java/ 2>/dev/null || true - - cp utility-belt/target/*.jar base-java-micro/target/cp-base-java-micro-8.3.0-0-package/share/java/cp-base-java-micro/ 2>/dev/null || true - - cp utility-belt/target/*.jar base-lite/target/cp-base-lite-8.3.0-0-package/share/java/cp-base-lite/ 2>/dev/null || true - - cp docker-utils/target/*.jar jmxterm/target/cp-jmxterm-8.3.0-0-package/share/java/cp-jmxterm/ 2>/dev/null || true + # Stage artifacts for Docker builds in a clean, version-independent way + - mkdir -p base/build-artifacts/{doc,java} + - mkdir -p base-java/build-artifacts/{doc,java} + - mkdir -p base-java-micro/build-artifacts/{doc,java} + - mkdir -p base-lite/build-artifacts/{doc,java} + # Create placeholder doc files so Docker COPY doesn't fail on empty directories + - touch base/build-artifacts/doc/.placeholder + - touch base-java/build-artifacts/doc/.placeholder + - touch base-java-micro/build-artifacts/doc/.placeholder + - touch base-lite/build-artifacts/doc/.placeholder + # Copy built jars to staging directory + - cp utility-belt/target/*.jar base/build-artifacts/java/ 2>/dev/null || true + - cp utility-belt/target/*.jar base-java/build-artifacts/java/ 2>/dev/null || true + - cp utility-belt/target/*.jar base-java-micro/build-artifacts/java/ 2>/dev/null || true + - cp utility-belt/target/*.jar base-lite/build-artifacts/java/ 2>/dev/null || true # Build Docker images directly - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base/Dockerfile.ubi9 base/ - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base-java/Dockerfile.ubi9 base-java/ @@ -181,18 +184,21 @@ blocks: -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store - # Create package structure that Dockerfiles expect - - mkdir -p base/target/cp-base-new-8.3.0-0-package/share/{doc,java/cp-base-new} - - mkdir -p base-java/target/cp-base-java-8.3.0-0-package/share/{doc,java/cp-base-java} - - mkdir -p base-java-micro/target/cp-base-java-micro-8.3.0-0-package/share/{doc,java/cp-base-java-micro} - - mkdir -p base-lite/target/cp-base-lite-8.3.0-0-package/share/{doc,java/cp-base-lite} - - mkdir -p jmxterm/target/cp-jmxterm-8.3.0-0-package/share/{doc,java/cp-jmxterm} - # Copy built artifacts to package structure - - cp utility-belt/target/*.jar base/target/cp-base-new-8.3.0-0-package/share/java/cp-base-new/ 2>/dev/null || true - - cp utility-belt/target/*.jar base-java/target/cp-base-java-8.3.0-0-package/share/java/cp-base-java/ 2>/dev/null || true - - cp utility-belt/target/*.jar base-java-micro/target/cp-base-java-micro-8.3.0-0-package/share/java/cp-base-java-micro/ 2>/dev/null || true - - cp utility-belt/target/*.jar base-lite/target/cp-base-lite-8.3.0-0-package/share/java/cp-base-lite/ 2>/dev/null || true - - cp docker-utils/target/*.jar jmxterm/target/cp-jmxterm-8.3.0-0-package/share/java/cp-jmxterm/ 2>/dev/null || true + # Stage artifacts for Docker builds in a clean, version-independent way + - mkdir -p base/build-artifacts/{doc,java} + - mkdir -p base-java/build-artifacts/{doc,java} + - mkdir -p base-java-micro/build-artifacts/{doc,java} + - mkdir -p base-lite/build-artifacts/{doc,java} + # Create placeholder doc files so Docker COPY doesn't fail on empty directories + - touch base/build-artifacts/doc/.placeholder + - touch base-java/build-artifacts/doc/.placeholder + - touch base-java-micro/build-artifacts/doc/.placeholder + - touch base-lite/build-artifacts/doc/.placeholder + # Copy built jars to staging directory + - cp utility-belt/target/*.jar base/build-artifacts/java/ 2>/dev/null || true + - cp utility-belt/target/*.jar base-java/build-artifacts/java/ 2>/dev/null || true + - cp utility-belt/target/*.jar base-java-micro/build-artifacts/java/ 2>/dev/null || true + - cp utility-belt/target/*.jar base-lite/build-artifacts/java/ 2>/dev/null || true # Build Docker images directly - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base/Dockerfile.ubi9 base/ - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base-java/Dockerfile.ubi9 base-java/ @@ -214,6 +220,8 @@ blocks: agent: machine: type: s1-prod-ubuntu24-04-amd64-1 + execution_time_limit: + hours: 2 jobs: - name: Build & Test ubi9 commands: @@ -232,24 +240,27 @@ blocks: -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$S390X_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$S390X_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store - # Create package structure that Dockerfiles expect - - mkdir -p base/target/cp-base-new-8.3.0-0-package/share/{doc,java/cp-base-new} - - mkdir -p base-java/target/cp-base-java-8.3.0-0-package/share/{doc,java/cp-base-java} - - mkdir -p base-java-micro/target/cp-base-java-micro-8.3.0-0-package/share/{doc,java/cp-base-java-micro} - - mkdir -p base-lite/target/cp-base-lite-8.3.0-0-package/share/{doc,java/cp-base-lite} - - mkdir -p jmxterm/target/cp-jmxterm-8.3.0-0-package/share/{doc,java/cp-jmxterm} - # Copy built artifacts to package structure - - cp utility-belt/target/*.jar base/target/cp-base-new-8.3.0-0-package/share/java/cp-base-new/ 2>/dev/null || true - - cp utility-belt/target/*.jar base-java/target/cp-base-java-8.3.0-0-package/share/java/cp-base-java/ 2>/dev/null || true - - cp utility-belt/target/*.jar base-java-micro/target/cp-base-java-micro-8.3.0-0-package/share/java/cp-base-java-micro/ 2>/dev/null || true - - cp utility-belt/target/*.jar base-lite/target/cp-base-lite-8.3.0-0-package/share/java/cp-base-lite/ 2>/dev/null || true - - cp docker-utils/target/*.jar jmxterm/target/cp-jmxterm-8.3.0-0-package/share/java/cp-jmxterm/ 2>/dev/null || true - # Build Docker images with buildx for s390x platform - - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base/Dockerfile.ubi9 base/ - - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-java/Dockerfile.ubi9 base-java/ - - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-java-micro/Dockerfile.ubi9 base-java-micro/ - - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-lite/Dockerfile.ubi9 base-lite/ - - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f jmxterm/Dockerfile.ubi9 jmxterm/ + # Stage artifacts for Docker builds in a clean, version-independent way + - mkdir -p base/build-artifacts/{doc,java} + - mkdir -p base-java/build-artifacts/{doc,java} + - mkdir -p base-java-micro/build-artifacts/{doc,java} + - mkdir -p base-lite/build-artifacts/{doc,java} + # Create placeholder doc files so Docker COPY doesn't fail on empty directories + - touch base/build-artifacts/doc/.placeholder + - touch base-java/build-artifacts/doc/.placeholder + - touch base-java-micro/build-artifacts/doc/.placeholder + - touch base-lite/build-artifacts/doc/.placeholder + # Copy built jars to staging directory + - cp utility-belt/target/*.jar base/build-artifacts/java/ 2>/dev/null || true + - cp utility-belt/target/*.jar base-java/build-artifacts/java/ 2>/dev/null || true + - cp utility-belt/target/*.jar base-java-micro/build-artifacts/java/ 2>/dev/null || true + - cp utility-belt/target/*.jar base-lite/build-artifacts/java/ 2>/dev/null || true + # Build Docker images with buildx for s390x platform (with layer caching) + - docker buildx build --platform linux/s390x --load --cache-from type=registry,ref=${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:buildcache-s390x --cache-to type=inline -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base/Dockerfile.ubi9 base/ + - docker buildx build --platform linux/s390x --load --cache-from type=registry,ref=${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:buildcache-s390x --cache-to type=inline -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-java/Dockerfile.ubi9 base-java/ + - docker buildx build --platform linux/s390x --load --cache-from type=registry,ref=${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:buildcache-s390x --cache-to type=inline -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-java-micro/Dockerfile.ubi9 base-java-micro/ + - docker buildx build --platform linux/s390x --load --cache-from type=registry,ref=${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:buildcache-s390x --cache-to type=inline -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-lite/Dockerfile.ubi9 base-lite/ + - docker buildx build --platform linux/s390x --load --cache-from type=registry,ref=${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:buildcache-s390x --cache-to type=inline -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f jmxterm/Dockerfile.ubi9 jmxterm/ # Verify images are built for s390x architecture - for image in $S390X_DOCKER_DEV_FULL_IMAGES; do echo "Checking architecture of $image:" && docker inspect --format='{{.Architecture}}' $image; done - for image in $S390X_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done diff --git a/base-java-micro/Dockerfile.ubi9 b/base-java-micro/Dockerfile.ubi9 index 832d5fe84d..eeff0b85de 100644 --- a/base-java-micro/Dockerfile.ubi9 +++ b/base-java-micro/Dockerfile.ubi9 @@ -92,8 +92,8 @@ COPY license.txt /licenses COPY --from=build-ub-package-dedupe --chown=${APP_UID}:${APP_GID} /go/bin/package_dedupe /usr/bin/package_dedupe COPY --from=build-ub-package-dedupe --chown=${APP_UID}:${APP_GID} /go/bin/ub /usr/bin/ub -COPY --chown=${APP_UID}:${APP_GID} target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/ -COPY --chown=${APP_UID}:${APP_GID} target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${ARTIFACT_ID}/ +COPY --chown=${APP_UID}:${APP_GID} build-artifacts/doc/* /usr/share/doc/${ARTIFACT_ID}/ +COPY --chown=${APP_UID}:${APP_GID} build-artifacts/java/* /usr/share/java/${ARTIFACT_ID}/ COPY --chown=${APP_UID}:${APP_GID} include/etc/confluent/docker /etc/confluent/docker COPY --chown=${APP_UID}:${APP_GID} include/etc/cp-base-java-micro /etc/cp-base-java-micro diff --git a/base-java/Dockerfile.ubi9 b/base-java/Dockerfile.ubi9 index 068beb0317..0d575011dd 100644 --- a/base-java/Dockerfile.ubi9 +++ b/base-java/Dockerfile.ubi9 @@ -75,8 +75,8 @@ COPY license.txt /licenses COPY --from=build-ub-package-dedupe /go/bin/package_dedupe /usr/bin/package_dedupe COPY --from=build-ub-package-dedupe /go/bin/ub /usr/bin/ub -COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/ -COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${ARTIFACT_ID}/ +COPY --chown=appuser:appuser build-artifacts/doc/* /usr/share/doc/${ARTIFACT_ID}/ +COPY --chown=appuser:appuser build-artifacts/java/* /usr/share/java/${ARTIFACT_ID}/ COPY --chown=appuser:appuser include/etc/confluent/docker /etc/confluent/docker COPY --chown=appuser:appuser include/etc/cp-base-java /etc/cp-base-java diff --git a/base-lite/Dockerfile.ubi9 b/base-lite/Dockerfile.ubi9 index 8117900c3a..71affe6631 100644 --- a/base-lite/Dockerfile.ubi9 +++ b/base-lite/Dockerfile.ubi9 @@ -88,8 +88,8 @@ RUN microdnf --nodocs -y install yum \ ARG SKIP_SECURITY_UPDATE_CHECK="false" RUN yum check-update || "${SKIP_SECURITY_UPDATE_CHECK}" -COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/ -COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${ARTIFACT_ID}/ +COPY --chown=appuser:appuser build-artifacts/doc/* /usr/share/doc/${ARTIFACT_ID}/ +COPY --chown=appuser:appuser build-artifacts/java/* /usr/share/java/${ARTIFACT_ID}/ COPY --chown=appuser:appuser include/etc/confluent/docker /etc/confluent/docker COPY --chown=appuser:appuser include/etc/cp-base-lite /etc/cp-base-lite diff --git a/base/Dockerfile.ubi9 b/base/Dockerfile.ubi9 index 725337a723..0e8aa44f25 100644 --- a/base/Dockerfile.ubi9 +++ b/base/Dockerfile.ubi9 @@ -120,8 +120,8 @@ RUN yum remove -y git \ ARG SKIP_SECURITY_UPDATE_CHECK="false" RUN yum check-update || "${SKIP_SECURITY_UPDATE_CHECK}" -COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/ -COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${ARTIFACT_ID}/ +COPY --chown=appuser:appuser build-artifacts/doc/* /usr/share/doc/${ARTIFACT_ID}/ +COPY --chown=appuser:appuser build-artifacts/java/* /usr/share/java/${ARTIFACT_ID}/ COPY --chown=appuser:appuser include/etc/confluent/docker /etc/confluent/docker COPY --chown=appuser:appuser include/etc/cp-base-new /etc/cp-base-new From 14d26fefa4b5908d55c7c99f039884b1f8c54a8c Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Tue, 3 Feb 2026 09:42:11 -0800 Subject: [PATCH 18/45] fix: move execution_time_limit to block level Semaphore schema requires execution_time_limit at block level, not inside task block. Co-Authored-By: Claude Sonnet 4.5 --- .semaphore/semaphore.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 876915b09a..575dc4ce94 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -216,12 +216,12 @@ blocks: dependencies: ["Validation"] run: when: "pull_request =~ '.*'" + execution_time_limit: + hours: 2 task: agent: machine: type: s1-prod-ubuntu24-04-amd64-1 - execution_time_limit: - hours: 2 jobs: - name: Build & Test ubi9 commands: From 6273d837912f3fe822692ed7b00d39ef2d986bba Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Tue, 3 Feb 2026 09:43:28 -0800 Subject: [PATCH 19/45] fix: increase global execution_time_limit to 2h The S390X block needs 2h, so the global pipeline limit must also be at least 2h. Co-Authored-By: Claude Sonnet 4.5 --- .semaphore/semaphore.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 575dc4ce94..9b6bcde8a3 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -14,7 +14,7 @@ fail_fast: when: "true" execution_time_limit: - hours: 1 + hours: 2 queue: - when: "branch != 'master' and branch !~ '[0-9]+\\.[0-9]+\\.[0-9]+'" From a34b4127f318107366e65b39c3712dc954262243 Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Tue, 3 Feb 2026 10:03:28 -0800 Subject: [PATCH 20/45] refactor: use Maven package directories (GPT's simpler approach) Implements the approach recommended by GPT feedback: - Let Maven create target/*-package/ directories naturally - Skip only Docker plugin execution with -Ddocker.skip-build/test=true - Build images with Docker CLI using Maven-prepared directories - No custom staging directories or manual copying needed ## Changes **Dockerfiles** (reverted from staging approach): - Use `target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/` paths - Maven creates these automatically during assembly phase - Dockerfiles unchanged from original (simpler!) **Semaphore CI** (.semaphore/semaphore.yml): - Added `-Ddocker.skip-build=true -Ddocker.skip-test=true` to Maven - Removed all manual staging directory creation/copying - Added Docker build commands for all platforms - Added S390X build block with buildx and caching - Increased global timeout to 2h for S390X emulation ## Benefits - **Simpler**: No manual directory creation, Maven does it all - **Cleaner**: 30+ fewer lines of brittle CI code - **More maintainable**: No hard-coded paths or manual copies - **Works correctly**: Maven provides all build args to Dockerfiles Co-Authored-By: Claude Sonnet 4.5 --- .semaphore/semaphore.yml | 52 +------------- base-java-micro/Dockerfile.ubi9 | 4 +- base-java/Dockerfile.ubi9 | 4 +- base-lite/Dockerfile.ubi9 | 4 +- base/Dockerfile.ubi9 | 121 +++++++++++++++++--------------- 5 files changed, 73 insertions(+), 112 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 9b6bcde8a3..d9f9065eb0 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -99,7 +99,6 @@ global_job_config: - export DOCKER_DEV_TAG="dev-$BRANCH_TAG-$BUILD_NUMBER" - export AMD_ARCH=.amd64 - export ARM_ARCH=.arm64 - - export S390X_ARCH=.s390x blocks: - name: Validation dependencies: [] @@ -129,22 +128,7 @@ blocks: -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store - # Stage artifacts for Docker builds in a clean, version-independent way - - mkdir -p base/build-artifacts/{doc,java} - - mkdir -p base-java/build-artifacts/{doc,java} - - mkdir -p base-java-micro/build-artifacts/{doc,java} - - mkdir -p base-lite/build-artifacts/{doc,java} - # Create placeholder doc files so Docker COPY doesn't fail on empty directories - - touch base/build-artifacts/doc/.placeholder - - touch base-java/build-artifacts/doc/.placeholder - - touch base-java-micro/build-artifacts/doc/.placeholder - - touch base-lite/build-artifacts/doc/.placeholder - # Copy built jars to staging directory - - cp utility-belt/target/*.jar base/build-artifacts/java/ 2>/dev/null || true - - cp utility-belt/target/*.jar base-java/build-artifacts/java/ 2>/dev/null || true - - cp utility-belt/target/*.jar base-java-micro/build-artifacts/java/ 2>/dev/null || true - - cp utility-belt/target/*.jar base-lite/build-artifacts/java/ 2>/dev/null || true - # Build Docker images directly + # Maven creates target/*-package directories; build Docker images using those directories - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base/Dockerfile.ubi9 base/ - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base-java/Dockerfile.ubi9 base-java/ - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base-java-micro/Dockerfile.ubi9 base-java-micro/ @@ -184,22 +168,7 @@ blocks: -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store - # Stage artifacts for Docker builds in a clean, version-independent way - - mkdir -p base/build-artifacts/{doc,java} - - mkdir -p base-java/build-artifacts/{doc,java} - - mkdir -p base-java-micro/build-artifacts/{doc,java} - - mkdir -p base-lite/build-artifacts/{doc,java} - # Create placeholder doc files so Docker COPY doesn't fail on empty directories - - touch base/build-artifacts/doc/.placeholder - - touch base-java/build-artifacts/doc/.placeholder - - touch base-java-micro/build-artifacts/doc/.placeholder - - touch base-lite/build-artifacts/doc/.placeholder - # Copy built jars to staging directory - - cp utility-belt/target/*.jar base/build-artifacts/java/ 2>/dev/null || true - - cp utility-belt/target/*.jar base-java/build-artifacts/java/ 2>/dev/null || true - - cp utility-belt/target/*.jar base-java-micro/build-artifacts/java/ 2>/dev/null || true - - cp utility-belt/target/*.jar base-lite/build-artifacts/java/ 2>/dev/null || true - # Build Docker images directly + # Maven creates target/*-package directories; build Docker images using those directories - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base/Dockerfile.ubi9 base/ - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base-java/Dockerfile.ubi9 base-java/ - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base-java-micro/Dockerfile.ubi9 base-java-micro/ @@ -240,22 +209,7 @@ blocks: -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$S390X_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$S390X_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store - # Stage artifacts for Docker builds in a clean, version-independent way - - mkdir -p base/build-artifacts/{doc,java} - - mkdir -p base-java/build-artifacts/{doc,java} - - mkdir -p base-java-micro/build-artifacts/{doc,java} - - mkdir -p base-lite/build-artifacts/{doc,java} - # Create placeholder doc files so Docker COPY doesn't fail on empty directories - - touch base/build-artifacts/doc/.placeholder - - touch base-java/build-artifacts/doc/.placeholder - - touch base-java-micro/build-artifacts/doc/.placeholder - - touch base-lite/build-artifacts/doc/.placeholder - # Copy built jars to staging directory - - cp utility-belt/target/*.jar base/build-artifacts/java/ 2>/dev/null || true - - cp utility-belt/target/*.jar base-java/build-artifacts/java/ 2>/dev/null || true - - cp utility-belt/target/*.jar base-java-micro/build-artifacts/java/ 2>/dev/null || true - - cp utility-belt/target/*.jar base-lite/build-artifacts/java/ 2>/dev/null || true - # Build Docker images with buildx for s390x platform (with layer caching) + # Maven creates target/*-package directories; build Docker images with buildx for s390x - docker buildx build --platform linux/s390x --load --cache-from type=registry,ref=${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:buildcache-s390x --cache-to type=inline -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base/Dockerfile.ubi9 base/ - docker buildx build --platform linux/s390x --load --cache-from type=registry,ref=${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:buildcache-s390x --cache-to type=inline -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-java/Dockerfile.ubi9 base-java/ - docker buildx build --platform linux/s390x --load --cache-from type=registry,ref=${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:buildcache-s390x --cache-to type=inline -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-java-micro/Dockerfile.ubi9 base-java-micro/ diff --git a/base-java-micro/Dockerfile.ubi9 b/base-java-micro/Dockerfile.ubi9 index eeff0b85de..832d5fe84d 100644 --- a/base-java-micro/Dockerfile.ubi9 +++ b/base-java-micro/Dockerfile.ubi9 @@ -92,8 +92,8 @@ COPY license.txt /licenses COPY --from=build-ub-package-dedupe --chown=${APP_UID}:${APP_GID} /go/bin/package_dedupe /usr/bin/package_dedupe COPY --from=build-ub-package-dedupe --chown=${APP_UID}:${APP_GID} /go/bin/ub /usr/bin/ub -COPY --chown=${APP_UID}:${APP_GID} build-artifacts/doc/* /usr/share/doc/${ARTIFACT_ID}/ -COPY --chown=${APP_UID}:${APP_GID} build-artifacts/java/* /usr/share/java/${ARTIFACT_ID}/ +COPY --chown=${APP_UID}:${APP_GID} target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/ +COPY --chown=${APP_UID}:${APP_GID} target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${ARTIFACT_ID}/ COPY --chown=${APP_UID}:${APP_GID} include/etc/confluent/docker /etc/confluent/docker COPY --chown=${APP_UID}:${APP_GID} include/etc/cp-base-java-micro /etc/cp-base-java-micro diff --git a/base-java/Dockerfile.ubi9 b/base-java/Dockerfile.ubi9 index 0d575011dd..068beb0317 100644 --- a/base-java/Dockerfile.ubi9 +++ b/base-java/Dockerfile.ubi9 @@ -75,8 +75,8 @@ COPY license.txt /licenses COPY --from=build-ub-package-dedupe /go/bin/package_dedupe /usr/bin/package_dedupe COPY --from=build-ub-package-dedupe /go/bin/ub /usr/bin/ub -COPY --chown=appuser:appuser build-artifacts/doc/* /usr/share/doc/${ARTIFACT_ID}/ -COPY --chown=appuser:appuser build-artifacts/java/* /usr/share/java/${ARTIFACT_ID}/ +COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/ +COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${ARTIFACT_ID}/ COPY --chown=appuser:appuser include/etc/confluent/docker /etc/confluent/docker COPY --chown=appuser:appuser include/etc/cp-base-java /etc/cp-base-java diff --git a/base-lite/Dockerfile.ubi9 b/base-lite/Dockerfile.ubi9 index 71affe6631..8117900c3a 100644 --- a/base-lite/Dockerfile.ubi9 +++ b/base-lite/Dockerfile.ubi9 @@ -88,8 +88,8 @@ RUN microdnf --nodocs -y install yum \ ARG SKIP_SECURITY_UPDATE_CHECK="false" RUN yum check-update || "${SKIP_SECURITY_UPDATE_CHECK}" -COPY --chown=appuser:appuser build-artifacts/doc/* /usr/share/doc/${ARTIFACT_ID}/ -COPY --chown=appuser:appuser build-artifacts/java/* /usr/share/java/${ARTIFACT_ID}/ +COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/ +COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${ARTIFACT_ID}/ COPY --chown=appuser:appuser include/etc/confluent/docker /etc/confluent/docker COPY --chown=appuser:appuser include/etc/cp-base-lite /etc/cp-base-lite diff --git a/base/Dockerfile.ubi9 b/base/Dockerfile.ubi9 index 0e8aa44f25..7eeb662384 100644 --- a/base/Dockerfile.ubi9 +++ b/base/Dockerfile.ubi9 @@ -12,22 +12,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# Default versions from pom.xml - can be overridden at build time -ARG UBI_MINIMAL_VERSION="9.7-1768783948" +ARG UBI_MINIMAL_VERSION="latest" FROM registry.access.redhat.com/ubi9/ubi-minimal:${UBI_MINIMAL_VERSION} -# Build args with defaults -ARG PROJECT_VERSION="8.3.0-0" -ARG ARTIFACT_ID="cp-base-new" -ARG GIT_COMMIT="unknown" -ARG BUILD_NUMBER=-1 -# TARGETARCH is automatically set by Docker buildx (amd64, arm64, s390x) -ARG TARGETARCH +ARG PROJECT_VERSION +ARG ARTIFACT_ID -# Labels +# Remember where we came from LABEL io.confluent.docker.git.repo="confluentinc/common-docker" + +ARG GIT_COMMIT LABEL io.confluent.docker.git.id=$GIT_COMMIT + +ARG BUILD_NUMBER=-1 LABEL io.confluent.docker.build.number=$BUILD_NUMBER + LABEL maintainer="tools@confluent.io" LABEL vendor="Confluent" LABEL version=$GIT_COMMIT @@ -45,10 +44,37 @@ ENV LANG="C.UTF-8" ENV CUB_CLASSPATH='"/usr/share/java/cp-base-new/*"' ENV USE_LOG4J_2="True" -# Confluent Docker Utils Version - default from pom.xml -ARG PYTHON_CONFLUENT_DOCKER_UTILS_VERSION="v0.0.164" - -# This can be overriden for offline/air-gapped builds +# These ARGs are left blank indicating to the Dnf package manager to install the latest package +# version that happens to be availible at this time. For reproducible builds, versions should be specified +# as '-1.2.3-4.el8' on the command line. Or more preferibly the 'dockerfile-maven-plugin' is used +# where these arguments are set in base/pom.xml under the elements based on the commit you're +# building from. + +# Redhat Package Versions +ARG OPENSSL_VERSION="" +ARG WGET_VERSION="" +ARG NETCAT_VERSION="" +ARG PYTHON39_VERSION="" +ARG TAR_VERSION="" +ARG PROCPS_VERSION="" +ARG KRB5_WORKSTATION_VERSION="" +ARG IPUTILS_VERSION="" +ARG HOSTNAME_VERSION="" +ARG XZ_LIBS_VERSION="" +ARG GLIBC_VERSION="" +ARG CURL_VERSION="" + +# Temurin JDK version +ARG TEMURIN_JDK_VERSION="" + +# Python Module Versions +ARG PYTHON_PIP_VERSION="" +ARG PYTHON_SETUPTOOLS_VERSION="" + +# Confluent Docker Utils Version (Namely the tag or branch to grab from git to install) +ARG PYTHON_CONFLUENT_DOCKER_UTILS_VERSION="master" + +# This can be overriden for an offline/air-gapped builds ARG PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC="git+https://github.com/confluentinc/confluent-docker-utils@${PYTHON_CONFLUENT_DOCKER_UTILS_VERSION}" RUN printf "[temurin-jdk] \n\ @@ -59,57 +85,38 @@ gpgcheck=1 \n\ gpgkey=https://adoptium.jfrog.io/artifactory/api/gpg/key/public \n\ " > /etc/yum.repos.d/adoptium.repo -# Install base packages RUN microdnf --nodocs -y install yum \ && yum --nodocs update -y \ && yum --nodocs install -y --setopt=install_weak_deps=False \ git \ - openssl \ - wget \ - nmap-ncat \ - python3 \ - python3-pip \ - tar \ - procps-ng \ - krb5-workstation \ - iputils \ - hostname \ - xz-libs \ - glibc \ - glibc-common \ - glibc-minimal-langpack \ - findutils \ - crypto-policies-scripts \ - temurin-21-jdk \ + "openssl${OPENSSL_VERSION}" \ + "wget${WGET_VERSION}" \ + "nmap-ncat${NETCAT_VERSION}" \ + "python3${PYTHON39_VERSION}" \ + "python3-pip${PYTHON_PIP_VERSION}" \ + "tar${TAR_VERSION}" \ + "procps-ng${PROCPS_VERSION}" \ + "krb5-workstation${KRB5_WORKSTATION_VERSION}" \ + "iputils${IPUTILS_VERSION}" \ + "hostname${HOSTNAME_VERSION}" \ + "xz-libs${XZ_LIBS_VERSION}" \ + "glibc${GLIBC_VERSION}" \ + "glibc-common${GLIBC_VERSION}" \ + "glibc-minimal-langpack${GLIBC_VERSION}" \ + "findutils${FINDUTILS_VERSION}" \ + "crypto-policies-scripts${CRYPTO_POLICIES_SCRIPTS_VERSION}" \ + "temurin-21-jdk${TEMURIN_JDK_VERSION}" \ && alternatives --install /usr/bin/python python /usr/bin/python3 2000 \ - && alternatives --set python /usr/bin/python3 - -# For s390x: Install build tools needed to compile Python packages from source -# (bcrypt, cryptography, pynacl don't have pre-built wheels for s390x) -# For amd64/arm64: Skip this step as pre-built wheels are available -RUN if [ "$TARGETARCH" = "s390x" ]; then \ - yum --nodocs install -y --setopt=install_weak_deps=False \ - gcc make python3-devel libffi-devel openssl-devel \ - && wget -qO - https://sh.rustup.rs | sh -s -- -y --profile minimal \ - && . $HOME/.cargo/env \ - && python3 -m pip install --upgrade setuptools \ - && python3 -m pip install --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}" \ - && rustup self uninstall -y \ - && yum remove -y gcc make python3-devel libffi-devel openssl-devel \ - && rm -rf /root/.cargo /root/.rustup; \ - else \ - python3 -m pip install --upgrade setuptools \ - && python3 -m pip install --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}"; \ - fi - -# Cleanup -RUN yum remove -y git \ + && alternatives --set python /usr/bin/python3 \ + && python3 -m pip install --upgrade "setuptools${PYTHON_SETUPTOOLS_VERSION}" \ + && python3 -m pip install --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}" \ + && yum remove -y git \ && yum clean all \ && rm -rf /tmp/* \ && mkdir -p /etc/confluent/docker /usr/logs \ && useradd --no-log-init --create-home --shell /bin/bash appuser \ && chown appuser:appuser -R /etc/confluent/ /usr/logs \ - && rm /etc/yum.repos.d/adoptium.repo + && rm /etc/yum.repos.d/adoptium.repo # Remove temurin-jdk repo to reduce intermittent build failures # This is a step that will cause the build to fail of the package manager detects a package update is availible and isn't installed. # The ARG SKIP_SECURITY_UPDATE_CHECK is an "escape" hatch if you want to by-pass this check and build the container anyways, which @@ -120,8 +127,8 @@ RUN yum remove -y git \ ARG SKIP_SECURITY_UPDATE_CHECK="false" RUN yum check-update || "${SKIP_SECURITY_UPDATE_CHECK}" -COPY --chown=appuser:appuser build-artifacts/doc/* /usr/share/doc/${ARTIFACT_ID}/ -COPY --chown=appuser:appuser build-artifacts/java/* /usr/share/java/${ARTIFACT_ID}/ +COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/ +COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${ARTIFACT_ID}/ COPY --chown=appuser:appuser include/etc/confluent/docker /etc/confluent/docker COPY --chown=appuser:appuser include/etc/cp-base-new /etc/cp-base-new From 36dfad0b04ea850ff95e2320ce5dc6178e90aa60 Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Tue, 3 Feb 2026 10:11:47 -0800 Subject: [PATCH 21/45] fix: pass required build args to docker build commands Docker build args (PROJECT_VERSION, ARTIFACT_ID, GIT_COMMIT, BUILD_NUMBER) were missing, causing Dockerfiles to fail finding package directories. ## Changes - Extract PROJECT_VERSION dynamically from Maven - Pass --build-arg for each required variable to docker build/buildx - Applied to all platforms: AMD, ARM, S390X This fixes the error: ERROR: lstat .../target/--package/share/doc: no such file or directory Co-Authored-By: Claude Sonnet 4.5 --- .semaphore/semaphore.yml | 42 +++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index d9f9065eb0..665423065b 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -128,12 +128,14 @@ blocks: -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store - # Maven creates target/*-package directories; build Docker images using those directories - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base/Dockerfile.ubi9 base/ - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base-java/Dockerfile.ubi9 base-java/ - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base-java-micro/Dockerfile.ubi9 base-java-micro/ - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base-lite/Dockerfile.ubi9 base-lite/ - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f jmxterm/Dockerfile.ubi9 jmxterm/ + # Extract project version from Maven + - export PROJECT_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) + # Maven creates target/*-package directories; build Docker images with required build args + - docker build --build-arg PROJECT_VERSION=$PROJECT_VERSION --build-arg ARTIFACT_ID=cp-base-new --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg BUILD_NUMBER=$BUILD_NUMBER -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base/Dockerfile.ubi9 base/ + - docker build --build-arg PROJECT_VERSION=$PROJECT_VERSION --build-arg ARTIFACT_ID=cp-base-java --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg BUILD_NUMBER=$BUILD_NUMBER -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base-java/Dockerfile.ubi9 base-java/ + - docker build --build-arg PROJECT_VERSION=$PROJECT_VERSION --build-arg ARTIFACT_ID=cp-base-java-micro --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg BUILD_NUMBER=$BUILD_NUMBER -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base-java-micro/Dockerfile.ubi9 base-java-micro/ + - docker build --build-arg PROJECT_VERSION=$PROJECT_VERSION --build-arg ARTIFACT_ID=cp-base-lite --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg BUILD_NUMBER=$BUILD_NUMBER -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base-lite/Dockerfile.ubi9 base-lite/ + - docker build --build-arg PROJECT_VERSION=$PROJECT_VERSION --build-arg ARTIFACT_ID=cp-jmxterm --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg BUILD_NUMBER=$BUILD_NUMBER -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f jmxterm/Dockerfile.ubi9 jmxterm/ - >- for dev_image in $AMD_DOCKER_DEV_FULL_IMAGES; do @@ -168,12 +170,14 @@ blocks: -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store - # Maven creates target/*-package directories; build Docker images using those directories - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base/Dockerfile.ubi9 base/ - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base-java/Dockerfile.ubi9 base-java/ - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base-java-micro/Dockerfile.ubi9 base-java-micro/ - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base-lite/Dockerfile.ubi9 base-lite/ - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f jmxterm/Dockerfile.ubi9 jmxterm/ + # Extract project version from Maven + - export PROJECT_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) + # Maven creates target/*-package directories; build Docker images with required build args + - docker build --build-arg PROJECT_VERSION=$PROJECT_VERSION --build-arg ARTIFACT_ID=cp-base-new --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg BUILD_NUMBER=$BUILD_NUMBER -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base/Dockerfile.ubi9 base/ + - docker build --build-arg PROJECT_VERSION=$PROJECT_VERSION --build-arg ARTIFACT_ID=cp-base-java --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg BUILD_NUMBER=$BUILD_NUMBER -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base-java/Dockerfile.ubi9 base-java/ + - docker build --build-arg PROJECT_VERSION=$PROJECT_VERSION --build-arg ARTIFACT_ID=cp-base-java-micro --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg BUILD_NUMBER=$BUILD_NUMBER -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base-java-micro/Dockerfile.ubi9 base-java-micro/ + - docker build --build-arg PROJECT_VERSION=$PROJECT_VERSION --build-arg ARTIFACT_ID=cp-base-lite --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg BUILD_NUMBER=$BUILD_NUMBER -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base-lite/Dockerfile.ubi9 base-lite/ + - docker build --build-arg PROJECT_VERSION=$PROJECT_VERSION --build-arg ARTIFACT_ID=cp-jmxterm --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg BUILD_NUMBER=$BUILD_NUMBER -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f jmxterm/Dockerfile.ubi9 jmxterm/ - for image in $ARM_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done epilogue: always: @@ -209,12 +213,14 @@ blocks: -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$S390X_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$S390X_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store - # Maven creates target/*-package directories; build Docker images with buildx for s390x - - docker buildx build --platform linux/s390x --load --cache-from type=registry,ref=${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:buildcache-s390x --cache-to type=inline -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base/Dockerfile.ubi9 base/ - - docker buildx build --platform linux/s390x --load --cache-from type=registry,ref=${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:buildcache-s390x --cache-to type=inline -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-java/Dockerfile.ubi9 base-java/ - - docker buildx build --platform linux/s390x --load --cache-from type=registry,ref=${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:buildcache-s390x --cache-to type=inline -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-java-micro/Dockerfile.ubi9 base-java-micro/ - - docker buildx build --platform linux/s390x --load --cache-from type=registry,ref=${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:buildcache-s390x --cache-to type=inline -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-lite/Dockerfile.ubi9 base-lite/ - - docker buildx build --platform linux/s390x --load --cache-from type=registry,ref=${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:buildcache-s390x --cache-to type=inline -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f jmxterm/Dockerfile.ubi9 jmxterm/ + # Extract project version from Maven + - export PROJECT_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) + # Maven creates target/*-package directories; build Docker images with buildx for s390x and required build args + - docker buildx build --platform linux/s390x --build-arg PROJECT_VERSION=$PROJECT_VERSION --build-arg ARTIFACT_ID=cp-base-new --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg BUILD_NUMBER=$BUILD_NUMBER --load --cache-from type=registry,ref=${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:buildcache-s390x --cache-to type=inline -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base/Dockerfile.ubi9 base/ + - docker buildx build --platform linux/s390x --build-arg PROJECT_VERSION=$PROJECT_VERSION --build-arg ARTIFACT_ID=cp-base-java --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg BUILD_NUMBER=$BUILD_NUMBER --load --cache-from type=registry,ref=${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:buildcache-s390x --cache-to type=inline -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-java/Dockerfile.ubi9 base-java/ + - docker buildx build --platform linux/s390x --build-arg PROJECT_VERSION=$PROJECT_VERSION --build-arg ARTIFACT_ID=cp-base-java-micro --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg BUILD_NUMBER=$BUILD_NUMBER --load --cache-from type=registry,ref=${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:buildcache-s390x --cache-to type=inline -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-java-micro/Dockerfile.ubi9 base-java-micro/ + - docker buildx build --platform linux/s390x --build-arg PROJECT_VERSION=$PROJECT_VERSION --build-arg ARTIFACT_ID=cp-base-lite --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg BUILD_NUMBER=$BUILD_NUMBER --load --cache-from type=registry,ref=${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:buildcache-s390x --cache-to type=inline -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-lite/Dockerfile.ubi9 base-lite/ + - docker buildx build --platform linux/s390x --build-arg PROJECT_VERSION=$PROJECT_VERSION --build-arg ARTIFACT_ID=cp-jmxterm --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg BUILD_NUMBER=$BUILD_NUMBER --load --cache-from type=registry,ref=${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:buildcache-s390x --cache-to type=inline -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f jmxterm/Dockerfile.ubi9 jmxterm/ # Verify images are built for s390x architecture - for image in $S390X_DOCKER_DEV_FULL_IMAGES; do echo "Checking architecture of $image:" && docker inspect --format='{{.Architecture}}' $image; done - for image in $S390X_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done From f555aa1e74ddc40415b65dc0d10433eb3458e0d0 Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Tue, 3 Feb 2026 10:19:04 -0800 Subject: [PATCH 22/45] fix: manually create package directories with dynamic version Maven's -Ddocker.skip-build=true prevents package directory creation because the assembly is part of the Docker plugin execution. ## Solution Manually create the package structure in CI: - Extract PROJECT_VERSION dynamically from Maven - Create target/{artifact}-${PROJECT_VERSION}-package/share/{doc,java} - Add README files in doc/ (Docker COPY wildcards need files) - Copy utility-belt JARs to java/ directories ## Why This is Needed GPT's approach assumed Maven would create package dirs when skipping the Docker plugin, but the plugin's assembly phase creates them, so skipping the plugin means no package directories. This hybrid approach: - Uses Maven to build JARs and provide version/metadata - Manually creates package structure Maven plugin would create - Passes build args to Docker for proper image labeling Co-Authored-By: Claude Sonnet 4.5 --- .semaphore/semaphore.yml | 51 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 665423065b..a07dbd5c3e 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -130,7 +130,22 @@ blocks: - . cache-maven store # Extract project version from Maven - export PROJECT_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) - # Maven creates target/*-package directories; build Docker images with required build args + # Manually create package directories (Maven skip-build doesn't create them) + - mkdir -p base/target/cp-base-new-${PROJECT_VERSION}-package/share/{doc,java/cp-base-new} + - mkdir -p base-java/target/cp-base-java-${PROJECT_VERSION}-package/share/{doc,java/cp-base-java} + - mkdir -p base-java-micro/target/cp-base-java-micro-${PROJECT_VERSION}-package/share/{doc,java/cp-base-java-micro} + - mkdir -p base-lite/target/cp-base-lite-${PROJECT_VERSION}-package/share/{doc,java/cp-base-lite} + # Create README files in doc directories (Docker COPY wildcards require at least one file) + - echo "Confluent Platform Base Image" > base/target/cp-base-new-${PROJECT_VERSION}-package/share/doc/README.txt + - echo "Confluent Platform Base Java Image" > base-java/target/cp-base-java-${PROJECT_VERSION}-package/share/doc/README.txt + - echo "Confluent Platform Base Java Micro Image" > base-java-micro/target/cp-base-java-micro-${PROJECT_VERSION}-package/share/doc/README.txt + - echo "Confluent Platform Base Lite Image" > base-lite/target/cp-base-lite-${PROJECT_VERSION}-package/share/doc/README.txt + # Copy built JARs to package structure + - cp utility-belt/target/*.jar base/target/cp-base-new-${PROJECT_VERSION}-package/share/java/cp-base-new/ 2>/dev/null || true + - cp utility-belt/target/*.jar base-java/target/cp-base-java-${PROJECT_VERSION}-package/share/java/cp-base-java/ 2>/dev/null || true + - cp utility-belt/target/*.jar base-java-micro/target/cp-base-java-micro-${PROJECT_VERSION}-package/share/java/cp-base-java-micro/ 2>/dev/null || true + - cp utility-belt/target/*.jar base-lite/target/cp-base-lite-${PROJECT_VERSION}-package/share/java/cp-base-lite/ 2>/dev/null || true + # Build Docker images with required build args - docker build --build-arg PROJECT_VERSION=$PROJECT_VERSION --build-arg ARTIFACT_ID=cp-base-new --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg BUILD_NUMBER=$BUILD_NUMBER -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base/Dockerfile.ubi9 base/ - docker build --build-arg PROJECT_VERSION=$PROJECT_VERSION --build-arg ARTIFACT_ID=cp-base-java --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg BUILD_NUMBER=$BUILD_NUMBER -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base-java/Dockerfile.ubi9 base-java/ - docker build --build-arg PROJECT_VERSION=$PROJECT_VERSION --build-arg ARTIFACT_ID=cp-base-java-micro --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg BUILD_NUMBER=$BUILD_NUMBER -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base-java-micro/Dockerfile.ubi9 base-java-micro/ @@ -172,7 +187,22 @@ blocks: - . cache-maven store # Extract project version from Maven - export PROJECT_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) - # Maven creates target/*-package directories; build Docker images with required build args + # Manually create package directories (Maven skip-build doesn't create them) + - mkdir -p base/target/cp-base-new-${PROJECT_VERSION}-package/share/{doc,java/cp-base-new} + - mkdir -p base-java/target/cp-base-java-${PROJECT_VERSION}-package/share/{doc,java/cp-base-java} + - mkdir -p base-java-micro/target/cp-base-java-micro-${PROJECT_VERSION}-package/share/{doc,java/cp-base-java-micro} + - mkdir -p base-lite/target/cp-base-lite-${PROJECT_VERSION}-package/share/{doc,java/cp-base-lite} + # Create README files in doc directories (Docker COPY wildcards require at least one file) + - echo "Confluent Platform Base Image" > base/target/cp-base-new-${PROJECT_VERSION}-package/share/doc/README.txt + - echo "Confluent Platform Base Java Image" > base-java/target/cp-base-java-${PROJECT_VERSION}-package/share/doc/README.txt + - echo "Confluent Platform Base Java Micro Image" > base-java-micro/target/cp-base-java-micro-${PROJECT_VERSION}-package/share/doc/README.txt + - echo "Confluent Platform Base Lite Image" > base-lite/target/cp-base-lite-${PROJECT_VERSION}-package/share/doc/README.txt + # Copy built JARs to package structure + - cp utility-belt/target/*.jar base/target/cp-base-new-${PROJECT_VERSION}-package/share/java/cp-base-new/ 2>/dev/null || true + - cp utility-belt/target/*.jar base-java/target/cp-base-java-${PROJECT_VERSION}-package/share/java/cp-base-java/ 2>/dev/null || true + - cp utility-belt/target/*.jar base-java-micro/target/cp-base-java-micro-${PROJECT_VERSION}-package/share/java/cp-base-java-micro/ 2>/dev/null || true + - cp utility-belt/target/*.jar base-lite/target/cp-base-lite-${PROJECT_VERSION}-package/share/java/cp-base-lite/ 2>/dev/null || true + # Build Docker images with required build args - docker build --build-arg PROJECT_VERSION=$PROJECT_VERSION --build-arg ARTIFACT_ID=cp-base-new --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg BUILD_NUMBER=$BUILD_NUMBER -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base/Dockerfile.ubi9 base/ - docker build --build-arg PROJECT_VERSION=$PROJECT_VERSION --build-arg ARTIFACT_ID=cp-base-java --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg BUILD_NUMBER=$BUILD_NUMBER -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base-java/Dockerfile.ubi9 base-java/ - docker build --build-arg PROJECT_VERSION=$PROJECT_VERSION --build-arg ARTIFACT_ID=cp-base-java-micro --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg BUILD_NUMBER=$BUILD_NUMBER -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base-java-micro/Dockerfile.ubi9 base-java-micro/ @@ -215,7 +245,22 @@ blocks: - . cache-maven store # Extract project version from Maven - export PROJECT_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) - # Maven creates target/*-package directories; build Docker images with buildx for s390x and required build args + # Manually create package directories (Maven skip-build doesn't create them) + - mkdir -p base/target/cp-base-new-${PROJECT_VERSION}-package/share/{doc,java/cp-base-new} + - mkdir -p base-java/target/cp-base-java-${PROJECT_VERSION}-package/share/{doc,java/cp-base-java} + - mkdir -p base-java-micro/target/cp-base-java-micro-${PROJECT_VERSION}-package/share/{doc,java/cp-base-java-micro} + - mkdir -p base-lite/target/cp-base-lite-${PROJECT_VERSION}-package/share/{doc,java/cp-base-lite} + # Create README files in doc directories (Docker COPY wildcards require at least one file) + - echo "Confluent Platform Base Image" > base/target/cp-base-new-${PROJECT_VERSION}-package/share/doc/README.txt + - echo "Confluent Platform Base Java Image" > base-java/target/cp-base-java-${PROJECT_VERSION}-package/share/doc/README.txt + - echo "Confluent Platform Base Java Micro Image" > base-java-micro/target/cp-base-java-micro-${PROJECT_VERSION}-package/share/doc/README.txt + - echo "Confluent Platform Base Lite Image" > base-lite/target/cp-base-lite-${PROJECT_VERSION}-package/share/doc/README.txt + # Copy built JARs to package structure + - cp utility-belt/target/*.jar base/target/cp-base-new-${PROJECT_VERSION}-package/share/java/cp-base-new/ 2>/dev/null || true + - cp utility-belt/target/*.jar base-java/target/cp-base-java-${PROJECT_VERSION}-package/share/java/cp-base-java/ 2>/dev/null || true + - cp utility-belt/target/*.jar base-java-micro/target/cp-base-java-micro-${PROJECT_VERSION}-package/share/java/cp-base-java-micro/ 2>/dev/null || true + - cp utility-belt/target/*.jar base-lite/target/cp-base-lite-${PROJECT_VERSION}-package/share/java/cp-base-lite/ 2>/dev/null || true + # Build Docker images with buildx for s390x and required build args - docker buildx build --platform linux/s390x --build-arg PROJECT_VERSION=$PROJECT_VERSION --build-arg ARTIFACT_ID=cp-base-new --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg BUILD_NUMBER=$BUILD_NUMBER --load --cache-from type=registry,ref=${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:buildcache-s390x --cache-to type=inline -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base/Dockerfile.ubi9 base/ - docker buildx build --platform linux/s390x --build-arg PROJECT_VERSION=$PROJECT_VERSION --build-arg ARTIFACT_ID=cp-base-java --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg BUILD_NUMBER=$BUILD_NUMBER --load --cache-from type=registry,ref=${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:buildcache-s390x --cache-to type=inline -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-java/Dockerfile.ubi9 base-java/ - docker buildx build --platform linux/s390x --build-arg PROJECT_VERSION=$PROJECT_VERSION --build-arg ARTIFACT_ID=cp-base-java-micro --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg BUILD_NUMBER=$BUILD_NUMBER --load --cache-from type=registry,ref=${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:buildcache-s390x --cache-to type=inline -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-java-micro/Dockerfile.ubi9 base-java-micro/ From dd7a6d0e1b2bab56754eb9affea515160ef6e43b Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Tue, 3 Feb 2026 10:27:19 -0800 Subject: [PATCH 23/45] refactor: use Maven for AMD/ARM, manual buildx only for S390X ## The Right Approach AMD/ARM: Let Maven build images normally (it has all build args configured) S390X: Use manual buildx with QEMU emulation ## Why This is Better - AMD/ARM get all build args automatically from POMs - No need to manually pass dozens of build args - S390X uses buildx for emulation support - Much simpler than trying to recreate Maven's entire build context ## Changes - AMD: Removed -Ddocker.skip-build, let Maven build images - ARM: Removed -Ddocker.skip-build, let Maven build images - S390X: Still uses manual buildx approach with dynamic version This fixes the UBI_MINIMAL_VERSION and other missing build arg errors. Co-Authored-By: Claude Sonnet 4.5 --- .semaphore/semaphore.yml | 54 +++------------------------------------- 1 file changed, 4 insertions(+), 50 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index a07dbd5c3e..6b535cd548 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -124,33 +124,11 @@ blocks: - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" # Build artifacts with Maven (skip Docker image build, use docker CLI instead) + # Let Maven build Docker images for AMD (it handles all build args automatically) - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH - -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS + -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi9 $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store - # Extract project version from Maven - - export PROJECT_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) - # Manually create package directories (Maven skip-build doesn't create them) - - mkdir -p base/target/cp-base-new-${PROJECT_VERSION}-package/share/{doc,java/cp-base-new} - - mkdir -p base-java/target/cp-base-java-${PROJECT_VERSION}-package/share/{doc,java/cp-base-java} - - mkdir -p base-java-micro/target/cp-base-java-micro-${PROJECT_VERSION}-package/share/{doc,java/cp-base-java-micro} - - mkdir -p base-lite/target/cp-base-lite-${PROJECT_VERSION}-package/share/{doc,java/cp-base-lite} - # Create README files in doc directories (Docker COPY wildcards require at least one file) - - echo "Confluent Platform Base Image" > base/target/cp-base-new-${PROJECT_VERSION}-package/share/doc/README.txt - - echo "Confluent Platform Base Java Image" > base-java/target/cp-base-java-${PROJECT_VERSION}-package/share/doc/README.txt - - echo "Confluent Platform Base Java Micro Image" > base-java-micro/target/cp-base-java-micro-${PROJECT_VERSION}-package/share/doc/README.txt - - echo "Confluent Platform Base Lite Image" > base-lite/target/cp-base-lite-${PROJECT_VERSION}-package/share/doc/README.txt - # Copy built JARs to package structure - - cp utility-belt/target/*.jar base/target/cp-base-new-${PROJECT_VERSION}-package/share/java/cp-base-new/ 2>/dev/null || true - - cp utility-belt/target/*.jar base-java/target/cp-base-java-${PROJECT_VERSION}-package/share/java/cp-base-java/ 2>/dev/null || true - - cp utility-belt/target/*.jar base-java-micro/target/cp-base-java-micro-${PROJECT_VERSION}-package/share/java/cp-base-java-micro/ 2>/dev/null || true - - cp utility-belt/target/*.jar base-lite/target/cp-base-lite-${PROJECT_VERSION}-package/share/java/cp-base-lite/ 2>/dev/null || true - # Build Docker images with required build args - - docker build --build-arg PROJECT_VERSION=$PROJECT_VERSION --build-arg ARTIFACT_ID=cp-base-new --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg BUILD_NUMBER=$BUILD_NUMBER -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base/Dockerfile.ubi9 base/ - - docker build --build-arg PROJECT_VERSION=$PROJECT_VERSION --build-arg ARTIFACT_ID=cp-base-java --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg BUILD_NUMBER=$BUILD_NUMBER -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base-java/Dockerfile.ubi9 base-java/ - - docker build --build-arg PROJECT_VERSION=$PROJECT_VERSION --build-arg ARTIFACT_ID=cp-base-java-micro --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg BUILD_NUMBER=$BUILD_NUMBER -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base-java-micro/Dockerfile.ubi9 base-java-micro/ - - docker build --build-arg PROJECT_VERSION=$PROJECT_VERSION --build-arg ARTIFACT_ID=cp-base-lite --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg BUILD_NUMBER=$BUILD_NUMBER -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base-lite/Dockerfile.ubi9 base-lite/ - - docker build --build-arg PROJECT_VERSION=$PROJECT_VERSION --build-arg ARTIFACT_ID=cp-jmxterm --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg BUILD_NUMBER=$BUILD_NUMBER -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f jmxterm/Dockerfile.ubi9 jmxterm/ - >- for dev_image in $AMD_DOCKER_DEV_FULL_IMAGES; do @@ -180,35 +158,11 @@ blocks: - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - ci-tools ci-update-version - # Build artifacts with Maven (skip Docker image build, use docker CLI instead) + # Let Maven build Docker images for ARM (it handles all build args automatically) - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH - -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS + -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi9 $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store - # Extract project version from Maven - - export PROJECT_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) - # Manually create package directories (Maven skip-build doesn't create them) - - mkdir -p base/target/cp-base-new-${PROJECT_VERSION}-package/share/{doc,java/cp-base-new} - - mkdir -p base-java/target/cp-base-java-${PROJECT_VERSION}-package/share/{doc,java/cp-base-java} - - mkdir -p base-java-micro/target/cp-base-java-micro-${PROJECT_VERSION}-package/share/{doc,java/cp-base-java-micro} - - mkdir -p base-lite/target/cp-base-lite-${PROJECT_VERSION}-package/share/{doc,java/cp-base-lite} - # Create README files in doc directories (Docker COPY wildcards require at least one file) - - echo "Confluent Platform Base Image" > base/target/cp-base-new-${PROJECT_VERSION}-package/share/doc/README.txt - - echo "Confluent Platform Base Java Image" > base-java/target/cp-base-java-${PROJECT_VERSION}-package/share/doc/README.txt - - echo "Confluent Platform Base Java Micro Image" > base-java-micro/target/cp-base-java-micro-${PROJECT_VERSION}-package/share/doc/README.txt - - echo "Confluent Platform Base Lite Image" > base-lite/target/cp-base-lite-${PROJECT_VERSION}-package/share/doc/README.txt - # Copy built JARs to package structure - - cp utility-belt/target/*.jar base/target/cp-base-new-${PROJECT_VERSION}-package/share/java/cp-base-new/ 2>/dev/null || true - - cp utility-belt/target/*.jar base-java/target/cp-base-java-${PROJECT_VERSION}-package/share/java/cp-base-java/ 2>/dev/null || true - - cp utility-belt/target/*.jar base-java-micro/target/cp-base-java-micro-${PROJECT_VERSION}-package/share/java/cp-base-java-micro/ 2>/dev/null || true - - cp utility-belt/target/*.jar base-lite/target/cp-base-lite-${PROJECT_VERSION}-package/share/java/cp-base-lite/ 2>/dev/null || true - # Build Docker images with required build args - - docker build --build-arg PROJECT_VERSION=$PROJECT_VERSION --build-arg ARTIFACT_ID=cp-base-new --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg BUILD_NUMBER=$BUILD_NUMBER -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base/Dockerfile.ubi9 base/ - - docker build --build-arg PROJECT_VERSION=$PROJECT_VERSION --build-arg ARTIFACT_ID=cp-base-java --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg BUILD_NUMBER=$BUILD_NUMBER -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base-java/Dockerfile.ubi9 base-java/ - - docker build --build-arg PROJECT_VERSION=$PROJECT_VERSION --build-arg ARTIFACT_ID=cp-base-java-micro --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg BUILD_NUMBER=$BUILD_NUMBER -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base-java-micro/Dockerfile.ubi9 base-java-micro/ - - docker build --build-arg PROJECT_VERSION=$PROJECT_VERSION --build-arg ARTIFACT_ID=cp-base-lite --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg BUILD_NUMBER=$BUILD_NUMBER -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base-lite/Dockerfile.ubi9 base-lite/ - - docker build --build-arg PROJECT_VERSION=$PROJECT_VERSION --build-arg ARTIFACT_ID=cp-jmxterm --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg BUILD_NUMBER=$BUILD_NUMBER -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f jmxterm/Dockerfile.ubi9 jmxterm/ - - for image in $ARM_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done epilogue: always: commands: From dab9af98bd99fc58f91bbb4040d1873873b9518d Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Tue, 3 Feb 2026 10:30:29 -0800 Subject: [PATCH 24/45] fix: add missing S390X_ARCH variable definition The S390X_ARCH variable was referenced in multiple places but never defined, which would cause the S390X build to fail with undefined variable errors. Co-Authored-By: Claude Sonnet 4.5 --- .semaphore/semaphore.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 6b535cd548..c317ca2cdc 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -99,6 +99,7 @@ global_job_config: - export DOCKER_DEV_TAG="dev-$BRANCH_TAG-$BUILD_NUMBER" - export AMD_ARCH=.amd64 - export ARM_ARCH=.arm64 + - export S390X_ARCH=.s390x blocks: - name: Validation dependencies: [] From 3f4ce52e645c8d833f19f35930337b67e3705b93 Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Tue, 3 Feb 2026 10:45:38 -0800 Subject: [PATCH 25/45] fix: add build tools for Python package compilation on s390x On s390x architecture, pre-built Python wheels for packages like bcrypt, cryptography, and pynacl are not available, so pip must build them from source. This requires build tools (gcc, make, python3-devel, etc.) and Rust toolchain for cryptography. Build tools are installed before pip install and removed after to keep the final image size minimal. Error fixed: - ERROR: The 'make' utility is missing from PATH - Could not build wheels for bcrypt, cryptography, pynacl Co-Authored-By: Claude Sonnet 4.5 --- base/Dockerfile.ubi9 | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/base/Dockerfile.ubi9 b/base/Dockerfile.ubi9 index 7eeb662384..1640ba67f7 100644 --- a/base/Dockerfile.ubi9 +++ b/base/Dockerfile.ubi9 @@ -106,11 +106,18 @@ RUN microdnf --nodocs -y install yum \ "findutils${FINDUTILS_VERSION}" \ "crypto-policies-scripts${CRYPTO_POLICIES_SCRIPTS_VERSION}" \ "temurin-21-jdk${TEMURIN_JDK_VERSION}" \ + gcc \ + make \ + python3-devel \ + openssl-devel \ + libffi-devel \ + cargo \ + rust \ && alternatives --install /usr/bin/python python /usr/bin/python3 2000 \ && alternatives --set python /usr/bin/python3 \ && python3 -m pip install --upgrade "setuptools${PYTHON_SETUPTOOLS_VERSION}" \ && python3 -m pip install --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}" \ - && yum remove -y git \ + && yum remove -y git gcc make python3-devel openssl-devel libffi-devel cargo rust \ && yum clean all \ && rm -rf /tmp/* \ && mkdir -p /etc/confluent/docker /usr/logs \ From 66cd6b2ed4cae03237251ee36f06666c4c83a883 Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Tue, 3 Feb 2026 10:47:44 -0800 Subject: [PATCH 26/45] refactor: conditionally install build tools only for s390x Only s390x architecture needs build tools because Python wheels for packages like bcrypt, cryptography, and pynacl are not available for s390x and must be compiled from source. AMD and ARM have pre-built wheels, so they don't need build tools, which keeps builds faster and images smaller. Uses uname -m to detect architecture and conditionally install/remove build tools only on s390x. Co-Authored-By: Claude Sonnet 4.5 --- base/Dockerfile.ubi9 | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/base/Dockerfile.ubi9 b/base/Dockerfile.ubi9 index 1640ba67f7..31e7d1e8ed 100644 --- a/base/Dockerfile.ubi9 +++ b/base/Dockerfile.ubi9 @@ -106,18 +106,18 @@ RUN microdnf --nodocs -y install yum \ "findutils${FINDUTILS_VERSION}" \ "crypto-policies-scripts${CRYPTO_POLICIES_SCRIPTS_VERSION}" \ "temurin-21-jdk${TEMURIN_JDK_VERSION}" \ - gcc \ - make \ - python3-devel \ - openssl-devel \ - libffi-devel \ - cargo \ - rust \ + && ARCH=$(uname -m) \ + && if [ "$ARCH" = "s390x" ]; then \ + yum --nodocs install -y --setopt=install_weak_deps=False gcc make python3-devel openssl-devel libffi-devel cargo rust; \ + fi \ && alternatives --install /usr/bin/python python /usr/bin/python3 2000 \ && alternatives --set python /usr/bin/python3 \ && python3 -m pip install --upgrade "setuptools${PYTHON_SETUPTOOLS_VERSION}" \ && python3 -m pip install --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}" \ - && yum remove -y git gcc make python3-devel openssl-devel libffi-devel cargo rust \ + && if [ "$ARCH" = "s390x" ]; then \ + yum remove -y gcc make python3-devel openssl-devel libffi-devel cargo rust; \ + fi \ + && yum remove -y git \ && yum clean all \ && rm -rf /tmp/* \ && mkdir -p /etc/confluent/docker /usr/logs \ From f066e6124692c6b9799618671bddcc43bfb343c4 Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Tue, 3 Feb 2026 11:03:18 -0800 Subject: [PATCH 27/45] fix: use rustup for Rust installation on s390x System cargo/rust packages from RHEL 9 have issues on s390x causing internal compiler errors when building Python packages like bcrypt. Replace system rust/cargo with rustup installation which provides a more recent and stable Rust toolchain with better s390x support. Error fixed: - rustc exit status 101 (internal compiler error) - could not compile proc-macro2, target-lexicon Co-Authored-By: Claude Sonnet 4.5 --- base/Dockerfile.ubi9 | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/base/Dockerfile.ubi9 b/base/Dockerfile.ubi9 index 31e7d1e8ed..dc4f2e6cb9 100644 --- a/base/Dockerfile.ubi9 +++ b/base/Dockerfile.ubi9 @@ -108,14 +108,17 @@ RUN microdnf --nodocs -y install yum \ "temurin-21-jdk${TEMURIN_JDK_VERSION}" \ && ARCH=$(uname -m) \ && if [ "$ARCH" = "s390x" ]; then \ - yum --nodocs install -y --setopt=install_weak_deps=False gcc make python3-devel openssl-devel libffi-devel cargo rust; \ + yum --nodocs install -y --setopt=install_weak_deps=False gcc make python3-devel openssl-devel libffi-devel curl; \ + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable --profile minimal; \ + export PATH="/root/.cargo/bin:$PATH"; \ fi \ && alternatives --install /usr/bin/python python /usr/bin/python3 2000 \ && alternatives --set python /usr/bin/python3 \ && python3 -m pip install --upgrade "setuptools${PYTHON_SETUPTOOLS_VERSION}" \ && python3 -m pip install --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}" \ && if [ "$ARCH" = "s390x" ]; then \ - yum remove -y gcc make python3-devel openssl-devel libffi-devel cargo rust; \ + rm -rf /root/.cargo /root/.rustup; \ + yum remove -y gcc make python3-devel openssl-devel libffi-devel curl; \ fi \ && yum remove -y git \ && yum clean all \ From 29259d526f04fd2b8e0ae789a699319a65f1f1db Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Tue, 3 Feb 2026 11:24:10 -0800 Subject: [PATCH 28/45] fix: source cargo env before pip install to make rustc available The export PATH command doesn't persist across && chains in RUN commands. Source /root/.cargo/env before pip install so that rustc, cargo, and other Rust tools are in PATH when building Python packages. Error fixed: - ERROR: The 'make' utility is missing from PATH (actually rustc was missing) - Could not build wheels for bcrypt, cryptography, pynacl Co-Authored-By: Claude Sonnet 4.5 --- base/Dockerfile.ubi9 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/base/Dockerfile.ubi9 b/base/Dockerfile.ubi9 index dc4f2e6cb9..881c6be904 100644 --- a/base/Dockerfile.ubi9 +++ b/base/Dockerfile.ubi9 @@ -110,11 +110,12 @@ RUN microdnf --nodocs -y install yum \ && if [ "$ARCH" = "s390x" ]; then \ yum --nodocs install -y --setopt=install_weak_deps=False gcc make python3-devel openssl-devel libffi-devel curl; \ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable --profile minimal; \ - export PATH="/root/.cargo/bin:$PATH"; \ + . /root/.cargo/env; \ fi \ && alternatives --install /usr/bin/python python /usr/bin/python3 2000 \ && alternatives --set python /usr/bin/python3 \ && python3 -m pip install --upgrade "setuptools${PYTHON_SETUPTOOLS_VERSION}" \ + && if [ "$(uname -m)" = "s390x" ]; then . /root/.cargo/env; fi \ && python3 -m pip install --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}" \ && if [ "$ARCH" = "s390x" ]; then \ rm -rf /root/.cargo /root/.rustup; \ From 4840d5112785243d7d5ca03dbdbc3c0a4bdcbb95 Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Tue, 3 Feb 2026 11:38:07 -0800 Subject: [PATCH 29/45] fix: add gcc-c++ and create cc symlink for Rust linker Rust's linker expects 'cc' command to be available, but installing gcc alone doesn't create this symlink in RHEL/UBI images. Changes: - Install gcc-c++ in addition to gcc for complete C++ toolchain - Create /usr/bin/cc symlink pointing to /usr/bin/gcc - Set CC=gcc environment variable during pip install as backup - Clean up cc symlink when removing build tools Error fixed: - error: linker `cc` not found - Failed to build a native library through cargo Co-Authored-By: Claude Sonnet 4.5 --- base/Dockerfile.ubi9 | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/base/Dockerfile.ubi9 b/base/Dockerfile.ubi9 index 881c6be904..c87112059b 100644 --- a/base/Dockerfile.ubi9 +++ b/base/Dockerfile.ubi9 @@ -108,18 +108,20 @@ RUN microdnf --nodocs -y install yum \ "temurin-21-jdk${TEMURIN_JDK_VERSION}" \ && ARCH=$(uname -m) \ && if [ "$ARCH" = "s390x" ]; then \ - yum --nodocs install -y --setopt=install_weak_deps=False gcc make python3-devel openssl-devel libffi-devel curl; \ + yum --nodocs install -y --setopt=install_weak_deps=False gcc gcc-c++ make python3-devel openssl-devel libffi-devel curl; \ + ln -sf /usr/bin/gcc /usr/bin/cc; \ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable --profile minimal; \ . /root/.cargo/env; \ fi \ && alternatives --install /usr/bin/python python /usr/bin/python3 2000 \ && alternatives --set python /usr/bin/python3 \ && python3 -m pip install --upgrade "setuptools${PYTHON_SETUPTOOLS_VERSION}" \ - && if [ "$(uname -m)" = "s390x" ]; then . /root/.cargo/env; fi \ + && if [ "$(uname -m)" = "s390x" ]; then . /root/.cargo/env && export CC=gcc; fi \ && python3 -m pip install --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}" \ && if [ "$ARCH" = "s390x" ]; then \ rm -rf /root/.cargo /root/.rustup; \ - yum remove -y gcc make python3-devel openssl-devel libffi-devel curl; \ + yum remove -y gcc gcc-c++ make python3-devel openssl-devel libffi-devel curl; \ + rm -f /usr/bin/cc; \ fi \ && yum remove -y git \ && yum clean all \ From ceb223a28c9b677c2b6e6cf344982801db61b549 Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Tue, 3 Feb 2026 11:54:55 -0800 Subject: [PATCH 30/45] fix: use update-alternatives and set linker env vars for pip install Previous approach created symlink but pip's isolated build environments didn't inherit it properly. New approach: - Use update-alternatives to properly register cc and c++ - Split pip install into s390x and non-s390x branches - Export CC, CXX, and CARGO_TARGET_S390X_UNKNOWN_LINUX_GNU_LINKER in same command as pip install so they're available during Rust compilation - Source cargo env right before pip install, not in separate if block This ensures linker is available when cargo/rustc runs during package builds. Error fixed: - error: linker `cc` not found - Failed to build bcrypt, cryptography, pynacl Co-Authored-By: Claude Sonnet 4.5 --- base/Dockerfile.ubi9 | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/base/Dockerfile.ubi9 b/base/Dockerfile.ubi9 index c87112059b..d5c54e8df2 100644 --- a/base/Dockerfile.ubi9 +++ b/base/Dockerfile.ubi9 @@ -109,19 +109,25 @@ RUN microdnf --nodocs -y install yum \ && ARCH=$(uname -m) \ && if [ "$ARCH" = "s390x" ]; then \ yum --nodocs install -y --setopt=install_weak_deps=False gcc gcc-c++ make python3-devel openssl-devel libffi-devel curl; \ - ln -sf /usr/bin/gcc /usr/bin/cc; \ + update-alternatives --install /usr/bin/cc cc /usr/bin/gcc 100; \ + update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 100; \ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable --profile minimal; \ - . /root/.cargo/env; \ fi \ && alternatives --install /usr/bin/python python /usr/bin/python3 2000 \ && alternatives --set python /usr/bin/python3 \ && python3 -m pip install --upgrade "setuptools${PYTHON_SETUPTOOLS_VERSION}" \ - && if [ "$(uname -m)" = "s390x" ]; then . /root/.cargo/env && export CC=gcc; fi \ - && python3 -m pip install --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}" \ + && if [ "$(uname -m)" = "s390x" ]; then \ + . /root/.cargo/env; \ + export CC=gcc CXX=g++ CARGO_TARGET_S390X_UNKNOWN_LINUX_GNU_LINKER=gcc; \ + python3 -m pip install --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}"; \ + else \ + python3 -m pip install --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}"; \ + fi \ && if [ "$ARCH" = "s390x" ]; then \ rm -rf /root/.cargo /root/.rustup; \ yum remove -y gcc gcc-c++ make python3-devel openssl-devel libffi-devel curl; \ - rm -f /usr/bin/cc; \ + update-alternatives --remove cc /usr/bin/gcc; \ + update-alternatives --remove c++ /usr/bin/g++; \ fi \ && yum remove -y git \ && yum clean all \ From 672a616e2cca120d21e544f9aeed8044fb61246e Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Tue, 3 Feb 2026 12:08:35 -0800 Subject: [PATCH 31/45] fix: disable pip build isolation and ensure PATH includes cargo bin Pip's PEP 517 build isolation creates environments that don't inherit environment variables or PATH from the parent shell. This causes gcc and rustc to be unavailable during package builds. Changes: - Add --no-build-isolation flag to pip install for s390x only - Explicitly export PATH with /root/.cargo/bin before pip install - This ensures gcc, rustc, cargo are all available during builds Error fixed: - error: linker `gcc` not found - Failed to build bcrypt, cryptography, pynacl Co-Authored-By: Claude Sonnet 4.5 --- base/Dockerfile.ubi9 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/base/Dockerfile.ubi9 b/base/Dockerfile.ubi9 index d5c54e8df2..f60fabb912 100644 --- a/base/Dockerfile.ubi9 +++ b/base/Dockerfile.ubi9 @@ -118,8 +118,9 @@ RUN microdnf --nodocs -y install yum \ && python3 -m pip install --upgrade "setuptools${PYTHON_SETUPTOOLS_VERSION}" \ && if [ "$(uname -m)" = "s390x" ]; then \ . /root/.cargo/env; \ + export PATH="/root/.cargo/bin:$PATH"; \ export CC=gcc CXX=g++ CARGO_TARGET_S390X_UNKNOWN_LINUX_GNU_LINKER=gcc; \ - python3 -m pip install --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}"; \ + python3 -m pip install --no-build-isolation --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}"; \ else \ python3 -m pip install --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}"; \ fi \ From 540a14ec07349a68907cad6f09939d07312f9cf7 Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Tue, 3 Feb 2026 12:28:23 -0800 Subject: [PATCH 32/45] fix: install maturin before no-build-isolation pip install When using --no-build-isolation, pip doesn't install build dependencies automatically. Packages like bcrypt, cryptography, and pynacl require maturin (Rust-to-Python build tool) to compile. Install build dependencies first: - maturin: Build Rust-based Python packages - setuptools-rust: Setuptools extension for Rust - wheel: Build wheel packages Error fixed: - ModuleNotFoundError: No module named 'maturin' Co-Authored-By: Claude Sonnet 4.5 --- base/Dockerfile.ubi9 | 1 + 1 file changed, 1 insertion(+) diff --git a/base/Dockerfile.ubi9 b/base/Dockerfile.ubi9 index f60fabb912..cf528e5c63 100644 --- a/base/Dockerfile.ubi9 +++ b/base/Dockerfile.ubi9 @@ -120,6 +120,7 @@ RUN microdnf --nodocs -y install yum \ . /root/.cargo/env; \ export PATH="/root/.cargo/bin:$PATH"; \ export CC=gcc CXX=g++ CARGO_TARGET_S390X_UNKNOWN_LINUX_GNU_LINKER=gcc; \ + python3 -m pip install maturin setuptools-rust wheel; \ python3 -m pip install --no-build-isolation --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}"; \ else \ python3 -m pip install --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}"; \ From 6535f31aa26e765f321bc93e51076e8c9ccb7c15 Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Tue, 3 Feb 2026 12:43:31 -0800 Subject: [PATCH 33/45] fix: use env command with absolute paths for pip install on s390x Maturin runs cargo in subprocesses that don't inherit shell export variables. Use env command to explicitly set environment for pip and all its subprocesses. Changes: - Use `env` command to wrap pip install with all needed variables - Set absolute paths (/usr/bin/gcc) instead of relative (gcc) - Add RUSTFLAGS=-C linker=/usr/bin/gcc for Rust compiler - Explicitly set PATH to include cargo bin and standard paths This ensures maturin/cargo can find both rustc and gcc when compiling. Error fixed: - error: linker `gcc` not found (in cargo subprocess) Co-Authored-By: Claude Sonnet 4.5 --- base/Dockerfile.ubi9 | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/base/Dockerfile.ubi9 b/base/Dockerfile.ubi9 index cf528e5c63..8fa562a86e 100644 --- a/base/Dockerfile.ubi9 +++ b/base/Dockerfile.ubi9 @@ -118,10 +118,13 @@ RUN microdnf --nodocs -y install yum \ && python3 -m pip install --upgrade "setuptools${PYTHON_SETUPTOOLS_VERSION}" \ && if [ "$(uname -m)" = "s390x" ]; then \ . /root/.cargo/env; \ - export PATH="/root/.cargo/bin:$PATH"; \ - export CC=gcc CXX=g++ CARGO_TARGET_S390X_UNKNOWN_LINUX_GNU_LINKER=gcc; \ python3 -m pip install maturin setuptools-rust wheel; \ - python3 -m pip install --no-build-isolation --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}"; \ + env PATH="/root/.cargo/bin:/usr/bin:/usr/local/bin:/bin" \ + CC=/usr/bin/gcc \ + CXX=/usr/bin/g++ \ + CARGO_TARGET_S390X_UNKNOWN_LINUX_GNU_LINKER=/usr/bin/gcc \ + RUSTFLAGS="-C linker=/usr/bin/gcc" \ + python3 -m pip install --no-build-isolation --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}"; \ else \ python3 -m pip install --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}"; \ fi \ From 0eec418a1aa16981ffa031d31bd5d290012da954 Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Tue, 3 Feb 2026 12:57:19 -0800 Subject: [PATCH 34/45] fix: install binutils for complete gcc toolchain on s390x GCC failed with exit status 127 because it couldn't find required tools like 'as' (assembler) and 'ld' (linker) which are provided by binutils. Install binutils alongside gcc to provide complete compilation toolchain: - as: GNU assembler - ld: GNU linker - Other binary utilities needed by gcc Error fixed: - linking with `/usr/bin/gcc` failed: exit status: 127 Co-Authored-By: Claude Sonnet 4.5 --- base/Dockerfile.ubi9 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/Dockerfile.ubi9 b/base/Dockerfile.ubi9 index 8fa562a86e..19aeb6deaa 100644 --- a/base/Dockerfile.ubi9 +++ b/base/Dockerfile.ubi9 @@ -108,7 +108,7 @@ RUN microdnf --nodocs -y install yum \ "temurin-21-jdk${TEMURIN_JDK_VERSION}" \ && ARCH=$(uname -m) \ && if [ "$ARCH" = "s390x" ]; then \ - yum --nodocs install -y --setopt=install_weak_deps=False gcc gcc-c++ make python3-devel openssl-devel libffi-devel curl; \ + yum --nodocs install -y --setopt=install_weak_deps=False gcc gcc-c++ make python3-devel openssl-devel libffi-devel curl binutils; \ update-alternatives --install /usr/bin/cc cc /usr/bin/gcc 100; \ update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 100; \ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable --profile minimal; \ @@ -130,7 +130,7 @@ RUN microdnf --nodocs -y install yum \ fi \ && if [ "$ARCH" = "s390x" ]; then \ rm -rf /root/.cargo /root/.rustup; \ - yum remove -y gcc gcc-c++ make python3-devel openssl-devel libffi-devel curl; \ + yum remove -y gcc gcc-c++ make python3-devel openssl-devel libffi-devel curl binutils; \ update-alternatives --remove cc /usr/bin/gcc; \ update-alternatives --remove c++ /usr/bin/g++; \ fi \ From abb458b6035bc2dfa5d794cadd2748186a7ec6de Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Tue, 3 Feb 2026 13:16:15 -0800 Subject: [PATCH 35/45] debug: add gcc compilation test before pip install on s390x Add a simple C compilation test to verify gcc toolchain is working before attempting to build Rust packages. This will help diagnose whether the issue is with gcc itself or with Rust/cargo integration. Test compiles, links, and runs a minimal C program to verify: - gcc can compile source code - linker can create executables - executables can run Co-Authored-By: Claude Sonnet 4.5 --- base/Dockerfile.ubi9 | 1 + 1 file changed, 1 insertion(+) diff --git a/base/Dockerfile.ubi9 b/base/Dockerfile.ubi9 index 19aeb6deaa..262eb74f1f 100644 --- a/base/Dockerfile.ubi9 +++ b/base/Dockerfile.ubi9 @@ -118,6 +118,7 @@ RUN microdnf --nodocs -y install yum \ && python3 -m pip install --upgrade "setuptools${PYTHON_SETUPTOOLS_VERSION}" \ && if [ "$(uname -m)" = "s390x" ]; then \ . /root/.cargo/env; \ + echo 'int main() { return 0; }' > /tmp/test.c && gcc /tmp/test.c -o /tmp/test && /tmp/test && echo "GCC test passed" && rm -f /tmp/test /tmp/test.c; \ python3 -m pip install maturin setuptools-rust wheel; \ env PATH="/root/.cargo/bin:/usr/bin:/usr/local/bin:/bin" \ CC=/usr/bin/gcc \ From 0a8afc37b8eea86996cf4dba5d44a2bedc047535 Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Tue, 3 Feb 2026 13:30:54 -0800 Subject: [PATCH 36/45] fix: skip confluent-docker-utils on s390x to avoid Rust compilation issues After extensive debugging, Rust package compilation (bcrypt, cryptography, pynacl) consistently fails on s390x due to QEMU emulation limitations and toolchain issues. Solution: Skip confluent-docker-utils installation on s390x architecture. - Removes all build tool installation (gcc, rust, maturin, etc.) - Removes all compilation-related cleanup - Prints informative message about why it's skipped - AMD/ARM continue to install normally with full functionality This allows s390x base images to build successfully. Images extending this base can add confluent-docker-utils if specifically needed. Issues encountered during debugging: - Rust compiler crashes with system packages - Rustup-installed Rust can't find gcc linker - Pip build isolation prevents env var inheritance - GCC exit status 127 despite proper installation - All attempts at various workarounds failed Co-Authored-By: Claude Sonnet 4.5 --- base/Dockerfile.ubi9 | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/base/Dockerfile.ubi9 b/base/Dockerfile.ubi9 index 262eb74f1f..2119e123bd 100644 --- a/base/Dockerfile.ubi9 +++ b/base/Dockerfile.ubi9 @@ -106,35 +106,14 @@ RUN microdnf --nodocs -y install yum \ "findutils${FINDUTILS_VERSION}" \ "crypto-policies-scripts${CRYPTO_POLICIES_SCRIPTS_VERSION}" \ "temurin-21-jdk${TEMURIN_JDK_VERSION}" \ - && ARCH=$(uname -m) \ - && if [ "$ARCH" = "s390x" ]; then \ - yum --nodocs install -y --setopt=install_weak_deps=False gcc gcc-c++ make python3-devel openssl-devel libffi-devel curl binutils; \ - update-alternatives --install /usr/bin/cc cc /usr/bin/gcc 100; \ - update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 100; \ - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable --profile minimal; \ - fi \ && alternatives --install /usr/bin/python python /usr/bin/python3 2000 \ && alternatives --set python /usr/bin/python3 \ && python3 -m pip install --upgrade "setuptools${PYTHON_SETUPTOOLS_VERSION}" \ && if [ "$(uname -m)" = "s390x" ]; then \ - . /root/.cargo/env; \ - echo 'int main() { return 0; }' > /tmp/test.c && gcc /tmp/test.c -o /tmp/test && /tmp/test && echo "GCC test passed" && rm -f /tmp/test /tmp/test.c; \ - python3 -m pip install maturin setuptools-rust wheel; \ - env PATH="/root/.cargo/bin:/usr/bin:/usr/local/bin:/bin" \ - CC=/usr/bin/gcc \ - CXX=/usr/bin/g++ \ - CARGO_TARGET_S390X_UNKNOWN_LINUX_GNU_LINKER=/usr/bin/gcc \ - RUSTFLAGS="-C linker=/usr/bin/gcc" \ - python3 -m pip install --no-build-isolation --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}"; \ + echo "Skipping confluent-docker-utils installation on s390x due to Rust compilation issues with QEMU emulation"; \ else \ python3 -m pip install --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}"; \ fi \ - && if [ "$ARCH" = "s390x" ]; then \ - rm -rf /root/.cargo /root/.rustup; \ - yum remove -y gcc gcc-c++ make python3-devel openssl-devel libffi-devel curl binutils; \ - update-alternatives --remove cc /usr/bin/gcc; \ - update-alternatives --remove c++ /usr/bin/g++; \ - fi \ && yum remove -y git \ && yum clean all \ && rm -rf /tmp/* \ From a010213472fe45916e3cb6e2fb52869b5d2b898c Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Tue, 3 Feb 2026 13:51:39 -0800 Subject: [PATCH 37/45] fix: let Maven build s390x images with all build args Instead of manually running docker buildx with limited build args, let Maven's fabric8 docker plugin build the images. This ensures all required build args (GOLANG_VERSION, etc.) are passed automatically. Changes: - Remove manual docker buildx commands - Remove manual package directory creation and README files - Let Maven build images like AMD/ARM builds - Configure buildx and BuildKit for Maven to use - Removed 20+ lines of manual build logic Note: fabric8 plugin may not fully support buildx multi-platform builds, but with QEMU and buildx configured, it should work for single platform. Co-Authored-By: Claude Sonnet 4.5 --- .semaphore/semaphore.yml | 34 ++++++++-------------------------- 1 file changed, 8 insertions(+), 26 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index c317ca2cdc..4cc7174cb6 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -185,7 +185,8 @@ blocks: commands: # Setup QEMU for s390x emulation - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - - docker buildx create --name s390x-builder --use || docker buildx use s390x-builder + - docker buildx create --name s390x-builder || true + - docker buildx use s390x-builder - docker buildx inspect --bootstrap - export OS_TAG="-ubi9" - export DOCKER_DEV_FULL_IMAGES=$DOCKER_DEV_REGISTRY${DOCKER_REPOS// /:$DOCKER_DEV_TAG$OS_TAG $DOCKER_DEV_REGISTRY}:$DOCKER_DEV_TAG$OS_TAG @@ -193,34 +194,15 @@ blocks: - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - ci-tools ci-update-version - # Build artifacts with Maven (skip Docker image build, use docker buildx instead) + # Configure buildx for Maven to use + - export DOCKER_BUILDKIT=1 + - export DOCKER_CLI_EXPERIMENTAL=enabled + # Let Maven build Docker images for S390X (it handles all build args automatically) + # Note: fabric8 docker-maven-plugin may not fully support buildx, so images might be AMD64 - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$S390X_ARCH - -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$S390X_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS + -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$S390X_ARCH -Ddocker.os_type=ubi9 $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store - # Extract project version from Maven - - export PROJECT_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) - # Manually create package directories (Maven skip-build doesn't create them) - - mkdir -p base/target/cp-base-new-${PROJECT_VERSION}-package/share/{doc,java/cp-base-new} - - mkdir -p base-java/target/cp-base-java-${PROJECT_VERSION}-package/share/{doc,java/cp-base-java} - - mkdir -p base-java-micro/target/cp-base-java-micro-${PROJECT_VERSION}-package/share/{doc,java/cp-base-java-micro} - - mkdir -p base-lite/target/cp-base-lite-${PROJECT_VERSION}-package/share/{doc,java/cp-base-lite} - # Create README files in doc directories (Docker COPY wildcards require at least one file) - - echo "Confluent Platform Base Image" > base/target/cp-base-new-${PROJECT_VERSION}-package/share/doc/README.txt - - echo "Confluent Platform Base Java Image" > base-java/target/cp-base-java-${PROJECT_VERSION}-package/share/doc/README.txt - - echo "Confluent Platform Base Java Micro Image" > base-java-micro/target/cp-base-java-micro-${PROJECT_VERSION}-package/share/doc/README.txt - - echo "Confluent Platform Base Lite Image" > base-lite/target/cp-base-lite-${PROJECT_VERSION}-package/share/doc/README.txt - # Copy built JARs to package structure - - cp utility-belt/target/*.jar base/target/cp-base-new-${PROJECT_VERSION}-package/share/java/cp-base-new/ 2>/dev/null || true - - cp utility-belt/target/*.jar base-java/target/cp-base-java-${PROJECT_VERSION}-package/share/java/cp-base-java/ 2>/dev/null || true - - cp utility-belt/target/*.jar base-java-micro/target/cp-base-java-micro-${PROJECT_VERSION}-package/share/java/cp-base-java-micro/ 2>/dev/null || true - - cp utility-belt/target/*.jar base-lite/target/cp-base-lite-${PROJECT_VERSION}-package/share/java/cp-base-lite/ 2>/dev/null || true - # Build Docker images with buildx for s390x and required build args - - docker buildx build --platform linux/s390x --build-arg PROJECT_VERSION=$PROJECT_VERSION --build-arg ARTIFACT_ID=cp-base-new --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg BUILD_NUMBER=$BUILD_NUMBER --load --cache-from type=registry,ref=${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:buildcache-s390x --cache-to type=inline -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base/Dockerfile.ubi9 base/ - - docker buildx build --platform linux/s390x --build-arg PROJECT_VERSION=$PROJECT_VERSION --build-arg ARTIFACT_ID=cp-base-java --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg BUILD_NUMBER=$BUILD_NUMBER --load --cache-from type=registry,ref=${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:buildcache-s390x --cache-to type=inline -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-java/Dockerfile.ubi9 base-java/ - - docker buildx build --platform linux/s390x --build-arg PROJECT_VERSION=$PROJECT_VERSION --build-arg ARTIFACT_ID=cp-base-java-micro --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg BUILD_NUMBER=$BUILD_NUMBER --load --cache-from type=registry,ref=${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:buildcache-s390x --cache-to type=inline -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-java-micro/Dockerfile.ubi9 base-java-micro/ - - docker buildx build --platform linux/s390x --build-arg PROJECT_VERSION=$PROJECT_VERSION --build-arg ARTIFACT_ID=cp-base-lite --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg BUILD_NUMBER=$BUILD_NUMBER --load --cache-from type=registry,ref=${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:buildcache-s390x --cache-to type=inline -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-lite/Dockerfile.ubi9 base-lite/ - - docker buildx build --platform linux/s390x --build-arg PROJECT_VERSION=$PROJECT_VERSION --build-arg ARTIFACT_ID=cp-jmxterm --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg BUILD_NUMBER=$BUILD_NUMBER --load --cache-from type=registry,ref=${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:buildcache-s390x --cache-to type=inline -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f jmxterm/Dockerfile.ubi9 jmxterm/ # Verify images are built for s390x architecture - for image in $S390X_DOCKER_DEV_FULL_IMAGES; do echo "Checking architecture of $image:" && docker inspect --format='{{.Architecture}}' $image; done - for image in $S390X_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done From 07b177da1d5ab0e5425a6add8414f9ac04d86769 Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Tue, 3 Feb 2026 14:25:58 -0800 Subject: [PATCH 38/45] fix: restore manual buildx with all Maven build args for s390x Spotify's dockerfile-maven-plugin does NOT support cross-platform builds. When Maven runs on AMD64 host, it always builds AMD64 images regardless of QEMU/buildx configuration. Correct approach for s390x: 1. Use Maven with -Ddocker.skip-build=true to build JAR artifacts only 2. Extract ALL build arg values from Maven properties using mvn help:evaluate 3. Manually run docker buildx with --platform linux/s390x and all extracted args Build args extracted and passed: - PROJECT_VERSION, GIT_COMMIT, BUILD_NUMBER - UBI9_MINIMAL_VERSION, UBI9_MICRO_VERSION, UBI9_VERSION - GOLANG_VERSION (required by base-lite and base-java) - CP_DOCKER_UTILS_VERSION, CONFLUENT_DOCKER_UTILS_VERSION This ensures actual s390x architecture images, not AMD64 images with s390x tag. Reverts: a01021347 which incorrectly tried to use Maven for s390x builds Co-Authored-By: Claude Sonnet 4.5 --- .semaphore/semaphore.yml | 94 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 86 insertions(+), 8 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 4cc7174cb6..5c36146e70 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -185,8 +185,7 @@ blocks: commands: # Setup QEMU for s390x emulation - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - - docker buildx create --name s390x-builder || true - - docker buildx use s390x-builder + - docker buildx create --name s390x-builder --use || docker buildx use s390x-builder - docker buildx inspect --bootstrap - export OS_TAG="-ubi9" - export DOCKER_DEV_FULL_IMAGES=$DOCKER_DEV_REGISTRY${DOCKER_REPOS// /:$DOCKER_DEV_TAG$OS_TAG $DOCKER_DEV_REGISTRY}:$DOCKER_DEV_TAG$OS_TAG @@ -194,15 +193,94 @@ blocks: - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - ci-tools ci-update-version - # Configure buildx for Maven to use - - export DOCKER_BUILDKIT=1 - - export DOCKER_CLI_EXPERIMENTAL=enabled - # Let Maven build Docker images for S390X (it handles all build args automatically) - # Note: fabric8 docker-maven-plugin may not fully support buildx, so images might be AMD64 + # Build artifacts with Maven (skip Docker image build, use docker buildx instead) - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$S390X_ARCH - -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$S390X_ARCH -Ddocker.os_type=ubi9 $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS + -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$S390X_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store + # Extract essential version properties from pom.xml for docker build args + - export PROJECT_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) + - export UBI9_MINIMAL_VERSION=$(mvn help:evaluate -Dexpression=ubi9-minimal.image.version -q -DforceStdout) + - export UBI9_MICRO_VERSION=$(mvn help:evaluate -Dexpression=ubi9-micro.image.version -q -DforceStdout) + - export UBI9_VERSION=$(mvn help:evaluate -Dexpression=ubi9.image.version -q -DforceStdout) + - export GOLANG_VERSION=$(mvn help:evaluate -Dexpression=golang.image.version -q -DforceStdout) + - export CP_DOCKER_UTILS_VERSION=$(mvn help:evaluate -Dexpression=git-repo.cp-docker-utils.tag -q -DforceStdout) + - export CONFLUENT_DOCKER_UTILS_VERSION=$(mvn help:evaluate -Dexpression=git-repo.confluent-docker-utils.tag -q -DforceStdout) + # Manually create package directories (Maven skip-build doesn't create them) + - mkdir -p base/target/cp-base-new-${PROJECT_VERSION}-package/share/{doc,java/cp-base-new} + - mkdir -p base-java/target/cp-base-java-${PROJECT_VERSION}-package/share/{doc,java/cp-base-java} + - mkdir -p base-java-micro/target/cp-base-java-micro-${PROJECT_VERSION}-package/share/{doc,java/cp-base-java-micro} + - mkdir -p base-lite/target/cp-base-lite-${PROJECT_VERSION}-package/share/{doc,java/cp-base-lite} + # Create README files in doc directories (Docker COPY wildcards require at least one file) + - echo "Confluent Platform Base Image" > base/target/cp-base-new-${PROJECT_VERSION}-package/share/doc/README.txt + - echo "Confluent Platform Base Java Image" > base-java/target/cp-base-java-${PROJECT_VERSION}-package/share/doc/README.txt + - echo "Confluent Platform Base Java Micro Image" > base-java-micro/target/cp-base-java-micro-${PROJECT_VERSION}-package/share/doc/README.txt + - echo "Confluent Platform Base Lite Image" > base-lite/target/cp-base-lite-${PROJECT_VERSION}-package/share/doc/README.txt + # Copy built JARs to package structure + - cp utility-belt/target/*.jar base/target/cp-base-new-${PROJECT_VERSION}-package/share/java/cp-base-new/ 2>/dev/null || true + - cp utility-belt/target/*.jar base-java/target/cp-base-java-${PROJECT_VERSION}-package/share/java/cp-base-java/ 2>/dev/null || true + - cp utility-belt/target/*.jar base-java-micro/target/cp-base-java-micro-${PROJECT_VERSION}-package/share/java/cp-base-java-micro/ 2>/dev/null || true + - cp utility-belt/target/*.jar base-lite/target/cp-base-lite-${PROJECT_VERSION}-package/share/java/cp-base-lite/ 2>/dev/null || true + # Build Docker images with docker buildx for s390x platform with all required build args + # Build cp-base-new (base image for others) + - | + docker buildx build --platform linux/s390x --load \ + -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \ + -f base/Dockerfile.ubi9 \ + --build-arg ARTIFACT_ID=cp-base-new \ + --build-arg PROJECT_VERSION=${PROJECT_VERSION} \ + --build-arg GIT_COMMIT=${GIT_COMMIT} \ + --build-arg BUILD_NUMBER=${BUILD_NUMBER} \ + --build-arg UBI_MINIMAL_VERSION=${UBI9_MINIMAL_VERSION} \ + --build-arg PYTHON_CONFLUENT_DOCKER_UTILS_VERSION=${CONFLUENT_DOCKER_UTILS_VERSION} \ + base/ + # Build cp-base-java + - | + docker buildx build --platform linux/s390x --load \ + -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \ + -f base-java/Dockerfile.ubi9 \ + --build-arg ARTIFACT_ID=cp-base-java \ + --build-arg PROJECT_VERSION=${PROJECT_VERSION} \ + --build-arg GIT_COMMIT=${GIT_COMMIT} \ + --build-arg BUILD_NUMBER=${BUILD_NUMBER} \ + --build-arg UBI_MINIMAL_VERSION=${UBI9_MINIMAL_VERSION} \ + --build-arg GOLANG_VERSION=${GOLANG_VERSION} \ + --build-arg CP_DOCKER_UTILS_VERSION=${CP_DOCKER_UTILS_VERSION} \ + base-java/ + # Build cp-base-java-micro + - | + docker buildx build --platform linux/s390x --load \ + -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \ + -f base-java-micro/Dockerfile.ubi9 \ + --build-arg ARTIFACT_ID=cp-base-java-micro \ + --build-arg PROJECT_VERSION=${PROJECT_VERSION} \ + --build-arg GIT_COMMIT=${GIT_COMMIT} \ + --build-arg BUILD_NUMBER=${BUILD_NUMBER} \ + --build-arg UBI_MICRO_VERSION=${UBI9_MICRO_VERSION} \ + --build-arg UBI9_VERSION=${UBI9_VERSION} \ + --build-arg GOLANG_VERSION=${GOLANG_VERSION} \ + --build-arg CP_DOCKER_UTILS_VERSION=${CP_DOCKER_UTILS_VERSION} \ + base-java-micro/ + # Build cp-base-lite + - | + docker buildx build --platform linux/s390x --load \ + -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \ + -f base-lite/Dockerfile.ubi9 \ + --build-arg ARTIFACT_ID=cp-base-lite \ + --build-arg PROJECT_VERSION=${PROJECT_VERSION} \ + --build-arg GIT_COMMIT=${GIT_COMMIT} \ + --build-arg BUILD_NUMBER=${BUILD_NUMBER} \ + --build-arg UBI_MINIMAL_VERSION=${UBI9_MINIMAL_VERSION} \ + --build-arg GOLANG_VERSION=${GOLANG_VERSION} \ + base-lite/ + # Build cp-jmxterm (depends on cp-base-new) + - | + docker buildx build --platform linux/s390x --load \ + -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \ + -f jmxterm/Dockerfile.ubi9 \ + --build-arg DOCKER_REGISTRY=${DOCKER_DEV_REGISTRY} \ + --build-arg DOCKER_TAG=${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \ + jmxterm/ # Verify images are built for s390x architecture - for image in $S390X_DOCKER_DEV_FULL_IMAGES; do echo "Checking architecture of $image:" && docker inspect --format='{{.Architecture}}' $image; done - for image in $S390X_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done From 0e0a790c18202609bbdd59897a31be1f5ba830fd Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Tue, 3 Feb 2026 14:51:30 -0800 Subject: [PATCH 39/45] fix: skip base-lite for s390x due to Go compiler segfault under QEMU Go compiler crashes with segmentation fault when cross-compiling for s390x under QEMU emulation: internal/asan: /usr/local/go/pkg/tool/linux_s390x/compile: signal: segmentation fault (core dumped) This is a known limitation of Go + QEMU for s390x architecture. Solutions considered: 1. Skip base-lite for s390x (current approach) 2. Use native s390x builder (requires infrastructure) 3. Pre-compile ub binary on native platform 4. Use older Go version (may not work) For now, skip base-lite on s390x. Other images (base-new, base-java, base-java-micro, jmxterm) build successfully. Co-Authored-By: Claude Sonnet 4.5 --- .semaphore/semaphore.yml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 5c36146e70..d845769043 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -261,18 +261,10 @@ blocks: --build-arg GOLANG_VERSION=${GOLANG_VERSION} \ --build-arg CP_DOCKER_UTILS_VERSION=${CP_DOCKER_UTILS_VERSION} \ base-java-micro/ - # Build cp-base-lite - - | - docker buildx build --platform linux/s390x --load \ - -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \ - -f base-lite/Dockerfile.ubi9 \ - --build-arg ARTIFACT_ID=cp-base-lite \ - --build-arg PROJECT_VERSION=${PROJECT_VERSION} \ - --build-arg GIT_COMMIT=${GIT_COMMIT} \ - --build-arg BUILD_NUMBER=${BUILD_NUMBER} \ - --build-arg UBI_MINIMAL_VERSION=${UBI9_MINIMAL_VERSION} \ - --build-arg GOLANG_VERSION=${GOLANG_VERSION} \ - base-lite/ + # Skip cp-base-lite for s390x - Go compiler crashes under QEMU emulation + # Error: internal/asan: compile: signal: segmentation fault (core dumped) + # TODO: Either use native s390x builder or pre-compile ub binary + - echo "Skipping cp-base-lite for s390x due to Go compiler issues under QEMU" # Build cp-jmxterm (depends on cp-base-new) - | docker buildx build --platform linux/s390x --load \ From fe603709ff4b6f7d809e721e57262a0988640a1f Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Tue, 3 Feb 2026 15:09:24 -0800 Subject: [PATCH 40/45] fix: enable Go cross-compilation for s390x instead of QEMU emulation Use tonistiigi/xx toolkit to cross-compile Go binaries natively instead of relying on QEMU emulation which causes segmentation faults. Changes: - base-java/Dockerfile.ubi9: Add xx toolkit for ub & package_dedupe - base-java-micro/Dockerfile.ubi9: Add xx toolkit for ub & package_dedupe - base-lite/Dockerfile.ubi9: Add xx toolkit for ub binary - .semaphore/semaphore.yml: Re-enable base-lite build for s390x This follows the approach from service-mesh PR #964: https://github.com/confluentinc/service-mesh/pull/964/changes Co-Authored-By: Claude Sonnet 4.5 --- .semaphore/semaphore.yml | 16 ++++++++++++---- base-java-micro/Dockerfile.ubi9 | 14 +++++++++++++- base-java/Dockerfile.ubi9 | 13 ++++++++++++- base-lite/Dockerfile.ubi9 | 12 +++++++++++- 4 files changed, 48 insertions(+), 7 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index d845769043..c580e33f1f 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -261,10 +261,18 @@ blocks: --build-arg GOLANG_VERSION=${GOLANG_VERSION} \ --build-arg CP_DOCKER_UTILS_VERSION=${CP_DOCKER_UTILS_VERSION} \ base-java-micro/ - # Skip cp-base-lite for s390x - Go compiler crashes under QEMU emulation - # Error: internal/asan: compile: signal: segmentation fault (core dumped) - # TODO: Either use native s390x builder or pre-compile ub binary - - echo "Skipping cp-base-lite for s390x due to Go compiler issues under QEMU" + # Build cp-base-lite (uses cross-compilation instead of QEMU emulation for Go binary) + - | + docker buildx build --platform linux/s390x --load \ + -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \ + -f base-lite/Dockerfile.ubi9 \ + --build-arg ARTIFACT_ID=cp-base-lite \ + --build-arg PROJECT_VERSION=${PROJECT_VERSION} \ + --build-arg GIT_COMMIT=${GIT_COMMIT} \ + --build-arg BUILD_NUMBER=${BUILD_NUMBER} \ + --build-arg UBI_MINIMAL_VERSION=${UBI9_MINIMAL_VERSION} \ + --build-arg GOLANG_VERSION=${GOLANG_VERSION} \ + base-lite/ # Build cp-jmxterm (depends on cp-base-new) - | docker buildx build --platform linux/s390x --load \ diff --git a/base-java-micro/Dockerfile.ubi9 b/base-java-micro/Dockerfile.ubi9 index 832d5fe84d..f37d2994be 100644 --- a/base-java-micro/Dockerfile.ubi9 +++ b/base-java-micro/Dockerfile.ubi9 @@ -1,3 +1,5 @@ +# syntax=docker/dockerfile:1 + ARG APP_UID=1000 ARG APP_GID=1000 @@ -5,11 +7,21 @@ ARG UBI_MICRO_VERSION ARG UBI9_VERSION ARG GOLANG_VERSION +# Helpers for cross-compilation using clang +FROM --platform=$BUILDPLATFORM tonistiigi/xx:1.5.0 AS xx + # --- Stage 1: Build Go Binaries --- -FROM docker.io/golang:${GOLANG_VERSION} AS build-ub-package-dedupe +FROM --platform=$BUILDPLATFORM docker.io/golang:${GOLANG_VERSION} AS build-ub-package-dedupe +COPY --from=xx / / +RUN apt-get update && apt-get install -y clang lld ENV GO_BIN="/go/bin" ARG CP_DOCKER_UTILS_VERSION +ARG TARGETPLATFORM + +RUN xx-apt-get install -y libc6-dev gcc g++ +# This wraps the go compiler to enable cross-compilation by default +RUN xx-go --wrap RUN CGO_ENABLED=0 go install -ldflags="-w -s" github.com/confluentinc/cp-docker-utils/cmd/ub@${CP_DOCKER_UTILS_VERSION} RUN CGO_ENABLED=0 go install -ldflags="-w -s" github.com/confluentinc/cp-docker-utils/cmd/package_dedupe@${CP_DOCKER_UTILS_VERSION} diff --git a/base-java/Dockerfile.ubi9 b/base-java/Dockerfile.ubi9 index 068beb0317..38c85a99a6 100644 --- a/base-java/Dockerfile.ubi9 +++ b/base-java/Dockerfile.ubi9 @@ -1,10 +1,21 @@ +# syntax=docker/dockerfile:1 + ARG GOLANG_VERSION ARG UBI_MINIMAL_VERSION +# Helpers for cross-compilation using clang +FROM --platform=$BUILDPLATFORM tonistiigi/xx:1.5.0 AS xx -FROM docker.io/golang:${GOLANG_VERSION} AS build-ub-package-dedupe +FROM --platform=$BUILDPLATFORM docker.io/golang:${GOLANG_VERSION} AS build-ub-package-dedupe +COPY --from=xx / / +RUN apt-get update && apt-get install -y clang lld ENV GO_BIN="/go/bin" ARG CP_DOCKER_UTILS_VERSION +ARG TARGETPLATFORM + +RUN xx-apt-get install -y libc6-dev gcc g++ +# This wraps the go compiler to enable cross-compilation by default +RUN xx-go --wrap RUN useradd --no-log-init --create-home --shell /bin/bash appuser diff --git a/base-lite/Dockerfile.ubi9 b/base-lite/Dockerfile.ubi9 index 8117900c3a..c585847ed5 100644 --- a/base-lite/Dockerfile.ubi9 +++ b/base-lite/Dockerfile.ubi9 @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1 # # Copyright 2017 Confluent Inc. # @@ -13,12 +14,21 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Helpers for cross-compilation using clang +FROM --platform=$BUILDPLATFORM tonistiigi/xx:1.5.0 AS xx + ARG UBI_MINIMAL_VERSION="latest" ARG GOLANG_VERSION ARG CURL_VERSION -FROM golang:${GOLANG_VERSION} AS build-ub +FROM --platform=$BUILDPLATFORM golang:${GOLANG_VERSION} AS build-ub +COPY --from=xx / / +RUN apt-get update && apt-get install -y clang lld WORKDIR /build +ARG TARGETPLATFORM +RUN xx-apt-get install -y libc6-dev gcc g++ +# This wraps the go compiler to enable cross-compilation by default +RUN xx-go --wrap RUN useradd --no-log-init --create-home --shell /bin/bash appuser COPY --chown=appuser:appuser ub/ ./ RUN CGO_ENABLED=0 go build -ldflags="-w -s" ./ub.go From dee01b0b60492e88171f468266c8eb249371fdcf Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Tue, 3 Feb 2026 15:22:07 -0800 Subject: [PATCH 41/45] fix: use go build instead of go install for cross-compilation When cross-compiling with xx toolkit, go install doesn't place binaries in the expected location. Use go build -o with explicit output paths. Changes: - Clone cp-docker-utils repo and build from source - Use go build -o /usr/local/bin/... for explicit binary placement - Update COPY commands to use /usr/local/bin instead of /go/bin Co-Authored-By: Claude Sonnet 4.5 --- base-java-micro/Dockerfile.ubi9 | 18 ++++++++++++------ base-java/Dockerfile.ubi9 | 17 +++++++++++------ 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/base-java-micro/Dockerfile.ubi9 b/base-java-micro/Dockerfile.ubi9 index f37d2994be..4b31b4e418 100644 --- a/base-java-micro/Dockerfile.ubi9 +++ b/base-java-micro/Dockerfile.ubi9 @@ -13,8 +13,8 @@ FROM --platform=$BUILDPLATFORM tonistiigi/xx:1.5.0 AS xx # --- Stage 1: Build Go Binaries --- FROM --platform=$BUILDPLATFORM docker.io/golang:${GOLANG_VERSION} AS build-ub-package-dedupe COPY --from=xx / / -RUN apt-get update && apt-get install -y clang lld -ENV GO_BIN="/go/bin" +RUN apt-get update && apt-get install -y clang lld git +WORKDIR /build ARG CP_DOCKER_UTILS_VERSION ARG TARGETPLATFORM @@ -23,8 +23,14 @@ RUN xx-apt-get install -y libc6-dev gcc g++ # This wraps the go compiler to enable cross-compilation by default RUN xx-go --wrap -RUN CGO_ENABLED=0 go install -ldflags="-w -s" github.com/confluentinc/cp-docker-utils/cmd/ub@${CP_DOCKER_UTILS_VERSION} -RUN CGO_ENABLED=0 go install -ldflags="-w -s" github.com/confluentinc/cp-docker-utils/cmd/package_dedupe@${CP_DOCKER_UTILS_VERSION} +# Clone and build cp-docker-utils binaries with cross-compilation +RUN git clone --depth 1 --branch ${CP_DOCKER_UTILS_VERSION} https://github.com/confluentinc/cp-docker-utils.git + +WORKDIR /build/cp-docker-utils/cmd/ub +RUN CGO_ENABLED=0 go build -ldflags="-w -s" -o /usr/local/bin/ub + +WORKDIR /build/cp-docker-utils/cmd/package_dedupe +RUN CGO_ENABLED=0 go build -ldflags="-w -s" -o /usr/local/bin/package_dedupe # --- Stage 2: Prepare the Micro Rootfs (using ubi9) --- FROM registry.access.redhat.com/ubi9:${UBI9_VERSION} AS jdk-builder @@ -101,8 +107,8 @@ RUN update-crypto-policies --set FIPS && \ COPY license.txt /licenses -COPY --from=build-ub-package-dedupe --chown=${APP_UID}:${APP_GID} /go/bin/package_dedupe /usr/bin/package_dedupe -COPY --from=build-ub-package-dedupe --chown=${APP_UID}:${APP_GID} /go/bin/ub /usr/bin/ub +COPY --from=build-ub-package-dedupe --chown=${APP_UID}:${APP_GID} /usr/local/bin/package_dedupe /usr/bin/package_dedupe +COPY --from=build-ub-package-dedupe --chown=${APP_UID}:${APP_GID} /usr/local/bin/ub /usr/bin/ub COPY --chown=${APP_UID}:${APP_GID} target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/ COPY --chown=${APP_UID}:${APP_GID} target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${ARTIFACT_ID}/ diff --git a/base-java/Dockerfile.ubi9 b/base-java/Dockerfile.ubi9 index 38c85a99a6..b947b363a2 100644 --- a/base-java/Dockerfile.ubi9 +++ b/base-java/Dockerfile.ubi9 @@ -8,8 +8,8 @@ FROM --platform=$BUILDPLATFORM tonistiigi/xx:1.5.0 AS xx FROM --platform=$BUILDPLATFORM docker.io/golang:${GOLANG_VERSION} AS build-ub-package-dedupe COPY --from=xx / / -RUN apt-get update && apt-get install -y clang lld -ENV GO_BIN="/go/bin" +RUN apt-get update && apt-get install -y clang lld git +WORKDIR /build ARG CP_DOCKER_UTILS_VERSION ARG TARGETPLATFORM @@ -19,13 +19,18 @@ RUN xx-go --wrap RUN useradd --no-log-init --create-home --shell /bin/bash appuser +# Clone and build cp-docker-utils binaries with cross-compilation +RUN git clone --depth 1 --branch ${CP_DOCKER_UTILS_VERSION} https://github.com/confluentinc/cp-docker-utils.git + # CGO_ENABLED=0 flag should be removed for FedRAMP compliance builds. # For more details, see https://go.dev/doc/security/fips140 -RUN CGO_ENABLED=0 go install -ldflags="-w -s" github.com/confluentinc/cp-docker-utils/cmd/ub@${CP_DOCKER_UTILS_VERSION} +WORKDIR /build/cp-docker-utils/cmd/ub +RUN CGO_ENABLED=0 go build -ldflags="-w -s" -o /usr/local/bin/ub # CGO_ENABLED=0 flag should be removed for FedRAMP compliance builds. # For more details, see https://go.dev/doc/security/fips140 -RUN CGO_ENABLED=0 go install -ldflags="-w -s" github.com/confluentinc/cp-docker-utils/cmd/package_dedupe@${CP_DOCKER_UTILS_VERSION} +WORKDIR /build/cp-docker-utils/cmd/package_dedupe +RUN CGO_ENABLED=0 go build -ldflags="-w -s" -o /usr/local/bin/package_dedupe FROM registry.access.redhat.com/ubi9-minimal:${UBI_MINIMAL_VERSION} AS REFRESH @@ -83,8 +88,8 @@ RUN echo "installing temurin-21-jre:${TEMURIN_JDK_VERSION}" \ RUN update-crypto-policies --set FIPS COPY license.txt /licenses -COPY --from=build-ub-package-dedupe /go/bin/package_dedupe /usr/bin/package_dedupe -COPY --from=build-ub-package-dedupe /go/bin/ub /usr/bin/ub +COPY --from=build-ub-package-dedupe /usr/local/bin/package_dedupe /usr/bin/package_dedupe +COPY --from=build-ub-package-dedupe /usr/local/bin/ub /usr/bin/ub COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/ COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${ARTIFACT_ID}/ From 48d56e20cc7818aaa7ccb219ecd9a8f94e4e79fb Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Tue, 3 Feb 2026 15:27:07 -0800 Subject: [PATCH 42/45] refactor: separate s390x Dockerfiles for cross-compilation AMD/ARM use Maven's Spotify plugin for native builds (doesn't support BuildKit syntax). S390X needs cross-compilation with xx toolkit. Changes: - Restored original Dockerfile.ubi9 for AMD/ARM (native builds via Maven) - Created Dockerfile.ubi9.s390x with cross-compilation for s390x - Updated semaphore.yml to use .s390x Dockerfiles for s390x builds This keeps cross-compilation complexity isolated to s390x only. Co-Authored-By: Claude Sonnet 4.5 --- .semaphore/semaphore.yml | 12 +-- base-java-micro/Dockerfile.ubi9 | 30 ++---- base-java-micro/Dockerfile.ubi9.s390x | 126 ++++++++++++++++++++++++++ base-java/Dockerfile.ubi9 | 28 ++---- base-java/Dockerfile.ubi9.s390x | 107 ++++++++++++++++++++++ base-lite/Dockerfile.ubi9 | 12 +-- base-lite/Dockerfile.ubi9.s390x | 112 +++++++++++++++++++++++ 7 files changed, 364 insertions(+), 63 deletions(-) create mode 100644 base-java-micro/Dockerfile.ubi9.s390x create mode 100644 base-java/Dockerfile.ubi9.s390x create mode 100644 base-lite/Dockerfile.ubi9.s390x diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index c580e33f1f..21901e6a4f 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -234,11 +234,11 @@ blocks: --build-arg UBI_MINIMAL_VERSION=${UBI9_MINIMAL_VERSION} \ --build-arg PYTHON_CONFLUENT_DOCKER_UTILS_VERSION=${CONFLUENT_DOCKER_UTILS_VERSION} \ base/ - # Build cp-base-java + # Build cp-base-java (uses s390x-specific Dockerfile with cross-compilation) - | docker buildx build --platform linux/s390x --load \ -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \ - -f base-java/Dockerfile.ubi9 \ + -f base-java/Dockerfile.ubi9.s390x \ --build-arg ARTIFACT_ID=cp-base-java \ --build-arg PROJECT_VERSION=${PROJECT_VERSION} \ --build-arg GIT_COMMIT=${GIT_COMMIT} \ @@ -247,11 +247,11 @@ blocks: --build-arg GOLANG_VERSION=${GOLANG_VERSION} \ --build-arg CP_DOCKER_UTILS_VERSION=${CP_DOCKER_UTILS_VERSION} \ base-java/ - # Build cp-base-java-micro + # Build cp-base-java-micro (uses s390x-specific Dockerfile with cross-compilation) - | docker buildx build --platform linux/s390x --load \ -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \ - -f base-java-micro/Dockerfile.ubi9 \ + -f base-java-micro/Dockerfile.ubi9.s390x \ --build-arg ARTIFACT_ID=cp-base-java-micro \ --build-arg PROJECT_VERSION=${PROJECT_VERSION} \ --build-arg GIT_COMMIT=${GIT_COMMIT} \ @@ -261,11 +261,11 @@ blocks: --build-arg GOLANG_VERSION=${GOLANG_VERSION} \ --build-arg CP_DOCKER_UTILS_VERSION=${CP_DOCKER_UTILS_VERSION} \ base-java-micro/ - # Build cp-base-lite (uses cross-compilation instead of QEMU emulation for Go binary) + # Build cp-base-lite (uses s390x-specific Dockerfile with cross-compilation) - | docker buildx build --platform linux/s390x --load \ -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \ - -f base-lite/Dockerfile.ubi9 \ + -f base-lite/Dockerfile.ubi9.s390x \ --build-arg ARTIFACT_ID=cp-base-lite \ --build-arg PROJECT_VERSION=${PROJECT_VERSION} \ --build-arg GIT_COMMIT=${GIT_COMMIT} \ diff --git a/base-java-micro/Dockerfile.ubi9 b/base-java-micro/Dockerfile.ubi9 index 4b31b4e418..832d5fe84d 100644 --- a/base-java-micro/Dockerfile.ubi9 +++ b/base-java-micro/Dockerfile.ubi9 @@ -1,5 +1,3 @@ -# syntax=docker/dockerfile:1 - ARG APP_UID=1000 ARG APP_GID=1000 @@ -7,30 +5,14 @@ ARG UBI_MICRO_VERSION ARG UBI9_VERSION ARG GOLANG_VERSION -# Helpers for cross-compilation using clang -FROM --platform=$BUILDPLATFORM tonistiigi/xx:1.5.0 AS xx - # --- Stage 1: Build Go Binaries --- -FROM --platform=$BUILDPLATFORM docker.io/golang:${GOLANG_VERSION} AS build-ub-package-dedupe -COPY --from=xx / / -RUN apt-get update && apt-get install -y clang lld git -WORKDIR /build +FROM docker.io/golang:${GOLANG_VERSION} AS build-ub-package-dedupe +ENV GO_BIN="/go/bin" ARG CP_DOCKER_UTILS_VERSION -ARG TARGETPLATFORM - -RUN xx-apt-get install -y libc6-dev gcc g++ -# This wraps the go compiler to enable cross-compilation by default -RUN xx-go --wrap - -# Clone and build cp-docker-utils binaries with cross-compilation -RUN git clone --depth 1 --branch ${CP_DOCKER_UTILS_VERSION} https://github.com/confluentinc/cp-docker-utils.git - -WORKDIR /build/cp-docker-utils/cmd/ub -RUN CGO_ENABLED=0 go build -ldflags="-w -s" -o /usr/local/bin/ub -WORKDIR /build/cp-docker-utils/cmd/package_dedupe -RUN CGO_ENABLED=0 go build -ldflags="-w -s" -o /usr/local/bin/package_dedupe +RUN CGO_ENABLED=0 go install -ldflags="-w -s" github.com/confluentinc/cp-docker-utils/cmd/ub@${CP_DOCKER_UTILS_VERSION} +RUN CGO_ENABLED=0 go install -ldflags="-w -s" github.com/confluentinc/cp-docker-utils/cmd/package_dedupe@${CP_DOCKER_UTILS_VERSION} # --- Stage 2: Prepare the Micro Rootfs (using ubi9) --- FROM registry.access.redhat.com/ubi9:${UBI9_VERSION} AS jdk-builder @@ -107,8 +89,8 @@ RUN update-crypto-policies --set FIPS && \ COPY license.txt /licenses -COPY --from=build-ub-package-dedupe --chown=${APP_UID}:${APP_GID} /usr/local/bin/package_dedupe /usr/bin/package_dedupe -COPY --from=build-ub-package-dedupe --chown=${APP_UID}:${APP_GID} /usr/local/bin/ub /usr/bin/ub +COPY --from=build-ub-package-dedupe --chown=${APP_UID}:${APP_GID} /go/bin/package_dedupe /usr/bin/package_dedupe +COPY --from=build-ub-package-dedupe --chown=${APP_UID}:${APP_GID} /go/bin/ub /usr/bin/ub COPY --chown=${APP_UID}:${APP_GID} target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/ COPY --chown=${APP_UID}:${APP_GID} target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${ARTIFACT_ID}/ diff --git a/base-java-micro/Dockerfile.ubi9.s390x b/base-java-micro/Dockerfile.ubi9.s390x new file mode 100644 index 0000000000..4b31b4e418 --- /dev/null +++ b/base-java-micro/Dockerfile.ubi9.s390x @@ -0,0 +1,126 @@ +# syntax=docker/dockerfile:1 + +ARG APP_UID=1000 +ARG APP_GID=1000 + +ARG UBI_MICRO_VERSION +ARG UBI9_VERSION +ARG GOLANG_VERSION + +# Helpers for cross-compilation using clang +FROM --platform=$BUILDPLATFORM tonistiigi/xx:1.5.0 AS xx + +# --- Stage 1: Build Go Binaries --- +FROM --platform=$BUILDPLATFORM docker.io/golang:${GOLANG_VERSION} AS build-ub-package-dedupe +COPY --from=xx / / +RUN apt-get update && apt-get install -y clang lld git +WORKDIR /build + +ARG CP_DOCKER_UTILS_VERSION +ARG TARGETPLATFORM + +RUN xx-apt-get install -y libc6-dev gcc g++ +# This wraps the go compiler to enable cross-compilation by default +RUN xx-go --wrap + +# Clone and build cp-docker-utils binaries with cross-compilation +RUN git clone --depth 1 --branch ${CP_DOCKER_UTILS_VERSION} https://github.com/confluentinc/cp-docker-utils.git + +WORKDIR /build/cp-docker-utils/cmd/ub +RUN CGO_ENABLED=0 go build -ldflags="-w -s" -o /usr/local/bin/ub + +WORKDIR /build/cp-docker-utils/cmd/package_dedupe +RUN CGO_ENABLED=0 go build -ldflags="-w -s" -o /usr/local/bin/package_dedupe + +# --- Stage 2: Prepare the Micro Rootfs (using ubi9) --- +FROM registry.access.redhat.com/ubi9:${UBI9_VERSION} AS jdk-builder +ARG APP_UID +ARG APP_GID +ARG TEMURIN_JDK_VERSION +ARG PROCPS_VERSION +ARG CRYPTO_POLICIES_SCRIPTS_VERSION +ARG FINDUTILS_VERSION +ARG HOSTNAME_VERSION +ARG SHADOW_UTILS_VERSION + +RUN printf "[temurin-jre] \n\ +name=temurin-jre \n\ +baseurl=https://adoptium.jfrog.io/artifactory/rpm/rhel/\$releasever/\$basearch \n\ +enabled=1 \n\ +gpgcheck=1 \n\ +gpgkey=https://adoptium.jfrog.io/artifactory/api/gpg/key/public \n\ +" > /etc/yum.repos.d/adoptium.repo + +RUN mkdir -p /microdir + +RUN echo "Installing temurin-21-jre:${TEMURIN_JDK_VERSION}" \ + && dnf install --installroot=/microdir --releasever=9 --setopt=install_weak_deps=False --nodocs -y \ + temurin-21-jre${TEMURIN_JDK_VERSION} \ + procps-ng${PROCPS_VERSION} \ + crypto-policies-scripts${CRYPTO_POLICIES_SCRIPTS_VERSION} \ + findutils${FINDUTILS_VERSION} \ + hostname${HOSTNAME_VERSION} \ + shadow-utils${SHADOW_UTILS_VERSION} \ + && dnf --installroot=/microdir clean all \ + && rm -rf /microdir/var/cache/* /microdir/var/log/dnf* /microdir/var/log/yum.* \ + && rm /etc/yum.repos.d/adoptium.repo # Remove temurin-jdk repo to reduce intermittent build failures + +# Create the user/group with EXPLICIT IDs inside the micro rootfs +RUN chroot /microdir groupadd -g ${APP_GID} appuser && \ + chroot /microdir useradd -u ${APP_UID} -g ${APP_GID} --no-log-init --create-home --shell /bin/bash appuser + +# --- Stage 3: Final Image (ubi9-micro) --- +FROM registry.access.redhat.com/ubi9-micro:${UBI_MICRO_VERSION} AS REFRESH + +# Re-declare ARGs to bring them into this scope +ARG APP_UID +ARG APP_GID +ARG PROJECT_VERSION +ARG ARTIFACT_ID +ARG GIT_COMMIT +ARG BUILD_NUMBER=-1 + +# EXPORT these as ENV so downstream images can use them via ${APP_UID} +ENV APP_UID=${APP_UID} +ENV APP_GID=${APP_GID} + +LABEL io.confluent.docker.git.repo="confluentinc/common-docker" \ + io.confluent.docker.git.id=$GIT_COMMIT \ + io.confluent.docker.build.number=$BUILD_NUMBER \ + maintainer="tools@confluent.io" \ + vendor="Confluent" \ + version=$GIT_COMMIT \ + release=$PROJECT_VERSION \ + name=$ARTIFACT_ID \ + summary="Common base image for new Confluent ultra-lightweight Docker images based on ubi9-micro." \ + description="Common base image for Confluent ultra-lightweight Docker images based on ubi9-micro." \ + io.confluent.docker=true + +ENV LANG="C.UTF-8" +ENV USE_LOG4J_2="True" + +COPY --from=jdk-builder /microdir/ / + +RUN update-crypto-policies --set FIPS && \ + mkdir -p /etc/confluent/docker /usr/logs /licenses && \ + chown ${APP_UID}:${APP_GID} -R /etc/confluent/ /usr/logs + +COPY license.txt /licenses + +COPY --from=build-ub-package-dedupe --chown=${APP_UID}:${APP_GID} /usr/local/bin/package_dedupe /usr/bin/package_dedupe +COPY --from=build-ub-package-dedupe --chown=${APP_UID}:${APP_GID} /usr/local/bin/ub /usr/bin/ub + +COPY --chown=${APP_UID}:${APP_GID} target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/ +COPY --chown=${APP_UID}:${APP_GID} target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${ARTIFACT_ID}/ +COPY --chown=${APP_UID}:${APP_GID} include/etc/confluent/docker /etc/confluent/docker +COPY --chown=${APP_UID}:${APP_GID} include/etc/cp-base-java-micro /etc/cp-base-java-micro + +# Some components have hardcoded paths to /usr/share/java/cp-base-new, so to keep backward compatibility a symlink is created +RUN ln -s /usr/share/java/${ARTIFACT_ID} /usr/share/java/cp-base-new + +# Disable setuid/setgid bits for security +RUN find / -perm /6000 -type f -exec chmod a-s {} \; 2>/dev/null || true + +# Switch to the numeric UID for runtime security +USER ${APP_UID} +WORKDIR /home/appuser diff --git a/base-java/Dockerfile.ubi9 b/base-java/Dockerfile.ubi9 index b947b363a2..068beb0317 100644 --- a/base-java/Dockerfile.ubi9 +++ b/base-java/Dockerfile.ubi9 @@ -1,36 +1,20 @@ -# syntax=docker/dockerfile:1 - ARG GOLANG_VERSION ARG UBI_MINIMAL_VERSION -# Helpers for cross-compilation using clang -FROM --platform=$BUILDPLATFORM tonistiigi/xx:1.5.0 AS xx -FROM --platform=$BUILDPLATFORM docker.io/golang:${GOLANG_VERSION} AS build-ub-package-dedupe -COPY --from=xx / / -RUN apt-get update && apt-get install -y clang lld git -WORKDIR /build +FROM docker.io/golang:${GOLANG_VERSION} AS build-ub-package-dedupe +ENV GO_BIN="/go/bin" ARG CP_DOCKER_UTILS_VERSION -ARG TARGETPLATFORM - -RUN xx-apt-get install -y libc6-dev gcc g++ -# This wraps the go compiler to enable cross-compilation by default -RUN xx-go --wrap RUN useradd --no-log-init --create-home --shell /bin/bash appuser -# Clone and build cp-docker-utils binaries with cross-compilation -RUN git clone --depth 1 --branch ${CP_DOCKER_UTILS_VERSION} https://github.com/confluentinc/cp-docker-utils.git - # CGO_ENABLED=0 flag should be removed for FedRAMP compliance builds. # For more details, see https://go.dev/doc/security/fips140 -WORKDIR /build/cp-docker-utils/cmd/ub -RUN CGO_ENABLED=0 go build -ldflags="-w -s" -o /usr/local/bin/ub +RUN CGO_ENABLED=0 go install -ldflags="-w -s" github.com/confluentinc/cp-docker-utils/cmd/ub@${CP_DOCKER_UTILS_VERSION} # CGO_ENABLED=0 flag should be removed for FedRAMP compliance builds. # For more details, see https://go.dev/doc/security/fips140 -WORKDIR /build/cp-docker-utils/cmd/package_dedupe -RUN CGO_ENABLED=0 go build -ldflags="-w -s" -o /usr/local/bin/package_dedupe +RUN CGO_ENABLED=0 go install -ldflags="-w -s" github.com/confluentinc/cp-docker-utils/cmd/package_dedupe@${CP_DOCKER_UTILS_VERSION} FROM registry.access.redhat.com/ubi9-minimal:${UBI_MINIMAL_VERSION} AS REFRESH @@ -88,8 +72,8 @@ RUN echo "installing temurin-21-jre:${TEMURIN_JDK_VERSION}" \ RUN update-crypto-policies --set FIPS COPY license.txt /licenses -COPY --from=build-ub-package-dedupe /usr/local/bin/package_dedupe /usr/bin/package_dedupe -COPY --from=build-ub-package-dedupe /usr/local/bin/ub /usr/bin/ub +COPY --from=build-ub-package-dedupe /go/bin/package_dedupe /usr/bin/package_dedupe +COPY --from=build-ub-package-dedupe /go/bin/ub /usr/bin/ub COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/ COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${ARTIFACT_ID}/ diff --git a/base-java/Dockerfile.ubi9.s390x b/base-java/Dockerfile.ubi9.s390x new file mode 100644 index 0000000000..b947b363a2 --- /dev/null +++ b/base-java/Dockerfile.ubi9.s390x @@ -0,0 +1,107 @@ +# syntax=docker/dockerfile:1 + +ARG GOLANG_VERSION +ARG UBI_MINIMAL_VERSION + +# Helpers for cross-compilation using clang +FROM --platform=$BUILDPLATFORM tonistiigi/xx:1.5.0 AS xx + +FROM --platform=$BUILDPLATFORM docker.io/golang:${GOLANG_VERSION} AS build-ub-package-dedupe +COPY --from=xx / / +RUN apt-get update && apt-get install -y clang lld git +WORKDIR /build +ARG CP_DOCKER_UTILS_VERSION +ARG TARGETPLATFORM + +RUN xx-apt-get install -y libc6-dev gcc g++ +# This wraps the go compiler to enable cross-compilation by default +RUN xx-go --wrap + +RUN useradd --no-log-init --create-home --shell /bin/bash appuser + +# Clone and build cp-docker-utils binaries with cross-compilation +RUN git clone --depth 1 --branch ${CP_DOCKER_UTILS_VERSION} https://github.com/confluentinc/cp-docker-utils.git + +# CGO_ENABLED=0 flag should be removed for FedRAMP compliance builds. +# For more details, see https://go.dev/doc/security/fips140 +WORKDIR /build/cp-docker-utils/cmd/ub +RUN CGO_ENABLED=0 go build -ldflags="-w -s" -o /usr/local/bin/ub + +# CGO_ENABLED=0 flag should be removed for FedRAMP compliance builds. +# For more details, see https://go.dev/doc/security/fips140 +WORKDIR /build/cp-docker-utils/cmd/package_dedupe +RUN CGO_ENABLED=0 go build -ldflags="-w -s" -o /usr/local/bin/package_dedupe + + +FROM registry.access.redhat.com/ubi9-minimal:${UBI_MINIMAL_VERSION} AS REFRESH +ARG PROJECT_VERSION +ARG ARTIFACT_ID +ARG TEMURIN_JDK_VERSION +ARG CRYPTO_POLICIES_SCRIPTS_VERSION +ARG FINDUTILS_VERSION +ARG HOSTNAME_VERSION + +# Remember where we came from +LABEL io.confluent.docker.git.repo="confluentinc/common-docker" + +ARG GIT_COMMIT +LABEL io.confluent.docker.git.id=$GIT_COMMIT + +ARG BUILD_NUMBER=-1 +LABEL io.confluent.docker.build.number=$BUILD_NUMBER + +LABEL maintainer="tools@confluent.io" +LABEL vendor="Confluent" +LABEL version=$GIT_COMMIT +LABEL release=$PROJECT_VERSION +LABEL name=$ARTIFACT_ID +LABEL summary="Common base image for new Confluent lightweight Docker images." +LABEL description="Common base image for Confluent lightweight Docker images." +LABEL io.confluent.docker=true +# This affects how strings in Java class files are interpreted. We want UTF-8 and this is the only locale in the +# base image that supports it +ENV LANG="C.UTF-8" +ENV USE_LOG4J_2="True" + +RUN printf "[temurin-jre] \n\ +name=temurin-jre \n\ +baseurl=https://adoptium.jfrog.io/artifactory/rpm/rhel/\$releasever/\$basearch \n\ +enabled=1 \n\ +gpgcheck=1 \n\ +gpgkey=https://adoptium.jfrog.io/artifactory/api/gpg/key/public \n\ +" > /etc/yum.repos.d/adoptium.repo + +RUN echo "installing temurin-21-jre:${TEMURIN_JDK_VERSION}" \ + && microdnf install -y temurin-21-jre${TEMURIN_JDK_VERSION} \ + && microdnf install -y procps-ng${PROCPS_VERSION} \ + && microdnf install -y crypto-policies-scripts${CRYPTO_POLICIES_SCRIPTS_VERSION} \ + && microdnf install -y findutils${FINDUTILS_VERSION} \ + && microdnf install -y hostname${HOSTNAME_VERSION} \ + && microdnf clean all \ + && useradd --no-log-init --create-home --shell /bin/bash appuser \ + && mkdir -p /etc/confluent/docker /usr/logs \ + && chown appuser:appuser -R /etc/confluent/ /usr/logs \ + && mkdir /licenses \ + && rm /etc/yum.repos.d/adoptium.repo # Remove temurin-jdk repo to reduce intermittent build failures + +# enable FIPS in docker image, this will only work if underlying OS has FIPS enabled as well else is a NO OP. +RUN update-crypto-policies --set FIPS + +COPY license.txt /licenses +COPY --from=build-ub-package-dedupe /usr/local/bin/package_dedupe /usr/bin/package_dedupe +COPY --from=build-ub-package-dedupe /usr/local/bin/ub /usr/bin/ub + +COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/ +COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${ARTIFACT_ID}/ +COPY --chown=appuser:appuser include/etc/confluent/docker /etc/confluent/docker +COPY --chown=appuser:appuser include/etc/cp-base-java /etc/cp-base-java + +# Some components have hardcoded paths to /usr/share/java/cp-base-new, so to keep backward compatibility a symlink is created +RUN ln -s /usr/share/java/${ARTIFACT_ID} /usr/share/java/cp-base-new + +# Disable setuid/setgid bits +RUN find / -perm /6000 -type f -exec chmod a-s {} \; || true +RUN find / -perm /6000 -type f -exec chmod g-s {} \; || true + +USER appuser +WORKDIR /home/appuser diff --git a/base-lite/Dockerfile.ubi9 b/base-lite/Dockerfile.ubi9 index c585847ed5..8117900c3a 100644 --- a/base-lite/Dockerfile.ubi9 +++ b/base-lite/Dockerfile.ubi9 @@ -1,4 +1,3 @@ -# syntax=docker/dockerfile:1 # # Copyright 2017 Confluent Inc. # @@ -14,21 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Helpers for cross-compilation using clang -FROM --platform=$BUILDPLATFORM tonistiigi/xx:1.5.0 AS xx - ARG UBI_MINIMAL_VERSION="latest" ARG GOLANG_VERSION ARG CURL_VERSION -FROM --platform=$BUILDPLATFORM golang:${GOLANG_VERSION} AS build-ub -COPY --from=xx / / -RUN apt-get update && apt-get install -y clang lld +FROM golang:${GOLANG_VERSION} AS build-ub WORKDIR /build -ARG TARGETPLATFORM -RUN xx-apt-get install -y libc6-dev gcc g++ -# This wraps the go compiler to enable cross-compilation by default -RUN xx-go --wrap RUN useradd --no-log-init --create-home --shell /bin/bash appuser COPY --chown=appuser:appuser ub/ ./ RUN CGO_ENABLED=0 go build -ldflags="-w -s" ./ub.go diff --git a/base-lite/Dockerfile.ubi9.s390x b/base-lite/Dockerfile.ubi9.s390x new file mode 100644 index 0000000000..c585847ed5 --- /dev/null +++ b/base-lite/Dockerfile.ubi9.s390x @@ -0,0 +1,112 @@ +# syntax=docker/dockerfile:1 +# +# Copyright 2017 Confluent Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Helpers for cross-compilation using clang +FROM --platform=$BUILDPLATFORM tonistiigi/xx:1.5.0 AS xx + +ARG UBI_MINIMAL_VERSION="latest" +ARG GOLANG_VERSION +ARG CURL_VERSION + +FROM --platform=$BUILDPLATFORM golang:${GOLANG_VERSION} AS build-ub +COPY --from=xx / / +RUN apt-get update && apt-get install -y clang lld +WORKDIR /build +ARG TARGETPLATFORM +RUN xx-apt-get install -y libc6-dev gcc g++ +# This wraps the go compiler to enable cross-compilation by default +RUN xx-go --wrap +RUN useradd --no-log-init --create-home --shell /bin/bash appuser +COPY --chown=appuser:appuser ub/ ./ +RUN CGO_ENABLED=0 go build -ldflags="-w -s" ./ub.go +USER appuser +RUN go test ./... + +FROM registry.access.redhat.com/ubi9/ubi-minimal:${UBI_MINIMAL_VERSION} + +ARG PROJECT_VERSION +ARG ARTIFACT_ID +ARG CURL_VERSION + +# Remember where we came from +LABEL io.confluent.docker.git.repo="confluentinc/common-docker" + +ARG GIT_COMMIT +LABEL io.confluent.docker.git.id=$GIT_COMMIT + +ARG BUILD_NUMBER=-1 +LABEL io.confluent.docker.build.number=$BUILD_NUMBER + +LABEL maintainer="tools@confluent.io" +LABEL vendor="Confluent" +LABEL version=$GIT_COMMIT +LABEL release=$PROJECT_VERSION +LABEL name=$ARTIFACT_ID +LABEL summary="Common base image for new Confluent lightweight Docker images." +LABEL description="Common base image for Confluent lightweight Docker images." +LABEL io.confluent.docker=true + +# This affects how strings in Java class files are interpreted. We want UTF-8 and this is the only locale in the +# base image that supports it +ENV LANG="C.UTF-8" +ENV USE_LOG4J_2="True" + +# Temurin JDK version +ARG TEMURIN_JDK_VERSION="" + +ENV UB_CLASSPATH=/usr/share/java/cp-base-lite/* + +RUN printf "[temurin-jdk] \n\ +name=temurin-jdk \n\ +baseurl=https://adoptium.jfrog.io/artifactory/rpm/rhel/\$releasever/\$basearch \n\ +enabled=1 \n\ +gpgcheck=1 \n\ +gpgkey=https://adoptium.jfrog.io/artifactory/api/gpg/key/public \n\ +" > /etc/yum.repos.d/adoptium.repo + +RUN microdnf --nodocs -y install yum \ + && yum --nodocs update -y \ + && yum --nodocs install -y --setopt=install_weak_deps=False \ + "temurin-21-jre${TEMURIN_JDK_VERSION}" \ + && microdnf clean all \ + && yum clean all \ + && rm -rf /tmp/* \ + && mkdir -p /etc/confluent/docker /usr/logs \ + && useradd --no-log-init --create-home --shell /bin/bash appuser \ + && chown appuser:appuser -R /etc/confluent/ /usr/logs \ + && rm /etc/yum.repos.d/adoptium.repo # Remove temurin-jdk repo to reduce intermittent build failures + +# This is a step that will cause the build to fail of the package manager detects a package update is availible and isn't installed. +# The ARG SKIP_SECURITY_UPDATE_CHECK is an "escape" hatch if you want to by-pass this check and build the container anyways, which +# is not advisable in terms of security posture. If set to false (which triggers a shell exit(1) if the check fails from the left +# hand of ||) this check will fail. If true (which triggers a right-hand || shell exit(0)), then this check will pass even if a +# security update is availible. We skip checks from TemurinJDK repos because Confluent pins those upstream versions for various reasons +# such as identified bugs in TemurinJDK's software. +ARG SKIP_SECURITY_UPDATE_CHECK="false" +RUN yum check-update || "${SKIP_SECURITY_UPDATE_CHECK}" + +COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/ +COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${ARTIFACT_ID}/ + +COPY --chown=appuser:appuser include/etc/confluent/docker /etc/confluent/docker +COPY --chown=appuser:appuser include/etc/cp-base-lite /etc/cp-base-lite +COPY --from=build-ub /build/ub /usr/bin + +RUN mkdir /licenses +COPY license.txt /licenses + +USER appuser +WORKDIR /home/appuser From ac2038feac2f64a8a886869ca30bf7263fd863ab Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Tue, 3 Feb 2026 15:41:09 -0800 Subject: [PATCH 43/45] fix: move ARG declarations before FROM in base-lite s390x Dockerfile ARG values must be declared before FROM statements that use them. This fixes the 'UndefinedArgInFrom' build error. Co-Authored-By: Claude Sonnet 4.5 --- base-lite/Dockerfile.ubi9.s390x | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/base-lite/Dockerfile.ubi9.s390x b/base-lite/Dockerfile.ubi9.s390x index c585847ed5..b98982f635 100644 --- a/base-lite/Dockerfile.ubi9.s390x +++ b/base-lite/Dockerfile.ubi9.s390x @@ -14,13 +14,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Helpers for cross-compilation using clang -FROM --platform=$BUILDPLATFORM tonistiigi/xx:1.5.0 AS xx - ARG UBI_MINIMAL_VERSION="latest" ARG GOLANG_VERSION ARG CURL_VERSION +# Helpers for cross-compilation using clang +FROM --platform=$BUILDPLATFORM tonistiigi/xx:1.5.0 AS xx + FROM --platform=$BUILDPLATFORM golang:${GOLANG_VERSION} AS build-ub COPY --from=xx / / RUN apt-get update && apt-get install -y clang lld From b3bd6448406e4e4d8bb4ea6346d642299428caa5 Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Tue, 3 Feb 2026 16:03:49 -0800 Subject: [PATCH 44/45] fix: push cp-base-new before building jmxterm for s390x jmxterm depends on cp-base-new as its base image and pulls from ECR. We need to push cp-base-new immediately after building it, before jmxterm tries to use it. Co-Authored-By: Claude Sonnet 4.5 --- .semaphore/semaphore.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 21901e6a4f..7a43fcb64c 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -234,6 +234,8 @@ blocks: --build-arg UBI_MINIMAL_VERSION=${UBI9_MINIMAL_VERSION} \ --build-arg PYTHON_CONFLUENT_DOCKER_UTILS_VERSION=${CONFLUENT_DOCKER_UTILS_VERSION} \ base/ + # Push cp-base-new immediately (jmxterm depends on it being in registry) + - docker push ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} # Build cp-base-java (uses s390x-specific Dockerfile with cross-compilation) - | docker buildx build --platform linux/s390x --load \ From e34f9a6aa7cc1fbc1edec8b27d7dc4098229121a Mon Sep 17 00:00:00 2001 From: Lan Yao Date: Tue, 3 Feb 2026 17:16:56 -0800 Subject: [PATCH 45/45] revert: remove cp_dockerfile_build.yml changes (not used for PR builds) --- .semaphore/cp_dockerfile_build.yml | 238 +---------------------------- 1 file changed, 7 insertions(+), 231 deletions(-) diff --git a/.semaphore/cp_dockerfile_build.yml b/.semaphore/cp_dockerfile_build.yml index f69957e07c..40c15a834e 100644 --- a/.semaphore/cp_dockerfile_build.yml +++ b/.semaphore/cp_dockerfile_build.yml @@ -106,7 +106,6 @@ global_job_config: - export DOCKER_DEV_TAG="dev-$BRANCH_TAG-$BUILD_NUMBER" - export AMD_ARCH=.amd64 - export ARM_ARCH=.arm64 - - export S390X_ARCH=.s390x blocks: - name: Validation dependencies: [] @@ -132,17 +131,10 @@ blocks: - ci-tools ci-update-version - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - # Build artifacts with Maven (skip Docker image build, use docker CLI instead) - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH - -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi9 -Ddockerfile.skip=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS + -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi9 $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store - # Build Docker images directly - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base/Dockerfile.ubi9 base/ - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base-java/Dockerfile.ubi9 base-java/ - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base-java-micro/Dockerfile.ubi9 base-java-micro/ - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f base-lite/Dockerfile.ubi9 base-lite/ - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${AMD_ARCH} -f jmxterm/Dockerfile.ubi9 jmxterm/ - >- for dev_image in $AMD_DOCKER_DEV_FULL_IMAGES; do @@ -323,17 +315,10 @@ blocks: - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - ci-tools ci-update-version - # Build artifacts with Maven (skip Docker image build, use docker CLI instead) - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH - -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi9 -Ddockerfile.skip=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS + -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi9 $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - . cache-maven store - # Build Docker images directly - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base/Dockerfile.ubi9 base/ - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base-java/Dockerfile.ubi9 base-java/ - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base-java-micro/Dockerfile.ubi9 base-java-micro/ - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f base-lite/Dockerfile.ubi9 base-lite/ - - docker build -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${ARM_ARCH} -f jmxterm/Dockerfile.ubi9 jmxterm/ - for image in $ARM_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done epilogue: always: @@ -506,217 +491,9 @@ blocks: - export LATEST_PUSH_TAG=$LATEST_TAG$OS_TAG$ARM_ARCH - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$LATEST_PUSH_TAG - docker push $PROD_IMAGE_NAME:$LATEST_PUSH_TAG - - name: Build & Test S390X - dependencies: ["Validation"] - run: - # don't run the tests on non-functional changes... - when: "change_in('/', {exclude: ['/.deployed-versions/', '.github/', 'service.yml', 'README.md'], default_branch: 'master'})" - task: - agent: - machine: - type: s1-prod-ubuntu24-04-amd64-1 - jobs: - - name: Build & Test ubi9 - commands: - # Setup QEMU for s390x emulation - - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - - docker buildx create --name s390x-builder --use || docker buildx use s390x-builder - - docker buildx inspect --bootstrap - - export OS_TAG="-ubi9" - - export DOCKER_DEV_FULL_IMAGES=$DOCKER_DEV_REGISTRY${DOCKER_REPOS// /:$DOCKER_DEV_TAG$OS_TAG $DOCKER_DEV_REGISTRY}:$DOCKER_DEV_TAG$OS_TAG - - export S390X_DOCKER_DEV_FULL_IMAGES=${DOCKER_DEV_FULL_IMAGES// /$S390X_ARCH }$S390X_ARCH - - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - - ci-tools ci-update-version - # Build artifacts with Maven (skip Docker image build, use docker buildx instead) - - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY - -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$S390X_ARCH - -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$S390X_ARCH -Ddocker.os_type=ubi9 -Ddockerfile.skip=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS - - . cache-maven store - # Build Docker images with buildx for s390x platform - - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base/Dockerfile.ubi9 base/ - - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-java/Dockerfile.ubi9 base-java/ - - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-java-micro/Dockerfile.ubi9 base-java-micro/ - - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f base-lite/Dockerfile.ubi9 base-lite/ - - docker buildx build --platform linux/s390x --load -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} -f jmxterm/Dockerfile.ubi9 jmxterm/ - # Verify images are built for s390x architecture - - for image in $S390X_DOCKER_DEV_FULL_IMAGES; do echo "Checking architecture of $image:" && docker inspect --format='{{.Architecture}}' $image; done - - for image in $S390X_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done - epilogue: - always: - commands: - - . publish-test-results - - artifact push workflow target/test-results - - artifact push workflow target --destination target-S390X - - name: Deploy S390X confluentinc/cp-base-java - dependencies: ["Build & Test S390X"] - run: - when: "branch = 'master' or branch =~ '^[0-9]+\\.[0-9]+\\.x$' or branch =~ '^[0-9]+\\.[0-9]+\\.[0-9]+(-cp[0-9]+)?(-rc[0-9]+)?$'" - task: - agent: - machine: - type: s1-prod-ubuntu24-04-amd64-1 - jobs: - - name: Deploy S390X confluentinc/cp-base-java ubi9 - env_vars: - - name: DOCKER_IMAGE - value: confluentinc/cp-base-java - commands: - - export OS_TAG="-ubi9" - - export PROD_IMAGE_NAME=${DOCKER_PROD_REGISTRY}confluentinc/cp-base-java - - export GIT_COMMIT_TAG=$GIT_COMMIT$OS_TAG$S390X_ARCH - - export BRANCH_BUILD_TAG=$BRANCH_TAG-$BUILD_NUMBER$OS_TAG$S390X_ARCH - - export DEV_IMAGE_FULL=${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:$DOCKER_DEV_TAG$OS_TAG$S390X_ARCH - - docker pull $DEV_IMAGE_FULL - - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$GIT_COMMIT_TAG - - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG - - docker push $PROD_IMAGE_NAME:$GIT_COMMIT_TAG - - docker push $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG - - sign-images $PROD_IMAGE_NAME:$GIT_COMMIT_TAG - - sign-images $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG - - export PACKAGE_TAG=$BRANCH_TAG-$PACKAGING_BUILD_NUMBER$OS_TAG$S390X_ARCH - - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$PACKAGE_TAG - - docker push $PROD_IMAGE_NAME:$PACKAGE_TAG - - sign-images $PROD_IMAGE_NAME:$PACKAGE_TAG - - export LATEST_PUSH_TAG=$LATEST_TAG$OS_TAG$S390X_ARCH - - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$LATEST_PUSH_TAG - - docker push $PROD_IMAGE_NAME:$LATEST_PUSH_TAG - - name: Deploy S390X confluentinc/cp-base-java-micro - dependencies: ["Build & Test S390X"] - run: - when: "branch = 'master' or branch =~ '^[0-9]+\\.[0-9]+\\.x$' or branch =~ '^[0-9]+\\.[0-9]+\\.[0-9]+(-cp[0-9]+)?(-rc[0-9]+)?$'" - task: - agent: - machine: - type: s1-prod-ubuntu24-04-amd64-1 - jobs: - - name: Deploy S390X confluentinc/cp-base-java-micro ubi9 - env_vars: - - name: DOCKER_IMAGE - value: confluentinc/cp-base-java-micro - commands: - - export OS_TAG="-ubi9" - - export PROD_IMAGE_NAME=${DOCKER_PROD_REGISTRY}confluentinc/cp-base-java-micro - - export GIT_COMMIT_TAG=$GIT_COMMIT$OS_TAG$S390X_ARCH - - export BRANCH_BUILD_TAG=$BRANCH_TAG-$BUILD_NUMBER$OS_TAG$S390X_ARCH - - export DEV_IMAGE_FULL=${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:$DOCKER_DEV_TAG$OS_TAG$S390X_ARCH - - docker pull $DEV_IMAGE_FULL - - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$GIT_COMMIT_TAG - - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG - - docker push $PROD_IMAGE_NAME:$GIT_COMMIT_TAG - - docker push $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG - - sign-images $PROD_IMAGE_NAME:$GIT_COMMIT_TAG - - sign-images $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG - - export PACKAGE_TAG=$BRANCH_TAG-$PACKAGING_BUILD_NUMBER$OS_TAG$S390X_ARCH - - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$PACKAGE_TAG - - docker push $PROD_IMAGE_NAME:$PACKAGE_TAG - - sign-images $PROD_IMAGE_NAME:$PACKAGE_TAG - - export LATEST_PUSH_TAG=$LATEST_TAG$OS_TAG$S390X_ARCH - - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$LATEST_PUSH_TAG - - docker push $PROD_IMAGE_NAME:$LATEST_PUSH_TAG - - name: Deploy S390X confluentinc/cp-base-new - dependencies: ["Build & Test S390X"] - run: - when: "branch = 'master' or branch =~ '^[0-9]+\\.[0-9]+\\.x$' or branch =~ '^[0-9]+\\.[0-9]+\\.[0-9]+(-cp[0-9]+)?(-rc[0-9]+)?$'" - task: - agent: - machine: - type: s1-prod-ubuntu24-04-amd64-1 - jobs: - - name: Deploy S390X confluentinc/cp-base-new ubi9 - env_vars: - - name: DOCKER_IMAGE - value: confluentinc/cp-base-new - commands: - - export OS_TAG="-ubi9" - - export PROD_IMAGE_NAME=${DOCKER_PROD_REGISTRY}confluentinc/cp-base-new - - export GIT_COMMIT_TAG=$GIT_COMMIT$OS_TAG$S390X_ARCH - - export BRANCH_BUILD_TAG=$BRANCH_TAG-$BUILD_NUMBER$OS_TAG$S390X_ARCH - - export DEV_IMAGE_FULL=${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:$DOCKER_DEV_TAG$OS_TAG$S390X_ARCH - - docker pull $DEV_IMAGE_FULL - - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$GIT_COMMIT_TAG - - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG - - docker push $PROD_IMAGE_NAME:$GIT_COMMIT_TAG - - docker push $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG - - sign-images $PROD_IMAGE_NAME:$GIT_COMMIT_TAG - - sign-images $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG - - export PACKAGE_TAG=$BRANCH_TAG-$PACKAGING_BUILD_NUMBER$OS_TAG$S390X_ARCH - - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$PACKAGE_TAG - - docker push $PROD_IMAGE_NAME:$PACKAGE_TAG - - sign-images $PROD_IMAGE_NAME:$PACKAGE_TAG - - export LATEST_PUSH_TAG=$LATEST_TAG$OS_TAG$S390X_ARCH - - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$LATEST_PUSH_TAG - - docker push $PROD_IMAGE_NAME:$LATEST_PUSH_TAG - - name: Deploy S390X confluentinc/cp-base-lite - dependencies: ["Build & Test S390X"] - run: - when: "branch = 'master' or branch =~ '^[0-9]+\\.[0-9]+\\.x$' or branch =~ '^[0-9]+\\.[0-9]+\\.[0-9]+(-cp[0-9]+)?(-rc[0-9]+)?$'" - task: - agent: - machine: - type: s1-prod-ubuntu24-04-amd64-1 - jobs: - - name: Deploy S390X confluentinc/cp-base-lite ubi9 - env_vars: - - name: DOCKER_IMAGE - value: confluentinc/cp-base-lite - commands: - - export OS_TAG="-ubi9" - - export PROD_IMAGE_NAME=${DOCKER_PROD_REGISTRY}confluentinc/cp-base-lite - - export GIT_COMMIT_TAG=$GIT_COMMIT$OS_TAG$S390X_ARCH - - export BRANCH_BUILD_TAG=$BRANCH_TAG-$BUILD_NUMBER$OS_TAG$S390X_ARCH - - export DEV_IMAGE_FULL=${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:$DOCKER_DEV_TAG$OS_TAG$S390X_ARCH - - docker pull $DEV_IMAGE_FULL - - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$GIT_COMMIT_TAG - - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG - - docker push $PROD_IMAGE_NAME:$GIT_COMMIT_TAG - - docker push $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG - - sign-images $PROD_IMAGE_NAME:$GIT_COMMIT_TAG - - sign-images $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG - - export PACKAGE_TAG=$BRANCH_TAG-$PACKAGING_BUILD_NUMBER$OS_TAG$S390X_ARCH - - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$PACKAGE_TAG - - docker push $PROD_IMAGE_NAME:$PACKAGE_TAG - - sign-images $PROD_IMAGE_NAME:$PACKAGE_TAG - - export LATEST_PUSH_TAG=$LATEST_TAG$OS_TAG$S390X_ARCH - - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$LATEST_PUSH_TAG - - docker push $PROD_IMAGE_NAME:$LATEST_PUSH_TAG - - name: Deploy S390X confluentinc/cp-jmxterm - dependencies: ["Build & Test S390X"] - run: - when: "branch = 'master' or branch =~ '^[0-9]+\\.[0-9]+\\.x$' or branch =~ '^[0-9]+\\.[0-9]+\\.[0-9]+(-cp[0-9]+)?(-rc[0-9]+)?$'" - task: - agent: - machine: - type: s1-prod-ubuntu24-04-amd64-1 - jobs: - - name: Deploy S390X confluentinc/cp-jmxterm ubi9 - env_vars: - - name: DOCKER_IMAGE - value: confluentinc/cp-jmxterm - commands: - - export OS_TAG="-ubi9" - - export PROD_IMAGE_NAME=${DOCKER_PROD_REGISTRY}confluentinc/cp-jmxterm - - export GIT_COMMIT_TAG=$GIT_COMMIT$OS_TAG$S390X_ARCH - - export BRANCH_BUILD_TAG=$BRANCH_TAG-$BUILD_NUMBER$OS_TAG$S390X_ARCH - - export DEV_IMAGE_FULL=${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:$DOCKER_DEV_TAG$OS_TAG$S390X_ARCH - - docker pull $DEV_IMAGE_FULL - - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$GIT_COMMIT_TAG - - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG - - docker push $PROD_IMAGE_NAME:$GIT_COMMIT_TAG - - docker push $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG - - sign-images $PROD_IMAGE_NAME:$GIT_COMMIT_TAG - - sign-images $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG - - export PACKAGE_TAG=$BRANCH_TAG-$PACKAGING_BUILD_NUMBER$OS_TAG$S390X_ARCH - - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$PACKAGE_TAG - - docker push $PROD_IMAGE_NAME:$PACKAGE_TAG - - sign-images $PROD_IMAGE_NAME:$PACKAGE_TAG - - export LATEST_PUSH_TAG=$LATEST_TAG$OS_TAG$S390X_ARCH - - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$LATEST_PUSH_TAG - - docker push $PROD_IMAGE_NAME:$LATEST_PUSH_TAG - name: Create Manifest and Maven Deploy dependencies: ["Deploy AMD confluentinc/cp-base-java", "Deploy AMD confluentinc/cp-base-java-micro", "Deploy AMD confluentinc/cp-base-new", "Deploy AMD confluentinc/cp-base-lite", "Deploy AMD confluentinc/cp-jmxterm", - "Deploy ARM confluentinc/cp-base-java", "Deploy ARM confluentinc/cp-base-java-micro", "Deploy ARM confluentinc/cp-base-new", "Deploy ARM confluentinc/cp-base-lite", "Deploy ARM confluentinc/cp-jmxterm", - "Deploy S390X confluentinc/cp-base-java", "Deploy S390X confluentinc/cp-base-java-micro", "Deploy S390X confluentinc/cp-base-new", "Deploy S390X confluentinc/cp-base-lite", "Deploy S390X confluentinc/cp-jmxterm"] + "Deploy ARM confluentinc/cp-base-java", "Deploy ARM confluentinc/cp-base-java-micro", "Deploy ARM confluentinc/cp-base-new", "Deploy ARM confluentinc/cp-base-lite", "Deploy ARM confluentinc/cp-jmxterm"] run: when: "branch = 'master' or branch =~ '^[0-9]+\\.[0-9]+\\.x$' or branch =~ '^[0-9]+\\.[0-9]+\\.[0-9]+(-cp[0-9]+)?(-rc[0-9]+)?$'" task: @@ -736,20 +513,20 @@ blocks: do export OS_TAG="-ubi9" export GIT_TAG=$GIT_COMMIT$OS_TAG - docker manifest create $image:$GIT_TAG $image:$GIT_TAG$AMD_ARCH $image:$GIT_TAG$ARM_ARCH $image:$GIT_TAG$S390X_ARCH + docker manifest create $image:$GIT_TAG $image:$GIT_TAG$AMD_ARCH $image:$GIT_TAG$ARM_ARCH docker manifest push $image:$GIT_TAG docker pull $image:$GIT_TAG sign-images $image:$GIT_TAG export BRANCH_BUILD_TAG=$BRANCH_TAG-$BUILD_NUMBER$OS_TAG - docker manifest create $image:$BRANCH_BUILD_TAG $image:$BRANCH_BUILD_TAG$AMD_ARCH $image:$BRANCH_BUILD_TAG$ARM_ARCH $image:$BRANCH_BUILD_TAG$S390X_ARCH + docker manifest create $image:$BRANCH_BUILD_TAG $image:$BRANCH_BUILD_TAG$AMD_ARCH $image:$BRANCH_BUILD_TAG$ARM_ARCH docker manifest push $image:$BRANCH_BUILD_TAG docker pull $image:$BRANCH_BUILD_TAG sign-images $image:$BRANCH_BUILD_TAG export PACKAGE_TAG=$BRANCH_TAG-$PACKAGING_BUILD_NUMBER$OS_TAG - docker manifest create $image:$PACKAGE_TAG $image:$PACKAGE_TAG$AMD_ARCH $image:$PACKAGE_TAG$ARM_ARCH $image:$PACKAGE_TAG$S390X_ARCH + docker manifest create $image:$PACKAGE_TAG $image:$PACKAGE_TAG$AMD_ARCH $image:$PACKAGE_TAG$ARM_ARCH docker manifest push $image:$PACKAGE_TAG export LATEST_MANIFEST_TAG=$LATEST_TAG$OS_TAG - docker manifest create $image:$LATEST_MANIFEST_TAG $image:$LATEST_MANIFEST_TAG$AMD_ARCH $image:$LATEST_MANIFEST_TAG$ARM_ARCH $image:$LATEST_MANIFEST_TAG$S390X_ARCH + docker manifest create $image:$LATEST_MANIFEST_TAG $image:$LATEST_MANIFEST_TAG$AMD_ARCH $image:$LATEST_MANIFEST_TAG$ARM_ARCH docker manifest push $image:$LATEST_MANIFEST_TAG done after_pipeline: @@ -769,5 +546,4 @@ after_pipeline: - checkout - artifact pull workflow target-AMD - artifact pull workflow target-ARM - - artifact pull workflow target-S390X - emit-sonarqube-data --run_only_sonar_scan