test emulation

.semaphore/semaphore.yml

@@ -14,7 +14,7 @@ fail_fast:
     when: "true"
 
 execution_time_limit:
-  hours: 1
+  hours: 2
 
 queue:
   - when: "branch != 'master' and branch !~ '[0-9]+\\.[0-9]+\\.[0-9]+'"
@@ -99,6 +99,7 @@ global_job_config:
       - export DOCKER_DEV_TAG="dev-$BRANCH_TAG-$BUILD_NUMBER"
       - export AMD_ARCH=.amd64
       - export ARM_ARCH=.arm64
+      - export S390X_ARCH=.s390x
 blocks:
   - name: Validation
     dependencies: []
@@ -123,8 +124,10 @@ blocks:
             - ci-tools ci-update-version
             - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g")
             - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL"
-            - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY 
-              -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH 
+            # Build artifacts with Maven (skip Docker image build, use docker CLI instead)
+            # Let Maven build Docker images for AMD (it handles all build args automatically)
+            - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY
+              -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH
               -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi9 $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS
             - . cache-maven store
             - >-
@@ -156,17 +159,139 @@ blocks:
             - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g")
             - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL"
             - ci-tools ci-update-version
-            - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY 
-              -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH 
+            # Let Maven build Docker images for ARM (it handles all build args automatically)
+            - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY
+              -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH
               -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi9 $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS
             - . cache-maven store
-            - for image in $ARM_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done
       epilogue:
         always:
           commands:
             - . publish-test-results
             - artifact push workflow target/test-results
             - artifact push workflow target --destination target-ARM
+  - name: Build & Test S390X
+    dependencies: ["Validation"]
+    run:
+      when: "pull_request =~ '.*'"
+    execution_time_limit:
+      hours: 2
+    task:
+      agent:
+        machine:
+          type: s1-prod-ubuntu24-04-amd64-1
+      jobs:
+        - name: Build & Test ubi9
+          commands:
+            # Setup QEMU for s390x emulation
+            - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+            - docker buildx create --name s390x-builder --use || docker buildx use s390x-builder
+            - docker buildx inspect --bootstrap
+            - export OS_TAG="-ubi9"
+            - export DOCKER_DEV_FULL_IMAGES=$DOCKER_DEV_REGISTRY${DOCKER_REPOS// /:$DOCKER_DEV_TAG$OS_TAG $DOCKER_DEV_REGISTRY}:$DOCKER_DEV_TAG$OS_TAG
+            - export S390X_DOCKER_DEV_FULL_IMAGES=${DOCKER_DEV_FULL_IMAGES// /$S390X_ARCH }$S390X_ARCH
+            - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g")
+            - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL"
+            - ci-tools ci-update-version
+            # Build artifacts with Maven (skip Docker image build, use docker buildx instead)
+            - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY
+              -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$S390X_ARCH
+              -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$S390X_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS
+            - . cache-maven store
+            # Extract essential version properties from pom.xml for docker build args
+            - export PROJECT_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
+            - export UBI9_MINIMAL_VERSION=$(mvn help:evaluate -Dexpression=ubi9-minimal.image.version -q -DforceStdout)
+            - export UBI9_MICRO_VERSION=$(mvn help:evaluate -Dexpression=ubi9-micro.image.version -q -DforceStdout)
+            - export UBI9_VERSION=$(mvn help:evaluate -Dexpression=ubi9.image.version -q -DforceStdout)
+            - export GOLANG_VERSION=$(mvn help:evaluate -Dexpression=golang.image.version -q -DforceStdout)
+            - export CP_DOCKER_UTILS_VERSION=$(mvn help:evaluate -Dexpression=git-repo.cp-docker-utils.tag -q -DforceStdout)
+            - export CONFLUENT_DOCKER_UTILS_VERSION=$(mvn help:evaluate -Dexpression=git-repo.confluent-docker-utils.tag -q -DforceStdout)
+            # Manually create package directories (Maven skip-build doesn't create them)
+            - mkdir -p base/target/cp-base-new-${PROJECT_VERSION}-package/share/{doc,java/cp-base-new}
+            - mkdir -p base-java/target/cp-base-java-${PROJECT_VERSION}-package/share/{doc,java/cp-base-java}
+            - mkdir -p base-java-micro/target/cp-base-java-micro-${PROJECT_VERSION}-package/share/{doc,java/cp-base-java-micro}
+            - mkdir -p base-lite/target/cp-base-lite-${PROJECT_VERSION}-package/share/{doc,java/cp-base-lite}
+            # Create README files in doc directories (Docker COPY wildcards require at least one file)
+            - echo "Confluent Platform Base Image" > base/target/cp-base-new-${PROJECT_VERSION}-package/share/doc/README.txt
+            - echo "Confluent Platform Base Java Image" > base-java/target/cp-base-java-${PROJECT_VERSION}-package/share/doc/README.txt
+            - echo "Confluent Platform Base Java Micro Image" > base-java-micro/target/cp-base-java-micro-${PROJECT_VERSION}-package/share/doc/README.txt
+            - echo "Confluent Platform Base Lite Image" > base-lite/target/cp-base-lite-${PROJECT_VERSION}-package/share/doc/README.txt
+            # Copy built JARs to package structure
+            - cp utility-belt/target/*.jar base/target/cp-base-new-${PROJECT_VERSION}-package/share/java/cp-base-new/ 2>/dev/null || true
+            - cp utility-belt/target/*.jar base-java/target/cp-base-java-${PROJECT_VERSION}-package/share/java/cp-base-java/ 2>/dev/null || true
+            - cp utility-belt/target/*.jar base-java-micro/target/cp-base-java-micro-${PROJECT_VERSION}-package/share/java/cp-base-java-micro/ 2>/dev/null || true
+            - cp utility-belt/target/*.jar base-lite/target/cp-base-lite-${PROJECT_VERSION}-package/share/java/cp-base-lite/ 2>/dev/null || true
+            # Build Docker images with docker buildx for s390x platform with all required build args
+            # Build cp-base-new (base image for others)
+            - |
+              docker buildx build --platform linux/s390x --load \
+                -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \
+                -f base/Dockerfile.ubi9 \
+                --build-arg ARTIFACT_ID=cp-base-new \
+                --build-arg PROJECT_VERSION=${PROJECT_VERSION} \
+                --build-arg GIT_COMMIT=${GIT_COMMIT} \
+                --build-arg BUILD_NUMBER=${BUILD_NUMBER} \
+                --build-arg UBI_MINIMAL_VERSION=${UBI9_MINIMAL_VERSION} \
+                --build-arg PYTHON_CONFLUENT_DOCKER_UTILS_VERSION=${CONFLUENT_DOCKER_UTILS_VERSION} \
+                base/
+            # Push cp-base-new immediately (jmxterm depends on it being in registry)
+            - docker push ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH}
+            # Build cp-base-java (uses s390x-specific Dockerfile with cross-compilation)
+            - |
+              docker buildx build --platform linux/s390x --load \
+                -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \
+                -f base-java/Dockerfile.ubi9.s390x \
+                --build-arg ARTIFACT_ID=cp-base-java \
+                --build-arg PROJECT_VERSION=${PROJECT_VERSION} \
+                --build-arg GIT_COMMIT=${GIT_COMMIT} \
+                --build-arg BUILD_NUMBER=${BUILD_NUMBER} \
+                --build-arg UBI_MINIMAL_VERSION=${UBI9_MINIMAL_VERSION} \
+                --build-arg GOLANG_VERSION=${GOLANG_VERSION} \
+                --build-arg CP_DOCKER_UTILS_VERSION=${CP_DOCKER_UTILS_VERSION} \
+                base-java/
+            # Build cp-base-java-micro (uses s390x-specific Dockerfile with cross-compilation)
+            - |
+              docker buildx build --platform linux/s390x --load \
+                -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \
+                -f base-java-micro/Dockerfile.ubi9.s390x \
+                --build-arg ARTIFACT_ID=cp-base-java-micro \
+                --build-arg PROJECT_VERSION=${PROJECT_VERSION} \
+                --build-arg GIT_COMMIT=${GIT_COMMIT} \
+                --build-arg BUILD_NUMBER=${BUILD_NUMBER} \
+                --build-arg UBI_MICRO_VERSION=${UBI9_MICRO_VERSION} \
+                --build-arg UBI9_VERSION=${UBI9_VERSION} \
+                --build-arg GOLANG_VERSION=${GOLANG_VERSION} \
+                --build-arg CP_DOCKER_UTILS_VERSION=${CP_DOCKER_UTILS_VERSION} \
+                base-java-micro/
+            # Build cp-base-lite (uses s390x-specific Dockerfile with cross-compilation)
+            - |
+              docker buildx build --platform linux/s390x --load \
+                -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \
+                -f base-lite/Dockerfile.ubi9.s390x \
+                --build-arg ARTIFACT_ID=cp-base-lite \
+                --build-arg PROJECT_VERSION=${PROJECT_VERSION} \
+                --build-arg GIT_COMMIT=${GIT_COMMIT} \
+                --build-arg BUILD_NUMBER=${BUILD_NUMBER} \
+                --build-arg UBI_MINIMAL_VERSION=${UBI9_MINIMAL_VERSION} \
+                --build-arg GOLANG_VERSION=${GOLANG_VERSION} \
+                base-lite/
+            # Build cp-jmxterm (depends on cp-base-new)
+            - |
+              docker buildx build --platform linux/s390x --load \
+                -t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \
+                -f jmxterm/Dockerfile.ubi9 \
+                --build-arg DOCKER_REGISTRY=${DOCKER_DEV_REGISTRY} \
+                --build-arg DOCKER_TAG=${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \
+                jmxterm/
+            # Verify images are built for s390x architecture
+            - for image in $S390X_DOCKER_DEV_FULL_IMAGES; do echo "Checking architecture of $image:" && docker inspect --format='{{.Architecture}}' $image; done
+            - for image in $S390X_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done
+      epilogue:
+        always:
+          commands:
+            - . publish-test-results
+            - artifact push workflow target/test-results
+            - artifact push workflow target --destination target-S390X
 after_pipeline:
   task:
     agent:
@@ -184,4 +309,5 @@ after_pipeline:
           - checkout
           - artifact pull workflow target-AMD
           - artifact pull workflow target-ARM
+          - artifact pull workflow target-S390X
           - emit-sonarqube-data --run_only_sonar_scan

base-java-micro/Dockerfile.ubi9.s390x

@@ -0,0 +1,126 @@
+# syntax=docker/dockerfile:1
+
+ARG APP_UID=1000
+ARG APP_GID=1000
+
+ARG UBI_MICRO_VERSION
+ARG UBI9_VERSION
+ARG GOLANG_VERSION
+
+# Helpers for cross-compilation using clang
+FROM --platform=$BUILDPLATFORM tonistiigi/xx:1.5.0 AS xx
+
+# --- Stage 1: Build Go Binaries ---
+FROM --platform=$BUILDPLATFORM docker.io/golang:${GOLANG_VERSION} AS build-ub-package-dedupe
+COPY --from=xx / /
+RUN apt-get update && apt-get install -y clang lld git
+WORKDIR /build
+
+ARG CP_DOCKER_UTILS_VERSION
+ARG TARGETPLATFORM
+
+RUN xx-apt-get install -y libc6-dev gcc g++
+# This wraps the go compiler to enable cross-compilation by default
+RUN xx-go --wrap
+
+# Clone and build cp-docker-utils binaries with cross-compilation
+RUN git clone --depth 1 --branch ${CP_DOCKER_UTILS_VERSION} https://github.com/confluentinc/cp-docker-utils.git
+
+WORKDIR /build/cp-docker-utils/cmd/ub
+RUN CGO_ENABLED=0 go build -ldflags="-w -s" -o /usr/local/bin/ub
+
+WORKDIR /build/cp-docker-utils/cmd/package_dedupe
+RUN CGO_ENABLED=0 go build -ldflags="-w -s" -o /usr/local/bin/package_dedupe
+
+# --- Stage 2: Prepare the Micro Rootfs (using ubi9) ---
+FROM registry.access.redhat.com/ubi9:${UBI9_VERSION} AS jdk-builder
+ARG APP_UID
+ARG APP_GID
+ARG TEMURIN_JDK_VERSION
+ARG PROCPS_VERSION
+ARG CRYPTO_POLICIES_SCRIPTS_VERSION
+ARG FINDUTILS_VERSION
+ARG HOSTNAME_VERSION
+ARG SHADOW_UTILS_VERSION
+
+RUN printf "[temurin-jre] \n\
+name=temurin-jre \n\
+baseurl=https://adoptium.jfrog.io/artifactory/rpm/rhel/\$releasever/\$basearch \n\
+enabled=1 \n\
+gpgcheck=1 \n\
+gpgkey=https://adoptium.jfrog.io/artifactory/api/gpg/key/public \n\
+" > /etc/yum.repos.d/adoptium.repo
+
+RUN mkdir -p /microdir
+
+RUN echo "Installing temurin-21-jre:${TEMURIN_JDK_VERSION}" \
+    && dnf install --installroot=/microdir --releasever=9 --setopt=install_weak_deps=False --nodocs -y \
+       temurin-21-jre${TEMURIN_JDK_VERSION} \
+       procps-ng${PROCPS_VERSION} \
+       crypto-policies-scripts${CRYPTO_POLICIES_SCRIPTS_VERSION} \
+       findutils${FINDUTILS_VERSION} \
+       hostname${HOSTNAME_VERSION} \
+       shadow-utils${SHADOW_UTILS_VERSION} \
+    && dnf --installroot=/microdir clean all \
+    && rm -rf /microdir/var/cache/* /microdir/var/log/dnf* /microdir/var/log/yum.* \
+    && rm /etc/yum.repos.d/adoptium.repo # Remove temurin-jdk repo to reduce intermittent build failures
+
+# Create the user/group with EXPLICIT IDs inside the micro rootfs
+RUN chroot /microdir groupadd -g ${APP_GID} appuser && \
+    chroot /microdir useradd -u ${APP_UID} -g ${APP_GID} --no-log-init --create-home --shell /bin/bash appuser
+
+# --- Stage 3: Final Image (ubi9-micro) ---
+FROM registry.access.redhat.com/ubi9-micro:${UBI_MICRO_VERSION} AS REFRESH
+
+# Re-declare ARGs to bring them into this scope
+ARG APP_UID
+ARG APP_GID
+ARG PROJECT_VERSION
+ARG ARTIFACT_ID
+ARG GIT_COMMIT
+ARG BUILD_NUMBER=-1
+
+# EXPORT these as ENV so downstream images can use them via ${APP_UID}
+ENV APP_UID=${APP_UID}
+ENV APP_GID=${APP_GID}
+
+LABEL io.confluent.docker.git.repo="confluentinc/common-docker" \
+      io.confluent.docker.git.id=$GIT_COMMIT \
+      io.confluent.docker.build.number=$BUILD_NUMBER \
+      maintainer="tools@confluent.io" \
+      vendor="Confluent" \
+      version=$GIT_COMMIT \
+      release=$PROJECT_VERSION \
+      name=$ARTIFACT_ID \
+      summary="Common base image for new Confluent ultra-lightweight Docker images based on ubi9-micro." \
+      description="Common base image for Confluent ultra-lightweight Docker images based on ubi9-micro." \
+      io.confluent.docker=true
+
+ENV LANG="C.UTF-8"
+ENV USE_LOG4J_2="True"
+
+COPY --from=jdk-builder /microdir/ /
+
+RUN update-crypto-policies --set FIPS && \
+    mkdir -p /etc/confluent/docker /usr/logs /licenses && \
+    chown ${APP_UID}:${APP_GID} -R /etc/confluent/ /usr/logs
+
+COPY license.txt /licenses
+
+COPY --from=build-ub-package-dedupe --chown=${APP_UID}:${APP_GID} /usr/local/bin/package_dedupe /usr/bin/package_dedupe
+COPY --from=build-ub-package-dedupe --chown=${APP_UID}:${APP_GID} /usr/local/bin/ub /usr/bin/ub
+
+COPY --chown=${APP_UID}:${APP_GID} target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/
+COPY --chown=${APP_UID}:${APP_GID} target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${ARTIFACT_ID}/
+COPY --chown=${APP_UID}:${APP_GID} include/etc/confluent/docker /etc/confluent/docker
+COPY --chown=${APP_UID}:${APP_GID} include/etc/cp-base-java-micro /etc/cp-base-java-micro
+
+# Some components have hardcoded paths to /usr/share/java/cp-base-new, so to keep backward compatibility a symlink is created
+RUN ln -s /usr/share/java/${ARTIFACT_ID} /usr/share/java/cp-base-new
+
+# Disable setuid/setgid bits for security
+RUN find / -perm /6000 -type f -exec chmod a-s {} \; 2>/dev/null || true
+
+# Switch to the numeric UID for runtime security
+USER ${APP_UID}
+WORKDIR /home/appuser

base-java-micro/pom.xml

@@ -141,36 +141,6 @@
                     </buildArgs>
                 </configuration>
             </plugin>
-            <plugin>
-                <groupId>io.fabric8</groupId>
-                <artifactId>docker-maven-plugin</artifactId>
-                <version>0.43.4</version>
-                <configuration>
-                    <images>
-                        <image>
-                            <build>
-                                <args>
-                                    <APP_UID>${app.uid}</APP_UID>
-                                    <APP_GID>${app.gid}</APP_GID>
-                                    <UBI_MICRO_VERSION>${ubi9-micro.image.version}</UBI_MICRO_VERSION>
-                                    <UBI9_VERSION>${ubi9.image.version}</UBI9_VERSION>
-                                    <TEMURIN_JDK_VERSION>-${ubi9.temurin-21-jdk.version}</TEMURIN_JDK_VERSION>
-                                    <PROCPS_VERSION>-${ubi9.procps-ng.version}</PROCPS_VERSION>
-                                    <SKIP_SECURITY_UPDATE_CHECK>
-                                        ${docker.skip-security-update-check}
-                                    </SKIP_SECURITY_UPDATE_CHECK>
-                                    <GOLANG_VERSION>${golang.image.version}</GOLANG_VERSION>
-                                    <CRYPTO_POLICIES_SCRIPTS_VERSION>-${ubi9.crypto-policies-scripts.version}</CRYPTO_POLICIES_SCRIPTS_VERSION>
-                                    <FINDUTILS_VERSION>-${ubi9.findutils.version}</FINDUTILS_VERSION>
-                                    <HOSTNAME_VERSION>-${ubi9.hostname.version}</HOSTNAME_VERSION>
-                                    <SHADOW_UTILS_VERSION>-${ubi9.shadow-utils.version}</SHADOW_UTILS_VERSION>
-                                    <CP_DOCKER_UTILS_VERSION>${git-repo.cp-docker-utils.tag}</CP_DOCKER_UTILS_VERSION>
-                                </args>
-                            </build>
-                        </image>
-                    </images>
-                </configuration>
-            </plugin>
         </plugins>
     </build>
 </project>