Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
138 changes: 132 additions & 6 deletions .semaphore/semaphore.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ fail_fast:
when: "true"

execution_time_limit:
hours: 1
hours: 2

queue:
- when: "branch != 'master' and branch !~ '[0-9]+\\.[0-9]+\\.[0-9]+'"
Expand Down Expand Up @@ -99,6 +99,7 @@ global_job_config:
- export DOCKER_DEV_TAG="dev-$BRANCH_TAG-$BUILD_NUMBER"
- export AMD_ARCH=.amd64
- export ARM_ARCH=.arm64
- export S390X_ARCH=.s390x
blocks:
- name: Validation
dependencies: []
Expand All @@ -123,8 +124,10 @@ blocks:
- ci-tools ci-update-version
- export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g")
- export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL"
- mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY
-Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH
# Build artifacts with Maven (skip Docker image build, use docker CLI instead)
# Let Maven build Docker images for AMD (it handles all build args automatically)
- mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY
-Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH
-Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi9 $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS
- . cache-maven store
- >-
Expand Down Expand Up @@ -156,17 +159,139 @@ blocks:
- export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g")
- export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL"
- ci-tools ci-update-version
- mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY
-Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH
# Let Maven build Docker images for ARM (it handles all build args automatically)
- mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY
-Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH
-Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi9 $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS
- . cache-maven store
- for image in $ARM_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done
epilogue:
always:
commands:
- . publish-test-results
- artifact push workflow target/test-results
- artifact push workflow target --destination target-ARM
- name: Build & Test S390X
dependencies: ["Validation"]
run:
when: "pull_request =~ '.*'"
execution_time_limit:
hours: 2
task:
agent:
machine:
type: s1-prod-ubuntu24-04-amd64-1
jobs:
- name: Build & Test ubi9
commands:
# Setup QEMU for s390x emulation
- docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
- docker buildx create --name s390x-builder --use || docker buildx use s390x-builder
- docker buildx inspect --bootstrap
- export OS_TAG="-ubi9"
- export DOCKER_DEV_FULL_IMAGES=$DOCKER_DEV_REGISTRY${DOCKER_REPOS// /:$DOCKER_DEV_TAG$OS_TAG $DOCKER_DEV_REGISTRY}:$DOCKER_DEV_TAG$OS_TAG
- export S390X_DOCKER_DEV_FULL_IMAGES=${DOCKER_DEV_FULL_IMAGES// /$S390X_ARCH }$S390X_ARCH
- export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g")
- export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL"
- ci-tools ci-update-version
# Build artifacts with Maven (skip Docker image build, use docker buildx instead)
- mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY
-Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$S390X_ARCH
-Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$S390X_ARCH -Ddocker.os_type=ubi9 -Ddocker.skip-build=true -Ddocker.skip-test=true $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS
- . cache-maven store
# Extract essential version properties from pom.xml for docker build args
- export PROJECT_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
- export UBI9_MINIMAL_VERSION=$(mvn help:evaluate -Dexpression=ubi9-minimal.image.version -q -DforceStdout)
- export UBI9_MICRO_VERSION=$(mvn help:evaluate -Dexpression=ubi9-micro.image.version -q -DforceStdout)
- export UBI9_VERSION=$(mvn help:evaluate -Dexpression=ubi9.image.version -q -DforceStdout)
- export GOLANG_VERSION=$(mvn help:evaluate -Dexpression=golang.image.version -q -DforceStdout)
- export CP_DOCKER_UTILS_VERSION=$(mvn help:evaluate -Dexpression=git-repo.cp-docker-utils.tag -q -DforceStdout)
- export CONFLUENT_DOCKER_UTILS_VERSION=$(mvn help:evaluate -Dexpression=git-repo.confluent-docker-utils.tag -q -DforceStdout)
# Manually create package directories (Maven skip-build doesn't create them)
- mkdir -p base/target/cp-base-new-${PROJECT_VERSION}-package/share/{doc,java/cp-base-new}
- mkdir -p base-java/target/cp-base-java-${PROJECT_VERSION}-package/share/{doc,java/cp-base-java}
- mkdir -p base-java-micro/target/cp-base-java-micro-${PROJECT_VERSION}-package/share/{doc,java/cp-base-java-micro}
- mkdir -p base-lite/target/cp-base-lite-${PROJECT_VERSION}-package/share/{doc,java/cp-base-lite}
# Create README files in doc directories (Docker COPY wildcards require at least one file)
- echo "Confluent Platform Base Image" > base/target/cp-base-new-${PROJECT_VERSION}-package/share/doc/README.txt
- echo "Confluent Platform Base Java Image" > base-java/target/cp-base-java-${PROJECT_VERSION}-package/share/doc/README.txt
- echo "Confluent Platform Base Java Micro Image" > base-java-micro/target/cp-base-java-micro-${PROJECT_VERSION}-package/share/doc/README.txt
- echo "Confluent Platform Base Lite Image" > base-lite/target/cp-base-lite-${PROJECT_VERSION}-package/share/doc/README.txt
# Copy built JARs to package structure
- cp utility-belt/target/*.jar base/target/cp-base-new-${PROJECT_VERSION}-package/share/java/cp-base-new/ 2>/dev/null || true
- cp utility-belt/target/*.jar base-java/target/cp-base-java-${PROJECT_VERSION}-package/share/java/cp-base-java/ 2>/dev/null || true
- cp utility-belt/target/*.jar base-java-micro/target/cp-base-java-micro-${PROJECT_VERSION}-package/share/java/cp-base-java-micro/ 2>/dev/null || true
- cp utility-belt/target/*.jar base-lite/target/cp-base-lite-${PROJECT_VERSION}-package/share/java/cp-base-lite/ 2>/dev/null || true
# Build Docker images with docker buildx for s390x platform with all required build args
# Build cp-base-new (base image for others)
- |
docker buildx build --platform linux/s390x --load \
-t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \
-f base/Dockerfile.ubi9 \
--build-arg ARTIFACT_ID=cp-base-new \
--build-arg PROJECT_VERSION=${PROJECT_VERSION} \
--build-arg GIT_COMMIT=${GIT_COMMIT} \
--build-arg BUILD_NUMBER=${BUILD_NUMBER} \
--build-arg UBI_MINIMAL_VERSION=${UBI9_MINIMAL_VERSION} \
--build-arg PYTHON_CONFLUENT_DOCKER_UTILS_VERSION=${CONFLUENT_DOCKER_UTILS_VERSION} \
base/
# Push cp-base-new immediately (jmxterm depends on it being in registry)
- docker push ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH}
# Build cp-base-java (uses s390x-specific Dockerfile with cross-compilation)
- |
docker buildx build --platform linux/s390x --load \
-t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \
-f base-java/Dockerfile.ubi9.s390x \
--build-arg ARTIFACT_ID=cp-base-java \
--build-arg PROJECT_VERSION=${PROJECT_VERSION} \
--build-arg GIT_COMMIT=${GIT_COMMIT} \
--build-arg BUILD_NUMBER=${BUILD_NUMBER} \
--build-arg UBI_MINIMAL_VERSION=${UBI9_MINIMAL_VERSION} \
--build-arg GOLANG_VERSION=${GOLANG_VERSION} \
--build-arg CP_DOCKER_UTILS_VERSION=${CP_DOCKER_UTILS_VERSION} \
base-java/
# Build cp-base-java-micro (uses s390x-specific Dockerfile with cross-compilation)
- |
docker buildx build --platform linux/s390x --load \
-t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java-micro:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \
-f base-java-micro/Dockerfile.ubi9.s390x \
--build-arg ARTIFACT_ID=cp-base-java-micro \
--build-arg PROJECT_VERSION=${PROJECT_VERSION} \
--build-arg GIT_COMMIT=${GIT_COMMIT} \
--build-arg BUILD_NUMBER=${BUILD_NUMBER} \
--build-arg UBI_MICRO_VERSION=${UBI9_MICRO_VERSION} \
--build-arg UBI9_VERSION=${UBI9_VERSION} \
--build-arg GOLANG_VERSION=${GOLANG_VERSION} \
--build-arg CP_DOCKER_UTILS_VERSION=${CP_DOCKER_UTILS_VERSION} \
base-java-micro/
# Build cp-base-lite (uses s390x-specific Dockerfile with cross-compilation)
- |
docker buildx build --platform linux/s390x --load \
-t ${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \
-f base-lite/Dockerfile.ubi9.s390x \
--build-arg ARTIFACT_ID=cp-base-lite \
--build-arg PROJECT_VERSION=${PROJECT_VERSION} \
--build-arg GIT_COMMIT=${GIT_COMMIT} \
--build-arg BUILD_NUMBER=${BUILD_NUMBER} \
--build-arg UBI_MINIMAL_VERSION=${UBI9_MINIMAL_VERSION} \
--build-arg GOLANG_VERSION=${GOLANG_VERSION} \
base-lite/
# Build cp-jmxterm (depends on cp-base-new)
- |
docker buildx build --platform linux/s390x --load \
-t ${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \
-f jmxterm/Dockerfile.ubi9 \
--build-arg DOCKER_REGISTRY=${DOCKER_DEV_REGISTRY} \
--build-arg DOCKER_TAG=${DOCKER_DEV_TAG}${OS_TAG}${S390X_ARCH} \
jmxterm/
# Verify images are built for s390x architecture
- for image in $S390X_DOCKER_DEV_FULL_IMAGES; do echo "Checking architecture of $image:" && docker inspect --format='{{.Architecture}}' $image; done
- for image in $S390X_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done
epilogue:
always:
commands:
- . publish-test-results
- artifact push workflow target/test-results
- artifact push workflow target --destination target-S390X
after_pipeline:
task:
agent:
Expand All @@ -184,4 +309,5 @@ after_pipeline:
- checkout
- artifact pull workflow target-AMD
- artifact pull workflow target-ARM
- artifact pull workflow target-S390X
- emit-sonarqube-data --run_only_sonar_scan
126 changes: 126 additions & 0 deletions base-java-micro/Dockerfile.ubi9.s390x
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
# syntax=docker/dockerfile:1

ARG APP_UID=1000
ARG APP_GID=1000

ARG UBI_MICRO_VERSION
ARG UBI9_VERSION
ARG GOLANG_VERSION

# Helpers for cross-compilation using clang
FROM --platform=$BUILDPLATFORM tonistiigi/xx:1.5.0 AS xx

# --- Stage 1: Build Go Binaries ---
FROM --platform=$BUILDPLATFORM docker.io/golang:${GOLANG_VERSION} AS build-ub-package-dedupe
COPY --from=xx / /
RUN apt-get update && apt-get install -y clang lld git
WORKDIR /build

ARG CP_DOCKER_UTILS_VERSION
ARG TARGETPLATFORM

RUN xx-apt-get install -y libc6-dev gcc g++
# This wraps the go compiler to enable cross-compilation by default
RUN xx-go --wrap

# Clone and build cp-docker-utils binaries with cross-compilation
RUN git clone --depth 1 --branch ${CP_DOCKER_UTILS_VERSION} https://github.com/confluentinc/cp-docker-utils.git

WORKDIR /build/cp-docker-utils/cmd/ub
RUN CGO_ENABLED=0 go build -ldflags="-w -s" -o /usr/local/bin/ub

WORKDIR /build/cp-docker-utils/cmd/package_dedupe
RUN CGO_ENABLED=0 go build -ldflags="-w -s" -o /usr/local/bin/package_dedupe

# --- Stage 2: Prepare the Micro Rootfs (using ubi9) ---
FROM registry.access.redhat.com/ubi9:${UBI9_VERSION} AS jdk-builder
ARG APP_UID
ARG APP_GID
ARG TEMURIN_JDK_VERSION
ARG PROCPS_VERSION
ARG CRYPTO_POLICIES_SCRIPTS_VERSION
ARG FINDUTILS_VERSION
ARG HOSTNAME_VERSION
ARG SHADOW_UTILS_VERSION

RUN printf "[temurin-jre] \n\
name=temurin-jre \n\
baseurl=https://adoptium.jfrog.io/artifactory/rpm/rhel/\$releasever/\$basearch \n\
enabled=1 \n\
gpgcheck=1 \n\
gpgkey=https://adoptium.jfrog.io/artifactory/api/gpg/key/public \n\
" > /etc/yum.repos.d/adoptium.repo

RUN mkdir -p /microdir

RUN echo "Installing temurin-21-jre:${TEMURIN_JDK_VERSION}" \
&& dnf install --installroot=/microdir --releasever=9 --setopt=install_weak_deps=False --nodocs -y \
temurin-21-jre${TEMURIN_JDK_VERSION} \
procps-ng${PROCPS_VERSION} \
crypto-policies-scripts${CRYPTO_POLICIES_SCRIPTS_VERSION} \
findutils${FINDUTILS_VERSION} \
hostname${HOSTNAME_VERSION} \
shadow-utils${SHADOW_UTILS_VERSION} \
&& dnf --installroot=/microdir clean all \
&& rm -rf /microdir/var/cache/* /microdir/var/log/dnf* /microdir/var/log/yum.* \
&& rm /etc/yum.repos.d/adoptium.repo # Remove temurin-jdk repo to reduce intermittent build failures

# Create the user/group with EXPLICIT IDs inside the micro rootfs
RUN chroot /microdir groupadd -g ${APP_GID} appuser && \
chroot /microdir useradd -u ${APP_UID} -g ${APP_GID} --no-log-init --create-home --shell /bin/bash appuser

# --- Stage 3: Final Image (ubi9-micro) ---
FROM registry.access.redhat.com/ubi9-micro:${UBI_MICRO_VERSION} AS REFRESH

# Re-declare ARGs to bring them into this scope
ARG APP_UID
ARG APP_GID
ARG PROJECT_VERSION
ARG ARTIFACT_ID
ARG GIT_COMMIT
ARG BUILD_NUMBER=-1

# EXPORT these as ENV so downstream images can use them via ${APP_UID}
ENV APP_UID=${APP_UID}
ENV APP_GID=${APP_GID}

LABEL io.confluent.docker.git.repo="confluentinc/common-docker" \
io.confluent.docker.git.id=$GIT_COMMIT \
io.confluent.docker.build.number=$BUILD_NUMBER \
maintainer="tools@confluent.io" \
vendor="Confluent" \
version=$GIT_COMMIT \
release=$PROJECT_VERSION \
name=$ARTIFACT_ID \
summary="Common base image for new Confluent ultra-lightweight Docker images based on ubi9-micro." \
description="Common base image for Confluent ultra-lightweight Docker images based on ubi9-micro." \
io.confluent.docker=true

ENV LANG="C.UTF-8"
ENV USE_LOG4J_2="True"

COPY --from=jdk-builder /microdir/ /

RUN update-crypto-policies --set FIPS && \
mkdir -p /etc/confluent/docker /usr/logs /licenses && \
chown ${APP_UID}:${APP_GID} -R /etc/confluent/ /usr/logs

COPY license.txt /licenses

COPY --from=build-ub-package-dedupe --chown=${APP_UID}:${APP_GID} /usr/local/bin/package_dedupe /usr/bin/package_dedupe
COPY --from=build-ub-package-dedupe --chown=${APP_UID}:${APP_GID} /usr/local/bin/ub /usr/bin/ub

COPY --chown=${APP_UID}:${APP_GID} target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/
COPY --chown=${APP_UID}:${APP_GID} target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${ARTIFACT_ID}/
COPY --chown=${APP_UID}:${APP_GID} include/etc/confluent/docker /etc/confluent/docker
COPY --chown=${APP_UID}:${APP_GID} include/etc/cp-base-java-micro /etc/cp-base-java-micro

# Some components have hardcoded paths to /usr/share/java/cp-base-new, so to keep backward compatibility a symlink is created
RUN ln -s /usr/share/java/${ARTIFACT_ID} /usr/share/java/cp-base-new

# Disable setuid/setgid bits for security
RUN find / -perm /6000 -type f -exec chmod a-s {} \; 2>/dev/null || true

# Switch to the numeric UID for runtime security
USER ${APP_UID}
WORKDIR /home/appuser
30 changes: 0 additions & 30 deletions base-java-micro/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -141,36 +141,6 @@
</buildArgs>
</configuration>
</plugin>
<plugin>
<groupId>io.fabric8</groupId>
<artifactId>docker-maven-plugin</artifactId>
<version>0.43.4</version>
<configuration>
<images>
<image>
<build>
<args>
<APP_UID>${app.uid}</APP_UID>
<APP_GID>${app.gid}</APP_GID>
<UBI_MICRO_VERSION>${ubi9-micro.image.version}</UBI_MICRO_VERSION>
<UBI9_VERSION>${ubi9.image.version}</UBI9_VERSION>
<TEMURIN_JDK_VERSION>-${ubi9.temurin-21-jdk.version}</TEMURIN_JDK_VERSION>
<PROCPS_VERSION>-${ubi9.procps-ng.version}</PROCPS_VERSION>
<SKIP_SECURITY_UPDATE_CHECK>
${docker.skip-security-update-check}
</SKIP_SECURITY_UPDATE_CHECK>
<GOLANG_VERSION>${golang.image.version}</GOLANG_VERSION>
<CRYPTO_POLICIES_SCRIPTS_VERSION>-${ubi9.crypto-policies-scripts.version}</CRYPTO_POLICIES_SCRIPTS_VERSION>
<FINDUTILS_VERSION>-${ubi9.findutils.version}</FINDUTILS_VERSION>
<HOSTNAME_VERSION>-${ubi9.hostname.version}</HOSTNAME_VERSION>
<SHADOW_UTILS_VERSION>-${ubi9.shadow-utils.version}</SHADOW_UTILS_VERSION>
<CP_DOCKER_UTILS_VERSION>${git-repo.cp-docker-utils.tag}</CP_DOCKER_UTILS_VERSION>
</args>
</build>
</image>
</images>
</configuration>
</plugin>
</plugins>
</build>
</project>
Loading