From a434707c790a6c0baf8901627c9e99e31a6a89f3 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Mon, 23 Mar 2026 08:03:08 -0500 Subject: [PATCH 1/2] fix: inline convert test Dockerfile data Skip downloading Dockerfiles for the convert test as they add instability to the tests when github is unreliable. Capture what we are fetching and the sha256sum values. Signed-off-by: Ryan Harper --- test/convert.bats | 32 ++-- test/data/elasticsearch-8.17.10.Dockerfile | 171 +++++++++++++++++++ test/data/python-3.11-alpine-3.22-Dockerfile | 140 +++++++++++++++ 3 files changed, 330 insertions(+), 13 deletions(-) create mode 100644 test/data/elasticsearch-8.17.10.Dockerfile create mode 100644 test/data/python-3.11-alpine-3.22-Dockerfile diff --git a/test/convert.bats b/test/convert.bats index a4728013..6f3bb6a7 100644 --- a/test/convert.bats +++ b/test/convert.bats @@ -57,9 +57,14 @@ EOF @test "alpine" { skip_slow_test - git clone https://github.com/alpinelinux/docker-alpine.git - chmod -R a+rwx docker-alpine - cd docker-alpine + # https://raw.githubusercontent.com/alpinelinux/docker-alpine/refs/heads/master/Dockerfile + cat > Dockerfile << EOF +FROM alpine:3.16 +RUN apk add --no-cache lua5.3 lua-filesystem lua-lyaml lua-http +COPY fetch-latest-releases.lua /usr/local/bin +VOLUME /out +ENTRYPOINT [ "/usr/local/bin/fetch-latest-releases.lua" ] +EOF TEMPDIR=$(mktemp -d) stacker convert --docker-file Dockerfile --output-file stacker.yaml --substitute-file stacker-subs.yaml stacker build -f stacker.yaml --substitute-file stacker-subs.yaml --substitute IMAGE=alpine --substitute STACKER_VOL1="$TEMPDIR" @@ -72,10 +77,10 @@ EOF @test "elasticsearch" { skip_slow_test - git clone --branch v8.17.10 --depth 1 https://github.com/elastic/dockerfiles.git - chmod -R a+rwx dockerfiles - cd dockerfiles/elasticsearch - stacker convert --docker-file Dockerfile --output-file stacker.yaml --substitute-file stacker-subs.yaml + # https://github.com/elastic/dockerfiles/archive/refs/tags/v8.17.10.tar.gz + # SHA256SUM=75f89195b53c9d02d6a45d2bc9d51a11d92b079dcc046893615b04455269ba1a dockerfiles-8.17.10/elasticsearch/Dockerfile + DOCKERFILE="data/elasticsearch-8.17.10.Dockerfile" + stacker convert --docker-file $DOCKERFILE --output-file stacker.yaml --substitute-file stacker-subs.yaml stacker build -f stacker.yaml --substitute-file stacker-subs.yaml --substitute IMAGE=elasticsearch if [ -nz "${REGISTRY_URL}" ]; then stacker publish -f stacker.yaml --substitute-file stacker-subs.yaml --substitute IMAGE=elasticsearch --skip-tls --url docker://${REGISTRY_URL} --image elasticsearch --tag latest @@ -85,13 +90,14 @@ EOF } @test "python" { skip_slow_test - git clone https://github.com/docker-library/python.git - cd python + # git clone https://github.com/docker-library/python.git + # cd python # pick a specific commit so we don't get broken by upstream changes: - git reset --hard aad39d215779f27b410b25f612b6680a75781edb - cd 3.11/alpine3.22 - chmod -R a+rw . - stacker convert --docker-file Dockerfile --output-file stacker.yaml --substitute-file stacker-subs.yaml + # git reset --hard aad39d215779f27b410b25f612b6680a75781edb + # cd 3.11/alpine3.22 + # SHA256SUM=ada7a604a8359de5914158bca13f539ea24c13d8a3492f511a68444a511a1db8 data/python-3.11-alpine-3.22-Dockerfile + DOCKERFILE="data/python-3.11-alpine-3.22-Dockerfile" + stacker convert --docker-file $DOCKERFILE --output-file stacker.yaml --substitute-file stacker-subs.yaml stacker build -f stacker.yaml --substitute-file stacker-subs.yaml --substitute IMAGE=python if [ -nz "${REGISTRY_URL}" ]; then stacker publish -f stacker.yaml --substitute-file stacker-subs.yaml --substitute IMAGE=python --skip-tls --url docker://${REGISTRY_URL} --image python --tag latest diff --git a/test/data/elasticsearch-8.17.10.Dockerfile b/test/data/elasticsearch-8.17.10.Dockerfile new file mode 100644 index 00000000..1a6e3ca3 --- /dev/null +++ b/test/data/elasticsearch-8.17.10.Dockerfile @@ -0,0 +1,171 @@ +################################################################################ +# This Dockerfile was generated from the template at distribution/src/docker/Dockerfile +# +# Beginning of multi stage Dockerfile +################################################################################ + +################################################################################ +# Build stage 1 `builder`: +# Extract Elasticsearch artifact +################################################################################ + +FROM ubuntu:24.04 AS builder + +# Install required packages to extract the Elasticsearch distribution + +RUN for iter in 1 2 3 4 5 6 7 8 9 10; do \ + apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y curl && \ + exit_code=0 && break || \ + exit_code=$? && echo "apt-get error: retry $iter in 10s" && sleep 10; \ + done; \ + exit $exit_code + + # `tini` is a tiny but valid init for containers. This is used to cleanly + # control how ES and any child processes are shut down. + # For wolfi we pick it from the blessed wolfi package registry. + # + # The tini GitHub page gives instructions for verifying the binary using + # gpg, but the keyservers are slow to return the key and this can fail the + # build. Instead, we check the binary against the published checksum. + RUN set -eux ; \ + tini_bin="" ; \ + case "$(arch)" in \ + aarch64) tini_bin='tini-arm64' ;; \ + x86_64) tini_bin='tini-amd64' ;; \ + *) echo >&2 ; echo >&2 "Unsupported architecture $(arch)" ; echo >&2 ; exit 1 ;; \ + esac ; \ + curl --retry 10 -S -L -O https://github.com/krallin/tini/releases/download/v0.19.0/${tini_bin} ; \ + curl --retry 10 -S -L -O https://github.com/krallin/tini/releases/download/v0.19.0/${tini_bin}.sha256sum ; \ + sha256sum -c ${tini_bin}.sha256sum ; \ + rm ${tini_bin}.sha256sum ; \ + mv ${tini_bin} /bin/tini ; \ + chmod 0555 /bin/tini + +RUN mkdir /usr/share/elasticsearch +WORKDIR /usr/share/elasticsearch + +RUN curl --retry 10 -S -L --output /tmp/elasticsearch.tar.gz https://artifacts-no-kpi.elastic.co/downloads/elasticsearch/elasticsearch-8.17.10-linux-$(arch).tar.gz + +RUN tar -zxf /tmp/elasticsearch.tar.gz --strip-components=1 + +# The distribution includes a `config` directory, no need to create it +COPY config/elasticsearch.yml config/ +COPY config/log4j2.properties config/log4j2.docker.properties + +# 1. Configure the distribution for Docker +# 2. Create required directory +# 3. Move the distribution's default logging config aside +# 4. Move the generated docker logging config so that it is the default +# 5. Reset permissions on all directories +# 6. Reset permissions on all files +# 7. Make CLI tools executable +# 8. Make some directories writable. `bin` must be writable because +# plugins can install their own CLI utilities. +# 9. Make some files writable +RUN sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' bin/elasticsearch-env && \ + mkdir data && \ + mv config/log4j2.properties config/log4j2.file.properties && \ + mv config/log4j2.docker.properties config/log4j2.properties && \ + find . -type d -exec chmod 0555 {} + && \ + find . -type f -exec chmod 0444 {} + && \ + chmod 0555 bin/* jdk/bin/* jdk/lib/jspawnhelper modules/x-pack-ml/platform/linux-*/bin/* && \ + chmod 0775 bin config config/jvm.options.d data logs plugins && \ + find config -type f -exec chmod 0664 {} + + +################################################################################ +# Build stage 2 (the actual Elasticsearch image): +# +# Copy elasticsearch from stage 1 +# Add entrypoint +################################################################################ + +FROM ubuntu:24.04 + +# Change default shell to bash, then install required packages with retries. +RUN yes no | dpkg-reconfigure dash && \ + for iter in 1 2 3 4 5 6 7 8 9 10; do \ + export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get upgrade -y && \ + apt-get install -y --no-install-recommends \ + ca-certificates curl netcat-openbsd p11-kit unzip zip && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* && \ + exit_code=0 && break || \ + exit_code=$? && echo "apt-get error: retry $iter in 10s" && sleep 10; \ + done; \ + exit $exit_code + +RUN userdel -r ubuntu && \ + groupadd -g 1000 elasticsearch && \ + useradd --uid 1000 --gid 1000 --home-dir /usr/share/elasticsearch --create-home elasticsearch && \ + usermod -aG root elasticsearch && \ + chown -R 0:0 /usr/share/elasticsearch + +ENV ELASTIC_CONTAINER true + +WORKDIR /usr/share/elasticsearch + +COPY --from=builder --chown=0:0 /usr/share/elasticsearch /usr/share/elasticsearch + +COPY --from=builder --chown=0:0 /bin/tini /bin/tini + +ENV PATH /usr/share/elasticsearch/bin:$PATH +ENV SHELL /bin/bash +COPY bin/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh + +# 1. Sync the user and group permissions of /etc/passwd +# 2. Set correct permissions of the entrypoint +# 3. Ensure that there are no files with setuid or setgid, in order to mitigate "stackclash" attacks. +# We've already run this in previous layers so it ought to be a no-op. +# 4. Replace OpenJDK's built-in CA certificate keystore with the one from the OS +# vendor. The latter is superior in several ways. +# REF: https://github.com/elastic/elasticsearch-docker/issues/171 +# 5. Tighten up permissions on the ES home dir (the permissions of the contents are handled earlier) +# 6. You can't install plugins that include configuration when running as `elasticsearch` and the `config` +# dir is owned by `root`, because the installed tries to manipulate the permissions on the plugin's +# config directory. +RUN chmod g=u /etc/passwd && \ + chmod 0555 /usr/local/bin/docker-entrypoint.sh && \ + find / -xdev -perm -4000 -exec chmod ug-s {} + && \ + chmod 0775 /usr/share/elasticsearch && \ + chown elasticsearch bin config config/jvm.options.d data logs plugins + +# Update "cacerts" bundle to use Ubuntu's CA certificates (and make sure it +# stays up-to-date with changes to Ubuntu's store) +COPY bin/docker-openjdk /etc/ca-certificates/update.d/docker-openjdk +RUN /etc/ca-certificates/update.d/docker-openjdk + +EXPOSE 9200 9300 + +LABEL org.label-schema.build-date="2025-08-08T08:36:52.872377389Z" \ + org.label-schema.license="Elastic-License-2.0" \ + org.label-schema.name="Elasticsearch" \ + org.label-schema.schema-version="1.0" \ + org.label-schema.url="https://www.elastic.co/products/elasticsearch" \ + org.label-schema.usage="https://www.elastic.co/guide/en/elasticsearch/reference/index.html" \ + org.label-schema.vcs-ref="e5c4b2af120c131ea2885730f6693cb7d40a0a63" \ + org.label-schema.vcs-url="https://github.com/elastic/elasticsearch" \ + org.label-schema.vendor="Elastic" \ + org.label-schema.version="8.17.10" \ + org.opencontainers.image.created="2025-08-08T08:36:52.872377389Z" \ + org.opencontainers.image.documentation="https://www.elastic.co/guide/en/elasticsearch/reference/index.html" \ + org.opencontainers.image.licenses="Elastic-License-2.0" \ + org.opencontainers.image.revision="e5c4b2af120c131ea2885730f6693cb7d40a0a63" \ + org.opencontainers.image.source="https://github.com/elastic/elasticsearch" \ + org.opencontainers.image.title="Elasticsearch" \ + org.opencontainers.image.url="https://www.elastic.co/products/elasticsearch" \ + org.opencontainers.image.vendor="Elastic" \ + org.opencontainers.image.version="8.17.10" + +# Our actual entrypoint is `tini`, a minimal but functional init program. It +# calls the entrypoint we provide, while correctly forwarding signals. +ENTRYPOINT ["/bin/tini", "--", "/usr/local/bin/docker-entrypoint.sh"] +# Dummy overridable parameter parsed by entrypoint +CMD ["eswrapper"] + +USER 1000:0 + +################################################################################ +# End of multi-stage Dockerfile +################################################################################ diff --git a/test/data/python-3.11-alpine-3.22-Dockerfile b/test/data/python-3.11-alpine-3.22-Dockerfile new file mode 100644 index 00000000..a88b9e32 --- /dev/null +++ b/test/data/python-3.11-alpine-3.22-Dockerfile @@ -0,0 +1,140 @@ +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# + +FROM alpine:3.22 + +# ensure local python is preferred over distribution python +ENV PATH /usr/local/bin:$PATH + +# cannot remove LANG even though https://bugs.python.org/issue19846 is fixed +# last attempted removal of LANG broke many users: +# https://github.com/docker-library/python/pull/570 +ENV LANG C.UTF-8 + +# runtime dependencies +RUN set -eux; \ + apk add --no-cache \ + ca-certificates \ + tzdata \ + ; + +ENV GPG_KEY A035C8C19219BA821ECEA86B64E628F8D684696D +ENV PYTHON_VERSION 3.11.14 +ENV PYTHON_SHA256 8d3ed8ec5c88c1c95f5e558612a725450d2452813ddad5e58fdb1a53b1209b78 + +RUN set -eux; \ + \ + apk add --no-cache --virtual .build-deps \ + bluez-dev \ + bzip2-dev \ + dpkg-dev dpkg \ + findutils \ + gcc \ + gdbm-dev \ + gnupg \ + libc-dev \ + libffi-dev \ + libnsl-dev \ + libtirpc-dev \ + linux-headers \ + make \ + ncurses-dev \ + openssl-dev \ + pax-utils \ + readline-dev \ + sqlite-dev \ + tar \ + tcl-dev \ + tk \ + tk-dev \ + util-linux-dev \ + xz \ + xz-dev \ + zlib-dev \ + ; \ + \ + wget -O python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz"; \ + echo "$PYTHON_SHA256 *python.tar.xz" | sha256sum -c -; \ + wget -O python.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc"; \ + GNUPGHOME="$(mktemp -d)"; export GNUPGHOME; \ + gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$GPG_KEY"; \ + gpg --batch --verify python.tar.xz.asc python.tar.xz; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME" python.tar.xz.asc; \ + mkdir -p /usr/src/python; \ + tar --extract --directory /usr/src/python --strip-components=1 --file python.tar.xz; \ + rm python.tar.xz; \ + \ + cd /usr/src/python; \ + gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)"; \ + ./configure \ + --build="$gnuArch" \ + --enable-loadable-sqlite-extensions \ + --enable-option-checking=fatal \ + --enable-shared \ + $(test "${gnuArch%%-*}" != 'riscv64' && echo '--with-lto') \ + --with-ensurepip \ + ; \ + nproc="$(nproc)"; \ +# set thread stack size to 1MB so we don't segfault before we hit sys.getrecursionlimit() +# https://github.com/alpinelinux/aports/commit/2026e1259422d4e0cf92391ca2d3844356c649d0 + EXTRA_CFLAGS="-DTHREAD_STACK_SIZE=0x100000"; \ + LDFLAGS="${LDFLAGS:--Wl},--strip-all"; \ + make -j "$nproc" \ + "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \ + "LDFLAGS=${LDFLAGS:-}" \ + ; \ +# https://github.com/docker-library/python/issues/784 +# prevent accidental usage of a system installed libpython of the same version + rm python; \ + make -j "$nproc" \ + "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \ + "LDFLAGS=${LDFLAGS:--Wl},-rpath='\$\$ORIGIN/../lib'" \ + python \ + ; \ + make install; \ + \ + cd /; \ + rm -rf /usr/src/python; \ + \ + find /usr/local -depth \ + \( \ + \( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \ + -o \( -type f -a \( -name '*.pyc' -o -name '*.pyo' -o -name 'libpython*.a' \) \) \ + \) -exec rm -rf '{}' + \ + ; \ + \ + find /usr/local -type f -executable -not \( -name '*tkinter*' \) -exec scanelf --needed --nobanner --format '%n#p' '{}' ';' \ + | tr ',' '\n' \ + | sort -u \ + | awk 'system("[ -e /usr/local/lib/" $1 " ]") == 0 { next } { print "so:" $1 }' \ + | xargs -rt apk add --no-network --virtual .python-rundeps \ + ; \ + apk del --no-network .build-deps; \ + \ + export PYTHONDONTWRITEBYTECODE=1; \ + python3 --version; \ + \ + pip3 install \ + --disable-pip-version-check \ + --no-cache-dir \ + --no-compile \ + 'setuptools==79.0.1' \ + # https://github.com/docker-library/python/issues/1023 + 'wheel<0.46' \ + ; \ + pip3 --version + +# make some useful symlinks that are expected to exist ("/usr/local/bin/python" and friends) +RUN set -eux; \ + for src in idle3 pip3 pydoc3 python3 python3-config; do \ + dst="$(echo "$src" | tr -d 3)"; \ + [ -s "/usr/local/bin/$src" ]; \ + [ ! -e "/usr/local/bin/$dst" ]; \ + ln -svT "$src" "/usr/local/bin/$dst"; \ + done + +CMD ["python3"] From 3b5a714c0c7ee54e6fd55d01028b796c454e5112 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Mon, 23 Mar 2026 09:16:00 -0500 Subject: [PATCH 2/2] fix: import from test/data to TEST_TMPDIR Signed-off-by: Ryan Harper --- test/convert.bats | 15 +++++++++++++-- test/helpers.bash | 10 ++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/test/convert.bats b/test/convert.bats index 6f3bb6a7..98501fe8 100644 --- a/test/convert.bats +++ b/test/convert.bats @@ -79,7 +79,12 @@ EOF skip_slow_test # https://github.com/elastic/dockerfiles/archive/refs/tags/v8.17.10.tar.gz # SHA256SUM=75f89195b53c9d02d6a45d2bc9d51a11d92b079dcc046893615b04455269ba1a dockerfiles-8.17.10/elasticsearch/Dockerfile - DOCKERFILE="data/elasticsearch-8.17.10.Dockerfile" + DOCKERFILE="elasticsearch-8.17.10.Dockerfile" + + # copy data file into TEST_TMPDIR + import_test_data_file "${DOCKERFILE}" + DOCKERFILE=${TEST_TMPDIR}/${DOCKERFILE} + stacker convert --docker-file $DOCKERFILE --output-file stacker.yaml --substitute-file stacker-subs.yaml stacker build -f stacker.yaml --substitute-file stacker-subs.yaml --substitute IMAGE=elasticsearch if [ -nz "${REGISTRY_URL}" ]; then @@ -88,6 +93,7 @@ EOF rm -f stacker.yaml stacker-subs.yaml stacker clean } + @test "python" { skip_slow_test # git clone https://github.com/docker-library/python.git @@ -96,7 +102,12 @@ EOF # git reset --hard aad39d215779f27b410b25f612b6680a75781edb # cd 3.11/alpine3.22 # SHA256SUM=ada7a604a8359de5914158bca13f539ea24c13d8a3492f511a68444a511a1db8 data/python-3.11-alpine-3.22-Dockerfile - DOCKERFILE="data/python-3.11-alpine-3.22-Dockerfile" + DOCKERFILE="python-3.11-alpine-3.22-Dockerfile" + + # copy data file into TEST_TMPDIR + import_test_data_file "${DOCKERFILE}" + DOCKERFILE=${TEST_TMPDIR}/${DOCKERFILE} + stacker convert --docker-file $DOCKERFILE --output-file stacker.yaml --substitute-file stacker-subs.yaml stacker build -f stacker.yaml --substitute-file stacker-subs.yaml --substitute IMAGE=python if [ -nz "${REGISTRY_URL}" ]; then diff --git a/test/helpers.bash b/test/helpers.bash index c3b0ec70..519fec3f 100644 --- a/test/helpers.bash +++ b/test/helpers.bash @@ -12,6 +12,16 @@ if [ "$(id -u)" != "0" ]; then exit 1 fi +function import_test_data_file() { + export TEST_TMPDIR=$(tmpd $BATS_TEST_NAME) + cd $TEST_TMPDIR + + if ! cp "${ROOT_DIR}/test/data/${1}" .; then + echo "Failed to import ${ROOT}/test/data/${1} to $TEST_TMPDIR"; + exit 1 + fi +} + function give_user_ownership() { if [ "$PRIVILEGE_LEVEL" = "priv" ]; then return