From 9d3f10f4e09876e7470d56d9d90c0ffd918a18d2 Mon Sep 17 00:00:00 2001 From: renner Date: Wed, 17 Dec 2025 00:33:27 +0100 Subject: [PATCH 1/6] feat: rechunk with rpm-ostree The legacy rechunker is basically unmaintained, this is a much safer option to reduce our (delta) image size. https://coreos.github.io/rpm-ostree/build-chunked-oci/ https://gitlab.com/fedora/bootc/base-images/-/blob/main/bootc-base-imagectl The metadata from a prior built image is used to construct the new one so that the difference between them is minimized if possible. --- .github/workflows/reusable-build.yml | 97 ++++++++++++---- Justfile | 161 +++------------------------ 2 files changed, 94 insertions(+), 164 deletions(-) diff --git a/.github/workflows/reusable-build.yml b/.github/workflows/reusable-build.yml index 6de608779..c248e9c25 100644 --- a/.github/workflows/reusable-build.yml +++ b/.github/workflows/reusable-build.yml @@ -130,31 +130,90 @@ jobs: echo "OUTPUT_PATH=${OUTPUT_PATH}" >> $GITHUB_OUTPUT sudo rm -rf ${OCI_DIR} - - name: Rechunk Image - id: rechunk-image - shell: bash + # this tries to make delta updates smaller by pulling + # the last image we published under this tag to the registry + # by minimizing the shuffling of content into different layers + - name: Check Base Image Availability + id: check-baseline env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - MATRIX_BASE_NAME: ${{ matrix.base_name }} - MATRIX_STREAM_NAME: ${{ matrix.stream_name }} - MATRIX_IMAGE_FLAVOR: ${{ matrix.image_flavor }} + BASELINE_IMAGE: ${{ env.IMAGE_REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEFAULT_TAG }} run: | - sudo -E $(command -v just) rechunk "${MATRIX_BASE_NAME}" \ - "${MATRIX_STREAM_NAME}" \ - "${MATRIX_IMAGE_FLAVOR}" \ - "1" + # TODO: check if this pulls stable-daily when we run stable-daily and stable for weeklies + # checks if image is rechunked here: + # https://github.com/coreos/rpm-ostree/blob/653e6ee0ba5953665b2bc2606bd6ab89c3871454/rust/src/compose.rs#L511-L519 + if skopeo inspect docker://"${BASELINE_IMAGE}" | jq -e '.LayersData[1:] | all(.Annotations?["ostree.components"]?)'; then + echo "found ${BASELINE_IMAGE} in registry." + echo "BASELINE_IMAGE=${BASELINE_IMAGE}" >> "$GITHUB_OUTPUT" + echo "PULL_BASELINE_IMAGE=true" >> "$GITHUB_OUTPUT" + else + echo "no baseline image with tag $BASELINE_IMAGE found in registry" + BASELINE_IMAGE="localhost/${{ env.IMAGE_NAME }}:${{ env.DEFAULT_TAG }}" + echo "BASELINE_IMAGE=${BASELINE_IMAGE}" >> "$GITHUB_OUTPUT" + echo "PULL_BASELINE_IMAGE=false" >> "$GITHUB_OUTPUT" + fi + + - name: Pull Baseline Image For Rechunker + id: pull-baseline + if: steps.check-baseline.outputs.PULL_BASELINE_IMAGE == 'true' + env: + BASELINE_IMAGE: ${{ steps.check-baseline.outputs.BASELINE_IMAGE }} + run: | + sudo podman pull ${BASELINE_IMAGE} + echo "BASELINE_IMAGE=${BASELINE_IMAGE}" >> "$GITHUB_OUTPUT" - - name: Load Image into Podman - id: load-rechunk - shell: bash + - name: debug + run: sudo podman images + + - name: Add debug symbols for rpm-ostree + run: | + cat <<'EOF' > debug.Containerfile + FROM quay.io/fedora/fedora-bootc:latest + RUN dnf install -y dnf5-plugins && \ + dnf5 config-manager setopt updates-debuginfo.enabled=1 && \ + dnf5 config-manager setopt fedora-debuginfo.enabled=1 && \ + dnf install -y rpm-ostree-debuginfo + EOF + + sudo podman build -t localhost/fedora-bootc-debug:latest -f debug.Containerfile . + + # This happens to rename localhost/aurora:stable to :stable-daily + # same behavior as before, we have to redo this logic when we are doing + # stable, testing, next anyway, leave the mess for now + - name: Rechunk Image with rpm-ostree + id: rechunker env: - MATRIX_BASE_NAME: ${{ matrix.base_name }} + IMAGE_NAME: ${{ env.IMAGE_NAME }} + BASELINE_IMAGE: ${{ steps.check-baseline.outputs.BASELINE_IMAGE }} + MATRIX_STREAM_NAME: ${{ matrix.stream_name }} + run: | + sudo podman run --rm \ + --privileged \ + --env=RUST_BACKTRACE=full \ + -v /var/lib/containers:/var/lib/containers \ + "localhost/fedora-bootc-debug:latest" \ + /usr/libexec/bootc-base-imagectl rechunk \ + localhost/${IMAGE_NAME}:${MATRIX_STREAM_NAME} \ + ${BASELINE_IMAGE} + + # we are prodcucing ghcr.io/ublue-os/aurora with the rechunker if we have a baseline image + # the rechunker needs to have the same target image tag for it to work + # ghcr.io/ublue-os/aurora:stable -> localhost/aurora + # later build steps expect localhost/aurora, this is essentially renaming the image again + - name: Rename Rechunker Baseline Image + id: retag-rechunk + if: steps.check-baseline.outputs.PULL_BASELINE_IMAGE == 'true' + env: + BASELINE_IMAGE: ${{ steps.pull-baseline.outputs.BASELINE_IMAGE }} + IMAGE_NAME: ${{ env.IMAGE_NAME }} DEFAULT_TAG: ${{ env.DEFAULT_TAG }} - MATRIX_IMAGE_FLAVOR: ${{ matrix.image_flavor }} run: | - sudo -E $(command -v just) load-rechunk "${MATRIX_BASE_NAME}" \ - "${DEFAULT_TAG}" \ - "${MATRIX_IMAGE_FLAVOR}" + sudo podman tag \ + ${BASELINE_IMAGE} \ + localhost/${IMAGE_NAME}:${DEFAULT_TAG} + sudo podman image rm -i ${BASELINE_IMAGE} + + - name: debug + run: sudo podman images - name: Secureboot Check id: secureboot diff --git a/Justfile b/Justfile index da3553040..221e6d4f4 100644 --- a/Justfile +++ b/Justfile @@ -238,7 +238,7 @@ build $image="aurora" $tag="latest" $flavor="main" rechunk="0" ghcr="0" pipeline elif [[ "{{ rechunk }}" == "1" && "{{ ghcr }}" == "1" ]]; then ${SUDOIF} {{ just }} rechunk "${image}" "${tag}" "${flavor}" 1 elif [[ "{{ rechunk }}" == "1" ]]; then - ${SUDOIF} {{ just }} rechunk "${image}" "${tag}" "${flavor}" + {{ just }} rechunk "${image}" "${tag}" "${flavor}" fi # Build Image and Rechunk @@ -283,133 +283,27 @@ rechunk $image="aurora" $tag="latest" $flavor="main" ghcr="0" pipeline="0": {{ just }} build "${image}" "${tag}" "${flavor}" fi - # Load into Rootful Podman - ID=$(${SUDOIF} ${PODMAN} images --filter reference=localhost/"${image_name}":"${tag}" --format "'{{ '{{.ID}}' }}'") - if [[ -z "$ID" && ! ${PODMAN} =~ docker ]]; then - COPYTMP=$(mktemp -p "${PWD}" -d -t podman_scp.XXXXXXXXXX) - ${SUDOIF} TMPDIR=${COPYTMP} ${PODMAN} image scp ${UID}@localhost::localhost/"${image_name}":"${tag}" root@localhost::localhost/"${image_name}":"${tag}" - rm -rf "${COPYTMP}" + # Delete the rechunked image if present, rpm-ostree shits itself for whatever reason + # workaround for https://github.com/coreos/rpm-ostree/issues/5545 + if ${SUDOIF} ${PODMAN} image exists "localhost/${image_name}:${tag}-chunked"; then + ${SUDOIF} ${PODMAN} image rm -f "localhost/${image_name}:${tag}-chunked" fi - # Prep Container - CREF=$(${SUDOIF} ${PODMAN} create localhost/"${image_name}":"${tag}" bash) - OLD_IMAGE=$(${SUDOIF} ${PODMAN} inspect $CREF | jq -r '.[].Image') - OUT_NAME="${image_name}_build" - MOUNT=$(${SUDOIF} ${PODMAN} mount "${CREF}") - - # Fedora Version - fedora_version=$(${SUDOIF} ${PODMAN} inspect $CREF | jq -r '.[].Config.Labels["ostree.linux"]' | grep -oP 'fc\K[0-9]+') - - # Label Version - VERSION=$(${SUDOIF} ${PODMAN} inspect $CREF | jq -r '.[].Config.Labels["org.opencontainers.image.version"]') - - # Git SHA - SHA="dedbeef" - if [[ -z "$(git status -s)" ]]; then - SHA=$(git rev-parse HEAD) - fi - - # Rest of Labels - LABELS=" - io.artifacthub.package.deprecated=false - io.artifacthub.package.keywords=bootc,fedora,aurora,ublue,universal-blue - io.artifacthub.package.logo-url=https://avatars.githubusercontent.com/u/120078124?s=200&v=4 - io.artifacthub.package.maintainers=[{\"name\": \"NiHaiden\", \"email\": \"me@nhaiden.io\"}] - io.artifacthub.package.readme-url=https://raw.githubusercontent.com/ublue-os/aurora/refs/heads/main/README.md - org.opencontainers.image.created=$(date -u +%Y\-%m\-%d\T%H\:%M\:%S\Z) - org.opencontainers.image.license=Apache-2.0 - org.opencontainers.image.source=https://raw.githubusercontent.com/ublue-os/aurora/refs/heads/main/Containerfile - org.opencontainers.image.title=${image_name} - org.opencontainers.image.url=https://getaurora.dev - org.opencontainers.image.vendor={{ repo_organization }} - ostree.linux=$(${SUDOIF} ${PODMAN} inspect $CREF | jq -r '.[].Config.Labels["ostree.linux"]') - containers.bootc=1 - " - - # Cleanup Space during Github Action - if [[ "{{ ghcr }}" == "1" ]]; then - base_image_name=kinoite - if [[ "${tag}" =~ stable ]]; then - tag="stable-daily" - fi - ID=$(${SUDOIF} ${PODMAN} images --filter reference=ghcr.io/{{ repo_organization }}/"${base_image_name}":${fedora_version} --format "{{ '{{.ID}}' }}") - if [[ -n "$ID" ]]; then - ${PODMAN} rmi "$ID" - fi + # Load into Rootful Podman + ID=$(${PODMAN} inspect --format={{ '{{.Digest}}' }} localhost/"${image_name}":"${tag}") + ID_ROOT=$(sudo ${PODMAN} inspect --format={{ '{{.Digest}}' }} localhost/"${image_name}":"${tag}") + if [[ ! "${PODMAN}" =~ "docker" ]] && [[ -n "$ID" ]] && [[ "$ID" != "$ID_ROOT" ]]; then + ${PODMAN} image scp $(whoami)@localhost::localhost/"${image_name}":"${tag}" fi - # Rechunk Container - rechunker="{{ rechunker_image }}" - - echo "::endgroup::" - echo "::group:: Prune" - - # Run Rechunker's Prune ${SUDOIF} ${PODMAN} run --rm \ --pull=${PULL_POLICY} \ - --security-opt label=disable \ - --volume "$MOUNT":/var/tree \ - --env TREE=/var/tree \ - --user 0:0 \ - "${rechunker}" \ - /sources/rechunk/1_prune.sh - - echo "::endgroup::" - echo "::group:: Create ostree tree" - - # Run Rechunker's Create - ${SUDOIF} ${PODMAN} run --rm \ - --security-opt label=disable \ - --volume "$MOUNT":/var/tree \ - --volume "cache_ostree:/var/ostree" \ - --env TREE=/var/tree \ - --env REPO=/var/ostree/repo \ - --env RESET_TIMESTAMP=1 \ - --user 0:0 \ - "${rechunker}" \ - /sources/rechunk/2_create.sh - - # Cleanup Temp Container Reference - ${SUDOIF} ${PODMAN} unmount "$CREF" - ${SUDOIF} ${PODMAN} rm "$CREF" - ${SUDOIF} ${PODMAN} rmi "$OLD_IMAGE" - - echo "::endgroup::" - echo "::group:: Rechunker" - - # Run Rechunker - ${SUDOIF} ${PODMAN} run --rm \ - --pull=${PULL_POLICY} \ - --security-opt label=disable \ - --volume "$PWD:/workspace" \ - --volume "$PWD:/var/git" \ - --volume cache_ostree:/var/ostree \ - --env REPO=/var/ostree/repo \ - --env PREV_REF=ghcr.io/ublue-os/"${image_name}":"${tag}" \ - --env OUT_NAME="$OUT_NAME" \ - --env LABELS="${LABELS}" \ - --env "DESCRIPTION='The ultimate productivity workstation'" \ - --env "VERSION=${VERSION}" \ - --env VERSION_FN=/workspace/version.txt \ - --env OUT_REF="oci:$OUT_NAME" \ - --env GIT_DIR="/var/git" \ - --env REVISION="$SHA" \ - --user 0:0 \ - "${rechunker}" \ - /sources/rechunk/3_chunk.sh - - # Fix Permissions of OCI - ${SUDOIF} find ${OUT_NAME} -type d -exec chmod 0755 {} \; || true - ${SUDOIF} find ${OUT_NAME}* -type f -exec chmod 0644 {} \; || true - - if [[ "${UID}" -gt "0" ]]; then - ${SUDOIF} chown "${UID}:${GROUPS}" -R "${PWD}" - elif [[ -n "${SUDO_UID:-}" ]]; then - chown "${SUDO_UID}":"${SUDO_GID}" -R "${PWD}" - fi - - # Remove cache_ostree - ${SUDOIF} ${PODMAN} volume rm cache_ostree + --privileged \ + -v "/var/lib/containers:/var/lib/containers" \ + "quay.io/fedora/fedora-bootc:latest" \ + /usr/libexec/bootc-base-imagectl rechunk \ + "localhost/${image_name}":"${tag}" \ + "localhost/${image_name}":"${tag}-chunked" echo "::endgroup::" @@ -419,27 +313,6 @@ rechunk $image="aurora" $tag="latest" $flavor="main" ghcr="0" pipeline="0": sudo -u "${SUDO_USER}" {{ just }} secureboot "${image}" "${tag}" "${flavor}" fi -# Load OCI into Podman Store -[group('Image')] -load-rechunk image="aurora" tag="latest" flavor="main": - #!/usr/bin/bash - set -eou pipefail - - # Validate - {{ just }} validate {{ image }} {{ tag }} {{ flavor }} - - # Image Name - image_name=$({{ just }} image_name {{ image }} {{ tag }} {{ flavor }}) - - # Load Image - OUT_NAME="${image_name}_build" - IMAGE=$(${PODMAN} pull oci:"${PWD}"/"${OUT_NAME}") - ${PODMAN} tag ${IMAGE} localhost/"${image_name}":{{ tag }} - - # Cleanup - rm -rf "${OUT_NAME}*" - rm -f previous.manifest.json - # Run Container [group('Image')] run $image="aurora" $tag="latest" $flavor="main": @@ -680,8 +553,6 @@ tag-images image_name="" default_tag="" tags="": ${PODMAN} tag $IMAGE {{ image_name }}:${tag} done - - # Show Images ${PODMAN} images From 606c6bdc5404ebe5464286b36981bfe6015fac04 Mon Sep 17 00:00:00 2001 From: renner Date: Wed, 17 Dec 2025 06:23:41 +0100 Subject: [PATCH 2/6] feat: add horrible workaround because of hhd-dev/rechunk https://github.com/ublue-os/main/issues/759 --- .../shared/usr/bin/rechunker-group-fix | 31 +++++++++++++++++++ .../system/rechunker-group-fix.service | 30 ++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100755 system_files/shared/usr/bin/rechunker-group-fix create mode 100644 system_files/shared/usr/lib/systemd/system/rechunker-group-fix.service diff --git a/system_files/shared/usr/bin/rechunker-group-fix b/system_files/shared/usr/bin/rechunker-group-fix new file mode 100755 index 000000000..ef53f1c52 --- /dev/null +++ b/system_files/shared/usr/bin/rechunker-group-fix @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +# See /usr/lib/systemd/system/rechunker-group-fix.service for more details +# https://github.com/zirconium-dev/zirconium/commit/85cc566697b3337d37e2b6c6305c51b7f1a9e9b1 +# thanks @tullilirockz @hecknt + +# To use this script, you'll want to put this in your systemd service: +# rm /etc/gshadow +# systemd-sysusers +# (run this script) +# systemd-tmpfiles --create --remove --boot --exclude-prefix=/dev +# This will populate /etc/group successfully, and then populate /etc/gshadow +# with any missing groups that we nuked when we removed /etc/gshadow + +GSHADOW_FILE="/etc/gshadow" +GROUP_FILE="/etc/group" + +for f in $(cat $GROUP_FILE); do + cut -f1 -d':' <(echo $f) | xargs -I{} grep ^"{}" $GSHADOW_FILE &>/dev/null || \ + echo $(cut -f1 -d':' <(echo $f)):'!*::' >> $GSHADOW_FILE +done && \ +cat > /etc/.aurora-rechunker-fix-do-not-remove < Date: Wed, 17 Dec 2025 16:35:52 +0100 Subject: [PATCH 3/6] feat: put big files in own layers enable when https://github.com/coreos/rpm-ostree/issues/5545 fixed --- build_files/base/20-layer-definitions.sh | 95 +++++++++++++++++++ build_files/base/{20-tests.sh => 21-tests.sh} | 0 build_files/shared/build.sh | 5 +- 3 files changed, 99 insertions(+), 1 deletion(-) create mode 100755 build_files/base/20-layer-definitions.sh rename build_files/base/{20-tests.sh => 21-tests.sh} (100%) diff --git a/build_files/base/20-layer-definitions.sh b/build_files/base/20-layer-definitions.sh new file mode 100755 index 000000000..abfd9d53b --- /dev/null +++ b/build_files/base/20-layer-definitions.sh @@ -0,0 +1,95 @@ +#!/usr/bin/bash + +# to reduce the indiviual layer size we put big files that are not shipped with RPMs in their own layers +# See: https://coreos.github.io/rpm-ostree/build-chunked-oci/ + +set -eoux pipefail + +# Enable all this when fixed: https://github.com/coreos/rpm-ostree/issues/5545 +# For now just hardcode the big files manually, keep the non-rpm layer amount static +# use systemd-escape -p +#setfattr -n user.component -v usr-share-doc-aurora-aurora.pdf /usr/share/doc/aurora/aurora.pdf +#setfattr -n user.component -v usr-share-homebrew.tar.zst /usr/share/homebrew.tar.zst +#setfattr -n user.component -v usr-bin-starhsip /usr/bin/starship +#setfattr -n user.component -v usr-share-backgrounds-aurora-aurora\x2dwallpaper\x2d3-contents-images-3840x2160.png /usr/share/backgrounds/aurora/aurora-wallpaper-3/contents/images/3840x2160.png +#setfattr -n user.component -v usr-share-backgrounds-aurora-aurora\x2dwallpaper\x2d4-contents-images-3840x2160.png /usr/share/backgrounds/aurora/aurora-wallpaper-4/contents/images/3840x2160.png +#setfattr -n user.component -v usr-share-backgrounds-aurora-aurora\x2dwallpaper\x2d2-contents-images-3840x2160.png /usr/share/backgrounds/aurora/aurora-wallpaper-2/contents/images/3840x2160.png +#setfattr -n user.component -v usr-share-backgrounds-aurora-aurora\x2dwallpaper\x2d7-contents-images-3840x2160.jxl /usr/share/backgrounds/aurora/aurora-wallpaper-7/contents/images/3840x2160.jxl +#setfattr -n user.component -v usr-lib-firmware-intel\x2ducode-06\x2dad\x2d01 /usr/lib/firmware/intel-ucode/06-ad-01 +#setfattr -n user.component -v usr-share-fonts-nerd\x2dfonts-NerdFontsSymbolsOnly-SymbolsNerdFontMono\x2dRegular.ttf /usr/share/fonts/nerd-fonts/NerdFontsSymbolsOnly/SymbolsNerdFontMono-Regular.ttf +#setfattr -n user.component -v usr-share-fonts-nerd\x2dfonts-NerdFontsSymbolsOnly-SymbolsNerdFont\x2dRegular.ttf /usr/share/fonts/nerd-fonts/NerdFontsSymbolsOnly/SymbolsNerdFont-Regular.ttf +#setfattr -n user.component -v usr-share-backgrounds-aurora-greg\x2drakozy\x2daurora-contents-images-5616x3744.jxl /usr/share/backgrounds/aurora/greg-rakozy-aurora/contents/images/5616x3744.jxl +#setfattr -n user.component -v usr-lib-firmware-asihpi-dsp8900.bin /usr/lib/firmware/asihpi/dsp8900.bin +#setfattr -n user.component -v usr-lib-firmware-intel\x2ducode-06\x2daf\x2d03 /usr/lib/firmware/intel-ucode/06-af-03 +#setfattr -n user.component -v usr-share-backgrounds-aurora-aurora\x2dwallpaper\x2d8-contents-images-3840x2160.jxl /usr/share/backgrounds/aurora/aurora-wallpaper-8/contents/images/3840x2160.jxl +#setfattr -n user.component -v usr-share-backgrounds-aurora-jonatan\x2dpie\x2daurora-contents-images-3944x2770.jxl /usr/share/backgrounds/aurora/jonatan-pie-aurora/contents/images/3944x2770.jxl +#setfattr -n user.component -v usr-lib-firmware-intel\x2ducode-06\x2d8f\x2d08 /usr/lib/firmware/intel-ucode/06-8f-08 +#setfattr -n user.component -v usr-share-sddm-themes-01\x2dbreeze\x2daurora-preview.png /usr/share/sddm/themes/01-breeze-aurora/preview.png +#setfattr -n user.component -v usr-share-color-icc-colord-framework16.icc /usr/share/color/icc/colord/framework16.icc +#setfattr -n user.component -v usr-share-color-icc-colord-framework13.icc /usr/share/color/icc/colord/framework13.icc +#setfattr -n user.component -v usr-share-plasma-avatars-lumina.png /usr/share/plasma/avatars/lumina.png +#setfattr -n user.component -v usr-share-backgrounds-aurora-aurora\x2dwallpaper\x2d6-contents-images-3840x2160.jxl /usr/share/backgrounds/aurora/aurora-wallpaper-6/contents/images/3840x2160.jxl +#setfattr -n user.component -v usr-lib-firmware-mixart-miXart8.elf /usr/lib/firmware/mixart/miXart8.elf +#setfattr -n user.component -v usr-share-plasma-avatars-vincent.png /usr/share/plasma/avatars/vincent.png +#setfattr -n user.component -v usr-lib-firmware-asihpi-dsp6200.bin /usr/lib/firmware/asihpi/dsp6200.bin +#setfattr -n user.component -v usr-share-backgrounds-aurora-xe_space_needle-contents-images-6000x4000.jxl /usr/share/backgrounds/aurora/xe_space_needle/contents/images/6000x4000.jxl +#setfattr -n user.component -v usr-share-plasma-avatars-echo.png /usr/share/plasma/avatars/echo.png +#setfattr -n user.component -v usr-share-plasma-avatars-phlip.png /usr/share/plasma/avatars/phlip.png +#setfattr -n user.component -v usr-share-plasma-avatars-scope.png /usr/share/plasma/avatars/scope.png +#setfattr -n user.component -v usr-share-plasma-avatars-tina.png /usr/share/plasma/avatars/tina.png +#setfattr -n user.component -v usr-lib-firmware-ctefx\x2ddesktop.bin /usr/lib/firmware/ctefx-desktop.bin +#setfattr -n user.component -v usr-lib-firmware-ctefx\x2dr3di.bin /usr/lib/firmware/ctefx-r3di.bin + +#COUNT=30 +#OUTPUT="/tmp/nonrpm.json" +## 600KiB +#MIN_SIZE=614400 +#IGNORE=( +# "^/usr/lib/sysimage/rpm-ostree-base-db/rpmdb.sqlite" +# "^/usr/share/rpm/rpmdb.sqlite" +# "^/usr/lib/modules" # spam +# "^/usr/bin/tailscaled" # packaging is bumd, hardlinks +# "^/usr/bin/zpool" +# "^/usr/lib/fontconfig/cache" +# "^/usr/lib64/.*\.so" # mostly useless files +# "^/usr/lib32/.*\.so" # can't be assed to make this one thing +# "^/usr/lib/.*\.so" # and do it properly +# "^/usr/lib/bootupd/" # not worth +#) +# +#IGNORE_REGEX=$(printf "|%s" "${IGNORE[@]}") +#IGNORE_REGEX="${IGNORE_REGEX:1}" +# +## which files are not tracked by the rpmdb +#RPM_FILES=$(mktemp) +#rpm -qa --qf '[%{FILENAMES}\n]' | grep -E "^/usr|^/opt" | sort > "$RPM_FILES" +# +#NON_RPM_FILES=$(comm -23 \ +# <(find /usr -type f 2>/dev/null | grep -Ev "$IGNORE_REGEX" | sort) \ +# "$RPM_FILES" +#) +# +#echo "$NON_RPM_FILES" | xargs -d '\n' du -b 2>/dev/null | jq -Rn --argjson min "$MIN_SIZE" ' +# [ +# inputs | split("\t") | { +# "path": .[1], +# "size": (.[0] | tonumber), +# "component": (.[1] | ltrimstr("/") | gsub("/"; "-")) +# } | select(.size >= $min) +# ] +#' > "$OUTPUT" +# +#echo "found $(jq 'length' "$OUTPUT") non-rpm tracked files" +# +## generated example command +## setfattr -n user.component -v usr-share-doc-aurora-aurora.pdf /usr/share/doc/aurora/aurora.pdf +#jq -c '.[]' "$OUTPUT" | while read -r item; do +# path=$(echo "$item" | jq -r '.path') +# comp=$(echo "$item" | jq -r '.component') +# setfattr -n user.component -v "$comp" "$path" +#done +# +#echo "listing $COUNT biggest ones" +#jq -r --arg n "$COUNT" ' sort_by(.size) | reverse | .[0:($n | tonumber)] | .[] | "\(.size)\t\(.path)" ' "$OUTPUT" | numfmt --to=iec --field=1 + +echo "::endgroup::" diff --git a/build_files/base/20-tests.sh b/build_files/base/21-tests.sh similarity index 100% rename from build_files/base/20-tests.sh rename to build_files/base/21-tests.sh diff --git a/build_files/shared/build.sh b/build_files/shared/build.sh index 6128c6023..803b63634 100755 --- a/build_files/shared/build.sh +++ b/build_files/shared/build.sh @@ -60,8 +60,11 @@ fi # Validate all repos are disabled before committing /ctx/build_files/shared/validate-repos.sh +# Set filesystem properties for rechunker +/ctx/build_files/base/20-layer-definitions.sh + # Clean Up /ctx/build_files/shared/clean-stage.sh # Simple Tests -/ctx/build_files/base/20-tests.sh +/ctx/build_files/base/21-tests.sh From 2624e02ef4f3feb366486af86519ae61a9c8d722 Mon Sep 17 00:00:00 2001 From: renner Date: Wed, 24 Dec 2025 03:14:12 +0100 Subject: [PATCH 4/6] fix: for layer annotations not being pushed We need a newer podman version than ubuntu 24.04 provides, homebrew is the most elegant solution. We need the layer annotations for the rpm-ostree rechunker so that updates are more efficient. workaround for: https://github.com/containers/podman/issues/27796 --- .github/workflows/reusable-build.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/reusable-build.yml b/.github/workflows/reusable-build.yml index c248e9c25..1170fd920 100644 --- a/.github/workflows/reusable-build.yml +++ b/.github/workflows/reusable-build.yml @@ -275,6 +275,14 @@ jobs: with: string: ${{ env.IMAGE_REGISTRY }} + # TODO: remove me when we have a new podman in 26.04 runners + # needed because old podman doesn't push layer annotations for + # the rpm-ostree rechunker at all + - name: install podman from brew + if: github.event_name != 'pull_request' + run: | + /home/linuxbrew/.linuxbrew/bin/brew install podman + - name: Login to GitHub Container Registry if: github.event_name != 'pull_request' run: | @@ -295,9 +303,15 @@ jobs: attempt_delay: 15000 command: | set -euox pipefail + # HACK: push a second time so layer annotations are pushed + # TODO: remove me when https://github.com/containers/podman/issues/27796 fixed + + for tag in ${ALIAS_TAGS}; do + sudo -E /home/linuxbrew/.linuxbrew/bin/podman push ${IMAGE_NAME}:${tag} ${LOWERCASE}/${IMAGE_NAME}:${tag} + done for tag in ${ALIAS_TAGS}; do - sudo -E podman push ${IMAGE_NAME}:${tag} ${LOWERCASE}/${IMAGE_NAME}:${tag} + sudo -E /home/linuxbrew/.linuxbrew/bin/podman push ${IMAGE_NAME}:${tag} ${LOWERCASE}/${IMAGE_NAME}:${tag} done digest=$(skopeo inspect docker://${LOWERCASE}/${IMAGE_NAME}:${DEFAULT_TAG} --format '{{.Digest}}') From 349ff89c1d8cc1c7eb86d1517799a9785b1e7886 Mon Sep 17 00:00:00 2001 From: renner Date: Sun, 8 Feb 2026 20:43:40 +0100 Subject: [PATCH 5/6] feat: use just on github ci --- .github/workflows/reusable-build.yml | 78 ++-------------------------- Justfile | 29 +++++++---- 2 files changed, 23 insertions(+), 84 deletions(-) diff --git a/.github/workflows/reusable-build.yml b/.github/workflows/reusable-build.yml index 766d5ec91..f26a56149 100644 --- a/.github/workflows/reusable-build.yml +++ b/.github/workflows/reusable-build.yml @@ -118,87 +118,19 @@ jobs: echo "OUTPUT_PATH=${OUTPUT_PATH}" >> $GITHUB_OUTPUT sudo rm -rf ${OCI_DIR} - # this tries to make delta updates smaller by pulling - # the last image we published under this tag to the registry - # by minimizing the shuffling of content into different layers - - name: Check Base Image Availability - id: check-baseline - env: - BASELINE_IMAGE: ${{ env.IMAGE_REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEFAULT_TAG }} - run: | - # TODO: check if this pulls stable-daily when we run stable-daily and stable for weeklies - # checks if image is rechunked here: - # https://github.com/coreos/rpm-ostree/blob/653e6ee0ba5953665b2bc2606bd6ab89c3871454/rust/src/compose.rs#L511-L519 - if skopeo inspect docker://"${BASELINE_IMAGE}" | jq -e '.LayersData[1:] | all(.Annotations?["ostree.components"]?)'; then - echo "found ${BASELINE_IMAGE} in registry." - echo "BASELINE_IMAGE=${BASELINE_IMAGE}" >> "$GITHUB_OUTPUT" - echo "PULL_BASELINE_IMAGE=true" >> "$GITHUB_OUTPUT" - else - echo "no baseline image with tag $BASELINE_IMAGE found in registry" - BASELINE_IMAGE="localhost/${{ env.IMAGE_NAME }}:${{ env.DEFAULT_TAG }}" - echo "BASELINE_IMAGE=${BASELINE_IMAGE}" >> "$GITHUB_OUTPUT" - echo "PULL_BASELINE_IMAGE=false" >> "$GITHUB_OUTPUT" - fi - - - name: Pull Baseline Image For Rechunker - id: pull-baseline - if: steps.check-baseline.outputs.PULL_BASELINE_IMAGE == 'true' - env: - BASELINE_IMAGE: ${{ steps.check-baseline.outputs.BASELINE_IMAGE }} - run: | - sudo podman pull ${BASELINE_IMAGE} - echo "BASELINE_IMAGE=${BASELINE_IMAGE}" >> "$GITHUB_OUTPUT" - - name: debug run: sudo podman images - - name: Add debug symbols for rpm-ostree - run: | - cat <<'EOF' > debug.Containerfile - FROM quay.io/fedora/fedora-bootc:latest - RUN dnf install -y dnf5-plugins && \ - dnf5 config-manager setopt updates-debuginfo.enabled=1 && \ - dnf5 config-manager setopt fedora-debuginfo.enabled=1 && \ - dnf install -y rpm-ostree-debuginfo - EOF - - sudo podman build -t localhost/fedora-bootc-debug:latest -f debug.Containerfile . - - # This happens to rename localhost/aurora:stable to :stable-daily - # same behavior as before, we have to redo this logic when we are doing - # stable, testing, next anyway, leave the mess for now - name: Rechunk Image with rpm-ostree id: rechunker env: - IMAGE_NAME: ${{ env.IMAGE_NAME }} - BASELINE_IMAGE: ${{ steps.check-baseline.outputs.BASELINE_IMAGE }} - MATRIX_STREAM_NAME: ${{ matrix.stream_name }} - run: | - sudo podman run --rm \ - --privileged \ - --env=RUST_BACKTRACE=full \ - -v /var/lib/containers:/var/lib/containers \ - "localhost/fedora-bootc-debug:latest" \ - /usr/libexec/bootc-base-imagectl rechunk \ - localhost/${IMAGE_NAME}:${MATRIX_STREAM_NAME} \ - ${BASELINE_IMAGE} - - # we are prodcucing ghcr.io/ublue-os/aurora with the rechunker if we have a baseline image - # the rechunker needs to have the same target image tag for it to work - # ghcr.io/ublue-os/aurora:stable -> localhost/aurora - # later build steps expect localhost/aurora, this is essentially renaming the image again - - name: Rename Rechunker Baseline Image - id: retag-rechunk - if: steps.check-baseline.outputs.PULL_BASELINE_IMAGE == 'true' - env: - BASELINE_IMAGE: ${{ steps.pull-baseline.outputs.BASELINE_IMAGE }} - IMAGE_NAME: ${{ env.IMAGE_NAME }} + MATRIX_BASE_NAME: ${{ matrix.base_name }} DEFAULT_TAG: ${{ env.DEFAULT_TAG }} + MATRIX_IMAGE_FLAVOR: ${{ matrix.image_flavor }} run: | - sudo podman tag \ - ${BASELINE_IMAGE} \ - localhost/${IMAGE_NAME}:${DEFAULT_TAG} - sudo podman image rm -i ${BASELINE_IMAGE} + sudo -E $(command -v just) rechunk "${MATRIX_BASE_NAME}" \ + "${DEFAULT_TAG}" \ + "${MATRIX_IMAGE_FLAVOR}" - name: debug run: sudo podman images diff --git a/Justfile b/Justfile index 5afff1453..c9ecff4c2 100644 --- a/Justfile +++ b/Justfile @@ -282,6 +282,7 @@ rechunk $image="aurora" $tag="latest" $flavor="main" ghcr="0" pipeline="0": # Image Name image_name=$({{ just }} image_name {{ image }} {{ tag }} {{ flavor }}) + fedora_version=$({{ just }} fedora_version '{{ image }}' '{{ tag }}' '{{ flavor }}') # Check if image is already built ID=$(${PODMAN} images --filter reference=localhost/"${image_name}":"${tag}" --format "'{{ '{{.ID}}' }}'") @@ -289,27 +290,33 @@ rechunk $image="aurora" $tag="latest" $flavor="main" ghcr="0" pipeline="0": {{ just }} build "${image}" "${tag}" "${flavor}" fi - # Delete the rechunked image if present, rpm-ostree shits itself for whatever reason - # workaround for https://github.com/coreos/rpm-ostree/issues/5545 - if ${SUDOIF} ${PODMAN} image exists "localhost/${image_name}:${tag}-chunked"; then - ${SUDOIF} ${PODMAN} image rm -f "localhost/${image_name}:${tag}-chunked" - fi + ## Delete the rechunked image if present, rpm-ostree shits itself for whatever reason + ## workaround for https://github.com/coreos/rpm-ostree/issues/5545 + #if ${SUDOIF} ${PODMAN} image exists "localhost/${image_name}:${tag}-chunked"; then + # ${SUDOIF} ${PODMAN} image rm -f "localhost/${image_name}:${tag}-chunked" + #fi # Load into Rootful Podman - ID=$(${PODMAN} inspect --format={{ '{{.Digest}}' }} localhost/"${image_name}":"${tag}") - ID_ROOT=$(sudo ${PODMAN} inspect --format={{ '{{.Digest}}' }} localhost/"${image_name}":"${tag}") + ID_ROOT=$(${SUDOIF} ${PODMAN} images --filter reference=localhost/"${image_name}":"${tag}" --format "'{{ '{{.ID}}' }}'") if [[ ! "${PODMAN}" =~ "docker" ]] && [[ -n "$ID" ]] && [[ "$ID" != "$ID_ROOT" ]]; then ${PODMAN} image scp $(whoami)@localhost::localhost/"${image_name}":"${tag}" fi + # 96 layers, conservative default, same what ci-test is using + # 499 is podman run limit + # not using base-imagectl, to avoid pulling 2GiB image for a wrapper script ${SUDOIF} ${PODMAN} run --rm \ --pull=${PULL_POLICY} \ --privileged \ -v "/var/lib/containers:/var/lib/containers" \ - "quay.io/fedora/fedora-bootc:latest" \ - /usr/libexec/bootc-base-imagectl rechunk \ - "localhost/${image_name}":"${tag}" \ - "localhost/${image_name}":"${tag}-chunked" + --entrypoint /usr/bin/rpm-ostree \ + "${base_image_org}/${base_image_name}:${fedora_version}" \ + compose build-chunked-oci \ + --max-layers 96 \ + --format-version=2 \ + --bootc \ + --from "localhost/${image_name}":"${tag}" \ + --output containers-storage:"localhost/${image_name}":"${tag}-chunked" echo "::endgroup::" From e14b421f07d2a6184f3491ef35f9c8e2da224954 Mon Sep 17 00:00:00 2001 From: renner Date: Tue, 10 Feb 2026 17:40:28 +0100 Subject: [PATCH 6/6] fix: use -chunked suffix for local builds --- Justfile | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/Justfile b/Justfile index c9ecff4c2..61377b4e5 100644 --- a/Justfile +++ b/Justfile @@ -302,6 +302,14 @@ rechunk $image="aurora" $tag="latest" $flavor="main" ghcr="0" pipeline="0": ${PODMAN} image scp $(whoami)@localhost::localhost/"${image_name}":"${tag}" fi + + # In CI this will replace the unrechunked image + if [[ {{ ghcr }} == "1" ]]; then + CHUNKED_IMAGE="localhost/"${image_name}":"${tag}"" + else + CHUNKED_IMAGE="localhost/"${image_name}":"${tag}"-chunked" + fi + # 96 layers, conservative default, same what ci-test is using # 499 is podman run limit # not using base-imagectl, to avoid pulling 2GiB image for a wrapper script @@ -315,8 +323,8 @@ rechunk $image="aurora" $tag="latest" $flavor="main" ghcr="0" pipeline="0": --max-layers 96 \ --format-version=2 \ --bootc \ - --from "localhost/${image_name}":"${tag}" \ - --output containers-storage:"localhost/${image_name}":"${tag}-chunked" + --from "localhost/"${image_name}":"${tag}"" \ + --output containers-storage:${CHUNKED_IMAGE} echo "::endgroup::"