Rework sealed build process

Main goal is to reduce signing logic duplication between the systemd-boot
and UKI generation.

However, this quickly snowballed into wanting to actually verify
by providing a custom secure boot keys to bcvk that things worked.
This depends on bootc-dev/bcvk#170

Now as part of that, I ran into what I think are bugs in pesign;
this cuts things back over to using sbsign. I'll file a tracker for that
separately.

Finally as part of this, just remove the TMT example that builds
a sealed image but doesn't actually verify it works - it's already
drifted from what we do outside here. Ultimately what we need
is to shift some of this into the Fedora examples and we just
fetch it here anyways.

Assisted-by: Claude Code (Sonnet 4.5)
Signed-off-by: Colin Walters <walters@verbum.org>

Author: cgwalters

.github/actions/bootc-ubuntu-setup/action.yml

@@ -65,19 +65,24 @@ runs:
     - name: Install libvirt and virtualization stack
       if: ${{ inputs.libvirt == 'true' }}
       shell: bash
+      env:
+        GH_TOKEN: ${{ github.token }}
       run: |
         set -xeuo pipefail
         export BCVK_VERSION=0.8.0
-        /bin/time -f '%E %C' sudo apt install -y libkrb5-dev pkg-config libvirt-dev genisoimage qemu-utils qemu-kvm virtiofsd libvirt-daemon-system
+        /bin/time -f '%E %C' sudo apt install -y libkrb5-dev pkg-config libvirt-dev genisoimage qemu-utils qemu-kvm virtiofsd libvirt-daemon-system python3-virt-firmware
         # Something in the stack is overriding this, but we want session right now for bcvk
         echo LIBVIRT_DEFAULT_URI=qemu:///session >> $GITHUB_ENV
         td=$(mktemp -d)
         cd $td
-        # Install bcvk
-        target=bcvk-$(arch)-unknown-linux-gnu
-        /bin/time -f '%E %C' curl -LO https://github.com/bootc-dev/bcvk/releases/download/v${BCVK_VERSION}/${target}.tar.gz
-        tar xzf ${target}.tar.gz
-        sudo install -T ${target} /usr/bin/bcvk
+        # Install bcvk from PR 172
+        gh run download 20107212783 --name bcvk-binary-tests --repo bootc-dev/bcvk
+        sudo install -m 755 bcvk /usr/bin/bcvk
+        # Install bcvk from release
+        # target=bcvk-$(arch)-unknown-linux-gnu
+        # /bin/time -f '%E %C' curl -LO https://github.com/bootc-dev/bcvk/releases/download/v${BCVK_VERSION}/${target}.tar.gz
+        # tar xzf ${target}.tar.gz
+        # sudo install -T ${target} /usr/bin/bcvk
         cd -
         rm -rf "$td"
 

Dockerfile

@@ -41,12 +41,28 @@ WORKDIR /src
 # First we download all of our Rust dependencies
 RUN --mount=type=cache,target=/src/target --mount=type=cache,target=/var/roothome cargo fetch
 
+FROM buildroot as sdboot-content
+# Writes to /out
+RUN /src/contrib/packaging/configure-systemdboot download
+
+# NOTE: Every RUN instruction past this point should use `--network=none`; we want to ensure
+# all external dependencies are clearly delineated.
+
 FROM buildroot as build
 # Version for RPM build (optional, computed from git in Justfile)
 ARG pkgversion
 # Build RPM directly from source, using cached target directory
 RUN --mount=type=cache,target=/src/target --mount=type=cache,target=/var/roothome --network=none RPM_VERSION="${pkgversion}" /src/contrib/packaging/build-rpm
 
+FROM buildroot as sdboot-signed
+# The secureboot key and cert are passed via Justfile
+# We write the signed binary into /out
+RUN --network=none \
+    --mount=type=bind,from=sdboot-content,target=/run/sdboot-package \
+    --mount=type=secret,id=secureboot_key \
+    --mount=type=secret,id=secureboot_cert \
+    /src/contrib/packaging/configure-systemdboot sign
+
 # This "build" includes our unit tests
 FROM build as units
 # A place that we're more likely to be able to set xattrs
@@ -62,7 +78,10 @@ RUN --mount=type=cache,target=/src/target --mount=type=cache,target=/var/roothom
 FROM base
 # See the Justfile for possible variants
 ARG variant
-RUN --mount=type=bind,from=packaging,target=/run/packaging /run/packaging/configure-variant "${variant}"
+RUN --network=none --mount=type=bind,from=packaging,target=/run/packaging \
+    --mount=type=bind,from=sdboot-content,target=/run/sdboot-content \
+    --mount=type=bind,from=sdboot-signed,target=/run/sdboot-signed \
+    /run/packaging/configure-variant "${variant}"
 # Support overriding the rootfs at build time conveniently
 ARG rootfs
 RUN --mount=type=bind,from=packaging,target=/run/packaging /run/packaging/configure-rootfs "${variant}" "${rootfs}"

Dockerfile.cfsuki

@@ -1,27 +1,25 @@
 # Override via --build-arg=base=<image> to use a different base
 ARG base=localhost/bootc
-# This is where we get the tools to build the UKI
-ARG buildroot=quay.io/centos/centos:stream10
-
 FROM $base AS base
 
-FROM $buildroot as buildroot-base
+FROM base as kernel
 RUN <<EORUN
 set -xeuo pipefail
-
-# systemd-udev is required for /usr/lib/systemd/systemd-measure which
-# is used by ukify as invoked with the `--measure` flag below.  Not
-# strictly required, but nice to have the measured PCR values in the
-# output.
-dnf install -y systemd-ukify systemd-udev pesign openssl
-dnf clean all
+. /usr/lib/os-release
+case $ID in
+  centos|rhel)
+    dnf config-manager --set-enabled crb
+    # Enable EPEL for sbsigntools
+    dnf -y install epel-release
+    ;;
+esac
+dnf -y install systemd-ukify sbsigntools
 EORUN
-
-FROM buildroot-base as kernel
 # Must be passed
 ARG COMPOSEFS_FSVERITY
-RUN --mount=type=secret,id=key \
-    --mount=type=secret,id=cert \
+RUN --network=none \
+    --mount=type=secret,id=secureboot_key \
+    --mount=type=secret,id=secureboot_cert \
     --mount=type=bind,from=base,target=/target \
      <<EOF
     set -xeuo pipefail
@@ -31,30 +29,24 @@ RUN --mount=type=secret,id=key \
 
     cmdline="composefs=${COMPOSEFS_FSVERITY} console=ttyS0,115200n8 console=hvc0 enforcing=0 rw"
 
-    # pesign uses NSS database so create it from input cert/key
-    mkdir pesign
-    certutil -N -d pesign --empty-password
-    openssl pkcs12 -export -password 'pass:' -inkey /run/secrets/key -in /run/secrets/cert -out db.p12
-    pk12util -i db.p12 -W '' -d pesign
-    subject=$(openssl x509 -in /run/secrets/cert -subject | grep '^subject=CN=' | sed 's/^subject=CN=//')
-
+    # Use sbsign to re-sign the entire UKI with our key
     kver=$(cd /target/usr/lib/modules && echo *)
     ukify build \
         --linux "/target/usr/lib/modules/$kver/vmlinuz" \
         --initrd "/target/usr/lib/modules/$kver/initramfs.img" \
         --uname="${kver}" \
         --cmdline "${cmdline}" \
         --os-release "@/target/usr/lib/os-release" \
-        --signtool pesign \
-        --secureboot-certificate-dir "pesign" \
-        --secureboot-certificate-name "${subject}" \
+        --signtool sbsign \
+        --secureboot-private-key "/run/secrets/secureboot_key" \
+        --secureboot-certificate "/run/secrets/secureboot_cert" \
         --measure \
         --json pretty \
         --output "/boot/$kver.efi"
 EOF
 
 FROM base as final
-RUN --mount=type=bind,from=kernel,target=/run/kernel <<EOF
+RUN --network=none --mount=type=bind,from=kernel,target=/run/kernel <<EOF
 set -xeuo pipefail
 kver=$(cd /usr/lib/modules && echo *)
 mkdir -p /boot/EFI/Linux

Dockerfile.sdboot

@@ -25,6 +25,8 @@ base := env("BOOTC_base", "quay.io/centos-bootc/centos-bootc:stream10")
 buildroot_base := env("BOOTC_buildroot_base", "quay.io/centos/centos:stream10")
 
 testimage_label := "bootc.testimage=1"
+# Images used by hack/lbi; keep in sync
+lbi_images := "quay.io/curl/curl:latest quay.io/curl/curl-base:latest registry.access.redhat.com/ubi9/podman:latest"
 # We used to have --jobs=4 here but sometimes that'd hit this
 # ```
 #   [2/3] STEP 2/2: RUN --mount=type=bind,from=context,target=/run/context <<EORUN (set -xeuo pipefail...)
@@ -34,21 +36,23 @@ testimage_label := "bootc.testimage=1"
 #   /bin/sh: line 3: cd: /run/context/: Permission denied
 # ```
 # TODO: Gather more info and file a buildah bug
-base_buildargs := ""
-buildargs := "--build-arg=base=" + base + " --build-arg=variant=" + variant
-
-# Build the container image from current sources.
+generic_buildargs := ""
+# Args for package building (no secrets needed, just builds RPMs)
+base_buildargs := generic_buildargs + " --build-arg=base=" + base + " --build-arg=variant=" + variant
+buildargs := base_buildargs + " --secret=id=secureboot_key,src=target/test-secureboot/db.key --secret=id=secureboot_cert,src=target/test-secureboot/db.crt"
+# Args for build-sealed (no base arg, it sets that itself)
+sealed_buildargs := "--build-arg=variant=" + variant + " --secret=id=secureboot_key,src=target/test-secureboot/db.key --secret=id=secureboot_cert,src=target/test-secureboot/db.crt"
+
+# The default target: build the container image from current sources.
 # Note commonly you might want to override the base image via e.g.
 # `just build --build-arg=base=quay.io/fedora/fedora-bootc:42`
-build: package
+build: package _keygen
     podman build {{base_buildargs}} -t {{base_img}}-bin {{buildargs}} .
-    ./tests/build-sealed {{variant}} {{base_img}}-bin {{base_img}} {{buildroot_base}}
+    ./hack/build-sealed {{variant}} {{base_img}}-bin {{base_img}} {{sealed_buildargs}}
 
-# Build the container image using pre-existing packages from PATH
-build-from-package PATH:
-    # @just copy-packages-from {{PATH}}
-    podman build {{base_buildargs}} -t {{base_img}}-bin {{buildargs}} .
-    ./tests/build-sealed {{variant}} {{base_img}}-bin {{base_img}} {{buildroot_base}}
+# Generate Secure Boot keys (only for our own CI/testing)
+_keygen:
+    ./hack/generate-secureboot-keys
 
 # Build a sealed image from current sources.
 build-sealed:
@@ -69,7 +73,7 @@ _packagecontainer:
         VERSION="${TIMESTAMP}.g${COMMIT}"
     fi
     echo "Building RPM with version: ${VERSION}"
-    podman build {{base_buildargs}} {{buildargs}} --build-arg=pkgversion=${VERSION} -t localhost/bootc-pkg --target=build .
+    podman build {{base_buildargs}} --build-arg=pkgversion=${VERSION} -t localhost/bootc-pkg --target=build .
 
 # Build packages (e.g. RPM) into target/packages/
 # Any old packages will be removed.
@@ -96,20 +100,28 @@ copy-packages-from PATH:
     chmod a+rx target target/packages
     chmod a+r target/packages/*.rpm
 
+# Build the container image using pre-existing packages from PATH
+# Note: The Dockerfile reads from target/packages/, so copy the packages there first
+# if they're in a different location.
+build-from-package PATH: _keygen
+    @if [ "{{PATH}}" != "target/packages" ]; then just copy-packages-from {{PATH}}; fi
+    podman build {{base_buildargs}} -t {{base_img}}-bin {{buildargs}} .
+    ./hack/build-sealed {{variant}} {{base_img}}-bin {{base_img}} {{sealed_buildargs}}
+
+# Pull images used by hack/lbi
+_pull-lbi-images:
+    podman pull -q --retry 5 --retry-delay 5s {{lbi_images}}
+
 # This container image has additional testing content and utilities
-build-integration-test-image: build
+build-integration-test-image: build _pull-lbi-images
     cd hack && podman build {{base_buildargs}} -t {{integration_img}}-bin -f Containerfile .
-    ./tests/build-sealed {{variant}} {{integration_img}}-bin {{integration_img}} {{buildroot_base}}
-    # Keep these in sync with what's used in hack/lbi
-    podman pull -q --retry 5 --retry-delay 5s quay.io/curl/curl:latest quay.io/curl/curl-base:latest registry.access.redhat.com/ubi9/podman:latest
+    ./hack/build-sealed {{variant}} {{integration_img}}-bin {{integration_img}} {{sealed_buildargs}}
 
 # Build integration test image using pre-existing packages from PATH
-build-integration-test-image-from-package PATH:
+build-integration-test-image-from-package PATH: _pull-lbi-images
     @just build-from-package {{PATH}}
     cd hack && podman build {{base_buildargs}} -t {{integration_img}}-bin -f Containerfile .
-    ./tests/build-sealed {{variant}} {{integration_img}}-bin {{integration_img}} {{buildroot_base}}
-    # Keep these in sync with what's used in hack/lbi
-    podman pull -q --retry 5 --retry-delay 5s quay.io/curl/curl:latest quay.io/curl/curl-base:latest registry.access.redhat.com/ubi9/podman:latest
+    ./hack/build-sealed {{variant}} {{integration_img}}-bin {{integration_img}} {{sealed_buildargs}}
 
 # Build+test using the `composefs-sealeduki-sdboot` variant.
 test-composefs:
@@ -146,7 +158,7 @@ test-tmt *ARGS: build-integration-test-image _build-upgrade-image
 # Generate a local synthetic upgrade
 _build-upgrade-image:
     cat tmt/tests/Dockerfile.upgrade | podman build -t {{integration_upgrade_img}}-bin --from={{integration_img}}-bin -
-    ./tests/build-sealed {{variant}} {{integration_upgrade_img}}-bin {{integration_upgrade_img}} {{buildroot_base}}
+    ./hack/build-sealed {{variant}} {{integration_upgrade_img}}-bin {{integration_upgrade_img}} {{sealed_buildargs}}
 
 # Assume the localhost/bootc-integration image is up to date, and just run tests.
 # Useful for iterating on tests quickly.

Justfile

@@ -0,0 +1,29 @@
+#!/bin/bash
+# Helper for signing and re-injecting systemd-boot
+set -euo pipefail
+op=$1
+shift
+
+sdboot="usr/lib/systemd/boot/efi/systemd-bootx64.efi"
+sdboot_bn=$(basename ${sdboot})
+
+case $op in 
+  download)
+    mkdir -p /out
+    cd /out
+    dnf -y download systemd-boot-unsigned
+    ;;
+  sign)
+    mkdir -p /out
+    rpm -Uvh /run/sdboot-package/out/*.rpm
+    # Sign with sbsign using db certificate and key
+    sbsign \
+           --key /run/secrets/secureboot_key \
+           --cert /run/secrets/secureboot_cert \
+           --output /out/${sdboot_bn} \
+           /${sdboot}
+    ls -al /out/${sdboot_bn}
+    ;;
+  *) echo "Unknown operation $op" 1>&2; exit 1
+  ;;
+esac

contrib/packaging/configure-systemdboot

@@ -2,6 +2,8 @@
 # Configure system for a specific bootc variant
 set -xeuo pipefail
 
+dn=$(dirname $0)
+
 VARIANT="${1:-}"
 
 if [ -z "$VARIANT" ]; then
@@ -12,8 +14,14 @@ fi
 # Handle variant-specific configuration
 case "${VARIANT}" in
   *-sdboot)
-    # Install systemd-boot and remove bootupd
-    dnf -y install systemd-boot-unsigned
+    # Install systemd-boot and remove bootupd;
+    # We downloaded this in an earlier phase
+    sdboot="usr/lib/systemd/boot/efi/systemd-bootx64.efi"
+    sdboot_bn=$(basename ${sdboot})
+    rpm -Uvh /run/sdboot-content/out/*.rpm
+      # And override with our signed binary
+    install -m 0644 /run/sdboot-signed/out/${sdboot_bn} /${sdboot}
+
     # Uninstall bootupd
     rpm -e bootupd
     rm -rf /usr/lib/bootupd/updates

contrib/packaging/configure-variant

@@ -7,3 +7,5 @@ git-core
 jq
 # We now always build a package in the container build
 rpm-build
+# Used for signing
+sbsigntools

contrib/packaging/fedora-extra.txt

@@ -0,0 +1,2 @@
+# This file defines the package name for systemd-boot
+systemd-boot-unsigned

contrib/packaging/fedora-systemd-boot.txt

@@ -4,7 +4,11 @@ set -xeuo pipefail
 cd $(dirname $0)
 . /usr/lib/os-release
 case $ID in
-  centos|rhel) dnf config-manager --set-enabled crb;;
+  centos|rhel)
+    dnf config-manager --set-enabled crb
+    # Enable EPEL for sbsigntools
+    dnf -y install epel-release
+    ;;
   fedora) dnf -y install dnf-utils 'dnf5-command(builddep)';;
 esac
 # Handle version skew, xref https://gitlab.com/redhat/centos-stream/containers/bootc/-/issues/1174