wip: l2 images

.gitignore

@@ -1,11 +1,22 @@
+# mkosi artifacts
+
 build/
-mkosi/
+build.*/
 env.json
-mkosi.packages/
-mkosi.cache/
 mkosi.builddir/
-*.qcow2
-.claudesync/
-.claudeignore
+mkosi.cache/
+mkosi.packages/
+mkosi/
+
+# temporary files
+
+.temp
 tmp/
+
+# IDEs/agents/whatnot
+
+.claudeignore
+.claudesync/
+.vscode
+*.qcow2
 NvVars

Makefile

@@ -37,16 +37,52 @@ check-perms: ## Check repository permissions
 setup: ## Install dependencies (Linux only)
 	@scripts/setup_deps.sh
 
+preflight:
+	@$(WRAPPER) echo "Ready to build"
+
 # Build module
 build: check-perms setup ## Build the specified module
-	$(WRAPPER) mkosi --force -I $(IMAGE).conf
+	time $(WRAPPER) mkosi --force -I $(patsubst %.conf,%,$(IMAGE)).conf
 
 # Build module with devtools profile
 build-dev: check-perms setup ## Build module with development tools
-	$(WRAPPER) mkosi --force --profile=devtools -I $(IMAGE).conf
+	time $(WRAPPER) mkosi --force --profile=devtools $(patsubst %.conf,%,$(IMAGE)).conf
 
 ##@ Utilities
 
+check-repro: ## Build same module twice and compare resulting images
+	@rm -rf build.1
+	@rm -rf build.2
+
+	@rm -rf build/* mkosi.builddir/* mkosi.cache/* mkosi.packages/*
+#	hack:  there's some race condition under lima that causes apt to fail while trying to
+#	       create a temp dir under apt cache
+	@sleep 15
+
+	@echo "Building image #1..."
+	time $(WRAPPER) mkosi --force -I $(patsubst %.conf,%,$(IMAGE)).conf
+	@mkdir -p build/cache
+	@mv mkosi.builddir/* build/cache/
+	@mv build build.1
+
+	@rm -rf build/* mkosi.builddir/* mkosi.cache/* mkosi.packages/*
+#	hack:  there's some race condition under lima that causes apt to fail while trying to
+#	       create a temp dir under apt cache
+	@sleep 15
+
+	@echo "Building image #2..."
+	time $(WRAPPER) mkosi --force -I $(patsubst %.conf,%,$(IMAGE)).conf
+	@mkdir -p build/cache
+	@mv mkosi.builddir/* build/cache/
+	@mv build build.2
+
+	@echo "Comparing..."
+	@for file in $$( find build.1 -type f ); do \
+		sha256sum $$file; \
+		sha256sum $${file/build1/build.2}; \
+		echo ""; \
+	done
+
 measure: ## Export TDX measurements for the built EFI file
 	@if [ ! -f build/tdx-debian.efi ]; then \
 		echo "Error: build/tdx-debian.efi not found. Run 'make build' first."; \
@@ -60,8 +96,7 @@ measure-gcp: ## Export TDX measurements for GCP
 		echo "Error: build/tdx-debian.efi not found. Run 'make build' first."; \
 		exit 1; \
 	fi
-	@$(WRAPPER) dstack-mr -uki build/tdx-debian.efi -json > build/gcp_measurements.json
-	echo "GCP Measurements exported to build/gcp_measurements.json"
+	@$(WRAPPER) dstack-mr -uki build/tdx-debian.efi
 
 # Clean build artifacts
 clean: ## Remove cache and build artifacts

README.md

@@ -26,7 +26,10 @@ For more information about this repository, see
 
 ### Prerequisites
 
-In order to build images, you'll need to install [Lima](https://lima-vm.io/) for your operating system. Building images without Lima is possible, but due to inconsistencies between distributions, it is not supported for generating official reproducible images.
+In order to build images, you'll need to install [Lima](https://lima-vm.io/) for
+your operating system. Building images without Lima is possible, but due to
+inconsistencies between distributions, it is not supported for generating
+official reproducible images.
 
 ### Building Images
 
@@ -37,8 +40,8 @@ make build IMAGE=bob
 # Build the Buildernet image
 make build IMAGE=buildernet
 
-# Build the l2 builder image
-make build IMAGE=l2-builder
+# Build the l2 op-rbuilder image
+make build IMAGE=l2/op-builder
 
 # Build with development tools
 make build-dev IMAGE=bob
@@ -94,7 +97,7 @@ This generates measurement files in the `build/` directory for attestation and v
   ```
 
 > [!NOTE]
-> 
+>
 > Depending on your Linux distro, these commands may require changing the
 > supplied OVMF paths or installing your distro's OVMF package.
 

base/add-backports.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# The mkosi sandbox environment should have a debian backports source list
+# that matches the archive timestamp of the main release.
+# See https://github.com/systemd/mkosi/issues/1755
+MIRROR=$(jq -r .Mirror /work/config.json)
+if [ "$MIRROR" = "null" ]; then
+    MIRROR="http://deb.debian.org/debian"
+fi
+
+cat > "$SRCDIR/mkosi.builddir/debian-backports.sources" <<EOF
+Types: deb deb-src
+URIs: $MIRROR
+Suites: ${RELEASE}-backports
+Components: main
+Enabled: yes
+Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg
+EOF

base/debloat.sh

@@ -1,6 +1,11 @@
 #!/bin/bash
 set -euo pipefail
 
+# Ensure deterministic ordering of uid and gids before debloating
+# See Debian issue #963788
+mkosi-chroot pwck  --sort >/dev/null
+mkosi-chroot grpck --sort >/dev/null
+
 # Remove all logs and cache, but keep directory structure intact
 find "$BUILDROOT/var/log" -type f -delete
 find "$BUILDROOT/var/cache" -type f -delete
@@ -34,12 +39,13 @@ debloat_paths=(
     "/usr/lib/systemd/network"
     "/usr/lib/pcrlock.d"
     "/usr/lib/tmpfiles.d"
+    "/var/lib/ucf"
     "/etc/systemd/network"
     "/etc/credstore"
     "/nix"
 )
 
-if [[ ! "$PROFILES" == *"devtools"* ]]; then
+if [[ ! "${PROFILES:-}" == *"devtools"* ]]; then
     debloat_paths+=(
         "/usr/share/bash-completion"
     )

base/mkosi.conf

@@ -5,6 +5,7 @@ Release=trixie
 
 [Build]
 PackageCacheDirectory=mkosi.cache
+SandboxTrees=mkosi.builddir/debian-backports.sources:/etc/apt/sources.list.d/debian-backports.sources
 Environment=KERNEL_IMAGE KERNEL_VERSION
 WithNetwork=true
 
@@ -23,6 +24,7 @@ FinalizeScripts=base/debloat.sh
 PostInstallationScripts=base/debloat-systemd.sh
 PostInstallationScripts=base/efi-stub.sh
 BuildScripts=kernel/mkosi.build
+SyncScripts=base/add-backports.sh
 
 CleanPackageMetadata=true
 Packages=kmod

flake.nix

@@ -44,10 +44,10 @@
       src = pkgs.fetchFromGitHub {
         owner = "flashbots";
         repo = "dstack-mr-gcp";
-        rev = "3d718ab28599ea0c05e65d0f742fdee9fc17a5c7";
-        sha256 = "sha256-KFo9wcQuG98Hi4mlMr5VS6D6/STW7jzZ9y1DyqsI820=";
+        rev = "503e7c506f89f9d81be04025c90921778b26f0a4";
+        sha256 = "sha256-z6STTgcOXatiqA2rlpzwRyvAwnXrK30oNDCJqtIp7/8=";
       };
-      vendorHash = "sha256-MxOQSXLAbWC1SOCPzPrNcU20WElbe7eUVdCLTutSYM8=";
+      vendorHash = "sha256-glOyRTrIF/zP78XGV+v58a1Bec6C3Fvc5c8G3PglzPM=";
     };
     mkosi = system: let
       pkgsForSystem = import nixpkgs {inherit system;};
@@ -71,6 +71,7 @@
             qemu-utils
             parted
             unzip
+            jq
           ]
           ++ [reprepro];
       };
@@ -92,6 +93,7 @@
         nativeBuildInputs = [(mkosi system) measured-boot measured-boot-gcp];
         shellHook = ''
           mkdir -p mkosi.packages mkosi.cache mkosi.builddir ~/.cache/mkosi
+          touch mkosi.builddir/debian-backports.sources
         '';
       };
     }) ["x86_64-linux" "aarch64-linux"]);

kernel/kernel-yocto.config

@@ -2,7 +2,7 @@
 # Automatically generated file; DO NOT EDIT.
 # Linux/x86 6.6.35 Kernel Configuration
 #
-CONFIG_CC_VERSION_TEXT="x86_64-poky-linux-gcc (GCC) 13.3.0"
+CONFIG_CC_VERSION_TEXT="gcc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0"
 CONFIG_CC_IS_GCC=y
 CONFIG_GCC_VERSION=130300
 CONFIG_CLANG_VERSION=0
@@ -11,12 +11,14 @@ CONFIG_AS_VERSION=24200
 CONFIG_LD_IS_BFD=y
 CONFIG_LD_VERSION=24200
 CONFIG_LLD_VERSION=0
+CONFIG_CC_CAN_LINK=y
+CONFIG_CC_CAN_LINK_STATIC=y
 CONFIG_CC_HAS_ASM_GOTO_OUTPUT=y
 CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT=y
 CONFIG_TOOLS_SUPPORT_RELR=y
 CONFIG_CC_HAS_ASM_INLINE=y
 CONFIG_CC_HAS_NO_PROFILE_FN_ATTR=y
-CONFIG_PAHOLE_VERSION=125
+CONFIG_PAHOLE_VERSION=0
 CONFIG_IRQ_WORK=y
 CONFIG_BUILDTIME_TABLE_SORT=y
 CONFIG_THREAD_INFO_IN_TASK=y
@@ -39,7 +41,7 @@ CONFIG_HAVE_KERNEL_LZ4=y
 CONFIG_HAVE_KERNEL_ZSTD=y
 # CONFIG_KERNEL_GZIP is not set
 # CONFIG_KERNEL_BZIP2 is not set
-CONFIG_KERNEL_LZMA=y
+# CONFIG_KERNEL_LZMA is not set
 # CONFIG_KERNEL_XZ is not set
 # CONFIG_KERNEL_LZO is not set
 # CONFIG_KERNEL_LZ4 is not set
@@ -316,7 +318,6 @@ CONFIG_X86_X2APIC=y
 # CONFIG_X86_MPPARSE is not set
 # CONFIG_GOLDFISH is not set
 # CONFIG_X86_CPU_RESCTRL is not set
-# CONFIG_QEMUX86 is not set
 CONFIG_X86_EXTENDED_PLATFORM=y
 # CONFIG_X86_VSMP is not set
 # CONFIG_X86_GOLDFISH is not set
@@ -747,7 +748,6 @@ CONFIG_ARCH_HAS_GCOV_PROFILE_ALL=y
 # end of GCOV-based kernel profiling
 
 CONFIG_HAVE_GCC_PLUGINS=y
-# CONFIG_GCC_PLUGINS is not set
 CONFIG_FUNCTION_ALIGNMENT_4B=y
 CONFIG_FUNCTION_ALIGNMENT_16B=y
 CONFIG_FUNCTION_ALIGNMENT=16
@@ -1178,6 +1178,7 @@ CONFIG_NF_DEFRAG_IPV6=y
 # CONFIG_NF_CONNTRACK_BRIDGE is not set
 # CONFIG_BRIDGE_NF_EBTABLES is not set
 # CONFIG_BPFILTER is not set
+# CONFIG_IP_DCCP is not set
 # CONFIG_IP_SCTP is not set
 # CONFIG_RDS is not set
 # CONFIG_TIPC is not set
@@ -3942,10 +3943,8 @@ CONFIG_OVERLAY_FS_METACOPY=y
 CONFIG_FAT_FS=y
 # CONFIG_MSDOS_FS is not set
 CONFIG_VFAT_FS=y
-# CONFIG_VFAT_FS_NO_DUALNAMES is not set
 CONFIG_FAT_DEFAULT_CODEPAGE=437
 CONFIG_FAT_DEFAULT_IOCHARSET="iso8859-1"
-# CONFIG_VFAT_NO_CREATE_WITH_LONGNAMES is not set
 # CONFIG_FAT_DEFAULT_UTF8 is not set
 # CONFIG_EXFAT_FS is not set
 # CONFIG_NTFS_FS is not set
@@ -3978,7 +3977,6 @@ CONFIG_EFIVAR_FS=y
 CONFIG_NETWORK_FILESYSTEMS=y
 CONFIG_NFS_FS=y
 # CONFIG_NFS_V2 is not set
-CONFIG_NFS_DEF_FILE_IO_SIZE=4096
 CONFIG_NFS_V3=y
 # CONFIG_NFS_V3_ACL is not set
 CONFIG_NFS_V4=y
@@ -4386,10 +4384,14 @@ CONFIG_CRC32_SLICEBY8=y
 # CONFIG_CRC7 is not set
 CONFIG_LIBCRC32C=m
 CONFIG_CRC8=m
+CONFIG_XXHASH=y
 # CONFIG_RANDOM32_SELFTEST is not set
 CONFIG_ZLIB_INFLATE=y
+CONFIG_ZSTD_COMMON=y
+CONFIG_ZSTD_DECOMPRESS=y
 # CONFIG_XZ_DEC is not set
 CONFIG_DECOMPRESS_GZIP=y
+CONFIG_DECOMPRESS_ZSTD=y
 CONFIG_GENERIC_ALLOCATOR=y
 CONFIG_INTERVAL_TREE=y
 CONFIG_ASSOCIATIVE_ARRAY=y
@@ -4471,10 +4473,9 @@ CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 # CONFIG_DEBUG_INFO_REDUCED is not set
 CONFIG_DEBUG_INFO_COMPRESSED_NONE=y
 # CONFIG_DEBUG_INFO_COMPRESSED_ZLIB is not set
+# CONFIG_DEBUG_INFO_COMPRESSED_ZSTD is not set
 # CONFIG_DEBUG_INFO_SPLIT is not set
 # CONFIG_DEBUG_INFO_BTF is not set
-CONFIG_PAHOLE_HAS_SPLIT_BTF=y
-CONFIG_PAHOLE_HAS_LANG_EXCLUDE=y
 # CONFIG_GDB_SCRIPTS is not set
 CONFIG_FRAME_WARN=2048
 # CONFIG_STRIP_ASM_SYMS is not set

kernel/mkosi.build

@@ -15,9 +15,20 @@ if [[ -n "${KERNEL_CONFIG_SNIPPETS:-}" ]]; then
     IFS=',' read -ra snippets <<< "$KERNEL_CONFIG_SNIPPETS"
     for snippet in "${snippets[@]}"; do
         snippet_file="$SRCDIR/$snippet"
-        [[ -f "$snippet_file" ]] && cat "$snippet_file" >> "$config_file"
+        if [[ -f "$snippet_file" ]]; then
+            cat "$snippet_file" >> "$config_file" || true
+        fi
     done
 fi
+for snippets_var in "${!KERNEL_CONFIG_SNIPPETS_@}"; do
+    IFS=',' read -ra snippets <<< "${!snippets_var}"
+    for snippet in "${snippets[@]}"; do
+        snippet_file="$SRCDIR/$snippet"
+        if [[ -f "$snippet_file" ]]; then
+            cat "$snippet_file" >> "$config_file" || true
+        fi
+    done
+done
 
 # Calculate cache key and paths
 config_hash=$(sha256sum "$config_file" | cut -d' ' -f1 | cut -c1-12)
@@ -37,12 +48,15 @@ else
     # Build kernel
     cd "$build_dir"
     cp "$config_file" .config
-    export KBUILD_BUILD_TIMESTAMP="$(date -u -d @${SOURCE_DATE_EPOCH:-$(date +%s)})"
+    export KBUILD_BUILD_TIMESTAMP="$(date -u -d @$(git log -1 --pretty=%ct))"
     export KBUILD_BUILD_USER="mkosi" KBUILD_BUILD_HOST="mkosi-builder"
 
     mkosi-chroot --chdir "/build/kernel-${KERNEL_VERSION}" make olddefconfig
     mkosi-chroot --chdir "/build/kernel-${KERNEL_VERSION}" make -j "$(nproc 2>/dev/null || echo 2)" bzImage ARCH=x86_64 CONFIG_EFI_STUB=y
 
+    echo "# kernel config:"
+    mkosi-chroot --chdir "/build/kernel-${KERNEL_VERSION}" cat .config
+
     # Cache result
     mkdir -p "$cache_dir"
     cp arch/x86_64/boot/bzImage "$cache_dir/"