feature: use Rosetta AOT Caching with CDI

This change introduces device configuration to enable Rosetta AOT Caching in Docker VMs.

- Modify Rosetta Caching Options from Abstract Socket to Unix Domain Socket:
  Unix Domain Socket can be mounted within a container using the Container Device Interface (CDI) mechanism.
  This requires merging the following pull request: Code-Hex/vz#195.

- Register Rosettad AOT Caching Daemon as a service:
  - `/etc/systemd/system/rosettad.service` on systemd
  - `/etc/init.d/rosettad` on OpenRC

- Add "lima.io/rosetta=cached" device specification to `{~/.config,/etc}/cdi/rosetta.yaml`
  see: https://github.com/cncf-tags/container-device-interface/blob/main/SPEC.md

- Add `{~/.config,/etc}/docker/daemon.json` to `docker{,-rootful}.yaml`
  - `.features.cdi = true` to enable CDI

- Add `enable_cdi = true` to `{~/.config,/etc}/containerd/config.toml`

To enable Rosetta AOT Caching in docker, use `--device=lima.io/rosetta=cached`.
see: https://docs.docker.com/build/building/cdi/

Signed-off-by: Norio Nomura <norio.nomura@gmail.com>

Author: norio-nomura

pkg/cidata/cidata.TEMPLATE.d/boot/40-install-containerd.sh

@@ -45,6 +45,8 @@ if [ "${LIMA_CIDATA_CONTAINERD_SYSTEM}" = 1 ]; then
 	mkdir -p /etc/containerd /etc/buildkit
 	cat >"/etc/containerd/config.toml" <<EOF
   version = 2
+  [plugins."io.containerd.grpc.v1.cri"]
+    enable_cdi = true
   [proxy_plugins]
     [proxy_plugins."stargz"]
       type = "snapshot"
@@ -67,6 +69,8 @@ if [ "${LIMA_CIDATA_CONTAINERD_USER}" = 1 ]; then
 		mkdir -p "${LIMA_CIDATA_HOME}/.config/containerd"
 		cat >"${LIMA_CIDATA_HOME}/.config/containerd/config.toml" <<EOF
   version = 2
+  [plugins."io.containerd.grpc.v1.cri"]
+    enable_cdi = true
   [proxy_plugins]
     [proxy_plugins."fuse-overlayfs"]
       type = "snapshot"

pkg/driver/vz/boot/05-rosetta-volume.sh

@@ -31,3 +31,81 @@ else
 	# remove binfmt.d(5) configuration if it exists
 	[ ! -f "$binfmtd_conf" ] || rm "$binfmtd_conf"
 fi
+
+if [ -x /mnt/lima-rosetta/rosettad ]; then
+	CACHE_DIRECTORY=/var/cache/rosettad
+	DEFAULT_SOCKET=${CACHE_DIRECTORY}/uds/rosetta.sock
+	EXPECTED_SOCKET=/run/rosettad/rosetta.sock
+
+	# Create rosettad service
+	if [ -f /sbin/openrc-run ]; then
+		cat >/etc/init.d/rosettad <<EOF
+#!/sbin/openrc-run
+name="rosettad"
+description="Rosetta AOT Caching Daemon"
+required_dirs=/mnt/lima-rosetta
+required_files=/mnt/lima-rosetta/rosettad
+command=/mnt/lima-rosetta/rosettad
+command_args="daemon ${CACHE_DIRECTORY}"
+command_background=true
+pidfile="/run/rosettad.pid"
+start_pre() {
+	# To detect creation of the socket by rosettad, remove the old socket before starting
+	test ! -e "${DEFAULT_SOCKET}" || rm -f "${DEFAULT_SOCKET}"
+}
+start_post() {
+	# Set the socket permission to world-writable
+	while ! chmod -f go+w "${DEFAULT_SOCKET}"; do sleep 1; done
+	# Create the symlink as expected by the configuration to enable Rosetta AOT caching
+	mkdir -p "$(dirname "${EXPECTED_SOCKET}")"
+	ln -sf "${DEFAULT_SOCKET}" "${EXPECTED_SOCKET}"
+}
+EOF
+		chmod 755 /etc/init.d/rosettad
+		rc-update add rosettad default
+		rc-service rosettad start
+	else
+		cat >/etc/systemd/system/rosettad.service <<EOF
+[Unit]
+Description=Rosetta AOT Caching Daemon
+RequiresMountsFor=/mnt/lima-rosetta
+[Service]
+RuntimeDirectory=rosettad
+CacheDirectory=rosettad
+# To detect creation of the socket by rosettad, remove the old socket
+ExecStartPre=sh -c "test ! -e \"${DEFAULT_SOCKET}\" || rm -f \"${DEFAULT_SOCKET}\""
+ExecStart=/mnt/lima-rosetta/rosettad daemon "${CACHE_DIRECTORY}"
+# Set the socket permission to world-writable and create the symlink as expected by the configuration to enable Rosetta AOT caching.
+ExecStartPost=sh -c "while ! chmod -f go+w \"${DEFAULT_SOCKET}\"; do sleep 1; done; ln -sf \"${DEFAULT_SOCKET}\" \"${EXPECTED_SOCKET}\""
+OOMPolicy=continue
+OOMScoreAdjust=-500
+[Install]
+WantedBy=default.target
+EOF
+		systemctl is-enabled rosettad || systemctl enable --now rosettad
+	fi
+
+	# Create CDI configuration for Rosetta
+	mkdir -p /etc/cdi /var/run/cdi /etc/buildkit/cdi
+	cat >/etc/cdi/rosetta.yaml <<EOF
+cdiVersion: "0.6.0"
+kind: "lima-vm.io/rosetta"
+devices:
+- name: cached
+  containerEdits:
+    mounts:
+    - hostPath: /var/cache/rosettad/uds/rosetta.sock
+      containerPath: /run/rosettad/rosetta.sock
+      options: [bind]
+annotations:
+  org.mobyproject.buildkit.device.autoallow: true
+EOF
+	# nerdctl requires user-specific CDI configuration directories
+	mkdir -p "${LIMA_CIDATA_HOME}/.config/cdi"
+	ln -sf /etc/cdi/rosetta.yaml "${LIMA_CIDATA_HOME}/.config/cdi/"
+	chown -R "${LIMA_CIDATA_USER}" "${LIMA_CIDATA_HOME}/.config"
+else
+	# Remove CDI configuration for Rosetta AOT Caching
+	[ ! -f /etc/cdi/rosetta.yaml ] || rm /etc/cdi/rosetta.yaml
+	[ ! -d "${LIMA_CIDATA_HOME}/.config/cdi/rosetta.yaml" ] || rm "${LIMA_CIDATA_HOME}/.config/cdi/rosetta.yaml"
+fi

pkg/driver/vz/rosetta_directory_share_arm64.go

@@ -44,7 +44,7 @@ func createRosettaDirectoryShareConfiguration() (*vz.VirtioFileSystemDeviceConfi
 		return nil, fmt.Errorf("failed to get macOS product version: %w", err)
 	}
 	if !macOSProductVersion.LessThan(*semver.New("14.0.0")) {
-		cachingOption, err := vz.NewLinuxRosettaAbstractSocketCachingOptions("rosetta")
+		cachingOption, err := vz.NewLinuxRosettaUnixSocketCachingOptions("/run/rosettad/rosetta.sock")
 		if err != nil {
 			return nil, fmt.Errorf("failed to create a new rosetta directory share caching option: %w", err)
 		}

templates/docker-rootful.yaml

@@ -38,6 +38,17 @@ provision:
       SocketUser={{.User}}
     EOF
     fi
+    daemon_json="/etc/docker/daemon.json"
+    if [ ! -f "${daemon_json}" ]; then
+      mkdir -p "$(dirname "${daemon_json}")"
+      cat > "${daemon_json}" <<EOF
+    {
+      "features": {
+        "cdi": true
+      }
+    }
+    EOF
+    fi
     export DEBIAN_FRONTEND=noninteractive
     curl -fsSL https://get.docker.com | sh
 probes:

templates/docker.yaml

@@ -39,6 +39,17 @@ provision:
   script: |
     #!/bin/bash
     set -eux -o pipefail
+    daemon_json="{{.Home}}/.config/docker/daemon.json"
+    if [ ! -f "${daemon_json}" ]; then
+      mkdir -p "$(dirname "${daemon_json}")"
+      cat > "${daemon_json}" <<EOF
+    {
+      "features": {
+        "cdi": true
+      }
+    }
+    EOF
+    fi
     systemctl --user start dbus
     dockerd-rootless-setuptool.sh install
     docker context use rootless