feature: use Rosetta AOT Caching with CDI

pkg/cidata/cidata.TEMPLATE.d/boot/40-install-containerd.sh

@@ -45,6 +45,8 @@ if [ "${LIMA_CIDATA_CONTAINERD_SYSTEM}" = 1 ]; then
 	mkdir -p /etc/containerd /etc/buildkit
 	cat >"/etc/containerd/config.toml" <<EOF
   version = 2
+  [plugins."io.containerd.grpc.v1.cri"]
+    enable_cdi = true
   [proxy_plugins]
     [proxy_plugins."stargz"]
       type = "snapshot"
@@ -67,6 +69,8 @@ if [ "${LIMA_CIDATA_CONTAINERD_USER}" = 1 ]; then
 		mkdir -p "${LIMA_CIDATA_HOME}/.config/containerd"
 		cat >"${LIMA_CIDATA_HOME}/.config/containerd/config.toml" <<EOF
   version = 2
+  [plugins."io.containerd.grpc.v1.cri"]
+    enable_cdi = true
   [proxy_plugins]
     [proxy_plugins."fuse-overlayfs"]
       type = "snapshot"

pkg/driver/vz/boot/05-rosetta-volume.sh

@@ -31,3 +31,81 @@ else
 	# remove binfmt.d(5) configuration if it exists
 	[ ! -f "$binfmtd_conf" ] || rm "$binfmtd_conf"
 fi
+
+if [ -x /mnt/lima-rosetta/rosettad ]; then
+	CACHE_DIRECTORY=/var/cache/rosettad
+	DEFAULT_SOCKET=${CACHE_DIRECTORY}/uds/rosetta.sock
+	EXPECTED_SOCKET=/run/rosettad/rosetta.sock
+
+	# Create rosettad service
+	if [ -f /sbin/openrc-run ]; then
+		cat >/etc/init.d/rosettad <<EOF
+#!/sbin/openrc-run
+name="rosettad"
+description="Rosetta AOT Caching Daemon"
+required_dirs=/mnt/lima-rosetta
+required_files=/mnt/lima-rosetta/rosettad
+command=/mnt/lima-rosetta/rosettad
+command_args="daemon ${CACHE_DIRECTORY}"
+command_background=true
+pidfile="/run/rosettad.pid"
+start_pre() {
+	# To detect creation of the socket by rosettad, remove the old socket before starting
+	test ! -e "${DEFAULT_SOCKET}" || rm -f "${DEFAULT_SOCKET}"
+}
+start_post() {
+	# Set the socket permission to world-writable
+	while ! chmod -f go+w "${DEFAULT_SOCKET}"; do sleep 1; done
+	# Create the symlink as expected by the configuration to enable Rosetta AOT caching
+	mkdir -p "$(dirname "${EXPECTED_SOCKET}")"
+	ln -sf "${DEFAULT_SOCKET}" "${EXPECTED_SOCKET}"
+}
+EOF
+		chmod 755 /etc/init.d/rosettad
+		rc-update add rosettad default
+		rc-service rosettad start
+	else
+		cat >/etc/systemd/system/rosettad.service <<EOF
+[Unit]
+Description=Rosetta AOT Caching Daemon
+RequiresMountsFor=/mnt/lima-rosetta
+[Service]
+RuntimeDirectory=rosettad
+CacheDirectory=rosettad
+# To detect creation of the socket by rosettad, remove the old socket
+ExecStartPre=sh -c "test ! -e \"${DEFAULT_SOCKET}\" || rm -f \"${DEFAULT_SOCKET}\""
+ExecStart=/mnt/lima-rosetta/rosettad daemon "${CACHE_DIRECTORY}"
+# Set the socket permission to world-writable and create the symlink as expected by the configuration to enable Rosetta AOT caching.
+ExecStartPost=sh -c "while ! chmod -f go+w \"${DEFAULT_SOCKET}\"; do sleep 1; done; ln -sf \"${DEFAULT_SOCKET}\" \"${EXPECTED_SOCKET}\""
+OOMPolicy=continue
+OOMScoreAdjust=-500
+[Install]
+WantedBy=default.target
+EOF
+		systemctl is-enabled rosettad || systemctl enable --now rosettad
+	fi
+
+	# Create CDI configuration for Rosetta
+	mkdir -p /etc/cdi /var/run/cdi /etc/buildkit/cdi
+	cat >/etc/cdi/rosetta.yaml <<EOF
+cdiVersion: "0.6.0"
+kind: "lima-vm.io/rosetta"
+devices:
+- name: cached
+  containerEdits:
+    mounts:
+    - hostPath: /var/cache/rosettad/uds/rosetta.sock
+      containerPath: /run/rosettad/rosetta.sock
+      options: [bind]
+annotations:
+  org.mobyproject.buildkit.device.autoallow: true
+EOF
+	# nerdctl requires user-specific CDI configuration directories
+	mkdir -p "${LIMA_CIDATA_HOME}/.config/cdi"
+	ln -sf /etc/cdi/rosetta.yaml "${LIMA_CIDATA_HOME}/.config/cdi/"
+	chown -R "${LIMA_CIDATA_USER}" "${LIMA_CIDATA_HOME}/.config"
+else
+	# Remove CDI configuration for Rosetta AOT Caching
+	[ ! -f /etc/cdi/rosetta.yaml ] || rm /etc/cdi/rosetta.yaml
+	[ ! -d "${LIMA_CIDATA_HOME}/.config/cdi/rosetta.yaml" ] || rm "${LIMA_CIDATA_HOME}/.config/cdi/rosetta.yaml"
+fi

pkg/driver/vz/rosetta_directory_share_arm64.go

@@ -44,7 +44,7 @@ func createRosettaDirectoryShareConfiguration() (*vz.VirtioFileSystemDeviceConfi
 		return nil, fmt.Errorf("failed to get macOS product version: %w", err)
 	}
 	if !macOSProductVersion.LessThan(*semver.New("14.0.0")) {
-		cachingOption, err := vz.NewLinuxRosettaAbstractSocketCachingOptions("rosetta")
+		cachingOption, err := vz.NewLinuxRosettaUnixSocketCachingOptions("/run/rosettad/rosetta.sock")
 		if err != nil {
 			return nil, fmt.Errorf("failed to create a new rosetta directory share caching option: %w", err)
 		}

templates/docker-rootful.yaml

@@ -38,7 +38,9 @@ provision:
   expression: .Socket.SocketUser="{{.User}}"
 - mode: yq
   path: "/etc/docker/daemon.json"
-  expression: .features.containerd-snapshotter = {{.Param.containerdSnapshotter}}
+  expression: |
+    .features.cdi = true |
+    .features.containerd-snapshotter = {{.Param.containerdSnapshotter}}
 probes:
 - script: |
     #!/bin/bash
@@ -67,5 +69,22 @@ message: |
   docker context use lima-{{.Name}}
   docker run hello-world
   ------
+  {{- if .Instance.Config.VMOpts.VZ.Rosetta.Enabled}}
+  Rosetta is enabled in this VM, so you can run x86_64 containers on Apple Silicon.
+  You can use Rosetta AOT Caching with the CDI spec:
+  - To run a container, add `--device=lima-vm.io/rosetta=cached` to your `docker run` command:
+    ------
+    docker run --platform=linux/amd64 --device=lima-vm.io/rosetta=cached ...
+    ------
+  - To build an image, add `# syntax=docker/dockerfile:1-labs` at the top of your Dockerfile,
+    and use `--device=lima-vm.io/rosetta=cached` in the `RUN` command:
+    ------
+    # syntax=docker/dockerfile:1-labs
+    FROM ...
+    ...
+    RUN --device=lima-vm.io/rosetta=cached <your amd64 command>
+    ------
+  See: https://lima-vm.io/docs/config/multi-arch/#rosetta-aot-caching
+  {{- end}}
 param:
   containerdSnapshotter: false

templates/docker.yaml

@@ -37,7 +37,9 @@ provision:
     apt-get install -y uidmap dbus-user-session
 - mode: yq
   path: "{{.Home}}/.config/docker/daemon.json"
-  expression: .features.containerd-snapshotter = {{.Param.containerdSnapshotter}}
+  expression: |
+    .features.cdi = true |
+    .features.containerd-snapshotter = {{.Param.containerdSnapshotter}}
   owner: "{{.User}}"
 - mode: user
   script: |
@@ -74,5 +76,22 @@ message: |
   docker context use lima-{{.Name}}
   docker run hello-world
   ------
+  {{- if .Instance.Config.VMOpts.VZ.Rosetta.Enabled}}
+  Rosetta is enabled in this VM, so you can run x86_64 containers on Apple Silicon.
+  You can use Rosetta AOT Caching with the CDI spec:
+  - To run a container, add `--device=lima-vm.io/rosetta=cached` to your `docker run` command:
+    ------
+    docker run --platform=linux/amd64 --device=lima-vm.io/rosetta=cached ...
+    ------
+  - To build an image, add `# syntax=docker/dockerfile:1-labs` at the top of your Dockerfile,
+    and use `--device=lima-vm.io/rosetta=cached` in the `RUN` command:
+    ------
+    # syntax=docker/dockerfile:1-labs
+    FROM ...
+    ...
+    RUN --device=lima-vm.io/rosetta=cached <your amd64 command>
+    ------
+  See: https://lima-vm.io/docs/config/multi-arch/#rosetta-aot-caching
+  {{- end}}
 param:
   containerdSnapshotter: false

website/content/en/docs/config/multi-arch.md

@@ -86,3 +86,52 @@ rosetta:
 ```
 {{% /tab %}}
 {{< /tabpane >}}
+
+### [Enable Rosetta AOT Caching with CDI spec](#rosetta-aot-caching)
+| ⚡ Requirement | Lima >= 2.0, macOS >= 14.0, ARM |
+|-------------------|----------------------------------|
+
+Rosetta AOT Caching speeds up containers by saving translated binaries, so they don't need to be translated again.  
+Learn more: [WWDC2023 video](https://developer.apple.com/videos/play/wwdc2023/10007/?time=721)
+
+**How to use Rosetta AOT Caching:**
+
+- **Run a container:**  
+  Add `--device=lima-vm.io/rosetta=cached` to your `docker run` command:
+  ```bash
+  docker run --platform=linux/amd64 --device=lima-vm.io/rosetta=cached ...
+  ```
+
+- **Build an image:**  
+  Add `# syntax=docker/dockerfile:1-labs` at the top of your Dockerfile to enable the `--device` option.  
+  Use `--device=lima-vm.io/rosetta=cached` in your `RUN` command:
+  ```Dockerfile
+  # syntax=docker/dockerfile:1-labs
+  FROM ...
+  ...
+  RUN --device=lima-vm.io/rosetta=cached <your amd64 command>
+  ```
+
+- **Check if caching works:**  
+  Look for cache files in the VM:
+  ```bash
+  limactl shell {{.Name}} ls -la /var/cache/rosettad
+  docker run --platform linux/amd64 --device=lima-vm.io/rosetta=cached ubuntu echo hello
+  limactl shell {{.Name}} ls -la /var/cache/rosettad
+  # You should see *.aotcache files here
+  ```
+
+- **Check if Docker recognizes the CDI device:**  
+  Look for CDI info in the output of `docker info`:
+  ```console
+  docker info
+  ...
+  CDI spec directories:
+    /etc/cdi
+    /var/run/cdi
+  Discovered Devices:
+    cdi: lima-vm.io/rosetta=cached
+  ```
+
+- **Learn more about CDI:**  
+  [CDI spec documentation](https://github.com/cncf-tags/container-device-interface/blob/main/SPEC.md)