feature: "yq" provision mode

```yaml
# `yq` creates or edits a file in the guest filesystem by using `/mnt/lima-cidata/provision/tool/yq`.
# `/mnt/lima-cidata/provision/tool/yq` is installed if `mode: yq` is specified in `provision`.
# The file specified by `path` will be updated with the specified `expression` using `yq -i`.
# If the file does not exist, it will be created using `yq -n`.
# `yq` v4.47.1 can write .ini, .json, .properties, .xml, .yaml, and .yml files.
# See https://github.com/mikefarah/yq for more info.
# Any missing directories will be created as needed using `mkdir -p`.
# The file permissions will be set to the specified value using `chmod`.
# The file and directory creation will be performed with the specified user privileges.
# If the existing file is not writable by the specified user, the operation will fail.
# `path` and `expression` are required.
# `user` and `permissions` are optional. Defaults to "root" and 644.
- mode: yq
  path: "{{.Home}}/.config/docker/daemon.json"
  expression: ".features.containerd-snapshotter = {{.Param.containerdSnapshotter}}"
  user: "{{.User}}"
  permissions: 644

# Override provisionTool
# 🟢 Builtin default: hard-coded URL with hard-coded digest (see the output of `limactl info | jq .defaultTemplate.provisionTool`)
provisionTool:
  yq:
  - location: "~/Downloads/yq_linux_amd64"
    arch: "x86_64"
    digest: "sha256:..."
```

Signed-off-by: Norio Nomura <norio.nomura@gmail.com>

Author: norio-nomura

cmd/limactl/hostagent.go

@@ -35,6 +35,7 @@ func newHostagentCommand() *cobra.Command {
 	hostagentCommand.Flags().Bool("run-gui", false, "Run GUI synchronously within hostagent")
 	hostagentCommand.Flags().String("guestagent", "", "Local file path (not URL) of lima-guestagent.OS-ARCH[.gz]")
 	hostagentCommand.Flags().String("nerdctl-archive", "", "Local file path (not URL) of nerdctl-full-VERSION-GOOS-GOARCH.tar.gz")
+	hostagentCommand.Flags().String("provision-tool", "", "<tool name>=<local file path>[,<tool name>=<local file path>[,...]]")
 	hostagentCommand.Flags().Bool("progress", false, "Show provision script progress by monitoring cloud-init logs")
 	return hostagentCommand
 }
@@ -96,6 +97,13 @@ func hostagentAction(cmd *cobra.Command, args []string) error {
 	if nerdctlArchive != "" {
 		opts = append(opts, hostagent.WithNerdctlArchive(nerdctlArchive))
 	}
+	provisionTool, err := cmd.Flags().GetString("provision-tool")
+	if err != nil {
+		return err
+	}
+	if provisionTool != "" {
+		opts = append(opts, hostagent.WithProvisionTool(provisionTool))
+	}
 	showProgress, err := cmd.Flags().GetBool("progress")
 	if err != nil {
 		return err

cmd/limactl/prune.go

@@ -113,6 +113,11 @@ func locationsFromLimaYAML(y *limayaml.LimaYAML) map[string]limayaml.File {
 	for _, f := range y.Containerd.Archives {
 		locations[downloader.CacheKey(f.Location)] = f
 	}
+	for _, files := range y.ProvisionTool {
+		for _, f := range files {
+			locations[downloader.CacheKey(f.Location)] = f
+		}
+	}
 	for _, f := range y.Firmware.Images {
 		locations[downloader.CacheKey(f.Location)] = f.File
 	}

hack/test-templates.sh

@@ -57,6 +57,7 @@ declare -A CHECKS=(
 	["mount-path-with-spaces"]=""
 	["provision-data"]=""
 	["param-env-variables"]=""
+	["provision-yq"]=""
 	["set-user"]=""
 )
 
@@ -90,6 +91,7 @@ case "$NAME" in
 	CHECKS["mount-path-with-spaces"]="1"
 	CHECKS["provision-data"]="1"
 	CHECKS["param-env-variables"]="1"
+	CHECKS["provision-yq"]="1"
 	CHECKS["set-user"]="1"
 	;;
 "docker")
@@ -208,6 +210,11 @@ if [[ -n ${CHECKS["param-env-variables"]} ]]; then
 	limactl shell "$NAME" test -e /tmp/param-user
 fi
 
+if [[ -n ${CHECKS["provision-yq"]} ]]; then
+	INFO 'Testing that /tmp/param-yq.json was created successfully on provision'
+	limactl shell "$NAME" grep -q '"YQ": "yq"' /tmp/param-yq.json
+fi
+
 if [[ -n ${CHECKS["set-user"]} ]]; then
 	INFO 'Testing that user settings can be provided by lima.yaml'
 	limactl shell "$NAME" grep "^john:x:4711:4711:John Doe:/home/john-john:/usr/bin/bash" /etc/passwd

hack/test-templates/test-misc.yaml

@@ -20,6 +20,7 @@ param:
   PROBE: probe
   SYSTEM: system
   USER: user
+  YQ: yq
 
 provision:
 - mode: ansible
@@ -37,6 +38,10 @@ provision:
   content: |
     fs.inotify.max_user_watches = 524288
     fs.inotify.max_user_instances = 512
+- mode: yq
+  path: "/tmp/param-{{.Param.YQ}}.json"
+  expression: .YQ = {{.Param.YQ}}
+  user: "{{.User}}"
 
 probes:
 - mode: readiness

pkg/cidata/cidata.TEMPLATE.d/boot.sh

@@ -77,6 +77,58 @@ if [ -d "${LIMA_CIDATA_MNT}"/provision/data ]; then
 	done
 fi
 
+if [ -d "${LIMA_CIDATA_MNT}"/provision/yq ]; then
+	yq="${LIMA_CIDATA_MNT}"/provision/tool/yq
+	if [ -x "${yq}" ]; then
+		for expression in "${LIMA_CIDATA_MNT}"/provision/yq/*; do
+			filename=$(basename "${expression}")
+			path=$(deref "LIMA_CIDATA_YQ_PROVISION_${filename}_PATH")
+			permissions=$(deref "LIMA_CIDATA_YQ_PROVISION_${filename}_PERMISSIONS")
+			user=$(deref "LIMA_CIDATA_YQ_PROVISION_${filename}_USER")
+			if ! sudo -iu "${user}" mkdir -p "$(dirname "${path}")"; then
+				WARNING "Failed to create directory for ${path} (as user ${user})"
+				CODE=1
+				continue
+			fi
+			# Since CIDATA is mounted with dmode=700,fmode=700,
+			# provision/tool/yq cannot be executed by non-root users,
+			# and provision/yq/* files cannot be read by non-root users.
+			if [ -f "${path}" ]; then
+				INFO "Updating ${path}"
+				# Relies on the fact that yq does not change the owner of the existing file.
+				if ! "${yq}" --inplace --from-file "${expression}" "${path}"; then
+					WARNING "Failed to update ${path} (as user ${user})"
+					CODE=1
+					continue
+				fi
+			else
+				# follow file extension
+				case "${path}" in
+				*.ini) output=ini ;;
+				*.json) output=json ;;
+				*.properties) output=properties ;;
+				*.xml) output=xml ;;
+				*.yaml | *.yml) output=yaml ;;
+				*) output=auto ;;
+				esac
+				INFO "Creating ${path}"
+				if ! "${yq}" --null-input --from-file "${expression}" --output-format "${output}" | sudo -iu "${user}" tee "${path}"; then
+					WARNING "Failed to create ${path} (as user ${user})"
+					CODE=1
+					continue
+				fi
+			fi
+			if ! sudo -iu "${user}" chmod "${permissions}" "${path}"; then
+				WARNING "Failed to set permissions for ${path} (as user ${user})"
+				CODE=1
+			fi
+		done
+	else
+		WARNING "${yq} is not executable, skipping yq provisioning"
+		CODE=1
+	fi
+fi
+
 if [ -d "${LIMA_CIDATA_MNT}"/provision/system ]; then
 	for f in "${LIMA_CIDATA_MNT}"/provision/system/*; do
 		INFO "Executing $f"

pkg/cidata/cidata.TEMPLATE.d/lima.env

@@ -25,6 +25,11 @@ LIMA_CIDATA_DATAFILE_{{$dataFile.FileName}}_OWNER={{$dataFile.Owner}}
 LIMA_CIDATA_DATAFILE_{{$dataFile.FileName}}_PATH={{$dataFile.Path}}
 LIMA_CIDATA_DATAFILE_{{$dataFile.FileName}}_PERMISSIONS={{$dataFile.Permissions}}
 {{- end}}
+{{- range $yqProvision := .YQProvisions}}
+LIMA_CIDATA_YQ_PROVISION_{{$yqProvision.FileName}}_PATH={{$yqProvision.Path}}
+LIMA_CIDATA_YQ_PROVISION_{{$yqProvision.FileName}}_PERMISSIONS={{$yqProvision.Permissions}}
+LIMA_CIDATA_YQ_PROVISION_{{$yqProvision.FileName}}_USER={{$yqProvision.User}}
+{{- end}}
 LIMA_CIDATA_GUEST_INSTALL_PREFIX={{ .GuestInstallPrefix }}
 {{- if .Containerd.User}}
 LIMA_CIDATA_CONTAINERD_USER=1

pkg/cidata/cidata.go

@@ -4,6 +4,7 @@
 package cidata
 
 import (
+	"bytes"
 	"compress/gzip"
 	"context"
 	"errors"
@@ -328,6 +329,14 @@ func templateArgs(ctx context.Context, bootScripts bool, instDir, name string, i
 				Permissions: *f.Permissions,
 			})
 		}
+		if f.Mode == limayaml.ProvisionModeYQ {
+			args.YQProvisions = append(args.YQProvisions, YQProvision{
+				FileName:    fmt.Sprintf("%08d", i),
+				Path:        *f.Path,
+				Permissions: *f.Permissions,
+				User:        *f.User,
+			})
+		}
 	}
 
 	return &args, nil
@@ -356,7 +365,7 @@ func GenerateCloudConfig(ctx context.Context, instDir, name string, instConfig *
 	return os.WriteFile(filepath.Join(instDir, filenames.CloudConfig), config, 0o444)
 }
 
-func GenerateISO9660(ctx context.Context, instDir, name string, instConfig *limayaml.LimaYAML, udpDNSLocalPort, tcpDNSLocalPort int, guestAgentBinary, nerdctlArchive string, vsockPort int, virtioPort string) error {
+func GenerateISO9660(ctx context.Context, instDir, name string, instConfig *limayaml.LimaYAML, udpDNSLocalPort, tcpDNSLocalPort int, guestAgentBinary, nerdctlArchive string, provisionTool map[string]string, vsockPort int, virtioPort string) error {
 	args, err := templateArgs(ctx, true, instDir, name, instConfig, udpDNSLocalPort, tcpDNSLocalPort, vsockPort, virtioPort)
 	if err != nil {
 		return err
@@ -383,6 +392,11 @@ func GenerateISO9660(ctx context.Context, instDir, name string, instConfig *lima
 				Path:   fmt.Sprintf("provision/%s/%08d", f.Mode, i),
 				Reader: strings.NewReader(*f.Content),
 			})
+		case limayaml.ProvisionModeYQ:
+			layout = append(layout, iso9660util.Entry{
+				Path:   fmt.Sprintf("provision/%s/%08d", f.Mode, i),
+				Reader: strings.NewReader(*f.Expression),
+			})
 		case limayaml.ProvisionModeBoot:
 			continue
 		case limayaml.ProvisionModeAnsible:
@@ -432,6 +446,28 @@ func GenerateISO9660(ctx context.Context, instDir, name string, instConfig *lima
 			Reader: nftgzR,
 		})
 	}
+	for name, path := range provisionTool {
+		f, err := os.Open(path)
+		if err != nil {
+			return err
+		}
+		buf := new(bytes.Buffer)
+		_, err = io.Copy(buf, f)
+		if err != nil {
+			_ = f.Close()
+			return err
+		}
+		_ = f.Close()
+		layout = append(layout, iso9660util.Entry{
+			// Place it at "provision/tool/<name>" instead of "util/<name>".
+			// Because "util" is in PATH while "boot.sh" is executing,
+			// that may override <name> with the Linux distribution providing binaries in VM.
+			// If some provision scripts install <name> from the distribution,
+			// it may lead to unexpected behavior for the provision scripts.
+			Path:   fmt.Sprintf("provision/tool/%s", name),
+			Reader: buf,
+		})
+	}
 
 	if args.VMType == limayaml.WSL2 {
 		layout = append(layout, iso9660util.Entry{

pkg/cidata/template.go

@@ -58,6 +58,13 @@ type DataFile struct {
 	Permissions string
 }
 
+type YQProvision struct {
+	FileName    string
+	Path        string
+	Permissions string
+	User        string
+}
+
 type Disk struct {
 	Name   string
 	Device string
@@ -93,6 +100,7 @@ type TemplateArgs struct {
 	Param                           map[string]string
 	BootScripts                     bool
 	DataFiles                       []DataFile
+	YQProvisions                    []YQProvision
 	DNSAddresses                    []string
 	CACerts                         CACerts
 	HostHomeMountPoint              string

pkg/driver/vz/vz_driver_darwin.go

@@ -57,6 +57,7 @@ var knownYamlProperties = []string{
 	"Probes",
 	"PropagateProxyEnv",
 	"Provision",
+	"ProvisionTool",
 	"Rosetta",
 	"SSH",
 	"TimeZone",

pkg/driver/wsl2/wsl_driver_windows.go

@@ -40,6 +40,7 @@ var knownYamlProperties = []string{
 	"Probes",
 	"PropagateProxyEnv",
 	"Provision",
+	"ProvisionTool",
 	"SSH",
 	"VMType",
 }