feature: "yq" provision mode

```yaml
# `yq` creates or edits a file in the guest filesystem by using `${LIMA_CIDATA_GUEST_INSTALL_PREFIX}/bin/lima-guestagent yq`.
# `${LIMA_CIDATA_GUEST_INSTALL_PREFIX}/bin/lima-guestagent yq` has the same functionality as https://github.com/mikefarah/yq.
# The file specified by `path` will be updated with the specified `expression` using `yq --inplace`.
# If the file does not exist, it will be created using `yq --null-input`.
# `format` is optional and defaults to "auto", which is passed to `yq` by `--input-format <format>`, `--output-format <format>`.
# One of the following is expected to work:
#   "auto", "a", "ini", "i", "json", "j", "props", "p", "xml", "x", "yaml", "y"
# `yq` v4.47.1 can write .ini, .json, .properties, .xml, .yaml, and .yml files.
# See https://github.com/mikefarah/yq for more info.
# Any missing directories will be created as needed using `mkdir -p`.
# The file permissions will be set to the specified value using `chmod`.
# The file and directory creation will be performed with the specified user privileges.
# If the existing file is not writable by the specified user, the operation will fail.
# `path` and `expression` are required.
# `user` and `permissions` are optional. Defaults to "root" and 644.
- mode: yq
  path: "{{.Home}}/.config/docker/daemon.json"
  expression: ".features.containerd-snapshotter = {{.Param.containerdSnapshotter}}"
  format: auto
  user: "{{.User}}"
  permissions: 644

Signed-off-by: Norio Nomura <norio.nomura@gmail.com>

Author: norio-nomura

hack/test-templates.sh

@@ -57,6 +57,7 @@ declare -A CHECKS=(
 	["mount-path-with-spaces"]=""
 	["provision-data"]=""
 	["param-env-variables"]=""
+	["provision-yq"]=""
 	["set-user"]=""
 	["preserve-env"]="1"
 )
@@ -91,6 +92,7 @@ case "$NAME" in
 	CHECKS["mount-path-with-spaces"]="1"
 	CHECKS["provision-data"]="1"
 	CHECKS["param-env-variables"]="1"
+	CHECKS["provision-yq"]="1"
 	CHECKS["set-user"]="1"
 	;;
 "docker")
@@ -209,6 +211,11 @@ if [[ -n ${CHECKS["param-env-variables"]} ]]; then
 	limactl shell "$NAME" test -e /tmp/param-user
 fi
 
+if [[ -n ${CHECKS["provision-yq"]} ]]; then
+	INFO 'Testing that /tmp/param-yq.json was created successfully on provision'
+	limactl shell "$NAME" grep -q '"YQ": "yq"' /tmp/param-yq.json
+fi
+
 if [[ -n ${CHECKS["set-user"]} ]]; then
 	INFO 'Testing that user settings can be provided by lima.yaml'
 	limactl shell "$NAME" grep "^john:x:4711:4711:John Doe:/home/john-john:/usr/bin/bash" /etc/passwd

hack/test-templates/test-misc.yaml

@@ -20,6 +20,7 @@ param:
   PROBE: probe
   SYSTEM: system
   USER: user
+  YQ: yq
 
 provision:
 - mode: ansible
@@ -37,6 +38,10 @@ provision:
   content: |
     fs.inotify.max_user_watches = 524288
     fs.inotify.max_user_instances = 512
+- mode: yq
+  path: "/tmp/param-{{.Param.YQ}}.json"
+  expression: .YQ = "{{.Param.YQ}}"
+  user: "{{.User}}"
 
 probes:
 - mode: readiness

pkg/cidata/cidata.TEMPLATE.d/boot.sh

@@ -77,6 +77,64 @@ if [ -d "${LIMA_CIDATA_MNT}"/provision.data ]; then
 	done
 fi
 
+if [ -d "${LIMA_CIDATA_MNT}"/provision.yq ]; then
+	yq="${LIMA_CIDATA_GUEST_INSTALL_PREFIX}/bin/lima-guestagent yq"
+	if ${yq} --version >/dev/null; then
+		for expression in "${LIMA_CIDATA_MNT}"/provision.yq/*; do
+			filename=$(basename "${expression}")
+			format=$(deref "LIMA_CIDATA_YQ_PROVISION_${filename}_FORMAT")
+			path=$(deref "LIMA_CIDATA_YQ_PROVISION_${filename}_PATH")
+			permissions=$(deref "LIMA_CIDATA_YQ_PROVISION_${filename}_PERMISSIONS")
+			user=$(deref "LIMA_CIDATA_YQ_PROVISION_${filename}_USER")
+			if ! sudo -iu "${user}" mkdir -p "$(dirname "${path}")"; then
+				WARNING "Failed to create directory for ${path} (as user ${user})"
+				CODE=1
+				continue
+			fi
+			# Since CIDATA is mounted with dmode=700,fmode=700,
+			# provision.tool/yq cannot be executed by non-root users,
+			# and provision.yq/* files cannot be read by non-root users.
+			if [ -f "${path}" ]; then
+				INFO "Updating ${path}"
+				# Relies on the fact that yq does not change the owner of the existing file.
+				if ! ${yq} --inplace --from-file "${expression}" --input-format "${format}" --output-format "${format}" "${path}"; then
+					WARNING "Failed to update ${path} (as user ${user})"
+					CODE=1
+					continue
+				fi
+			else
+				if [ "${format}" = "auto" ]; then
+					# yq can't determine the output format from non-existing files
+					case "${path}" in
+					*.ini) format=ini ;;
+					*.json) format=json ;;
+					*.properties) format=properties ;;
+					*.xml) format=xml ;;
+					*.yaml | *.yml) format=yaml ;;
+					*)
+						format=yaml
+						WARNING "Cannot determine file type for ${path}, using yaml format"
+						;;
+					esac
+				fi
+				INFO "Creating ${path}"
+				if ! ${yq} --null-input --from-file "${expression}" --output-format "${format}" | sudo -iu "${user}" tee "${path}"; then
+					WARNING "Failed to create ${path} (as user ${user})"
+					CODE=1
+					continue
+				fi
+			fi
+			if ! sudo -iu "${user}" chmod "${permissions}" "${path}"; then
+				WARNING "Failed to set permissions for ${path} (as user ${user})"
+				CODE=1
+			fi
+		done
+	else
+		WARNING "${yq} is not executable, skipping yq provisioning"
+		CODE=1
+	fi
+fi
+
 if [ -d "${LIMA_CIDATA_MNT}"/provision.system ]; then
 	for f in "${LIMA_CIDATA_MNT}"/provision.system/*; do
 		INFO "Executing $f"

pkg/cidata/cidata.TEMPLATE.d/lima.env

@@ -25,6 +25,12 @@ LIMA_CIDATA_DATAFILE_{{$dataFile.FileName}}_OWNER={{$dataFile.Owner}}
 LIMA_CIDATA_DATAFILE_{{$dataFile.FileName}}_PATH={{$dataFile.Path}}
 LIMA_CIDATA_DATAFILE_{{$dataFile.FileName}}_PERMISSIONS={{$dataFile.Permissions}}
 {{- end}}
+{{- range $yqProvision := .YQProvisions}}
+LIMA_CIDATA_YQ_PROVISION_{{$yqProvision.FileName}}_FORMAT={{$yqProvision.Format}}
+LIMA_CIDATA_YQ_PROVISION_{{$yqProvision.FileName}}_PATH={{$yqProvision.Path}}
+LIMA_CIDATA_YQ_PROVISION_{{$yqProvision.FileName}}_PERMISSIONS={{$yqProvision.Permissions}}
+LIMA_CIDATA_YQ_PROVISION_{{$yqProvision.FileName}}_USER={{$yqProvision.User}}
+{{- end}}
 LIMA_CIDATA_GUEST_INSTALL_PREFIX={{ .GuestInstallPrefix }}
 {{- if .Containerd.User}}
 LIMA_CIDATA_CONTAINERD_USER=1

pkg/cidata/cidata.go

@@ -335,6 +335,15 @@ func templateArgs(ctx context.Context, bootScripts bool, instDir, name string, i
 				Permissions: *f.Permissions,
 			})
 		}
+		if f.Mode == limatype.ProvisionModeYQ {
+			args.YQProvisions = append(args.YQProvisions, YQProvision{
+				FileName:    fmt.Sprintf("%08d", i),
+				Format:      *f.Format,
+				Path:        *f.Path,
+				Permissions: *f.Permissions,
+				User:        *f.User,
+			})
+		}
 	}
 
 	return &args, nil
@@ -402,6 +411,11 @@ func GenerateISO9660(ctx context.Context, drv driver.Driver, instDir, name strin
 				Path:   fmt.Sprintf("provision.%s/%08d", f.Mode, i),
 				Reader: strings.NewReader(*f.Content),
 			})
+		case limatype.ProvisionModeYQ:
+			layout = append(layout, iso9660util.Entry{
+				Path:   fmt.Sprintf("provision.%s/%08d", f.Mode, i),
+				Reader: strings.NewReader(*f.Expression),
+			})
 		case limatype.ProvisionModeBoot:
 			continue
 		case limatype.ProvisionModeAnsible:

pkg/cidata/template.go

@@ -58,6 +58,14 @@ type DataFile struct {
 	Permissions string
 }
 
+type YQProvision struct {
+	FileName    string
+	Format      string
+	Path        string
+	Permissions string
+	User        string
+}
+
 type Disk struct {
 	Name   string
 	Device string
@@ -93,6 +101,7 @@ type TemplateArgs struct {
 	Param                           map[string]string
 	BootScripts                     bool
 	DataFiles                       []DataFile
+	YQProvisions                    []YQProvision
 	DNSAddresses                    []string
 	CACerts                         CACerts
 	HostHomeMountPoint              string

pkg/limatmpl/embed.go

@@ -648,6 +648,11 @@ func (tmpl *Template) embedAllScripts(ctx context.Context, embedAll bool) error
 			if p.Content != nil {
 				continue
 			}
+		case limatype.ProvisionModeYQ:
+			newName = "expression"
+			if p.Expression != nil {
+				continue
+			}
 		default:
 			if p.Script != "" {
 				continue

pkg/limatmpl/embed_test.go

@@ -343,6 +343,9 @@ provision:
 - mode: data
   file: base1.sh # This comment will move to the "content" key
   path: /tmp/data
+- mode: yq
+  file: base1.sh # This comment will move to the "expression" key
+  path: /tmp/yq
 `,
 		`
 # base0.yaml is ignored
@@ -364,6 +367,11 @@ provision:
     #!/usr/bin/env bash
     echo "This is base1.sh"
   path: /tmp/data
+- mode: yq
+  expression: |-  # This comment will move to the "expression" key
+    #!/usr/bin/env bash
+    echo "This is base1.sh"
+  path: /tmp/yq
 
 # base0.yaml is ignored
 `,

pkg/limatype/lima_yaml.go

@@ -235,6 +235,7 @@ const (
 	ProvisionModeDependency ProvisionMode = "dependency"
 	ProvisionModeAnsible    ProvisionMode = "ansible" // DEPRECATED
 	ProvisionModeData       ProvisionMode = "data"
+	ProvisionModeYQ         ProvisionMode = "yq"
 )
 
 type Provision struct {
@@ -245,6 +246,10 @@ type Provision struct {
 	Playbook                        string             `yaml:"playbook,omitempty" json:"playbook,omitempty"` // DEPRECATED
 	// All ProvisionData fields must be nil unless Mode is ProvisionModeData
 	ProvisionData `yaml:",inline"` // Flatten fields for "strict" YAML mode
+	// ProvisionModeYQ borrows Path and Permissions from ProvisionData
+	Expression *string `yaml:"expression,omitempty" json:"expression,omitempty" jsonschema:"nullable"`
+	User       *string `yaml:"user,omitempty" json:"user,omitempty" jsonschema:"nullable"`
+	Format     *string `yaml:"format,omitempty" json:"format,omitempty" jsonschema:"nullable"`
 }
 
 type ProvisionData struct {

pkg/limayaml/defaults.go

@@ -414,6 +414,29 @@ func FillDefault(ctx context.Context, y, d, o *limatype.LimaYAML, filePath strin
 					logrus.WithError(err).Warnf("Couldn't owner %q as a template", *provision.Owner)
 				}
 			}
+		}
+		if provision.Mode == limatype.ProvisionModeYQ {
+			if provision.Expression != nil {
+				if out, err := executeGuestTemplate(*provision.Expression, instDir, y.User, y.Param); err == nil {
+					provision.Expression = ptr.Of(out.String())
+				} else {
+					logrus.WithError(err).Warnf("Couldn't process expression %q as a template", *provision.Expression)
+				}
+			}
+			if provision.Format == nil {
+				provision.Format = ptr.Of("auto")
+			}
+			if provision.User == nil {
+				provision.User = ptr.Of("root")
+			} else {
+				if out, err := executeGuestTemplate(*provision.User, instDir, y.User, y.Param); err == nil {
+					provision.User = ptr.Of(out.String())
+				} else {
+					logrus.WithError(err).Warnf("Couldn't process user %q as a template", *provision.User)
+				}
+			}
+		}
+		if provision.Mode == limatype.ProvisionModeData || provision.Mode == limatype.ProvisionModeYQ {
 			// Path is required; validation will throw an error when it is nil
 			if provision.Path != nil {
 				if out, err := executeGuestTemplate(*provision.Path, instDir, y.User, y.Param); err == nil {