add: volume preloading via ?preload=true directive in Acornfile (manager#1598) (#2351)

Signed-off-by: Thorsten Klein <tk@thklein.io>

Author: iwilltry42

Dockerfile

@@ -14,7 +14,7 @@ RUN --mount=type=cache,target=/go/pkg --mount=type=cache,target=/root/.cache/go-
 
 FROM ghcr.io/acorn-io/images-mirror/golang:1.21-alpine AS loglevel
 WORKDIR /usr/src
-RUN apk -U add curl
+RUN apk -U add curl && rm -rf /var/cache/apk/*
 RUN curl -sfL https://github.com/acorn-io/loglevel/archive/refs/tags/v0.1.6.tar.gz | tar xzf - --strip-components=1
 RUN --mount=type=cache,target=/go/pkg --mount=type=cache,target=/root/.cache/go-build CGO_ENABLED=0 go build -o /usr/local/bin/loglevel -ldflags "-s -w"
 
@@ -25,14 +25,14 @@ COPY --from=sleep /sleep /src/pkg/controller/appdefinition/embed/acorn-sleep
 RUN --mount=type=cache,target=/go/pkg --mount=type=cache,target=/root/.cache/go-build GO_TAGS=netgo,image make build
 
 FROM ghcr.io/acorn-io/images-mirror/nginx:1.23.2-alpine AS base
-RUN apk add --no-cache ca-certificates iptables ip6tables fuse3 git openssh pigz xz \
+RUN apk add --no-cache ca-certificates iptables ip6tables fuse3 git openssh pigz xz busybox-static \
   && ln -s fusermount3 /usr/bin/fusermount
 RUN adduser -D acorn
 RUN mkdir apiserver.local.config && chown acorn apiserver.local.config
 RUN --mount=from=binfmt,src=/usr/bin,target=/usr/src for i in aarch64 x86_64; do if [ -e /usr/src/qemu-$i ]; then cp /usr/src/qemu-$i /usr/bin; fi; done
 RUN --mount=from=buildkit,src=/usr/bin,target=/usr/src for i in aarch64 x86_64; do if [ -e /usr/src/buildkit-qemu-$i ]; then cp /usr/src/buildkit-qemu-$i /usr/bin; fi; done
 COPY --from=binfmt /usr/bin/binfmt /usr/local/bin
-COPY --from=buildkit /usr/bin/buildkitd /usr/bin/buildctl /usr/bin/buildkit-runc /usr/local/bin
+COPY --from=buildkit /usr/bin/buildkitd /usr/bin/buildctl /usr/bin/buildkit-runc /usr/local/bin/
 COPY --from=registry /etc/docker/registry/config.yml /etc/docker/registry/config.yml
 COPY --from=registry /bin/registry /usr/local/bin
 COPY --from=klipper-lb /usr/bin/entry /usr/local/bin/klipper-lb
@@ -43,6 +43,7 @@ COPY --from=loglevel /usr/local/bin/loglevel /usr/local/bin/
 VOLUME /var/lib/buildkit
 
 COPY /scripts/acorn-helper-init /usr/local/bin
+COPY /scripts/acorn-busybox-init /usr/local/bin
 COPY /scripts/acorn-job-helper-init /usr/local/bin
 COPY /scripts/acorn-job-helper-shutdown /usr/local/bin
 COPY /scripts/acorn-job-get-output /usr/local/bin

integration/client/imagerules/testdata/nested-perms/Acornfile

@@ -1,6 +1,6 @@
 containers: {
   "rootapp": {
-      image: "nginx:latest"
+      image: "ghcr.io/acorn-io/images-mirror/nginx:latest"
       permissions: rules: [{
         verbs: ["get"]
         apiGroups: ["foo.bar.com"]

integration/client/imagerules/testdata/nested-perms/nested.acorn

@@ -1,6 +1,6 @@
 containers: {
   "awsapp": {
-      image: "nginx:latest"
+      image: "ghcr.io/acorn-io/images-mirror/nginx:latest"
       permissions: {
         rules: [{
           verbs: ["get"]

pkg/apis/internal.acorn.io/v1/appspec.go

@@ -165,6 +165,7 @@ type VolumeMount struct {
 	Volume     string            `json:"volume,omitempty"`
 	SubPath    string            `json:"subPath,omitempty"`
 	ContextDir string            `json:"contextDir,omitempty"`
+	Preload    bool              `json:"preload,omitempty"`
 	Secret     VolumeSecretMount `json:"secret,omitempty"`
 }
 

pkg/apis/internal.acorn.io/v1/unmarshal.go

@@ -974,7 +974,7 @@ func (in *VolumeMount) UnmarshalJSON(data []byte) error {
 	} else if strings.HasPrefix(s, "./") {
 		in.ContextDir = s
 	} else {
-		in.Volume, in.SubPath, err = parseVolumeReference(s)
+		in.Volume, in.SubPath, in.Preload, err = parseVolumeReference(s)
 		if err != nil {
 			return err
 		}
@@ -1411,14 +1411,19 @@ func parseVolumeDefinition(anonName, s string) (VolumeBinding, error) {
 	return result, nil
 }
 
-func parseVolumeReference(s string) (string, string, error) {
+// parseVolumeReference parses a volume reference string into its components, including query parameters
+// @return string - volume reference (unmodified)
+// @return string - subpath that should be mounted (query param)
+// @return bool 	- preload: if the volume should be pre-populated with data from the container image (query param)
+// @return error 	- parse error
+func parseVolumeReference(s string) (string, string, bool, error) {
 	if !strings.HasPrefix(s, "volume://") && !strings.HasPrefix(s, "ephemeral://") {
-		return s, "", nil
+		return s, "", false, nil
 	}
 
 	u, err := url.Parse(s)
 	if err != nil {
-		return "", "", fmt.Errorf("parsing volume reference %s: %w", s, err)
+		return "", "", false, fmt.Errorf("parsing volume reference %s: %w", s, err)
 	}
 
 	subPath := u.Query().Get("subPath")
@@ -1429,7 +1434,16 @@ func parseVolumeReference(s string) (string, string, error) {
 		subPath = u.Query().Get("sub-path")
 	}
 
-	return s, subPath, nil
+	preload := false
+	preloadStr := u.Query().Get("preload")
+	if preloadStr != "" {
+		preload, err = strconv.ParseBool(preloadStr)
+		if err != nil {
+			return "", "", false, fmt.Errorf("malformed ?preload value %q: %v", preloadStr, err)
+		}
+	}
+
+	return s, subPath, preload, nil
 }
 
 func MustParseResourceQuantity(s Quantity) *resource.Quantity {

pkg/apis/internal.acorn.io/v1/unmarshal_test.go

@@ -190,3 +190,29 @@ func FuzzUserContextUnmarshalJSON(f *testing.F) {
 		}
 	})
 }
+
+func TestParseVolumeReference(t *testing.T) {
+	s, subPath, preload, err := parseVolumeReference("name")
+	require.NoError(t, err)
+	require.Equal(t, "name", s)
+	require.Empty(t, subPath)
+	require.False(t, preload)
+
+	_, subPath, preload, err = parseVolumeReference("volume://foo?subPath=bar")
+	require.NoError(t, err)
+	require.False(t, preload)
+	require.Equal(t, "bar", subPath)
+
+	_, subPath, preload, err = parseVolumeReference("volume://foo?preload=true&subPath=bar")
+	require.NoError(t, err)
+	require.Equal(t, "bar", subPath)
+	require.True(t, preload)
+
+	_, subPath, preload, err = parseVolumeReference("volume://foo?preload=false")
+	require.NoError(t, err)
+	require.Empty(t, subPath)
+	require.False(t, preload)
+
+	_, _, _, err = parseVolumeReference("volume://foo?preload=foo")
+	require.Error(t, err)
+}

pkg/controller/appdefinition/deploy.go

@@ -31,6 +31,7 @@ import (
 	"github.com/acorn-io/z"
 	"github.com/google/go-containerregistry/pkg/name"
 	"github.com/rancher/wrangler/pkg/data/convert"
+	wname "github.com/rancher/wrangler/pkg/name"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	apierror "k8s.io/apimachinery/pkg/api/errors"
@@ -213,7 +214,7 @@ func hasContextDir(container v1.Container) bool {
 	return false
 }
 
-func toContainers(app *v1.AppInstance, tag name.Reference, name string, container v1.Container, interpolator *secrets.Interpolator, addWait bool) ([]corev1.Container, []corev1.Container) {
+func toContainers(app *v1.AppInstance, tag name.Reference, name string, container v1.Container, interpolator *secrets.Interpolator, addWait, addBusybox bool) ([]corev1.Container, []corev1.Container) {
 	var (
 		containers     []corev1.Container
 		initContainers []corev1.Container
@@ -234,7 +235,49 @@ func toContainers(app *v1.AppInstance, tag name.Reference, name string, containe
 		})
 	}
 
+	if addBusybox {
+		// Drop the static busybox binary into a shared volume so that we can use it in initContainers.
+		initContainers = append(initContainers, corev1.Container{
+			Name:    wname.SafeConcatName("acorn-helper-busybox", string(app.UID)),
+			Image:   system.DefaultImage(),
+			Command: []string{"acorn-busybox-init"},
+			VolumeMounts: []corev1.VolumeMount{
+				{
+					Name:      sanitizeVolumeName(AcornHelper),
+					MountPath: AcornHelperPath,
+				},
+			},
+		},
+		)
+	}
+
 	newContainer := toContainer(app, tag, name, container, interpolator, addWait && len(container.Ports) > 0)
+	for src, dir := range container.Dirs {
+		if dir.Preload {
+			// If a directory is marked for preload, then add an initContainer that copies the directory into the shared volume.
+			// Data will be copied to the data/ subdirectory and we'll drop a .preload-done file in the root to indicate that
+			// the copy has been completed (and should not be repeated).
+			initContainers = append(initContainers, corev1.Container{
+				Name:            wname.SafeConcatName("acorn-preload-dir", sanitizeVolumeName(src), string(app.UID)),
+				Image:           newContainer.Image,
+				Command:         []string{AcornHelperBusyboxPath, "sh", "-c"},
+				ImagePullPolicy: corev1.PullIfNotPresent,
+				// cp -aT == copy recursively, following symlinks and preserving file attributes, only copy the contents of the source directory
+				Args: []string{fmt.Sprintf("if [ ! -f /dest/.preload-done ]; then mkdir -p /dest/data && cp -aT %s /dest/data && date > /dest/.preload-done; fi", src)},
+				VolumeMounts: []corev1.VolumeMount{
+					{
+						Name:      sanitizeVolumeName(dir.Volume),
+						MountPath: "/dest",
+						SubPath:   dir.SubPath,
+					},
+					{
+						Name:      sanitizeVolumeName(AcornHelper),
+						MountPath: AcornHelperPath,
+					},
+				},
+			})
+		}
+	}
 	containers = append(containers, newContainer)
 	for _, entry := range typed.Sorted(container.Sidecars) {
 		newContainer = toContainer(app, tag, entry.Key, entry.Value, interpolator, addWait && len(entry.Value.Ports) > 0)
@@ -293,11 +336,18 @@ func toMounts(app *v1.AppInstance, container v1.Container, interpolation *secret
 				helperMounted = true
 			}
 		} else if mount.Secret.Name == "" {
-			result = append(result, corev1.VolumeMount{
+			vm := corev1.VolumeMount{
 				Name:      sanitizeVolumeName(mount.Volume),
 				MountPath: path.Join("/", mountPath),
 				SubPath:   mount.SubPath,
-			})
+			}
+			if mount.Preload {
+				// Preloaded contents land in the data/ subdirectory, since we have to drop the .preload-done file
+				// in the root of the volume to indicate that the copy has been completed
+				// and that may cause issues with applications that dislike unknown files in their path.
+				vm.SubPath = path.Join(vm.SubPath, "data")
+			}
+			result = append(result, vm)
 		} else {
 			result = append(result, corev1.VolumeMount{
 				Name:      secretPodVolName(mount.Secret.Name),
@@ -670,20 +720,27 @@ func getSecretAnnotations(req router.Request, appInstance *v1.AppInstance, conta
 
 func toDeployment(req router.Request, appInstance *v1.AppInstance, tag name.Reference, name string, container v1.Container, pullSecrets *PullSecrets, interpolator *secrets.Interpolator) (*appsv1.Deployment, error) {
 	var (
-		stateful = isStateful(appInstance, container)
-		addWait  = !stateful && len(acornSleepBinary) > 0 && z.Dereference(container.Scale) > 1 && !appInstance.Status.GetDevMode()
+		stateful   = isStateful(appInstance, container)
+		addWait    = !stateful && len(acornSleepBinary) > 0 && z.Dereference(container.Scale) > 1 && !appInstance.Status.GetDevMode()
+		addBusybox = false
 	)
 
+	for _, v := range container.Dirs {
+		if v.Preload {
+			addBusybox = true
+		}
+	}
+
 	interpolator = interpolator.ForContainer(name)
 
-	containers, initContainers := toContainers(appInstance, tag, name, container, interpolator, addWait)
+	containers, initContainers := toContainers(appInstance, tag, name, container, interpolator, addWait, addBusybox)
 
 	secretAnnotations, err := getSecretAnnotations(req, appInstance, container, interpolator)
 	if err != nil {
 		return nil, err
 	}
 
-	volumes, err := toVolumes(appInstance, container, interpolator, addWait)
+	volumes, err := toVolumes(appInstance, container, interpolator, addWait, addBusybox)
 	if err != nil {
 		return nil, err
 	}

pkg/controller/appdefinition/jobs.go

@@ -40,7 +40,13 @@ func toJobs(req router.Request, appInstance *v1.AppInstance, pullSecrets *PullSe
 		if err != nil {
 			return nil, err
 		}
-		job, err := toJob(req, appInstance, pullSecrets, tag, jobName, jobDef, interpolator)
+		addBusybox := false
+		for _, v := range jobDef.Dirs {
+			if v.Preload {
+				addBusybox = true
+			}
+		}
+		job, err := toJob(req, appInstance, pullSecrets, tag, jobName, jobDef, interpolator, addBusybox)
 		if err != nil {
 			return nil, err
 		}
@@ -85,7 +91,7 @@ func setSecretOutputVolume(containers []corev1.Container) (result []corev1.Conta
 	return
 }
 
-func toJob(req router.Request, appInstance *v1.AppInstance, pullSecrets *PullSecrets, tag name.Reference, name string, container v1.Container, interpolator *secrets.Interpolator) (kclient.Object, error) {
+func toJob(req router.Request, appInstance *v1.AppInstance, pullSecrets *PullSecrets, tag name.Reference, name string, container v1.Container, interpolator *secrets.Interpolator, addBusybox bool) (kclient.Object, error) {
 	interpolator = interpolator.ForJob(name)
 	jobEventName := jobs.GetEvent(name, appInstance)
 
@@ -100,7 +106,7 @@ func toJob(req router.Request, appInstance *v1.AppInstance, pullSecrets *PullSec
 		return nil, nil
 	}
 
-	containers, initContainers := toContainers(appInstance, tag, name, container, interpolator, false)
+	containers, initContainers := toContainers(appInstance, tag, name, container, interpolator, false, addBusybox)
 
 	containers = append(containers, corev1.Container{
 		Name:            jobs.Helper,
@@ -114,7 +120,7 @@ func toJob(req router.Request, appInstance *v1.AppInstance, pullSecrets *PullSec
 		return nil, err
 	}
 
-	volumes, err := toVolumes(appInstance, container, interpolator, false)
+	volumes, err := toVolumes(appInstance, container, interpolator, false, addBusybox)
 	if err != nil {
 		return nil, err
 	}