Add support for multiple build contexts in Dockerfile builds

ibuildthecloud · ibuildthecloud · commit 063071bfef75 · 2023-07-14T16:26:12.000-07:00
Signed-off-by: Darren Shepherd &lt;darren@acorn.io&gt;
diff --git a/go.mod b/go.mod
@@ -13,7 +13,7 @@ replace (
 require (
 	cuelang.org/go v0.5.0
 	github.com/AlecAivazis/survey/v2 v2.3.6
-	github.com/acorn-io/aml v0.0.0-20230707055340-d9d7a8c927b2
+	github.com/acorn-io/aml v0.0.0-20230714230100-7aded86d1c3e
 	github.com/acorn-io/baaah v0.0.0-20230707151126-5d519d272865
 	github.com/acorn-io/mink v0.0.0-20230523184405-ceaaa366d500
 	github.com/acorn-io/namegenerator v0.0.0-20220915160418-9e3d5a0ffe78
diff --git a/go.sum b/go.sum
@@ -92,8 +92,8 @@ github.com/ThalesIgnite/crypto11 v1.2.5 h1:1IiIIEqYmBvUYFeMnHqRft4bwf/O36jryEUpY
 github.com/ThalesIgnite/crypto11 v1.2.5/go.mod h1:ILDKtnCKiQ7zRoNxcp36Y1ZR8LBPmR2E23+wTQe/MlE=
 github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=
 github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
-github.com/acorn-io/aml v0.0.0-20230707055340-d9d7a8c927b2 h1:zx1YhJI+jecZ1nQ3MzFRblMU4KdACUG8DQfd0zB9XvY=
-github.com/acorn-io/aml v0.0.0-20230707055340-d9d7a8c927b2/go.mod h1:UEx5RRLFjryCEHN2pM59+d8A0mPJ3VAxggJOTzPymwg=
+github.com/acorn-io/aml v0.0.0-20230714230100-7aded86d1c3e h1:jx4uFAazHUwX2uMeCHNtOFSTXBMPjQtflQOnPjmkhms=
+github.com/acorn-io/aml v0.0.0-20230714230100-7aded86d1c3e/go.mod h1:UEx5RRLFjryCEHN2pM59+d8A0mPJ3VAxggJOTzPymwg=
 github.com/acorn-io/baaah v0.0.0-20230707151126-5d519d272865 h1:BPPGCEBgPxn7crFFWqLDJUlzdHQ23olFkdUqlXd3KA8=
 github.com/acorn-io/baaah v0.0.0-20230707151126-5d519d272865/go.mod h1:LtwaWrYK/VuGptWxeD5Sgl0sgJV1ksicpTzyLilow1U=
 github.com/acorn-io/mink v0.0.0-20230523184405-ceaaa366d500 h1:tiM36bM+iMWuW9HM+YlM1GfNDXC7f565z8Be5epO0qM=
diff --git a/integration/build/build_test.go b/integration/build/build_test.go
@@ -170,6 +170,17 @@ func TestBuildDefault(t *testing.T) {
 	assert.Len(t, image.ImageData.Containers, 1)
 }
 
+func TestBuildMulticontext(t *testing.T) {
+	c := helper.BuilderClient(t, system.DefaultUserNamespace)
+	image, err := c.AcornImageBuild(helper.GetCTX(t), "./testdata/multicontextdir/Acornfile", &client.AcornImageBuildOptions{
+		Cwd: "./testdata/multicontextdir",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Len(t, image.ImageData.Containers, 1)
+}
+
 func TestMultiArch(t *testing.T) {
 	helper.StartController(t)
 	cfg := helper.StartAPI(t)
diff --git a/integration/build/testdata/multicontextdir/Acornfile b/integration/build/testdata/multicontextdir/Acornfile
@@ -0,0 +1,11 @@
+containers: {
+	simple: {
+		build: {
+            dockerfile: "Dockerfile"
+			context: "./files"
+			additionalContexts: {
+				"other-context": "./files/subdir"
+			}
+		}
+	}
+}
diff --git a/integration/build/testdata/multicontextdir/Dockerfile b/integration/build/testdata/multicontextdir/Dockerfile
@@ -0,0 +1,4 @@
+FROM ghcr.io/acorn-io/images-mirror/busybox:latest AS not-default
+COPY test.sh /test.sh
+COPY --from=other-context /token /
+RUN test -f /test.sh && sh /test.sh
diff --git a/integration/build/testdata/multicontextdir/files/subdir/token b/integration/build/testdata/multicontextdir/files/subdir/token
@@ -0,0 +1 @@
+hi
diff --git a/integration/build/testdata/multicontextdir/files/test.sh b/integration/build/testdata/multicontextdir/files/test.sh
@@ -0,0 +1,4 @@
+#!/bin/sh
+set -e -x
+test -f /token
+[ "hi" = "$(cat /token)" ]
diff --git a/pkg/apis/internal.acorn.io/v1/appspec.go b/pkg/apis/internal.acorn.io/v1/appspec.go
@@ -41,6 +41,7 @@ type AcornBuild struct {
 
 type Build struct {
 	Context            string            `json:"context,omitempty"`
+	AdditionalContexts map[string]string `json:"additionalContexts,omitempty"`
 	Dockerfile         string            `json:"dockerfile,omitempty"`
 	DockerfileContents string            `json:"dockerfileContents,omitempty"`
 	Target             string            `json:"target,omitempty"`
diff --git a/pkg/apis/internal.acorn.io/v1/zz_generated.deepcopy.go b/pkg/apis/internal.acorn.io/v1/zz_generated.deepcopy.go
diff --git a/pkg/appdefinition/appdefinition_test.go b/pkg/appdefinition/appdefinition_test.go
@@ -1017,35 +1017,39 @@ images: {
 			},
 			"build": {
 				Build: &v1.Build{
-					BuildArgs:  map[string]string{},
-					Context:    ".",
-					Dockerfile: "Dockerfile",
+					BuildArgs:          map[string]string{},
+					AdditionalContexts: map[string]string{},
+					Context:            ".",
+					Dockerfile:         "Dockerfile",
 				},
 				Sidecars: map[string]v1.ContainerImageBuilderSpec{
 					"side": {
 						Build: &v1.Build{
-							BuildArgs:  map[string]string{},
-							Context:    ".",
-							Dockerfile: "Dockerfile",
+							BuildArgs:          map[string]string{},
+							AdditionalContexts: map[string]string{},
+							Context:            ".",
+							Dockerfile:         "Dockerfile",
 						},
 					},
 				},
 			},
 			"buildcontext": {
 				Build: &v1.Build{
-					BuildArgs:  map[string]string{},
-					Context:    ".",
-					Dockerfile: "Dockerfile",
+					BuildArgs:          map[string]string{},
+					AdditionalContexts: map[string]string{},
+					Context:            ".",
+					Dockerfile:         "Dockerfile",
 					ContextDirs: map[string]string{
 						"/var/tmp": "./foo/bar",
 					},
 				},
 				Sidecars: map[string]v1.ContainerImageBuilderSpec{
 					"side": {
 						Build: &v1.Build{
-							BuildArgs:  map[string]string{},
-							Context:    ".",
-							Dockerfile: "Dockerfile",
+							BuildArgs:          map[string]string{},
+							AdditionalContexts: map[string]string{},
+							Context:            ".",
+							Dockerfile:         "Dockerfile",
 							ContextDirs: map[string]string{
 								"/var/tmp": "./foo/bar",
 							},
diff --git a/pkg/build/buildkit/build.go b/pkg/build/buildkit/build.go
@@ -97,9 +97,15 @@ func Build(ctx context.Context, pushRepo string, local bool, cwd string, platfor
 				"dockerfile": filepath.Dir(filepath.Join(cwd, build.Dockerfile)),
 			}
 		} else {
+			additionalContext := map[string]string{}
+			for k, v := range build.AdditionalContexts {
+				options.FrontendAttrs["context:"+k] = "local:" + k
+				additionalContext[k] = filepath.Join(cwd, v)
+			}
 			options.Session = append(options.Session,
 				buildclient.NewFileServer(messages,
 					filepath.Join(cwd, build.Context),
+					additionalContext,
 					filepath.Join(cwd, build.Dockerfile),
 					build.DockerfileContents))
 		}
diff --git a/pkg/buildclient/filesync.go b/pkg/buildclient/filesync.go
@@ -20,14 +20,16 @@ const (
 type FileServer struct {
 	messages           Messages
 	context            string
+	additionalContexts map[string]string
 	dockerfilePath     string
 	dockerfileContents string
 }
 
-func NewFileServer(messages Messages, context, dockerfilePath, dockerFileContents string) *FileServer {
+func NewFileServer(messages Messages, context string, additionalContexts map[string]string, dockerfilePath, dockerFileContents string) *FileServer {
 	return &FileServer{
 		messages:           messages,
 		context:            context,
+		additionalContexts: additionalContexts,
 		dockerfilePath:     dockerfilePath,
 		dockerfileContents: dockerFileContents,
 	}
@@ -49,6 +51,7 @@ func (f *FileServer) DiffCopy(server filesync.FileSync_DiffCopyServer) error {
 		SyncOptions: &SyncOptions{
 			Compress:           true,
 			Context:            f.context,
+			AdditionalContexts: f.additionalContexts,
 			Dockerfile:         f.dockerfilePath,
 			DockerfileContents: f.dockerfileContents,
 			OverrideExcludes:   ctx.Get(keyOverrideExcludes),
@@ -77,8 +80,8 @@ func (f *FileServer) DiffCopy(server filesync.FileSync_DiffCopyServer) error {
 	}()
 
 	for msg := range msgs {
-		logrus.Tracef("file sync message msg.FileSessionID=%s sessionID=%s close=%v", msg.FileSessionID, sessionID, msg.FileSessionClose)
 		if msg.FileSessionID == sessionID {
+			logrus.Tracef("file sync message msg.FileSessionID=%s sessionID=%s close=%v", msg.FileSessionID, sessionID, msg.FileSessionClose)
 			if msg.FileSessionClose {
 				cancel()
 			} else {
diff --git a/pkg/buildclient/filesyncclient.go b/pkg/buildclient/filesyncclient.go
@@ -89,6 +89,7 @@ func createFileMapInput(cwd string, opts *SyncOptions) (string, map[string]strin
 		err        error
 		context    = filepath.Join(cwd, opts.Context)
 		dockerfile = filepath.Dir(filepath.Join(cwd, opts.Dockerfile))
+		dirs       = map[string]string{}
 	)
 	if opts.DockerfileContents != "" {
 		tempDir, err = os.MkdirTemp("", "acorn")
@@ -101,10 +102,17 @@ func createFileMapInput(cwd string, opts *SyncOptions) (string, map[string]strin
 			return "", nil, err
 		}
 	}
-	return tempDir, map[string]string{
-		"context":    context,
-		"dockerfile": dockerfile,
-	}, nil
+
+	if len(opts.DirName) > 0 {
+		if opts.DirName[0] == "context" {
+			dirs["context"] = context
+		} else if dir, ok := opts.AdditionalContexts[opts.DirName[0]]; ok {
+			dirs[opts.DirName[0]] = filepath.Join(cwd, dir)
+		}
+	}
+
+	dirs["dockerfile"] = dockerfile
+	return tempDir, dirs, nil
 }
 
 func prepareSyncedDirs(localDirs map[string]string, dirNames []string, followPaths []string) (filesync.StaticDirSource, error) {
@@ -122,7 +130,7 @@ func prepareSyncedDirs(localDirs map[string]string, dirNames []string, followPat
 			return nil, fmt.Errorf("%s not a directory", d)
 		}
 		for _, dirName := range dirNames {
-			if dirName == "context" && localDirName == dirName {
+			if dirName != "dockerfile" && localDirName == dirName {
 				for _, followPath := range followPaths {
 					if ignoreLocalFiles[followPath] {
 						continue
@@ -204,10 +212,10 @@ func (s *fileSyncClient) RecvMsg(m interface{}) error {
 		if !ok {
 			return io.EOF
 		}
-		logrus.Tracef("fileSyncClient msg.fileSessionID=%s sessionID=%s packetNil=%v", nextMessage.FileSessionID, s.sessionID, nextMessage.Packet == nil)
 		if nextMessage.Packet == nil || nextMessage.FileSessionID != s.sessionID {
 			continue
 		}
+		logrus.Tracef("fileSyncClient msg.fileSessionID=%s sessionID=%s packetNil=%v", nextMessage.FileSessionID, s.sessionID, nextMessage.Packet == nil)
 		n := m.(*types.Packet)
 		*n = *nextMessage.Packet
 		return nil
diff --git a/pkg/buildclient/messages.go b/pkg/buildclient/messages.go
@@ -132,6 +132,7 @@ func (m *Message) String() string {
 
 type SyncOptions struct {
 	Context            string
+	AdditionalContexts map[string]string
 	Dockerfile         string
 	DockerfileContents string
 
diff --git a/pkg/openapi/generated/openapi_generated.go b/pkg/openapi/generated/openapi_generated.go
