Merge pull request #174 from AkihiroSuda/dev-compose

compose: support build

Author: AkihiroSuda

README.md

@@ -221,6 +221,7 @@ It does not necessarily mean that the corresponding features are missing in cont
     - [:whale: nerdctl compose](#whale-nerdctl-compose)
     - [:whale: nerdctl compose up](#whale-nerdctl-compose-up)
     - [:whale: nerdctl compose logs](#whale-nerdctl-compose-logs)
+    - [:whale: nerdctl compose build](#whale-nerdctl-compose-build)
     - [:whale: nerdctl compose down](#whale-nerdctl-compose-down)
   - [Global flags](#global-flags)
   - [Unimplemented Docker commands](#unimplemented-docker-commands)
@@ -777,9 +778,10 @@ Flags:
 - :whale: `-d, --detach`: Detached mode: Run containers in the background
 - :whale: `--no-color`: Produce monochrome output
 - :whale: `--no-log-prefix`: Don't print prefix in logs
+- :whale: `build`: Build images before starting containers.
 
 Unimplemented `docker-compose up` flags: `--quiet-pull`, `--no-deps`, `--force-recreate`, `--always-recreate-deps`, `--no-recreate`,
-`--no-start`, `--build`, `--abort-on-container-exit`, `--attach-dependencies`, `--timeout`, `--renew-anon-volumes`, `--remove-orphans`, `--exit-code-from`,
+`--no-start`, `--abort-on-container-exit`, `--attach-dependencies`, `--timeout`, `--renew-anon-volumes`, `--remove-orphans`, `--exit-code-from`,
 `--scale`
 
 ### :whale: nerdctl compose logs
@@ -793,6 +795,18 @@ Flags:
 
 Unimplemented `docker-compose logs` flags:  `--timestamps`, `--tail`
 
+### :whale: nerdctl compose build
+Build or rebuild services.
+
+Usage: `nerdctl compose build [OPTIONS]`
+
+Flags:
+- :whale: `--build-arg`: Set build-time variables for services
+- :whale: `--no-cache`: Do not use cache when building the image
+- :whale: `--progress`: Set type of progress output (auto, plain, tty)
+
+Unimplemented `docker-compose build` flags:  `--compress`, `--force-rm`, `--memory`, `--no-rm`, `--parallel`, `--pull`, `--quiet`
+
 ### :whale: nerdctl compose down
 Remove containers and associated resources
 
@@ -850,7 +864,7 @@ Registry:
 - `docker search`
 
 Compose:
-- `docker-compose build|config|create|events|exec|images|kill|logs|pause|port|ps|pull|push|restart|rm|run|scale|start|stop|top|unpause`
+- `docker-compose config|create|events|exec|images|kill|pause|port|ps|pull|push|restart|rm|run|scale|start|stop|top|unpause`
 
 Others:
 - `docker system df`

compose.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/errdefs"
+	refdocker "github.com/containerd/containerd/reference/docker"
 	"github.com/containerd/nerdctl/pkg/composer"
 	"github.com/containerd/nerdctl/pkg/imgutil"
 	"github.com/containerd/nerdctl/pkg/netutil"
@@ -34,6 +35,7 @@ var composeCommand = &cli.Command{
 	Subcommands: []*cli.Command{
 		composeUpCommand,
 		composeLogsCommand,
+		composeBuildCommand,
 		composeDownCommand,
 	},
 
@@ -94,6 +96,21 @@ func getComposer(clicontext *cli.Context, client *containerd.Client) (*composer.
 		}
 	}
 
+	o.ImageExists = func(ctx context.Context, rawRef string) (bool, error) {
+		named, err := refdocker.ParseDockerRef(rawRef)
+		if err != nil {
+			return false, err
+		}
+		ref := named.String()
+		if _, err := client.ImageService().Get(ctx, ref); err != nil {
+			if errors.Is(err, errdefs.ErrNotFound) {
+				return false, nil
+			}
+			return false, err
+		}
+		return true, nil
+	}
+
 	insecure := clicontext.Bool("insecure-registry")
 	o.EnsureImage = func(ctx context.Context, imageName, pullMode string) error {
 		_, imgErr := imgutil.EnsureImage(ctx, client, clicontext.App.Writer, clicontext.String("snapshotter"), imageName,

compose_build.go

@@ -0,0 +1,67 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package main
+
+import (
+	"github.com/containerd/nerdctl/pkg/composer"
+	"github.com/pkg/errors"
+	"github.com/urfave/cli/v2"
+)
+
+var composeBuildCommand = &cli.Command{
+	Name:   "build",
+	Usage:  "Build or rebuild services",
+	Action: composeBuildAction,
+	Flags: []cli.Flag{
+		&cli.StringSliceFlag{
+			Name:  "build-arg",
+			Usage: "Set build-time variables for services.",
+		},
+		&cli.BoolFlag{
+			Name:  "no-cache",
+			Usage: "Do not use cache when building the image.",
+		},
+		&cli.StringFlag{
+			Name:  "progress",
+			Usage: "Set type of progress output",
+		},
+	},
+}
+
+func composeBuildAction(clicontext *cli.Context) error {
+	if clicontext.NArg() != 0 {
+		// TODO: support specifying service names as args
+		return errors.Errorf("arguments %v not supported", clicontext.Args())
+	}
+
+	client, ctx, cancel, err := newClient(clicontext)
+	if err != nil {
+		return err
+	}
+	defer cancel()
+
+	c, err := getComposer(clicontext, client)
+	if err != nil {
+		return err
+	}
+	bo := composer.BuildOptions{
+		Args:     clicontext.StringSlice("build-arg"),
+		NoCache:  clicontext.Bool("no-cache"),
+		Progress: clicontext.String("progress"),
+	}
+	return c.Build(ctx, bo)
+}

compose_up.go

@@ -39,6 +39,10 @@ var composeUpCommand = &cli.Command{
 			Name:  "no-log-prefix",
 			Usage: "Don't print prefix in logs",
 		},
+		&cli.BoolFlag{
+			Name:  "build",
+			Usage: "Build images before starting containers.",
+		},
 	},
 }
 
@@ -57,6 +61,7 @@ func composeUpAction(clicontext *cli.Context) error {
 		Detach:      clicontext.Bool("detach"),
 		NoColor:     clicontext.Bool("no-color"),
 		NoLogPrefix: clicontext.Bool("no-log-prefix"),
+		ForceBuild:  clicontext.Bool("build"),
 	}
 	return c.Up(ctx, uo)
 }

compose_up_test.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/containerd/nerdctl/pkg/testutil"
 	"github.com/pkg/errors"
+	"gotest.tools/v3/assert"
 )
 
 func TestComposeUp(t *testing.T) {
@@ -112,3 +113,36 @@ volumes:
 	base.Cmd("volume", "inspect", fmt.Sprintf("%s_db", projectName)).AssertFail()
 	base.Cmd("network", "inspect", fmt.Sprintf("%s_default", projectName)).AssertFail()
 }
+
+func TestComposeUpBuild(t *testing.T) {
+	testutil.RequiresBuild(t)
+	base := testutil.NewBase(t)
+
+	const dockerComposeYAML = `
+services:
+  web:
+    build: .
+    ports:
+    - 8080:80
+`
+	dockerfile := fmt.Sprintf(`FROM %s
+COPY index.html /usr/share/nginx/html/index.html
+`, testutil.NginxAlpineImage)
+	indexHTML := t.Name()
+
+	comp := testutil.NewComposeDir(t, dockerComposeYAML)
+	defer comp.CleanUp()
+
+	comp.WriteFile("Dockerfile", dockerfile)
+	comp.WriteFile("index.html", indexHTML)
+
+	base.ComposeCmd("-f", comp.YAMLFullPath(), "up", "-d", "--build").AssertOK()
+	defer base.ComposeCmd("-f", comp.YAMLFullPath(), "down", "-v").Run()
+
+	resp, err := httpGet("http://127.0.0.1:8080", 50)
+	assert.NilError(t, err)
+	respBody, err := ioutil.ReadAll(resp.Body)
+	assert.NilError(t, err)
+	t.Logf("respBody=%q", respBody)
+	assert.Assert(t, strings.Contains(string(respBody), indexHTML))
+}

docs/compose.md

@@ -18,7 +18,7 @@ which was derived from [Docker Compose file version 3 specification](https://doc
 
 ### Unimplemented YAML fields
 - Fields that correspond to unimplemented `docker run` flags, e.g., `services.<SERVICE>.links` (corresponds to `docker run --link`)
-- `services.<SERVICE>.build`
+- Fields that correspond to unimplemented `docker build` flags, e.g., `services.<SERVICE>.build.labels` (corresponds to `docker build --label`)
 - `services.<SERVICE>.credential_spec`
 - `services.<SERVICE>.deploy.update_config`
 - `services.<SERVICE>.deploy.rollback_config`
@@ -33,6 +33,9 @@ which was derived from [Docker Compose file version 3 specification](https://doc
 - `secrets.<SECRET>.external`
 
 ### Incompatibility
+#### `services.<SERVICE>.build.context`
+- The value must be a local directory path, not a URL.
+
 #### `services.<SERVICE>.entrypoint`
 - Multiple entrypoint strings cannot be specified.
 

go.mod

@@ -3,7 +3,7 @@ module github.com/containerd/nerdctl
 go 1.16
 
 require (
-	github.com/compose-spec/compose-go v0.0.0-20210408102153-fe76f8471db2
+	github.com/compose-spec/compose-go v0.0.0-20210415072938-109132373677
 	github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68
 	github.com/containerd/console v1.0.2
 	github.com/containerd/containerd v1.5.0-rc.1

go.sum

@@ -97,8 +97,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
-github.com/compose-spec/compose-go v0.0.0-20210408102153-fe76f8471db2 h1:z1lMW1kukwXXuTxwqXEZUuNrkTb7NA/gnpcsEtYpTsQ=
-github.com/compose-spec/compose-go v0.0.0-20210408102153-fe76f8471db2/go.mod h1:6eIT9U2OgdHmkRD6szmqatCrWWEEUSwl/j2iJYH4jLo=
+github.com/compose-spec/compose-go v0.0.0-20210415072938-109132373677 h1:WXbccjCgmE1RtrWGbtlLz6NF4VMIZrXsM+FX+UzFmd0=
+github.com/compose-spec/compose-go v0.0.0-20210415072938-109132373677/go.mod h1:6eIT9U2OgdHmkRD6szmqatCrWWEEUSwl/j2iJYH4jLo=
 github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE=
 github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU=
 github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU=

pkg/composer/build.go

@@ -0,0 +1,73 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package composer
+
+import (
+	"context"
+	"os"
+
+	"github.com/compose-spec/compose-go/types"
+	"github.com/containerd/nerdctl/pkg/composer/serviceparser"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+type BuildOptions struct {
+	Args     []string // --build-arg strings
+	NoCache  bool
+	Progress string
+}
+
+func (c *Composer) Build(ctx context.Context, bo BuildOptions) error {
+	return c.project.WithServices(nil, func(svc types.ServiceConfig) error {
+		ps, err := serviceparser.Parse(c.project, svc)
+		if err != nil {
+			return err
+		}
+		if ps.Build != nil {
+			return c.buildServiceImage(ctx, ps.Image, ps.Build, bo)
+		}
+		return nil
+	})
+}
+
+func (c *Composer) buildServiceImage(ctx context.Context, image string, b *serviceparser.Build, bo BuildOptions) error {
+	logrus.Infof("Building image %s", image)
+
+	var args []string // nolint: prealloc
+	for _, a := range bo.Args {
+		args = append(args, "--build-arg="+a)
+	}
+	if bo.NoCache {
+		args = append(args, "--no-cache")
+	}
+	if bo.Progress != "" {
+		args = append(args, "--progress="+bo.Progress)
+	}
+	args = append(args, b.BuildArgs...)
+
+	cmd := c.createNerdctlCmd(ctx, append([]string{"build"}, args...)...)
+	if c.DebugPrintFull {
+		logrus.Debugf("Running %v", cmd.Args)
+	}
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return errors.Wrapf(err, "error while building image %s", image)
+	}
+	return nil
+}

pkg/composer/composer.go

@@ -38,6 +38,7 @@ type Options struct {
 	NerdctlArgs    []string
 	NetworkExists  func(string) (bool, error)
 	VolumeExists   func(string) (bool, error)
+	ImageExists    func(ctx context.Context, imageName string) (bool, error)
 	EnsureImage    func(ctx context.Context, imageName, pullMode string) error
 	DebugPrintFull bool // full debug print, may leak secret env var to logs
 }