From 283bd44da054b2244dcb6d34943303d357793290 Mon Sep 17 00:00:00 2001 From: Cory Bennett Date: Mon, 4 May 2020 00:05:16 -0700 Subject: [PATCH 1/2] [wip] capture output from run to use elsewhere --- builtin/builtin.go | 3 ++ codegen/chain.go | 65 ++++++++++++++++++++++++++++++++++++++------ codegen/codegen.go | 2 ++ docs/reference.md | 6 ++++ language/builtin.hlb | 2 ++ solver/solve.go | 39 ++++++++++++++++++++++++++ 6 files changed, 108 insertions(+), 9 deletions(-) diff --git a/builtin/builtin.go b/builtin/builtin.go index fd60852f..f07ffa48 100644 --- a/builtin/builtin.go +++ b/builtin/builtin.go @@ -328,6 +328,9 @@ var ( }, "option::run": LookupByType{ Func: map[string]FuncLookup{ + "capture": FuncLookup{ + Params: []*parser.Field{}, + }, "readonlyRootfs": FuncLookup{ Params: []*parser.Field{}, }, diff --git a/codegen/chain.go b/codegen/chain.go index c8302c19..8de15ecb 100644 --- a/codegen/chain.go +++ b/codegen/chain.go @@ -349,13 +349,22 @@ func (cg *CodeGen) EmitFilesystemBuiltinChainStmt(ctx context.Context, scope *pa // to be in the context of a specific function run is in. case with.Expr.FuncLit != nil: for _, stmt := range with.Expr.FuncLit.Body.NonEmptyStmts() { - if stmt.Call.Func.Name() != "mount" || stmt.Call.Alias == nil { + if stmt.Call.Alias == nil { continue } - target, err := cg.EmitStringExpr(ctx, scope, stmt.Call.Args[1]) - if err != nil { - return fc, err + var target string + switch stmt.Call.Func.Name() { + case "mount": + target, err = cg.EmitStringExpr(ctx, scope, stmt.Call.Args[1]) + if err != nil { + return fc, err + } + case "capture": + target = "capture" + } + if target == "" { + continue } calls[target] = stmt.Call @@ -381,7 +390,43 @@ func (cg *CodeGen) EmitFilesystemBuiltinChainStmt(ctx context.Context, scope *pa for _, target := range targets { // Mounts are unique by its mountpoint, and its vertex representing the // mount after execing can be aliased. - cont := ac(calls[target], exec.GetMount(target)) + var cont bool + switch calls[target].Func.Name() { + case "mount": + cont = ac(calls[target], exec.GetMount(target)) + case "capture": + cont = ac(calls[target], func() (string, error) { + st := exec.Root() + pw := cg.mw.WithPrefix("", false) + + s, err := cg.newSession(ctx) + if err != nil { + return "", err + } + + g, ctx := errgroup.WithContext(ctx) + + g.Go(func() error { + return s.Run(ctx, cg.cln.Dialer()) + }) + + var captureBuf strings.Builder + g.Go(func() error { + opts, err := cg.SolveOptions(ctx, st) + if err != nil { + return err + } + opts = append(opts, solver.WithOutputCapture(&captureBuf)) + def, err := st.Marshal(ctx, llb.LinuxAmd64) + if err != nil { + return err + } + return solver.Solve(ctx, cg.cln, s, pw, def, opts...) + }) + err = g.Wait() + return captureBuf.String(), err + }) + } if !cont { return exec.Root(), ErrAliasReached } @@ -940,11 +985,13 @@ func (cg *CodeGen) EmitStringChainStmt(ctx context.Context, scope *parser.Scope, return nil, err } return func(_ string) (string, error) { - str, ok := v.(string) - if !ok { - return str, errors.WithStack(ErrCodeGen{obj.Node, ErrBadCast}) + switch s := v.(type) { + case string: + return s, nil + case func() (string, error): + return s() } - return str, nil + return "", errors.WithStack(ErrCodeGen{obj.Node, ErrBadCast}) }, nil } } diff --git a/codegen/codegen.go b/codegen/codegen.go index eec6ca91..9d49725a 100644 --- a/codegen/codegen.go +++ b/codegen/codegen.go @@ -1075,6 +1075,8 @@ func (cg *CodeGen) EmitExecOptions(ctx context.Context, scope *parser.Scope, op opts = append(opts, llb.Security(securityMode)) case "shlex": opts = append(opts, &shlexOption{}) + case "capture": + // no op, only relevant if aliased, handled in alias callback case "host": host, err := cg.EmitStringExpr(ctx, scope, args[0]) if err != nil { diff --git a/docs/reference.md b/docs/reference.md index 2ffdb2bd..d0ce4de1 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -480,6 +480,7 @@ If more than one arg is given, it will be executed directly, without a shell. #!hlb fs default() { run "arg" with option { + capture dir "path" env "key" "value" forward "src" "dest" @@ -497,6 +498,11 @@ If more than one arg is given, it will be executed directly, without a shell. } +#### option::run capture() + + + + #### option::run dir(string path) !!! info "string path" diff --git a/language/builtin.hlb b/language/builtin.hlb index e05b66a9..be7449fd 100644 --- a/language/builtin.hlb +++ b/language/builtin.hlb @@ -120,6 +120,8 @@ fs shell(variadic string arg) # @return the filesystem after the command has executed. fs run(variadic string arg) +option::run capture() + # Sets the rootfs as read-only for the duration of the run command. # # @return an option to set the rootfs as read-only. diff --git a/solver/solve.go b/solver/solve.go index 9b3b85f5..323d4aa3 100644 --- a/solver/solve.go +++ b/solver/solve.go @@ -3,6 +3,7 @@ package solver import ( "context" "encoding/json" + "io" "github.com/docker/buildx/util/progress" "github.com/moby/buildkit/client" @@ -26,6 +27,7 @@ type SolveInfo struct { Callbacks []func() error `json:"-"` ImageSpec *specs.Image Entitlements []entitlements.Entitlement + OutputCapture io.Writer } func WithDownloadDockerTarball(ref string) SolveOption { @@ -63,6 +65,13 @@ func WithDownloadOCITarball() SolveOption { } } +func WithOutputCapture(w io.Writer) SolveOption { + return func(info *SolveInfo) error { + info.OutputCapture = w + return nil + } +} + func WithCallback(fn func() error) SolveOption { return func(info *SolveInfo) error { info.Callbacks = append(info.Callbacks, fn) @@ -174,6 +183,36 @@ func Build(ctx context.Context, c *client.Client, s *session.Session, pw progres statusCh = pw.Status() } + if info.OutputCapture != nil { + captureStatusCh := make(chan *client.SolveStatus) + go func(origStatusCh chan *client.SolveStatus) { + defer func() { + if origStatusCh != nil { + close(origStatusCh) + } + }() + for { + select { + case <-pw.Done(): + return + case <-ctx.Done(): + return + case status, ok := <-captureStatusCh: + if !ok { + return + } + for _, log := range status.Logs { + info.OutputCapture.Write(log.Data) + } + if origStatusCh != nil { + origStatusCh <- status + } + } + } + }(statusCh) + statusCh = captureStatusCh + } + g.Go(func() error { _, err := c.Build(ctx, solveOpt, "", f, statusCh) return err From 85186e83286da791e83183f3f7179daf46c7a38f Mon Sep 17 00:00:00 2001 From: Cory Bennett Date: Tue, 5 May 2020 09:58:15 -0700 Subject: [PATCH 2/2] only collect output for desired vertex --- codegen/chain.go | 7 ++++++- solver/solve.go | 9 +++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/codegen/chain.go b/codegen/chain.go index 8de15ecb..c913dbea 100644 --- a/codegen/chain.go +++ b/codegen/chain.go @@ -15,6 +15,7 @@ import ( gateway "github.com/moby/buildkit/frontend/gateway/client" "github.com/moby/buildkit/session/filesync" "github.com/moby/buildkit/solver/pb" + digest "github.com/opencontainers/go-digest" "github.com/openllb/hlb/local" "github.com/openllb/hlb/parser" "github.com/openllb/hlb/solver" @@ -416,11 +417,15 @@ func (cg *CodeGen) EmitFilesystemBuiltinChainStmt(ctx context.Context, scope *pa if err != nil { return err } - opts = append(opts, solver.WithOutputCapture(&captureBuf)) def, err := st.Marshal(ctx, llb.LinuxAmd64) if err != nil { return err } + + // * last def is for constraint + // * second to last def is for the vertex we are capturing + dgst := digest.FromBytes(def.Def[len(def.Def)-2]) + opts = append(opts, solver.WithOutputCapture(dgst, &captureBuf)) return solver.Solve(ctx, cg.cln, s, pw, def, opts...) }) err = g.Wait() diff --git a/solver/solve.go b/solver/solve.go index 323d4aa3..089bd142 100644 --- a/solver/solve.go +++ b/solver/solve.go @@ -12,6 +12,7 @@ import ( gateway "github.com/moby/buildkit/frontend/gateway/client" "github.com/moby/buildkit/session" "github.com/moby/buildkit/util/entitlements" + digest "github.com/opencontainers/go-digest" specs "github.com/opencontainers/image-spec/specs-go/v1" "golang.org/x/sync/errgroup" ) @@ -28,6 +29,7 @@ type SolveInfo struct { ImageSpec *specs.Image Entitlements []entitlements.Entitlement OutputCapture io.Writer + OutputCaptureDigest digest.Digest } func WithDownloadDockerTarball(ref string) SolveOption { @@ -65,8 +67,9 @@ func WithDownloadOCITarball() SolveOption { } } -func WithOutputCapture(w io.Writer) SolveOption { +func WithOutputCapture(dgst digest.Digest, w io.Writer) SolveOption { return func(info *SolveInfo) error { + info.OutputCaptureDigest = dgst info.OutputCapture = w return nil } @@ -202,7 +205,9 @@ func Build(ctx context.Context, c *client.Client, s *session.Session, pw progres return } for _, log := range status.Logs { - info.OutputCapture.Write(log.Data) + if log.Vertex.String() == info.OutputCaptureDigest.String() { + info.OutputCapture.Write(log.Data) + } } if origStatusCh != nil { origStatusCh <- status