diff --git a/build/build.go b/build/build.go
index 04665f5207db..8325ccf08f44 100644
--- a/build/build.go
+++ b/build/build.go
@@ -6,6 +6,7 @@ import (
_ "crypto/sha256" // ensure digests can be computed
"encoding/base64"
"encoding/json"
+ stderrors "errors"
"fmt"
"io"
"maps"
@@ -468,9 +469,20 @@ func prepareMultiDriverExports(so *client.SolveOpt, pushNames *string, insecureP
}
type (
+ ExecutionMode string
+ Execution struct {
+ Mode ExecutionMode
+ Parallel int
+ }
EvaluateFunc func(ctx context.Context, name string, c gateway.Client, res *gateway.Result, opt Options) error
Handler struct {
- Evaluate EvaluateFunc
+ Evaluate EvaluateFunc
+ Execution Execution
+ Completed func(name string, result TargetResult)
+ }
+ TargetResult struct {
+ Err error
+ Aborted bool
}
linkedTargetState struct {
results *waitmap.Map
@@ -479,6 +491,22 @@ type (
parents map[string][]string
children map[string][]string
}
+ linkedTargetHooks struct {
+ preEvaluate func() error
+ evaluate func() error
+ postEvaluate func() error
+ }
+ syncTargetState struct {
+ targets []string
+ results *waitmap.Map
+ evaluated *waitmap.Map
+ }
+)
+
+const (
+ ExecutionModeFailFast ExecutionMode = "fail-fast"
+ ExecutionModeDeferOutput ExecutionMode = "defer-output"
+ ExecutionModeDeferError ExecutionMode = "defer-error"
)
func newLinkedTargetState(parents, children map[string][]string) *linkedTargetState {
@@ -495,41 +523,141 @@ func (s *linkedTargetState) isLinked(key string) bool {
return len(s.parents[key]) > 0 || len(s.children[key]) > 0
}
-func (s *linkedTargetState) run(ctx context.Context, key string, result any, evaluate func() error) error {
+func (s *linkedTargetState) hasLinks() bool {
+ return len(s.parents) > 0 || len(s.children) > 0
+}
+
+func (s *linkedTargetState) fail(key string, err error) {
+ s.results.Set(key, err)
+ s.evaluated.Set(key, err)
+ s.completed.Set(key, err)
+}
+
+func (s *linkedTargetState) run(ctx context.Context, key string, result any, hooks linkedTargetHooks) error {
// Registration flows from parents to children. Waiting for every direct child
// here preserves external-cache lookup before evaluation begins.
s.results.Set(key, result)
children := s.children[key]
- if _, err := s.results.Get(ctx, children...); err != nil {
+ if res, err := s.results.Get(ctx, children...); err != nil {
+ return err
+ } else if err := wrapResultError(res, "aborted: dependent target failed"); err != nil {
return err
}
+ if hooks.preEvaluate != nil {
+ if err := hooks.preEvaluate(); err != nil {
+ return err
+ }
+ }
// Evaluation follows dependency order so the target's own session is attached
// to shared solver vertices before a dependent can evaluate them.
- if _, err := s.evaluated.Get(ctx, s.parents[key]...); err != nil {
+ if res, err := s.evaluated.Get(ctx, s.parents[key]...); err != nil {
+ return err
+ } else if err := wrapResultError(res, "aborted: dependency target failed"); err != nil {
return err
}
- if err := evaluate(); err != nil {
+ if err := hooks.evaluate(); err != nil {
return err
}
s.evaluated.Set(key, struct{}{})
+ if hooks.postEvaluate != nil {
+ if err := hooks.postEvaluate(); err != nil {
+ return err
+ }
+ }
// Completion flows back from children to parents, retaining each parent job
// and its session until every dependent has finished evaluating.
- if _, err := s.completed.Get(ctx, children...); err != nil {
+ if res, err := s.completed.Get(ctx, children...); err != nil {
+ return err
+ } else if err := wrapResultError(res, "aborted: dependent target failed"); err != nil {
return err
}
s.completed.Set(key, struct{}{})
return nil
}
-func Build(ctx context.Context, nodes []builder.Node, opts map[string]Options, docker *dockerutil.Client, cfg *confutil.Config, w progress.Writer) (resp map[string]*client.SolveResponse, err error) {
- return BuildWithResultHandler(ctx, nodes, opts, docker, cfg, w, nil)
+func resultError(results map[string]any) error {
+ for _, key := range slices.Sorted(maps.Keys(results)) {
+ result := results[key]
+ if err, ok := result.(error); ok {
+ return err
+ }
+ }
+ return nil
+}
+
+type targetAbortError struct{ error }
+
+func (e targetAbortError) Unwrap() error {
+ return e.error
+}
+
+func wrapResultError(results map[string]any, msg string) error {
+ if err := resultError(results); err != nil {
+ var abortErr targetAbortError
+ if stderrors.As(err, &abortErr) {
+ return err
+ }
+ return targetAbortError{errors.Wrap(err, msg)}
+ }
+ return nil
+}
+
+func newSyncTargetState(opts map[string]Options, drivers map[string][]*noderesolver.ResolvedNode) *syncTargetState {
+ targets := make([]string, 0, len(opts))
+ for k := range opts {
+ for _, dp := range drivers[k] {
+ targets = append(targets, resultKey(dp, k))
+ }
+ }
+ return &syncTargetState{
+ targets: targets,
+ results: waitmap.New(),
+ evaluated: waitmap.New(),
+ }
+}
+
+func (s *syncTargetState) fail(key string, err error) {
+ s.results.Set(key, err)
+ s.evaluated.Set(key, err)
+}
+
+func (s *syncTargetState) waitResult(ctx context.Context, key string, result any) error {
+ s.results.Set(key, result)
+ results, err := s.results.Get(ctx, s.targets...)
+ if err != nil {
+ return err
+ }
+ return wrapResultError(results, "aborted: another target failed")
+}
+
+func (s *syncTargetState) waitEvaluated(ctx context.Context, key string, result any) error {
+ s.evaluated.Set(key, result)
+ results, err := s.evaluated.Get(ctx, s.targets...)
+ if err != nil {
+ return err
+ }
+ return wrapResultError(results, "aborted: another target failed")
}
-func BuildWithResultHandler(ctx context.Context, nodes []builder.Node, opts map[string]Options, docker *dockerutil.Client, cfg *confutil.Config, w progress.Writer, bh *Handler) (resp map[string]*client.SolveResponse, err error) {
+func (h *Handler) completed(ctx context.Context, name string, err error) {
+ if h == nil || h.Completed == nil {
+ return
+ }
+ var abortErr targetAbortError
+ h.Completed(name, TargetResult{
+ Err: err,
+ Aborted: err != nil && (context.Cause(ctx) != nil || stderrors.As(err, &abortErr) || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded)),
+ })
+}
+
+func Build(ctx context.Context, nodes []builder.Node, opts map[string]Options, docker *dockerutil.Client, cfg *confutil.Config, w progress.Writer, bh *Handler) (resp map[string]*client.SolveResponse, err error) {
if len(nodes) == 0 {
return nil, errors.Errorf("driver required for build")
}
+ ctx, cancel := context.WithCancelCause(ctx)
+ defer func() { cancel(err) }()
+
nodes, err = filterAvailableNodes(nodes)
if err != nil {
return nil, errors.Wrapf(err, "no valid drivers found")
@@ -543,7 +671,12 @@ func BuildWithResultHandler(ctx context.Context, nodes []builder.Node, opts map[
return nil, err
}
- eg, ctx := errgroup.WithContext(ctx)
+ var eg *errgroup.Group
+ if bh != nil && bh.Execution.Mode == ExecutionModeDeferError {
+ eg = &errgroup.Group{}
+ } else {
+ eg, ctx = errgroup.WithContext(ctx)
+ }
reqForNodes, release, err := newBuildRequests(ctx, docker, cfg, drivers, w, opts)
if err != nil {
return nil, err
@@ -563,6 +696,14 @@ func BuildWithResultHandler(ctx context.Context, nodes []builder.Node, opts map[
return nil, err
}
sharedSessionsWG := map[string]*sync.WaitGroup{}
+ var sharedSessionHolds []func()
+ releaseSharedSessionHolds := func() {
+ for _, release := range sharedSessionHolds {
+ release()
+ }
+ sharedSessionHolds = nil
+ }
+ defer releaseSharedSessionHolds()
resp = map[string]*client.SolveResponse{}
var respMu sync.Mutex
@@ -573,8 +714,29 @@ func BuildWithResultHandler(ctx context.Context, nodes []builder.Node, opts map[
// loop below, before any goroutines are spawned; no mutex needed.
linkedClients := make(map[string]*client.Client)
- for k, opt := range opts {
- err := func(k string) (err error) {
+ var syncState *syncTargetState
+ if bh != nil && bh.Execution.Mode == ExecutionModeDeferOutput {
+ if bh.Execution.Parallel > 0 && bh.Execution.Parallel < len(opts) {
+ return nil, errors.Errorf("defer-output execution requires parallelism to be unlimited or at least the number of targets, including targets referenced by target contexts")
+ }
+ // Sync waits for every solve result before any ref evaluation starts and
+ // every ref evaluation before exporters can run, so output is only written
+ // after all targets have reached the output boundary successfully.
+ syncState = newSyncTargetState(opts, drivers)
+ }
+
+ var targetLimit chan struct{}
+ if bh != nil && bh.Execution.Parallel > 0 {
+ if bh.Execution.Parallel < len(opts) && linkedTargets.hasLinks() {
+ return nil, errors.Errorf("limited parallelism is not supported with linked targets")
+ }
+ targetLimit = make(chan struct{}, bh.Execution.Parallel)
+ }
+
+ targets := slices.Sorted(maps.Keys(opts))
+ for _, k := range targets {
+ opt := opts[k]
+ err = func(k string) (err error) {
dps := drivers[k]
multiDriver := len(drivers[k]) > 1
@@ -585,14 +747,33 @@ func BuildWithResultHandler(ctx context.Context, nodes []builder.Node, opts map[
}
baseCtx := ctx
+ res := make([]*client.SolveResponse, len(dps))
+ eg2, ctx := errgroup.WithContext(ctx)
+ var releaseTarget func()
+ if targetLimit != nil {
+ select {
+ case targetLimit <- struct{}{}:
+ releaseTarget = func() { <-targetLimit }
+ defer func() {
+ if err != nil {
+ releaseTarget()
+ }
+ }()
+ case <-ctx.Done():
+ return context.Cause(ctx)
+ }
+ }
+
if multiTarget {
defer func() {
err = errors.Wrapf(err, "target %s", k)
}()
}
-
- res := make([]*client.SolveResponse, len(dps))
- eg2, ctx := errgroup.WithContext(ctx)
+ defer func() {
+ if err != nil {
+ bh.completed(baseCtx, k, err)
+ }
+ }()
var pushNames string
var insecurePush bool
@@ -636,6 +817,8 @@ func BuildWithResultHandler(ctx context.Context, nodes []builder.Node, opts map[
} else {
wg = &sync.WaitGroup{}
wg.Add(1)
+ sharedSessionHolds = append(sharedSessionHolds, wg.Done)
+ wg.Add(1)
sharedSessionsWG[node.Name] = wg
for _, s := range sessions {
eg.Go(func() error {
@@ -652,7 +835,15 @@ func BuildWithResultHandler(ctx context.Context, nodes []builder.Node, opts map[
done = wg.Done
}
- eg2.Go(func() error {
+ eg2.Go(func() (err error) {
+ defer func() {
+ if err != nil {
+ if syncState != nil {
+ syncState.fail(rKey, err)
+ }
+ linkedTargets.fail(rKey, err)
+ }
+ }()
if done != nil {
defer done()
}
@@ -688,7 +879,7 @@ func BuildWithResultHandler(ctx context.Context, nodes []builder.Node, opts map[
callRes map[string][]byte
frontendErr error
)
- buildFunc := func(ctx context.Context, c gateway.Client) (_ *gateway.Result, retErr error) {
+ buildFunc := func(solveCtx context.Context, c gateway.Client) (_ *gateway.Result, retErr error) {
// Capture the error from this build function.
defer catchFrontendError(&retErr, &frontendErr)
@@ -701,7 +892,7 @@ func BuildWithResultHandler(ctx context.Context, nodes []builder.Node, opts map[
req.FrontendOpt["requestid"] = "frontend." + opt.CallFunc.Name
}
- res, err := solve(ctx, c, req)
+ res, err := solve(solveCtx, c, req)
if err != nil {
return nil, err
}
@@ -710,17 +901,31 @@ func BuildWithResultHandler(ctx context.Context, nodes []builder.Node, opts map[
callRes = res.Metadata
}
- if err := linkedTargets.run(ctx, rKey, res, func() error {
- // invoke custom evaluate handler if it is present
- if bh != nil && bh.Evaluate != nil {
- return bh.Evaluate(ctx, k, c, res, opt)
+ var preEvaluate, postEvaluate func() error
+ if syncState != nil {
+ preEvaluate = func() error {
+ return syncState.waitResult(ctx, rKey, res)
}
- if linkedTargets.isLinked(rKey) {
- return eachRefParallel(ctx, res, func(ctx context.Context, ref gateway.Reference) error {
- return ref.Evaluate(ctx)
- })
+ postEvaluate = func() error {
+ return syncState.waitEvaluated(ctx, rKey, struct{}{})
}
- return nil
+ }
+
+ if err := linkedTargets.run(ctx, rKey, res, linkedTargetHooks{
+ preEvaluate: preEvaluate,
+ evaluate: func() error {
+ // invoke custom evaluate handler if it is present
+ if bh != nil && bh.Evaluate != nil {
+ return bh.Evaluate(solveCtx, k, c, res, opt)
+ }
+ if syncState != nil || linkedTargets.isLinked(rKey) {
+ return eachRefParallel(solveCtx, res, func(ctx context.Context, ref gateway.Reference) error {
+ return ref.Evaluate(ctx)
+ })
+ }
+ return nil
+ },
+ postEvaluate: postEvaluate,
}); err != nil {
return nil, err
}
@@ -734,6 +939,13 @@ func BuildWithResultHandler(ctx context.Context, nodes []builder.Node, opts map[
if errors.Is(frontendErr, ErrRestart) {
err = ErrRestart
}
+
+ // Preserve abort classification across the gateway's RPC boundary.
+ var abortErr targetAbortError
+ if err != nil && stderrors.As(frontendErr, &abortErr) {
+ err = targetAbortError{err}
+ }
+
tracing.FinishWithError(span, err)
if !so.Internal && desktop.BuildBackendEnabled() && node.Driver.HistoryAPISupported(ctx) {
@@ -812,7 +1024,13 @@ func BuildWithResultHandler(ctx context.Context, nodes []builder.Node, opts map[
}
eg.Go(func() (err error) {
+ defer func() {
+ bh.completed(baseCtx, k, err)
+ }()
ctx := baseCtx
+ if releaseTarget != nil {
+ defer releaseTarget()
+ }
defer func() {
if span != nil {
tracing.FinishWithError(span, err)
@@ -948,15 +1166,17 @@ func BuildWithResultHandler(ctx context.Context, nodes []builder.Node, opts map[
return nil
}(k)
if err != nil {
- return nil, err
+ cancel(err)
+ break
}
}
- if err := eg.Wait(); err != nil {
- return nil, err
- }
+ releaseSharedSessionHolds()
- return resp, nil
+ if waitErr := eg.Wait(); err == nil {
+ err = waitErr
+ }
+ return resp, err
}
func extractIndexAnnotations(exports []client.ExportEntry) (map[exptypes.AnnotationKey]string, error) {
@@ -1251,6 +1471,9 @@ func waitContextDeps(ctx context.Context, node *noderesolver.ResolvedNode, resul
if err != nil {
return err
}
+ if err := resultError(res); err != nil {
+ return targetAbortError{err}
+ }
for k, contexts := range m {
r, ok := res[k]
@@ -1259,7 +1482,7 @@ func waitContextDeps(ctx context.Context, node *noderesolver.ResolvedNode, resul
}
rr, ok := r.(*gateway.Result)
if !ok {
- return errors.Errorf("invalid result type %T", rr)
+ return errors.Errorf("invalid result type %T", r)
}
if so.FrontendAttrs == nil {
so.FrontendAttrs = map[string]string{}
diff --git a/build/build_test.go b/build/build_test.go
index 30450584782f..5ce892811178 100644
--- a/build/build_test.go
+++ b/build/build_test.go
@@ -2,16 +2,58 @@ package build
import (
"bytes"
+ "context"
+ stderrors "errors"
"testing"
"github.com/docker/buildx/builder"
"github.com/docker/buildx/driver"
"github.com/moby/buildkit/client"
+ "github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
)
+func TestHandlerCompleted(t *testing.T) {
+ failure := stderrors.New("failed")
+ for _, tt := range []struct {
+ name string
+ err error
+ canceled bool
+ aborted bool
+ }{
+ {name: "success"},
+ {name: "failure", err: failure},
+ {name: "dependency", err: targetAbortError{failure}, aborted: true},
+ {name: "wrapped dependency", err: errors.Wrap(targetAbortError{failure}, "target"), aborted: true},
+ {name: "canceled", err: context.Canceled, aborted: true},
+ {name: "deadline", err: context.DeadlineExceeded, aborted: true},
+ {name: "canceled with cause", err: failure, canceled: true, aborted: true},
+ {name: "canceled RPC", err: status.Error(codes.Canceled, "canceled"), canceled: true, aborted: true},
+ {name: "success before cancellation", canceled: true},
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ ctx, cancel := context.WithCancelCause(t.Context())
+ defer cancel(nil)
+ if tt.canceled {
+ cancel(failure)
+ }
+ calls := 0
+ h := &Handler{Completed: func(name string, result TargetResult) {
+ calls++
+ require.Equal(t, "target", name)
+ require.Equal(t, tt.err, result.Err)
+ require.Equal(t, tt.aborted, result.Aborted)
+ }}
+ h.completed(ctx, "target", tt.err)
+ require.Equal(t, 1, calls)
+ })
+ }
+}
+
type warnOutputFactory struct {
driver.Factory
name string
diff --git a/build/linked_targets_test.go b/build/linked_targets_test.go
index 40460a09fb3e..13ebf91631df 100644
--- a/build/linked_targets_test.go
+++ b/build/linked_targets_test.go
@@ -5,6 +5,7 @@ import (
"testing"
"time"
+ "github.com/docker/buildx/util/waitmap"
"github.com/pkg/errors"
"github.com/stretchr/testify/require"
)
@@ -39,16 +40,22 @@ func TestLinkedTargetStateChainRetainsParents(t *testing.T) {
}
go func() {
- done["root"] <- state.run(t.Context(), "root", struct{}{}, func() error { return nil })
+ done["root"] <- state.run(t.Context(), "root", struct{}{}, linkedTargetHooks{
+ evaluate: func() error { return nil },
+ })
}()
go func() {
- done["middle"] <- state.run(t.Context(), "middle", struct{}{}, func() error { return nil })
+ done["middle"] <- state.run(t.Context(), "middle", struct{}{}, linkedTargetHooks{
+ evaluate: func() error { return nil },
+ })
}()
go func() {
- done["leaf"] <- state.run(t.Context(), "leaf", struct{}{}, func() error {
- close(leafStarted)
- <-releaseLeaf
- return nil
+ done["leaf"] <- state.run(t.Context(), "leaf", struct{}{}, linkedTargetHooks{
+ evaluate: func() error {
+ close(leafStarted)
+ <-releaseLeaf
+ return nil
+ },
})
}()
@@ -83,7 +90,9 @@ func TestLinkedTargetStateDiamondEvaluatesBranchesInParallel(t *testing.T) {
done := make(chan error, 4)
run := func(key string, evaluate func() error) {
go func() {
- done <- state.run(t.Context(), key, struct{}{}, evaluate)
+ done <- state.run(t.Context(), key, struct{}{}, linkedTargetHooks{
+ evaluate: evaluate,
+ })
}()
}
run("root", func() error { return nil })
@@ -126,12 +135,204 @@ func TestLinkedTargetStateCancellation(t *testing.T) {
cause := errors.New("target failed")
done := make(chan error, 1)
go func() {
- done <- state.run(ctx, "child", struct{}{}, func() error { return nil })
+ done <- state.run(ctx, "child", struct{}{}, linkedTargetHooks{
+ evaluate: func() error { return nil },
+ })
}()
cancel(cause)
require.ErrorIs(t, <-done, cause)
}
+func TestLinkedTargetStatePropagatesDependencyErrors(t *testing.T) {
+ state := newLinkedTargetState(
+ map[string][]string{"child": {"parent"}},
+ map[string][]string{"parent": {"child"}},
+ )
+ cause := errors.New("parent failed")
+ state.fail("parent", cause)
+
+ err := state.run(t.Context(), "child", struct{}{}, linkedTargetHooks{
+ evaluate: func() error { return nil },
+ })
+ require.ErrorIs(t, err, cause)
+}
+
+func TestSyncEvaluateWaitsForAllTargets(t *testing.T) {
+ targets := []string{"foo", "bar"}
+ results := waitmap.New()
+
+ fooStarted := make(chan struct{})
+ done := map[string]chan error{
+ "foo": make(chan error, 1),
+ "bar": make(chan error, 1),
+ }
+
+ go func() {
+ results.Set("foo", struct{}{})
+ if _, err := results.Get(t.Context(), targets...); err != nil {
+ done["foo"] <- err
+ return
+ }
+ close(fooStarted)
+ done["foo"] <- nil
+ }()
+
+ assertNotSignaled(t, fooStarted)
+ assertNotCompleted(t, done["foo"])
+
+ go func() {
+ results.Set("bar", struct{}{})
+ if _, err := results.Get(t.Context(), targets...); err != nil {
+ done["bar"] <- err
+ return
+ }
+ done["bar"] <- nil
+ }()
+
+ require.NoError(t, <-done["foo"])
+ require.NoError(t, <-done["bar"])
+}
+
+func TestSyncEvaluateCancellation(t *testing.T) {
+ results := waitmap.New()
+ ctx, cancel := context.WithCancelCause(t.Context())
+ cause := errors.New("target failed")
+
+ done := make(chan error, 1)
+ go func() {
+ results.Set("foo", struct{}{})
+ _, err := results.Get(ctx, "foo", "bar")
+ done <- err
+ }()
+
+ cancel(cause)
+ require.ErrorIs(t, <-done, cause)
+}
+
+func TestSyncEvaluateDoesNotDeadlockLinkedTargets(t *testing.T) {
+ linked := newLinkedTargetState(
+ map[string][]string{"child": {"parent"}},
+ map[string][]string{"parent": {"child"}},
+ )
+ results := waitmap.New()
+ evaluated := waitmap.New()
+
+ done := map[string]chan error{
+ "parent": make(chan error, 1),
+ "child": make(chan error, 1),
+ }
+
+ for _, key := range []string{"parent", "child"} {
+ go func() {
+ done[key] <- linked.run(t.Context(), key, struct{}{}, linkedTargetHooks{
+ preEvaluate: func() error {
+ results.Set(key, struct{}{})
+ _, err := results.Get(t.Context(), "parent", "child")
+ return err
+ },
+ evaluate: func() error {
+ return nil
+ },
+ postEvaluate: func() error {
+ evaluated.Set(key, struct{}{})
+ _, err := evaluated.Get(t.Context(), "parent", "child")
+ return err
+ },
+ })
+ }()
+ }
+
+ require.NoError(t, <-done["parent"])
+ require.NoError(t, <-done["child"])
+}
+
+func TestSyncEvaluatePropagatesEvaluationErrors(t *testing.T) {
+ linked := newLinkedTargetState(map[string][]string{}, map[string][]string{})
+ evaluated := waitmap.New()
+ cause := errors.New("target failed")
+ done := map[string]chan error{
+ "success": make(chan error, 1),
+ "failure": make(chan error, 1),
+ }
+
+ go func() {
+ done["success"] <- linked.run(t.Context(), "success", struct{}{}, linkedTargetHooks{
+ evaluate: func() error {
+ return nil
+ },
+ postEvaluate: func() error {
+ evaluated.Set("success", struct{}{})
+ results, err := evaluated.Get(t.Context(), "success", "failure")
+ if err != nil {
+ return err
+ }
+ return wrapResultError(results, "aborted: another target failed")
+ },
+ })
+ }()
+
+ assertNotCompleted(t, done["success"])
+
+ go func() {
+ err := linked.run(t.Context(), "failure", struct{}{}, linkedTargetHooks{
+ evaluate: func() error {
+ return cause
+ },
+ })
+ evaluated.Set("failure", err)
+ done["failure"] <- err
+ }()
+
+ err := <-done["success"]
+ require.ErrorContains(t, err, "aborted: another target failed")
+ require.ErrorIs(t, err, cause)
+ require.ErrorIs(t, <-done["failure"], cause)
+}
+
+func TestResultErrorReturnsFirstErrorInKeyOrder(t *testing.T) {
+ alpha := errors.New("alpha failed")
+ beta := errors.New("beta failed")
+
+ err := resultError(map[string]any{
+ "b": beta,
+ "ok": struct{}{},
+ "a": alpha,
+ })
+
+ require.EqualError(t, err, "alpha failed")
+ require.ErrorIs(t, err, alpha)
+}
+
+func TestSyncTargetStateWrapsPropagatedErrors(t *testing.T) {
+ state := &syncTargetState{
+ targets: []string{"success", "failure"},
+ results: waitmap.New(),
+ evaluated: waitmap.New(),
+ }
+ cause := errors.New("target failed")
+ done := make(chan error, 1)
+
+ go func() {
+ done <- state.waitResult(t.Context(), "success", struct{}{})
+ }()
+
+ assertNotCompleted(t, done)
+ state.fail("failure", cause)
+
+ err := <-done
+ require.ErrorContains(t, err, "aborted: another target failed")
+ require.ErrorIs(t, err, cause)
+}
+
+func TestWrapResultErrorDoesNotNestAbortErrors(t *testing.T) {
+ cause := errors.New("target failed")
+ first := wrapResultError(map[string]any{"root": cause}, "aborted: dependency target failed")
+ second := wrapResultError(map[string]any{"mid": first}, "aborted: dependency target failed")
+
+ require.EqualError(t, second, "aborted: dependency target failed: target failed")
+ require.ErrorIs(t, second, cause)
+}
+
func assertNotCompleted(t *testing.T, ch <-chan error) {
t.Helper()
select {
diff --git a/commands/bake.go b/commands/bake.go
index 2924177f24d0..906825327840 100644
--- a/commands/bake.go
+++ b/commands/bake.go
@@ -7,8 +7,10 @@ import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
+ stderrors "errors"
"fmt"
"io"
+ "maps"
"os"
"slices"
"sort"
@@ -16,6 +18,7 @@ import (
"strings"
"sync"
"text/tabwriter"
+ "time"
"github.com/containerd/console"
"github.com/containerd/containerd/v2/pkg/epoch"
@@ -38,6 +41,7 @@ import (
"github.com/docker/buildx/util/tracing"
"github.com/docker/buildx/util/urlutil"
"github.com/docker/cli/cli/command"
+ "github.com/moby/buildkit/client"
"github.com/moby/buildkit/identity"
"github.com/moby/buildkit/session/auth/authprovider"
"github.com/moby/buildkit/util/progress/progressui"
@@ -68,6 +72,7 @@ type bakeOptions struct {
exportPush bool
exportLoad bool
callFunc string
+ jobs string
print bool
list string
@@ -97,6 +102,11 @@ func runBake(ctx context.Context, dockerCli command.Cli, targets []string, in ba
targets = []string{"default"}
}
+ execution, err := parseBakeJobs(in.jobs)
+ if err != nil {
+ return err
+ }
+
callFunc, err := buildflags.ParseCallFunc(in.callFunc)
if err != nil {
return err
@@ -361,7 +371,26 @@ func runBake(ctx context.Context, dockerCli command.Cli, targets []string, in ba
}
done := timeBuildCommand(mp, attributes)
- resp, retErr := build.Build(ctx, nodes, bo, dockerutil.NewClient(dockerCli), confutil.NewConfig(dockerCli), printer)
+ var bh *build.Handler
+ if execution.Mode != build.ExecutionModeFailFast || execution.Parallel > 0 {
+ bh = &build.Handler{
+ Execution: execution,
+ }
+ }
+ var targetResults map[string]build.TargetResult
+ if execution.Mode == build.ExecutionModeDeferError && len(bo) > 1 {
+ targetResults = make(map[string]build.TargetResult, len(bo))
+ var resultsMu sync.Mutex
+ bh.Completed = func(name string, result build.TargetResult) {
+ resultsMu.Lock()
+ defer resultsMu.Unlock()
+ targetResults[name] = result
+ }
+ }
+ resp, retErr := build.Build(ctx, nodes, bo, dockerutil.NewClient(dockerCli), confutil.NewConfig(dockerCli), printer, bh)
+ if retErr != nil && len(targetResults) > 0 {
+ writeBakeTargetSummary(printer.Write, slices.Sorted(maps.Keys(bo)), targetResults)
+ }
if err := printer.Wait(); retErr == nil {
retErr = err
}
@@ -370,14 +399,10 @@ func runBake(ctx context.Context, dockerCli command.Cli, targets []string, in ba
}
done(err)
- if err != nil {
- return err
- }
-
- if progressMode != progressui.QuietMode && progressMode != progressui.RawJSONMode {
+ if err == nil && progressMode != progressui.QuietMode && progressMode != progressui.RawJSONMode {
desktop.PrintBuildDetails(os.Stderr, printer.BuildRefs(), term)
}
- if len(in.metadataFile) > 0 {
+ if len(in.metadataFile) > 0 && (err == nil || execution.Mode == build.ExecutionModeDeferError) {
dt := make(map[string]any)
for t, r := range resp {
dt[t] = decodeExporterResponse(r.ExporterResponse)
@@ -387,10 +412,16 @@ func runBake(ctx context.Context, dockerCli command.Cli, targets []string, in ba
dt["buildx.build.warnings"] = warnings
}
}
- if err := writeMetadataFile(in.metadataFile, dt); err != nil {
- return err
+ if metaErr := writeMetadataFile(in.metadataFile, dt); metaErr != nil {
+ if err != nil {
+ return stderrors.Join(err, metaErr)
+ }
+ return metaErr
}
}
+ if err != nil {
+ return err
+ }
var callFormatJSON bool
jsonResults := map[string]map[string]any{}
@@ -508,6 +539,81 @@ func runBake(ctx context.Context, dockerCli command.Cli, targets []string, in ba
return nil
}
+func writeBakeTargetSummary(log progress.Logger, names []string, results map[string]build.TargetResult) {
+ progress.Wrap("[internal] target results", log, func(sub progress.SubLogger) error {
+ for _, name := range names {
+ status := "not started"
+ if result, ok := results[name]; ok {
+ switch {
+ case result.Aborted:
+ status = "aborted"
+ case result.Err != nil:
+ status = "failed"
+ default:
+ status = "succeeded"
+ }
+ }
+ now := time.Now()
+ sub.SetStatus(&client.VertexStatus{
+ ID: name + ": " + status,
+ Name: status,
+ Timestamp: now,
+ Started: &now,
+ Completed: &now,
+ })
+ }
+ return nil
+ })
+}
+
+func parseBakeJobs(value string) (build.Execution, error) {
+ execution := build.Execution{Mode: build.ExecutionModeFailFast}
+ value = strings.TrimSpace(value)
+ if value == "" {
+ return execution, nil
+ }
+ if _, err := strconv.Atoi(value); err == nil {
+ value = "parallel=" + value
+ }
+
+ fields, err := csvvalue.Fields(value, nil)
+ if err != nil {
+ return execution, errors.Wrap(err, "invalid jobs option")
+ }
+
+ seen := map[string]bool{}
+ for _, field := range fields {
+ key, val, ok := strings.Cut(strings.TrimSpace(field), "=")
+ if !ok {
+ key, val = "mode", key
+ }
+ key = strings.ToLower(strings.TrimSpace(key))
+ val = strings.TrimSpace(val)
+ if seen[key] {
+ return execution, errors.Errorf("duplicate jobs option %q", key)
+ }
+ seen[key] = true
+ switch key {
+ case "mode":
+ switch mode := build.ExecutionMode(val); mode {
+ case build.ExecutionModeFailFast, build.ExecutionModeDeferOutput, build.ExecutionModeDeferError:
+ execution.Mode = mode
+ default:
+ return execution, errors.Errorf("invalid jobs mode %q", val)
+ }
+ case "parallel":
+ n, err := strconv.Atoi(val)
+ if err != nil || n < 0 {
+ return execution, errors.Errorf("invalid jobs parallel value %q: must be a non-negative integer", val)
+ }
+ execution.Parallel = n
+ default:
+ return execution, errors.Errorf("unknown jobs option %q", key)
+ }
+ }
+ return execution, nil
+}
+
func bakeCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {
var options bakeOptions
var cFlags commonFlags
@@ -561,6 +667,7 @@ func bakeCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {
flags.StringArrayVar(&options.vars, "var", nil, `Set a variable value (e.g., "name=value")`)
flags.StringVar(&options.callFunc, "call", "build", `Set method for evaluating build ("check", "outline", "targets")`)
flags.StringArrayVar(&options.allow, "allow", nil, "Allow build to access specified resources")
+ flags.StringVarP(&options.jobs, "jobs", "j", "fail-fast", `Set target execution behavior (format: "N" or "mode[,parallel=N]")`)
flags.VarPF(callAlias(&options.callFunc, "check"), "check", "", `Shorthand for "--call=check"`)
flags.Lookup("check").NoOptDefVal = "true"
diff --git a/commands/bake_test.go b/commands/bake_test.go
new file mode 100644
index 000000000000..7c85b19b55cb
--- /dev/null
+++ b/commands/bake_test.go
@@ -0,0 +1,149 @@
+package commands
+
+import (
+ "bytes"
+ "encoding/json"
+ stderrors "errors"
+ "io"
+ "testing"
+
+ "github.com/docker/buildx/build"
+ "github.com/docker/buildx/util/progress"
+ "github.com/docker/cli/cli/command"
+ "github.com/moby/buildkit/client"
+ "github.com/moby/buildkit/util/progress/progressui"
+ "github.com/stretchr/testify/require"
+)
+
+func TestWriteBakeTargetSummary(t *testing.T) {
+ names := []string{"a-success", "b-failure", "c-dependent", "d-pending"}
+ results := map[string]build.TargetResult{
+ "a-success": {},
+ "b-failure": {Err: stderrors.New("failed")},
+ "c-dependent": {Err: stderrors.New("dependency failed"), Aborted: true},
+ }
+ for _, mode := range []progressui.DisplayMode{progressui.PlainMode, progressui.RawJSONMode, progressui.QuietMode} {
+ t.Run(string(mode), func(t *testing.T) {
+ var out bytes.Buffer
+ printer, err := progress.NewPrinter(t.Context(), &out, mode)
+ require.NoError(t, err)
+ writeBakeTargetSummary(printer.Write, names, results)
+ require.NoError(t, printer.Wait())
+ switch mode {
+ case progressui.PlainMode:
+ require.Contains(t, out.String(), "#1 [internal] target results\n")
+ for _, row := range []string{"a-success: succeeded", "b-failure: failed", "c-dependent: aborted", "d-pending: not started"} {
+ require.Contains(t, out.String(), "#1 "+row+" done\n")
+ }
+ case progressui.RawJSONMode:
+ dec := json.NewDecoder(&out)
+ var vertices []*client.Vertex
+ var statuses []*client.VertexStatus
+ for {
+ var event client.SolveStatus
+ err := dec.Decode(&event)
+ if err == io.EOF {
+ break
+ }
+ require.NoError(t, err)
+ vertices = append(vertices, event.Vertexes...)
+ statuses = append(statuses, event.Statuses...)
+ }
+ require.Len(t, vertices, 2)
+ require.Equal(t, "[internal] target results", vertices[0].Name)
+ require.Equal(t, vertices[0].Digest, vertices[1].Digest)
+ require.NotNil(t, vertices[1].Completed)
+ require.Empty(t, vertices[1].Error)
+ require.Len(t, statuses, len(names))
+ for i, outcome := range []string{"succeeded", "failed", "aborted", "not started"} {
+ require.Equal(t, vertices[0].Digest, statuses[i].Vertex)
+ require.Equal(t, names[i]+": "+outcome, statuses[i].ID)
+ require.Equal(t, outcome, statuses[i].Name)
+ require.NotNil(t, statuses[i].Completed)
+ }
+ case progressui.QuietMode:
+ require.Empty(t, out.String())
+ }
+ })
+ }
+}
+
+func TestBakeJobsFlag(t *testing.T) {
+ tests := []struct {
+ name string
+ args []string
+ jobs string
+ }{
+ {name: "default", jobs: "fail-fast"},
+ {name: "long", args: []string{"--jobs=2"}, jobs: "2"},
+ {name: "short", args: []string{"-j=2"}, jobs: "2"},
+ {name: "attached", args: []string{"-j2"}, jobs: "2"},
+ {name: "unlimited", args: []string{"-j=0"}, jobs: "0"},
+ {name: "parallel", args: []string{"--jobs=parallel=2"}, jobs: "parallel=2"},
+ {name: "with mode", args: []string{"--jobs=defer-error,parallel=2"}, jobs: "defer-error,parallel=2"},
+ {name: "short with mode", args: []string{"-j=defer-output"}, jobs: "defer-output"},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ dockerCli, err := command.NewDockerCli()
+ require.NoError(t, err)
+ cmd := bakeCmd(dockerCli, &rootOptions{})
+ require.NoError(t, cmd.ParseFlags(tt.args))
+ jobs, err := cmd.Flags().GetString("jobs")
+ require.NoError(t, err)
+ require.Equal(t, tt.jobs, jobs)
+ })
+ }
+}
+
+func TestBakeJobsNegative(t *testing.T) {
+ dockerCli, err := command.NewDockerCli()
+ require.NoError(t, err)
+ cmd := bakeCmd(dockerCli, &rootOptions{})
+ require.NoError(t, cmd.ParseFlags([]string{"-j=-1"}))
+ require.EqualError(t, cmd.RunE(cmd, nil), `invalid jobs parallel value "-1": must be a non-negative integer`)
+}
+
+func TestParseBakeJobs(t *testing.T) {
+ for _, tt := range []struct {
+ value string
+ mode build.ExecutionMode
+ parallel int
+ }{
+ {value: "", mode: build.ExecutionModeFailFast},
+ {value: "fail-fast", mode: build.ExecutionModeFailFast},
+ {value: "defer-output", mode: build.ExecutionModeDeferOutput},
+ {value: "defer-error", mode: build.ExecutionModeDeferError},
+ {value: "0", mode: build.ExecutionModeFailFast},
+ {value: "2", mode: build.ExecutionModeFailFast, parallel: 2},
+ {value: "parallel=0", mode: build.ExecutionModeFailFast},
+ {value: "parallel=2", mode: build.ExecutionModeFailFast, parallel: 2},
+ {value: "defer-error,parallel=2", mode: build.ExecutionModeDeferError, parallel: 2},
+ {value: "parallel=2,defer-output", mode: build.ExecutionModeDeferOutput, parallel: 2},
+ {value: "mode=defer-error,parallel=2", mode: build.ExecutionModeDeferError, parallel: 2},
+ {value: " 2 ", mode: build.ExecutionModeFailFast, parallel: 2},
+ {value: " mode = defer-error , PARALLEL = 2 ", mode: build.ExecutionModeDeferError, parallel: 2},
+ {value: `"defer-error","parallel=2"`, mode: build.ExecutionModeDeferError, parallel: 2},
+ } {
+ t.Run(tt.value, func(t *testing.T) {
+ got, err := parseBakeJobs(tt.value)
+ require.NoError(t, err)
+ require.Equal(t, build.Execution{Mode: tt.mode, Parallel: tt.parallel}, got)
+ })
+ }
+}
+
+func TestParseBakeJobsInvalid(t *testing.T) {
+ for _, value := range []string{
+ "-1", "parallel=-1", "parallel=", "parallel=abc", "parallel=1.5",
+ "parallel=999999999999999999999999999999", "999999999999999999999999999999",
+ "unknown", "mode=", "mode=unknown", "Defer-Error", "jobs=2",
+ "defer-error,defer-output", "fail-fast,mode=fail-fast", "parallel=1,parallel=2",
+ "parallel=1,PARALLEL=2", "defer-error,", ",defer-error", `"defer-error`,
+ } {
+ t.Run(value, func(t *testing.T) {
+ _, err := parseBakeJobs(value)
+ require.Error(t, err)
+ })
+ }
+}
diff --git a/commands/build.go b/commands/build.go
index 5f267704052a..6b059f467fed 100644
--- a/commands/build.go
+++ b/commands/build.go
@@ -1203,7 +1203,7 @@ func RunBuild(ctx context.Context, dockerCli command.Cli, in *BuildOptions, inSt
var inputs *build.Inputs
buildOptions := map[string]build.Options{defaultTargetName: opts}
- resp, err := build.BuildWithResultHandler(ctx, nodes, buildOptions, dockerutil.NewClient(dockerCli), confutil.NewConfig(dockerCli), progress, bh)
+ resp, err := build.Build(ctx, nodes, buildOptions, dockerutil.NewClient(dockerCli), confutil.NewConfig(dockerCli), progress, bh)
err = wrapBuildError(err, false)
if err != nil {
return nil, nil, err
diff --git a/docs/reference/buildx_bake.md b/docs/reference/buildx_bake.md
index 42e3a78cd144..1b81cf604eee 100644
--- a/docs/reference/buildx_bake.md
+++ b/docs/reference/buildx_bake.md
@@ -13,27 +13,28 @@ Build from a file
### Options
-| Name | Type | Default | Description |
-|:------------------------------------|:--------------|:--------|:----------------------------------------------------------------------------------------------------------------------|
-| [`--allow`](#allow) | `stringArray` | | Allow build to access specified resources |
-| [`--builder`](#builder) | `string` | | Override the configured builder instance |
-| [`--call`](#call) | `string` | `build` | Set method for evaluating build (`check`, `outline`, `targets`) |
-| [`--check`](#check) | `bool` | | Shorthand for `--call=check` |
-| `-D`, `--debug` | `bool` | | Enable debug logging |
-| [`-f`](#file), [`--file`](#file) | `stringArray` | | Build definition file |
-| [`--list`](#list) | `string` | | List targets or variables |
-| [`--load`](#load) | `bool` | | Shorthand for `--set=*.output=type=docker`. Conditional. |
-| [`--metadata-file`](#metadata-file) | `string` | | Write build result metadata to a file |
-| [`--no-cache`](#no-cache) | `bool` | | Do not use cache when building the image |
-| `--policy` | `stringArray` | | Global policy evaluation options (format: `[disabled=true\|false][,strict=true\|false][,log-level=level]`) |
-| [`--print`](#print) | `bool` | | Print the options without building |
-| [`--progress`](#progress) | `string` | `auto` | Set type of progress output (`auto`, `none`, `plain`, `quiet`, `rawjson`, `tty`). Use plain to show container output |
-| [`--provenance`](#provenance) | `string` | | Shorthand for `--set=*.attest=type=provenance` |
-| [`--pull`](#pull) | `bool` | | Always attempt to pull all referenced images |
-| [`--push`](#push) | `bool` | | Shorthand for `--set=*.output=type=registry`. Conditional. |
-| [`--sbom`](#sbom) | `string` | | Shorthand for `--set=*.attest=type=sbom` |
-| [`--set`](#set) | `stringArray` | | Override target value (e.g., `targetpattern.key=value`) |
-| `--var` | `stringArray` | | Set a variable value (e.g., `name=value`) |
+| Name | Type | Default | Description |
+|:------------------------------------|:--------------|:------------|:----------------------------------------------------------------------------------------------------------------------|
+| [`--allow`](#allow) | `stringArray` | | Allow build to access specified resources |
+| [`--builder`](#builder) | `string` | | Override the configured builder instance |
+| [`--call`](#call) | `string` | `build` | Set method for evaluating build (`check`, `outline`, `targets`) |
+| [`--check`](#check) | `bool` | | Shorthand for `--call=check` |
+| `-D`, `--debug` | `bool` | | Enable debug logging |
+| [`-f`](#file), [`--file`](#file) | `stringArray` | | Build definition file |
+| [`-j`](#jobs), [`--jobs`](#jobs) | `string` | `fail-fast` | Set target execution behavior (format: `N` or `mode[,parallel=N]`) |
+| [`--list`](#list) | `string` | | List targets or variables |
+| [`--load`](#load) | `bool` | | Shorthand for `--set=*.output=type=docker`. Conditional. |
+| [`--metadata-file`](#metadata-file) | `string` | | Write build result metadata to a file |
+| [`--no-cache`](#no-cache) | `bool` | | Do not use cache when building the image |
+| `--policy` | `stringArray` | | Global policy evaluation options (format: `[disabled=true\|false][,strict=true\|false][,log-level=level]`) |
+| [`--print`](#print) | `bool` | | Print the options without building |
+| [`--progress`](#progress) | `string` | `auto` | Set type of progress output (`auto`, `none`, `plain`, `quiet`, `rawjson`, `tty`). Use plain to show container output |
+| [`--provenance`](#provenance) | `string` | | Shorthand for `--set=*.attest=type=provenance` |
+| [`--pull`](#pull) | `bool` | | Always attempt to pull all referenced images |
+| [`--push`](#push) | `bool` | | Shorthand for `--set=*.output=type=registry`. Conditional. |
+| [`--sbom`](#sbom) | `string` | | Shorthand for `--set=*.attest=type=sbom` |
+| [`--set`](#set) | `stringArray` | | Override target value (e.g., `targetpattern.key=value`) |
+| `--var` | `stringArray` | | Set a variable value (e.g., `name=value`) |
@@ -137,6 +138,66 @@ Same as [`build --call`](buildx_build.md#call).
Same as [`build --check`](buildx_build.md#check).
+### Configure target execution behavior (--jobs, -j)
+
+```text
+--jobs=N
+--jobs=[mode,]parallel=N
+--jobs=mode
+```
+
+The `--jobs` flag (shorthand `-j`) controls target concurrency, output
+synchronization, and how Bake handles target failures. It accepts an integer
+as shorthand for `parallel=N`, or comma-separated mode and parallel options.
+For example, `-j=2` is equivalent to `--jobs=parallel=2`, and
+`--jobs=defer-error,parallel=2` combines a mode with a concurrency limit.
+The mode can also be written as `mode=defer-error`.
+
+The default mode is `fail-fast`, which stops the build when a target
+fails and cancels targets that are still running. Outputs already written by
+successful targets are not removed.
+
+The `defer-output` mode waits until all participating targets have successfully
+evaluated their build results before any target begins exporting output. This is
+useful when multiple targets produce related artifacts and you want to withhold
+their outputs if a build fails. Exports are not transactional: an export failure
+can still leave partial output, and completed exports are not rolled back.
+
+For multi-node targets, `defer-output` applies to the BuildKit solve and export
+boundary for each target. Any manifest list merge or registry push that Buildx
+performs after the per-node solves complete is not part of this synchronization
+barrier.
+
+The `defer-error` mode allows independent targets to keep running after another
+target fails. Bake still returns an error after all possible targets complete,
+but successful targets can finish and write their outputs. Targets that depend
+on a failed target cannot complete successfully. When `--metadata-file` is set,
+Bake writes metadata for successful targets even if another target fails.
+
+```console
+$ docker buildx bake --jobs=defer-output # wait for all build results before exporting
+$ docker buildx bake --jobs=defer-error # let independent targets finish before returning an error
+```
+
+The `parallel` option limits how many Bake targets run at the same time. It
+requires a non-negative integer. When omitted or set to `0`, Bake doesn't apply
+a target concurrency limit. This controls target scheduling, not BuildKit's
+internal parallelism for build steps within a target. Specify the mode and limit
+in the same flag value; repeating `--jobs` replaces the previous value.
+
+```console
+$ docker buildx bake -j=2 # run at most two targets at the same time
+$ docker buildx bake --jobs=parallel=1 # run targets sequentially
+$ docker buildx bake --jobs=defer-error,parallel=2 # limit concurrency and let independent targets finish after a failure
+```
+
+With `defer-output`, every participating target must be able to reach the output
+boundary before any target completes. A nonzero jobs limit smaller than the
+number of participating targets is therefore rejected. The same restriction
+applies when linked targets are present, because their solves may need to be
+active together. The count includes implicit targets referenced through
+`target:` contexts.
+
### Specify a build definition file (-f, --file)
Use the `-f` / `--file` option to specify the build definition file to use.
diff --git a/tests/bake.go b/tests/bake.go
index 279bc3c1143e..f44da2b88d05 100644
--- a/tests/bake.go
+++ b/tests/bake.go
@@ -1,6 +1,7 @@
package tests
import (
+ "bufio"
"bytes"
"crypto/rand"
"crypto/rsa"
@@ -10,9 +11,11 @@ import (
"fmt"
"os"
"path/filepath"
+ "runtime"
"strconv"
"strings"
"testing"
+ "time"
"github.com/containerd/continuity/fs/fstest"
"github.com/docker/buildx/bake"
@@ -46,6 +49,12 @@ var bakeTests = []func(t *testing.T, sb integration.Sandbox){
testBakePrintRemoteContextSubdir,
testBakeLocal,
testBakeLocalMulti,
+ testBakeDeferOutput,
+ testBakeFailFast,
+ testBakeDeferError,
+ testBakeDeferErrorSummary,
+ testBakeDeferErrorCancel,
+ testBakeParallel,
testBakeFileRelativePaths,
testBakeLocalExportDeleteMode,
testBakeRemote,
@@ -724,6 +733,295 @@ services:
require.FileExists(t, filepath.Join(dirDest2, "foo"))
}
+func testBakeDeferOutput(t *testing.T, sb integration.Sandbox) {
+ dir := bakeExecutionFailureDir(t, []byte(`
+FROM scratch
+COPY foo /foo
+`))
+
+ out, err := bakeCmd(sb, withDir(dir), withArgs("--jobs=defer-output"))
+ require.Error(t, err, out)
+ require.Contains(t, out, "b-failure")
+ require.NoFileExists(t, filepath.Join(dir, "out", "foo"))
+
+ dir = bakeExecutionSuccessDir(t)
+ out, err = bakeCmd(sb, withDir(dir), withArgs("--jobs=defer-output"))
+ require.NoError(t, err, out)
+ require.FileExists(t, filepath.Join(dir, "out", "a", "foo"))
+ require.FileExists(t, filepath.Join(dir, "out", "b", "foo"))
+}
+
+func testBakeFailFast(t *testing.T, sb integration.Sandbox) {
+ dir := bakeExecutionFailureDir(t, []byte(`
+FROM busybox
+RUN sleep 2
+COPY foo /foo
+`))
+
+ out, err := bakeCmd(sb, withDir(dir), withArgs("--jobs=fail-fast"))
+ require.Error(t, err, out)
+ require.Contains(t, out, "b-failure")
+ require.NoFileExists(t, filepath.Join(dir, "out", "foo"))
+}
+
+func testBakeDeferError(t *testing.T, sb integration.Sandbox) {
+ dir := bakeExecutionFailureDir(t, []byte(`
+FROM busybox
+RUN sleep 2
+COPY foo /foo
+`))
+
+ metadataFile := filepath.Join(dir, "metadata.json")
+ out, err := bakeCmd(sb, withDir(dir), withArgs("--jobs=defer-error", "--metadata-file", metadataFile))
+ require.Error(t, err, out)
+ require.Contains(t, out, "b-failure")
+ require.FileExists(t, filepath.Join(dir, "out", "foo"))
+ require.FileExists(t, metadataFile)
+
+ dt, err := os.ReadFile(metadataFile)
+ require.NoError(t, err)
+
+ var metadata map[string]any
+ require.NoError(t, json.Unmarshal(dt, &metadata))
+ require.Contains(t, metadata, "a-success")
+ require.NotContains(t, metadata, "b-failure")
+}
+
+func testBakeDeferErrorSummary(t *testing.T, sb integration.Sandbox) {
+ for _, tt := range []struct {
+ name string
+ mode string
+ progress string
+ env string
+ target string
+ summary bool
+ }{
+ {name: "plain", mode: "defer-error", progress: "plain", summary: true},
+ {name: "auto", mode: "defer-error", progress: "auto", summary: true},
+ {name: "quiet", mode: "defer-error", progress: "quiet"},
+ {name: "rawjson", mode: "defer-error", progress: "rawjson", summary: true},
+ {name: "env quiet", mode: "defer-error", progress: "auto", env: "quiet"},
+ {name: "env rawjson", mode: "defer-error", progress: "auto", env: "rawjson", summary: true},
+ {name: "fail fast", mode: "fail-fast", progress: "plain"},
+ {name: "defer output", mode: "defer-output", progress: "plain"},
+ {name: "single target", mode: "defer-error", progress: "plain", target: "b-failure"},
+ {name: "single target rawjson", mode: "defer-error", progress: "rawjson", target: "b-failure"},
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ dir := bakeExecutionFailureDir(t, []byte("FROM scratch\nCOPY foo /foo\n"))
+ var stdout, stderr bytes.Buffer
+ args := []string{"bake", "--jobs=" + tt.mode, "--progress=" + tt.progress}
+ if tt.target != "" {
+ args = append(args, tt.target)
+ }
+ cmd := buildxCmd(sb, withDir(dir), withArgs(args...), withEnv("BUILDKIT_PROGRESS="+tt.env))
+ cmd.Stdout = &stdout
+ cmd.Stderr = &stderr
+ require.Error(t, cmd.Run(), stderr.String())
+ require.NotContains(t, stdout.String(), "target results")
+ if tt.summary && (tt.progress == "rawjson" || tt.env == "rawjson") {
+ var summary *client.Vertex
+ var statuses []*client.VertexStatus
+ for line := range strings.SplitSeq(stderr.String(), "\n") {
+ if !strings.HasPrefix(line, "{") {
+ continue // The final CLI error is not a progress event.
+ }
+ var event client.SolveStatus
+ require.NoError(t, json.Unmarshal([]byte(line), &event))
+ for _, v := range event.Vertexes {
+ if v.Name == "[internal] target results" {
+ summary = v
+ }
+ }
+ statuses = append(statuses, event.Statuses...)
+ }
+ require.NotNil(t, summary, stderr.String())
+ require.NotNil(t, summary.Completed)
+ require.Empty(t, summary.Error)
+ outcomes := map[string]string{}
+ for _, status := range statuses {
+ if status.Vertex == summary.Digest {
+ outcomes[status.ID] = status.Name
+ }
+ }
+ require.Equal(t, map[string]string{"a-success: succeeded": "succeeded", "b-failure: failed": "failed"}, outcomes)
+ } else if tt.summary {
+ require.Regexp(t, `(?m)^#[0-9]+ \[internal\] target results$`, stderr.String())
+ require.Regexp(t, `(?m)^#[0-9]+ a-success: succeeded done$`, stderr.String())
+ require.Regexp(t, `(?m)^#[0-9]+ b-failure: failed done$`, stderr.String())
+ } else {
+ require.NotContains(t, stderr.String(), "target results")
+ }
+ })
+ }
+ t.Run("all succeed", func(t *testing.T) {
+ dir := bakeExecutionSuccessDir(t)
+ out, err := bakeCmd(sb, withDir(dir), withArgs("--jobs=defer-error", "--progress=plain"))
+ require.NoError(t, err, out)
+ require.NotContains(t, out, "target results")
+ })
+ t.Run("linked failure", func(t *testing.T) {
+ dir := tmpdir(t,
+ fstest.CreateFile("docker-bake.hcl", []byte(`
+group "default" {
+ targets = ["a-success", "c-dependent"]
+}
+target "a-success" {
+ dockerfile-inline = "FROM scratch\nCOPY foo /foo"
+ output = ["type=local,dest=out"]
+}
+target "b-failure" {
+ dockerfile-inline = "FROM scratch\nCOPY missing /missing"
+}
+target "c-dependent" {
+ dockerfile-inline = "FROM base"
+ contexts = { base = "target:b-failure" }
+}
+`), 0600),
+ fstest.CreateFile("foo", []byte("foo"), 0600),
+ )
+ out, err := bakeCmd(sb, withDir(dir), withArgs("--jobs=defer-error", "--progress=plain"))
+ require.Error(t, err, out)
+ require.Regexp(t, `(?m)^#[0-9]+ \[internal\] target results$`, out)
+ require.Contains(t, out, "a-success: succeeded done")
+ require.Contains(t, out, "b-failure: failed done")
+ require.Contains(t, out, "c-dependent: aborted done")
+ require.FileExists(t, filepath.Join(dir, "out", "foo"))
+ })
+}
+
+func testBakeDeferErrorCancel(t *testing.T, sb integration.Sandbox) {
+ if runtime.GOOS == "windows" {
+ t.Skip("sending os.Interrupt is not supported on Windows")
+ }
+ dir := tmpdir(t,
+ fstest.CreateFile("docker-bake.hcl", []byte(`
+group "default" {
+ targets = ["a-success", "b-waiting", "c-queued"]
+}
+target "a-success" {
+ dockerfile-inline = "FROM scratch\nCOPY foo /foo"
+}
+target "b-waiting" {
+ dockerfile-inline = "FROM busybox\nRUN echo b-waiting-ready && sleep 300"
+}
+target "c-queued" {
+ dockerfile-inline = "FROM scratch\nCOPY foo /foo"
+}
+`), 0600),
+ fstest.CreateFile("foo", []byte("foo"), 0600),
+ )
+ cmd := buildxCmd(sb, withDir(dir), withArgs("bake", "--jobs=defer-error,parallel=1", "--progress=plain"))
+ stderr, err := cmd.StderrPipe()
+ require.NoError(t, err)
+ require.NoError(t, cmd.Start())
+ defer func() {
+ if cmd.ProcessState == nil {
+ cmd.Process.Kill()
+ cmd.Wait()
+ }
+ }()
+ timer := time.AfterFunc(2*time.Minute, func() { cmd.Process.Kill() })
+ defer timer.Stop()
+
+ var out strings.Builder
+ interrupted := false
+ scanner := bufio.NewScanner(stderr)
+ for scanner.Scan() {
+ line := scanner.Text()
+ fmt.Fprintln(&out, line)
+ if !interrupted && strings.HasSuffix(line, " b-waiting-ready") {
+ require.NoError(t, cmd.Process.Signal(os.Interrupt))
+ interrupted = true
+ }
+ }
+ require.NoError(t, scanner.Err())
+ require.Error(t, cmd.Wait(), out.String())
+ require.True(t, interrupted, out.String())
+ require.Contains(t, out.String(), "[internal] target results")
+ require.Contains(t, out.String(), "a-success: succeeded done")
+ require.Contains(t, out.String(), "b-waiting: aborted done")
+ require.Contains(t, out.String(), "c-queued: not started done")
+ require.Contains(t, out.String(), "ERROR: got SIGTERM/SIGINT, forcing shutdown")
+ require.NotContains(t, out.String(), "ERROR: target c-queued:")
+}
+
+func testBakeParallel(t *testing.T, sb integration.Sandbox) {
+ dir := bakeExecutionFailureDir(t, []byte(`
+FROM scratch
+COPY foo /foo
+`))
+
+ out, err := bakeCmd(sb, withDir(dir), withArgs("--jobs=defer-error,parallel=1"))
+ require.Error(t, err, out)
+ require.FileExists(t, filepath.Join(dir, "out", "foo"))
+
+ for _, flag := range []string{"-j=1", "--jobs=parallel=1"} {
+ t.Run(flag, func(t *testing.T) {
+ dir := bakeExecutionSuccessDir(t)
+ out, err := bakeCmd(sb, withDir(dir), withArgs(flag))
+ require.NoError(t, err, out)
+ require.FileExists(t, filepath.Join(dir, "out", "a", "foo"))
+ require.FileExists(t, filepath.Join(dir, "out", "b", "foo"))
+ })
+ }
+}
+
+func bakeExecutionSuccessDir(t *testing.T) string {
+ return tmpdir(
+ t,
+ fstest.CreateFile("docker-bake.hcl", []byte(`
+group "default" {
+ targets = ["a", "b"]
+}
+
+target "a" {
+ dockerfile = "Dockerfile"
+ output = ["type=local,dest=out/a"]
+}
+
+target "b" {
+ dockerfile = "Dockerfile"
+ output = ["type=local,dest=out/b"]
+}
+`), 0600),
+ fstest.CreateFile("Dockerfile", []byte(`
+FROM scratch
+COPY foo /foo
+`), 0600),
+ fstest.CreateFile("foo", []byte("bar"), 0600),
+ )
+}
+
+func bakeExecutionFailureDir(t *testing.T, dockerfile []byte) string {
+ failureDockerfile := []byte(`
+FROM scratch
+COPY missing /missing
+`)
+ bakefile := []byte(`
+group "default" {
+ targets = ["a-success", "b-failure"]
+}
+
+target "a-success" {
+ dockerfile = "Dockerfile"
+ output = ["type=local,dest=out"]
+}
+
+target "b-failure" {
+ dockerfile = "failure.Dockerfile"
+ output = ["type=cacheonly"]
+}
+`)
+ return tmpdir(
+ t,
+ fstest.CreateFile("docker-bake.hcl", bakefile, 0600),
+ fstest.CreateFile("Dockerfile", dockerfile, 0600),
+ fstest.CreateFile("failure.Dockerfile", failureDockerfile, 0600),
+ fstest.CreateFile("foo", []byte("foo"), 0600),
+ )
+}
+
func testBakeFileRelativePaths(t *testing.T, sb integration.Sandbox) {
t.Run("compose context", func(t *testing.T) {
dockerfile := []byte(`