Skip to content

Commit

Permalink
feat(bake): allow concurrent projects with x-depot bake extension
Browse files Browse the repository at this point in the history 
Adding x-depot to compose files allows different services to
be built on different projects.  This allows faster concurrent
builds.

Signed-off-by: Chris Goller <[email protected]>
  • Loading branch information
@goller
goller committed Jun 5, 2024
1 parent 872736a commit 7e7944b
Show file tree
Hide file tree
Showing 5 changed files with 211 additions and 69 deletions.
48 changes: 42 additions & 6 deletions pkg/buildx/bake/bake.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -628,6 +628,8 @@ type Target struct {

// linked is a private field to mark a target used as a linked one
linked bool

ProjectID string `json:"project_id,omitempty" hcl:"project_id,optional" cty:"project_id"`
}

var _ hclparser.WithEvalContexts = &Target{}
Expand Down Expand Up @@ -730,6 +732,9 @@ func (t *Target) Merge(t2 *Target) {
if t2.NoCacheFilter != nil { // merge
t.NoCacheFilter = append(t.NoCacheFilter, t2.NoCacheFilter...)
}
if t2.ProjectID != "" {
t.ProjectID = t2.ProjectID
}
t.Inherits = append(t.Inherits, t2.Inherits...)
}

Expand Down Expand Up @@ -927,16 +932,47 @@ func (t *Target) GetName(ectx *hcl.EvalContext, block *hcl.Block, loadDeps func(
return value.AsString(), nil
}

func TargetsToBuildOpt(m map[string]*Target, inp *Input) (map[string]build.Options, error) {
m2 := make(map[string]build.Options, len(m))
for k, v := range m {
bo, err := toBuildOpt(v, inp)
type DepotBakeOptions struct {
ProjectTargetOptions map[string]map[string]build.Options
}

// input is only used for remote bake.
func NewDepotBakeOptions(defaultProjectID string, targets map[string]*Target, input *Input) (*DepotBakeOptions, error) {
opts := &DepotBakeOptions{
ProjectTargetOptions: map[string]map[string]build.Options{},
}

for targetName, target := range targets {
projectID := target.ProjectID
if projectID == "" {
projectID = defaultProjectID
}
buildOpt, err := toBuildOpt(target, input)
if err != nil {
return nil, err
}
m2[k] = *bo

if _, ok := opts.ProjectTargetOptions[projectID]; !ok {
opts.ProjectTargetOptions[projectID] = map[string]build.Options{}
}
opts.ProjectTargetOptions[projectID][targetName] = *buildOpt
}

return opts, nil
}

// ProjectOpts returns the targeted build options for a specific project ID.
func (o *DepotBakeOptions) ProjectOpts(id string) map[string]build.Options {
return o.ProjectTargetOptions[id]
}

// ProjectIDs returns the x-depot project IDs.
func (o *DepotBakeOptions) ProjectIDs() []string {
projectIDs := make([]string, 0, len(o.ProjectTargetOptions))
for projectID := range o.ProjectTargetOptions {
projectIDs = append(projectIDs, projectID)
}
return m2, nil
return projectIDs
}

func updateContext(t *build.Inputs, inp *Input) {
Expand Down
16 changes: 16 additions & 0 deletions pkg/buildx/bake/compose.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -232,6 +232,10 @@ type xbake struct {
// docs/manuals/bake/compose-file.md#extension-field-with-x-bake
}

type XDepot struct {
ProjectID string `yaml:"project-id,omitempty"`
}

type stringMap map[string]string
type stringArray []string

Expand All @@ -253,6 +257,18 @@ func (sa *stringArray) UnmarshalYAML(unmarshal func(interface{}) error) error {
// composeExtTarget converts Compose build extension x-bake to bake Target
// https://github.com/compose-spec/compose-spec/blob/master/spec.md#extension
func (t *Target) composeExtTarget(exts map[string]interface{}) error {
var xdepot XDepot
buf, ok := exts["x-depot"]
if ok && buf != nil {
yb, _ := yaml.Marshal(buf)
if err := yaml.Unmarshal(yb, &xdepot); err != nil {
return err
}
if xdepot.ProjectID != "" {
t.ProjectID = xdepot.ProjectID
}
}

var xb xbake

ext, ok := exts["x-bake"]
Expand Down
137 changes: 76 additions & 61 deletions pkg/buildx/commands/bake.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ type BakeOptions struct {
DepotOptions
}

func RunBake(dockerCli command.Cli, in BakeOptions, validator BakeValidator) (err error) {
func RunBake(dockerCli command.Cli, in BakeOptions, validator BakeValidator, printer *progresshelper.SharedPrinter) (err error) {
ctx := appcontext.Context()

ctx, end, err := tracing.TraceCurrentCommand(ctx, "bake")
Expand All @@ -53,29 +53,16 @@ func RunBake(dockerCli command.Cli, in BakeOptions, validator BakeValidator) (er
end(err)
}()

ctx2, cancel := context.WithCancel(context.TODO())

printer, err := progress.NewPrinter(ctx2, os.Stderr, os.Stderr, in.progress)
if err != nil {
cancel()
return err
}

defer func() {
// There is extra logic far below that will also do a printer.Wait()
// if there are no errors. We want to control when the buildx printer
// finishes writing so that we can write our own information such as
// linting without it being interleaved.
if printer != nil && err != nil {
err1 := printer.Wait()
if err == nil && !errors.Is(err1, context.Canceled) {
err = err1
}
_ = printer.Wait()
}
}()

defer cancel()

if os.Getenv("DEPOT_NO_SUMMARY_LINK") == "" {
progress.Write(printer, "[depot] build: "+in.buildURL, func() error { return err })
}
Expand All @@ -95,11 +82,16 @@ func RunBake(dockerCli command.Cli, in BakeOptions, validator BakeValidator) (er
return err
}

buildOpts, requestedTargets, err := validator.Validate(ctx, nodes, printer)
validatedOpts, requestedTargets, err := validator.Validate(ctx, nodes, printer)
if err != nil {
return err
}

buildOpts := validatedOpts.ProjectOpts(in.project)
if buildOpts == nil {
return fmt.Errorf("project %s build options not found", in.project)
}

var (
pullOpts map[string]load.PullOptions
// Only used for failures to pull images.
Expand Down Expand Up @@ -165,13 +157,14 @@ func RunBake(dockerCli command.Cli, in BakeOptions, validator BakeValidator) (er
}
dt[buildRes.Name] = metadata
}
if err := writeMetadataFile(in.metadataFile, in.project, in.buildID, requestedTargets, dt); err != nil {
err = writeMetadataFile(in.metadataFile, in.project, in.buildID, requestedTargets, dt)
if err != nil {
return err
}
}

if in.sbomDir != "" {
err := sbom.Save(ctx, in.sbomDir, resp)
err = sbom.Save(ctx, in.sbomDir, resp)
if err != nil {
return err
}
Expand All @@ -197,7 +190,7 @@ func RunBake(dockerCli command.Cli, in BakeOptions, validator BakeValidator) (er
}(i, requestedTargets)
}

err := eg.Wait()
err = eg.Wait()
if err != nil && !errors.Is(err, context.Canceled) {
// For now, we will fallback by rebuilding with load.
if in.exportLoad {
Expand Down Expand Up @@ -267,7 +260,7 @@ func BakeCmd(dockerCli command.Cli) *cobra.Command {

var (
validator BakeValidator
validatedOpts map[string]buildx.Options
validatedOpts *bake.DepotBakeOptions
)
if isRemoteTarget(args) {
validator = NewRemoteBakeValidator(options, args)
Expand All @@ -280,49 +273,71 @@ func BakeCmd(dockerCli command.Cli) *cobra.Command {
}
}

req := helpers.NewBakeRequest(
options.project,
validatedOpts,
helpers.UsingDepotFeatures{
Push: options.exportPush,
Load: options.exportLoad,
Save: options.save,
Lint: options.lint,
},
)
build, err := helpers.BeginBuild(context.Background(), req, token)
projectIDs := validatedOpts.ProjectIDs()

printer, err := progresshelper.NewSharedPrinter(options.progress)
if err != nil {
return err
}
var buildErr error
defer func() {
build.Finish(buildErr)
PrintBuildURL(build.BuildURL, options.progress)
}()

options.builderOptions = []builder.Option{builder.WithDepotOptions(buildPlatform, build)}

buildProject := build.BuildProject()
if buildProject != "" {
options.project = buildProject
}
if options.save {
options.additionalCredentials = build.AdditionalCredentials()
options.additionalTags = build.AdditionalTags()
for range projectIDs {
printer.Add()
}
options.buildID = build.ID
options.buildURL = build.BuildURL
options.token = build.Token
options.build = &build

if options.allowNoOutput {
_ = os.Setenv("BUILDX_NO_DEFAULT_LOAD", "1")
eg, ctx := errgroup.WithContext(context.Background())
for _, projectID := range projectIDs {
options.project = projectID
bakeOpts := validatedOpts.ProjectOpts(projectID)

req := helpers.NewBakeRequest(
options.project,
bakeOpts,
helpers.UsingDepotFeatures{
Push: options.exportPush,
Load: options.exportLoad,
Save: options.save,
Lint: options.lint,
},
)
build, err := helpers.BeginBuild(context.Background(), req, token)
if err != nil {
return err
}
var buildErr error
defer func() {
build.Finish(buildErr)
PrintBuildURL(build.BuildURL, options.progress)
}()

options.builderOptions = []builder.Option{builder.WithDepotOptions(buildPlatform, build)}

buildProject := build.BuildProject()
if buildProject != "" {
options.project = buildProject
}
if options.save {
options.additionalCredentials = build.AdditionalCredentials()
options.additionalTags = build.AdditionalTags()
}
options.buildID = build.ID
options.buildURL = build.BuildURL
options.token = build.Token
options.build = &build

if options.allowNoOutput {
_ = os.Setenv("BUILDX_NO_DEFAULT_LOAD", "1")
}

eg.Go(func() error {
buildErr = retryRetryableErrors(ctx, func() error {
return RunBake(dockerCli, options, validator, printer)
})

return rewriteFriendlyErrors(buildErr)
})
}

buildErr = retryRetryableErrors(context.Background(), func() error {
return RunBake(dockerCli, options, validator)
})
return rewriteFriendlyErrors(buildErr)
return eg.Wait()
},
}

Expand Down Expand Up @@ -379,15 +394,15 @@ var (

// BakeValidator returns either local or remote build options for targets as well as the targets themselves.
type BakeValidator interface {
Validate(ctx context.Context, nodes []builder.Node, pw progress.Writer) (opts map[string]buildx.Options, targets []string, err error)
Validate(ctx context.Context, nodes []builder.Node, pw progress.Writer) (opts *bake.DepotBakeOptions, targets []string, err error)
}

type LocalBakeValidator struct {
options BakeOptions
bakeTargets bakeTargets

once sync.Once
buildOpts map[string]buildx.Options
buildOpts *bake.DepotBakeOptions
targets []string
err error
}
Expand All @@ -399,7 +414,7 @@ func NewLocalBakeValidator(options BakeOptions, args []string) *LocalBakeValidat
}
}

func (t *LocalBakeValidator) Validate(ctx context.Context, _ []builder.Node, _ progress.Writer) (map[string]buildx.Options, []string, error) {
func (t *LocalBakeValidator) Validate(ctx context.Context, _ []builder.Node, _ progress.Writer) (*bake.DepotBakeOptions, []string, error) {
// Using a sync.Once because I _think_ the bake file may not always be read
// more than one time such as passed over stdin.
t.once.Do(func() {
Expand Down Expand Up @@ -448,7 +463,7 @@ func (t *LocalBakeValidator) Validate(ctx context.Context, _ []builder.Node, _ p
}
}

t.buildOpts, t.err = bake.TargetsToBuildOpt(targets, nil)
t.buildOpts, t.err = bake.NewDepotBakeOptions(t.options.project, targets, nil)
})

return t.buildOpts, t.targets, t.err
Expand All @@ -466,7 +481,7 @@ func NewRemoteBakeValidator(options BakeOptions, args []string) *RemoteBakeValid
}
}

func (t *RemoteBakeValidator) Validate(ctx context.Context, nodes []builder.Node, pw progress.Writer) (map[string]buildx.Options, []string, error) {
func (t *RemoteBakeValidator) Validate(ctx context.Context, nodes []builder.Node, pw progress.Writer) (*bake.DepotBakeOptions, []string, error) {
files, inp, err := bake.ReadRemoteFiles(ctx, builder.ToBuildxNodes(nodes), t.bakeTargets.FileURL, t.options.files, pw)
if err != nil {
return nil, nil, err
Expand Down Expand Up @@ -499,7 +514,7 @@ func (t *RemoteBakeValidator) Validate(ctx context.Context, nodes []builder.Node
requestedTargets = append(requestedTargets, target)
}

opts, err := bake.TargetsToBuildOpt(targets, inp)
opts, err := bake.NewDepotBakeOptions(t.options.project, targets, inp)
return opts, requestedTargets, err
}

Expand Down
4 changes: 2 additions & 2 deletions pkg/buildx/commands/lint.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,13 +84,13 @@ type Linter struct {
FailureMode LintFailure
Clients []*client.Client
BuildxNodes []builder.Node
printer *progress.Printer
printer progress.Writer

mu sync.Mutex
issues map[string][]client.VertexWarning
}

func NewLinter(printer *progress.Printer, failureMode LintFailure, clients []*client.Client, nodes []builder.Node) *Linter {
func NewLinter(printer progress.Writer, failureMode LintFailure, clients []*client.Client, nodes []builder.Node) *Linter {
return &Linter{
FailureMode: failureMode,
Clients: clients,
Expand Down
Loading

0 comments on commit 7e7944b

Please sign in to comment.