Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

buildinfo: merge build sources for deps #2684

Merged
merged 2 commits into from
Mar 9, 2022
Merged

buildinfo: merge build sources for deps #2684

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c61838f
imageutil: helper to return buildinfo from image config
crazy-max Feb 26, 2022
49aa39c
buildinfo: merge build sources for deps
crazy-max Mar 9, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 43 additions & 31 deletions frontend/dockerfile/builder/build.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
"github.com/moby/buildkit/solver/errdefs"
"github.com/moby/buildkit/solver/pb"
"github.com/moby/buildkit/util/apicaps"
binfotypes "github.com/moby/buildkit/util/buildinfo/types"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
Expand Down Expand Up @@ -811,11 +812,11 @@ func warnOpts(sm *llb.SourceMap, r *parser.Range, detail [][]byte, url string) c
return opts
}

func contextByNameFunc(c client.Client, p *ocispecs.Platform) func(context.Context, string) (*llb.State, *dockerfile2llb.Image, error) {
return func(ctx context.Context, name string) (*llb.State, *dockerfile2llb.Image, error) {
func contextByNameFunc(c client.Client, p *ocispecs.Platform) func(context.Context, string) (*llb.State, *dockerfile2llb.Image, *binfotypes.BuildInfo, error) {
return func(ctx context.Context, name string) (*llb.State, *dockerfile2llb.Image, *binfotypes.BuildInfo, error) {
named, err := reference.ParseNormalizedNamed(name)
if err != nil {
return nil, nil, errors.Wrapf(err, "invalid context name %s", name)
return nil, nil, nil, errors.Wrapf(err, "invalid context name %s", name)
}
name = strings.TrimSuffix(reference.FamiliarString(named), ":latest")

Expand All @@ -825,28 +826,28 @@ func contextByNameFunc(c client.Client, p *ocispecs.Platform) func(context.Conte
}
if p != nil {
name := name + "::" + platforms.Format(platforms.Normalize(*p))
st, img, err := contextByName(ctx, c, name, p)
st, img, bi, err := contextByName(ctx, c, name, p)
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}
if st != nil {
return st, img, nil
return st, img, bi, nil
}
}
return contextByName(ctx, c, name, p)
}
}

func contextByName(ctx context.Context, c client.Client, name string, platform *ocispecs.Platform) (*llb.State, *dockerfile2llb.Image, error) {
func contextByName(ctx context.Context, c client.Client, name string, platform *ocispecs.Platform) (*llb.State, *dockerfile2llb.Image, *binfotypes.BuildInfo, error) {
opts := c.BuildOpts().Opts
v, ok := opts["context:"+name]
if !ok {
return nil, nil, nil
return nil, nil, nil, nil
}

vv := strings.SplitN(v, ":", 2)
if len(vv) != 2 {
return nil, nil, errors.Errorf("invalid context specifier %s for %s", v, name)
return nil, nil, nil, errors.Errorf("invalid context specifier %s for %s", v, name)
}
switch vv[0] {
case "docker-image":
Expand All @@ -859,20 +860,20 @@ func contextByName(ctx context.Context, c client.Client, name string, platform *
imgOpt = append(imgOpt, llb.Platform(*platform))
}
st := llb.Image(ref, imgOpt...)
return &st, nil, nil
return &st, nil, nil, nil
case "git":
st, ok := detectGitContext(v, "1")
if !ok {
return nil, nil, errors.Errorf("invalid git context %s", v)
return nil, nil, nil, errors.Errorf("invalid git context %s", v)
}
return st, nil, nil
return st, nil, nil, nil
case "http", "https":
st, ok := detectGitContext(v, "1")
if !ok {
httpst := llb.HTTP(v, llb.WithCustomName("[context "+name+"] "+v))
st = &httpst
}
return st, nil, nil
return st, nil, nil, nil
case "local":
st := llb.Local(vv[1],
llb.SessionID(c.BuildOpts().SessionID),
Expand All @@ -883,18 +884,18 @@ func contextByName(ctx context.Context, c client.Client, name string, platform *
)
def, err := st.Marshal(ctx)
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}
res, err := c.Solve(ctx, client.SolveRequest{
Evaluate: true,
Definition: def.ToPB(),
})
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}
ref, err := res.SingleRef()
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}
dt, _ := ref.ReadFile(ctx, client.ReadRequest{
Filename: dockerignoreFilename,
Expand All @@ -903,7 +904,7 @@ func contextByName(ctx context.Context, c client.Client, name string, platform *
if len(dt) != 0 {
excludes, err = dockerignore.ReadAll(bytes.NewBuffer(dt))
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}
}
st = llb.Local(vv[1],
Expand All @@ -912,38 +913,49 @@ func contextByName(ctx context.Context, c client.Client, name string, platform *
llb.SharedKeyHint("context:"+name),
llb.ExcludePatterns(excludes),
)
return &st, nil, nil
return &st, nil, nil, nil
case "input":
inputs, err := c.Inputs(ctx)
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}
st, ok := inputs[vv[1]]
if !ok {
return nil, nil, errors.Errorf("invalid input %s for %s", vv[1], name)
return nil, nil, nil, errors.Errorf("invalid input %s for %s", vv[1], name)
}
md, ok := opts["input-metadata:"+vv[1]]
if ok {
m := make(map[string][]byte)
if err := json.Unmarshal([]byte(md), &m); err != nil {
return nil, nil, errors.Wrapf(err, "failed to parse input metadata %s", md)
return nil, nil, nil, errors.Wrapf(err, "failed to parse input metadata %s", md)
}
dt, ok := m["containerimage.config"]
if ok {
st, err = st.WithImageConfig([]byte(dt))
var bi *binfotypes.BuildInfo
if dtbi, ok := m[exptypes.ExporterBuildInfo]; ok {
var depbi binfotypes.BuildInfo
if err := json.Unmarshal(dtbi, &depbi); err != nil {
return nil, nil, nil, errors.Wrapf(err, "failed to parse buildinfo for %s", name)
}
bi = &binfotypes.BuildInfo{
Deps: map[string]binfotypes.BuildInfo{
strings.SplitN(vv[1], "::", 2)[0]: depbi,
},
}
}
var img *dockerfile2llb.Image
if dtic, ok := m[exptypes.ExporterImageConfigKey]; ok {
st, err = st.WithImageConfig(dtic)
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}
var img dockerfile2llb.Image
if err := json.Unmarshal(dt, &img); err != nil {
return nil, nil, errors.Wrapf(err, "failed to parse image config for %s", name)
if err := json.Unmarshal(dtic, &img); err != nil {
return nil, nil, nil, errors.Wrapf(err, "failed to parse image config for %s", name)
}
return &st, &img, nil
}
return &st, img, bi, nil
}
return &st, nil, nil
return &st, nil, nil, nil
default:
return nil, nil, errors.Errorf("unsupported context source %s for %s", vv[0], name)
return nil, nil, nil, errors.Errorf("unsupported context source %s for %s", vv[0], name)
}
}

Expand Down
50 changes: 38 additions & 12 deletions frontend/dockerfile/dockerfile2llb/convert.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,18 +70,31 @@ type ConvertOpt struct {
SourceMap *llb.SourceMap
Hostname string
Warn func(short, url string, detail [][]byte, location *parser.Range)
ContextByName func(context.Context, string) (*llb.State, *Image, error)
ContextByName func(context.Context, string) (*llb.State, *Image, *binfotypes.BuildInfo, error)
}

func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, *Image, *binfotypes.BuildInfo, error) {
buildInfo := &binfotypes.BuildInfo{}
contextByName := opt.ContextByName
opt.ContextByName = func(ctx context.Context, name string) (*llb.State, *Image, error) {
opt.ContextByName = func(ctx context.Context, name string) (*llb.State, *Image, *binfotypes.BuildInfo, error) {
if !strings.EqualFold(name, "scratch") && !strings.EqualFold(name, "context") {
if contextByName != nil {
return contextByName(ctx, name)
st, img, bi, err := contextByName(ctx, name)
if err != nil {
return nil, nil, nil, err
}
if bi != nil && bi.Deps != nil {
for k := range bi.Deps {
if buildInfo.Deps == nil {
buildInfo.Deps = make(map[string]binfotypes.BuildInfo)
}
buildInfo.Deps[k] = bi.Deps[k]
}
}
return st, img, bi, nil
}
}
return nil, nil, nil
return nil, nil, nil, nil
}

if len(dt) == 0 {
Expand Down Expand Up @@ -152,7 +165,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
}

if st.Name != "" {
s, img, err := opt.ContextByName(ctx, st.Name)
s, img, bi, err := opt.ContextByName(ctx, st.Name)
if err != nil {
return nil, nil, nil, err
}
Expand All @@ -162,6 +175,9 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
if img != nil {
ds.image = *img
}
if bi != nil {
ds.buildInfo = *bi
}
allDispatchStates.addState(ds)
continue
}
Expand Down Expand Up @@ -291,7 +307,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
d.stage.BaseName = reference.TagNameOnly(ref).String()

var isScratch bool
st, img, err := opt.ContextByName(ctx, d.stage.BaseName)
st, img, bi, err := opt.ContextByName(ctx, d.stage.BaseName)
if err != nil {
return err
}
Expand All @@ -301,6 +317,9 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
} else {
d.image = emptyImage(platformOpt.targetPlatform)
}
if bi != nil {
d.buildInfo = *bi
}
d.state = *st
d.platform = platform
return nil
Expand Down Expand Up @@ -349,12 +368,12 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
if !isScratch {
// if image not scratch set original image name as ref
// and actual reference as alias in binfotypes.Source
d.buildSource = &binfotypes.Source{
d.buildInfo.Sources = append(d.buildInfo.Sources, binfotypes.Source{
Type: binfotypes.SourceTypeDockerImage,
Ref: origName,
Alias: ref.String(),
Pin: dgst.String(),
}
})
}
d.image = img
}
Expand Down Expand Up @@ -382,16 +401,23 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,

buildContext := &mutableOutput{}
ctxPaths := map[string]struct{}{}
buildInfo := &binfotypes.BuildInfo{}

for _, d := range allDispatchStates.states {
if !isReachable(target, d) {
continue
}

// collect build sources and dependencies
if d.buildSource != nil {
buildInfo.Sources = append(buildInfo.Sources, *d.buildSource)
if len(d.buildInfo.Sources) > 0 {
buildInfo.Sources = append(buildInfo.Sources, d.buildInfo.Sources...)
}
if d.buildInfo.Deps != nil {
for name, bi := range d.buildInfo.Deps {
if buildInfo.Deps == nil {
buildInfo.Deps = make(map[string]binfotypes.BuildInfo)
}
buildInfo.Deps[name] = bi
}
}

if d.base != nil {
Expand Down Expand Up @@ -701,7 +727,7 @@ type dispatchState struct {
cmdIndex int
cmdTotal int
prefixPlatform bool
buildSource *binfotypes.Source
buildInfo binfotypes.BuildInfo
}

type dispatchStates struct {
Expand Down
Loading