From 1d43a00be3b55223b15b24209c82372a2bafd030 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Wed, 17 Aug 2022 15:12:41 +0200 Subject: [PATCH] bake(compose): fix unskipped services without build context Signed-off-by: CrazyMax --- bake/bake.go | 76 +++++++++++++++++++++++--------------------- bake/bake_test.go | 3 +- bake/compose.go | 40 ++--------------------- bake/compose_test.go | 43 ++++++++++++++----------- bake/hcl.go | 12 +++---- 5 files changed, 75 insertions(+), 99 deletions(-) diff --git a/bake/bake.go b/bake/bake.go index 573556cb14fa..cc33c8c10750 100644 --- a/bake/bake.go +++ b/bake/bake.go @@ -3,7 +3,6 @@ package bake import ( "context" "encoding/csv" - "fmt" "io" "os" "path" @@ -13,6 +12,7 @@ import ( "strconv" "strings" + compose "github.com/compose-spec/compose-go/types" "github.com/docker/buildx/bake/hclparser" "github.com/docker/buildx/build" "github.com/docker/buildx/util/buildflags" @@ -200,33 +200,32 @@ func ParseFiles(files []File, defaults map[string]string) (_ *Config, err error) }() var c Config - var fs []*hcl.File + var composeFiles []File + var hclFiles []*hcl.File for _, f := range files { - cfg, isCompose, composeErr := ParseComposeFile(f.Data, f.Name) - if isCompose { - if composeErr != nil { - return nil, composeErr - } - c = mergeConfig(c, *cfg) - c = dedupeConfig(c) - } - if !isCompose { - hf, isHCL, err := ParseHCLFile(f.Data, f.Name) - if isHCL { - if err != nil { - return nil, err - } - fs = append(fs, hf) - } else if composeErr != nil { - return nil, fmt.Errorf("failed to parse %s: parsing yaml: %v, parsing hcl: %w", f.Name, composeErr, err) - } else { + switch filepath.Ext(strings.ToLower(f.Name)) { + case ".yml", ".yaml": + composeFiles = append(composeFiles, f) + default: + hf, err := ParseHCLFile(f.Data, f.Name) + if err != nil { return nil, err } + hclFiles = append(hclFiles, hf) } } - if len(fs) > 0 { - if err := hclparser.Parse(hcl.MergeFiles(fs), hclparser.Opt{ + if len(composeFiles) > 0 { + cfg, cmperr := ParseComposeFiles(composeFiles) + if cmperr != nil { + return nil, errors.Wrap(cmperr, "failed to parse compose file") + } + c = mergeConfig(c, *cfg) + c = dedupeConfig(c) + } + + if len(hclFiles) > 0 { + if err := hclparser.Parse(hcl.MergeFiles(hclFiles), hclparser.Opt{ LookupVar: os.LookupEnv, Vars: defaults, ValidateLabel: validateTargetName, @@ -234,18 +233,25 @@ func ParseFiles(files []File, defaults map[string]string) (_ *Config, err error) return nil, err } } + return &c, nil } func dedupeConfig(c Config) Config { c2 := c + c2.Groups = make([]*Group, 0, len(c2.Groups)) + for _, g := range c.Groups { + g1 := *g + g1.Targets = dedupSlice(g1.Targets) + c2.Groups = append(c2.Groups, &g1) + } c2.Targets = make([]*Target, 0, len(c2.Targets)) - m := map[string]*Target{} + mt := map[string]*Target{} for _, t := range c.Targets { - if t2, ok := m[t.Name]; ok { + if t2, ok := mt[t.Name]; ok { t2.Merge(t) } else { - m[t.Name] = t + mt[t.Name] = t c2.Targets = append(c2.Targets, t) } } @@ -256,24 +262,22 @@ func ParseFile(dt []byte, fn string) (*Config, error) { return ParseFiles([]File{{Data: dt, Name: fn}}, nil) } -func ParseComposeFile(dt []byte, fn string) (*Config, bool, error) { +func ParseComposeFiles(fs []File) (*Config, error) { envs := sliceToMap(os.Environ()) if wd, err := os.Getwd(); err == nil { envs, err = loadDotEnv(envs, wd) if err != nil { - return nil, true, err + return nil, err } } - fnl := strings.ToLower(fn) - if strings.HasSuffix(fnl, ".yml") || strings.HasSuffix(fnl, ".yaml") { - cfg, err := ParseCompose(dt, envs) - return cfg, true, err - } - if strings.HasSuffix(fnl, ".json") || strings.HasSuffix(fnl, ".hcl") { - return nil, false, nil + var cfgs []compose.ConfigFile + for _, f := range fs { + cfgs = append(cfgs, compose.ConfigFile{ + Filename: f.Name, + Content: f.Data, + }) } - cfg, err := ParseCompose(dt, envs) - return cfg, err == nil, err + return ParseCompose(cfgs, envs) } type Config struct { diff --git a/bake/bake_test.go b/bake/bake_test.go index 4441a2caec5e..c0b900591cd4 100644 --- a/bake/bake_test.go +++ b/bake/bake_test.go @@ -530,7 +530,8 @@ func TestReadEmptyTargets(t *testing.T) { Name: "docker-compose.yml", Data: []byte(` services: - app2: {} + app2: + build: {} `), } diff --git a/bake/compose.go b/bake/compose.go index d89c5d8a4d8e..004f3e3e0dd2 100644 --- a/bake/compose.go +++ b/bake/compose.go @@ -1,7 +1,6 @@ package bake import ( - "fmt" "os" "path/filepath" "strings" @@ -13,27 +12,16 @@ import ( "gopkg.in/yaml.v3" ) -// errComposeInvalid is returned when a compose file is invalid -var errComposeInvalid = errors.New("invalid compose file") - -func ParseCompose(dt []byte, envs map[string]string) (*Config, error) { +func ParseCompose(cfgs []compose.ConfigFile, envs map[string]string) (*Config, error) { cfg, err := loader.Load(compose.ConfigDetails{ - ConfigFiles: []compose.ConfigFile{ - { - Content: dt, - }, - }, + ConfigFiles: cfgs, Environment: envs, }, func(options *loader.Options) { options.SkipNormalization = true - options.SkipConsistencyCheck = true }) if err != nil { return nil, err } - if err = composeValidate(cfg); err != nil { - return nil, err - } var c Config if len(cfg.Services) > 0 { @@ -44,7 +32,7 @@ func ParseCompose(dt []byte, envs map[string]string) (*Config, error) { for _, s := range cfg.Services { if s.Build == nil { - s.Build = &compose.BuildConfig{} + continue } targetName := sanitizeTargetName(s.Name) @@ -248,28 +236,6 @@ func (t *Target) composeExtTarget(exts map[string]interface{}) error { return nil } -// composeValidate validates a compose file -func composeValidate(project *compose.Project) error { - for _, s := range project.Services { - if s.Build != nil { - for _, secret := range s.Build.Secrets { - if _, ok := project.Secrets[secret.Source]; !ok { - return errors.Wrap(errComposeInvalid, fmt.Sprintf("service %q refers to undefined build secret %s", sanitizeTargetName(s.Name), secret.Source)) - } - } - } - } - for name, secret := range project.Secrets { - if secret.External.External { - continue - } - if secret.File == "" && secret.Environment == "" { - return errors.Wrap(errComposeInvalid, fmt.Sprintf("secret %q must declare either `file` or `environment`", name)) - } - } - return nil -} - // composeToBuildkitSecret converts secret from compose format to buildkit's // csv format. func composeToBuildkitSecret(inp compose.ServiceSecretConfig, psecret compose.SecretConfig) (string, error) { diff --git a/bake/compose_test.go b/bake/compose_test.go index 4a075ab22451..12a6c8dea9c6 100644 --- a/bake/compose_test.go +++ b/bake/compose_test.go @@ -6,6 +6,7 @@ import ( "sort" "testing" + compose "github.com/compose-spec/compose-go/types" "github.com/stretchr/testify/require" ) @@ -38,7 +39,7 @@ secrets: file: /root/.aws/credentials `) - c, err := ParseCompose(dt, nil) + c, err := ParseCompose([]compose.ConfigFile{{Content: dt}}, nil) require.NoError(t, err) require.Equal(t, 1, len(c.Groups)) @@ -76,9 +77,10 @@ services: webapp: build: ./db `) - c, err := ParseCompose(dt, nil) + c, err := ParseCompose([]compose.ConfigFile{{Content: dt}}, nil) require.NoError(t, err) require.Equal(t, 1, len(c.Groups)) + require.Equal(t, 1, len(c.Targets)) } func TestParseComposeTarget(t *testing.T) { @@ -94,7 +96,7 @@ services: target: webapp `) - c, err := ParseCompose(dt, nil) + c, err := ParseCompose([]compose.ConfigFile{{Content: dt}}, nil) require.NoError(t, err) require.Equal(t, 2, len(c.Targets)) @@ -119,7 +121,7 @@ services: target: webapp `) - c, err := ParseCompose(dt, nil) + c, err := ParseCompose([]compose.ConfigFile{{Content: dt}}, nil) require.NoError(t, err) require.Equal(t, 2, len(c.Targets)) sort.Slice(c.Targets, func(i, j int) bool { @@ -153,7 +155,7 @@ services: os.Setenv("ZZZ_BAR", "zzz_foo") defer os.Unsetenv("ZZZ_BAR") - c, err := ParseCompose(dt, sliceToMap(os.Environ())) + c, err := ParseCompose([]compose.ConfigFile{{Content: dt}}, sliceToMap(os.Environ())) require.NoError(t, err) require.Equal(t, "bar", c.Targets[0].Args["FOO"]) require.Equal(t, "zzz_foo", c.Targets[0].Args["BAR"]) @@ -167,8 +169,8 @@ services: entrypoint: echo 1 `) - _, err := ParseCompose(dt, nil) - require.NoError(t, err) + _, err := ParseCompose([]compose.ConfigFile{{Content: dt}}, nil) + require.Error(t, err) } func TestAdvancedNetwork(t *testing.T) { @@ -192,7 +194,7 @@ networks: gateway: 10.5.0.254 `) - _, err := ParseCompose(dt, nil) + _, err := ParseCompose([]compose.ConfigFile{{Content: dt}}, nil) require.NoError(t, err) } @@ -209,7 +211,7 @@ services: - bar `) - c, err := ParseCompose(dt, nil) + c, err := ParseCompose([]compose.ConfigFile{{Content: dt}}, nil) require.NoError(t, err) require.Equal(t, []string{"foo", "bar"}, c.Targets[0].Tags) } @@ -246,7 +248,7 @@ networks: name: test-net `) - _, err := ParseCompose(dt, nil) + _, err := ParseCompose([]compose.ConfigFile{{Content: dt}}, nil) require.NoError(t, err) } @@ -299,7 +301,7 @@ services: no-cache: true `) - c, err := ParseCompose(dt, nil) + c, err := ParseCompose([]compose.ConfigFile{{Content: dt}}, nil) require.NoError(t, err) require.Equal(t, 2, len(c.Targets)) sort.Slice(c.Targets, func(i, j int) bool { @@ -343,7 +345,7 @@ services: - type=local,dest=path/to/cache `) - c, err := ParseCompose(dt, nil) + c, err := ParseCompose([]compose.ConfigFile{{Content: dt}}, nil) require.NoError(t, err) require.Equal(t, 1, len(c.Targets)) require.Equal(t, []string{"ct-addon:foo", "ct-addon:baz"}, c.Targets[0].Tags) @@ -376,7 +378,7 @@ services: - ` + envf.Name() + ` `) - c, err := ParseCompose(dt, nil) + c, err := ParseCompose([]compose.ConfigFile{{Content: dt}}, nil) require.NoError(t, err) require.Equal(t, map[string]string{"CT_ECR": "foo", "FOO": "bsdf -csdf", "NODE_ENV": "test"}, c.Targets[0].Args) } @@ -397,7 +399,10 @@ services: `) chdir(t, tmpdir) - c, _, err := ParseComposeFile(dt, "docker-compose.yml") + c, err := ParseComposeFiles([]File{{ + Name: "docker-compose.yml", + Data: dt, + }}) require.NoError(t, err) require.Equal(t, map[string]string{"FOO": "bar"}, c.Targets[0].Args) } @@ -419,7 +424,7 @@ services: published: "3306" protocol: tcp `) - _, err := ParseCompose(dt, nil) + _, err := ParseCompose([]compose.ConfigFile{{Content: dt}}, nil) require.NoError(t, err) } @@ -465,12 +470,12 @@ func TestServiceName(t *testing.T) { for _, tt := range cases { tt := tt t.Run(tt.svc, func(t *testing.T) { - _, err := ParseCompose([]byte(` + _, err := ParseCompose([]compose.ConfigFile{{Content: []byte(` services: - `+tt.svc+`: + ` + tt.svc + `: build: context: . -`), nil) +`)}}, nil) if tt.wantErr { require.Error(t, err) } else { @@ -536,7 +541,7 @@ services: for _, tt := range cases { tt := tt t.Run(tt.name, func(t *testing.T) { - _, err := ParseCompose(tt.dt, nil) + _, err := ParseCompose([]compose.ConfigFile{{Content: tt.dt}}, nil) if tt.wantErr { require.Error(t, err) } else { diff --git a/bake/hcl.go b/bake/hcl.go index 9c059cd8bcc0..a3f3530f8fc1 100644 --- a/bake/hcl.go +++ b/bake/hcl.go @@ -9,31 +9,31 @@ import ( "github.com/moby/buildkit/solver/pb" ) -func ParseHCLFile(dt []byte, fn string) (*hcl.File, bool, error) { +func ParseHCLFile(dt []byte, fn string) (*hcl.File, error) { var err error if strings.HasSuffix(fn, ".json") { f, diags := hclparse.NewParser().ParseJSON(dt, fn) if diags.HasErrors() { err = diags } - return f, true, err + return f, err } if strings.HasSuffix(fn, ".hcl") { f, diags := hclparse.NewParser().ParseHCL(dt, fn) if diags.HasErrors() { err = diags } - return f, true, err + return f, err } f, diags := hclparse.NewParser().ParseHCL(dt, fn+".hcl") if diags.HasErrors() { f, diags2 := hclparse.NewParser().ParseJSON(dt, fn+".json") if !diags2.HasErrors() { - return f, true, nil + return f, nil } - return nil, false, diags + return nil, diags } - return f, true, nil + return f, nil } func formatHCLError(err error, files []File) error {