bake(compose): fix unskipped services without build context

Signed-off-by: CrazyMax <[email protected]>

bake/bake.go

@@ -3,7 +3,6 @@ package bake
 import (
 	"context"
 	"encoding/csv"
-	"fmt"
 	"io"
 	"os"
 	"path"
@@ -13,6 +12,7 @@ import (
 	"strconv"
 	"strings"
 
+	compose "github.com/compose-spec/compose-go/types"
 	"github.com/docker/buildx/bake/hclparser"
 	"github.com/docker/buildx/build"
 	"github.com/docker/buildx/util/buildflags"
@@ -200,52 +200,58 @@ func ParseFiles(files []File, defaults map[string]string) (_ *Config, err error)
 	}()
 
 	var c Config
-	var fs []*hcl.File
+	var composeFiles []File
+	var hclFiles []*hcl.File
 	for _, f := range files {
-		cfg, isCompose, composeErr := ParseComposeFile(f.Data, f.Name)
-		if isCompose {
-			if composeErr != nil {
-				return nil, composeErr
-			}
-			c = mergeConfig(c, *cfg)
-			c = dedupeConfig(c)
-		}
-		if !isCompose {
-			hf, isHCL, err := ParseHCLFile(f.Data, f.Name)
-			if isHCL {
-				if err != nil {
-					return nil, err
-				}
-				fs = append(fs, hf)
-			} else if composeErr != nil {
-				return nil, fmt.Errorf("failed to parse %s: parsing yaml: %v, parsing hcl: %w", f.Name, composeErr, err)
-			} else {
+		switch filepath.Ext(strings.ToLower(f.Name)) {
+		case ".yml", ".yaml":
+			composeFiles = append(composeFiles, f)
+		default:
+			hf, err := ParseHCLFile(f.Data, f.Name)
+			if err != nil {
 				return nil, err
 			}
+			hclFiles = append(hclFiles, hf)
 		}
 	}
 
-	if len(fs) > 0 {
-		if err := hclparser.Parse(hcl.MergeFiles(fs), hclparser.Opt{
+	if len(composeFiles) > 0 {
+		cfg, cmperr := ParseComposeFiles(composeFiles)
+		if cmperr != nil {
+			return nil, errors.Wrap(cmperr, "failed to parse compose file")
+		}
+		c = mergeConfig(c, *cfg)
+		c = dedupeConfig(c)
+	}
+
+	if len(hclFiles) > 0 {
+		if err := hclparser.Parse(hcl.MergeFiles(hclFiles), hclparser.Opt{
 			LookupVar:     os.LookupEnv,
 			Vars:          defaults,
 			ValidateLabel: validateTargetName,
 		}, &c); err.HasErrors() {
 			return nil, err
 		}
 	}
+
 	return &c, nil
 }
 
 func dedupeConfig(c Config) Config {
 	c2 := c
+	c2.Groups = make([]*Group, 0, len(c2.Groups))
+	for _, g := range c.Groups {
+		g1 := *g
+		g1.Targets = dedupSlice(g1.Targets)
+		c2.Groups = append(c2.Groups, &g1)
+	}
 	c2.Targets = make([]*Target, 0, len(c2.Targets))
-	m := map[string]*Target{}
+	mt := map[string]*Target{}
 	for _, t := range c.Targets {
-		if t2, ok := m[t.Name]; ok {
+		if t2, ok := mt[t.Name]; ok {
 			t2.Merge(t)
 		} else {
-			m[t.Name] = t
+			mt[t.Name] = t
 			c2.Targets = append(c2.Targets, t)
 		}
 	}
@@ -256,24 +262,22 @@ func ParseFile(dt []byte, fn string) (*Config, error) {
 	return ParseFiles([]File{{Data: dt, Name: fn}}, nil)
 }
 
-func ParseComposeFile(dt []byte, fn string) (*Config, bool, error) {
+func ParseComposeFiles(fs []File) (*Config, error) {
 	envs := sliceToMap(os.Environ())
 	if wd, err := os.Getwd(); err == nil {
 		envs, err = loadDotEnv(envs, wd)
 		if err != nil {
-			return nil, true, err
+			return nil, err
 		}
 	}
-	fnl := strings.ToLower(fn)
-	if strings.HasSuffix(fnl, ".yml") || strings.HasSuffix(fnl, ".yaml") {
-		cfg, err := ParseCompose(dt, envs)
-		return cfg, true, err
-	}
-	if strings.HasSuffix(fnl, ".json") || strings.HasSuffix(fnl, ".hcl") {
-		return nil, false, nil
+	var cfgs []compose.ConfigFile
+	for _, f := range fs {
+		cfgs = append(cfgs, compose.ConfigFile{
+			Filename: f.Name,
+			Content:  f.Data,
+		})
 	}
-	cfg, err := ParseCompose(dt, envs)
-	return cfg, err == nil, err
+	return ParseCompose(cfgs, envs)
 }
 
 type Config struct {

bake/bake_test.go

@@ -530,7 +530,8 @@ func TestReadEmptyTargets(t *testing.T) {
 		Name: "docker-compose.yml",
 		Data: []byte(`
 services:
-  app2: {}
+  app2:
+    build: {}
 `),
 	}
 

bake/compose.go

@@ -1,7 +1,6 @@
 package bake
 
 import (
-	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
@@ -13,27 +12,16 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
-// errComposeInvalid is returned when a compose file is invalid
-var errComposeInvalid = errors.New("invalid compose file")
-
-func ParseCompose(dt []byte, envs map[string]string) (*Config, error) {
+func ParseCompose(cfgs []compose.ConfigFile, envs map[string]string) (*Config, error) {
 	cfg, err := loader.Load(compose.ConfigDetails{
-		ConfigFiles: []compose.ConfigFile{
-			{
-				Content: dt,
-			},
-		},
+		ConfigFiles: cfgs,
 		Environment: envs,
 	}, func(options *loader.Options) {
 		options.SkipNormalization = true
-		options.SkipConsistencyCheck = true
 	})
 	if err != nil {
 		return nil, err
 	}
-	if err = composeValidate(cfg); err != nil {
-		return nil, err
-	}
 
 	var c Config
 	if len(cfg.Services) > 0 {
@@ -44,7 +32,7 @@ func ParseCompose(dt []byte, envs map[string]string) (*Config, error) {
 
 		for _, s := range cfg.Services {
 			if s.Build == nil {
-				s.Build = &compose.BuildConfig{}
+				continue
 			}
 
 			targetName := sanitizeTargetName(s.Name)
@@ -248,28 +236,6 @@ func (t *Target) composeExtTarget(exts map[string]interface{}) error {
 	return nil
 }
 
-// composeValidate validates a compose file
-func composeValidate(project *compose.Project) error {
-	for _, s := range project.Services {
-		if s.Build != nil {
-			for _, secret := range s.Build.Secrets {
-				if _, ok := project.Secrets[secret.Source]; !ok {
-					return errors.Wrap(errComposeInvalid, fmt.Sprintf("service %q refers to undefined build secret %s", sanitizeTargetName(s.Name), secret.Source))
-				}
-			}
-		}
-	}
-	for name, secret := range project.Secrets {
-		if secret.External.External {
-			continue
-		}
-		if secret.File == "" && secret.Environment == "" {
-			return errors.Wrap(errComposeInvalid, fmt.Sprintf("secret %q must declare either `file` or `environment`", name))
-		}
-	}
-	return nil
-}
-
 // composeToBuildkitSecret converts secret from compose format to buildkit's
 // csv format.
 func composeToBuildkitSecret(inp compose.ServiceSecretConfig, psecret compose.SecretConfig) (string, error) {

bake/compose_test.go

@@ -6,6 +6,7 @@ import (
 	"sort"
 	"testing"
 
+	compose "github.com/compose-spec/compose-go/types"
 	"github.com/stretchr/testify/require"
 )
 
@@ -38,7 +39,7 @@ secrets:
     file: /root/.aws/credentials
 `)
 
-	c, err := ParseCompose(dt, nil)
+	c, err := ParseCompose([]compose.ConfigFile{{Content: dt}}, nil)
 	require.NoError(t, err)
 
 	require.Equal(t, 1, len(c.Groups))
@@ -76,9 +77,10 @@ services:
     webapp:
         build: ./db
 `)
-	c, err := ParseCompose(dt, nil)
+	c, err := ParseCompose([]compose.ConfigFile{{Content: dt}}, nil)
 	require.NoError(t, err)
 	require.Equal(t, 1, len(c.Groups))
+	require.Equal(t, 1, len(c.Targets))
 }
 
 func TestParseComposeTarget(t *testing.T) {
@@ -94,7 +96,7 @@ services:
       target: webapp
 `)
 
-	c, err := ParseCompose(dt, nil)
+	c, err := ParseCompose([]compose.ConfigFile{{Content: dt}}, nil)
 	require.NoError(t, err)
 
 	require.Equal(t, 2, len(c.Targets))
@@ -119,7 +121,7 @@ services:
       target: webapp
 `)
 
-	c, err := ParseCompose(dt, nil)
+	c, err := ParseCompose([]compose.ConfigFile{{Content: dt}}, nil)
 	require.NoError(t, err)
 	require.Equal(t, 2, len(c.Targets))
 	sort.Slice(c.Targets, func(i, j int) bool {
@@ -153,7 +155,7 @@ services:
 	os.Setenv("ZZZ_BAR", "zzz_foo")
 	defer os.Unsetenv("ZZZ_BAR")
 
-	c, err := ParseCompose(dt, sliceToMap(os.Environ()))
+	c, err := ParseCompose([]compose.ConfigFile{{Content: dt}}, sliceToMap(os.Environ()))
 	require.NoError(t, err)
 	require.Equal(t, "bar", c.Targets[0].Args["FOO"])
 	require.Equal(t, "zzz_foo", c.Targets[0].Args["BAR"])
@@ -167,8 +169,8 @@ services:
     entrypoint: echo 1
 `)
 
-	_, err := ParseCompose(dt, nil)
-	require.NoError(t, err)
+	_, err := ParseCompose([]compose.ConfigFile{{Content: dt}}, nil)
+	require.Error(t, err)
 }
 
 func TestAdvancedNetwork(t *testing.T) {
@@ -192,7 +194,7 @@ networks:
           gateway: 10.5.0.254
 `)
 
-	_, err := ParseCompose(dt, nil)
+	_, err := ParseCompose([]compose.ConfigFile{{Content: dt}}, nil)
 	require.NoError(t, err)
 }
 
@@ -209,7 +211,7 @@ services:
         - bar
 `)
 
-	c, err := ParseCompose(dt, nil)
+	c, err := ParseCompose([]compose.ConfigFile{{Content: dt}}, nil)
 	require.NoError(t, err)
 	require.Equal(t, []string{"foo", "bar"}, c.Targets[0].Tags)
 }
@@ -246,7 +248,7 @@ networks:
     name: test-net
 `)
 
-	_, err := ParseCompose(dt, nil)
+	_, err := ParseCompose([]compose.ConfigFile{{Content: dt}}, nil)
 	require.NoError(t, err)
 }
 
@@ -299,7 +301,7 @@ services:
         no-cache: true
 `)
 
-	c, err := ParseCompose(dt, nil)
+	c, err := ParseCompose([]compose.ConfigFile{{Content: dt}}, nil)
 	require.NoError(t, err)
 	require.Equal(t, 2, len(c.Targets))
 	sort.Slice(c.Targets, func(i, j int) bool {
@@ -343,7 +345,7 @@ services:
           - type=local,dest=path/to/cache
 `)
 
-	c, err := ParseCompose(dt, nil)
+	c, err := ParseCompose([]compose.ConfigFile{{Content: dt}}, nil)
 	require.NoError(t, err)
 	require.Equal(t, 1, len(c.Targets))
 	require.Equal(t, []string{"ct-addon:foo", "ct-addon:baz"}, c.Targets[0].Tags)
@@ -376,7 +378,7 @@ services:
       - ` + envf.Name() + `
 `)
 
-	c, err := ParseCompose(dt, nil)
+	c, err := ParseCompose([]compose.ConfigFile{{Content: dt}}, nil)
 	require.NoError(t, err)
 	require.Equal(t, map[string]string{"CT_ECR": "foo", "FOO": "bsdf -csdf", "NODE_ENV": "test"}, c.Targets[0].Args)
 }
@@ -397,7 +399,10 @@ services:
 `)
 
 	chdir(t, tmpdir)
-	c, _, err := ParseComposeFile(dt, "docker-compose.yml")
+	c, err := ParseComposeFiles([]File{{
+		Name: "docker-compose.yml",
+		Data: dt,
+	}})
 	require.NoError(t, err)
 	require.Equal(t, map[string]string{"FOO": "bar"}, c.Targets[0].Args)
 }
@@ -419,7 +424,7 @@ services:
         published: "3306"
         protocol: tcp
 `)
-	_, err := ParseCompose(dt, nil)
+	_, err := ParseCompose([]compose.ConfigFile{{Content: dt}}, nil)
 	require.NoError(t, err)
 }
 
@@ -465,12 +470,12 @@ func TestServiceName(t *testing.T) {
 	for _, tt := range cases {
 		tt := tt
 		t.Run(tt.svc, func(t *testing.T) {
-			_, err := ParseCompose([]byte(`
+			_, err := ParseCompose([]compose.ConfigFile{{Content: []byte(`
 services:
-  `+tt.svc+`:
+  ` + tt.svc + `:
     build:
       context: .
-`), nil)
+`)}}, nil)
 			if tt.wantErr {
 				require.Error(t, err)
 			} else {
@@ -536,7 +541,7 @@ services:
 	for _, tt := range cases {
 		tt := tt
 		t.Run(tt.name, func(t *testing.T) {
-			_, err := ParseCompose(tt.dt, nil)
+			_, err := ParseCompose([]compose.ConfigFile{{Content: tt.dt}}, nil)
 			if tt.wantErr {
 				require.Error(t, err)
 			} else {