diff --git a/README.md b/README.md index f76ade72ef2e..271e9600890a 100644 --- a/README.md +++ b/README.md @@ -293,6 +293,50 @@ COPY --from=builder /usr/src/app/testresult.xml . buildctl build ... --opt target=testresult --output type=local,dest=path/to/output-dir ``` +With a [multi-platform build](docs/multi-platform.md), a subfolder matching +each target platform will be created in the destination directory: + +```dockerfile +FROM busybox AS build +ARG TARGETOS +ARG TARGETARCH +RUN mkdir /out && echo foo > /out/hello-$TARGETOS-$TARGETARCH + +FROM scratch +COPY --from=build /out / +``` + +```bash +$ buildctl build \ + --frontend dockerfile.v0 \ + --opt platform=linux/amd64,linux/arm64 \ + --output type=local,dest=./bin/release + +$ tree ./bin +./bin/ +└── release + ├── linux_amd64 + │ └── hello-linux-amd64 + └── linux_arm64 + └── hello-linux-arm64 +``` + +You can set `platform-split=false` to merge files from all platforms together +into same directory: + +```bash +$ buildctl build \ + --frontend dockerfile.v0 \ + --opt platform=linux/amd64,linux/arm64 \ + --output type=local,dest=./bin/release,platform-split=false + +$ tree ./bin +./bin/ +└── release + ├── hello-linux-amd64 + └── hello-linux-arm64 +``` + Tar exporter is similar to local exporter but transfers the files through a tarball. ```bash diff --git a/client/client_test.go b/client/client_test.go index ae445cd524a5..180037bcc46b 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -200,6 +200,8 @@ func TestIntegration(t *testing.T) { testLLBMountPerformance, testClientCustomGRPCOpts, testMultipleRecordsWithSameLayersCacheImportExport, + testExportLocalNoPlatformSplit, + testExportLocalNoPlatformSplitOverwrite, ) } @@ -5249,6 +5251,152 @@ func testMultipleRecordsWithSameLayersCacheImportExport(t *testing.T, sb integra ensurePruneAll(t, c, sb) } +func testExportLocalNoPlatformSplit(t *testing.T, sb integration.Sandbox) { + integration.CheckFeatureCompat(t, sb, integration.FeatureOCIExporter, integration.FeatureMultiPlatform) + c, err := New(sb.Context(), sb.Address()) + require.NoError(t, err) + defer c.Close() + + platformsToTest := []string{"linux/amd64", "linux/arm64"} + frontend := func(ctx context.Context, c gateway.Client) (*gateway.Result, error) { + res := gateway.NewResult() + expPlatforms := &exptypes.Platforms{ + Platforms: make([]exptypes.Platform, len(platformsToTest)), + } + for i, platform := range platformsToTest { + st := llb.Scratch().File( + llb.Mkfile("hello-"+strings.ReplaceAll(platform, "/", "-"), 0600, []byte(platform)), + ) + + def, err := st.Marshal(ctx) + if err != nil { + return nil, err + } + + r, err := c.Solve(ctx, gateway.SolveRequest{ + Definition: def.ToPB(), + }) + if err != nil { + return nil, err + } + + ref, err := r.SingleRef() + if err != nil { + return nil, err + } + + _, err = ref.ToState() + if err != nil { + return nil, err + } + res.AddRef(platform, ref) + + expPlatforms.Platforms[i] = exptypes.Platform{ + ID: platform, + Platform: platforms.MustParse(platform), + } + } + dt, err := json.Marshal(expPlatforms) + if err != nil { + return nil, err + } + res.AddMeta(exptypes.ExporterPlatformsKey, dt) + + return res, nil + } + + destDir := t.TempDir() + _, err = c.Build(sb.Context(), SolveOpt{ + Exports: []ExportEntry{ + { + Type: ExporterLocal, + OutputDir: destDir, + Attrs: map[string]string{ + "platform-split": "false", + }, + }, + }, + }, "", frontend, nil) + require.NoError(t, err) + + dt, err := os.ReadFile(filepath.Join(destDir, "hello-linux-amd64")) + require.NoError(t, err) + require.Equal(t, "linux/amd64", string(dt)) + + dt, err = os.ReadFile(filepath.Join(destDir, "hello-linux-arm64")) + require.NoError(t, err) + require.Equal(t, "linux/arm64", string(dt)) +} + +func testExportLocalNoPlatformSplitOverwrite(t *testing.T, sb integration.Sandbox) { + integration.CheckFeatureCompat(t, sb, integration.FeatureOCIExporter, integration.FeatureMultiPlatform) + c, err := New(sb.Context(), sb.Address()) + require.NoError(t, err) + defer c.Close() + + platformsToTest := []string{"linux/amd64", "linux/arm64"} + frontend := func(ctx context.Context, c gateway.Client) (*gateway.Result, error) { + res := gateway.NewResult() + expPlatforms := &exptypes.Platforms{ + Platforms: make([]exptypes.Platform, len(platformsToTest)), + } + for i, platform := range platformsToTest { + st := llb.Scratch().File( + llb.Mkfile("hello-linux", 0600, []byte(platform)), + ) + + def, err := st.Marshal(ctx) + if err != nil { + return nil, err + } + + r, err := c.Solve(ctx, gateway.SolveRequest{ + Definition: def.ToPB(), + }) + if err != nil { + return nil, err + } + + ref, err := r.SingleRef() + if err != nil { + return nil, err + } + + _, err = ref.ToState() + if err != nil { + return nil, err + } + res.AddRef(platform, ref) + + expPlatforms.Platforms[i] = exptypes.Platform{ + ID: platform, + Platform: platforms.MustParse(platform), + } + } + dt, err := json.Marshal(expPlatforms) + if err != nil { + return nil, err + } + res.AddMeta(exptypes.ExporterPlatformsKey, dt) + + return res, nil + } + + destDir := t.TempDir() + _, err = c.Build(sb.Context(), SolveOpt{ + Exports: []ExportEntry{ + { + Type: ExporterLocal, + OutputDir: destDir, + Attrs: map[string]string{ + "platform-split": "false", + }, + }, + }, + }, "", frontend, nil) + require.Error(t, err) +} + func readFileInImage(ctx context.Context, t *testing.T, c *Client, ref, path string) ([]byte, error) { def, err := llb.Image(ref).Marshal(ctx) if err != nil { diff --git a/exporter/local/export.go b/exporter/local/export.go index 7157c63dac80..771b7aaf2252 100644 --- a/exporter/local/export.go +++ b/exporter/local/export.go @@ -4,6 +4,7 @@ import ( "context" "os" "strings" + "sync" "time" "github.com/moby/buildkit/cache" @@ -92,6 +93,9 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source now := time.Now().Truncate(time.Second) + visitedPath := map[string]string{} + var visitedMu sync.Mutex + export := func(ctx context.Context, k string, ref cache.ImmutableRef, attestations []exporter.Attestation) func() error { return func() error { outputFS, cleanup, err := CreateFS(ctx, sessionID, k, ref, attestations, now, e.opts) @@ -102,20 +106,43 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source defer cleanup() } + if !e.opts.PlatformSplit { + // check for duplicate paths + err = outputFS.Walk(ctx, func(p string, fi os.FileInfo, err error) error { + if fi.IsDir() { + return nil + } + if err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + visitedMu.Lock() + defer visitedMu.Unlock() + if vp, ok := visitedPath[p]; ok { + return errors.Errorf("cannot overwrite %s from %s with %s when split option is disabled", p, vp, k) + } + visitedPath[p] = k + return nil + }) + if err != nil { + return err + } + } + lbl := "copying files" if isMap { lbl += " " + k - st := fstypes.Stat{ - Mode: uint32(os.ModeDir | 0755), - Path: strings.Replace(k, "/", "_", -1), - } - if e.opts.Epoch != nil { - st.ModTime = e.opts.Epoch.UnixNano() - } - - outputFS, err = fsutil.SubDirFS([]fsutil.Dir{{FS: outputFS, Stat: st}}) - if err != nil { - return err + if e.opts.PlatformSplit { + st := fstypes.Stat{ + Mode: uint32(os.ModeDir | 0755), + Path: strings.Replace(k, "/", "_", -1), + } + if e.opts.Epoch != nil { + st.ModTime = e.opts.Epoch.UnixNano() + } + outputFS, err = fsutil.SubDirFS([]fsutil.Dir{{FS: outputFS, Stat: st}}) + if err != nil { + return err + } } } diff --git a/exporter/local/fs.go b/exporter/local/fs.go index 9caf6f73af60..d8e4703ac146 100644 --- a/exporter/local/fs.go +++ b/exporter/local/fs.go @@ -3,11 +3,13 @@ package local import ( "context" "encoding/json" + "fmt" "io" "io/fs" "os" "path" "strconv" + "strings" "time" "github.com/docker/docker/pkg/idtools" @@ -28,15 +30,20 @@ import ( const ( keyAttestationPrefix = "attestation-prefix" + // keyPlatformSplit is an exporter option which can be used to split result + // in subfolders when multiple platform references are exported. + keyPlatformSplit = "platform-split" ) type CreateFSOpts struct { Epoch *time.Time AttestationPrefix string + PlatformSplit bool } func (c *CreateFSOpts) Load(opt map[string]string) (map[string]string, error) { rest := make(map[string]string) + c.PlatformSplit = true var err error c.Epoch, opt, err = epoch.ParseExporterAttrs(opt) @@ -48,6 +55,12 @@ func (c *CreateFSOpts) Load(opt map[string]string) (map[string]string, error) { switch k { case keyAttestationPrefix: c.AttestationPrefix = v + case keyPlatformSplit: + b, err := strconv.ParseBool(v) + if err != nil { + return nil, errors.Wrapf(err, "non-bool value for %s: %s", keyPlatformSplit, v) + } + c.PlatformSplit = b default: rest[k] = v } @@ -164,6 +177,11 @@ func CreateFS(ctx context.Context, sessionID string, k string, ref cache.Immutab } name := opt.AttestationPrefix + path.Base(attestations[i].Path) + if !opt.PlatformSplit { + nameExt := path.Ext(name) + namBase := strings.TrimSuffix(name, nameExt) + name = fmt.Sprintf("%s.%s%s", namBase, strings.Replace(k, "/", "_", -1), nameExt) + } if _, ok := names[name]; ok { return nil, nil, errors.Errorf("duplicate attestation path name %s", name) }