From 41ee238815d5d86f580899830f5443553215defe Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Mon, 2 Jun 2025 21:52:33 -0700 Subject: [PATCH 1/4] local: fix platform-split=true option Currently the option only worked when platform-split=false was set for multi-platform build but not for setting single platform build to use platform-split=true. Signed-off-by: Tonis Tiigi --- exporter/local/export.go | 32 ++++++++++++++------------------ exporter/local/fs.go | 17 ++++++++++++----- exporter/tar/export.go | 4 ++-- 3 files changed, 28 insertions(+), 25 deletions(-) diff --git a/exporter/local/export.go b/exporter/local/export.go index 29c6afbac990..d802c36a185f 100644 --- a/exporter/local/export.go +++ b/exporter/local/export.go @@ -117,7 +117,7 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source export := func(ctx context.Context, k string, ref cache.ImmutableRef, attestations []exporter.Attestation) func() error { return func() error { - outputFS, cleanup, err := CreateFS(ctx, sessionID, k, ref, attestations, now, e.opts) + outputFS, cleanup, err := CreateFS(ctx, sessionID, k, ref, attestations, now, isMap, e.opts) if err != nil { return err } @@ -125,7 +125,8 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source defer cleanup() } - if !e.opts.PlatformSplit { + lbl := "copying files" + if !e.opts.UsePlatformSplit(isMap) { // check for duplicate paths err = outputFS.Walk(ctx, "", func(p string, entry os.DirEntry, err error) error { if entry.IsDir() { @@ -145,23 +146,18 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source if err != nil { return err } - } - - lbl := "copying files" - if isMap { + } else { lbl += " " + k - if e.opts.PlatformSplit { - st := &fstypes.Stat{ - Mode: uint32(os.ModeDir | 0755), - Path: strings.ReplaceAll(k, "/", "_"), - } - if e.opts.Epoch != nil { - st.ModTime = e.opts.Epoch.UnixNano() - } - outputFS, err = fsutil.SubDirFS([]fsutil.Dir{{FS: outputFS, Stat: st}}) - if err != nil { - return err - } + st := &fstypes.Stat{ + Mode: uint32(os.ModeDir | 0755), + Path: strings.ReplaceAll(k, "/", "_"), + } + if e.opts.Epoch != nil { + st.ModTime = e.opts.Epoch.UnixNano() + } + outputFS, err = fsutil.SubDirFS([]fsutil.Dir{{FS: outputFS, Stat: st}}) + if err != nil { + return err } } diff --git a/exporter/local/fs.go b/exporter/local/fs.go index 1985ed0096dc..6e68b0aa2a59 100644 --- a/exporter/local/fs.go +++ b/exporter/local/fs.go @@ -38,12 +38,18 @@ const ( type CreateFSOpts struct { Epoch *time.Time AttestationPrefix string - PlatformSplit bool + PlatformSplit *bool +} + +func (c *CreateFSOpts) UsePlatformSplit(isMap bool) bool { + if c.PlatformSplit == nil { + return isMap + } + return *c.PlatformSplit } func (c *CreateFSOpts) Load(opt map[string]string) (map[string]string, error) { rest := make(map[string]string) - c.PlatformSplit = true var err error c.Epoch, opt, err = epoch.ParseExporterAttrs(opt) @@ -60,7 +66,7 @@ func (c *CreateFSOpts) Load(opt map[string]string) (map[string]string, error) { if err != nil { return nil, errors.Wrapf(err, "non-bool value for %s: %s", keyPlatformSplit, v) } - c.PlatformSplit = b + c.PlatformSplit = &b default: rest[k] = v } @@ -69,7 +75,7 @@ func (c *CreateFSOpts) Load(opt map[string]string) (map[string]string, error) { return rest, nil } -func CreateFS(ctx context.Context, sessionID string, k string, ref cache.ImmutableRef, attestations []exporter.Attestation, defaultTime time.Time, opt CreateFSOpts) (fsutil.FS, func() error, error) { +func CreateFS(ctx context.Context, sessionID string, k string, ref cache.ImmutableRef, attestations []exporter.Attestation, defaultTime time.Time, isMap bool, opt CreateFSOpts) (fsutil.FS, func() error, error) { var cleanup func() error var src string var err error @@ -174,6 +180,7 @@ func CreateFS(ctx context.Context, sessionID string, k string, ref cache.Immutab return nil, nil, err } stmtFS := staticfs.NewFS() + split := opt.UsePlatformSplit(isMap) names := map[string]struct{}{} for i, stmt := range stmts { @@ -183,7 +190,7 @@ func CreateFS(ctx context.Context, sessionID string, k string, ref cache.Immutab } name := opt.AttestationPrefix + path.Base(attestations[i].Path) - if !opt.PlatformSplit { + if !split { nameExt := path.Ext(name) namBase := strings.TrimSuffix(name, nameExt) name = fmt.Sprintf("%s.%s%s", namBase, strings.ReplaceAll(k, "/", "_"), nameExt) diff --git a/exporter/tar/export.go b/exporter/tar/export.go index 770b38254d10..57d9eef10464 100644 --- a/exporter/tar/export.go +++ b/exporter/tar/export.go @@ -95,9 +95,10 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source } now := time.Now().Truncate(time.Second) + isMap := len(inp.Refs) > 0 getDir := func(ctx context.Context, k string, ref cache.ImmutableRef, attestations []exporter.Attestation) (*fsutil.Dir, error) { - outputFS, cleanup, err := local.CreateFS(ctx, sessionID, k, ref, attestations, now, e.opts) + outputFS, cleanup, err := local.CreateFS(ctx, sessionID, k, ref, attestations, now, isMap, e.opts) if err != nil { return nil, err } @@ -119,7 +120,6 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source }, nil } - isMap := len(inp.Refs) > 0 if _, ok := inp.Metadata[exptypes.ExporterPlatformsKey]; isMap && !ok { return nil, nil, errors.Errorf("unable to export multiple refs, missing platforms mapping") } From 40ddcfca90b69adb398f97a977f77e8d92291609 Mon Sep 17 00:00:00 2001 From: Anthony Nandaa Date: Thu, 5 Jun 2025 17:55:01 +0300 Subject: [PATCH 2/4] fix: wcow: privileges for local exporter Signed-off-by: Anthony Nandaa --- exporter/local/export.go | 2 +- exporter/local/export_unix.go | 14 ++++++++++++++ exporter/local/export_windows.go | 17 +++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 exporter/local/export_unix.go create mode 100644 exporter/local/export_windows.go diff --git a/exporter/local/export.go b/exporter/local/export.go index d802c36a185f..10e3a87ce3a0 100644 --- a/exporter/local/export.go +++ b/exporter/local/export.go @@ -128,7 +128,7 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source lbl := "copying files" if !e.opts.UsePlatformSplit(isMap) { // check for duplicate paths - err = outputFS.Walk(ctx, "", func(p string, entry os.DirEntry, err error) error { + err = fsWalk(ctx, outputFS, "", func(p string, entry os.DirEntry, err error) error { if entry.IsDir() { return nil } diff --git a/exporter/local/export_unix.go b/exporter/local/export_unix.go new file mode 100644 index 000000000000..d2612182fd38 --- /dev/null +++ b/exporter/local/export_unix.go @@ -0,0 +1,14 @@ +//go:build !windows + +package local + +import ( + "context" + gofs "io/fs" + + "github.com/tonistiigi/fsutil" +) + +func fsWalk(ctx context.Context, fs fsutil.FS, s string, walkFn gofs.WalkDirFunc) error { + return fs.Walk(ctx, s, walkFn) +} diff --git a/exporter/local/export_windows.go b/exporter/local/export_windows.go new file mode 100644 index 000000000000..dea1821a71ad --- /dev/null +++ b/exporter/local/export_windows.go @@ -0,0 +1,17 @@ +package local + +import ( + "context" + gofs "io/fs" + + "github.com/Microsoft/go-winio" + "github.com/tonistiigi/fsutil" +) + +func fsWalk(ctx context.Context, fs fsutil.FS, s string, walkFn gofs.WalkDirFunc) error { + // Windows has some special files that require + // SeBackupPrivilege to be accessed. Ref #4994 + return winio.RunWithPrivilege(winio.SeBackupPrivilege, func() error { + return fs.Walk(ctx, s, walkFn) + }) +} From 3eb4e72cefcb1e18e677044b8ce3d18c38763d42 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Tue, 3 Jun 2025 15:56:31 -0700 Subject: [PATCH 3/4] client: add test for forcing platform-split=true on single-arch build Signed-off-by: Tonis Tiigi --- client/client_test.go | 57 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/client/client_test.go b/client/client_test.go index 6c49b5e8c07d..ab8c3d2a7994 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -227,6 +227,7 @@ var allTests = []func(t *testing.T, sb integration.Sandbox){ testSnapshotWithMultipleBlobs, testExportLocalNoPlatformSplit, testExportLocalNoPlatformSplitOverwrite, + testExportLocalForcePlatformSplit, testSolverOptLocalDirsStillWorks, testOCIIndexMediatype, testLayerLimitOnMounts, @@ -6951,6 +6952,62 @@ func testExportLocalNoPlatformSplitOverwrite(t *testing.T, sb integration.Sandbo }, }, "", frontend, nil) require.Error(t, err) + require.ErrorContains(t, err, "cannot overwrite hello-linux from") + require.ErrorContains(t, err, "when split option is disabled") +} + +func testExportLocalForcePlatformSplit(t *testing.T, sb integration.Sandbox) { + workers.CheckFeatureCompat(t, sb, workers.FeatureOCIExporter, workers.FeatureMultiPlatform) + c, err := New(sb.Context(), sb.Address()) + require.NoError(t, err) + defer c.Close() + + frontend := func(ctx context.Context, c gateway.Client) (*gateway.Result, error) { + st := llb.Scratch().File( + llb.Mkfile("foo", 0600, []byte("hello")), + ) + + def, err := st.Marshal(ctx) + if err != nil { + return nil, err + } + + return c.Solve(ctx, gateway.SolveRequest{ + Definition: def.ToPB(), + }) + } + + destDir := t.TempDir() + _, err = c.Build(sb.Context(), SolveOpt{ + Exports: []ExportEntry{ + { + Type: ExporterLocal, + OutputDir: destDir, + Attrs: map[string]string{ + "platform-split": "true", + }, + }, + }, + }, "", frontend, nil) + require.NoError(t, err) + + fis, err := os.ReadDir(destDir) + require.NoError(t, err) + + require.Len(t, fis, 1, "expected one files in the output directory") + + expPlatform := runtime.GOOS + "_" + runtime.GOARCH + _, err = os.Stat(filepath.Join(destDir, expPlatform+"/")) + require.NoError(t, err) + + fis, err = os.ReadDir(filepath.Join(destDir, expPlatform)) + require.NoError(t, err) + + require.Len(t, fis, 1, "expected one files in the output directory for platform "+expPlatform) + + dt, err := os.ReadFile(filepath.Join(destDir, expPlatform, "foo")) + require.NoError(t, err) + require.Equal(t, "hello", string(dt)) } func readFileInImage(ctx context.Context, t *testing.T, c *Client, ref, path string) ([]byte, error) { From db03322b421bfefaec5d033b01ce61bdecfada10 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Wed, 11 Jun 2025 12:53:17 +0200 Subject: [PATCH 4/4] test: use platforms spec instead of runtime const Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- client/client_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/client_test.go b/client/client_test.go index ab8c3d2a7994..541b285848ca 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -6996,7 +6996,7 @@ func testExportLocalForcePlatformSplit(t *testing.T, sb integration.Sandbox) { require.Len(t, fis, 1, "expected one files in the output directory") - expPlatform := runtime.GOOS + "_" + runtime.GOARCH + expPlatform := strings.ReplaceAll(platforms.FormatAll(platforms.DefaultSpec()), "/", "_") _, err = os.Stat(filepath.Join(destDir, expPlatform+"/")) require.NoError(t, err)