source: add metadata-only transfer option for local source

client/client_test.go

@@ -9,6 +9,7 @@ import (
 	"crypto/rsa"
 	"crypto/x509"
 	"encoding/base64"
+	"encoding/binary"
 	"encoding/json"
 	"encoding/pem"
 	"fmt"
@@ -70,6 +71,7 @@ import (
 	"github.com/spdx/tools-golang/spdx"
 	"github.com/stretchr/testify/require"
 	"github.com/tonistiigi/fsutil"
+	fsutiltypes "github.com/tonistiigi/fsutil/types"
 	"golang.org/x/crypto/ssh/agent"
 	"golang.org/x/mod/semver"
 	"golang.org/x/sync/errgroup"
@@ -225,6 +227,7 @@ var allTests = []func(t *testing.T, sb integration.Sandbox){
 	testFrontendLintSkipVerifyPlatforms,
 	testRunValidExitCodes,
 	testFileOpSymlink,
+	testMetadataOnlyLocal,
 }
 
 func TestIntegration(t *testing.T) {
@@ -8452,6 +8455,150 @@ func testParallelLocalBuilds(t *testing.T, sb integration.Sandbox) {
 	require.NoError(t, err)
 }
 
+func testMetadataOnlyLocal(t *testing.T, sb integration.Sandbox) {
+	ctx, cancel := context.WithCancelCause(sb.Context())
+	defer func() { cancel(errors.WithStack(context.Canceled)) }()
+
+	c, err := New(ctx, sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	srcDir := integration.Tmpdir(
+		t,
+		fstest.CreateFile("data", []byte("contents"), 0600),
+		fstest.CreateDir("dir", 0700),
+		fstest.CreateFile("dir/file1", []byte("file1"), 0600),
+		fstest.CreateFile("dir/file2", []byte("file2"), 0600),
+		fstest.CreateDir("dir/subdir", 0700),
+		fstest.CreateDir("dir/subdir2", 0700),
+		fstest.CreateFile("dir/subdir/bar1", []byte("bar1"), 0600),
+		fstest.CreateFile("dir/subdir/bar2", []byte("bar2"), 0600),
+		fstest.CreateFile("dir/subdir2/bar3", []byte("bar3"), 0600),
+		fstest.CreateFile("foo", []byte("foo"), 0602),
+	)
+
+	def, err := llb.Local("source", llb.MetadataOnlyTransfer([]string{"dir/**/*1", "foo"})).Marshal(sb.Context())
+	require.NoError(t, err)
+
+	destDir := t.TempDir()
+
+	_, err = c.Solve(ctx, def, SolveOpt{
+		Exports: []ExportEntry{
+			{
+				Type:      ExporterLocal,
+				OutputDir: destDir,
+			},
+		},
+		LocalMounts: map[string]fsutil.FS{
+			"source": srcDir,
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	_, err = os.ReadFile(filepath.Join(destDir, "data"))
+	require.Error(t, err)
+	require.True(t, os.IsNotExist(err))
+
+	act, err := os.ReadFile(filepath.Join(destDir, "dir/file1"))
+	require.NoError(t, err)
+	require.Equal(t, "file1", string(act))
+
+	act, err = os.ReadFile(filepath.Join(destDir, "foo"))
+	require.NoError(t, err)
+	require.Equal(t, "foo", string(act))
+
+	_, err = os.ReadFile(filepath.Join(destDir, "dir/file2"))
+	require.Error(t, err)
+	require.True(t, os.IsNotExist(err))
+
+	act, err = os.ReadFile(filepath.Join(destDir, "dir/subdir/bar1"))
+	require.NoError(t, err)
+	require.Equal(t, "bar1", string(act))
+
+	_, err = os.Stat(filepath.Join(destDir, "dir/subdir2"))
+	require.Error(t, err)
+	require.True(t, os.IsNotExist(err))
+
+	_, err = os.ReadFile(filepath.Join(destDir, "dir/subdir/bar2"))
+	require.Error(t, err)
+	require.True(t, os.IsNotExist(err))
+
+	dt, err := os.ReadFile(filepath.Join(destDir, ".fsutil-metadata"))
+	require.NoError(t, err)
+
+	stats := parseFSMetadata(t, dt)
+	require.Equal(t, 10, len(stats))
+
+	require.Equal(t, "data", stats[0].Path)
+	require.Equal(t, "dir", stats[1].Path)
+	require.Equal(t, "dir/file1", stats[2].Path)
+	require.Equal(t, "dir/file2", stats[3].Path)
+	require.Equal(t, "dir/subdir", stats[4].Path)
+	require.Equal(t, "dir/subdir/bar1", stats[5].Path)
+	require.Equal(t, "dir/subdir/bar2", stats[6].Path)
+	require.Equal(t, "dir/subdir2", stats[7].Path)
+	require.Equal(t, "dir/subdir2/bar3", stats[8].Path)
+	require.Equal(t, "foo", stats[9].Path)
+
+	err = os.RemoveAll(filepath.Join(srcDir.Name, "dir/subdir"))
+	require.NoError(t, err)
+
+	err = os.WriteFile(filepath.Join(srcDir.Name, "dir/file1"), []byte("file1-updated"), 0600)
+	require.NoError(t, err)
+
+	err = os.WriteFile(filepath.Join(srcDir.Name, "dir/bar1"), []byte("bar1"), 0600)
+	require.NoError(t, err)
+
+	def, err = llb.Local("source", llb.MetadataOnlyTransfer([]string{"dir/**/*1", "foo"})).Marshal(sb.Context())
+	require.NoError(t, err)
+
+	destDir = t.TempDir()
+
+	_, err = c.Solve(ctx, def, SolveOpt{
+		Exports: []ExportEntry{
+			{
+				Type:      ExporterLocal,
+				OutputDir: destDir,
+			},
+		},
+		LocalMounts: map[string]fsutil.FS{
+			"source": srcDir,
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	_, err = os.ReadFile(filepath.Join(destDir, "data"))
+	require.Error(t, err)
+	require.True(t, os.IsNotExist(err))
+
+	act, err = os.ReadFile(filepath.Join(destDir, "dir/file1"))
+	require.NoError(t, err)
+	require.Equal(t, "file1-updated", string(act))
+
+	act, err = os.ReadFile(filepath.Join(destDir, "dir/bar1"))
+	require.NoError(t, err)
+	require.Equal(t, "bar1", string(act))
+
+	_, err = os.Stat(filepath.Join(destDir, "dir/subdir"))
+	require.Error(t, err)
+	require.True(t, os.IsNotExist(err))
+
+	dt, err = os.ReadFile(filepath.Join(destDir, ".fsutil-metadata"))
+	require.NoError(t, err)
+
+	stats = parseFSMetadata(t, dt)
+	require.Equal(t, 8, len(stats))
+
+	require.Equal(t, "data", stats[0].Path)
+	require.Equal(t, "dir", stats[1].Path)
+	require.Equal(t, "dir/bar1", stats[2].Path)
+	require.Equal(t, "dir/file1", stats[3].Path)
+	require.Equal(t, "dir/file2", stats[4].Path)
+	require.Equal(t, "dir/subdir2", stats[5].Path)
+	require.Equal(t, "dir/subdir2/bar3", stats[6].Path)
+	require.Equal(t, "foo", stats[7].Path)
+}
+
 // testRelativeMountpoint is a test that relative paths for mountpoints don't
 // fail when runc is upgraded to at least rc95, which introduces an error when
 // mountpoints are not absolute. Relative paths should be transformed to
@@ -11662,3 +11809,17 @@ devices:
 	require.NotContains(t, strings.TrimSpace(string(dt)), `BAZ=injected`)
 	require.NotContains(t, strings.TrimSpace(string(dt)), `QUX=injected`)
 }
+
+func parseFSMetadata(t *testing.T, dt []byte) []fsutiltypes.Stat {
+	var m []fsutiltypes.Stat
+	for len(dt) > 0 {
+		var s fsutiltypes.Stat
+		n := binary.LittleEndian.Uint32(dt[:4])
+		dt = dt[4:]
+		err := s.Unmarshal(dt[:n])
+		require.NoError(t, err)
+		m = append(m, *s.CloneVT())
+		dt = dt[n:]
+	}
+	return m
+}

client/llb/source.go

@@ -435,6 +435,13 @@ func Local(name string, opts ...LocalOption) State {
 			addCap(&gi.Constraints, pb.CapSourceLocalDiffer)
 		}
 	}
+	if gi.MetadataOnlyCollector {
+		attrs[pb.AttrMetadataTransfer] = "true"
+		if gi.MetadataOnlyExceptions != "" {
+			attrs[pb.AttrMetadataTransferExclude] = gi.MetadataOnlyExceptions
+		}
+		addCap(&gi.Constraints, pb.CapSourceMetadataTransfer)
+	}
 
 	addCap(&gi.Constraints, pb.CapSourceLocal)
 
@@ -506,6 +513,18 @@ func Differ(t DiffType, required bool) LocalOption {
 	})
 }
 
+func MetadataOnlyTransfer(exceptions []string) LocalOption {
+	return localOptionFunc(func(li *LocalInfo) {
+		li.MetadataOnlyCollector = true
+		if len(exceptions) == 0 {
+			li.MetadataOnlyExceptions = ""
+		} else {
+			dt, _ := json.Marshal(exceptions) // empty on error
+			li.MetadataOnlyExceptions = string(dt)
+		}
+	})
+}
+
 func OCILayout(ref string, opts ...OCILayoutOption) State {
 	gi := &OCILayoutInfo{}
 
@@ -578,12 +597,14 @@ type DifferInfo struct {
 
 type LocalInfo struct {
 	constraintsWrapper
-	SessionID       string
-	IncludePatterns string
-	ExcludePatterns string
-	FollowPaths     string
-	SharedKeyHint   string
-	Differ          DifferInfo
+	SessionID              string
+	IncludePatterns        string
+	ExcludePatterns        string
+	FollowPaths            string
+	SharedKeyHint          string
+	Differ                 DifferInfo
+	MetadataOnlyCollector  bool
+	MetadataOnlyExceptions string
 }
 
 func HTTP(url string, opts ...HTTPOption) State {

go.mod

@@ -73,7 +73,7 @@ require (
 	github.com/spdx/tools-golang v0.5.3
 	github.com/stretchr/testify v1.10.0
 	github.com/tonistiigi/dchapes-mode v0.0.0-20250318174251-73d941a28323
-	github.com/tonistiigi/fsutil v0.0.0-20250318190121-d73a4b3b8a7e
+	github.com/tonistiigi/fsutil v0.0.0-20250410151801-5b74a7ad7583
 	github.com/tonistiigi/go-actions-cache v0.0.0-20250228231703-3e9a6642607f
 	github.com/tonistiigi/go-archvariant v1.0.0
 	github.com/tonistiigi/go-csvvalue v0.0.0-20240710180619-ddb21b71c0b4

go.sum

@@ -397,8 +397,8 @@ github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtse
 github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/tonistiigi/dchapes-mode v0.0.0-20250318174251-73d941a28323 h1:r0p7fK56l8WPequOaR3i9LBqfPtEdXIQbUTzT55iqT4=
 github.com/tonistiigi/dchapes-mode v0.0.0-20250318174251-73d941a28323/go.mod h1:3Iuxbr0P7D3zUzBMAZB+ois3h/et0shEz0qApgHYGpY=
-github.com/tonistiigi/fsutil v0.0.0-20250318190121-d73a4b3b8a7e h1:AiXT0JHwQA52AEOVMsxRytSI9mdJSie5gUp6OQ1R8fU=
-github.com/tonistiigi/fsutil v0.0.0-20250318190121-d73a4b3b8a7e/go.mod h1:BKdcez7BiVtBvIcef90ZPc6ebqIWr4JWD7+EvLm6J98=
+github.com/tonistiigi/fsutil v0.0.0-20250410151801-5b74a7ad7583 h1:mK+ZskNt7SG4dxfKIi27C7qHAQzyjAVt1iyTf0hmsNc=
+github.com/tonistiigi/fsutil v0.0.0-20250410151801-5b74a7ad7583/go.mod h1:BKdcez7BiVtBvIcef90ZPc6ebqIWr4JWD7+EvLm6J98=
 github.com/tonistiigi/go-actions-cache v0.0.0-20250228231703-3e9a6642607f h1:q/SWz3Bz0KtAsqaBo73CHVXjaz5O8PDnmD2JHVhgYnE=
 github.com/tonistiigi/go-actions-cache v0.0.0-20250228231703-3e9a6642607f/go.mod h1:h0oRlVs3NoFIHysRQ4rU1+RG4QmU0M2JVSwTYrB4igk=
 github.com/tonistiigi/go-archvariant v1.0.0 h1:5LC1eDWiBNflnTF1prCiX09yfNHIxDC/aukdhCdTyb0=

session/filesync/diffcopy.go

@@ -83,7 +83,7 @@ func (wc *streamWriterCloser) Close() error {
 	return nil
 }
 
-func recvDiffCopy(ds grpc.ClientStream, dest string, cu CacheUpdater, progress progressCb, differ fsutil.DiffType, filter func(string, *fstypes.Stat) bool) (err error) {
+func recvDiffCopy(ds grpc.ClientStream, dest string, cu CacheUpdater, progress progressCb, differ fsutil.DiffType, filter, metadataOnlyFilter func(string, *fstypes.Stat) bool) (err error) {
 	st := time.Now()
 	defer func() {
 		bklog.G(ds.Context()).Debugf("diffcopy took: %v", time.Since(st))
@@ -107,6 +107,7 @@ func recvDiffCopy(ds grpc.ClientStream, dest string, cu CacheUpdater, progress p
 		ProgressCb:    progress,
 		Filter:        fsutil.FilterFunc(filter),
 		Differ:        differ,
+		MetadataOnly:  metadataOnlyFilter,
 	}))
 }
 

session/filesync/filesync.go

@@ -3,7 +3,7 @@ package filesync
 import (
 	"context"
 	"fmt"
-	io "io"
+	"io"
 	"net/url"
 	"os"
 	"strconv"
@@ -145,7 +145,7 @@ type progressCb func(int, bool)
 type protocol struct {
 	name   string
 	sendFn func(stream Stream, fs fsutil.FS, progress progressCb) error
-	recvFn func(stream grpc.ClientStream, destDir string, cu CacheUpdater, progress progressCb, differ fsutil.DiffType, mapFunc func(string, *fstypes.Stat) bool) error
+	recvFn func(stream grpc.ClientStream, destDir string, cu CacheUpdater, progress progressCb, differ fsutil.DiffType, mapFunc, metadataOnlyFilter func(string, *fstypes.Stat) bool) error
 }
 
 var supportedProtocols = []protocol{
@@ -158,15 +158,17 @@ var supportedProtocols = []protocol{
 
 // FSSendRequestOpt defines options for FSSend request
 type FSSendRequestOpt struct {
-	Name            string
-	IncludePatterns []string
-	ExcludePatterns []string
-	FollowPaths     []string
-	DestDir         string
-	CacheUpdater    CacheUpdater
-	ProgressCb      func(int, bool)
-	Filter          func(string, *fstypes.Stat) bool
-	Differ          fsutil.DiffType
+	Name               string
+	IncludePatterns    []string
+	ExcludePatterns    []string
+	FollowPaths        []string
+	DestDir            string
+	CacheUpdater       CacheUpdater
+	ProgressCb         func(int, bool)
+	Filter             func(string, *fstypes.Stat) bool
+	Differ             fsutil.DiffType
+	MetadataOnly       bool
+	MetadataOnlyFilter func(string, *fstypes.Stat) bool
 }
 
 // CacheUpdater is an object capable of sending notifications for the cache hash changes
@@ -233,7 +235,16 @@ func FSSync(ctx context.Context, c session.Caller, opt FSSendRequestOpt) error {
 		panic(fmt.Sprintf("invalid protocol: %q", pr.name))
 	}
 
-	return pr.recvFn(stream, opt.DestDir, opt.CacheUpdater, opt.ProgressCb, opt.Differ, opt.Filter)
+	var metadataOnlyFilter func(string, *fstypes.Stat) bool
+	if opt.MetadataOnly {
+		if opt.MetadataOnlyFilter != nil {
+			metadataOnlyFilter = opt.MetadataOnlyFilter
+		} else {
+			metadataOnlyFilter = func(string, *fstypes.Stat) bool { return false }
+		}
+	}
+
+	return pr.recvFn(stream, opt.DestDir, opt.CacheUpdater, opt.ProgressCb, opt.Differ, opt.Filter, metadataOnlyFilter)
 }
 
 type FSSyncTarget interface {

solver/pb/attr.go

@@ -12,6 +12,8 @@ const AttrIncludePatterns = "local.includepattern"
 const AttrFollowPaths = "local.followpaths"
 const AttrExcludePatterns = "local.excludepatterns"
 const AttrSharedKeyHint = "local.sharedkeyhint"
+const AttrMetadataTransfer = "local.metadatatransfer"
+const AttrMetadataTransferExclude = "local.metadatatransferexclude"
 
 const AttrLLBDefinitionFilename = "llbbuild.filename"
 

solver/pb/caps.go

@@ -21,6 +21,7 @@ const (
 	CapSourceLocalExcludePatterns apicaps.CapID = "source.local.excludepatterns"
 	CapSourceLocalSharedKeyHint   apicaps.CapID = "source.local.sharedkeyhint"
 	CapSourceLocalDiffer          apicaps.CapID = "source.local.differ"
+	CapSourceMetadataTransfer     apicaps.CapID = "source.local.metadatatransfer"
 
 	CapSourceGit              apicaps.CapID = "source.git"
 	CapSourceGitKeepDir       apicaps.CapID = "source.git.keepgitdir"
@@ -172,6 +173,12 @@ func init() {
 		Status:  apicaps.CapStatusExperimental,
 	})
 
+	Caps.Init(apicaps.Cap{
+		ID:      CapSourceMetadataTransfer,
+		Enabled: true,
+		Status:  apicaps.CapStatusExperimental,
+	})
+
 	Caps.Init(apicaps.Cap{
 		ID:      CapSourceGit,
 		Enabled: true,

source/local/identifier.go

@@ -9,13 +9,15 @@ import (
 )
 
 type LocalIdentifier struct {
-	Name            string
-	SessionID       string
-	IncludePatterns []string
-	ExcludePatterns []string
-	FollowPaths     []string
-	SharedKeyHint   string
-	Differ          fsutil.DiffType
+	Name               string
+	SessionID          string
+	IncludePatterns    []string
+	ExcludePatterns    []string
+	FollowPaths        []string
+	SharedKeyHint      string
+	Differ             fsutil.DiffType
+	MetadataOnly       bool
+	MetadataExceptions []string
 }
 
 func NewLocalIdentifier(str string) (*LocalIdentifier, error) {