Skip to content

GitBlobstore: remote-managed fetch/merge/push sync #10458

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
coffeegoddd merged 11 commits into from
Feb 10, 2026
Merged

GitBlobstore: remote-managed fetch/merge/push sync #10458

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
a8b2fdd
/go/store/blobstore: add git fetch push and merge primitives
coffeegoddd Feb 9, 2026
77724ee
/go/store/blobstore: remote managed sync integration
coffeegoddd Feb 9, 2026
4acca03
/go/store/blobstore/git_blobstore_test.go: tests
coffeegoddd Feb 9, 2026
425a63d
/go/store/blobstore: some trimming
coffeegoddd Feb 9, 2026
9212865
/go/store/blobstore: comment reduction
coffeegoddd Feb 9, 2026
e971d39
/go/store/blobstore: fix chunk width
coffeegoddd Feb 10, 2026
3bc5c6d
/go/store/blobstore: reuse temp index file
coffeegoddd Feb 10, 2026
0b9a547
/go/store/blobstore: address feedback
coffeegoddd Feb 10, 2026
b0406ea
/go/store/blobstore: remove merge logic, force fetch push with lease
coffeegoddd Feb 10, 2026
dc5cea6
/go/store/{blobstore,nbs}: remove remote managed gate
coffeegoddd Feb 10, 2026
adfab24
/go/store/blobstore: three-ref model
coffeegoddd Feb 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
386 changes: 202 additions & 184 deletions go/store/blobstore/git_blobstore.go
View file
Open in desktop

Large diffs are not rendered by default.

29 changes: 24 additions & 5 deletions go/store/blobstore/git_blobstore_chunked_checkandput_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,17 +23,27 @@ import (

"github.com/stretchr/testify/require"

git "github.com/dolthub/dolt/go/store/blobstore/internal/git"
"github.com/dolthub/dolt/go/store/testutils/gitrepo"
)

func TestGitBlobstore_CheckAndPut_ChunkedRoundTrip_CreateOnly(t *testing.T) {
requireGitOnPath(t)

ctx := context.Background()
repo, err := gitrepo.InitBare(ctx, t.TempDir()+"/repo.git")
remoteRepo, err := gitrepo.InitBare(ctx, t.TempDir()+"/remote.git")
require.NoError(t, err)
_, err = remoteRepo.SetRefToTree(ctx, DoltDataRef, nil, "seed empty")
require.NoError(t, err)

localRepo, err := gitrepo.InitBare(ctx, t.TempDir()+"/local.git")
require.NoError(t, err)
localRunner, err := git.NewRunner(localRepo.GitDir)
require.NoError(t, err)
_, err = localRunner.Run(ctx, git.RunOptions{}, "remote", "add", "origin", remoteRepo.GitDir)
require.NoError(t, err)

bs, err := NewGitBlobstoreWithOptions(repo.GitDir, DoltDataRef, GitBlobstoreOptions{
bs, err := NewGitBlobstoreWithOptions(localRepo.GitDir, DoltDataRef, GitBlobstoreOptions{
Identity: testIdentity(),
MaxPartSize: 3,
})
Expand All @@ -60,16 +70,25 @@ func TestGitBlobstore_CheckAndPut_MismatchDoesNotConsumeReader_WithChunkingEnabl
requireGitOnPath(t)

ctx := context.Background()
repo, err := gitrepo.InitBare(ctx, t.TempDir()+"/repo.git")
remoteRepo, err := gitrepo.InitBare(ctx, t.TempDir()+"/remote.git")
require.NoError(t, err)
_, err = remoteRepo.SetRefToTree(ctx, DoltDataRef, nil, "seed empty")
require.NoError(t, err)

localRepo, err := gitrepo.InitBare(ctx, t.TempDir()+"/local.git")
require.NoError(t, err)
localRunner, err := git.NewRunner(localRepo.GitDir)
require.NoError(t, err)
_, err = localRunner.Run(ctx, git.RunOptions{}, "remote", "add", "origin", remoteRepo.GitDir)
require.NoError(t, err)

// Seed any commit so actualVersion != "".
bs0, err := NewGitBlobstoreWithOptions(repo.GitDir, DoltDataRef, GitBlobstoreOptions{Identity: testIdentity()})
bs0, err := NewGitBlobstoreWithOptions(localRepo.GitDir, DoltDataRef, GitBlobstoreOptions{Identity: testIdentity()})
require.NoError(t, err)
_, err = bs0.Put(ctx, "x", 1, bytes.NewReader([]byte("x")))
require.NoError(t, err)

bs, err := NewGitBlobstoreWithOptions(repo.GitDir, DoltDataRef, GitBlobstoreOptions{
bs, err := NewGitBlobstoreWithOptions(localRepo.GitDir, DoltDataRef, GitBlobstoreOptions{
Identity: testIdentity(),
MaxPartSize: 3,
})
Expand Down
30 changes: 22 additions & 8 deletions go/store/blobstore/git_blobstore_chunked_get_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,24 +31,31 @@ func TestGitBlobstore_Get_ChunkedTree_AllAndRanges(t *testing.T) {
}

ctx := context.Background()
repo, err := gitrepo.InitBare(ctx, t.TempDir()+"/repo.git")
remoteRepo, err := gitrepo.InitBare(ctx, t.TempDir()+"/remote.git")
require.NoError(t, err)

part1 := []byte("abc")
part2 := []byte("defgh")
commitOID, err := repo.SetRefToTree(ctx, DoltDataRef, map[string][]byte{
commitOID, err := remoteRepo.SetRefToTree(ctx, DoltDataRef, map[string][]byte{
"chunked/0001": part1,
"chunked/0002": part2,
}, "seed chunked tree")
require.NoError(t, err)

runner, err := git.NewRunner(repo.GitDir)
remoteRunner, err := git.NewRunner(remoteRepo.GitDir)
require.NoError(t, err)
api := git.NewGitAPIImpl(runner)
api := git.NewGitAPIImpl(remoteRunner)
treeOID, _, err := api.ResolvePathObject(ctx, git.OID(commitOID), "chunked")
require.NoError(t, err)

bs, err := NewGitBlobstore(repo.GitDir, DoltDataRef)
localRepo, err := gitrepo.InitBare(ctx, t.TempDir()+"/local.git")
require.NoError(t, err)
localRunner, err := git.NewRunner(localRepo.GitDir)
require.NoError(t, err)
_, err = localRunner.Run(ctx, git.RunOptions{}, "remote", "add", "origin", remoteRepo.GitDir)
require.NoError(t, err)

bs, err := NewGitBlobstore(localRepo.GitDir, DoltDataRef)
require.NoError(t, err)

wantAll := append(append([]byte(nil), part1...), part2...)
Expand Down Expand Up @@ -82,17 +89,24 @@ func TestGitBlobstore_Get_ChunkedTree_InvalidPartsError(t *testing.T) {
requireGitOnPath(t)

ctx := context.Background()
repo, err := gitrepo.InitBare(ctx, t.TempDir()+"/repo.git")
remoteRepo, err := gitrepo.InitBare(ctx, t.TempDir()+"/remote.git")
require.NoError(t, err)

// Gap: 0001, 0003
_, err = repo.SetRefToTree(ctx, DoltDataRef, map[string][]byte{
_, err = remoteRepo.SetRefToTree(ctx, DoltDataRef, map[string][]byte{
"chunked/0001": []byte("a"),
"chunked/0003": []byte("b"),
}, "seed invalid chunked tree")
require.NoError(t, err)

bs, err := NewGitBlobstore(repo.GitDir, DoltDataRef)
localRepo, err := gitrepo.InitBare(ctx, t.TempDir()+"/local.git")
require.NoError(t, err)
localRunner, err := git.NewRunner(localRepo.GitDir)
require.NoError(t, err)
_, err = localRunner.Run(ctx, git.RunOptions{}, "remote", "add", "origin", remoteRepo.GitDir)
require.NoError(t, err)

bs, err := NewGitBlobstore(localRepo.GitDir, DoltDataRef)
require.NoError(t, err)

_, _, err = GetBytes(ctx, bs, "chunked", AllRange)
Expand Down
34 changes: 26 additions & 8 deletions go/store/blobstore/git_blobstore_chunked_put_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,19 @@ func TestGitBlobstore_Put_ChunkedWritesTreeParts(t *testing.T) {
requireGitOnPath(t)

ctx := context.Background()
repo, err := gitrepo.InitBare(ctx, t.TempDir()+"/repo.git")
remoteRepo, err := gitrepo.InitBare(ctx, t.TempDir()+"/remote.git")
require.NoError(t, err)
_, err = remoteRepo.SetRefToTree(ctx, DoltDataRef, nil, "seed empty")
require.NoError(t, err)

localRepo, err := gitrepo.InitBare(ctx, t.TempDir()+"/local.git")
require.NoError(t, err)
localRunner, err := git.NewRunner(localRepo.GitDir)
require.NoError(t, err)
_, err = localRunner.Run(ctx, git.RunOptions{}, "remote", "add", "origin", remoteRepo.GitDir)
require.NoError(t, err)

bs, err := NewGitBlobstoreWithOptions(repo.GitDir, DoltDataRef, GitBlobstoreOptions{
bs, err := NewGitBlobstoreWithOptions(localRepo.GitDir, DoltDataRef, GitBlobstoreOptions{
Identity: testIdentity(),
MaxPartSize: 3,
})
Expand All @@ -47,7 +56,7 @@ func TestGitBlobstore_Put_ChunkedWritesTreeParts(t *testing.T) {
require.Equal(t, ver, ver2)
require.Equal(t, want, got)

runner, err := git.NewRunner(repo.GitDir)
runner, err := git.NewRunner(remoteRepo.GitDir)
require.NoError(t, err)
api := git.NewGitAPIImpl(runner)

Expand All @@ -62,24 +71,33 @@ func TestGitBlobstore_Put_ChunkedWritesTreeParts(t *testing.T) {
entries, err := api.ListTree(ctx, head, "big")
require.NoError(t, err)
require.Len(t, entries, 4)
require.Equal(t, "00000001", entries[0].Name)
require.Equal(t, "00000004", entries[3].Name)
require.Equal(t, "0001", entries[0].Name)
require.Equal(t, "0004", entries[3].Name)
}

func TestGitBlobstore_Put_IdempotentDoesNotChangeExistingRepresentation(t *testing.T) {
requireGitOnPath(t)

ctx := context.Background()
repo, err := gitrepo.InitBare(ctx, t.TempDir()+"/repo.git")
remoteRepo, err := gitrepo.InitBare(ctx, t.TempDir()+"/remote.git")
require.NoError(t, err)
_, err = remoteRepo.SetRefToTree(ctx, DoltDataRef, nil, "seed empty")
require.NoError(t, err)

localRepo, err := gitrepo.InitBare(ctx, t.TempDir()+"/local.git")
require.NoError(t, err)
localRunner, err := git.NewRunner(localRepo.GitDir)
require.NoError(t, err)
_, err = localRunner.Run(ctx, git.RunOptions{}, "remote", "add", "origin", remoteRepo.GitDir)
require.NoError(t, err)

bs, err := NewGitBlobstoreWithOptions(repo.GitDir, DoltDataRef, GitBlobstoreOptions{
bs, err := NewGitBlobstoreWithOptions(localRepo.GitDir, DoltDataRef, GitBlobstoreOptions{
Identity: testIdentity(),
MaxPartSize: 3,
})
require.NoError(t, err)

runner, err := git.NewRunner(repo.GitDir)
runner, err := git.NewRunner(remoteRepo.GitDir)
require.NoError(t, err)
api := git.NewGitAPIImpl(runner)

Expand Down
22 changes: 18 additions & 4 deletions go/store/blobstore/git_blobstore_helpers_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ type fakeGitAPI struct {
listTree func(ctx context.Context, commit git.OID, treePath string) ([]git.TreeEntry, error)
blobSize func(ctx context.Context, oid git.OID) (int64, error)
blobReader func(ctx context.Context, oid git.OID) (io.ReadCloser, error)
fetchRef func(ctx context.Context, remote string, srcRef string, dstRef string) error
pushRefWithLease func(ctx context.Context, remote string, srcRef string, dstRef string, expectedDstOID git.OID) error
}

func (f fakeGitAPI) TryResolveRefCommit(ctx context.Context, ref string) (git.OID, bool, error) {
Expand Down Expand Up @@ -86,6 +88,18 @@ func (f fakeGitAPI) UpdateRefCAS(ctx context.Context, ref string, newOID git.OID
func (f fakeGitAPI) UpdateRef(ctx context.Context, ref string, newOID git.OID, msg string) error {
panic("unexpected call")
}
func (f fakeGitAPI) FetchRef(ctx context.Context, remote string, srcRef string, dstRef string) error {
if f.fetchRef == nil {
panic("unexpected call")
}
return f.fetchRef(ctx, remote, srcRef, dstRef)
}
func (f fakeGitAPI) PushRefWithLease(ctx context.Context, remote string, srcRef string, dstRef string, expectedDstOID git.OID) error {
if f.pushRefWithLease == nil {
panic("unexpected call")
}
return f.pushRefWithLease(ctx, remote, srcRef, dstRef, expectedDstOID)
}

func TestGitBlobstoreHelpers_resolveCommitForGet(t *testing.T) {
ctx := context.Background()
Expand All @@ -97,7 +111,7 @@ func TestGitBlobstoreHelpers_resolveCommitForGet(t *testing.T) {
return git.OID("0123456789abcdef0123456789abcdef01234567"), true, nil
},
}
gbs := &GitBlobstore{ref: DoltDataRef, api: api}
gbs := &GitBlobstore{localRef: DoltDataRef, api: api}

commit, err := gbs.resolveCommitForGet(ctx, "k")
require.NoError(t, err)
Expand All @@ -110,7 +124,7 @@ func TestGitBlobstoreHelpers_resolveCommitForGet(t *testing.T) {
return git.OID(""), false, nil
},
}
gbs := &GitBlobstore{ref: DoltDataRef, api: api}
gbs := &GitBlobstore{localRef: DoltDataRef, api: api}

_, err := gbs.resolveCommitForGet(ctx, "manifest")
var nf NotFound
Expand All @@ -124,7 +138,7 @@ func TestGitBlobstoreHelpers_resolveCommitForGet(t *testing.T) {
return git.OID(""), false, nil
},
}
gbs := &GitBlobstore{ref: DoltDataRef, api: api}
gbs := &GitBlobstore{localRef: DoltDataRef, api: api}

_, err := gbs.resolveCommitForGet(ctx, "somekey")
var rnf *git.RefNotFoundError
Expand All @@ -139,7 +153,7 @@ func TestGitBlobstoreHelpers_resolveCommitForGet(t *testing.T) {
return git.OID(""), false, sentinel
},
}
gbs := &GitBlobstore{ref: DoltDataRef, api: api}
gbs := &GitBlobstore{localRef: DoltDataRef, api: api}

_, err := gbs.resolveCommitForGet(ctx, "k")
require.ErrorIs(t, err, sentinel)
Expand Down
Loading
Loading