From c8259c61abbfae9f070491135c2cdda7089a182c Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 30 Jan 2026 12:18:33 -0800 Subject: [PATCH 01/10] Fix bug when do LFS GC --- models/git/lfs.go | 27 +++++---------- models/git/lfs_test.go | 61 +++++++++++++++++++++++++++++++++ services/repository/lfs.go | 6 ++-- services/repository/lfs_test.go | 27 +++++++++++++++ 4 files changed, 99 insertions(+), 22 deletions(-) create mode 100644 models/git/lfs_test.go diff --git a/models/git/lfs.go b/models/git/lfs.go index a4ae3e7beebe8..1f3d086ac2588 100644 --- a/models/git/lfs.go +++ b/models/git/lfs.go @@ -312,15 +312,12 @@ func IterateRepositoryIDsWithLFSMetaObjects(ctx context.Context, f func(ctx cont // IterateLFSMetaObjectsForRepoOptions provides options for IterateLFSMetaObjectsForRepo type IterateLFSMetaObjectsForRepoOptions struct { - OlderThan timeutil.TimeStamp - UpdatedLessRecentlyThan timeutil.TimeStamp - OrderByUpdated bool - LoopFunctionAlwaysUpdates bool + OlderThan timeutil.TimeStamp + UpdatedLessRecentlyThan timeutil.TimeStamp } // IterateLFSMetaObjectsForRepo provides a iterator for LFSMetaObjects per Repo func IterateLFSMetaObjectsForRepo(ctx context.Context, repoID int64, f func(context.Context, *LFSMetaObject, int64) error, opts *IterateLFSMetaObjectsForRepoOptions) error { - var start int batchSize := setting.Database.IterateBufferSize engine := db.GetEngine(ctx) type CountLFSMetaObject struct { @@ -328,7 +325,7 @@ func IterateLFSMetaObjectsForRepo(ctx context.Context, repoID int64, f func(cont LFSMetaObject `xorm:"extends"` } - id := int64(0) + lastID := int64(0) for { beans := make([]*CountLFSMetaObject, 0, batchSize) @@ -341,29 +338,23 @@ func IterateLFSMetaObjectsForRepo(ctx context.Context, repoID int64, f func(cont if !opts.UpdatedLessRecentlyThan.IsZero() { sess.And("`lfs_meta_object`.updated_unix < ?", opts.UpdatedLessRecentlyThan) } - sess.GroupBy("`lfs_meta_object`.id") - if opts.OrderByUpdated { - sess.OrderBy("`lfs_meta_object`.updated_unix ASC") - } else { - sess.And("`lfs_meta_object`.id > ?", id) - sess.OrderBy("`lfs_meta_object`.id ASC") - } - if err := sess.Limit(batchSize, start).Find(&beans); err != nil { + sess.GroupBy("`lfs_meta_object`.id"). + And("`lfs_meta_object`.id > ?", lastID). + OrderBy("`lfs_meta_object`.id ASC") + + if err := sess.Limit(batchSize).Find(&beans); err != nil { return err } if len(beans) == 0 { return nil } - if !opts.LoopFunctionAlwaysUpdates { - start += len(beans) - } for _, bean := range beans { if err := f(ctx, &bean.LFSMetaObject, bean.Count); err != nil { return err } } - id = beans[len(beans)-1].ID + lastID = beans[len(beans)-1].ID } } diff --git a/models/git/lfs_test.go b/models/git/lfs_test.go new file mode 100644 index 0000000000000..f8f5b19262952 --- /dev/null +++ b/models/git/lfs_test.go @@ -0,0 +1,61 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package git_test + +import ( + "bytes" + "context" + "strconv" + "testing" + "time" + + "code.gitea.io/gitea/models/db" + git_model "code.gitea.io/gitea/models/git" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unittest" + "code.gitea.io/gitea/modules/lfs" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" + "code.gitea.io/gitea/modules/timeutil" + + "github.com/stretchr/testify/assert" +) + +func TestIterateLFSMetaObjectsForRepoUpdatesDoNotSkip(t *testing.T) { + unittest.PrepareTestEnv(t) + + ctx := t.Context() + repo, err := repo_model.GetRepositoryByOwnerAndName(ctx, "user2", "repo1") + assert.NoError(t, err) + + test.MockVariableValue(&setting.Database.IterateBufferSize, 1) + + created := make([]*git_model.LFSMetaObject, 0, 3) + for i := 0; i < 3; i++ { + content := []byte("gitea-lfs-" + strconv.Itoa(i)) + pointer, err := lfs.GeneratePointer(bytes.NewReader(content)) + assert.NoError(t, err) + + meta, err := git_model.NewLFSMetaObject(ctx, repo.ID, pointer) + assert.NoError(t, err) + created = append(created, meta) + } + + iterated := make([]int64, 0, len(created)) + cutoff := time.Now().Add(24 * time.Hour) + iterErr := git_model.IterateLFSMetaObjectsForRepo(ctx, repo.ID, func(ctx context.Context, meta *git_model.LFSMetaObject, count int64) error { + iterated = append(iterated, meta.ID) + _, err := db.GetEngine(ctx).ID(meta.ID).Cols("updated_unix").Update(&git_model.LFSMetaObject{ + UpdatedUnix: timeutil.TimeStamp(time.Now().Unix()), + }) + return err + }, &git_model.IterateLFSMetaObjectsForRepoOptions{ + OlderThan: timeutil.TimeStamp(cutoff.Unix()), + UpdatedLessRecentlyThan: timeutil.TimeStamp(cutoff.Unix()), + }) + assert.NoError(t, iterErr) + + expected := []int64{created[0].ID, created[1].ID, created[2].ID} + assert.Equal(t, expected, iterated) +} diff --git a/services/repository/lfs.go b/services/repository/lfs.go index 4d48881b87f63..5ef2dbdac44b9 100644 --- a/services/repository/lfs.go +++ b/services/repository/lfs.go @@ -123,10 +123,8 @@ func GarbageCollectLFSMetaObjectsForRepo(ctx context.Context, repo *repo_model.R // // It is likely that a week is potentially excessive but it should definitely be enough that any // unassociated LFS object is genuinely unassociated. - OlderThan: timeutil.TimeStamp(opts.OlderThan.Unix()), - UpdatedLessRecentlyThan: timeutil.TimeStamp(opts.UpdatedLessRecentlyThan.Unix()), - OrderByUpdated: true, - LoopFunctionAlwaysUpdates: true, + OlderThan: timeutil.TimeStamp(opts.OlderThan.Unix()), + UpdatedLessRecentlyThan: timeutil.TimeStamp(opts.UpdatedLessRecentlyThan.Unix()), }) if err == errStop { diff --git a/services/repository/lfs_test.go b/services/repository/lfs_test.go index 7fb202f42d68b..4bcc99e3bc8d6 100644 --- a/services/repository/lfs_test.go +++ b/services/repository/lfs_test.go @@ -14,6 +14,7 @@ import ( "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" + "code.gitea.io/gitea/modules/test" repo_service "code.gitea.io/gitea/services/repository" "github.com/stretchr/testify/assert" @@ -46,6 +47,32 @@ func TestGarbageCollectLFSMetaObjects(t *testing.T) { assert.ErrorIs(t, err, git_model.ErrLFSObjectNotExist) } +func TestGarbageCollectLFSMetaObjectsForRepoAutoFix(t *testing.T) { + unittest.PrepareTestEnv(t) + + test.MockVariableValue(&setting.LFS.StartServer, true) + + err := storage.Init() + assert.NoError(t, err) + + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) + + // add lfs object + lfsContent := []byte("gitea2") + lfsOid := storeObjectInRepo(t, repo.ID, &lfsContent) + + err = repo_service.GarbageCollectLFSMetaObjectsForRepo(t.Context(), repo, repo_service.GarbageCollectLFSMetaObjectsOptions{ + LogDetail: func(string, ...any) {}, + AutoFix: true, + OlderThan: time.Now().Add(24 * time.Hour * 7), + UpdatedLessRecentlyThan: time.Now().Add(24 * time.Hour * 3), + }) + assert.NoError(t, err) + + _, err = git_model.GetLFSMetaObjectByOid(t.Context(), repo.ID, lfsOid) + assert.Equal(t, git_model.ErrLFSObjectNotExist, err) +} + func storeObjectInRepo(t *testing.T, repositoryID int64, content *[]byte) string { pointer, err := lfs.GeneratePointer(bytes.NewReader(*content)) assert.NoError(t, err) From 2be938ce18aa7d716c0719573ca9c40a084e2725 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 30 Jan 2026 12:22:33 -0800 Subject: [PATCH 02/10] correct copyright year --- models/git/lfs_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/git/lfs_test.go b/models/git/lfs_test.go index f8f5b19262952..0ab4da9427c87 100644 --- a/models/git/lfs_test.go +++ b/models/git/lfs_test.go @@ -1,4 +1,4 @@ -// Copyright 2024 The Gitea Authors. All rights reserved. +// Copyright 2026 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT package git_test From 6bf7e47c521db7bbc5141204ad2eb5a4b9e7b4c5 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 30 Jan 2026 13:02:26 -0800 Subject: [PATCH 03/10] Fix test --- models/git/lfs_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/git/lfs_test.go b/models/git/lfs_test.go index 0ab4da9427c87..b3066bd6ea553 100644 --- a/models/git/lfs_test.go +++ b/models/git/lfs_test.go @@ -32,7 +32,7 @@ func TestIterateLFSMetaObjectsForRepoUpdatesDoNotSkip(t *testing.T) { test.MockVariableValue(&setting.Database.IterateBufferSize, 1) created := make([]*git_model.LFSMetaObject, 0, 3) - for i := 0; i < 3; i++ { + for i := range 3 { content := []byte("gitea-lfs-" + strconv.Itoa(i)) pointer, err := lfs.GeneratePointer(bytes.NewReader(content)) assert.NoError(t, err) From 1bb4b9ae2ebffad828c2dc71740b4b62f22f8b88 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 30 Jan 2026 16:37:34 -0800 Subject: [PATCH 04/10] Follow reviews --- models/git/lfs_test.go | 2 +- services/repository/lfs_test.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/models/git/lfs_test.go b/models/git/lfs_test.go index b3066bd6ea553..a59e702e50edd 100644 --- a/models/git/lfs_test.go +++ b/models/git/lfs_test.go @@ -23,7 +23,7 @@ import ( ) func TestIterateLFSMetaObjectsForRepoUpdatesDoNotSkip(t *testing.T) { - unittest.PrepareTestEnv(t) + assert.NoError(t, unittest.PrepareTestDatabase()) ctx := t.Context() repo, err := repo_model.GetRepositoryByOwnerAndName(ctx, "user2", "repo1") diff --git a/services/repository/lfs_test.go b/services/repository/lfs_test.go index 4bcc99e3bc8d6..23ce64c5e61ce 100644 --- a/services/repository/lfs_test.go +++ b/services/repository/lfs_test.go @@ -23,7 +23,8 @@ import ( func TestGarbageCollectLFSMetaObjects(t *testing.T) { unittest.PrepareTestEnv(t) - setting.LFS.StartServer = true + test.MockVariableValue(&setting.LFS.StartServer, true) + err := storage.Init() assert.NoError(t, err) From e6ad8bfe7488909db0acf07f5fea3be10b83e26e Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 31 Jan 2026 10:37:01 -0800 Subject: [PATCH 05/10] improvement --- services/repository/lfs_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/services/repository/lfs_test.go b/services/repository/lfs_test.go index 23ce64c5e61ce..4348a0e6d8e11 100644 --- a/services/repository/lfs_test.go +++ b/services/repository/lfs_test.go @@ -14,7 +14,6 @@ import ( "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" - "code.gitea.io/gitea/modules/test" repo_service "code.gitea.io/gitea/services/repository" "github.com/stretchr/testify/assert" @@ -23,7 +22,7 @@ import ( func TestGarbageCollectLFSMetaObjects(t *testing.T) { unittest.PrepareTestEnv(t) - test.MockVariableValue(&setting.LFS.StartServer, true) + setting.LFS.StartServer = true err := storage.Init() assert.NoError(t, err) @@ -51,7 +50,7 @@ func TestGarbageCollectLFSMetaObjects(t *testing.T) { func TestGarbageCollectLFSMetaObjectsForRepoAutoFix(t *testing.T) { unittest.PrepareTestEnv(t) - test.MockVariableValue(&setting.LFS.StartServer, true) + setting.LFS.StartServer = true err := storage.Init() assert.NoError(t, err) From aaf9470f460e9e8ae2727df59d2e39d43d368e80 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 31 Jan 2026 10:37:50 -0800 Subject: [PATCH 06/10] remove unnecessary change --- services/repository/lfs_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/services/repository/lfs_test.go b/services/repository/lfs_test.go index 4348a0e6d8e11..8385cedc0751c 100644 --- a/services/repository/lfs_test.go +++ b/services/repository/lfs_test.go @@ -51,7 +51,6 @@ func TestGarbageCollectLFSMetaObjectsForRepoAutoFix(t *testing.T) { unittest.PrepareTestEnv(t) setting.LFS.StartServer = true - err := storage.Init() assert.NoError(t, err) From f4005746438e7f957f4080e6fdd3ac0437a6d762 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 31 Jan 2026 10:38:19 -0800 Subject: [PATCH 07/10] remove unnecessary change --- services/repository/lfs_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/services/repository/lfs_test.go b/services/repository/lfs_test.go index 8385cedc0751c..00c812f0d1ed2 100644 --- a/services/repository/lfs_test.go +++ b/services/repository/lfs_test.go @@ -23,7 +23,6 @@ func TestGarbageCollectLFSMetaObjects(t *testing.T) { unittest.PrepareTestEnv(t) setting.LFS.StartServer = true - err := storage.Init() assert.NoError(t, err) From 0d9d5f8f3cbc1b0e84d5f9ae0e840287fd895657 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 31 Jan 2026 22:11:43 -0800 Subject: [PATCH 08/10] Fix bug --- models/git/lfs_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/git/lfs_test.go b/models/git/lfs_test.go index a59e702e50edd..4c0242f439cf9 100644 --- a/models/git/lfs_test.go +++ b/models/git/lfs_test.go @@ -29,7 +29,7 @@ func TestIterateLFSMetaObjectsForRepoUpdatesDoNotSkip(t *testing.T) { repo, err := repo_model.GetRepositoryByOwnerAndName(ctx, "user2", "repo1") assert.NoError(t, err) - test.MockVariableValue(&setting.Database.IterateBufferSize, 1) + defer test.MockVariableValue(&setting.Database.IterateBufferSize, 1)() created := make([]*git_model.LFSMetaObject, 0, 3) for i := range 3 { From 0fd0be6a6d1a09f02d357d604e413be2298b4604 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 12 Feb 2026 09:43:06 -0800 Subject: [PATCH 09/10] Update services/repository/lfs_test.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Lunny Xiao --- services/repository/lfs_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/repository/lfs_test.go b/services/repository/lfs_test.go index 00c812f0d1ed2..273553885752f 100644 --- a/services/repository/lfs_test.go +++ b/services/repository/lfs_test.go @@ -68,7 +68,7 @@ func TestGarbageCollectLFSMetaObjectsForRepoAutoFix(t *testing.T) { assert.NoError(t, err) _, err = git_model.GetLFSMetaObjectByOid(t.Context(), repo.ID, lfsOid) - assert.Equal(t, git_model.ErrLFSObjectNotExist, err) + assert.ErrorIs(t, err, git_model.ErrLFSObjectNotExist) } func storeObjectInRepo(t *testing.T, repositoryID int64, content *[]byte) string { From d50a7402b786b3c5ba3df22b00c47c3ff22f690b Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 12 Feb 2026 11:30:16 -0800 Subject: [PATCH 10/10] Fix test --- services/repository/lfs_test.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/services/repository/lfs_test.go b/services/repository/lfs_test.go index 00c812f0d1ed2..9637cefdbde95 100644 --- a/services/repository/lfs_test.go +++ b/services/repository/lfs_test.go @@ -14,6 +14,7 @@ import ( "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" + "code.gitea.io/gitea/modules/test" repo_service "code.gitea.io/gitea/services/repository" "github.com/stretchr/testify/assert" @@ -22,7 +23,8 @@ import ( func TestGarbageCollectLFSMetaObjects(t *testing.T) { unittest.PrepareTestEnv(t) - setting.LFS.StartServer = true + defer test.MockVariableValue(&setting.LFS.StartServer, true)() + err := storage.Init() assert.NoError(t, err) @@ -49,7 +51,8 @@ func TestGarbageCollectLFSMetaObjects(t *testing.T) { func TestGarbageCollectLFSMetaObjectsForRepoAutoFix(t *testing.T) { unittest.PrepareTestEnv(t) - setting.LFS.StartServer = true + defer test.MockVariableValue(&setting.LFS.StartServer, true)() + err := storage.Init() assert.NoError(t, err)