From 36582e77bd9abf0310861a308f0bb9699af446bc Mon Sep 17 00:00:00 2001 From: Nick Reiley Date: Thu, 18 Feb 2021 04:24:09 +0500 Subject: [PATCH] Add LFS data mirroring --- modules/forms/repo_form.go | 1 + modules/migrations/base/options.go | 1 + modules/migrations/gitea_uploader.go | 1 + modules/repository/repo.go | 71 ++++++++++++++++++++++++++++ modules/structs/repo.go | 1 + options/locale/locale_en-US.ini | 4 +- routers/api/v1/repo/migrate.go | 1 + routers/repo/migrate.go | 2 + templates/repo/migrate/git.tmpl | 15 ++++-- templates/repo/migrate/gitea.tmpl | 13 +++-- templates/repo/migrate/github.tmpl | 15 ++++-- templates/repo/migrate/gitlab.tmpl | 15 ++++-- templates/repo/migrate/gogs.tmpl | 13 +++-- templates/swagger/v1_json.tmpl | 8 ++++ 14 files changed, 136 insertions(+), 25 deletions(-) diff --git a/modules/forms/repo_form.go b/modules/forms/repo_form.go index 48af3450f371..d99bb835c2a1 100644 --- a/modules/forms/repo_form.go +++ b/modules/forms/repo_form.go @@ -73,6 +73,7 @@ type MigrateRepoForm struct { // required: true RepoName string `json:"repo_name" binding:"Required;AlphaDashDot;MaxSize(100)"` Mirror bool `json:"mirror"` + LFS bool `json:"lfs"` Private bool `json:"private"` Description string `json:"description" binding:"MaxSize(255)"` Wiki bool `json:"wiki"` diff --git a/modules/migrations/base/options.go b/modules/migrations/base/options.go index 168f9848c813..92b23ca51b43 100644 --- a/modules/migrations/base/options.go +++ b/modules/migrations/base/options.go @@ -20,6 +20,7 @@ type MigrateOptions struct { // required: true RepoName string `json:"repo_name" binding:"Required"` Mirror bool `json:"mirror"` + LFS bool `json:"lfs"` Private bool `json:"private"` Description string `json:"description"` OriginalURL string diff --git a/modules/migrations/gitea_uploader.go b/modules/migrations/gitea_uploader.go index 3be49b5c6ced..6d2e70994297 100644 --- a/modules/migrations/gitea_uploader.go +++ b/modules/migrations/gitea_uploader.go @@ -117,6 +117,7 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate OriginalURL: repo.OriginalURL, GitServiceType: opts.GitServiceType, Mirror: repo.IsMirror, + LFS: opts.LFS, CloneAddr: repo.CloneURL, Private: repo.IsPrivate, Wiki: opts.Wiki, diff --git a/modules/repository/repo.go b/modules/repository/repo.go index ede714673ab1..a1f7c996bf1b 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -7,7 +7,9 @@ package repository import ( "context" "fmt" + "os" "path" + "path/filepath" "strings" "time" @@ -70,6 +72,75 @@ func MigrateRepositoryGitData(ctx context.Context, u *models.User, repo *models. return repo, fmt.Errorf("Clone: %v", err) } + if opts.LFS { + _, err = git.NewCommand("lfs", "fetch", opts.CloneAddr).RunInDir(repoPath) + if err != nil { + return repo, fmt.Errorf("LFS fetch failed %s: %v", opts.CloneAddr, err) + } + + lfsSrc := path.Join(repoPath, "lfs", "objects") + lfsDst := path.Join(setting.LFS.Path) + + // move LFS files + err := filepath.Walk(lfsSrc, func(path string, info os.FileInfo, err error) error { + var relSrcPath = strings.Replace(path, lfsSrc, "", 1) + if relSrcPath == "" { + return nil + } + if err != nil { + return err + } + lfsSrcFull := filepath.Join(lfsSrc, relSrcPath) + lfsDstFull := filepath.Join(lfsDst, relSrcPath) + + if _, err := os.Stat(lfsDstFull); !os.IsNotExist(err) { + return nil + } + + if info.IsDir() { + return os.Mkdir(lfsDstFull, 0755) + } + + // generate and associate LFS OIDs + file, err := os.Open(lfsSrcFull) + if err != nil { + return err + } + defer file.Close() + + oid, err := models.GenerateLFSOid(file) + if err != nil { + return err + } + fileInfo, err := file.Stat() + if err != nil { + return err + } + + lfsDstFull = filepath.Join(lfsDst, oid[0:2], oid[2:4], oid[4:]) + err = os.Rename(lfsSrcFull, lfsDstFull) + if err != nil { + return err + } + + _, err = models.NewLFSMetaObject(&models.LFSMetaObject{Oid: oid, Size: fileInfo.Size(), RepositoryID: repo.ID}) + if err != nil { + log.Error("Unable to write LFS OID[%s] size %d meta object in %v/%v to database. Error: %v", oid, fileInfo.Size(), u.Name, repoPath, err) + return err + } + + return nil + }) + if err != nil { + return repo, fmt.Errorf("Failed to move LFS files %s: %v", lfsSrc, err) + } + + err = os.RemoveAll(path.Join(repoPath, "lfs")) + if err != nil { + return repo, fmt.Errorf("Failed to remove LFS files %s: %v", repoPath, err) + } + } + if opts.Wiki { wikiPath := models.WikiPath(u.Name, opts.RepoName) wikiRemotePath := WikiRemoteURL(opts.CloneAddr) diff --git a/modules/structs/repo.go b/modules/structs/repo.go index d588813b2188..5bd5314cb1a4 100644 --- a/modules/structs/repo.go +++ b/modules/structs/repo.go @@ -253,6 +253,7 @@ type MigrateRepoOptions struct { AuthToken string `json:"auth_token"` Mirror bool `json:"mirror"` + LFS bool `json:"lfs"` Private bool `json:"private"` Description string `json:"description" binding:"MaxSize(255)"` Wiki bool `json:"wiki"` diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 767696cfb901..e6df0c928864 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -754,6 +754,7 @@ need_auth = Clone Authorization migrate_options = Migration Options migrate_service = Migration Service migrate_options_mirror_helper = This repository will be a mirror +migrate_options_mirror_lfs = Mirror LFS data migrate_options_mirror_disabled = Your site administrator has disabled new mirrors. migrate_items = Migration Items migrate_items_wiki = Wiki @@ -770,7 +771,6 @@ migrate.clone_local_path = or a local server path migrate.permission_denied = You are not allowed to import local repositories. migrate.invalid_local_path = "The local path is invalid. It does not exist or is not a directory." migrate.failed = Migration failed: %v -migrate.lfs_mirror_unsupported = Mirroring LFS objects is not supported - use 'git lfs fetch --all' and 'git lfs push --all' instead. migrate.migrate_items_options = Access Token is required to migrate additional items migrated_from = Migrated from %[2]s migrated_from_fake = Migrated From %[1]s @@ -931,7 +931,7 @@ ext_issues = Ext. Issues ext_issues.desc = Link to an external issue tracker. projects = Projects -projects.desc = Manage issues and pulls in project boards. +projects.desc = Manage issues and pulls in project boards. projects.description = Description (optional) projects.description_placeholder = Description projects.create = Create Project diff --git a/routers/api/v1/repo/migrate.go b/routers/api/v1/repo/migrate.go index 61cd12b991cb..f4c5b0ee1b7b 100644 --- a/routers/api/v1/repo/migrate.go +++ b/routers/api/v1/repo/migrate.go @@ -134,6 +134,7 @@ func Migrate(ctx *context.APIContext) { Description: form.Description, Private: form.Private || setting.Repository.ForcePrivate, Mirror: form.Mirror, + LFS: form.LFS, AuthUsername: form.AuthUsername, AuthPassword: form.AuthPassword, AuthToken: form.AuthToken, diff --git a/routers/repo/migrate.go b/routers/repo/migrate.go index 89452de0fabf..15df1dfdee88 100644 --- a/routers/repo/migrate.go +++ b/routers/repo/migrate.go @@ -47,6 +47,7 @@ func Migrate(ctx *context.Context) { ctx.Data["IsForcedPrivate"] = setting.Repository.ForcePrivate ctx.Data["DisableMirrors"] = setting.Repository.DisableMirrors ctx.Data["mirror"] = ctx.Query("mirror") == "1" + ctx.Data["LFS"] = ctx.Query("lfs") == "1" ctx.Data["wiki"] = ctx.Query("wiki") == "1" ctx.Data["milestones"] = ctx.Query("milestones") == "1" ctx.Data["labels"] = ctx.Query("labels") == "1" @@ -172,6 +173,7 @@ func MigratePost(ctx *context.Context) { Description: form.Description, Private: form.Private || setting.Repository.ForcePrivate, Mirror: form.Mirror && !setting.Repository.DisableMirrors, + LFS: form.LFS, AuthUsername: form.AuthUsername, AuthPassword: form.AuthPassword, AuthToken: form.AuthToken, diff --git a/templates/repo/migrate/git.tmpl b/templates/repo/migrate/git.tmpl index 233a01943530..26d231147120 100644 --- a/templates/repo/migrate/git.tmpl +++ b/templates/repo/migrate/git.tmpl @@ -15,7 +15,6 @@ {{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}} - {{if .LFSActive}}
{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}}
@@ -30,15 +29,21 @@
+ {{if .DisableMirrors}}
- {{if .DisableMirrors}} - {{else}} - +
+ {{else}} +
+ - {{end}}
+
+ + +
+ {{end}}
diff --git a/templates/repo/migrate/gitea.tmpl b/templates/repo/migrate/gitea.tmpl index b21e6b18ffdf..b984e54cd69d 100644 --- a/templates/repo/migrate/gitea.tmpl +++ b/templates/repo/migrate/gitea.tmpl @@ -15,7 +15,6 @@ {{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}} - {{if .LFSActive}}
{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}}
@@ -27,15 +26,21 @@
+ {{if .DisableMirrors}}
- {{if .DisableMirrors}} - {{else}} +
+ {{else}} +
- {{end}}
+
+ + +
+ {{end}}
{{.i18n.Tr "repo.migrate.migrate_items_options"}} diff --git a/templates/repo/migrate/github.tmpl b/templates/repo/migrate/github.tmpl index 06f76d72980e..f0103f74a9c6 100644 --- a/templates/repo/migrate/github.tmpl +++ b/templates/repo/migrate/github.tmpl @@ -15,7 +15,6 @@ {{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}} - {{if .LFSActive}}
{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}} @@ -27,15 +26,21 @@
+ {{if .DisableMirrors}}
- {{if .DisableMirrors}} - {{else}} - +
+ {{else}} +
+ - {{end}}
+
+ + +
+ {{end}}
{{.i18n.Tr "repo.migrate.migrate_items_options"}} diff --git a/templates/repo/migrate/gitlab.tmpl b/templates/repo/migrate/gitlab.tmpl index 545a1ff43717..ff8f3637370d 100644 --- a/templates/repo/migrate/gitlab.tmpl +++ b/templates/repo/migrate/gitlab.tmpl @@ -15,7 +15,6 @@ {{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}} - {{if .LFSActive}}
{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}} @@ -27,15 +26,21 @@
+ {{if .DisableMirrors}}
- {{if .DisableMirrors}} - {{else}} - +
+ {{else}} +
+ - {{end}}
+
+ + +
+ {{end}}
{{.i18n.Tr "repo.migrate.migrate_items_options"}} diff --git a/templates/repo/migrate/gogs.tmpl b/templates/repo/migrate/gogs.tmpl index ac81872b9227..84156310aba1 100644 --- a/templates/repo/migrate/gogs.tmpl +++ b/templates/repo/migrate/gogs.tmpl @@ -15,7 +15,6 @@ {{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}} - {{if .LFSActive}}
{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}} @@ -27,15 +26,21 @@
+ {{if .DisableMirrors}}
- {{if .DisableMirrors}} - {{else}} +
+ {{else}} +
- {{end}}
+
+ + +
+ {{end}}
{{.i18n.Tr "repo.migrate.migrate_items_options"}} diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index 2dedb56d1ec2..21f3db90a5c1 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -14636,6 +14636,10 @@ "type": "boolean", "x-go-name": "Labels" }, + "lfs": { + "type": "boolean", + "x-go-name": "LFS" + }, "milestones": { "type": "boolean", "x-go-name": "Milestones" @@ -14715,6 +14719,10 @@ "type": "boolean", "x-go-name": "Labels" }, + "lfs": { + "type": "boolean", + "x-go-name": "LFS" + }, "milestones": { "type": "boolean", "x-go-name": "Milestones"