Skip to content

Commit

Permalink
Add LFS data mirroring
Browse files Browse the repository at this point in the history
  • Loading branch information
@bl00mber
bl00mber committed Feb 18, 2021
1 parent 0a9a484 commit 36582e7
Show file tree
Hide file tree
Showing 14 changed files with 136 additions and 25 deletions.
1 change: 1 addition & 0 deletions modules/forms/repo_form.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ type MigrateRepoForm struct {
// required: true
RepoName string `json:"repo_name" binding:"Required;AlphaDashDot;MaxSize(100)"`
Mirror bool `json:"mirror"`
LFS bool `json:"lfs"`
Private bool `json:"private"`
Description string `json:"description" binding:"MaxSize(255)"`
Wiki bool `json:"wiki"`
Expand Down
1 change: 1 addition & 0 deletions modules/migrations/base/options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ type MigrateOptions struct {
// required: true
RepoName string `json:"repo_name" binding:"Required"`
Mirror bool `json:"mirror"`
LFS bool `json:"lfs"`
Private bool `json:"private"`
Description string `json:"description"`
OriginalURL string
Expand Down
1 change: 1 addition & 0 deletions modules/migrations/gitea_uploader.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,7 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate
OriginalURL: repo.OriginalURL,
GitServiceType: opts.GitServiceType,
Mirror: repo.IsMirror,
LFS: opts.LFS,
CloneAddr: repo.CloneURL,
Private: repo.IsPrivate,
Wiki: opts.Wiki,
Expand Down
71 changes: 71 additions & 0 deletions modules/repository/repo.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,9 @@ package repository
import (
"context"
"fmt"
"os"
"path"
"path/filepath"
"strings"
"time"

Expand Down Expand Up @@ -70,6 +72,75 @@ func MigrateRepositoryGitData(ctx context.Context, u *models.User, repo *models.
return repo, fmt.Errorf("Clone: %v", err)
}

if opts.LFS {
_, err = git.NewCommand("lfs", "fetch", opts.CloneAddr).RunInDir(repoPath)
if err != nil {
return repo, fmt.Errorf("LFS fetch failed %s: %v", opts.CloneAddr, err)
}

lfsSrc := path.Join(repoPath, "lfs", "objects")
lfsDst := path.Join(setting.LFS.Path)

// move LFS files
err := filepath.Walk(lfsSrc, func(path string, info os.FileInfo, err error) error {
var relSrcPath = strings.Replace(path, lfsSrc, "", 1)
if relSrcPath == "" {
return nil
}
if err != nil {
return err
}
lfsSrcFull := filepath.Join(lfsSrc, relSrcPath)
lfsDstFull := filepath.Join(lfsDst, relSrcPath)

if _, err := os.Stat(lfsDstFull); !os.IsNotExist(err) {
return nil
}

if info.IsDir() {
return os.Mkdir(lfsDstFull, 0755)
}

// generate and associate LFS OIDs
file, err := os.Open(lfsSrcFull)
if err != nil {
return err
}
defer file.Close()

oid, err := models.GenerateLFSOid(file)
if err != nil {
return err
}
fileInfo, err := file.Stat()
if err != nil {
return err
}

lfsDstFull = filepath.Join(lfsDst, oid[0:2], oid[2:4], oid[4:])
err = os.Rename(lfsSrcFull, lfsDstFull)
if err != nil {
return err
}

_, err = models.NewLFSMetaObject(&models.LFSMetaObject{Oid: oid, Size: fileInfo.Size(), RepositoryID: repo.ID})
if err != nil {
log.Error("Unable to write LFS OID[%s] size %d meta object in %v/%v to database. Error: %v", oid, fileInfo.Size(), u.Name, repoPath, err)
return err
}

return nil
})
if err != nil {
return repo, fmt.Errorf("Failed to move LFS files %s: %v", lfsSrc, err)
}

err = os.RemoveAll(path.Join(repoPath, "lfs"))
if err != nil {
return repo, fmt.Errorf("Failed to remove LFS files %s: %v", repoPath, err)
}
}

if opts.Wiki {
wikiPath := models.WikiPath(u.Name, opts.RepoName)
wikiRemotePath := WikiRemoteURL(opts.CloneAddr)
Expand Down
1 change: 1 addition & 0 deletions modules/structs/repo.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -253,6 +253,7 @@ type MigrateRepoOptions struct {
AuthToken string `json:"auth_token"`

Mirror bool `json:"mirror"`
LFS bool `json:"lfs"`
Private bool `json:"private"`
Description string `json:"description" binding:"MaxSize(255)"`
Wiki bool `json:"wiki"`
Expand Down
4 changes: 2 additions & 2 deletions options/locale/locale_en-US.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -754,6 +754,7 @@ need_auth = Clone Authorization
migrate_options = Migration Options
migrate_service = Migration Service
migrate_options_mirror_helper = This repository will be a <span class="text blue">mirror</span>
migrate_options_mirror_lfs = Mirror LFS data
migrate_options_mirror_disabled = Your site administrator has disabled new mirrors.
migrate_items = Migration Items
migrate_items_wiki = Wiki
Expand All @@ -770,7 +771,6 @@ migrate.clone_local_path = or a local server path
migrate.permission_denied = You are not allowed to import local repositories.
migrate.invalid_local_path = "The local path is invalid. It does not exist or is not a directory."
migrate.failed = Migration failed: %v
migrate.lfs_mirror_unsupported = Mirroring LFS objects is not supported - use 'git lfs fetch --all' and 'git lfs push --all' instead.
migrate.migrate_items_options = Access Token is required to migrate additional items
migrated_from = Migrated from <a href="%[1]s">%[2]s</a>
migrated_from_fake = Migrated From %[1]s
Expand Down Expand Up @@ -931,7 +931,7 @@ ext_issues = Ext. Issues
ext_issues.desc = Link to an external issue tracker.
projects = Projects
projects.desc = Manage issues and pulls in project boards.
projects.desc = Manage issues and pulls in project boards.
projects.description = Description (optional)
projects.description_placeholder = Description
projects.create = Create Project
Expand Down
1 change: 1 addition & 0 deletions routers/api/v1/repo/migrate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,7 @@ func Migrate(ctx *context.APIContext) {
Description: form.Description,
Private: form.Private || setting.Repository.ForcePrivate,
Mirror: form.Mirror,
LFS: form.LFS,
AuthUsername: form.AuthUsername,
AuthPassword: form.AuthPassword,
AuthToken: form.AuthToken,
Expand Down
2 changes: 2 additions & 0 deletions routers/repo/migrate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ func Migrate(ctx *context.Context) {
ctx.Data["IsForcedPrivate"] = setting.Repository.ForcePrivate
ctx.Data["DisableMirrors"] = setting.Repository.DisableMirrors
ctx.Data["mirror"] = ctx.Query("mirror") == "1"
ctx.Data["LFS"] = ctx.Query("lfs") == "1"
ctx.Data["wiki"] = ctx.Query("wiki") == "1"
ctx.Data["milestones"] = ctx.Query("milestones") == "1"
ctx.Data["labels"] = ctx.Query("labels") == "1"
Expand Down Expand Up @@ -172,6 +173,7 @@ func MigratePost(ctx *context.Context) {
Description: form.Description,
Private: form.Private || setting.Repository.ForcePrivate,
Mirror: form.Mirror && !setting.Repository.DisableMirrors,
LFS: form.LFS,
AuthUsername: form.AuthUsername,
AuthPassword: form.AuthPassword,
AuthToken: form.AuthToken,
Expand Down
15 changes: 10 additions & 5 deletions templates/repo/migrate/git.tmpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
<input id="clone_addr" name="clone_addr" value="{{.clone_addr}}" autofocus required>
<span class="help">
{{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}}
{{if .LFSActive}}<br/>{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}}
</span>
</div>
<div class="inline field {{if .Err_Auth}}error{{end}}">
Expand All @@ -30,15 +29,21 @@

<div class="inline field">
<label>{{.i18n.Tr "repo.migrate_options"}}</label>
{{if .DisableMirrors}}
<div class="ui checkbox">
{{if .DisableMirrors}}
<input id="mirror" name="mirror" type="checkbox" readonly>
<label>{{.i18n.Tr "repo.migrate_options_mirror_disabled"}}</label>
{{else}}
<input id="mirror" name="mirror" type="checkbox" {{if .mirror}}checked{{end}}>
</div>
{{else}}
<div class="ui checkbox">
<input id="mirror" name="mirror" type="checkbox" {{if .mirror}} checked{{end}}>
<label>{{.i18n.Tr "repo.migrate_options_mirror_helper" | Safe}}</label>
{{end}}
</div>
<div class="ui checkbox">
<input id="lfs" name="lfs" type="checkbox" {{if .lfs}} checked{{end}}>
<label>{{.i18n.Tr "repo.migrate_options_mirror_lfs" | Safe}}</label>
</div>
{{end}}
</div>

<div class="ui divider"></div>
Expand Down
13 changes: 9 additions & 4 deletions templates/repo/migrate/gitea.tmpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
<input id="clone_addr" name="clone_addr" value="{{.clone_addr}}" autofocus required>
<span class="help">
{{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}}
{{if .LFSActive}}<br />{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}}
</span>
</div>

Expand All @@ -27,15 +26,21 @@

<div class="inline field">
<label>{{.i18n.Tr "repo.migrate_options"}}</label>
{{if .DisableMirrors}}
<div class="ui checkbox">
{{if .DisableMirrors}}
<input id="mirror" name="mirror" type="checkbox" readonly>
<label>{{.i18n.Tr "repo.migrate_options_mirror_disabled"}}</label>
{{else}}
</div>
{{else}}
<div class="ui checkbox">
<input id="mirror" name="mirror" type="checkbox" {{if .mirror}} checked{{end}}>
<label>{{.i18n.Tr "repo.migrate_options_mirror_helper" | Safe}}</label>
{{end}}
</div>
<div class="ui checkbox">
<input id="lfs" name="lfs" type="checkbox" {{if .lfs}} checked{{end}}>
<label>{{.i18n.Tr "repo.migrate_options_mirror_lfs" | Safe}}</label>
</div>
{{end}}
</div>

<span class="help">{{.i18n.Tr "repo.migrate.migrate_items_options"}}</span>
Expand Down
15 changes: 10 additions & 5 deletions templates/repo/migrate/github.tmpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
<input id="clone_addr" name="clone_addr" value="{{.clone_addr}}" autofocus required>
<span class="help">
{{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}}
{{if .LFSActive}}<br/>{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}}
</span>
</div>

Expand All @@ -27,15 +26,21 @@

<div class="inline field">
<label>{{.i18n.Tr "repo.migrate_options"}}</label>
{{if .DisableMirrors}}
<div class="ui checkbox">
{{if .DisableMirrors}}
<input id="mirror" name="mirror" type="checkbox" readonly>
<label>{{.i18n.Tr "repo.migrate_options_mirror_disabled"}}</label>
{{else}}
<input id="mirror" name="mirror" type="checkbox" {{if .mirror}}checked{{end}}>
</div>
{{else}}
<div class="ui checkbox">
<input id="mirror" name="mirror" type="checkbox" {{if .mirror}} checked{{end}}>
<label>{{.i18n.Tr "repo.migrate_options_mirror_helper" | Safe}}</label>
{{end}}
</div>
<div class="ui checkbox">
<input id="lfs" name="lfs" type="checkbox" {{if .lfs}} checked{{end}}>
<label>{{.i18n.Tr "repo.migrate_options_mirror_lfs" | Safe}}</label>
</div>
{{end}}
</div>

<span class="help">{{.i18n.Tr "repo.migrate.migrate_items_options"}}</span>
Expand Down
15 changes: 10 additions & 5 deletions templates/repo/migrate/gitlab.tmpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
<input id="clone_addr" name="clone_addr" value="{{.clone_addr}}" autofocus required>
<span class="help">
{{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}}
{{if .LFSActive}}<br/>{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}}
</span>
</div>

Expand All @@ -27,15 +26,21 @@

<div class="inline field">
<label>{{.i18n.Tr "repo.migrate_options"}}</label>
{{if .DisableMirrors}}
<div class="ui checkbox">
{{if .DisableMirrors}}
<input id="mirror" name="mirror" type="checkbox" readonly>
<label>{{.i18n.Tr "repo.migrate_options_mirror_disabled"}}</label>
{{else}}
<input id="mirror" name="mirror" type="checkbox" {{if .mirror}}checked{{end}}>
</div>
{{else}}
<div class="ui checkbox">
<input id="mirror" name="mirror" type="checkbox" {{if .mirror}} checked{{end}}>
<label>{{.i18n.Tr "repo.migrate_options_mirror_helper" | Safe}}</label>
{{end}}
</div>
<div class="ui checkbox">
<input id="lfs" name="lfs" type="checkbox" {{if .lfs}} checked{{end}}>
<label>{{.i18n.Tr "repo.migrate_options_mirror_lfs" | Safe}}</label>
</div>
{{end}}
</div>

<span class="help">{{.i18n.Tr "repo.migrate.migrate_items_options"}}</span>
Expand Down
13 changes: 9 additions & 4 deletions templates/repo/migrate/gogs.tmpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
<input id="clone_addr" name="clone_addr" value="{{.clone_addr}}" autofocus required>
<span class="help">
{{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}}
{{if .LFSActive}}<br />{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}}
</span>
</div>

Expand All @@ -27,15 +26,21 @@

<div class="inline field">
<label>{{.i18n.Tr "repo.migrate_options"}}</label>
{{if .DisableMirrors}}
<div class="ui checkbox">
{{if .DisableMirrors}}
<input id="mirror" name="mirror" type="checkbox" readonly>
<label>{{.i18n.Tr "repo.migrate_options_mirror_disabled"}}</label>
{{else}}
</div>
{{else}}
<div class="ui checkbox">
<input id="mirror" name="mirror" type="checkbox" {{if .mirror}} checked{{end}}>
<label>{{.i18n.Tr "repo.migrate_options_mirror_helper" | Safe}}</label>
{{end}}
</div>
<div class="ui checkbox">
<input id="lfs" name="lfs" type="checkbox" {{if .lfs}} checked{{end}}>
<label>{{.i18n.Tr "repo.migrate_options_mirror_lfs" | Safe}}</label>
</div>
{{end}}
</div>

<span class="help">{{.i18n.Tr "repo.migrate.migrate_items_options"}}</span>
Expand Down
8 changes: 8 additions & 0 deletions templates/swagger/v1_json.tmpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14636,6 +14636,10 @@
"type": "boolean",
"x-go-name": "Labels"
},
"lfs": {
"type": "boolean",
"x-go-name": "LFS"
},
"milestones": {
"type": "boolean",
"x-go-name": "Milestones"
Expand Down Expand Up @@ -14715,6 +14719,10 @@
"type": "boolean",
"x-go-name": "Labels"
},
"lfs": {
"type": "boolean",
"x-go-name": "LFS"
},
"milestones": {
"type": "boolean",
"x-go-name": "Milestones"
Expand Down

0 comments on commit 36582e7

Please sign in to comment.