Sleep longer if request speed is over github limitation (#9335)

* Sleep longer if request speed is over github limitation

* improve code

* remove unused code

* fix lint

* Use github's rate limit remain value to determine how long to sleep

* Save reset time when finished github api request

* fix bug

* fix lint

* Add context.Context for sleep

* fix test

* improve code

* fix bug and lint

* fix import order

Author: lunny

modules/migrations/base/downloader.go

@@ -6,13 +6,15 @@
 package base
 
 import (
+	"context"
 	"time"
 
 	"code.gitea.io/gitea/modules/structs"
 )
 
 // Downloader downloads the site repo informations
 type Downloader interface {
+	SetContext(context.Context)
 	GetRepoInfo() (*Repository, error)
 	GetTopics() ([]string, error)
 	GetMilestones() ([]*Milestone, error)
@@ -30,6 +32,10 @@ type DownloaderFactory interface {
 	GitServiceType() structs.GitServiceType
 }
 
+var (
+	_ Downloader = &RetryDownloader{}
+)
+
 // RetryDownloader retry the downloads
 type RetryDownloader struct {
 	Downloader
@@ -46,6 +52,11 @@ func NewRetryDownloader(downloader Downloader, retryTimes, retryDelay int) *Retr
 	}
 }
 
+// SetContext set context
+func (d *RetryDownloader) SetContext(ctx context.Context) {
+	d.Downloader.SetContext(ctx)
+}
+
 // GetRepoInfo returns a repository information with retry
 func (d *RetryDownloader) GetRepoInfo() (*Repository, error) {
 	var (

modules/migrations/git.go

@@ -5,6 +5,8 @@
 package migrations
 
 import (
+	"context"
+
 	"code.gitea.io/gitea/modules/migrations/base"
 )
 
@@ -28,6 +30,10 @@ func NewPlainGitDownloader(ownerName, repoName, remoteURL string) *PlainGitDownl
 	}
 }
 
+// SetContext set context
+func (g *PlainGitDownloader) SetContext(ctx context.Context) {
+}
+
 // GetRepoInfo returns a repository information
 func (g *PlainGitDownloader) GetRepoInfo() (*base.Repository, error) {
 	// convert github repo to stand Repo

modules/migrations/gitea.go

@@ -6,6 +6,7 @@
 package migrations
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"net/http"
@@ -35,6 +36,7 @@ var (
 
 // GiteaLocalUploader implements an Uploader to gitea sites
 type GiteaLocalUploader struct {
+	ctx            context.Context
 	doer           *models.User
 	repoOwner      string
 	repoName       string
@@ -49,8 +51,9 @@ type GiteaLocalUploader struct {
 }
 
 // NewGiteaLocalUploader creates an gitea Uploader via gitea API v1
-func NewGiteaLocalUploader(doer *models.User, repoOwner, repoName string) *GiteaLocalUploader {
+func NewGiteaLocalUploader(ctx context.Context, doer *models.User, repoOwner, repoName string) *GiteaLocalUploader {
 	return &GiteaLocalUploader{
+		ctx:         ctx,
 		doer:        doer,
 		repoOwner:   repoOwner,
 		repoName:    repoName,

modules/migrations/gitea_test.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/graceful"
 	"code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/util"
 
@@ -27,7 +28,7 @@ func TestGiteaUploadRepo(t *testing.T) {
 	var (
 		downloader = NewGithubDownloaderV3("", "", "go-xorm", "builder")
 		repoName   = "builder-" + time.Now().Format("2006-01-02-15-04-05")
-		uploader   = NewGiteaLocalUploader(user, user.Name, repoName)
+		uploader   = NewGiteaLocalUploader(graceful.GetManager().HammerContext(), user, user.Name, repoName)
 	)
 
 	err := migrateRepository(downloader, uploader, structs.MigrateRepoOption{

modules/migrations/github.go

@@ -11,6 +11,7 @@ import (
 	"net/http"
 	"net/url"
 	"strings"
+	"time"
 
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/migrations/base"
@@ -73,6 +74,7 @@ type GithubDownloaderV3 struct {
 	repoName  string
 	userName  string
 	password  string
+	rate      *github.Rate
 }
 
 // NewGithubDownloaderV3 creates a github Downloader via github v3 API
@@ -107,12 +109,39 @@ func NewGithubDownloaderV3(userName, password, repoOwner, repoName string) *Gith
 	return &downloader
 }
 
+// SetContext set context
+func (g *GithubDownloaderV3) SetContext(ctx context.Context) {
+	g.ctx = ctx
+}
+
+func (g *GithubDownloaderV3) sleep() {
+	for g.rate != nil && g.rate.Remaining <= 0 {
+		timer := time.NewTimer(time.Until(g.rate.Reset.Time))
+		select {
+		case <-g.ctx.Done():
+			timer.Stop()
+			return
+		case <-timer.C:
+		}
+
+		rates, _, err := g.client.RateLimits(g.ctx)
+		if err != nil {
+			log.Error("g.client.RateLimits: %s", err)
+		}
+
+		g.rate = rates.GetCore()
+	}
+}
+
 // GetRepoInfo returns a repository information
 func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) {
-	gr, _, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName)
+	g.sleep()
+	gr, resp, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName)
 	if err != nil {
 		return nil, err
 	}
+	g.rate = &resp.Rate
+
 	// convert github repo to stand Repo
 	return &base.Repository{
 		Owner:       g.repoOwner,
@@ -126,16 +155,22 @@ func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) {
 
 // GetTopics return github topics
 func (g *GithubDownloaderV3) GetTopics() ([]string, error) {
-	r, _, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName)
-	return r.Topics, err
+	g.sleep()
+	r, resp, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName)
+	if err != nil {
+		return nil, err
+	}
+	g.rate = &resp.Rate
+	return r.Topics, nil
 }
 
 // GetMilestones returns milestones
 func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) {
 	var perPage = 100
 	var milestones = make([]*base.Milestone, 0, perPage)
 	for i := 1; ; i++ {
-		ms, _, err := g.client.Issues.ListMilestones(g.ctx, g.repoOwner, g.repoName,
+		g.sleep()
+		ms, resp, err := g.client.Issues.ListMilestones(g.ctx, g.repoOwner, g.repoName,
 			&github.MilestoneListOptions{
 				State: "all",
 				ListOptions: github.ListOptions{
@@ -145,6 +180,7 @@ func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) {
 		if err != nil {
 			return nil, err
 		}
+		g.rate = &resp.Rate
 
 		for _, m := range ms {
 			var desc string
@@ -189,14 +225,16 @@ func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) {
 	var perPage = 100
 	var labels = make([]*base.Label, 0, perPage)
 	for i := 1; ; i++ {
-		ls, _, err := g.client.Issues.ListLabels(g.ctx, g.repoOwner, g.repoName,
+		g.sleep()
+		ls, resp, err := g.client.Issues.ListLabels(g.ctx, g.repoOwner, g.repoName,
 			&github.ListOptions{
 				Page:    i,
 				PerPage: perPage,
 			})
 		if err != nil {
 			return nil, err
 		}
+		g.rate = &resp.Rate
 
 		for _, label := range ls {
 			labels = append(labels, convertGithubLabel(label))
@@ -260,14 +298,16 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) {
 	var perPage = 100
 	var releases = make([]*base.Release, 0, perPage)
 	for i := 1; ; i++ {
-		ls, _, err := g.client.Repositories.ListReleases(g.ctx, g.repoOwner, g.repoName,
+		g.sleep()
+		ls, resp, err := g.client.Repositories.ListReleases(g.ctx, g.repoOwner, g.repoName,
 			&github.ListOptions{
 				Page:    i,
 				PerPage: perPage,
 			})
 		if err != nil {
 			return nil, err
 		}
+		g.rate = &resp.Rate
 
 		for _, release := range ls {
 			releases = append(releases, g.convertGithubRelease(release))
@@ -304,11 +344,12 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool,
 	}
 
 	var allIssues = make([]*base.Issue, 0, perPage)
-
-	issues, _, err := g.client.Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt)
+	g.sleep()
+	issues, resp, err := g.client.Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt)
 	if err != nil {
 		return nil, false, fmt.Errorf("error while listing repos: %v", err)
 	}
+	g.rate = &resp.Rate
 	for _, issue := range issues {
 		if issue.IsPullRequest() {
 			continue
@@ -365,10 +406,12 @@ func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, er
 		},
 	}
 	for {
+		g.sleep()
 		comments, resp, err := g.client.Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(issueNumber), opt)
 		if err != nil {
 			return nil, fmt.Errorf("error while listing repos: %v", err)
 		}
+		g.rate = &resp.Rate
 		for _, comment := range comments {
 			var email string
 			if comment.User.Email != nil {
@@ -408,11 +451,12 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq
 		},
 	}
 	var allPRs = make([]*base.PullRequest, 0, perPage)
-
-	prs, _, err := g.client.PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt)
+	g.sleep()
+	prs, resp, err := g.client.PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt)
 	if err != nil {
 		return nil, fmt.Errorf("error while listing repos: %v", err)
 	}
+	g.rate = &resp.Rate
 	for _, pr := range prs {
 		var body string
 		if pr.Body != nil {

modules/migrations/migrate.go

@@ -6,6 +6,7 @@
 package migrations
 
 import (
+	"context"
 	"fmt"
 
 	"code.gitea.io/gitea/models"
@@ -28,10 +29,10 @@ func RegisterDownloaderFactory(factory base.DownloaderFactory) {
 }
 
 // MigrateRepository migrate repository according MigrateOptions
-func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOptions) (*models.Repository, error) {
+func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, opts base.MigrateOptions) (*models.Repository, error) {
 	var (
 		downloader base.Downloader
-		uploader   = NewGiteaLocalUploader(doer, ownerName, opts.RepoName)
+		uploader   = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName)
 		theFactory base.DownloaderFactory
 	)
 
@@ -69,6 +70,8 @@ func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOpt
 		downloader = base.NewRetryDownloader(downloader, setting.Migrations.MaxAttempts, setting.Migrations.RetryBackoff)
 	}
 
+	downloader.SetContext(ctx)
+
 	if err := migrateRepository(downloader, uploader, opts); err != nil {
 		if err1 := uploader.Rollback(); err1 != nil {
 			log.Error("rollback failed: %v", err1)

modules/task/migrate.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 
 	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/graceful"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/migrations"
 	"code.gitea.io/gitea/modules/notification"
@@ -95,7 +96,7 @@ func runMigrateTask(t *models.Task) (err error) {
 	}
 
 	opts.MigrateToRepoID = t.RepoID
-	repo, err := migrations.MigrateRepository(t.Doer, t.Owner.Name, *opts)
+	repo, err := migrations.MigrateRepository(graceful.GetManager().HammerContext(), t.Doer, t.Owner.Name, *opts)
 	if err == nil {
 		log.Trace("Repository migrated [%d]: %s/%s", repo.ID, t.Owner.Name, repo.Name)
 		return nil

routers/api/v1/repo/repo.go

@@ -18,6 +18,7 @@ import (
 	"code.gitea.io/gitea/modules/context"
 	"code.gitea.io/gitea/modules/convert"
 	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/graceful"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/migrations"
 	"code.gitea.io/gitea/modules/notification"
@@ -481,7 +482,7 @@ func Migrate(ctx *context.APIContext, form auth.MigrateRepoForm) {
 		}
 	}()
 
-	if _, err = migrations.MigrateRepository(ctx.User, ctxUser.Name, opts); err != nil {
+	if _, err = migrations.MigrateRepository(graceful.GetManager().HammerContext(), ctx.User, ctxUser.Name, opts); err != nil {
 		handleMigrateError(ctx, ctxUser, remoteAddr, err)
 		return
 	}