From 6bc317d975121ed5e6d9a7b32957e463af376a57 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 20 Oct 2019 12:52:47 +0100 Subject: [PATCH 01/19] make notifyWatchers work on multiple actions --- models/repo_watch.go | 90 ++++++++++++++++++++++---------------------- 1 file changed, 46 insertions(+), 44 deletions(-) diff --git a/models/repo_watch.go b/models/repo_watch.go index 2279dcb1156e..e65013325d03 100644 --- a/models/repo_watch.go +++ b/models/repo_watch.go @@ -164,68 +164,70 @@ func (repo *Repository) GetWatchers(page int) ([]*User, error) { return users, sess.Find(&users) } -func notifyWatchers(e Engine, act *Action) error { - // Add feeds for user self and all watchers. - watches, err := getWatchers(e, act.RepoID) - if err != nil { - return fmt.Errorf("get watchers: %v", err) - } - - // Add feed for actioner. - act.UserID = act.ActUserID - if _, err = e.InsertOne(act); err != nil { - return fmt.Errorf("insert new actioner: %v", err) - } - - act.loadRepo() - // check repo owner exist. - if err := act.Repo.getOwner(e); err != nil { - return fmt.Errorf("can't get repo owner: %v", err) - } +func notifyWatchers(e Engine, actions ...*Action) error { + for _, act := range actions { + // Add feeds for user self and all watchers. + watches, err := getWatchers(e, act.RepoID) + if err != nil { + return fmt.Errorf("get watchers: %v", err) + } - // Add feed for organization - if act.Repo.Owner.IsOrganization() && act.ActUserID != act.Repo.Owner.ID { - act.ID = 0 - act.UserID = act.Repo.Owner.ID + // Add feed for actioner. + act.UserID = act.ActUserID if _, err = e.InsertOne(act); err != nil { return fmt.Errorf("insert new actioner: %v", err) } - } - for i := range watches { - if act.ActUserID == watches[i].UserID { - continue + act.loadRepo() + // check repo owner exist. + if err := act.Repo.getOwner(e); err != nil { + return fmt.Errorf("can't get repo owner: %v", err) } - act.ID = 0 - act.UserID = watches[i].UserID - act.Repo.Units = nil - - switch act.OpType { - case ActionCommitRepo, ActionPushTag, ActionDeleteTag, ActionDeleteBranch: - if !act.Repo.checkUnitUser(e, act.UserID, false, UnitTypeCode) { - continue + // Add feed for organization + if act.Repo.Owner.IsOrganization() && act.ActUserID != act.Repo.Owner.ID { + act.ID = 0 + act.UserID = act.Repo.Owner.ID + if _, err = e.InsertOne(act); err != nil { + return fmt.Errorf("insert new actioner: %v", err) } - case ActionCreateIssue, ActionCommentIssue, ActionCloseIssue, ActionReopenIssue: - if !act.Repo.checkUnitUser(e, act.UserID, false, UnitTypeIssues) { + } + + for i := range watches { + if act.ActUserID == watches[i].UserID { continue } - case ActionCreatePullRequest, ActionMergePullRequest, ActionClosePullRequest, ActionReopenPullRequest: - if !act.Repo.checkUnitUser(e, act.UserID, false, UnitTypePullRequests) { - continue + + act.ID = 0 + act.UserID = watches[i].UserID + act.Repo.Units = nil + + switch act.OpType { + case ActionCommitRepo, ActionPushTag, ActionDeleteTag, ActionDeleteBranch: + if !act.Repo.checkUnitUser(e, act.UserID, false, UnitTypeCode) { + continue + } + case ActionCreateIssue, ActionCommentIssue, ActionCloseIssue, ActionReopenIssue: + if !act.Repo.checkUnitUser(e, act.UserID, false, UnitTypeIssues) { + continue + } + case ActionCreatePullRequest, ActionMergePullRequest, ActionClosePullRequest, ActionReopenPullRequest: + if !act.Repo.checkUnitUser(e, act.UserID, false, UnitTypePullRequests) { + continue + } } - } - if _, err = e.InsertOne(act); err != nil { - return fmt.Errorf("insert new action: %v", err) + if _, err = e.InsertOne(act); err != nil { + return fmt.Errorf("insert new action: %v", err) + } } } return nil } // NotifyWatchers creates batch of actions for every watcher. -func NotifyWatchers(act *Action) error { - return notifyWatchers(x, act) +func NotifyWatchers(actions ...*Action) error { + return notifyWatchers(x, actions...) } // NotifyWatchersActions creates batch of actions for every watcher. From 743498637967596bc9d8c0ac25d0d7c9086860fa Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 20 Oct 2019 13:56:43 +0100 Subject: [PATCH 02/19] more efficient multiple notifyWatchers --- models/repo_watch.go | 103 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 89 insertions(+), 14 deletions(-) diff --git a/models/repo_watch.go b/models/repo_watch.go index e65013325d03..cee32be7392d 100644 --- a/models/repo_watch.go +++ b/models/repo_watch.go @@ -165,11 +165,22 @@ func (repo *Repository) GetWatchers(page int) ([]*User, error) { } func notifyWatchers(e Engine, actions ...*Action) error { + var watchers []*Watch + var repo *Repository + var err error + var permCode []bool + var permIssue []bool + var permPR []bool + for _, act := range actions { - // Add feeds for user self and all watchers. - watches, err := getWatchers(e, act.RepoID) - if err != nil { - return fmt.Errorf("get watchers: %v", err) + repoChanged := repo == nil || repo.ID != act.RepoID + + if repoChanged { + // Add feeds for user self and all watchers. + watchers, err = getWatchers(e, act.RepoID) + if err != nil { + return fmt.Errorf("get watchers: %v", err) + } } // Add feed for actioner. @@ -178,10 +189,16 @@ func notifyWatchers(e Engine, actions ...*Action) error { return fmt.Errorf("insert new actioner: %v", err) } - act.loadRepo() - // check repo owner exist. - if err := act.Repo.getOwner(e); err != nil { - return fmt.Errorf("can't get repo owner: %v", err) + if repoChanged { + act.loadRepo() + repo = act.Repo + + // check repo owner exist. + if err := act.Repo.getOwner(e); err != nil { + return fmt.Errorf("can't get repo owner: %v", err) + } + } else if act.Repo == nil { + act.Repo = repo } // Add feed for organization @@ -193,26 +210,84 @@ func notifyWatchers(e Engine, actions ...*Action) error { } } - for i := range watches { - if act.ActUserID == watches[i].UserID { + if len(actions) == 1 { + // More efficient to just check the code and not cache + for _, watcher := range watchers { + if act.ActUserID == watcher.UserID { + continue + } + + act.ID = 0 + act.UserID = watcher.UserID + act.Repo.Units = nil + + switch act.OpType { + case ActionCommitRepo, ActionPushTag, ActionDeleteTag, ActionDeleteBranch: + if !act.Repo.checkUnitUser(e, act.UserID, false, UnitTypeCode) { + continue + } + case ActionCreateIssue, ActionCommentIssue, ActionCloseIssue, ActionReopenIssue: + if !act.Repo.checkUnitUser(e, act.UserID, false, UnitTypeIssues) { + continue + } + case ActionCreatePullRequest, ActionMergePullRequest, ActionClosePullRequest, ActionReopenPullRequest: + if !act.Repo.checkUnitUser(e, act.UserID, false, UnitTypePullRequests) { + continue + } + } + + if _, err = e.InsertOne(act); err != nil { + return fmt.Errorf("insert new action: %v", err) + } + } + return nil + } + + if repoChanged { + permCode = make([]bool, len(watchers)) + permIssue = make([]bool, len(watchers)) + permPR = make([]bool, len(watchers)) + for i, watcher := range watchers { + user, err := getUserByID(e, watcher.UserID) + if err != nil { + permCode[i] = false + permIssue[i] = false + permPR[i] = false + continue + } + perm, err := getUserRepoPermission(e, repo, user) + if err != nil { + permCode[i] = false + permIssue[i] = false + permPR[i] = false + continue + } + permCode[i] = perm.CanRead(UnitTypeCode) + permIssue[i] = perm.CanRead(UnitTypeIssues) + permPR[i] = perm.CanRead(UnitTypePullRequests) + } + } + + for i, watcher := range watchers { + if act.ActUserID == watcher.UserID { continue } act.ID = 0 - act.UserID = watches[i].UserID + act.UserID = watcher.UserID act.Repo.Units = nil switch act.OpType { case ActionCommitRepo, ActionPushTag, ActionDeleteTag, ActionDeleteBranch: - if !act.Repo.checkUnitUser(e, act.UserID, false, UnitTypeCode) { + if !permCode[i] { continue } case ActionCreateIssue, ActionCommentIssue, ActionCloseIssue, ActionReopenIssue: - if !act.Repo.checkUnitUser(e, act.UserID, false, UnitTypeIssues) { + if !permIssue[i] { continue } case ActionCreatePullRequest, ActionMergePullRequest, ActionClosePullRequest, ActionReopenPullRequest: - if !act.Repo.checkUnitUser(e, act.UserID, false, UnitTypePullRequests) { + if !permPR[i] { continue } } From 2ed8271a1ee0b35dbc185ed58ab63d50c7344300 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 3 Nov 2019 18:54:18 +0000 Subject: [PATCH 03/19] Make CommitRepoAction take advantage of multiple actions --- modules/repofiles/action.go | 190 ++++++++++++++++++++---------------- 1 file changed, 105 insertions(+), 85 deletions(-) diff --git a/modules/repofiles/action.go b/modules/repofiles/action.go index a5a5e151cbd2..a09a48df8318 100644 --- a/modules/repofiles/action.go +++ b/modules/repofiles/action.go @@ -159,112 +159,132 @@ type CommitRepoActionOptions struct { // CommitRepoAction adds new commit action to the repository, and prepare // corresponding webhooks. -func CommitRepoAction(opts CommitRepoActionOptions) error { - pusher, err := models.GetUserByName(opts.PusherName) - if err != nil { - return fmt.Errorf("GetUserByName [%s]: %v", opts.PusherName, err) - } - - repo, err := models.GetRepositoryByName(opts.RepoOwnerID, opts.RepoName) - if err != nil { - return fmt.Errorf("GetRepositoryByName [owner_id: %d, name: %s]: %v", opts.RepoOwnerID, opts.RepoName, err) - } - - refName := git.RefEndName(opts.RefFullName) - - // Change default branch and empty status only if pushed ref is non-empty branch. - if repo.IsEmpty && opts.NewCommitID != git.EmptySHA && strings.HasPrefix(opts.RefFullName, git.BranchPrefix) { - repo.DefaultBranch = refName - repo.IsEmpty = false - if refName != "master" { - gitRepo, err := git.OpenRepository(repo.RepoPath()) +func CommitRepoAction(optsList ...CommitRepoActionOptions) error { + var pusher *models.User + var repo *models.Repository + actions := make([]*models.Action, len(optsList)) + + for i, opts := range optsList { + if pusher == nil || pusher.Name != opts.PusherName { + var err error + pusher, err = models.GetUserByName(opts.PusherName) if err != nil { - return err + return fmt.Errorf("GetUserByName [%s]: %v", opts.PusherName, err) } - if err := gitRepo.SetDefaultBranch(repo.DefaultBranch); err != nil { - if !git.IsErrUnsupportedVersion(err) { - gitRepo.Close() + } + + if repo == nil || repo.OwnerID != opts.RepoOwnerID || repo.Name != opts.RepoName { + var err error + if repo != nil { + // Change repository empty status and update last updated time. + if err := models.UpdateRepository(repo, false); err != nil { + return fmt.Errorf("UpdateRepository: %v", err) + } + repo, err = models.GetRepositoryByName(opts.RepoOwnerID, opts.RepoName) + if err != nil { + return fmt.Errorf("GetRepositoryByName [owner_id: %d, name: %s]: %v", opts.RepoOwnerID, opts.RepoName, err) + } + } + } + refName := git.RefEndName(opts.RefFullName) + + // Change default branch and empty status only if pushed ref is non-empty branch. + if repo.IsEmpty && opts.NewCommitID != git.EmptySHA && strings.HasPrefix(opts.RefFullName, git.BranchPrefix) { + repo.DefaultBranch = refName + repo.IsEmpty = false + if refName != "master" { + gitRepo, err := git.OpenRepository(repo.RepoPath()) + if err != nil { return err } + if err := gitRepo.SetDefaultBranch(repo.DefaultBranch); err != nil { + if !git.IsErrUnsupportedVersion(err) { + gitRepo.Close() + return err + } + } + gitRepo.Close() } - gitRepo.Close() } - } - // Change repository empty status and update last updated time. - if err = models.UpdateRepository(repo, false); err != nil { - return fmt.Errorf("UpdateRepository: %v", err) - } + isNewBranch := false + opType := models.ActionCommitRepo - isNewBranch := false - opType := models.ActionCommitRepo - // Check it's tag push or branch. - if strings.HasPrefix(opts.RefFullName, git.TagPrefix) { - opType = models.ActionPushTag - if opts.NewCommitID == git.EmptySHA { - opType = models.ActionDeleteTag - } - opts.Commits = &models.PushCommits{} - } else if opts.NewCommitID == git.EmptySHA { - opType = models.ActionDeleteBranch - opts.Commits = &models.PushCommits{} - } else { - // if not the first commit, set the compare URL. - if opts.OldCommitID == git.EmptySHA { - isNewBranch = true + // Check it's tag push or branch. + if strings.HasPrefix(opts.RefFullName, git.TagPrefix) { + opType = models.ActionPushTag + if opts.NewCommitID == git.EmptySHA { + opType = models.ActionDeleteTag + } + opts.Commits = &models.PushCommits{} + } else if opts.NewCommitID == git.EmptySHA { + opType = models.ActionDeleteBranch + opts.Commits = &models.PushCommits{} } else { - opts.Commits.CompareURL = repo.ComposeCompareURL(opts.OldCommitID, opts.NewCommitID) - } + // if not the first commit, set the compare URL. + if opts.OldCommitID == git.EmptySHA { + isNewBranch = true + } else { + opts.Commits.CompareURL = repo.ComposeCompareURL(opts.OldCommitID, opts.NewCommitID) + } - if err = UpdateIssuesCommit(pusher, repo, opts.Commits.Commits, refName); err != nil { - log.Error("updateIssuesCommit: %v", err) + if err := UpdateIssuesCommit(pusher, repo, opts.Commits.Commits, refName); err != nil { + log.Error("updateIssuesCommit: %v", err) + } } - } - if len(opts.Commits.Commits) > setting.UI.FeedMaxCommitNum { - opts.Commits.Commits = opts.Commits.Commits[:setting.UI.FeedMaxCommitNum] - } - - data, err := json.Marshal(opts.Commits) - if err != nil { - return fmt.Errorf("Marshal: %v", err) - } + if len(opts.Commits.Commits) > setting.UI.FeedMaxCommitNum { + opts.Commits.Commits = opts.Commits.Commits[:setting.UI.FeedMaxCommitNum] + } - if err = models.NotifyWatchers(&models.Action{ - ActUserID: pusher.ID, - ActUser: pusher, - OpType: opType, - Content: string(data), - RepoID: repo.ID, - Repo: repo, - RefName: refName, - IsPrivate: repo.IsPrivate, - }); err != nil { - return fmt.Errorf("NotifyWatchers: %v", err) - } + data, err := json.Marshal(opts.Commits) + if err != nil { + return fmt.Errorf("Marshal: %v", err) + } - var isHookEventPush = true - switch opType { - case models.ActionCommitRepo: // Push - if isNewBranch { - notification.NotifyCreateRef(pusher, repo, "branch", opts.RefFullName) + actions[i] = &models.Action{ + ActUserID: pusher.ID, + ActUser: pusher, + OpType: opType, + Content: string(data), + RepoID: repo.ID, + Repo: repo, + RefName: refName, + IsPrivate: repo.IsPrivate, } - case models.ActionDeleteBranch: // Delete Branch - notification.NotifyDeleteRef(pusher, repo, "branch", opts.RefFullName) + var isHookEventPush = true + switch opType { + case models.ActionCommitRepo: // Push + if isNewBranch { + notification.NotifyCreateRef(pusher, repo, "branch", opts.RefFullName) + } + case models.ActionDeleteBranch: // Delete Branch + notification.NotifyDeleteRef(pusher, repo, "branch", opts.RefFullName) + + case models.ActionPushTag: // Create + notification.NotifyCreateRef(pusher, repo, "tag", opts.RefFullName) - case models.ActionPushTag: // Create - notification.NotifyCreateRef(pusher, repo, "tag", opts.RefFullName) + case models.ActionDeleteTag: // Delete Tag + notification.NotifyDeleteRef(pusher, repo, "tag", opts.RefFullName) + default: + isHookEventPush = false + } - case models.ActionDeleteTag: // Delete Tag - notification.NotifyDeleteRef(pusher, repo, "tag", opts.RefFullName) - default: - isHookEventPush = false + if isHookEventPush { + notification.NotifyPushCommits(pusher, repo, opts.RefFullName, opts.OldCommitID, opts.NewCommitID, opts.Commits) + } } - if isHookEventPush { - notification.NotifyPushCommits(pusher, repo, opts.RefFullName, opts.OldCommitID, opts.NewCommitID, opts.Commits) + if repo != nil { + // Change repository empty status and update last updated time. + if err := models.UpdateRepository(repo, false); err != nil { + return fmt.Errorf("UpdateRepository: %v", err) + } } + if err := models.NotifyWatchers(actions...); err != nil { + return fmt.Errorf("NotifyWatchers: %v", err) + } return nil } From ca0a0467fc9057985eb2458d0191e72e017286e4 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 20 Oct 2019 18:15:59 +0100 Subject: [PATCH 04/19] Batch post and pre-receive results --- cmd/hook.go | 143 +++++++++---- modules/private/hook.go | 51 ++--- modules/repofiles/update.go | 122 ++++++++--- routers/private/hook.go | 393 ++++++++++++++++++++---------------- routers/private/internal.go | 8 +- 5 files changed, 450 insertions(+), 267 deletions(-) diff --git a/cmd/hook.go b/cmd/hook.go index 9f547362da9b..929d2126a970 100644 --- a/cmd/hook.go +++ b/cmd/hook.go @@ -21,6 +21,10 @@ import ( "github.com/urfave/cli" ) +const ( + hookBatchSize = 200 +) + var ( // CmdHook represents the available hooks sub-command. CmdHook = cli.Command{ @@ -75,12 +79,23 @@ Gitea or set your environment appropriately.`, "") prID, _ := strconv.ParseInt(os.Getenv(models.ProtectedBranchPRID), 10, 64) isDeployKey, _ := strconv.ParseBool(os.Getenv(models.EnvIsDeployKey)) - buf := bytes.NewBuffer(nil) + hookOptions := private.HookOptions{ + UserID: userID, + GitAlternativeObjectDirectories: os.Getenv(private.GitAlternativeObjectDirectories), + GitObjectDirectory: os.Getenv(private.GitObjectDirectory), + GitQuarantinePath: os.Getenv(private.GitQuarantinePath), + ProtectedBranchID: prID, + IsDeployKey: isDeployKey, + } + scanner := bufio.NewScanner(os.Stdin) - for scanner.Scan() { - buf.Write(scanner.Bytes()) - buf.WriteByte('\n') + oldCommitIDs := make([]string, hookBatchSize) + newCommitIDs := make([]string, hookBatchSize) + refFullNames := make([]string, hookBatchSize) + count := 0 + + for scanner.Scan() { // TODO: support news feeds for wiki if isWiki { continue @@ -97,26 +112,40 @@ Gitea or set your environment appropriately.`, "") // If the ref is a branch, check if it's protected if strings.HasPrefix(refFullName, git.BranchPrefix) { - statusCode, msg := private.HookPreReceive(username, reponame, private.HookOptions{ - OldCommitID: oldCommitID, - NewCommitID: newCommitID, - RefFullName: refFullName, - UserID: userID, - GitAlternativeObjectDirectories: os.Getenv(private.GitAlternativeObjectDirectories), - GitObjectDirectory: os.Getenv(private.GitObjectDirectory), - GitQuarantinePath: os.Getenv(private.GitQuarantinePath), - ProtectedBranchID: prID, - IsDeployKey: isDeployKey, - }) - switch statusCode { - case http.StatusInternalServerError: - fail("Internal Server Error", msg) - case http.StatusForbidden: - fail(msg, "") + oldCommitIDs[count] = oldCommitID + newCommitIDs[count] = newCommitID + refFullNames[count] = refFullName + count++ + if count >= hookBatchSize { + hookOptions.OldCommitIDs = oldCommitIDs + hookOptions.NewCommitIDs = newCommitIDs + hookOptions.RefFullNames = refFullNames + statusCode, msg := private.HookPreReceive(username, reponame, hookOptions) + switch statusCode { + case http.StatusInternalServerError: + fail("Internal Server Error", msg) + case http.StatusForbidden: + fail(msg, "") + } + count = 0 } } } + if count > 0 { + hookOptions.OldCommitIDs = oldCommitIDs[:count] + hookOptions.NewCommitIDs = newCommitIDs[:count] + hookOptions.RefFullNames = refFullNames[:count] + + statusCode, msg := private.HookPreReceive(username, reponame, hookOptions) + switch statusCode { + case http.StatusInternalServerError: + fail("Internal Server Error", msg) + case http.StatusForbidden: + fail(msg, "") + } + } + return nil } @@ -156,6 +185,18 @@ Gitea or set your environment appropriately.`, "") pusherID, _ := strconv.ParseInt(os.Getenv(models.EnvPusherID), 10, 64) pusherName := os.Getenv(models.EnvPusherName) + hookOptions := private.HookOptions{ + UserName: pusherName, + UserID: pusherID, + GitAlternativeObjectDirectories: os.Getenv(private.GitAlternativeObjectDirectories), + GitObjectDirectory: os.Getenv(private.GitObjectDirectory), + GitQuarantinePath: os.Getenv(private.GitQuarantinePath), + } + oldCommitIDs := make([]string, hookBatchSize) + newCommitIDs := make([]string, hookBatchSize) + refFullNames := make([]string, hookBatchSize) + count := 0 + buf := bytes.NewBuffer(nil) scanner := bufio.NewScanner(os.Stdin) for scanner.Scan() { @@ -172,33 +213,61 @@ Gitea or set your environment appropriately.`, "") continue } - oldCommitID := string(fields[0]) - newCommitID := string(fields[1]) - refFullName := string(fields[2]) + oldCommitIDs[count] = string(fields[0]) + newCommitIDs[count] = string(fields[1]) + refFullNames[count] = string(fields[2]) + count++ - res, err := private.HookPostReceive(repoUser, repoName, private.HookOptions{ - OldCommitID: oldCommitID, - NewCommitID: newCommitID, - RefFullName: refFullName, - UserID: pusherID, - UserName: pusherName, - }) + if count >= hookBatchSize { + hookOptions.OldCommitIDs = oldCommitIDs + hookOptions.NewCommitIDs = newCommitIDs + hookOptions.RefFullNames = refFullNames + resps, err := private.HookPostReceive(repoUser, repoName, hookOptions) + if resps == nil { + fail("Internal Server Error", err) + } + for _, res := range resps { + if !res.Message { + continue + } - if res == nil { - fail("Internal Server Error", err) + fmt.Fprintln(os.Stderr, "") + if res.Create { + fmt.Fprintf(os.Stderr, "Create a new pull request for '%s':\n", res.Branch) + fmt.Fprintf(os.Stderr, " %s\n", res.URL) + } else { + fmt.Fprint(os.Stderr, "Visit the existing pull request:\n") + fmt.Fprintf(os.Stderr, " %s\n", res.URL) + } + fmt.Fprintln(os.Stderr, "") + } + count = 0 } + } - if res["message"] == false { + if count == 0 { + return nil + } + + hookOptions.OldCommitIDs = oldCommitIDs[:count] + hookOptions.NewCommitIDs = newCommitIDs[:count] + hookOptions.RefFullNames = refFullNames[:count] + resps, err := private.HookPostReceive(repoUser, repoName, hookOptions) + if resps == nil { + fail("Internal Server Error", err) + } + for _, res := range resps { + if !res.Message { continue } fmt.Fprintln(os.Stderr, "") - if res["create"] == true { - fmt.Fprintf(os.Stderr, "Create a new pull request for '%s':\n", res["branch"]) - fmt.Fprintf(os.Stderr, " %s\n", res["url"]) + if res.Create { + fmt.Fprintf(os.Stderr, "Create a new pull request for '%s':\n", res.Branch) + fmt.Fprintf(os.Stderr, " %s\n", res.URL) } else { fmt.Fprint(os.Stderr, "Visit the existing pull request:\n") - fmt.Fprintf(os.Stderr, " %s\n", res["url"]) + fmt.Fprintf(os.Stderr, " %s\n", res.URL) } fmt.Fprintln(os.Stderr, "") } diff --git a/modules/private/hook.go b/modules/private/hook.go index cc9703cc77ec..85dad7720ff0 100644 --- a/modules/private/hook.go +++ b/modules/private/hook.go @@ -22,9 +22,9 @@ const ( // HookOptions represents the options for the Hook calls type HookOptions struct { - OldCommitID string - NewCommitID string - RefFullName string + OldCommitIDs []string + NewCommitIDs []string + RefFullNames []string UserID int64 UserName string GitObjectDirectory string @@ -34,23 +34,26 @@ type HookOptions struct { IsDeployKey bool } +// HookPostReceiveResult represents an individual result from PostReceive +type HookPostReceiveResult struct { + Message bool + Create bool + Branch string + URL string + Err string +} + // HookPreReceive check whether the provided commits are allowed func HookPreReceive(ownerName, repoName string, opts HookOptions) (int, string) { - reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/pre-receive/%s/%s?old=%s&new=%s&ref=%s&userID=%d&gitObjectDirectory=%s&gitAlternativeObjectDirectories=%s&gitQuarantinePath=%s&prID=%d&isDeployKey=%t", + reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/pre-receive/%s/%s", url.PathEscape(ownerName), url.PathEscape(repoName), - url.QueryEscape(opts.OldCommitID), - url.QueryEscape(opts.NewCommitID), - url.QueryEscape(opts.RefFullName), - opts.UserID, - url.QueryEscape(opts.GitObjectDirectory), - url.QueryEscape(opts.GitAlternativeObjectDirectories), - url.QueryEscape(opts.GitQuarantinePath), - opts.ProtectedBranchID, - opts.IsDeployKey, ) - - resp, err := newInternalRequest(reqURL, "GET").Response() + req := newInternalRequest(reqURL, "POST") + req = req.Header("Content-Type", "application/json") + jsonBytes, _ := json.Marshal(opts) + req.Body(jsonBytes) + resp, err := req.Response() if err != nil { return http.StatusInternalServerError, fmt.Sprintf("Unable to contact gitea: %v", err.Error()) } @@ -64,17 +67,17 @@ func HookPreReceive(ownerName, repoName string, opts HookOptions) (int, string) } // HookPostReceive updates services and users -func HookPostReceive(ownerName, repoName string, opts HookOptions) (map[string]interface{}, string) { - reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/post-receive/%s/%s?old=%s&new=%s&ref=%s&userID=%d&username=%s", +func HookPostReceive(ownerName, repoName string, opts HookOptions) ([]HookPostReceiveResult, string) { + reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/post-receive/%s/%s", url.PathEscape(ownerName), url.PathEscape(repoName), - url.QueryEscape(opts.OldCommitID), - url.QueryEscape(opts.NewCommitID), - url.QueryEscape(opts.RefFullName), - opts.UserID, - url.QueryEscape(opts.UserName)) + ) - resp, err := newInternalRequest(reqURL, "GET").Response() + req := newInternalRequest(reqURL, "POST") + req = req.Header("Content-Type", "application/json") + jsonBytes, _ := json.Marshal(opts) + req.Body(jsonBytes) + resp, err := req.Response() if err != nil { return nil, fmt.Sprintf("Unable to contact gitea: %v", err.Error()) } @@ -83,7 +86,7 @@ func HookPostReceive(ownerName, repoName string, opts HookOptions) (map[string]i if resp.StatusCode != http.StatusOK { return nil, decodeJSONError(resp).Err } - res := map[string]interface{}{} + res := []HookPostReceiveResult{} _ = json.NewDecoder(resp.Body).Decode(&res) return res, "" diff --git a/modules/repofiles/update.go b/modules/repofiles/update.go index 4d2f1d5f045d..29ee0038e525 100644 --- a/modules/repofiles/update.go +++ b/modules/repofiles/update.go @@ -419,6 +419,7 @@ type PushUpdateOptions struct { RefFullName string OldCommitID string NewCommitID string + Branch string } // PushUpdate must be called for any push actions in order to @@ -447,21 +448,103 @@ func PushUpdate(repo *models.Repository, branch string, opts PushUpdateOptions) log.Error("Failed to update size for repository: %v", err) } + commitRepoActionOptions, err := createCommitRepoActionOption(repo, gitRepo, &opts) + if err != nil { + return err + } + + if err := CommitRepoAction(*commitRepoActionOptions); err != nil { + return fmt.Errorf("CommitRepoAction: %v", err) + } + + pusher, err := models.GetUserByID(opts.PusherID) + if err != nil { + return err + } + + log.Trace("TriggerTask '%s/%s' by %s", repo.Name, branch, pusher.Name) + + go pull_service.AddTestPullRequestTask(pusher, repo.ID, branch, true) + + if err = models.WatchIfAuto(opts.PusherID, repo.ID, true); err != nil { + log.Warn("Fail to perform auto watch on user %v for repo %v: %v", opts.PusherID, repo.ID, err) + } + + return nil +} + +// PushUpdates generates push action history feeds for push updating multiple refs +func PushUpdates(repo *models.Repository, optsList []PushUpdateOptions) error { + actions := make([]CommitRepoActionOptions, len(optsList)) + + repoPath := repo.RepoPath() + _, err := git.NewCommand("update-server-info").RunInDir(repoPath) + if err != nil { + return fmt.Errorf("Failed to call 'git update-server-info': %v", err) + } + gitRepo, err := git.OpenRepository(repoPath) + if err != nil { + return fmt.Errorf("OpenRepository: %v", err) + } + if err = repo.UpdateSize(); err != nil { + log.Error("Failed to update size for repository: %v", err) + } + + for i, opts := range optsList { + commitRepoActionOptions, err := createCommitRepoActionOption(repo, gitRepo, &opts) + if err != nil { + return err + } + + actions[i] = *commitRepoActionOptions + } + if err := CommitRepoAction(actions...); err != nil { + return fmt.Errorf("CommitRepoAction: %v", err) + } + + var pusher *models.User + + for _, opts := range optsList { + if pusher == nil || pusher.ID != opts.PusherID { + var err error + pusher, err = models.GetUserByID(opts.PusherID) + if err != nil { + return err + } + } + + log.Trace("TriggerTask '%s/%s' by %s", repo.Name, opts.Branch, pusher.Name) + + go pull_service.AddTestPullRequestTask(pusher, repo.ID, opts.Branch, true) + + if err = models.WatchIfAuto(opts.PusherID, repo.ID, true); err != nil { + log.Warn("Fail to perform auto watch on user %v for repo %v: %v", opts.PusherID, repo.ID, err) + } + } + + return nil +} + +func createCommitRepoActionOption(repo *models.Repository, gitRepo *git.Repository, opts *PushUpdateOptions) (*CommitRepoActionOptions, error) { + isNewRef := opts.OldCommitID == git.EmptySHA + isDelRef := opts.NewCommitID == git.EmptySHA + if isNewRef && isDelRef { + return nil, fmt.Errorf("Old and new revisions are both %s", git.EmptySHA) + } + var commits = &models.PushCommits{} if strings.HasPrefix(opts.RefFullName, git.TagPrefix) { // If is tag reference tagName := opts.RefFullName[len(git.TagPrefix):] if isDelRef { - err = models.PushUpdateDeleteTag(repo, tagName) - if err != nil { - return fmt.Errorf("PushUpdateDeleteTag: %v", err) + if err := models.PushUpdateDeleteTag(repo, tagName); err != nil { + return nil, fmt.Errorf("PushUpdateDeleteTag: %v", err) } } else { // Clear cache for tag commit count cache.Remove(repo.GetCommitsCountCacheKey(tagName, true)) - err = models.PushUpdateAddTag(repo, gitRepo, tagName) - if err != nil { - return fmt.Errorf("PushUpdateAddTag: %v", err) + if err := models.PushUpdateAddTag(repo, gitRepo, tagName); err != nil { + return nil, fmt.Errorf("PushUpdateAddTag: %v", err) } } } else if !isDelRef { @@ -472,7 +555,7 @@ func PushUpdate(repo *models.Repository, branch string, opts PushUpdateOptions) newCommit, err := gitRepo.GetCommit(opts.NewCommitID) if err != nil { - return fmt.Errorf("gitRepo.GetCommit: %v", err) + return nil, fmt.Errorf("gitRepo.GetCommit: %v", err) } // Push new branch. @@ -480,19 +563,19 @@ func PushUpdate(repo *models.Repository, branch string, opts PushUpdateOptions) if isNewRef { l, err = newCommit.CommitsBeforeLimit(10) if err != nil { - return fmt.Errorf("newCommit.CommitsBeforeLimit: %v", err) + return nil, fmt.Errorf("newCommit.CommitsBeforeLimit: %v", err) } } else { l, err = newCommit.CommitsBeforeUntil(opts.OldCommitID) if err != nil { - return fmt.Errorf("newCommit.CommitsBeforeUntil: %v", err) + return nil, fmt.Errorf("newCommit.CommitsBeforeUntil: %v", err) } } commits = models.ListToPushCommits(l) } - if err := CommitRepoAction(CommitRepoActionOptions{ + return &CommitRepoActionOptions{ PusherName: opts.PusherName, RepoOwnerID: repo.OwnerID, RepoName: repo.Name, @@ -500,22 +583,5 @@ func PushUpdate(repo *models.Repository, branch string, opts PushUpdateOptions) OldCommitID: opts.OldCommitID, NewCommitID: opts.NewCommitID, Commits: commits, - }); err != nil { - return fmt.Errorf("CommitRepoAction: %v", err) - } - - pusher, err := models.GetUserByID(opts.PusherID) - if err != nil { - return err - } - - log.Trace("TriggerTask '%s/%s' by %s", repo.Name, branch, pusher.Name) - - go pull_service.AddTestPullRequestTask(pusher, repo.ID, branch, true) - - if err = models.WatchIfAuto(opts.PusherID, repo.ID, true); err != nil { - log.Warn("Fail to perform auto watch on user %v for repo %v: %v", opts.PusherID, repo.ID, err) - } - - return nil + }, nil } diff --git a/routers/private/hook.go b/routers/private/hook.go index 2644302eadcd..7e0c9bf6dee0 100644 --- a/routers/private/hook.go +++ b/routers/private/hook.go @@ -22,20 +22,9 @@ import ( ) // HookPreReceive checks whether a individual commit is acceptable -func HookPreReceive(ctx *macaron.Context) { +func HookPreReceive(ctx *macaron.Context, opts private.HookOptions) { ownerName := ctx.Params(":owner") repoName := ctx.Params(":repo") - oldCommitID := ctx.QueryTrim("old") - newCommitID := ctx.QueryTrim("new") - refFullName := ctx.QueryTrim("ref") - userID := ctx.QueryInt64("userID") - gitObjectDirectory := ctx.QueryTrim("gitObjectDirectory") - gitAlternativeObjectDirectories := ctx.QueryTrim("gitAlternativeObjectDirectories") - gitQuarantinePath := ctx.QueryTrim("gitQuarantinePath") - prID := ctx.QueryInt64("prID") - isDeployKey := ctx.QueryBool("isDeployKey") - - branchName := strings.TrimPrefix(refFullName, git.BranchPrefix) repo, err := models.GetRepositoryByOwnerAndName(ownerName, repoName) if err != nil { log.Error("Unable to get repository: %s/%s Error: %v", ownerName, repoName, err) @@ -45,206 +34,258 @@ func HookPreReceive(ctx *macaron.Context) { return } repo.OwnerName = ownerName - protectBranch, err := models.GetProtectedBranchBy(repo.ID, branchName) - if err != nil { - log.Error("Unable to get protected branch: %s in %-v Error: %v", branchName, repo, err) - ctx.JSON(500, map[string]interface{}{ - "err": err.Error(), - }) - return - } - if protectBranch != nil && protectBranch.IsProtected() { - // check and deletion - if newCommitID == git.EmptySHA { - log.Warn("Forbidden: Branch: %s in %-v is protected from deletion", branchName, repo) - ctx.JSON(http.StatusForbidden, map[string]interface{}{ - "err": fmt.Sprintf("branch %s is protected from deletion", branchName), + + for i := range opts.OldCommitIDs { + oldCommitID := opts.OldCommitIDs[i] + newCommitID := opts.NewCommitIDs[i] + refFullName := opts.RefFullNames[i] + + branchName := strings.TrimPrefix(refFullName, git.BranchPrefix) + protectBranch, err := models.GetProtectedBranchBy(repo.ID, branchName) + if err != nil { + log.Error("Unable to get protected branch: %s in %-v Error: %v", branchName, repo, err) + ctx.JSON(500, map[string]interface{}{ + "err": err.Error(), }) return } - - // detect force push - if git.EmptySHA != oldCommitID { - env := os.Environ() - if gitAlternativeObjectDirectories != "" { - env = append(env, - private.GitAlternativeObjectDirectories+"="+gitAlternativeObjectDirectories) - } - if gitObjectDirectory != "" { - env = append(env, - private.GitObjectDirectory+"="+gitObjectDirectory) - } - if gitQuarantinePath != "" { - env = append(env, - private.GitQuarantinePath+"="+gitQuarantinePath) - } - - output, err := git.NewCommand("rev-list", "--max-count=1", oldCommitID, "^"+newCommitID).RunInDirWithEnv(repo.RepoPath(), env) - if err != nil { - log.Error("Unable to detect force push between: %s and %s in %-v Error: %v", oldCommitID, newCommitID, repo, err) - ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ - "err": fmt.Sprintf("Fail to detect force push: %v", err), - }) - return - } else if len(output) > 0 { - log.Warn("Forbidden: Branch: %s in %-v is protected from force push", branchName, repo) + if protectBranch != nil && protectBranch.IsProtected() { + // check and deletion + if newCommitID == git.EmptySHA { + log.Warn("Forbidden: Branch: %s in %-v is protected from deletion", branchName, repo) ctx.JSON(http.StatusForbidden, map[string]interface{}{ - "err": fmt.Sprintf("branch %s is protected from force push", branchName), + "err": fmt.Sprintf("branch %s is protected from deletion", branchName), }) return - } - } - canPush := false - if isDeployKey { - canPush = protectBranch.CanPush && (!protectBranch.EnableWhitelist || protectBranch.WhitelistDeployKeys) - } else { - canPush = protectBranch.CanUserPush(userID) - } - if !canPush && prID > 0 { - pr, err := models.GetPullRequestByID(prID) - if err != nil { - log.Error("Unable to get PullRequest %d Error: %v", prID, err) - ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ - "err": fmt.Sprintf("Unable to get PullRequest %d Error: %v", prID, err), - }) - return + // detect force push + if git.EmptySHA != oldCommitID { + env := os.Environ() + if opts.GitAlternativeObjectDirectories != "" { + env = append(env, + private.GitAlternativeObjectDirectories+"="+opts.GitAlternativeObjectDirectories) + } + if opts.GitObjectDirectory != "" { + env = append(env, + private.GitObjectDirectory+"="+opts.GitObjectDirectory) + } + if opts.GitQuarantinePath != "" { + env = append(env, + private.GitQuarantinePath+"="+opts.GitQuarantinePath) + } + + output, err := git.NewCommand("rev-list", "--max-count=1", oldCommitID, "^"+newCommitID).RunInDirWithEnv(repo.RepoPath(), env) + if err != nil { + log.Error("Unable to detect force push between: %s and %s in %-v Error: %v", oldCommitID, newCommitID, repo, err) + ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ + "err": fmt.Sprintf("Fail to detect force push: %v", err), + }) + return + } else if len(output) > 0 { + log.Warn("Forbidden: Branch: %s in %-v is protected from force push", branchName, repo) + ctx.JSON(http.StatusForbidden, map[string]interface{}{ + "err": fmt.Sprintf("branch %s is protected from force push", branchName), + }) + return + + } } - if !protectBranch.HasEnoughApprovals(pr) { - log.Warn("Forbidden: User %d cannot push to protected branch: %s in %-v and pr #%d does not have enough approvals", userID, branchName, repo, pr.Index) + canPush := false + if opts.IsDeployKey { + canPush = protectBranch.CanPush && (!protectBranch.EnableWhitelist || protectBranch.WhitelistDeployKeys) + } else { + canPush = protectBranch.CanUserPush(opts.UserID) + } + if !canPush && opts.ProtectedBranchID > 0 { + pr, err := models.GetPullRequestByID(opts.ProtectedBranchID) + if err != nil { + log.Error("Unable to get PullRequest %d Error: %v", opts.ProtectedBranchID, err) + ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ + "err": fmt.Sprintf("Unable to get PullRequest %d Error: %v", opts.ProtectedBranchID, err), + }) + return + } + if !protectBranch.HasEnoughApprovals(pr) { + log.Warn("Forbidden: User %d cannot push to protected branch: %s in %-v and pr #%d does not have enough approvals", opts.UserID, branchName, repo, pr.Index) + ctx.JSON(http.StatusForbidden, map[string]interface{}{ + "err": fmt.Sprintf("protected branch %s can not be pushed to and pr #%d does not have enough approvals", branchName, opts.ProtectedBranchID), + }) + return + } + } else if !canPush { + log.Warn("Forbidden: User %d cannot push to protected branch: %s in %-v", opts.UserID, branchName, repo) ctx.JSON(http.StatusForbidden, map[string]interface{}{ - "err": fmt.Sprintf("protected branch %s can not be pushed to and pr #%d does not have enough approvals", branchName, prID), + "err": fmt.Sprintf("protected branch %s can not be pushed to", branchName), }) return } - } else if !canPush { - log.Warn("Forbidden: User %d cannot push to protected branch: %s in %-v", userID, branchName, repo) - ctx.JSON(http.StatusForbidden, map[string]interface{}{ - "err": fmt.Sprintf("protected branch %s can not be pushed to", branchName), - }) - return } } + ctx.PlainText(http.StatusOK, []byte("ok")) } // HookPostReceive updates services and users -func HookPostReceive(ctx *macaron.Context) { +func HookPostReceive(ctx *macaron.Context, opts private.HookOptions) { ownerName := ctx.Params(":owner") repoName := ctx.Params(":repo") - oldCommitID := ctx.Query("old") - newCommitID := ctx.Query("new") - refFullName := ctx.Query("ref") - userID := ctx.QueryInt64("userID") - userName := ctx.Query("username") - - branch := refFullName - if strings.HasPrefix(refFullName, git.BranchPrefix) { - branch = strings.TrimPrefix(refFullName, git.BranchPrefix) - } else if strings.HasPrefix(refFullName, git.TagPrefix) { - branch = strings.TrimPrefix(refFullName, git.TagPrefix) - } - // Only trigger activity updates for changes to branches or - // tags. Updates to other refs (eg, refs/notes, refs/changes, - // or other less-standard refs spaces are ignored since there - // may be a very large number of them). - if strings.HasPrefix(refFullName, git.BranchPrefix) || strings.HasPrefix(refFullName, git.TagPrefix) { - repo, err := models.GetRepositoryByOwnerAndName(ownerName, repoName) - if err != nil { - log.Error("Failed to get repository: %s/%s Error: %v", ownerName, repoName, err) - ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ - "err": fmt.Sprintf("Failed to get repository: %s/%s Error: %v", ownerName, repoName, err), - }) - return + var repo *models.Repository + updates := make([]repofiles.PushUpdateOptions, 0, len(opts.OldCommitIDs)) + + for i := range opts.OldCommitIDs { + refFullName := opts.RefFullNames[i] + branch := opts.RefFullNames[i] + if strings.HasPrefix(branch, git.BranchPrefix) { + branch = strings.TrimPrefix(branch, git.BranchPrefix) + } else { + branch = strings.TrimPrefix(branch, git.TagPrefix) } - if err := repofiles.PushUpdate(repo, branch, repofiles.PushUpdateOptions{ - RefFullName: refFullName, - OldCommitID: oldCommitID, - NewCommitID: newCommitID, - PusherID: userID, - PusherName: userName, - RepoUserName: ownerName, - RepoName: repoName, - }); err != nil { - log.Error("Failed to Update: %s/%s Branch: %s Error: %v", ownerName, repoName, branch, err) - ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ - "err": fmt.Sprintf("Failed to Update: %s/%s Branch: %s Error: %v", ownerName, repoName, branch, err), + + // Only trigger activity updates for changes to branches or + // tags. Updates to other refs (eg, refs/notes, refs/changes, + // or other less-standard refs spaces are ignored since there + // may be a very large number of them). + if strings.HasPrefix(refFullName, git.BranchPrefix) || strings.HasPrefix(refFullName, git.TagPrefix) { + if repo == nil { + var err error + repo, err = models.GetRepositoryByOwnerAndName(ownerName, repoName) + if err != nil { + log.Error("Failed to get repository: %s/%s Error: %v", ownerName, repoName, err) + ctx.JSON(http.StatusInternalServerError, []private.HookPostReceiveResult{ + { + Err: fmt.Sprintf("Failed to get repository: %s/%s Error: %v", ownerName, repoName, err), + }, + }) + return + } + if repo.OwnerName == "" { + repo.OwnerName = ownerName + } + } + updates = append(updates, repofiles.PushUpdateOptions{ + RefFullName: refFullName, + OldCommitID: opts.OldCommitIDs[i], + NewCommitID: opts.NewCommitIDs[i], + Branch: branch, + PusherID: opts.UserID, + PusherName: opts.UserName, + RepoUserName: ownerName, + RepoName: repoName, }) - return } } - if newCommitID != git.EmptySHA && strings.HasPrefix(refFullName, git.BranchPrefix) { - repo, err := models.GetRepositoryByOwnerAndName(ownerName, repoName) - if err != nil { - log.Error("Failed to get repository: %s/%s Error: %v", ownerName, repoName, err) - ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ - "err": fmt.Sprintf("Failed to get repository: %s/%s Error: %v", ownerName, repoName, err), - }) - return + if err := repofiles.PushUpdates(repo, updates); err != nil { + log.Error("Failed to Update: %s/%s Total Updates: %d", ownerName, repoName, len(updates)) + for i, update := range updates { + log.Error("Failed to Update: %s/%s Update: %d/%d: Branch: %s", ownerName, repoName, i, len(updates), update.Branch) } - repo.OwnerName = ownerName + log.Error("Failed to Update: %s/%s Error: %v", ownerName, repoName, err) - pullRequestAllowed := repo.AllowsPulls() - if !pullRequestAllowed { - ctx.JSON(http.StatusOK, map[string]interface{}{ - "message": false, - }) - return + ctx.JSON(http.StatusInternalServerError, []private.HookPostReceiveResult{ + { + Err: fmt.Sprintf("Failed to Update: %s/%s Error: %v", ownerName, repoName, err), + }, + }) + return + } + + results := make([]private.HookPostReceiveResult, 0, len(opts.OldCommitIDs)) + + // We have to reload the repo in case its state is changed above + repo = nil + var baseRepo *models.Repository + + for i := range opts.OldCommitIDs { + refFullName := opts.RefFullNames[i] + newCommitID := opts.NewCommitIDs[i] + + branch := opts.RefFullNames[i] + if strings.HasPrefix(branch, git.BranchPrefix) { + branch = strings.TrimPrefix(branch, git.BranchPrefix) + } else { + branch = strings.TrimPrefix(branch, git.TagPrefix) } - baseRepo := repo - if repo.IsFork { - if err := repo.GetBaseRepo(); err != nil { - log.Error("Failed to get Base Repository of Forked repository: %-v Error: %v", repo, err) - ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ - "err": fmt.Sprintf("Failed to get Base Repository of Forked repository: %-v Error: %v", repo, err), - }) - return + if newCommitID != git.EmptySHA && strings.HasPrefix(refFullName, git.BranchPrefix) { + if repo == nil { + var err error + repo, err = models.GetRepositoryByOwnerAndName(ownerName, repoName) + if err != nil { + log.Error("Failed to get repository: %s/%s Error: %v", ownerName, repoName, err) + ctx.JSON(http.StatusInternalServerError, []private.HookPostReceiveResult{ + { + Err: fmt.Sprintf("Failed to get repository: %s/%s Error: %v", ownerName, repoName, err), + }, + }) + return + } + if repo.OwnerName == "" { + repo.OwnerName = ownerName + } + + pullRequestAllowed := repo.AllowsPulls() + if !pullRequestAllowed { + // We can stop there's no need to go any further + ctx.JSON(http.StatusOK, []private.HookPostReceiveResult{ + {}, + }) + return + } + baseRepo = repo + + if repo.IsFork { + if err := repo.GetBaseRepo(); err != nil { + log.Error("Failed to get Base Repository of Forked repository: %-v Error: %v", repo, err) + ctx.JSON(http.StatusInternalServerError, []private.HookPostReceiveResult{ + { + Err: fmt.Sprintf("Failed to get Base Repository of Forked repository: %-v Error: %v", repo, err), + }, + }) + return + } + baseRepo = repo.BaseRepo + } } - baseRepo = repo.BaseRepo - } - if !repo.IsFork && branch == baseRepo.DefaultBranch { - ctx.JSON(http.StatusOK, map[string]interface{}{ - "message": false, - }) - return - } + if !repo.IsFork && branch == baseRepo.DefaultBranch { + results = append(results, private.HookPostReceiveResult{}) + continue + } - pr, err := models.GetUnmergedPullRequest(repo.ID, baseRepo.ID, branch, baseRepo.DefaultBranch) - if err != nil && !models.IsErrPullRequestNotExist(err) { - log.Error("Failed to get active PR in: %-v Branch: %s to: %-v Branch: %s Error: %v", repo, branch, baseRepo, baseRepo.DefaultBranch, err) - ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ - "err": fmt.Sprintf( - "Failed to get active PR in: %-v Branch: %s to: %-v Branch: %s Error: %v", repo, branch, baseRepo, baseRepo.DefaultBranch, err), - }) - return - } + pr, err := models.GetUnmergedPullRequest(repo.ID, baseRepo.ID, branch, baseRepo.DefaultBranch) + if err != nil && !models.IsErrPullRequestNotExist(err) { + log.Error("Failed to get active PR in: %-v Branch: %s to: %-v Branch: %s Error: %v", repo, branch, baseRepo, baseRepo.DefaultBranch, err) + ctx.JSON(http.StatusInternalServerError, []private.HookPostReceiveResult{ + { + Err: fmt.Sprintf( + "Failed to get active PR in: %-v Branch: %s to: %-v Branch: %s Error: %v", repo, branch, baseRepo, baseRepo.DefaultBranch, err), + }, + }) + return + } - if pr == nil { - if repo.IsFork { - branch = fmt.Sprintf("%s:%s", repo.OwnerName, branch) + if pr == nil { + if repo.IsFork { + branch = fmt.Sprintf("%s:%s", repo.OwnerName, branch) + } + results = append(results, private.HookPostReceiveResult{ + Message: true, + Create: true, + Branch: branch, + URL: fmt.Sprintf("%s/compare/%s...%s", baseRepo.HTMLURL(), util.PathEscapeSegments(baseRepo.DefaultBranch), util.PathEscapeSegments(branch)), + }) + } else { + results = append(results, private.HookPostReceiveResult{ + Message: true, + Create: false, + Branch: branch, + URL: fmt.Sprintf("%s/pulls/%d", baseRepo.HTMLURL(), pr.Index), + }) } - ctx.JSON(http.StatusOK, map[string]interface{}{ - "message": true, - "create": true, - "branch": branch, - "url": fmt.Sprintf("%s/compare/%s...%s", baseRepo.HTMLURL(), util.PathEscapeSegments(baseRepo.DefaultBranch), util.PathEscapeSegments(branch)), - }) - } else { - ctx.JSON(http.StatusOK, map[string]interface{}{ - "message": true, - "create": false, - "branch": branch, - "url": fmt.Sprintf("%s/pulls/%d", baseRepo.HTMLURL(), pr.Index), - }) } - return } - ctx.JSON(http.StatusOK, map[string]interface{}{ - "message": false, - }) + ctx.JSON(http.StatusOK, results) } diff --git a/routers/private/internal.go b/routers/private/internal.go index 3a48f5384d8c..2ba8fc4d894b 100644 --- a/routers/private/internal.go +++ b/routers/private/internal.go @@ -9,8 +9,10 @@ import ( "strings" "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/private" "code.gitea.io/gitea/modules/setting" + "gitea.com/macaron/binding" "gitea.com/macaron/macaron" ) @@ -75,10 +77,12 @@ func CheckUnitUser(ctx *macaron.Context) { // RegisterRoutes registers all internal APIs routes to web application. // These APIs will be invoked by internal commands for example `gitea serv` and etc. func RegisterRoutes(m *macaron.Macaron) { + bind := binding.Bind + m.Group("/", func() { m.Post("/ssh/:id/update/:repoid", UpdatePublicKeyInRepo) - m.Get("/hook/pre-receive/:owner/:repo", HookPreReceive) - m.Get("/hook/post-receive/:owner/:repo", HookPostReceive) + m.Post("/hook/pre-receive/:owner/:repo", bind(private.HookOptions{}), HookPreReceive) + m.Post("/hook/post-receive/:owner/:repo", bind(private.HookOptions{}), HookPostReceive) m.Get("/serv/none/:keyid", ServNoCommand) m.Get("/serv/command/:keyid/:owner/:repo", ServCommand) }, CheckInternalToken) From b2dbf87b46e953ea28c805d2be129a7597d1785d Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Mon, 21 Oct 2019 16:29:31 +0100 Subject: [PATCH 05/19] Set batch to 30 --- cmd/hook.go | 6 +----- modules/repository/repo.go | 6 +++--- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/cmd/hook.go b/cmd/hook.go index 929d2126a970..690b90409508 100644 --- a/cmd/hook.go +++ b/cmd/hook.go @@ -22,7 +22,7 @@ import ( ) const ( - hookBatchSize = 200 + hookBatchSize = 30 ) var ( @@ -197,12 +197,8 @@ Gitea or set your environment appropriately.`, "") refFullNames := make([]string, hookBatchSize) count := 0 - buf := bytes.NewBuffer(nil) scanner := bufio.NewScanner(os.Stdin) for scanner.Scan() { - buf.Write(scanner.Bytes()) - buf.WriteByte('\n') - // TODO: support news feeds for wiki if isWiki { continue diff --git a/modules/repository/repo.go b/modules/repository/repo.go index ea526a1e3033..9351ab397e39 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -197,11 +197,11 @@ func SyncReleasesWithTags(repo *models.Repository, gitRepo *git.Repository) erro } commitID, err := gitRepo.GetTagCommitID(rel.TagName) if err != nil && !git.IsErrNotExist(err) { - return fmt.Errorf("GetTagCommitID: %v", err) + return fmt.Errorf("GetTagCommitID: %s: %v", rel.TagName, err) } if git.IsErrNotExist(err) || commitID != rel.Sha1 { if err := models.PushUpdateDeleteTag(repo, rel.TagName); err != nil { - return fmt.Errorf("PushUpdateDeleteTag: %v", err) + return fmt.Errorf("PushUpdateDeleteTag: %s: %v", rel.TagName, err) } } else { existingRelTags[strings.ToLower(rel.TagName)] = struct{}{} @@ -215,7 +215,7 @@ func SyncReleasesWithTags(repo *models.Repository, gitRepo *git.Repository) erro for _, tagName := range tags { if _, ok := existingRelTags[strings.ToLower(tagName)]; !ok { if err := models.PushUpdateAddTag(repo, gitRepo, tagName); err != nil { - return fmt.Errorf("pushUpdateAddTag: %v", err) + return fmt.Errorf("pushUpdateAddTag: %s: %v", tagName, err) } } } From ce4609bdc8f253f3238d50c696aad3c28c0b2458 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Tue, 22 Oct 2019 20:17:46 +0100 Subject: [PATCH 06/19] Auto adjust timeout & add logging --- cmd/hook.go | 43 ++++++++++++++++++++++++----------------- modules/private/hook.go | 3 +++ 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/cmd/hook.go b/cmd/hook.go index 690b90409508..65a4016e500a 100644 --- a/cmd/hook.go +++ b/cmd/hook.go @@ -196,6 +196,8 @@ Gitea or set your environment appropriately.`, "") newCommitIDs := make([]string, hookBatchSize) refFullNames := make([]string, hookBatchSize) count := 0 + total := 0 + results := make([]private.HookPostReceiveResult, 0) scanner := bufio.NewScanner(os.Stdin) for scanner.Scan() { @@ -209,34 +211,26 @@ Gitea or set your environment appropriately.`, "") continue } + fmt.Fprintf(os.Stderr, ".") oldCommitIDs[count] = string(fields[0]) newCommitIDs[count] = string(fields[1]) refFullNames[count] = string(fields[2]) count++ + total++ + os.Stderr.Sync() if count >= hookBatchSize { + fmt.Fprintf(os.Stderr, " Processing %d references\n", count) + os.Stderr.Sync() hookOptions.OldCommitIDs = oldCommitIDs hookOptions.NewCommitIDs = newCommitIDs hookOptions.RefFullNames = refFullNames resps, err := private.HookPostReceive(repoUser, repoName, hookOptions) if resps == nil { + hookPrintResults(results) fail("Internal Server Error", err) } - for _, res := range resps { - if !res.Message { - continue - } - - fmt.Fprintln(os.Stderr, "") - if res.Create { - fmt.Fprintf(os.Stderr, "Create a new pull request for '%s':\n", res.Branch) - fmt.Fprintf(os.Stderr, " %s\n", res.URL) - } else { - fmt.Fprint(os.Stderr, "Visit the existing pull request:\n") - fmt.Fprintf(os.Stderr, " %s\n", res.URL) - } - fmt.Fprintln(os.Stderr, "") - } + results = append(results, resps...) count = 0 } } @@ -248,11 +242,25 @@ Gitea or set your environment appropriately.`, "") hookOptions.OldCommitIDs = oldCommitIDs[:count] hookOptions.NewCommitIDs = newCommitIDs[:count] hookOptions.RefFullNames = refFullNames[:count] + + fmt.Fprintf(os.Stderr, " Processing %d references\n", count) + os.Stderr.Sync() + fmt.Fprintf(os.Stderr, "Processed %d references in total\n", total) + os.Stderr.Sync() + resps, err := private.HookPostReceive(repoUser, repoName, hookOptions) if resps == nil { + hookPrintResults(results) fail("Internal Server Error", err) } - for _, res := range resps { + results = append(results, resps...) + hookPrintResults(results) + + return nil +} + +func hookPrintResults(results []private.HookPostReceiveResult) { + for _, res := range results { if !res.Message { continue } @@ -266,7 +274,6 @@ Gitea or set your environment appropriately.`, "") fmt.Fprintf(os.Stderr, " %s\n", res.URL) } fmt.Fprintln(os.Stderr, "") + os.Stderr.Sync() } - - return nil } diff --git a/modules/private/hook.go b/modules/private/hook.go index 85dad7720ff0..798cffe34528 100644 --- a/modules/private/hook.go +++ b/modules/private/hook.go @@ -9,6 +9,7 @@ import ( "fmt" "net/http" "net/url" + "time" "code.gitea.io/gitea/modules/setting" ) @@ -53,6 +54,7 @@ func HookPreReceive(ownerName, repoName string, opts HookOptions) (int, string) req = req.Header("Content-Type", "application/json") jsonBytes, _ := json.Marshal(opts) req.Body(jsonBytes) + req.SetTimeout(60*time.Second, time.Duration(60+len(opts.OldCommitIDs))*time.Second) resp, err := req.Response() if err != nil { return http.StatusInternalServerError, fmt.Sprintf("Unable to contact gitea: %v", err.Error()) @@ -75,6 +77,7 @@ func HookPostReceive(ownerName, repoName string, opts HookOptions) ([]HookPostRe req := newInternalRequest(reqURL, "POST") req = req.Header("Content-Type", "application/json") + req.SetTimeout(60*time.Second, time.Duration(60+len(opts.OldCommitIDs))*time.Second) jsonBytes, _ := json.Marshal(opts) req.Body(jsonBytes) resp, err := req.Response() From fff0a1de76485e389553a002e8a71f7ae03948db Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Wed, 23 Oct 2019 13:14:41 +0100 Subject: [PATCH 07/19] adjust processing message --- cmd/hook.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/cmd/hook.go b/cmd/hook.go index 65a4016e500a..f82834351200 100644 --- a/cmd/hook.go +++ b/cmd/hook.go @@ -236,6 +236,10 @@ Gitea or set your environment appropriately.`, "") } if count == 0 { + + fmt.Fprintf(os.Stderr, "Processed %d references in total\n", total) + os.Stderr.Sync() + return nil } @@ -245,8 +249,6 @@ Gitea or set your environment appropriately.`, "") fmt.Fprintf(os.Stderr, " Processing %d references\n", count) os.Stderr.Sync() - fmt.Fprintf(os.Stderr, "Processed %d references in total\n", total) - os.Stderr.Sync() resps, err := private.HookPostReceive(repoUser, repoName, hookOptions) if resps == nil { @@ -254,6 +256,10 @@ Gitea or set your environment appropriately.`, "") fail("Internal Server Error", err) } results = append(results, resps...) + + fmt.Fprintf(os.Stderr, "Processed %d references in total\n", total) + os.Stderr.Sync() + hookPrintResults(results) return nil From 66d80188305530bd9f8b715dcd005299faea1cfc Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Wed, 23 Oct 2019 14:34:36 +0100 Subject: [PATCH 08/19] Add some messages to pre-receive --- cmd/hook.go | 49 +++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/cmd/hook.go b/cmd/hook.go index f82834351200..b6d188c0c91b 100644 --- a/cmd/hook.go +++ b/cmd/hook.go @@ -94,6 +94,8 @@ Gitea or set your environment appropriately.`, "") newCommitIDs := make([]string, hookBatchSize) refFullNames := make([]string, hookBatchSize) count := 0 + total := 0 + lastline := 0 for scanner.Scan() { // TODO: support news feeds for wiki @@ -109,6 +111,8 @@ Gitea or set your environment appropriately.`, "") oldCommitID := string(fields[0]) newCommitID := string(fields[1]) refFullName := string(fields[2]) + total++ + lastline++ // If the ref is a branch, check if it's protected if strings.HasPrefix(refFullName, git.BranchPrefix) { @@ -116,7 +120,13 @@ Gitea or set your environment appropriately.`, "") newCommitIDs[count] = newCommitID refFullNames[count] = refFullName count++ + fmt.Fprintf(os.Stdout, "*") + os.Stdout.Sync() + if count >= hookBatchSize { + fmt.Fprintf(os.Stdout, " Checking %d branches\n", count) + os.Stdout.Sync() + hookOptions.OldCommitIDs = oldCommitIDs hookOptions.NewCommitIDs = newCommitIDs hookOptions.RefFullNames = refFullNames @@ -128,7 +138,16 @@ Gitea or set your environment appropriately.`, "") fail(msg, "") } count = 0 + lastline = 0 } + } else { + fmt.Fprintf(os.Stdout, ".") + os.Stdout.Sync() + } + if lastline >= hookBatchSize { + fmt.Fprintf(os.Stdout, "\n") + os.Stdout.Sync() + lastline = 0 } } @@ -137,6 +156,9 @@ Gitea or set your environment appropriately.`, "") hookOptions.NewCommitIDs = newCommitIDs[:count] hookOptions.RefFullNames = refFullNames[:count] + fmt.Fprintf(os.Stdout, " Checking %d branches\n", count) + os.Stdout.Sync() + statusCode, msg := private.HookPreReceive(username, reponame, hookOptions) switch statusCode { case http.StatusInternalServerError: @@ -144,8 +166,15 @@ Gitea or set your environment appropriately.`, "") case http.StatusForbidden: fail(msg, "") } + } else if lastline > 0 { + fmt.Fprintf(os.Stdout, "\n") + os.Stdout.Sync() + lastline = 0 } + fmt.Fprintf(os.Stdout, "Checked %d references in total\n", total) + os.Stdout.Sync() + return nil } @@ -211,17 +240,17 @@ Gitea or set your environment appropriately.`, "") continue } - fmt.Fprintf(os.Stderr, ".") + fmt.Fprintf(os.Stdout, ".") oldCommitIDs[count] = string(fields[0]) newCommitIDs[count] = string(fields[1]) refFullNames[count] = string(fields[2]) count++ total++ - os.Stderr.Sync() + os.Stdout.Sync() if count >= hookBatchSize { - fmt.Fprintf(os.Stderr, " Processing %d references\n", count) - os.Stderr.Sync() + fmt.Fprintf(os.Stdout, " Processing %d references\n", count) + os.Stdout.Sync() hookOptions.OldCommitIDs = oldCommitIDs hookOptions.NewCommitIDs = newCommitIDs hookOptions.RefFullNames = refFullNames @@ -237,8 +266,8 @@ Gitea or set your environment appropriately.`, "") if count == 0 { - fmt.Fprintf(os.Stderr, "Processed %d references in total\n", total) - os.Stderr.Sync() + fmt.Fprintf(os.Stdout, "Processed %d references in total\n", total) + os.Stdout.Sync() return nil } @@ -247,8 +276,8 @@ Gitea or set your environment appropriately.`, "") hookOptions.NewCommitIDs = newCommitIDs[:count] hookOptions.RefFullNames = refFullNames[:count] - fmt.Fprintf(os.Stderr, " Processing %d references\n", count) - os.Stderr.Sync() + fmt.Fprintf(os.Stdout, " Processing %d references\n", count) + os.Stdout.Sync() resps, err := private.HookPostReceive(repoUser, repoName, hookOptions) if resps == nil { @@ -257,8 +286,8 @@ Gitea or set your environment appropriately.`, "") } results = append(results, resps...) - fmt.Fprintf(os.Stderr, "Processed %d references in total\n", total) - os.Stderr.Sync() + fmt.Fprintf(os.Stdout, "Processed %d references in total\n", total) + os.Stdout.Sync() hookPrintResults(results) From 0114ae21b1b40cb330167f843f98301ab513a8d1 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 3 Nov 2019 18:59:51 +0000 Subject: [PATCH 09/19] Make any non-200 status code from pre-receive an error --- cmd/hook.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/hook.go b/cmd/hook.go index b6d188c0c91b..d53ac06e3518 100644 --- a/cmd/hook.go +++ b/cmd/hook.go @@ -132,9 +132,11 @@ Gitea or set your environment appropriately.`, "") hookOptions.RefFullNames = refFullNames statusCode, msg := private.HookPreReceive(username, reponame, hookOptions) switch statusCode { + case http.StatusOK: + // no-op case http.StatusInternalServerError: fail("Internal Server Error", msg) - case http.StatusForbidden: + default: fail(msg, "") } count = 0 From 83ef52dc627ddb7978788e4903721c3694c2a5af Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 3 Nov 2019 19:05:06 +0000 Subject: [PATCH 10/19] Add missing hookPrintResults --- cmd/hook.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/hook.go b/cmd/hook.go index d53ac06e3518..d09e3915f091 100644 --- a/cmd/hook.go +++ b/cmd/hook.go @@ -267,10 +267,10 @@ Gitea or set your environment appropriately.`, "") } if count == 0 { - fmt.Fprintf(os.Stdout, "Processed %d references in total\n", total) os.Stdout.Sync() + hookPrintResults(results) return nil } From f177f164656661dc3f3fbdfaf04b6552ac94adbc Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 3 Nov 2019 19:20:56 +0000 Subject: [PATCH 11/19] Remove shortcut for single action --- models/repo_watch.go | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/models/repo_watch.go b/models/repo_watch.go index cee32be7392d..3992828a2000 100644 --- a/models/repo_watch.go +++ b/models/repo_watch.go @@ -210,39 +210,6 @@ func notifyWatchers(e Engine, actions ...*Action) error { } } - if len(actions) == 1 { - // More efficient to just check the code and not cache - for _, watcher := range watchers { - if act.ActUserID == watcher.UserID { - continue - } - - act.ID = 0 - act.UserID = watcher.UserID - act.Repo.Units = nil - - switch act.OpType { - case ActionCommitRepo, ActionPushTag, ActionDeleteTag, ActionDeleteBranch: - if !act.Repo.checkUnitUser(e, act.UserID, false, UnitTypeCode) { - continue - } - case ActionCreateIssue, ActionCommentIssue, ActionCloseIssue, ActionReopenIssue: - if !act.Repo.checkUnitUser(e, act.UserID, false, UnitTypeIssues) { - continue - } - case ActionCreatePullRequest, ActionMergePullRequest, ActionClosePullRequest, ActionReopenPullRequest: - if !act.Repo.checkUnitUser(e, act.UserID, false, UnitTypePullRequests) { - continue - } - } - - if _, err = e.InsertOne(act); err != nil { - return fmt.Errorf("insert new action: %v", err) - } - } - return nil - } - if repoChanged { permCode = make([]bool, len(watchers)) permIssue = make([]bool, len(watchers)) From c0bb1f0fc69fe20ad80ae1a188841ba90a1f6913 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 3 Nov 2019 21:19:21 +0000 Subject: [PATCH 12/19] mistaken merge fix --- modules/repofiles/action.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/repofiles/action.go b/modules/repofiles/action.go index a09a48df8318..6507d39ea03a 100644 --- a/modules/repofiles/action.go +++ b/modules/repofiles/action.go @@ -185,6 +185,10 @@ func CommitRepoAction(optsList ...CommitRepoActionOptions) error { return fmt.Errorf("GetRepositoryByName [owner_id: %d, name: %s]: %v", opts.RepoOwnerID, opts.RepoName, err) } } + repo, err = models.GetRepositoryByName(opts.RepoOwnerID, opts.RepoName) + if err != nil { + return fmt.Errorf("GetRepositoryByName [owner_id: %d, name: %s]: %v", opts.RepoOwnerID, opts.RepoName, err) + } } refName := git.RefEndName(opts.RefFullName) From 52bc4ce3d0b443d3f8afbf068bb1eda45212b9a7 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Mon, 11 Nov 2019 21:12:10 +0000 Subject: [PATCH 13/19] oops --- modules/repofiles/action.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/modules/repofiles/action.go b/modules/repofiles/action.go index 6507d39ea03a..5cb64d3ca01f 100644 --- a/modules/repofiles/action.go +++ b/modules/repofiles/action.go @@ -180,10 +180,6 @@ func CommitRepoAction(optsList ...CommitRepoActionOptions) error { if err := models.UpdateRepository(repo, false); err != nil { return fmt.Errorf("UpdateRepository: %v", err) } - repo, err = models.GetRepositoryByName(opts.RepoOwnerID, opts.RepoName) - if err != nil { - return fmt.Errorf("GetRepositoryByName [owner_id: %d, name: %s]: %v", opts.RepoOwnerID, opts.RepoName, err) - } } repo, err = models.GetRepositoryByName(opts.RepoOwnerID, opts.RepoName) if err != nil { From ad0f4559c4dfdd40909bca503bcf22760e9c750b Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 22 Dec 2019 19:11:11 +0000 Subject: [PATCH 14/19] Move master branch to the front --- modules/repofiles/update.go | 4 ++-- routers/private/hook.go | 39 +++++++++++++++++++++++-------------- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/modules/repofiles/update.go b/modules/repofiles/update.go index 29ee0038e525..80c838732f63 100644 --- a/modules/repofiles/update.go +++ b/modules/repofiles/update.go @@ -474,7 +474,7 @@ func PushUpdate(repo *models.Repository, branch string, opts PushUpdateOptions) } // PushUpdates generates push action history feeds for push updating multiple refs -func PushUpdates(repo *models.Repository, optsList []PushUpdateOptions) error { +func PushUpdates(repo *models.Repository, optsList []*PushUpdateOptions) error { actions := make([]CommitRepoActionOptions, len(optsList)) repoPath := repo.RepoPath() @@ -491,7 +491,7 @@ func PushUpdates(repo *models.Repository, optsList []PushUpdateOptions) error { } for i, opts := range optsList { - commitRepoActionOptions, err := createCommitRepoActionOption(repo, gitRepo, &opts) + commitRepoActionOptions, err := createCommitRepoActionOption(repo, gitRepo, opts) if err != nil { return err } diff --git a/routers/private/hook.go b/routers/private/hook.go index 7e0c9bf6dee0..6c8fb4ff69b1 100644 --- a/routers/private/hook.go +++ b/routers/private/hook.go @@ -132,7 +132,7 @@ func HookPostReceive(ctx *macaron.Context, opts private.HookOptions) { repoName := ctx.Params(":repo") var repo *models.Repository - updates := make([]repofiles.PushUpdateOptions, 0, len(opts.OldCommitIDs)) + updates := make([]*repofiles.PushUpdateOptions, 0, len(opts.OldCommitIDs)) for i := range opts.OldCommitIDs { refFullName := opts.RefFullNames[i] @@ -164,7 +164,8 @@ func HookPostReceive(ctx *macaron.Context, opts private.HookOptions) { repo.OwnerName = ownerName } } - updates = append(updates, repofiles.PushUpdateOptions{ + + option := repofiles.PushUpdateOptions{ RefFullName: refFullName, OldCommitID: opts.OldCommitIDs[i], NewCommitID: opts.NewCommitIDs[i], @@ -173,23 +174,31 @@ func HookPostReceive(ctx *macaron.Context, opts private.HookOptions) { PusherName: opts.UserName, RepoUserName: ownerName, RepoName: repoName, - }) + } + updates = append(updates, &option) + if repo.IsEmpty && branch == "master" && strings.HasPrefix(refFullName, git.BranchPrefix) { + // put the master branch first + copy(updates[1:], updates) + updates[0] = &option + } } } - if err := repofiles.PushUpdates(repo, updates); err != nil { - log.Error("Failed to Update: %s/%s Total Updates: %d", ownerName, repoName, len(updates)) - for i, update := range updates { - log.Error("Failed to Update: %s/%s Update: %d/%d: Branch: %s", ownerName, repoName, i, len(updates), update.Branch) - } - log.Error("Failed to Update: %s/%s Error: %v", ownerName, repoName, err) + if repo != nil && len(updates) > 0 { + if err := repofiles.PushUpdates(repo, updates); err != nil { + log.Error("Failed to Update: %s/%s Total Updates: %d", ownerName, repoName, len(updates)) + for i, update := range updates { + log.Error("Failed to Update: %s/%s Update: %d/%d: Branch: %s", ownerName, repoName, i, len(updates), update.Branch) + } + log.Error("Failed to Update: %s/%s Error: %v", ownerName, repoName, err) - ctx.JSON(http.StatusInternalServerError, []private.HookPostReceiveResult{ - { - Err: fmt.Sprintf("Failed to Update: %s/%s Error: %v", ownerName, repoName, err), - }, - }) - return + ctx.JSON(http.StatusInternalServerError, []private.HookPostReceiveResult{ + { + Err: fmt.Sprintf("Failed to Update: %s/%s Error: %v", ownerName, repoName, err), + }, + }) + return + } } results := make([]private.HookPostReceiveResult, 0, len(opts.OldCommitIDs)) From dad23e78ff5cbd5ddb0ad85d6ee1794ca1ce1302 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 22 Dec 2019 19:55:18 +0000 Subject: [PATCH 15/19] If repo was empty and the master branch is pushed ensure that that is set as the default branch --- cmd/hook.go | 24 +++++++-- modules/private/hook.go | 36 ++++++++++++-- routers/private/hook.go | 99 ++++++++++++++++++++++++++----------- routers/private/internal.go | 1 + 4 files changed, 123 insertions(+), 37 deletions(-) diff --git a/cmd/hook.go b/cmd/hook.go index d09e3915f091..b8dec9164049 100644 --- a/cmd/hook.go +++ b/cmd/hook.go @@ -228,7 +228,9 @@ Gitea or set your environment appropriately.`, "") refFullNames := make([]string, hookBatchSize) count := 0 total := 0 - results := make([]private.HookPostReceiveResult, 0) + wasEmpty := false + masterPushed := false + results := make([]private.HookPostReceiveBranchResult, 0) scanner := bufio.NewScanner(os.Stdin) for scanner.Scan() { @@ -246,6 +248,9 @@ Gitea or set your environment appropriately.`, "") oldCommitIDs[count] = string(fields[0]) newCommitIDs[count] = string(fields[1]) refFullNames[count] = string(fields[2]) + if refFullNames[count] == git.BranchPrefix+"master" && newCommitIDs[count] != git.EmptySHA && count == total { + masterPushed = true + } count++ total++ os.Stdout.Sync() @@ -256,16 +261,25 @@ Gitea or set your environment appropriately.`, "") hookOptions.OldCommitIDs = oldCommitIDs hookOptions.NewCommitIDs = newCommitIDs hookOptions.RefFullNames = refFullNames - resps, err := private.HookPostReceive(repoUser, repoName, hookOptions) - if resps == nil { + resp, err := private.HookPostReceive(repoUser, repoName, hookOptions) + if resp == nil { hookPrintResults(results) fail("Internal Server Error", err) } - results = append(results, resps...) + wasEmpty = wasEmpty || resp.RepoWasEmpty + results = append(results, resp.Results...) count = 0 } } + if wasEmpty && masterPushed { + // We need to tell the repo to reset the default branch to master + err := private.SetDefaultBranch(repoUser, repoName, "master") + if err != nil { + fail("Internal Server Error", err) + } + } + if count == 0 { fmt.Fprintf(os.Stdout, "Processed %d references in total\n", total) os.Stdout.Sync() @@ -296,7 +310,7 @@ Gitea or set your environment appropriately.`, "") return nil } -func hookPrintResults(results []private.HookPostReceiveResult) { +func hookPrintResults(results []private.HookPostReceiveBranchResult) { for _, res := range results { if !res.Message { continue diff --git a/modules/private/hook.go b/modules/private/hook.go index 798cffe34528..f9fa3bb6b9df 100644 --- a/modules/private/hook.go +++ b/modules/private/hook.go @@ -37,11 +37,17 @@ type HookOptions struct { // HookPostReceiveResult represents an individual result from PostReceive type HookPostReceiveResult struct { + Results []HookPostReceiveBranchResult + RepoWasEmpty bool + Err string +} + +// HookPostReceiveBranchResult represents an individual branch result from PostReceive +type HookPostReceiveBranchResult struct { Message bool Create bool Branch string URL string - Err string } // HookPreReceive check whether the provided commits are allowed @@ -69,7 +75,7 @@ func HookPreReceive(ownerName, repoName string, opts HookOptions) (int, string) } // HookPostReceive updates services and users -func HookPostReceive(ownerName, repoName string, opts HookOptions) ([]HookPostReceiveResult, string) { +func HookPostReceive(ownerName, repoName string, opts HookOptions) (*HookPostReceiveResult, string) { reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/post-receive/%s/%s", url.PathEscape(ownerName), url.PathEscape(repoName), @@ -89,8 +95,30 @@ func HookPostReceive(ownerName, repoName string, opts HookOptions) ([]HookPostRe if resp.StatusCode != http.StatusOK { return nil, decodeJSONError(resp).Err } - res := []HookPostReceiveResult{} - _ = json.NewDecoder(resp.Body).Decode(&res) + res := &HookPostReceiveResult{} + _ = json.NewDecoder(resp.Body).Decode(res) return res, "" } + +// SetDefaultBranch will set the default branch to the provided branch for the provided repository +func SetDefaultBranch(ownerName, repoName, branch string) error { + reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/set-default-branch/%s/%s/%s", + url.PathEscape(ownerName), + url.PathEscape(repoName), + url.PathEscape(branch), + ) + req := newInternalRequest(reqURL, "POST") + req = req.Header("Content-Type", "application/json") + + req.SetTimeout(60*time.Second, time.Duration(60+len(opts.OldCommitIDs))*time.Second) + resp, err := req.Response() + if err != nil { + return fmt.Errorf("Unable to contact gitea: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("Error returned from gitea: %v", decodeJSONError(resp).Err) + } + return nil +} diff --git a/routers/private/hook.go b/routers/private/hook.go index 6c8fb4ff69b1..5ed09a898e00 100644 --- a/routers/private/hook.go +++ b/routers/private/hook.go @@ -133,6 +133,7 @@ func HookPostReceive(ctx *macaron.Context, opts private.HookOptions) { var repo *models.Repository updates := make([]*repofiles.PushUpdateOptions, 0, len(opts.OldCommitIDs)) + wasEmpty := false for i := range opts.OldCommitIDs { refFullName := opts.RefFullNames[i] @@ -153,16 +154,15 @@ func HookPostReceive(ctx *macaron.Context, opts private.HookOptions) { repo, err = models.GetRepositoryByOwnerAndName(ownerName, repoName) if err != nil { log.Error("Failed to get repository: %s/%s Error: %v", ownerName, repoName, err) - ctx.JSON(http.StatusInternalServerError, []private.HookPostReceiveResult{ - { - Err: fmt.Sprintf("Failed to get repository: %s/%s Error: %v", ownerName, repoName, err), - }, + ctx.JSON(http.StatusInternalServerError, private.HookPostReceiveResult{ + Err: fmt.Sprintf("Failed to get repository: %s/%s Error: %v", ownerName, repoName, err), }) return } if repo.OwnerName == "" { repo.OwnerName = ownerName } + wasEmpty = repo.IsEmpty } option := repofiles.PushUpdateOptions{ @@ -192,16 +192,14 @@ func HookPostReceive(ctx *macaron.Context, opts private.HookOptions) { } log.Error("Failed to Update: %s/%s Error: %v", ownerName, repoName, err) - ctx.JSON(http.StatusInternalServerError, []private.HookPostReceiveResult{ - { - Err: fmt.Sprintf("Failed to Update: %s/%s Error: %v", ownerName, repoName, err), - }, + ctx.JSON(http.StatusInternalServerError, private.HookPostReceiveResult{ + Err: fmt.Sprintf("Failed to Update: %s/%s Error: %v", ownerName, repoName, err), }) return } } - results := make([]private.HookPostReceiveResult, 0, len(opts.OldCommitIDs)) + results := make([]private.HookPostReceiveBranchResult, 0, len(opts.OldCommitIDs)) // We have to reload the repo in case its state is changed above repo = nil @@ -224,10 +222,9 @@ func HookPostReceive(ctx *macaron.Context, opts private.HookOptions) { repo, err = models.GetRepositoryByOwnerAndName(ownerName, repoName) if err != nil { log.Error("Failed to get repository: %s/%s Error: %v", ownerName, repoName, err) - ctx.JSON(http.StatusInternalServerError, []private.HookPostReceiveResult{ - { - Err: fmt.Sprintf("Failed to get repository: %s/%s Error: %v", ownerName, repoName, err), - }, + ctx.JSON(http.StatusInternalServerError, private.HookPostReceiveResult{ + Err: fmt.Sprintf("Failed to get repository: %s/%s Error: %v", ownerName, repoName, err), + RepoWasEmpty: wasEmpty, }) return } @@ -238,8 +235,8 @@ func HookPostReceive(ctx *macaron.Context, opts private.HookOptions) { pullRequestAllowed := repo.AllowsPulls() if !pullRequestAllowed { // We can stop there's no need to go any further - ctx.JSON(http.StatusOK, []private.HookPostReceiveResult{ - {}, + ctx.JSON(http.StatusOK, private.HookPostReceiveResult{ + RepoWasEmpty: wasEmpty, }) return } @@ -248,10 +245,9 @@ func HookPostReceive(ctx *macaron.Context, opts private.HookOptions) { if repo.IsFork { if err := repo.GetBaseRepo(); err != nil { log.Error("Failed to get Base Repository of Forked repository: %-v Error: %v", repo, err) - ctx.JSON(http.StatusInternalServerError, []private.HookPostReceiveResult{ - { - Err: fmt.Sprintf("Failed to get Base Repository of Forked repository: %-v Error: %v", repo, err), - }, + ctx.JSON(http.StatusInternalServerError, private.HookPostReceiveResult{ + Err: fmt.Sprintf("Failed to get Base Repository of Forked repository: %-v Error: %v", repo, err), + RepoWasEmpty: wasEmpty, }) return } @@ -260,18 +256,17 @@ func HookPostReceive(ctx *macaron.Context, opts private.HookOptions) { } if !repo.IsFork && branch == baseRepo.DefaultBranch { - results = append(results, private.HookPostReceiveResult{}) + results = append(results, private.HookPostReceiveBranchResult{}) continue } pr, err := models.GetUnmergedPullRequest(repo.ID, baseRepo.ID, branch, baseRepo.DefaultBranch) if err != nil && !models.IsErrPullRequestNotExist(err) { log.Error("Failed to get active PR in: %-v Branch: %s to: %-v Branch: %s Error: %v", repo, branch, baseRepo, baseRepo.DefaultBranch, err) - ctx.JSON(http.StatusInternalServerError, []private.HookPostReceiveResult{ - { - Err: fmt.Sprintf( - "Failed to get active PR in: %-v Branch: %s to: %-v Branch: %s Error: %v", repo, branch, baseRepo, baseRepo.DefaultBranch, err), - }, + ctx.JSON(http.StatusInternalServerError, private.HookPostReceiveResult{ + Err: fmt.Sprintf( + "Failed to get active PR in: %-v Branch: %s to: %-v Branch: %s Error: %v", repo, branch, baseRepo, baseRepo.DefaultBranch, err), + RepoWasEmpty: wasEmpty, }) return } @@ -280,14 +275,14 @@ func HookPostReceive(ctx *macaron.Context, opts private.HookOptions) { if repo.IsFork { branch = fmt.Sprintf("%s:%s", repo.OwnerName, branch) } - results = append(results, private.HookPostReceiveResult{ + results = append(results, private.HookPostReceiveBranchResult{ Message: true, Create: true, Branch: branch, URL: fmt.Sprintf("%s/compare/%s...%s", baseRepo.HTMLURL(), util.PathEscapeSegments(baseRepo.DefaultBranch), util.PathEscapeSegments(branch)), }) } else { - results = append(results, private.HookPostReceiveResult{ + results = append(results, private.HookPostReceiveBranchResult{ Message: true, Create: false, Branch: branch, @@ -296,5 +291,53 @@ func HookPostReceive(ctx *macaron.Context, opts private.HookOptions) { } } } - ctx.JSON(http.StatusOK, results) + ctx.JSON(http.StatusOK, private.HookPostReceiveResult{ + Results: results, + RepoWasEmpty: wasEmpty, + }) +} + +// SetDefaultBranch updates the default branch +func SetDefaultBranch(ctx *macaron.Context, opts private.HookOptions) { + ownerName := ctx.Params(":owner") + repoName := ctx.Params(":repo") + branch := ctx.Params(":branch") + repo, err := models.GetRepositoryByOwnerAndName(ownerName, repoName) + if err != nil { + log.Error("Failed to get repository: %s/%s Error: %v", ownerName, repoName, err) + ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ + "Err": fmt.Sprintf("Failed to get repository: %s/%s Error: %v", ownerName, repoName, err), + }) + return + } + if repo.OwnerName == "" { + repo.OwnerName = ownerName + } + + repo.DefaultBranch = branch + gitRepo, err := git.OpenRepository(repo.RepoPath()) + if err != nil { + ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ + "Err": fmt.Sprintf("Failed to get git repository: %s/%s Error: %v", ownerName, repoName, err), + }) + return + } + if err := gitRepo.SetDefaultBranch(repo.DefaultBranch); err != nil { + if !git.IsErrUnsupportedVersion(err) { + gitRepo.Close() + ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ + "Err": fmt.Sprintf("Unable to set default branch onrepository: %s/%s Error: %v", ownerName, repoName, err), + }) + return + } + } + gitRepo.Close() + + if err := repo.UpdateDefaultBranch(); err != nil { + ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ + "Err": fmt.Sprintf("Unable to set default branch onrepository: %s/%s Error: %v", ownerName, repoName, err), + }) + return + } + ctx.PlainText(200, []byte("success")) } diff --git a/routers/private/internal.go b/routers/private/internal.go index 7a60209b4ace..a7a5a14b4b21 100644 --- a/routers/private/internal.go +++ b/routers/private/internal.go @@ -84,6 +84,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/ssh/:id/update/:repoid", UpdatePublicKeyInRepo) m.Post("/hook/pre-receive/:owner/:repo", bind(private.HookOptions{}), HookPreReceive) m.Post("/hook/post-receive/:owner/:repo", bind(private.HookOptions{}), HookPostReceive) + m.Post("/hook/set-default-branch/:owner/:repo/:branch", SetDefaultBranch) m.Get("/serv/none/:keyid", ServNoCommand) m.Get("/serv/command/:keyid/:owner/:repo", ServCommand) }, CheckInternalToken) From c633620c50f383e2301b2839a407bf3de97e6154 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 22 Dec 2019 20:32:51 +0000 Subject: [PATCH 16/19] fixup --- modules/private/hook.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/private/hook.go b/modules/private/hook.go index f9fa3bb6b9df..3908773e221b 100644 --- a/modules/private/hook.go +++ b/modules/private/hook.go @@ -111,7 +111,7 @@ func SetDefaultBranch(ownerName, repoName, branch string) error { req := newInternalRequest(reqURL, "POST") req = req.Header("Content-Type", "application/json") - req.SetTimeout(60*time.Second, time.Duration(60+len(opts.OldCommitIDs))*time.Second) + req.SetTimeout(60*time.Second, time.Duration(60*time.Second) resp, err := req.Response() if err != nil { return fmt.Errorf("Unable to contact gitea: %v", err) From 3d62415367b7b4fe922a9c879472fb4ec2372a37 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 22 Dec 2019 20:39:36 +0000 Subject: [PATCH 17/19] fixup --- cmd/hook.go | 30 +++++++++++++++++++----------- modules/private/hook.go | 2 +- routers/private/hook.go | 10 ++-------- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/cmd/hook.go b/cmd/hook.go index b8dec9164049..1951394c9f0d 100644 --- a/cmd/hook.go +++ b/cmd/hook.go @@ -272,15 +272,14 @@ Gitea or set your environment appropriately.`, "") } } - if wasEmpty && masterPushed { - // We need to tell the repo to reset the default branch to master - err := private.SetDefaultBranch(repoUser, repoName, "master") - if err != nil { - fail("Internal Server Error", err) - } - } - if count == 0 { + if wasEmpty && masterPushed { + // We need to tell the repo to reset the default branch to master + err := private.SetDefaultBranch(repoUser, repoName, "master") + if err != nil { + fail("Internal Server Error", "SetDefaultBranch failed with Error: %v", err) + } + } fmt.Fprintf(os.Stdout, "Processed %d references in total\n", total) os.Stdout.Sync() @@ -295,16 +294,25 @@ Gitea or set your environment appropriately.`, "") fmt.Fprintf(os.Stdout, " Processing %d references\n", count) os.Stdout.Sync() - resps, err := private.HookPostReceive(repoUser, repoName, hookOptions) - if resps == nil { + resp, err := private.HookPostReceive(repoUser, repoName, hookOptions) + if resp == nil { hookPrintResults(results) fail("Internal Server Error", err) } - results = append(results, resps...) + wasEmpty = wasEmpty || resp.RepoWasEmpty + results = append(results, resp.Results...) fmt.Fprintf(os.Stdout, "Processed %d references in total\n", total) os.Stdout.Sync() + if wasEmpty && masterPushed { + // We need to tell the repo to reset the default branch to master + err := private.SetDefaultBranch(repoUser, repoName, "master") + if err != nil { + fail("Internal Server Error", "SetDefaultBranch failed with Error: %v", err) + } + } + hookPrintResults(results) return nil diff --git a/modules/private/hook.go b/modules/private/hook.go index 3908773e221b..010fc4d72453 100644 --- a/modules/private/hook.go +++ b/modules/private/hook.go @@ -111,7 +111,7 @@ func SetDefaultBranch(ownerName, repoName, branch string) error { req := newInternalRequest(reqURL, "POST") req = req.Header("Content-Type", "application/json") - req.SetTimeout(60*time.Second, time.Duration(60*time.Second) + req.SetTimeout(60*time.Second, 60*time.Second) resp, err := req.Response() if err != nil { return fmt.Errorf("Unable to contact gitea: %v", err) diff --git a/routers/private/hook.go b/routers/private/hook.go index 5ed09a898e00..243cca8b19f9 100644 --- a/routers/private/hook.go +++ b/routers/private/hook.go @@ -209,12 +209,7 @@ func HookPostReceive(ctx *macaron.Context, opts private.HookOptions) { refFullName := opts.RefFullNames[i] newCommitID := opts.NewCommitIDs[i] - branch := opts.RefFullNames[i] - if strings.HasPrefix(branch, git.BranchPrefix) { - branch = strings.TrimPrefix(branch, git.BranchPrefix) - } else { - branch = strings.TrimPrefix(branch, git.TagPrefix) - } + branch := git.RefEndName(opts.RefFullNames[i]) if newCommitID != git.EmptySHA && strings.HasPrefix(refFullName, git.BranchPrefix) { if repo == nil { @@ -232,8 +227,7 @@ func HookPostReceive(ctx *macaron.Context, opts private.HookOptions) { repo.OwnerName = ownerName } - pullRequestAllowed := repo.AllowsPulls() - if !pullRequestAllowed { + if !repo.AllowsPulls() { // We can stop there's no need to go any further ctx.JSON(http.StatusOK, private.HookPostReceiveResult{ RepoWasEmpty: wasEmpty, From 1613c8004d5bbcdba02951b263253e6420fe94b6 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 22 Dec 2019 20:55:07 +0000 Subject: [PATCH 18/19] Missed HookOptions in setdefaultbranch --- routers/private/hook.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/private/hook.go b/routers/private/hook.go index 243cca8b19f9..dc5001ad4e5a 100644 --- a/routers/private/hook.go +++ b/routers/private/hook.go @@ -292,7 +292,7 @@ func HookPostReceive(ctx *macaron.Context, opts private.HookOptions) { } // SetDefaultBranch updates the default branch -func SetDefaultBranch(ctx *macaron.Context, opts private.HookOptions) { +func SetDefaultBranch(ctx *macaron.Context) { ownerName := ctx.Params(":owner") repoName := ctx.Params(":repo") branch := ctx.Params(":branch") From ac282e6765d6039b68d6ce08dcfed6145f81e577 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Wed, 25 Dec 2019 13:44:05 +0000 Subject: [PATCH 19/19] Batch PushUpdateAddTag and PushUpdateDelTag --- models/update.go | 179 +++++++++++++++++++++++++++++++ modules/repofiles/action.go | 2 +- modules/repofiles/action_test.go | 4 +- modules/repofiles/update.go | 78 ++++++++++++-- 4 files changed, 250 insertions(+), 13 deletions(-) diff --git a/models/update.go b/models/update.go index deac91b6dcfa..1105c9a82895 100644 --- a/models/update.go +++ b/models/update.go @@ -53,6 +53,66 @@ func ListToPushCommits(l *list.List) *PushCommits { return &PushCommits{l.Len(), commits, "", make(map[string]string), make(map[string]*User)} } +// PushUpdateAddDeleteTags updates a number of added and delete tags +func PushUpdateAddDeleteTags(repo *Repository, gitRepo *git.Repository, addTags, delTags []string) error { + sess := x.NewSession() + defer sess.Close() + if err := sess.Begin(); err != nil { + return fmt.Errorf("Unable to begin sess in PushUpdateDeleteTags: %v", err) + } + if err := pushUpdateDeleteTags(sess, repo, delTags); err != nil { + return err + } + if err := pushUpdateAddTags(sess, repo, gitRepo, addTags); err != nil { + return err + } + + return sess.Commit() +} + +// PushUpdateDeleteTags updates a number of delete tags +func PushUpdateDeleteTags(repo *Repository, tags []string) error { + sess := x.NewSession() + defer sess.Close() + if err := sess.Begin(); err != nil { + return fmt.Errorf("Unable to begin sess in PushUpdateDeleteTags: %v", err) + } + if err := pushUpdateDeleteTags(sess, repo, tags); err != nil { + return err + } + + return sess.Commit() +} + +func pushUpdateDeleteTags(e Engine, repo *Repository, tags []string) error { + if len(tags) == 0 { + return nil + } + lowerTags := make([]string, 0, len(tags)) + for _, tag := range tags { + lowerTags = append(lowerTags, strings.ToLower(tag)) + } + + if _, err := e. + Where("repo_id = ? AND is_tag = ?", repo.ID, true). + In("lower_tag_name", lowerTags). + Delete(new(Release)); err != nil { + return fmt.Errorf("Delete: %v", err) + } + + if _, err := e. + Where("repo_id = ? AND is_tag = ?", repo.ID, false). + In("lower_tag_name", lowerTags). + SetExpr("is_draft", true). + SetExpr("num_commits", 0). + SetExpr("sha1", ""). + Update(new(Release)); err != nil { + return fmt.Errorf("Update: %v", err) + } + + return nil +} + // PushUpdateDeleteTag must be called for any push actions to delete tag func PushUpdateDeleteTag(repo *Repository, tagName string) error { rel, err := GetRelease(repo.ID, tagName) @@ -78,6 +138,125 @@ func PushUpdateDeleteTag(repo *Repository, tagName string) error { return nil } +// PushUpdateAddTags updates a number of add tags +func PushUpdateAddTags(repo *Repository, gitRepo *git.Repository, tags []string) error { + sess := x.NewSession() + defer sess.Close() + if err := sess.Begin(); err != nil { + return fmt.Errorf("Unable to begin sess in PushUpdateAddTags: %v", err) + } + if err := pushUpdateAddTags(sess, repo, gitRepo, tags); err != nil { + return err + } + + return sess.Commit() +} +func pushUpdateAddTags(e Engine, repo *Repository, gitRepo *git.Repository, tags []string) error { + if len(tags) == 0 { + return nil + } + + lowerTags := make([]string, 0, len(tags)) + for _, tag := range tags { + lowerTags = append(lowerTags, strings.ToLower(tag)) + } + + releases := make([]Release, 0, len(tags)) + if err := e.Where("repo_id = ?", repo.ID). + In("lower_tag_name", lowerTags).Find(&releases); err != nil { + return fmt.Errorf("GetRelease: %v", err) + } + relMap := make(map[string]*Release) + for _, rel := range releases { + relMap[rel.LowerTagName] = &rel + } + + newReleases := make([]*Release, 0, len(lowerTags)-len(relMap)) + + emailToUser := make(map[string]*User) + + for i, lowerTag := range lowerTags { + tag, err := gitRepo.GetTag(tags[i]) + if err != nil { + return fmt.Errorf("GetTag: %v", err) + } + commit, err := tag.Commit() + if err != nil { + return fmt.Errorf("Commit: %v", err) + } + + sig := tag.Tagger + if sig == nil { + sig = commit.Author + } + if sig == nil { + sig = commit.Committer + } + var author *User + var createdAt = time.Unix(1, 0) + + if sig != nil { + var ok bool + author, ok = emailToUser[sig.Email] + if !ok { + author, err = GetUserByEmail(sig.Email) + if err != nil && !IsErrUserNotExist(err) { + return fmt.Errorf("GetUserByEmail: %v", err) + } + } + createdAt = sig.When + } + + commitsCount, err := commit.CommitsCount() + if err != nil { + return fmt.Errorf("CommitsCount: %v", err) + } + + rel, has := relMap[lowerTag] + + if !has { + rel = &Release{ + RepoID: repo.ID, + Title: "", + TagName: tags[i], + LowerTagName: lowerTag, + Target: "", + Sha1: commit.ID.String(), + NumCommits: commitsCount, + Note: "", + IsDraft: false, + IsPrerelease: false, + IsTag: true, + CreatedUnix: timeutil.TimeStamp(createdAt.Unix()), + } + if author != nil { + rel.PublisherID = author.ID + } + + newReleases = append(newReleases, rel) + } else { + rel.Sha1 = commit.ID.String() + rel.CreatedUnix = timeutil.TimeStamp(createdAt.Unix()) + rel.NumCommits = commitsCount + rel.IsDraft = false + if rel.IsTag && author != nil { + rel.PublisherID = author.ID + } + if _, err = e.ID(rel.ID).AllCols().Update(rel); err != nil { + return fmt.Errorf("Update: %v", err) + } + } + } + + if len(newReleases) > 0 { + if _, err := e.Insert(newReleases); err != nil { + return fmt.Errorf("Insert: %v", err) + } + } + + return nil +} + // PushUpdateAddTag must be called for any push actions to add tag func PushUpdateAddTag(repo *Repository, gitRepo *git.Repository, tagName string) error { rel, err := GetRelease(repo.ID, tagName) diff --git a/modules/repofiles/action.go b/modules/repofiles/action.go index 5cb64d3ca01f..d207247114a9 100644 --- a/modules/repofiles/action.go +++ b/modules/repofiles/action.go @@ -159,7 +159,7 @@ type CommitRepoActionOptions struct { // CommitRepoAction adds new commit action to the repository, and prepare // corresponding webhooks. -func CommitRepoAction(optsList ...CommitRepoActionOptions) error { +func CommitRepoAction(optsList ...*CommitRepoActionOptions) error { var pusher *models.User var repo *models.Repository actions := make([]*models.Action, len(optsList)) diff --git a/modules/repofiles/action_test.go b/modules/repofiles/action_test.go index 5a4c6231f3b1..97ac1c45e92b 100644 --- a/modules/repofiles/action_test.go +++ b/modules/repofiles/action_test.go @@ -13,7 +13,7 @@ import ( "github.com/stretchr/testify/assert" ) -func testCorrectRepoAction(t *testing.T, opts CommitRepoActionOptions, actionBean *models.Action) { +func testCorrectRepoAction(t *testing.T, opts *CommitRepoActionOptions, actionBean *models.Action) { models.AssertNotExistsBean(t, actionBean) assert.NoError(t, CommitRepoAction(opts)) models.AssertExistsAndLoadBean(t, actionBean) @@ -121,7 +121,7 @@ func TestCommitRepoAction(t *testing.T) { s.action.Repo = repo s.action.IsPrivate = repo.IsPrivate - testCorrectRepoAction(t, s.commitRepoActionOptions, &s.action) + testCorrectRepoAction(t, &s.commitRepoActionOptions, &s.action) } } diff --git a/modules/repofiles/update.go b/modules/repofiles/update.go index 80c838732f63..d39f49d9e851 100644 --- a/modules/repofiles/update.go +++ b/modules/repofiles/update.go @@ -453,7 +453,7 @@ func PushUpdate(repo *models.Repository, branch string, opts PushUpdateOptions) return err } - if err := CommitRepoAction(*commitRepoActionOptions); err != nil { + if err := CommitRepoAction(commitRepoActionOptions); err != nil { return fmt.Errorf("CommitRepoAction: %v", err) } @@ -475,8 +475,6 @@ func PushUpdate(repo *models.Repository, branch string, opts PushUpdateOptions) // PushUpdates generates push action history feeds for push updating multiple refs func PushUpdates(repo *models.Repository, optsList []*PushUpdateOptions) error { - actions := make([]CommitRepoActionOptions, len(optsList)) - repoPath := repo.RepoPath() _, err := git.NewCommand("update-server-info").RunInDir(repoPath) if err != nil { @@ -490,13 +488,9 @@ func PushUpdates(repo *models.Repository, optsList []*PushUpdateOptions) error { log.Error("Failed to update size for repository: %v", err) } - for i, opts := range optsList { - commitRepoActionOptions, err := createCommitRepoActionOption(repo, gitRepo, opts) - if err != nil { - return err - } - - actions[i] = *commitRepoActionOptions + actions, err := createCommitRepoActions(repo, gitRepo, optsList) + if err != nil { + return err } if err := CommitRepoAction(actions...); err != nil { return fmt.Errorf("CommitRepoAction: %v", err) @@ -525,6 +519,70 @@ func PushUpdates(repo *models.Repository, optsList []*PushUpdateOptions) error { return nil } +func createCommitRepoActions(repo *models.Repository, gitRepo *git.Repository, optsList []*PushUpdateOptions) ([]*CommitRepoActionOptions, error) { + addTags := make([]string, 0, len(optsList)) + delTags := make([]string, 0, len(optsList)) + actions := make([]*CommitRepoActionOptions, 0, len(optsList)) + + for _, opts := range optsList { + isNewRef := opts.OldCommitID == git.EmptySHA + isDelRef := opts.NewCommitID == git.EmptySHA + if isNewRef && isDelRef { + return nil, fmt.Errorf("Old and new revisions are both %s", git.EmptySHA) + } + var commits = &models.PushCommits{} + if strings.HasPrefix(opts.RefFullName, git.TagPrefix) { + // If is tag reference + tagName := opts.RefFullName[len(git.TagPrefix):] + if isDelRef { + delTags = append(delTags, tagName) + } else { + cache.Remove(repo.GetCommitsCountCacheKey(tagName, true)) + addTags = append(addTags, tagName) + } + } else if !isDelRef { + // If is branch reference + + // Clear cache for branch commit count + cache.Remove(repo.GetCommitsCountCacheKey(opts.RefFullName[len(git.BranchPrefix):], true)) + + newCommit, err := gitRepo.GetCommit(opts.NewCommitID) + if err != nil { + return nil, fmt.Errorf("gitRepo.GetCommit: %v", err) + } + + // Push new branch. + var l *list.List + if isNewRef { + l, err = newCommit.CommitsBeforeLimit(10) + if err != nil { + return nil, fmt.Errorf("newCommit.CommitsBeforeLimit: %v", err) + } + } else { + l, err = newCommit.CommitsBeforeUntil(opts.OldCommitID) + if err != nil { + return nil, fmt.Errorf("newCommit.CommitsBeforeUntil: %v", err) + } + } + + commits = models.ListToPushCommits(l) + } + actions = append(actions, &CommitRepoActionOptions{ + PusherName: opts.PusherName, + RepoOwnerID: repo.OwnerID, + RepoName: repo.Name, + RefFullName: opts.RefFullName, + OldCommitID: opts.OldCommitID, + NewCommitID: opts.NewCommitID, + Commits: commits, + }) + } + if err := models.PushUpdateAddDeleteTags(repo, gitRepo, addTags, delTags); err != nil { + return nil, fmt.Errorf("PushUpdateAddDeleteTags: %v", err) + } + return actions, nil +} + func createCommitRepoActionOption(repo *models.Repository, gitRepo *git.Repository, opts *PushUpdateOptions) (*CommitRepoActionOptions, error) { isNewRef := opts.OldCommitID == git.EmptySHA isDelRef := opts.NewCommitID == git.EmptySHA