diff --git a/cmd/releasebot/README.md b/cmd/releasebot/README.md deleted file mode 100644 index 382846c597..0000000000 --- a/cmd/releasebot/README.md +++ /dev/null @@ -1,21 +0,0 @@ -# golang.org/x/build/cmd/releasebot - -Command releasebot runs a Go release. - -The release happens in two stages: - -* the `prepare` stage checks preconditions, makes the release commit and mails it for review (if applicable), and performs testing; -* the `release` stage runs after the release commit (if any) is merged, and it tags, builds and cleans up the release. - -## Permissions - -The user running a release will need: - -* A GitHub personal access token with the `public_repo` scope in `~/.github-issue-token`, and an account with write access to golang/go -* gomote access and a token in your name -* gcloud application default credentials, and an account with GCS access to golang-org for bucket golang-release-staging -* **`release-manager` group membership on Gerrit** -* for `-mode=mail-dl-cl` only, Secret Manager access to Gerrit API secret -* for `-mode=tweet-*` only, Secret Manager access to Twitter API secret - -NOTE: all but the Gerrit permission are ensured by the bot on startup. diff --git a/cmd/releasebot/gcs.go b/cmd/releasebot/gcs.go deleted file mode 100644 index 9e4fb138b8..0000000000 --- a/cmd/releasebot/gcs.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2017 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "bytes" - "context" - "crypto/md5" - "fmt" - "io" - "log" - "os" - "time" - - "cloud.google.com/go/storage" -) - -const releaseBucket = "golang-release-staging" - -var gcsClient *storage.Client - -func loadGCSAuth() { - ctx := context.Background() - client, err := storage.NewClient(ctx) - if err != nil { - log.Fatal(err) - } - - // Test early that we can write to the bucket. - name := fmt.Sprintf(".writable/%d", time.Now().UnixNano()) - err = client.Bucket(releaseBucket).Object(name).NewWriter(ctx).Close() - if err != nil { - log.Fatalf("cannot write to %s: %v", releaseBucket, err) - } - err = client.Bucket(releaseBucket).Object(name).Delete(ctx) - if err != nil { - log.Fatalf("cannot delete from %s: %v", releaseBucket, err) - } - - gcsClient = client -} - -func gcsUpload(src, dst string) error { - f, err := os.Open(src) - if err != nil { - return err - } - defer f.Close() - - h := md5.New() - if _, err := io.Copy(h, f); err != nil { - return err - } - sum := h.Sum(nil) - if _, err := f.Seek(0, 0); err != nil { - return err - } - - ctx := context.Background() - obj := gcsClient.Bucket("golang-release-staging").Object(dst) - if attrs, err := obj.Attrs(ctx); err == nil && bytes.Equal(attrs.MD5, sum[:]) { - return nil - } - - cloud := obj.NewWriter(ctx) - if _, err := io.Copy(cloud, f); err != nil { - cloud.Close() - return err - } - if err := cloud.Close(); err != nil { - return err - } - if attrs, err := obj.Attrs(ctx); err != nil || !bytes.Equal(attrs.MD5, sum[:]) { - if err == nil { - err = fmt.Errorf("md5 mismatch") - } - return fmt.Errorf("upload %s: %v", dst, err) - } - - return nil -} diff --git a/cmd/releasebot/gerrit.go b/cmd/releasebot/gerrit.go deleted file mode 100644 index 46a0baf618..0000000000 --- a/cmd/releasebot/gerrit.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "context" - - "golang.org/x/build/buildenv" - "golang.org/x/build/gerrit" - "golang.org/x/build/internal/secret" -) - -const ( - // gerritAPIURL is the Gerrit API URL. - gerritAPIURL = "https://go-review.googlesource.com" -) - -// loadGerritAuth loads Gerrit API credentials. -func loadGerritAuth() (gerrit.Auth, error) { - sc, err := secret.NewClientInProject(buildenv.Production.ProjectName) - if err != nil { - return nil, err - } - defer sc.Close() - token, err := sc.Retrieve(context.Background(), secret.NameGobotPassword) - if err != nil { - return nil, err - } - return gerrit.BasicAuth("git-gobot.golang.org", token), nil -} diff --git a/cmd/releasebot/git.go b/cmd/releasebot/git.go deleted file mode 100644 index baaa2ae1bb..0000000000 --- a/cmd/releasebot/git.go +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright 2017 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "fmt" - "os" - "path/filepath" - "strings" -) - -// gitCheckout sets w.Dir to the work directory -// at $HOME/go-releasebot-work/ (where is a string like go1.8.5), -// creating it if it doesn't exist, -// and sets up a fresh git checkout in which to work, -// in $HOME/go-releasebot-work//gitwork. -// -// The first time it is run for a particular release, -// gitCheckout also creates a clean checkout in -// $HOME/go-releasebot-work//gitmirror, -// to use as an object cache to speed future checkouts. -// -// For beta releases in -mode=release, it sets w.Dir but doesn't -// fetch the latest master branch. This way the exact commit that -// was tested in prepare mode is also used in release mode. -func (w *Work) gitCheckout() { - w.Dir = filepath.Join(os.Getenv("HOME"), "go-releasebot-work/"+strings.ToLower(w.Version)) - w.log.Printf("working in %s\n", w.Dir) - if err := os.MkdirAll(w.Dir, 0777); err != nil { - w.log.Panic(err) - } - - if w.BetaRelease && !w.Prepare { - // We don't want to fetch the latest "master" branch for - // a beta release. Instead, reuse the same commit that was - // already fetched from origin and tested in prepare mode. - return - } - - const origin = "https://go.googlesource.com/go" - - // Check out a local mirror to work-mirror, to speed future checkouts for this point release. - mirror := filepath.Join(w.Dir, "gitmirror") - r := w.runner(mirror) - if _, err := os.Stat(mirror); err != nil { - w.runner(w.Dir).run("git", "clone", origin, mirror) - r.run("git", "config", "gc.auto", "0") // don't throw away refs we fetch - } else { - r.run("git", "fetch", "origin", "master") - } - r.run("git", "fetch", "origin", w.ReleaseBranch) - - // Clone real Gerrit, but using local mirror for most objects. - gitDir := filepath.Join(w.Dir, "gitwork") - if err := os.RemoveAll(gitDir); err != nil { - w.log.Panic(err) - } - w.runner(w.Dir).run("git", "clone", "--reference", mirror, "-b", w.ReleaseBranch, origin, gitDir) - - r = w.runner(gitDir) - r.run("git", "codereview", "change", "relwork") - r.run("git", "config", "gc.auto", "0") // don't throw away refs we fetch -} - -// gitTagExists returns whether a git tag is already present in the repository. -func (w *Work) gitTagExists() bool { - _, err := w.runner(filepath.Join(w.Dir, "gitwork")).runErr("git", "rev-parse", w.Version) - return err == nil -} - -// gitTagVersion tags the release candidate or release in Git. -func (w *Work) gitTagVersion() { - r := w.runner(filepath.Join(w.Dir, "gitwork")) - if w.gitTagExists() { - out := r.runOut("git", "rev-parse", w.Version) - w.VersionCommit = strings.TrimSpace(string(out)) - w.log.Printf("Git tag already exists (%s), resuming release.", w.VersionCommit) - return - } - out := r.runOut("git", "rev-parse", "HEAD") - w.VersionCommit = strings.TrimSpace(string(out)) - out = r.runOut("git", "show", w.VersionCommit) - fmt.Printf("About to tag the following commit as %s:\n\n%s\n\nOk? (Y/n) ", w.Version, out) - if dryRun { - fmt.Println("dry-run") - return - } - var resp string - if _, err := fmt.Scanln(&resp); err != nil { - w.log.Panic(err) - } else if resp != "Y" && resp != "y" { - w.log.Fatal("stopped as requested") - } - out, err := r.runErr("git", "tag", w.Version, w.VersionCommit) - if err != nil { - w.logError("git tag failed: %s\n%s", err, out) - return - } - r.run("git", "push", "origin", w.Version) -} - -// gitHeadCommit returns the hash of the HEAD commit. -func (w *Work) gitHeadCommit() string { - r := w.runner(filepath.Join(w.Dir, "gitwork")) - out := r.runOut("git", "rev-parse", "HEAD") - return strings.TrimSpace(string(out)) -} diff --git a/cmd/releasebot/github.go b/cmd/releasebot/github.go deleted file mode 100644 index f4a920d65c..0000000000 --- a/cmd/releasebot/github.go +++ /dev/null @@ -1,315 +0,0 @@ -// Copyright 2017 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "context" - "errors" - "fmt" - "io/ioutil" - "log" - "net/http" - "os" - "path/filepath" - "strconv" - "strings" - - "github.com/google/go-github/github" - "golang.org/x/build/maintner" - "golang.org/x/build/maintner/godata" - "golang.org/x/oauth2" -) - -const ( - projectOwner = "golang" - projectRepo = "go" -) - -var githubClient *github.Client - -// GitHub personal access token, from https://github.com/settings/applications. -var githubAuthToken string - -var goRepo *maintner.GitHubRepo - -func loadMaintner() { - corpus, err := godata.Get(context.Background()) - if err != nil { - log.Fatal("failed to load maintner data:", err) - } - goRepo = corpus.GitHub().Repo(projectOwner, projectRepo) -} - -func loadGithubAuth() { - const short = ".github-issue-token" - filename := filepath.Clean(os.Getenv("HOME") + "/" + short) - shortFilename := filepath.Clean("$HOME/" + short) - data, err := ioutil.ReadFile(filename) - if err != nil { - log.Fatal("reading token: ", err, "\n\n"+ - "Please create a personal access token at https://github.com/settings/tokens/new\n"+ - "and write it to ", shortFilename, " to use this program.\n"+ - "** The token only needs the public_repo scope. **\n"+ - "The benefit of using a personal access token over using your GitHub\n"+ - "password directly is that you can limit its use and revoke it at any time.\n\n") - } - fi, err := os.Stat(filename) - if err != nil { - log.Fatalln("reading token:", err) - } - if fi.Mode()&0077 != 0 { - log.Fatalf("reading token: %s mode is %#o, want %#o", shortFilename, fi.Mode()&0777, fi.Mode()&0700) - } - githubAuthToken = strings.TrimSpace(string(data)) - t := &oauth2.Transport{ - Source: &tokenSource{AccessToken: githubAuthToken}, - } - githubClient = github.NewClient(&http.Client{Transport: t}) -} - -// releaseStatusTitle returns the title of the release status issue -// for the given milestone. -// If you change this function, releasebot will not be able to find an -// existing tracking issue using the old name and will create a new one. -func (w *Work) releaseStatusTitle() string { - return "all: " + strings.Replace(w.Version, "go", "Go ", -1) + " release status" -} - -type tokenSource oauth2.Token - -func (t *tokenSource) Token() (*oauth2.Token, error) { - return (*oauth2.Token)(t), nil -} - -func (w *Work) findOrCreateReleaseIssue() { - w.log.Printf("Release status issue title: %q", w.releaseStatusTitle()) - if dryRun { - return - } - if w.ReleaseIssue == 0 { - title := w.releaseStatusTitle() - body := fmt.Sprintf("Issue tracking the %s release by releasebot.", w.Version) - num, err := w.createGitHubIssue(title, body) - if err != nil { - w.log.Panic(err) - } - w.ReleaseIssue = num - w.log.Printf("Release status issue: https://golang.org/issue/%d", num) - } -} - -// createGitHubIssue creates an issue in the release milestone and returns its number. -func (w *Work) createGitHubIssue(title, msg string) (int, error) { - if dryRun { - return 0, errors.New("attempted write operation in dry-run mode") - } - var dup int - goRepo.ForeachIssue(func(gi *maintner.GitHubIssue) error { - if gi.Title == title { - dup = int(gi.Number) - return errors.New("stop iteration") - } - return nil - }) - if dup != 0 { - return dup, nil - } - opts := &github.IssueListByRepoOptions{ - State: "all", - ListOptions: github.ListOptions{PerPage: 100}, - } - if !w.BetaRelease && !w.RCRelease { - opts.Milestone = strconv.Itoa(int(w.Milestone.Number)) - } - is, _, err := githubClient.Issues.ListByRepo(context.TODO(), "golang", "go", opts) - if err != nil { - return 0, err - } - for _, i := range is { - if i.GetTitle() == title { - // Dup. - return i.GetNumber(), nil - } - } - copts := &github.IssueRequest{ - Title: github.String(title), - Body: github.String(msg), - } - if !w.BetaRelease && !w.RCRelease { - copts.Milestone = github.Int(int(w.Milestone.Number)) - } - i, _, err := githubClient.Issues.Create(context.TODO(), "golang", "go", copts) - return i.GetNumber(), err -} - -// pushIssues moves open issues to the milestone of the next release of the same kind, -// creating the milestone if it doesn't already exist. -// For major releases, it's the milestone of the next major release (e.g., 1.14 → 1.15). -// For minor releases, it's the milestone of the next minor release (e.g., 1.14.1 → 1.14.2). -// For other release types, it does nothing. -// -// For major releases, it also creates the first minor release milestone if it doesn't already exist. -func (w *Work) pushIssues() { - if w.BetaRelease || w.RCRelease { - // Nothing to do. - return - } - - // Get the milestone for the next release. - var nextMilestone *github.Milestone - nextV, err := nextVersion(w.Version) - if err != nil { - w.logError("error determining next version: %v", err) - return - } - nextMilestone, err = w.findOrCreateMilestone(nextV) - if err != nil { - w.logError("error finding or creating %s, the next GitHub milestone after release %s: %v", nextV, w.Version, err) - return - } - - // For major releases (go1.X), also create the first minor release milestone (go1.X.1). See issue 44404. - if strings.Count(w.Version, ".") == 1 { - firstMinor := w.Version + ".1" - _, err := w.findOrCreateMilestone(firstMinor) - if err != nil { - // Log this error, but continue executing the rest of the task. - w.logError("error finding or creating %s, the first minor release GitHub milestone after major release %s: %v", firstMinor, w.Version, err) - } - } - - if err := goRepo.ForeachIssue(func(gi *maintner.GitHubIssue) error { - if gi.Milestone == nil || gi.Milestone.ID != w.Milestone.ID { - return nil - } - if gi.Number == int32(w.ReleaseIssue) { - return nil - } - if gi.Closed { - return nil - } - w.log.Printf("changing milestone of issue %d to %s", gi.Number, nextMilestone.GetTitle()) - if dryRun { - return nil - } - _, _, err := githubClient.Issues.Edit(context.TODO(), projectOwner, projectRepo, int(gi.Number), &github.IssueRequest{ - Milestone: github.Int(nextMilestone.GetNumber()), - }) - if err != nil { - return fmt.Errorf("#%d: %s", gi.Number, err) - } - return nil - }); err != nil { - w.logError("error moving issues to the next minor release: %v", err) - return - } -} - -// findOrCreateMilestone finds or creates a GitHub milestone corresponding -// to the specified Go version. This is done via the GitHub API, using githubClient. -// If the milestone exists but isn't open, an error is returned. -func (w *Work) findOrCreateMilestone(version string) (*github.Milestone, error) { - // Look for an existing open milestone corresponding to version, - // and return it if found. - for opt := (&github.MilestoneListOptions{ListOptions: github.ListOptions{PerPage: 100}}); ; { - ms, resp, err := githubClient.Issues.ListMilestones(context.Background(), projectOwner, projectRepo, opt) - if err != nil { - return nil, err - } - for _, m := range ms { - if strings.ToLower(m.GetTitle()) == version { - // Found an existing milestone. - return m, nil - } - } - if resp.NextPage == 0 { - break - } - opt.Page = resp.NextPage - } - - // Create a new milestone. - // For historical reasons, Go milestone titles use a capital "Go1.n" format, - // in contrast to go versions which are like "go1.n". Do the same here. - title := strings.Replace(version, "go", "Go", 1) - w.log.Printf("creating milestone titled %q", title) - if dryRun { - return &github.Milestone{Title: github.String(title)}, nil - } - m, _, err := githubClient.Issues.CreateMilestone(context.Background(), projectOwner, projectRepo, &github.Milestone{ - Title: github.String(title), - }) - if e := (*github.ErrorResponse)(nil); errors.As(err, &e) && e.Response != nil && e.Response.StatusCode == http.StatusUnprocessableEntity && len(e.Errors) == 1 && e.Errors[0].Code == "already_exists" { - // We'll run into an already_exists error here if the milestone exists, - // but it wasn't found in the loop above because the milestone isn't open. - // That shouldn't happen under normal circumstances, so if it does, - // let humans figure out how to best deal with it. - return nil, errors.New("a closed milestone with the same title already exists") - } else if err != nil { - return nil, err - } - return m, nil -} - -// closeMilestone closes the milestone for the current release. -func (w *Work) closeMilestone() { - w.log.Printf("closing milestone %s", w.Milestone.Title) - if dryRun { - return - } - closed := "closed" - _, _, err := githubClient.Issues.EditMilestone(context.TODO(), projectOwner, projectRepo, int(w.Milestone.Number), &github.Milestone{ - State: &closed, - }) - if err != nil { - w.logError("closing milestone: %v", err) - } - -} - -// removeOkayAfterBeta1 cleans up the okay-after-beta1 label after the beta1 release is out. -func (w *Work) removeOkayAfterBeta1() { - if !w.BetaRelease || !strings.HasSuffix(w.Version, "beta1") { - // Nothing to do. - return - } - - if err := goRepo.ForeachIssue(func(gi *maintner.GitHubIssue) error { - if gi.Milestone == nil || gi.Milestone.ID != w.Milestone.ID { - return nil - } - if gi.Number == int32(w.ReleaseIssue) { - return nil - } - if gi.Closed || !gi.HasLabel("okay-after-beta1") { - return nil - } - w.log.Printf("removing okay-after-beta1 label in issue %d", gi.Number) - if dryRun { - return nil - } - _, err := githubClient.Issues.RemoveLabelForIssue(context.Background(), - projectOwner, projectRepo, int(gi.Number), "okay-after-beta1") - if err != nil { - return fmt.Errorf("#%d: %s", gi.Number, err) - } - return nil - }); err != nil { - w.logError("error removing okay-after-beta1 label from issues in current milestone: %v", err) - return - } -} - -const githubCommentCharacterLimit = 65536 // discovered in golang.org/issue/45998 - -func postGithubComment(number int, body string) error { - if dryRun { - return errors.New("attempted write operation in dry-run mode") - } - _, _, err := githubClient.Issues.CreateComment(context.TODO(), projectOwner, projectRepo, number, &github.IssueComment{ - Body: &body, - }) - return err -} diff --git a/cmd/releasebot/http.go b/cmd/releasebot/http.go deleted file mode 100644 index 22b2dd0160..0000000000 --- a/cmd/releasebot/http.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2017 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "fmt" - "net/http" - "os" - "strings" - "sync" - "time" -) - -func newLogger(t http.RoundTripper) http.RoundTripper { - return &loggingTransport{transport: t} -} - -type loggingTransport struct { - transport http.RoundTripper - mu sync.Mutex - active []byte -} - -func (t *loggingTransport) RoundTrip(r *http.Request) (*http.Response, error) { - t.mu.Lock() - index := len(t.active) - start := time.Now() - fmt.Fprintf(os.Stderr, "HTTP: %s %s+ %s\n", timeFormat1(start), t.active, r.URL) - t.active = append(t.active, '|') - t.mu.Unlock() - - resp, err := t.transport.RoundTrip(r) - - last := r.URL.Path - if i := strings.LastIndex(last, "/"); i >= 0 { - last = last[i:] - } - display := last - if resp != nil { - display += " " + resp.Status - } - if err != nil { - display += " error: " + err.Error() - } - now := time.Now() - - t.mu.Lock() - t.active[index] = '-' - fmt.Fprintf(os.Stderr, "HTTP: %s %s %s (%.3fs)\n", timeFormat1(now), t.active, display, now.Sub(start).Seconds()) - t.active[index] = ' ' - n := len(t.active) - for n%4 == 0 && n >= 4 && t.active[n-1] == ' ' && t.active[n-2] == ' ' && t.active[n-3] == ' ' && t.active[n-4] == ' ' { - t.active = t.active[:n-4] - n -= 4 - } - t.mu.Unlock() - - return resp, err -} - -func timeFormat1(t time.Time) string { - return t.Format("15:04:05.000") -} diff --git a/cmd/releasebot/main.go b/cmd/releasebot/main.go deleted file mode 100644 index 4d122227a9..0000000000 --- a/cmd/releasebot/main.go +++ /dev/null @@ -1,955 +0,0 @@ -// Copyright 2017 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Releasebot manages the process of defining, -// packaging, and publishing Go releases. -package main - -import ( - "bytes" - "context" - "crypto/sha1" - "crypto/sha256" - "encoding/json" - "errors" - "flag" - "fmt" - "io" - "io/ioutil" - "log" - "net/http" - "os" - "os/exec" - "path/filepath" - "runtime/debug" - "strconv" - "strings" - "sync" - "time" - - "golang.org/x/build/buildenv" - "golang.org/x/build/gerrit" - "golang.org/x/build/internal/envutil" - "golang.org/x/build/internal/releasetargets" - "golang.org/x/build/internal/task" - "golang.org/x/build/internal/workflow" - "golang.org/x/build/maintner" -) - -// A Target is a release target. -type Target struct { - Name string // Target name as accepted by cmd/release. For example, "linux-amd64". - SkipTests bool // Skip tests. -} - -var releaseModes = map[string]bool{ - "prepare": true, - "release": true, - - "mail-dl-cl": true, - - "tweet-minor": true, - "tweet-beta": true, - "tweet-rc": true, - "tweet-major": true, -} - -func usage() { - fmt.Fprintln(os.Stderr, "usage: releasebot -mode {prepare|release|mail-dl-cl|tweet-{minor,beta,rc,major}} [-security] [-dry-run] {go1.8.5|go1.10beta2|go1.11rc1}") - flag.PrintDefaults() - os.Exit(2) -} - -var ( - skipTestFlag = flag.String("skip-test", "", "space-separated list of targets for which to skip tests (only use if sufficient testing was done elsewhere)") - skipTargetFlag = flag.String("skip-target", "", "space-separated list of targets to skip. This will require manual intervention to create artifacts for a target after releasing.") - skipAllTestsFlag = flag.Bool("skip-all-tests", false, "skip all tests for all targets (only use if tests were verified elsewhere)") -) - -var ( - dryRun bool // only perform pre-flight checks, only log to terminal -) - -func main() { - modeFlag := flag.String("mode", "", "release mode (prepare, release)") - flag.BoolVar(&dryRun, "dry-run", false, "only perform pre-flight checks, only log to terminal") - flag.Usage = usage - flag.Parse() - if !releaseModes[*modeFlag] { - fmt.Fprintln(os.Stderr, "need to provide a valid mode") - usage() - } else if *modeFlag == "mail-dl-cl" { - mailDLCL() - return - } else if strings.HasPrefix(*modeFlag, "tweet-") { - kind := (*modeFlag)[len("tweet-"):] - postTweet(kind) - return - } else if flag.NArg() != 1 { - fmt.Fprintln(os.Stderr, "need to provide a release name") - usage() - } - releaseVersion := flag.Arg(0) - releaseTargets, ok := releasetargets.TargetsForVersion(releaseVersion) - if !ok { - fmt.Fprintf(os.Stderr, "could not parse release name %q\n", releaseVersion) - usage() - } - for _, target := range strings.Fields(*skipTestFlag) { - t, ok := releaseTargets[target] - if !ok { - fmt.Fprintf(os.Stderr, "target %q in -skip-test=%q is not a known target\n", target, *skipTestFlag) - usage() - } - t.LongTestBuilder = "" - t.BuildOnly = true - } - for _, target := range strings.Fields(*skipTargetFlag) { - if _, ok := releaseTargets[target]; !ok { - fmt.Fprintf(os.Stderr, "target %q in -skip-target=%q is not a known target\n", target, *skipTargetFlag) - usage() - } - delete(releaseTargets, target) - } - - http.DefaultTransport = newLogger(http.DefaultTransport) - - buildenv.CheckUserCredentials() - checkForGitCodereview() - loadMaintner() - loadGomoteUser() - loadGithubAuth() - loadGCSAuth() - - w := &Work{ - Prepare: *modeFlag == "prepare", - Version: releaseVersion, - BetaRelease: strings.Contains(releaseVersion, "beta"), - RCRelease: strings.Contains(releaseVersion, "rc"), - } - - // Validate release version types. - if w.BetaRelease { - w.ReleaseBranch = "master" - } else if w.RCRelease { - shortRel := strings.Split(w.Version, "rc")[0] - w.ReleaseBranch = "release-branch." + shortRel - } else if strings.Count(w.Version, ".") == 1 { - // Major release like "go1.X". - w.ReleaseBranch = "release-branch." + w.Version - } else if strings.Count(w.Version, ".") == 2 { - // Minor release or security release like "go1.X.Y". - shortRel := w.Version[:strings.LastIndex(w.Version, ".")] - w.ReleaseBranch = "release-branch." + shortRel - } else { - log.Fatalf("cannot understand version %q", w.Version) - } - - w.ReleaseTargets = []Target{{Name: "src"}} - for name, release := range releaseTargets { - w.ReleaseTargets = append(w.ReleaseTargets, Target{Name: name, SkipTests: release.BuildOnly || *skipAllTestsFlag}) - } - - // Find milestone. - var err error - w.Milestone, err = findMilestone(w.Version) - if err != nil { - log.Fatalf("cannot find the GitHub milestone for release %s: %v", w.Version, err) - } - - w.doRelease() -} - -// mailDLCL parses command-line arguments for the mail-dl-cl mode, -// and runs it. -func mailDLCL() { - if flag.NArg() != 1 && flag.NArg() != 2 { - fmt.Fprintln(os.Stderr, "need to provide 1 or 2 versions") - usage() - } - versions := flag.Args() - - versionTasks := &task.VersionTasks{} - if !dryRun { - auth, err := loadGerritAuth() - if err != nil { - log.Fatalln("error loading Gerrit API credentials:", err) - } - versionTasks.Gerrit = &task.RealGerritClient{Client: gerrit.NewClient(gerritAPIURL, auth)} - } - - fmt.Printf("About to create a golang.org/dl CL for the following Go versions:\n\n\t• %s\n\nOk? (Y/n) ", strings.Join(versions, "\n\t• ")) - var resp string - if _, err := fmt.Scanln(&resp); err != nil { - log.Fatalln(err) - } else if resp != "Y" && resp != "y" { - log.Fatalln("stopped as requested") - } - changeID, err := versionTasks.MailDLCL(&workflow.TaskContext{Context: context.Background(), Logger: log.Default()}, versions, dryRun) - if err != nil { - log.Fatalf(`task.MailDLCL(ctx, %#v, extCfg) failed: - - %v - -If it's necessary to perform it manually as a workaround, -consider the following steps: - - git clone https://go.googlesource.com/dl && cd dl - # create files displayed in the log above - git add . - git commit -m "dl: add goX.Y.Z and goX.A.B" - git codereview mail -trybot - -Discuss with the secondary release coordinator as needed.`, versions, err) - } - fmt.Printf("\nPlease review and submit %s\nand then refer to the playbook for the next steps.\n\n", task.ChangeLink(changeID)) -} - -// postTweet parses command-line arguments for the tweet-* modes, -// and runs it. -// kind must be one of "minor", "beta", "rc", or "major". -func postTweet(kind string) { - if flag.NArg() != 1 { - fmt.Fprintln(os.Stderr, "need to provide 1 release tweet JSON object") - usage() - } - var tweet task.ReleaseTweet - err := json.Unmarshal([]byte(flag.Arg(0)), &tweet) - if err != nil { - log.Fatalln("error parsing release tweet JSON object:", err) - } - - var tasks task.TweetTasks - if !dryRun { - twitterAPI, err := loadTwitterAuth() - if err != nil { - log.Fatalln("error loading Twitter API credentials:", err) - } - tasks.TwitterClient = task.NewTwitterClient(twitterAPI) - } - - versions := []string{tweet.Version} - if tweet.SecondaryVersion != "" { - versions = append(versions, tweet.SecondaryVersion+" (secondary)") - } - fmt.Printf("About to tweet about the release of the following Go versions:\n\n\t• %s\n\n", strings.Join(versions, "\n\t• ")) - if tweet.Security != "" { - fmt.Printf("with the following security sentence (%d characters long):\n\n\t%s\n\n", len([]rune(tweet.Security)), tweet.Security) - } else { - fmt.Print("with no security fixes being mentioned,\n\n") - } - if tweet.Announcement != "" { - fmt.Printf("and with the following announcement URL:\n\n\t%s\n\n", tweet.Announcement) - } - fmt.Print("Ok? (Y/n) ") - var resp string - if _, err = fmt.Scanln(&resp); err != nil { - log.Fatalln(err) - } else if resp != "Y" && resp != "y" { - log.Fatalln("stopped as requested") - } - tweetRelease := map[string]func(*workflow.TaskContext, task.ReleaseTweet) (string, error){ - "minor": tasks.TweetMinorRelease, - "beta": tasks.TweetBetaRelease, - "rc": tasks.TweetRCRelease, - "major": tasks.TweetMajorRelease, - }[kind] - tweetURL, err := tweetRelease(&workflow.TaskContext{Context: context.Background(), Logger: log.Default()}, tweet) - if errors.Is(err, task.ErrTweetTooLong) && len([]rune(tweet.Security)) > 120 { - log.Fatalf(`A tweet was not created because it's too long. - -The provided security sentence is somewhat long (%d characters), -so try making it shorter to avoid exceeding Twitter's limits.`, len([]rune(tweet.Security))) - } else if err != nil { - log.Fatalf(`tweetRelease(ctx, %#v, extCfg) failed: - - %v - -If it's necessary to perform it manually as a workaround, -consider the following options: - - • use the template displayed in the log above (if any) - • use the same format as the last tweet for the release - of the same kind - -Discuss with the secondary release coordinator as needed.`, tweet, err) - } - fmt.Printf("\nPlease check that %s looks okay\nand then refer to the playbook for the next steps.\n\n", tweetURL) -} - -// checkForGitCodereview exits the program if git-codereview is not installed -// in the user's path. -func checkForGitCodereview() { - cmd := exec.Command("which", "git-codereview") - if err := cmd.Run(); err != nil { - log.Fatal("could not find git-codereivew: ", cmd.Args, ": ", err, "\n\n"+ - "Please install it via go install golang.org/x/review/git-codereview@latest\n"+ - "to use this program.") - } -} - -var gomoteUser string - -func loadGomoteUser() { - tokenPath := filepath.Join(os.Getenv("HOME"), ".config/gomote") - files, _ := ioutil.ReadDir(tokenPath) - for _, file := range files { - if file.IsDir() { - continue - } - name := file.Name() - if strings.HasSuffix(name, ".token") && strings.HasPrefix(name, "user-") { - gomoteUser = strings.TrimPrefix(strings.TrimSuffix(name, ".token"), "user-") - return - } - } - log.Fatal("missing gomote token - cannot build releases.\n**FIX**: Download https://build-dot-golang-org.appspot.com/key?builder=user-YOURNAME\nand store in ~/.config/gomote/user-YOURNAME.token") -} - -// findMilestone finds the GitHub milestone corresponding to the specified Go version. -// If there isn't exactly one open GitHub milestone that matches, an error is returned. -func findMilestone(version string) (*maintner.GitHubMilestone, error) { - // Pre-release versions of Go share the same milestone as the - // release version, so trim the pre-release suffix, if any. - if i := strings.Index(version, "beta"); i != -1 { - version = version[:i] - } else if i := strings.Index(version, "rc"); i != -1 { - version = version[:i] - } - - var open, closed []*maintner.GitHubMilestone - goRepo.ForeachMilestone(func(m *maintner.GitHubMilestone) error { - if strings.ToLower(m.Title) != version { - return nil - } - if !m.Closed { - open = append(open, m) - } else { - closed = append(closed, m) - } - return nil - }) - if len(open) == 1 { - // Happy path: found exactly one open matching milestone. - return open[0], nil - } else if len(open) == 0 && len(closed) == 0 { - return nil, errors.New("no milestone found") - } - // Something's really unexpected. - // Include all relevant information to help the human who'll need to sort it out. - var buf strings.Builder - buf.WriteString("found duplicate or closed milestones:\n") - for _, m := range open { - fmt.Fprintf(&buf, "\t• open milestone %q (https://github.com/golang/go/milestone/%d)\n", m.Title, m.Number) - } - for _, m := range closed { - fmt.Fprintf(&buf, "\t• closed milestone %q (https://github.com/golang/go/milestone/%d)\n", m.Title, m.Number) - } - return nil, errors.New(buf.String()) -} - -func nextVersion(version string) (string, error) { - parts := strings.Split(version, ".") - n, err := strconv.Atoi(parts[len(parts)-1]) - if err != nil { - return "", err - } - parts[len(parts)-1] = strconv.Itoa(n + 1) - return strings.Join(parts, "."), nil -} - -// Work collects all the work state for managing a particular release. -// The intent is that the code could be used in a setting where one program -// is managing multiple releases, although the current releasebot command line -// only accepts a single release. -type Work struct { - logBuf *bytes.Buffer - log *log.Logger - - Prepare bool // create the release commit and submit it for review - BetaRelease bool - RCRelease bool - - ReleaseIssue int // Release status issue number - ReleaseBranch string // "master" for beta releases - Dir string // work directory ($HOME/go-releasebot-work/) - StagingDir string // staging directory (a temporary directory inside /release-staging) - Errors []string - ReleaseBinary string - ReleaseTargets []Target // Selected release targets for this release. - Version string - VersionCommit string - - releaseMu sync.Mutex - ReleaseInfo map[string]*ReleaseInfo // map and info protected by releaseMu - - Milestone *maintner.GitHubMilestone // Milestone for the current release. -} - -// ReleaseInfo describes a release build for a specific target. -type ReleaseInfo struct { - Outputs []*ReleaseOutput - Msg string -} - -// ReleaseOutput describes a single release file. -type ReleaseOutput struct { - File string - Suffix string - Link string - Error string -} - -// logError records an error. -// The error is always shown in the "PROBLEMS WITH RELEASE" -// section at the top of the status page. -// If cl is not nil, the error is also shown in that CL's summary. -func (w *Work) logError(msg string, a ...interface{}) { - w.Errors = append(w.Errors, fmt.Sprintf(msg, a...)) -} - -// finally should be deferred at the top of each goroutine using a Work -// (as in "defer w.finally()"). It catches and logs panics and posts -// the log. -func (w *Work) finally() { - if err := recover(); err != nil { - w.log.Printf("\n\nPANIC: %v\n\n%s", err, debug.Stack()) - } - w.postSummary() -} - -type runner struct { - w *Work - dir string - extraEnv []string -} - -func (w *Work) runner(dir string, env ...string) *runner { - return &runner{ - w: w, - dir: dir, - extraEnv: env, - } -} - -// run runs the command and requires that it succeeds. -// If not, it logs the failure and aborts the work. -// It logs the command line. -func (r *runner) run(args ...string) { - out, err := r.runErr(args...) - if err != nil { - r.w.log.Printf("command failed: %s\n%s", err, out) - panic("command failed") - } -} - -// runOut runs the command, requires that it succeeds, -// and returns the command's output. -// It does not log the command line except in case of failure. -// Not logging these commands avoids filling the log with -// runs of side-effect-free commands like "git cat-file commit HEAD". -func (r *runner) runOut(args ...string) []byte { - cmd := exec.Command(args[0], args[1:]...) - envutil.SetDir(cmd, r.dir) - out, err := cmd.CombinedOutput() - if err != nil { - r.w.log.Printf("$ %s\n", strings.Join(args, " ")) - r.w.log.Printf("command failed: %s\n%s", err, out) - panic("command failed") - } - return out -} - -// runErr runs the given command and returns the output and status (error). -// It logs the command line. -func (r *runner) runErr(args ...string) ([]byte, error) { - r.w.log.Printf("$ %s\n", strings.Join(args, " ")) - cmd := exec.Command(args[0], args[1:]...) - envutil.SetDir(cmd, r.dir) - envutil.SetEnv(cmd, r.extraEnv...) - return cmd.CombinedOutput() -} - -func (w *Work) doRelease() { - w.logBuf = new(bytes.Buffer) - w.log = log.New(io.MultiWriter(os.Stdout, w.logBuf), "", log.LstdFlags) - defer w.finally() - - w.log.Printf("starting") - - w.checkSpelling() - w.gitCheckout() - - // In release mode we carry on even if the tag exists, in case we - // need to resume a failed build. - if w.Prepare && w.gitTagExists() { - w.logError("%s tag already exists in Go repository!", w.Version) - w.logError("**Found errors during release. Stopping!**") - return - } - if w.BetaRelease || w.RCRelease { - // TODO: go tool api -allow_new=false - if strings.HasSuffix(w.Version, "beta1") { - w.checkBeta1ReleaseBlockers() - } - } else { - w.checkReleaseBlockers() - } - w.findOrCreateReleaseIssue() - if len(w.Errors) > 0 && !dryRun { - w.logError("**Found errors during release. Stopping!**") - return - } - - if w.Prepare { - var changeID string - if !w.BetaRelease { - changeID = w.writeVersion() - } - - // Create release archives and run all.bash tests on the builders. - w.VersionCommit = w.gitHeadCommit() - w.buildReleases() - if len(w.Errors) > 0 { - w.logError("**Found errors during release. Stopping!**") - return - } - - if w.BetaRelease { - w.nextStepsBeta() - } else { - w.nextStepsPrepare(changeID) - } - } else { - if !w.BetaRelease { - w.checkVersion() - } - if len(w.Errors) > 0 { - w.logError("**Found errors during release. Stopping!**") - return - } - - // Create and push the Git tag for the release, then create or reuse release archives. - // (Tests are skipped here since they ran during the prepare mode.) - w.gitTagVersion() - w.buildReleases() - if len(w.Errors) > 0 { - w.logError("**Found errors during release. Stopping!**") - return - } - - switch { - case !w.BetaRelease && !w.RCRelease: - w.pushIssues() - w.closeMilestone() - case w.BetaRelease && strings.HasSuffix(w.Version, "beta1"): - w.removeOkayAfterBeta1() - } - w.nextStepsRelease() - } -} - -func (w *Work) checkSpelling() { - if w.Version != strings.ToLower(w.Version) { - w.logError("release name should be lowercase: %q", w.Version) - } - if strings.Contains(w.Version, " ") { - w.logError("release name should not contain any spaces: %q", w.Version) - } - if !strings.HasPrefix(w.Version, "go") { - w.logError("release name should have 'go' prefix: %q", w.Version) - } -} - -func (w *Work) checkReleaseBlockers() { - if err := goRepo.ForeachIssue(func(gi *maintner.GitHubIssue) error { - if gi.Milestone == nil || gi.Milestone.ID != w.Milestone.ID { - return nil - } - if !gi.Closed && gi.HasLabel("release-blocker") { - w.logError("open issue #%d is tagged release-blocker", gi.Number) - } - return nil - }); err != nil { - w.logError("error checking release-blockers: %v", err.Error()) - return - } -} - -func (w *Work) checkBeta1ReleaseBlockers() { - if err := goRepo.ForeachIssue(func(gi *maintner.GitHubIssue) error { - if gi.Milestone == nil || gi.Milestone.ID != w.Milestone.ID { - return nil - } - if !gi.Closed && gi.HasLabel("release-blocker") && !gi.HasLabel("okay-after-beta1") { - w.logError("open issue #%d is tagged release-blocker and not okay after beta1", gi.Number) - } - return nil - }); err != nil { - w.logError("error checking release-blockers: %v", err.Error()) - return - } -} - -func (w *Work) nextStepsPrepare(changeID string) { - w.log.Printf(` - -The prepare stage has completed. - -Please review and submit https://go-review.googlesource.com/q/%s -and then run the release stage. - -`, changeID) -} - -func (w *Work) nextStepsBeta() { - w.log.Printf(` - -The prepare stage has completed. - -Please run the release stage next. - -`) -} - -func (w *Work) nextStepsRelease() { - w.log.Printf(` - -The release stage has completed. Thanks for riding with releasebot today! - -Please refer to the playbook for the next steps. - -`) -} - -func (w *Work) postSummary() { - var md bytes.Buffer - - if len(w.Errors) > 0 { - fmt.Fprintf(&md, "## PROBLEMS WITH RELEASE\n\n") - for _, e := range w.Errors { - fmt.Fprintf(&md, " - ") - fmt.Fprintf(&md, "%s\n", strings.Replace(strings.TrimRight(e, "\n"), "\n", "\n ", -1)) - } - } - - if !w.Prepare { - fmt.Fprintf(&md, "\n## Latest build: %s\n\n", mdEscape(w.Version)) - w.printReleaseTable(&md) - } - - fmt.Fprintf(&md, "\n## Log\n\n ") - md.WriteString(strings.Replace(w.logBuf.String(), "\n", "\n ", -1)) - fmt.Fprintf(&md, "\n\n") - - if len(w.Errors) > 0 { - fmt.Fprintf(&md, "There were problems with the release, see above for details.\n") - } - - body := md.String() - fmt.Printf("%s", body) - if dryRun { - return - } - - // Ensure that the entire body can be posted to the issue by splitting it into multiple - // GitHub comments if necessary. See golang.org/issue/45998. - bodyParts := splitLogMessage(body, githubCommentCharacterLimit) - for _, b := range bodyParts { - err := postGithubComment(w.ReleaseIssue, b) - if err != nil { - fmt.Printf("error posting update comment: %v\n", err) - } - } -} - -func (w *Work) printReleaseTable(md *bytes.Buffer) { - // TODO: print sha256 - w.releaseMu.Lock() - defer w.releaseMu.Unlock() - for _, target := range w.ReleaseTargets { - fmt.Fprintf(md, "- %s", mdEscape(target.Name)) - info := w.ReleaseInfo[target.Name] - if info == nil { - fmt.Fprintf(md, " - not started\n") - continue - } - if len(info.Outputs) == 0 { - fmt.Fprintf(md, " - not built") - } - for _, out := range info.Outputs { - if out.Link != "" { - fmt.Fprintf(md, " ([%s](%s))", mdEscape(out.Suffix), out.Link) - } else { - fmt.Fprintf(md, " (~~%s~~)", mdEscape(out.Suffix)) - } - } - fmt.Fprintf(md, "\n") - if info.Msg != "" { - fmt.Fprintf(md, " - %s\n", strings.Replace(strings.TrimRight(info.Msg, "\n"), "\n", "\n ", -1)) - } - } -} - -func (w *Work) writeVersion() (changeID string) { - changeID = fmt.Sprintf("I%x", sha1.Sum([]byte(fmt.Sprintf("cmd/release-version-%s", w.Version)))) - - err := ioutil.WriteFile(filepath.Join(w.Dir, "gitwork", "VERSION"), []byte(w.Version), 0666) - if err != nil { - w.log.Panic(err) - } - - desc := w.Version + "\n\n" - desc += "Change-Id: " + changeID + "\n" - - r := w.runner(filepath.Join(w.Dir, "gitwork")) - r.run("git", "add", "VERSION") - r.run("git", "commit", "-m", desc, "VERSION") - if dryRun { - fmt.Printf("\n### VERSION commit\n\n%s\n", r.runOut("git", "show", "HEAD")) - } else { - r.run("git", "codereview", "mail", "-trybot") - } - return -} - -// checkVersion makes sure that the version commit has been submitted. -func (w *Work) checkVersion() { - ver, err := ioutil.ReadFile(filepath.Join(w.Dir, "gitwork", "VERSION")) - if err != nil { - w.log.Panic(err) - } - if string(ver) != w.Version { - w.logError("VERSION is %q; want %q. Did you run prepare and submit the CL?", string(ver), w.Version) - } -} - -func (w *Work) buildReleaseBinary() { - gopath := filepath.Join(w.Dir, "gopath") - r := w.runner(w.Dir, "GOPATH="+gopath, "GOBIN="+filepath.Join(gopath, "bin")) - r.run("go", "clean", "-modcache") - if err := os.RemoveAll(gopath); err != nil { - w.log.Panic(err) - } - if err := os.MkdirAll(gopath, 0777); err != nil { - w.log.Panic(err) - } - r.run("go", "install", "golang.org/x/build/cmd/release@latest") - w.ReleaseBinary = filepath.Join(gopath, "bin/release") -} - -func (w *Work) buildReleases() { - w.buildReleaseBinary() - if err := os.MkdirAll(filepath.Join(w.Dir, "release", w.VersionCommit), 0777); err != nil { - w.log.Panic(err) - } - if err := os.MkdirAll(filepath.Join(w.Dir, "release-staging"), 0777); err != nil { - w.log.Panic(err) - } - stagingDir, err := ioutil.TempDir(filepath.Join(w.Dir, "release-staging"), w.VersionCommit+"_") - if err != nil { - w.log.Panic(err) - } - w.StagingDir = stagingDir - w.ReleaseInfo = make(map[string]*ReleaseInfo) - - var wg sync.WaitGroup - for _, target := range w.ReleaseTargets { - w.releaseMu.Lock() - w.ReleaseInfo[target.Name] = new(ReleaseInfo) - w.releaseMu.Unlock() - - wg.Add(1) - target := target - go func() { - defer wg.Done() - defer func() { - if err := recover(); err != nil { - stk := strings.TrimSpace(string(debug.Stack())) - msg := fmt.Sprintf("PANIC: %v\n\n %s\n", mdEscape(fmt.Sprint(err)), strings.Replace(stk, "\n", "\n ", -1)) - w.logError(msg) - w.log.Printf("\n\nBuilding %s: PANIC: %v\n\n%s", target.Name, err, debug.Stack()) - w.releaseMu.Lock() - w.ReleaseInfo[target.Name].Msg = msg - w.releaseMu.Unlock() - } - }() - w.buildRelease(target) - }() - } - wg.Wait() - - // Check for release errors and stop if any. - w.releaseMu.Lock() - for _, target := range w.ReleaseTargets { - for _, out := range w.ReleaseInfo[target.Name].Outputs { - if out.Error != "" || len(w.Errors) > 0 { - w.logError("RELEASE BUILD FAILED\n") - w.releaseMu.Unlock() - return - } - } - } - w.releaseMu.Unlock() -} - -// buildRelease builds the release packaging for a given target. Because the -// "release" program can be flaky, it tries multiple times before stopping. -// The release files are first written to a staging directory specified in w.StagingDir -// (a temporary directory inside $HOME/go-releasebot-work/go1.2.3/release-staging), -// then after the all.bash tests complete successfully (or get skipped), -// they get moved to the final release directory -// ($HOME/go-releasebot-work/go1.2.3/release/COMMIT_HASH). -// -// If files for the current version commit are already present in the release directory, -// they are reused instead of being rebuilt. In release mode, buildRelease then uploads -// the release packaging to the gs://golang-release-staging bucket, along with files -// containing the SHA256 hash of the releases, for eventual use by the download page. -func (w *Work) buildRelease(target Target) { - log.Printf("BUILDRELEASE %s %s\n", w.Version, target.Name) - defer log.Printf("DONE BUILDRELEASE %s %s\n", w.Version, target.Name) - releaseDir := filepath.Join(w.Dir, "release", w.VersionCommit) - prefix := fmt.Sprintf("%s.%s.", w.Version, target.Name) - var files []string - switch { - case strings.HasPrefix(target.Name, "windows-"): - files = []string{prefix + "zip", prefix + "msi"} - default: - files = []string{prefix + "tar.gz"} - } - var outs []*ReleaseOutput - haveFiles := true - for _, file := range files { - out := &ReleaseOutput{ - File: file, - Suffix: strings.TrimPrefix(file, prefix), - } - outs = append(outs, out) - _, err := os.Stat(filepath.Join(releaseDir, file)) - if err != nil { - haveFiles = false - } - } - w.releaseMu.Lock() - w.ReleaseInfo[target.Name].Outputs = outs - w.releaseMu.Unlock() - - if haveFiles { - w.log.Printf("release -target=%q: already have %v; not rebuilding files", target.Name, files) - } else { - failures := 0 - for { - args := []string{w.ReleaseBinary, "-target", target.Name, "-user", gomoteUser, - "-version", w.Version, "-staging_dir", w.StagingDir, "-rev", w.VersionCommit} - // The prepare step will run the tests on a commit that has the same - // tree (but maybe different message) as the one that the release - // step will process, so we can skip tests the second time. - if !w.Prepare || target.SkipTests { - args = append(args, "-skip_tests") - } - releaseOutput, releaseError := w.runner(releaseDir, "GOPATH="+filepath.Join(w.Dir, "gopath")).runErr(args...) - // Exit code from release binary is apparently unreliable. - // Look to see if the files we expected were created instead. - failed := false - w.releaseMu.Lock() - for _, out := range outs { - if _, err := os.Stat(filepath.Join(releaseDir, out.File)); err != nil { - failed = true - } - } - w.releaseMu.Unlock() - if !failed { - w.log.Printf("release -target=%q: build succeeded (after %d retries)\n", target.Name, failures) - break - } - w.log.Printf("release -target=%q did not produce expected output files %v:\nerror from cmd/release binary = %v\noutput from cmd/release binary:\n%s", target.Name, files, releaseError, releaseOutput) - if failures++; failures >= 3 { - w.log.Printf("release -target=%q: too many failed attempts, stopping\n", target.Name) - for _, out := range outs { - w.releaseMu.Lock() - out.Error = fmt.Sprintf("release -target=%q: build failed", target.Name) - w.releaseMu.Unlock() - } - return - } - w.log.Printf("release -target=%q: waiting a bit and trying again\n", target.Name) - time.Sleep(1 * time.Minute) - } - } - - if dryRun || w.Prepare { - return - } - - for _, out := range outs { - if err := w.uploadStagingRelease(target, out); err != nil { - w.log.Printf("error uploading release %s to staging bucket: %s", target.Name, err) - w.releaseMu.Lock() - out.Error = err.Error() - w.releaseMu.Unlock() - } - } -} - -// uploadStagingRelease uploads target to the release staging bucket. -// If successful, it records the corresponding URL in out.Link. -// In addition to uploading target, it creates and uploads a file -// named ".sha256" containing the hex sha256 hash -// of the target file. This is needed for the release signing process -// and also displayed on the eventual download page. -func (w *Work) uploadStagingRelease(target Target, out *ReleaseOutput) error { - if dryRun { - return errors.New("attempted write operation in dry-run mode") - } - - src := filepath.Join(w.Dir, "release", w.VersionCommit, out.File) - h := sha256.New() - f, err := os.Open(src) - if err != nil { - return err - } - _, err = io.Copy(h, f) - f.Close() - if err != nil { - return err - } - if err := ioutil.WriteFile(src+".sha256", []byte(fmt.Sprintf("%x", h.Sum(nil))), 0666); err != nil { - return err - } - - dst := w.Version + "/" + out.File - if err := gcsUpload(src, dst); err != nil { - return err - } - if err := gcsUpload(src+".sha256", dst+".sha256"); err != nil { - return err - } - - w.releaseMu.Lock() - out.Link = "https://" + releaseBucket + ".storage.googleapis.com/" + dst - w.releaseMu.Unlock() - return nil -} - -// splitLogMessage splits a string into n number of strings of maximum size maxStrLen. -// It naively attempts to split the string along the boundaries of new line characters in order -// to make each individual string as readable as possible. -func splitLogMessage(s string, maxStrLen int) []string { - sl := []string{} - for len(s) > maxStrLen { - end := strings.LastIndex(s[:maxStrLen], "\n") - if end == -1 { - end = maxStrLen - } - sl = append(sl, s[:end]) - - if string(s[end]) == "\n" { - s = s[end+1:] - } else { - s = s[end:] - } - } - sl = append(sl, s) - return sl -} diff --git a/cmd/releasebot/main_test.go b/cmd/releasebot/main_test.go deleted file mode 100644 index ce025aee28..0000000000 --- a/cmd/releasebot/main_test.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2020 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "testing" - - "github.com/google/go-cmp/cmp" -) - -func TestSplitLogMessage(t *testing.T) { - testCases := []struct { - desc string - str string - maxLen int - want []string - }{ - { - desc: "string matches max size", - str: "the quicks", - maxLen: 10, - want: []string{"the quicks"}, - }, - { - desc: "string greater than max size", - str: "the quick brown fox", - maxLen: 10, - want: []string{"the quick ", "brown fox"}, - }, - { - desc: "string smaller than max size", - str: "the quick", - maxLen: 20, - want: []string{"the quick"}, - }, - { - desc: "string matches max size with return", - str: "the quick\n", - maxLen: 10, - want: []string{"the quick\n"}, - }, - { - desc: "string greater than max size with return", - str: "the quick\n brown fox", - maxLen: 10, - want: []string{"the quick", " brown fox"}, - }, - { - desc: "string smaller than max size with return", - str: "the \nquick", - maxLen: 20, - want: []string{"the \nquick"}, - }, - { - desc: "string is multiples of max size", - str: "000000000011111111112222222222", - maxLen: 10, - want: []string{"0000000000", "1111111111", "2222222222"}, - }, - { - desc: "string is multiples of max size with return", - str: "000000000\n111111111\n222222222\n", - maxLen: 10, - want: []string{"000000000", "111111111", "222222222\n"}, - }, - { - desc: "string is multiples of max size with extra return", - str: "000000000\n111111111\n222222222\n\n", - maxLen: 10, - want: []string{"000000000", "111111111", "222222222", "\n"}, - }, - } - for _, tc := range testCases { - t.Run(tc.desc, func(t *testing.T) { - got := splitLogMessage(tc.str, tc.maxLen) - if !cmp.Equal(tc.want, got) { - t.Errorf("splitStringToSlice(%q, %d) =\ngot \t %#v\nwant \t %#v", tc.str, tc.maxLen, got, tc.want) - } - }) - } -} diff --git a/cmd/releasebot/markdown.go b/cmd/releasebot/markdown.go deleted file mode 100644 index 7f9b861a31..0000000000 --- a/cmd/releasebot/markdown.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2017 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "strings" -) - -// mdEscape escapes text so that it does not have any special meaning in Markdown. -func mdEscape(text string) string { - return mdEscaper.Replace(text) -} - -var mdEscaper = strings.NewReplacer( - `\`, `\\`, - `{`, `\{`, - `}`, `\}`, - "`", "\\`", - `#`, `\#`, - `*`, `\*`, - `+`, `\+`, - `_`, `\_`, - `-`, `\-`, - `(`, `\(`, - `)`, `\)`, - `.`, `\.`, - `[`, `\[`, - `]`, `\]`, - `!`, `\!`, -) diff --git a/cmd/releasebot/twitter.go b/cmd/releasebot/twitter.go deleted file mode 100644 index 27aba7064c..0000000000 --- a/cmd/releasebot/twitter.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "context" - "encoding/json" - - "golang.org/x/build/buildenv" - "golang.org/x/build/internal/secret" -) - -// loadTwitterAuth loads Twitter API credentials. -func loadTwitterAuth() (secret.TwitterCredentials, error) { - sc, err := secret.NewClientInProject(buildenv.Production.ProjectName) - if err != nil { - return secret.TwitterCredentials{}, err - } - defer sc.Close() - secretJSON, err := sc.Retrieve(context.Background(), secret.NameTwitterAPISecret) - if err != nil { - return secret.TwitterCredentials{}, err - } - var v secret.TwitterCredentials - err = json.Unmarshal([]byte(secretJSON), &v) - if err != nil { - return secret.TwitterCredentials{}, err - } - return v, nil -}