From f3352ae4dbbff06b414b9e3275b2f4d190e0349e Mon Sep 17 00:00:00 2001
From: "W. Trevor King" <wking@tremily.us>
Date: Tue, 20 Oct 2020 14:12:36 -0700
Subject: [PATCH] cmd/pj-rehearse: Truncate extra rehearsals

Before this commit, pj-rehearse would just give up [1]:

  $ curl -s https://storage.googleapis.com/origin-ci-test/pr-logs/pull/openshift_release/12959/pull-ci-openshift-release-master-pj-rehearse/1318651994902106112/build-log.txt | tail -n2
  time="2020-10-20T20:36:51Z" level=info msg="Created a rehearsal job to be submitted" org=openshift rehearsal-job=rehearse-12959-pull-ci-openshift-kubernetes-master-e2e-cmd repo=release target-job=pull-ci-openshift-kubernetes-master-e2e-cmd target-repo=openshift/kubernetes
  time="2020-10-20T20:36:51Z" level=info msg="Would rehearse too many jobs, will not proceed" org=openshift rehearsal-jobs=68 rehearsal-threshold=45 repo=release

But skipping rehearsals for a change that touches tons of jobs, like
we have done since ef4f47ebfe (Limit rehearsals to 15 jobs,
2019-02-13, openshift/ci-operator-prowgen#78), makes it possible to
break a whole lot of things without failing a warning rehearsal.  With
this commit, we pivot from "give up and test nothing" to "test as many
of the touched jobs as we can afford".

It would be nice to intelligently truncate, e.g. if we touch a few
types of jobs, or a few different workflows.  But grouping seems
complicated, so for this commit I'm just randomly shuffling and then
dumping the tail.

[1]: https://prow.ci.openshift.org/view/gs/origin-ci-test/pr-logs/pull/openshift_release/12959/pull-ci-openshift-release-master-pj-rehearse/1318651994902106112
---
 cmd/pj-rehearse/main.go | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/cmd/pj-rehearse/main.go b/cmd/pj-rehearse/main.go
index 3b0d73c8791..5706736dd0f 100644
--- a/cmd/pj-rehearse/main.go
+++ b/cmd/pj-rehearse/main.go
@@ -5,6 +5,7 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"math/rand"
 	"os"
 	"path/filepath"
 	"strings"
@@ -67,7 +68,7 @@ func gatherOptions() (options, error) {
 	fs.BoolVar(&o.noRegistry, "no-registry", false, "If true, do not attempt to compare step registry content")
 	fs.BoolVar(&o.noClusterProfiles, "no-cluster-profiles", false, "If true, do not attempt to compare cluster profiles")
 
-	fs.IntVar(&o.rehearsalLimit, "rehearsal-limit", 15, "Upper limit of jobs attempted to rehearse (if more jobs would be rehearsed, none will)")
+	fs.IntVar(&o.rehearsalLimit, "rehearsal-limit", 15, "Upper limit of jobs attempted to rehearse (if more jobs are being touched, only this many will be rehearsed)")
 
 	if err := fs.Parse(os.Args[1:]); err != nil {
 		return o, fmt.Errorf("failed to parse flags: %w", err)
@@ -320,8 +321,13 @@ func rehearseMain() error {
 	}
 	apihelper.MergeImageStreamTagMaps(imagestreamtags, periodicImageStreamTags)
 
-	rehearsals := len(presubmitsToRehearse) + len(periodicsToRehearse)
-	if rehearsals == 0 {
+	periodicPresubmits, err := jobConfigurer.ConvertPeriodicsToPresubmits(periodicsToRehearse)
+	if err != nil {
+		return err
+	}
+	presubmitsToRehearse = append(presubmitsToRehearse, periodicPresubmits...)
+
+	if rehearsals := len(presubmitsToRehearse); rehearsals == 0 {
 		logger.Info("no jobs to rehearse have been found")
 		return nil
 	} else if rehearsals > o.rehearsalLimit {
@@ -329,15 +335,13 @@ func rehearseMain() error {
 			"rehearsal-threshold": o.rehearsalLimit,
 			"rehearsal-jobs":      rehearsals,
 		}
-		logger.WithFields(jobCountFields).Info("Would rehearse too many jobs, will not proceed")
-		return nil
+		logger.WithFields(jobCountFields).Info("Would rehearse too many jobs, randomly selecting a subset")
+		rand.Shuffle(len(presubmitsToRehearse), func(i, j int) {
+			presubmitsToRehearse[i], presubmitsToRehearse[j] = presubmitsToRehearse[j], presubmitsToRehearse[i]
+		})
+		presubmitsToRehearse = presubmitsToRehearse[0:o.rehearsalLimit]
 	}
 
-	periodicPresubmits, err := jobConfigurer.ConvertPeriodicsToPresubmits(periodicsToRehearse)
-	if err != nil {
-		return err
-	}
-	presubmitsToRehearse = append(presubmitsToRehearse, periodicPresubmits...)
 	if prConfig.Prow.JobConfig.PresubmitsStatic == nil {
 		prConfig.Prow.JobConfig.PresubmitsStatic = map[string][]prowconfig.Presubmit{}
 	}