pj-rehearse: implement rehearsing of changed registry steps

cmd/ci-operator-prowgen/main.go

@@ -20,6 +20,7 @@ import (
 
 	cioperatorapi "github.com/openshift/ci-tools/pkg/api"
 	"github.com/openshift/ci-tools/pkg/config"
+	"github.com/openshift/ci-tools/pkg/jobconfig"
 	jc "github.com/openshift/ci-tools/pkg/jobconfig"
 	"github.com/openshift/ci-tools/pkg/promotion"
 )
@@ -32,10 +33,6 @@ const (
 	sentryDsnMountPath  = "/etc/sentry-dsn"
 	sentryDsnSecretPath = "/etc/sentry-dsn/ci-operator"
 
-	presubmitPrefix  = "pull"
-	postsubmitPrefix = "branch"
-	periodicPrefix   = "periodic"
-
 	openshiftInstallerRandomCmd = `set -eux
 target=$(awk < /usr/local/e2e-targets \
     --assign "r=$RANDOM" \
@@ -558,11 +555,10 @@ func generateJobBase(name, prefix string, info *prowgenInfo, label jc.ProwgenLab
 		labels[jc.CanBeRehearsedLabel] = string(jc.Generated)
 	}
 
-	jobPrefix := fmt.Sprintf("%s-ci-%s-%s-%s-", prefix, info.Org, info.Repo, info.Branch)
+	jobName := info.Info.JobName(prefix, name)
 	if len(info.Variant) > 0 {
 		labels[prowJobLabelVariant] = info.Variant
 	}
-	jobName := fmt.Sprintf("%s%s", jobPrefix, name)
 	newTrue := true
 	dc := &v1.DecorationConfig{SkipCloning: &newTrue}
 	base := prowconfig.JobBase{
@@ -588,7 +584,7 @@ func generatePresubmitForTest(name string, info *prowgenInfo, label jc.ProwgenLa
 	if len(info.Variant) > 0 {
 		name = fmt.Sprintf("%s-%s", info.Variant, name)
 	}
-	base := generateJobBase(name, presubmitPrefix, info, label, podSpec, rehearsable, pathAlias)
+	base := generateJobBase(name, jobconfig.PresubmitPrefix, info, label, podSpec, rehearsable, pathAlias)
 	return &prowconfig.Presubmit{
 		JobBase:   base,
 		AlwaysRun: true,
@@ -605,7 +601,7 @@ func generatePostsubmitForTest(name string, info *prowgenInfo, label jc.ProwgenL
 	if len(info.Variant) > 0 {
 		name = fmt.Sprintf("%s-%s", info.Variant, name)
 	}
-	base := generateJobBase(name, postsubmitPrefix, info, label, podSpec, false, pathAlias)
+	base := generateJobBase(name, jobconfig.PostsubmitPrefix, info, label, podSpec, false, pathAlias)
 	return &prowconfig.Postsubmit{
 		JobBase:  base,
 		Brancher: prowconfig.Brancher{Branches: []string{makeBranchExplicit(info.Branch)}},
@@ -616,7 +612,7 @@ func generatePeriodicForTest(name string, info *prowgenInfo, label jc.ProwgenLab
 	if len(info.Variant) > 0 {
 		name = fmt.Sprintf("%s-%s", info.Variant, name)
 	}
-	base := generateJobBase(name, periodicPrefix, info, label, podSpec, rehearsable, nil)
+	base := generateJobBase(name, jobconfig.PeriodicPrefix, info, label, podSpec, rehearsable, nil)
 	// periodics are not associated with a repo per se, but we can add in an
 	// extra ref so that periodics which want to access the repo tha they are
 	// defined for can have that information

cmd/pj-rehearse/main.go

@@ -185,6 +185,26 @@ func rehearseMain() int {
 		metrics.RecordChangedCiopConfigs(changedCiopConfigData)
 	}
 
+	refs, chains, workflows, err := load.Registry(filepath.Join(o.releaseRepoPath, config.RegistryPath), false)
+	if err != nil {
+		logger.WithError(err).Error("could not load step registry")
+		return gracefulExit(o.noFail, misconfigurationOutput)
+	}
+	graph, err := registry.NewGraph(refs, chains, workflows)
+	if err != nil {
+		logger.WithError(err).Error("could not create step registry graph")
+		return gracefulExit(o.noFail, misconfigurationOutput)
+	}
+	changedRegistrySteps, err := config.GetChangedRegistrySteps(o.releaseRepoPath, jobSpec.Refs.BaseSHA, graph)
+	if err != nil {
+		logger.WithError(err).Error("could not get step registry differences")
+		return gracefulExit(o.noFail, misconfigurationOutput)
+	}
+	if len(changedRegistrySteps) != 0 {
+		logger.WithField("registry", changedRegistrySteps).Info("registry steps changed")
+		metrics.RecordChangedRegistryElements(changedRegistrySteps)
+	}
+
 	changedTemplates, err := config.GetChangedTemplates(o.releaseRepoPath, jobSpec.Refs.BaseSHA)
 	if err != nil {
 		logger.WithError(err).Error("could not get template differences")
@@ -269,13 +289,11 @@ func rehearseMain() int {
 	metrics.RecordPresubmitsOpportunity(toRehearseClusterProfiles, "cluster-profile-change")
 	toRehearse.AddAll(toRehearseClusterProfiles)
 
-	refs, chains, workflows, err := load.Registry(filepath.Join(o.releaseRepoPath, config.RegistryPath), false)
-	if err != nil {
-		logger.WithError(err).Error("could not load step registry")
-		return gracefulExit(o.noFail, misconfigurationOutput)
-	}
 	resolver := registry.NewResolver(refs, chains, workflows)
 	jobConfigurer := rehearse.NewJobConfigurer(prConfig.CiOperator, resolver, prNumber, loggers, o.allowVolumes, changedTemplates, changedClusterProfiles, jobSpec.Refs)
+	presubmitsWithChangedRegistry := rehearse.AddRandomJobsForChangedRegistry(changedRegistrySteps, graph, prConfig.Prow.JobConfig.PresubmitsStatic, filepath.Join(o.releaseRepoPath, diffs.CIOperatorConfigInRepoPath), loggers)
+	metrics.RecordPresubmitsOpportunity(presubmitsWithChangedRegistry, "registry-change")
+	toRehearse.AddAll(presubmitsWithChangedRegistry)
 
 	presubmitsToRehearse := jobConfigurer.ConfigurePresubmitRehearsals(toRehearse)
 	periodicsToRehearse := jobConfigurer.ConfigurePeriodicRehearsals(changedPeriodics)

pkg/config/load.go

@@ -49,6 +49,10 @@ type Info struct {
 	RepoPath string
 }
 
+func (i *Info) JobName(prefix, name string) string {
+	return fmt.Sprintf("%s-ci-%s-%s-%s-%s", prefix, i.Org, i.Repo, i.Branch, name)
+}
+
 // Basename returns the unique name for this file in the config
 func (i *Info) Basename() string {
 	basename := strings.Join([]string{i.Org, i.Repo, i.Branch}, "-")

pkg/config/release.go

@@ -7,10 +7,11 @@ import (
 	"strings"
 
 	"github.com/sirupsen/logrus"
-
 	pjapi "k8s.io/test-infra/prow/apis/prowjobs/v1"
 	prowconfig "k8s.io/test-infra/prow/config"
 	pjdwapi "k8s.io/test-infra/prow/pod-utils/downwardapi"
+
+	"github.com/openshift/ci-tools/pkg/registry"
 )
 
 const (
@@ -165,6 +166,48 @@ func GetChangedTemplates(path, baseRev string) ([]ConfigMapSource, error) {
 	return ret, nil
 }
 
+func loadRegistryStep(filename string, graph, changes registry.NodeByName) error {
+	// if a commands script changed, mark reference as changed
+	filename = strings.ReplaceAll(filename, "-commands.sh", "-ref.yaml")
+	name := ""
+	if strings.HasSuffix(filename, "-ref.yaml") {
+		name = strings.TrimSuffix(filename, "-ref.yaml")
+	}
+	if strings.HasSuffix(filename, "-chain.yaml") {
+		name = strings.TrimSuffix(filename, "-chain.yaml")
+	}
+	if strings.HasSuffix(filename, "-workflow.yaml") {
+		name = strings.TrimSuffix(filename, "-workflow.yaml")
+	}
+	if name == "" {
+		return fmt.Errorf("invalid step filename: %s", filename)
+	}
+	node, ok := graph[name]
+	if !ok {
+		return fmt.Errorf("could not find registry component in registry graph: %s", name)
+	}
+	changes[name] = node
+	return nil
+}
+
+// GetChangedRegistrySteps identifies all registry components (refs, chains, and workflows) that changed.
+func GetChangedRegistrySteps(path, baseRev string, graph registry.NodeByName) (registry.NodeByName, error) {
+	changes := make(registry.NodeByName)
+	revChanges, err := getRevChanges(path, RegistryPath, baseRev, true)
+	if err != nil {
+		return changes, err
+	}
+	for _, c := range revChanges {
+		if filepath.Ext(c.Filename) == ".yaml" || strings.HasSuffix(c.Filename, "-commands.sh") {
+			err := loadRegistryStep(filepath.Base(c.Filename), graph, changes)
+			if err != nil {
+				return changes, err
+			}
+		}
+	}
+	return changes, nil
+}
+
 func GetChangedClusterProfiles(path, baseRev string) ([]ConfigMapSource, error) {
 	return getRevChanges(path, ClusterProfilesPath, baseRev, false)
 }

pkg/jobconfig/files.go

@@ -13,7 +13,7 @@ import (
 	"github.com/openshift/ci-tools/pkg/promotion"
 	"github.com/sirupsen/logrus"
 
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
 	prowconfig "k8s.io/test-infra/prow/config"
 )
@@ -25,6 +25,9 @@ const (
 	CanBeRehearsedLabel                = "pj-rehearse.openshift.io/can-be-rehearsed"
 	Generated             ProwgenLabel = "true"
 	New                   ProwgenLabel = "newly-generated"
+	PresubmitPrefix                    = "pull"
+	PostsubmitPrefix                   = "branch"
+	PeriodicPrefix                     = "periodic"
 )
 
 // DataWithInfo describes the metadata for a Prow job configuration file

pkg/rehearse/jobs.go

@@ -3,6 +3,7 @@ package rehearse
 import (
 	"fmt"
 	"path/filepath"
+	"reflect"
 	"sort"
 	"strconv"
 	"strings"
@@ -32,7 +33,9 @@ import (
 	prowconfig "k8s.io/test-infra/prow/config"
 	"k8s.io/test-infra/prow/pjutil"
 
+	"github.com/openshift/ci-tools/pkg/api"
 	"github.com/openshift/ci-tools/pkg/config"
+	"github.com/openshift/ci-tools/pkg/jobconfig"
 	"github.com/openshift/ci-tools/pkg/registry"
 )
 
@@ -408,6 +411,129 @@ func AddRandomJobsForChangedTemplates(templates []config.ConfigMapSource, toBeRe
 	return rehearsals
 }
 
+func getPresubmitByJobName(presubmits []prowconfig.Presubmit, name string) (prowconfig.Presubmit, error) {
+	for _, presubmit := range presubmits {
+		if presubmit.Name == name {
+			return presubmit, nil
+		}
+	}
+	return prowconfig.Presubmit{}, fmt.Errorf("could not find presubmit with name: %s", name)
+}
+
+func getPresubmitsForRegistryStep(node registry.Node, configs config.ByFilename, prConfigPresubmits map[string][]prowconfig.Presubmit, addedConfigs []*api.MultiStageTestConfiguration) (map[string][]prowconfig.Presubmit, []*api.MultiStageTestConfiguration, error) {
+	toTest := make(map[string][]prowconfig.Presubmit)
+	// get sorted list of configs keys to make the function deterministic
+	var keys []string
+	for k := range configs {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, key := range keys {
+		ciopConfig := configs[key]
+		tests := ciopConfig.Configuration.Tests
+		orgRepo := fmt.Sprintf("%s/%s", ciopConfig.Info.Org, ciopConfig.Info.Repo)
+		repoPresubmits := prConfigPresubmits[orgRepo]
+		for _, test := range tests {
+			if test.MultiStageTestConfiguration == nil {
+				continue
+			}
+			skip := false
+			for _, added := range addedConfigs {
+				if reflect.DeepEqual(test.MultiStageTestConfiguration, added) {
+					skip = true
+					break
+				}
+			}
+			if skip {
+				continue
+			}
+			jobName := ciopConfig.Info.JobName(jobconfig.PresubmitPrefix, test.As)
+			// TODO: Handle workflows with overridden fields.
+			// Workflows can have overridden fields and thus may have overridden the field that made the workflow an ancestor.
+			// This should be handled to reduce the number of rehearsals being done, but requires much more information than
+			// the graph alone provides.
+			if test.MultiStageTestConfiguration.Workflow != nil && node.Type() == registry.Workflow && node.Name() == *test.MultiStageTestConfiguration.Workflow {
+				presubmit, err := getPresubmitByJobName(repoPresubmits, jobName)
+				if err != nil {
+					return toTest, addedConfigs, err
+				}
+				addedConfigs = append(addedConfigs, test.MultiStageTestConfiguration)
+				toTest[orgRepo] = append(toTest[orgRepo], presubmit)
+				// continue to check other tests
+				continue
+			}
+			testSteps := append(test.MultiStageTestConfiguration.Pre, append(test.MultiStageTestConfiguration.Test, test.MultiStageTestConfiguration.Post...)...)
+			for _, testStep := range testSteps {
+				if testStep.Reference != nil && node.Type() == registry.Reference && node.Name() == *testStep.Reference {
+					presubmit, err := getPresubmitByJobName(repoPresubmits, jobName)
+					if err != nil {
+						return toTest, addedConfigs, err
+					}
+					addedConfigs = append(addedConfigs, test.MultiStageTestConfiguration)
+					toTest[orgRepo] = append(toTest[orgRepo], presubmit)
+					// found step; break
+					break
+				}
+				if testStep.Chain != nil && node.Type() == registry.Chain && node.Name() == *testStep.Chain {
+					presubmit, err := getPresubmitByJobName(repoPresubmits, jobName)
+					if err != nil {
+						return toTest, addedConfigs, err
+					}
+					addedConfigs = append(addedConfigs, test.MultiStageTestConfiguration)
+					toTest[orgRepo] = append(toTest[orgRepo], presubmit)
+					// found step; break
+					break
+				}
+			}
+		}
+	}
+	return toTest, addedConfigs, nil
+}
+
+// expandAncestors takes a graph of changed steps and adds all ancestors of
+// the existing steps to the changed steps graph
+func expandAncestors(changed, graph registry.NodeByName) {
+	for _, node := range changed {
+		for name := range node.AncestorNames() {
+			changed[name] = graph[name]
+		}
+	}
+}
+
+func AddRandomJobsForChangedRegistry(regSteps, graph registry.NodeByName, prConfigPresubmits map[string][]prowconfig.Presubmit, configPath string, loggers Loggers) config.Presubmits {
+	configsByFilename, err := config.LoadConfigByFilename(configPath)
+	if err != nil {
+		loggers.Debug.Errorf("Failed to load config by filename in AddRandomJobsForChangedRegistry: %v", err)
+	}
+	expandAncestors(regSteps, graph)
+	rehearsals := make(config.Presubmits)
+	// get sorted list of regSteps keys to make the function deterministic
+	var keys []string
+	for k := range regSteps {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	// make list to store MultiStageTestConfigurations that we've already added to the test list
+	addedConfigs := []*api.MultiStageTestConfiguration{}
+	for _, key := range keys {
+		step := regSteps[key]
+		var presubmitsMap map[string][]prowconfig.Presubmit
+		presubmitsMap, addedConfigs, err = getPresubmitsForRegistryStep(step, configsByFilename, prConfigPresubmits, addedConfigs)
+		if err != nil {
+			loggers.Debug.Errorf("Error getting presubmits in AddRandomJobsForChangedRegistry: %v", err)
+		}
+		if len(presubmitsMap) == 0 {
+			// if the code reaches this point, then no config contains the step or the step has already been tested
+			loggers.Debug.Warnf("No config found containing step: %+v", step)
+		}
+		for repo, presubmits := range presubmitsMap {
+			rehearsals[repo] = append(rehearsals[repo], presubmits...)
+			continue
+		}
+	}
+	return rehearsals
+}
+
 func getClusterTypes(jobs map[string][]prowconfig.Presubmit) []string {
 	ret := sets.NewString()
 	for _, jobs := range jobs {