On a template change, rehearse all cluster types.

cmd/pj-rehearse/main.go

@@ -253,9 +253,14 @@ func rehearseMain() int {
 	metrics.RecordChangedPresubmits(toRehearse)
 	metrics.RecordOpportunity(toRehearse, "direct-change")
 
-	presubmitsWitchChangedCiopConfigs := diffs.GetPresubmitsForCiopConfigs(prConfig.Prow, changedCiopConfigs, logger, affectedJobs)
-	metrics.RecordOpportunity(presubmitsWitchChangedCiopConfigs, "ci-operator-config-change")
-	toRehearse.AddAll(presubmitsWitchChangedCiopConfigs)
+	presubmitsWithChangedCiopConfigs := diffs.GetPresubmitsForCiopConfigs(prConfig.Prow, changedCiopConfigs, logger, affectedJobs)
+	metrics.RecordOpportunity(presubmitsWithChangedCiopConfigs, "ci-operator-config-change")
+	toRehearse.AddAll(presubmitsWithChangedCiopConfigs)
+
+	presubmitsWithChangedTemplates := rehearse.AddRandomJobsForChangedTemplates(changedTemplates, prConfig.Prow.JobConfig.Presubmits, loggers, prNumber)
+	metrics.RecordOpportunity(presubmitsWithChangedTemplates, "templates-change")
+	toRehearse.AddAll(presubmitsWithChangedTemplates)
+
 	toRehearseClusterProfiles := diffs.GetPresubmitsForClusterProfiles(prConfig.Prow, changedClusterProfiles, logger)
 	metrics.RecordOpportunity(toRehearseClusterProfiles, "cluster-profile-change")
 	toRehearse.AddAll(toRehearseClusterProfiles)

pkg/rehearse/jobs.go

@@ -38,6 +38,8 @@ const (
 	logRehearsalJob              = "rehearsal-job"
 	logCiopConfigFile            = "ciop-config-file"
 	logCiopConfigRepo            = "ciop-config-repo"
+
+	clusterTypeEnvName = "CLUSTER_TYPE"
 )
 
 // Loggers holds the two loggers that will be used for normal and debug logging respectively.
@@ -203,17 +205,14 @@ func ConfigureRehearsalJobs(toBeRehearsed config.Presubmits, ciopConfigs config.
 				continue
 			}
 
-			exists, index, templateKey := hasChangedTemplateVolume(rehearsal.Spec.Containers[0].VolumeMounts, rehearsal.Spec.Volumes, templates)
-			if exists {
-				if templateData, err := config.GetTemplateData(templates[templateKey]); err != nil {
-					jobLogger.WithError(err).WithField("template-name", templates[templateKey].Name).Warn("couldn't get template's data. Job won't be rehearsed")
+			if allowVolumes {
+				if err := replaceCMTemplateName(rehearsal.Spec.Containers[0].VolumeMounts, rehearsal.Spec.Volumes, templates); err != nil {
+					jobLogger.WithError(err).Warn("Failed to replace the configmap name to the temporary one")
 					continue
-				} else {
-					templateName := config.GetTemplateName(templateKey)
-					rehearsal.Spec.Volumes[index].VolumeSource.ConfigMap.Name = config.GetTempCMName(templateName, templateKey, templateData)
 				}
+
+				replaceClusterProfiles(rehearsal.Spec.Volumes, profiles, loggers.Debug.WithField("name", job.Name))
 			}
-			replaceClusterProfiles(rehearsal.Spec.Volumes, profiles, loggers.Debug.WithField("name", job.Name))
 
 			jobLogger.WithField(logRehearsalJob, rehearsal.Name).Info("Created a rehearsal job to be submitted")
 			rehearsals = append(rehearsals, rehearsal)
@@ -223,24 +222,93 @@ func ConfigureRehearsalJobs(toBeRehearsed config.Presubmits, ciopConfigs config.
 	return rehearsals
 }
 
-func hasChangedTemplateVolume(volumeMounts []v1.VolumeMount, volumes []v1.Volume, templates config.CiTemplates) (bool, int, string) {
-	var templateKey string
-	var volumeName string
+// AddRandomJobsForChangedTemplates finds jobs from the PR config that are using a specific template with a specific cluster type.
+// The job selection is done by iterating in an unspecified order, which avoids picking the same job
+// So if a template will be changed, find the jobs that are using a template in combination with the `aws`,`openstack`,`gcs` and `libvirt` cluster types.
+func AddRandomJobsForChangedTemplates(templates config.CiTemplates, prConfigPresubmits map[string][]prowconfig.Presubmit, loggers Loggers, prNumber int) config.Presubmits {
+	rehearsals := make(config.Presubmits)
+
+	for templateFile := range templates {
+		for _, clusterType := range []string{"aws", "gcs", "openstack", "libvirt"} {
+			if repo, job := pickTemplateJob(prConfigPresubmits, templateFile, clusterType); job != nil {
+				jobLogger := loggers.Job.WithFields(logrus.Fields{"target-repo": repo, "target-job": job.Name})
+				jobLogger.Info("Picking job to rehearse the template changes")
+				rehearsals[repo] = append(rehearsals[repo], *job)
+			}
+		}
+	}
+	return rehearsals
+}
+
+func replaceCMTemplateName(volumeMounts []v1.VolumeMount, volumes []v1.Volume, templates config.CiTemplates) error {
+	replace := func(v *v1.Volume) error {
+		volumeName, templateKey := hasChangedTemplateVolume(volumeMounts, *v, templates)
+		if len(volumeName) == 0 || len(templateKey) == 0 || volumeName != v.Name {
+			return nil
+		}
+
+		templateData, err := config.GetTemplateData(templates[templateKey])
+		if err != nil {
+			return fmt.Errorf("couldn't get template's data: %s: %v", templates[templateKey], err)
+		}
+
+		name := config.GetTempCMName(config.GetTemplateName(templateKey), templateKey, templateData)
+		v.VolumeSource.ConfigMap.Name = name
+
+		return nil
+	}
+
+	for _, volume := range volumes {
+		if err := replace(&volume); err != nil {
+			return err
+		}
+	}
 
+	return nil
+}
+
+func hasChangedTemplateVolume(volumeMounts []v1.VolumeMount, volume v1.Volume, templates config.CiTemplates) (string, string) {
 	for _, volumeMount := range volumeMounts {
 		if _, ok := templates[volumeMount.SubPath]; ok {
-			templateKey = volumeMount.SubPath
-			volumeName = volumeMount.Name
+			return volumeMount.Name, volumeMount.SubPath
 		}
 	}
+	return "", ""
+}
 
-	for index, volume := range volumes {
-		if volume.Name == volumeName {
-			return true, index, templateKey
+func pickTemplateJob(presubmits map[string][]prowconfig.Presubmit, templateFile, clusterType string) (string, *prowconfig.Presubmit) {
+	for repo, jobs := range presubmits {
+		for _, job := range jobs {
+			if job.Agent != string(pjapi.KubernetesAgent) {
+				continue
+			}
+
+			if hasClusterType(job, clusterType) && hasTemplateFile(job, templateFile) {
+				return repo, &job
+			}
 		}
 	}
+	return "", nil
+}
+
+func hasClusterType(job prowconfig.Presubmit, clusterType string) bool {
+	for _, env := range job.Spec.Containers[0].Env {
+		if env.Name == clusterTypeEnvName && env.Value == clusterType {
+			return true
+		}
+	}
+	return false
+}
 
-	return false, 0, ""
+func hasTemplateFile(job prowconfig.Presubmit, templateFile string) bool {
+	if job.Spec.Containers[0].VolumeMounts != nil {
+		for _, volumeMount := range job.Spec.Containers[0].VolumeMounts {
+			if volumeMount.SubPath == templateFile {
+				return true
+			}
+		}
+	}
+	return false
 }
 
 func replaceClusterProfiles(volumes []v1.Volume, profiles []config.ClusterProfile, logger *logrus.Entry) {

pkg/rehearse/jobs_test.go

@@ -827,7 +827,9 @@ func makeBasePresubmit() *prowconfig.Presubmit {
 	}
 }
 
-func TestHasChangedTemplateVolume(t *testing.T) {
+func TestReplaceCMTemplateName(t *testing.T) {
+	const tempCMName = "rehearse-pv8j80dg-test-template"
+
 	templates := config.CiTemplates{
 		"test-template.yaml": &templateapi.Template{
 			ObjectMeta: metav1.ObjectMeta{
@@ -846,23 +848,17 @@ func TestHasChangedTemplateVolume(t *testing.T) {
 		},
 	}
 
-	type expectedToFind struct {
-		exists      bool
-		index       int
-		templateKey string
-	}
-
 	testCases := []struct {
 		description     string
 		jobVolumeMounts []v1.VolumeMount
 		jobVolumes      []v1.Volume
-		expectedToFind  expectedToFind
+		expectedToFind  func() []v1.Volume
 	}{
 		{
 			description:     "no volumes",
 			jobVolumeMounts: []v1.VolumeMount{},
 			jobVolumes:      []v1.Volume{},
-			expectedToFind:  expectedToFind{},
+			expectedToFind:  func() []v1.Volume { return []v1.Volume{} },
 		},
 		{
 			description: "find one in multiple volumes",
@@ -878,10 +874,14 @@ func TestHasChangedTemplateVolume(t *testing.T) {
 				},
 			},
 			jobVolumes: createVolumesHelper("job-definition", "test-template.yaml"),
-			expectedToFind: expectedToFind{
-				exists:      true,
-				index:       2,
-				templateKey: "test-template.yaml",
+			expectedToFind: func() []v1.Volume {
+				volumes := createVolumesHelper("job-definition", "test-template.yaml")
+				for _, volume := range volumes {
+					if volume.Name == "job-definition" {
+						volume.VolumeSource.ConfigMap.Name = tempCMName
+					}
+				}
+				return volumes
 			},
 		},
 		{
@@ -898,10 +898,10 @@ func TestHasChangedTemplateVolume(t *testing.T) {
 				},
 			},
 			jobVolumes: append(createVolumesHelper("job-definition", "test-template.yaml"), createVolumesHelper("job-definition2", "test-template2.yaml")...),
-			expectedToFind: expectedToFind{
-				exists:      true,
-				index:       2,
-				templateKey: "test-template.yaml",
+			expectedToFind: func() []v1.Volume {
+				volumes := append(createVolumesHelper("job-definition", "test-template.yaml"), createVolumesHelper("job-definition2", "test-template2.yaml")...)
+				volumes[2].VolumeSource.ConfigMap.Name = tempCMName
+				return volumes
 			},
 		},
 		{
@@ -917,21 +917,22 @@ func TestHasChangedTemplateVolume(t *testing.T) {
 					SubPath:   "test-template5.yaml",
 				},
 			},
-			jobVolumes:     createVolumesHelper("job-definition", "test-template5.yaml"),
-			expectedToFind: expectedToFind{},
+			jobVolumes: createVolumesHelper("job-definition", "test-template5.yaml"),
+			expectedToFind: func() []v1.Volume {
+				return createVolumesHelper("job-definition", "test-template5.yaml")
+			},
 		},
 	}
 
 	for _, testCase := range testCases {
 		t.Run(testCase.description, func(t *testing.T) {
-			exists, index, templateKey := hasChangedTemplateVolume(testCase.jobVolumeMounts, testCase.jobVolumes, templates)
-			found := expectedToFind{
-				exists:      exists,
-				index:       index,
-				templateKey: templateKey,
+			if err := replaceCMTemplateName(testCase.jobVolumeMounts, testCase.jobVolumes, templates); err != nil {
+				t.Fatal()
 			}
-			if !reflect.DeepEqual(testCase.expectedToFind, found) {
-				t.Fatalf("Expected:%v\nFound:%v", testCase.expectedToFind, found)
+
+			expected := testCase.expectedToFind()
+			if !reflect.DeepEqual(expected, testCase.jobVolumes) {
+				t.Fatalf("Diff found %v", diff.ObjectReflectDiff(expected, testCase.jobVolumes))
 			}
 		})
 	}
@@ -960,7 +961,6 @@ func createVolumesHelper(name, key string) []v1.Volume {
 			},
 		},
 	}
-
 	volumes = append(volumes, v1.Volume{
 		Name: name,
 		VolumeSource: v1.VolumeSource{

test/pj-rehearse-integration/candidate/ci-operator/jobs/super/duper/super-duper-master-presubmits.yaml

@@ -201,3 +201,117 @@ presubmits:
     name: test_pull_request_origin_extended_gssapi
     rerun_command: /test extended_gssapi
     trigger: ((?m)^/test extended_gssapi,?(\s+|$))
+  - agent: kubernetes
+    always_run: true
+    branches:
+    - master
+    context: ci/prow/test-template-e2e-aws
+    decorate: true
+    decoration_config:
+      skip_cloning: true
+    name: pull-ci-super-duper-master-test-template-e2e-aws
+    rerun_command: /test test-template-e2e-aws
+    spec:
+      containers:
+      - args:
+        - --artifact-dir=$(ARTIFACTS)
+        - --give-pr-author-access-to-namespace=true
+        - --secret-dir=/usr/local/e2e-aws-cluster-profile
+        - --target=e2e-aws
+        - --template=/usr/local/e2e-aws
+        command:
+        - ci-operator
+        env:
+        - name: CLUSTER_TYPE
+          value: aws
+        - name: CONFIG_SPEC
+          valueFrom:
+            configMapKeyRef:
+              key: super-duper-master.yaml
+              name: ci-operator-master-configs
+        - name: JOB_NAME_SAFE
+          value: e2e-aws
+        - name: TEST_COMMAND
+          value: TEST_SUITE=openshift/conformance/parallel run-tests
+        image: ci-operator:latest
+        imagePullPolicy: Always
+        name: ""
+        resources:
+          limits:
+            cpu: 500m
+          requests:
+            cpu: 10m
+        volumeMounts:
+        - mountPath: /usr/local/e2e-aws-cluster-profile
+          name: cluster-profile
+        - mountPath: /usr/local/e2e-aws
+          name: job-definition
+          subPath: test-template.yaml
+      serviceAccountName: ci-operator
+      volumes:
+      - name: cluster-profile
+        projected:
+          sources:
+          - secret:
+              name: cluster-secrets-aws
+      - configMap:
+          name: prow-job-cluster-launch-installer-e2e
+        name: job-definition
+    trigger: ((?m)^/test( all| test-template-e2e-aws),?(\s+|$))
+  - agent: kubernetes
+    always_run: true
+    branches:
+    - master
+    context: ci/prow/test-template-e2e-openstack
+    decorate: true
+    decoration_config:
+      skip_cloning: true
+    name: pull-ci-super-duper-master-test-template-e2e-openstack
+    rerun_command: /test test-template-e2e-openstack
+    spec:
+      containers:
+      - args:
+        - --artifact-dir=$(ARTIFACTS)
+        - --give-pr-author-access-to-namespace=true
+        - --secret-dir=/usr/local/e2e-aws-cluster-profile
+        - --target=e2e-aws
+        - --template=/usr/local/e2e-aws-openstack
+        command:
+        - ci-operator
+        env:
+        - name: CLUSTER_TYPE
+          value: openstack
+        - name: CONFIG_SPEC
+          valueFrom:
+            configMapKeyRef:
+              key: super-duper-master.yaml
+              name: ci-operator-master-configs
+        - name: JOB_NAME_SAFE
+          value: e2e-aws
+        - name: TEST_COMMAND
+          value: TEST_SUITE=openshift/conformance/parallel run-tests
+        image: ci-operator:latest
+        imagePullPolicy: Always
+        name: ""
+        resources:
+          limits:
+            cpu: 500m
+          requests:
+            cpu: 10m
+        volumeMounts:
+        - mountPath: /usr/local/e2e-openstack-cluster-profile
+          name: cluster-profile
+        - mountPath: /usr/local/e2e-openstack
+          name: job-definition
+          subPath: test-template.yaml
+      serviceAccountName: ci-operator
+      volumes:
+      - name: cluster-profile
+        projected:
+          sources:
+          - secret:
+              name: cluster-secrets-openstack
+      - configMap:
+          name: prow-job-cluster-launch-installer-e2e
+        name: job-definition
+    trigger: ((?m)^/test( all| test-template-e2e-openstack),?(\s+|$))

test/pj-rehearse-integration/candidate/ci-operator/templates/test-template.yaml

@@ -83,6 +83,7 @@ objects:
       command:
       - echo
       - test
+      - CHANGED
 
     # Runs an install
     - name: setup