ci-operator multi-stage: MultiStageTestStep

pkg/api/config.go

@@ -103,33 +103,39 @@ func validateTestStepConfiguration(fieldRoot string, input []TestStepConfigurati
 	validationErrors = append(validationErrors, searchForTestDuplicates(input)...)
 
 	for num, test := range input {
+		fieldRootN := fmt.Sprintf("%s[%d]", fieldRoot, num)
 		if len(test.As) == 0 {
-			validationErrors = append(validationErrors, fmt.Errorf("%s[%d].as: is required", fieldRoot, num))
+			validationErrors = append(validationErrors, fmt.Errorf("%s.as: is required", fieldRootN))
 		} else if test.As == "images" {
-			validationErrors = append(validationErrors, fmt.Errorf("%s[%d].as: should not be called 'images' because it gets confused with '[images]' target", fieldRoot, num))
+			validationErrors = append(validationErrors, fmt.Errorf("%s.as: should not be called 'images' because it gets confused with '[images]' target", fieldRootN))
 		} else if ok := regexp.MustCompile("^[a-zA-Z0-9_.-]*$").MatchString(test.As); !ok {
-			validationErrors = append(validationErrors, fmt.Errorf("%s[%d].as: '%s' is not valid value, should be [a-zA-Z0-9_.-]", fieldRoot, num, test.As))
+			validationErrors = append(validationErrors, fmt.Errorf("%s.as: '%s' is not valid value, should be [a-zA-Z0-9_.-]", fieldRootN, test.As))
 		}
-
-		if len(test.Commands) == 0 {
-			validationErrors = append(validationErrors, fmt.Errorf("%s[%d].commands: is required", fieldRoot, num))
+		if hasCommands, hasSteps := len(test.Commands) != 0, test.MultiStageTestConfiguration != nil; !hasCommands && !hasSteps {
+			validationErrors = append(validationErrors, fmt.Errorf("%s: either `commands` or `steps` should be set", fieldRootN))
+		} else if hasCommands && hasSteps {
+			validationErrors = append(validationErrors, fmt.Errorf("%s: `commands` and `steps` are mutually exclusive", fieldRootN))
+		} else if hasSteps {
+			validationErrors = append(validationErrors, validateTestSteps(fieldRootN+".steps.pre", test.MultiStageTestConfiguration.Pre)...)
+			validationErrors = append(validationErrors, validateTestSteps(fieldRootN+".steps.test", test.MultiStageTestConfiguration.Test)...)
+			validationErrors = append(validationErrors, validateTestSteps(fieldRootN+".steps.post", test.MultiStageTestConfiguration.Post)...)
 		}
 
 		if test.Secret != nil {
 			// TODO: Move to upstream validation when vendoring is fixed
 			// currently checking against DNS RFC 1123 regexp
 			if ok := regexp.MustCompile("^[a-z0-9]([-a-z0-9]*[a-z0-9])?$").MatchString(test.Secret.Name); !ok {
-				validationErrors = append(validationErrors, fmt.Errorf("%s[%d].name: '%s' secret name is not valid value, should be [a-z0-9]([-a-z0-9]*[a-z0-9]", fieldRoot, num, test.Secret.Name))
+				validationErrors = append(validationErrors, fmt.Errorf("%s.name: '%s' secret name is not valid value, should be [a-z0-9]([-a-z0-9]*[a-z0-9]", fieldRootN, test.Secret.Name))
 			}
 			// validate path only if name is passed
 			if test.Secret.MountPath != "" {
 				if ok := filepath.IsAbs(test.Secret.MountPath); !ok {
-					validationErrors = append(validationErrors, fmt.Errorf("%s[%d].path: '%s' secret mount path is not valid value, should be ^((\\/*)\\w+)+", fieldRoot, num, test.Secret.MountPath))
+					validationErrors = append(validationErrors, fmt.Errorf("%s.path: '%s' secret mount path is not valid value, should be ^((\\/*)\\w+)+", fieldRootN, test.Secret.MountPath))
 				}
 			}
 		}
 
-		validationErrors = append(validationErrors, validateTestConfigurationType(fmt.Sprintf("%s[%d]", fieldRoot, num), test, release)...)
+		validationErrors = append(validationErrors, validateTestConfigurationType(fieldRootN, test, release)...)
 	}
 	return validationErrors
 }

pkg/api/job_spec.go

@@ -1,6 +1,7 @@
 package api
 
 import (
+	"crypto/sha256"
 	"encoding/json"
 	"fmt"
 
@@ -50,6 +51,10 @@ func (s *JobSpec) Inputs() InputDefinition {
 	return InputDefinition{string(raw)}
 }
 
+func (s JobSpec) JobNameHash() string {
+	return fmt.Sprintf("%x", sha256.Sum256([]byte(s.Job)))[:5]
+}
+
 // ResolveSpecFromEnv will determine the Refs being
 // tested in by parsing Prow environment variable contents
 func ResolveSpecFromEnv() (*JobSpec, error) {

pkg/api/types.go

@@ -27,8 +27,8 @@ type ReleaseBuildConfiguration struct {
 	// command. The created RPMs will then be served via HTTP to the "base" image
 	// via an injected rpm.repo in the standard location at /etc/yum.repos.d.
 	RpmBuildCommands string `json:"rpm_build_commands,omitempty"`
-	// RpmBuildLocation is where RPms are deposited/ after being built. If
-	// unset, this will default/ under the repository root to
+	// RpmBuildLocation is where RPms are deposited after being built. If
+	// unset, this will default under the repository root to
 	// _output/local/releases/rpms/.
 	RpmBuildLocation string `json:"rpm_build_location,omitempty"`
 
@@ -66,6 +66,39 @@ type ReleaseBuildConfiguration struct {
 	Resources ResourceConfiguration `json:"resources,omitempty"`
 }
 
+// BuildsImage checks if an image is built by the release configuration.
+func (c ReleaseBuildConfiguration) BuildsImage(name string) bool {
+	for _, i := range c.Images {
+		if string(i.To) == name {
+			return true
+		}
+	}
+	return false
+}
+
+// IsPipelineImage checks if `name` will be a tag in the pipeline image stream.
+func (c ReleaseBuildConfiguration) IsPipelineImage(name string) bool {
+	for i := range c.BaseImages {
+		if i == name {
+			return true
+		}
+	}
+	for i := range c.BaseRPMImages {
+		if i == name {
+			return true
+		}
+	}
+	switch name {
+	case string(PipelineImageStreamTagReferenceRoot),
+		string(PipelineImageStreamTagReferenceSource),
+		string(PipelineImageStreamTagReferenceBinaries),
+		string(PipelineImageStreamTagReferenceTestBinaries),
+		string(PipelineImageStreamTagReferenceRPMs):
+		return true
+	}
+	return false
+}
+
 // ResourceConfiguration defines resource overrides for jobs run
 // by the operator.
 type ResourceConfiguration map[string]ResourceRequirements

pkg/api/types_test.go

@@ -134,3 +134,65 @@ func TestOverlay(t *testing.T) {
 		})
 	}
 }
+
+func TestBuildsImage(t *testing.T) {
+	conf := ReleaseBuildConfiguration{
+		Images: []ProjectDirectoryImageBuildStepConfiguration{
+			{To: "this-image-is-in-the-images-field"},
+		},
+	}
+	for _, tc := range []struct {
+		name  string
+		image string
+		want  bool
+	}{{
+		name:  "not in `images`",
+		image: "this-image-is-not-in-the-images-field",
+	}, {
+		name:  "in `images`",
+		image: "this-image-is-in-the-images-field",
+		want:  true,
+	}} {
+		t.Run(tc.name, func(t *testing.T) {
+			if ret := conf.BuildsImage(tc.image); ret != tc.want {
+				t.Errorf("got %v, want %v", ret, tc.want)
+			}
+		})
+	}
+}
+
+func TestIsPipelineImage(t *testing.T) {
+	conf := ReleaseBuildConfiguration{
+		InputConfiguration: InputConfiguration{
+			BaseImages: map[string]ImageStreamTagReference{
+				"base-img": {},
+			},
+			BaseRPMImages: map[string]ImageStreamTagReference{
+				"base-rpm-img": {},
+			},
+		},
+		BinaryBuildCommands:     "make",
+		TestBinaryBuildCommands: "make test-bin",
+		RpmBuildCommands:        "make rpms",
+		Images:                  []ProjectDirectoryImageBuildStepConfiguration{{To: "img"}},
+	}
+	for _, tc := range []struct {
+		name string
+		want bool
+	}{
+		{name: "base-img", want: true},
+		{name: "base-rpm-img", want: true},
+		{name: "root", want: true},
+		{name: "bin", want: true},
+		{name: "test-bin", want: true},
+		{name: "rpms", want: true},
+		{name: "img"},
+		{name: "404"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			if ret := conf.IsPipelineImage(tc.name); ret != tc.want {
+				t.Errorf("want %v, got %v", tc.want, ret)
+			}
+		})
+	}
+}

pkg/defaults/defaults.go

@@ -1,7 +1,6 @@
 package defaults
 
 import (
-	"crypto/sha256"
 	"fmt"
 	"log"
 	"net/url"
@@ -99,7 +98,7 @@ func FromConfig(
 
 	params := api.NewDeferredParameters()
 	params.Add("JOB_NAME", nil, func() (string, error) { return jobSpec.Job, nil })
-	params.Add("JOB_NAME_HASH", nil, func() (string, error) { return fmt.Sprintf("%x", sha256.Sum256([]byte(jobSpec.Job)))[:5], nil })
+	params.Add("JOB_NAME_HASH", nil, func() (string, error) { return jobSpec.JobNameHash(), nil })
 	params.Add("JOB_NAME_SAFE", nil, func() (string, error) { return strings.Replace(jobSpec.Job, "_", "-", -1), nil })
 	params.Add("NAMESPACE", nil, func() (string, error) { return jobSpec.Namespace, nil })
 
@@ -155,7 +154,9 @@ func FromConfig(
 			buildSteps = append(buildSteps, initialReleaseStep)
 
 		} else if testStep := rawStep.TestStepConfiguration; testStep != nil {
-			if testStep.OpenshiftInstallerClusterTestConfiguration != nil {
+			if testStep.MultiStageTestConfiguration != nil {
+				step = steps.MultiStageTestStep(*testStep, config, podClient, jobSpec)
+			} else if testStep.OpenshiftInstallerClusterTestConfiguration != nil {
 				if testStep.OpenshiftInstallerClusterTestConfiguration.Upgrade {
 					var err error
 					step, err = clusterinstall.E2ETestStep(*testStep.OpenshiftInstallerClusterTestConfiguration, *testStep, params, podClient, templateClient, secretGetter, artifactDir, jobSpec)
@@ -413,10 +414,7 @@ func stepConfigsForBuild(config *api.ReleaseBuildConfiguration, jobSpec *api.Job
 
 	for i := range config.Tests {
 		test := &config.Tests[i]
-		switch {
-		case test.ContainerTestConfiguration != nil:
-			buildSteps = append(buildSteps, api.StepConfiguration{TestStepConfiguration: test})
-		case test.OpenshiftInstallerClusterTestConfiguration != nil && test.OpenshiftInstallerClusterTestConfiguration.Upgrade:
+		if test.ContainerTestConfiguration != nil || test.MultiStageTestConfiguration != nil || (test.OpenshiftInstallerClusterTestConfiguration != nil && test.OpenshiftInstallerClusterTestConfiguration.Upgrade) {
 			buildSteps = append(buildSteps, api.StepConfiguration{TestStepConfiguration: test})
 		}
 	}