Implement multi-stage test registry interface

pkg/api/config.go

@@ -271,6 +271,12 @@ func validateTestConfigurationType(fieldRoot string, test TestStepConfiguration,
 		typeCount++
 		validationErrors = append(validationErrors, validateClusterProfile(fmt.Sprintf("%s", fieldRoot), testConfig.ClusterProfile)...)
 	}
+	if testConfig := test.MultiStageTestConfiguration; testConfig != nil {
+		typeCount++
+		if testConfig.ClusterProfile != "" {
+			validationErrors = append(validationErrors, validateClusterProfile(fmt.Sprintf("%s", fieldRoot), testConfig.ClusterProfile)...)
+		}
+	}
 	if test.OpenshiftInstallerRandomClusterTestConfiguration != nil {
 		typeCount++
 	}

pkg/api/types.go

@@ -294,6 +294,7 @@ type TestStepConfiguration struct {
 
 	// Only one of the following can be not-null.
 	ContainerTestConfiguration                                *ContainerTestConfiguration                                `json:"container,omitempty"`
+	MultiStageTestConfiguration                               *MultiStageTestConfiguration                               `json:"steps,omitempty"`
 	OpenshiftAnsibleClusterTestConfiguration                  *OpenshiftAnsibleClusterTestConfiguration                  `json:"openshift_ansible,omitempty"`
 	OpenshiftAnsibleSrcClusterTestConfiguration               *OpenshiftAnsibleSrcClusterTestConfiguration               `json:"openshift_ansible_src,omitempty"`
 	OpenshiftAnsibleCustomClusterTestConfiguration            *OpenshiftAnsibleCustomClusterTestConfiguration            `json:"openshift_ansible_custom,omitempty"`
@@ -307,12 +308,25 @@ type TestStepConfiguration struct {
 	OpenshiftInstallerCustomTestImageClusterTestConfiguration *OpenshiftInstallerCustomTestImageClusterTestConfiguration `json:"openshift_installer_custom_test_image,omitempty"`
 }
 
+// TestStep is the external representation of a test step that allows users to
+// write less verbose test configs. It gets converted to an internal TestStep
+// struct that represents the full configuration that ci-operator can use.
 type TestStep struct {
-	Name        string               `json:"name,omitempty"`
-	Image       string               `json:"image,omitempty"`
-	Commands    string               `json:"commands,omitempty"`
-	ArtifactDir string               `json:"artifact_dir,omitempty"`
-	Resources   ResourceRequirements `json:"resources,omitempty"`
+	Name          string               `json:"name,omitempty"`
+	Documentation string               `json:"documentation,omitempty"`
+	Image         string               `json:"image,omitempty"`
+	Commands      string               `json:"commands,omitempty"`
+	ArtifactDir   string               `json:"artifact_dir,omitempty"`
+	Resources     ResourceRequirements `json:"resources,omitempty"`
+}
+
+// MultiStageTestConfiguration is a flexible configuration mode that allows tighter control over
+// the multiple stages of end to end tests
+type MultiStageTestConfiguration struct {
+	ClusterProfile ClusterProfile `json:"cluster_profile"`
+	Pre            []TestStep     `json:"pre,omitempty"`
+	Test           []TestStep     `json:"test,omitempty"`
+	Post           []TestStep     `json:"post,omitempty"`
 }
 
 // Secret describes a secret to be mounted inside a test

pkg/registry/resolver.go

@@ -0,0 +1,55 @@
+package registry
+
+import (
+	api "github.com/openshift/ci-tools/pkg/api"
+	types "github.com/openshift/ci-tools/pkg/steps/types"
+	"k8s.io/apimachinery/pkg/util/errors"
+)
+
+type Resolver interface {
+	Resolve(config api.MultiStageTestConfiguration) (types.TestFlow, error)
+}
+
+// Registry will hold all the registry information needed to convert between the
+// user provided configs referencing the registry and the internal, complete
+// representation
+type registry struct{}
+
+func NewResolver() Resolver {
+	return &registry{}
+}
+
+func (r *registry) Resolve(config api.MultiStageTestConfiguration) (types.TestFlow, error) {
+	var resolveErrors []error
+	expandedFlow := types.TestFlow{
+		ClusterProfile: config.ClusterProfile,
+	}
+	for _, external := range config.Pre {
+		newStep := toInternal(external)
+		expandedFlow.Pre = append(expandedFlow.Pre, newStep)
+	}
+	for _, external := range config.Test {
+		newStep := toInternal(external)
+		expandedFlow.Test = append(expandedFlow.Test, newStep)
+	}
+	for _, external := range config.Post {
+		newStep := toInternal(external)
+		expandedFlow.Post = append(expandedFlow.Post, newStep)
+	}
+	// This currently never gets run as we don't have any situations that can result in errors yet.
+	// This will become used as we add more functionality.
+	if resolveErrors != nil {
+		return types.TestFlow{}, errors.NewAggregate(resolveErrors)
+	}
+	return expandedFlow, nil
+}
+
+func toInternal(input api.TestStep) types.TestStep {
+	return types.TestStep{
+		Name:        input.Name,
+		Image:       input.Image,
+		Commands:    input.Commands,
+		ArtifactDir: input.ArtifactDir,
+		Resources:   input.Resources,
+	}
+}

pkg/registry/resolver_test.go

@@ -0,0 +1,96 @@
+package registry
+
+import (
+	"reflect"
+	"testing"
+
+	api "github.com/openshift/ci-tools/pkg/api"
+	types "github.com/openshift/ci-tools/pkg/steps/types"
+	"k8s.io/apimachinery/pkg/util/diff"
+)
+
+func TestResolve(t *testing.T) {
+	for _, testCase := range []struct {
+		name        string
+		config      api.MultiStageTestConfiguration
+		expectedRes types.TestFlow
+		expectErr   bool
+	}{{
+		// This is a full config that should not change (other than struct) when passed to the Resolver
+		name: "Full AWS IPI",
+		config: api.MultiStageTestConfiguration{
+			ClusterProfile: api.ClusterProfileAWS,
+			Pre: []api.TestStep{{
+				Name:     "ipi-install",
+				Image:    "installer",
+				Commands: "openshift-cluster install",
+				Resources: api.ResourceRequirements{
+					Requests: api.ResourceList{"cpu": "1000m"},
+					Limits:   api.ResourceList{"memory": "2Gi"},
+				},
+			}},
+			Test: []api.TestStep{{
+				Name:     "e2e",
+				Image:    "my-image",
+				Commands: "make custom-e2e",
+				Resources: api.ResourceRequirements{
+					Requests: api.ResourceList{"cpu": "1000m"},
+					Limits:   api.ResourceList{"memory": "2Gi"},
+				},
+			}},
+			Post: []api.TestStep{{
+				Name:     "ipi-teardown",
+				Image:    "installer",
+				Commands: "openshift-cluster destroy",
+				Resources: api.ResourceRequirements{
+					Requests: api.ResourceList{"cpu": "1000m"},
+					Limits:   api.ResourceList{"memory": "2Gi"},
+				},
+			}},
+		},
+		expectedRes: types.TestFlow{
+			ClusterProfile: api.ClusterProfileAWS,
+			Pre: []types.TestStep{{
+				Name:     "ipi-install",
+				Image:    "installer",
+				Commands: "openshift-cluster install",
+				Resources: api.ResourceRequirements{
+					Requests: api.ResourceList{"cpu": "1000m"},
+					Limits:   api.ResourceList{"memory": "2Gi"},
+				},
+			}},
+			Test: []types.TestStep{{
+				Name:     "e2e",
+				Image:    "my-image",
+				Commands: "make custom-e2e",
+				Resources: api.ResourceRequirements{
+					Requests: api.ResourceList{"cpu": "1000m"},
+					Limits:   api.ResourceList{"memory": "2Gi"},
+				},
+			}},
+			Post: []types.TestStep{{
+				Name:     "ipi-teardown",
+				Image:    "installer",
+				Commands: "openshift-cluster destroy",
+				Resources: api.ResourceRequirements{
+					Requests: api.ResourceList{"cpu": "1000m"},
+					Limits:   api.ResourceList{"memory": "2Gi"},
+				},
+			}},
+		},
+		expectErr: false,
+	}, {}} {
+		t.Run(testCase.name, func(t *testing.T) {
+			ret, err := NewResolver().Resolve(testCase.config)
+			if !testCase.expectErr && err != nil {
+				t.Errorf("%s: expected no error but got: %s", testCase.name, err)
+			}
+			if testCase.expectErr && err == nil {
+				t.Errorf("%s: expected error but got none", testCase.name)
+			}
+			if !reflect.DeepEqual(ret, testCase.expectedRes) {
+				t.Errorf("%s: fo incorrect output: %s", testCase.name, diff.ObjectReflectDiff(ret, testCase.expectedRes))
+			}
+		})
+	}
+}

pkg/steps/types/types.go

@@ -0,0 +1,23 @@
+package steptypes
+
+import (
+	api "github.com/openshift/ci-tools/pkg/api"
+)
+
+// TestStep contains a full definition of a test step from a MultiStageTestConfiguration
+type TestStep struct {
+	Name        string
+	Image       string
+	Commands    string
+	ArtifactDir string
+	Resources   api.ResourceRequirements
+}
+
+// TestFlow contains the separate stages of a MultiStageTestConfigurations with full TestSteps in each stage
+// ClusterProfile can be used to define an infrastructure/profile to use (i.e. aws, azure4, gcp, etc.)
+type TestFlow struct {
+	ClusterProfile api.ClusterProfile
+	Pre            []TestStep
+	Test           []TestStep
+	Post           []TestStep
+}