ci-operator multi-stage: add step parameters

pkg/api/config.go

@@ -347,9 +347,9 @@ func validateTestConfigurationType(fieldRoot string, test TestStepConfiguration,
 			validationErrors = append(validationErrors, validateClusterProfile(fieldRoot, testConfig.ClusterProfile)...)
 		}
 		seen := sets.NewString()
-		validationErrors = append(validationErrors, validateTestSteps(fmt.Sprintf("%s.Pre", fieldRoot), testConfig.Pre, seen)...)
-		validationErrors = append(validationErrors, validateTestSteps(fmt.Sprintf("%s.Test", fieldRoot), testConfig.Test, seen)...)
-		validationErrors = append(validationErrors, validateTestSteps(fmt.Sprintf("%s.Post", fieldRoot), testConfig.Post, seen)...)
+		validationErrors = append(validationErrors, validateTestSteps(fmt.Sprintf("%s.Pre", fieldRoot), testConfig.Pre, seen, testConfig.Environment)...)
+		validationErrors = append(validationErrors, validateTestSteps(fmt.Sprintf("%s.Test", fieldRoot), testConfig.Test, seen, testConfig.Environment)...)
+		validationErrors = append(validationErrors, validateTestSteps(fmt.Sprintf("%s.Post", fieldRoot), testConfig.Post, seen, testConfig.Environment)...)
 	}
 	if testConfig := test.MultiStageTestConfigurationLiteral; testConfig != nil {
 		typeCount++
@@ -359,15 +359,15 @@ func validateTestConfigurationType(fieldRoot string, test TestStepConfiguration,
 		seen := sets.NewString()
 		for i, s := range testConfig.Pre {
 			fieldRootI := fmt.Sprintf("%s.Pre[%d]", fieldRoot, i)
-			validationErrors = append(validationErrors, validateLiteralTestStep(fieldRootI, s, seen)...)
+			validationErrors = append(validationErrors, validateLiteralTestStep(fieldRootI, s, seen, testConfig.Environment)...)
 		}
 		for i, s := range testConfig.Test {
 			fieldRootI := fmt.Sprintf("%s.Test[%d]", fieldRoot, i)
-			validationErrors = append(validationErrors, validateLiteralTestStep(fieldRootI, s, seen)...)
+			validationErrors = append(validationErrors, validateLiteralTestStep(fieldRootI, s, seen, testConfig.Environment)...)
 		}
 		for i, s := range testConfig.Post {
 			fieldRootI := fmt.Sprintf("%s.Post[%d]", fieldRoot, i)
-			validationErrors = append(validationErrors, validateLiteralTestStep(fieldRootI, s, seen)...)
+			validationErrors = append(validationErrors, validateLiteralTestStep(fieldRootI, s, seen, testConfig.Environment)...)
 		}
 	}
 	if test.OpenshiftInstallerRandomClusterTestConfiguration != nil {
@@ -386,7 +386,7 @@ func validateTestConfigurationType(fieldRoot string, test TestStepConfiguration,
 	return validationErrors
 }
 
-func validateTestSteps(fieldRoot string, steps []TestStep, seen sets.String) (ret []error) {
+func validateTestSteps(fieldRoot string, steps []TestStep, seen sets.String, env TestEnvironment) (ret []error) {
 	for i, s := range steps {
 		fieldRootI := fmt.Sprintf("%s[%d]", fieldRoot, i)
 		if (s.LiteralTestStep != nil && s.Reference != nil) ||
@@ -418,13 +418,13 @@ func validateTestSteps(fieldRoot string, steps []TestStep, seen sets.String) (re
 			}
 		}
 		if s.LiteralTestStep != nil {
-			ret = append(ret, validateLiteralTestStep(fieldRootI, *s.LiteralTestStep, seen)...)
+			ret = append(ret, validateLiteralTestStep(fieldRootI, *s.LiteralTestStep, seen, env)...)
 		}
 	}
 	return
 }
 
-func validateLiteralTestStep(fieldRoot string, step LiteralTestStep, seen sets.String) (ret []error) {
+func validateLiteralTestStep(fieldRoot string, step LiteralTestStep, seen sets.String, env TestEnvironment) (ret []error) {
 	if len(step.As) == 0 {
 		ret = append(ret, fmt.Errorf("%s: `as` is required", fieldRoot))
 	} else if seen.Has(step.As) {
@@ -454,6 +454,9 @@ func validateLiteralTestStep(fieldRoot string, step LiteralTestStep, seen sets.S
 	}
 	ret = append(ret, validateResourceRequirements(fieldRoot+".resources", step.Resources)...)
 	ret = append(ret, validateCredentials(fieldRoot, step.Credentials)...)
+	if err := validateParameters(fieldRoot, step.Environment, env); err != nil {
+		ret = append(ret, err)
+	}
 	return
 }
 
@@ -502,6 +505,22 @@ func validateCredentials(fieldRoot string, credentials []CredentialReference) []
 	return errs
 }
 
+func validateParameters(fieldRoot string, params []StepParameter, env TestEnvironment) error {
+	var missing []string
+	for _, param := range params {
+		if param.Default != "" {
+			continue
+		}
+		if _, ok := env[param.Name]; !ok {
+			missing = append(missing, param.Name)
+		}
+	}
+	if missing != nil {
+		return fmt.Errorf("%s: unresolved parameter(s): %s", fieldRoot, missing)
+	}
+	return nil
+}
+
 func validateReleaseBuildConfiguration(input *ReleaseBuildConfiguration, org, repo string) []error {
 	var validationErrors []error
 

pkg/api/config_test.go

@@ -687,14 +687,53 @@ func TestValidateTestSteps(t *testing.T) {
 			if seen == nil {
 				seen = sets.NewString()
 			}
-			ret := validateTestSteps("test", tc.steps, seen)
+			ret := validateTestSteps("test", tc.steps, seen, nil)
 			if !reflect.DeepEqual(ret, tc.errs) {
 				t.Fatal(diff.ObjectReflectDiff(ret, tc.errs))
 			}
 		})
 	}
 }
 
+func TestValidateParameters(t *testing.T) {
+	for _, tc := range []struct {
+		name   string
+		params []StepParameter
+		env    TestEnvironment
+		err    []error
+	}{{
+		name: "no parameters",
+	}, {
+		name:   "has parameter, parameter provided",
+		params: []StepParameter{{Name: "TEST"}},
+		env:    TestEnvironment{"TEST": "test"},
+	}, {
+		name:   "has parameter with default, no parameter provided",
+		params: []StepParameter{{Name: "TEST", Default: "default"}},
+	}, {
+		name:   "has parameters, some not provided",
+		params: []StepParameter{{Name: "TEST0"}, {Name: "TEST1"}},
+		env:    TestEnvironment{"TEST0": "test0"},
+		err:    []error{errors.New("test: unresolved parameter(s): [TEST1]")},
+	}} {
+		t.Run(tc.name, func(t *testing.T) {
+			err := validateLiteralTestStep("test", LiteralTestStep{
+				As:       "as",
+				From:     "from",
+				Commands: "commands",
+				Resources: ResourceRequirements{
+					Requests: ResourceList{"cpu": "1"},
+					Limits:   ResourceList{"memory": "1m"},
+				},
+				Environment: tc.params,
+			}, sets.NewString(), tc.env)
+			if diff := diff.ObjectReflectDiff(err, tc.err); diff != "<no diffs>" {
+				t.Errorf("incorrect error: %s", diff)
+			}
+		})
+	}
+}
+
 func TestValidateResources(t *testing.T) {
 	for _, testCase := range []struct {
 		name        string

pkg/api/types.go

@@ -523,6 +523,8 @@ type RegistryChain struct {
 	Steps []TestStep `json:"steps"`
 	// Documentation describes what the chain does.
 	Documentation string `json:"documentation,omitempty"`
+	// Environment lists parameters that should be set by the test.
+	Environment []StepParameter `json:"env,omitempty"`
 }
 
 // RegistryWorkflowConfig is the struct that workflow references are unmarshalled into.
@@ -560,6 +562,18 @@ type LiteralTestStep struct {
 	Resources ResourceRequirements `json:"resources,omitempty"`
 	// Credentials defines the credentials we'll mount into this step.
 	Credentials []CredentialReference `json:"credentials,omitempty"`
+	// Environment lists parameters that should be set by the test.
+	Environment []StepParameter `json:"env,omitempty"`
+}
+
+// StepParameter is a variable set by the test, with an optional default.
+type StepParameter struct {
+	// Name of the environment variable.
+	Name string `json:"name"`
+	// Default if not set, optional, makes the parameter not required if set.
+	Default string `json:"default,omitempty"`
+	// Documentation is a textual description of the parameter.
+	Documentation string `json:"documentation,omitempty"`
 }
 
 // CredentialReference defines a secret to mount into a step and where to mount it.
@@ -608,6 +622,8 @@ type MultiStageTestConfiguration struct {
 	// Workflow is the name of the workflow to be used for this configuration. For fields defined in both
 	// the config and the workflow, the fields from the config will override what is set in Workflow.
 	Workflow *string `json:"workflow,omitempty"`
+	// Environment has the values of parameters for the steps.
+	Environment TestEnvironment `json:"env,omitempty"`
 }
 
 // MultiStageTestConfigurationLiteral is a form of the MultiStageTestConfiguration that does not include
@@ -623,8 +639,13 @@ type MultiStageTestConfigurationLiteral struct {
 	// Post is the array of test steps run after the tests finish and teardown/deprovision resources.
 	// Post steps always run, even if previous steps fail.
 	Post []LiteralTestStep `json:"post,omitempty"`
+	// Environment has the values of parameters for the steps.
+	Environment TestEnvironment `json:"env,omitempty"`
 }
 
+// TestEnvironment has the values of parameters for multi-stage tests.
+type TestEnvironment map[string]string
+
 // Secret describes a secret to be mounted inside a test
 // container.
 type Secret struct {

pkg/registry/resolver.go

@@ -53,11 +53,18 @@ func (r *registry) Resolve(name string, config api.MultiStageTestConfiguration)
 		if config.Post == nil {
 			config.Post = workflow.Post
 		}
+		if config.Environment == nil {
+			config.Environment = make(api.TestEnvironment, len(workflow.Environment))
+			for k, v := range workflow.Environment {
+				config.Environment[k] = v
+			}
+		}
 	}
 	expandedFlow := api.MultiStageTestConfigurationLiteral{
 		ClusterProfile: config.ClusterProfile,
 	}
-	stack := []stackRecord{stackRecord(name)}
+	rec := stackRecordForTest(name, config.Environment)
+	stack := []stackRecord{rec}
 	pre, errs := r.process(config.Pre, sets.NewString(), stack)
 	expandedFlow.Pre = append(expandedFlow.Pre, pre...)
 	resolveErrors = append(resolveErrors, errs...)
@@ -69,19 +76,41 @@ func (r *registry) Resolve(name string, config api.MultiStageTestConfiguration)
 	post, errs := r.process(config.Post, sets.NewString(), stack)
 	expandedFlow.Post = append(expandedFlow.Post, post...)
 	resolveErrors = append(resolveErrors, errs...)
-
+	for u := range rec.unused {
+		resolveErrors = append(resolveErrors, stackErrorf(stack, "no step declares parameter %q", u))
+	}
 	if resolveErrors != nil {
 		return api.MultiStageTestConfigurationLiteral{}, errors.NewAggregate(resolveErrors)
 	}
 	return expandedFlow, nil
 }
 
-type stackRecord string
+type stackRecord struct {
+	name   string
+	env    []api.StepParameter
+	unused sets.String
+}
+
+func stackRecordForStep(name string, env []api.StepParameter) stackRecord {
+	unused := sets.NewString()
+	for _, x := range env {
+		unused.Insert(x.Name)
+	}
+	return stackRecord{name: name, env: env, unused: unused}
+}
+
+func stackRecordForTest(name string, env api.TestEnvironment) stackRecord {
+	params := make([]api.StepParameter, 0, len(env))
+	for k, v := range env {
+		params = append(params, api.StepParameter{Name: k, Default: v})
+	}
+	return stackRecordForStep(name, params)
+}
 
 func stackErrorf(s []stackRecord, format string, args ...interface{}) error {
 	var b strings.Builder
 	for i := range s {
-		b.WriteString(string(s[i]))
+		b.WriteString(s[i].name)
 		b.WriteString(": ")
 	}
 	args = append([]interface{}{b.String()}, args...)
@@ -95,9 +124,9 @@ func (r *registry) process(steps []api.TestStep, seen sets.String, stack []stack
 			errs = append(errs, err...)
 			ret = append(ret, steps...)
 		} else {
-			if step, err := r.processStep(&step, seen, stack); err != nil {
-				errs = append(errs, err)
-			} else {
+			step, err := r.processStep(&step, seen, stack)
+			errs = append(errs, err...)
+			if err == nil {
 				ret = append(ret, step)
 			}
 		}
@@ -111,26 +140,55 @@ func (r *registry) processChain(step *api.TestStep, seen sets.String, stack []st
 	if !ok {
 		return nil, []error{stackErrorf(stack, "unknown step chain: %s", name)}
 	}
-	return r.process(chain.Steps, seen, append(stack, stackRecord(name)))
+	rec := stackRecordForStep(name, chain.Environment)
+	stack = append(stack, rec)
+	ret, err := r.process(chain.Steps, seen, stack)
+	for u := range rec.unused {
+		err = append(err, stackErrorf(stack, "no step declares parameter %q", u))
+	}
+	return ret, err
 }
 
-func (r *registry) processStep(step *api.TestStep, seen sets.String, stack []stackRecord) (ret api.LiteralTestStep, err error) {
+func (r *registry) processStep(step *api.TestStep, seen sets.String, stack []stackRecord) (ret api.LiteralTestStep, err []error) {
 	if ref := step.Reference; ref != nil {
 		var ok bool
 		ret, ok = r.stepsByName[*ref]
 		if !ok {
-			return api.LiteralTestStep{}, stackErrorf(stack, "invalid step reference: %s", *ref)
+			return api.LiteralTestStep{}, []error{stackErrorf(stack, "invalid step reference: %s", *ref)}
 		}
 	} else if step.LiteralTestStep != nil {
 		ret = *step.LiteralTestStep
 	} else {
-		return api.LiteralTestStep{}, stackErrorf(stack, "encountered TestStep where both `Reference` and `LiteralTestStep` are nil")
+		return api.LiteralTestStep{}, []error{stackErrorf(stack, "encountered TestStep where both `Reference` and `LiteralTestStep` are nil")}
 	}
 	if seen.Has(ret.As) {
-		return api.LiteralTestStep{}, stackErrorf(stack, "duplicate name: %s", ret.As)
+		return api.LiteralTestStep{}, []error{stackErrorf(stack, "duplicate name: %s", ret.As)}
 	}
 	seen.Insert(ret.As)
-	return ret, nil
+	var errs []error
+	for i, e := range ret.Environment {
+		if v := resolveVariable(e.Name, stack); v != nil {
+			ret.Environment[i].Default = *v
+		} else if e.Default == "" {
+			errs = append(errs, stackErrorf(stack, "%s: unresolved parameter: %s", ret.As, e.Name))
+		}
+
+	}
+	return ret, errs
+}
+
+func resolveVariable(name string, stack []stackRecord) *string {
+	for _, r := range stack {
+		for j, e := range r.env {
+			if e.Name == name {
+				for _, r := range stack {
+					r.unused.Delete(e.Name)
+				}
+				return &r.env[j].Default
+			}
+		}
+	}
+	return nil
 }
 
 // ResolveConfig uses a resolver to resolve an entire ci-operator config