diff --git a/pkg/defaults/defaults.go b/pkg/defaults/defaults.go index cd7e9a989f6..53062c95e64 100644 --- a/pkg/defaults/defaults.go +++ b/pkg/defaults/defaults.go @@ -155,7 +155,7 @@ func FromConfig( } else if testStep := rawStep.TestStepConfiguration; testStep != nil { if testStep.MultiStageTestConfiguration != nil { - step = steps.MultiStageTestStep(*testStep, config, podClient, artifactDir, jobSpec) + step = steps.MultiStageTestStep(*testStep, config, params, podClient, secretGetter, artifactDir, jobSpec) } else if testStep.OpenshiftInstallerClusterTestConfiguration != nil { if testStep.OpenshiftInstallerClusterTestConfiguration.Upgrade { var err error diff --git a/pkg/steps/multi_stage.go b/pkg/steps/multi_stage.go index a0df4f46d2f..5224998cdd7 100644 --- a/pkg/steps/multi_stage.go +++ b/pkg/steps/multi_stage.go @@ -21,14 +21,21 @@ import ( ) const ( - multiStageTestLabel = "ci.openshift.io/multi-stage-test" + multiStageTestLabel = "ci.openshift.io/multi-stage-test" + clusterProfileMountPath = "/var/run/secrets/ci.openshift.io/cluster-profile" + secretMountPath = "/var/run/secrets/ci.openshift.io/multi-stage" ) type multiStageTestStep struct { dry bool name string + releaseInitial string + releaseLatest string + profile api.ClusterProfile config *api.ReleaseBuildConfiguration + params api.Parameters podClient PodClient + secretClient coreclientset.SecretsGetter artifactDir string jobSpec *api.JobSpec pre, test, post []api.TestStep @@ -38,17 +45,21 @@ type multiStageTestStep struct { func MultiStageTestStep( testConfig api.TestStepConfiguration, config *api.ReleaseBuildConfiguration, + params api.Parameters, podClient PodClient, + secretClient coreclientset.SecretsGetter, artifactDir string, jobSpec *api.JobSpec, ) api.Step { - return newMultiStageTestStep(testConfig, config, podClient, artifactDir, jobSpec) + return newMultiStageTestStep(testConfig, config, params, podClient, secretClient, artifactDir, jobSpec) } func newMultiStageTestStep( testConfig api.TestStepConfiguration, config *api.ReleaseBuildConfiguration, + params api.Parameters, podClient PodClient, + secretClient coreclientset.SecretsGetter, artifactDir string, jobSpec *api.JobSpec, ) *multiStageTestStep { @@ -56,14 +67,17 @@ func newMultiStageTestStep( artifactDir = filepath.Join(artifactDir, testConfig.As) } return &multiStageTestStep{ - name: testConfig.As, - config: config, - podClient: podClient, - artifactDir: artifactDir, - jobSpec: jobSpec, - pre: testConfig.MultiStageTestConfiguration.Pre, - test: testConfig.MultiStageTestConfiguration.Test, - post: testConfig.MultiStageTestConfiguration.Post, + name: testConfig.As, + profile: testConfig.MultiStageTestConfiguration.ClusterProfile, + config: config, + params: params, + podClient: podClient, + secretClient: secretClient, + artifactDir: artifactDir, + jobSpec: jobSpec, + pre: testConfig.MultiStageTestConfiguration.Pre, + test: testConfig.MultiStageTestConfiguration.Test, + post: testConfig.MultiStageTestConfiguration.Post, } } @@ -73,6 +87,24 @@ func (s *multiStageTestStep) Inputs(ctx context.Context, dry bool) (api.InputDef func (s *multiStageTestStep) Run(ctx context.Context, dry bool) error { s.dry = dry + if s.profile != "" { + if !dry { + profileSecret := fmt.Sprintf("%s-cluster-profile", s.name) + if _, err := s.secretClient.Secrets(s.jobSpec.Namespace).Get(profileSecret, meta.GetOptions{}); err != nil { + return fmt.Errorf("could not find secret %q: %v", profileSecret, err) + } + } + var err error + if s.releaseInitial, err = s.params.Get("RELEASE_IMAGE_INITIAL"); err != nil { + return err + } + if s.releaseLatest, err = s.params.Get("RELEASE_IMAGE_LATEST"); err != nil { + return err + } + } + if err := createSecret(s.secretClient.Secrets(s.jobSpec.Namespace), s.name, s.dry); err != nil { + return fmt.Errorf("failed to create secret: %v", err) + } var errs []error if err := s.runSteps(ctx, s.pre, true); err != nil { errs = append(errs, fmt.Errorf("%q pre steps failed: %v", s.name, err)) @@ -108,6 +140,10 @@ func (s *multiStageTestStep) Requires() (ret []api.StepLink) { if needsRelease { ret = append(ret, api.ReleaseImagesLink()) } + if s.profile != "" { + ret = append(ret, s.params.Links("RELEASE_IMAGE_INITIAL")...) + ret = append(ret, s.params.Links("RELEASE_IMAGE_LATEST")...) + } return } @@ -154,18 +190,61 @@ func (s *multiStageTestStep) generatePods(steps []api.TestStep) ([]coreapi.Pod, if owner := s.jobSpec.Owner(); owner != nil { pod.OwnerReferences = append(pod.OwnerReferences, *owner) } + if s.profile != "" { + addProfile(s.name, s.profile, pod) + container.Env = append(container.Env, []coreapi.EnvVar{ + {Name: "KUBECONFIG", Value: filepath.Join(secretMountPath, "kubeconfig")}, + {Name: "RELEASE_IMAGE_INITIAL", Value: s.releaseInitial}, + {Name: "RELEASE_IMAGE_LATEST", Value: s.releaseLatest}, + }...) + } if s.artifactDir != "" && step.ArtifactDir != "" { - pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, coreapi.VolumeMount{ + container.VolumeMounts = append(container.VolumeMounts, coreapi.VolumeMount{ Name: "artifacts", MountPath: step.ArtifactDir, }) addArtifactsContainer(pod) } + addSecret(s.name, pod) ret = append(ret, *pod) } return ret, utilerrors.NewAggregate(errs) } +func addSecret(secret string, pod *coreapi.Pod) { + pod.Spec.Volumes = append(pod.Spec.Volumes, coreapi.Volume{ + Name: secret, + VolumeSource: coreapi.VolumeSource{ + Secret: &coreapi.SecretVolumeSource{SecretName: secret}, + }, + }) + pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, coreapi.VolumeMount{ + Name: secret, + MountPath: secretMountPath, + }) +} + +func addProfile(name string, profile api.ClusterProfile, pod *coreapi.Pod) { + volumeName := "cluster-profile" + pod.Spec.Volumes = append(pod.Spec.Volumes, coreapi.Volume{ + Name: volumeName, + VolumeSource: coreapi.VolumeSource{ + Secret: &coreapi.SecretVolumeSource{ + SecretName: name + "-cluster-profile", + }, + }, + }) + container := &pod.Spec.Containers[0] + container.VolumeMounts = append(container.VolumeMounts, coreapi.VolumeMount{ + Name: volumeName, + MountPath: clusterProfileMountPath, + }) + container.Env = append(container.Env, coreapi.EnvVar{ + Name: "CLUSTER_TYPE", + Value: strings.Split(string(profile), "-")[0], + }) +} + func (s *multiStageTestStep) runPods(ctx context.Context, pods []coreapi.Pod, shortCircuit bool) error { go func() { <-ctx.Done() @@ -232,6 +311,19 @@ func deletePods(client coreclientset.PodInterface, test string) error { return nil } +func createSecret(client coreclientset.SecretInterface, name string, dry bool) error { + log.Printf("Creating multi-stage test secret %q", name) + secret := coreapi.Secret{ObjectMeta: meta.ObjectMeta{Name: name}} + if dry { + return dumpObject(&secret) + } + if err := client.Delete(name, &meta.DeleteOptions{}); err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("cannot delete secret %q: %v", name, err) + } + _, err := client.Create(&secret) + return err +} + func dumpObject(obj runtime.Object) error { j, err := json.MarshalIndent(obj, "", " ") if err != nil { diff --git a/pkg/steps/multi_stage_test.go b/pkg/steps/multi_stage_test.go index 8525a90ee1c..1b699cd925c 100644 --- a/pkg/steps/multi_stage_test.go +++ b/pkg/steps/multi_stage_test.go @@ -73,6 +73,7 @@ func TestGeneratePods(t *testing.T) { Tests: []api.TestStepConfiguration{{ As: "test", MultiStageTestConfiguration: &api.MultiStageTestConfiguration{ + ClusterProfile: api.ClusterProfileAWS, Test: []api.TestStep{{ LiteralTestStep: &api.LiteralTestStep{As: "step0", From: "image0", Commands: "command0"}, }, { @@ -110,6 +111,10 @@ func TestGeneratePods(t *testing.T) { {Name: "NAMESPACE", Value: "namespace"}, {Name: "JOB_NAME_SAFE", Value: "test"}, {Name: "JOB_NAME_HASH", Value: "5e8c9"}, + {Name: "CLUSTER_TYPE", Value: "aws"}, + {Name: "KUBECONFIG", Value: "/var/run/secrets/ci.openshift.io/multi-stage/kubeconfig"}, + {Name: "RELEASE_IMAGE_INITIAL", Value: "release:initial"}, + {Name: "RELEASE_IMAGE_LATEST", Value: "release:latest"}, } jobSpec := api.JobSpec{ JobSpec: prowdapi.JobSpec{ @@ -126,7 +131,9 @@ func TestGeneratePods(t *testing.T) { }, Namespace: "namespace", } - step := newMultiStageTestStep(config.Tests[0], &config, nil, "artifact_dir", &jobSpec) + step := newMultiStageTestStep(config.Tests[0], &config, nil, nil, nil, "artifact_dir", &jobSpec) + step.releaseInitial = "release:initial" + step.releaseLatest = "release:latest" ret, err := step.generatePods(config.Tests[0].MultiStageTestConfiguration.Test) if err != nil { t.Fatal(err) @@ -149,6 +156,28 @@ func TestGeneratePods(t *testing.T) { Env: env, Resources: coreapi.ResourceRequirements{}, TerminationMessagePolicy: "FallbackToLogsOnError", + VolumeMounts: []coreapi.VolumeMount{{ + Name: "cluster-profile", + MountPath: "/var/run/secrets/ci.openshift.io/cluster-profile", + }, { + Name: "test", + MountPath: "/var/run/secrets/ci.openshift.io/multi-stage", + }}, + }}, + Volumes: []coreapi.Volume{{ + Name: "cluster-profile", + VolumeSource: coreapi.VolumeSource{ + Secret: &coreapi.SecretVolumeSource{ + SecretName: "test-cluster-profile", + }, + }, + }, { + Name: "test", + VolumeSource: coreapi.VolumeSource{ + Secret: &coreapi.SecretVolumeSource{ + SecretName: "test", + }, + }, }}, }, }, { @@ -170,8 +199,14 @@ func TestGeneratePods(t *testing.T) { Resources: coreapi.ResourceRequirements{}, TerminationMessagePolicy: "FallbackToLogsOnError", VolumeMounts: []coreapi.VolumeMount{{ + Name: "cluster-profile", + MountPath: "/var/run/secrets/ci.openshift.io/cluster-profile", + }, { Name: "artifacts", MountPath: "/artifact/dir", + }, { + Name: "test", + MountPath: "/var/run/secrets/ci.openshift.io/multi-stage", }}, }, { Name: "artifacts", @@ -199,10 +234,24 @@ done }}, }}, Volumes: []coreapi.Volume{{ + Name: "cluster-profile", + VolumeSource: coreapi.VolumeSource{ + Secret: &coreapi.SecretVolumeSource{ + SecretName: "test-cluster-profile", + }, + }, + }, { Name: "artifacts", VolumeSource: coreapi.VolumeSource{ EmptyDir: &coreapi.EmptyDirVolumeSource{}, }, + }, { + Name: "test", + VolumeSource: coreapi.VolumeSource{ + Secret: &coreapi.SecretVolumeSource{ + SecretName: "test", + }, + }, }}, }, }} @@ -285,10 +334,19 @@ func TestRun(t *testing.T) { return false, nil, nil }) step.podClient = NewPodClient(fakecs.CoreV1(), nil, nil) + step.secretClient = fakecs.CoreV1() if err := step.Run(context.Background(), false); tc.failures == nil && err != nil { t.Error(err) return } + secrets, err := step.secretClient.Secrets(step.jobSpec.Namespace).List(meta.ListOptions{}) + if err != nil { + t.Error(err) + return + } + if l := secrets.Items; len(l) != 1 || l[0].ObjectMeta.Name != step.name { + t.Errorf("unexpected secrets: %#v", l) + } var names []string for _, pods := range pods { names = append(names, pods.ObjectMeta.Name) @@ -344,8 +402,10 @@ func TestArtifacts(t *testing.T) { } return false, nil, nil }) - client := fakePodClient{PodsGetter: fakecs.CoreV1()} - step.podClient = &client + client := fakecs.CoreV1() + podClient := fakePodClient{PodsGetter: client} + step.podClient = &podClient + step.secretClient = client if err := step.Run(context.Background(), false); err != nil { t.Fatal(err) }