From 049996bf2473ba273e1fd14f21f673b6b3fecafe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Barcarol=20Guimar=C3=A3es?= Date: Tue, 30 Jul 2019 12:12:14 +0000 Subject: [PATCH 1/3] ci-operator multi-stage: create secret --- pkg/defaults/defaults.go | 2 +- pkg/steps/multi_stage.go | 53 +++++++++++++++++++++++++++++------ pkg/steps/multi_stage_test.go | 39 ++++++++++++++++++++++++-- 3 files changed, 81 insertions(+), 13 deletions(-) diff --git a/pkg/defaults/defaults.go b/pkg/defaults/defaults.go index cd7e9a989f6..932198ec38b 100644 --- a/pkg/defaults/defaults.go +++ b/pkg/defaults/defaults.go @@ -155,7 +155,7 @@ func FromConfig( } else if testStep := rawStep.TestStepConfiguration; testStep != nil { if testStep.MultiStageTestConfiguration != nil { - step = steps.MultiStageTestStep(*testStep, config, podClient, artifactDir, jobSpec) + step = steps.MultiStageTestStep(*testStep, config, podClient, secretGetter, artifactDir, jobSpec) } else if testStep.OpenshiftInstallerClusterTestConfiguration != nil { if testStep.OpenshiftInstallerClusterTestConfiguration.Upgrade { var err error diff --git a/pkg/steps/multi_stage.go b/pkg/steps/multi_stage.go index a0df4f46d2f..967751978a8 100644 --- a/pkg/steps/multi_stage.go +++ b/pkg/steps/multi_stage.go @@ -22,6 +22,7 @@ import ( const ( multiStageTestLabel = "ci.openshift.io/multi-stage-test" + secretMountPath = "/var/run/secrets/ci.openshift.io/multi-stage" ) type multiStageTestStep struct { @@ -29,6 +30,7 @@ type multiStageTestStep struct { name string config *api.ReleaseBuildConfiguration podClient PodClient + secretClient coreclientset.SecretsGetter artifactDir string jobSpec *api.JobSpec pre, test, post []api.TestStep @@ -39,16 +41,18 @@ func MultiStageTestStep( testConfig api.TestStepConfiguration, config *api.ReleaseBuildConfiguration, podClient PodClient, + secretClient coreclientset.SecretsGetter, artifactDir string, jobSpec *api.JobSpec, ) api.Step { - return newMultiStageTestStep(testConfig, config, podClient, artifactDir, jobSpec) + return newMultiStageTestStep(testConfig, config, podClient, secretClient, artifactDir, jobSpec) } func newMultiStageTestStep( testConfig api.TestStepConfiguration, config *api.ReleaseBuildConfiguration, podClient PodClient, + secretClient coreclientset.SecretsGetter, artifactDir string, jobSpec *api.JobSpec, ) *multiStageTestStep { @@ -56,14 +60,15 @@ func newMultiStageTestStep( artifactDir = filepath.Join(artifactDir, testConfig.As) } return &multiStageTestStep{ - name: testConfig.As, - config: config, - podClient: podClient, - artifactDir: artifactDir, - jobSpec: jobSpec, - pre: testConfig.MultiStageTestConfiguration.Pre, - test: testConfig.MultiStageTestConfiguration.Test, - post: testConfig.MultiStageTestConfiguration.Post, + name: testConfig.As, + config: config, + podClient: podClient, + secretClient: secretClient, + artifactDir: artifactDir, + jobSpec: jobSpec, + pre: testConfig.MultiStageTestConfiguration.Pre, + test: testConfig.MultiStageTestConfiguration.Test, + post: testConfig.MultiStageTestConfiguration.Post, } } @@ -73,6 +78,9 @@ func (s *multiStageTestStep) Inputs(ctx context.Context, dry bool) (api.InputDef func (s *multiStageTestStep) Run(ctx context.Context, dry bool) error { s.dry = dry + if err := createSecret(s.secretClient.Secrets(s.jobSpec.Namespace), s.name, s.dry); err != nil { + return fmt.Errorf("failed to create secret: %v", err) + } var errs []error if err := s.runSteps(ctx, s.pre, true); err != nil { errs = append(errs, fmt.Errorf("%q pre steps failed: %v", s.name, err)) @@ -161,11 +169,25 @@ func (s *multiStageTestStep) generatePods(steps []api.TestStep) ([]coreapi.Pod, }) addArtifactsContainer(pod) } + addSecret(s.name, pod) ret = append(ret, *pod) } return ret, utilerrors.NewAggregate(errs) } +func addSecret(secret string, pod *coreapi.Pod) { + pod.Spec.Volumes = append(pod.Spec.Volumes, coreapi.Volume{ + Name: secret, + VolumeSource: coreapi.VolumeSource{ + Secret: &coreapi.SecretVolumeSource{SecretName: secret}, + }, + }) + pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, coreapi.VolumeMount{ + Name: secret, + MountPath: secretMountPath, + }) +} + func (s *multiStageTestStep) runPods(ctx context.Context, pods []coreapi.Pod, shortCircuit bool) error { go func() { <-ctx.Done() @@ -232,6 +254,19 @@ func deletePods(client coreclientset.PodInterface, test string) error { return nil } +func createSecret(client coreclientset.SecretInterface, name string, dry bool) error { + log.Printf("Creating multi-stage test secret %q", name) + secret := coreapi.Secret{ObjectMeta: meta.ObjectMeta{Name: name}} + if dry { + return dumpObject(&secret) + } + if err := client.Delete(name, &meta.DeleteOptions{}); err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("cannot delete secret %q: %v", name, err) + } + _, err := client.Create(&secret) + return err +} + func dumpObject(obj runtime.Object) error { j, err := json.MarshalIndent(obj, "", " ") if err != nil { diff --git a/pkg/steps/multi_stage_test.go b/pkg/steps/multi_stage_test.go index 8525a90ee1c..60cec6f2837 100644 --- a/pkg/steps/multi_stage_test.go +++ b/pkg/steps/multi_stage_test.go @@ -126,7 +126,7 @@ func TestGeneratePods(t *testing.T) { }, Namespace: "namespace", } - step := newMultiStageTestStep(config.Tests[0], &config, nil, "artifact_dir", &jobSpec) + step := newMultiStageTestStep(config.Tests[0], &config, nil, nil, "artifact_dir", &jobSpec) ret, err := step.generatePods(config.Tests[0].MultiStageTestConfiguration.Test) if err != nil { t.Fatal(err) @@ -149,6 +149,18 @@ func TestGeneratePods(t *testing.T) { Env: env, Resources: coreapi.ResourceRequirements{}, TerminationMessagePolicy: "FallbackToLogsOnError", + VolumeMounts: []coreapi.VolumeMount{{ + Name: "test", + MountPath: "/var/run/secrets/ci.openshift.io/multi-stage", + }}, + }}, + Volumes: []coreapi.Volume{{ + Name: "test", + VolumeSource: coreapi.VolumeSource{ + Secret: &coreapi.SecretVolumeSource{ + SecretName: "test", + }, + }, }}, }, }, { @@ -172,6 +184,9 @@ func TestGeneratePods(t *testing.T) { VolumeMounts: []coreapi.VolumeMount{{ Name: "artifacts", MountPath: "/artifact/dir", + }, { + Name: "test", + MountPath: "/var/run/secrets/ci.openshift.io/multi-stage", }}, }, { Name: "artifacts", @@ -203,6 +218,13 @@ done VolumeSource: coreapi.VolumeSource{ EmptyDir: &coreapi.EmptyDirVolumeSource{}, }, + }, { + Name: "test", + VolumeSource: coreapi.VolumeSource{ + Secret: &coreapi.SecretVolumeSource{ + SecretName: "test", + }, + }, }}, }, }} @@ -285,10 +307,19 @@ func TestRun(t *testing.T) { return false, nil, nil }) step.podClient = NewPodClient(fakecs.CoreV1(), nil, nil) + step.secretClient = fakecs.CoreV1() if err := step.Run(context.Background(), false); tc.failures == nil && err != nil { t.Error(err) return } + secrets, err := step.secretClient.Secrets(step.jobSpec.Namespace).List(meta.ListOptions{}) + if err != nil { + t.Error(err) + return + } + if l := secrets.Items; len(l) != 1 || l[0].ObjectMeta.Name != step.name { + t.Errorf("unexpected secrets: %#v", l) + } var names []string for _, pods := range pods { names = append(names, pods.ObjectMeta.Name) @@ -344,8 +375,10 @@ func TestArtifacts(t *testing.T) { } return false, nil, nil }) - client := fakePodClient{PodsGetter: fakecs.CoreV1()} - step.podClient = &client + client := fakecs.CoreV1() + podClient := fakePodClient{PodsGetter: client} + step.podClient = &podClient + step.secretClient = client if err := step.Run(context.Background(), false); err != nil { t.Fatal(err) } From ed6cf37b7ee54dcbccdb3f5b5cfc641ce97e2239 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Barcarol=20Guimar=C3=A3es?= Date: Thu, 22 Aug 2019 08:38:56 +0000 Subject: [PATCH 2/3] ci-operator multi-stage: add cluster profile --- pkg/steps/multi_stage.go | 45 ++++++++++++++++++++++++++++++++--- pkg/steps/multi_stage_test.go | 23 ++++++++++++++++++ 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/pkg/steps/multi_stage.go b/pkg/steps/multi_stage.go index 967751978a8..bee4ba8b7b5 100644 --- a/pkg/steps/multi_stage.go +++ b/pkg/steps/multi_stage.go @@ -21,13 +21,15 @@ import ( ) const ( - multiStageTestLabel = "ci.openshift.io/multi-stage-test" - secretMountPath = "/var/run/secrets/ci.openshift.io/multi-stage" + multiStageTestLabel = "ci.openshift.io/multi-stage-test" + clusterProfileMountPath = "/var/run/secrets/ci.openshift.io/cluster-profile" + secretMountPath = "/var/run/secrets/ci.openshift.io/multi-stage" ) type multiStageTestStep struct { dry bool name string + profile api.ClusterProfile config *api.ReleaseBuildConfiguration podClient PodClient secretClient coreclientset.SecretsGetter @@ -61,6 +63,7 @@ func newMultiStageTestStep( } return &multiStageTestStep{ name: testConfig.As, + profile: testConfig.MultiStageTestConfiguration.ClusterProfile, config: config, podClient: podClient, secretClient: secretClient, @@ -78,6 +81,14 @@ func (s *multiStageTestStep) Inputs(ctx context.Context, dry bool) (api.InputDef func (s *multiStageTestStep) Run(ctx context.Context, dry bool) error { s.dry = dry + if s.profile != "" { + if !dry { + profileSecret := fmt.Sprintf("%s-cluster-profile", s.name) + if _, err := s.secretClient.Secrets(s.jobSpec.Namespace).Get(profileSecret, meta.GetOptions{}); err != nil { + return fmt.Errorf("could not find secret %q: %v", profileSecret, err) + } + } + } if err := createSecret(s.secretClient.Secrets(s.jobSpec.Namespace), s.name, s.dry); err != nil { return fmt.Errorf("failed to create secret: %v", err) } @@ -162,8 +173,15 @@ func (s *multiStageTestStep) generatePods(steps []api.TestStep) ([]coreapi.Pod, if owner := s.jobSpec.Owner(); owner != nil { pod.OwnerReferences = append(pod.OwnerReferences, *owner) } + if s.profile != "" { + addProfile(s.name, s.profile, pod) + container.Env = append(container.Env, coreapi.EnvVar{ + Name: "KUBECONFIG", + Value: filepath.Join(secretMountPath, "kubeconfig"), + }) + } if s.artifactDir != "" && step.ArtifactDir != "" { - pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, coreapi.VolumeMount{ + container.VolumeMounts = append(container.VolumeMounts, coreapi.VolumeMount{ Name: "artifacts", MountPath: step.ArtifactDir, }) @@ -188,6 +206,27 @@ func addSecret(secret string, pod *coreapi.Pod) { }) } +func addProfile(name string, profile api.ClusterProfile, pod *coreapi.Pod) { + volumeName := "cluster-profile" + pod.Spec.Volumes = append(pod.Spec.Volumes, coreapi.Volume{ + Name: volumeName, + VolumeSource: coreapi.VolumeSource{ + Secret: &coreapi.SecretVolumeSource{ + SecretName: name + "-cluster-profile", + }, + }, + }) + container := &pod.Spec.Containers[0] + container.VolumeMounts = append(container.VolumeMounts, coreapi.VolumeMount{ + Name: volumeName, + MountPath: clusterProfileMountPath, + }) + container.Env = append(container.Env, coreapi.EnvVar{ + Name: "CLUSTER_TYPE", + Value: strings.Split(string(profile), "-")[0], + }) +} + func (s *multiStageTestStep) runPods(ctx context.Context, pods []coreapi.Pod, shortCircuit bool) error { go func() { <-ctx.Done() diff --git a/pkg/steps/multi_stage_test.go b/pkg/steps/multi_stage_test.go index 60cec6f2837..c491f6cc0c1 100644 --- a/pkg/steps/multi_stage_test.go +++ b/pkg/steps/multi_stage_test.go @@ -73,6 +73,7 @@ func TestGeneratePods(t *testing.T) { Tests: []api.TestStepConfiguration{{ As: "test", MultiStageTestConfiguration: &api.MultiStageTestConfiguration{ + ClusterProfile: api.ClusterProfileAWS, Test: []api.TestStep{{ LiteralTestStep: &api.LiteralTestStep{As: "step0", From: "image0", Commands: "command0"}, }, { @@ -110,6 +111,8 @@ func TestGeneratePods(t *testing.T) { {Name: "NAMESPACE", Value: "namespace"}, {Name: "JOB_NAME_SAFE", Value: "test"}, {Name: "JOB_NAME_HASH", Value: "5e8c9"}, + {Name: "CLUSTER_TYPE", Value: "aws"}, + {Name: "KUBECONFIG", Value: "/var/run/secrets/ci.openshift.io/multi-stage/kubeconfig"}, } jobSpec := api.JobSpec{ JobSpec: prowdapi.JobSpec{ @@ -150,11 +153,21 @@ func TestGeneratePods(t *testing.T) { Resources: coreapi.ResourceRequirements{}, TerminationMessagePolicy: "FallbackToLogsOnError", VolumeMounts: []coreapi.VolumeMount{{ + Name: "cluster-profile", + MountPath: "/var/run/secrets/ci.openshift.io/cluster-profile", + }, { Name: "test", MountPath: "/var/run/secrets/ci.openshift.io/multi-stage", }}, }}, Volumes: []coreapi.Volume{{ + Name: "cluster-profile", + VolumeSource: coreapi.VolumeSource{ + Secret: &coreapi.SecretVolumeSource{ + SecretName: "test-cluster-profile", + }, + }, + }, { Name: "test", VolumeSource: coreapi.VolumeSource{ Secret: &coreapi.SecretVolumeSource{ @@ -182,6 +195,9 @@ func TestGeneratePods(t *testing.T) { Resources: coreapi.ResourceRequirements{}, TerminationMessagePolicy: "FallbackToLogsOnError", VolumeMounts: []coreapi.VolumeMount{{ + Name: "cluster-profile", + MountPath: "/var/run/secrets/ci.openshift.io/cluster-profile", + }, { Name: "artifacts", MountPath: "/artifact/dir", }, { @@ -214,6 +230,13 @@ done }}, }}, Volumes: []coreapi.Volume{{ + Name: "cluster-profile", + VolumeSource: coreapi.VolumeSource{ + Secret: &coreapi.SecretVolumeSource{ + SecretName: "test-cluster-profile", + }, + }, + }, { Name: "artifacts", VolumeSource: coreapi.VolumeSource{ EmptyDir: &coreapi.EmptyDirVolumeSource{}, From 02559fde96e2928fd3d0aeb6577bb20c805eaad9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Barcarol=20Guimar=C3=A3es?= Date: Thu, 22 Aug 2019 11:57:34 +0000 Subject: [PATCH 3/3] ci-operator multi-stage: use release image --- pkg/defaults/defaults.go | 2 +- pkg/steps/multi_stage.go | 28 +++++++++++++++++++++++----- pkg/steps/multi_stage_test.go | 6 +++++- 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/pkg/defaults/defaults.go b/pkg/defaults/defaults.go index 932198ec38b..53062c95e64 100644 --- a/pkg/defaults/defaults.go +++ b/pkg/defaults/defaults.go @@ -155,7 +155,7 @@ func FromConfig( } else if testStep := rawStep.TestStepConfiguration; testStep != nil { if testStep.MultiStageTestConfiguration != nil { - step = steps.MultiStageTestStep(*testStep, config, podClient, secretGetter, artifactDir, jobSpec) + step = steps.MultiStageTestStep(*testStep, config, params, podClient, secretGetter, artifactDir, jobSpec) } else if testStep.OpenshiftInstallerClusterTestConfiguration != nil { if testStep.OpenshiftInstallerClusterTestConfiguration.Upgrade { var err error diff --git a/pkg/steps/multi_stage.go b/pkg/steps/multi_stage.go index bee4ba8b7b5..5224998cdd7 100644 --- a/pkg/steps/multi_stage.go +++ b/pkg/steps/multi_stage.go @@ -29,8 +29,11 @@ const ( type multiStageTestStep struct { dry bool name string + releaseInitial string + releaseLatest string profile api.ClusterProfile config *api.ReleaseBuildConfiguration + params api.Parameters podClient PodClient secretClient coreclientset.SecretsGetter artifactDir string @@ -42,17 +45,19 @@ type multiStageTestStep struct { func MultiStageTestStep( testConfig api.TestStepConfiguration, config *api.ReleaseBuildConfiguration, + params api.Parameters, podClient PodClient, secretClient coreclientset.SecretsGetter, artifactDir string, jobSpec *api.JobSpec, ) api.Step { - return newMultiStageTestStep(testConfig, config, podClient, secretClient, artifactDir, jobSpec) + return newMultiStageTestStep(testConfig, config, params, podClient, secretClient, artifactDir, jobSpec) } func newMultiStageTestStep( testConfig api.TestStepConfiguration, config *api.ReleaseBuildConfiguration, + params api.Parameters, podClient PodClient, secretClient coreclientset.SecretsGetter, artifactDir string, @@ -65,6 +70,7 @@ func newMultiStageTestStep( name: testConfig.As, profile: testConfig.MultiStageTestConfiguration.ClusterProfile, config: config, + params: params, podClient: podClient, secretClient: secretClient, artifactDir: artifactDir, @@ -88,6 +94,13 @@ func (s *multiStageTestStep) Run(ctx context.Context, dry bool) error { return fmt.Errorf("could not find secret %q: %v", profileSecret, err) } } + var err error + if s.releaseInitial, err = s.params.Get("RELEASE_IMAGE_INITIAL"); err != nil { + return err + } + if s.releaseLatest, err = s.params.Get("RELEASE_IMAGE_LATEST"); err != nil { + return err + } } if err := createSecret(s.secretClient.Secrets(s.jobSpec.Namespace), s.name, s.dry); err != nil { return fmt.Errorf("failed to create secret: %v", err) @@ -127,6 +140,10 @@ func (s *multiStageTestStep) Requires() (ret []api.StepLink) { if needsRelease { ret = append(ret, api.ReleaseImagesLink()) } + if s.profile != "" { + ret = append(ret, s.params.Links("RELEASE_IMAGE_INITIAL")...) + ret = append(ret, s.params.Links("RELEASE_IMAGE_LATEST")...) + } return } @@ -175,10 +192,11 @@ func (s *multiStageTestStep) generatePods(steps []api.TestStep) ([]coreapi.Pod, } if s.profile != "" { addProfile(s.name, s.profile, pod) - container.Env = append(container.Env, coreapi.EnvVar{ - Name: "KUBECONFIG", - Value: filepath.Join(secretMountPath, "kubeconfig"), - }) + container.Env = append(container.Env, []coreapi.EnvVar{ + {Name: "KUBECONFIG", Value: filepath.Join(secretMountPath, "kubeconfig")}, + {Name: "RELEASE_IMAGE_INITIAL", Value: s.releaseInitial}, + {Name: "RELEASE_IMAGE_LATEST", Value: s.releaseLatest}, + }...) } if s.artifactDir != "" && step.ArtifactDir != "" { container.VolumeMounts = append(container.VolumeMounts, coreapi.VolumeMount{ diff --git a/pkg/steps/multi_stage_test.go b/pkg/steps/multi_stage_test.go index c491f6cc0c1..1b699cd925c 100644 --- a/pkg/steps/multi_stage_test.go +++ b/pkg/steps/multi_stage_test.go @@ -113,6 +113,8 @@ func TestGeneratePods(t *testing.T) { {Name: "JOB_NAME_HASH", Value: "5e8c9"}, {Name: "CLUSTER_TYPE", Value: "aws"}, {Name: "KUBECONFIG", Value: "/var/run/secrets/ci.openshift.io/multi-stage/kubeconfig"}, + {Name: "RELEASE_IMAGE_INITIAL", Value: "release:initial"}, + {Name: "RELEASE_IMAGE_LATEST", Value: "release:latest"}, } jobSpec := api.JobSpec{ JobSpec: prowdapi.JobSpec{ @@ -129,7 +131,9 @@ func TestGeneratePods(t *testing.T) { }, Namespace: "namespace", } - step := newMultiStageTestStep(config.Tests[0], &config, nil, nil, "artifact_dir", &jobSpec) + step := newMultiStageTestStep(config.Tests[0], &config, nil, nil, nil, "artifact_dir", &jobSpec) + step.releaseInitial = "release:initial" + step.releaseLatest = "release:latest" ret, err := step.generatePods(config.Tests[0].MultiStageTestConfiguration.Test) if err != nil { t.Fatal(err)