From 3e08c233e30077640b86cad99dfaa550bca276e8 Mon Sep 17 00:00:00 2001 From: Vivek Lakshmanan Date: Thu, 25 Mar 2021 00:15:13 -0700 Subject: [PATCH] Switch to envRefs model --- deploy/crds/pulumi.com_stacks.yaml | 64 ++++++++- pkg/apis/pulumi/v1alpha1/stack_types.go | 122 ++++++++++++++++- .../pulumi/v1alpha1/zz_generated.deepcopy.go | 125 +++++++++++++++++- pkg/controller/stack/stack_controller.go | 63 +++++++-- test/stack_controller_test.go | 60 ++++----- test/suite_test.go | 1 + 6 files changed, 374 insertions(+), 61 deletions(-) diff --git a/deploy/crds/pulumi.com_stacks.yaml b/deploy/crds/pulumi.com_stacks.yaml index 213a117a..2fe11906 100644 --- a/deploy/crds/pulumi.com_stacks.yaml +++ b/deploy/crds/pulumi.com_stacks.yaml @@ -52,18 +52,68 @@ spec: destroyOnFinalize: description: (optional) DestroyOnFinalize can be set to true to destroy the stack completely upon deletion of the CRD. type: boolean + envRefs: + additionalProperties: + description: ResourceRef identifies a resource from which information can be loaded. + properties: + env: + description: EnvSelector identifies the environment variable to load information from. + properties: + name: + description: Name of the environment variable + type: string + required: + - name + type: object + filesystem: + description: FSSelector identifies the path to load information from. + properties: + path: + description: Path on the filesystem to use to load information from. + type: string + required: + - path + type: object + literal: + description: LiteralRef identifies a literal value to load. + properties: + value: + description: Value to load + type: string + required: + - value + type: object + secret: + description: SecretSelector identifies the information to load from a Kubernetes secret. + properties: + key: + description: Key within the secret to use. + type: string + name: + description: Name of the secret + type: string + namespace: + description: Namespace where the secret is stored. Defaults to 'default' if omitted. + type: string + required: + - key + - name + type: object + type: + description: 'SelectorType is required and signifies the type of selector. Must be one of: Env, FS, Secret, Literal' + type: string + required: + - type + type: object + description: (optional) EnvRefs is an optional map containing environment variables as keys and stores descriptors to where the variables' values should be loaded from (one of literal, environment variable, file on the filesystem, or Kubernetes secret) as values. + type: object envSecrets: - description: (optional) SecretEnvs is an optional array of secret names containing environment variables to set. + description: '(optional) SecretEnvs is an optional array of secret names containing environment variables to set. Deprecated: use EnvRefs instead.' items: type: string type: array - envSecretsFromPath: - additionalProperties: - type: string - description: (optional) SecretEnvsFromPath is an optional map of environment variables whose values are secrets read from paths on the filesystem. The paths could be injected through Kubernetes secret volume mounts, CSI drivers, etc. This is an alternative to passing secret environment variables to the stack through SecretEnvs which doesn't directly depend on Kubernetes Secrets. - type: object envs: - description: (optional) Envs is an optional array of config maps containing environment variables to set. + description: '(optional) Envs is an optional array of config maps containing environment variables to set. Deprecated: use EnvRefs instead.' items: type: string type: array diff --git a/pkg/apis/pulumi/v1alpha1/stack_types.go b/pkg/apis/pulumi/v1alpha1/stack_types.go index 7ddccc5b..a7f3c05b 100644 --- a/pkg/apis/pulumi/v1alpha1/stack_types.go +++ b/pkg/apis/pulumi/v1alpha1/stack_types.go @@ -16,15 +16,20 @@ type StackSpec struct { // Deprecated: use SecretEnvsFromPath with PULUMI_ACCESS_TOKEN as key or SecretEnvs with a secret entry key // PULUMI_ACCESS_KEY instead. AccessTokenSecret string `json:"accessTokenSecret,omitempty"` + // (optional) Envs is an optional array of config maps containing environment variables to set. + // Deprecated: use EnvRefs instead. Envs []string `json:"envs,omitempty"` + + // (optional) EnvRefs is an optional map containing environment variables as keys and stores descriptors to where + // the variables' values should be loaded from (one of literal, environment variable, file on the + // filesystem, or Kubernetes secret) as values. + EnvRefs map[string]ResourceRef `json:"envRefs,omitempty"` + // (optional) SecretEnvs is an optional array of secret names containing environment variables to set. + // Deprecated: use EnvRefs instead. SecretEnvs []string `json:"envSecrets,omitempty"` - // (optional) SecretEnvsFromPath is an optional map of environment variables whose values are secrets - // read from paths on the filesystem. The paths could be injected through Kubernetes secret volume mounts, - // CSI drivers, etc. This is an alternative to passing secret environment variables to the stack through - // SecretEnvs which doesn't directly depend on Kubernetes Secrets. - SecretEnvsFromPath map[string]string `json:"envSecretsFromPath,omitempty"` + // (optional) Backend is an optional backend URL to use for all Pulumi operations. // Examples: // - Pulumi Service: "https://app.pulumi.com" (default) @@ -98,6 +103,113 @@ type StackSpec struct { RetryOnUpdateConflict bool `json:"retryOnUpdateConflict,omitempty"` } +// ResourceRef identifies a resource from which information can be loaded. +type ResourceRef struct { + // SelectorType is required and signifies the type of selector. Must be one of: + // Env, FS, Secret, Literal + SelectorType ResourceSelectorType `json:"type"` + ResourceSelector `json:",inline"` +} + +// NewEnvResourceRef creates a new environment variable resource ref. +func NewEnvResourceRef(envVarName string) ResourceRef { + return ResourceRef{ + SelectorType: ResourceSelectorEnv, + ResourceSelector: ResourceSelector{ + Env: &EnvSelector{ + Name: envVarName, + }, + }, + } +} + +// NewFileSystemResourceRef creates a new file system resource ref. +func NewFileSystemResourceRef(path string) ResourceRef { + return ResourceRef{ + SelectorType: ResourceSelectorFS, + ResourceSelector: ResourceSelector{ + FileSystem: &FSSelector{ + Path: path, + }, + }, + } +} + +// NewSecretResourceRef creates a new secret resource ref. +func NewSecretResourceRef(namespace, name, key string) ResourceRef { + return ResourceRef{ + SelectorType: ResourceSelectorSecret, + ResourceSelector: ResourceSelector{ + SecretRef: &SecretSelector{ + Namespace: namespace, + Name: name, + Key: key, + }, + }, + } +} + +// NewLiteralResourceRef creates a new literal resource ref. +func NewLiteralResourceRef(value string) ResourceRef { + return ResourceRef{ + SelectorType: ResourceSelectorLiteral, + ResourceSelector: ResourceSelector{ + LiteralRef: &LiteralRef{ + Value: value, + }, + }, + } +} + +// ResourceSelectorType identifies the type of the resource reference in +type ResourceSelectorType string + +const ( + // ResourceSelectorEnv indicates the resource is an environment variable + ResourceSelectorEnv = ResourceSelectorType("Env") + // ResourceSelectorFS indicates the resource is on the filesystem + ResourceSelectorFS = ResourceSelectorType("FS") + // ResourceSelectorSecret indicates the resource is a Kubernetes secret + ResourceSelectorSecret = ResourceSelectorType("Secret") + // ResourceSelectorLiteral indicates the resource is a literal + ResourceSelectorLiteral = ResourceSelectorType("Literal") +) + +type ResourceSelector struct { + FileSystem *FSSelector `json:"filesystem,omitempty"` + Env *EnvSelector `json:"env,omitempty"` + SecretRef *SecretSelector `json:"secret,omitempty"` + LiteralRef *LiteralRef `json:"literal,omitempty"` +} + +// FSSelector identifies the path to load information from. +type FSSelector struct { + // Path on the filesystem to use to load information from. + Path string `json:"path"` +} + +// EnvSelector identifies the environment variable to load information from. +type EnvSelector struct { + // Name of the environment variable + Name string `json:"name"` +} + +// SecretSelector identifies the information to load from a Kubernetes secret. +type SecretSelector struct { + // Namespace where the secret is stored. Defaults to 'default' if omitted. + Namespace string `json:"namespace,omitempty"` + // Name of the secret + Name string `json:"name"` + // Key within the secret to use. + Key string `json:"key"` +} + +// LiteralRef identifies a literal value to load. +type LiteralRef struct { + // Value to load + Value string `json:"value"` +} + // StackStatus defines the observed state of Stack type StackStatus struct { // Outputs contains the exported stack output variables resulting from a deployment. diff --git a/pkg/apis/pulumi/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/pulumi/v1alpha1/zz_generated.deepcopy.go index 71569151..af2e707f 100644 --- a/pkg/apis/pulumi/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/pulumi/v1alpha1/zz_generated.deepcopy.go @@ -8,6 +8,117 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EnvSelector) DeepCopyInto(out *EnvSelector) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EnvSelector. +func (in *EnvSelector) DeepCopy() *EnvSelector { + if in == nil { + return nil + } + out := new(EnvSelector) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FSSelector) DeepCopyInto(out *FSSelector) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FSSelector. +func (in *FSSelector) DeepCopy() *FSSelector { + if in == nil { + return nil + } + out := new(FSSelector) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LiteralRef) DeepCopyInto(out *LiteralRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LiteralRef. +func (in *LiteralRef) DeepCopy() *LiteralRef { + if in == nil { + return nil + } + out := new(LiteralRef) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResourceRef) DeepCopyInto(out *ResourceRef) { + *out = *in + in.ResourceSelector.DeepCopyInto(&out.ResourceSelector) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceRef. +func (in *ResourceRef) DeepCopy() *ResourceRef { + if in == nil { + return nil + } + out := new(ResourceRef) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResourceSelector) DeepCopyInto(out *ResourceSelector) { + *out = *in + if in.FileSystem != nil { + in, out := &in.FileSystem, &out.FileSystem + *out = new(FSSelector) + **out = **in + } + if in.Env != nil { + in, out := &in.Env, &out.Env + *out = new(EnvSelector) + **out = **in + } + if in.SecretRef != nil { + in, out := &in.SecretRef, &out.SecretRef + *out = new(SecretSelector) + **out = **in + } + if in.LiteralRef != nil { + in, out := &in.LiteralRef, &out.LiteralRef + *out = new(LiteralRef) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceSelector. +func (in *ResourceSelector) DeepCopy() *ResourceSelector { + if in == nil { + return nil + } + out := new(ResourceSelector) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecretSelector) DeepCopyInto(out *SecretSelector) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretSelector. +func (in *SecretSelector) DeepCopy() *SecretSelector { + if in == nil { + return nil + } + out := new(SecretSelector) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Stack) DeepCopyInto(out *Stack) { *out = *in @@ -96,18 +207,18 @@ func (in *StackSpec) DeepCopyInto(out *StackSpec) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.EnvRefs != nil { + in, out := &in.EnvRefs, &out.EnvRefs + *out = make(map[string]ResourceRef, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } if in.SecretEnvs != nil { in, out := &in.SecretEnvs, &out.SecretEnvs *out = make([]string, len(*in)) copy(*out, *in) } - if in.SecretEnvsFromPath != nil { - in, out := &in.SecretEnvsFromPath, &out.SecretEnvsFromPath - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } if in.Config != nil { in, out := &in.Config, &out.Config *out = make(map[string]string, len(*in)) diff --git a/pkg/controller/stack/stack_controller.go b/pkg/controller/stack/stack_controller.go index 5e60af4d..1bd1b3b3 100644 --- a/pkg/controller/stack/stack_controller.go +++ b/pkg/controller/stack/stack_controller.go @@ -169,10 +169,6 @@ func (r *ReconcileStack) Reconcile(request reconcile.Request) (reconcile.Result, reqLogger.Error(err, "Could not find Secret for SecretEnvs") return reconcile.Result{}, err } - if err = sess.SetSecretEnvsFromPath(stack.SecretEnvsFromPath); err != nil { - reqLogger.Error(err, "Could not find Secret for SecretEnvsFromPath") - return reconcile.Result{}, err - } // Check if the Stack instance is marked to be deleted, which is // indicated by the deletion timestamp being set. @@ -407,15 +403,54 @@ func (sess *reconcileStackSession) SetSecretEnvs(secrets []string, namespace str return nil } -// SetSecretEnvsFromPath uses the provided input map where each entry consists of the environment variable and the value -// consists of the filesystem path containing the secret content. -func (sess *reconcileStackSession) SetSecretEnvsFromPath(secretsFromPaths map[string]string) error { - for envVar, path := range secretsFromPaths { - contents, err := os.ReadFile(path) - if err != nil { - return errors.Wrapf(err, "reading secret path for env var: %s: %s", envVar, path) +// SetEnvRefsForWorkspace populates environment variables for workspace using items in +// the EnvRefs field in the stack specification. +func (sess *reconcileStackSession) SetEnvRefsForWorkspace(w auto.Workspace) error { + envRefs := sess.stack.EnvRefs + for envVar, ref := range envRefs { + switch ref.SelectorType { + case pulumiv1alpha1.ResourceSelectorEnv: + if ref.Env != nil { + envVarVal := os.Getenv(ref.Env.Name) + w.SetEnvVar(envVar, envVarVal) + } else { + return errors.Errorf("Missing env reference in ResourceRef for '%s'", envVar) + } + case pulumiv1alpha1.ResourceSelectorLiteral: + if ref.LiteralRef != nil { + w.SetEnvVar(envVar, ref.LiteralRef.Value) + } else { + return errors.Errorf("Missing literal reference in ResourceRef for '%s'", envVar) + } + case pulumiv1alpha1.ResourceSelectorFS: + if ref.FileSystem != nil { + contents, err := os.ReadFile(ref.FileSystem.Path) + if err != nil { + return errors.Wrapf(err, "reading path for env var: %s: %s", envVar, ref.FileSystem.Path) + } + w.SetEnvVar(envVar, string(contents)) + } else { + return errors.Errorf("Missing filesystem reference in ResourceRef for '%s'", envVar) + } + case pulumiv1alpha1.ResourceSelectorSecret: + if ref.SecretRef != nil { + config := &corev1.Secret{} + namespace := ref.SecretRef.Namespace + if namespace == "" { + namespace = "default" + } + if err := sess.getLatestResource(config, types.NamespacedName{Name: ref.SecretRef.Name, Namespace: namespace}); err != nil { + return errors.Wrapf(err, "Namespace=%s Name=%s", ref.SecretRef.Namespace, ref.SecretRef.Name) + } + val, ok := config.Data[ref.SecretRef.Key] + if !ok { + return errors.Errorf("No key %s found in secret %s/%s", ref.SecretRef.Key, ref.SecretRef.Namespace, ref.SecretRef.Name) + } + w.SetEnvVar(envVar, string(val)) + } else { + return errors.Errorf("Mising secret reference in ResourceRef for '%s'", envVar) + } } - sess.autoStack.Workspace().SetEnvVar(envVar, string(contents)) } return nil } @@ -497,6 +532,7 @@ func (sess *reconcileStackSession) lookupPulumiAccessToken() (string, bool) { } return accessToken, true } + return "", false } @@ -522,6 +558,9 @@ func (sess *reconcileStackSession) SetupPulumiWorkdir(gitAuth *auto.GitAuth) err if accessToken, found := sess.lookupPulumiAccessToken(); found { w.SetEnvVar("PULUMI_ACCESS_TOKEN", accessToken) } + if err = sess.SetEnvRefsForWorkspace(w); err != nil { + return err + } sess.workdir = w.WorkDir() // Create a new stack if the stack does not already exist, or fall back to diff --git a/test/stack_controller_test.go b/test/stack_controller_test.go index f280521a..e9a0cbb5 100644 --- a/test/stack_controller_test.go +++ b/test/stack_controller_test.go @@ -114,6 +114,25 @@ var _ = Describe("Stack Controller", func() { }) AfterEach(func() { + By("Deleting left over stacks") + deletionPolicy := metav1.DeletePropagationForeground + Expect(k8sClient.DeleteAllOf( + ctx, + &pulumiv1alpha1.Stack{}, + client.InNamespace(namespace), + &client.DeleteAllOfOptions{ + DeleteOptions: client.DeleteOptions{PropagationPolicy: &deletionPolicy}, + }, + )).Should(Succeed()) + + Eventually(func() bool { + var stacksList pulumiv1alpha1.StackList + if err = k8sClient.List(ctx, &stacksList, client.InNamespace(namespace)); err != nil { + return false + } + return len(stacksList.Items) == 0 + }, timeout, interval).Should(BeTrue()) + if pulumiAPISecret != nil { By("Deleting the Stack Pulumi API Secret") Expect(k8sClient.Delete(ctx, pulumiAPISecret)).Should(Succeed()) @@ -138,17 +157,6 @@ var _ = Describe("Stack Controller", func() { return k8serrors.IsNotFound(err) }, timeout, interval).Should(BeTrue()) } - - By("Deleting left over stacks") - deletionPolicy := metav1.DeletePropagationForeground - Expect(k8sClient.DeleteAllOf(ctx, &pulumiv1alpha1.Stack{}, client.InNamespace(namespace), &client.DeleteAllOfOptions{DeleteOptions: client.DeleteOptions{PropagationPolicy: &deletionPolicy}})).Should(Succeed()) - Eventually(func() bool { - var stacksList pulumiv1alpha1.StackList - if err = k8sClient.List(ctx, &stacksList, client.InNamespace(namespace)); err != nil { - return false - } - return len(stacksList.Items) == 0 - }, timeout, interval).Should(BeTrue()) }) // Tip: avoid adding tests for vanilla CRUD operations because they would @@ -308,7 +316,7 @@ var _ = Describe("Stack Controller", func() { return true }, timeout, interval).Should(BeTrue()) - // Check that the stack update attempted but succeeded after the region fix + // Check that the stack update attempted and succeeded after the region fix Eventually(func() bool { err := k8sClient.Get(ctx, types.NamespacedName{Name: stack.Name, Namespace: namespace}, fetched) if err != nil { @@ -333,36 +341,28 @@ var _ = Describe("Stack Controller", func() { }, timeout, interval).Should(BeTrue()) }) - It("Should deploy an AWS S3 Stack successfully with file based secrets", func() { + It("Should deploy an AWS S3 Stack successfully with credentials passed through EnvRefs", func() { var stack *pulumiv1alpha1.Stack - stackName := fmt.Sprintf("%s/s3-op-project/dev-file-secrets-%s", stackOrg, randString()) + stackName := fmt.Sprintf("%s/s3-op-project/dev-env-ref-%s", stackOrg, randString()) fmt.Fprintf(GinkgoWriter, "Stack.Name: %s\n", stackName) - // Write the contents of the secrets to /tmp in the container. This is a stand-in for other - // mechanism to reify the secrets on the file system. This is not a recommended way to store/pass secrets. + // Write a secret to a temp directory. This is a stand-in for other mechanisms to reify the secrets + // on the file system. This is not a recommended way to store/pass secrets. Eventually(func() bool { if err = os.WriteFile(filepath.Join(secretsDir, pulumiAPISecretName), []byte(pulumiAccessToken), 0600); err != nil { return false } - if err = os.WriteFile(filepath.Join(secretsDir, "aws-access-key-id"), []byte(awsAccessKeyID), 0600); err != nil { - return false - } - if err = os.WriteFile(filepath.Join(secretsDir, "aws-secret-access-key"), []byte(awsSecretAccessKey), 0600); err != nil { - return false - } - if err = os.WriteFile(filepath.Join(secretsDir, "aws-session-token"), []byte(awsSessionToken), 0600); err != nil { - return false - } return true }, timeout, interval).Should(BeTrue()) // Define the stack spec spec := pulumiv1alpha1.StackSpec{ - SecretEnvsFromPath: map[string]string{ - "PULUMI_ACCESS_TOKEN": filepath.Join(secretsDir, pulumiAPISecretName), - "AWS_ACCESS_KEY_ID": filepath.Join(secretsDir, "aws-access-key-id"), - "AWS_SECRET_ACCESS_KEY": filepath.Join(secretsDir, "aws-secret-access-key"), - "AWS_SESSION_TOKEN": filepath.Join(secretsDir, "aws-session-token"), + // Cover all variations of resource refs + EnvRefs: map[string]pulumiv1alpha1.ResourceRef{ + "PULUMI_ACCESS_TOKEN": pulumiv1alpha1.NewFileSystemResourceRef(filepath.Join(secretsDir, pulumiAPISecretName)), + "AWS_ACCESS_KEY_ID": pulumiv1alpha1.NewLiteralResourceRef(awsAccessKeyID), + "AWS_SECRET_ACCESS_KEY": pulumiv1alpha1.NewSecretResourceRef(namespace, pulumiAWSSecret.Name, "AWS_SECRET_ACCESS_KEY"), + "AWS_SESSION_TOKEN": pulumiv1alpha1.NewEnvResourceRef("AWS_SESSION_TOKEN"), }, Config: map[string]string{ "aws:region": "us-east-2", diff --git a/test/suite_test.go b/test/suite_test.go index 6a281129..eb0246ab 100644 --- a/test/suite_test.go +++ b/test/suite_test.go @@ -99,6 +99,7 @@ var _ = BeforeSuite(func(done Done) { k8sClient = k8sManager.GetClient() Expect(k8sClient).ToNot(BeNil()) + By("Creating directory to store secrets") secretsDir, err = os.MkdirTemp("", "secrets") if err != nil { Fail("Failed to create secret temp directory")