From dfe16b4a7c64aa64f18f3b257473b866b16cd003 Mon Sep 17 00:00:00 2001 From: Bayan Taani <86984560+btaani@users.noreply.github.com> Date: Fri, 19 Apr 2024 10:21:27 +0200 Subject: [PATCH 1/2] feat(operator): Add support for Swift TLS CA configuration --- .../internal/manifests/storage/configure.go | 66 ++++-- .../manifests/storage/configure_test.go | 202 +++++++++++++++++- 2 files changed, 245 insertions(+), 23 deletions(-) diff --git a/operator/internal/manifests/storage/configure.go b/operator/internal/manifests/storage/configure.go index a8d0bf69f82b9..2d90485c8ad6d 100644 --- a/operator/internal/manifests/storage/configure.go +++ b/operator/internal/manifests/storage/configure.go @@ -29,17 +29,23 @@ var ( // based on the object storage type. Currently supported amendments: // - All: Ensure object storage secret mounted and auth projected as env vars. // - GCS: Ensure env var GOOGLE_APPLICATION_CREDENTIALS in container -// - S3: Ensure mounting custom CA configmap if any TLSConfig given +// - S3 & Swift: Ensure mounting custom CA configmap if any TLSConfig given func ConfigureDeployment(d *appsv1.Deployment, opts Options) error { switch opts.SharedStore { - case lokiv1.ObjectStorageSecretAlibabaCloud, lokiv1.ObjectStorageSecretAzure, lokiv1.ObjectStorageSecretGCS, lokiv1.ObjectStorageSecretSwift: + case lokiv1.ObjectStorageSecretAlibabaCloud, lokiv1.ObjectStorageSecretAzure, lokiv1.ObjectStorageSecretGCS: return configureDeployment(d, opts) case lokiv1.ObjectStorageSecretS3: err := configureDeployment(d, opts) if err != nil { return err } - return configureDeploymentCA(d, opts.TLS) + return configureDeploymentCA(d, opts.TLS, lokiv1.ObjectStorageSecretS3) + case lokiv1.ObjectStorageSecretSwift: + err := configureDeployment(d, opts) + if err != nil { + return err + } + return configureDeploymentCA(d, opts.TLS, lokiv1.ObjectStorageSecretSwift) default: return nil } @@ -49,16 +55,21 @@ func ConfigureDeployment(d *appsv1.Deployment, opts Options) error { // based on the object storage type. Currently supported amendments: // - All: Ensure object storage secret mounted and auth projected as env vars. // - GCS: Ensure env var GOOGLE_APPLICATION_CREDENTIALS in container -// - S3: Ensure mounting custom CA configmap if any TLSConfig given +// - S3 & Swift: Ensure mounting custom CA configmap if any TLSConfig given func ConfigureStatefulSet(d *appsv1.StatefulSet, opts Options) error { switch opts.SharedStore { - case lokiv1.ObjectStorageSecretAlibabaCloud, lokiv1.ObjectStorageSecretAzure, lokiv1.ObjectStorageSecretGCS, lokiv1.ObjectStorageSecretSwift: + case lokiv1.ObjectStorageSecretAlibabaCloud, lokiv1.ObjectStorageSecretAzure, lokiv1.ObjectStorageSecretGCS: return configureStatefulSet(d, opts) case lokiv1.ObjectStorageSecretS3: if err := configureStatefulSet(d, opts); err != nil { return err } - return configureStatefulSetCA(d, opts.TLS) + return configureStatefulSetCA(d, opts.TLS, lokiv1.ObjectStorageSecretS3) + case lokiv1.ObjectStorageSecretSwift: + if err := configureStatefulSet(d, opts); err != nil { + return err + } + return configureStatefulSetCA(d, opts.TLS, lokiv1.ObjectStorageSecretSwift) default: return nil } @@ -75,16 +86,22 @@ func configureDeployment(d *appsv1.Deployment, opts Options) error { return nil } -// ConfigureDeploymentCA merges a S3 CA ConfigMap volume into the deployment spec. -func configureDeploymentCA(d *appsv1.Deployment, tls *TLSConfig) error { +// ConfigureDeploymentCA merges a S3 or Swift CA ConfigMap volume into the deployment spec. +func configureDeploymentCA(d *appsv1.Deployment, tls *TLSConfig, secretType lokiv1.ObjectStorageSecretType) error { if tls == nil { return nil } - p := ensureCAForS3(&d.Spec.Template.Spec, tls) + var p corev1.PodSpec + switch secretType { + case lokiv1.ObjectStorageSecretS3: + p = ensureCAForObjectStorage(&d.Spec.Template.Spec, tls, lokiv1.ObjectStorageSecretS3) + case lokiv1.ObjectStorageSecretSwift: + p = ensureCAForObjectStorage(&d.Spec.Template.Spec, tls, lokiv1.ObjectStorageSecretSwift) + } if err := mergo.Merge(&d.Spec.Template.Spec, p, mergo.WithOverride); err != nil { - return kverrors.Wrap(err, "failed to merge s3 object storage ca options ") + return kverrors.Wrap(err, "failed to merge object storage ca options ") } return nil @@ -101,16 +118,22 @@ func configureStatefulSet(s *appsv1.StatefulSet, opts Options) error { return nil } -// ConfigureStatefulSetCA merges a S3 CA ConfigMap volume into the statefulset spec. -func configureStatefulSetCA(s *appsv1.StatefulSet, tls *TLSConfig) error { +// ConfigureStatefulSetCA merges a S3 or Swift CA ConfigMap volume into the statefulset spec. +func configureStatefulSetCA(s *appsv1.StatefulSet, tls *TLSConfig, secretType lokiv1.ObjectStorageSecretType) error { if tls == nil { return nil } + var p corev1.PodSpec - p := ensureCAForS3(&s.Spec.Template.Spec, tls) + switch secretType { + case lokiv1.ObjectStorageSecretS3: + p = ensureCAForObjectStorage(&s.Spec.Template.Spec, tls, lokiv1.ObjectStorageSecretS3) + case lokiv1.ObjectStorageSecretSwift: + p = ensureCAForObjectStorage(&s.Spec.Template.Spec, tls, lokiv1.ObjectStorageSecretSwift) + } if err := mergo.Merge(&s.Spec.Template.Spec, p, mergo.WithOverride); err != nil { - return kverrors.Wrap(err, "failed to merge s3 object storage ca options ") + return kverrors.Wrap(err, "failed to merge object storage ca options ") } return nil @@ -254,7 +277,7 @@ func serverSideEncryption(opts Options) []corev1.EnvVar { } } -func ensureCAForS3(p *corev1.PodSpec, tls *TLSConfig) corev1.PodSpec { +func ensureCAForObjectStorage(p *corev1.PodSpec, tls *TLSConfig, secretType lokiv1.ObjectStorageSecretType) corev1.PodSpec { container := p.Containers[0].DeepCopy() volumes := p.Volumes @@ -275,9 +298,16 @@ func ensureCAForS3(p *corev1.PodSpec, tls *TLSConfig) corev1.PodSpec { MountPath: caDirectory, }) - container.Args = append(container.Args, - fmt.Sprintf("-s3.http.ca-file=%s", path.Join(caDirectory, tls.Key)), - ) + switch secretType { + case lokiv1.ObjectStorageSecretS3: + container.Args = append(container.Args, + fmt.Sprintf("-s3.http.ca-file=%s", path.Join(caDirectory, tls.Key)), + ) + case lokiv1.ObjectStorageSecretSwift: + container.Args = append(container.Args, + fmt.Sprintf("-swift.http.ca-file=%s", path.Join(caDirectory, tls.Key)), + ) + } return corev1.PodSpec{ Containers: []corev1.Container{ diff --git a/operator/internal/manifests/storage/configure_test.go b/operator/internal/manifests/storage/configure_test.go index 2c4ef5636d9c0..69db7126eb04d 100644 --- a/operator/internal/manifests/storage/configure_test.go +++ b/operator/internal/manifests/storage/configure_test.go @@ -2600,6 +2600,102 @@ func TestConfigureDeploymentForStorageCA(t *testing.T) { }, }, }, + { + desc: "object storage Swift", + opts: Options{ + SecretName: "test", + SharedStore: lokiv1.ObjectStorageSecretSwift, + TLS: &TLSConfig{ + CA: "test", + Key: "service-ca.crt", + }, + }, + dpl: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "loki-querier", + }, + }, + }, + }, + }, + }, + want: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "loki-querier", + VolumeMounts: []corev1.VolumeMount{ + { + Name: "test", + ReadOnly: false, + MountPath: "/etc/storage/secrets", + }, + { + Name: "storage-tls", + ReadOnly: false, + MountPath: "/etc/storage/ca", + }, + }, + Args: []string{ + "-swift.http.ca-file=/etc/storage/ca/service-ca.crt", + }, + Env: []corev1.EnvVar{ + { + Name: EnvSwiftUsername, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test", + }, + Key: KeySwiftUsername, + }, + }, + }, + { + Name: EnvSwiftPassword, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test", + }, + Key: KeySwiftPassword, + }, + }, + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "test", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "test", + }, + }, + }, + { + Name: "storage-tls", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, } for _, tc := range tc { @@ -2625,7 +2721,7 @@ func TestConfigureStatefulSetForStorageCA(t *testing.T) { desc: "object storage other than S3", opts: Options{ SecretName: "test", - SharedStore: lokiv1.ObjectStorageSecretSwift, + SharedStore: lokiv1.ObjectStorageSecretAzure, TLS: &TLSConfig{ CA: "test", }, @@ -2659,24 +2755,24 @@ func TestConfigureStatefulSetForStorageCA(t *testing.T) { }, Env: []corev1.EnvVar{ { - Name: EnvSwiftUsername, + Name: EnvAzureStorageAccountName, ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{ Name: "test", }, - Key: KeySwiftUsername, + Key: KeyAzureStorageAccountName, }, }, }, { - Name: EnvSwiftPassword, + Name: EnvAzureStorageAccountKey, ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{ Name: "test", }, - Key: KeySwiftPassword, + Key: KeyAzureStorageAccountKey, }, }, }, @@ -2794,6 +2890,102 @@ func TestConfigureStatefulSetForStorageCA(t *testing.T) { }, }, }, + { + desc: "object storage Swift", + opts: Options{ + SecretName: "test", + SharedStore: lokiv1.ObjectStorageSecretSwift, + TLS: &TLSConfig{ + CA: "test", + Key: "service-ca.crt", + }, + }, + sts: &appsv1.StatefulSet{ + Spec: appsv1.StatefulSetSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "loki-ingester", + }, + }, + }, + }, + }, + }, + want: &appsv1.StatefulSet{ + Spec: appsv1.StatefulSetSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "loki-ingester", + VolumeMounts: []corev1.VolumeMount{ + { + Name: "test", + ReadOnly: false, + MountPath: "/etc/storage/secrets", + }, + { + Name: "storage-tls", + ReadOnly: false, + MountPath: "/etc/storage/ca", + }, + }, + Args: []string{ + "-swift.http.ca-file=/etc/storage/ca/service-ca.crt", + }, + Env: []corev1.EnvVar{ + { + Name: EnvSwiftUsername, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test", + }, + Key: KeySwiftUsername, + }, + }, + }, + { + Name: EnvSwiftPassword, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test", + }, + Key: KeySwiftPassword, + }, + }, + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "test", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "test", + }, + }, + }, + { + Name: "storage-tls", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, } for _, tc := range tc { From 9178aeb881833e26b8a6eabbe6e339b95484cd98 Mon Sep 17 00:00:00 2001 From: Joao Marcal Date: Wed, 4 Dec 2024 17:00:19 +0000 Subject: [PATCH 2/2] update loki cli args for swift ca to match the ones introduced in https://github.com/grafana/loki/pull/11672 --- operator/internal/manifests/storage/configure.go | 2 +- operator/internal/manifests/storage/configure_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/operator/internal/manifests/storage/configure.go b/operator/internal/manifests/storage/configure.go index 2d90485c8ad6d..0d0e67fa95e4d 100644 --- a/operator/internal/manifests/storage/configure.go +++ b/operator/internal/manifests/storage/configure.go @@ -305,7 +305,7 @@ func ensureCAForObjectStorage(p *corev1.PodSpec, tls *TLSConfig, secretType loki ) case lokiv1.ObjectStorageSecretSwift: container.Args = append(container.Args, - fmt.Sprintf("-swift.http.ca-file=%s", path.Join(caDirectory, tls.Key)), + fmt.Sprintf("-swift.http.tls-ca-path=%s", path.Join(caDirectory, tls.Key)), ) } diff --git a/operator/internal/manifests/storage/configure_test.go b/operator/internal/manifests/storage/configure_test.go index 69db7126eb04d..365844330bbe4 100644 --- a/operator/internal/manifests/storage/configure_test.go +++ b/operator/internal/manifests/storage/configure_test.go @@ -2643,7 +2643,7 @@ func TestConfigureDeploymentForStorageCA(t *testing.T) { }, }, Args: []string{ - "-swift.http.ca-file=/etc/storage/ca/service-ca.crt", + "-swift.http.tls-ca-path=/etc/storage/ca/service-ca.crt", }, Env: []corev1.EnvVar{ { @@ -2933,7 +2933,7 @@ func TestConfigureStatefulSetForStorageCA(t *testing.T) { }, }, Args: []string{ - "-swift.http.ca-file=/etc/storage/ca/service-ca.crt", + "-swift.http.tls-ca-path=/etc/storage/ca/service-ca.crt", }, Env: []corev1.EnvVar{ {