Skip to content

Commit 8427c41

Browse files
ChunyiLyuChunyiLyu
committed
Mount tls secrets using projected volume not subpath
- secrets and configmaps mounted with subpath do not not get updated in pods when changed, as reported in issue: kubernetes/kubernetes#50345 - RabbitMQ itself supports TLS credential rotation without restart. By mounting tls secrets without using subpath, rabbitmq pods will pick up cert changes and support tls rotation without server restart. - certificate rotation tested in system tests
1 parent d8a14c8 commit 8427c41

File tree

6 files changed

+213
-188
lines changed

6 files changed

+213
-188
lines changed

‎controllers/reconcile_tls_test.go‎

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package controllers_test
33
import (
44
"context"
55
"fmt"
6+
"k8s.io/utils/pointer"
67

78
rabbitmqv1beta1 "github.com/rabbitmq/cluster-operator/api/v1beta1"
89

@@ -43,13 +44,31 @@ var _ = Describe("Reconcile TLS", func() {
4344

4445
sts, err := clientSet.AppsV1().StatefulSets(cluster.Namespace).Get(ctx, cluster.ChildResourceName("server"), metav1.GetOptions{})
4546
Expect(err).NotTo(HaveOccurred())
46-
volumeMount := corev1.VolumeMount{
47+
48+
Expect(sts.Spec.Template.Spec.Volumes).To(ContainElement(corev1.Volume{
49+
Name: "rabbitmq-tls",
50+
VolumeSource: corev1.VolumeSource{
51+
Projected: &corev1.ProjectedVolumeSource{
52+
Sources: []corev1.VolumeProjection{
53+
{
54+
Secret: &corev1.SecretProjection{
55+
LocalObjectReference: corev1.LocalObjectReference{
56+
Name: "tls-secret",
57+
},
58+
Optional: pointer.BoolPtr(true),
59+
},
60+
},
61+
},
62+
DefaultMode: pointer.Int32Ptr(400),
63+
},
64+
},
65+
}))
66+
67+
Expect(extractContainer(sts.Spec.Template.Spec.Containers, "rabbitmq").VolumeMounts).To(ContainElement(corev1.VolumeMount{
4768
Name: "rabbitmq-tls",
48-
MountPath: "/etc/rabbitmq-tls/ca.crt",
49-
SubPath: "ca.crt",
69+
MountPath: "/etc/rabbitmq-tls/",
5070
ReadOnly: true,
51-
}
52-
Expect(sts.Spec.Template.Spec.Containers[0].VolumeMounts).To(ContainElement(volumeMount))
71+
}))
5372
})
5473

5574
It("Does not deploy if the cert name does not match the contents of the secret", func() {

‎go.mod‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ require (
2424
github.com/streadway/amqp v0.0.0-20200108173154-1c71cc93ed71
2525
go.uber.org/multierr v1.2.0 // indirect
2626
golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb
27+
golang.org/x/tools v0.0.0-20200616195046-dc31b401abb5
2728
gopkg.in/ini.v1 v1.62.0
2829
k8s.io/api v0.18.6
2930
k8s.io/apimachinery v0.18.8

‎internal/resource/statefulset.go‎

Lines changed: 29 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -421,67 +421,44 @@ func (builder *StatefulSetBuilder) podTemplateSpec(previousPodAnnotations map[st
421421

422422
tlsSpec := builder.Instance.Spec.TLS
423423
if builder.Instance.TLSEnabled() {
424-
// add tls volume
425-
filePermissions := int32(400)
426-
secretEnforced := true
427-
tlsSecretName := tlsSpec.SecretName
428-
volumes = append(volumes, corev1.Volume{
429-
Name: "rabbitmq-tls",
430-
VolumeSource: corev1.VolumeSource{
431-
Secret: &corev1.SecretVolumeSource{
432-
SecretName: tlsSecretName,
433-
DefaultMode: &filePermissions,
434-
Optional: &secretEnforced,
435-
},
436-
},
437-
})
438-
439-
// add volume mount
440424
rabbitmqContainerVolumeMounts = append(rabbitmqContainerVolumeMounts, corev1.VolumeMount{
441425
Name: "rabbitmq-tls",
442-
MountPath: "/etc/rabbitmq-tls/tls.crt",
443-
SubPath: "tls.crt",
444-
ReadOnly: true,
445-
})
446-
rabbitmqContainerVolumeMounts = append(rabbitmqContainerVolumeMounts, corev1.VolumeMount{
447-
Name: "rabbitmq-tls",
448-
MountPath: "/etc/rabbitmq-tls/tls.key",
449-
SubPath: "tls.key",
426+
MountPath: "/etc/rabbitmq-tls/",
450427
ReadOnly: true,
451428
})
452429

453-
if builder.Instance.MutualTLSEnabled() {
454-
if builder.Instance.SingleTLSSecret() {
455-
//Mount CaCert in TLS Secret
456-
rabbitmqContainerVolumeMounts = append(rabbitmqContainerVolumeMounts, corev1.VolumeMount{
457-
Name: "rabbitmq-tls",
458-
MountPath: "/etc/rabbitmq-tls/ca.crt",
459-
SubPath: "ca.crt",
460-
ReadOnly: true,
461-
})
462-
} else {
463-
// add tls volume
464-
filePermissions := int32(400)
465-
secretEnforced := true
466-
volumes = append(volumes, corev1.Volume{
467-
Name: "rabbitmq-mutual-tls",
468-
VolumeSource: corev1.VolumeSource{
469-
Secret: &corev1.SecretVolumeSource{
470-
SecretName: tlsSpec.CaSecretName,
471-
DefaultMode: &filePermissions,
472-
Optional: &secretEnforced,
430+
secretEnforced := true
431+
filePermissions := pointer.Int32Ptr(400)
432+
tlsProjectedVolume := corev1.Volume{
433+
Name: "rabbitmq-tls",
434+
VolumeSource: corev1.VolumeSource{
435+
Projected: &corev1.ProjectedVolumeSource{
436+
Sources: []corev1.VolumeProjection{
437+
{
438+
Secret: &corev1.SecretProjection{
439+
LocalObjectReference: corev1.LocalObjectReference{
440+
Name: tlsSpec.SecretName,
441+
},
442+
Optional: &secretEnforced,
443+
},
473444
},
474445
},
475-
})
476-
//Mount new volume to same path as TLS cert and key
477-
rabbitmqContainerVolumeMounts = append(rabbitmqContainerVolumeMounts, corev1.VolumeMount{
478-
Name: "rabbitmq-mutual-tls",
479-
MountPath: "/etc/rabbitmq-tls/ca.crt",
480-
SubPath: "ca.crt",
481-
ReadOnly: true,
482-
})
446+
DefaultMode: filePermissions,
447+
},
448+
},
449+
}
450+
451+
if builder.Instance.MutualTLSEnabled() && !builder.Instance.SingleTLSSecret() {
452+
caSecretProjection := corev1.VolumeProjection{
453+
Secret: &corev1.SecretProjection{
454+
LocalObjectReference: corev1.LocalObjectReference{Name: tlsSpec.CaSecretName},
455+
Optional: &secretEnforced,
456+
},
483457
}
458+
tlsProjectedVolume.VolumeSource.Projected.Sources = append(tlsProjectedVolume.VolumeSource.Projected.Sources, caSecretProjection)
484459
}
460+
461+
volumes = append(volumes, tlsProjectedVolume)
485462
}
486463

487464
return corev1.PodTemplateSpec{

‎internal/resource/statefulset_test.go‎

Lines changed: 39 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -563,39 +563,38 @@ var _ = Describe("StatefulSet", func() {
563563
})
564564

565565
Context("TLS", func() {
566-
It("adds a TLS volume to the pod template spec", func() {
566+
It("adds a TLS projected volume to the pod template spec", func() {
567567
instance.Spec.TLS.SecretName = "tls-secret"
568568
Expect(stsBuilder.Update(statefulSet)).To(Succeed())
569569

570-
filePermissions := int32(400)
571-
secretEnforced := true
572570
Expect(statefulSet.Spec.Template.Spec.Volumes).To(ContainElement(corev1.Volume{
573571
Name: "rabbitmq-tls",
574572
VolumeSource: corev1.VolumeSource{
575-
Secret: &corev1.SecretVolumeSource{
576-
SecretName: "tls-secret",
577-
DefaultMode: &filePermissions,
578-
Optional: &secretEnforced,
573+
Projected: &corev1.ProjectedVolumeSource{
574+
Sources: []corev1.VolumeProjection{
575+
{
576+
Secret: &corev1.SecretProjection{
577+
LocalObjectReference: corev1.LocalObjectReference{
578+
Name: "tls-secret",
579+
},
580+
Optional: pointer.BoolPtr(true),
581+
},
582+
},
583+
},
584+
DefaultMode: pointer.Int32Ptr(400),
579585
},
580586
},
581587
}))
582588
})
583589

584-
It("adds two TLS volume mounts to the rabbitmq container", func() {
590+
It("adds a TLS volume mount to the rabbitmq container", func() {
585591
instance.Spec.TLS.SecretName = "tls-secret"
586592
Expect(stsBuilder.Update(statefulSet)).To(Succeed())
587593

588594
rabbitmqContainerSpec := extractContainer(statefulSet.Spec.Template.Spec.Containers, "rabbitmq")
589595
Expect(rabbitmqContainerSpec.VolumeMounts).To(ContainElement(corev1.VolumeMount{
590596
Name: "rabbitmq-tls",
591-
MountPath: "/etc/rabbitmq-tls/tls.crt",
592-
SubPath: "tls.crt",
593-
ReadOnly: true,
594-
}))
595-
Expect(rabbitmqContainerSpec.VolumeMounts).To(ContainElement(corev1.VolumeMount{
596-
Name: "rabbitmq-tls",
597-
MountPath: "/etc/rabbitmq-tls/tls.key",
598-
SubPath: "tls.key",
597+
MountPath: "/etc/rabbitmq-tls/",
599598
ReadOnly: true,
600599
}))
601600
})
@@ -636,21 +635,7 @@ var _ = Describe("StatefulSet", func() {
636635
}))
637636
})
638637

639-
Context("Mutual TLS (same secret)", func() {
640-
It("add a TLS CA cert volume mount to the rabbitmq container", func() {
641-
instance.Spec.TLS.SecretName = "tls-secret"
642-
instance.Spec.TLS.CaSecretName = "tls-secret"
643-
Expect(stsBuilder.Update(statefulSet)).To(Succeed())
644-
645-
rabbitmqContainerSpec := extractContainer(statefulSet.Spec.Template.Spec.Containers, "rabbitmq")
646-
Expect(rabbitmqContainerSpec.VolumeMounts).To(ContainElement(corev1.VolumeMount{
647-
Name: "rabbitmq-tls",
648-
MountPath: "/etc/rabbitmq-tls/ca.crt",
649-
SubPath: "ca.crt",
650-
ReadOnly: true,
651-
}))
652-
})
653-
638+
When("Mutual TLS (same secret) is enabled", func() {
654639
It("opens tls ports when rabbitmq_web_mqtt and rabbitmq_web_stomp are configured", func() {
655640
instance.Spec.TLS.SecretName = "tls-secret"
656641
instance.Spec.TLS.CaSecretName = "tls-secret"
@@ -672,35 +657,35 @@ var _ = Describe("StatefulSet", func() {
672657
})
673658
})
674659

675-
Context("Mutual TLS (different secret)", func() {
676-
It("add a TLS CA cert volume mount to the rabbitmq container", func() {
677-
instance.Spec.TLS.SecretName = "tls-secret"
678-
instance.Spec.TLS.CaSecretName = "mutual-tls-secret"
679-
Expect(stsBuilder.Update(statefulSet)).To(Succeed())
680-
681-
rabbitmqContainerSpec := extractContainer(statefulSet.Spec.Template.Spec.Containers, "rabbitmq")
682-
Expect(rabbitmqContainerSpec.VolumeMounts).To(ContainElement(corev1.VolumeMount{
683-
Name: "rabbitmq-mutual-tls",
684-
MountPath: "/etc/rabbitmq-tls/ca.crt",
685-
SubPath: "ca.crt",
686-
ReadOnly: true,
687-
}))
688-
})
689-
690-
It("adds a mutual TLS volume to the pod template spec", func() {
660+
When("Mutual TLS (different secret) is enabled", func() {
661+
It("adds the CA cert secret to tls project volume", func() {
691662
instance.Spec.TLS.SecretName = "tls-secret"
692663
instance.Spec.TLS.CaSecretName = "mutual-tls-secret"
693664
Expect(stsBuilder.Update(statefulSet)).To(Succeed())
694665

695-
filePermissions := int32(400)
696-
secretEnforced := true
697666
Expect(statefulSet.Spec.Template.Spec.Volumes).To(ContainElement(corev1.Volume{
698-
Name: "rabbitmq-mutual-tls",
667+
Name: "rabbitmq-tls",
699668
VolumeSource: corev1.VolumeSource{
700-
Secret: &corev1.SecretVolumeSource{
701-
SecretName: "mutual-tls-secret",
702-
DefaultMode: &filePermissions,
703-
Optional: &secretEnforced,
669+
Projected: &corev1.ProjectedVolumeSource{
670+
Sources: []corev1.VolumeProjection{
671+
{
672+
Secret: &corev1.SecretProjection{
673+
LocalObjectReference: corev1.LocalObjectReference{
674+
Name: "tls-secret",
675+
},
676+
Optional: pointer.BoolPtr(true),
677+
},
678+
},
679+
{
680+
Secret: &corev1.SecretProjection{
681+
LocalObjectReference: corev1.LocalObjectReference{
682+
Name: "mutual-tls-secret",
683+
},
684+
Optional: pointer.BoolPtr(true),
685+
},
686+
},
687+
},
688+
DefaultMode: pointer.Int32Ptr(400),
704689
},
705690
},
706691
}))

0 commit comments

Comments
 (0)