Set podManagementPolicy to parallel

- use cluster_formation.randomized_startup_delay_range
to avoid race condition between nodes during initial
cluster formation
- this change solves the problem of failing to restart
the cluster when all pods are deleted and recreated at
once; with podManagementPolicy set to 'OrderedReady',
pod 0 is also the first one getting recreated, but it
may not be the last node to shut down.

Author: ChunyiLyu

internal/resource/configmap.go

@@ -34,7 +34,9 @@ cluster_formation.k8s.host = kubernetes.default
 cluster_formation.k8s.address_type = hostname
 cluster_partition_handling = pause_minority
 queue_master_locator = min-masters
-disk_free_limit.absolute = 2GB`
+disk_free_limit.absolute = 2GB
+cluster_formation.randomized_startup_delay_range.min = 5
+cluster_formation.randomized_startup_delay_range.max = 30`
 
 	defaultTLSConf = `
 ssl_options.certfile = /etc/rabbitmq-tls/tls.crt

internal/resource/configmap_test.go

@@ -33,6 +33,8 @@ cluster_formation.k8s.address_type       = hostname
 cluster_partition_handling               = pause_minority
 queue_master_locator                     = min-masters
 disk_free_limit.absolute                 = 2GB
+cluster_formation.randomized_startup_delay_range.min = 5
+cluster_formation.randomized_startup_delay_range.max = 30
 cluster_name                             = ` + instanceName)
 }
 

internal/resource/statefulset.go

@@ -67,6 +67,7 @@ func (builder *StatefulSetBuilder) Build() (client.Object, error) {
 				MatchLabels: metadata.LabelSelector(builder.Instance.Name),
 			},
 			VolumeClaimTemplates: pvc,
+			PodManagementPolicy:  appsv1.ParallelPodManagement,
 		},
 	}
 

internal/resource/statefulset_test.go

@@ -65,6 +65,7 @@ var _ = Describe("StatefulSet", func() {
 
 			Expect(statefulSet.Spec.ServiceName).To(Equal(instance.ChildResourceName("nodes")))
 		})
+
 		It("adds the correct label selector", func() {
 			obj, err := stsBuilder.Build()
 			Expect(err).NotTo(HaveOccurred())
@@ -74,7 +75,15 @@ var _ = Describe("StatefulSet", func() {
 			Expect(labels["app.kubernetes.io/name"]).To(Equal(instance.Name))
 		})
 
-		It("references the storageclassname when specified", func() {
+		It("sets pod management policy to 'Parallel' ", func() {
+			obj, err := stsBuilder.Build()
+			Expect(err).NotTo(HaveOccurred())
+			statefulSet := obj.(*appsv1.StatefulSet)
+
+			Expect(statefulSet.Spec.PodManagementPolicy).To(Equal(appsv1.ParallelPodManagement))
+		})
+
+		It("references the storage class name when specified", func() {
 			storageClassName := "my-storage-class"
 			builder.Instance.Spec.Persistence.StorageClassName = &storageClassName