|
| 1 | +package controllers |
| 2 | + |
| 3 | +import ( |
| 4 | + "errors" |
| 5 | + "context" |
| 6 | + "fmt" |
| 7 | + "github.com/go-logr/logr" |
| 8 | + rabbitmqv1beta1 "github.com/rabbitmq/cluster-operator/api/v1beta1" |
| 9 | + "github.com/rabbitmq/cluster-operator/internal/resource" |
| 10 | + appsv1 "k8s.io/api/apps/v1" |
| 11 | + corev1 "k8s.io/api/core/v1" |
| 12 | + k8serrors "k8s.io/apimachinery/pkg/api/errors" |
| 13 | + k8sresource "k8s.io/apimachinery/pkg/api/resource" |
| 14 | + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
| 15 | + "k8s.io/apimachinery/pkg/types" |
| 16 | + ctrl "sigs.k8s.io/controller-runtime" |
| 17 | + "sigs.k8s.io/controller-runtime/pkg/client" |
| 18 | + "time" |
| 19 | +) |
| 20 | + |
| 21 | + |
| 22 | +func (r *RabbitmqClusterReconciler) reconcilePVC(ctx context.Context, builder resource.ResourceBuilder, cluster *rabbitmqv1beta1.RabbitmqCluster, resource client.Object) error { |
| 23 | + logger := ctrl.LoggerFrom(ctx) |
| 24 | + |
| 25 | + switch sts := resource.(type) { |
| 26 | + case *appsv1.StatefulSet: |
| 27 | + current, err := r.statefulSet(ctx, cluster) |
| 28 | + if client.IgnoreNotFound(err) != nil { |
| 29 | + return err |
| 30 | + } else if k8serrors.IsNotFound(err) { |
| 31 | + logger.Info("statefulSet not created yet, skipping checks to expand PersistentVolumeClaims") |
| 32 | + return nil |
| 33 | + } |
| 34 | + |
| 35 | + if err := builder.Update(sts); err != nil { |
| 36 | + return err |
| 37 | + } |
| 38 | + |
| 39 | + resize, err := r.needsPVCResize(current, sts) |
| 40 | + |
| 41 | + if err != nil { |
| 42 | + return err |
| 43 | + } |
| 44 | + |
| 45 | + if resize { |
| 46 | + if err := r.expandPVC(ctx, cluster, current, sts); err != nil { |
| 47 | + logger.Error(err, "Failed to expand PersistentVolumeClaims", "statefulSet", cluster.ChildResourceName("server")) |
| 48 | + return err |
| 49 | + } |
| 50 | + } |
| 51 | + } |
| 52 | + return nil |
| 53 | +} |
| 54 | + |
| 55 | +func (r *RabbitmqClusterReconciler) expandPVC(ctx context.Context, rmq *rabbitmqv1beta1.RabbitmqCluster, current, desired *appsv1.StatefulSet) error { |
| 56 | + logger := ctrl.LoggerFrom(ctx) |
| 57 | + |
| 58 | + currentCapacity, err := persistenceStorageCapacity(current.Spec.VolumeClaimTemplates) |
| 59 | + if err != nil { |
| 60 | + return err |
| 61 | + } |
| 62 | + |
| 63 | + desiredCapacity, err := persistenceStorageCapacity(desired.Spec.VolumeClaimTemplates) |
| 64 | + if err != nil { |
| 65 | + return err |
| 66 | + } |
| 67 | + |
| 68 | + logger.Info(fmt.Sprintf("updating storage capacity from %v to %v", currentCapacity, desiredCapacity)) |
| 69 | + |
| 70 | + if err := r.deleteSts(ctx, rmq); err != nil { |
| 71 | + return err |
| 72 | + } |
| 73 | + |
| 74 | + if err := r.updatePVC(ctx, rmq, *current.Spec.Replicas, desiredCapacity); err != nil { |
| 75 | + return err |
| 76 | + } |
| 77 | + |
| 78 | + return nil |
| 79 | +} |
| 80 | + |
| 81 | +func (r *RabbitmqClusterReconciler) updatePVC(ctx context.Context, rmq *rabbitmqv1beta1.RabbitmqCluster, replicas int32, desiredCapacity k8sresource.Quantity) error { |
| 82 | + logger := ctrl.LoggerFrom(ctx) |
| 83 | + logger.Info("expanding PersistentVolumeClaims") |
| 84 | + |
| 85 | + for i := 0; i < int(replicas); i++ { |
| 86 | + PVCName := rmq.PVCName(i) |
| 87 | + PVC := corev1.PersistentVolumeClaim{} |
| 88 | + |
| 89 | + if err := r.Client.Get(ctx, types.NamespacedName{Namespace: rmq.Namespace, Name: PVCName}, &PVC); err != nil { |
| 90 | + logger.Error(err, "failed to get PersistentVolumeClaim") |
| 91 | + return err |
| 92 | + } |
| 93 | + PVC.Spec.Resources.Requests[corev1.ResourceStorage] = desiredCapacity |
| 94 | + if err := r.Client.Update(ctx, &PVC, &client.UpdateOptions{}); err != nil { |
| 95 | + logger.Error(err, "failed to update PersistentVolumeClaim") |
| 96 | + return err |
| 97 | + } |
| 98 | + logger.Info("successfully expanded", "PVC", PVCName) |
| 99 | + } |
| 100 | + return nil |
| 101 | +} |
| 102 | + |
| 103 | +func (r *RabbitmqClusterReconciler) needsPVCResize(current, desired *appsv1.StatefulSet) (bool, error) { |
| 104 | + currentCapacity, err := persistenceStorageCapacity(current.Spec.VolumeClaimTemplates) |
| 105 | + if err != nil { |
| 106 | + return false, err |
| 107 | + } |
| 108 | + |
| 109 | + desiredCapacity, err := persistenceStorageCapacity(desired.Spec.VolumeClaimTemplates) |
| 110 | + if err != nil { |
| 111 | + return false, err |
| 112 | + } |
| 113 | + |
| 114 | + if currentCapacity.Cmp(desiredCapacity) != 0 { |
| 115 | + return true, nil |
| 116 | + } |
| 117 | + |
| 118 | + return false, nil |
| 119 | +} |
| 120 | + |
| 121 | +func persistenceStorageCapacity(templates []corev1.PersistentVolumeClaim) (k8sresource.Quantity, error) { |
| 122 | + for _, t := range templates { |
| 123 | + if t.Name == "persistence" { |
| 124 | + return t.Spec.Resources.Requests[corev1.ResourceStorage], nil |
| 125 | + } |
| 126 | + } |
| 127 | + return k8sresource.Quantity{}, errors.New("cannot find PersistentVolumeClaim 'persistence'") |
| 128 | +} |
| 129 | + |
| 130 | + |
| 131 | +// deleteSts deletes a sts without deleting pods and PVCs |
| 132 | +// using DeletePropagationPolicy set to 'Orphan' |
| 133 | +func (r *RabbitmqClusterReconciler) deleteSts(ctx context.Context, rmq *rabbitmqv1beta1.RabbitmqCluster) error { |
| 134 | + logger := ctrl.LoggerFrom(ctx) |
| 135 | + logger.Info("deleting statefulSet (pods won't be deleted)", "statefulSet", rmq.ChildResourceName("server")) |
| 136 | + deletePropagationPolicy := metav1.DeletePropagationOrphan |
| 137 | + deleteOptions := &client.DeleteOptions{PropagationPolicy: &deletePropagationPolicy} |
| 138 | + stsName := rmq.ChildResourceName("server") |
| 139 | + current, err := r.statefulSet(ctx, rmq) |
| 140 | + if err != nil { |
| 141 | + return err |
| 142 | + } |
| 143 | + if err := r.Delete(ctx, current, deleteOptions); err != nil { |
| 144 | + logger.Error(err, "failed to delete statefulSet", "statefulSet", stsName) |
| 145 | + return err |
| 146 | + } |
| 147 | + |
| 148 | + if err := retryWithInterval(logger, "delete statefulSet", 10, 3*time.Second, func() bool { |
| 149 | + _, getErr := r.statefulSet(ctx, rmq) |
| 150 | + if k8serrors.IsNotFound(getErr) { |
| 151 | + return true |
| 152 | + } |
| 153 | + return false |
| 154 | + }); err != nil { |
| 155 | + logger.Error(err, "statefulSet not deleting after 50 seconds", "statefulSet", stsName) |
| 156 | + return err |
| 157 | + } |
| 158 | + logger.Info("statefulSet deleted", "statefulSet", stsName) |
| 159 | + return nil |
| 160 | +} |
| 161 | + |
| 162 | +func retryWithInterval(logger logr.Logger, msg string, retry int, interval time.Duration, f func() bool) (err error) { |
| 163 | + for i := 0; i < retry; i++ { |
| 164 | + if ok := f(); ok { |
| 165 | + return |
| 166 | + } |
| 167 | + time.Sleep(interval) |
| 168 | + logger.Info("retrying again", "action", msg, "interval", interval, "attempt", i+1) |
| 169 | + } |
| 170 | + return fmt.Errorf("failed to %s after %d retries", msg, retry) |
| 171 | +} |
0 commit comments