fix #648: delete the statefulset with precondition set to the correct UID

Author: srteam2020

controllers/reconcile_finalizer.go

@@ -41,6 +41,10 @@ func (r *RabbitmqClusterReconciler) removeFinalizer(ctx context.Context, rabbitm
 func (r *RabbitmqClusterReconciler) prepareForDeletion(ctx context.Context, rabbitmqCluster *rabbitmqv1beta1.RabbitmqCluster) error {
 	if controllerutil.ContainsFinalizer(rabbitmqCluster, deletionFinalizer) {
 		if err := clientretry.RetryOnConflict(clientretry.DefaultRetry, func() error {
+			uid, err := r.statefulSetUID(ctx, rabbitmqCluster);
+			if err != nil {
+				return err
+			}
 			sts := &appsv1.StatefulSet{
 				ObjectMeta: metav1.ObjectMeta{
 					Name:      rabbitmqCluster.ChildResourceName("server"),
@@ -54,7 +58,7 @@ func (r *RabbitmqClusterReconciler) prepareForDeletion(ctx context.Context, rabb
 			// Delete StatefulSet immediately after changing pod labels to minimize risk of them respawning.
 			// There is a window where the StatefulSet could respawn Pods without the deletion label in this order.
 			// But we can't delete it before because the DownwardAPI doesn't update once a Pod enters Terminating.
-			if err := r.Client.Delete(ctx, sts); client.IgnoreNotFound(err) != nil {
+			if err := r.Client.Delete(ctx, sts, &client.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}); client.IgnoreNotFound(err) != nil {
 				return fmt.Errorf("cannot delete StatefulSet: %s", err.Error())
 			}
 

controllers/utils.go

@@ -2,9 +2,12 @@ package controllers
 
 import (
 	"context"
+	"fmt"
+
 	rabbitmqv1beta1 "github.com/rabbitmq/cluster-operator/api/v1beta1"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apimachinery/pkg/types"
@@ -64,6 +67,18 @@ func (r *RabbitmqClusterReconciler) statefulSet(ctx context.Context, rmq *rabbit
 	return sts, nil
 }
 
+func (r *RabbitmqClusterReconciler) statefulSetUID(ctx context.Context, rmq *rabbitmqv1beta1.RabbitmqCluster) (types.UID, error) {
+	uid := types.UID("")
+	if sts, err := r.statefulSet(ctx, rmq); err == nil {
+		if ref := metav1.GetControllerOf(sts); ref != nil {
+			if string(rmq.GetUID()) == string(ref.UID) {
+				return sts.UID, nil
+			}
+		}
+	}
+	return uid, fmt.Errorf("failed to get the uid of the statefulset owned by the current rabbitmqCluster")
+}
+
 func (r *RabbitmqClusterReconciler) configMap(ctx context.Context, rmq *rabbitmqv1beta1.RabbitmqCluster, name string) (*corev1.ConfigMap, error) {
 	configMap := &corev1.ConfigMap{}
 	if err := r.Get(ctx, types.NamespacedName{Namespace: rmq.Namespace, Name: name}, configMap); err != nil {