re-apply secret OwnerRef on pivot

Signed-off-by: Andrew Sy Kim <[email protected]>

cmd/clusterctl/clusterdeployer/clusterclient/clusterclient.go

@@ -32,8 +32,10 @@ import (
 	autoscalingv1 "k8s.io/api/autoscaling/v1"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
 	_ "k8s.io/client-go/plugin/pkg/client/auth" // nolint
 	tcmd "k8s.io/client-go/tools/clientcmd"
 	"k8s.io/klog"
@@ -105,6 +107,7 @@ type Client interface {
 	ScaleDeployment(namespace, name string, scale int32) error
 	WaitForClusterV1alpha2Ready() error
 	WaitForResourceStatuses() error
+	SetClusterOwnerRef(runtime.Object, *clusterv1.Cluster) error
 }
 
 type client struct {
@@ -676,6 +679,29 @@ func (c *client) WaitForResourceStatuses() error {
 	})
 }
 
+func (c *client) SetClusterOwnerRef(obj runtime.Object, cluster *clusterv1.Cluster) error {
+	meta, err := meta.Accessor(obj)
+	if err != nil {
+		return err
+	}
+
+	meta.SetOwnerReferences([]metav1.OwnerReference{
+		{
+			APIVersion: clusterv1.GroupVersion.String(),
+			Kind:       "Cluster",
+			Name:       cluster.Name,
+			UID:        cluster.UID,
+		},
+	})
+
+	if err := c.clientSet.Update(ctx, obj); err != nil {
+		return errors.Wrapf(err, "error updating object [%s] %s/%s with cluster OwnerRef",
+			obj.GetObjectKind().GroupVersionKind(), meta.GetNamespace(), meta.GetName())
+	}
+
+	return nil
+}
+
 func (c *client) GetClusterSecrets(cluster *clusterv1.Cluster) ([]*corev1.Secret, error) {
 	list := &corev1.SecretList{}
 	if err := c.clientSet.List(ctx, list, ctrlclient.InNamespace(cluster.Namespace)); err != nil {

cmd/clusterctl/clusterdeployer/clusterdeployer_test.go

@@ -27,6 +27,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/kubernetes"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha2"
 	"sigs.k8s.io/cluster-api/cmd/clusterctl/clusterdeployer/clusterclient"
@@ -560,6 +561,10 @@ func (c *testClusterClient) WaitForResourceStatuses() error {
 	return nil
 }
 
+func (c *testClusterClient) SetClusterOwnerRef(obj runtime.Object, cluster *clusterv1.Cluster) error {
+	return nil
+}
+
 func (c *testClusterClient) CreateSecret(secret *corev1.Secret) error {
 	if c.CreateSecretErr == nil {
 		c.secrets = append(c.secrets, secret)

cmd/clusterctl/phases/pivot.go

@@ -25,6 +25,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/yaml"
 	"k8s.io/klog"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha2"
@@ -61,10 +62,12 @@ type targetClient interface {
 	CreateMachines([]*clusterv1.Machine, string) error
 	CreateMachineSets([]*clusterv1.MachineSet, string) error
 	CreateUnstructuredObject(*unstructured.Unstructured) error
+	GetCluster(string, string) (*clusterv1.Cluster, error)
 	EnsureNamespace(string) error
 	GetMachineDeployment(namespace, name string) (*clusterv1.MachineDeployment, error)
 	GetMachineSet(string, string) (*clusterv1.MachineSet, error)
 	WaitForClusterV1alpha2Ready() error
+	SetClusterOwnerRef(runtime.Object, *clusterv1.Cluster) error
 }
 
 // Pivot deploys the provided provider components to a target cluster and then migrates
@@ -178,11 +181,6 @@ func moveCluster(from sourceClient, to targetClient, cluster *clusterv1.Cluster)
 		return errors.Wrapf(err, "unable to ensure namespace %q in target cluster", cluster.Namespace)
 	}
 
-	// Move the cluster's secrets first.
-	if err := moveClusterSecrets(from, to, cluster); err != nil {
-		return errors.Wrapf(err, "failed to move Secrets for Cluster %s/%s to target cluster", cluster.Namespace, cluster.Name)
-	}
-
 	// New objects cannot have a specified resource version. Clear it out.
 	cluster.SetResourceVersion("")
 	if err := to.CreateClusterObject(cluster); err != nil {
@@ -197,6 +195,12 @@ func moveCluster(from sourceClient, to targetClient, cluster *clusterv1.Cluster)
 		}
 	}
 
+	// Move the cluster's secrets only after the target cluster resource is created
+	// since we have to update the Secret's OwnerRef
+	if err := moveClusterSecrets(from, to, cluster); err != nil {
+		return errors.Wrapf(err, "failed to move Secrets for Cluster %s/%s to target cluster", cluster.Namespace, cluster.Name)
+	}
+
 	klog.V(4).Infof("Retrieving list of MachineDeployments to move for Cluster %s/%s", cluster.Namespace, cluster.Name)
 	machineDeployments, err := from.GetMachineDeploymentsForCluster(cluster)
 	if err != nil {
@@ -239,24 +243,26 @@ func moveClusterSecrets(from sourceClient, to targetClient, cluster *clusterv1.C
 		return err
 	}
 
+	toCluster, err := to.GetCluster(cluster.Name, cluster.Namespace)
+	if err != nil {
+		return err
+	}
+
 	for _, secret := range secrets {
-		if err := moveSecret(from, to, secret); err != nil {
+		if err := moveSecret(from, to, secret, toCluster); err != nil {
 			return errors.Wrapf(err, "failed to move Secret %s/%s", secret.Namespace, secret.Name)
 		}
 	}
 	return nil
 }
 
-func moveSecret(from sourceClient, to targetClient, secret *corev1.Secret) error {
+func moveSecret(from sourceClient, to targetClient, secret *corev1.Secret, toCluster *clusterv1.Cluster) error {
 	klog.V(4).Infof("Moving secret %s/%s", secret.Namespace, secret.Name)
 
 	// New objects cannot have a specified resource version. Clear it out.
 	secret.SetResourceVersion("")
-
-	// remove the UID from ownerReferences as it will be different across clusters
-	for i := 0; i < len(secret.OwnerReferences); i++ {
-		secret.OwnerReferences[i].UID = ""
-	}
+	// Set the cluster owner ref based on target cluster's Cluster resource
+	to.SetClusterOwnerRef(secret, toCluster)
 
 	if err := to.CreateSecret(secret); err != nil {
 		return errors.Wrapf(err, "error copying Secret %s/%s to target cluster", secret.Namespace, secret.Name)

cmd/clusterctl/phases/pivot_test.go

@@ -25,6 +25,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha2"
 )
 
@@ -556,6 +557,10 @@ func (t *target) EnsureNamespace(string) error {
 	return nil
 }
 
+func (t *target) GetCluster(name, ns string) (*clusterv1.Cluster, error) {
+	return nil, nil
+}
+
 func (t *target) GetMachineDeployment(ns, name string) (*clusterv1.MachineDeployment, error) {
 	for _, deployment := range t.machineDeployments[ns] {
 		if deployment.Name == name {
@@ -578,6 +583,10 @@ func (t *target) WaitForClusterV1alpha2Ready() error {
 	return nil
 }
 
+func (t *target) SetClusterOwnerRef(obj runtime.Object, cluster *clusterv1.Cluster) error {
+	return nil
+}
+
 func (t *target) CreateUnstructuredObject(u *unstructured.Unstructured) error {
 	ns := u.GetNamespace()