Surface kubeconfig via hostedcluster.status

api/v1alpha1/hostedcluster_types.go

@@ -64,6 +64,11 @@ type HostedClusterStatus struct {
 
 	// +optional
 	Ready bool `json:"ready,omitempty"`
+
+	// KubeConfig is a reference to the secret containing the default kubeconfig
+	// for the cluster.
+	// +optional
+	KubeConfig *corev1.LocalObjectReference `json:"kubeconfig,omitempty"`
 }
 
 // ClusterVersionStatus reports the status of the cluster versioning,

cmd/install/assets/hypershift-operator/hypershift.openshift.io_hostedclusters.yaml

@@ -81,6 +81,13 @@ spec:
           status:
             description: HostedClusterStatus defines the observed state of HostedCluster
             properties:
+              kubeconfig:
+                description: KubeConfig is a reference to the secret containing the default kubeconfig for the cluster.
+                properties:
+                  name:
+                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'
+                    type: string
+                type: object
               ready:
                 type: boolean
               version:

hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go

@@ -26,6 +26,7 @@ import (
 	"github.com/go-logr/logr"
 	configv1 "github.com/openshift/api/config/v1"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/equality"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
@@ -210,7 +211,7 @@ func (r *HostedClusterReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 		capiAwsProviderDeployment,
 	}
 
-	err = r.applyObjects(ctx, capiManagerObjects)
+	err = r.applyObjects(ctx, capiManagerObjects...)
 	if err != nil {
 		r.Log.Error(err, "failed to apply cluster api resources")
 		return ctrl.Result{}, err
@@ -285,7 +286,7 @@ func (r *HostedClusterReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 		controlPlaneOperatorDeployment,
 	}
 
-	err = r.applyObjects(ctx, controlPlaneObjects)
+	err = r.applyObjects(ctx, controlPlaneObjects...)
 	if err != nil {
 		r.Log.Error(err, "failed to apply control plane resources")
 		return ctrl.Result{}, err
@@ -350,34 +351,80 @@ func (r *HostedClusterReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 		return ctrl.Result{Requeue: true}, nil
 	}
 
-	// Roll out an auto scaler
-	// TODO: this should only happen when the HCP is ready and publishes a kubeconfig
-	autoScalerRole := autoscaler.Role{Namespace: targetNamespace}.Build()
-	autoScalerServiceAccount := autoscaler.ServiceAccount{Namespace: targetNamespace}.Build()
-	autoScalerRoleBinding := autoscaler.RoleBinding{
-		Role:           autoScalerRole,
-		ServiceAccount: autoScalerServiceAccount,
-	}.Build()
-	autoScalerDeployment := autoscaler.Deployment{
-		Namespace:      targetNamespace,
-		ServiceAccount: autoScalerServiceAccount,
-		Image:          "k8s.gcr.io/autoscaling/cluster-autoscaler:v1.20.0",
-		//The client used by CAPI machine controller expects the kubeconfig to follow this naming convention
-		//https://github.com/kubernetes-sigs/cluster-api/blob/5c85a0a01ee44ecf7c8a3c3fdc867a88af87d73c/util/secret/secret.go#L29-L33
-		TargetClusterKubeconfigSecretName: fmt.Sprintf("%s-kubeconfig", capiCluster.GetName()),
-	}.Build()
-	autoScalerObjects := []ctrlclient.Object{
-		autoScalerRole,
-		autoScalerServiceAccount,
-		autoScalerRoleBinding,
-		autoScalerDeployment,
+	// When the hosted control plane kubeconfig secret is available, copy it to the
+	// hostedcluster namespace and update status
+	if hcp.Status.KubeConfig != nil {
+		var targetKubeConfigSecret corev1.Secret
+		targetKubeConfigSecretName := types.NamespacedName{Namespace: targetNamespace.Name, Name: hcp.Status.KubeConfig.Name}
+		err := r.Client.Get(ctx, targetKubeConfigSecretName, &targetKubeConfigSecret)
+		if err != nil {
+			return ctrl.Result{}, fmt.Errorf("failed to get target kubeconfig secret %q: %w", targetKubeConfigSecretName, err)
+		}
+		// Build a kubeconfig secret scoped to the hostedcluster's namespace
+		// which has the same contents as the target secret.
+		// TODO: Leaky abstraction, publish this key through HCP status?
+		targetSecretData, ok := targetKubeConfigSecret.Data["value"]
+		if !ok {
+			return ctrl.Result{}, fmt.Errorf("target kubeconfig secret %q is missing key %q", targetKubeConfigSecretName, "value")
+		}
+		kubeConfigSecret := manifests.KubeConfigSecret{
+			HostedCluster: hcluster,
+			Data:          targetSecretData,
+		}.Build()
+		// Update the hostedcluster's copy of the secret.
+		err = r.applyObjects(ctx, kubeConfigSecret)
+		if err != nil {
+			return ctrl.Result{}, fmt.Errorf("failed to apply kubeconfig secret: %w", err)
+		}
+		// Ensure the hostedcluster has a reference to the secret.
+		updatedReference := &corev1.LocalObjectReference{Name: kubeConfigSecret.Name}
+		if !equality.Semantic.DeepEqual(hcluster.Status.KubeConfig, updatedReference) {
+			hcluster.Status.KubeConfig = updatedReference
+			if err = r.Status().Update(ctx, hcluster); err != nil {
+				r.Log.Error(err, "failed to update version in hosted cluster status")
+				return ctrl.Result{}, fmt.Errorf("failed to update version in hosted cluster status: %w", err)
+			}
+			r.Log.Info("updated hostedcluster version, requeueing")
+			return ctrl.Result{Requeue: true}, nil
+		}
 	}
-	err = r.applyObjects(ctx, autoScalerObjects)
-	if err != nil {
-		r.Log.Error(err, "failed to apply auto scaler resources")
-		return ctrl.Result{}, err
+
+	// Roll out an auto scaler once the kubeconfig is available
+	if hcp.Status.KubeConfig != nil {
+		var targetKubeConfigSecret corev1.Secret
+		targetKubeConfigSecretName := types.NamespacedName{Namespace: targetNamespace.Name, Name: hcp.Status.KubeConfig.Name}
+		err := r.Client.Get(ctx, targetKubeConfigSecretName, &targetKubeConfigSecret)
+		if err != nil {
+			return ctrl.Result{}, fmt.Errorf("failed to get target kubeconfig secret %q: %w", targetKubeConfigSecretName, err)
+		}
+		autoScalerRole := autoscaler.Role{Namespace: targetNamespace}.Build()
+		autoScalerServiceAccount := autoscaler.ServiceAccount{Namespace: targetNamespace}.Build()
+		autoScalerRoleBinding := autoscaler.RoleBinding{
+			Role:           autoScalerRole,
+			ServiceAccount: autoScalerServiceAccount,
+		}.Build()
+		autoScalerDeployment := autoscaler.Deployment{
+			Namespace:        targetNamespace,
+			ServiceAccount:   autoScalerServiceAccount,
+			Image:            "k8s.gcr.io/autoscaling/cluster-autoscaler:v1.20.0",
+			TargetKubeConfig: &targetKubeConfigSecret,
+		}.Build()
+		autoScalerObjects := []ctrlclient.Object{
+			autoScalerRole,
+			autoScalerServiceAccount,
+			autoScalerRoleBinding,
+			autoScalerDeployment,
+		}
+		err = r.applyObjects(ctx, autoScalerObjects...)
+		if err != nil {
+			r.Log.Error(err, "failed to apply auto scaler resources")
+			return ctrl.Result{}, err
+		}
+		r.Log.Info("created all autoscaler resources")
+	} else {
+		// TODO: status?
+		r.Log.Info("autoscaler rollout pending kubeconfig availability")
 	}
-	r.Log.Info("Created all autoscaler resources")
 
 	// Check for readiness and update status
 	var currentCluster capiv1.Cluster
@@ -406,7 +453,7 @@ func (r *HostedClusterReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 	return ctrl.Result{}, nil
 }
 
-func (r *HostedClusterReconciler) applyObjects(ctx context.Context, objects []ctrlclient.Object) error {
+func (r *HostedClusterReconciler) applyObjects(ctx context.Context, objects ...ctrlclient.Object) error {
 	for i := range objects {
 		object := objects[i]
 		var objectBytes bytes.Buffer

hypershift-operator/controllers/hostedcluster/manifests/autoscaler/manifests.go

@@ -9,10 +9,10 @@ import (
 )
 
 type Deployment struct {
-	Namespace                         *corev1.Namespace
-	ServiceAccount                    *corev1.ServiceAccount
-	Image                             string
-	TargetClusterKubeconfigSecretName string
+	Namespace        *corev1.Namespace
+	ServiceAccount   *corev1.ServiceAccount
+	Image            string
+	TargetKubeConfig *corev1.Secret
 }
 
 func (o Deployment) Build() *appsv1.Deployment {
@@ -52,12 +52,11 @@ func (o Deployment) Build() *appsv1.Deployment {
 							Name: "target-kubeconfig",
 							VolumeSource: corev1.VolumeSource{
 								Secret: &corev1.SecretVolumeSource{
-									// TODO: this should come from HCP status
-									SecretName: o.TargetClusterKubeconfigSecretName,
+									SecretName: o.TargetKubeConfig.Name,
 									Items: []corev1.KeyToPath{
 										{
-											// TODO: this should come from HCP status
-											Key:  "value",
+											// TODO: should the key be published on status?
+											Key:  "kubeconfig",
 											Path: "target-kubeconfig",
 										},
 									},

hypershift-operator/controllers/hostedcluster/manifests/manifests.go

@@ -115,3 +115,24 @@ func (o DefaultNodePool) Build() *hyperv1.NodePool {
 	}
 	return nodePool
 }
+
+type KubeConfigSecret struct {
+	HostedCluster *hyperv1.HostedCluster
+	Data          []byte
+}
+
+func (o KubeConfigSecret) Build() *corev1.Secret {
+	secret := &corev1.Secret{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Secret",
+			APIVersion: corev1.SchemeGroupVersion.String(),
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: o.HostedCluster.Namespace,
+			Name:      o.HostedCluster.Name + "-admin-kubeconfig",
+		},
+		Type: corev1.SecretTypeOpaque,
+		Data: map[string][]byte{"kubeconfig": o.Data},
+	}
+	return secret
+}

test/e2e/quick_start_test.go

@@ -4,7 +4,6 @@ package e2e
 
 import (
 	"context"
-	"fmt"
 	"io/ioutil"
 	"time"
 
@@ -19,6 +18,7 @@ import (
 
 	hyperapi "openshift.io/hypershift/api"
 	apifixtures "openshift.io/hypershift/api/fixtures"
+	hyperv1 "openshift.io/hypershift/api/v1alpha1"
 	"openshift.io/hypershift/test/e2e/internal/log"
 	"openshift.io/hypershift/version"
 )
@@ -128,21 +128,31 @@ func QuickStartSpec(ctx context.Context, inputGetter func() QuickStartSpecInput)
 		By("Ensuring the guest cluster exposes a valid kubeconfig")
 
 		log.Logf("Waiting for guest kubeconfig to become available")
-		guestKubeConfigSecret := &corev1.Secret{}
+		var guestKubeConfigSecret corev1.Secret
 		Eventually(func() bool {
+			var currentCluster hyperv1.HostedCluster
+			err := input.Client.Get(ctx, ctrl.ObjectKeyFromObject(example.Cluster), &currentCluster)
+			if err != nil {
+				log.Logf("error getting cluster: %w", err)
+				return false
+			}
+			if currentCluster.Status.KubeConfig == nil {
+				return false
+			}
 			key := ctrl.ObjectKey{
-				// TODO: This resource needs extracted into a library function
-				Namespace: example.Cluster.GetName(),
-				Name:      fmt.Sprintf("%s-kubeconfig", example.Cluster.GetName()),
+				Namespace: currentCluster.Namespace,
+				Name:      currentCluster.Status.KubeConfig.Name,
 			}
-			if err := input.Client.Get(ctx, key, guestKubeConfigSecret); err != nil {
+			if err := input.Client.Get(ctx, key, &guestKubeConfigSecret); err != nil {
+				log.Logf("error getting guest kubeconfig secret %s: %w", key, err)
 				return false
 			}
 			return true
 		}, 5*time.Minute, 1*time.Second).Should(BeTrue(), "couldn't find guest kubeconfig secret")
 
-		guestKubeConfigSecretData, hasData := guestKubeConfigSecret.Data["value"]
-		Expect(hasData).To(BeTrue(), "guest guest kubeconfig secret is missing value key")
+		// TODO: this key should probably be published or an API constant
+		guestKubeConfigSecretData, hasData := guestKubeConfigSecret.Data["kubeconfig"]
+		Expect(hasData).To(BeTrue(), "guest kubeconfig secret is missing kubeconfig key")
 
 		guestConfig, err := clientcmd.RESTConfigFromKubeConfig(guestKubeConfigSecretData)
 		Expect(err).NotTo(HaveOccurred(), "couldn't load guest kubeconfig")