CCO-388: control-plane-operator: reconcile the CCO

availability-prober/availability_prober.go

@@ -121,7 +121,7 @@ func check(log logr.Logger, target *url.URL, requestTimeout time.Duration, sleep
 			log.WithValues("statuscode", response.StatusCode).Info("Request didn't return a 2XX status code, retrying...")
 			continue
 		}
-		log.Info("Success", "statuscode", response.StatusCode)
+		log.Info("URI probing succeeded", "uri", target.String(), "statuscode", response.StatusCode)
 
 		if len(requiredAPIs) > 0 {
 			_, apis, err := discoveryClient.ServerGroupsAndResources()
@@ -183,6 +183,7 @@ func check(log logr.Logger, target *url.URL, requestTimeout time.Duration, sleep
 			}
 		}
 
+		log.Info("all checks successful, exiting...")
 		return
 	}
 }

control-plane-operator/controllers/hostedcontrolplane/cco/reconcile.go

@@ -0,0 +1,171 @@
+package cco
+
+import (
+	"path"
+
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/utils/pointer"
+
+	hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1"
+	"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/imageprovider"
+	"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/kas"
+	"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/manifests"
+	"github.com/openshift/hypershift/support/config"
+	"github.com/openshift/hypershift/support/proxy"
+	"github.com/openshift/hypershift/support/util"
+)
+
+const (
+	WorkerNamespace      = "openshift-cloud-credential-operator"
+	WorkerServiceAccount = "cloud-credential-operator"
+)
+
+func selectorLabels() map[string]string {
+	return map[string]string{
+		"app":                         "cloud-credential-operator",
+		hyperv1.ControlPlaneComponent: "cloud-credential-operator",
+	}
+}
+
+var (
+	volumeMounts = util.PodVolumeMounts{
+		containerMain().Name: {
+			volumeServiceAccountKubeconfig().Name: "/etc/kubernetes",
+		},
+	}
+)
+
+type Params struct {
+	operatorImage           string
+	kubeRbacProxyImage      string
+	availabilityProberImage string
+
+	deploymentConfig config.DeploymentConfig
+	releaseVersion   string
+	issuerURL        string
+	apiPort          *int32
+
+	config.OwnerRef
+}
+
+func NewParams(hcp *hyperv1.HostedControlPlane, version string, releaseImageProvider *imageprovider.ReleaseImageProvider, setDefaultSecurityContext bool) Params {
+	params := Params{
+		operatorImage:           releaseImageProvider.GetImage("cloud-credential-operator"),
+		kubeRbacProxyImage:      releaseImageProvider.GetImage("kube-rbac-proxy"),
+		availabilityProberImage: releaseImageProvider.GetImage(util.AvailabilityProberImageName),
+		releaseVersion:          version,
+		issuerURL:               hcp.Spec.IssuerURL,
+		OwnerRef:                config.OwnerRefFrom(hcp),
+		apiPort:                 pointer.Int32(util.KASPodPort(hcp)),
+		deploymentConfig: config.DeploymentConfig{
+			Scheduling: config.Scheduling{
+				PriorityClass: config.DefaultPriorityClass,
+			},
+			SetDefaultSecurityContext: setDefaultSecurityContext,
+			Resources: config.ResourcesSpec{
+				containerMain().Name: {
+					Requests: corev1.ResourceList{
+						corev1.ResourceCPU:    resource.MustParse("10m"),
+						corev1.ResourceMemory: resource.MustParse("75Mi"),
+					},
+				},
+			},
+		},
+	}
+	params.deploymentConfig.SetRestartAnnotation(hcp.ObjectMeta)
+	if hcp.Annotations[hyperv1.ControlPlanePriorityClass] != "" {
+		params.deploymentConfig.Scheduling.PriorityClass = hcp.Annotations[hyperv1.ControlPlanePriorityClass]
+	}
+	params.deploymentConfig.SetDefaults(hcp, selectorLabels(), pointer.Int(1))
+	params.deploymentConfig.SetReleaseImageAnnotation(hcp.Spec.ReleaseImage)
+	return params
+}
+
+func ReconcileDeployment(deployment *appsv1.Deployment, params Params) error {
+	params.OwnerRef.ApplyTo(deployment)
+	deployment.Spec = appsv1.DeploymentSpec{
+		Selector: &metav1.LabelSelector{
+			MatchLabels: selectorLabels(),
+		},
+		Strategy: appsv1.DeploymentStrategy{
+			Type: appsv1.RecreateDeploymentStrategyType,
+		},
+		Template: corev1.PodTemplateSpec{
+			ObjectMeta: metav1.ObjectMeta{
+				Labels: selectorLabels(),
+			},
+			Spec: corev1.PodSpec{
+				AutomountServiceAccountToken: pointer.Bool(false),
+				Containers: []corev1.Container{
+					util.BuildContainer(containerMain(), buildMainContainer(params.operatorImage, params.releaseVersion)),
+				},
+				Volumes: []corev1.Volume{
+					util.BuildVolume(volumeServiceAccountKubeconfig(), buildVolumeServiceAccountKubeconfig),
+				},
+			},
+		},
+	}
+
+	params.deploymentConfig.ApplyTo(deployment)
+	util.AvailabilityProber(kas.InClusterKASReadyURL(), params.availabilityProberImage, &deployment.Spec.Template.Spec, func(o *util.AvailabilityProberOpts) {
+		o.KubeconfigVolumeName = volumeServiceAccountKubeconfig().Name
+		o.WaitForInfrastructureResource = true
+		o.RequiredAPIs = []schema.GroupVersionKind{
+			{Group: "operator.openshift.io", Version: "v1", Kind: "CloudCredential"},
+		}
+	})
+	return nil
+}
+
+func containerMain() *corev1.Container {
+	return &corev1.Container{
+		Name: "cloud-credential-operator",
+	}
+}
+
+func buildMainContainer(image, releaseVersion string) func(*corev1.Container) {
+	return func(c *corev1.Container) {
+		c.Image = image
+		c.Command = []string{
+			"/usr/bin/cloud-credential-operator",
+		}
+		c.Args = []string{
+			"operator",
+			"--kubeconfig=" + path.Join(volumeMounts.Path(containerMain().Name, volumeServiceAccountKubeconfig().Name), util.KubeconfigKey),
+		}
+		c.Env = []corev1.EnvVar{
+			{
+				Name:  "RELEASE_VERSION",
+				Value: releaseVersion,
+			},
+			{
+				Name:  "KUBECONFIG",
+				Value: path.Join(volumeMounts.Path(containerMain().Name, volumeServiceAccountKubeconfig().Name), util.KubeconfigKey),
+			},
+		}
+		proxy.SetEnvVars(&c.Env)
+		c.VolumeMounts = volumeMounts.ContainerMounts(c.Name)
+		c.TerminationMessagePolicy = corev1.TerminationMessageFallbackToLogsOnError
+		c.SecurityContext = &corev1.SecurityContext{
+			Capabilities:             &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}},
+			AllowPrivilegeEscalation: pointer.Bool(false),
+		}
+	}
+}
+
+func volumeServiceAccountKubeconfig() *corev1.Volume {
+	return &corev1.Volume{
+		Name: "service-account-kubeconfig",
+	}
+}
+
+func buildVolumeServiceAccountKubeconfig(v *corev1.Volume) {
+	v.Secret = &corev1.SecretVolumeSource{
+		SecretName:  manifests.CloudCredentialOperatorKubeconfig("").Name,
+		DefaultMode: pointer.Int32(0640),
+	}
+}

control-plane-operator/controllers/hostedcontrolplane/cco/reconcile_test.go

@@ -0,0 +1,48 @@
+package cco
+
+import (
+	"testing"
+
+	"github.com/openshift/hypershift/api"
+	hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1"
+	"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/imageprovider"
+	"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/manifests"
+	"github.com/openshift/hypershift/support/testutil"
+	"github.com/openshift/hypershift/support/util"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/pointer"
+)
+
+func TestReconcileDeployment(t *testing.T) {
+	hcp := &hyperv1.HostedControlPlane{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "test-namespace",
+		},
+		Spec: hyperv1.HostedControlPlaneSpec{
+			ReleaseImage: "quay.io/ocp-dev/test-release-image:latest",
+			IssuerURL:    "https://www.example.com",
+			Networking: hyperv1.ClusterNetworking{
+				APIServer: &hyperv1.APIServerNetworking{
+					Port: pointer.Int32(1234),
+				},
+			},
+		},
+	}
+	images := map[string]string{
+		"cloud-credential-operator": "quay.io/openshift/cloud-credential-operator:latest",
+		"token-minter":              "quay.io/openshift/token-minter:latest",
+		"availability-prober":       "quay.io/openshift/availability-prober:latest",
+	}
+	deployment := manifests.CloudCredentialOperatorDeployment("test-namespace")
+	imageProvider := imageprovider.NewFromImages(images)
+	params := NewParams(hcp, "1.0.0", imageProvider, true)
+	if err := ReconcileDeployment(deployment, params); err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	deploymentYaml, err := util.SerializeResource(deployment, api.Scheme)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	testutil.CompareWithFixture(t, deploymentYaml)
+}

control-plane-operator/controllers/hostedcontrolplane/cco/testdata/zz_fixture_TestReconcileDeployment.yaml

@@ -0,0 +1,119 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  creationTimestamp: null
+  labels:
+    hypershift.openshift.io/managed-by: control-plane-operator
+  name: cloud-credential-operator
+  namespace: test-namespace
+  ownerReferences:
+  - apiVersion: hypershift.openshift.io/v1beta1
+    blockOwnerDeletion: true
+    controller: true
+    kind: HostedControlPlane
+    name: test
+    uid: ""
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: cloud-credential-operator
+      hypershift.openshift.io/control-plane-component: cloud-credential-operator
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      annotations:
+        hypershift.openshift.io/release-image: quay.io/ocp-dev/test-release-image:latest
+      creationTimestamp: null
+      labels:
+        app: cloud-credential-operator
+        hypershift.openshift.io/control-plane-component: cloud-credential-operator
+        hypershift.openshift.io/hosted-control-plane: test-namespace
+    spec:
+      affinity:
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - preference:
+              matchExpressions:
+              - key: hypershift.openshift.io/control-plane
+                operator: In
+                values:
+                - "true"
+            weight: 50
+          - preference:
+              matchExpressions:
+              - key: hypershift.openshift.io/cluster
+                operator: In
+                values:
+                - test-namespace
+            weight: 100
+        podAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - podAffinityTerm:
+              labelSelector:
+                matchLabels:
+                  hypershift.openshift.io/hosted-control-plane: test-namespace
+              topologyKey: kubernetes.io/hostname
+            weight: 100
+      automountServiceAccountToken: false
+      containers:
+      - args:
+        - operator
+        - --kubeconfig=/etc/kubernetes/kubeconfig
+        command:
+        - /usr/bin/cloud-credential-operator
+        env:
+        - name: RELEASE_VERSION
+          value: 1.0.0
+        - name: KUBECONFIG
+          value: /etc/kubernetes/kubeconfig
+        image: quay.io/openshift/cloud-credential-operator:latest
+        name: cloud-credential-operator
+        resources:
+          requests:
+            cpu: 10m
+            memory: 75Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+        terminationMessagePolicy: FallbackToLogsOnError
+        volumeMounts:
+        - mountPath: /etc/kubernetes
+          name: service-account-kubeconfig
+      initContainers:
+      - command:
+        - /usr/bin/control-plane-operator
+        - availability-prober
+        - --target
+        - https://kube-apiserver:6443/readyz
+        - --kubeconfig=/var/kubeconfig/kubeconfig
+        - --required-api=operator.openshift.io,v1,CloudCredential
+        - --wait-for-infrastructure-resource
+        image: quay.io/openshift/availability-prober:latest
+        imagePullPolicy: IfNotPresent
+        name: availability-prober
+        resources: {}
+        volumeMounts:
+        - mountPath: /var/kubeconfig
+          name: service-account-kubeconfig
+      priorityClassName: hypershift-control-plane
+      securityContext:
+        runAsUser: 1001
+      tolerations:
+      - effect: NoSchedule
+        key: hypershift.openshift.io/control-plane
+        operator: Equal
+        value: "true"
+      - effect: NoSchedule
+        key: hypershift.openshift.io/cluster
+        operator: Equal
+        value: test-namespace
+      volumes:
+      - name: service-account-kubeconfig
+        secret:
+          defaultMode: 416
+          secretName: cloud-credential-operator-kubeconfig
+status: {}