[release-4.9] Bug 2015503: User CA bundle sync controller

Dockerfile

@@ -5,7 +5,7 @@ RUN make build
 
 FROM registry.ci.openshift.org/ocp/4.9:base
 COPY --from=builder /go/src/github.com/openshift/cluster-cloud-controller-manager-operator/bin/cluster-controller-manager-operator .
-COPY --from=builder /go/src/github.com/openshift/cluster-cloud-controller-manager-operator/bin/cloud-config-sync-controller .
+COPY --from=builder /go/src/github.com/openshift/cluster-cloud-controller-manager-operator/bin/config-sync-controllers .
 COPY --from=builder /go/src/github.com/openshift/cluster-cloud-controller-manager-operator/bin/azure-config-credentials-injector .
 COPY --from=builder /go/src/github.com/openshift/cluster-cloud-controller-manager-operator/manifests manifests
 

Makefile

@@ -22,13 +22,13 @@ unit:
 	hack/unit-tests.sh
 
 # Build operator binaries
-build: operator cloud-config-sync-controller azure-config-credentials-injector
+build: operator config-sync-controllers azure-config-credentials-injector
 
 operator:
 	go build -o bin/cluster-controller-manager-operator cmd/cluster-cloud-controller-manager-operator/main.go
 
-cloud-config-sync-controller:
-	go build -o bin/cloud-config-sync-controller cmd/cloud-config-sync-controller/main.go
+config-sync-controllers:
+	go build -o bin/config-sync-controllers cmd/config-sync-controllers/main.go
 
 azure-config-credentials-injector:
 	go build -o bin/azure-config-credentials-injector cmd/azure-config-credentials-injector/main.go

cmd/cloud-config-sync-controller/main.go → cmd/config-sync-controllers/main.go

@@ -91,7 +91,7 @@ func main() {
 	options.BindLeaderElectionFlags(&leaderElectionConfig, pflag.CommandLine)
 	pflag.Parse()
 
-	ctrl.SetLogger(klogr.New().WithName("CCMOCloudConfigSyncController"))
+	ctrl.SetLogger(klogr.New().WithName("CCCMOConfigSyncControllers"))
 
 	syncPeriod := 10 * time.Minute
 	cacheBuilder := cache.MultiNamespacedCacheBuilder([]string{
@@ -119,12 +119,22 @@ func main() {
 	if err = (&controllers.CloudConfigReconciler{
 		Client:          mgr.GetClient(),
 		Scheme:          mgr.GetScheme(),
-		Recorder:        mgr.GetEventRecorderFor("cloud-controller-manager-operator-config-sync-controller"),
+		Recorder:        mgr.GetEventRecorderFor("cloud-controller-manager-operator-cloud-config-sync-controller"),
 		TargetNamespace: *managedNamespace,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create cloud-config sync controller", "controller", "ClusterOperator")
 		os.Exit(1)
 	}
+
+	if err = (&controllers.TrustedCABundleReconciler{
+		Client:          mgr.GetClient(),
+		Scheme:          mgr.GetScheme(),
+		Recorder:        mgr.GetEventRecorderFor("cloud-controller-manager-operator-ca-sync-controller"),
+		TargetNamespace: *managedNamespace,
+	}).SetupWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to create Trusted CA sync controller", "controller", "ClusterOperator")
+		os.Exit(1)
+	}
 	// +kubebuilder:scaffold:builder
 
 	if err := mgr.AddHealthzCheck("health", healthz.Ping); err != nil {

docs/dev/cloud-config-sync.md

@@ -10,7 +10,7 @@ There are two places where this ConfigMap is stored on a running cluster at the
 1. `kube-cloud-config` ConfigMap in `openshift-config-managed` namespace.
 2. ConfigMap with an arbitrary name in `openshift-config` namespace. Such name might be taken from the `cluster` Infrastructure resource spec.
 
-This ConfigMap should be copied from one of the places described above and kept in sync within the CCCMO managed namespace for further mounting onto cloud provider pods. For such purposes `cloud-config-sync-controller` has been introduced as a [separate binary](https://github.com/openshift/cluster-cloud-controller-manager-operator/pull/86) in CCCMO pod.
+This ConfigMap should be copied from one of the places described above and kept in sync within the CCCMO managed namespace for further mounting onto cloud provider pods. For such purposes `config-sync-controllers` has been introduced as a [separate binary](https://github.com/openshift/cluster-cloud-controller-manager-operator/pull/86) in CCCMO pod.
 
 ## Implementation Description
 
@@ -26,3 +26,7 @@ If `openshift-config-managed/kube-cloud-config` does not exists - the controller
 ## Links
 - [library-go implementation](https://github.com/openshift/library-go/blob/master/pkg/operator/configobserver/cloudprovider/observe_cloudprovider.go#L82)
 - [cluster-config-operator repository](https://github.com/openshift/cluster-config-operator)
+
+## Notable changes
+
+- 10.11.2021: `cloud-config-sync-controller` binary was renamed to `config-sync-controllers`

docs/dev/trusted_ca_bundle_sync.md

@@ -0,0 +1,45 @@
+# Managing trusted ca for CCMs
+
+## Intro
+
+On some, mainly on-prem platforms, such as OpenStack, vSphere, or Azure Stack, 
+privately or self-signed SSL certificates might be used for its endpoints.
+
+In order to be able to communicate with the underlying platform, CA certificates should be
+propagated into CCMs pods trust store.
+
+Openshift documentation [prescribes](https://docs.openshift.com/container-platform/4.8/networking/configuring-a-custom-pki.html)
+to use cluster-wide **_Proxy_** object as a common way to configure trusted CA for an entire cluster.
+
+All necessary logic around `trustedCA` management already exists in Openshift platform, but can not be leveraged by CCM/CCCMO
+in its current state due to CCCMO's role in the cluster bootstrap process.
+All necessary steps are performing by [cluster-network-operator](https://github.com/openshift/cluster-network-operator/), only once control plane nodes have been initialized by the CCM.
+Such node initialization might require communication with the underlying platform (OpenStack, vSphere, ASH),
+which would not be successful without configured trust to platform endpoint certificates.
+
+For solving this 'chicken-and-egg' problem with a minimum amount of risk, a separate `trusted_ca_bundle_controller` was introduced.
+
+Moreover, [historically](https://github.com/openshift/installer/pull/5251) `additionalTrustBundle` from installer-config
+does not always end up in the **_Proxy_** object, instead this CA goes to cloud-config for future consuming by cloud provider.
+
+## Implementation Description
+
+Implementation mostly replicates logic from `cluster-network-operator`.
+
+The controller has been [introduced](https://github.com/openshift/cluster-cloud-controller-manager-operator/pull/136) as a part of `config-sync-controllers` binary in CCCMO pod and lives as separate control loop along with [cloud-config-sync](cloud-config-sync.md) controller. 
+
+The controller performs sync and merges CA from user defined ConfigMap
+(located in `openshift-config` and referenced by cluster scoped Proxy resource) and `ca-bundle.pem` key of [synced
+cloud-config configmap](cloud-config-sync.md) with the system bundle.
+Merged CA bundle will be written to `ccm-trusted-ca` ConfigMap in `openshift-cloud-controller-manager` namespace and intended to be mounted in all CCM pods.
+
+Top-level overview:
+- In case when Proxy resource contains the `trustedCA` parameter in its spec, user's CA will be taken from a config map with a name specified by `trustedCA` parameter.
+- In case if `ca-bundle.pem` key is presented in `cloud-config` ConfigMap within CCMs namespace, it would be added to merged CA as well.
+- In case if Proxy resource does not contain the `trustedCA` parameter, CA bundle from `cloud-config` pod will be used along with system one.
+- In case if user defined CAs is invalid (PEM can not be parsed, ConfigMap format is unexpected) or not presented only the system bundle from the CCCMO pod will be used
+
+# Links
+- [cluster-network-operator implementation](https://github.com/openshift/cluster-network-operator/blob/master/pkg/controller/proxyconfig/controller.go#L91)
+- [related openshift documentation](https://docs.openshift.com/container-platform/4.8/networking/configuring-a-custom-pki.html)
+- [installer part with cloud-config shaping](https://github.com/openshift/installer/blob/master/pkg/asset/manifests/cloudproviderconfig.go#L99)

manifests/0000_26_cloud-controller-manager-operator_11_deployment.yaml

@@ -58,7 +58,7 @@ spec:
         - mountPath: /etc/kubernetes
           name: host-etc-kube
           readOnly: true
-      - name: cloud-config-sync-controller
+      - name: config-sync-controllers
         image: quay.io/openshift/origin-cluster-cloud-controller-manager-operator
         command:
           - /bin/bash
@@ -71,7 +71,7 @@ spec:
             else
               URL_ONLY_KUBECONFIG=/etc/kubernetes/kubeconfig
             fi
-            exec /cloud-config-sync-controller \
+            exec /config-sync-controllers \
             --leader-elect=true \
             --leader-elect-lease-duration=137s \
             --leader-elect-renew-deadline=107s \
@@ -185,7 +185,7 @@ spec:
         - mountPath: /etc/kubernetes
           name: host-etc-kube
           readOnly: true
-      - name: cloud-config-sync-controller
+      - name: config-sync-controllers
         image: quay.io/openshift/origin-cluster-cloud-controller-manager-operator
         command:
           - /bin/bash
@@ -196,7 +196,7 @@ spec:
             if [[ -f /etc/kubernetes/apiserver-url.env ]]; then
               source /etc/kubernetes/apiserver-url.env
             fi
-            exec /cloud-config-sync-controller \
+            exec /config-sync-controllers \
             --leader-elect \
             --metrics-bind-address=:8081 \
             --health-addr=:9441

pkg/cloud/aws/assets/deployment.yaml

@@ -55,6 +55,9 @@ spec:
         - mountPath: /etc/kubernetes
           name: host-etc-kube
           readOnly: true
+        - name: trusted-ca
+          mountPath: /etc/pki/ca-trust/extracted/pem
+          readOnly: true
       hostNetwork: true
       nodeSelector:
         node-role.kubernetes.io/master: ""
@@ -85,6 +88,12 @@ spec:
         key: node.kubernetes.io/not-ready
         operator: Exists
       volumes:
+      - name: trusted-ca
+        configMap:
+          name: ccm-trusted-ca
+          items:
+            - key: ca-bundle.crt
+              path: tls-ca-bundle.pem
       - name: host-etc-kube
         hostPath:
           path: /etc/kubernetes

pkg/cloud/azure/assets/cloud-controller-manager-deployment.yaml

@@ -93,13 +93,22 @@ spec:
             - name: config-accm
               mountPath: /etc/kubernetes-cloud-config
               readOnly: true
+            - name: trusted-ca
+              mountPath: /etc/pki/ca-trust/extracted/pem
+              readOnly: true
       volumes:
         - name: config-accm
           configMap:
             name: cloud-conf
             items:
               - key: cloud.conf
                 path: cloud.conf
+        - name: trusted-ca
+          configMap:
+            name: ccm-trusted-ca
+            items:
+              - key: ca-bundle.crt
+                path: tls-ca-bundle.pem
         - name: host-etc-kube
           hostPath:
             path: /etc/kubernetes

pkg/cloud/azure/assets/cloud-node-manager-daemonset.yaml

@@ -73,11 +73,20 @@ spec:
             - name: host-etc-kube
               mountPath: /etc/kubernetes
               readOnly: true
+            - name: trusted-ca
+              mountPath: /etc/pki/ca-trust/extracted/pem
+              readOnly: true
           resources:
             requests:
               cpu: 50m
               memory: 50Mi
       volumes:
+        - name: trusted-ca
+          configMap:
+            name: ccm-trusted-ca
+            items:
+              - key: ca-bundle.crt
+                path: tls-ca-bundle.pem
         - name: host-etc-kube
           hostPath:
             path: /etc/kubernetes

pkg/cloud/azurestack/assets/cloud-controller-manager-deployment.yaml

@@ -123,6 +123,9 @@ spec:
             - name: cloud-config
               mountPath: /etc/cloud-config
               readOnly: true
+            - name: trusted-ca
+              mountPath: /etc/pki/ca-trust/extracted/pem
+              readOnly: true
       volumes:
         - name: config-accm
           configMap:
@@ -136,5 +139,11 @@ spec:
           hostPath:
             path: /etc/kubernetes
             type: Directory
+        - name: trusted-ca
+          configMap:
+            name: ccm-trusted-ca
+            items:
+              - key: ca-bundle.crt
+                path: tls-ca-bundle.pem
         - name: cloud-config
           emptyDir: {}

pkg/cloud/azurestack/assets/cloud-node-manager-daemonset.yaml

@@ -111,6 +111,9 @@ spec:
             - name: cloud-config
               mountPath: /etc/cloud-config
               readOnly: true
+            - name: trusted-ca
+              mountPath: /etc/pki/ca-trust/extracted/pem
+              readOnly: true
           resources:
             requests:
               cpu: 50m
@@ -128,5 +131,11 @@ spec:
                 path: cloud.conf
               - key: endpoints
                 path: endpoints.conf
+        - name: trusted-ca
+          configMap:
+            name: ccm-trusted-ca
+            items:
+              - key: ca-bundle.crt
+                path: tls-ca-bundle.pem
         - name: cloud-config
           emptyDir: {}

pkg/cloud/cloud_test.go

@@ -157,6 +157,7 @@ func TestPodSpec(t *testing.T) {
 				checkResourceRunsBeforeCNI(t, podSpec)
 				checkLeaderElection(t, podSpec)
 				checkCloudControllerManagerFlags(t, podSpec)
+				checkTrustedCAMounted(t, podSpec)
 			}
 		})
 	}
@@ -422,3 +423,22 @@ func checkDeploymentStrategy(t *testing.T, strategy appsv1.DeploymentStrategy) {
 		t.Errorf("Deployment should set strategy type to \"Recreate\"")
 	}
 }
+
+func checkTrustedCAMounted(t *testing.T, podSpec corev1.PodSpec) {
+	trustedCAVolume := corev1.Volume{
+		Name: "trusted-ca",
+		VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{
+			LocalObjectReference: corev1.LocalObjectReference{Name: "ccm-trusted-ca"},
+			Items:                []corev1.KeyToPath{{Key: "ca-bundle.crt", Path: "tls-ca-bundle.pem"}},
+		}},
+	}
+	trustedCAVolumeMount := corev1.VolumeMount{
+		MountPath: "/etc/pki/ca-trust/extracted/pem",
+		Name:      "trusted-ca",
+		ReadOnly:  true,
+	}
+	assert.Contains(t, podSpec.Volumes, trustedCAVolume, "PodSpec %s volumes should contain trusted-ca volume")
+	for _, c := range podSpec.Containers {
+		assert.Contains(t, c.VolumeMounts, trustedCAVolumeMount, "Container VolumeMounts should contain trusted ca volume mount")
+	}
+}

pkg/cloud/openstack/assets/deployment.yaml

@@ -89,6 +89,9 @@ spec:
             - name: config-occm
               mountPath: /etc/openstack/config
               readOnly: true
+            - name: trusted-ca
+              mountPath: /etc/pki/ca-trust/extracted/pem
+              readOnly: true
       volumes:
         - name: host-etc-kube
           hostPath:
@@ -100,6 +103,12 @@ spec:
             items:
               - key: clouds.yaml
                 path: clouds.yaml
+        - name: trusted-ca
+          configMap:
+            name: ccm-trusted-ca
+            items:
+              - key: ca-bundle.crt
+                path: tls-ca-bundle.pem
         - name: config-occm
           configMap:
             name: openstack-cloud-controller-manager-config

pkg/controllers/cloud_config_sync_controller.go

@@ -22,8 +22,7 @@ import (
 const (
 	managedCloudConfigMapName = "kube-cloud-config"
 
-	cloudConfigMapName = "cloud-conf"
-	defaultConfigKey   = "cloud.conf"
+	defaultConfigKey = "cloud.conf"
 )
 
 type CloudConfigReconciler struct {
@@ -68,7 +67,7 @@ func (r *CloudConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	targetCM := &corev1.ConfigMap{}
 	targetConfigMapKey := client.ObjectKey{
 		Namespace: r.TargetNamespace,
-		Name:      cloudConfigMapName,
+		Name:      syncedCloudConfigMapName,
 	}
 
 	// If the config does not exist, it will be created later, so we can ignore a Not Found error
@@ -116,7 +115,7 @@ func (r *CloudConfigReconciler) isCloudConfigEqual(source *corev1.ConfigMap, tar
 }
 
 func (r *CloudConfigReconciler) syncCloudConfigData(ctx context.Context, source *corev1.ConfigMap, target *corev1.ConfigMap) error {
-	target.SetName(cloudConfigMapName)
+	target.SetName(syncedCloudConfigMapName)
 	target.SetNamespace(r.TargetNamespace)
 	target.Data = source.Data
 	target.BinaryData = source.BinaryData