Bug 2015493: [OCPCLOUD-1306] User CA bundle sync controller

Dockerfile

@@ -5,7 +5,7 @@ RUN make build
 
 FROM registry.ci.openshift.org/ocp/4.10:base
 COPY --from=builder /go/src/github.com/openshift/cluster-cloud-controller-manager-operator/bin/cluster-controller-manager-operator .
-COPY --from=builder /go/src/github.com/openshift/cluster-cloud-controller-manager-operator/bin/cloud-config-sync-controller .
+COPY --from=builder /go/src/github.com/openshift/cluster-cloud-controller-manager-operator/bin/config-sync-controllers .
 COPY --from=builder /go/src/github.com/openshift/cluster-cloud-controller-manager-operator/bin/azure-config-credentials-injector .
 COPY --from=builder /go/src/github.com/openshift/cluster-cloud-controller-manager-operator/manifests manifests
 

Makefile

@@ -26,13 +26,13 @@ unit:
 	hack/unit-tests.sh
 
 # Build operator binaries
-build: operator cloud-config-sync-controller azure-config-credentials-injector
+build: operator config-sync-controllers azure-config-credentials-injector
 
 operator:
 	go build -o bin/cluster-controller-manager-operator cmd/cluster-cloud-controller-manager-operator/main.go
 
-cloud-config-sync-controller:
-	go build -o bin/cloud-config-sync-controller cmd/cloud-config-sync-controller/main.go
+config-sync-controllers:
+	go build -o bin/config-sync-controllers cmd/config-sync-controllers/main.go
 
 azure-config-credentials-injector:
 	go build -o bin/azure-config-credentials-injector cmd/azure-config-credentials-injector/main.go

cmd/cloud-config-sync-controller/main.go → cmd/config-sync-controllers/main.go

@@ -92,7 +92,7 @@ func main() {
 	options.BindLeaderElectionFlags(&leaderElectionConfig, pflag.CommandLine)
 	pflag.Parse()
 
-	ctrl.SetLogger(klogr.New().WithName("CCMOCloudConfigSyncController"))
+	ctrl.SetLogger(klogr.New().WithName("CCCMOConfigSyncControllers"))
 
 	syncPeriod := 10 * time.Minute
 	cacheBuilder := cache.MultiNamespacedCacheBuilder([]string{
@@ -120,12 +120,22 @@ func main() {
 	if err = (&controllers.CloudConfigReconciler{
 		Client:          mgr.GetClient(),
 		Scheme:          mgr.GetScheme(),
-		Recorder:        mgr.GetEventRecorderFor("cloud-controller-manager-operator-config-sync-controller"),
+		Recorder:        mgr.GetEventRecorderFor("cloud-controller-manager-operator-cloud-config-sync-controller"),
 		TargetNamespace: *managedNamespace,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create cloud-config sync controller", "controller", "ClusterOperator")
 		os.Exit(1)
 	}
+
+	if err = (&controllers.TrustedCABundleReconciler{
+		Client:          mgr.GetClient(),
+		Scheme:          mgr.GetScheme(),
+		Recorder:        mgr.GetEventRecorderFor("cloud-controller-manager-operator-ca-sync-controller"),
+		TargetNamespace: *managedNamespace,
+	}).SetupWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to create Trusted CA sync controller", "controller", "ClusterOperator")
+		os.Exit(1)
+	}
 	// +kubebuilder:scaffold:builder
 
 	if err := mgr.AddHealthzCheck("health", healthz.Ping); err != nil {

docs/dev/cloud-config-sync.md

@@ -10,7 +10,7 @@ There are two places where this ConfigMap is stored on a running cluster at the
 1. `kube-cloud-config` ConfigMap in `openshift-config-managed` namespace.
 2. ConfigMap with an arbitrary name in `openshift-config` namespace. Such name might be taken from the `cluster` Infrastructure resource spec.
 
-This ConfigMap should be copied from one of the places described above and kept in sync within the CCCMO managed namespace for further mounting onto cloud provider pods. For such purposes `cloud-config-sync-controller` has been introduced as a [separate binary](https://github.com/openshift/cluster-cloud-controller-manager-operator/pull/86) in CCCMO pod.
+This ConfigMap should be copied from one of the places described above and kept in sync within the CCCMO managed namespace for further mounting onto cloud provider pods. For such purposes `config-sync-controllers` has been introduced as a [separate binary](https://github.com/openshift/cluster-cloud-controller-manager-operator/pull/86) in CCCMO pod.
 
 ## Implementation Description
 
@@ -26,3 +26,7 @@ If `openshift-config-managed/kube-cloud-config` does not exists - the controller
 ## Links
 - [library-go implementation](https://github.com/openshift/library-go/blob/master/pkg/operator/configobserver/cloudprovider/observe_cloudprovider.go#L82)
 - [cluster-config-operator repository](https://github.com/openshift/cluster-config-operator)
+
+## Notable changes
+
+- 10.11.2021: `cloud-config-sync-controller` binary was renamed to `config-sync-controllers`

docs/dev/trusted_ca_bundle_sync.md

@@ -0,0 +1,39 @@
+# Managing trusted ca for CCMs
+
+## Intro
+
+On some, mainly on-prem platforms, such as OpenStack, vSphere, or Azure Stack, 
+privately or self-signed SSL certificates might be used for its endpoints.
+
+In order to be able to communicate with the underlying platform, CA certificates should be
+propagated into CCMs pods trust store.
+
+Openshift documentation [prescribes](https://docs.openshift.com/container-platform/4.8/networking/configuring-a-custom-pki.html)
+to use cluster-wide **_Proxy_** object as a common way to configure trusted CA for an entire cluster.
+
+All necessary logic around `trustedCA` management already exists in Openshift platform, but can not be leveraged by CCM/CCCMO
+in its current state due to CCCMO's role in the cluster bootstrap process.
+All necessary steps are performing by [cluster-network-operator](https://github.com/openshift/cluster-network-operator/), only once control plane nodes have been initialized by the CCM.
+Such node initialization might require communication with the underlying platform (OpenStack, vSphere, ASH),
+which would not be successful without configured trust to platform endpoint certificates.
+
+For solving this 'chicken-and-egg' problem with a minimum amount of risk, a separate `trusted_ca_bundle_controller` was introduced.
+
+## Implementation Description
+
+Implementation mostly replicates logic from `cluster-network-operator`.
+
+The controller has been [introduced](https://github.com/openshift/cluster-cloud-controller-manager-operator/pull/136) as a part of `config-sync-controllers` binary in CCCMO pod and lives as separate control loop along with [cloud-config-sync](cloud-config-sync.md) controller. 
+
+The controller performs sync and merges CA from user defined ConfigMap
+(located in `openshift-config` and referenced by cluster scoped Proxy resource) with system bundle.
+Merged CA bundle will be written to `ccm-trusted-ca` ConfigMap in `openshift-cloud-controller-manager` namespace and intended to be mounted in all CCM pods.
+
+Top-level overview:
+- In case when Proxy resource contains the `trustedCA` parameter in its spec, user's CA will be taken from a config map with a name specified by `trustedCA` parameter.
+- In case if Proxy resource does not contain the `trustedCA` parameter, only the system bundle from the CCCMO pod will be used.
+- In case if user defined CA is invalid (PEM can not be parsed, ConfigMap format is unexpected) only the system bundle from the CCCMO pod will be used
+
+# Links
+- [cluster-network-operator implementation](https://github.com/openshift/cluster-network-operator/blob/master/pkg/controller/proxyconfig/controller.go#L91)
+- [related openshift documentation](https://docs.openshift.com/container-platform/4.8/networking/configuring-a-custom-pki.html)

manifests/0000_26_cloud-controller-manager-operator_11_deployment.yaml

@@ -58,7 +58,7 @@ spec:
         - mountPath: /etc/kubernetes
           name: host-etc-kube
           readOnly: true
-      - name: cloud-config-sync-controller
+      - name: config-sync-controllers
         image: quay.io/openshift/origin-cluster-cloud-controller-manager-operator
         command:
           - /bin/bash
@@ -71,7 +71,7 @@ spec:
             else
               URL_ONLY_KUBECONFIG=/etc/kubernetes/kubeconfig
             fi
-            exec /cloud-config-sync-controller \
+            exec /config-sync-controllers \
             --leader-elect=true \
             --leader-elect-lease-duration=137s \
             --leader-elect-renew-deadline=107s \
@@ -185,7 +185,7 @@ spec:
         - mountPath: /etc/kubernetes
           name: host-etc-kube
           readOnly: true
-      - name: cloud-config-sync-controller
+      - name: config-sync-controllers
         image: quay.io/openshift/origin-cluster-cloud-controller-manager-operator
         command:
           - /bin/bash
@@ -196,7 +196,7 @@ spec:
             if [[ -f /etc/kubernetes/apiserver-url.env ]]; then
               source /etc/kubernetes/apiserver-url.env
             fi
-            exec /cloud-config-sync-controller \
+            exec /config-sync-controllers \
             --leader-elect \
             --metrics-bind-address=:8081 \
             --health-addr=:9441

pkg/cloud/aws/assets/deployment.yaml

@@ -55,6 +55,9 @@ spec:
         - mountPath: /etc/kubernetes
           name: host-etc-kube
           readOnly: true
+        - name: trusted-ca
+          mountPath: /etc/pki/ca-trust/extracted/pem
+          readOnly: true
       hostNetwork: true
       nodeSelector:
         node-role.kubernetes.io/master: ""
@@ -85,6 +88,12 @@ spec:
         key: node.kubernetes.io/not-ready
         operator: Exists
       volumes:
+      - name: trusted-ca
+        configMap:
+          name: ccm-trusted-ca
+          items:
+            - key: ca-bundle.crt
+              path: tls-ca-bundle.pem
       - name: host-etc-kube
         hostPath:
           path: /etc/kubernetes

pkg/cloud/azure/assets/cloud-controller-manager-deployment.yaml

@@ -93,13 +93,22 @@ spec:
             - name: config-accm
               mountPath: /etc/kubernetes-cloud-config
               readOnly: true
+            - name: trusted-ca
+              mountPath: /etc/pki/ca-trust/extracted/pem
+              readOnly: true
       volumes:
         - name: config-accm
           configMap:
             name: cloud-conf
             items:
               - key: cloud.conf
                 path: cloud.conf
+        - name: trusted-ca
+          configMap:
+            name: ccm-trusted-ca
+            items:
+              - key: ca-bundle.crt
+                path: tls-ca-bundle.pem
         - name: host-etc-kube
           hostPath:
             path: /etc/kubernetes

pkg/cloud/azure/assets/cloud-node-manager-daemonset.yaml

@@ -73,11 +73,20 @@ spec:
             - name: host-etc-kube
               mountPath: /etc/kubernetes
               readOnly: true
+            - name: trusted-ca
+              mountPath: /etc/pki/ca-trust/extracted/pem
+              readOnly: true
           resources:
             requests:
               cpu: 50m
               memory: 50Mi
       volumes:
+        - name: trusted-ca
+          configMap:
+            name: ccm-trusted-ca
+            items:
+              - key: ca-bundle.crt
+                path: tls-ca-bundle.pem
         - name: host-etc-kube
           hostPath:
             path: /etc/kubernetes

pkg/cloud/azurestack/assets/cloud-controller-manager-deployment.yaml

@@ -123,6 +123,9 @@ spec:
             - name: cloud-config
               mountPath: /etc/cloud-config
               readOnly: true
+            - name: trusted-ca
+              mountPath: /etc/pki/ca-trust/extracted/pem
+              readOnly: true
       volumes:
         - name: config-accm
           configMap:
@@ -136,5 +139,11 @@ spec:
           hostPath:
             path: /etc/kubernetes
             type: Directory
+        - name: trusted-ca
+          configMap:
+            name: ccm-trusted-ca
+            items:
+              - key: ca-bundle.crt
+                path: tls-ca-bundle.pem
         - name: cloud-config
           emptyDir: {}

pkg/cloud/azurestack/assets/cloud-node-manager-daemonset.yaml

@@ -111,6 +111,9 @@ spec:
             - name: cloud-config
               mountPath: /etc/cloud-config
               readOnly: true
+            - name: trusted-ca
+              mountPath: /etc/pki/ca-trust/extracted/pem
+              readOnly: true
           resources:
             requests:
               cpu: 50m
@@ -128,5 +131,11 @@ spec:
                 path: cloud.conf
               - key: endpoints
                 path: endpoints.conf
+        - name: trusted-ca
+          configMap:
+            name: ccm-trusted-ca
+            items:
+              - key: ca-bundle.crt
+                path: tls-ca-bundle.pem
         - name: cloud-config
           emptyDir: {}

pkg/cloud/cloud_test.go

@@ -213,6 +213,7 @@ func TestPodSpec(t *testing.T) {
 				checkResourceRunsBeforeCNI(t, podSpec)
 				checkLeaderElection(t, podSpec)
 				checkCloudControllerManagerFlags(t, podSpec)
+				checkTrustedCAMounted(t, podSpec)
 			}
 		})
 	}
@@ -441,3 +442,22 @@ func checkDeploymentStrategy(t *testing.T, strategy appsv1.DeploymentStrategy) {
 		t.Errorf("Deployment should set strategy type to \"Recreate\"")
 	}
 }
+
+func checkTrustedCAMounted(t *testing.T, podSpec corev1.PodSpec) {
+	trustedCAVolume := corev1.Volume{
+		Name: "trusted-ca",
+		VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{
+			LocalObjectReference: corev1.LocalObjectReference{Name: "ccm-trusted-ca"},
+			Items:                []corev1.KeyToPath{{Key: "ca-bundle.crt", Path: "tls-ca-bundle.pem"}},
+		}},
+	}
+	trustedCAVolumeMount := corev1.VolumeMount{
+		MountPath: "/etc/pki/ca-trust/extracted/pem",
+		Name:      "trusted-ca",
+		ReadOnly:  true,
+	}
+	assert.Contains(t, podSpec.Volumes, trustedCAVolume, "PodSpec %s volumes should contain trusted-ca volume")
+	for _, c := range podSpec.Containers {
+		assert.Contains(t, c.VolumeMounts, trustedCAVolumeMount, "Container VolumeMounts should contain trusted ca volume mount")
+	}
+}

pkg/cloud/ibm/assets/deployment.yaml

@@ -109,7 +109,16 @@ spec:
             name: cloud-conf
           - mountPath: /etc/vpc
             name: ibm-cloud-credentials
+          - name: trusted-ca
+            mountPath: /etc/pki/ca-trust/extracted/pem
+            readOnly: true
       volumes:
+        - name: trusted-ca
+          configMap:
+            name: ccm-trusted-ca
+            items:
+              - key: ca-bundle.crt
+                path: tls-ca-bundle.pem
         - name: host-etc-kube
           hostPath:
             path: /etc/kubernetes

pkg/cloud/openstack/assets/deployment.yaml

@@ -89,6 +89,9 @@ spec:
             - name: config-occm
               mountPath: /etc/openstack/config
               readOnly: true
+            - name: trusted-ca
+              mountPath: /etc/pki/ca-trust/extracted/pem
+              readOnly: true
       volumes:
         - name: host-etc-kube
           hostPath:
@@ -100,6 +103,12 @@ spec:
             items:
               - key: clouds.yaml
                 path: clouds.yaml
+        - name: trusted-ca
+          configMap:
+            name: ccm-trusted-ca
+            items:
+              - key: ca-bundle.crt
+                path: tls-ca-bundle.pem
         - name: config-occm
           configMap:
             name: openstack-cloud-controller-manager-config

pkg/controllers/clusteroperator_controller.go

@@ -40,7 +40,6 @@ import (
 
 const (
 	externalFeatureGateName = "cluster"
-	proxyResourceName       = "cluster"
 	kcmResourceName         = "cluster"
 
 	// Condition type for Cloud Controller ownership

pkg/controllers/common_consts.go

@@ -7,4 +7,6 @@ const (
 
 	OpenshiftConfigNamespace        = "openshift-config"
 	OpenshiftManagedConfigNamespace = "openshift-config-managed"
+
+	proxyResourceName = "cluster"
 )

pkg/controllers/fixtures/additional_ca_amazon.pem

@@ -0,0 +1,14 @@
+# Random amazon cert from fedora trust store
+# Amazon Root CA 3
+-----BEGIN CERTIFICATE-----
+MIIBtjCCAVugAwIBAgITBmyf1XSXNmY/Owua2eiedgPySjAKBggqhkjOPQQDAjA5
+MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6b24g
+Um9vdCBDQSAzMB4XDTE1MDUyNjAwMDAwMFoXDTQwMDUyNjAwMDAwMFowOTELMAkG
+A1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZMBcGA1UEAxMQQW1hem9uIFJvb3Qg
+Q0EgMzBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABCmXp8ZBf8ANm+gBG1bG8lKl
+ui2yEujSLtf6ycXYqm0fc4E7O5hrOXwzpcVOho6AF2hiRVd9RFgdszflZwjrZt6j
+QjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGGMB0GA1UdDgQWBBSr
+ttvXBp43rDCGB5Fwx5zEGbF4wDAKBggqhkjOPQQDAgNJADBGAiEA4IWSoxe3jfkr
+BqWTrBqYaGFy+uGh0PsceGCmQ5nFuMQCIQCcAu/xlJyzlvnrxir4tiz+OpAUFteM
+YyRIHN8wfdVoOw==
+-----END CERTIFICATE-----

pkg/controllers/fixtures/additional_ca_ms.pem

@@ -0,0 +1,17 @@
+# Random ms cert from fedora trust store
+# Microsoft ECC Root Certificate Authority 2017
+-----BEGIN CERTIFICATE-----
+MIICWTCCAd+gAwIBAgIQZvI9r4fei7FK6gxXMQHC7DAKBggqhkjOPQQDAzBlMQsw
+CQYDVQQGEwJVUzEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMTYwNAYD
+VQQDEy1NaWNyb3NvZnQgRUNDIFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5IDIw
+MTcwHhcNMTkxMjE4MjMwNjQ1WhcNNDIwNzE4MjMxNjA0WjBlMQswCQYDVQQGEwJV
+UzEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMTYwNAYDVQQDEy1NaWNy
+b3NvZnQgRUNDIFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5IDIwMTcwdjAQBgcq
+hkjOPQIBBgUrgQQAIgNiAATUvD0CQnVBEyPNgASGAlEvaqiBYgtlzPbKnR5vSmZR
+ogPZnZH6thaxjG7efM3beaYvzrvOcS/lpaso7GMEZpn4+vKTEAXhgShC48Zo9OYb
+hGBKia/teQ87zvH2RPUBeMCjVDBSMA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8E
+BTADAQH/MB0GA1UdDgQWBBTIy5lycFIM+Oa+sgRXKSrPQhDtNTAQBgkrBgEEAYI3
+FQEEAwIBADAKBggqhkjOPQQDAwNoADBlAjBY8k3qDPlfXu5gKcs68tvWMoQZP3zV
+L8KxzJOuULsJMsbG7X7JNpQS5GiFBqIb0C8CMQCZ6Ra0DvpWSNSkMBaReNtUjGUB
+iudQZsIxtzm6uBoiB078a1QWIP8rtedMDE2mT3M=
+-----END CERTIFICATE-----

pkg/controllers/fixtures/trust_bundle_invalid.pem

@@ -0,0 +1 @@
+this one is broken