Bug 1788712: publish a router-ca that can be used to verify routes in golang clients

pkg/operator/controller/certificate/controller.go

@@ -21,6 +21,7 @@ import (
 	"k8s.io/client-go/tools/record"
 
 	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
 
 	operatorv1 "github.com/openshift/api/operator/v1"
 
@@ -109,12 +110,32 @@ func (r *reconciler) Reconcile(request reconcile.Request) (reconcile.Result, err
 		}
 	}
 
+	// This section creates the legacy router-ca configmap which only ever contains the operator-managed default signing
+	// cert used for signing the default wildcard serving cert
 	ingresses := &operatorv1.IngressControllerList{}
 	if err := r.cache.List(context.TODO(), ingresses, client.InNamespace(r.operatorNamespace)); err != nil {
 		errs = append(errs, fmt.Errorf("failed to list ingresscontrollers: %v", err))
 	} else if err := r.ensureRouterCAConfigMap(ca, ingresses.Items); err != nil {
 		errs = append(errs, fmt.Errorf("failed to publish router CA: %v", err))
 	}
 
+	// We need to construct the CA bundle that can be used to verify the ingress used to serve the console and the oauth-server.
+	// In an operator maintained cluster, this is always `oc get -n openshift-ingress-operator ingresscontroller/default`, skip the rest and return here.
+	// TODO if network-edge wishes to expand the scope of the CA bundle (and you could legitimately see a need/desire to have one CA that verifies all ingress traffic).
+	// TODO this could be accomplished using union logic similar to the kube-apiserver's join of multiple CAs.
+	if ingress == nil || ingress.Namespace != "openshift-ingress-operator" || ingress.Name != "default" {
+		return result, utilerrors.NewAggregate(errs)
+	}
+
+	wildcardServingCertKeySecret := &corev1.Secret{}
+	if err := r.client.Get(context.TODO(), controller.RouterEffectiveDefaultCertificateSecretName(ingress, "openshift-ingress"), wildcardServingCertKeySecret); err != nil {
+		errs = append(errs, fmt.Errorf("failed to lookup wildcard cert: %v", err))
+		return result, utilerrors.NewAggregate(errs)
+	}
+	caBundle := string(wildcardServingCertKeySecret.Data["tls.crt"])
+	if err := r.ensureDefaultIngressCertConfigMap(caBundle); err != nil {
+		errs = append(errs, fmt.Errorf("failed to publish router CA: %v", err))
+	}
+
 	return result, utilerrors.NewAggregate(errs)
 }

pkg/operator/controller/certificate/publish_ca.go

@@ -10,16 +10,37 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 )
 
+// ensureDefaultIngressCertConfigMap will create or update the configmap containing the public half of the default ingress wildcard certificate
+func (r *reconciler) ensureDefaultIngressCertConfigMap(caBundle string) error {
+	name := controller.DefaultIngressCertConfigMapName()
+	desired := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name.Name,
+			Namespace: name.Namespace,
+		},
+		Data: map[string]string{
+			"ca-bundle.crt": caBundle,
+		},
+	}
+	return r.ensureConfigMap(name, desired)
+}
+
 // ensureRouterCAConfigMap will create, update, or delete the configmap for the
 // router CA as appropriate.
 func (r *reconciler) ensureRouterCAConfigMap(secret *corev1.Secret, ingresses []operatorv1.IngressController) error {
 	desired, err := desiredRouterCAConfigMap(secret, ingresses)
 	if err != nil {
 		return err
 	}
-	current, err := r.currentRouterCAConfigMap()
+	return r.ensureConfigMap(controller.RouterCAConfigMapName(), desired)
+}
+
+// ensureConfigMap will create, update, or delete the configmap as appropriate.
+func (r *reconciler) ensureConfigMap(name types.NamespacedName, desired *corev1.ConfigMap) error {
+	current, err := r.currentConfigMap(name)
 	if err != nil {
 		return err
 	}
@@ -28,25 +49,25 @@ func (r *reconciler) ensureRouterCAConfigMap(secret *corev1.Secret, ingresses []
 		// Nothing to do.
 	case desired == nil && current != nil:
 		if deleted, err := r.deleteRouterCAConfigMap(current); err != nil {
-			return fmt.Errorf("failed to ensure router CA was unpublished: %v", err)
+			return fmt.Errorf("failed to ensure %q in %q was unpublished: %v", name.Name, name.Namespace, err)
 		} else if deleted {
-			r.recorder.Eventf(current, "Normal", "UnpublishedDefaultRouterCA", "Unpublished default router CA")
+			r.recorder.Eventf(current, "Normal", "UnpublishedRouterCA", "Unpublished %q in %q", name.Name, name.Namespace)
 		}
 	case desired != nil && current == nil:
 		if created, err := r.createRouterCAConfigMap(desired); err != nil {
-			return fmt.Errorf("failed to ensure router CA was published: %v", err)
+			return fmt.Errorf("failed to ensure %q in %q was published: %v", desired.Name, desired.Namespace, err)
 		} else if created {
-			new, err := r.currentRouterCAConfigMap()
+			new, err := r.currentConfigMap(name)
 			if err != nil {
 				return err
 			}
-			r.recorder.Eventf(new, "Normal", "PublishedDefaultRouterCA", "Published default router CA")
+			r.recorder.Eventf(new, "Normal", "PublishedRouterCA", "Published %q in %q", desired.Name, desired.Namespace)
 		}
 	case desired != nil && current != nil:
 		if updated, err := r.updateRouterCAConfigMap(current, desired); err != nil {
-			return fmt.Errorf("failed to update published router CA: %v", err)
+			return fmt.Errorf("failed to update published %q in %q: %v", desired.Name, desired.Namespace, err)
 		} else if updated {
-			r.recorder.Eventf(current, "Normal", "UpdatedPublishedDefaultRouterCA", "Updated the published default router CA")
+			r.recorder.Eventf(current, "Normal", "UpdatedPublishedRouterCA", "Updated the published %q in %q", desired.Name, desired.Namespace)
 		}
 	}
 	return nil
@@ -82,9 +103,8 @@ func shouldPublishRouterCA(ingresses []operatorv1.IngressController) bool {
 	return false
 }
 
-// currentRouterCAConfigMap returns the current router CA configmap.
-func (r *reconciler) currentRouterCAConfigMap() (*corev1.ConfigMap, error) {
-	name := controller.RouterCAConfigMapName()
+// currentConfigMap returns the current state of the desired configmap namespace/name.
+func (r *reconciler) currentConfigMap(name types.NamespacedName) (*corev1.ConfigMap, error) {
 	cm := &corev1.ConfigMap{}
 	if err := r.client.Get(context.TODO(), name, cm); err != nil {
 		if errors.IsNotFound(err) {

pkg/operator/controller/names.go

@@ -63,6 +63,16 @@ func RouterCAConfigMapName() types.NamespacedName {
 	}
 }
 
+// DefaultIngressCertConfigMapName returns the namespaced name for the default ingress cert configmap.
+// The operator uses this configmap to publish the public key that golang clients can use to trust
+// the default ingress wildcard serving cert.
+func DefaultIngressCertConfigMapName() types.NamespacedName {
+	return types.NamespacedName{
+		Namespace: GlobalMachineSpecifiedConfigNamespace,
+		Name:      "default-ingress-cert",
+	}
+}
+
 // RouterCertsGlobalSecretName returns the namespaced name for the router certs
 // secret.  The operator uses this secret to publish the default certificates and
 // their keys, so that the authentication operator can configure the OAuth server

test/e2e/operator_test.go

@@ -192,8 +192,12 @@ func TestUpdateDefaultIngressController(t *testing.T) {
 		t.Fatalf("failed to observe expected conditions: %v", err)
 	}
 
-	configmap := &corev1.ConfigMap{}
-	if err := kclient.Get(context.TODO(), controller.RouterCAConfigMapName(), configmap); err != nil {
+	routerCAConfigmap := &corev1.ConfigMap{}
+	if err := kclient.Get(context.TODO(), controller.RouterCAConfigMapName(), routerCAConfigmap); err != nil {
+		t.Fatalf("failed to get CA certificate configmap: %v", err)
+	}
+	defaultIngressCAConfigmap := &corev1.ConfigMap{}
+	if err := kclient.Get(context.TODO(), controller.DefaultIngressCertConfigMapName(), defaultIngressCAConfigmap); err != nil {
 		t.Fatalf("failed to get CA certificate configmap: %v", err)
 	}
 
@@ -244,7 +248,7 @@ func TestUpdateDefaultIngressController(t *testing.T) {
 
 	// Wait for the CA certificate configmap to be deleted.
 	err = wait.PollImmediate(1*time.Second, 10*time.Second, func() (bool, error) {
-		if err := kclient.Get(context.TODO(), controller.RouterCAConfigMapName(), configmap); err != nil {
+		if err := kclient.Get(context.TODO(), controller.RouterCAConfigMapName(), routerCAConfigmap); err != nil {
 			if errors.IsNotFound(err) {
 				return true, nil
 			}
@@ -255,6 +259,20 @@ func TestUpdateDefaultIngressController(t *testing.T) {
 	if err != nil {
 		t.Fatalf("failed to observe clean-up of CA certificate configmap: %v", err)
 	}
+	// Wait for the default ingress configmap to be updated
+	previousDefaultIngressCAConfigmap := defaultIngressCAConfigmap.DeepCopy()
+	err = wait.PollImmediate(1*time.Second, 10*time.Second, func() (bool, error) {
+		if err := kclient.Get(context.TODO(), controller.DefaultIngressCertConfigMapName(), defaultIngressCAConfigmap); err != nil {
+			return false, err
+		}
+		if defaultIngressCAConfigmap.Data["ca-bundle.crt"] == previousDefaultIngressCAConfigmap.Data["ca-bundle.crt"] {
+			return false, nil
+		}
+		return true, nil
+	})
+	if err != nil {
+		t.Fatalf("failed to observe update of default ingress CA certificate configmap: %v", err)
+	}
 
 	// Reset .spec.defaultCertificate to its original value.
 	if err := kclient.Get(context.TODO(), defaultName, ic); err != nil {
@@ -267,7 +285,7 @@ func TestUpdateDefaultIngressController(t *testing.T) {
 
 	// Wait for the CA certificate configmap to be recreated.
 	err = wait.PollImmediate(1*time.Second, 10*time.Second, func() (bool, error) {
-		if err := kclient.Get(context.TODO(), controller.RouterCAConfigMapName(), configmap); err != nil {
+		if err := kclient.Get(context.TODO(), controller.RouterCAConfigMapName(), routerCAConfigmap); err != nil {
 			if !errors.IsNotFound(err) {
 				t.Logf("failed to get CA certificate configmap, will retry: %v", err)
 			}
@@ -278,6 +296,21 @@ func TestUpdateDefaultIngressController(t *testing.T) {
 	if err != nil {
 		t.Fatalf("failed to get recreated CA certificate configmap: %v", err)
 	}
+	// Wait for the default ingress configmap to be updated back to the original
+	previousDefaultIngressCAConfigmap = defaultIngressCAConfigmap.DeepCopy()
+	err = wait.PollImmediate(1*time.Second, 10*time.Second, func() (bool, error) {
+		if err := kclient.Get(context.TODO(), controller.DefaultIngressCertConfigMapName(), defaultIngressCAConfigmap); err != nil {
+			return false, err
+		}
+		if defaultIngressCAConfigmap.Data["ca-bundle.crt"] == previousDefaultIngressCAConfigmap.Data["ca-bundle.crt"] {
+			return false, nil
+		}
+		return true, nil
+	})
+	if err != nil {
+		t.Logf("secret content=%v", string(secret.Data["tls.crt"]))
+		t.Fatalf("failed to observe update of default ingress CA certificate configmap: %v\noriginal=%v\ncurrent=%v", err, previousDefaultIngressCAConfigmap.Data["ca-bundle.crt"], defaultIngressCAConfigmap.Data["ca-bundle.crt"])
+	}
 }
 
 // TestIngressControllerScale exercises a simple scale up/down scenario.