add internal load balancer cert

bindata/bootkube/config/bootstrap-config-overrides.yaml

@@ -33,6 +33,8 @@ servingInfo:
       keyFile: /etc/kubernetes/secrets/kube-apiserver-localhost-server.key
     - certFile: /etc/kubernetes/secrets/kube-apiserver-lb-server.crt
       keyFile: /etc/kubernetes/secrets/kube-apiserver-lb-server.key
+    - certFile: /etc/kubernetes/secrets/kube-apiserver-internal-lb-server.crt
+      keyFile: /etc/kubernetes/secrets/kube-apiserver-internal-lb-server.key
 storageConfig:
   ca: /etc/kubernetes/secrets/{{.EtcdServingCA}} # origin 3.11: ca.crt
   certFile: /etc/kubernetes/secrets/etcd-client.crt # origin 3.11: master.etcd-client.crt

pkg/operator/certrotationcontroller/certrotationcontroller.go

@@ -33,8 +33,11 @@ type CertRotationController struct {
 	serviceNetwork        *DynamicServingRotation
 	serviceHostnamesQueue workqueue.RateLimitingInterface
 
-	loadBalancer               *DynamicServingRotation
-	loadBalancerHostnamesQueue workqueue.RateLimitingInterface
+	externalLoadBalancer               *DynamicServingRotation
+	externalLoadBalancerHostnamesQueue workqueue.RateLimitingInterface
+
+	internalLoadBalancer               *DynamicServingRotation
+	internalLoadBalancerHostnamesQueue workqueue.RateLimitingInterface
 
 	cachesSynced []cache.InformerSynced
 }
@@ -54,8 +57,11 @@ func NewCertRotationController(
 		serviceHostnamesQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "ServiceHostnames"),
 		serviceNetwork:        &DynamicServingRotation{hostnamesChanged: make(chan struct{}, 10)},
 
-		loadBalancerHostnamesQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "LoadBalancerHostnames"),
-		loadBalancer:               &DynamicServingRotation{hostnamesChanged: make(chan struct{}, 10)},
+		externalLoadBalancerHostnamesQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "ExternalLoadBalancerHostnames"),
+		externalLoadBalancer:               &DynamicServingRotation{hostnamesChanged: make(chan struct{}, 10)},
+
+		internalLoadBalancerHostnamesQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "InternalLoadBalancerHostnames"),
+		internalLoadBalancer:               &DynamicServingRotation{hostnamesChanged: make(chan struct{}, 10)},
 
 		cachesSynced: []cache.InformerSynced{
 			configInformer.Config().V1().Networks().Informer().HasSynced,
@@ -64,7 +70,7 @@ func NewCertRotationController(
 	}
 
 	configInformer.Config().V1().Networks().Informer().AddEventHandler(ret.serviceHostnameEventHandler())
-	configInformer.Config().V1().Infrastructures().Informer().AddEventHandler(ret.loadBalancerHostnameEventHandler())
+	configInformer.Config().V1().Infrastructures().Informer().AddEventHandler(ret.externalLoadBalancerHostnameEventHandler())
 
 	rotationDay := defaultRotationDay
 	if day != time.Duration(0) {
@@ -195,7 +201,7 @@ func NewCertRotationController(
 	ret.certRotators = append(ret.certRotators, certRotator)
 
 	certRotator, err = certrotation.NewCertRotationController(
-		"LoadBalancerServing",
+		"ExternalLoadBalancerServing",
 		certrotation.SigningRotation{
 			Namespace:     operatorclient.OperatorNamespace,
 			Name:          "loadbalancer-serving-signer",
@@ -220,8 +226,49 @@ func NewCertRotationController(
 			Validity:  30 * rotationDay,
 			Refresh:   15 * rotationDay,
 			CertCreator: &certrotation.ServingRotation{
-				Hostnames:        ret.loadBalancer.GetHostnames,
-				HostnamesChanged: ret.loadBalancer.hostnamesChanged,
+				Hostnames:        ret.externalLoadBalancer.GetHostnames,
+				HostnamesChanged: ret.externalLoadBalancer.hostnamesChanged,
+			},
+			Informer:      kubeInformersForNamespaces.InformersFor(operatorclient.TargetNamespace).Core().V1().Secrets(),
+			Lister:        kubeInformersForNamespaces.InformersFor(operatorclient.TargetNamespace).Core().V1().Secrets().Lister(),
+			Client:        kubeClient.CoreV1(),
+			EventRecorder: eventRecorder,
+		},
+		operatorClient,
+	)
+	if err != nil {
+		return nil, err
+	}
+	ret.certRotators = append(ret.certRotators, certRotator)
+
+	certRotator, err = certrotation.NewCertRotationController(
+		"InternalLoadBalancerServing",
+		certrotation.SigningRotation{
+			Namespace:     operatorclient.OperatorNamespace,
+			Name:          "loadbalancer-serving-signer",
+			Validity:      10 * 365 * rotationDay, // this comes from the installer
+			Refresh:       8 * 365 * rotationDay,  // this means we effectively do not rotate
+			Informer:      kubeInformersForNamespaces.InformersFor(operatorclient.OperatorNamespace).Core().V1().Secrets(),
+			Lister:        kubeInformersForNamespaces.InformersFor(operatorclient.OperatorNamespace).Core().V1().Secrets().Lister(),
+			Client:        kubeClient.CoreV1(),
+			EventRecorder: eventRecorder,
+		},
+		certrotation.CABundleRotation{
+			Namespace:     operatorclient.OperatorNamespace,
+			Name:          "loadbalancer-serving-ca",
+			Informer:      kubeInformersForNamespaces.InformersFor(operatorclient.OperatorNamespace).Core().V1().ConfigMaps(),
+			Lister:        kubeInformersForNamespaces.InformersFor(operatorclient.OperatorNamespace).Core().V1().ConfigMaps().Lister(),
+			Client:        kubeClient.CoreV1(),
+			EventRecorder: eventRecorder,
+		},
+		certrotation.TargetRotation{
+			Namespace: operatorclient.TargetNamespace,
+			Name:      "internal-loadbalancer-serving-certkey",
+			Validity:  30 * rotationDay,
+			Refresh:   15 * rotationDay,
+			CertCreator: &certrotation.ServingRotation{
+				Hostnames:        ret.internalLoadBalancer.GetHostnames,
+				HostnamesChanged: ret.internalLoadBalancer.hostnamesChanged,
 			},
 			Informer:      kubeInformersForNamespaces.InformersFor(operatorclient.TargetNamespace).Core().V1().Secrets(),
 			Lister:        kubeInformersForNamespaces.InformersFor(operatorclient.TargetNamespace).Core().V1().Secrets().Lister(),
@@ -374,12 +421,16 @@ func (c *CertRotationController) Run(workers int, stopCh <-chan struct{}) {
 	if err := c.syncServiceHostnames(); err != nil {
 		panic(err)
 	}
-	if err := c.syncLoadBalancerHostnames(); err != nil {
+	if err := c.syncExternalLoadBalancerHostnames(); err != nil {
+		panic(err)
+	}
+	if err := c.syncInternalLoadBalancerHostnames(); err != nil {
 		panic(err)
 	}
 
 	go wait.Until(c.runServiceHostnames, time.Second, stopCh)
-	go wait.Until(c.runLoadBalancerHostnames, time.Second, stopCh)
+	go wait.Until(c.runExternalLoadBalancerHostnames, time.Second, stopCh)
+	go wait.Until(c.runInternalLoadBalancerHostnames, time.Second, stopCh)
 
 	for _, certRotator := range c.certRotators {
 		go certRotator.Run(workers, stopCh)

pkg/operator/certrotationcontroller/externalloadbalancer.go

@@ -0,0 +1,58 @@
+package certrotationcontroller
+
+import (
+	"fmt"
+	"strings"
+
+	"k8s.io/klog"
+
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/client-go/tools/cache"
+)
+
+func (c *CertRotationController) syncExternalLoadBalancerHostnames() error {
+	infrastructureConfig, err := c.infrastructureLister.Get("cluster")
+	if err != nil {
+		return err
+	}
+	hostname := infrastructureConfig.Status.APIServerURL
+	hostname = strings.Replace(hostname, "https://", "", 1)
+	hostname = hostname[0:strings.LastIndex(hostname, ":")]
+
+	klog.V(2).Infof("syncing external loadbalancer hostnames: %v", hostname)
+	c.externalLoadBalancer.setHostnames([]string{hostname})
+	return nil
+}
+
+func (c *CertRotationController) runExternalLoadBalancerHostnames() {
+	for c.processExternalLoadBalancerHostnames() {
+	}
+}
+
+func (c *CertRotationController) processExternalLoadBalancerHostnames() bool {
+	dsKey, quit := c.externalLoadBalancerHostnamesQueue.Get()
+	if quit {
+		return false
+	}
+	defer c.externalLoadBalancerHostnamesQueue.Done(dsKey)
+
+	err := c.syncExternalLoadBalancerHostnames()
+	if err == nil {
+		c.externalLoadBalancerHostnamesQueue.Forget(dsKey)
+		return true
+	}
+
+	utilruntime.HandleError(fmt.Errorf("%v failed with : %v", dsKey, err))
+	c.externalLoadBalancerHostnamesQueue.AddRateLimited(dsKey)
+
+	return true
+}
+
+// eventHandler queues the operator to check spec and status
+func (c *CertRotationController) externalLoadBalancerHostnameEventHandler() cache.ResourceEventHandler {
+	return cache.ResourceEventHandlerFuncs{
+		AddFunc:    func(obj interface{}) { c.externalLoadBalancerHostnamesQueue.Add(workQueueKey) },
+		UpdateFunc: func(old, new interface{}) { c.externalLoadBalancerHostnamesQueue.Add(workQueueKey) },
+		DeleteFunc: func(obj interface{}) { c.externalLoadBalancerHostnamesQueue.Add(workQueueKey) },
+	}
+}

pkg/operator/certrotationcontroller/internalloadbalancer.go

@@ -0,0 +1,59 @@
+package certrotationcontroller
+
+import (
+	"fmt"
+	"strings"
+
+	"k8s.io/klog"
+
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/client-go/tools/cache"
+)
+
+func (c *CertRotationController) syncInternalLoadBalancerHostnames() error {
+	infrastructureConfig, err := c.infrastructureLister.Get("cluster")
+	if err != nil {
+		return err
+	}
+	hostname := infrastructureConfig.Status.APIServerURL
+	hostname = strings.Replace(hostname, "https://", "", 1)
+	hostname = hostname[0:strings.LastIndex(hostname, ":")]
+	hostname = strings.Replace(hostname, "api.", "api-int.", 1)
+
+	klog.V(2).Infof("syncing internal loadbalancer hostnames: %v", hostname)
+	c.internalLoadBalancer.setHostnames([]string{hostname})
+	return nil
+}
+
+func (c *CertRotationController) runInternalLoadBalancerHostnames() {
+	for c.processExternalLoadBalancerHostnames() {
+	}
+}
+
+func (c *CertRotationController) processInternalLoadBalancerHostnames() bool {
+	dsKey, quit := c.internalLoadBalancerHostnamesQueue.Get()
+	if quit {
+		return false
+	}
+	defer c.internalLoadBalancerHostnamesQueue.Done(dsKey)
+
+	err := c.syncInternalLoadBalancerHostnames()
+	if err == nil {
+		c.internalLoadBalancerHostnamesQueue.Forget(dsKey)
+		return true
+	}
+
+	utilruntime.HandleError(fmt.Errorf("%v failed with : %v", dsKey, err))
+	c.internalLoadBalancerHostnamesQueue.AddRateLimited(dsKey)
+
+	return true
+}
+
+// eventHandler queues the operator to check spec and status
+func (c *CertRotationController) internalLoadBalancerHostnameEventHandler() cache.ResourceEventHandler {
+	return cache.ResourceEventHandlerFuncs{
+		AddFunc:    func(obj interface{}) { c.internalLoadBalancerHostnamesQueue.Add(workQueueKey) },
+		UpdateFunc: func(old, new interface{}) { c.internalLoadBalancerHostnamesQueue.Add(workQueueKey) },
+		DeleteFunc: func(obj interface{}) { c.internalLoadBalancerHostnamesQueue.Add(workQueueKey) },
+	}
+}

pkg/operator/configobservation/apiserver/observe_apiserver.go

@@ -145,6 +145,9 @@ func observeNamedCertificates(apiServer *configv1.APIServer, recorder events.Rec
 	observedNamedCertificates = append(observedNamedCertificates, map[string]interface{}{
 		"certFile": "/etc/kubernetes/static-pod-certs/secrets/external-loadbalancer-serving-certkey/tls.crt",
 		"keyFile":  "/etc/kubernetes/static-pod-certs/secrets/external-loadbalancer-serving-certkey/tls.key"})
+	observedNamedCertificates = append(observedNamedCertificates, map[string]interface{}{
+		"certFile": "/etc/kubernetes/static-pod-certs/secrets/internal-loadbalancer-serving-certkey/tls.crt",
+		"keyFile":  "/etc/kubernetes/static-pod-certs/secrets/internal-loadbalancer-serving-certkey/tls.key"})
 
 	for index, namedCertificate := range namedCertificates {
 		observedNamedCertificate := map[string]interface{}{}

pkg/operator/configobservation/apiserver/observe_apiserver_test.go

@@ -214,6 +214,10 @@ func TestObserveNamedCertificates(t *testing.T) {
 							"certFile": "/etc/kubernetes/static-pod-certs/secrets/external-loadbalancer-serving-certkey/tls.crt",
 							"keyFile":  "/etc/kubernetes/static-pod-certs/secrets/external-loadbalancer-serving-certkey/tls.key",
 						},
+						map[string]interface{}{
+							"certFile": "/etc/kubernetes/static-pod-certs/secrets/internal-loadbalancer-serving-certkey/tls.crt",
+							"keyFile":  "/etc/kubernetes/static-pod-certs/secrets/internal-loadbalancer-serving-certkey/tls.key",
+						},
 					},
 				},
 			},
@@ -254,6 +258,10 @@ func TestObserveNamedCertificates(t *testing.T) {
 							"certFile": "/etc/kubernetes/static-pod-certs/secrets/external-loadbalancer-serving-certkey/tls.crt",
 							"keyFile":  "/etc/kubernetes/static-pod-certs/secrets/external-loadbalancer-serving-certkey/tls.key",
 						},
+						map[string]interface{}{
+							"certFile": "/etc/kubernetes/static-pod-certs/secrets/internal-loadbalancer-serving-certkey/tls.crt",
+							"keyFile":  "/etc/kubernetes/static-pod-certs/secrets/internal-loadbalancer-serving-certkey/tls.key",
+						},
 						map[string]interface{}{
 							"certFile": "/etc/kubernetes/static-pod-certs/secrets/user-serving-cert-000/tls.crt",
 							"keyFile":  "/etc/kubernetes/static-pod-certs/secrets/user-serving-cert-000/tls.key",
@@ -298,6 +306,10 @@ func TestObserveNamedCertificates(t *testing.T) {
 							"certFile": "/etc/kubernetes/static-pod-certs/secrets/external-loadbalancer-serving-certkey/tls.crt",
 							"keyFile":  "/etc/kubernetes/static-pod-certs/secrets/external-loadbalancer-serving-certkey/tls.key",
 						},
+						map[string]interface{}{
+							"certFile": "/etc/kubernetes/static-pod-certs/secrets/internal-loadbalancer-serving-certkey/tls.crt",
+							"keyFile":  "/etc/kubernetes/static-pod-certs/secrets/internal-loadbalancer-serving-certkey/tls.key",
+						},
 						map[string]interface{}{
 							"certFile": "/etc/kubernetes/static-pod-certs/secrets/user-serving-cert-000/tls.crt",
 							"keyFile":  "/etc/kubernetes/static-pod-certs/secrets/user-serving-cert-000/tls.key",
@@ -344,6 +356,10 @@ func TestObserveNamedCertificates(t *testing.T) {
 							"certFile": "/etc/kubernetes/static-pod-certs/secrets/external-loadbalancer-serving-certkey/tls.crt",
 							"keyFile":  "/etc/kubernetes/static-pod-certs/secrets/external-loadbalancer-serving-certkey/tls.key",
 						},
+						map[string]interface{}{
+							"certFile": "/etc/kubernetes/static-pod-certs/secrets/internal-loadbalancer-serving-certkey/tls.crt",
+							"keyFile":  "/etc/kubernetes/static-pod-certs/secrets/internal-loadbalancer-serving-certkey/tls.key",
+						},
 						map[string]interface{}{
 							"certFile": "/etc/kubernetes/static-pod-certs/secrets/user-serving-cert-000/tls.crt",
 							"keyFile":  "/etc/kubernetes/static-pod-certs/secrets/user-serving-cert-000/tls.key",
@@ -397,6 +413,10 @@ func TestObserveNamedCertificates(t *testing.T) {
 							"certFile": "/etc/kubernetes/static-pod-certs/secrets/external-loadbalancer-serving-certkey/tls.crt",
 							"keyFile":  "/etc/kubernetes/static-pod-certs/secrets/external-loadbalancer-serving-certkey/tls.key",
 						},
+						map[string]interface{}{
+							"certFile": "/etc/kubernetes/static-pod-certs/secrets/internal-loadbalancer-serving-certkey/tls.crt",
+							"keyFile":  "/etc/kubernetes/static-pod-certs/secrets/internal-loadbalancer-serving-certkey/tls.key",
+						},
 						map[string]interface{}{
 							"certFile": "/etc/kubernetes/static-pod-certs/secrets/user-serving-cert-000/tls.crt",
 							"keyFile":  "/etc/kubernetes/static-pod-certs/secrets/user-serving-cert-000/tls.key",

pkg/operator/starter.go

@@ -212,6 +212,7 @@ var CertSecrets = []revision.RevisionResource{
 	{Name: "localhost-serving-cert-certkey"},
 	{Name: "service-network-serving-certkey"},
 	{Name: "external-loadbalancer-serving-certkey"},
+	{Name: "internal-loadbalancer-serving-certkey"},
 
 	{Name: "user-serving-cert", Optional: true},
 	{Name: "user-serving-cert-000", Optional: true},