Dataplane-trust Adding mTLS and TLS To Activator

cmd/activator/main.go

@@ -18,7 +18,6 @@ package main
 
 import (
 	"context"
-	"crypto/tls"
 	"errors"
 	"fmt"
 	"log"
@@ -145,7 +144,6 @@ func main() {
 	// (via keep-alive) to send real requests, avoiding needing an extra
 	// reconnect for the first request after the probe succeeds.
 	logger.Debugf("MaxIdleProxyConns: %d, MaxIdleProxyConnsPerHost: %d", env.MaxIdleProxyConns, env.MaxIdleProxyConnsPerHost)
-	transport := pkgnet.NewProxyAutoTransport(env.MaxIdleProxyConns, env.MaxIdleProxyConnsPerHost)
 
 	// Fetch networking configuration to determine whether EnableMeshPodAddressability
 	// is enabled or not.
@@ -158,18 +156,19 @@ func main() {
 		logger.Fatalw("Failed to construct network config", zap.Error(err))
 	}
 
-	// Enable TLS against queue-proxy when internal-encryption is enabled.
-	tlsEnabled := networkConfig.InternalEncryption
-
+	// Enable TLS both as client and as server if DataplaneTrust != TrustDisabled
+	tlsEnabled := networkConfig.DataplaneTrust != netcfg.TrustDisabled
+	var transport http.RoundTripper
 	var certCache *certificate.CertCache
 
-	// Enable TLS client when queue-proxy-ca is specified.
 	// At this moment activator with TLS does not disable HTTP.
 	// See also https://github.com/knative/serving/issues/12808.
 	if tlsEnabled {
-		logger.Info("Internal Encryption is enabled")
-		certCache = certificate.NewCertCache(ctx)
-		transport = pkgnet.NewProxyAutoTLSTransport(env.MaxIdleProxyConns, env.MaxIdleProxyConnsPerHost, &certCache.TLSConf)
+		logger.Info("Dataplane trust %q is used", networkConfig.DataplaneTrust)
+		certCache = certificate.NewCertCache(ctx, networkConfig.DataplaneTrust)
+		transport = activatorhandler.NewProxyAutoTLSTransport(env.MaxIdleProxyConns, env.MaxIdleProxyConnsPerHost, &certCache.ClientTLSConf)
+	} else {
+		transport = activatorhandler.NewProxyAutoTransport(env.MaxIdleProxyConns, env.MaxIdleProxyConnsPerHost)
 	}
 
 	// Start throttler.
@@ -189,7 +188,7 @@ func main() {
 
 	// Set up our config store
 	configMapWatcher := configmapinformer.NewInformedWatcher(kubeClient, system.Namespace())
-	configStore := activatorconfig.NewStore(logger, tracerUpdater)
+	configStore := activatorconfig.NewStore(logger, networkConfig.DataplaneTrust, tracerUpdater)
 	configStore.WatchConfigs(configMapWatcher)
 
 	statCh := make(chan []asmetrics.StatMessage)
@@ -278,16 +277,13 @@ func main() {
 		}(name, server)
 	}
 
-	// Enable TLS server when internal-encryption is specified.
+	// Enable TLS server when DataPlaneTrust is not netcfg.TrustDisabled.
 	// At this moment activator with TLS does not disable HTTP.
 	// See also https://github.com/knative/serving/issues/12808.
 	if tlsEnabled {
 		name, server := "https", pkgnet.NewServer(":"+strconv.Itoa(networking.BackendHTTPSPort), ah)
 		go func(name string, s *http.Server) {
-			s.TLSConfig = &tls.Config{
-				MinVersion:     tls.VersionTLS12,
-				GetCertificate: certCache.GetCertificate,
-			}
+			s.TLSConfig = &certCache.ServerTLSConf
 			// Don't forward ErrServerClosed as that indicates we're already shutting down.
 			if err := s.ListenAndServeTLS("", ""); err != nil && !errors.Is(err, http.ErrServerClosed) {
 				errCh <- fmt.Errorf("%s server failed: %w", name, err)

pkg/activator/certificate/cache.go

@@ -20,8 +20,7 @@ import (
 	"context"
 	"crypto/tls"
 	"crypto/x509"
-	"encoding/pem"
-	"sync"
+	"fmt"
 
 	"go.uber.org/zap"
 	corev1 "k8s.io/api/core/v1"
@@ -34,44 +33,79 @@ import (
 	secretinformer "knative.dev/pkg/injection/clients/namespacedkube/informers/core/v1/secret"
 	"knative.dev/pkg/logging"
 	"knative.dev/pkg/system"
+	"knative.dev/serving/pkg/activator/handler"
 )
 
 // CertCache caches certificates and CA pool.
 type CertCache struct {
 	secretInformer v1.SecretInformer
 	logger         *zap.SugaredLogger
+	trustConfig    netcfg.Trust
 
-	certificate *tls.Certificate
-	TLSConf     tls.Config
-
-	certificatesMux sync.RWMutex
+	certificate   *tls.Certificate
+	ClientTLSConf tls.Config
+	ServerTLSConf tls.Config
 }
 
-// NewCertCache starts secretInformer.
-func NewCertCache(ctx context.Context) *CertCache {
-	secretInformer := secretinformer.Get(ctx)
-
+func newCertCache(ctx context.Context, trust netcfg.Trust, secretInformer v1.SecretInformer) *CertCache {
 	cr := &CertCache{
 		secretInformer: secretInformer,
 		logger:         logging.FromContext(ctx),
+		trustConfig:    trust,
 	}
-
-	secret, err := cr.secretInformer.Lister().Secrets(system.Namespace()).Get(netcfg.ServingInternalCertName)
-	if err != nil {
-		cr.logger.Warnw("failed to get secret", zap.Error(err))
-		return nil
+	cr.ClientTLSConf.ServerName = certificates.LegacyFakeDnsName
+	cr.ClientTLSConf.MinVersion = tls.VersionTLS13
+	cr.ClientTLSConf.RootCAs = x509.NewCertPool()
+	cr.ClientTLSConf.GetClientCertificate = cr.GetClientCertificate
+
+	cr.ServerTLSConf.MinVersion = tls.VersionTLS12
+	cr.ServerTLSConf.ClientCAs = x509.NewCertPool()
+	cr.ServerTLSConf.GetCertificate = cr.GetCertificate
+	cr.ServerTLSConf.GetConfigForClient = cr.GetConfigForClient
+	switch cr.trustConfig {
+	case netcfg.TrustIdentity, netcfg.TrustMutual:
+		cr.ServerTLSConf.ClientAuth = tls.RequireAndVerifyClientCert
+		cr.ServerTLSConf.VerifyConnection = func(cs tls.ConnectionState) error {
+			if len(cs.PeerCertificates) == 0 {
+				// Should never happen on a server side
+				cr.logger.Info("mTLS: Failed to verify client connection. Certificate is missing\n")
+				return fmt.Errorf("mTLS: Failed to verify client connection. Certificate is missing")
+			}
+			for _, match := range cs.PeerCertificates[0].DNSNames {
+				// Activator currently supports a single routingId which is the default "0"
+				// Working with other routingId is not yet implemented
+				if match == certificates.DataPlaneRoutingName("0") {
+					return nil
+				}
+			}
+
+			cr.logger.Info("mTLS: Failed to verify client connection for DNSNames: %v\n", cs.PeerCertificates[0].DNSNames)
+			return fmt.Errorf("mTLS: Failed to verify client connection for DNSNames: %v", cs.PeerCertificates[0].DNSNames)
+		}
 	}
 
-	cr.updateCache(secret)
-
 	secretInformer.Informer().AddEventHandler(cache.FilteringResourceEventHandler{
-		FilterFunc: controller.FilterWithNameAndNamespace(system.Namespace(), netcfg.ServingInternalCertName),
+		FilterFunc: controller.FilterWithNameAndNamespace(system.Namespace(), netcfg.ServingRoutingCertName),
 		Handler: cache.ResourceEventHandlerFuncs{
 			UpdateFunc: cr.handleCertificateUpdate,
 			AddFunc:    cr.handleCertificateAdd,
 		},
 	})
+	return cr
+}
+
+// NewCertCache starts secretInformer.
+func NewCertCache(ctx context.Context, trust netcfg.Trust) *CertCache {
+	secretInformer := secretinformer.Get(ctx)
+	cr := newCertCache(ctx, trust, secretInformer)
 
+	secret, err := cr.secretInformer.Lister().Secrets(system.Namespace()).Get(netcfg.ServingRoutingCertName)
+	if err != nil {
+		cr.logger.Warnw("failed to get secret", zap.Error(err))
+		return nil
+	}
+
+	cr.updateCache(secret)
 	return cr
 }
 
@@ -82,8 +116,8 @@ func (cr *CertCache) handleCertificateAdd(added interface{}) {
 }
 
 func (cr *CertCache) updateCache(secret *corev1.Secret) {
-	cr.certificatesMux.Lock()
-	defer cr.certificatesMux.Unlock()
+	handler.TLSConfLock()
+	defer handler.TLSConfUnlock()
 
 	cert, err := tls.X509KeyPair(secret.Data[certificates.CertName], secret.Data[certificates.PrivateKeyName])
 	if err != nil {
@@ -92,18 +126,8 @@ func (cr *CertCache) updateCache(secret *corev1.Secret) {
 	}
 	cr.certificate = &cert
 
-	pool := x509.NewCertPool()
-	block, _ := pem.Decode(secret.Data[certificates.CaCertName])
-	ca, err := x509.ParseCertificate(block.Bytes)
-	if err != nil {
-		cr.logger.Warnw("failed to parse CA", zap.Error(err))
-		return
-	}
-	pool.AddCert(ca)
-
-	cr.TLSConf.RootCAs = pool
-	cr.TLSConf.ServerName = certificates.LegacyFakeDnsName
-	cr.TLSConf.MinVersion = tls.VersionTLS13
+	cr.ClientTLSConf.RootCAs.AppendCertsFromPEM(secret.Data[certificates.CaCertName])
+	cr.ServerTLSConf.ClientCAs.AppendCertsFromPEM(secret.Data[certificates.CaCertName])
 }
 
 func (cr *CertCache) handleCertificateUpdate(_, new interface{}) {
@@ -112,5 +136,22 @@ func (cr *CertCache) handleCertificateUpdate(_, new interface{}) {
 
 // GetCertificate returns the cached certificates.
 func (cr *CertCache) GetCertificate(_ *tls.ClientHelloInfo) (*tls.Certificate, error) {
+	handler.TLSConfLock()
+	defer handler.TLSConfUnlock()
 	return cr.certificate, nil
 }
+
+func (cr *CertCache) GetClientCertificate(*tls.CertificateRequestInfo) (*tls.Certificate, error) {
+	handler.TLSConfLock()
+	defer handler.TLSConfUnlock()
+	return cr.certificate, nil
+}
+
+func (cr *CertCache) GetConfigForClient(*tls.ClientHelloInfo) (*tls.Config, error) {
+	handler.TLSConfLock()
+	defer handler.TLSConfUnlock()
+	// Clone the certificate Pool such that the one used by the server will be different from the one that will get updated is CA is replaced.
+	serverTLSConf := cr.ServerTLSConf.Clone()
+	serverTLSConf.ClientCAs = serverTLSConf.ClientCAs.Clone()
+	return serverTLSConf, nil
+}