diff --git a/cmd/activator/main.go b/cmd/activator/main.go index ee227c7dc0ea..098a48643905 100644 --- a/cmd/activator/main.go +++ b/cmd/activator/main.go @@ -18,7 +18,6 @@ package main import ( "context" - "crypto/tls" "errors" "fmt" "log" @@ -145,7 +144,6 @@ func main() { // (via keep-alive) to send real requests, avoiding needing an extra // reconnect for the first request after the probe succeeds. logger.Debugf("MaxIdleProxyConns: %d, MaxIdleProxyConnsPerHost: %d", env.MaxIdleProxyConns, env.MaxIdleProxyConnsPerHost) - transport := pkgnet.NewProxyAutoTransport(env.MaxIdleProxyConns, env.MaxIdleProxyConnsPerHost) // Fetch networking configuration to determine whether EnableMeshPodAddressability // is enabled or not. @@ -158,18 +156,19 @@ func main() { logger.Fatalw("Failed to construct network config", zap.Error(err)) } - // Enable TLS against queue-proxy when internal-encryption is enabled. - tlsEnabled := networkConfig.InternalEncryption - + // Enable TLS both as client and as server if DataplaneTrust != TrustDisabled + tlsEnabled := networkConfig.DataplaneTrust != netcfg.TrustDisabled + var transport http.RoundTripper var certCache *certificate.CertCache - // Enable TLS client when queue-proxy-ca is specified. // At this moment activator with TLS does not disable HTTP. // See also https://github.com/knative/serving/issues/12808. if tlsEnabled { - logger.Info("Internal Encryption is enabled") - certCache = certificate.NewCertCache(ctx) - transport = pkgnet.NewProxyAutoTLSTransport(env.MaxIdleProxyConns, env.MaxIdleProxyConnsPerHost, &certCache.TLSConf) + logger.Info("Dataplane trust %q is used", networkConfig.DataplaneTrust) + certCache = certificate.NewCertCache(ctx, networkConfig.DataplaneTrust) + transport = activatorhandler.NewProxyAutoTLSTransport(env.MaxIdleProxyConns, env.MaxIdleProxyConnsPerHost, &certCache.ClientTLSConf) + } else { + transport = activatorhandler.NewProxyAutoTransport(env.MaxIdleProxyConns, env.MaxIdleProxyConnsPerHost) } // Start throttler. @@ -189,7 +188,7 @@ func main() { // Set up our config store configMapWatcher := configmapinformer.NewInformedWatcher(kubeClient, system.Namespace()) - configStore := activatorconfig.NewStore(logger, tracerUpdater) + configStore := activatorconfig.NewStore(logger, networkConfig.DataplaneTrust, tracerUpdater) configStore.WatchConfigs(configMapWatcher) statCh := make(chan []asmetrics.StatMessage) @@ -278,16 +277,13 @@ func main() { }(name, server) } - // Enable TLS server when internal-encryption is specified. + // Enable TLS server when DataPlaneTrust is not netcfg.TrustDisabled. // At this moment activator with TLS does not disable HTTP. // See also https://github.com/knative/serving/issues/12808. if tlsEnabled { name, server := "https", pkgnet.NewServer(":"+strconv.Itoa(networking.BackendHTTPSPort), ah) go func(name string, s *http.Server) { - s.TLSConfig = &tls.Config{ - MinVersion: tls.VersionTLS12, - GetCertificate: certCache.GetCertificate, - } + s.TLSConfig = &certCache.ServerTLSConf // Don't forward ErrServerClosed as that indicates we're already shutting down. if err := s.ListenAndServeTLS("", ""); err != nil && !errors.Is(err, http.ErrServerClosed) { errCh <- fmt.Errorf("%s server failed: %w", name, err) diff --git a/pkg/activator/certificate/cache.go b/pkg/activator/certificate/cache.go index 62f43e4f8229..22c32c9917f0 100644 --- a/pkg/activator/certificate/cache.go +++ b/pkg/activator/certificate/cache.go @@ -20,8 +20,7 @@ import ( "context" "crypto/tls" "crypto/x509" - "encoding/pem" - "sync" + "fmt" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" @@ -34,44 +33,79 @@ import ( secretinformer "knative.dev/pkg/injection/clients/namespacedkube/informers/core/v1/secret" "knative.dev/pkg/logging" "knative.dev/pkg/system" + "knative.dev/serving/pkg/activator/handler" ) // CertCache caches certificates and CA pool. type CertCache struct { secretInformer v1.SecretInformer logger *zap.SugaredLogger + trustConfig netcfg.Trust - certificate *tls.Certificate - TLSConf tls.Config - - certificatesMux sync.RWMutex + certificate *tls.Certificate + ClientTLSConf tls.Config + ServerTLSConf tls.Config } -// NewCertCache starts secretInformer. -func NewCertCache(ctx context.Context) *CertCache { - secretInformer := secretinformer.Get(ctx) - +func newCertCache(ctx context.Context, trust netcfg.Trust, secretInformer v1.SecretInformer) *CertCache { cr := &CertCache{ secretInformer: secretInformer, logger: logging.FromContext(ctx), + trustConfig: trust, } - - secret, err := cr.secretInformer.Lister().Secrets(system.Namespace()).Get(netcfg.ServingInternalCertName) - if err != nil { - cr.logger.Warnw("failed to get secret", zap.Error(err)) - return nil + cr.ClientTLSConf.ServerName = certificates.LegacyFakeDnsName + cr.ClientTLSConf.MinVersion = tls.VersionTLS13 + cr.ClientTLSConf.RootCAs = x509.NewCertPool() + cr.ClientTLSConf.GetClientCertificate = cr.GetClientCertificate + + cr.ServerTLSConf.MinVersion = tls.VersionTLS12 + cr.ServerTLSConf.ClientCAs = x509.NewCertPool() + cr.ServerTLSConf.GetCertificate = cr.GetCertificate + cr.ServerTLSConf.GetConfigForClient = cr.GetConfigForClient + switch cr.trustConfig { + case netcfg.TrustIdentity, netcfg.TrustMutual: + cr.ServerTLSConf.ClientAuth = tls.RequireAndVerifyClientCert + cr.ServerTLSConf.VerifyConnection = func(cs tls.ConnectionState) error { + if len(cs.PeerCertificates) == 0 { + // Should never happen on a server side + cr.logger.Info("mTLS: Failed to verify client connection. Certificate is missing\n") + return fmt.Errorf("mTLS: Failed to verify client connection. Certificate is missing") + } + for _, match := range cs.PeerCertificates[0].DNSNames { + // Activator currently supports a single routingId which is the default "0" + // Working with other routingId is not yet implemented + if match == certificates.DataPlaneRoutingName("0") { + return nil + } + } + + cr.logger.Info("mTLS: Failed to verify client connection for DNSNames: %v\n", cs.PeerCertificates[0].DNSNames) + return fmt.Errorf("mTLS: Failed to verify client connection for DNSNames: %v", cs.PeerCertificates[0].DNSNames) + } } - cr.updateCache(secret) - secretInformer.Informer().AddEventHandler(cache.FilteringResourceEventHandler{ - FilterFunc: controller.FilterWithNameAndNamespace(system.Namespace(), netcfg.ServingInternalCertName), + FilterFunc: controller.FilterWithNameAndNamespace(system.Namespace(), netcfg.ServingRoutingCertName), Handler: cache.ResourceEventHandlerFuncs{ UpdateFunc: cr.handleCertificateUpdate, AddFunc: cr.handleCertificateAdd, }, }) + return cr +} + +// NewCertCache starts secretInformer. +func NewCertCache(ctx context.Context, trust netcfg.Trust) *CertCache { + secretInformer := secretinformer.Get(ctx) + cr := newCertCache(ctx, trust, secretInformer) + secret, err := cr.secretInformer.Lister().Secrets(system.Namespace()).Get(netcfg.ServingRoutingCertName) + if err != nil { + cr.logger.Warnw("failed to get secret", zap.Error(err)) + return nil + } + + cr.updateCache(secret) return cr } @@ -82,8 +116,8 @@ func (cr *CertCache) handleCertificateAdd(added interface{}) { } func (cr *CertCache) updateCache(secret *corev1.Secret) { - cr.certificatesMux.Lock() - defer cr.certificatesMux.Unlock() + handler.TLSConfLock() + defer handler.TLSConfUnlock() cert, err := tls.X509KeyPair(secret.Data[certificates.CertName], secret.Data[certificates.PrivateKeyName]) if err != nil { @@ -92,18 +126,8 @@ func (cr *CertCache) updateCache(secret *corev1.Secret) { } cr.certificate = &cert - pool := x509.NewCertPool() - block, _ := pem.Decode(secret.Data[certificates.CaCertName]) - ca, err := x509.ParseCertificate(block.Bytes) - if err != nil { - cr.logger.Warnw("failed to parse CA", zap.Error(err)) - return - } - pool.AddCert(ca) - - cr.TLSConf.RootCAs = pool - cr.TLSConf.ServerName = certificates.LegacyFakeDnsName - cr.TLSConf.MinVersion = tls.VersionTLS13 + cr.ClientTLSConf.RootCAs.AppendCertsFromPEM(secret.Data[certificates.CaCertName]) + cr.ServerTLSConf.ClientCAs.AppendCertsFromPEM(secret.Data[certificates.CaCertName]) } func (cr *CertCache) handleCertificateUpdate(_, new interface{}) { @@ -112,5 +136,22 @@ func (cr *CertCache) handleCertificateUpdate(_, new interface{}) { // GetCertificate returns the cached certificates. func (cr *CertCache) GetCertificate(_ *tls.ClientHelloInfo) (*tls.Certificate, error) { + handler.TLSConfLock() + defer handler.TLSConfUnlock() return cr.certificate, nil } + +func (cr *CertCache) GetClientCertificate(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + handler.TLSConfLock() + defer handler.TLSConfUnlock() + return cr.certificate, nil +} + +func (cr *CertCache) GetConfigForClient(*tls.ClientHelloInfo) (*tls.Config, error) { + handler.TLSConfLock() + defer handler.TLSConfUnlock() + // Clone the certificate Pool such that the one used by the server will be different from the one that will get updated is CA is replaced. + serverTLSConf := cr.ServerTLSConf.Clone() + serverTLSConf.ClientCAs = serverTLSConf.ClientCAs.Clone() + return serverTLSConf, nil +} diff --git a/pkg/activator/certificate/cache_test.go b/pkg/activator/certificate/cache_test.go index 5378bb2c6526..cfa8ffb05aa8 100644 --- a/pkg/activator/certificate/cache_test.go +++ b/pkg/activator/certificate/cache_test.go @@ -29,40 +29,34 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/tools/cache" "knative.dev/networking/pkg/certificates" netcfg "knative.dev/networking/pkg/config" fakekubeclient "knative.dev/pkg/client/injection/kube/client/fake" fakesecretinformer "knative.dev/pkg/client/injection/kube/informers/core/v1/secret/fake" - "knative.dev/pkg/controller" - "knative.dev/pkg/logging" rtesting "knative.dev/pkg/reconciler/testing" "knative.dev/pkg/system" + "knative.dev/serving/pkg/activator/handler" ) -func fakeCertCache(ctx context.Context) *CertCache { - secretInformer := fakesecretinformer.Get(ctx) - - cr := &CertCache{ - secretInformer: secretInformer, - certificate: nil, - TLSConf: tls.Config{}, - logger: logging.FromContext(ctx), +func AddCert(t *testing.T, c *x509.CertPool, cert []byte) { + block, _ := pem.Decode(cert) + ca, err := x509.ParseCertificate(block.Bytes) + if err != nil { + t.Errorf("Failed to parse CA: %v", zap.Error(err)) + return } + c.AddCert(ca) +} - secretInformer.Informer().AddEventHandler(cache.FilteringResourceEventHandler{ - FilterFunc: controller.FilterWithNameAndNamespace(system.Namespace(), netcfg.ServingInternalCertName), - Handler: cache.ResourceEventHandlerFuncs{ - UpdateFunc: cr.handleCertificateUpdate, - AddFunc: cr.handleCertificateAdd, - }, - }) - - return cr +func fakeCertCache(ctx context.Context) *CertCache { + secretInformer := fakesecretinformer.Get(ctx) + return newCertCache(ctx, netcfg.TrustMutual, secretInformer) } -func TestReconcile(t *testing.T) { +func TestFakeReconcile(t *testing.T) { + caCertPool := x509.NewCertPool() + ctx, cancel, informers := rtesting.SetupFakeContextWithCancel(t) cr := fakeCertCache(ctx) @@ -78,7 +72,7 @@ func TestReconcile(t *testing.T) { secret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: netcfg.ServingInternalCertName, + Name: netcfg.ServingRoutingCertName, Namespace: system.Namespace(), }, Data: map[string][]byte{ @@ -94,13 +88,39 @@ func TestReconcile(t *testing.T) { // Wait for the resources to be created and the handler is called. if err := wait.PollImmediate(10*time.Millisecond, 2*time.Second, func() (bool, error) { // To access cert.Certificate, take a lock. - cr.certificatesMux.RLock() - defer cr.certificatesMux.RUnlock() - cert, _ := cr.GetCertificate(nil) - return cert != nil, nil + cert1, _ := cr.GetCertificate(nil) + cert2, _ := cr.GetClientCertificate(nil) + return cert1 != nil && cert2 != nil, nil }); err != nil { t.Fatal("timeout to get the secret:", err) } + block, _ := pem.Decode(ca) + cacert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + t.Errorf("faield to parse ca %v", err) + } + + caCertPool.AddCert(cacert) + if cr.ClientTLSConf.ClientCAs.Equal(caCertPool) { + t.Errorf("cr.ClientTLSConf.ClientCAs different expected") + } + if cr.ServerTLSConf.RootCAs.Equal(caCertPool) { + t.Errorf("cr.ClientTLSConf.ClientCAs different expected") + } + + cs := tls.ConnectionState{PeerCertificates: []*x509.Certificate{{DNSNames: []string{"ddd", "xxx", certificates.DataPlaneRoutingName("0"), "ddd", "xxx"}}}} + if verifyErr := cr.ServerTLSConf.VerifyConnection(cs); verifyErr != nil { + t.Error("cr.ServerTLSConf.VerifyConnection: expected to find kn-routing-0") + } + + cs = tls.ConnectionState{PeerCertificates: []*x509.Certificate{{DNSNames: []string{"ddd", "xxx"}}}} + if verifyErr := cr.ServerTLSConf.VerifyConnection(cs); verifyErr == nil { + t.Error("cr.ServerTLSConf.VerifyConnection: expected error") + } + cs = tls.ConnectionState{PeerCertificates: []*x509.Certificate{{DNSNames: []string{}}}} + if verifyErr := cr.ServerTLSConf.VerifyConnection(cs); verifyErr == nil { + t.Error("cr.ServerTLSConf.VerifyConnection: expected error") + } // Update cert and key but keep using old CA, then the error is expected. secret.Data[certificates.CertName] = newTLSCrt @@ -109,37 +129,84 @@ func TestReconcile(t *testing.T) { fakekubeclient.Get(ctx).CoreV1().Secrets(system.Namespace()).Update(ctx, secret, metav1.UpdateOptions{}) if err := wait.PollImmediate(10*time.Millisecond, 5*time.Second, func() (bool, error) { - // To access cert.Certificate, take a lock. - cr.certificatesMux.RLock() - defer cr.certificatesMux.RUnlock() - cert, err := cr.GetCertificate(nil) - return err == nil && reflect.DeepEqual(newCert.Certificate, cert.Certificate), nil + // To access cr.TLSConf.RootCAs, take a lock. + cert1, _ := cr.GetCertificate(nil) + cert2, _ := cr.GetClientCertificate(nil) + return err == nil && reflect.DeepEqual(newCert.Certificate, cert1.Certificate) && reflect.DeepEqual(newCert.Certificate, cert2.Certificate), nil }); err != nil { t.Fatalf("timeout to update the cert: %v", err) } + if cr.ClientTLSConf.ClientCAs.Equal(caCertPool) { + t.Errorf("cr.ClientTLSConf.ClientCAs different expected") + } + if cr.ServerTLSConf.RootCAs.Equal(caCertPool) { + t.Errorf("cr.ClientTLSConf.ClientCAs different expected") + } + // Update CA, now the error is gone. secret.Data[certificates.CaCertName] = newCA pool := x509.NewCertPool() - block, _ := pem.Decode(secret.Data[certificates.CaCertName]) - ca, err := x509.ParseCertificate(block.Bytes) + AddCert(t, pool, ca) + AddCert(t, pool, newCA) + + fakekubeclient.Get(ctx).CoreV1().Secrets(system.Namespace()).Update(ctx, secret, metav1.UpdateOptions{}) + if err := wait.PollImmediate(10*time.Millisecond, 10*time.Second, func() (bool, error) { + // To access cr.TLSConf.RootCAs, take a lock. + handler.TLSConfLock() + defer handler.TLSConfUnlock() + return err == nil && pool.Equal(cr.ClientTLSConf.RootCAs) && pool.Equal(cr.ServerTLSConf.ClientCAs), nil + }); err != nil { + t.Fatalf("timeout to update the cert: %v", err) + } + + block2, _ := pem.Decode(newCA) + cacert2, err := x509.ParseCertificate(block2.Bytes) if err != nil { - cr.logger.Warnw("Failed to parse CA: %v", zap.Error(err)) - return + t.Errorf("faield to parse ca %v", err) + } + + caCertPool.AddCert(cacert2) + if cr.ClientTLSConf.ClientCAs.Equal(caCertPool) { + t.Errorf("cr.ClientTLSConf.ClientCAs different expected") + } + if cr.ServerTLSConf.RootCAs.Equal(caCertPool) { + t.Errorf("cr.ClientTLSConf.ClientCAs different expected") } - pool.AddCert(ca) + // Update bad CA. + secret.Data[certificates.CaCertName] = badPem fakekubeclient.Get(ctx).CoreV1().Secrets(system.Namespace()).Update(ctx, secret, metav1.UpdateOptions{}) if err := wait.PollImmediate(10*time.Millisecond, 10*time.Second, func() (bool, error) { // To access cr.TLSConf.RootCAs, take a lock. - cr.certificatesMux.RLock() - defer cr.certificatesMux.RUnlock() - return err == nil && pool.Equal(cr.TLSConf.RootCAs), nil + handler.TLSConfLock() + defer handler.TLSConfUnlock() + return err == nil && pool.Equal(cr.ClientTLSConf.RootCAs) && pool.Equal(cr.ServerTLSConf.ClientCAs), nil }); err != nil { t.Fatalf("timeout to update the cert: %v", err) } + + // Update bad key pair. + secret.Data[certificates.CertName] = badPem + secret.Data[certificates.PrivateKeyName] = badPem + fakekubeclient.Get(ctx).CoreV1().Secrets(system.Namespace()).Update(ctx, secret, metav1.UpdateOptions{}) + if err := wait.PollImmediate(10*time.Millisecond, 5*time.Second, func() (bool, error) { + // To access cr.TLSConf.RootCAs, take a lock. + cert1, _ := cr.GetCertificate(nil) + cert2, _ := cr.GetClientCertificate(nil) + return err == nil && reflect.DeepEqual(newCert.Certificate, cert1.Certificate) && reflect.DeepEqual(newCert.Certificate, cert2.Certificate), nil + }); err != nil { + t.Fatalf("timeout to update the cert: %v", err) + } + + if cr.ClientTLSConf.ClientCAs.Equal(caCertPool) { + t.Errorf("cr.ClientTLSConf.ClientCAs different expected") + } + if cr.ServerTLSConf.RootCAs.Equal(caCertPool) { + t.Errorf("cr.ClientTLSConf.ClientCAs different expected") + } } var ca = []byte( @@ -288,3 +355,6 @@ uNLDuUL4EPGk084uoa/rwQxUDwWQ05aw81c/Q0ssPeyekgLNfet4HX4lzBDJWZEQ FUu9LuwG/tVRBIecvo/IcUuQ1/UObbRAXpp0Y8aO56UVeBvOb9bG2/wjRJmrgR+1 vCCFsBlglA== -----END CERTIFICATE-----`) + +var badPem = []byte( + `----------`) diff --git a/pkg/activator/config/store.go b/pkg/activator/config/store.go index ade9962cb766..759a5a1973bf 100644 --- a/pkg/activator/config/store.go +++ b/pkg/activator/config/store.go @@ -20,6 +20,7 @@ import ( "context" "go.uber.org/atomic" + netcfg "knative.dev/networking/pkg/config" "knative.dev/pkg/configmap" tracingconfig "knative.dev/pkg/tracing/config" ) @@ -29,6 +30,7 @@ type cfgKey struct{} // Config is the configuration for the activator. type Config struct { Tracing *tracingconfig.Config + Trust netcfg.Trust } // FromContext obtains a Config injected into the passed context. @@ -45,7 +47,7 @@ type Store struct { } // NewStore creates a new configuration Store. -func NewStore(logger configmap.Logger, onAfterStore ...func(name string, value interface{})) *Store { +func NewStore(logger configmap.Logger, trust netcfg.Trust, onAfterStore ...func(name string, value interface{})) *Store { s := &Store{} // Append an update function to run after a ConfigMap has updated to update the @@ -53,6 +55,7 @@ func NewStore(logger configmap.Logger, onAfterStore ...func(name string, value i onAfterStore = append(onAfterStore, func(_ string, _ interface{}) { s.current.Store(&Config{ Tracing: s.UntypedLoad(tracingconfig.ConfigName).(*tracingconfig.Config).DeepCopy(), + Trust: trust, }) }) s.UntypedStore = configmap.NewUntypedStore( diff --git a/pkg/activator/config/store_test.go b/pkg/activator/config/store_test.go index 94c84658f358..96d95ec25e8a 100644 --- a/pkg/activator/config/store_test.go +++ b/pkg/activator/config/store_test.go @@ -37,7 +37,7 @@ var tracingConfig = &corev1.ConfigMap{ func TestStore(t *testing.T) { logger := ltesting.TestLogger(t) - store := NewStore(logger) + store := NewStore(logger, "Disabled") store.OnConfigChanged(tracingConfig) ctx := store.ToContext(context.Background()) @@ -68,7 +68,7 @@ func TestStore(t *testing.T) { func BenchmarkStoreToContext(b *testing.B) { logger := ltesting.TestLogger(b) - store := NewStore(logger) + store := NewStore(logger, "Disabled") store.OnConfigChanged(tracingConfig) b.Run("sequential", func(b *testing.B) { diff --git a/pkg/activator/handler/handler_test.go b/pkg/activator/handler/handler_test.go index c206b46304dc..cdd493ba66cb 100644 --- a/pkg/activator/handler/handler_test.go +++ b/pkg/activator/handler/handler_test.go @@ -70,19 +70,26 @@ func (ft fakeThrottler) Try(_ context.Context, _ types.NamespacedName, f func(st func TestActivationHandler(t *testing.T) { tests := []struct { - name string - wantBody string - wantCode int - wantErr error - probeErr error - probeCode int - probeResp []string - throttler Throttler + name string + wantBody string + wantCode int + wantErr error + probeErr error + probeCode int + probeResp []string + throttler Throttler + tlsEnabled bool }{{ name: "active endpoint", wantBody: wantBody, wantCode: http.StatusOK, throttler: fakeThrottler{}, + }, { + name: "active TLS endpoint", + wantBody: wantBody, + wantCode: http.StatusOK, + throttler: fakeThrottler{}, + tlsEnabled: true, }, { name: "request error", wantBody: "request error\n", @@ -123,7 +130,7 @@ func TestActivationHandler(t *testing.T) { ctx, cancel, _ := rtesting.SetupFakeContextWithCancel(t) defer cancel() - handler := New(ctx, test.throttler, rt, false /*usePassthroughLb*/, logging.FromContext(ctx), false /* TLS */) + handler := New(ctx, test.throttler, rt, false /*usePassthroughLb*/, logging.FromContext(ctx), test.tlsEnabled /* TLS */) resp := httptest.NewRecorder() req := httptest.NewRequest(http.MethodPost, "http://example.com", nil) @@ -201,7 +208,7 @@ func TestActivationHandlerPassthroughLb(t *testing.T) { req := httptest.NewRequest(http.MethodPost, "http://example.com", nil) // Set up config store to populate context. - configStore := activatorconfig.NewStore(logging.FromContext(ctx)) + configStore := activatorconfig.NewStore(logging.FromContext(ctx), "disabled") configStore.OnConfigChanged(tracingConfig(false)) ctx = configStore.ToContext(req.Context()) ctx = WithRevisionAndID(ctx, nil, types.NamespacedName{Namespace: testNamespace, Name: testRevName}) @@ -324,7 +331,7 @@ func revision(namespace, name string) *v1.Revision { } func setupConfigStore(t testing.TB, logger *zap.SugaredLogger) *activatorconfig.Store { - configStore := activatorconfig.NewStore(logger) + configStore := activatorconfig.NewStore(logger, "disabled") configStore.OnConfigChanged(tracingConfig(false)) return configStore } diff --git a/pkg/activator/handler/tracing_handler_test.go b/pkg/activator/handler/tracing_handler_test.go index 55263066477e..73cd1f3f6018 100644 --- a/pkg/activator/handler/tracing_handler_test.go +++ b/pkg/activator/handler/tracing_handler_test.go @@ -63,7 +63,7 @@ func TestTracingHandler(t *testing.T) { t.Fatal("Failed to parse tracing config", err) } - configStore := activatorconfig.NewStore(logging.FromContext(ctx)) + configStore := activatorconfig.NewStore(logging.FromContext(ctx), "disabled") configStore.OnConfigChanged(cm) ctx = configStore.ToContext(ctx) diff --git a/pkg/activator/handler/transport.go b/pkg/activator/handler/transport.go new file mode 100644 index 000000000000..24932a6e0175 --- /dev/null +++ b/pkg/activator/handler/transport.go @@ -0,0 +1,160 @@ +/* +Copyright 2018 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "net" + "net/http" + "sync" + + "golang.org/x/net/http2" + "knative.dev/networking/pkg/certificates" + netcfg "knative.dev/networking/pkg/config" + pkgnet "knative.dev/pkg/network" + activatorconfig "knative.dev/serving/pkg/activator/config" +) + +func verifySanConnection(san string) func(tls.ConnectionState) error { + return func(cs tls.ConnectionState) error { + if len(cs.PeerCertificates) == 0 { + return errors.New("server certificate could not be verified during VerifyConnection. No PeerCertificates provided") + } + for _, name := range cs.PeerCertificates[0].DNSNames { + if name == san { + return nil + } + } + return fmt.Errorf("server with SAN %s does not have a matching name in certificate. Names provided in certificate: %s", san, cs.PeerCertificates[0].DNSNames) + } +} + +// tlsWrapper is a tls.Config wrapper with a TLS dialer for HTTP1 +type tlsWrapper struct { + tlsConf *tls.Config +} + +// dialTLSContextHTTP1 handles HTTPS:HTTP1 dialer +func (tw *tlsWrapper) dialTLSContextHTTP1(ctx context.Context, network, addr string) (net.Conn, error) { + return dialTLSContext(ctx, network, addr, tw.tlsConf) +} + +// dialTLSContext handles HTTPS:HTTP1 and HTTP2 dialers +// Depends on the activator handler's RevIDFrom +func dialTLSContext(ctx context.Context, network, addr string, tlsConf *tls.Config) (net.Conn, error) { + config := activatorconfig.FromContext(ctx) + trust := config.Trust + if trust != netcfg.TrustDisabled { + TLSConfLock() + tlsConf = tlsConf.Clone() + // Clone the certificate Pool such that the one used by the client will be different from the one that will get updated is CA is replaced. + tlsConf.RootCAs = tlsConf.RootCAs.Clone() + TLSConfUnlock() + san := certificates.LegacyFakeDnsName + if trust != netcfg.TrustMinimal { + revID := RevIDFrom(ctx) + san = certificates.DataPlaneUserName(revID.Namespace) + } + tlsConf.VerifyConnection = verifySanConnection(san) + } else { + tlsConf = nil + } + return pkgnet.DialTLSWithBackOff(ctx, network, addr, tlsConf) +} + +// newAutoTransport is a duplicate of the unexported knative/pkg's network newAutoTransport +func newAutoTransport(v1, v2 http.RoundTripper) http.RoundTripper { + return pkgnet.RoundTripperFunc(func(r *http.Request) (*http.Response, error) { + t := v1 + if r.ProtoMajor == 2 { + t = v2 + } + return t.RoundTrip(r) + }) +} + +// newHTTPTransport is a duplicate of the unexported knative/pkg's network newHTTPTransport +func newHTTPTransport(disableKeepAlives, disableCompression bool, maxIdle, maxIdlePerHost int) http.RoundTripper { + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.DialContext = pkgnet.DialWithBackOff + transport.DisableKeepAlives = disableKeepAlives + transport.MaxIdleConns = maxIdle + transport.MaxIdleConnsPerHost = maxIdlePerHost + transport.ForceAttemptHTTP2 = false + transport.DisableCompression = disableCompression + return transport +} + +// Depends on the activator handler's RevIDFrom +func newHTTPSTransport(disableKeepAlives, disableCompression bool, maxIdle, maxIdlePerHost int, tlsConf *tls.Config) http.RoundTripper { + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.DisableKeepAlives = disableKeepAlives + transport.MaxIdleConns = maxIdle + transport.MaxIdleConnsPerHost = maxIdlePerHost + transport.ForceAttemptHTTP2 = false + transport.DisableCompression = disableCompression + + tw := tlsWrapper{tlsConf: tlsConf} + transport.DialTLSContext = tw.dialTLSContextHTTP1 + + return transport +} + +// Depends on the activator handler's RevIDFrom +func newH2CTransport(disableCompression bool) http.RoundTripper { + return &http2.Transport{ + AllowHTTP: true, + DisableCompression: disableCompression, + DialTLSContext: dialTLSContext, + } +} + +// Depends on the activator handler's RevIDFrom +func newH2Transport(disableCompression bool, tlsConf *tls.Config) http.RoundTripper { + return &http2.Transport{ + DisableCompression: disableCompression, + DialTLSContext: dialTLSContext, + TLSClientConfig: tlsConf, + } +} + +// Depends on the activator handler's RevIDFrom +func NewProxyAutoTLSTransport(maxIdle, maxIdlePerHost int, tlsConf *tls.Config) http.RoundTripper { + return newAutoTransport( + newHTTPSTransport(false, true, maxIdle, maxIdlePerHost, tlsConf), + newH2Transport(true, tlsConf)) +} + +// Depends on the activator handler's RevIDFrom +func NewProxyAutoTransport(maxIdle, maxIdlePerHost int) http.RoundTripper { + return newAutoTransport( + newHTTPTransport(false, true, maxIdle, maxIdlePerHost), + newH2CTransport(true)) +} + +var certificatesMux sync.RWMutex + +func TLSConfLock() { + certificatesMux.Lock() +} + +func TLSConfUnlock() { + certificatesMux.Unlock() +} diff --git a/pkg/activator/handler/transport_test.go b/pkg/activator/handler/transport_test.go new file mode 100644 index 000000000000..e102049b99c9 --- /dev/null +++ b/pkg/activator/handler/transport_test.go @@ -0,0 +1,73 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "net/http" + "testing" + + "k8s.io/apimachinery/pkg/util/sets" +) + +type RoundTripperFunc func(*http.Request) (*http.Response, error) + +// RoundTrip implements http.RoundTripper. +func (rt RoundTripperFunc) RoundTrip(r *http.Request) (*http.Response, error) { + return rt(r) +} + +func TestHTTPRoundTripper(t *testing.T) { + wants := sets.NewString() + frt := func(key string) http.RoundTripper { + return RoundTripperFunc(func(r *http.Request) (*http.Response, error) { + wants.Insert(key) + return nil, nil + }) + } + + rt := newAutoTransport(frt("v1"), frt("v2")) + + examples := []struct { + label string + protoMajor int + want string + }{{ + label: "use default transport for HTTP1", + protoMajor: 1, + want: "v1", + }, { + label: "use h2c transport for HTTP2", + protoMajor: 2, + want: "v2", + }, { + label: "use default transport for all others", + protoMajor: 99, + want: "v1", + }} + + for _, e := range examples { + t.Run(e.label, func(t *testing.T) { + wants.Delete(e.want) + r := &http.Request{ProtoMajor: e.protoMajor} + rt.RoundTrip(r) + + if !wants.Has(e.want) { + t.Error("Wrong transport selected for request.") + } + }) + } +} diff --git a/pkg/reconciler/revision/resources/deploy.go b/pkg/reconciler/revision/resources/deploy.go index 8ba95784dc12..893017096324 100644 --- a/pkg/reconciler/revision/resources/deploy.go +++ b/pkg/reconciler/revision/resources/deploy.go @@ -38,6 +38,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" + netcfg "knative.dev/networking/pkg/config" apiconfig "knative.dev/serving/pkg/apis/config" ) @@ -193,7 +194,7 @@ func makePodSpec(rev *v1.Revision, cfg *config.Config) (*corev1.PodSpec, error) extraVolumes = append(extraVolumes, *tokenVolume) } - if cfg.Network.InternalEncryption { + if cfg.Network.DataplaneTrust != netcfg.TrustDisabled { queueContainer.VolumeMounts = append(queueContainer.VolumeMounts, varCertVolumeMount) extraVolumes = append(extraVolumes, certVolume(networking.ServingCertName)) } diff --git a/pkg/reconciler/revision/resources/queue_test.go b/pkg/reconciler/revision/resources/queue_test.go index 8c4f33a2721c..6ad9d552fd8b 100644 --- a/pkg/reconciler/revision/resources/queue_test.go +++ b/pkg/reconciler/revision/resources/queue_test.go @@ -998,9 +998,12 @@ func revConfig() *config.Config { Defaults: defaults, Features: &apicfg.Features{}, }, - Deployment: &deploymentConfig, - Logging: &logConfig, - Network: &netcfg.Config{}, + Deployment: &deploymentConfig, + Logging: &logConfig, + Network: &netcfg.Config{ + DataplaneTrust: netcfg.TrustDisabled, + ControlplaneTrust: netcfg.TrustDisabled, + }, Observability: &obsConfig, Tracing: &traceConfig, } diff --git a/pkg/reconciler/revision/revision.go b/pkg/reconciler/revision/revision.go index 69fa9275807f..7d2b38a88cd0 100644 --- a/pkg/reconciler/revision/revision.go +++ b/pkg/reconciler/revision/revision.go @@ -32,6 +32,7 @@ import ( "k8s.io/client-go/kubernetes" appsv1listers "k8s.io/client-go/listers/apps/v1" cachingclientset "knative.dev/caching/pkg/client/clientset/versioned" + netcfg "knative.dev/networking/pkg/config" clientset "knative.dev/serving/pkg/client/clientset/versioned" revisionreconciler "knative.dev/serving/pkg/client/injection/reconciler/serving/v1/revision" @@ -138,8 +139,8 @@ func (c *Reconciler) ReconcileKind(ctx context.Context, rev *v1.Revision) pkgrec logger.Debug("Revision meta: " + spew.Sdump(rev.ObjectMeta)) } - // Deploy certificate when internal-encryption is enabled. - if config.FromContext(ctx).Network.InternalEncryption { + // Deploy certificate when dataplane-trust is not disabled. + if config.FromContext(ctx).Network.DataplaneTrust != netcfg.TrustDisabled { if err := c.reconcileSecret(ctx, rev); err != nil { return err } diff --git a/pkg/reconciler/revision/table_test.go b/pkg/reconciler/revision/table_test.go index 026854c90267..5e8815625385 100644 --- a/pkg/reconciler/revision/table_test.go +++ b/pkg/reconciler/revision/table_test.go @@ -892,6 +892,9 @@ func reconcilerTestConfig() *config.Config { }, Logging: &logging.Config{}, Tracing: &tracingconfig.Config{}, - Network: &netcfg.Config{}, + Network: &netcfg.Config{ + DataplaneTrust: netcfg.TrustDisabled, + ControlplaneTrust: netcfg.TrustDisabled, + }, } }