feat(extension manager): dynamically reload CA certificates

go.mod

@@ -63,6 +63,7 @@ require (
 	golang.org/x/net v0.39.0
 	google.golang.org/genproto/googleapis/api v0.0.0-20250218202821-56aae31c358a
 	google.golang.org/grpc v1.71.1
+	google.golang.org/grpc/security/advancedtls v1.0.0
 	google.golang.org/protobuf v1.36.6
 	gopkg.in/yaml.v3 v3.0.1
 	helm.sh/helm/v3 v3.17.3

go.sum

@@ -1693,6 +1693,10 @@ google.golang.org/grpc v1.71.1 h1:ffsFWr7ygTUscGPI0KKK6TLrGz0476KUvvsbqWK0rPI=
 google.golang.org/grpc v1.71.1/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.5.1 h1:F29+wU6Ee6qgu9TddPgooOdaqsxTMunOoj8KA5yuS5A=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.5.1/go.mod h1:5KF+wpkbTSbGcR9zteSqZV6fqFOWBl4Yde8En8MryZA=
+google.golang.org/grpc/examples v0.0.0-20201112215255-90f1b3ee835b h1:NuxyvVZoDfHZwYW9LD4GJiF5/nhiSyP4/InTrvw9Ibk=
+google.golang.org/grpc/examples v0.0.0-20201112215255-90f1b3ee835b/go.mod h1:IBqQ7wSUJ2Ep09a8rMWFsg4fmI2r38zwsq8a0GgxXpM=
+google.golang.org/grpc/security/advancedtls v1.0.0 h1:/KQ7VP/1bs53/aopk9QhuPyFAp9Dm9Ejix3lzYkCrDA=
+google.golang.org/grpc/security/advancedtls v1.0.0/go.mod h1:o+s4go+e1PJ2AjuQMY5hU82W7lDlefjJA6FqEHRVHWk=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=

internal/extension/registry/extension_manager.go

@@ -17,6 +17,7 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/security/advancedtls"
 	"google.golang.org/grpc/test/bufconn"
 	corev1 "k8s.io/api/core/v1"
 	k8scli "sigs.k8s.io/controller-runtime/pkg/client"
@@ -245,18 +246,6 @@ func (m *Manager) CleanupHookConns() {
 	}
 }
 
-func parseCA(caSecret *corev1.Secret) (*x509.CertPool, error) {
-	caCertPEMBytes, ok := caSecret.Data[corev1.TLSCertKey]
-	if !ok {
-		return nil, errors.New("no cert found in CA secret")
-	}
-	cp := x509.NewCertPool()
-	if ok := cp.AppendCertsFromPEM(caCertPEMBytes); !ok {
-		return nil, errors.New("failed to append certificates")
-	}
-	return cp, nil
-}
-
 func setupGRPCOpts(ctx context.Context, client k8scli.Client, ext *egv1a1.ExtensionManager, namespace string) ([]grpc.DialOption, error) {
 	// These two errors shouldn't happen since we check these conditions when loading the extension
 	if ext == nil {
@@ -267,20 +256,16 @@ func setupGRPCOpts(ctx context.Context, client k8scli.Client, ext *egv1a1.Extens
 	}
 
 	var opts []grpc.DialOption
-	var creds credentials.TransportCredentials
 	if ext.Service.TLS != nil {
-		certRef := ext.Service.TLS.CertificateRef
-		secret, secretNamespace, err := kubernetes.ValidateSecretObjectReference(ctx, client, &certRef, namespace)
+		// Sanity check to ensure that the extension manager has a valid certificate reference
+		_, err := getCertPoolFromSecret(ctx, client, ext, namespace)
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("failed to get root CA certificates: %w", err)
 		}
-
-		cp, err := parseCA(secret)
+		creds, err := getGRPCCredentials(client, ext, namespace)
 		if err != nil {
-			return nil, fmt.Errorf("error parsing cert in Secret %s in namespace %s", string(certRef.Name), secretNamespace)
+			return nil, fmt.Errorf("failed to get gRPC TLS credentials: %w", err)
 		}
-
-		creds = credentials.NewClientTLSFromCert(cp, "")
 		opts = append(opts, grpc.WithTransportCredentials(creds))
 	} else {
 		opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials()))
@@ -300,3 +285,42 @@ func setupGRPCOpts(ctx context.Context, client k8scli.Client, ext *egv1a1.Extens
 
 	return opts, nil
 }
+
+func getGRPCCredentials(client k8scli.Client, ext *egv1a1.ExtensionManager, namespace string) (credentials.TransportCredentials, error) {
+	return advancedtls.NewClientCreds(&advancedtls.Options{
+		RootOptions: advancedtls.RootCertificateOptions{
+			// A callback function that dynamically loads root CA certificates from secret
+			GetRootCertificates: createGetRootCertificatesHandler(client, ext, namespace),
+		},
+	})
+}
+
+func createGetRootCertificatesHandler(client k8scli.Client, ext *egv1a1.ExtensionManager, namespace string) func(*advancedtls.ConnectionInfo) (*advancedtls.RootCertificates, error) {
+	return func(params *advancedtls.ConnectionInfo) (*advancedtls.RootCertificates, error) {
+		ctx := context.Background()
+		cp, err := getCertPoolFromSecret(ctx, client, ext, namespace)
+		if err != nil {
+			return nil, err
+		}
+
+		return &advancedtls.RootCertificates{TrustCerts: cp}, nil
+	}
+}
+
+func getCertPoolFromSecret(ctx context.Context, client k8scli.Client, ext *egv1a1.ExtensionManager, namespace string) (*x509.CertPool, error) {
+	certRef := ext.Service.TLS.CertificateRef
+	secret, _, err := kubernetes.ValidateSecretObjectReference(ctx, client, &certRef, namespace)
+	if err != nil {
+		return nil, fmt.Errorf("failed to validate TLS certificate reference: %w", err)
+	}
+
+	caCertPEMBytes, ok := secret.Data[corev1.TLSCertKey]
+	if !ok {
+		return nil, fmt.Errorf("no CA certificate found in Kubernetes Secret %s in namespace %s", secret.GetName(), secret.GetNamespace())
+	}
+	cp := x509.NewCertPool()
+	if ok := cp.AppendCertsFromPEM(caCertPEMBytes); !ok {
+		return nil, errors.New("failed to append certificates from CA secret")
+	}
+	return cp, nil
+}

internal/extension/registry/extension_manager_test.go

@@ -7,18 +7,28 @@ package registry
 
 import (
 	"context"
+	"crypto/tls"
+	"crypto/x509"
 	"fmt"
 	"math"
+	"net"
+	"os"
 	"testing"
 
+	v3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
 	"github.com/stretchr/testify/require"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/utils/ptr"
 	fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake"
+	gwapiv1 "sigs.k8s.io/gateway-api/apis/v1"
 
 	egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1"
 	"github.com/envoyproxy/gateway/internal/envoygateway"
+	"github.com/envoyproxy/gateway/proto/extension"
 )
 
 func TestGetExtensionServerAddress(t *testing.T) {
@@ -150,3 +160,96 @@ func Test_setupGRPCOpts(t *testing.T) {
 		})
 	}
 }
+
+type testServer struct {
+	extension.UnimplementedEnvoyGatewayExtensionServer
+}
+
+func (s *testServer) PostRouteModify(ctx context.Context, req *extension.PostRouteModifyRequest) (*extension.PostRouteModifyResponse, error) {
+	return &extension.PostRouteModifyResponse{
+		Route: req.Route,
+	}, nil
+}
+
+func Test_TLS(t *testing.T) {
+	testDir := "testdata"
+	caFile := testDir + "/ca.pem"
+	certFile := testDir + "/cert.pem"
+	keyFile := testDir + "/key.pem"
+
+	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
+	require.NoError(t, err)
+
+	caCert, err := os.ReadFile(caFile)
+	require.NoError(t, err)
+	caPool := x509.NewCertPool()
+	ok := caPool.AppendCertsFromPEM(caCert)
+	require.True(t, ok)
+
+	lis, err := net.Listen("tcp", "localhost:0")
+	require.NoError(t, err)
+	defer lis.Close()
+
+	port := lis.Addr().(*net.TCPAddr).Port
+	server := grpc.NewServer(grpc.Creds(credentials.NewTLS(&tls.Config{
+		Certificates: []tls.Certificate{cert},
+		ClientAuth:   tls.NoClientCert,
+		MinVersion:   tls.VersionTLS12,
+	})))
+	extension.RegisterEnvoyGatewayExtensionServer(server, &testServer{})
+	go func() {
+		_ = server.Serve(lis)
+		defer server.GracefulStop()
+	}()
+
+	extManager := &egv1a1.ExtensionManager{
+		Service: &egv1a1.ExtensionService{
+			BackendEndpoint: egv1a1.BackendEndpoint{
+				IP: &egv1a1.IPEndpoint{
+					Address: "localhost",
+					Port:    int32(port),
+				},
+			},
+			TLS: &egv1a1.ExtensionTLS{
+				CertificateRef: gwapiv1.SecretObjectReference{
+					Name:      "cert",
+					Namespace: ptr.To(gwapiv1.Namespace("default")),
+				},
+			},
+		},
+	}
+
+	secret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "cert",
+			Namespace: "default",
+		},
+		Type: corev1.SecretTypeTLS,
+		Data: map[string][]byte{
+			corev1.TLSCertKey: caCert,
+		},
+	}
+
+	fakeClient := fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects(secret).Build()
+
+	opts, err := setupGRPCOpts(context.Background(), fakeClient, extManager, "test-ns")
+	require.NoError(t, err)
+	require.NotEmpty(t, opts)
+
+	conn, err := grpc.DialContext(context.Background(), fmt.Sprintf("localhost:%d", port),
+		opts...,
+	)
+	require.NoError(t, err)
+	defer conn.Close()
+
+	client := extension.NewEnvoyGatewayExtensionClient(conn)
+	require.NotNil(t, client)
+
+	response, err := client.PostRouteModify(context.Background(), &extension.PostRouteModifyRequest{
+		Route: &v3.Route{
+			Name: "test-route",
+		},
+	})
+	require.NoError(t, err)
+	require.Equal(t, "test-route", response.Route.Name)
+}

internal/extension/registry/testdata/ca.pem

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDITCCAgmgAwIBAgIUGkbKo/xIQFgfIzSWmWIr6IIt6hEwDQYJKoZIhvcNAQEL
+BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MCAXDTI1MDQwNjEyMjEzOFoYDzIxMjUw
+MzEzMTIyMTM4WjAUMRIwEAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEB
+AQUAA4IBDwAwggEKAoIBAQDfkT1bhPFG90raDJLSU5nwVGWJK7mtiT+aMQzaBuHr
+hXQPWNWF+SflslO/tsVf67gX4eChUwXiUmoTA6Ahh9nlJ7fTWmK7QJkdcTjKv1N+
+DiEFWUvOpssviBfHavXkJjMsorBLBGCGS03q4xznDVYf9mfwUIiprkVBGQtYcH+M
+BBb4ws2SxhTaFK0rr9cwwavOf+omHiE0cvQBd73+G8nFEq8vKTnDsub3ozk+GSXf
+5Hu81reePUcU1lWzksik4U5ZcsL+/B4sU1TwEhL6zLhQ9kr3KVyyW2wVESQzKk4O
+XrYrPnQdULXWnAQBFPUOfiD2kHCGcylTZ1D4pcp4Q8EnAgMBAAGjaTBnMB0GA1Ud
+DgQWBBSkuvsTM4xhiNIzJBk0cCtfj4KhWzAfBgNVHSMEGDAWgBSkuvsTM4xhiNIz
+JBk0cCtfj4KhWzAPBgNVHRMBAf8EBTADAQH/MBQGA1UdEQQNMAuCCWxvY2FsaG9z
+dDANBgkqhkiG9w0BAQsFAAOCAQEAwk/AvTJcjmDQ5n0CQfqEUNnzHd24yPLob4rG
+GbrBgLO7YRS0l4ot6oTRau5PUAJ2eL0v3eGITaxZ+N7IWpa5Y0TFCcyFXBwtW9hq
+i2+v1CfMfz6+/HjbWq1VvgRwATMctRbcl1XGR1DNs6jKKakTBoYDped9wAkKeSRb
++4ZK989M4zCuwLwcej/sEhRdazuEykm+QgUSabb45CrSnJb4gOYd0CdOilXr98EQ
+Z2k4Hno434aD1CEIm4Fu7iwiKSR6TLFHHqC0bBd2aknB6DHLLMZMo1dXJchjQrkN
+EO9DtJxCTVRLTbEA0589h50fW8/L1GtwYgSo721fntphbgw3VA==
+-----END CERTIFICATE-----

internal/extension/registry/testdata/cert.pem

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDITCCAgmgAwIBAgIUGkbKo/xIQFgfIzSWmWIr6IIt6hEwDQYJKoZIhvcNAQEL
+BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MCAXDTI1MDQwNjEyMjEzOFoYDzIxMjUw
+MzEzMTIyMTM4WjAUMRIwEAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEB
+AQUAA4IBDwAwggEKAoIBAQDfkT1bhPFG90raDJLSU5nwVGWJK7mtiT+aMQzaBuHr
+hXQPWNWF+SflslO/tsVf67gX4eChUwXiUmoTA6Ahh9nlJ7fTWmK7QJkdcTjKv1N+
+DiEFWUvOpssviBfHavXkJjMsorBLBGCGS03q4xznDVYf9mfwUIiprkVBGQtYcH+M
+BBb4ws2SxhTaFK0rr9cwwavOf+omHiE0cvQBd73+G8nFEq8vKTnDsub3ozk+GSXf
+5Hu81reePUcU1lWzksik4U5ZcsL+/B4sU1TwEhL6zLhQ9kr3KVyyW2wVESQzKk4O
+XrYrPnQdULXWnAQBFPUOfiD2kHCGcylTZ1D4pcp4Q8EnAgMBAAGjaTBnMB0GA1Ud
+DgQWBBSkuvsTM4xhiNIzJBk0cCtfj4KhWzAfBgNVHSMEGDAWgBSkuvsTM4xhiNIz
+JBk0cCtfj4KhWzAPBgNVHRMBAf8EBTADAQH/MBQGA1UdEQQNMAuCCWxvY2FsaG9z
+dDANBgkqhkiG9w0BAQsFAAOCAQEAwk/AvTJcjmDQ5n0CQfqEUNnzHd24yPLob4rG
+GbrBgLO7YRS0l4ot6oTRau5PUAJ2eL0v3eGITaxZ+N7IWpa5Y0TFCcyFXBwtW9hq
+i2+v1CfMfz6+/HjbWq1VvgRwATMctRbcl1XGR1DNs6jKKakTBoYDped9wAkKeSRb
++4ZK989M4zCuwLwcej/sEhRdazuEykm+QgUSabb45CrSnJb4gOYd0CdOilXr98EQ
+Z2k4Hno434aD1CEIm4Fu7iwiKSR6TLFHHqC0bBd2aknB6DHLLMZMo1dXJchjQrkN
+EO9DtJxCTVRLTbEA0589h50fW8/L1GtwYgSo721fntphbgw3VA==
+-----END CERTIFICATE-----

internal/extension/registry/testdata/key.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDfkT1bhPFG90ra
+DJLSU5nwVGWJK7mtiT+aMQzaBuHrhXQPWNWF+SflslO/tsVf67gX4eChUwXiUmoT
+A6Ahh9nlJ7fTWmK7QJkdcTjKv1N+DiEFWUvOpssviBfHavXkJjMsorBLBGCGS03q
+4xznDVYf9mfwUIiprkVBGQtYcH+MBBb4ws2SxhTaFK0rr9cwwavOf+omHiE0cvQB
+d73+G8nFEq8vKTnDsub3ozk+GSXf5Hu81reePUcU1lWzksik4U5ZcsL+/B4sU1Tw
+EhL6zLhQ9kr3KVyyW2wVESQzKk4OXrYrPnQdULXWnAQBFPUOfiD2kHCGcylTZ1D4
+pcp4Q8EnAgMBAAECggEAad3jCkWH460SuvKdTA/eo5mzgyM7h9uoTKHRjiUYMmk9
+rk7IvepgZDrvwacRC/0ZbeW+c2ZXMPcJ1/sthvNH0WXffKweeMF8QB7vX8L+DIEw
+TdIdPbxqFxYA/KO/0MvuC8L2bG+kLbPXL9VgSGFeWUBbs8bPEFq/3LW5U6XiIDCk
++tVmyjL8YgqFouzrMvfdy1CHChmC3Kk9vPlJIJJ90ru7+p1l7NK8fYMWe49CKNcR
+ayfPLedI3zNQwbywkpvFebzAM04/YGDMqwhyfajc0hHOpvroWA3/UNIssvHLULk6
+K4INTcMVn+F1PI4+iDEneWJvrjU02p0MjgM7XeOEkQKBgQDzvgQNDGL2NUwmqfia
+/qvqusWDVyH9DdCuPilt0GsFcvjdNhD1Jvp3V/B5bqFclR8+iPa71beYVqzGjVxp
+b+B5nJ2BMQ0D6p9ItZ0czO+ZHj1nQI13m90lG1D/Yfu9EQNpnDCUP7jL3oE3Aw+U
+7vZOkGI4JuDI4vBpXEr+XRsOBQKBgQDqz3z+djzKOzR1TcZ+qsrUkWhSKhDngLIf
+sz6wUE535Ebvu4u3gK9RethZK7ogkNxI4aN7g3Dr8k+jEflPZCbbuGjhyRA5mQal
+EUj7V5e17tgZC4LwXVVAd/b4grrMKfJD5CptBjmRFhbU/mNjiN+C/VpPiABLWzc+
+5KJjK1pOOwKBgQCOmY65K5QCQ3BH9o0x8OkXrSm9C48hA6IhLtECJDtYtskOcoE9
+TA6hH9vaz3SsO7pJ1cu0XPbKs3rltvJn5UJJI+2qFc5tiiiiW06N+P/8bwqxi7y4
+S4H4IaEjqGmlVXzPnsd1FErDS2wBiVLmaV/E2wf7nhNItCy/F3XwlvwrGQKBgQC4
+wusnjwHmXw+3/arioKFZAdGEVXVXs/x01SXOtmIIFKd8m6Ykji1lf7Qc9jtOxK2Q
+63soBRUlk1T2i34Q6k1pNoHQp9UMfUytNhynKRVHItkHW/d8CvS59atvf+5cF+V5
+Zl+7ydoiP69XiZPLDjhRaelWCz2wDeY6pszgG4zDYQKBgHgagAqV3qlWScVJBf6F
+FyqRjt3aXVxM11f8LakXTI/3crFFdWp33HGhEw8Pv5EKu4ynJnBbYc4EJ8N1KfTB
+iZ6rSpNYhkB6JRsSBO/RZjcceHc4wcyJkBciD3G+ZhwI5Ydrgt4/+pJX/9fA0wks
+DF883l7zYtiBvGQXpKYwup6t
+-----END PRIVATE KEY-----