Implement a cert watcher to use new certificates without restarting

go.mod

@@ -27,6 +27,7 @@ require (
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/emicklei/go-restful/v3 v3.12.2 // indirect
+	github.com/fsnotify/fsnotify v1.9.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.9.0 // indirect
 	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
@@ -66,6 +67,7 @@ require (
 	gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/apiextensions-apiserver v0.34.0 // indirect
 	k8s.io/component-base v0.35.0 // indirect
 	k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 // indirect
 	sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect

go.sum

@@ -14,6 +14,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU=
 github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
+github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
+github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
 github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM=
 github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
@@ -177,6 +179,8 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 k8s.io/api v0.35.0 h1:iBAU5LTyBI9vw3L5glmat1njFK34srdLmktWwLTprlY=
 k8s.io/api v0.35.0/go.mod h1:AQ0SNTzm4ZAczM03QH42c7l3bih1TbAXYo0DkF8ktnA=
+k8s.io/apiextensions-apiserver v0.35.0 h1:3xHk2rTOdWXXJM+RDQZJvdx0yEOgC0FgQ1PlJatA5T4=
+k8s.io/apiextensions-apiserver v0.35.0/go.mod h1:E1Ahk9SADaLQ4qtzYFkwUqusXTcaV2uw3l14aqpL2LU=
 k8s.io/apimachinery v0.35.0 h1:Z2L3IHvPVv/MJ7xRxHEtk6GoJElaAqDCCU0S6ncYok8=
 k8s.io/apimachinery v0.35.0/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns=
 k8s.io/apiserver v0.35.0 h1:CUGo5o+7hW9GcAEF3x3usT3fX4f9r8xmgQeCBDaOgX4=

pkg/internal/server/grpc/server.go

@@ -18,6 +18,7 @@ package grpc
 
 import (
 	"crypto/tls"
+	"errors"
 	"fmt"
 	"net"
 	"strconv"
@@ -26,6 +27,7 @@ import (
 	"time"
 
 	snapshot "github.com/kubernetes-csi/external-snapshotter/client/v8/clientset/versioned"
+	cw "github.com/kubernetes-csi/external-snapshotter/v8/pkg/webhook"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/health"
@@ -53,6 +55,8 @@ type ServerConfig struct {
 	// the client do not complete in this time.
 	// If not set then HandlerDefaultMaxStreamDuration is used.
 	MaxStreamDur time.Duration
+
+	Certwatcher *cw.CertWatcher
 }
 
 type Server struct {
@@ -65,18 +69,26 @@ type Server struct {
 }
 
 func NewServer(config ServerConfig) (*Server, error) {
-	options, err := buildOptions(config)
-	if err != nil {
-		return nil, err
-	}
-
 	if config.MaxStreamDur <= 0 {
 		config.MaxStreamDur = HandlerDefaultMaxStreamDuration
 	}
 
+	if config.Certwatcher == nil {
+		return nil, errors.New("the certificate watcher/provider for the gRPC server is unset.")
+	}
+
 	return &Server{
-		config:       config,
-		grpcServer:   grpc.NewServer(options...),
+		config: config,
+		grpcServer: grpc.NewServer(
+			grpc.Creds(
+				credentials.NewTLS(
+					&tls.Config{
+						GetCertificate: config.Certwatcher.GetCertificate,
+						ClientAuth:     tls.NoClientCert,
+					},
+				),
+			),
+		),
 		healthServer: newHealthServer(),
 	}, nil
 }
@@ -105,31 +117,6 @@ func (s *Server) audience() string {
 	return s.config.Runtime.Audience
 }
 
-func buildOptions(config ServerConfig) ([]grpc.ServerOption, error) {
-	tlsOptions, err := buildTLSOption(config.Runtime.TLSCertFile, config.Runtime.TLSKeyFile)
-	if err != nil {
-		return nil, err
-	}
-
-	return []grpc.ServerOption{
-		tlsOptions,
-	}, nil
-}
-
-func buildTLSOption(cert, key string) (grpc.ServerOption, error) {
-	serverCert, err := tls.LoadX509KeyPair(cert, key)
-	if err != nil {
-		return nil, fmt.Errorf("failed to load tls certificates: %v", err)
-	}
-
-	config := &tls.Config{
-		Certificates: []tls.Certificate{serverCert},
-		ClientAuth:   tls.NoClientCert,
-	}
-
-	return grpc.Creds(credentials.NewTLS(config)), nil
-}
-
 // Start start the gRPC server in its own goroutine.
 // The method guarantees that on successful return the gRPC server goroutine is running.
 // The invoker should use the Stop() method to terminate the server when desired.

pkg/internal/server/grpc/server_test.go

@@ -22,6 +22,7 @@ import (
 	"testing"
 	"time"
 
+	cw "github.com/kubernetes-csi/external-snapshotter/v8/pkg/webhook"
 	"github.com/stretchr/testify/assert"
 	"google.golang.org/grpc/codes"
 	healthpb "google.golang.org/grpc/health/grpc_health_v1"
@@ -56,7 +57,13 @@ func TestNewServer(t *testing.T) {
 		rt.TLSCertFile = rta.TLSCertFile
 		rt.TLSKeyFile = rta.TLSKeyFile + "foo" // invalid path
 
-		server, err := NewServer(ServerConfig{Runtime: &rt})
+		// Should fail to load the invalid cert
+		cw, err := cw.NewCertWatcher(rt.TLSCertFile, rt.TLSKeyFile)
+		assert.Error(t, err)
+		assert.Nil(t, cw)
+
+		// Show fail to start due to missing certwatcher
+		server, err := NewServer(ServerConfig{Runtime: &rt, Certwatcher: cw})
 		assert.Error(t, err)
 		assert.Nil(t, server)
 	})
@@ -71,7 +78,11 @@ func TestNewServer(t *testing.T) {
 		rt.TLSKeyFile = rta.TLSKeyFile
 		rt.GRPCPort = -1 // invalid port
 
-		s, err := NewServer(ServerConfig{Runtime: &rt})
+		cw, err := cw.NewCertWatcher(rt.TLSCertFile, rt.TLSKeyFile)
+		assert.NoError(t, err)
+		assert.NotNil(t, cw)
+
+		s, err := NewServer(ServerConfig{Runtime: &rt, Certwatcher: cw})
 		assert.NoError(t, err)
 		assert.NotNil(t, s)
 		assert.NotNil(t, s.grpcServer)
@@ -91,10 +102,15 @@ func TestNewServer(t *testing.T) {
 		rt.TLSCertFile = rta.TLSCertFile
 		rt.TLSKeyFile = rta.TLSKeyFile
 
+		cw, err := cw.NewCertWatcher(rt.TLSCertFile, rt.TLSKeyFile)
+		assert.NoError(t, err)
+		assert.NotNil(t, cw)
+
 		expMaxStreamDur := HandlerDefaultMaxStreamDuration + time.Minute
 		s, err := NewServer(ServerConfig{
 			Runtime:      &rt,
 			MaxStreamDur: expMaxStreamDur,
+			Certwatcher:  cw,
 		})
 		assert.NoError(t, err)
 		assert.NotNil(t, s)

pkg/sidecar/sidecar.go

@@ -17,6 +17,7 @@ limitations under the License.
 package sidecar
 
 import (
+	"context"
 	"flag"
 	"fmt"
 	"net/http"
@@ -26,6 +27,7 @@ import (
 	"syscall"
 	"time"
 
+	cw "github.com/kubernetes-csi/external-snapshotter/v8/pkg/webhook"
 	"k8s.io/klog/v2"
 
 	"github.com/kubernetes-csi/external-snapshot-metadata/pkg/internal/runtime"
@@ -90,7 +92,17 @@ func Run(argv []string, version string) int {
 
 	klog.Infof("CSI driver name: %q", rt.DriverName)
 
-	grpcServer, err := startGRPCServerAndValidateCSIDriver(s.createServerConfig(rt))
+	// Setup a certificate watcher
+	ctx, cancel := context.WithCancel(context.TODO())
+	defer cancel()
+
+	cw, err := cw.NewCertWatcher(rt.TLSCertFile, rt.TLSKeyFile)
+	if err != nil {
+		klog.Errorf("failed to start certwatcher: %v", err)
+		return 1
+	}
+
+	grpcServer, err := startGRPCServerAndValidateCSIDriver(s.createServerConfig(rt, cw))
 	if err != nil {
 		klog.Error(err)
 		return 1
@@ -123,6 +135,13 @@ func Run(argv []string, version string) int {
 		}
 	}()
 
+	// Dispatch the go routine for certificate watcher
+	go func() {
+		if err := cw.Start(ctx); err != nil {
+			klog.Errorf("error in certificate watcher: %v", err)
+		}
+	}()
+
 	shutdownOnTerminationSignal(grpcServer)
 
 	return 0
@@ -265,10 +284,11 @@ func (s *sidecarFlagSet) runtimeArgsToArgv(progName string, rta runtime.Args) []
 	return argv
 }
 
-func (s *sidecarFlagSet) createServerConfig(rt *runtime.Runtime) grpc.ServerConfig {
+func (s *sidecarFlagSet) createServerConfig(rt *runtime.Runtime, cw *cw.CertWatcher) grpc.ServerConfig {
 	return grpc.ServerConfig{
 		Runtime:      rt,
 		MaxStreamDur: time.Duration(*s.maxStreamingDurMin) * time.Minute,
+		Certwatcher:  cw,
 	}
 }
 

pkg/sidecar/sidecar_test.go

@@ -31,6 +31,7 @@ import (
 	"time"
 
 	"github.com/container-storage-interface/spec/lib/go/csi"
+	cw "github.com/kubernetes-csi/external-snapshotter/v8/pkg/webhook"
 	"github.com/stretchr/testify/assert"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
@@ -119,7 +120,7 @@ func TestSidecarFlagSet(t *testing.T) {
 		assert.Equal(t, expRTA, rta)
 
 		rt := &runtime.Runtime{}
-		config := sfs.createServerConfig(rt)
+		config := sfs.createServerConfig(rt, nil)
 		assert.Equal(t, rt, config.Runtime)
 		assert.Equal(t, time.Duration(defaultMaxStreamingDurationMin)*time.Minute, config.MaxStreamDur)
 	})
@@ -157,7 +158,7 @@ func TestSidecarFlagSet(t *testing.T) {
 		assert.Equal(t, expRTA, rta)
 
 		rt := &runtime.Runtime{}
-		config := sfs.createServerConfig(rt)
+		config := sfs.createServerConfig(rt, nil)
 		assert.Equal(t, rt, config.Runtime)
 		assert.Equal(t, time.Duration(defaultMaxStreamingDurationMin)*time.Minute, config.MaxStreamDur)
 	})
@@ -188,9 +189,15 @@ func TestStartGRPCServerAndValidateCSIDriver(t *testing.T) {
 
 		rt := rth.RuntimeForFakeCSIDriver(t)
 
+		rt.TLSCertFile = rth.RuntimeArgs().TLSCertFile
+		rt.TLSKeyFile = rth.RuntimeArgs().TLSKeyFile
 		rt.GRPCPort = -1 // invalid port
 
-		s, err := startGRPCServerAndValidateCSIDriver(grpc.ServerConfig{Runtime: rt})
+		cw, err := cw.NewCertWatcher(rt.TLSCertFile, rt.TLSKeyFile)
+		assert.NoError(t, err)
+		assert.NotNil(t, cw)
+
+		s, err := startGRPCServerAndValidateCSIDriver(grpc.ServerConfig{Runtime: rt, Certwatcher: cw})
 		assert.Error(t, err)
 		assert.Contains(t, err.Error(), "invalid port")
 		assert.Nil(t, s)
@@ -204,7 +211,14 @@ func TestStartGRPCServerAndValidateCSIDriver(t *testing.T) {
 
 		rt := rth.RuntimeForFakeCSIDriver(t)
 
-		s, err := startGRPCServerAndValidateCSIDriver(grpc.ServerConfig{Runtime: rt})
+		rt.TLSCertFile = rth.RuntimeArgs().TLSCertFile
+		rt.TLSKeyFile = rth.RuntimeArgs().TLSKeyFile
+
+		cw, err := cw.NewCertWatcher(rt.TLSCertFile, rt.TLSKeyFile)
+		assert.NoError(t, err)
+		assert.NotNil(t, cw)
+
+		s, err := startGRPCServerAndValidateCSIDriver(grpc.ServerConfig{Runtime: rt, Certwatcher: cw})
 		assert.Error(t, err)
 		assert.Contains(t, err.Error(), "error waiting for CSI driver to become ready") // probe unimplemented.
 		assert.Nil(t, s)