diff --git a/vendor/k8s.io/kubernetes/cmd/kubelet/app/server.go b/vendor/k8s.io/kubernetes/cmd/kubelet/app/server.go index 64d48f1deffa..95c4bd9d5009 100644 --- a/vendor/k8s.io/kubernetes/cmd/kubelet/app/server.go +++ b/vendor/k8s.io/kubernetes/cmd/kubelet/app/server.go @@ -50,12 +50,12 @@ import ( utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/apiserver/pkg/util/flag" clientset "k8s.io/client-go/kubernetes" + certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" v1core "k8s.io/client-go/kubernetes/typed/core/v1" restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/tools/record" certutil "k8s.io/client-go/util/cert" - "k8s.io/client-go/util/certificate" "k8s.io/kubernetes/cmd/kubelet/app/options" "k8s.io/kubernetes/pkg/api/legacyscheme" api "k8s.io/kubernetes/pkg/apis/core" @@ -544,12 +544,6 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan return err } - if s.BootstrapKubeconfig != "" { - if err := bootstrap.LoadClientCert(s.KubeConfig, s.BootstrapKubeconfig, s.CertDirectory, nodeName); err != nil { - return err - } - } - // if in standalone mode, indicate as much by setting all clients to nil if standaloneMode { kubeDeps.KubeClient = nil @@ -557,73 +551,39 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan kubeDeps.EventClient = nil kubeDeps.HeartbeatClient = nil glog.Warningf("standalone mode, no API client") - } else if kubeDeps.KubeClient == nil || kubeDeps.ExternalKubeClient == nil || kubeDeps.EventClient == nil || kubeDeps.HeartbeatClient == nil { - // initialize clients if not standalone mode and any of the clients are not provided - var kubeClient clientset.Interface - var eventClient v1core.EventsGetter - var heartbeatClient v1core.CoreV1Interface - var externalKubeClient clientset.Interface - clientConfig, err := createAPIServerClientConfig(s) - if err != nil { - return fmt.Errorf("invalid kubeconfig: %v", err) - } + } else if kubeDeps.KubeClient == nil || kubeDeps.ExternalKubeClient == nil || kubeDeps.EventClient == nil || kubeDeps.HeartbeatClient == nil { - var clientCertificateManager certificate.Manager - if s.RotateCertificates && utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletClientCertificate) { - clientCertificateManager, err = kubeletcertificate.NewKubeletClientCertificateManager(s.CertDirectory, nodeName, clientConfig.CertData, clientConfig.KeyData, clientConfig.CertFile, clientConfig.KeyFile) - if err != nil { - return err - } - } - // we set exitAfter to five minutes because we use this client configuration to request new certs - if we are unable - // to request new certs, we will be unable to continue normal operation. Exiting the process allows a wrapper - // or the bootstrapping credentials to potentially lay down new initial config. - closeAllConns, err := kubeletcertificate.UpdateTransport(wait.NeverStop, clientConfig, clientCertificateManager, 5*time.Minute) + clientConfig, closeAllConns, err := buildKubeletClientConfig(s, nodeName) if err != nil { return err } + kubeDeps.OnHeartbeatFailure = closeAllConns - kubeClient, err = clientset.NewForConfig(clientConfig) + kubeDeps.KubeClient, err = clientset.NewForConfig(clientConfig) if err != nil { - glog.Warningf("New kubeClient from clientConfig error: %v", err) - } else if kubeClient.CertificatesV1beta1() != nil && clientCertificateManager != nil { - glog.V(2).Info("Starting client certificate rotation.") - clientCertificateManager.SetCertificateSigningRequestClient(kubeClient.CertificatesV1beta1().CertificateSigningRequests()) - clientCertificateManager.Start() - } - externalKubeClient, err = clientset.NewForConfig(clientConfig) - if err != nil { - glog.Warningf("New kubeClient from clientConfig error: %v", err) + return fmt.Errorf("failed to initialize kubelet client: %v", err) } + kubeDeps.ExternalKubeClient = kubeDeps.KubeClient // make a separate client for events eventClientConfig := *clientConfig eventClientConfig.QPS = float32(s.EventRecordQPS) eventClientConfig.Burst = int(s.EventBurst) - eventClient, err = v1core.NewForConfig(&eventClientConfig) + kubeDeps.EventClient, err = v1core.NewForConfig(&eventClientConfig) if err != nil { - glog.Warningf("Failed to create API Server client for Events: %v", err) + return fmt.Errorf("failed to initialize kubelet event client: %v", err) } // make a separate client for heartbeat with throttling disabled and a timeout attached heartbeatClientConfig := *clientConfig heartbeatClientConfig.Timeout = s.KubeletConfiguration.NodeStatusUpdateFrequency.Duration heartbeatClientConfig.QPS = float32(-1) - heartbeatClient, err = v1core.NewForConfig(&heartbeatClientConfig) + heartbeatClient, err := clientset.NewForConfig(&heartbeatClientConfig) if err != nil { - glog.Warningf("Failed to create API Server client for heartbeat: %v", err) - } - - kubeDeps.KubeClient = kubeClient - kubeDeps.ExternalKubeClient = externalKubeClient - if heartbeatClient != nil { - kubeDeps.HeartbeatClient = heartbeatClient - kubeDeps.OnHeartbeatFailure = closeAllConns - } - if eventClient != nil { - kubeDeps.EventClient = eventClient + return fmt.Errorf("failed to initialize kubelet heartbeat client: %v", err) } + kubeDeps.HeartbeatClient = heartbeatClient.CoreV1() } // If the kubelet config controller is available, and dynamic config is enabled, start the config and status sync loops @@ -762,6 +722,74 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan return nil } +// buildKubeletClientConfig constructs the appropriate client config for the kubelet depending on whether +// bootstrapping is enabled or client certificate rotation is enabled. +func buildKubeletClientConfig(s *options.KubeletServer, nodeName types.NodeName) (*restclient.Config, func(), error) { + if s.RotateCertificates && utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletClientCertificate) { + certConfig, clientConfig, err := bootstrap.LoadClientConfig(s.KubeConfig, s.BootstrapKubeconfig, s.CertDirectory) + if err != nil { + return nil, nil, err + } + + newClientFn := func(current *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) { + // If we have a valid certificate, use that to fetch CSRs. Otherwise use the bootstrap + // credentials. + // XXX: When an external bootstrap source is available, it should be possible to always use that source + // to retrieve new credentials. + config := certConfig + if current != nil { + config = clientConfig + } + client, err := clientset.NewForConfig(config) + if err != nil { + return nil, err + } + return client.CertificatesV1beta1().CertificateSigningRequests(), nil + } + + clientCertificateManager, err := kubeletcertificate.NewKubeletClientCertificateManager( + s.CertDirectory, + nodeName, + clientConfig.CertFile, + clientConfig.KeyFile, + newClientFn, + ) + if err != nil { + return nil, nil, err + } + + // the rotating transport will use the cert from the cert manager instead of these files + transportConfig := *clientConfig + transportConfig.CertFile = "" + transportConfig.KeyFile = "" + + // we set exitAfter to five minutes because we use this client configuration to request new certs - if we are unable + // to request new certs, we will be unable to continue normal operation. Exiting the process allows a wrapper + // or the bootstrapping credentials to potentially lay down new initial config. + closeAllConns, err := kubeletcertificate.UpdateTransport(wait.NeverStop, &transportConfig, clientCertificateManager, 5*time.Minute) + if err != nil { + return nil, nil, err + } + + glog.V(2).Info("Starting client certificate rotation.") + clientCertificateManager.Start() + + return &transportConfig, closeAllConns, nil + } + + if len(s.BootstrapKubeconfig) > 0 { + if err := bootstrap.LoadClientCert(s.KubeConfig, s.BootstrapKubeconfig, s.CertDirectory, nodeName); err != nil { + return nil, nil, err + } + } + + clientConfig, err := createAPIServerClientConfig(s) + if err != nil { + return nil, nil, fmt.Errorf("invalid kubeconfig: %v", err) + } + return clientConfig, nil, nil +} + // getNodeName returns the node name according to the cloud provider // if cloud provider is specified. Otherwise, returns the hostname of the node. func getNodeName(cloud cloudprovider.Interface, hostname string) (types.NodeName, error) { @@ -856,11 +884,10 @@ func kubeconfigClientConfig(s *options.KubeletServer) (*restclient.Config, error // createClientConfig creates a client configuration from the command line arguments. // If --kubeconfig is explicitly set, it will be used. func createClientConfig(s *options.KubeletServer) (*restclient.Config, error) { - if s.BootstrapKubeconfig != "" || len(s.KubeConfig) > 0 { + if len(s.BootstrapKubeconfig) > 0 || len(s.KubeConfig) > 0 { return kubeconfigClientConfig(s) - } else { - return nil, fmt.Errorf("createClientConfig called in standalone mode") } + return nil, fmt.Errorf("createClientConfig called in standalone mode") } // createAPIServerClientConfig generates a client.Config from command line flags diff --git a/vendor/k8s.io/kubernetes/pkg/kubelet/certificate/bootstrap/bootstrap.go b/vendor/k8s.io/kubernetes/pkg/kubelet/certificate/bootstrap/bootstrap.go index 350e4ed212b4..84b482f24877 100644 --- a/vendor/k8s.io/kubernetes/pkg/kubelet/certificate/bootstrap/bootstrap.go +++ b/vendor/k8s.io/kubernetes/pkg/kubelet/certificate/bootstrap/bootstrap.go @@ -38,6 +38,55 @@ import ( const tmpPrivateKeyFile = "kubelet-client.key.tmp" +// LoadClientConfig tries to load the appropriate client config for retrieving certs and for use by users. +// If bootstrapPath is empty, only kubeconfigPath is checked. If bootstrap path is set and the contents +// of kubeconfigPath are valid, both certConfig and userConfig will point to that file. Otherwise the +// kubeconfigPath on disk is populated based on bootstrapPath but pointing to the location of the client cert +// in certDir. This preserves the historical behavior of bootstrapping where on subsequent restarts the +// most recent client cert is used to request new client certs instead of the initial token. +func LoadClientConfig(kubeconfigPath string, bootstrapPath string, certDir string) (certConfig, userConfig *restclient.Config, err error) { + if len(bootstrapPath) == 0 { + clientConfig, err := loadRESTClientConfig(kubeconfigPath) + if err != nil { + return nil, nil, fmt.Errorf("unable to load kubeconfig: %v", err) + } + return clientConfig, clientConfig, nil + } + + store, err := certificate.NewFileStore("kubelet-client", certDir, certDir, "", "") + if err != nil { + return nil, nil, fmt.Errorf("unable to build bootstrap cert store") + } + + ok, err := verifyBootstrapClientConfig(kubeconfigPath) + if err != nil { + return nil, nil, err + } + + // use the current client config + if ok { + clientConfig, err := loadRESTClientConfig(kubeconfigPath) + if err != nil { + return nil, nil, fmt.Errorf("unable to load kubeconfig: %v", err) + } + return clientConfig, clientConfig, nil + } + + bootstrapClientConfig, err := loadRESTClientConfig(bootstrapPath) + if err != nil { + return nil, nil, fmt.Errorf("unable to load bootstrap kubeconfig: %v", err) + } + + clientConfig := restclient.AnonymousClientConfig(bootstrapClientConfig) + pemPath := store.CurrentPath() + clientConfig.KeyFile = pemPath + clientConfig.CertFile = pemPath + if err := writeKubeconfigFromBootstrapping(clientConfig, kubeconfigPath, pemPath); err != nil { + return nil, nil, err + } + return bootstrapClientConfig, clientConfig, nil +} + // LoadClientCert requests a client cert for kubelet if the kubeconfigPath file does not exist. // The kubeconfig at bootstrapPath is used to request a client certificate from the API server. // On success, a kubeconfig file referencing the generated key and obtained certificate is written to kubeconfigPath. @@ -59,6 +108,7 @@ func LoadClientCert(kubeconfigPath string, bootstrapPath string, certDir string, if err != nil { return fmt.Errorf("unable to load bootstrap kubeconfig: %v", err) } + bootstrapClient, err := certificates.NewForConfig(bootstrapClientConfig) if err != nil { return fmt.Errorf("unable to create certificates signing request client: %v", err) @@ -103,8 +153,10 @@ func LoadClientCert(kubeconfigPath string, bootstrapPath string, certDir string, glog.V(2).Infof("failed cleaning up private key file %q: %v", privKeyPath, err) } - pemPath := store.CurrentPath() + return writeKubeconfigFromBootstrapping(bootstrapClientConfig, kubeconfigPath, store.CurrentPath()) +} +func writeKubeconfigFromBootstrapping(bootstrapClientConfig *restclient.Config, kubeconfigPath, pemPath string) error { // Get the CA data from the bootstrap client config. caFile, caData := bootstrapClientConfig.CAFile, []byte{} if len(caFile) == 0 { diff --git a/vendor/k8s.io/kubernetes/pkg/kubelet/certificate/kubelet.go b/vendor/k8s.io/kubernetes/pkg/kubelet/certificate/kubelet.go index b8530074c44f..850c104fd575 100644 --- a/vendor/k8s.io/kubernetes/pkg/kubelet/certificate/kubelet.go +++ b/vendor/k8s.io/kubernetes/pkg/kubelet/certificate/kubelet.go @@ -17,14 +17,17 @@ limitations under the License. package certificate import ( + "crypto/tls" "crypto/x509" "crypto/x509/pkix" "fmt" "net" + "sort" "github.com/prometheus/client_golang/prometheus" certificates "k8s.io/api/certificates/v1beta1" + "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" clientset "k8s.io/client-go/kubernetes" clientcertificates "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" @@ -60,7 +63,9 @@ func NewKubeletServerCertificateManager(kubeClient clientset.Interface, kubeCfg prometheus.MustRegister(certificateExpiration) m, err := certificate.NewManager(&certificate.Config{ - CertificateSigningRequestClient: certSigningRequestClient, + ClientFn: func(current *tls.Certificate) (clientcertificates.CertificateSigningRequestInterface, error) { + return certSigningRequestClient, nil + }, Template: &x509.CertificateRequest{ Subject: pkix.Name{ CommonName: fmt.Sprintf("system:node:%s", nodeName), @@ -92,11 +97,48 @@ func NewKubeletServerCertificateManager(kubeClient clientset.Interface, kubeCfg return m, nil } +func addressesToHostnamesAndIPs(addresses []v1.NodeAddress) (dnsNames []string, ips []net.IP) { + seenDNSNames := map[string]bool{} + seenIPs := map[string]bool{} + for _, address := range addresses { + if len(address.Address) == 0 { + continue + } + + switch address.Type { + case v1.NodeHostName: + if ip := net.ParseIP(address.Address); ip != nil { + seenIPs[address.Address] = true + } else { + seenDNSNames[address.Address] = true + } + case v1.NodeExternalIP, v1.NodeInternalIP: + if ip := net.ParseIP(address.Address); ip != nil { + seenIPs[address.Address] = true + } + case v1.NodeExternalDNS, v1.NodeInternalDNS: + seenDNSNames[address.Address] = true + } + } + + for dnsName := range seenDNSNames { + dnsNames = append(dnsNames, dnsName) + } + for ip := range seenIPs { + ips = append(ips, net.ParseIP(ip)) + } + + // return in stable order + sort.Strings(dnsNames) + sort.Slice(ips, func(i, j int) bool { return ips[i].String() < ips[j].String() }) + + return dnsNames, ips +} + // NewKubeletClientCertificateManager sets up a certificate manager without a -// client that can be used to sign new certificates (or rotate). It answers with -// whatever certificate it is initialized with. If a CSR client is set later, it -// may begin rotating/renewing the client cert -func NewKubeletClientCertificateManager(certDirectory string, nodeName types.NodeName, certData []byte, keyData []byte, certFile string, keyFile string) (certificate.Manager, error) { +// client that can be used to sign new certificates (or rotate). If a CSR +// client is set later, it may begin rotating/renewing the client cert. +func NewKubeletClientCertificateManager(certDirectory string, nodeName types.NodeName, certFile string, keyFile string, clientFn certificate.CSRClientFunc) (certificate.Manager, error) { certificateStore, err := certificate.NewFileStore( "kubelet-client", certDirectory, @@ -117,6 +159,7 @@ func NewKubeletClientCertificateManager(certDirectory string, nodeName types.Nod prometheus.MustRegister(certificateExpiration) m, err := certificate.NewManager(&certificate.Config{ + ClientFn: clientFn, Template: &x509.CertificateRequest{ Subject: pkix.Name{ CommonName: fmt.Sprintf("system:node:%s", nodeName), @@ -138,10 +181,8 @@ func NewKubeletClientCertificateManager(certDirectory string, nodeName types.Nod // authenticate itself to the TLS server. certificates.UsageClientAuth, }, - CertificateStore: certificateStore, - BootstrapCertificatePEM: certData, - BootstrapKeyPEM: keyData, - CertificateExpiration: certificateExpiration, + CertificateStore: certificateStore, + CertificateExpiration: certificateExpiration, }) if err != nil { return nil, fmt.Errorf("failed to initialize client certificate manager: %v", err) diff --git a/vendor/k8s.io/kubernetes/staging/src/k8s.io/client-go/util/certificate/certificate_manager.go b/vendor/k8s.io/kubernetes/staging/src/k8s.io/client-go/util/certificate/certificate_manager.go index e189c84799b3..e7e0ba149a17 100644 --- a/vendor/k8s.io/kubernetes/staging/src/k8s.io/client-go/util/certificate/certificate_manager.go +++ b/vendor/k8s.io/kubernetes/staging/src/k8s.io/client-go/util/certificate/certificate_manager.go @@ -24,6 +24,7 @@ import ( "crypto/x509" "encoding/pem" "fmt" + "reflect" "sync" "time" @@ -32,6 +33,7 @@ import ( certificates "k8s.io/api/certificates/v1beta1" "k8s.io/apimachinery/pkg/api/errors" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" "k8s.io/client-go/util/cert" @@ -46,9 +48,6 @@ var certificateWaitBackoff = wait.Backoff{Duration: 30 * time.Second, Steps: 4, // manager. In the background it communicates with the API server to get new // certificates for certificates about to expire. type Manager interface { - // CertificateSigningRequestClient sets the client interface that is used for - // signing new certificates generated as part of rotation. - SetCertificateSigningRequestClient(certificatesclient.CertificateSigningRequestInterface) error // Start the API server status sync loop. Start() // Current returns the currently selected certificate from the @@ -65,16 +64,23 @@ type Manager interface { // Config is the set of configuration parameters available for a new Manager. type Config struct { - // CertificateSigningRequestClient will be used for signing new certificate - // requests generated when a key rotation occurs. It must be set either at - // initialization or by using CertificateSigningRequestClient before - // Manager.Start() is called. - CertificateSigningRequestClient certificatesclient.CertificateSigningRequestInterface + // ClientFn will be used to create a client for + // signing new certificate requests generated when a key rotation occurs. + // It must be set at initialization. The function will never be invoked + // in parallel. It is passed the current client certificate if one exists. + ClientFn CSRClientFunc // Template is the CertificateRequest that will be used as a template for // generating certificate signing requests for all new keys generated as // part of rotation. It follows the same rules as the template parameter of // crypto.x509.CreateCertificateRequest in the Go standard libraries. Template *x509.CertificateRequest + // GetTemplate returns the CertificateRequest that will be used as a template for + // generating certificate signing requests for all new keys generated as + // part of rotation. It follows the same rules as the template parameter of + // crypto.x509.CreateCertificateRequest in the Go standard libraries. + // If no template is available, nil may be returned, and no certificate will be requested. + // If specified, takes precedence over Template. + GetTemplate func() *x509.CertificateRequest // Usages is the types of usages that certificates generated by the manager // can be used for. Usages []certificates.KeyUsage @@ -132,18 +138,30 @@ type Gauge interface { // NoCertKeyError indicates there is no cert/key currently available. type NoCertKeyError string +type CSRClientFunc func(current *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) + func (e *NoCertKeyError) Error() string { return string(*e) } type manager struct { - certSigningRequestClient certificatesclient.CertificateSigningRequestInterface - template *x509.CertificateRequest - usages []certificates.KeyUsage - certStore Store - certAccessLock sync.RWMutex - cert *tls.Certificate - forceRotation bool - certificateExpiration Gauge - serverHealth bool + getTemplate func() *x509.CertificateRequest + lastRequestLock sync.Mutex + lastRequest *x509.CertificateRequest + dynamicTemplate bool + usages []certificates.KeyUsage + forceRotation bool + + certStore Store + + certificateExpiration Gauge + + // the following variables must only be accessed under certAccessLock + certAccessLock sync.RWMutex + cert *tls.Certificate + serverHealth bool + + // the clientFn must only be accessed under the clientAccessLock + clientAccessLock sync.Mutex + clientFn CSRClientFunc } // NewManager returns a new certificate manager. A certificate manager is @@ -158,14 +176,20 @@ func NewManager(config *Config) (Manager, error) { return nil, err } + getTemplate := config.GetTemplate + if getTemplate == nil { + getTemplate = func() *x509.CertificateRequest { return config.Template } + } + m := manager{ - certSigningRequestClient: config.CertificateSigningRequestClient, - template: config.Template, - usages: config.Usages, - certStore: config.CertificateStore, - cert: cert, - forceRotation: forceRotation, - certificateExpiration: config.CertificateExpiration, + clientFn: config.ClientFn, + getTemplate: getTemplate, + dynamicTemplate: config.GetTemplate != nil, + usages: config.Usages, + certStore: config.CertificateStore, + cert: cert, + forceRotation: forceRotation, + certificateExpiration: config.CertificateExpiration, } return &m, nil @@ -189,38 +213,44 @@ func (m *manager) ServerHealthy() bool { return m.serverHealth } -// SetCertificateSigningRequestClient sets the client interface that is used -// for signing new certificates generated as part of rotation. It must be -// called before Start() and can not be used to change the -// CertificateSigningRequestClient that has already been set. This method is to -// support the one specific scenario where the CertificateSigningRequestClient -// uses the CertificateManager. -func (m *manager) SetCertificateSigningRequestClient(certSigningRequestClient certificatesclient.CertificateSigningRequestInterface) error { - if m.certSigningRequestClient == nil { - m.certSigningRequestClient = certSigningRequestClient - return nil - } - return fmt.Errorf("property CertificateSigningRequestClient is already set") -} - // Start will start the background work of rotating the certificates. func (m *manager) Start() { // Certificate rotation depends on access to the API server certificate // signing API, so don't start the certificate manager if we don't have a // client. - if m.certSigningRequestClient == nil { + if m.clientFn == nil { glog.V(2).Infof("Certificate rotation is not enabled, no connection to the apiserver.") return } glog.V(2).Infof("Certificate rotation is enabled.") + templateChanged := make(chan struct{}) go wait.Forever(func() { deadline := m.nextRotationDeadline() if sleepInterval := deadline.Sub(time.Now()); sleepInterval > 0 { glog.V(2).Infof("Waiting %v for next certificate rotation", sleepInterval) - time.Sleep(sleepInterval) + + timer := time.NewTimer(sleepInterval) + defer timer.Stop() + + select { + case <-timer.C: + // unblock when deadline expires + case <-templateChanged: + if reflect.DeepEqual(m.getLastRequest(), m.getTemplate()) { + // if the template now matches what we last requested, restart the rotation deadline loop + return + } + glog.V(2).Infof("Certificate template changed, rotating") + } + } + + // Don't enter rotateCerts and trigger backoff if we don't even have a template to request yet + if m.getTemplate() == nil { + return } + backoff := wait.Backoff{ Duration: 2 * time.Second, Factor: 2, @@ -231,7 +261,18 @@ func (m *manager) Start() { utilruntime.HandleError(fmt.Errorf("Reached backoff limit, still unable to rotate certs: %v", err)) wait.PollInfinite(32*time.Second, m.rotateCerts) } - }, 0) + }, time.Second) + + if m.dynamicTemplate { + go wait.Forever(func() { + // check if the current template matches what we last requested + if !reflect.DeepEqual(m.getLastRequest(), m.getTemplate()) { + // if the template is different, queue up an interrupt of the rotation deadline loop. + // if we've requested a CSR that matches the new template by the time the interrupt is handled, the interrupt is disregarded. + templateChanged <- struct{}{} + } + }, time.Second) + } } func getCurrentCertificateOrBootstrap( @@ -278,6 +319,13 @@ func getCurrentCertificateOrBootstrap( return &bootstrapCert, true, nil } +func (m *manager) getClient() (certificatesclient.CertificateSigningRequestInterface, error) { + current := m.Current() + m.clientAccessLock.Lock() + defer m.clientAccessLock.Unlock() + return m.clientFn(current) +} + // rotateCerts attempts to request a client cert from the server, wait a reasonable // period of time for it to be signed, and then update the cert on disk. If it cannot // retrieve a cert, it will return false. It will only return error in exceptional cases. @@ -286,20 +334,30 @@ func getCurrentCertificateOrBootstrap( func (m *manager) rotateCerts() (bool, error) { glog.V(2).Infof("Rotating certificates") - csrPEM, keyPEM, privateKey, err := m.generateCSR() + template, csrPEM, keyPEM, privateKey, err := m.generateCSR() if err != nil { utilruntime.HandleError(fmt.Errorf("Unable to generate a certificate signing request: %v", err)) return false, nil } + // request the client each time + client, err := m.getClient() + if err != nil { + utilruntime.HandleError(fmt.Errorf("Unable to load a client to request certificates: %v", err)) + return false, nil + } + // Call the Certificate Signing Request API to get a certificate for the // new private key. - req, err := csr.RequestCertificate(m.certSigningRequestClient, csrPEM, "", m.usages, privateKey) + req, err := csr.RequestCertificate(client, csrPEM, "", m.usages, privateKey) if err != nil { utilruntime.HandleError(fmt.Errorf("Failed while requesting a signed certificate from the master: %v", err)) return false, m.updateServerError(err) } + // Once we've successfully submitted a CSR for this template, record that we did so + m.setLastRequest(template) + // Wait for the certificate to be signed. Instead of one long watch, we retry with slightly longer // intervals each time in order to tolerate failures from the server AND to preserve the liveliness // of the cert manager loop. This creates slightly more traffic against the API server in return @@ -307,7 +365,7 @@ func (m *manager) rotateCerts() (bool, error) { var crtPEM []byte watchDuration := time.Minute if err := wait.ExponentialBackoff(certificateWaitBackoff, func() (bool, error) { - data, err := csr.WaitForCertificate(m.certSigningRequestClient, req, watchDuration) + data, err := csr.WaitForCertificate(client, req, watchDuration) switch { case err == nil: crtPEM = data @@ -353,6 +411,36 @@ func (m *manager) nextRotationDeadline() time.Time { return time.Now() } + // Ensure the currently held certificate satisfies the requested subject CN and SANs + if template := m.getTemplate(); template != nil { + if template.Subject.CommonName != m.cert.Leaf.Subject.CommonName { + glog.V(2).Infof("Current certificate CN (%s) does not match requested CN (%s), rotating now", m.cert.Leaf.Subject.CommonName, template.Subject.CommonName) + return time.Now() + } + + currentDNSNames := sets.NewString(m.cert.Leaf.DNSNames...) + desiredDNSNames := sets.NewString(template.DNSNames...) + missingDNSNames := desiredDNSNames.Difference(currentDNSNames) + if len(missingDNSNames) > 0 { + glog.V(2).Infof("Current certificate is missing requested DNS names %v, rotating now", missingDNSNames.List()) + return time.Now() + } + + currentIPs := sets.NewString() + for _, ip := range m.cert.Leaf.IPAddresses { + currentIPs.Insert(ip.String()) + } + desiredIPs := sets.NewString() + for _, ip := range template.IPAddresses { + desiredIPs.Insert(ip.String()) + } + missingIPs := desiredIPs.Difference(currentIPs) + if len(missingIPs) > 0 { + glog.V(2).Infof("Current certificate is missing requested IP addresses %v, rotating now", missingIPs.List()) + return time.Now() + } + } + notAfter := m.cert.Leaf.NotAfter totalDuration := float64(notAfter.Sub(m.cert.Leaf.NotBefore)) deadline := m.cert.Leaf.NotBefore.Add(jitteryDuration(totalDuration)) @@ -408,22 +496,38 @@ func (m *manager) updateServerError(err error) error { return nil } -func (m *manager) generateCSR() (csrPEM []byte, keyPEM []byte, key interface{}, err error) { +func (m *manager) generateCSR() (template *x509.CertificateRequest, csrPEM []byte, keyPEM []byte, key interface{}, err error) { // Generate a new private key. privateKey, err := ecdsa.GenerateKey(elliptic.P256(), cryptorand.Reader) if err != nil { - return nil, nil, nil, fmt.Errorf("unable to generate a new private key: %v", err) + return nil, nil, nil, nil, fmt.Errorf("unable to generate a new private key: %v", err) } der, err := x509.MarshalECPrivateKey(privateKey) if err != nil { - return nil, nil, nil, fmt.Errorf("unable to marshal the new key to DER: %v", err) + return nil, nil, nil, nil, fmt.Errorf("unable to marshal the new key to DER: %v", err) } keyPEM = pem.EncodeToMemory(&pem.Block{Type: cert.ECPrivateKeyBlockType, Bytes: der}) - csrPEM, err = cert.MakeCSRFromTemplate(privateKey, m.template) + template = m.getTemplate() + if template == nil { + return nil, nil, nil, nil, fmt.Errorf("unable to create a csr, no template available") + } + csrPEM, err = cert.MakeCSRFromTemplate(privateKey, template) if err != nil { - return nil, nil, nil, fmt.Errorf("unable to create a csr from the private key: %v", err) + return nil, nil, nil, nil, fmt.Errorf("unable to create a csr from the private key: %v", err) } - return csrPEM, keyPEM, privateKey, nil + return template, csrPEM, keyPEM, privateKey, nil +} + +func (m *manager) getLastRequest() *x509.CertificateRequest { + m.lastRequestLock.Lock() + defer m.lastRequestLock.Unlock() + return m.lastRequest +} + +func (m *manager) setLastRequest(r *x509.CertificateRequest) { + m.lastRequestLock.Lock() + defer m.lastRequestLock.Unlock() + m.lastRequest = r } diff --git a/vendor/k8s.io/kubernetes/staging/src/k8s.io/client-go/util/certificate/certificate_manager_test.go b/vendor/k8s.io/kubernetes/staging/src/k8s.io/client-go/util/certificate/certificate_manager_test.go index 6a23f04260d0..ac63ab35a3cc 100644 --- a/vendor/k8s.io/kubernetes/staging/src/k8s.io/client-go/util/certificate/certificate_manager_test.go +++ b/vendor/k8s.io/kubernetes/staging/src/k8s.io/client-go/util/certificate/certificate_manager_test.go @@ -186,7 +186,7 @@ func TestSetRotationDeadline(t *testing.T) { NotAfter: tc.notAfter, }, }, - template: &x509.CertificateRequest{}, + getTemplate: func() *x509.CertificateRequest { return &x509.CertificateRequest{} }, usages: []certificates.KeyUsage{}, certificateExpiration: &g, } @@ -221,10 +221,10 @@ func TestRotateCertCreateCSRError(t *testing.T) { NotAfter: now.Add(-1 * time.Hour), }, }, - template: &x509.CertificateRequest{}, - usages: []certificates.KeyUsage{}, - certSigningRequestClient: fakeClient{ - failureType: createError, + getTemplate: func() *x509.CertificateRequest { return &x509.CertificateRequest{} }, + usages: []certificates.KeyUsage{}, + clientFn: func(_ *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) { + return fakeClient{failureType: createError}, nil }, } @@ -244,10 +244,10 @@ func TestRotateCertWaitingForResultError(t *testing.T) { NotAfter: now.Add(-1 * time.Hour), }, }, - template: &x509.CertificateRequest{}, - usages: []certificates.KeyUsage{}, - certSigningRequestClient: fakeClient{ - failureType: watchError, + getTemplate: func() *x509.CertificateRequest { return &x509.CertificateRequest{} }, + usages: []certificates.KeyUsage{}, + clientFn: func(_ *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) { + return fakeClient{failureType: watchError}, nil }, } @@ -456,6 +456,11 @@ func TestInitializeCertificateSigningRequestClient(t *testing.T) { CertificateStore: certificateStore, BootstrapCertificatePEM: tc.bootstrapCert.certificatePEM, BootstrapKeyPEM: tc.bootstrapCert.keyPEM, + ClientFn: func(_ *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) { + return &fakeClient{ + certificatePEM: tc.apiCert.certificatePEM, + }, nil + }, }) if err != nil { t.Errorf("Got %v, wanted no error.", err) @@ -465,11 +470,6 @@ func TestInitializeCertificateSigningRequestClient(t *testing.T) { if !certificatesEqual(certificate, tc.expectedCertBeforeStart.certificate) { t.Errorf("Got %v, wanted %v", certificateString(certificate), certificateString(tc.expectedCertBeforeStart.certificate)) } - if err := certificateManager.SetCertificateSigningRequestClient(&fakeClient{ - certificatePEM: tc.apiCert.certificatePEM, - }); err != nil { - t.Errorf("Got error %v, expected none.", err) - } if m, ok := certificateManager.(*manager); !ok { t.Errorf("Expected a '*manager' from 'NewManager'") @@ -556,8 +556,10 @@ func TestInitializeOtherRESTClients(t *testing.T) { CertificateStore: certificateStore, BootstrapCertificatePEM: tc.bootstrapCert.certificatePEM, BootstrapKeyPEM: tc.bootstrapCert.keyPEM, - CertificateSigningRequestClient: &fakeClient{ - certificatePEM: tc.apiCert.certificatePEM, + ClientFn: func(_ *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) { + return &fakeClient{ + certificatePEM: tc.apiCert.certificatePEM, + }, nil }, }) if err != nil { @@ -708,10 +710,12 @@ func TestServerHealth(t *testing.T) { CertificateStore: certificateStore, BootstrapCertificatePEM: tc.bootstrapCert.certificatePEM, BootstrapKeyPEM: tc.bootstrapCert.keyPEM, - CertificateSigningRequestClient: &fakeClient{ - certificatePEM: tc.apiCert.certificatePEM, - failureType: tc.failureType, - err: tc.clientErr, + ClientFn: func(_ *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) { + return &fakeClient{ + certificatePEM: tc.apiCert.certificatePEM, + failureType: tc.failureType, + err: tc.clientErr, + }, nil }, }) if err != nil {