certwatcher: add metrics to monitor certificate reads

Author: isitinschi

pkg/certwatcher/certwatcher.go

@@ -22,6 +22,7 @@ import (
 	"sync"
 
 	"github.com/fsnotify/fsnotify"
+	"sigs.k8s.io/controller-runtime/pkg/certwatcher/metrics"
 	logf "sigs.k8s.io/controller-runtime/pkg/internal/log"
 )
 
@@ -116,8 +117,10 @@ func (cw *CertWatcher) Watch() {
 // and updates the current certificate on the watcher.  If a callback is set, it
 // is invoked with the new certificate.
 func (cw *CertWatcher) ReadCertificate() error {
+	metrics.ReadCertificateTotal.Inc()
 	cert, err := tls.LoadX509KeyPair(cw.certPath, cw.keyPath)
 	if err != nil {
+		metrics.ReadCertificateErrors.Inc()
 		return err
 	}
 

pkg/certwatcher/certwatcher_test.go

@@ -23,14 +23,17 @@ import (
 	"crypto/x509"
 	"crypto/x509/pkix"
 	"encoding/pem"
+	"fmt"
 	"math/big"
 	"net"
 	"os"
 	"time"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	"github.com/prometheus/client_golang/prometheus/testutil"
 	"sigs.k8s.io/controller-runtime/pkg/certwatcher"
+	"sigs.k8s.io/controller-runtime/pkg/certwatcher/metrics"
 )
 
 var _ = Describe("CertWatcher", func() {
@@ -109,6 +112,64 @@ var _ = Describe("CertWatcher", func() {
 			ctxCancel()
 			Eventually(doneCh, "4s").Should(BeClosed())
 		})
+
+		Context("prometheus metric read_certificate_total", func() {
+			var readCertificateTotalBefore float64
+			var readCertificateErrorsBefore float64
+
+			BeforeEach(func() {
+				readCertificateTotalBefore = testutil.ToFloat64(metrics.ReadCertificateTotal)
+				readCertificateErrorsBefore = testutil.ToFloat64(metrics.ReadCertificateErrors)
+			})
+
+			It("should get updated on successful certificate read", func() {
+				doneCh := startWatcher()
+
+				Eventually(func() error {
+					readCertificateTotalAfter := testutil.ToFloat64(metrics.ReadCertificateTotal)
+					if readCertificateTotalAfter != readCertificateTotalBefore+1.0 {
+						return fmt.Errorf("metric read certificate total expected: %v and got: %v", readCertificateTotalBefore+1.0, readCertificateTotalAfter)
+					}
+					return nil
+				}, "4s").Should(Succeed())
+
+				ctxCancel()
+				Eventually(doneCh, "4s").Should(BeClosed())
+			})
+
+			It("should get updated on read certificate errors", func() {
+				doneCh := startWatcher()
+
+				Eventually(func() error {
+					readCertificateTotalAfter := testutil.ToFloat64(metrics.ReadCertificateTotal)
+					if readCertificateTotalAfter != readCertificateTotalBefore+1.0 {
+						return fmt.Errorf("metric read certificate total expected: %v and got: %v", readCertificateTotalBefore+1.0, readCertificateTotalAfter)
+					}
+					readCertificateTotalBefore = readCertificateTotalAfter
+					return nil
+				}, "4s").Should(Succeed())
+
+				Expect(os.Remove(keyPath)).To(BeNil())
+
+				Eventually(func() error {
+					readCertificateTotalAfter := testutil.ToFloat64(metrics.ReadCertificateTotal)
+					if readCertificateTotalAfter != readCertificateTotalBefore+1.0 {
+						return fmt.Errorf("metric read certificate total expected: %v and got: %v", readCertificateTotalBefore+1.0, readCertificateTotalAfter)
+					}
+					return nil
+				}, "4s").Should(Succeed())
+				Eventually(func() error {
+					readCertificateErrorsAfter := testutil.ToFloat64(metrics.ReadCertificateErrors)
+					if readCertificateErrorsAfter != readCertificateErrorsBefore+1.0 {
+						return fmt.Errorf("metric read certificate errors expected: %v and got: %v", readCertificateErrorsBefore+1.0, readCertificateErrorsAfter)
+					}
+					return nil
+				}, "4s").Should(Succeed())
+
+				ctxCancel()
+				Eventually(doneCh, "4s").Should(BeClosed())
+			})
+		})
 	})
 })
 

pkg/certwatcher/metrics/metrics.go

@@ -0,0 +1,45 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
+)
+
+var (
+	// ReadCertificateTotal is a prometheus counter metrics which holds the total
+	// number of certificate reads.
+	ReadCertificateTotal = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "certwatcher_read_certificate_total",
+		Help: "Total number of certificate reads",
+	})
+
+	// ReadCertificateErrors is a prometheus counter metrics which holds the total
+	// number of errors from certificate read.
+	ReadCertificateErrors = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "certwatcher_read_certificate_errors_total",
+		Help: "Total number of certificate read errors",
+	})
+)
+
+func init() {
+	metrics.Registry.MustRegister(
+		ReadCertificateTotal,
+		ReadCertificateErrors,
+	)
+}