Bug 1962502: Create custom metrics to list unmanaged Routes and Ingresses without IngressClassName

pkg/route/ingress/ingress.go

@@ -33,6 +33,7 @@ import (
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/client-go/util/workqueue"
+	"k8s.io/component-base/metrics/legacyregistry"
 	"k8s.io/kubernetes/pkg/api/legacyscheme"
 
 	routev1 "github.com/openshift/api/route/v1"
@@ -96,6 +97,11 @@ type Controller struct {
 	// expectationDelay controls how long the controller waits to observe its
 	// own creates. Exposed only for testing.
 	expectationDelay time.Duration
+
+	// Prometheus metrics
+	metricsCreated    bool
+	metricsCreateOnce sync.Once
+	metricsCreateLock sync.RWMutex
 }
 
 // expectations track an upcoming change to a named resource related
@@ -251,6 +257,11 @@ func NewController(eventsClient kv1core.EventsGetter, routeClient routeclient.Ro
 		},
 	})
 
+	if !c.MetricsCreated() {
+		legacyregistry.MustRegister(c)
+		klog.Info("ingress-to-route metrics registered with prometheus")
+	}
+
 	return c
 }
 
@@ -373,30 +384,12 @@ func (c *Controller) sync(key queueKey) error {
 		return err
 	}
 
-	// If the ingress specifies an ingressclass and the ingressclass does
-	// not specify openshift.io/ingress-to-route as its controller, ignore
-	// the ingress.
-	var ingressClassName *string
-	if v, ok := ingress.Annotations["kubernetes.io/ingress.class"]; ok {
-		ingressClassName = &v
-	} else {
-		ingressClassName = ingress.Spec.IngressClassName
+	managed, err := c.ingressManaged(ingress)
+	if err != nil {
+		return err
 	}
-	if ingressClassName != nil {
-		ingressclass, err := c.ingressclassLister.Get(*ingressClassName)
-		if kerrors.IsNotFound(err) {
-			return nil
-		}
-		if err != nil {
-			return err
-		}
-		// TODO Replace "openshift.io/ingress-to-route" with
-		// routev1.IngressToRouteIngressClassControllerName once
-		// openshift-controller-manager bumps openshift/api to a version
-		// that defines it.
-		if ingressclass.Spec.Controller != "openshift.io/ingress-to-route" {
-			return nil
-		}
+	if !managed {
+		return nil
 	}
 
 	// find all matching routes
@@ -577,6 +570,36 @@ func hasIngressOwnerRef(owners []metav1.OwnerReference) (string, bool) {
 	return "", false
 }
 
+func (c *Controller) ingressManaged(ingress *networkingv1.Ingress) (bool, error) {
+	// If the ingress specifies an ingressclass and the ingressclass does
+	// not specify openshift.io/ingress-to-route as its controller, ignore
+	// the ingress.
+	var ingressClassName *string
+	if v, ok := ingress.Annotations["kubernetes.io/ingress.class"]; ok {
+		ingressClassName = &v
+	} else {
+		ingressClassName = ingress.Spec.IngressClassName
+	}
+	if ingressClassName != nil {
+		ingressclass, err := c.ingressclassLister.Get(*ingressClassName)
+		if kerrors.IsNotFound(err) {
+			return false, nil
+		}
+		if err != nil {
+			return false, err
+		}
+		// TODO Replace "openshift.io/ingress-to-route" with
+		// routev1.IngressToRouteIngressClassControllerName once
+		// openshift-controller-manager bumps openshift/api to a version
+		// that defines it.
+		if ingressclass.Spec.Controller != "openshift.io/ingress-to-route" {
+			return false, nil
+		}
+	}
+
+	return true, nil
+}
+
 func newRouteForIngress(
 	ingress *networkingv1.Ingress,
 	rule *networkingv1.IngressRule,

pkg/route/ingress/metrics.go

@@ -0,0 +1,103 @@
+package ingress
+
+import (
+	"k8s.io/apimachinery/pkg/labels"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+
+	"github.com/blang/semver/v4"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	routeController               = "openshift_ingress_to_route_controller"
+	metricRouteWithUnmanagedOwner = routeController + "_route_with_unmanaged_owner"
+	metricIngressWithoutClassName = routeController + "_ingress_without_class_name"
+)
+
+var (
+	unmanagedRoutes = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: metricRouteWithUnmanagedOwner,
+		Help: "Report the number of routes owned by ingresses no longer managed.",
+	}, []string{"name", "host"})
+
+	ingressesWithoutClassName = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: metricIngressWithoutClassName,
+		Help: "Report the number of ingresses that do not specify ingressClassName.",
+	}, []string{"name"})
+)
+
+func (c *Controller) Create(v *semver.Version) bool {
+	c.metricsCreateOnce.Do(func() {
+		c.metricsCreateLock.Lock()
+		defer c.metricsCreateLock.Unlock()
+		c.metricsCreated = true
+	})
+	return c.MetricsCreated()
+}
+
+func (c *Controller) MetricsCreated() bool {
+	return c.metricsCreated
+}
+
+func (c *Controller) ClearState() {
+	c.metricsCreateLock.Lock()
+	defer c.metricsCreateLock.Unlock()
+	c.metricsCreated = false
+}
+
+// FQName returns the fully-qualified metric name of the collector.
+func (c *Controller) FQName() string {
+	return routeController
+}
+
+func (c *Controller) Describe(ch chan<- *prometheus.Desc) {
+	unmanagedRoutes.Describe(ch)
+	ingressesWithoutClassName.Describe(ch)
+}
+
+func (c *Controller) Collect(ch chan<- prometheus.Metric) {
+	// collect ingresses that do not specify ingressClassName
+	ingressInstances, err := c.ingressLister.List(labels.Everything())
+	if err != nil {
+		utilruntime.HandleError(err)
+		return
+	}
+
+	for _, ingressInstance := range ingressInstances {
+		labelVal := 0
+		if ingressInstance.Spec.IngressClassName == nil {
+			labelVal = 1
+		}
+		ingressesWithoutClassName.WithLabelValues(ingressInstance.Name).Set(float64(labelVal))
+	}
+
+	ingressesWithoutClassName.Collect(ch)
+
+	// collect routes owned by ingresses no longer managed
+	routeInstances, err := c.routeLister.List(labels.Everything())
+	if err != nil {
+		utilruntime.HandleError(err)
+		return
+	}
+
+	for _, routeInstance := range routeInstances {
+		labelVal := 0
+		if owner, have := hasIngressOwnerRef(routeInstance.OwnerReferences); have {
+			for _, ingressInstance := range ingressInstances {
+				if ingressInstance.Name == owner {
+					managed, err := c.ingressManaged(ingressInstance)
+					if err != nil {
+						utilruntime.HandleError(err)
+						return
+					}
+					if !managed {
+						labelVal = 1
+					}
+				}
+			}
+		}
+		unmanagedRoutes.WithLabelValues(routeInstance.Name, routeInstance.Spec.Host).Set(float64(labelVal))
+	}
+
+	unmanagedRoutes.Collect(ch)
+}

pkg/route/ingress/metrics_test.go

@@ -0,0 +1,143 @@
+package ingress
+
+import (
+	"bytes"
+	"net/http"
+	"strings"
+	"testing"
+
+	networkingv1 "k8s.io/api/networking/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/component-base/metrics/legacyregistry"
+
+	routev1 "github.com/openshift/api/route/v1"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+type fakeResponseWriter struct {
+	bytes.Buffer
+	statusCode int
+	header     http.Header
+}
+
+func (f *fakeResponseWriter) Header() http.Header {
+	return f.header
+}
+
+func (f *fakeResponseWriter) WriteHeader(statusCode int) {
+	f.statusCode = statusCode
+}
+
+func TestMetrics(t *testing.T) {
+	expectedResponse := []string{
+		"# HELP openshift_ingress_to_route_controller_ingress_without_class_name Report the number of ingresses that do not specify ingressClassName.",
+		"# TYPE openshift_ingress_to_route_controller_ingress_without_class_name gauge",
+		"openshift_ingress_to_route_controller_ingress_without_class_name{name=\"without-ingressclassname\"} 1",
+		"openshift_ingress_to_route_controller_ingress_without_class_name{name=\"not-managed\"} 0",
+		"openshift_ingress_to_route_controller_ingress_without_class_name{name=\"managed\"} 0",
+		"# HELP openshift_ingress_to_route_controller_route_with_unmanaged_owner Report the number of routes owned by ingresses no longer managed.",
+		"# TYPE openshift_ingress_to_route_controller_route_with_unmanaged_owner gauge",
+		"openshift_ingress_to_route_controller_route_with_unmanaged_owner{host=\"test.com\",name=\"owned-by-unmanaged\"} 1",
+		"openshift_ingress_to_route_controller_route_with_unmanaged_owner{host=\"test.com\",name=\"owned-by-managed\"} 0",
+	}
+
+	boolTrue := true
+	customIngressClassName := "custom"
+	openshiftDefaultIngressClassName := "openshift-default"
+
+	r := &routeLister{
+		Items: []*routev1.Route{
+			{ // Route owned by an Ingress that is not managed.
+				ObjectMeta: metav1.ObjectMeta{
+					Name:            "owned-by-unmanaged",
+					Namespace:       "test",
+					OwnerReferences: []metav1.OwnerReference{{APIVersion: "networking.k8s.io/v1", Kind: "Ingress", Name: "not-managed", Controller: &boolTrue}},
+				},
+				Spec: routev1.RouteSpec{
+					Host: "test.com",
+				},
+			},
+			{ // Route owned by an Ingress that is managed.
+				ObjectMeta: metav1.ObjectMeta{
+					Name:            "owned-by-managed",
+					Namespace:       "test",
+					OwnerReferences: []metav1.OwnerReference{{APIVersion: "networking.k8s.io/v1", Kind: "Ingress", Name: "managed", Controller: &boolTrue}},
+				},
+				Spec: routev1.RouteSpec{
+					Host: "test.com",
+				},
+			},
+		},
+	}
+	i := &ingressLister{
+		Items: []*networkingv1.Ingress{
+			{ // Ingress without IngressClassName.
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "without-ingressclassname",
+					Namespace: "test",
+				},
+				Spec: networkingv1.IngressSpec{
+					IngressClassName: nil,
+				},
+			},
+			{ // Ingress with IngressClassName that is not managed.
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "not-managed",
+					Namespace: "test",
+				},
+				Spec: networkingv1.IngressSpec{
+					IngressClassName: &customIngressClassName,
+				},
+			},
+			{ // Ingress with IngressClassName that is managed.
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "managed",
+					Namespace: "test",
+				},
+				Spec: networkingv1.IngressSpec{
+					IngressClassName: &openshiftDefaultIngressClassName,
+				},
+			},
+		},
+	}
+
+	ic := &ingressclassLister{
+		Items: []*networkingv1.IngressClass{
+			{ // IngressClass specifying "openshift.io/ingress-to-route" controller
+				ObjectMeta: metav1.ObjectMeta{
+					Name: openshiftDefaultIngressClassName,
+				},
+				Spec: networkingv1.IngressClassSpec{
+					Controller: "openshift.io/ingress-to-route",
+				},
+			},
+			{ // IngressClass specifying "acme.io/ingress-controller" controller
+				ObjectMeta: metav1.ObjectMeta{
+					Name: customIngressClassName,
+				},
+				Spec: networkingv1.IngressClassSpec{
+					Controller: "acme.io/ingress-controller",
+				},
+			},
+		},
+	}
+
+	c := &Controller{
+		routeLister:        r,
+		ingressLister:      i,
+		ingressclassLister: ic,
+	}
+
+	legacyregistry.MustRegister(c)
+	h := promhttp.HandlerFor(legacyregistry.DefaultGatherer, promhttp.HandlerOpts{ErrorHandling: promhttp.PanicOnError})
+
+	rw := &fakeResponseWriter{header: http.Header{}}
+	h.ServeHTTP(rw, &http.Request{})
+
+	respStr := rw.String()
+	for _, s := range expectedResponse {
+		if !strings.Contains(respStr, s) {
+			t.Errorf("expected string %s did not appear in %s", s, respStr)
+		}
+	}
+}