From f9539f7f0e23f6af307e80c3ecb136e38fd0aaec Mon Sep 17 00:00:00 2001
From: nghialv <nghialv2607@gmail.com>
Date: Thu, 30 Jul 2020 22:03:06 +0900
Subject: [PATCH] Add metrics about received resource events from kubernetes
 api server

---
 .../livestatestore/kubernetes/metrics.go      | 32 ++++++++++++++++---
 .../livestatestore/kubernetes/reflector.go    | 15 +++++++--
 2 files changed, 40 insertions(+), 7 deletions(-)

diff --git a/pkg/app/piped/livestatestore/kubernetes/metrics.go b/pkg/app/piped/livestatestore/kubernetes/metrics.go
index b8ca19188f..0aca74b658 100644
--- a/pkg/app/piped/livestatestore/kubernetes/metrics.go
+++ b/pkg/app/piped/livestatestore/kubernetes/metrics.go
@@ -15,14 +15,18 @@
 package kubernetes
 
 import (
+	"strconv"
+
 	"github.com/prometheus/client_golang/prometheus"
 	"k8s.io/client-go/tools/metrics"
 )
 
 const (
-	metricsLabelHost   = "host"
-	metricsLabelMethod = "method"
-	metricsLabelCode   = "code"
+	metricsLabelHost         = "host"
+	metricsLabelMethod       = "method"
+	metricsLabelCode         = "code"
+	metricsLabelEvent        = "event"
+	metricsLabelEventHandled = "handled"
 )
 
 var (
@@ -37,10 +41,23 @@ var (
 			metricsLabelCode,
 		},
 	)
+	metricsResourceEvents = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "livestatestore_kubernetes_resource_events_total",
+			Help: "Number of resource events received from kubernetes server.",
+		},
+		[]string{
+			metricsLabelEvent,
+			metricsLabelEventHandled,
+		},
+	)
 )
 
 func registerMetrics() {
-	prometheus.MustRegister(metricsAPIRequests)
+	prometheus.MustRegister(
+		metricsAPIRequests,
+		metricsResourceEvents,
+	)
 
 	opts := metrics.RegisterOpts{
 		RequestResult: requestResultCollector{},
@@ -58,3 +75,10 @@ func (c requestResultCollector) Increment(code string, method string, host strin
 		metricsLabelCode:   code,
 	}).Inc()
 }
+
+func incrementResourceEventCounter(event string, handled bool) {
+	metricsResourceEvents.With(prometheus.Labels{
+		metricsLabelEvent:        event,
+		metricsLabelEventHandled: strconv.FormatBool(handled),
+	}).Inc()
+}
diff --git a/pkg/app/piped/livestatestore/kubernetes/reflector.go b/pkg/app/piped/livestatestore/kubernetes/reflector.go
index 305a12bed5..47a4bb2efb 100644
--- a/pkg/app/piped/livestatestore/kubernetes/reflector.go
+++ b/pkg/app/piped/livestatestore/kubernetes/reflector.go
@@ -235,17 +235,20 @@ func (r *reflector) onObjectAdd(obj interface{}) {
 
 	// Ignore all predefined ones.
 	if _, ok := ignoreResourceKeys[key.String()]; ok {
+		incrementResourceEventCounter("add", false)
 		return
 	}
 
 	// Ignore all objects that are not handled by this piped.
 	pipedID := u.GetAnnotations()[provider.LabelPiped]
 	if pipedID != "" && pipedID != r.pipedConfig.PipedID {
+		incrementResourceEventCounter("add", false)
 		return
 	}
 
-	r.logger.Info(fmt.Sprintf("received add event for %s", key.String()))
+	r.logger.Debug(fmt.Sprintf("received add event for %s", key.String()))
 	r.onAdd(u)
+	incrementResourceEventCounter("add", true)
 }
 
 func (r *reflector) onObjectUpdate(oldObj, obj interface{}) {
@@ -255,17 +258,20 @@ func (r *reflector) onObjectUpdate(oldObj, obj interface{}) {
 	// Ignore all predefined ones.
 	key := provider.MakeResourceKey(u)
 	if _, ok := ignoreResourceKeys[key.String()]; ok {
+		incrementResourceEventCounter("update", false)
 		return
 	}
 
 	// Ignore all objects that are not handled by this piped.
 	pipedID := u.GetAnnotations()[provider.LabelPiped]
 	if pipedID != "" && pipedID != r.pipedConfig.PipedID {
+		incrementResourceEventCounter("update", false)
 		return
 	}
 
-	r.logger.Info(fmt.Sprintf("received update event for %s", key.String()))
+	r.logger.Debug(fmt.Sprintf("received update event for %s", key.String()))
 	r.onUpdate(oldU, u)
+	incrementResourceEventCounter("update", true)
 }
 
 func (r *reflector) onObjectDelete(obj interface{}) {
@@ -274,17 +280,20 @@ func (r *reflector) onObjectDelete(obj interface{}) {
 
 	// Ignore all predefined ones.
 	if _, ok := ignoreResourceKeys[key.String()]; ok {
+		incrementResourceEventCounter("delete", false)
 		return
 	}
 
 	// Ignore all objects that are not handled by this piped.
 	pipedID := u.GetAnnotations()[provider.LabelPiped]
 	if pipedID != "" && pipedID != r.pipedConfig.PipedID {
+		incrementResourceEventCounter("delete", false)
 		return
 	}
 
-	r.logger.Info(fmt.Sprintf("received delete event for %s", key.String()))
+	r.logger.Debug(fmt.Sprintf("received delete event for %s", key.String()))
 	r.onDelete(u)
+	incrementResourceEventCounter("delete", true)
 }
 
 func isSupportedWatch(r metav1.APIResource) bool {