Added metrics for lock acquiring attempts

pkg/server/prometheus_exporter.go

@@ -1,41 +1,55 @@
 package server
 
 import (
-	"github.com/prometheus/client_golang/prometheus"
 	"strings"
+
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 var (
+	buildInfoGaugeVec = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "sidecache_admission_build_info",
+			Help: "Build info for sidecache admission webhook",
+		}, []string{"version"})
+
 	cacheHitCounter = prometheus.NewCounter(
 		prometheus.CounterOpts{
 			Namespace: "sidecache_" + ProjectName,
 			Name:      "cache_hit_counter",
 			Help:      "Cache hit count",
 		})
 
+	lockAcquiringAttemptsHistogram = prometheus.NewHistogram(
+		prometheus.HistogramOpts{
+			Namespace: "sidecache_" + ProjectName,
+			Name:      "lock_acquiring_attempts_histogram",
+			Help:      "Lock acquiring attempts histogram",
+			Buckets:   []float64{0.999, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 20, 100, 200, 300, 500, 1000},
+		})
+
 	totalRequestCounter = prometheus.NewCounter(
 		prometheus.CounterOpts{
 			Namespace: "sidecache_" + ProjectName,
 			Name:      "all_request_hit_counter",
 			Help:      "All request hit counter",
 		})
-
-	buildInfoGaugeVec = prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "sidecache_admission_build_info",
-			Help: "Build info for sidecache admission webhook",
-		}, []string{"version"})
 )
 
 type Prometheus struct {
-	CacheHitCounter     prometheus.Counter
-	TotalRequestCounter prometheus.Counter
+	CacheHitCounter                prometheus.Counter
+	LockAcquiringAttemptsHistogram prometheus.Histogram
+	TotalRequestCounter            prometheus.Counter
 }
 
 func NewPrometheusClient() *Prometheus {
-	prometheus.MustRegister(cacheHitCounter, totalRequestCounter, buildInfoGaugeVec)
+	prometheus.MustRegister(buildInfoGaugeVec, cacheHitCounter, lockAcquiringAttemptsHistogram, totalRequestCounter)
 
-	return &Prometheus{TotalRequestCounter: totalRequestCounter, CacheHitCounter: cacheHitCounter}
+	return &Prometheus{
+		CacheHitCounter:                cacheHitCounter,
+		LockAcquiringAttemptsHistogram: lockAcquiringAttemptsHistogram,
+		TotalRequestCounter:            totalRequestCounter,
+	}
 }
 
 func BuildInfo(admission string) {

pkg/server/server.go

@@ -7,10 +7,8 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"errors"
-	"github.com/zeriontech/sidecache/pkg/lock"
 	"io"
 	"io/ioutil"
-	"math"
 	"net/http"
 	"net/http/httputil"
 	"net/url"
@@ -20,6 +18,7 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/zeriontech/sidecache/pkg/cache"
+	"github.com/zeriontech/sidecache/pkg/lock"
 	"go.uber.org/zap"
 )
 
@@ -145,14 +144,14 @@ func (server CacheServer) CacheHandler(w http.ResponseWriter, r *http.Request) {
 	}()
 
 	path := strings.Split(r.URL.Path, "/")
-	key := "lock:" + path[1]
+	key := "lock:" + path[1] + "/" + path[2]
 	resultKey := server.HashURL(server.ReorderQueryString(r.URL))
 
 	if UseLock {
-		attempt := 0
+		attempt := 1
 		for {
 			// check the cache
-			server.Logger.Info("checking the cache", zap.String("resultKey", resultKey), zap.Int("attempt", attempt+1))
+			server.Logger.Info("checking the cache", zap.String("resultKey", resultKey), zap.Int("attempt", attempt))
 			if cachedDataBytes := server.CheckCache(resultKey); cachedDataBytes != nil {
 				serveFromCache(cachedDataBytes, server, w, r)
 				return
@@ -161,7 +160,8 @@ func (server CacheServer) CacheHandler(w http.ResponseWriter, r *http.Request) {
 			// try to acquire the lock
 			server.Logger.Info("acquiring the lock", zap.String("key", key))
 			if err := server.LockMgr.Acquire(key, LockTtl); err == nil {
-				server.Logger.Info("lock acquired", zap.String("key", key))
+				server.Prometheus.LockAcquiringAttemptsHistogram.Observe(float64(attempt))
+				server.Logger.Info("lock acquired", zap.String("key", key), zap.Int("attempt", attempt))
 				defer func() {
 					// release the lock
 					if err := server.LockMgr.Release(key); err != nil {
@@ -252,9 +252,8 @@ func (server CacheServer) ReorderQueryString(url *url.URL) string {
 }
 
 func (server CacheServer) GetBackoff(attempt int) time.Duration {
-	multiplier := 1
-	if attempt%2 != 0 {
-		multiplier = 5
+	if attempt <= 10 {
+		return 100 * time.Millisecond
 	}
-	return time.Duration(multiplier*int(math.Pow(10, float64(attempt/2+1)))) * time.Millisecond
+	return 500 * time.Millisecond
 }