Restore cache metrics

lib/auth/accesspoint/accesspoint.go

@@ -23,17 +23,15 @@ package accesspoint
 
 import (
 	"context"
-	"log/slog"
 	"slices"
 	"time"
 
 	"github.com/gravitational/trace"
+	"github.com/prometheus/client_golang/prometheus"
 	oteltrace "go.opentelemetry.io/otel/trace"
 
 	"github.com/gravitational/teleport"
 	"github.com/gravitational/teleport/api/types"
-	"github.com/gravitational/teleport/lib/backend"
-	"github.com/gravitational/teleport/lib/backend/memory"
 	"github.com/gravitational/teleport/lib/cache"
 	"github.com/gravitational/teleport/lib/observability/tracing"
 	"github.com/gravitational/teleport/lib/services"
@@ -63,6 +61,8 @@ type Config struct {
 	// TracingProvider is the provider to be used for exporting
 	// traces. No-op tracers will be used if no provider is set.
 	TracingProvider *tracing.Provider
+	// Registerer is used to register prometheus metrics.
+	Registerer prometheus.Registerer
 
 	// The following services are provided to the Cache to allow it to
 	// populate its resource collections. They will either be the local service
@@ -132,27 +132,11 @@ func NewCache(cfg Config) (*cache.Cache, error) {
 	if err := cfg.CheckAndSetDefaults(); err != nil {
 		return nil, trace.Wrap(err)
 	}
-	slog.DebugContext(cfg.Context, "Creating in-memory backend cache.", "cache_name", cfg.CacheName)
-	mem, err := memory.New(memory.Config{
-		Context:   cfg.Context,
-		EventsOff: !cfg.EventsSystem,
-		Mirror:    true,
-	})
-	if err != nil {
-		return nil, trace.Wrap(err)
-	}
+
 	var tracer oteltrace.Tracer
 	if cfg.TracingProvider != nil {
 		tracer = cfg.TracingProvider.Tracer(teleport.ComponentCache)
 	}
-	reporter, err := backend.NewReporter(backend.ReporterConfig{
-		Component: teleport.ComponentCache,
-		Backend:   mem,
-		Tracer:    tracer,
-	})
-	if err != nil {
-		return nil, trace.Wrap(err)
-	}
 
 	component := slices.Clone(cfg.CacheName)
 	if cfg.ProcessID != "" {
@@ -163,14 +147,13 @@ func NewCache(cfg Config) (*cache.Cache, error) {
 	metricComponent := append(slices.Clone(cfg.CacheName), teleport.ComponentCache)
 
 	cacheCfg := cache.Config{
-		Context:         cfg.Context,
-		Backend:         reporter,
-		Component:       teleport.Component(component...),
-		MetricComponent: teleport.Component(metricComponent...),
-		Tracer:          tracer,
-		MaxRetryPeriod:  cfg.MaxRetryPeriod,
-		Unstarted:       cfg.Unstarted,
-
+		Context:                 cfg.Context,
+		Component:               teleport.Component(component...),
+		MetricComponent:         teleport.Component(metricComponent...),
+		Tracer:                  tracer,
+		Registerer:              cfg.Registerer,
+		MaxRetryPeriod:          cfg.MaxRetryPeriod,
+		Unstarted:               cfg.Unstarted,
 		Access:                  cfg.Access,
 		AccessLists:             cfg.AccessLists,
 		AccessMonitoringRules:   cfg.AccessMonitoringRules,

lib/backend/atomicwrite.go

@@ -218,6 +218,8 @@ const (
 	// MaxAtomicWriteSize is the maximum number of conditional actions that may
 	// be applied via a single atomic write. The exact number is subject to change
 	// but must always be less than the minimum value supported across all backends.
+	//
+	// NOTE:The buckets for backendmetrics.AtomicWriteSize must stay in sync with this constant.
 	MaxAtomicWriteSize = 64
 )
 

lib/backend/backendmetrics/metrics.go

@@ -0,0 +1,255 @@
+// Teleport
+// Copyright (C) 2025 Gravitational, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package backendmetrics
+
+import (
+	"github.com/gravitational/trace"
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/gravitational/teleport"
+	"github.com/gravitational/teleport/lib/observability/metrics"
+)
+
+var (
+	Requests = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: teleport.MetricBackendRequests,
+			Help: "Number of requests to the backend (reads, writes, and keepalives)",
+		},
+		[]string{teleport.ComponentLabel, teleport.TagReq, teleport.TagRange},
+	)
+	Watchers = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: teleport.MetricBackendWatchers,
+			Help: "Number of active backend watchers",
+		},
+		[]string{teleport.ComponentLabel},
+	)
+	WatcherQueues = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: teleport.MetricBackendWatcherQueues,
+			Help: "Watcher queue sizes",
+		},
+		[]string{teleport.ComponentLabel},
+	)
+	WriteRequests = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: teleport.MetricBackendWriteRequests,
+			Help: "Number of write requests to the backend",
+		},
+		[]string{teleport.ComponentLabel},
+	)
+	Writes = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: teleport.MetricNamespace,
+			Name:      teleport.MetricBackendWrites,
+			Help:      "Number of individual items written to the backend",
+		},
+		[]string{teleport.ComponentLabel},
+	)
+	WriteRequestsFailed = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: teleport.MetricBackendWriteFailedRequests,
+			Help: "Number of failed write requests to the backend",
+		},
+		[]string{teleport.ComponentLabel},
+	)
+	WriteRequestsFailedPrecondition = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: teleport.MetricNamespace,
+			Name:      teleport.MetricBackendWriteFailedPreconditionRequests,
+			Help:      "Number of write requests that failed due to a precondition (existence, revision, value, etc)",
+		},
+		[]string{teleport.ComponentLabel},
+	)
+	AtomicWriteRequests = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: teleport.MetricNamespace,
+			Name:      teleport.MetricBackendAtomicWriteRequests,
+			Help:      "Number of atomic write requests to the backend",
+		},
+		[]string{teleport.ComponentLabel},
+	)
+	AtomicWriteRequestsFailed = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: teleport.MetricNamespace,
+			Name:      teleport.MetricBackendAtomicWriteFailedRequests,
+			Help:      "Number of failed atomic write requests to the backend",
+		},
+		[]string{teleport.ComponentLabel},
+	)
+	AtomicWriteConditionFailed = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: teleport.MetricNamespace,
+			Name:      teleport.MetricBackendAtomicWriteConditionFailed,
+			Help:      "Number of times an atomic write request results in condition failure",
+		},
+		[]string{teleport.ComponentLabel},
+	)
+	AtomicWriteLatencies = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Namespace: teleport.MetricNamespace,
+			Name:      teleport.MetricBackendAtomicWriteHistogram,
+			Help:      "Latency for backend atomic write operations",
+			// lowest bucket start of upper bound 0.001 sec (1 ms) with factor 2
+			// highest bucket start of 0.001 sec * 2^15 == 32.768 sec
+			Buckets: prometheus.ExponentialBuckets(0.001, 2, 16),
+		},
+		[]string{teleport.ComponentLabel},
+	)
+	AtomicWriteSize = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Namespace: teleport.MetricNamespace,
+			Name:      teleport.MetricBackendAtomicWriteSize,
+			Help:      "Atomic write batch size",
+			// buckets of the form 1, 2, 4, 8, 16, etc...
+			Buckets: prometheus.ExponentialBuckets(1, 2, 8),
+		},
+		[]string{teleport.ComponentLabel},
+	)
+	AtomicWriteContention = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: teleport.MetricNamespace,
+			Name:      teleport.MetricBackendAtomicWriteContention,
+			Help:      "Number of times atomic write requests experience contention",
+		},
+		[]string{teleport.ComponentLabel},
+	)
+	BatchWriteRequests = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: teleport.MetricBackendBatchWriteRequests,
+			Help: "Number of batch write requests to the backend",
+		},
+		[]string{teleport.ComponentLabel},
+	)
+	BatchWriteRequestsFailed = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: teleport.MetricBackendBatchFailedWriteRequests,
+			Help: "Number of failed write requests to the backend",
+		},
+		[]string{teleport.ComponentLabel},
+	)
+	ReadRequests = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: teleport.MetricBackendReadRequests,
+			Help: "Number of read requests to the backend",
+		},
+		[]string{teleport.ComponentLabel},
+	)
+	Reads = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: teleport.MetricNamespace,
+			Name:      teleport.MetricBackendReads,
+			Help:      "Number of individual items read from the backend",
+		},
+		[]string{teleport.ComponentLabel},
+	)
+	ReadRequestsFailed = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: teleport.MetricBackendFailedReadRequests,
+			Help: "Number of failed read requests to the backend",
+		},
+		[]string{teleport.ComponentLabel},
+	)
+	StreamingRequests = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: teleport.MetricNamespace,
+			Subsystem: "backend",
+			Name:      "stream_requests",
+			Help:      "Number of inflight stream requests to the backend",
+		},
+		[]string{teleport.ComponentLabel},
+	)
+	StreamingRequestsFailed = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: teleport.MetricNamespace,
+			Subsystem: "backend",
+			Name:      "stream_requests_failed",
+			Help:      "Number of failed stream requests to the backend",
+		},
+		[]string{teleport.ComponentLabel},
+	)
+	BatchReadRequests = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: teleport.MetricBackendBatchReadRequests,
+			Help: "Number of read requests to the backend",
+		},
+		[]string{teleport.ComponentLabel},
+	)
+	BatchReadRequestsFailed = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: teleport.MetricBackendBatchFailedReadRequests,
+			Help: "Number of failed read requests to the backend",
+		},
+		[]string{teleport.ComponentLabel},
+	)
+	WriteLatencies = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name: teleport.MetricBackendWriteHistogram,
+			Help: "Latency for backend write operations",
+			// lowest bucket start of upper bound 0.001 sec (1 ms) with factor 2
+			// highest bucket start of 0.001 sec * 2^15 == 32.768 sec
+			Buckets: prometheus.ExponentialBuckets(0.001, 2, 16),
+		},
+		[]string{teleport.ComponentLabel},
+	)
+	BatchWriteLatencies = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name: teleport.MetricBackendBatchWriteHistogram,
+			Help: "Latency for backend batch write operations",
+			// lowest bucket start of upper bound 0.001 sec (1 ms) with factor 2
+			// highest bucket start of 0.001 sec * 2^15 == 32.768 sec
+			Buckets: prometheus.ExponentialBuckets(0.001, 2, 16),
+		},
+		[]string{teleport.ComponentLabel},
+	)
+	BatchReadLatencies = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name: teleport.MetricBackendBatchReadHistogram,
+			Help: "Latency for batch read operations",
+			// lowest bucket start of upper bound 0.001 sec (1 ms) with factor 2
+			// highest bucket start of 0.001 sec * 2^15 == 32.768 sec
+			Buckets: prometheus.ExponentialBuckets(0.001, 2, 16),
+		},
+		[]string{teleport.ComponentLabel},
+	)
+	ReadLatencies = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name: teleport.MetricBackendReadHistogram,
+			Help: "Latency for read operations",
+			// lowest bucket start of upper bound 0.001 sec (1 ms) with factor 2
+			// highest bucket start of 0.001 sec * 2^15 == 32.768 sec
+			Buckets: prometheus.ExponentialBuckets(0.001, 2, 16),
+		},
+		[]string{teleport.ComponentLabel},
+	)
+)
+
+// RegisterCollectors ensures all backend metrics are registered
+// with the provided [prometheus.Registerer].
+func RegisterCollectors(reg prometheus.Registerer) error {
+	return trace.Wrap(metrics.RegisterCollectors(reg,
+		Watchers, WatcherQueues, Requests, WriteRequests,
+		WriteRequestsFailed, BatchWriteRequests, BatchWriteRequestsFailed, ReadRequests,
+		ReadRequestsFailed, BatchReadRequests, BatchReadRequestsFailed, WriteLatencies,
+		WriteRequestsFailedPrecondition,
+		AtomicWriteRequests, AtomicWriteRequestsFailed, AtomicWriteConditionFailed, AtomicWriteLatencies,
+		AtomicWriteContention, AtomicWriteSize, Reads, Writes,
+		BatchWriteLatencies, BatchReadLatencies, ReadLatencies,
+		StreamingRequests, StreamingRequestsFailed,
+	))
+}

lib/backend/buffer.go

@@ -33,6 +33,7 @@ import (
 
 	"github.com/gravitational/teleport"
 	"github.com/gravitational/teleport/api/types"
+	"github.com/gravitational/teleport/lib/backend/backendmetrics"
 	logutils "github.com/gravitational/teleport/lib/utils/log"
 )
 
@@ -204,7 +205,7 @@ func (c *CircularBuffer) fanOutEvent(r Event) {
 	var watchersToDelete []*BufferWatcher
 	c.watchers.walkPath(r.Item.Key.String(), func(watcher *BufferWatcher) {
 		if watcher.MetricComponent != "" {
-			watcherQueues.WithLabelValues(watcher.MetricComponent).Set(float64(len(watcher.eventsC)))
+			backendmetrics.WatcherQueues.WithLabelValues(watcher.MetricComponent).Set(float64(len(watcher.eventsC)))
 		}
 		if !watcher.emit(r) {
 			watchersToDelete = append(watchersToDelete, watcher)

lib/backend/dynamo/atomicwrite.go

@@ -34,6 +34,7 @@ import (
 	"github.com/gravitational/teleport"
 	"github.com/gravitational/teleport/api/utils/retryutils"
 	"github.com/gravitational/teleport/lib/backend"
+	"github.com/gravitational/teleport/lib/backend/backendmetrics"
 )
 
 const (
@@ -250,7 +251,7 @@ TxnLoop:
 		}
 
 		if i > 0 {
-			backend.AtomicWriteContention.WithLabelValues(teleport.ComponentDynamoDB).Add(float64(i))
+			backendmetrics.AtomicWriteContention.WithLabelValues(teleport.ComponentDynamoDB).Add(float64(i))
 		}
 
 		if n := i + 1; n > 2 {

lib/backend/firestore/atomicwrite.go

@@ -29,6 +29,7 @@ import (
 
 	"github.com/gravitational/teleport"
 	"github.com/gravitational/teleport/lib/backend"
+	"github.com/gravitational/teleport/lib/backend/backendmetrics"
 )
 
 func (b *Backend) AtomicWrite(ctx context.Context, condacts []backend.ConditionalAction) (revision string, err error) {
@@ -146,7 +147,7 @@ func (b *Backend) AtomicWrite(ctx context.Context, condacts []backend.Conditiona
 	}
 
 	if n > 1 {
-		backend.AtomicWriteContention.WithLabelValues(teleport.ComponentFirestore).Add(float64(n - 1))
+		backendmetrics.AtomicWriteContention.WithLabelValues(teleport.ComponentFirestore).Add(float64(n - 1))
 	}
 
 	if n > 2 {