[v18] Backport metrics.Registry and entra metrics

lib/auth/middleware.go

@@ -55,6 +55,7 @@ import (
 	"github.com/gravitational/teleport/lib/limiter"
 	"github.com/gravitational/teleport/lib/multiplexer"
 	"github.com/gravitational/teleport/lib/observability/metrics"
+	grpcmetrics "github.com/gravitational/teleport/lib/observability/metrics/grpc"
 	"github.com/gravitational/teleport/lib/tlsca"
 	"github.com/gravitational/teleport/lib/utils"
 	logutils "github.com/gravitational/teleport/lib/utils/log"
@@ -162,7 +163,7 @@ func NewTLSServer(ctx context.Context, cfg TLSServerConfig) (*TLSServer, error)
 	}
 
 	// sets up gRPC metrics interceptor
-	grpcMetrics := metrics.CreateGRPCServerMetrics(cfg.Metrics.GRPCServerLatency, prometheus.Labels{teleport.TagServer: "teleport-auth"})
+	grpcMetrics := grpcmetrics.CreateGRPCServerMetrics(cfg.Metrics.GRPCServerLatency, prometheus.Labels{teleport.TagServer: "teleport-auth"})
 	err = metrics.RegisterPrometheusCollectors(grpcMetrics)
 	if err != nil {
 		return nil, trace.Wrap(err)

lib/msgraph/client.go

@@ -43,6 +43,7 @@ import (
 	"github.com/gravitational/teleport/api/types"
 	"github.com/gravitational/teleport/api/utils/retryutils"
 	"github.com/gravitational/teleport/lib/defaults"
+	"github.com/gravitational/teleport/lib/observability/metrics"
 	"github.com/gravitational/teleport/lib/utils"
 )
 
@@ -98,6 +99,9 @@ type Config struct {
 	// GraphEndpoint specifies root domain of the Graph API.
 	GraphEndpoint string
 	Logger        *slog.Logger
+	// MetricsRegistry configures where metrics should be registered.
+	// When nil, metrics are created but not registered.
+	MetricsRegistry *metrics.Registry
 }
 
 // SetDefaults sets the default values for optional fields.
@@ -120,6 +124,9 @@ func (cfg *Config) SetDefaults() {
 	if cfg.Logger == nil {
 		cfg.Logger = slog.With(teleport.ComponentKey, "msgraph")
 	}
+	if cfg.MetricsRegistry == nil {
+		cfg.MetricsRegistry = metrics.NoopRegistry()
+	}
 }
 
 // Validate checks that required fields are set.
@@ -144,6 +151,7 @@ type Client struct {
 	baseURL       *url.URL
 	pageSize      int
 	logger        *slog.Logger
+	metrics       *clientMetrics
 }
 
 // NewClient returns a new client for the given config.
@@ -156,6 +164,13 @@ func NewClient(cfg Config) (*Client, error) {
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
+
+	m := newMetrics(cfg.MetricsRegistry)
+	// gracefully handle not being given a metric registry
+	if err := m.register(cfg.MetricsRegistry); err != nil {
+		cfg.Logger.ErrorContext(context.Background(), "Failed to register metrics.", "error", err)
+	}
+
 	return &Client{
 		httpClient:    cfg.HTTPClient,
 		tokenProvider: cfg.TokenProvider,
@@ -164,6 +179,7 @@ func NewClient(cfg Config) (*Client, error) {
 		baseURL:       base.JoinPath(graphVersion),
 		pageSize:      cfg.PageSize,
 		logger:        cfg.Logger,
+		metrics:       m,
 	}, nil
 }
 
@@ -201,7 +217,8 @@ func (c *Client) request(ctx context.Context, method string, uri string, header
 	}
 
 	var lastErr error
-	for i := 0; i < maxRetries; i++ {
+	var start time.Time
+	for range maxRetries {
 		if retryAfter > 0 {
 			select {
 			case <-c.clock.After(retryAfter):
@@ -231,10 +248,13 @@ func (c *Client) request(ctx context.Context, method string, uri string, header
 		// https://learn.microsoft.com/en-us/graph/best-practices-concept#reliability-and-support
 		req.Header.Set("client-request-id", requestID)
 
+		start = c.clock.Now()
 		resp, err := c.httpClient.Do(req)
 		if err != nil {
 			return nil, trace.Wrap(err) // hard I/O error, bail
 		}
+		c.metrics.requestDuration.WithLabelValues(method).Observe(c.clock.Since(start).Seconds())
+		c.metrics.requestTotal.WithLabelValues(method, strconv.Itoa(resp.StatusCode))
 
 		if resp.StatusCode >= 200 && resp.StatusCode < 400 {
 			return resp, nil

lib/msgraph/metrics.go

@@ -0,0 +1,68 @@
+/*
+ * Teleport
+ * Copyright (C) 2025  Gravitational, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package msgraph
+
+import (
+	"github.com/gravitational/trace"
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/gravitational/teleport/lib/observability/metrics"
+)
+
+type clientMetrics struct {
+	// requestsTotal keeps track of the number of requests done by the client
+	// This metric is labeled by status code.
+	requestTotal *prometheus.CounterVec
+	// requestDuration keeps track of the request duration, in seconds.
+	requestDuration *prometheus.HistogramVec
+}
+
+const (
+	metricsLabelStatus  = "status"
+	metricsLabelsMethod = "method"
+)
+
+func newMetrics(reg *metrics.Registry) *clientMetrics {
+	var namespace, subsystem string
+	if reg != nil {
+		namespace = reg.Namespace()
+		subsystem = reg.Subsystem()
+	}
+	return &clientMetrics{
+		requestTotal: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: subsystem,
+			Name:      "request_total",
+			Help:      "Total number of requests made to MS Graph",
+		}, []string{metricsLabelsMethod, metricsLabelStatus}),
+		requestDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
+			Namespace: namespace,
+			Subsystem: subsystem,
+			Name:      "request_duration_seconds",
+			Help:      "Request to MS Graph duration in seconds.",
+		}, []string{metricsLabelsMethod}),
+	}
+}
+
+func (metrics *clientMetrics) register(r prometheus.Registerer) error {
+	return trace.NewAggregate(
+		r.Register(metrics.requestTotal),
+		r.Register(metrics.requestDuration),
+	)
+}

lib/observability/metrics/gatherers.go

@@ -0,0 +1,58 @@
+/*
+ * Teleport
+ * Copyright (C) 2025  Gravitational, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package metrics
+
+import (
+	"sync"
+
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+)
+
+// SyncGatherers wraps a [prometheus.Gatherers] so it can be accessed
+// in a thread-safe fashion. Gatherer can be added with AddGatherer.
+type SyncGatherers struct {
+	mutex     sync.Mutex
+	gatherers prometheus.Gatherers
+}
+
+// NewSyncGatherers returns a thread-safe [prometheus.Gatherers].
+func NewSyncGatherers(gatherers ...prometheus.Gatherer) *SyncGatherers {
+	return &SyncGatherers{
+		gatherers: gatherers,
+	}
+}
+
+// AddGatherer adds a gatherer to the SyncGatherers slice.
+func (s *SyncGatherers) AddGatherer(g prometheus.Gatherer) {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+
+	s.gatherers = append(s.gatherers, g)
+}
+
+// Gather implements [prometheus.Gatherer].
+func (s *SyncGatherers) Gather() ([]*dto.MetricFamily, error) {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	if s.gatherers == nil {
+		return nil, nil
+	}
+	return s.gatherers.Gather()
+}

lib/observability/metrics/grpc/grpc.go

@@ -0,0 +1,65 @@
+/*
+ * Teleport
+ * Copyright (C) 2025  Gravitational, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package grpcmetrics
+
+import (
+	grpcprom "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// CreateGRPCServerMetrics creates server gRPC metrics configuration that is to be registered and used by the caller
+// in an openmetrics unary and/or stream interceptor
+func CreateGRPCServerMetrics(
+	latencyEnabled bool, labels prometheus.Labels,
+) *grpcprom.ServerMetrics {
+	serverOpts := []grpcprom.ServerMetricsOption{
+		grpcprom.WithServerCounterOptions(grpcprom.WithConstLabels(labels)),
+	}
+	if latencyEnabled {
+		histOpts := grpcHistogramOpts(labels)
+		serverOpts = append(
+			serverOpts, grpcprom.WithServerHandlingTimeHistogram(histOpts...),
+		)
+	}
+	return grpcprom.NewServerMetrics(serverOpts...)
+}
+
+// CreateGRPCClientMetrics creates client gRPC metrics configuration that is to be registered and used by the caller
+// in an openmetrics unary and/or stream interceptor
+func CreateGRPCClientMetrics(
+	latencyEnabled bool, labels prometheus.Labels,
+) *grpcprom.ClientMetrics {
+	clientOpts := []grpcprom.ClientMetricsOption{
+		grpcprom.WithClientCounterOptions(grpcprom.WithConstLabels(labels)),
+	}
+	if latencyEnabled {
+		histOpts := grpcHistogramOpts(labels)
+		clientOpts = append(
+			clientOpts, grpcprom.WithClientHandlingTimeHistogram(histOpts...),
+		)
+	}
+	return grpcprom.NewClientMetrics(clientOpts...)
+}
+
+func grpcHistogramOpts(labels prometheus.Labels) []grpcprom.HistogramOption {
+	return []grpcprom.HistogramOption{
+		grpcprom.WithHistogramBuckets(prometheus.ExponentialBuckets(0.001, 2, 16)),
+		grpcprom.WithHistogramConstLabels(labels),
+	}
+}

lib/observability/metrics/prometheus.go

@@ -23,7 +23,6 @@ import (
 	"runtime"
 
 	"github.com/gravitational/trace"
-	grpcprom "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus"
 	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/gravitational/teleport"
@@ -70,44 +69,3 @@ func BuildCollector() prometheus.Collector {
 		func() float64 { return 1 },
 	)
 }
-
-// CreateGRPCServerMetrics creates server gRPC metrics configuration that is to be registered and used by the caller
-// in an openmetrics unary and/or stream interceptor
-func CreateGRPCServerMetrics(
-	latencyEnabled bool, labels prometheus.Labels,
-) *grpcprom.ServerMetrics {
-	serverOpts := []grpcprom.ServerMetricsOption{
-		grpcprom.WithServerCounterOptions(grpcprom.WithConstLabels(labels)),
-	}
-	if latencyEnabled {
-		histOpts := grpcHistogramOpts(labels)
-		serverOpts = append(
-			serverOpts, grpcprom.WithServerHandlingTimeHistogram(histOpts...),
-		)
-	}
-	return grpcprom.NewServerMetrics(serverOpts...)
-}
-
-// CreateGRPCClientMetrics creates client gRPC metrics configuration that is to be registered and used by the caller
-// in an openmetrics unary and/or stream interceptor
-func CreateGRPCClientMetrics(
-	latencyEnabled bool, labels prometheus.Labels,
-) *grpcprom.ClientMetrics {
-	clientOpts := []grpcprom.ClientMetricsOption{
-		grpcprom.WithClientCounterOptions(grpcprom.WithConstLabels(labels)),
-	}
-	if latencyEnabled {
-		histOpts := grpcHistogramOpts(labels)
-		clientOpts = append(
-			clientOpts, grpcprom.WithClientHandlingTimeHistogram(histOpts...),
-		)
-	}
-	return grpcprom.NewClientMetrics(clientOpts...)
-}
-
-func grpcHistogramOpts(labels prometheus.Labels) []grpcprom.HistogramOption {
-	return []grpcprom.HistogramOption{
-		grpcprom.WithHistogramBuckets(prometheus.ExponentialBuckets(0.001, 2, 16)),
-		grpcprom.WithHistogramConstLabels(labels),
-	}
-}