diff --git a/lib/auth/middleware.go b/lib/auth/middleware.go
index d4d22a28a5186..6b34f7a8e716d 100644
--- a/lib/auth/middleware.go
+++ b/lib/auth/middleware.go
@@ -52,6 +52,7 @@ import (
"github.com/gravitational/teleport/lib/limiter"
"github.com/gravitational/teleport/lib/multiplexer"
"github.com/gravitational/teleport/lib/observability/metrics"
+ grpcmetrics "github.com/gravitational/teleport/lib/observability/metrics/grpc"
"github.com/gravitational/teleport/lib/utils"
logutils "github.com/gravitational/teleport/lib/utils/log"
)
@@ -150,7 +151,7 @@ func NewTLSServer(ctx context.Context, cfg TLSServerConfig) (*TLSServer, error)
}
// sets up gRPC metrics interceptor
- grpcMetrics := metrics.CreateGRPCServerMetrics(cfg.Metrics.GRPCServerLatency, prometheus.Labels{teleport.TagServer: "teleport-auth"})
+ grpcMetrics := grpcmetrics.CreateGRPCServerMetrics(cfg.Metrics.GRPCServerLatency, prometheus.Labels{teleport.TagServer: "teleport-auth"})
err = metrics.RegisterPrometheusCollectors(grpcMetrics)
if err != nil {
return nil, trace.Wrap(err)
diff --git a/lib/msgraph/client.go b/lib/msgraph/client.go
index 081d6b28da239..4f74ed1531ce3 100644
--- a/lib/msgraph/client.go
+++ b/lib/msgraph/client.go
@@ -37,13 +37,13 @@ import (
"github.com/google/uuid"
"github.com/gravitational/trace"
"github.com/jonboulle/clockwork"
- "github.com/prometheus/client_golang/prometheus"
"github.com/gravitational/teleport"
apidefaults "github.com/gravitational/teleport/api/defaults"
"github.com/gravitational/teleport/api/types"
"github.com/gravitational/teleport/api/utils/retryutils"
"github.com/gravitational/teleport/lib/defaults"
+ "github.com/gravitational/teleport/lib/observability/metrics"
"github.com/gravitational/teleport/lib/utils"
)
@@ -101,7 +101,7 @@ type Config struct {
Logger *slog.Logger
// MetricsRegistry configures where metrics should be registered.
// When nil, metrics are created but not registered.
- MetricsRegistry prometheus.Registerer
+ MetricsRegistry *metrics.Registry
}
// SetDefaults sets the default values for optional fields.
@@ -124,6 +124,9 @@ func (cfg *Config) SetDefaults() {
if cfg.Logger == nil {
cfg.Logger = slog.With(teleport.ComponentKey, "msgraph")
}
+ if cfg.MetricsRegistry == nil {
+ cfg.MetricsRegistry = metrics.NoopRegistry()
+ }
}
// Validate checks that required fields are set.
@@ -162,12 +165,10 @@ func NewClient(cfg Config) (*Client, error) {
return nil, trace.Wrap(err)
}
- metrics := newMetrics()
+ m := newMetrics(cfg.MetricsRegistry)
// gracefully handle not being given a metric registry
- if cfg.MetricsRegistry != nil {
- if err := metrics.register(cfg.MetricsRegistry); err != nil {
- return nil, trace.Wrap(err, "registering metrics")
- }
+ if err := m.register(cfg.MetricsRegistry); err != nil {
+ cfg.Logger.ErrorContext(context.Background(), "Failed to register metrics.", "error", err)
}
return &Client{
@@ -178,7 +179,7 @@ func NewClient(cfg Config) (*Client, error) {
baseURL: base.JoinPath(graphVersion),
pageSize: cfg.PageSize,
logger: cfg.Logger,
- metrics: metrics,
+ metrics: m,
}, nil
}
diff --git a/lib/msgraph/metrics.go b/lib/msgraph/metrics.go
index 2c762ea17ff18..d7bca2ee9f223 100644
--- a/lib/msgraph/metrics.go
+++ b/lib/msgraph/metrics.go
@@ -22,7 +22,7 @@ import (
"github.com/gravitational/trace"
"github.com/prometheus/client_golang/prometheus"
- "github.com/gravitational/teleport"
+ "github.com/gravitational/teleport/lib/observability/metrics"
)
type clientMetrics struct {
@@ -34,22 +34,26 @@ type clientMetrics struct {
}
const (
- metricSubsystem = "msgraph"
metricsLabelStatus = "status"
metricsLabelsMethod = "method"
)
-func newMetrics() *clientMetrics {
+func newMetrics(reg *metrics.Registry) *clientMetrics {
+ var namespace, subsystem string
+ if reg != nil {
+ namespace = reg.Namespace()
+ subsystem = reg.Subsystem()
+ }
return &clientMetrics{
requestTotal: prometheus.NewCounterVec(prometheus.CounterOpts{
- Namespace: teleport.MetricNamespace,
- Subsystem: metricSubsystem,
+ Namespace: namespace,
+ Subsystem: subsystem,
Name: "request_total",
Help: "Total number of requests made to MS Graph",
}, []string{metricsLabelsMethod, metricsLabelStatus}),
requestDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
- Namespace: teleport.MetricNamespace,
- Subsystem: metricSubsystem,
+ Namespace: namespace,
+ Subsystem: subsystem,
Name: "request_duration_seconds",
Help: "Request to MS Graph duration in seconds.",
}, []string{metricsLabelsMethod}),
diff --git a/lib/metrics/gatherers.go b/lib/observability/metrics/gatherers.go
similarity index 100%
rename from lib/metrics/gatherers.go
rename to lib/observability/metrics/gatherers.go
diff --git a/lib/observability/metrics/grpc/grpc.go b/lib/observability/metrics/grpc/grpc.go
new file mode 100644
index 0000000000000..426b7c30f988c
--- /dev/null
+++ b/lib/observability/metrics/grpc/grpc.go
@@ -0,0 +1,65 @@
+/*
+ * Teleport
+ * Copyright (C) 2025 Gravitational, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+package grpcmetrics
+
+import (
+ grpcprom "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus"
+ "github.com/prometheus/client_golang/prometheus"
+)
+
+// CreateGRPCServerMetrics creates server gRPC metrics configuration that is to be registered and used by the caller
+// in an openmetrics unary and/or stream interceptor
+func CreateGRPCServerMetrics(
+ latencyEnabled bool, labels prometheus.Labels,
+) *grpcprom.ServerMetrics {
+ serverOpts := []grpcprom.ServerMetricsOption{
+ grpcprom.WithServerCounterOptions(grpcprom.WithConstLabels(labels)),
+ }
+ if latencyEnabled {
+ histOpts := grpcHistogramOpts(labels)
+ serverOpts = append(
+ serverOpts, grpcprom.WithServerHandlingTimeHistogram(histOpts...),
+ )
+ }
+ return grpcprom.NewServerMetrics(serverOpts...)
+}
+
+// CreateGRPCClientMetrics creates client gRPC metrics configuration that is to be registered and used by the caller
+// in an openmetrics unary and/or stream interceptor
+func CreateGRPCClientMetrics(
+ latencyEnabled bool, labels prometheus.Labels,
+) *grpcprom.ClientMetrics {
+ clientOpts := []grpcprom.ClientMetricsOption{
+ grpcprom.WithClientCounterOptions(grpcprom.WithConstLabels(labels)),
+ }
+ if latencyEnabled {
+ histOpts := grpcHistogramOpts(labels)
+ clientOpts = append(
+ clientOpts, grpcprom.WithClientHandlingTimeHistogram(histOpts...),
+ )
+ }
+ return grpcprom.NewClientMetrics(clientOpts...)
+}
+
+func grpcHistogramOpts(labels prometheus.Labels) []grpcprom.HistogramOption {
+ return []grpcprom.HistogramOption{
+ grpcprom.WithHistogramBuckets(prometheus.ExponentialBuckets(0.001, 2, 16)),
+ grpcprom.WithHistogramConstLabels(labels),
+ }
+}
diff --git a/lib/observability/metrics/prometheus.go b/lib/observability/metrics/prometheus.go
index d190c7ca1f5dc..440a4078c696c 100644
--- a/lib/observability/metrics/prometheus.go
+++ b/lib/observability/metrics/prometheus.go
@@ -23,7 +23,6 @@ import (
"runtime"
"github.com/gravitational/trace"
- grpcprom "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus"
"github.com/prometheus/client_golang/prometheus"
"github.com/gravitational/teleport"
@@ -70,44 +69,3 @@ func BuildCollector() prometheus.Collector {
func() float64 { return 1 },
)
}
-
-// CreateGRPCServerMetrics creates server gRPC metrics configuration that is to be registered and used by the caller
-// in an openmetrics unary and/or stream interceptor
-func CreateGRPCServerMetrics(
- latencyEnabled bool, labels prometheus.Labels,
-) *grpcprom.ServerMetrics {
- serverOpts := []grpcprom.ServerMetricsOption{
- grpcprom.WithServerCounterOptions(grpcprom.WithConstLabels(labels)),
- }
- if latencyEnabled {
- histOpts := grpcHistogramOpts(labels)
- serverOpts = append(
- serverOpts, grpcprom.WithServerHandlingTimeHistogram(histOpts...),
- )
- }
- return grpcprom.NewServerMetrics(serverOpts...)
-}
-
-// CreateGRPCClientMetrics creates client gRPC metrics configuration that is to be registered and used by the caller
-// in an openmetrics unary and/or stream interceptor
-func CreateGRPCClientMetrics(
- latencyEnabled bool, labels prometheus.Labels,
-) *grpcprom.ClientMetrics {
- clientOpts := []grpcprom.ClientMetricsOption{
- grpcprom.WithClientCounterOptions(grpcprom.WithConstLabels(labels)),
- }
- if latencyEnabled {
- histOpts := grpcHistogramOpts(labels)
- clientOpts = append(
- clientOpts, grpcprom.WithClientHandlingTimeHistogram(histOpts...),
- )
- }
- return grpcprom.NewClientMetrics(clientOpts...)
-}
-
-func grpcHistogramOpts(labels prometheus.Labels) []grpcprom.HistogramOption {
- return []grpcprom.HistogramOption{
- grpcprom.WithHistogramBuckets(prometheus.ExponentialBuckets(0.001, 2, 16)),
- grpcprom.WithHistogramConstLabels(labels),
- }
-}
diff --git a/lib/observability/metrics/registry.go b/lib/observability/metrics/registry.go
new file mode 100644
index 0000000000000..d329ab9876212
--- /dev/null
+++ b/lib/observability/metrics/registry.go
@@ -0,0 +1,122 @@
+/*
+ * Teleport
+ * Copyright (C) 2025 Gravitational, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+package metrics
+
+import (
+ "errors"
+
+ "github.com/prometheus/client_golang/prometheus"
+)
+
+// Registry is a [prometheus.Registerer] for a Teleport process that
+// allows propagating additional information such as:
+// - the metric namespace (`teleport`, `teleport_bot`, `teleport_plugins`)
+// - an optional subsystem
+//
+// This should be passed anywhere that needs to register a metric.
+type Registry struct {
+ prometheus.Registerer
+
+ namespace string
+ subsystem string
+}
+
+// Namespace returns the namespace that should be used by metrics registered
+// in this Registry. Common namespaces are "teleport", "tbot", and
+// "teleport_plugins".
+func (r *Registry) Namespace() string {
+ return r.namespace
+}
+
+// Subsystem is the subsystem base that should be used by metrics registered in
+// this Registry. Subsystem parts can be added with WrapWithSubsystem.
+func (r *Registry) Subsystem() string {
+ return r.subsystem
+}
+
+// Wrap wraps a Registry by adding a component to its subsystem.
+// This should be used before passing a registry to a sub-component.
+// Example usage:
+//
+// rootReg := prometheus.NewRegistry()
+// process.AddGatherer(rootReg)
+// reg, err := NewRegistry(rootReg, "teleport_plugins", "")
+// go runFooService(ctx, log, reg.Wrap("foo"))
+// go runBarService(ctx, log, reg.Wrap("bar"))
+func (r *Registry) Wrap(subsystem string) *Registry {
+ newReg := &Registry{
+ Registerer: r.Registerer,
+ namespace: r.namespace,
+ subsystem: r.subsystem + "_" + subsystem,
+ }
+ return newReg
+}
+
+// NewRegistry creates a new Registry wrapping a prometheus registry.
+// This should only be called when starting the service management routines such
+// as: service.NewTeleport(), tbot.New(), or the hosted plugin manager.
+// Services and sub-services should take the registry as a parameter, like they
+// already do for the logger.
+// Example usage:
+//
+// rootReg := prometheus.NewRegistry()
+// process.AddGatherer(rootReg)
+// reg, err := NewRegistry(rootReg, "teleport_plugins", "")
+// go runFooService(ctx, log, reg.Wrap("foo"))
+// go runBarService(ctx, log, reg.Wrap("bar"))
+func NewRegistry(reg prometheus.Registerer, namespace, subsystem string) (*Registry, error) {
+ if reg == nil {
+ return nil, errors.New("nil prometheus.Registerer (this is a bug)")
+ }
+ if namespace == "" {
+ return nil, errors.New("namespace is required (this is a bug)")
+ }
+ return &Registry{
+ Registerer: reg,
+ namespace: namespace,
+ subsystem: subsystem,
+ }, nil
+}
+
+// NoopRegistry returns a Registry that doesn't register metrics.
+// This can be used in tests, or to provide backward compatibility when a nil
+// Registry is passed.
+func NoopRegistry() *Registry {
+ return &Registry{
+ Registerer: noopRegistry{},
+ namespace: "noop",
+ subsystem: "",
+ }
+}
+
+type noopRegistry struct{}
+
+// Register implements [prometheus.Registerer].
+func (b noopRegistry) Register(collector prometheus.Collector) error {
+ return nil
+}
+
+// MustRegister implements [prometheus.Registerer].
+func (b noopRegistry) MustRegister(collector ...prometheus.Collector) {
+}
+
+// Unregister implements [prometheus.Registerer].
+func (b noopRegistry) Unregister(collector prometheus.Collector) bool {
+ return true
+}
diff --git a/lib/service/connect.go b/lib/service/connect.go
index 1815691d1594f..2f1b938eef36f 100644
--- a/lib/service/connect.go
+++ b/lib/service/connect.go
@@ -57,6 +57,7 @@ import (
"github.com/gravitational/teleport/lib/defaults"
"github.com/gravitational/teleport/lib/join/joinclient"
"github.com/gravitational/teleport/lib/observability/metrics"
+ grpcmetrics "github.com/gravitational/teleport/lib/observability/metrics/grpc"
"github.com/gravitational/teleport/lib/openssh"
"github.com/gravitational/teleport/lib/reversetunnelclient"
servicebreaker "github.com/gravitational/teleport/lib/service/breaker"
@@ -1507,7 +1508,7 @@ func (process *TeleportProcess) newClientDirect(authServers []utils.NetAddr, tls
var dialOpts []grpc.DialOption
if role == types.RoleProxy {
- grpcMetrics := metrics.CreateGRPCClientMetrics(process.Config.Metrics.GRPCClientLatency, prometheus.Labels{teleport.TagClient: "teleport-proxy"})
+ grpcMetrics := grpcmetrics.CreateGRPCClientMetrics(process.Config.Metrics.GRPCClientLatency, prometheus.Labels{teleport.TagClient: "teleport-proxy"})
if err := metrics.RegisterPrometheusCollectors(grpcMetrics); err != nil {
return nil, nil, trace.Wrap(err)
}
diff --git a/lib/service/service.go b/lib/service/service.go
index ff5d024075a3b..8652583700629 100644
--- a/lib/service/service.go
+++ b/lib/service/service.go
@@ -143,9 +143,9 @@ import (
kubeproxy "github.com/gravitational/teleport/lib/kube/proxy"
"github.com/gravitational/teleport/lib/labels"
"github.com/gravitational/teleport/lib/limiter"
- "github.com/gravitational/teleport/lib/metrics"
"github.com/gravitational/teleport/lib/modules"
"github.com/gravitational/teleport/lib/multiplexer"
+ "github.com/gravitational/teleport/lib/observability/metrics"
"github.com/gravitational/teleport/lib/observability/tracing"
"github.com/gravitational/teleport/lib/openssh"
"github.com/gravitational/teleport/lib/pam"
diff --git a/lib/service/service_test.go b/lib/service/service_test.go
index bc71c1d103dac..6229a0436744f 100644
--- a/lib/service/service_test.go
+++ b/lib/service/service_test.go
@@ -70,10 +70,10 @@ import (
"github.com/gravitational/teleport/lib/events/athena"
"github.com/gravitational/teleport/lib/integrations/externalauditstorage"
"github.com/gravitational/teleport/lib/limiter"
- "github.com/gravitational/teleport/lib/metrics"
"github.com/gravitational/teleport/lib/modules"
"github.com/gravitational/teleport/lib/modules/modulestest"
"github.com/gravitational/teleport/lib/multiplexer"
+ "github.com/gravitational/teleport/lib/observability/metrics"
"github.com/gravitational/teleport/lib/reversetunnelclient"
"github.com/gravitational/teleport/lib/service/servicecfg"
"github.com/gravitational/teleport/lib/services"
diff --git a/lib/services/reconciler.go b/lib/services/reconciler.go
index 26bdc7170b7d3..a7cae8d16f3db 100644
--- a/lib/services/reconciler.go
+++ b/lib/services/reconciler.go
@@ -28,6 +28,7 @@ import (
"github.com/gravitational/teleport"
"github.com/gravitational/teleport/api/types"
+ "github.com/gravitational/teleport/lib/observability/metrics"
logutils "github.com/gravitational/teleport/lib/utils/log"
)
@@ -105,7 +106,7 @@ func (c *GenericReconcilerConfig[K, T]) CheckAndSetDefaults() error {
var err error
// If we are not given metrics, we create our own so we don't
// panic when trying to increment/observe.
- c.Metrics, err = NewReconcilerMetrics("unknown")
+ c.Metrics, err = NewReconcilerMetrics(metrics.NoopRegistry().Wrap("unknown"))
if err != nil {
return trace.Wrap(err)
}
@@ -136,20 +137,23 @@ const (
// The caller is responsible for registering them into an appropriate registry.
// The same ReconcilerMetrics can be used across different reconcilers.
// The metrics subsystem cannot be empty.
-func NewReconcilerMetrics(subsystem string) (*ReconcilerMetrics, error) {
- if subsystem == "" {
- return nil, trace.BadParameter("missing reconciler metric subsystem (this is a bug)")
+func NewReconcilerMetrics(reg *metrics.Registry) (*ReconcilerMetrics, error) {
+ if reg == nil {
+ return nil, trace.BadParameter("missing metrics registry (this is a bug)")
+ }
+ if reg.Subsystem() == "" {
+ return nil, trace.BadParameter("missing metrics subsystem (this is a bug)")
}
return &ReconcilerMetrics{
reconciliationTotal: prometheus.NewCounterVec(prometheus.CounterOpts{
- Namespace: teleport.MetricNamespace,
- Subsystem: subsystem,
+ Namespace: reg.Namespace(),
+ Subsystem: reg.Subsystem(),
Name: "reconciliation_total",
Help: "Total number of individual resource reconciliations.",
}, []string{metricLabelKind, metricLabelOperation, metricLabelResult}),
reconciliationDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
- Namespace: teleport.MetricNamespace,
- Subsystem: subsystem,
+ Namespace: reg.Namespace(),
+ Subsystem: reg.Subsystem(),
Name: "reconciliation_duration_seconds",
Help: "The duration of individual resource reconciliation in seconds.",
}, []string{metricLabelKind, metricLabelOperation}),
diff --git a/lib/tbot/services/workloadidentity/workload_api.go b/lib/tbot/services/workloadidentity/workload_api.go
index 23a26f00d936c..2dc7bacb98cf9 100644
--- a/lib/tbot/services/workloadidentity/workload_api.go
+++ b/lib/tbot/services/workloadidentity/workload_api.go
@@ -46,6 +46,7 @@ import (
apiclient "github.com/gravitational/teleport/api/client"
"github.com/gravitational/teleport/lib/defaults"
"github.com/gravitational/teleport/lib/observability/metrics"
+ grpcmetrics "github.com/gravitational/teleport/lib/observability/metrics/grpc"
"github.com/gravitational/teleport/lib/tbot/bot"
"github.com/gravitational/teleport/lib/tbot/client"
"github.com/gravitational/teleport/lib/tbot/internal"
@@ -168,7 +169,7 @@ func (s *WorkloadAPIService) Run(ctx context.Context) error {
defer s.client.Close()
s.log.DebugContext(ctx, "Completed pre-run initialization")
- srvMetrics := metrics.CreateGRPCServerMetrics(
+ srvMetrics := grpcmetrics.CreateGRPCServerMetrics(
true, prometheus.Labels{
teleport.TagServer: "tbot-workload-identity-api",
},
diff --git a/lib/tbot/tbot.go b/lib/tbot/tbot.go
index f8f9a628a19d4..0229419b00954 100644
--- a/lib/tbot/tbot.go
+++ b/lib/tbot/tbot.go
@@ -35,6 +35,7 @@ import (
"github.com/gravitational/teleport/api/types"
"github.com/gravitational/teleport/lib/modules"
"github.com/gravitational/teleport/lib/observability/metrics"
+ grpcmetrics "github.com/gravitational/teleport/lib/observability/metrics/grpc"
"github.com/gravitational/teleport/lib/tbot/bot"
"github.com/gravitational/teleport/lib/tbot/bot/connection"
"github.com/gravitational/teleport/lib/tbot/config"
@@ -57,7 +58,7 @@ import (
var tracer = otel.Tracer("github.com/gravitational/teleport/lib/tbot")
-var clientMetrics = metrics.CreateGRPCClientMetrics(
+var clientMetrics = grpcmetrics.CreateGRPCClientMetrics(
false,
prometheus.Labels{},
)