Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion lib/auth/middleware.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ import (
"github.com/gravitational/teleport/lib/limiter"
"github.com/gravitational/teleport/lib/multiplexer"
"github.com/gravitational/teleport/lib/observability/metrics"
grpcmetrics "github.com/gravitational/teleport/lib/observability/metrics/grpc"
"github.com/gravitational/teleport/lib/utils"
logutils "github.com/gravitational/teleport/lib/utils/log"
)
Expand Down Expand Up @@ -150,7 +151,7 @@ func NewTLSServer(ctx context.Context, cfg TLSServerConfig) (*TLSServer, error)
}

// sets up gRPC metrics interceptor
grpcMetrics := metrics.CreateGRPCServerMetrics(cfg.Metrics.GRPCServerLatency, prometheus.Labels{teleport.TagServer: "teleport-auth"})
grpcMetrics := grpcmetrics.CreateGRPCServerMetrics(cfg.Metrics.GRPCServerLatency, prometheus.Labels{teleport.TagServer: "teleport-auth"})
err = metrics.RegisterPrometheusCollectors(grpcMetrics)
if err != nil {
return nil, trace.Wrap(err)
Expand Down
17 changes: 9 additions & 8 deletions lib/msgraph/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,13 +37,13 @@ import (
"github.com/google/uuid"
"github.com/gravitational/trace"
"github.com/jonboulle/clockwork"
"github.com/prometheus/client_golang/prometheus"

"github.com/gravitational/teleport"
apidefaults "github.com/gravitational/teleport/api/defaults"
"github.com/gravitational/teleport/api/types"
"github.com/gravitational/teleport/api/utils/retryutils"
"github.com/gravitational/teleport/lib/defaults"
"github.com/gravitational/teleport/lib/observability/metrics"
"github.com/gravitational/teleport/lib/utils"
)

Expand Down Expand Up @@ -101,7 +101,7 @@ type Config struct {
Logger *slog.Logger
// MetricsRegistry configures where metrics should be registered.
// When nil, metrics are created but not registered.
MetricsRegistry prometheus.Registerer
MetricsRegistry *metrics.Registry
}

// SetDefaults sets the default values for optional fields.
Expand All @@ -124,6 +124,9 @@ func (cfg *Config) SetDefaults() {
if cfg.Logger == nil {
cfg.Logger = slog.With(teleport.ComponentKey, "msgraph")
}
if cfg.MetricsRegistry == nil {
cfg.MetricsRegistry = metrics.NoopRegistry()
}
}

// Validate checks that required fields are set.
Expand Down Expand Up @@ -162,12 +165,10 @@ func NewClient(cfg Config) (*Client, error) {
return nil, trace.Wrap(err)
}

metrics := newMetrics()
m := newMetrics(cfg.MetricsRegistry)
// gracefully handle not being given a metric registry
if cfg.MetricsRegistry != nil {
if err := metrics.register(cfg.MetricsRegistry); err != nil {
return nil, trace.Wrap(err, "registering metrics")
}
if err := m.register(cfg.MetricsRegistry); err != nil {
cfg.Logger.ErrorContext(context.Background(), "Failed to register metrics.", "error", err)
}

return &Client{
Expand All @@ -178,7 +179,7 @@ func NewClient(cfg Config) (*Client, error) {
baseURL: base.JoinPath(graphVersion),
pageSize: cfg.PageSize,
logger: cfg.Logger,
metrics: metrics,
metrics: m,
}, nil
}

Expand Down
18 changes: 11 additions & 7 deletions lib/msgraph/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import (
"github.com/gravitational/trace"
"github.com/prometheus/client_golang/prometheus"

"github.com/gravitational/teleport"
"github.com/gravitational/teleport/lib/observability/metrics"
)

type clientMetrics struct {
Expand All @@ -34,22 +34,26 @@ type clientMetrics struct {
}

const (
metricSubsystem = "msgraph"
metricsLabelStatus = "status"
metricsLabelsMethod = "method"
)

func newMetrics() *clientMetrics {
func newMetrics(reg *metrics.Registry) *clientMetrics {
var namespace, subsystem string
if reg != nil {
namespace = reg.Namespace()
subsystem = reg.Subsystem()
}
return &clientMetrics{
requestTotal: prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: teleport.MetricNamespace,
Subsystem: metricSubsystem,
Namespace: namespace,
Subsystem: subsystem,
Name: "request_total",
Help: "Total number of requests made to MS Graph",
}, []string{metricsLabelsMethod, metricsLabelStatus}),
requestDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: teleport.MetricNamespace,
Subsystem: metricSubsystem,
Namespace: namespace,
Subsystem: subsystem,
Name: "request_duration_seconds",
Help: "Request to MS Graph duration in seconds.",
}, []string{metricsLabelsMethod}),
Expand Down
File renamed without changes.
65 changes: 65 additions & 0 deletions lib/observability/metrics/grpc/grpc.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
/*
* Teleport
* Copyright (C) 2025 Gravitational, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/

package grpcmetrics

import (
grpcprom "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus"
"github.com/prometheus/client_golang/prometheus"
)

// CreateGRPCServerMetrics creates server gRPC metrics configuration that is to be registered and used by the caller
// in an openmetrics unary and/or stream interceptor
func CreateGRPCServerMetrics(
latencyEnabled bool, labels prometheus.Labels,
) *grpcprom.ServerMetrics {
serverOpts := []grpcprom.ServerMetricsOption{
grpcprom.WithServerCounterOptions(grpcprom.WithConstLabels(labels)),
}
if latencyEnabled {
histOpts := grpcHistogramOpts(labels)
serverOpts = append(
serverOpts, grpcprom.WithServerHandlingTimeHistogram(histOpts...),
)
}
return grpcprom.NewServerMetrics(serverOpts...)
}

// CreateGRPCClientMetrics creates client gRPC metrics configuration that is to be registered and used by the caller
// in an openmetrics unary and/or stream interceptor
func CreateGRPCClientMetrics(
latencyEnabled bool, labels prometheus.Labels,
) *grpcprom.ClientMetrics {
clientOpts := []grpcprom.ClientMetricsOption{
grpcprom.WithClientCounterOptions(grpcprom.WithConstLabels(labels)),
}
if latencyEnabled {
histOpts := grpcHistogramOpts(labels)
clientOpts = append(
clientOpts, grpcprom.WithClientHandlingTimeHistogram(histOpts...),
)
}
return grpcprom.NewClientMetrics(clientOpts...)
}

func grpcHistogramOpts(labels prometheus.Labels) []grpcprom.HistogramOption {
return []grpcprom.HistogramOption{
grpcprom.WithHistogramBuckets(prometheus.ExponentialBuckets(0.001, 2, 16)),
grpcprom.WithHistogramConstLabels(labels),
}
}
42 changes: 0 additions & 42 deletions lib/observability/metrics/prometheus.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ import (
"runtime"

"github.com/gravitational/trace"
grpcprom "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus"
"github.com/prometheus/client_golang/prometheus"

"github.com/gravitational/teleport"
Expand Down Expand Up @@ -70,44 +69,3 @@ func BuildCollector() prometheus.Collector {
func() float64 { return 1 },
)
}

// CreateGRPCServerMetrics creates server gRPC metrics configuration that is to be registered and used by the caller
// in an openmetrics unary and/or stream interceptor
func CreateGRPCServerMetrics(
latencyEnabled bool, labels prometheus.Labels,
) *grpcprom.ServerMetrics {
serverOpts := []grpcprom.ServerMetricsOption{
grpcprom.WithServerCounterOptions(grpcprom.WithConstLabels(labels)),
}
if latencyEnabled {
histOpts := grpcHistogramOpts(labels)
serverOpts = append(
serverOpts, grpcprom.WithServerHandlingTimeHistogram(histOpts...),
)
}
return grpcprom.NewServerMetrics(serverOpts...)
}

// CreateGRPCClientMetrics creates client gRPC metrics configuration that is to be registered and used by the caller
// in an openmetrics unary and/or stream interceptor
func CreateGRPCClientMetrics(
latencyEnabled bool, labels prometheus.Labels,
) *grpcprom.ClientMetrics {
clientOpts := []grpcprom.ClientMetricsOption{
grpcprom.WithClientCounterOptions(grpcprom.WithConstLabels(labels)),
}
if latencyEnabled {
histOpts := grpcHistogramOpts(labels)
clientOpts = append(
clientOpts, grpcprom.WithClientHandlingTimeHistogram(histOpts...),
)
}
return grpcprom.NewClientMetrics(clientOpts...)
}

func grpcHistogramOpts(labels prometheus.Labels) []grpcprom.HistogramOption {
return []grpcprom.HistogramOption{
grpcprom.WithHistogramBuckets(prometheus.ExponentialBuckets(0.001, 2, 16)),
grpcprom.WithHistogramConstLabels(labels),
}
}
122 changes: 122 additions & 0 deletions lib/observability/metrics/registry.go
Comment thread
hugoShaka marked this conversation as resolved.
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
/*
* Teleport
* Copyright (C) 2025 Gravitational, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/

package metrics

import (
"errors"

"github.com/prometheus/client_golang/prometheus"
)

// Registry is a [prometheus.Registerer] for a Teleport process that
// allows propagating additional information such as:
// - the metric namespace (`teleport`, `teleport_bot`, `teleport_plugins`)
// - an optional subsystem
//
// This should be passed anywhere that needs to register a metric.
type Registry struct {
prometheus.Registerer

namespace string
subsystem string
Comment on lines +33 to +37
Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can easily add a debug prometheus.Registerer later, so the scale team can expose tctl debug-only super high cardinality metrics.

}

// Namespace returns the namespace that should be used by metrics registered
// in this Registry. Common namespaces are "teleport", "tbot", and
// "teleport_plugins".
func (r *Registry) Namespace() string {
return r.namespace
}

// Subsystem is the subsystem base that should be used by metrics registered in
// this Registry. Subsystem parts can be added with WrapWithSubsystem.
func (r *Registry) Subsystem() string {
return r.subsystem
}

// Wrap wraps a Registry by adding a component to its subsystem.
// This should be used before passing a registry to a sub-component.
// Example usage:
//
// rootReg := prometheus.NewRegistry()
// process.AddGatherer(rootReg)
// reg, err := NewRegistry(rootReg, "teleport_plugins", "")
// go runFooService(ctx, log, reg.Wrap("foo"))
// go runBarService(ctx, log, reg.Wrap("bar"))
func (r *Registry) Wrap(subsystem string) *Registry {
newReg := &Registry{
Registerer: r.Registerer,
namespace: r.namespace,
subsystem: r.subsystem + "_" + subsystem,
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should this handle an empty r.subsystem?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🤦

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

}
return newReg
}

// NewRegistry creates a new Registry wrapping a prometheus registry.
// This should only be called when starting the service management routines such
// as: service.NewTeleport(), tbot.New(), or the hosted plugin manager.
// Services and sub-services should take the registry as a parameter, like they
// already do for the logger.
// Example usage:
//
// rootReg := prometheus.NewRegistry()
// process.AddGatherer(rootReg)
// reg, err := NewRegistry(rootReg, "teleport_plugins", "")
// go runFooService(ctx, log, reg.Wrap("foo"))
// go runBarService(ctx, log, reg.Wrap("bar"))
func NewRegistry(reg prometheus.Registerer, namespace, subsystem string) (*Registry, error) {
if reg == nil {
return nil, errors.New("nil prometheus.Registerer (this is a bug)")
}
if namespace == "" {
return nil, errors.New("namespace is required (this is a bug)")
}
return &Registry{
Registerer: reg,
namespace: namespace,
subsystem: subsystem,
}, nil
}

// NoopRegistry returns a Registry that doesn't register metrics.
// This can be used in tests, or to provide backward compatibility when a nil
// Registry is passed.
func NoopRegistry() *Registry {
return &Registry{
Registerer: noopRegistry{},
namespace: "noop",
subsystem: "",
}
}

type noopRegistry struct{}

// Register implements [prometheus.Registerer].
func (b noopRegistry) Register(collector prometheus.Collector) error {
return nil
}

// MustRegister implements [prometheus.Registerer].
func (b noopRegistry) MustRegister(collector ...prometheus.Collector) {
}

// Unregister implements [prometheus.Registerer].
func (b noopRegistry) Unregister(collector prometheus.Collector) bool {
return true
}
3 changes: 2 additions & 1 deletion lib/service/connect.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ import (
"github.com/gravitational/teleport/lib/defaults"
"github.com/gravitational/teleport/lib/join/joinclient"
"github.com/gravitational/teleport/lib/observability/metrics"
grpcmetrics "github.com/gravitational/teleport/lib/observability/metrics/grpc"
"github.com/gravitational/teleport/lib/openssh"
"github.com/gravitational/teleport/lib/reversetunnelclient"
servicebreaker "github.com/gravitational/teleport/lib/service/breaker"
Expand Down Expand Up @@ -1507,7 +1508,7 @@ func (process *TeleportProcess) newClientDirect(authServers []utils.NetAddr, tls

var dialOpts []grpc.DialOption
if role == types.RoleProxy {
grpcMetrics := metrics.CreateGRPCClientMetrics(process.Config.Metrics.GRPCClientLatency, prometheus.Labels{teleport.TagClient: "teleport-proxy"})
grpcMetrics := grpcmetrics.CreateGRPCClientMetrics(process.Config.Metrics.GRPCClientLatency, prometheus.Labels{teleport.TagClient: "teleport-proxy"})
if err := metrics.RegisterPrometheusCollectors(grpcMetrics); err != nil {
return nil, nil, trace.Wrap(err)
}
Expand Down
2 changes: 1 addition & 1 deletion lib/service/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -143,9 +143,9 @@ import (
kubeproxy "github.com/gravitational/teleport/lib/kube/proxy"
"github.com/gravitational/teleport/lib/labels"
"github.com/gravitational/teleport/lib/limiter"
"github.com/gravitational/teleport/lib/metrics"
"github.com/gravitational/teleport/lib/modules"
"github.com/gravitational/teleport/lib/multiplexer"
"github.com/gravitational/teleport/lib/observability/metrics"
"github.com/gravitational/teleport/lib/observability/tracing"
"github.com/gravitational/teleport/lib/openssh"
"github.com/gravitational/teleport/lib/pam"
Expand Down
2 changes: 1 addition & 1 deletion lib/service/service_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,10 +70,10 @@ import (
"github.com/gravitational/teleport/lib/events/athena"
"github.com/gravitational/teleport/lib/integrations/externalauditstorage"
"github.com/gravitational/teleport/lib/limiter"
"github.com/gravitational/teleport/lib/metrics"
"github.com/gravitational/teleport/lib/modules"
"github.com/gravitational/teleport/lib/modules/modulestest"
"github.com/gravitational/teleport/lib/multiplexer"
"github.com/gravitational/teleport/lib/observability/metrics"
"github.com/gravitational/teleport/lib/reversetunnelclient"
"github.com/gravitational/teleport/lib/service/servicecfg"
"github.com/gravitational/teleport/lib/services"
Expand Down
Loading
Loading