From 1e089154e31159231b78cf087a7c3389aab96cd2 Mon Sep 17 00:00:00 2001 From: hugoShaka Date: Thu, 16 Jan 2025 16:34:27 -0500 Subject: [PATCH 1/7] RFD 197 - Prometheus metrics guidelines --- rfd/0197-prometheus-metrics.md | 226 +++++++++++++++++++++++++++++++++ 1 file changed, 226 insertions(+) create mode 100644 rfd/0197-prometheus-metrics.md diff --git a/rfd/0197-prometheus-metrics.md b/rfd/0197-prometheus-metrics.md new file mode 100644 index 0000000000000..8d2d5be17452b --- /dev/null +++ b/rfd/0197-prometheus-metrics.md @@ -0,0 +1,226 @@ +--- +authors: Hugo Hervieux (hugo.hervieux@goteleport.com) +state: draft +--- + +# RFD 197 - Prometheus metrics guidelines + +## Required Approvers + +* Engineering: @codingllama && @rosstimothy && @zmb3 + +## What + +This RFD covers the recommended way of adding metrics to Teleport and other services such as +tbot, the Teleport Kube Agent Updater, the Teleport Kubernetes Operator, or plugins +(slack, teams, pagerduty, event-handler, ...). + +## Why + +Every component currently relies on the global metrics registry and package-scoped metrics. +This poses several challenges: +- We pick up and serve arbitrary metrics declared by any of our dependencies. This can cause registration conflicts + (errors, flakiness, panics), high cardinality, confusing metric names, and data loss. +- Several Teleport components cannot run in the same go process without causing metric conflicts. This is the case in + integration tests (running both tbot and Teleport), or even in user-facing binaries (tctl uses `embeddedtbot`). +- The same Teleport component cannot be started multiple times without causing metric conflicts, or inaccurate metrics. + For example, integration tests starting several Teleport instances will merge their metrics together. + +## Details + +### Metrics Registry + +#### Do + +- Take the local in-process registry as an argument in your service constructor/main routine, like you would receive a + logger, and register your metrics against it. +- Pass the registry as a `promtetheus.Registerer` + +```golang +func NewService(log *slog.Logger, reg prometheus.Registerer) (Service, error) { + myMetric := prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: teleport.MetricNamespace, + Subsystem: metricsSubsystem, + Name: "my_metric", + Help: "Measures the number of foos doing bars.", + }), + if err := reg.Register(myMetric); err != nil { + return nil, trace.Wrap(err, "registering metric") + } + // ... +} +``` + +#### Don't + +- Register against the global prometheus registry + ```golang + func NewService(log *slog.Logger) (Service, error) { + myMetric := prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: teleport.MetricNamespace, + Subsystem: metricsSubsystem, + Name: "my_metric", + Help: "Measures the number of foos doing bars.", + }), + if err := prometheus.Register(myMetric); err != nil { + return nil, trace.Wrap(err, "registering metric") + } + // ... + } + ``` + +- Pass the registry as a `*prometheus.Registry` + + ```golang + func NewService(log *slog.Logger, reg *prometheus.Registry) (Service, error) { + myMetric := prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: teleport.MetricNamespace, + Subsystem: metricsSubsystem, + Name: "my_metric", + Help: "Measures the number of foos doing bars.", + }), + if err := reg.Register(myMetric); err != nil { + return nil, trace.Wrap(err, "registering metric") + } + // ... + } + ``` + +### Storing metrics + +#### Do + +- Store metrics in a private struct in your package + +```golang +type metrics struct { + currentFoo prometheus.Gauge + barCounter *prometheus.CounterVec +} + +func newMetrics(reg prometheus.Registerer) (*metrics, error) { + m := metrics{ + currentFoo: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: teleport.MetricNamespace, + Subsystem: metricsSubsystem, + Name: "foo_current", + Help: "Measures the number of foos.", + }), + barCounter: // ... + } + + errs := trace.NewAggregate( + reg.Register(m.currentFoo), + reg.Register(m.barCounter), + ) + + if errs != nil { + return trace.Wrap(err, "registering metrics") + } + + return &m +} + +``` + +#### Don't + +- Store metrics in a package-scoped variable + +```golang +var ( + currentFoo = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: teleport.MetricNamespace, + Subsystem: metricsSubsystem, + Name: "foo_current", + Help: "Measures the number of foos.", + }) + // ... +) +``` + +### Naming metrics + +#### Do + +- Use `teleport.MetricsNamespace` as the namespace. +- Use a subsystem name unique to your component. +- Follow [the prometheus metrics naming guidelines](https://prometheus.io/docs/practices/naming/), + especially always specify the unit and use a suffix to clarify the metric type (`_total`, `_info`). + +```golang +type metrics struct { + currentFoo prometheus.Gauge + barCounter *prometheus.CounterVec +} + +const metricsSubsystem = "my_service" + +func newMetrics(reg prometheus.Registerer) (*metrics, error) { + m := metrics{ + currentFoo: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: teleport.MetricNamespace, + Subsystem: metricsSubsystem, + Name: "foo_timestamp_seconds", + Help: "Represents the foo time, in seconds.", + }), + barCounter: prometheus.NewCounter(prometheus.GaugeOpts{ + Namespace: teleport.MetricNamespace, + Subsystem: metricsSubsystem, + Name: "bar_total", + Help: "Number of times bar happened.", + }), + } + // ... +} +``` + +#### Don't + +- Manually namespace metrics +- Create non-namespaced metrics + +```golang +type metrics struct { + currentFoo prometheus.Gauge + barCounter *prometheus.CounterVec +} + +func newMetrics(reg prometheus.Registerer) (*metrics, error) { + m := metrics{ + currentFoo: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "teleport_foo_timestamp_seconds", + Help: "Represents the foo time, in seconds.", + }), + barCounter: prometheus.NewCounter(prometheus.GaugeOpts{ + Name: "bar_total", + Help: "Number of times bar happened.", + }), + } + // ... +} +``` + +### Metric registration + +#### Do + +- Use `reg.Register()` to register the metric. +- Aggregate errors and fail early if you can't register metrics. + +#### Don't + +- Use the package helpers `prometheus.Register()` or `prometheus.MustRegister`. +- Use `reg.MustRegister` as it panics in case of conflict and doesn't clearly indicate which metric descriptor conflicted. + +### Labeling + +#### Do + +- Ensure no secret information is present in the labels. +- Avoid high cardinality labels + +#### Don't + +- Put user input directly in the labels +- Create large (1k+) metric combinations From b9669aff57f1ff9a521a56115b49761905da00cb Mon Sep 17 00:00:00 2001 From: hugoShaka Date: Thu, 16 Jan 2025 17:34:19 -0500 Subject: [PATCH 2/7] Fix indent and trailing spaces --- rfd/0197-prometheus-metrics.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/rfd/0197-prometheus-metrics.md b/rfd/0197-prometheus-metrics.md index 8d2d5be17452b..ff637258e1625 100644 --- a/rfd/0197-prometheus-metrics.md +++ b/rfd/0197-prometheus-metrics.md @@ -40,12 +40,12 @@ This poses several challenges: func NewService(log *slog.Logger, reg prometheus.Registerer) (Service, error) { myMetric := prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: teleport.MetricNamespace, - Subsystem: metricsSubsystem, + Subsystem: metricsSubsystem, Name: "my_metric", Help: "Measures the number of foos doing bars.", }), if err := reg.Register(myMetric); err != nil { - return nil, trace.Wrap(err, "registering metric") + return nil, trace.Wrap(err, "registering metric") } // ... } @@ -58,12 +58,12 @@ func NewService(log *slog.Logger, reg prometheus.Registerer) (Service, error) { func NewService(log *slog.Logger) (Service, error) { myMetric := prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: teleport.MetricNamespace, - Subsystem: metricsSubsystem, + Subsystem: metricsSubsystem, Name: "my_metric", Help: "Measures the number of foos doing bars.", }), if err := prometheus.Register(myMetric); err != nil { - return nil, trace.Wrap(err, "registering metric") + return nil, trace.Wrap(err, "registering metric") } // ... } @@ -75,12 +75,12 @@ func NewService(log *slog.Logger, reg prometheus.Registerer) (Service, error) { func NewService(log *slog.Logger, reg *prometheus.Registry) (Service, error) { myMetric := prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: teleport.MetricNamespace, - Subsystem: metricsSubsystem, + Subsystem: metricsSubsystem, Name: "my_metric", Help: "Measures the number of foos doing bars.", }), if err := reg.Register(myMetric); err != nil { - return nil, trace.Wrap(err, "registering metric") + return nil, trace.Wrap(err, "registering metric") } // ... } @@ -102,12 +102,12 @@ func newMetrics(reg prometheus.Registerer) (*metrics, error) { m := metrics{ currentFoo: prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: teleport.MetricNamespace, - Subsystem: metricsSubsystem, + Subsystem: metricsSubsystem, Name: "foo_current", Help: "Measures the number of foos.", }), barCounter: // ... - } + } errs := trace.NewAggregate( reg.Register(m.currentFoo), @@ -131,7 +131,7 @@ func newMetrics(reg prometheus.Registerer) (*metrics, error) { var ( currentFoo = prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: teleport.MetricNamespace, - Subsystem: metricsSubsystem, + Subsystem: metricsSubsystem, Name: "foo_current", Help: "Measures the number of foos.", }) @@ -160,17 +160,17 @@ func newMetrics(reg prometheus.Registerer) (*metrics, error) { m := metrics{ currentFoo: prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: teleport.MetricNamespace, - Subsystem: metricsSubsystem, + Subsystem: metricsSubsystem, Name: "foo_timestamp_seconds", Help: "Represents the foo time, in seconds.", }), barCounter: prometheus.NewCounter(prometheus.GaugeOpts{ Namespace: teleport.MetricNamespace, - Subsystem: metricsSubsystem, + Subsystem: metricsSubsystem, Name: "bar_total", Help: "Number of times bar happened.", }), - } + } // ... } ``` @@ -196,7 +196,7 @@ func newMetrics(reg prometheus.Registerer) (*metrics, error) { Name: "bar_total", Help: "Number of times bar happened.", }), - } + } // ... } ``` From 09447cfef2af7c43b489e83cfec4de3091059be3 Mon Sep 17 00:00:00 2001 From: hugoShaka Date: Thu, 16 Jan 2025 17:45:49 -0500 Subject: [PATCH 3/7] Add section about lint --- rfd/0197-prometheus-metrics.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/rfd/0197-prometheus-metrics.md b/rfd/0197-prometheus-metrics.md index ff637258e1625..2db7175c96b6a 100644 --- a/rfd/0197-prometheus-metrics.md +++ b/rfd/0197-prometheus-metrics.md @@ -26,7 +26,7 @@ This poses several challenges: - The same Teleport component cannot be started multiple times without causing metric conflicts, or inaccurate metrics. For example, integration tests starting several Teleport instances will merge their metrics together. -## Details +## Guidelines ### Metrics Registry @@ -224,3 +224,14 @@ func newMetrics(reg prometheus.Registerer) (*metrics, error) { - Put user input directly in the labels - Create large (1k+) metric combinations + +## Enforcing the guidelines + +Some guidelines can be enforced by setting up linters: +- [promlinter](https://golangci-lint.run/usage/linters/#promlinter) to ensure that metric naming and labeling follows + the Prometheus guidelines. +- [forbidigo](https://golangci-lint.run/usage/linters/#forbidigo) to reject usages of `prometheus.DefaultRegisterer`, + `prometheus.(Must)Register`. `reg.MustRegister` conflicts with `backend.MustRegister`, we might not be able to detect + it (`fobidigo.analyze-types` might not be sufficient) + +Existing non-compliant metrics and edge case usages will be allowed via `//nolint` comments. \ No newline at end of file From 268976ab3f49392b28c567f973d803f072caacaa Mon Sep 17 00:00:00 2001 From: hugoShaka Date: Tue, 11 Nov 2025 16:52:32 -0500 Subject: [PATCH 4/7] refresh to use metrics.Registry --- rfd/0197-prometheus-metrics.md | 133 ++++++++++++++++++--------------- 1 file changed, 73 insertions(+), 60 deletions(-) diff --git a/rfd/0197-prometheus-metrics.md b/rfd/0197-prometheus-metrics.md index 2db7175c96b6a..c05a3cd865fb1 100644 --- a/rfd/0197-prometheus-metrics.md +++ b/rfd/0197-prometheus-metrics.md @@ -34,13 +34,13 @@ This poses several challenges: - Take the local in-process registry as an argument in your service constructor/main routine, like you would receive a logger, and register your metrics against it. -- Pass the registry as a `promtetheus.Registerer` +- Pass the registry as a `*metrics.Registry` ```golang -func NewService(log *slog.Logger, reg prometheus.Registerer) (Service, error) { +func NewService(log *slog.Logger, reg *metrics.Registry) (Service, error) { myMetric := prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: teleport.MetricNamespace, - Subsystem: metricsSubsystem, + Namespace: reg.Namespace(), + Subsystem: reg.Subsystem(), Name: "my_metric", Help: "Measures the number of foos doing bars.", }), @@ -53,7 +53,9 @@ func NewService(log *slog.Logger, reg prometheus.Registerer) (Service, error) { #### Don't -- Register against the global prometheus registry +-
+ Register against the global prometheus registry + ```golang func NewService(log *slog.Logger) (Service, error) { myMetric := prometheus.NewGauge(prometheus.GaugeOpts{ @@ -68,8 +70,9 @@ func NewService(log *slog.Logger, reg prometheus.Registerer) (Service, error) { // ... } ``` - -- Pass the registry as a `*prometheus.Registry` +
+-
+ Pass the registry as a `*prometheus.Registry` or `prometheus.Registerer` ```golang func NewService(log *slog.Logger, reg *prometheus.Registry) (Service, error) { @@ -85,6 +88,7 @@ func NewService(log *slog.Logger, reg prometheus.Registerer) (Service, error) { // ... } ``` +
### Storing metrics @@ -93,12 +97,16 @@ func NewService(log *slog.Logger, reg prometheus.Registerer) (Service, error) { - Store metrics in a private struct in your package ```golang -type metrics struct { +type fooMetrics struct { currentFoo prometheus.Gauge barCounter *prometheus.CounterVec } -func newMetrics(reg prometheus.Registerer) (*metrics, error) { +type fooService struct { + metrics *fooMetrics +} + +func newMetrics(reg *metrics.Registry) (metrics, error) { m := metrics{ currentFoo: prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: teleport.MetricNamespace, @@ -109,15 +117,6 @@ func newMetrics(reg prometheus.Registerer) (*metrics, error) { barCounter: // ... } - errs := trace.NewAggregate( - reg.Register(m.currentFoo), - reg.Register(m.barCounter), - ) - - if errs != nil { - return trace.Wrap(err, "registering metrics") - } - return &m } @@ -125,26 +124,28 @@ func newMetrics(reg prometheus.Registerer) (*metrics, error) { #### Don't -- Store metrics in a package-scoped variable +-
+ Store metrics in a package-scoped variable -```golang -var ( - currentFoo = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: teleport.MetricNamespace, - Subsystem: metricsSubsystem, - Name: "foo_current", - Help: "Measures the number of foos.", - }) - // ... -) -``` + ```golang + var ( + currentFoo = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: teleport.MetricNamespace, + Subsystem: metricsSubsystem, + Name: "foo_current", + Help: "Measures the number of foos.", + }) + // ... + ) + ``` +
### Naming metrics #### Do -- Use `teleport.MetricsNamespace` as the namespace. -- Use a subsystem name unique to your component. +- Honour the namespace and subsystem from the `metrics.Registry` +- Wrap the `metrics.Registry` to add component-level information to the metrics subsystem. - Follow [the prometheus metrics naming guidelines](https://prometheus.io/docs/practices/naming/), especially always specify the unit and use a suffix to clarify the metric type (`_total`, `_info`). @@ -154,9 +155,7 @@ type metrics struct { barCounter *prometheus.CounterVec } -const metricsSubsystem = "my_service" - -func newMetrics(reg prometheus.Registerer) (*metrics, error) { +func newMetrics(reg *metrics.Registry) (*metrics, error) { m := metrics{ currentFoo: prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: teleport.MetricNamespace, @@ -173,40 +172,52 @@ func newMetrics(reg prometheus.Registerer) (*metrics, error) { } // ... } -``` - -#### Don't -- Manually namespace metrics -- Create non-namespaced metrics - -```golang -type metrics struct { - currentFoo prometheus.Gauge - barCounter *prometheus.CounterVec +func newService(reg *metrics.Registry) { + go runComponentA(reg.Wrap("component_a")) } -func newMetrics(reg prometheus.Registerer) (*metrics, error) { - m := metrics{ - currentFoo: prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "teleport_foo_timestamp_seconds", - Help: "Represents the foo time, in seconds.", - }), - barCounter: prometheus.NewCounter(prometheus.GaugeOpts{ - Name: "bar_total", - Help: "Number of times bar happened.", - }), - } - // ... +func newComponentA(reg *metrics.Registry) { + m := newMetrics(reg) + err := m.register(reg) } ``` +#### Don't + +- Manually namespace metrics +
+ Manually namespace metrics or create non-namespaced metrics + + ```golang + type metrics struct { + currentFoo prometheus.Gauge + barCounter *prometheus.CounterVec + } + + func newMetrics(reg prometheus.Registerer) (*metrics, error) { + m := metrics{ + currentFoo: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "teleport" + Name: "foo_timestamp_seconds", + Help: "Represents the foo time, in seconds.", + }), + barCounter: prometheus.NewCounter(prometheus.GaugeOpts{ + Name: "bar_total", + Help: "Number of times bar happened.", + }), + } + // ... + } + ``` +
+ ### Metric registration #### Do - Use `reg.Register()` to register the metric. -- Aggregate errors and fail early if you can't register metrics. +- Aggregate errors and fail early if you can't register metrics? #### Don't @@ -224,14 +235,16 @@ func newMetrics(reg prometheus.Registerer) (*metrics, error) { - Put user input directly in the labels - Create large (1k+) metric combinations +- Create metrics with an ever growing number of labels ## Enforcing the guidelines Some guidelines can be enforced by setting up linters: - [promlinter](https://golangci-lint.run/usage/linters/#promlinter) to ensure that metric naming and labeling follows - the Prometheus guidelines. + the Prometheus guidelines. Note: we might not be able to use the strict mode as namespacesa and subsystems are + passed from the caller. - [forbidigo](https://golangci-lint.run/usage/linters/#forbidigo) to reject usages of `prometheus.DefaultRegisterer`, `prometheus.(Must)Register`. `reg.MustRegister` conflicts with `backend.MustRegister`, we might not be able to detect it (`fobidigo.analyze-types` might not be sufficient) -Existing non-compliant metrics and edge case usages will be allowed via `//nolint` comments. \ No newline at end of file +Existing non-compliant metrics and edge case usages will be allowed via `//nolint` comments. From f8c758a3dcdc2675feddb6925e36fe917b4385f0 Mon Sep 17 00:00:00 2001 From: hugoShaka Date: Tue, 11 Nov 2025 17:25:15 -0500 Subject: [PATCH 5/7] lint --- rfd/0197-prometheus-metrics.md | 4 ++-- rfd/cspell.json | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/rfd/0197-prometheus-metrics.md b/rfd/0197-prometheus-metrics.md index c05a3cd865fb1..72f85521acf57 100644 --- a/rfd/0197-prometheus-metrics.md +++ b/rfd/0197-prometheus-metrics.md @@ -241,10 +241,10 @@ func newComponentA(reg *metrics.Registry) { Some guidelines can be enforced by setting up linters: - [promlinter](https://golangci-lint.run/usage/linters/#promlinter) to ensure that metric naming and labeling follows - the Prometheus guidelines. Note: we might not be able to use the strict mode as namespacesa and subsystems are + the Prometheus guidelines. Note: we might not be able to use the strict mode as namespaces and subsystems are passed from the caller. - [forbidigo](https://golangci-lint.run/usage/linters/#forbidigo) to reject usages of `prometheus.DefaultRegisterer`, `prometheus.(Must)Register`. `reg.MustRegister` conflicts with `backend.MustRegister`, we might not be able to detect - it (`fobidigo.analyze-types` might not be sufficient) + it (`forbidigo.analyze-types` might not be sufficient) Existing non-compliant metrics and edge case usages will be allowed via `//nolint` comments. diff --git a/rfd/cspell.json b/rfd/cspell.json index 06f1ab60ab75b..6b4370b8be53c 100644 --- a/rfd/cspell.json +++ b/rfd/cspell.json @@ -223,6 +223,7 @@ "eksctl", "Elastisearch", "elif", + "embeddedtbot", "endpointslices", "enduml", "Enes", @@ -260,6 +261,7 @@ "Foos", "footgun", "foov", + "forbidigo", "Fprintln", "Frazar", "fspmarshall", @@ -490,6 +492,7 @@ "nklaassen", "noahstride", "nodename", + "nolint", "nologin", "NOPASSWD", "noup", @@ -596,6 +599,7 @@ "proce", "producting", "programmerq", + "promlinter", "Proto", "protobuf", "protobufs", From 7fff9728e1db03f1b7f0e2d2723885eefad016ea Mon Sep 17 00:00:00 2001 From: hugoShaka Date: Tue, 11 Nov 2025 17:35:21 -0500 Subject: [PATCH 6/7] add conflicts sections --- rfd/0197-prometheus-metrics.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/rfd/0197-prometheus-metrics.md b/rfd/0197-prometheus-metrics.md index 72f85521acf57..9377f25cab8be 100644 --- a/rfd/0197-prometheus-metrics.md +++ b/rfd/0197-prometheus-metrics.md @@ -26,6 +26,21 @@ This poses several challenges: - The same Teleport component cannot be started multiple times without causing metric conflicts, or inaccurate metrics. For example, integration tests starting several Teleport instances will merge their metrics together. +## Conflicts + +Metrics conflicts can happen in several cases: + +1. The same metric is registered with different labels by several teleport processes. This is what happens with the + grpc metrics currently as teleport and tbot register global metrics with the same name but different labels. This can + be avoided in tests by using per-process registries. +2. The same metric is registered several times by different components. This can be avoided by namespacing and + adding metrics subsystem. +3. The same metric is registered both in the local and global registries. In this case, the process registry takes + precedence, the gathering succeeds but logs errors. +4. The same component is being started and stopped several times and re-register its metrics. This is the case for + hosted plugins. We work around by creating a dedicared registry, and registering/unregistering it as a collector. + See https://github.com/prometheus/client_golang/pull/1766 for an example. + ## Guidelines ### Metrics Registry From f614d341a8350241232a7d330b044e8095354b25 Mon Sep 17 00:00:00 2001 From: Hugo Shaka Date: Mon, 17 Nov 2025 17:37:50 -0500 Subject: [PATCH 7/7] Update rfd/0197-prometheus-metrics.md Co-authored-by: Alan Parra <12500300+codingllama@users.noreply.github.com> --- rfd/0197-prometheus-metrics.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rfd/0197-prometheus-metrics.md b/rfd/0197-prometheus-metrics.md index 9377f25cab8be..2ce07de8bdba4 100644 --- a/rfd/0197-prometheus-metrics.md +++ b/rfd/0197-prometheus-metrics.md @@ -38,7 +38,7 @@ Metrics conflicts can happen in several cases: 3. The same metric is registered both in the local and global registries. In this case, the process registry takes precedence, the gathering succeeds but logs errors. 4. The same component is being started and stopped several times and re-register its metrics. This is the case for - hosted plugins. We work around by creating a dedicared registry, and registering/unregistering it as a collector. + hosted plugins. We work around by creating a dedicated registry, and registering/unregistering it as a collector. See https://github.com/prometheus/client_golang/pull/1766 for an example. ## Guidelines