From c2441125484a2ac27770adef56cd245d8265de19 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 19 Jun 2025 09:39:25 +0200 Subject: [PATCH 1/5] bitswap/httpnet: limit metric cardinality Currently, we label the StatusCounter metric with the hostname of the http endpoint. If there was a high number of endpoints, this would cause high cardinality for this metric. This is a non-problem now as there are very few HTTP endpoints. This commit addresses it anyways by only labelling the metric hostnames when the Allowlist is enabled, therefore limiting labels to hostnames in the allowlist. --- bitswap/network/httpnet/httpnet.go | 1 - bitswap/network/httpnet/metrics.go | 7 +++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/bitswap/network/httpnet/httpnet.go b/bitswap/network/httpnet/httpnet.go index 907fba703..60fab739f 100644 --- a/bitswap/network/httpnet/httpnet.go +++ b/bitswap/network/httpnet/httpnet.go @@ -241,7 +241,6 @@ func New(host host.Host, opts ...Option) network.BitSwapNetwork { opt(htnet) } - // TODO: take allowlist into account! htnet.metrics = newMetrics(htnet.allowlist) reqTracker := newRequestTracker() diff --git a/bitswap/network/httpnet/metrics.go b/bitswap/network/httpnet/metrics.go index 077aa727d..6d217d9a2 100644 --- a/bitswap/network/httpnet/metrics.go +++ b/bitswap/network/httpnet/metrics.go @@ -93,10 +93,9 @@ func newMetrics(endpoints map[string]struct{}) *metrics { func (m *metrics) updateStatusCounter(method string, statusCode int, host string) { m.RequestsTotal.Inc() - // Track all for the moment. - // if _, ok := m.trackedEndpoints[host]; !ok { - // host = "other" - // } + if _, ok := m.trackedEndpoints[host]; !ok { + host = "other" + } var statusStr string From d82d0f1e3b19653be851efc8b2a486e3a68222f7 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Wed, 25 Jun 2025 16:20:33 +0200 Subject: [PATCH 2/5] bitswap/httpnet: new WithMetricsLabelsForEndpoints option. Allows assigning host labels to metrics. To limit unwanted cardinality growth it is disabled by default. --- bitswap/network/httpnet/httpnet.go | 20 +++++++++++++++++++- bitswap/network/httpnet/metrics.go | 21 ++++++++++++++++++--- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/bitswap/network/httpnet/httpnet.go b/bitswap/network/httpnet/httpnet.go index 60fab739f..a959143a8 100644 --- a/bitswap/network/httpnet/httpnet.go +++ b/bitswap/network/httpnet/httpnet.go @@ -170,6 +170,23 @@ func WithMaxDontHaveErrors(threshold int) Option { } } +// WithMetricsLabelsForHosts allows to label some metrics that support it +// with the endpoint name that they relate to. For example, this allows +// tracking respose statuses by endpoint. Using '*' means that all endpoints +// are tracked. By default, no endpoints are tracked. Endpoints that are not +// tracked are assigned the label "other". In a scenario where we are making +// requests to many different endpoints, logging all of them with '*' can +// cause the metric cardinality to grow accordingly, and end up affecting +// the performance of the metrics collector (i.e. Prometheus). +func WithMetricsLabelsForEndpoints(hosts []string) Option { + return func(net *Network) { + net.trackedEndpoints = make(map[string]struct{}) + for _, h := range hosts { + net.trackedEndpoints[h] = struct{}{} + } + } +} + type Network struct { // NOTE: Stats must be at the top of the heap allocation to ensure 64bit // alignment. @@ -200,6 +217,7 @@ type Network struct { httpWorkers int allowlist map[string]struct{} denylist map[string]struct{} + trackedEndpoints map[string]struct{} metrics *metrics httpRequests chan httpRequestInfo @@ -241,7 +259,7 @@ func New(host host.Host, opts ...Option) network.BitSwapNetwork { opt(htnet) } - htnet.metrics = newMetrics(htnet.allowlist) + htnet.metrics = newMetrics(htnet.trackedEndpoints) reqTracker := newRequestTracker() htnet.requestTracker = reqTracker diff --git a/bitswap/network/httpnet/metrics.go b/bitswap/network/httpnet/metrics.go index 6d217d9a2..592803c14 100644 --- a/bitswap/network/httpnet/metrics.go +++ b/bitswap/network/httpnet/metrics.go @@ -56,7 +56,9 @@ func status(ctx context.Context) imetrics.CounterVec { } type metrics struct { - trackedEndpoints map[string]struct{} + trackedEndpoints map[string]struct{} + trackAllEndpoints bool // avoid unnecessary map-lookups + trackSomeEndpoints bool // avoid unnecessary map-lookups RequestsInFlight imetrics.Gauge RequestsTotal imetrics.Counter @@ -74,7 +76,12 @@ type metrics struct { func newMetrics(endpoints map[string]struct{}) *metrics { ctx := imetrics.CtxScope(context.Background(), "exchange_httpnet") + _, trackAll := endpoints["*"] + trackSome := len(endpoints) > 0 && !trackAll + return &metrics{ + trackAllEndpoints: trackAll, + trackSomeEndpoints: trackSome, trackedEndpoints: endpoints, RequestsInFlight: requestsInFlight(ctx), RequestsTotal: requestsTotal(ctx), @@ -93,8 +100,16 @@ func newMetrics(endpoints map[string]struct{}) *metrics { func (m *metrics) updateStatusCounter(method string, statusCode int, host string) { m.RequestsTotal.Inc() - if _, ok := m.trackedEndpoints[host]; !ok { - host = "other" + // Set host == other if we are: + // - not tracking all hosts + // - not tracking any hosts + // - the host is not among those we are tracking + if !m.trackAllEndpoints { + if !m.trackSomeEndpoints { + host = "other" + } else if _, ok := m.trackedEndpoints[host]; !ok { + host = "other" + } } var statusStr string From 24399491cc2cb0565deebcf3224b7f7fc4aa1dba Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Wed, 25 Jun 2025 16:25:40 +0200 Subject: [PATCH 3/5] Explain new metric labelling option in the changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e0a9f740..1222b1175 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ The following emojis are used to highlight certain changes: ### Added +- `bitswap/network/httpnet`: New `WithMetricsLabelsForEndpoints` allows defining which hosts/endpoints can be used for labelling metrics that support such label. '*' enables this for all endpoints receiving HTTP requests, but may cause metric cardinality growth when too many endpoints exist. These labels allow tracking, for example, number or requests per respose status AND endpoint used. + ### Changed - upgrade to `go-libp2p` [v0.42.0](https://github.com/libp2p/go-libp2p/releases/tag/v0.42.0) From 5ac828a13aa80d0982af4e0e75c9e3b07e88daf5 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 25 Jun 2025 21:31:54 +0200 Subject: [PATCH 4/5] docs: clarify the default behavior --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1222b1175..b7087285d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ The following emojis are used to highlight certain changes: ### Added -- `bitswap/network/httpnet`: New `WithMetricsLabelsForEndpoints` allows defining which hosts/endpoints can be used for labelling metrics that support such label. '*' enables this for all endpoints receiving HTTP requests, but may cause metric cardinality growth when too many endpoints exist. These labels allow tracking, for example, number or requests per respose status AND endpoint used. +- `bitswap/network/httpnet`: New `WithMetricsLabelsForEndpoints` allows defining which hosts/endpoints can be used for labelling metrics that support such label. '*' enables this for all endpoints receiving HTTP requests, but may cause metric cardinality growth when too many endpoints exist. These labels allow tracking, for example, number or requests per response status AND endpoint used. If no endpoints are specified via `WithMetricsLabelsForEndpoints` or `WithAllowlist`, all requests are metered under the label 'other' to keep cardinality in check. ### Changed From b051253e087d30a00eb8b72e03d9ac1bbc5c857b Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Fri, 27 Jun 2025 10:31:39 +0200 Subject: [PATCH 5/5] changelog: minor correction about http metric cardinality --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7087285d..14362ad04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ The following emojis are used to highlight certain changes: ### Added -- `bitswap/network/httpnet`: New `WithMetricsLabelsForEndpoints` allows defining which hosts/endpoints can be used for labelling metrics that support such label. '*' enables this for all endpoints receiving HTTP requests, but may cause metric cardinality growth when too many endpoints exist. These labels allow tracking, for example, number or requests per response status AND endpoint used. If no endpoints are specified via `WithMetricsLabelsForEndpoints` or `WithAllowlist`, all requests are metered under the label 'other' to keep cardinality in check. +- `bitswap/network/httpnet`: New `WithMetricsLabelsForEndpoints` allows defining which hosts/endpoints can be used for labelling metrics that support such label. '*' enables this for all endpoints receiving HTTP requests, but may cause metric cardinality growth when too many endpoints exist. These labels allow tracking, for example, number or requests per response status AND endpoint used. Non-labelled request hosts are labelled with same value: `other`. ### Changed