diff --git a/dashboards/autonatv2/autonatv2.json b/dashboards/autonatv2/autonatv2.json index 11186c09d4..82697edc26 100644 --- a/dashboards/autonatv2/autonatv2.json +++ b/dashboards/autonatv2/autonatv2.json @@ -24,9 +24,219 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 4, + "id": 5, "links": [], "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "sum (increase(libp2p_autonatv2_client_requests_completed_total{instance=~\"$instance\",dial_refused=\"false\"}[5m])) by (instance, ip_or_dns_version, transport, reachability)", + "legendFormat": "{{instance}} {{ip_or_dns_version}} {{transport}} {{reachability}}", + "range": true, + "refId": "A" + } + ], + "title": "Requests by Reachability", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "increase(libp2p_autonatv2_client_requests_total{instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "{{instance}} {{outcome}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "editorMode": "code", + "expr": "sum (increase(libp2p_autonatv2_client_requests_completed_total{instance=~\"$instance\", dial_refused=\"true\"}[$__rate_interval])) by (instance)", + "hide": false, + "instant": false, + "legendFormat": "{{instance}} refused", + "range": true, + "refId": "B" + } + ], + "title": "All Requests", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 4, + "panels": [], + "title": "Server", + "type": "row" + }, { "datasource": { "type": "prometheus", @@ -45,6 +255,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -75,8 +286,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -122,7 +332,7 @@ "h": 8, "w": 12, "x": 0, - "y": 0 + "y": 9 }, "id": 1, "options": { @@ -133,11 +343,13 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "maxHeight": 600, "mode": "single", "sort": "none" } }, + "pluginVersion": "11.6.1", "targets": [ { "datasource": { @@ -173,6 +385,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -203,8 +416,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -265,7 +477,7 @@ "h": 8, "w": 12, "x": 12, - "y": 0 + "y": 9 }, "id": 2, "options": { @@ -276,11 +488,13 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "maxHeight": 600, "mode": "single", "sort": "none" } }, + "pluginVersion": "11.6.1", "targets": [ { "datasource": { @@ -316,6 +530,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -346,8 +561,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -408,7 +622,7 @@ "h": 8, "w": 12, "x": 5, - "y": 8 + "y": 17 }, "id": 3, "options": { @@ -419,11 +633,13 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "maxHeight": 600, "mode": "single", "sort": "none" } }, + "pluginVersion": "11.6.1", "targets": [ { "datasource": { @@ -442,24 +658,20 @@ "type": "timeseries" } ], + "preload": false, "refresh": "", - "schemaVersion": 39, + "schemaVersion": 41, "tags": [], "templating": { "list": [ { - "allValue": "", "current": { - "selected": true, - "text": [ - "All" - ], + "text": "All", "value": [ "$__all" ] }, "definition": "label_values(up,instance)", - "hide": 0, "includeAll": true, "label": "instance", "multi": true, @@ -472,22 +684,19 @@ }, "refresh": 1, "regex": "", - "skipUrlSync": false, - "sort": 0, "type": "query" }, { - "hide": 0, + "current": { + "text": "prometheus", + "value": "${data_source}" + }, "includeAll": false, - "label": "", - "multi": false, "name": "data_source", "options": [], "query": "prometheus", - "queryValue": "", "refresh": 1, "regex": "", - "skipUrlSync": false, "type": "datasource" } ] @@ -496,11 +705,9 @@ "from": "now-1h", "to": "now" }, - "timeRangeUpdatedDuringEditOrView": false, "timepicker": {}, "timezone": "browser", "title": "go-libp2p autoNATv2", "uid": "cdpusyp3xtfcwa", - "version": 1, - "weekStart": "" + "version": 9 } \ No newline at end of file diff --git a/dashboards/host-addrs/host-addrs.json b/dashboards/host-addrs/host-addrs.json new file mode 100644 index 0000000000..6a79282a01 --- /dev/null +++ b/dashboards/host-addrs/host-addrs.json @@ -0,0 +1,279 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 16, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "bdpgk86mw6jgga" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "bdpgk86mw6jgga" + }, + "editorMode": "code", + "expr": "libp2p_host_addrs_reachable{instance=~\"$instance\"}", + "legendFormat": "{{instance}} {{ipv}}, {{transport}}", + "range": true, + "refId": "A" + } + ], + "title": "Reachable Addrs", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "bdpgk86mw6jgga" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "bdpgk86mw6jgga" + }, + "editorMode": "code", + "expr": "libp2p_host_addrs_unreachable{instance=~\"$instance\"}", + "legendFormat": "{{instance}} {{ipv}}, {{transport}}", + "range": true, + "refId": "A" + } + ], + "title": "Unreachable Addrs", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "bdpgk86mw6jgga" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "bdpgk86mw6jgga" + }, + "editorMode": "code", + "expr": "libp2p_host_addrs_unknown{instance=~\"$instance\"}", + "legendFormat": "{{instance}} {{ipv}}, {{transport}}", + "range": true, + "refId": "A" + } + ], + "title": "Unknown Reachability Addrs", + "type": "stat" + } + ], + "preload": false, + "schemaVersion": 41, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "definition": "label_values(up,instance)", + "includeAll": true, + "label": "instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(up,instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "prometheus", + "value": "bdpgk86mw6jgga" + }, + "includeAll": false, + "name": "data_source", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "go-libp2p Host Addresses", + "uid": "beon8z59rh7nkf", + "version": 5 +} \ No newline at end of file diff --git a/p2p/host/basic/addrs_manager.go b/p2p/host/basic/addrs_manager.go index b1ee831c20..46d2c49662 100644 --- a/p2p/host/basic/addrs_manager.go +++ b/p2p/host/basic/addrs_manager.go @@ -20,6 +20,7 @@ import ( "github.com/libp2p/go-netroute" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" + "github.com/prometheus/client_golang/prometheus" ) const maxObservedAddrsPerListenAddr = 5 @@ -75,6 +76,8 @@ func newAddrsManager( observedAddrsManager observedAddrsManager, addrsUpdatedChan chan struct{}, client autonatv2Client, + enableMetrics bool, + registerer prometheus.Registerer, ) (*addrsManager, error) { ctx, cancel := context.WithCancel(context.Background()) as := &addrsManager{ @@ -95,7 +98,11 @@ func newAddrsManager( as.hostReachability.Store(&unknownReachability) if client != nil { - as.addrsReachabilityTracker = newAddrsReachabilityTracker(client, as.triggerReachabilityUpdate, nil) + var metricsTracker MetricsTracker + if enableMetrics { + metricsTracker = newMetricsTracker(withRegisterer(registerer)) + } + as.addrsReachabilityTracker = newAddrsReachabilityTracker(client, as.triggerReachabilityUpdate, nil, metricsTracker) } return as, nil } diff --git a/p2p/host/basic/addrs_manager_test.go b/p2p/host/basic/addrs_manager_test.go index f2532d9920..299a9d814f 100644 --- a/p2p/host/basic/addrs_manager_test.go +++ b/p2p/host/basic/addrs_manager_test.go @@ -14,6 +14,7 @@ import ( "github.com/libp2p/go-libp2p/p2p/protocol/autonatv2" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" + "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -194,7 +195,7 @@ func newAddrsManagerTestCase(t *testing.T, args addrsManagerArgs) addrsManagerTe } addrsUpdatedChan := make(chan struct{}, 1) am, err := newAddrsManager( - eb, args.NATManager, args.AddrsFactory, args.ListenAddrs, nil, args.ObservedAddrsManager, addrsUpdatedChan, args.AutoNATClient, + eb, args.NATManager, args.AddrsFactory, args.ListenAddrs, nil, args.ObservedAddrsManager, addrsUpdatedChan, args.AutoNATClient, true, prometheus.DefaultRegisterer, ) require.NoError(t, err) diff --git a/p2p/host/basic/addrs_metrics.go b/p2p/host/basic/addrs_metrics.go new file mode 100644 index 0000000000..6c04a6b362 --- /dev/null +++ b/p2p/host/basic/addrs_metrics.go @@ -0,0 +1,154 @@ +package basichost + +import ( + "maps" + + "github.com/libp2p/go-libp2p/p2p/metricshelper" + ma "github.com/multiformats/go-multiaddr" + + "github.com/prometheus/client_golang/prometheus" +) + +const metricNamespace = "libp2p_host_addrs" + +var ( + reachableAddrs = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricNamespace, + Name: "reachable", + Help: "Number of reachable addresses by transport", + }, + []string{"ipv", "transport"}, + ) + unreachableAddrs = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricNamespace, + Name: "unreachable", + Help: "Number of unreachable addresses by transport", + }, + []string{"ipv", "transport"}, + ) + unknownAddrs = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricNamespace, + Name: "unknown", + Help: "Number of addresses with unknown reachability by transport", + }, + []string{"ipv", "transport"}, + ) + collectors = []prometheus.Collector{ + reachableAddrs, + unreachableAddrs, + unknownAddrs, + } +) + +// MetricsTracker tracks autonatv2 reachability metrics +type MetricsTracker interface { + // ConfirmedAddrsChanged updates metrics with current address reachability status + ConfirmedAddrsChanged(reachable, unreachable, unknown []ma.Multiaddr) + // ReachabilityTrackerClosed updated metrics on host close + ReachabilityTrackerClosed() +} + +type metricsTracker struct { + prevReachableCounts map[metricKey]int + prevUnreachableCounts map[metricKey]int + prevUnknownCounts map[metricKey]int + currentReachable map[metricKey]int + currentUnreachable map[metricKey]int + currentUnknown map[metricKey]int +} + +var _ MetricsTracker = &metricsTracker{} + +type metricsTrackerSetting struct { + reg prometheus.Registerer +} + +type metricsTrackerOption func(*metricsTrackerSetting) + +// withRegisterer sets the prometheus registerer for the metrics +func withRegisterer(reg prometheus.Registerer) metricsTrackerOption { + return func(s *metricsTrackerSetting) { + if reg != nil { + s.reg = reg + } + } +} + +type metricKey struct { + ipv string + transport string +} + +// newMetricsTracker creates a new metrics tracker for autonatv2 +func newMetricsTracker(opts ...metricsTrackerOption) MetricsTracker { + setting := &metricsTrackerSetting{reg: prometheus.DefaultRegisterer} + for _, opt := range opts { + opt(setting) + } + metricshelper.RegisterCollectors(setting.reg, collectors...) + return &metricsTracker{ + prevReachableCounts: make(map[metricKey]int), + prevUnreachableCounts: make(map[metricKey]int), + prevUnknownCounts: make(map[metricKey]int), + currentReachable: make(map[metricKey]int), + currentUnreachable: make(map[metricKey]int), + currentUnknown: make(map[metricKey]int), + } +} + +func (t *metricsTracker) ReachabilityTrackerClosed() { + resetMetric(reachableAddrs, t.currentReachable, t.prevReachableCounts) + resetMetric(unreachableAddrs, t.currentUnreachable, t.prevUnreachableCounts) + resetMetric(unknownAddrs, t.currentUnknown, t.prevUnknownCounts) +} + +// ConfirmedAddrsChanged updates the metrics with current address reachability counts by transport +func (t *metricsTracker) ConfirmedAddrsChanged(reachable, unreachable, unknown []ma.Multiaddr) { + updateMetric(reachableAddrs, reachable, t.currentReachable, t.prevReachableCounts) + updateMetric(unreachableAddrs, unreachable, t.currentUnreachable, t.prevUnreachableCounts) + updateMetric(unknownAddrs, unknown, t.currentUnknown, t.prevUnknownCounts) +} + +func updateMetric(metric *prometheus.GaugeVec, addrs []ma.Multiaddr, current map[metricKey]int, prev map[metricKey]int) { + clear(prev) + maps.Copy(prev, current) + clear(current) + for _, addr := range addrs { + transport := metricshelper.GetTransport(addr) + ipv := metricshelper.GetIPVersion(addr) + key := metricKey{ipv: ipv, transport: transport} + current[key]++ + } + + tags := metricshelper.GetStringSlice() + defer metricshelper.PutStringSlice(tags) + + for k, v := range current { + *tags = append(*tags, k.ipv, k.transport) + metric.WithLabelValues(*tags...).Set(float64(v)) + *tags = (*tags)[:0] + } + for k := range prev { + if _, ok := current[k]; ok { + continue + } + *tags = append(*tags, k.ipv, k.transport) + metric.WithLabelValues(*tags...).Set(0) + *tags = (*tags)[:0] + } +} + +func resetMetric(metric *prometheus.GaugeVec, current map[metricKey]int, prev map[metricKey]int) { + tags := metricshelper.GetStringSlice() + defer metricshelper.PutStringSlice(tags) + for k := range current { + *tags = append(*tags, k.ipv, k.transport) + metric.WithLabelValues(*tags...).Set(0) + *tags = (*tags)[:0] + } + clear(current) + clear(prev) +} diff --git a/p2p/host/basic/addrs_metrics_test.go b/p2p/host/basic/addrs_metrics_test.go new file mode 100644 index 0000000000..7022ea35b9 --- /dev/null +++ b/p2p/host/basic/addrs_metrics_test.go @@ -0,0 +1,46 @@ +//go:build nocover + +package basichost + +import ( + "math/rand" + "testing" + + ma "github.com/multiformats/go-multiaddr" + "github.com/prometheus/client_golang/prometheus" +) + +func TestMetricsNoAllocNoCover(t *testing.T) { + addrs := []ma.Multiaddr{ + ma.StringCast("/ip4/1.2.3.4/tcp/1"), + ma.StringCast("/ip4/1.2.3.4/tcp/2"), + ma.StringCast("/ip4/1.2.3.4/udp/2345/quic"), + ma.StringCast("/ip4/1.2.3.4/udp/2346/webrtc-direct"), + ma.StringCast("/ip4/1.2.3.4/tcp/80/ws"), + ma.StringCast("/ip4/1.2.3.4/tcp/443/wss"), + ma.StringCast("/ip4/1.2.3.4/udp/443/quic-v1/webtransport"), + } + + randAddrs := func() []ma.Multiaddr { + n := rand.Intn(len(addrs)) + k := n + rand.Intn(len(addrs)-n) + return addrs[n:k] + } + + mt := newMetricsTracker(withRegisterer(prometheus.DefaultRegisterer)) + tests := map[string]func(){ + "ConfirmedAddrsChanged": func() { + mt.ConfirmedAddrsChanged(randAddrs(), randAddrs(), randAddrs()) + }, + "ReachabilityTrackerClosed": func() { + mt.ReachabilityTrackerClosed() + }, + } + + for method, f := range tests { + allocs := testing.AllocsPerRun(1000, f) + if allocs > 0 { + t.Fatalf("Alloc Test: %s, got: %0.2f, expected: 0 allocs", method, allocs) + } + } +} diff --git a/p2p/host/basic/addrs_reachability_tracker.go b/p2p/host/basic/addrs_reachability_tracker.go index 923b04cf70..b74feb27b9 100644 --- a/p2p/host/basic/addrs_reachability_tracker.go +++ b/p2p/host/basic/addrs_reachability_tracker.go @@ -50,6 +50,7 @@ type addrsReachabilityTracker struct { probeManager *probeManager newAddrs chan []ma.Multiaddr clock clock.Clock + metricsTracker MetricsTracker mx sync.Mutex reachableAddrs []ma.Multiaddr @@ -59,7 +60,7 @@ type addrsReachabilityTracker struct { // newAddrsReachabilityTracker returns a new addrsReachabilityTracker. // reachabilityUpdateCh is notified when reachability for any of the tracked address changes. -func newAddrsReachabilityTracker(client autonatv2Client, reachabilityUpdateCh chan struct{}, cl clock.Clock) *addrsReachabilityTracker { +func newAddrsReachabilityTracker(client autonatv2Client, reachabilityUpdateCh chan struct{}, cl clock.Clock, metricsTracker MetricsTracker) *addrsReachabilityTracker { ctx, cancel := context.WithCancel(context.Background()) if cl == nil { cl = clock.New() @@ -74,6 +75,7 @@ func newAddrsReachabilityTracker(client autonatv2Client, reachabilityUpdateCh ch maxConcurrency: defaultMaxConcurrency, newAddrs: make(chan []ma.Multiaddr, 1), clock: cl, + metricsTracker: metricsTracker, } } @@ -171,11 +173,17 @@ func (r *addrsReachabilityTracker) background() { <-task.BackoffCh task = reachabilityTask{} } + if r.metricsTracker != nil { + r.metricsTracker.ReachabilityTrackerClosed() + } return } currReachable, currUnreachable, currUnknown = r.appendConfirmedAddrs(currReachable[:0], currUnreachable[:0], currUnknown[:0]) if areAddrsDifferent(prevReachable, currReachable) || areAddrsDifferent(prevUnreachable, currUnreachable) || areAddrsDifferent(prevUnknown, currUnknown) { + if r.metricsTracker != nil { + r.metricsTracker.ConfirmedAddrsChanged(currReachable, currUnreachable, currUnknown) + } r.notify() } prevReachable = append(prevReachable[:0], currReachable...) @@ -205,6 +213,7 @@ func (r *addrsReachabilityTracker) appendConfirmedAddrs(reachable, unreachable, r.unreachableAddrs = append(r.unreachableAddrs[:0], unreachable...) r.unknownAddrs = append(r.unknownAddrs[:0], unknown...) r.mx.Unlock() + return reachable, unreachable, unknown } diff --git a/p2p/host/basic/addrs_reachability_tracker_test.go b/p2p/host/basic/addrs_reachability_tracker_test.go index b2916a0de7..74f9828d2d 100644 --- a/p2p/host/basic/addrs_reachability_tracker_test.go +++ b/p2p/host/basic/addrs_reachability_tracker_test.go @@ -925,7 +925,7 @@ func FuzzAddrsReachabilityTracker(f *testing.F) { cl := clock.NewMock() f.Fuzz(func(t *testing.T, numAddrs int, ips, protos, hostNames, autonatResponses []byte) { - tr := newAddrsReachabilityTracker(newMockClient(autonatResponses), nil, cl) + tr := newAddrsReachabilityTracker(newMockClient(autonatResponses), nil, cl, nil) require.NoError(t, tr.Start()) tr.UpdateAddrs(getAddrs(numAddrs, ips, protos, hostNames)) diff --git a/p2p/host/basic/basic_host.go b/p2p/host/basic/basic_host.go index 476ea9b275..ee55054a03 100644 --- a/p2p/host/basic/basic_host.go +++ b/p2p/host/basic/basic_host.go @@ -153,6 +153,8 @@ type HostOpts struct { EnableMetrics bool // PrometheusRegisterer is the PrometheusRegisterer used for metrics PrometheusRegisterer prometheus.Registerer + // AutoNATv2MetricsTracker tracks AutoNATv2 address reachability metrics + AutoNATv2MetricsTracker MetricsTracker // DisableIdentifyAddressDiscovery disables address discovery using peer provided observed addresses in identify DisableIdentifyAddressDiscovery bool @@ -245,7 +247,18 @@ func NewHost(n network.Network, opts *HostOpts) (*BasicHost, error) { if h.autonatv2 != nil { autonatv2Client = h.autonatv2 } - h.addressManager, err = newAddrsManager(h.eventbus, natmgr, addrFactory, h.Network().ListenAddresses, tfl, h.ids, h.addrsUpdatedChan, autonatv2Client) + h.addressManager, err = newAddrsManager( + h.eventbus, + natmgr, + addrFactory, + h.Network().ListenAddresses, + tfl, + h.ids, + h.addrsUpdatedChan, + autonatv2Client, + opts.EnableMetrics, + opts.PrometheusRegisterer, + ) if err != nil { return nil, fmt.Errorf("failed to create address service: %w", err) } diff --git a/p2p/protocol/autonatv2/autonat.go b/p2p/protocol/autonatv2/autonat.go index 866c02381f..bec135f689 100644 --- a/p2p/protocol/autonatv2/autonat.go +++ b/p2p/protocol/autonatv2/autonat.go @@ -108,7 +108,7 @@ func New(dialerHost host.Host, opts ...AutoNATOption) (*AutoNAT, error) { ctx: ctx, cancel: cancel, srv: newServer(dialerHost, s), - cli: newClient(), + cli: newClient(s), allowPrivateAddrs: s.allowPrivateAddrs, peers: newPeersMap(), throttlePeer: make(map[peer.ID]time.Time), diff --git a/p2p/protocol/autonatv2/client.go b/p2p/protocol/autonatv2/client.go index 7cd0dba5f0..d0cb02e681 100644 --- a/p2p/protocol/autonatv2/client.go +++ b/p2p/protocol/autonatv2/client.go @@ -24,6 +24,7 @@ type client struct { host host.Host dialData []byte normalizeMultiaddr func(ma.Multiaddr) ma.Multiaddr + metricsTracer MetricsTracer mu sync.Mutex // dialBackQueues maps nonce to the channel for providing the local multiaddr of the connection @@ -35,10 +36,11 @@ type normalizeMultiaddrer interface { NormalizeMultiaddr(ma.Multiaddr) ma.Multiaddr } -func newClient() *client { +func newClient(s *autoNATSettings) *client { return &client{ dialData: make([]byte, 4000), dialBackQueues: make(map[uint64]chan ma.Multiaddr), + metricsTracer: s.metricsTracer, } } @@ -58,6 +60,17 @@ func (ac *client) Close() { // GetReachability verifies address reachability with a AutoNAT v2 server p. func (ac *client) GetReachability(ctx context.Context, p peer.ID, reqs []Request) (Result, error) { + result, err := ac.getReachability(ctx, p, reqs) + + // Track metrics + if ac.metricsTracer != nil { + ac.metricsTracer.ClientCompletedRequest(reqs, result, err) + } + + return result, err +} + +func (ac *client) getReachability(ctx context.Context, p peer.ID, reqs []Request) (Result, error) { ctx, cancel := context.WithTimeout(ctx, streamTimeout) defer cancel() diff --git a/p2p/protocol/autonatv2/metrics.go b/p2p/protocol/autonatv2/metrics.go index baa467f6d2..55ba003529 100644 --- a/p2p/protocol/autonatv2/metrics.go +++ b/p2p/protocol/autonatv2/metrics.go @@ -1,6 +1,9 @@ package autonatv2 import ( + "strconv" + + "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/p2p/metricshelper" "github.com/libp2p/go-libp2p/p2p/protocol/autonatv2/pb" ma "github.com/multiformats/go-multiaddr" @@ -9,6 +12,7 @@ import ( type MetricsTracer interface { CompletedRequest(EventDialRequestCompleted) + ClientCompletedRequest([]Request, Result, error) } const metricNamespace = "libp2p_autonatv2" @@ -22,13 +26,29 @@ var ( }, []string{"server_error", "response_status", "dial_status", "dial_data_required", "ip_or_dns_version", "transport"}, ) + clientRequestsCompleted = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: metricNamespace, + Name: "client_requests_completed_total", + Help: "Client Requests Completed", + }, + []string{"ip_or_dns_version", "transport", "addr_count", "dial_refused", "reachability"}, + ) + clientRequestsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: metricNamespace, + Name: "client_requests_total", + Help: "Client Requests Total", + }, + []string{"outcome"}, + ) ) type metricsTracer struct { } func NewMetricsTracer(reg prometheus.Registerer) MetricsTracer { - metricshelper.RegisterCollectors(reg, requestsCompleted) + metricshelper.RegisterCollectors(reg, requestsCompleted, clientRequestsCompleted, clientRequestsTotal) return &metricsTracer{} } @@ -60,26 +80,63 @@ func (m *metricsTracer) CompletedRequest(e EventDialRequestCompleted) { requestsCompleted.WithLabelValues(*labels...).Inc() } +func (m *metricsTracer) ClientCompletedRequest(reqs []Request, result Result, err error) { + labels := metricshelper.GetStringSlice() + defer metricshelper.PutStringSlice(labels) + + if err != nil { + clientRequestsTotal.WithLabelValues("failure").Inc() + return + } + clientRequestsTotal.WithLabelValues("success").Inc() + + addrCount := len(reqs) + dialRefused := "false" + if result.AllAddrsRefused { + dialRefused = "true" + } + reachability := "unknown" + switch result.Reachability { + case network.ReachabilityPublic: + reachability = "public" + case network.ReachabilityPrivate: + reachability = "private" + } + + ipOrDNSVersion := "unknown" + transport := "unknown" + if result.Addr != nil { + ipOrDNSVersion = getIPOrDNSVersion(result.Addr) + transport = metricshelper.GetTransport(result.Addr) + } + + *labels = append(*labels, + ipOrDNSVersion, + transport, + strconv.Itoa(addrCount), + dialRefused, + reachability, + ) + clientRequestsCompleted.WithLabelValues(*labels...).Inc() +} + func getIPOrDNSVersion(a ma.Multiaddr) string { - if a == nil { + if len(a) == 0 { return "" } res := "unknown" - ma.ForEach(a, func(c ma.Component) bool { - switch c.Protocol().Code { - case ma.P_IP4: - res = "ip4" - case ma.P_IP6: - res = "ip6" - case ma.P_DNS, ma.P_DNSADDR: - res = "dns" - case ma.P_DNS4: - res = "dns4" - case ma.P_DNS6: - res = "dns6" - } - return false - }) + switch a[0].Protocol().Code { + case ma.P_DNS, ma.P_DNSADDR: + res = "dns" + case ma.P_DNS4: + res = "dns4" + case ma.P_DNS6: + res = "dns6" + case ma.P_IP4: + res = "ip4" + case ma.P_IP6: + res = "ip6" + } return res } diff --git a/p2p/protocol/autonatv2/metrics_test.go b/p2p/protocol/autonatv2/metrics_test.go index 786715143b..92d311e141 100644 --- a/p2p/protocol/autonatv2/metrics_test.go +++ b/p2p/protocol/autonatv2/metrics_test.go @@ -5,6 +5,7 @@ import ( "math/rand" "testing" + "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/p2p/protocol/autonatv2/pb" ma "github.com/multiformats/go-multiaddr" "github.com/prometheus/client_golang/prometheus" @@ -31,6 +32,11 @@ func TestMetricsNoAllocNoCover(t *testing.T) { ma.StringCast("/ip4/1.2.3.4/udp/1/quic-v1"), ma.StringCast("/ip4/1.1.1.1/tcp/1/"), } + reqs := [][]Request{ + {{Addr: addrs[0]}, {Addr: addrs[1], SendDialData: true}}, + {{Addr: addrs[1]}, {Addr: addrs[2]}}, + } + tests := map[string]func(){ "CompletedRequest": func() { mt.CompletedRequest(EventDialRequestCompleted{ @@ -41,6 +47,9 @@ func TestMetricsNoAllocNoCover(t *testing.T) { DialedAddr: addrs[rand.Intn(len(addrs))], }) }, + "CompletedClientRequest": func() { + mt.ClientCompletedRequest(reqs[rand.Intn(len(reqs))], Result{AllAddrsRefused: rand.Intn(2) == 1, Reachability: network.Reachability(rand.Intn(2)), Addr: addrs[rand.Intn(len(addrs))]}, errs[rand.Intn(len(errs))]) + }, } for method, f := range tests { allocs := testing.AllocsPerRun(10000, f)