From f9361bdb3a1be39cdf8d763d746089ec19e9e691 Mon Sep 17 00:00:00 2001 From: Alexandru Vasile Date: Mon, 30 Mar 2026 10:58:02 +0000 Subject: [PATCH 1/7] metrics: Count reserved and non reserved Signed-off-by: Alexandru Vasile --- .../src/litep2p/shim/notification/peerset.rs | 30 +++++++++++++ .../client/network/src/service/metrics.rs | 42 +++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/substrate/client/network/src/litep2p/shim/notification/peerset.rs b/substrate/client/network/src/litep2p/shim/notification/peerset.rs index 8fdf65fb2ad6..c97e732d3ad5 100644 --- a/substrate/client/network/src/litep2p/shim/notification/peerset.rs +++ b/substrate/client/network/src/litep2p/shim/notification/peerset.rs @@ -931,6 +931,36 @@ impl Peerset { } } + /// Report connected peer counts to metrics, split by direction and reservation status. + fn update_slot_metrics(&self) { + let (mut in_reserved, mut in_non_reserved) = (0usize, 0usize); + let (mut out_reserved, mut out_non_reserved) = (0usize, 0usize); + for state in self.peers.values() { + match state { + PeerState::Connected { direction: Direction::Inbound(Reserved::Yes) } => { + in_reserved += 1 + }, + PeerState::Connected { direction: Direction::Inbound(Reserved::No) } => { + in_non_reserved += 1 + }, + PeerState::Connected { direction: Direction::Outbound(Reserved::Yes) } => { + out_reserved += 1 + }, + PeerState::Connected { direction: Direction::Outbound(Reserved::No) } => { + out_non_reserved += 1 + }, + _ => {}, + } + } + self.metrics.set_peerset_num_connected( + &self.protocol, + in_reserved, + in_non_reserved, + out_reserved, + out_non_reserved, + ); + } + /// Connect to all reserved peers. /// /// Under the following conditions: diff --git a/substrate/client/network/src/service/metrics.rs b/substrate/client/network/src/service/metrics.rs index a34b7418fbad..6867a2ff160e 100644 --- a/substrate/client/network/src/service/metrics.rs +++ b/substrate/client/network/src/service/metrics.rs @@ -365,6 +365,35 @@ impl NotificationMetrics { .observe(size as f64); } } + + /// Update the number of connected peers per direction and reservation status. + pub fn set_peerset_num_connected( + &self, + protocol: &ProtocolName, + in_reserved: usize, + in_non_reserved: usize, + out_reserved: usize, + out_non_reserved: usize, + ) { + if let Some(metrics) = &self.metrics { + metrics + .peerset_num_connected + .with_label_values(&["in", "reserved", protocol]) + .set(in_reserved as u64); + metrics + .peerset_num_connected + .with_label_values(&["in", "non-reserved", protocol]) + .set(in_non_reserved as u64); + metrics + .peerset_num_connected + .with_label_values(&["out", "reserved", protocol]) + .set(out_reserved as u64); + metrics + .peerset_num_connected + .with_label_values(&["out", "non-reserved", protocol]) + .set(out_non_reserved as u64); + } + } } /// Notification metrics. @@ -378,6 +407,9 @@ struct InnerNotificationMetrics { /// In/outbound notification sizes. pub notifications_sizes: HistogramVec, + + /// Number of connected peers per direction, reservation status and protocol. + pub peerset_num_connected: GaugeVec, } impl InnerNotificationMetrics { @@ -417,6 +449,16 @@ impl InnerNotificationMetrics { )?, registry, )?, + peerset_num_connected: prometheus::register( + GaugeVec::new( + Opts::new( + "substrate_sub_libp2p_peerset_num_connected", + "Number of connected peers per direction, reservation status and protocol", + ), + &["direction", "kind", "protocol"], + )?, + registry, + )?, }) } } From b6e8bbc151719b9e41349167d32a7fb578fd8669 Mon Sep 17 00:00:00 2001 From: Alexandru Vasile Date: Tue, 31 Mar 2026 10:48:58 +0000 Subject: [PATCH 2/7] metrics: Simplify peerset metric propagation Signed-off-by: Alexandru Vasile --- .../src/litep2p/shim/notification/peerset.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/substrate/client/network/src/litep2p/shim/notification/peerset.rs b/substrate/client/network/src/litep2p/shim/notification/peerset.rs index c97e732d3ad5..8d67eccd6304 100644 --- a/substrate/client/network/src/litep2p/shim/notification/peerset.rs +++ b/substrate/client/network/src/litep2p/shim/notification/peerset.rs @@ -40,7 +40,10 @@ use crate::{ peer_store::{PeerStoreProvider, ProtocolHandle}, - service::traits::{self, ValidationResult}, + service::{ + metrics::NotificationMetrics, + traits::{self, ValidationResult}, + }, ProtocolName, ReputationChange as Reputation, }; @@ -931,10 +934,11 @@ impl Peerset { } } - /// Report connected peer counts to metrics, split by direction and reservation status. + /// Report connected peer counts to metrics. fn update_slot_metrics(&self) { let (mut in_reserved, mut in_non_reserved) = (0usize, 0usize); let (mut out_reserved, mut out_non_reserved) = (0usize, 0usize); + for state in self.peers.values() { match state { PeerState::Connected { direction: Direction::Inbound(Reserved::Yes) } => { @@ -952,6 +956,7 @@ impl Peerset { _ => {}, } } + self.metrics.set_peerset_num_connected( &self.protocol, in_reserved, @@ -1545,10 +1550,14 @@ impl Stream for Peerset { } } - // start timer for the next allocation and if there were peers which the `Peerset` + // Start timer for the next allocation and if there were peers which the `Peerset` // wasn't connected but should be, send command to litep2p to start opening substreams. self.next_slot_allocation = Delay::new(SLOT_ALLOCATION_FREQUENCY); + // Update metrics on every tick of slot allocation. This ensures metrics are + // eventually consistent at 1s intervals. + self.update_slot_metrics(); + if !connect_to.is_empty() { log::trace!( target: LOG_TARGET, From da51493e885fecb777bbdb50dffca53902ea6eba Mon Sep 17 00:00:00 2001 From: Alexandru Vasile Date: Tue, 31 Mar 2026 10:57:29 +0000 Subject: [PATCH 3/7] Enhance metrics with num disconnected and backoff Signed-off-by: Alexandru Vasile --- .../src/litep2p/shim/notification/peerset.rs | 6 +++++ .../client/network/src/service/metrics.rs | 24 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/substrate/client/network/src/litep2p/shim/notification/peerset.rs b/substrate/client/network/src/litep2p/shim/notification/peerset.rs index 8d67eccd6304..c744f9fbcc0b 100644 --- a/substrate/client/network/src/litep2p/shim/notification/peerset.rs +++ b/substrate/client/network/src/litep2p/shim/notification/peerset.rs @@ -938,6 +938,7 @@ impl Peerset { fn update_slot_metrics(&self) { let (mut in_reserved, mut in_non_reserved) = (0usize, 0usize); let (mut out_reserved, mut out_non_reserved) = (0usize, 0usize); + let (mut num_disconnected, mut num_backoff) = (0usize, 0usize); for state in self.peers.values() { match state { @@ -953,6 +954,9 @@ impl Peerset { PeerState::Connected { direction: Direction::Outbound(Reserved::No) } => { out_non_reserved += 1 }, + PeerState::Disconnected => num_disconnected += 1, + PeerState::Backoff => num_backoff += 1, + _ => {}, } } @@ -963,6 +967,8 @@ impl Peerset { in_non_reserved, out_reserved, out_non_reserved, + num_disconnected, + num_backoff, ); } diff --git a/substrate/client/network/src/service/metrics.rs b/substrate/client/network/src/service/metrics.rs index 6867a2ff160e..aea3b587eb34 100644 --- a/substrate/client/network/src/service/metrics.rs +++ b/substrate/client/network/src/service/metrics.rs @@ -374,6 +374,8 @@ impl NotificationMetrics { in_non_reserved: usize, out_reserved: usize, out_non_reserved: usize, + num_disconnected: usize, + num_backoff: usize, ) { if let Some(metrics) = &self.metrics { metrics @@ -392,6 +394,15 @@ impl NotificationMetrics { .peerset_num_connected .with_label_values(&["out", "non-reserved", protocol]) .set(out_non_reserved as u64); + + metrics + .peerset_num_state + .with_label_values(&["disconnected", protocol]) + .set(num_disconnected as u64); + metrics + .peerset_num_state + .with_label_values(&["backoff", protocol]) + .set(num_backoff as u64); } } } @@ -410,6 +421,9 @@ struct InnerNotificationMetrics { /// Number of connected peers per direction, reservation status and protocol. pub peerset_num_connected: GaugeVec, + + /// Number of disconnected and backed off peers. + pub peerset_num_state: GaugeVec, } impl InnerNotificationMetrics { @@ -459,6 +473,16 @@ impl InnerNotificationMetrics { )?, registry, )?, + peerset_num_state: prometheus::register( + GaugeVec::new( + Opts::new( + "substrate_sub_libp2p_peerset_num_state", + "Number of peers per state in the peerset manager", + ), + &["state", "protocol"], + )?, + registry, + )?, }) } } From f0fd6c04c021c74c7be656d8781a98d5b03d1b2e Mon Sep 17 00:00:00 2001 From: "cmd[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 31 Mar 2026 11:07:20 +0000 Subject: [PATCH 4/7] Update from github-actions[bot] running command 'prdoc --audience node_dev --bump patch' --- prdoc/pr_11574.prdoc | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 prdoc/pr_11574.prdoc diff --git a/prdoc/pr_11574.prdoc b/prdoc/pr_11574.prdoc new file mode 100644 index 000000000000..d28035aab9fb --- /dev/null +++ b/prdoc/pr_11574.prdoc @@ -0,0 +1,14 @@ +title: 'peerset/metrics: Add notification layer metrics for peer states' +doc: +- audience: Node Dev + description: |- + This PR adds two new metrics to enhance our monitoring insights into the notification protocol. + + The data exposed: + - connected peers grouped by in / out and kind (reserve / non-reserve) + - disconnected and backed off peers + + Because the slot counts are tightly coupled with the full peerset state machine, I've picked an eventually consistent model to easily bump the metrics at 1s tick intervals. +crates: +- name: sc-network + bump: patch From ddf2d07cff65c7a8d7d4858f88d4e19a459fb0bb Mon Sep 17 00:00:00 2001 From: Alexandru Vasile Date: Tue, 31 Mar 2026 16:19:18 +0000 Subject: [PATCH 5/7] Fix testing Signed-off-by: Alexandru Vasile --- .../src/litep2p/shim/notification/config.rs | 1 + .../src/litep2p/shim/notification/peerset.rs | 5 +++++ .../src/litep2p/shim/notification/tests/fuzz.rs | 3 ++- .../litep2p/shim/notification/tests/peerset.rs | 16 +++++++++++++++- 4 files changed, 23 insertions(+), 2 deletions(-) diff --git a/substrate/client/network/src/litep2p/shim/notification/config.rs b/substrate/client/network/src/litep2p/shim/notification/config.rs index 70e136da4ed9..9c1dda7e34c4 100644 --- a/substrate/client/network/src/litep2p/shim/notification/config.rs +++ b/substrate/client/network/src/litep2p/shim/notification/config.rs @@ -96,6 +96,7 @@ impl NotificationProtocolConfig { set_config.reserved_nodes.iter().map(|address| address.peer_id).collect(), Arc::clone(&connected_peers), peerstore_handle, + metrics.clone(), ); // create `litep2p` notification protocol configuration for the protocol diff --git a/substrate/client/network/src/litep2p/shim/notification/peerset.rs b/substrate/client/network/src/litep2p/shim/notification/peerset.rs index c744f9fbcc0b..a1d7e91d101d 100644 --- a/substrate/client/network/src/litep2p/shim/notification/peerset.rs +++ b/substrate/client/network/src/litep2p/shim/notification/peerset.rs @@ -371,6 +371,9 @@ pub struct Peerset { /// Next time when [`Peerset`] should perform slot allocation. next_slot_allocation: Delay, + + /// Notification metrics. + metrics: NotificationMetrics, } macro_rules! decrement_or_warn { @@ -415,6 +418,7 @@ impl Peerset { reserved_peers: HashSet, connected_peers: Arc, peerstore_handle: Arc, + metrics: NotificationMetrics, ) -> (Self, TracingUnboundedSender) { let (cmd_tx, cmd_rx) = tracing_unbounded("mpsc-peerset-protocol", 100_000); let peers = reserved_peers @@ -450,6 +454,7 @@ impl Peerset { connected_peers, pending_backoffs: FuturesUnordered::new(), next_slot_allocation: Delay::new(SLOT_ALLOCATION_FREQUENCY), + metrics, }, cmd_tx, ) diff --git a/substrate/client/network/src/litep2p/shim/notification/tests/fuzz.rs b/substrate/client/network/src/litep2p/shim/notification/tests/fuzz.rs index c183dae6fa2b..dadcaaee74c6 100644 --- a/substrate/client/network/src/litep2p/shim/notification/tests/fuzz.rs +++ b/substrate/client/network/src/litep2p/shim/notification/tests/fuzz.rs @@ -24,7 +24,7 @@ use crate::{ peerstore::Peerstore, shim::notification::peerset::{OpenResult, Peerset, PeersetCommand}, }, - service::traits::{Direction, PeerStore, ValidationResult}, + service::{metrics::NotificationMetrics, traits::{Direction, PeerStore, ValidationResult}}, ProtocolName, }; @@ -94,6 +94,7 @@ async fn test_once() { .collect(), Default::default(), Arc::clone(&peer_store_handle), + NotificationMetrics::new(None), ); tokio::spawn(peerstore.run()); diff --git a/substrate/client/network/src/litep2p/shim/notification/tests/peerset.rs b/substrate/client/network/src/litep2p/shim/notification/tests/peerset.rs index b56c82f950a5..10385d0e8c0e 100644 --- a/substrate/client/network/src/litep2p/shim/notification/tests/peerset.rs +++ b/substrate/client/network/src/litep2p/shim/notification/tests/peerset.rs @@ -23,7 +23,7 @@ use crate::{ Direction, OpenResult, PeerState, Peerset, PeersetCommand, Reserved, }, }, - service::traits::{self, ValidationResult}, + service::{metrics::NotificationMetrics, traits::{self, ValidationResult}}, ProtocolName, }; @@ -65,6 +65,7 @@ async fn inbound_substream_for_outbound_peer() { Default::default(), Default::default(), peerstore_handle, + NotificationMetrics::new(None), ); assert_eq!(peerset.num_in(), 0usize); assert_eq!(peerset.num_out(), 0usize); @@ -114,6 +115,7 @@ async fn canceled_peer_gets_banned() { peers.clone(), Default::default(), peerstore_handle, + NotificationMetrics::new(None), ); assert_eq!(peerset.num_in(), 0usize); assert_eq!(peerset.num_out(), 0usize); @@ -168,6 +170,7 @@ async fn peer_added_and_removed_from_peerset() { Default::default(), Default::default(), peerstore_handle, + NotificationMetrics::new(None), ); assert_eq!(peerset.num_in(), 0usize); assert_eq!(peerset.num_out(), 0usize); @@ -290,6 +293,7 @@ async fn set_reserved_peers() { reserved.clone(), Default::default(), Arc::new(peerstore_handle_test()), + NotificationMetrics::new(None), ); assert_eq!(peerset.num_in(), 0usize); assert_eq!(peerset.num_out(), 0usize); @@ -374,6 +378,7 @@ async fn set_reserved_peers_one_peer_already_in_the_set() { reserved.clone(), Default::default(), Arc::new(peerstore_handle_test()), + NotificationMetrics::new(None), ); assert_eq!(peerset.num_in(), 0usize); assert_eq!(peerset.num_out(), 0usize); @@ -478,6 +483,7 @@ async fn add_reserved_peers_one_peer_already_in_the_set() { reserved.iter().cloned().collect(), Default::default(), peerstore_handle, + NotificationMetrics::new(None), ); assert_eq!(peerset.num_in(), 0usize); assert_eq!(peerset.num_out(), 0usize); @@ -570,6 +576,7 @@ async fn opening_peer_gets_canceled_and_disconnected() { Default::default(), Arc::clone(&num_connected), peerstore_handle, + NotificationMetrics::new(None), ); assert_eq!(peerset.num_in(), 0); assert_eq!(peerset.num_out(), 0); @@ -649,6 +656,7 @@ async fn open_failure_for_canceled_peer() { Default::default(), Default::default(), peerstore_handle, + NotificationMetrics::new(None), ); assert_eq!(peerset.num_in(), 0usize); assert_eq!(peerset.num_out(), 0usize); @@ -713,6 +721,7 @@ async fn peer_disconnected_when_being_validated_then_rejected() { Default::default(), Default::default(), peerstore_handle, + NotificationMetrics::new(None), ); assert_eq!(peerset.num_in(), 0usize); assert_eq!(peerset.num_out(), 0usize); @@ -745,6 +754,7 @@ async fn removed_reserved_peer_kept_due_to_free_slots() { peers.clone(), Default::default(), peerstore_handle, + NotificationMetrics::new(None), ); assert_eq!(peerset.num_in(), 0usize); assert_eq!(peerset.num_out(), 0usize); @@ -815,6 +825,7 @@ async fn set_reserved_peers_but_available_slots() { Default::default(), Default::default(), peerstore_handle, + NotificationMetrics::new(None), ); assert_eq!(peerset.num_in(), 0usize); assert_eq!(peerset.num_out(), 0usize); @@ -919,6 +930,7 @@ async fn set_reserved_peers_move_previously_reserved() { known_peers.clone(), Default::default(), peerstore_handle, + NotificationMetrics::new(None), ); assert_eq!(peerset.num_in(), 0usize); assert_eq!(peerset.num_out(), 0usize); @@ -1058,6 +1070,7 @@ async fn set_reserved_peers_cannot_move_previously_reserved() { known_peers.clone(), Default::default(), peerstore_handle, + NotificationMetrics::new(None), ); assert_eq!(peerset.num_in(), 0usize); assert_eq!(peerset.num_out(), 0usize); @@ -1157,6 +1170,7 @@ async fn reserved_only_rejects_non_reserved_peers() { reserved_peers.clone(), connected_peers.clone(), peerstore_handle, + NotificationMetrics::new(None), ); assert_eq!(peerset.num_in(), 0usize); assert_eq!(peerset.num_out(), 0usize); From 15dceccfbcafd0b311ee0dc77ecf32afc2f50376 Mon Sep 17 00:00:00 2001 From: Alexandru Vasile Date: Wed, 1 Apr 2026 09:30:59 +0000 Subject: [PATCH 6/7] notif: Fix fmt Signed-off-by: Alexandru Vasile --- .../network/src/litep2p/shim/notification/tests/fuzz.rs | 5 ++++- .../network/src/litep2p/shim/notification/tests/peerset.rs | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/substrate/client/network/src/litep2p/shim/notification/tests/fuzz.rs b/substrate/client/network/src/litep2p/shim/notification/tests/fuzz.rs index dadcaaee74c6..1dc5e4a53d5b 100644 --- a/substrate/client/network/src/litep2p/shim/notification/tests/fuzz.rs +++ b/substrate/client/network/src/litep2p/shim/notification/tests/fuzz.rs @@ -24,7 +24,10 @@ use crate::{ peerstore::Peerstore, shim::notification::peerset::{OpenResult, Peerset, PeersetCommand}, }, - service::{metrics::NotificationMetrics, traits::{Direction, PeerStore, ValidationResult}}, + service::{ + metrics::NotificationMetrics, + traits::{Direction, PeerStore, ValidationResult}, + }, ProtocolName, }; diff --git a/substrate/client/network/src/litep2p/shim/notification/tests/peerset.rs b/substrate/client/network/src/litep2p/shim/notification/tests/peerset.rs index 10385d0e8c0e..ec828677e580 100644 --- a/substrate/client/network/src/litep2p/shim/notification/tests/peerset.rs +++ b/substrate/client/network/src/litep2p/shim/notification/tests/peerset.rs @@ -23,7 +23,10 @@ use crate::{ Direction, OpenResult, PeerState, Peerset, PeersetCommand, Reserved, }, }, - service::{metrics::NotificationMetrics, traits::{self, ValidationResult}}, + service::{ + metrics::NotificationMetrics, + traits::{self, ValidationResult}, + }, ProtocolName, }; From 6c57035d33f28f6ad8aae7226838ea2341a5ce50 Mon Sep 17 00:00:00 2001 From: Alexandru Vasile <60601340+lexnv@users.noreply.github.com> Date: Tue, 19 May 2026 14:37:16 +0300 Subject: [PATCH 7/7] prdoc: Bump to minor --- prdoc/pr_11574.prdoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prdoc/pr_11574.prdoc b/prdoc/pr_11574.prdoc index d28035aab9fb..cdf87f6e2b89 100644 --- a/prdoc/pr_11574.prdoc +++ b/prdoc/pr_11574.prdoc @@ -11,4 +11,4 @@ doc: Because the slot counts are tightly coupled with the full peerset state machine, I've picked an eventually consistent model to easily bump the metrics at 1s tick intervals. crates: - name: sc-network - bump: patch + bump: minor