Skip to content
This repository was archived by the owner on Nov 15, 2023. It is now read-only.

client/network/service: Add primary dimension to connection metrics #6472

Merged
gavofyork merged 6 commits into from
Jun 23, 2020
Merged

client/network/service: Add primary dimension to connection metrics #6472

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
68c4f31
client/network/service: Add primary dimension to connection metrics
mxinden Jun 22, 2020
2d2f93e
.maintain/monitoring: Ensure to sum over all connections_closed variants
mxinden Jun 22, 2020
2ca29a8
client/network/service: Rename is_primary to is_first
mxinden Jun 23, 2020
b03b12d
client/network/service: Split by metric name with two additional metrics
mxinden Jun 23, 2020
9f3f33e
Revert ".maintain/monitoring: Ensure to sum over all connections_clos…
mxinden Jun 23, 2020
779a06e
client/network/service: Remove labels from distinct metrics
mxinden Jun 23, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .maintain/monitoring/grafana-dashboards/substrate-networking.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1790,7 +1790,7 @@
"steppedLine": false,
"targets": [
{
"expr": "avg(increase(${metric_namespace}_sub_libp2p_connections_closed_total{instance=~\"${nodename}\"}[$__interval])) by (reason)",
"expr": "avg(sum(rate(${metric_namespace}_sub_libp2p_connections_closed_total{instance=~\"${nodename}\"}[$__interval])) by (instance, reason)) by (reason)",
"interval": "",
"legendFormat": "{{reason}}",
"refId": "A"
Expand Down Expand Up @@ -2719,4 +2719,4 @@
"list": []
},
"version": 103
}
}
52 changes: 27 additions & 25 deletions client/network/src/service.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -889,16 +889,16 @@ impl Metrics {
connections_closed_total: register(CounterVec::new(
Opts::new(
"sub_libp2p_connections_closed_total",
"Total number of connections closed, by reason and direction"
"Total number of connections closed, by direction, reason and by being the first or not"
),
&["direction", "reason"]
&["direction", "reason", "was_first"]
Copy link
Contributor

@tomaka tomaka Jun 23, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
&["direction", "reason", "was_first"]
&["direction", "reason", "was_last"]

Connections can close in a different order as they were opened.
What we report is whether the connection that has closed was the last one still open, not whether it is the one that was opened the first.

)?, registry)?,
connections_opened_total: register(CounterVec::new(
Opts::new(
"sub_libp2p_connections_opened_total",
"Total number of connections opened"
"Total number of connections opened by direction and by being the first or not"
),
&["direction"]
&["direction", "is_first"]
)?, registry)?,
import_queue_blocks_submitted: register(Counter::new(
"import_queue_blocks_submitted",
Expand Down Expand Up @@ -1214,41 +1214,43 @@ impl<B: BlockT + 'static, H: ExHashT> Future for NetworkWorker<B, H> {
}
this.event_streams.send(ev);
},
Poll::Ready(SwarmEvent::ConnectionEstablished { peer_id, endpoint, .. }) => {
Poll::Ready(SwarmEvent::ConnectionEstablished { peer_id, endpoint, num_established }) => {
trace!(target: "sub-libp2p", "Libp2p => Connected({:?})", peer_id);

if let Some(metrics) = this.metrics.as_ref() {
match endpoint {
ConnectedPoint::Dialer { .. } =>
metrics.connections_opened_total.with_label_values(&["out"]).inc(),
ConnectedPoint::Listener { .. } =>
metrics.connections_opened_total.with_label_values(&["in"]).inc(),
}
let direction = match endpoint {
ConnectedPoint::Dialer { .. } => "out",
ConnectedPoint::Listener { .. } => "in",
};
let is_first = if num_established.get() == 1 { "true" } else { "false" };

metrics.connections_opened_total.with_label_values(&[direction, is_first]).inc();
}
},
Poll::Ready(SwarmEvent::ConnectionClosed { peer_id, cause, endpoint, .. }) => {
Poll::Ready(SwarmEvent::ConnectionClosed { peer_id, cause, endpoint, num_established }) => {
trace!(target: "sub-libp2p", "Libp2p => Disconnected({:?}, {:?})", peer_id, cause);
if let Some(metrics) = this.metrics.as_ref() {
let dir = match endpoint {
let direction = match endpoint {
ConnectedPoint::Dialer { .. } => "out",
ConnectedPoint::Listener { .. } => "in",
};

match cause {
ConnectionError::IO(_) =>
metrics.connections_closed_total.with_label_values(&[dir, "transport-error"]).inc(),
let reason = match cause {
ConnectionError::IO(_) => "transport-error",
ConnectionError::Handler(NodeHandlerWrapperError::Handler(EitherError::A(EitherError::A(
EitherError::A(EitherError::A(EitherError::B(
EitherError::A(PingFailure::Timeout)))))))) =>
metrics.connections_closed_total.with_label_values(&[dir, "ping-timeout"]).inc(),
EitherError::A(PingFailure::Timeout)))))))) => "ping-timeout",
ConnectionError::Handler(NodeHandlerWrapperError::Handler(EitherError::A(EitherError::A(
EitherError::A(EitherError::A(EitherError::A(
EitherError::B(LegacyConnectionKillError)))))))) =>
metrics.connections_closed_total.with_label_values(&[dir, "force-closed"]).inc(),
ConnectionError::Handler(NodeHandlerWrapperError::Handler(_)) =>
metrics.connections_closed_total.with_label_values(&[dir, "protocol-error"]).inc(),
ConnectionError::Handler(NodeHandlerWrapperError::KeepAliveTimeout) =>
metrics.connections_closed_total.with_label_values(&[dir, "keep-alive-timeout"]).inc(),
}
EitherError::B(LegacyConnectionKillError)))))))) => "force-closed",
ConnectionError::Handler(NodeHandlerWrapperError::Handler(_)) => "protocol-error",
ConnectionError::Handler(NodeHandlerWrapperError::KeepAliveTimeout) => "keep-alive-timeout",
};

// `num_established` represents the number of *remaining* connections.
let was_first = if num_established == 0 { "true" } else { "false" };

metrics.connections_closed_total.with_label_values(&[direction, reason, was_first]).inc();
}
},
Poll::Ready(SwarmEvent::NewListenAddr(addr)) => {
Expand Down