Fix incorrect peer port autodetection and substream limit exceeded errors

Cargo.toml

@@ -817,7 +817,7 @@ hyper-rustls = { version = "0.27.3", default-features = false, features = ["http
 hyper-util = { version = "0.1.5", default-features = false }
 impl-serde = { version = "0.5.0", default-features = false }
 impl-trait-for-tuples = { version = "0.2.2" }
-indexmap = { version = "2.0.0" }
+indexmap = { version = "2.7.1" }
 indicatif = { version = "0.17.7" }
 integer-sqrt = { version = "0.1.2" }
 ip_network = { version = "0.4.1" }
@@ -846,7 +846,7 @@ linked-hash-map = { version = "0.5.4" }
 linked_hash_set = { version = "0.1.4" }
 linregress = { version = "0.5.1" }
 lite-json = { version = "0.2.0", default-features = false }
-litep2p = { version = "0.9.0", features = ["websocket"] }
+litep2p = { version = "0.9.1", features = ["websocket"] }
 log = { version = "0.4.22", default-features = false }
 macro_magic = { version = "0.5.1" }
 maplit = { version = "1.0.2" }
@@ -1359,7 +1359,7 @@ tt-call = { version = "1.0.8" }
 tuplex = { version = "0.1", default-features = false }
 twox-hash = { version = "1.6.3", default-features = false }
 unsigned-varint = { version = "0.7.2" }
-url = { version = "2.4.0" }
+url = { version = "2.5.4" }
 void = { version = "1.0.2" }
 w3f-bls = { version = "0.1.3", default-features = false }
 wait-timeout = { version = "0.2" }

prdoc/pr_7338.prdoc

@@ -0,0 +1,10 @@
+title: '[net/libp2p] Use raw `Identify` observed addresses to discover external addresses'
+doc:
+- audience: Node Dev
+  description: |-
+    Instead of using libp2p-provided external address candidates, susceptible to address translation issues, use litep2p-backend approach based on confirming addresses observed by multiple peers as external.
+
+    Fixes https://github.com/paritytech/polkadot-sdk/issues/7207.
+crates:
+- name: sc-network
+  bump: major

prdoc/pr_7640.prdoc

@@ -0,0 +1,25 @@
+title: Bring the latest compatibility fixes via litep2p v0.9.1
+
+doc:
+  - audience: [Node Dev, Node Operator]
+    description: |
+      This release enhances compatibility between litep2p and libp2p by using the latest Yamux upstream version.
+      Additionally, it includes various improvements and fixes to boost the stability and performance of the WebSocket stream and the multistream-select protocol.
+
+crates:
+  - name: sc-network
+    bump: minor
+  - name: cumulus-client-cli
+    bump: minor
+  - name: sc-network-types
+    bump: minor
+  - name: sc-transaction-pool-api
+    bump: minor
+  - name: sc-transaction-pool
+    bump: minor
+  - name: polkadot-statement-distribution
+    bump: minor
+  - name: polkadot-dispute-distribution
+    bump: minor
+  - name: cumulus-relay-chain-rpc-interface
+    bump: minor

prdoc/pr_7724.prdoc

@@ -0,0 +1,13 @@
+title: Terminate libp2p the outbound notification substream on io errors
+
+doc:
+  - audience: [Node Dev, Node Operator]
+    description: |
+      This PR handles a case where we called the poll_next on an outbound substream notification to check if the stream is closed.
+      It is entirely possible that the poll_next would return an io::error, for example end of file.
+      This PR ensures that we make the distinction between unexpected incoming data, and error originated from poll_next.
+      While at it, the bulk of the PR change propagates the PeerID from the network behavior, through the notification handler, to the notification outbound stream for logging purposes.
+
+crates:
+  - name: sc-network
+    bump: patch

substrate/client/network/src/behaviour.rs

@@ -184,6 +184,7 @@ impl<B: BlockT> Behaviour<B> {
 		request_response_protocols: Vec<ProtocolConfig>,
 		peer_store_handle: Arc<dyn PeerStoreProvider>,
 		external_addresses: Arc<Mutex<HashSet<Multiaddr>>>,
+		public_addresses: Vec<Multiaddr>,
 		connection_limits: ConnectionLimits,
 	) -> Result<Self, request_responses::RegisterError> {
 		Ok(Self {
@@ -192,6 +193,7 @@ impl<B: BlockT> Behaviour<B> {
 				user_agent,
 				local_public_key,
 				external_addresses,
+				public_addresses,
 			),
 			discovery: disco_config.finish(),
 			request_responses: request_responses::RequestResponsesBehaviour::new(

substrate/client/network/src/lib.rs

@@ -291,6 +291,9 @@ pub use service::{
 };
 pub use types::ProtocolName;
 
+/// Log target for `sc-network`.
+const LOG_TARGET: &str = "sub-libp2p";
+
 /// The maximum allowed number of established connections per peer.
 ///
 /// Typically, and by design of the network behaviours in this crate,

substrate/client/network/src/litep2p/discovery.rs

@@ -50,6 +50,7 @@ use schnellru::{ByLength, LruMap};
 use std::{
 	cmp,
 	collections::{HashMap, HashSet, VecDeque},
+	iter,
 	num::NonZeroUsize,
 	pin::Pin,
 	sync::Arc,
@@ -72,11 +73,9 @@ const GET_RECORD_REDUNDANCY_FACTOR: usize = 4;
 /// The maximum number of tracked external addresses we allow.
 const MAX_EXTERNAL_ADDRESSES: u32 = 32;
 
-/// Minimum number of confirmations received before an address is verified.
-///
-/// Note: all addresses are confirmed by libp2p on the first encounter. This aims to make
-/// addresses a bit more robust.
-const MIN_ADDRESS_CONFIRMATIONS: usize = 2;
+/// Number of times observed address is received from different peers before it is confirmed as
+/// external.
+const MIN_ADDRESS_CONFIRMATIONS: usize = 3;
 
 /// Discovery events.
 #[derive(Debug)]
@@ -486,7 +485,7 @@ impl Discovery {
 					.flatten()
 					.flatten();
 
-				self.address_confirmations.insert(address.clone(), Default::default());
+				self.address_confirmations.insert(address.clone(), iter::once(peer).collect());
 
 				return (false, oldest)
 			},

substrate/client/network/src/litep2p/mod.rs

@@ -39,7 +39,6 @@ use crate::{
 		},
 	},
 	peer_store::PeerStoreProvider,
-	protocol,
 	service::{
 		metrics::{register_without_sources, MetricSources, Metrics, NotificationMetrics},
 		out_events,
@@ -266,57 +265,6 @@ impl Litep2pNetworkBackend {
 		};
 		let config_builder = ConfigBuilder::new();
 
-		// The yamux buffer size limit is configured to be equal to the maximum frame size
-		// of all protocols. 10 bytes are added to each limit for the length prefix that
-		// is not included in the upper layer protocols limit but is still present in the
-		// yamux buffer. These 10 bytes correspond to the maximum size required to encode
-		// a variable-length-encoding 64bits number. In other words, we make the
-		// assumption that no notification larger than 2^64 will ever be sent.
-		let yamux_maximum_buffer_size = {
-			let requests_max = config
-				.request_response_protocols
-				.iter()
-				.map(|cfg| usize::try_from(cfg.max_request_size).unwrap_or(usize::MAX));
-			let responses_max = config
-				.request_response_protocols
-				.iter()
-				.map(|cfg| usize::try_from(cfg.max_response_size).unwrap_or(usize::MAX));
-			let notifs_max = config
-				.notification_protocols
-				.iter()
-				.map(|cfg| usize::try_from(cfg.max_notification_size()).unwrap_or(usize::MAX));
-
-			// A "default" max is added to cover all the other protocols: ping, identify,
-			// kademlia, block announces, and transactions.
-			let default_max = cmp::max(
-				1024 * 1024,
-				usize::try_from(protocol::BLOCK_ANNOUNCES_TRANSACTIONS_SUBSTREAM_SIZE)
-					.unwrap_or(usize::MAX),
-			);
-
-			iter::once(default_max)
-				.chain(requests_max)
-				.chain(responses_max)
-				.chain(notifs_max)
-				.max()
-				.expect("iterator known to always yield at least one element; qed")
-				.saturating_add(10)
-		};
-
-		let yamux_config = {
-			let mut yamux_config = litep2p::yamux::Config::default();
-			// Enable proper flow-control: window updates are only sent when
-			// buffered data has been consumed.
-			yamux_config.set_window_update_mode(litep2p::yamux::WindowUpdateMode::OnRead);
-			yamux_config.set_max_buffer_size(yamux_maximum_buffer_size);
-
-			if let Some(yamux_window_size) = config.network_config.yamux_window_size {
-				yamux_config.set_receive_window(yamux_window_size);
-			}
-
-			yamux_config
-		};
-
 		let (tcp, websocket): (Vec<Option<_>>, Vec<Option<_>>) = config
 			.network_config
 			.listen_addresses
@@ -365,13 +313,13 @@ impl Litep2pNetworkBackend {
 		config_builder
 			.with_websocket(WebSocketTransportConfig {
 				listen_addresses: websocket.into_iter().flatten().map(Into::into).collect(),
-				yamux_config: yamux_config.clone(),
+				yamux_config: litep2p::yamux::Config::default(),
 				nodelay: true,
 				..Default::default()
 			})
 			.with_tcp(TcpTransportConfig {
 				listen_addresses: tcp.into_iter().flatten().map(Into::into).collect(),
-				yamux_config,
+				yamux_config: litep2p::yamux::Config::default(),
 				nodelay: true,
 				..Default::default()
 			})