From 56e89b67523baa18de28f4a4d64ae51b19ddf241 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Sat, 4 Apr 2026 17:57:52 +0200 Subject: [PATCH 01/43] perf: tighten network timeouts for faster connection establishment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reduce connection strategy timeouts based on RTT analysis (300ms worst-case cross-globe RTT). The previous values were overly conservative, causing up to 89s worst-case connection time to unreachable peers. Changes: - Direct connection: 5s → 2s (1-2 RTTs + margin) - Hole-punch round: 15s → 3s (3 RTTs + margin) - Max hole-punch rounds: 3 → 2 - Relay establishment: 30s → 10s - Post-hole-punch direct retry: 3s → 1s (warm NAT pinhole) - Default send ACK timeout: 1s → 500ms - Remove redundant hardcoded 15s deadline in try_hole_punch; outer strategy.holepunch_timeout() is now the single source of truth Worst-case connection path drops from 89s to ~21s. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/config/nat_timeouts.rs | 2 +- src/connection_strategy.rs | 20 ++++++++++---------- src/p2p_endpoint.rs | 23 ++++++++++++----------- 3 files changed, 23 insertions(+), 22 deletions(-) diff --git a/src/config/nat_timeouts.rs b/src/config/nat_timeouts.rs index 861f7fa1..f07011c2 100644 --- a/src/config/nat_timeouts.rs +++ b/src/config/nat_timeouts.rs @@ -132,7 +132,7 @@ impl Default for RelayTimeouts { } /// Default time to wait for the peer to acknowledge stream data after a send. -const DEFAULT_SEND_ACK_TIMEOUT: Duration = Duration::from_secs(1); +const DEFAULT_SEND_ACK_TIMEOUT: Duration = Duration::from_millis(500); /// Fast-network send ACK timeout. const FAST_SEND_ACK_TIMEOUT: Duration = Duration::from_millis(500); diff --git a/src/connection_strategy.rs b/src/connection_strategy.rs index 11c9bed8..12e1b4fb 100644 --- a/src/connection_strategy.rs +++ b/src/connection_strategy.rs @@ -178,11 +178,11 @@ pub struct StrategyConfig { impl Default for StrategyConfig { fn default() -> Self { Self { - ipv4_timeout: Duration::from_secs(5), - ipv6_timeout: Duration::from_secs(5), - holepunch_timeout: Duration::from_secs(15), - relay_timeout: Duration::from_secs(30), - max_holepunch_rounds: 3, + ipv4_timeout: Duration::from_secs(2), + ipv6_timeout: Duration::from_secs(2), + holepunch_timeout: Duration::from_secs(3), + relay_timeout: Duration::from_secs(10), + max_holepunch_rounds: 2, ipv6_enabled: true, relay_enabled: true, coordinator: None, @@ -484,11 +484,11 @@ mod tests { #[test] fn test_default_config() { let config = StrategyConfig::default(); - assert_eq!(config.ipv4_timeout, Duration::from_secs(5)); - assert_eq!(config.ipv6_timeout, Duration::from_secs(5)); - assert_eq!(config.holepunch_timeout, Duration::from_secs(15)); - assert_eq!(config.relay_timeout, Duration::from_secs(30)); - assert_eq!(config.max_holepunch_rounds, 3); + assert_eq!(config.ipv4_timeout, Duration::from_secs(2)); + assert_eq!(config.ipv6_timeout, Duration::from_secs(2)); + assert_eq!(config.holepunch_timeout, Duration::from_secs(3)); + assert_eq!(config.relay_timeout, Duration::from_secs(10)); + assert_eq!(config.max_holepunch_rounds, 2); assert!(config.ipv6_enabled); assert!(config.relay_enabled); } diff --git a/src/p2p_endpoint.rs b/src/p2p_endpoint.rs index 2c1b25f0..ca07f5e9 100644 --- a/src/p2p_endpoint.rs +++ b/src/p2p_endpoint.rs @@ -87,6 +87,11 @@ const EVENT_CHANNEL_CAPACITY: usize = 256; /// event-driven reader-exit detection. const STALE_REAPER_INTERVAL: Duration = Duration::from_secs(10); +/// Quick direct connection attempt after a failed hole-punch round. +/// If the target's outgoing packets created a NAT binding, a QUIC handshake +/// through the pinhole needs only 1-2 RTTs (~600ms at 300ms worst-case RTT). +const POST_HOLEPUNCH_DIRECT_RETRY_TIMEOUT: Duration = Duration::from_secs(1); + use crate::SHUTDOWN_DRAIN_TIMEOUT; /// Extract the raw SPKI (SubjectPublicKeyInfo) bytes from a QUIC connection's @@ -1544,7 +1549,8 @@ impl P2pEndpoint { // the target's outgoing packets even though our // try_hole_punch didn't detect the connection. if let Ok(Ok(peer_conn)) = - timeout(Duration::from_secs(3), self.connect(target)).await + timeout(POST_HOLEPUNCH_DIRECT_RETRY_TIMEOUT, self.connect(target)) + .await { info!("✓ Post-hole-punch direct connect succeeded to {}", target); return Ok(( @@ -1578,7 +1584,8 @@ impl P2pEndpoint { Err(_) => { // Same: try a quick direct connect after timeout if let Ok(Ok(peer_conn)) = - timeout(Duration::from_secs(3), self.connect(target)).await + timeout(POST_HOLEPUNCH_DIRECT_RETRY_TIMEOUT, self.connect(target)) + .await { info!("✓ Post-hole-punch direct connect succeeded to {}", target); return Ok(( @@ -1816,7 +1823,8 @@ impl P2pEndpoint { // Poll for the connection to appear. The target node will receive // the relayed PUNCH_ME_NOW and initiate a QUIC connection to us, // which gets accepted by saorsa-core's transport handler. - let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + // No internal deadline — the outer strategy.holepunch_timeout() + // cancels this future when it expires. let mut poll_count = 0u32; loop { @@ -1873,7 +1881,7 @@ impl P2pEndpoint { } } - // Wait briefly then re-check, or timeout + // Wait briefly then re-check; the outer timeout cancels us on expiry tokio::select! { _ = self.inner.connection_notify().notified() => { debug!("try_hole_punch: connection_notify fired for {}", target); @@ -1881,13 +1889,6 @@ impl P2pEndpoint { _ = self.shutdown.cancelled() => { return Err(EndpointError::ShuttingDown); } - _ = tokio::time::sleep_until(deadline) => { - info!( - "try_hole_punch: TIMEOUT after 15s for {} (polled {} times)", - target, poll_count - ); - return Err(EndpointError::Timeout); - } // Wake periodically to drive session and re-check connections _ = tokio::time::sleep(Duration::from_millis(500)) => {} } From eb1eb64e0745087834ff35c3ab18fed3def29969 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Sat, 4 Apr 2026 18:13:37 +0200 Subject: [PATCH 02/43] =?UTF-8?q?fix:=20differentiate=20FAST=5FSEND=5FACK?= =?UTF-8?q?=5FTIMEOUT=20from=20default=20(500ms=20=E2=86=92=20250ms)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After lowering DEFAULT_SEND_ACK_TIMEOUT to 500ms, both constants held the same value, making TimeoutConfig::fast() identical to default for send_ack_timeout. Lower the fast variant to 250ms to match the halved-interval pattern used throughout the fast profile. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/config/nat_timeouts.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/config/nat_timeouts.rs b/src/config/nat_timeouts.rs index f07011c2..ceba2347 100644 --- a/src/config/nat_timeouts.rs +++ b/src/config/nat_timeouts.rs @@ -134,8 +134,8 @@ impl Default for RelayTimeouts { /// Default time to wait for the peer to acknowledge stream data after a send. const DEFAULT_SEND_ACK_TIMEOUT: Duration = Duration::from_millis(500); -/// Fast-network send ACK timeout. -const FAST_SEND_ACK_TIMEOUT: Duration = Duration::from_millis(500); +/// Fast-network send ACK timeout (halved from default, matching the fast profile pattern). +const FAST_SEND_ACK_TIMEOUT: Duration = Duration::from_millis(250); /// Master timeout configuration #[derive(Debug, Clone, Serialize, Deserialize)] From e29f32464c308b1d61a8481b16f7970f4a583688 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Sat, 4 Apr 2026 20:27:45 +0100 Subject: [PATCH 03/43] perf: increase timeouts based on 50-node multi-region testnet data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Observed hole-punch success times of 1-6s across regions (London → Tokyo coordinator → Hetzner NAT node). The 3s holepunch_timeout was cutting off punches that needed 3-6s of cross-continental coordination, causing fallback to relay which then also struggled. - ipv4_timeout: 2s → 3s (cross-region direct needs margin) - ipv6_timeout: 2s → 3s - holepunch_timeout: 3s → 8s (2x margin over observed 1-6s range) Worst-case connection path: 3+3+8+1+8+1+10 ≈ 34s Co-Authored-By: Claude Opus 4.6 (1M context) --- src/connection_strategy.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/connection_strategy.rs b/src/connection_strategy.rs index 12e1b4fb..a3e9953d 100644 --- a/src/connection_strategy.rs +++ b/src/connection_strategy.rs @@ -178,9 +178,9 @@ pub struct StrategyConfig { impl Default for StrategyConfig { fn default() -> Self { Self { - ipv4_timeout: Duration::from_secs(2), - ipv6_timeout: Duration::from_secs(2), - holepunch_timeout: Duration::from_secs(3), + ipv4_timeout: Duration::from_secs(3), + ipv6_timeout: Duration::from_secs(3), + holepunch_timeout: Duration::from_secs(8), relay_timeout: Duration::from_secs(10), max_holepunch_rounds: 2, ipv6_enabled: true, @@ -484,9 +484,9 @@ mod tests { #[test] fn test_default_config() { let config = StrategyConfig::default(); - assert_eq!(config.ipv4_timeout, Duration::from_secs(2)); - assert_eq!(config.ipv6_timeout, Duration::from_secs(2)); - assert_eq!(config.holepunch_timeout, Duration::from_secs(3)); + assert_eq!(config.ipv4_timeout, Duration::from_secs(3)); + assert_eq!(config.ipv6_timeout, Duration::from_secs(3)); + assert_eq!(config.holepunch_timeout, Duration::from_secs(8)); assert_eq!(config.relay_timeout, Duration::from_secs(10)); assert_eq!(config.max_holepunch_rounds, 2); assert!(config.ipv6_enabled); From da74da9c8a80788b6639ce092abadbe7cc3e6b18 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Sat, 4 Apr 2026 22:37:36 +0100 Subject: [PATCH 04/43] fix: race condition in hole-punch peer ID matching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit hole_punch_target_peer_id was a single shared field on P2pEndpoint. When multiple concurrent dial_candidate calls ran in parallel (the DHT lookup uses ALPHA=3 parallel queries), they all overwrote the same field. try_hole_punch then used find_connection_by_peer_id which returned whatever connection matched the LAST peer ID written, not the one being hole-punched. This caused hole-punches to "succeed" but return connections to the wrong peer. The DHT layer then couldn't use the connection (address mismatch), re-dialled, and timed out — blocking uploads indefinitely. Fix: replace the single shared Option<[u8; 32]> with a per-target DashMap. Each concurrent dial gets its own entry keyed by the target address it's connecting to. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/p2p_endpoint.rs | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/p2p_endpoint.rs b/src/p2p_endpoint.rs index ca07f5e9..b2586ca7 100644 --- a/src/p2p_endpoint.rs +++ b/src/p2p_endpoint.rs @@ -168,10 +168,11 @@ pub struct P2pEndpoint { /// Registered when ConnectionAccepted/Established fires for constrained transports. constrained_peer_addrs: Arc>>, - /// Target peer ID for the next hole-punch attempt. When set, the - /// PUNCH_ME_NOW uses this instead of wire_id_from_addr, allowing the - /// coordinator to match by peer identity (works for symmetric NAT). - hole_punch_target_peer_id: Arc>>, + /// Per-target peer IDs for hole-punch attempts. When set for a target + /// address, the PUNCH_ME_NOW uses the peer ID instead of wire_id_from_addr, + /// allowing the coordinator to match by peer identity. Keyed by target + /// address so concurrent dials don't race on shared state. + hole_punch_target_peer_ids: Arc>, /// Channel sender for data received from QUIC reader tasks and constrained poller data_tx: mpsc::Sender<(SocketAddr, Vec)>, @@ -761,7 +762,7 @@ impl P2pEndpoint { router: Arc::new(RwLock::new(router)), constrained_connections: Arc::new(RwLock::new(HashMap::new())), constrained_peer_addrs: Arc::new(RwLock::new(HashMap::new())), - hole_punch_target_peer_id: Arc::new(tokio::sync::Mutex::new(None)), + hole_punch_target_peer_ids: Arc::new(dashmap::DashMap::new()), data_tx, data_rx: Arc::new(tokio::sync::Mutex::new(data_rx)), reader_tasks, @@ -1196,15 +1197,15 @@ impl P2pEndpoint { /// ConnectionMethod::Relayed { relay } => println!("Relayed via {}", relay), /// } /// ``` - /// Set the target peer ID for the next hole-punch attempt. When set, the - /// PUNCH_ME_NOW frame carries the peer ID instead of a socket-address-derived - /// wire ID, allowing the coordinator to find the target connection by - /// authenticated identity — essential for symmetric NAT where the address - /// differs per peer. + /// Set the target peer ID for a hole-punch attempt to a specific address. + /// When set, the PUNCH_ME_NOW frame carries the peer ID instead of a + /// socket-address-derived wire ID, allowing the coordinator to find the + /// target connection by authenticated identity. /// - /// Cleared after each `connect_with_fallback` call. - pub async fn set_hole_punch_target_peer_id(&self, peer_id: Option<[u8; 32]>) { - *self.hole_punch_target_peer_id.lock().await = peer_id; + /// Keyed by target address so concurrent dials to different peers each + /// get their own peer ID without racing on shared state. + pub async fn set_hole_punch_target_peer_id(&self, target: SocketAddr, peer_id: [u8; 32]) { + self.hole_punch_target_peer_ids.insert(target, peer_id); } /// Connect with automatic fallback: Direct → HolePunch → Relay. @@ -1785,10 +1786,9 @@ impl P2pEndpoint { } // Initiate NAT traversal — sends PUNCH_ME_NOW to coordinator. - // Use the stored target peer ID if available (for symmetric NAT routing). - // Clone (not take) — the peer ID is needed across multiple - // hole-punch rounds within connect_with_fallback. - let target_peer_id = *self.hole_punch_target_peer_id.lock().await; + // Look up the target peer ID from the per-target map. This avoids + // races when multiple concurrent connections share the same P2pEndpoint. + let target_peer_id = self.hole_punch_target_peer_ids.get(&target).map(|v| *v); if target_peer_id.is_some() { info!( "try_hole_punch: calling initiate_nat_traversal({}, {}) with peer ID", @@ -3287,7 +3287,7 @@ impl Clone for P2pEndpoint { router: Arc::clone(&self.router), constrained_connections: Arc::clone(&self.constrained_connections), constrained_peer_addrs: Arc::clone(&self.constrained_peer_addrs), - hole_punch_target_peer_id: Arc::clone(&self.hole_punch_target_peer_id), + hole_punch_target_peer_ids: Arc::clone(&self.hole_punch_target_peer_ids), data_tx: self.data_tx.clone(), data_rx: Arc::clone(&self.data_rx), reader_tasks: Arc::clone(&self.reader_tasks), From f65af2e5a04ed8cc6476baab80c665f7dae33b53 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Sun, 5 Apr 2026 01:09:49 +0100 Subject: [PATCH 05/43] debug: add peer ID tracing to hole-punch coordination path Log the actual peer ID value at three points: 1. dial_candidate: what's inserted into the DashMap 2. try_hole_punch: what's read from the DashMap 3. send_coordination_request_with_peer_id: what goes on the wire This will show whether the DashMap fix is working or if the peer ID is being corrupted elsewhere in the path. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/nat_traversal_api.rs | 7 +++++-- src/p2p_endpoint.rs | 10 +++++----- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/nat_traversal_api.rs b/src/nat_traversal_api.rs index aa72f2da..e875ca7c 100644 --- a/src/nat_traversal_api.rs +++ b/src/nat_traversal_api.rs @@ -6064,8 +6064,11 @@ impl NatTraversalEndpoint { // wire_id_from_addr (works for cone NAT where address is stable). let target_wire_id = target_peer_id.unwrap_or_else(|| Self::wire_id_from_addr(target_addr)); info!( - "Sending PUNCH_ME_NOW coordination request for {} to coordinator {}", - target_addr, coordinator + "Sending PUNCH_ME_NOW coordination request for {} to coordinator {} (wire_id={}, from_peer_id={}, from_addr={})", + target_addr, coordinator, + hex::encode(&target_wire_id[..8]), + target_peer_id.map(|p| hex::encode(&p[..8])).unwrap_or_else(|| "none".to_string()), + !target_peer_id.is_some(), ); // Get our external address - this is where the target peer should punch to diff --git a/src/p2p_endpoint.rs b/src/p2p_endpoint.rs index b2586ca7..31fa5613 100644 --- a/src/p2p_endpoint.rs +++ b/src/p2p_endpoint.rs @@ -1789,15 +1789,15 @@ impl P2pEndpoint { // Look up the target peer ID from the per-target map. This avoids // races when multiple concurrent connections share the same P2pEndpoint. let target_peer_id = self.hole_punch_target_peer_ids.get(&target).map(|v| *v); - if target_peer_id.is_some() { + if let Some(ref pid) = target_peer_id { info!( - "try_hole_punch: calling initiate_nat_traversal({}, {}) with peer ID", - target, coordinator + "try_hole_punch: calling initiate_nat_traversal({}, {}) with peer ID {} (dashmap key={})", + target, coordinator, hex::encode(&pid[..8]), target ); } else { info!( - "try_hole_punch: calling initiate_nat_traversal({}, {}) with address-based wire ID", - target, coordinator + "try_hole_punch: calling initiate_nat_traversal({}, {}) with address-based wire ID (no dashmap entry for key={})", + target, coordinator, target ); } self.inner From d99e5ab11d95fefaf04b7e3f8024f67859925608 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Sun, 5 Apr 2026 07:02:13 +0100 Subject: [PATCH 06/43] feat: add preferred coordinator for hole-punch relay and improve relay diagnostics Add a per-target preferred coordinator DashMap so the DHT referrer (the node that returned the target in a FindNode response) is used as the first-choice coordinator for PUNCH_ME_NOW relay. This ensures the coordinator has an active connection to the target. Also improve the "No connection found" log to include hex-encoded peer IDs of all known connections, making it possible to definitively confirm when a coordinator cannot relay. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/endpoint.rs | 15 +++++++++++++-- src/p2p_endpoint.rs | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/src/endpoint.rs b/src/endpoint.rs index 8df1f395..d1d4e49c 100644 --- a/src/endpoint.rs +++ b/src/endpoint.rs @@ -375,9 +375,20 @@ impl Endpoint { ); } } else { + let known_peers: Vec = self + .connections + .iter() + .filter_map(|(_, meta)| { + meta.peer_id + .as_ref() + .map(|pid| hex::encode(&pid.0[..8])) + }) + .collect(); tracing::warn!( - "No connection found for PUNCH_ME_NOW relay target (peer_id {:?})", - &target_peer_id[..8] + "No connection found for PUNCH_ME_NOW relay target peer_id={}, checked {} connections. Known peers: [{}]", + hex::encode(&target_peer_id[..8]), + self.connections.len(), + known_peers.join(", ") ); } } diff --git a/src/p2p_endpoint.rs b/src/p2p_endpoint.rs index 31fa5613..4e3fa94c 100644 --- a/src/p2p_endpoint.rs +++ b/src/p2p_endpoint.rs @@ -174,6 +174,13 @@ pub struct P2pEndpoint { /// address so concurrent dials don't race on shared state. hole_punch_target_peer_ids: Arc>, + /// Per-target preferred coordinator for hole-punch relay. When the DHT + /// lookup discovers a peer via a FindNode response from another node, that + /// responding node (the "referrer") has a connection to the discovered peer + /// and is an ideal coordinator for PUNCH_ME_NOW relay. Keyed by target + /// address, value is the referrer's socket address. + hole_punch_preferred_coordinators: Arc>, + /// Channel sender for data received from QUIC reader tasks and constrained poller data_tx: mpsc::Sender<(SocketAddr, Vec)>, @@ -763,6 +770,7 @@ impl P2pEndpoint { constrained_connections: Arc::new(RwLock::new(HashMap::new())), constrained_peer_addrs: Arc::new(RwLock::new(HashMap::new())), hole_punch_target_peer_ids: Arc::new(dashmap::DashMap::new()), + hole_punch_preferred_coordinators: Arc::new(dashmap::DashMap::new()), data_tx, data_rx: Arc::new(tokio::sync::Mutex::new(data_rx)), reader_tasks, @@ -1208,6 +1216,18 @@ impl P2pEndpoint { self.hole_punch_target_peer_ids.insert(target, peer_id); } + /// Set a preferred coordinator for hole-punching to a specific target. + /// The preferred coordinator is a peer that referred us to the target + /// during a DHT lookup, so it has a connection to the target. + pub async fn set_hole_punch_preferred_coordinator( + &self, + target: SocketAddr, + coordinator: SocketAddr, + ) { + self.hole_punch_preferred_coordinators + .insert(target, coordinator); + } + /// Connect with automatic fallback: Direct → HolePunch → Relay. pub async fn connect_with_fallback( &self, @@ -1312,6 +1332,25 @@ impl P2pEndpoint { } } + // If the DHT referrer set a preferred coordinator for this target, + // move it to the front of the candidate list so round 1 uses it. + if let Some(target_addr) = target { + if let Some(preferred) = self.hole_punch_preferred_coordinators.get(&target_addr) { + let preferred_addr = *preferred; + coordinator_candidates.retain(|a| *a != preferred_addr); + coordinator_candidates.insert(0, preferred_addr); + info!( + "Using preferred coordinator {} for target {} (DHT referrer)", + preferred_addr, target_addr + ); + } else { + info!( + "No preferred coordinator for target {} (not discovered via DHT referral)", + target_addr + ); + } + } + if config.coordinator.is_none() { config.coordinator = coordinator_candidates.first().copied(); if let Some(coord) = config.coordinator { @@ -3288,6 +3327,7 @@ impl Clone for P2pEndpoint { constrained_connections: Arc::clone(&self.constrained_connections), constrained_peer_addrs: Arc::clone(&self.constrained_peer_addrs), hole_punch_target_peer_ids: Arc::clone(&self.hole_punch_target_peer_ids), + hole_punch_preferred_coordinators: Arc::clone(&self.hole_punch_preferred_coordinators), data_tx: self.data_tx.clone(), data_rx: Arc::clone(&self.data_rx), reader_tasks: Arc::clone(&self.reader_tasks), From 9009e64afed6693c7dd4951257fe4d02af6e26de Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Sun, 5 Apr 2026 10:26:33 +0100 Subject: [PATCH 07/43] fix: scale send ACK timeout with payload size MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fixed 500ms send_ack_timeout causes all large chunk transfers to fail. QUIC slow-start over a high-RTT path (e.g. London→Tokyo, 250ms RTT) needs 9-10 round trips to ramp the congestion window enough for 4MB, taking 2-3 seconds minimum. The 500ms timeout fires well before the data is delivered. Add a size-proportional budget: base_timeout + (payload_bytes / 1024)ms. This gives ~500ms for small DHT messages (still catches dead connections quickly) and ~4.5s for a 4MB chunk. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/p2p_endpoint.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/p2p_endpoint.rs b/src/p2p_endpoint.rs index 4e3fa94c..79b56a5f 100644 --- a/src/p2p_endpoint.rs +++ b/src/p2p_endpoint.rs @@ -2320,7 +2320,16 @@ impl P2pEndpoint { // Without this, finish() only buffers a FIN locally — if the // connection is dead the caller would see Ok(()) despite the // data never arriving. - let ack_timeout = self.config.timeouts.send_ack_timeout; + // + // The base timeout handles small messages and dead-connection + // detection. For large payloads we add time proportional to + // size: QUIC slow-start over a high-RTT path needs multiple + // round trips to ramp the congestion window, so a 4 MB chunk + // over a 250 ms RTT link can take 2-3 s just to transmit. + let base_timeout = self.config.timeouts.send_ack_timeout; + let size_budget = + std::time::Duration::from_millis((data.len() as u64).saturating_div(1024)); + let ack_timeout = base_timeout + size_budget; match timeout(ack_timeout, send_stream.stopped()).await { Ok(Ok(None)) => {} Ok(Ok(Some(stop_code))) => { From e828dbc2ca5216543020fbdba6ff585d2efa06f6 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Sun, 5 Apr 2026 11:38:45 +0100 Subject: [PATCH 08/43] debug: check connection health before queuing PUNCH_ME_NOW relay The PUNCH_ME_NOW relay path queues frames directly on the connection's pending queue without checking if the connection is alive. If the connection has silently died (error set or drained), the frame goes into a queue that will never be transmitted and no error is surfaced. Add conn.error and is_drained() checks before queuing, with logging to diagnose why some PUNCH_ME_NOW frames are queued but never arrive at the coordinator. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/high_level/connection.rs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/high_level/connection.rs b/src/high_level/connection.rs index 0e6bf83a..355c21ba 100644 --- a/src/high_level/connection.rs +++ b/src/high_level/connection.rs @@ -656,6 +656,27 @@ impl Connection { round: u32, ) -> Result<(), crate::ConnectionError> { let conn = &mut *self.0.state.lock("send_nat_punch_via_relay"); + + // Check connection health before queuing — a dead connection will + // silently swallow the frame. + if let Some(ref err) = conn.error { + tracing::warn!( + "send_nat_punch_via_relay: connection has error BEFORE queuing: {}", + err + ); + return Err(err.clone()); + } + if conn.inner.is_drained() { + tracing::warn!("send_nat_punch_via_relay: connection is drained"); + return Err(crate::ConnectionError::LocallyClosed); + } + + tracing::info!( + "send_nat_punch_via_relay: connection alive, queuing frame (target_peer={}, remote={})", + hex::encode(&target_peer_id[..8]), + conn.inner.remote_address(), + ); + conn.inner .send_nat_punch_via_relay(target_peer_id, our_address, round)?; // Wake the connection driver so it transmits the queued frame From 6c4088f8b691e496db761214d734729e0f38a796 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Sun, 5 Apr 2026 11:54:52 +0100 Subject: [PATCH 09/43] debug: log when PUNCH_ME_NOW relay frame is encoded into QUIC packet Frames queued on a live connection sometimes don't arrive at the coordinator. The previous diagnostic confirmed the connection is alive at queue time. This log covers the next step: whether poll_transmit actually pops the frame from the pending queue and encodes it into a QUIC packet. Only logs relay frames (target_peer_id is Some), not direct punches. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/connection/mod.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/connection/mod.rs b/src/connection/mod.rs index d908d696..3b05685f 100644 --- a/src/connection/mod.rs +++ b/src/connection/mod.rs @@ -4173,11 +4173,15 @@ impl Connection { Some(x) => x, None => break, }; - trace!( - round = %punch_me_now.round, - paired_with_sequence_number = %punch_me_now.paired_with_sequence_number, - "PUNCH_ME_NOW" - ); + if let Some(ref target) = punch_me_now.target_peer_id { + info!( + "populate_packet: ENCODING PUNCH_ME_NOW relay frame target_peer={} remote={} buf_len={} max_size={}", + hex::encode(&target[..8]), + self.path.remote, + buf.len(), + max_size, + ); + } // Use the correct encoding format based on negotiated configuration if self.nat_traversal_frame_config.use_rfc_format { encode_or_close!(punch_me_now.try_encode_rfc(buf), "PUNCH_ME_NOW (rfc)"); From c670c5498ad75977ca834ade3d13cd809e87aac6 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Sun, 5 Apr 2026 12:12:06 +0100 Subject: [PATCH 10/43] fix: always overwrite NAT API connection DashMap with newer connection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous logic skipped overwriting a DashMap entry when the existing connection had no close_reason. But connections can become zombies — their driver is no longer polling them, yet close_reason() still returns None. Frames queued on zombie connections are never encoded into QUIC packets and silently lost. Diagnostic data from a 50-node testnet showed that only 1 out of 5 coordinators was actually encoding PUNCH_ME_NOW frames. The other 4 had stale Connection objects in the DashMap that accepted frames into their pending queues but had no active driver to transmit them. Always overwrite with the newest connection, which is the one most likely to have an active driver. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/nat_traversal_api.rs | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/src/nat_traversal_api.rs b/src/nat_traversal_api.rs index e875ca7c..b546293d 100644 --- a/src/nat_traversal_api.rs +++ b/src/nat_traversal_api.rs @@ -3947,27 +3947,20 @@ impl NatTraversalEndpoint { ) -> Result<(), NatTraversalError> { let observed = connection.observed_address(); info!("add_connection: {} observed_address={:?}", addr, observed); - // Only insert if no existing LIVE connection. This prevents - // an outgoing hole-punch connection (which may die quickly) - // from overwriting an incoming connection that the reader task - // is actively using. The reader task has a clone of the - // connection object and continues to receive data even if the - // DashMap entry is replaced — but the send path looks up the - // DashMap, so we must keep the live connection there. - if let Some(existing) = self.connections.get(&addr) { - if existing.value().close_reason().is_none() { - info!( - "add_connection: {} already has a live connection, skipping overwrite", - addr - ); - drop(existing); - } else { - drop(existing); - self.connections.insert(addr, connection); - } - } else { - self.connections.insert(addr, connection); + // Always overwrite with the newer connection. The previous + // logic skipped overwrite when the existing connection had no + // close_reason, but a connection can become a zombie (driver no + // longer polling it) while still reporting close_reason=None. + // Frames queued on such a connection are never transmitted. + // The newest connection is the one most likely to have an active + // driver, so always use it. + if self.connections.contains_key(&addr) { + info!( + "add_connection: {} replacing existing connection with newer one", + addr + ); } + self.connections.insert(addr, connection); info!( "add_connection: now have {} connections", self.connections.len() From 5516b063d6ec0d518d53d0ff48f4bcddfb5a6245 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Sun, 5 Apr 2026 12:33:55 +0100 Subject: [PATCH 11/43] fix: detect zombie connections before queuing PUNCH_ME_NOW relay Before queuing a PUNCH_ME_NOW on a coordinator connection from the DashMap, verify the low-level endpoint still tracks it. Zombie connections pass close_reason() and is_drained() checks but their driver is no longer polling, so frames queued on them are never encoded into QUIC packets. When a zombie is detected, remove it from the DashMap and fall through to establish a new connection. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/high_level/endpoint.rs | 19 ++++++++++ src/nat_traversal_api.rs | 76 ++++++++++++++++++++++++-------------- 2 files changed, 68 insertions(+), 27 deletions(-) diff --git a/src/high_level/endpoint.rs b/src/high_level/endpoint.rs index e6129e98..56bd85cc 100644 --- a/src/high_level/endpoint.rs +++ b/src/high_level/endpoint.rs @@ -424,6 +424,25 @@ impl Endpoint { .local_addr() } + /// Check whether the low-level endpoint still has an active connection + /// to the given address. Returns `false` for zombie connections that have + /// been removed from the endpoint but still exist in higher-level caches. + pub fn has_active_connection(&self, addr: &SocketAddr) -> bool { + let Ok(state) = self.inner.state.lock() else { + return true; // mutex poisoned, assume live + }; + let normalized = crate::shared::normalize_socket_addr(*addr); + if state.inner.connection_handle_for_addr(&normalized).is_some() { + return true; + } + if let Some(alt) = crate::shared::dual_stack_alternate(&normalized) { + if state.inner.connection_handle_for_addr(&alt).is_some() { + return true; + } + } + false + } + /// Get the number of connections that are currently open pub fn open_connections(&self) -> usize { self.inner diff --git a/src/nat_traversal_api.rs b/src/nat_traversal_api.rs index b546293d..170d1a46 100644 --- a/src/nat_traversal_api.rs +++ b/src/nat_traversal_api.rs @@ -6089,9 +6089,11 @@ impl NatTraversalEndpoint { our_external_address ); - // Find the connection to the coordinator via direct lookup instead of - // iterating all shards. Try the normalized address first, then the - // dual-stack alternate (IPv4 ↔ IPv4-mapped IPv6). + // Find the connection to the coordinator. Prefer the DashMap (fast), + // but verify it's still actively driven by the low-level endpoint. + // Connections can become zombies — their driver stopped polling but + // close_reason() still returns None. Frames queued on zombies are + // never encoded into QUIC packets. let normalized_coordinator = normalize_socket_addr(coordinator); let coord_conn = self.connections.get(&normalized_coordinator).or_else(|| { dual_stack_alternate(&normalized_coordinator).and_then(|alt| self.connections.get(&alt)) @@ -6099,31 +6101,51 @@ impl NatTraversalEndpoint { if let Some(entry) = coord_conn { let conn = entry.value(); - info!( - "Sending PUNCH_ME_NOW via coordinator {} (normalized: {}) to target {}", - coordinator, normalized_coordinator, target_addr - ); - // Use round 1 for initial coordination - match conn.send_nat_punch_via_relay(target_wire_id, our_external_address, 1) { - Ok(()) => { - // Wake the connection driver immediately so the queued - // PUNCH_ME_NOW frame is transmitted without waiting for - // the next keep-alive or scheduled poll. Without this, - // idle connections delay transmission by up to 15s. - conn.wake_transmit(); - info!( - "Successfully queued PUNCH_ME_NOW for relay to {}", - target_addr - ); - return Ok(()); - } - Err(e) => { - warn!("Failed to queue PUNCH_ME_NOW frame: {:?}", e); - return Err(NatTraversalError::CoordinationFailed(format!( - "Failed to send PUNCH_ME_NOW: {:?}", - e - ))); + // Verify the low-level endpoint still tracks this connection. + // If it doesn't, the connection is a zombie — remove it and + // fall through to establish a new one. + let is_live = if let Some(ep) = &self.inner_endpoint { + ep.has_active_connection(&normalized_coordinator) + } else { + true // no endpoint, assume live + }; + + if !is_live { + warn!( + "Coordinator connection {} is zombie (not in low-level endpoint), removing from DashMap", + normalized_coordinator + ); + drop(entry); + self.connections.remove(&normalized_coordinator); + // Fall through to "establish new connection" below + } else { + info!( + "Sending PUNCH_ME_NOW via coordinator {} (normalized: {}) to target {}", + coordinator, normalized_coordinator, target_addr + ); + + // Use round 1 for initial coordination + match conn.send_nat_punch_via_relay(target_wire_id, our_external_address, 1) { + Ok(()) => { + // Wake the connection driver immediately so the queued + // PUNCH_ME_NOW frame is transmitted without waiting for + // the next keep-alive or scheduled poll. Without this, + // idle connections delay transmission by up to 15s. + conn.wake_transmit(); + info!( + "Successfully queued PUNCH_ME_NOW for relay to {}", + target_addr + ); + return Ok(()); + } + Err(e) => { + warn!("Failed to queue PUNCH_ME_NOW frame: {:?}", e); + return Err(NatTraversalError::CoordinationFailed(format!( + "Failed to send PUNCH_ME_NOW: {:?}", + e + ))); + } } } } From 0216038681d39d97c17010c55949f4911de10339 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Sun, 5 Apr 2026 13:13:14 +0100 Subject: [PATCH 12/43] fix: detect stale coordinator connections by comparing handle indices MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The DashMap can hold a connection whose handle index differs from the endpoint's current connection to the same address. This happens when the coordinator reconnects — the endpoint gets a new connection but the DashMap keeps the old one. Frames encoded on the old connection use stale connection IDs that the coordinator drops silently. Compare the DashMap connection's handle_index() against the endpoint's connection_stable_id_for_addr(). If they differ, log the mismatch (STALE) and remove the DashMap entry, falling through to establish a new connection. Logging distinguishes three cases: - STALE: DashMap and endpoint both have connections, but different ones - ORPHAN: DashMap has a connection but the endpoint doesn't - verified: handles match, connection is the same one being driven Co-Authored-By: Claude Opus 4.6 (1M context) --- src/endpoint.rs | 6 +++++ src/high_level/connection.rs | 7 ++++++ src/high_level/endpoint.rs | 27 ++++++++++++++-------- src/nat_traversal_api.rs | 45 +++++++++++++++++++++++++++--------- 4 files changed, 64 insertions(+), 21 deletions(-) diff --git a/src/endpoint.rs b/src/endpoint.rs index d1d4e49c..1291d4f2 100644 --- a/src/endpoint.rs +++ b/src/endpoint.rs @@ -278,6 +278,12 @@ impl Endpoint { None } + /// Get a stable identifier for a connection by handle. This is the slab + /// index, which is stable for the lifetime of the connection. + pub fn connection_stable_id(&self, handle: ConnectionHandle) -> usize { + handle.0 + } + /// Get relay statistics for monitoring pub fn relay_stats(&self) -> &RelayStats { &self.relay_stats diff --git a/src/high_level/connection.rs b/src/high_level/connection.rs index 355c21ba..26894bec 100644 --- a/src/high_level/connection.rs +++ b/src/high_level/connection.rs @@ -793,6 +793,13 @@ impl Connection { self.0.stable_id() } + /// Get the low-level connection handle index. This can be compared against + /// the endpoint's `connection_stable_id_for_addr()` to detect when the + /// endpoint has replaced the connection with a newer one. + pub fn handle_index(&self) -> usize { + self.0.state.lock("handle_index").handle.0 + } + /// Returns true if this connection negotiated post-quantum settings. /// /// This reflects either explicit PQC algorithms advertised via transport diff --git a/src/high_level/endpoint.rs b/src/high_level/endpoint.rs index 56bd85cc..8f94e76b 100644 --- a/src/high_level/endpoint.rs +++ b/src/high_level/endpoint.rs @@ -428,19 +428,26 @@ impl Endpoint { /// to the given address. Returns `false` for zombie connections that have /// been removed from the endpoint but still exist in higher-level caches. pub fn has_active_connection(&self, addr: &SocketAddr) -> bool { + self.connection_stable_id_for_addr(addr).is_some() + } + + /// Get the stable ID of the low-level endpoint's connection to the given + /// address. Returns `None` if no connection exists. The stable ID uniquely + /// identifies a specific QUIC connection and can be compared against a + /// cached Connection's stable_id() to detect stale references. + pub fn connection_stable_id_for_addr(&self, addr: &SocketAddr) -> Option { let Ok(state) = self.inner.state.lock() else { - return true; // mutex poisoned, assume live + return None; }; let normalized = crate::shared::normalize_socket_addr(*addr); - if state.inner.connection_handle_for_addr(&normalized).is_some() { - return true; - } - if let Some(alt) = crate::shared::dual_stack_alternate(&normalized) { - if state.inner.connection_handle_for_addr(&alt).is_some() { - return true; - } - } - false + let handle = state + .inner + .connection_handle_for_addr(&normalized) + .or_else(|| { + crate::shared::dual_stack_alternate(&normalized) + .and_then(|alt| state.inner.connection_handle_for_addr(&alt)) + }); + handle.map(|h| state.inner.connection_stable_id(h)) } /// Get the number of connections that are currently open diff --git a/src/nat_traversal_api.rs b/src/nat_traversal_api.rs index 170d1a46..e5152058 100644 --- a/src/nat_traversal_api.rs +++ b/src/nat_traversal_api.rs @@ -6102,20 +6102,43 @@ impl NatTraversalEndpoint { if let Some(entry) = coord_conn { let conn = entry.value(); - // Verify the low-level endpoint still tracks this connection. - // If it doesn't, the connection is a zombie — remove it and - // fall through to establish a new one. - let is_live = if let Some(ep) = &self.inner_endpoint { - ep.has_active_connection(&normalized_coordinator) + // Verify this is the SAME connection the endpoint is driving. + // The DashMap may hold a stale connection while the endpoint has + // a newer one to the same address. Frames encoded on the stale + // connection are sent with old connection IDs that the coordinator + // no longer recognises. + let dashmap_handle = conn.handle_index(); + let endpoint_handle = if let Some(ep) = &self.inner_endpoint { + ep.connection_stable_id_for_addr(&normalized_coordinator) } else { - true // no endpoint, assume live + None }; - if !is_live { - warn!( - "Coordinator connection {} is zombie (not in low-level endpoint), removing from DashMap", - normalized_coordinator - ); + let is_stale = match endpoint_handle { + Some(ep_handle) if ep_handle != dashmap_handle => { + warn!( + "Coordinator connection {} is STALE: DashMap handle={} but endpoint handle={}. Removing stale entry.", + normalized_coordinator, dashmap_handle, ep_handle + ); + true + } + None => { + warn!( + "Coordinator connection {} is ORPHAN: DashMap handle={} but endpoint has no connection. Removing.", + normalized_coordinator, dashmap_handle + ); + true + } + Some(ep_handle) => { + info!( + "Coordinator connection {} verified: handle={} matches endpoint", + normalized_coordinator, ep_handle + ); + false + } + }; + + if is_stale { drop(entry); self.connections.remove(&normalized_coordinator); // Fall through to "establish new connection" below From e861d3a5cb607a3e0162687ac5a7a23f0596e696 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Sun, 5 Apr 2026 13:45:30 +0100 Subject: [PATCH 13/43] fix: keep preferred coordinator on hole-punch retry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, round 2 switched to coordinator_candidates[1] — a random fallback that likely has no connection to the target. The preferred coordinator (index 0) was chosen because the DHT referrer has a known connection to the target. If round 1 timed out, the relay was sent but the return connection didn't arrive in time. Retrying with the same coordinator gives it another chance rather than wasting 5 seconds on a coordinator that can't relay. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/p2p_endpoint.rs | 35 +++++++++++++---------------------- 1 file changed, 13 insertions(+), 22 deletions(-) diff --git a/src/p2p_endpoint.rs b/src/p2p_endpoint.rs index 79b56a5f..a3d7afc9 100644 --- a/src/p2p_endpoint.rs +++ b/src/p2p_endpoint.rs @@ -1600,21 +1600,15 @@ impl P2pEndpoint { } strategy.record_holepunch_error(round, e.to_string()); if strategy.should_retry_holepunch() { - // Try a different coordinator for the next round. - // The current coordinator may be behind NAT itself - // and unable to relay PUNCH_ME_NOW to the target. - if let Some(next) = coordinator_candidates.get(round as usize) { - info!( - "Hole-punch round {} failed, trying coordinator {} next", - round, next - ); - strategy.set_coordinator(*next); - } else { - debug!( - "Hole-punch round {} failed, no more coordinators", - round - ); - } + // Keep the same coordinator for retries. The preferred + // coordinator (index 0) was chosen because it has a + // known connection to the target. Switching to a random + // fallback wastes another round on a coordinator that + // likely can't relay to the target. + info!( + "Hole-punch round {} failed, retrying with same coordinator", + round + ); strategy.increment_round(); } else { debug!("Hole-punch failed after {} rounds", round); @@ -1635,13 +1629,10 @@ impl P2pEndpoint { } strategy.record_holepunch_error(round, "Timeout".to_string()); if strategy.should_retry_holepunch() { - if let Some(next) = coordinator_candidates.get(round as usize) { - info!( - "Hole-punch round {} timed out, trying coordinator {} next", - round, next - ); - strategy.set_coordinator(*next); - } + info!( + "Hole-punch round {} timed out, retrying with same coordinator", + round + ); strategy.increment_round(); } else { debug!("Hole-punch timed out after {} rounds", round); From 5280123d5f1d15570cfc37eab4147d54f5825121 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Mon, 6 Apr 2026 15:39:33 +0200 Subject: [PATCH 14/43] refactor!: remove dead NatTraversalEndpoint hole-punch chain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deletes the unused duplicate connect_with_fallback on NatTraversalEndpoint plus its supporting "PATH_CHALLENGE hole-punch" chain (attempt_hole_punching, attempt_quic_hole_punching, get_candidate_pairs_for_addr, calculate_candidate_pair_priority, create_path_challenge_packet, store_successful_candidate_pair, get_successful_candidate_address) and the orphaned successful_candidates DashMap field. The deleted code was reachable only from the duplicate connect_with_fallback (also removed). The actual production hole-punch path lives in P2pEndpoint::connect_with_fallback_inner, which is what LinkTransport::dial_addr (link_transport_impl.rs) and the saorsa-transport example bin both call. The duplicate had no in-tree or downstream consumers — verified across this crate, saorsa-core, and ant-node. The deleted hole-punch implementation could not have worked even if it had been wired in: 1. attempt_quic_hole_punching bound a fresh std::net::UdpSocket to the local candidate address — always fails on a real node because Quinn already owns the port (UDP binds are exclusive). 2. The "QUIC packet" it sent was a hand-rolled byte sequence (0x40 [0,0,0,1] 0x1a <8 random>) that is not a valid encrypted QUIC packet, so any receiving Quinn endpoint silently dropped it during connection-id lookup. 3. The success branch then waited 100 ms blocking on recv_from for a "response" no compliant peer would ever send. The #[allow(dead_code)] markers on every function in the chain disguised that nothing in the path could ever succeed, actively misleading anyone debugging hole-punch issues via grep. Comment blocks left in place point readers at the production path. Also fixes a pre-existing nonminimal_bool clippy warning in the same file (`!target_peer_id.is_some()` -> `target_peer_id.is_none()`) so the file now passes `cargo clippy --lib -- -D warnings`. The fix is a single character but is called out here so it isn't hidden inside the bulk deletion. The `!` indicates removal of a `pub fn`. No in-tree or downstream consumer depended on it, but the public API surface is technically narrower. Verified: cargo test --lib 1459/1459 passed (3 ignored, unchanged) cargo clippy --lib -- -D warnings clean rustfmt --edition 2024 --check clean Downstream saorsa-core test suite still green Co-Authored-By: Claude Opus 4.6 (1M context) --- src/nat_traversal_api.rs | 499 ++++----------------------------------- 1 file changed, 47 insertions(+), 452 deletions(-) diff --git a/src/nat_traversal_api.rs b/src/nat_traversal_api.rs index e5152058..89ff67a4 100644 --- a/src/nat_traversal_api.rs +++ b/src/nat_traversal_api.rs @@ -161,7 +161,7 @@ impl TransportCandidate { } } -use tracing::{debug, error, info, trace, warn}; +use tracing::{debug, error, info, warn}; use std::sync::atomic::{AtomicBool, Ordering}; // Use parking_lot for faster, non-poisoning locks that work better with async code @@ -298,10 +298,6 @@ pub struct NatTraversalEndpoint { /// MASQUE relay server - every node provides relay services (symmetric P2P) /// Per ADR-004: All nodes are equal and participate in relaying with resource budgets relay_server: Option>, - /// Successful candidate pairs discovered via hole punching - /// Maps remote SocketAddr to the validated address that successfully responded - /// Uses DashMap for fine-grained concurrent access without blocking workers - successful_candidates: Arc>, /// Transport candidates received from peers (multi-transport support) /// Maps remote SocketAddr to all known transport candidates for that peer /// Enables routing decisions based on transport type and capabilities @@ -1393,7 +1389,6 @@ impl NatTraversalEndpoint { relay_manager, relay_sessions: Arc::new(dashmap::DashMap::new()), relay_server, - successful_candidates: Arc::new(dashmap::DashMap::new()), transport_candidates: Arc::new(dashmap::DashMap::new()), transport_registry, peer_address_update_rx: TokioMutex::new(peer_addr_rx), @@ -1803,7 +1798,6 @@ impl NatTraversalEndpoint { relay_manager, relay_sessions: Arc::new(dashmap::DashMap::new()), relay_server, - successful_candidates: Arc::new(dashmap::DashMap::new()), transport_candidates: Arc::new(dashmap::DashMap::new()), transport_registry, peer_address_update_rx: TokioMutex::new(peer_addr_rx), @@ -3242,163 +3236,20 @@ impl NatTraversalEndpoint { Ok(connection) } - /// Attempt connection with automatic fallback strategies - /// - /// Connection attempts follow this priority order: - /// 1. **Direct connection** - simple QUIC connect to the target address - /// 2. **Hole punching** - coordinated NAT traversal with candidate discovery - /// 3. **Relay** - last resort via MASQUE through connected peers (symmetric P2P) - /// - /// # Symmetric P2P Relay Strategy - /// When relay is needed: - /// - First try connected peers as relays (any peer can relay) - /// - Fall back to configured relay_nodes (for bootstrap scenarios only) - pub async fn connect_with_fallback( - &self, - server_name: &str, - remote_addr: SocketAddr, - ) -> Result { - // Step 1: Try direct connection first - info!("Attempting direct connection to {}", remote_addr); - match self.connect_to(server_name, remote_addr).await { - Ok(conn) => { - info!("Direct connection to {} succeeded", remote_addr); - return Ok(conn); - } - Err(e) => { - info!( - "Direct connection to {} failed ({:?}), trying hole punching", - remote_addr, e - ); - } - } - - // Step 2: Try hole punching (coordinated NAT traversal) - info!("Attempting hole punching for {}", remote_addr); - match self.attempt_hole_punching(remote_addr) { - Ok(()) => { - // Hole punching succeeded - NAT mappings are established - // Now try to connect again using the discovered path - info!( - "Hole punching succeeded for {}, retrying connection", - remote_addr - ); - - // Get the successful candidate pair address if available - let connect_addr = self - .get_successful_candidate_address(remote_addr) - .unwrap_or(remote_addr); - - match self.connect_to(server_name, connect_addr).await { - Ok(conn) => { - info!("Connection via hole punching to {} succeeded", remote_addr); - return Ok(conn); - } - Err(e) => { - info!( - "Connection after hole punching failed ({:?}), trying relay", - e - ); - } - } - } - Err(e) => { - info!( - "Hole punching for {} failed ({:?}), trying relay", - remote_addr, e - ); - } - } - - // Step 3: Relay is the last resort - info!( - "Attempting relay connection to {} (last resort)", - remote_addr - ); - - // Symmetric P2P: Collect connected peers to use as potential relays - // Any connected peer can provide relay services - // DashMap provides lock-free concurrent access - let connected_peers: Vec = self - .connections - .iter() - .filter(|entry| entry.value().close_reason().is_none()) // Only active connections - .map(|entry| entry.value().remote_address()) - .filter(|addr| *addr != remote_addr) // Don't try to relay through the target - .collect(); - - info!( - "Found {} connected peers to try as relays", - connected_peers.len() - ); - - // Also add configured relay nodes as fallback (for bootstrapping) - let mut relay_candidates: Vec = connected_peers; - if let Some(ref manager) = self.relay_manager { - let configured_relays = manager.available_relays().await; - for relay in configured_relays { - if !relay_candidates.contains(&relay) { - relay_candidates.push(relay); - } - } - } - - if relay_candidates.is_empty() { - return Err(NatTraversalError::ConnectionFailed( - "No connected peers or relay nodes available".to_string(), - )); - } - - // Try each relay in order - let mut last_error = None; - for relay_addr in relay_candidates { - info!("Attempting connection via relay: {}", relay_addr); - - // Establish relay session (CONNECT-UDP Bind) - match self.establish_relay_session(relay_addr).await { - Ok(public_addr) => { - info!( - "Relay session established via {} with public address {:?}", - relay_addr, public_addr - ); - - // Now attempt the connection through the relay - // The relay session is stored and the connection can use datagram forwarding - // For now, we attempt a direct connection to the peer using our relay public address - // The peer should be able to reach us through the relay - - // Try connecting to the peer - the relay will forward our traffic - match self.connect_to(server_name, remote_addr).await { - Ok(conn) => { - info!( - "Connected to {} via relay {} (public addr: {:?})", - remote_addr, relay_addr, public_addr - ); - return Ok(conn); - } - Err(e) => { - warn!( - "Connection via relay {} failed: {:?}, trying next relay", - relay_addr, e - ); - last_error = Some(e); - } - } - } - Err(e) => { - warn!( - "Failed to establish relay session with {}: {:?}", - relay_addr, e - ); - last_error = Some(e); - } - } - } - - Err(last_error.unwrap_or_else(|| { - NatTraversalError::ConnectionFailed("All relay attempts failed".to_string()) - })) - } + // ───────────────────────────────────────────────────────────────────── + // Note: the historical `NatTraversalEndpoint::connect_with_fallback` + // method that lived here has been removed. It was an unused duplicate + // of `P2pEndpoint::connect_with_fallback` (in `p2p_endpoint.rs`), which + // is the actual production entry point reached through `LinkTransport:: + // dial_addr` and the `saorsa-transport` example binary. The removed + // copy delegated to `attempt_hole_punching` (also removed below), an + // implementation that crafted a hand-rolled "PATH_CHALLENGE" UDP + // datagram on a freshly bound socket — both unworkable in practice + // (the bind raced Quinn for the port; the bytes were not a valid + // QUIC packet so the receiver dropped them) and misleading during + // debugging because the surrounding `#[allow(dead_code)]` markers + // disguised that nothing in the path could ever succeed. + // ───────────────────────────────────────────────────────────────────── /// Get the relay manager for advanced relay operations /// @@ -4820,291 +4671,32 @@ impl NatTraversalEndpoint { Ok(frame) } - #[allow(dead_code)] - fn attempt_hole_punching(&self, target_addr: SocketAddr) -> Result<(), NatTraversalError> { - debug!("Attempting hole punching for {}", target_addr); - - // Get candidate pairs for this target - let candidate_pairs = self.get_candidate_pairs_for_addr(target_addr)?; - - if candidate_pairs.is_empty() { - return Err(NatTraversalError::NoCandidatesFound); - } - - info!( - "Generated {} candidate pairs for hole punching with {}", - candidate_pairs.len(), - target_addr - ); - - // Attempt hole punching with each candidate pair - - self.attempt_quic_hole_punching(target_addr, candidate_pairs) - } - - /// Generate candidate pairs for hole punching based on ICE-like algorithm - #[allow(dead_code)] - fn get_candidate_pairs_for_addr( - &self, - target_addr: SocketAddr, - ) -> Result, NatTraversalError> { - let discovery_session_id = DiscoverySessionId::Remote(target_addr); - - // Get discovered candidates from the discovery manager - // parking_lot::Mutex doesn't poison - let discovery_candidates = { - let discovery = self.discovery_manager.lock(); - discovery.get_candidates(discovery_session_id) - }; - - if discovery_candidates.is_empty() { - return Err(NatTraversalError::NoCandidatesFound); - } - - // Create candidate pairs with priorities (ICE-like pairing) - let mut candidate_pairs = Vec::new(); - let local_candidates = discovery_candidates - .iter() - .filter(|c| matches!(c.source, CandidateSource::Local)) - .collect::>(); - let remote_candidates = discovery_candidates - .iter() - .filter(|c| !matches!(c.source, CandidateSource::Local)) - .collect::>(); - - // Pair each local candidate with each remote candidate - // Skip cross-family pairs (IPv4 ↔ IPv6) as they cannot connect at the socket level - for local in &local_candidates { - for remote in &remote_candidates { - // Cross-family pairs will always fail - skip them - let local_is_v4 = local.address.ip().is_ipv4(); - let remote_is_v4 = remote.address.ip().is_ipv4(); - if local_is_v4 != remote_is_v4 { - trace!( - "Skipping cross-family candidate pair: {} ↔ {}", - local.address, remote.address - ); - continue; - } - - let pair_priority = self.calculate_candidate_pair_priority(local, remote); - candidate_pairs.push(CandidatePair { - local_candidate: (*local).clone(), - remote_candidate: (*remote).clone(), - priority: pair_priority, - state: CandidatePairState::Waiting, - }); - } - } - - // Sort by priority (highest first) - candidate_pairs.sort_by(|a, b| b.priority.cmp(&a.priority)); - - // Limit to reasonable number for initial attempts - candidate_pairs.truncate(8); - - Ok(candidate_pairs) - } - - /// Calculate candidate pair priority using ICE algorithm - #[allow(dead_code)] - fn calculate_candidate_pair_priority( - &self, - local: &CandidateAddress, - remote: &CandidateAddress, - ) -> u64 { - // ICE candidate pair priority formula: min(G,D) * 2^32 + max(G,D) * 2 + (G>D ? 1 : 0) - // Where G is controlling agent priority, D is controlled agent priority - - let local_type_preference = match local.source { - CandidateSource::Local => 126, - CandidateSource::Observed { .. } => 100, - CandidateSource::Predicted => 75, - CandidateSource::Peer => 50, - }; - - let remote_type_preference = match remote.source { - CandidateSource::Local => 126, - CandidateSource::Observed { .. } => 100, - CandidateSource::Predicted => 75, - CandidateSource::Peer => 50, - }; - - // Simplified priority calculation - let local_priority = (local_type_preference as u64) << 8 | local.priority as u64; - let remote_priority = (remote_type_preference as u64) << 8 | remote.priority as u64; - - let min_priority = local_priority.min(remote_priority); - let max_priority = local_priority.max(remote_priority); - - (min_priority << 32) - | (max_priority << 1) - | if local_priority > remote_priority { - 1 - } else { - 0 - } - } - - /// Real QUIC-based hole punching implementation - #[allow(dead_code)] - fn attempt_quic_hole_punching( - &self, - target_addr: SocketAddr, - candidate_pairs: Vec, - ) -> Result<(), NatTraversalError> { - let _endpoint = self.inner_endpoint.as_ref().ok_or_else(|| { - NatTraversalError::ConfigError("QUIC endpoint not initialized".to_string()) - })?; - - for pair in candidate_pairs { - debug!( - "Attempting hole punch with candidate pair: {} -> {}", - pair.local_candidate.address, pair.remote_candidate.address - ); - - // Create PATH_CHALLENGE frame data (8 random bytes) - let mut challenge_data = [0u8; 8]; - for byte in &mut challenge_data { - *byte = rand::random(); - } - - // Create a raw UDP socket bound to the local candidate address - let local_socket = - std::net::UdpSocket::bind(pair.local_candidate.address).map_err(|e| { - NatTraversalError::NetworkError(format!( - "Failed to bind to local candidate: {e}" - )) - })?; - - // Craft a minimal QUIC packet with PATH_CHALLENGE frame - let path_challenge_packet = self.create_path_challenge_packet(challenge_data)?; - - // Send the packet to the remote candidate address - match local_socket.send_to(&path_challenge_packet, pair.remote_candidate.address) { - Ok(bytes_sent) => { - debug!( - "Sent {} bytes for hole punch from {} to {}", - bytes_sent, pair.local_candidate.address, pair.remote_candidate.address - ); - - // Set a short timeout for response - local_socket - .set_read_timeout(Some(Duration::from_millis(100))) - .map_err(|e| { - NatTraversalError::NetworkError(format!("Failed to set timeout: {e}")) - })?; - - // Try to receive a response - let mut response_buffer = [0u8; 1024]; - match local_socket.recv_from(&mut response_buffer) { - Ok((_bytes_received, response_addr)) => { - if response_addr == pair.remote_candidate.address { - info!( - "Hole punch succeeded for {}: {} <-> {}", - target_addr, - pair.local_candidate.address, - pair.remote_candidate.address - ); - - // Store successful candidate pair for connection establishment - self.store_successful_candidate_pair(target_addr, pair)?; - return Ok(()); - } else { - debug!( - "Received response from unexpected address: {}", - response_addr - ); - } - } - Err(e) - if e.kind() == std::io::ErrorKind::WouldBlock - || e.kind() == std::io::ErrorKind::TimedOut => - { - debug!("No response received for hole punch attempt"); - } - Err(e) => { - debug!("Error receiving hole punch response: {}", e); - } - } - } - Err(e) => { - debug!("Failed to send hole punch packet: {}", e); - } - } - } - - // If we get here, all hole punch attempts failed - Err(NatTraversalError::HolePunchingFailed) - } - - /// Create a minimal QUIC packet with PATH_CHALLENGE frame for hole punching - fn create_path_challenge_packet( - &self, - challenge_data: [u8; 8], - ) -> Result, NatTraversalError> { - // Create a minimal QUIC packet structure - // This is a simplified implementation - in production, you'd use proper QUIC packet construction - let mut packet = Vec::new(); - - // QUIC packet header (simplified) - packet.push(0x40); // Short header, fixed bit set - packet.extend_from_slice(&[0, 0, 0, 1]); // Connection ID (simplified) - - // PATH_CHALLENGE frame - packet.push(0x1a); // PATH_CHALLENGE frame type - packet.extend_from_slice(&challenge_data); // 8-byte challenge data - - Ok(packet) - } - - /// Store successful candidate pair for later connection establishment - fn store_successful_candidate_pair( - &self, - target_addr: SocketAddr, - pair: CandidatePair, - ) -> Result<(), NatTraversalError> { - debug!( - "Storing successful candidate pair for {}: {} <-> {}", - target_addr, pair.local_candidate.address, pair.remote_candidate.address - ); - - // Store the successful remote address for use in connection establishment - // DashMap provides lock-free .insert() - self.successful_candidates - .insert(target_addr, pair.remote_candidate.address); - info!( - "Stored successful candidate for {}: {}", - target_addr, pair.remote_candidate.address - ); - - // Emit events to notify the application - if let Some(ref callback) = self.event_callback { - callback(NatTraversalEvent::PathValidated { - remote_address: target_addr, - rtt: Duration::from_millis(50), // Estimated RTT - }); - - callback(NatTraversalEvent::TraversalSucceeded { - remote_address: target_addr, - final_address: pair.remote_candidate.address, - total_time: Duration::from_secs(1), // Estimated total time - }); - } - - Ok(()) - } - - /// Get the successful candidate address for a target (discovered via hole punching) - /// - /// Returns the remote address that successfully responded during hole punching. - /// This address should be used for establishing the actual QUIC connection. - fn get_successful_candidate_address(&self, target_addr: SocketAddr) -> Option { - // DashMap provides lock-free .get() that returns Option> - self.successful_candidates - .get(&target_addr) - .map(|r| *r.value()) - } + // ───────────────────────────────────────────────────────────────────── + // Removed: the dead `attempt_hole_punching` / + // `attempt_quic_hole_punching` / `get_candidate_pairs_for_addr` / + // `calculate_candidate_pair_priority` / `create_path_challenge_packet` + // / `store_successful_candidate_pair` / `get_successful_candidate_address` + // chain. These were only ever called from the duplicate + // `NatTraversalEndpoint::connect_with_fallback` (also removed above) + // and could not have worked in production: + // + // 1. `attempt_quic_hole_punching` bound a fresh `std::net::UdpSocket` + // to the local candidate address, which always fails on a real + // node because Quinn already owns the port — UDP binds are + // exclusive. + // 2. The "QUIC packet" it sent was a hand-rolled byte sequence + // (`0x40 [0,0,0,1] 0x1a <8 random>`) that is not a valid + // encrypted QUIC packet, so any receiving Quinn endpoint + // silently dropped it. + // 3. The success branch then waited 100 ms on a blocking + // `recv_from` for a "response" that no compliant peer would + // ever send. + // + // Production hole-punch coordination lives in + // `crate::p2p_endpoint::P2pEndpoint::connect_with_fallback_inner`, + // which uses the proper coordinator-mediated PUNCH_ME_NOW flow + // implemented elsewhere in this file. + // ───────────────────────────────────────────────────────────────────── /// Attempt connection to a specific candidate address fn attempt_connection_to_candidate( @@ -6058,10 +5650,13 @@ impl NatTraversalEndpoint { let target_wire_id = target_peer_id.unwrap_or_else(|| Self::wire_id_from_addr(target_addr)); info!( "Sending PUNCH_ME_NOW coordination request for {} to coordinator {} (wire_id={}, from_peer_id={}, from_addr={})", - target_addr, coordinator, + target_addr, + coordinator, hex::encode(&target_wire_id[..8]), - target_peer_id.map(|p| hex::encode(&p[..8])).unwrap_or_else(|| "none".to_string()), - !target_peer_id.is_some(), + target_peer_id + .map(|p| hex::encode(&p[..8])) + .unwrap_or_else(|| "none".to_string()), + target_peer_id.is_none(), ); // Get our external address - this is where the target peer should punch to From 7314d0b373c1352995c16f6d1c7fb5ecf04fb1d3 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Mon, 6 Apr 2026 15:41:13 +0200 Subject: [PATCH 15/43] feat: best-effort UPnP IGD port mapping for NAT traversal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds an optional UPnP IGD client that opportunistically asks the local router to forward the endpoint's UDP port and surfaces the resulting public address as a high-priority candidate alongside locally-discovered and peer-observed candidates. The integration is strictly additive: failure is silent (no router, UPnP disabled, refused mapping, untrusted external IP) and behaviour matches the pre-UPnP build exactly when the gateway is unavailable. The endpoint construction never blocks on the probe — discovery runs in a background task with a 2s deadline. Key design points: * Single-shot probe per session. A router that did not answer SSDP is left alone for the rest of the session — no aggressive re-probing. * Lease renewed at half the lease duration (default 1h, so refresh every 30 minutes). Crash-path safety: 1h lease bounds the worst-case leak. * Gateway-claimed external IPs are validated as plausibly public before use. RFC1918, CGNAT (100.64/10), loopback, link-local, documentation, broadcast and multicast all rejected so a misbehaving router cannot poison candidate discovery. * Endpoint owns the service exclusively; the discovery manager only holds a read-only UpnpStateRx (watch::Receiver). This lets the shutdown path move out the service and call DeletePortMapping directly with a 500ms budget. * PortMapped slots into priority calculation between Host (126) and ServerReflexive (100) at 110, and gets discovery priority 70_000 so it outranks the bound-address promotion (60_000). Drive-by: clippy caught a pre-existing `!is_some()` in the PUNCH_ME_NOW coordination path that was blocking warnings-as-errors after a clippy upgrade — replaced with `.is_none()`. Tests: 1468 lib tests pass with --features upnp. New unit tests cover config defaults, the disabled path, the IP plausibility classifier across IPv4/IPv6 special ranges, and the discovery integration with a pinned UpnpStateRx. Co-Authored-By: Claude Opus 4.6 (1M context) --- Cargo.lock | 107 ++++- Cargo.toml | 15 +- benches/nat_traversal.rs | 2 + src/bin/saorsa-transport.rs | 17 + src/candidate_discovery.rs | 226 +++++++++ src/connection/nat_traversal.rs | 9 + src/endpoint.rs | 4 +- src/lib.rs | 6 + src/nat_traversal_api.rs | 94 +++- src/p2p_endpoint.rs | 5 +- src/unified_config.rs | 8 + src/upnp.rs | 708 +++++++++++++++++++++++++++++ tests/relay_queue_tests.rs | 5 + tests/security_regression_tests.rs | 9 + 14 files changed, 1194 insertions(+), 21 deletions(-) create mode 100644 src/upnp.rs diff --git a/Cargo.lock b/Cargo.lock index 5de40c3e..6ce39384 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -20,7 +20,7 @@ checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", "cipher", - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] @@ -209,6 +209,18 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "attohttpc" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e2cdb6d5ed835199484bb92bb8b3edd526effe995c61732580439c1a67e2e9" +dependencies = [ + "base64", + "http", + "log", + "url", +] + [[package]] name = "autocfg" version = "1.5.0" @@ -287,7 +299,7 @@ dependencies = [ "cc", "cfg-if", "constant_time_eq", - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] @@ -433,7 +445,18 @@ checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" dependencies = [ "cfg-if", "cipher", - "cpufeatures", + "cpufeatures 0.2.17", +] + +[[package]] +name = "chacha20" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.0", ] [[package]] @@ -443,7 +466,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" dependencies = [ "aead", - "chacha20", + "chacha20 0.9.1", "cipher", "poly1305", "zeroize", @@ -622,6 +645,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "criterion" version = "0.5.1" @@ -734,7 +766,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "curve25519-dalek-derive", "digest", "fiat-crypto", @@ -1288,6 +1320,25 @@ dependencies = [ "subtle", ] +[[package]] +name = "h2" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "half" version = "2.7.1" @@ -1455,6 +1506,7 @@ dependencies = [ "bytes", "futures-channel", "futures-core", + "h2", "http", "http-body", "httparse", @@ -1643,6 +1695,26 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "igd-next" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bac9a3c8278f43b4cd8463380f4a25653ac843e5b177e1d3eaf849cc9ba10d4d" +dependencies = [ + "attohttpc", + "bytes", + "futures", + "http", + "http-body-util", + "hyper", + "hyper-util", + "log", + "rand 0.10.0", + "tokio", + "url", + "xmltree", +] + [[package]] name = "indexmap" version = "2.13.0" @@ -1822,7 +1894,7 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb26cec98cce3a3d96cbb7bced3c4b16e3d13f27ec56dbd62cbc8f39cfb9d653" dependencies = [ - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] @@ -2244,7 +2316,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" dependencies = [ - "cpufeatures", + "cpufeatures 0.2.17", "opaque-debug", "universal-hash", ] @@ -2256,7 +2328,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "opaque-debug", "universal-hash", ] @@ -2514,6 +2586,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc266eb313df6c5c09c1c7b1fbe2510961e5bcd3add930c1e31f7ed9da0feff8" dependencies = [ + "chacha20 0.10.0", "getrandom 0.4.2", "rand_core 0.10.0", ] @@ -2953,6 +3026,7 @@ dependencies = [ "futures-util", "hex", "hex-literal", + "igd-next", "indexmap", "keyring", "lazy_static", @@ -3180,7 +3254,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -4602,6 +4676,21 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8aa498d22c9bbaf482329839bc5620c46be275a19a812e9a22a2b07529a642a" +[[package]] +name = "xml-rs" +version = "0.8.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae8337f8a065cfc972643663ea4279e04e7256de865aa66fe25cec5fb912d3f" + +[[package]] +name = "xmltree" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7d8a75eaf6557bb84a65ace8609883db44a29951042ada9b393151532e41fcb" +dependencies = [ + "xml-rs", +] + [[package]] name = "yasna" version = "0.5.2" diff --git a/Cargo.toml b/Cargo.toml index 066235b0..75326e6a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,11 +39,18 @@ exclude = [ [features] # Default features include essential functionality with 100% PQC support # v0.15.0: Simplified feature flags - crypto is always enabled -default = ["platform-verifier", "network-discovery"] +default = ["platform-verifier", "network-discovery", "upnp"] # Platform-specific certificate verification platform-verifier = ["dep:rustls-platform-verifier"] +# UPnP IGD port mapping for best-effort NAT traversal assistance. +# When enabled, the endpoint will opportunistically request a UDP port +# mapping from a local Internet Gateway Device. Failure is silent and +# non-fatal — the endpoint behaves identically to a non-UPnP build when +# no gateway is available. +upnp = ["dep:igd-next"] + # Configure `tracing` to log events via `log` if no `tracing` subscriber exists log = ["tracing/log"] @@ -113,6 +120,12 @@ rustls-post-quantum = { version = "0.2", features = ["aws-lc-rs-unstable"] } socket2 = { version = "0.5", optional = true } nix = { version = "0.29", features = ["resource", "net"], optional = true } +# UPnP IGD port mapping (optional) +# Used by the `upnp` feature for best-effort UDP port mapping. The +# implementation never blocks startup and silently degrades when the +# router does not support or has disabled UPnP IGD. +igd-next = { version = "0.17", default-features = false, features = ["aio_tokio"], optional = true } + # BLE transport dependencies (cross-platform, optional) # btleplug supports Linux (BlueZ), macOS (Core Bluetooth), and Windows (WinRT) btleplug = { version = "0.11", optional = true } diff --git a/benches/nat_traversal.rs b/benches/nat_traversal.rs index bdc573e4..f841dad0 100644 --- a/benches/nat_traversal.rs +++ b/benches/nat_traversal.rs @@ -565,6 +565,7 @@ fn bench_pair_generation(c: &mut Criterion) { CandidateSource::Observed { .. } => 1, CandidateSource::Peer => 2, CandidateSource::Predicted => 3, + CandidateSource::PortMapped => 4, }; for remote in &remote_candidates { @@ -585,6 +586,7 @@ fn bench_pair_generation(c: &mut Criterion) { CandidateSource::Observed { .. } => 1, CandidateSource::Peer => 2, CandidateSource::Predicted => 3, + CandidateSource::PortMapped => 4, }; // Calculate priority diff --git a/src/bin/saorsa-transport.rs b/src/bin/saorsa-transport.rs index c5be2293..f29038d5 100644 --- a/src/bin/saorsa-transport.rs +++ b/src/bin/saorsa-transport.rs @@ -170,6 +170,14 @@ struct Args { /// Chunk size for data generation/verification (bytes) #[arg(long, default_value = "65536")] chunk_size: usize, + + /// Disable best-effort UPnP IGD port mapping. By default the endpoint + /// asks the local router to forward its UDP port — pass this flag to + /// skip the UPnP probe entirely (useful when the router is known to + /// be hostile or when running on infrastructure that does not need + /// it). NAT traversal still works without UPnP via hole punching. + #[arg(long)] + no_upnp: bool, } /// CLI subcommands @@ -371,6 +379,15 @@ async fn main() -> anyhow::Result<()> { } // v0.13.0: No mode-based NAT config - all nodes are symmetric + if args.no_upnp { + let nat = saorsa_transport::unified_config::NatConfig { + upnp: saorsa_transport::upnp::UpnpConfig::disabled(), + ..saorsa_transport::unified_config::NatConfig::default() + }; + builder = builder.nat(nat); + info!("UPnP IGD port mapping disabled (--no-upnp)"); + } + let config = builder.build()?; // Create endpoint diff --git a/src/candidate_discovery.rs b/src/candidate_discovery.rs index 448904fc..393e3d2c 100644 --- a/src/candidate_discovery.rs +++ b/src/candidate_discovery.rs @@ -28,6 +28,14 @@ use crate::{ nat_traversal_api::{BootstrapNode, CandidateAddress}, }; +/// Discovery-side priority assigned to UPnP port-mapped candidates. +/// +/// Slotted strictly above the bound-address promotion (`60_000`) so that +/// a router-confirmed public mapping always outranks any host-side +/// candidate during pairing. The constant lives here so the priority +/// scale stays in one file alongside the other discovery priorities. +const PORT_MAPPED_DISCOVERY_PRIORITY: u32 = 70_000; + /// Session identifier for the candidate discovery manager. /// /// Replaces the legacy `PeerId` key. Each discovery session is either for @@ -66,6 +74,7 @@ fn convert_to_nat_source(discovery_source: DiscoverySourceType) -> CandidateSour DiscoverySourceType::Local => CandidateSource::Local, DiscoverySourceType::ServerReflexive => CandidateSource::Observed { by_node: None }, DiscoverySourceType::Predicted => CandidateSource::Predicted, + DiscoverySourceType::PortMapped => CandidateSource::PortMapped, } } @@ -92,6 +101,14 @@ pub enum DiscoverySourceType { /// These are algorithmically predicted addresses that might work based on /// observed NAT traversal patterns and port prediction algorithms. Predicted, + + /// Public address obtained from a router-side port mapping (UPnP IGD). + /// + /// The gateway has explicitly committed to forwarding the mapped port to + /// our local socket for the lease duration, so these candidates are + /// strictly more reliable than [`Self::ServerReflexive`] addresses + /// observed via peer reports. + PortMapped, } /// IPv6 address type classification for priority calculation. @@ -211,6 +228,17 @@ pub struct CandidateDiscoveryManager { active_sessions: HashMap, /// Cached local interface results (shared across all sessions) cached_local_candidates: Option<(Instant, Vec)>, + /// Optional read-only handle to the UPnP mapping service. When set, + /// the current mapping is surfaced as a high-priority candidate + /// during the local-scanning phase. The handle is purely additive — + /// when absent or in [`crate::upnp::UpnpState::Unavailable`], + /// discovery proceeds exactly as it would in a non-UPnP build. + /// + /// This is a `UpnpStateRx` rather than `Arc` so + /// the discovery manager only borrows the state, leaving the + /// `NatTraversalEndpoint` as the sole owner of the service for + /// graceful shutdown. + upnp: Option, } /// Configuration for candidate discovery behavior @@ -680,9 +708,21 @@ impl CandidateDiscoveryManager { interface_discovery, active_sessions: HashMap::new(), cached_local_candidates: None, + upnp: None, } } + /// Attach a read-only handle to the UPnP mapping service whose current + /// state should be surfaced as a discovery candidate during local + /// scanning. + /// + /// Calling this is optional and best-effort — if the handle never + /// reaches [`crate::upnp::UpnpState::Mapped`], discovery behaves + /// identically to a manager without UPnP attached. + pub fn set_upnp_state_rx(&mut self, state_rx: crate::upnp::UpnpStateRx) { + self.upnp = Some(state_rx); + } + /// Set the actual bound address of the local endpoint pub fn set_bound_address(&mut self, address: SocketAddr) { self.config.bound_address = Some(address); @@ -690,6 +730,58 @@ impl CandidateDiscoveryManager { self.cached_local_candidates = None; } + /// Snapshot the UPnP mapping (if any) as a [`DiscoveryCandidate`]. + /// + /// Returns `None` when no service is attached, when the service is + /// still probing, or when it has reached the sticky `Unavailable` + /// state. The peek is a single atomic load on the underlying watch + /// channel and is cheap to call from the discovery hot path. + fn upnp_candidate(&self) -> Option { + let state_rx = self.upnp.as_ref()?; + match state_rx.current() { + crate::upnp::UpnpState::Mapped { external, .. } => Some(DiscoveryCandidate { + address: external, + priority: PORT_MAPPED_DISCOVERY_PRIORITY, + source: DiscoverySourceType::PortMapped, + state: CandidateState::New, + }), + crate::upnp::UpnpState::Probing | crate::upnp::UpnpState::Unavailable => None, + } + } + + /// Idempotently push the current UPnP candidate (if any) into `session`, + /// emitting a `LocalCandidateDiscovered` event the first time it appears. + /// + /// Safe to call repeatedly from the same `poll()` invocation — duplicate + /// candidates with the same external address are detected and skipped, + /// matching the dedup discipline used for bound-address promotion. + fn try_publish_upnp_candidate( + upnp_candidate: Option<&DiscoveryCandidate>, + session: &mut DiscoverySession, + events: &mut Vec, + ) -> bool { + let Some(candidate) = upnp_candidate else { + return false; + }; + let already_present = session + .discovered_candidates + .iter() + .any(|existing| existing.address == candidate.address); + if already_present { + return false; + } + session.discovered_candidates.push(candidate.clone()); + session.statistics.local_candidates_found += 1; + events.push(DiscoveryEvent::LocalCandidateDiscovered { + candidate: candidate.to_candidate_address(), + }); + debug!( + "Added UPnP-mapped public address {} as PortMapped candidate", + candidate.address + ); + true + } + /// Discover local network interface candidates synchronously pub fn discover_local_candidates(&mut self) -> Result, DiscoveryError> { // Start interface scan @@ -816,6 +908,12 @@ impl CandidateDiscoveryManager { } }); + // Snapshot the current UPnP mapping (if any) once per poll — + // we will publish it to the session below alongside the + // bound address. Computed before any session borrows so the + // borrow checker is happy. + let upnp_candidate = self.upnp_candidate(); + if let Some(bound_addr) = bound_candidate { if let Some(session) = self.active_sessions.get_mut(&session_id) { let already_present = session @@ -841,6 +939,12 @@ impl CandidateDiscoveryManager { bound_addr, session_id ); } + + Self::try_publish_upnp_candidate( + upnp_candidate.as_ref(), + session, + &mut all_events, + ); } } @@ -896,6 +1000,23 @@ impl CandidateDiscoveryManager { } } + // Surface the UPnP mapping (if any) at scan completion. + // Re-snapshot here because the mapping may have become + // available between the early-promotion site above and + // this point. The new snapshot lives in a local because + // `try_publish_upnp_candidate` cannot borrow `self` + // while we hold a mutable session reference. + let upnp_candidate_now = self.upnp_candidate(); + if let Some(session) = self.active_sessions.get_mut(&session_id) { + if Self::try_publish_upnp_candidate( + upnp_candidate_now.as_ref(), + session, + &mut all_events, + ) { + candidates_added += 1; + } + } + // Process discovered interfaces // Get the bound port to use for interface addresses (they come with port 0) let bound_port = self.config.bound_address.map(|a| a.port()).unwrap_or(9000); @@ -1008,6 +1129,7 @@ impl CandidateDiscoveryManager { } }); + let upnp_candidate_now = self.upnp_candidate(); if let Some(session) = self.active_sessions.get_mut(&session_id) { if let Some(bound_addr) = bound_candidate { let already_present = session @@ -1035,6 +1157,12 @@ impl CandidateDiscoveryManager { } } + Self::try_publish_upnp_candidate( + upnp_candidate_now.as_ref(), + session, + &mut all_events, + ); + let final_candidates: Vec = session .discovered_candidates .iter() @@ -1632,6 +1760,7 @@ pub mod test_utils { #[cfg(test)] mod tests { use super::*; + use crate::upnp::{UpnpState, UpnpStateRx}; fn create_test_manager() -> CandidateDiscoveryManager { CandidateDiscoveryManager::new(DiscoveryConfig::test_default()) @@ -2453,4 +2582,101 @@ mod tests { Err(CandidateValidationError::IPv4MappedAddress) )); } + + #[test] + fn upnp_mapped_state_surfaces_port_mapped_candidate() { + let mut manager = create_test_manager(); + let session_id = test_session_id(); + + // Pin the UPnP state to Mapped. The address must look public to + // pass downstream candidate validation; 1.1.1.1 is outside every + // reserved range. + let external: SocketAddr = "1.1.1.1:42000".parse().unwrap(); + manager.set_upnp_state_rx(UpnpStateRx::for_test(UpnpState::Mapped { + external, + lease_expires_at: Instant::now() + Duration::from_secs(3600), + })); + + manager + .start_discovery(session_id, vec![]) + .expect("start_discovery should succeed in test"); + + // Drive the local-scanning poll loop until the session reaches + // Completed or we exhaust the test budget. The poll path adds + // both the bound address and the UPnP candidate, then transitions + // the session to Completed once the local interface scan finishes. + let mut events = Vec::new(); + for _ in 0..50 { + events.extend(manager.poll(Instant::now())); + let phase = manager + .active_sessions + .get(&session_id) + .map(|s| s.current_phase.clone()); + if matches!(phase, Some(DiscoveryPhase::Completed { .. })) { + break; + } + std::thread::sleep(Duration::from_millis(20)); + } + + let session = manager + .active_sessions + .get(&session_id) + .expect("session should still exist after polling"); + let port_mapped: Vec<_> = session + .discovered_candidates + .iter() + .filter(|c| matches!(c.source, DiscoverySourceType::PortMapped)) + .collect(); + assert_eq!( + port_mapped.len(), + 1, + "exactly one PortMapped candidate should be surfaced, got {port_mapped:?}", + ); + assert_eq!(port_mapped[0].address, external); + assert_eq!( + port_mapped[0].priority, PORT_MAPPED_DISCOVERY_PRIORITY, + "PortMapped candidate should use the documented priority slot" + ); + + let saw_event = events.iter().any(|e| { + matches!( + e, + DiscoveryEvent::LocalCandidateDiscovered { candidate } + if candidate.address == external + ) + }); + assert!( + saw_event, + "LocalCandidateDiscovered event should be emitted for the UPnP mapping" + ); + } + + #[test] + fn upnp_unavailable_state_does_not_add_candidate() { + let mut manager = create_test_manager(); + let session_id = test_session_id(); + manager.set_upnp_state_rx(UpnpStateRx::for_test(UpnpState::Unavailable)); + + manager + .start_discovery(session_id, vec![]) + .expect("start_discovery should succeed in test"); + + for _ in 0..20 { + manager.poll(Instant::now()); + std::thread::sleep(Duration::from_millis(20)); + } + + let session = manager + .active_sessions + .get(&session_id) + .expect("session should still exist"); + let any_port_mapped = session + .discovered_candidates + .iter() + .any(|c| matches!(c.source, DiscoverySourceType::PortMapped)); + assert!( + !any_port_mapped, + "Unavailable UPnP state must not contribute candidates" + ); + } } diff --git a/src/connection/nat_traversal.rs b/src/connection/nat_traversal.rs index ef9fb456..5067ed10 100644 --- a/src/connection/nat_traversal.rs +++ b/src/connection/nat_traversal.rs @@ -95,6 +95,11 @@ pub enum CandidateSource { Peer, /// Generated prediction for symmetric NAT Predicted, + /// Public address obtained via a router-side port mapping + /// (e.g. UPnP IGD AddPortMapping). Treated like a server-reflexive + /// candidate but with higher confidence because the gateway has + /// explicitly committed to forwarding the port for the lease duration. + PortMapped, } /// Current state of a candidate address #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -296,6 +301,10 @@ fn classify_candidate_type(source: CandidateSource) -> CandidateType { CandidateSource::Observed { .. } => CandidateType::ServerReflexive, CandidateSource::Peer => CandidateType::PeerReflexive, CandidateSource::Predicted => CandidateType::ServerReflexive, // Symmetric NAT prediction + // Port-mapped candidates are reflexive — they describe our public + // address as the gateway sees it, just with a deterministic guarantee + // that the gateway will forward traffic for the lease duration. + CandidateSource::PortMapped => CandidateType::ServerReflexive, } } /// Determine pair type from individual candidate types diff --git a/src/endpoint.rs b/src/endpoint.rs index 1291d4f2..99cf2417 100644 --- a/src/endpoint.rs +++ b/src/endpoint.rs @@ -385,9 +385,7 @@ impl Endpoint { .connections .iter() .filter_map(|(_, meta)| { - meta.peer_id - .as_ref() - .map(|pid| hex::encode(&pid.0[..8])) + meta.peer_id.as_ref().map(|pid| hex::encode(&pid.0[..8])) }) .collect(); tracing::warn!( diff --git a/src/lib.rs b/src/lib.rs index 7415fa8f..3f98961a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -197,6 +197,12 @@ mod token; mod token_memory_cache; /// Zero-cost tracing and event logging system pub mod tracing; +/// Best-effort UPnP IGD port mapping for NAT traversal assistance. +/// +/// This module is feature-gated behind `upnp` (enabled by default). When +/// disabled, [`UpnpMappingService`] is still present but is a no-op stub +/// that always reports [`UpnpState::Unavailable`]. +pub mod upnp; // Public modules with new structure /// Constrained protocol engine for low-bandwidth transports (BLE, LoRa) diff --git a/src/nat_traversal_api.rs b/src/nat_traversal_api.rs index e5152058..d4f0f8d2 100644 --- a/src/nat_traversal_api.rs +++ b/src/nat_traversal_api.rs @@ -346,6 +346,14 @@ pub struct NatTraversalEndpoint { /// Tracks when each connection was first observed as closed. /// Used to enforce a grace period before removing dead connections. closed_at: dashmap::DashMap, + /// Best-effort UPnP IGD port mapping service. + /// + /// The endpoint is the sole owner of the service — the discovery + /// manager only holds a [`crate::upnp::UpnpStateRx`] read handle — + /// so [`Self::shutdown`] can `take()` the service and call + /// [`crate::upnp::UpnpMappingService::shutdown`] for graceful + /// teardown including the gateway-side `DeletePortMapping` request. + upnp_service: parking_lot::Mutex>, } /// Configuration for NAT traversal behavior @@ -487,6 +495,18 @@ pub struct NatTraversalConfig { /// Default: `false` #[serde(default)] pub allow_loopback: bool, + + /// Best-effort UPnP IGD port mapping configuration. + /// + /// When enabled, the endpoint asks the local Internet Gateway Device + /// (UPnP-capable router) to forward its UDP port. The mapping is + /// surfaced as a high-priority NAT traversal candidate when the + /// gateway cooperates, and silently degrades to a no-op when the + /// gateway is absent, has UPnP disabled, or refuses the request. + /// + /// Default: enabled with a one-hour lease. + #[serde(default)] + pub upnp: crate::upnp::UpnpConfig, } fn default_max_message_size() -> usize { @@ -1100,6 +1120,7 @@ impl Default for NatTraversalConfig { transport_registry: None, // Use direct UDP binding by default max_message_size: crate::unified_config::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: false, + upnp: crate::upnp::UpnpConfig::default(), } } } @@ -1297,11 +1318,25 @@ impl NatTraversalEndpoint { let (inner_endpoint, event_tx, event_rx, local_addr, relay_server_config) = Self::create_inner_endpoint(&config, token_store, registry_ref, None).await?; - // Update discovery manager with the actual bound address + // Spawn the best-effort UPnP service against the actual bound port + // before installing the read handle on the discovery manager. The + // service starts a background task that probes the local IGD + // gateway and never blocks endpoint construction — failure + // transitions to `Unavailable` and is invisible to the rest of + // the endpoint. The endpoint owns the service exclusively so + // shutdown can reclaim it for graceful unmap. + let upnp_service = + crate::upnp::UpnpMappingService::start(local_addr.port(), config.upnp.clone()); + let upnp_state_rx = upnp_service.subscribe(); + + // Update discovery manager with the actual bound address and + // attach the UPnP read handle so port-mapped candidates flow + // through local-phase scans. { // parking_lot::Mutex doesn't poison - no need for map_err let mut discovery = discovery_manager.lock(); discovery.set_bound_address(local_addr); + discovery.set_upnp_state_rx(upnp_state_rx); info!( "Updated discovery manager with bound address: {}", local_addr @@ -1411,6 +1446,7 @@ impl NatTraversalEndpoint { handshake_tx: hs_tx, handshake_rx: TokioMutex::new(hs_rx), closed_at: dashmap::DashMap::new(), + upnp_service: parking_lot::Mutex::new(Some(upnp_service)), }; // Multi-transport listening: Spawn receive tasks for all online transports @@ -1707,11 +1743,25 @@ impl NatTraversalEndpoint { let (inner_endpoint, event_tx, event_rx, local_addr, relay_server_config) = Self::create_inner_endpoint(&config, token_store, registry_ref, quinn_socket).await?; - // Update discovery manager with the actual bound address + // Spawn the best-effort UPnP service against the actual bound port + // before installing the read handle on the discovery manager. The + // service starts a background task that probes the local IGD + // gateway and never blocks endpoint construction — failure + // transitions to `Unavailable` and is invisible to the rest of + // the endpoint. The endpoint owns the service exclusively so + // shutdown can reclaim it for graceful unmap. + let upnp_service = + crate::upnp::UpnpMappingService::start(local_addr.port(), config.upnp.clone()); + let upnp_state_rx = upnp_service.subscribe(); + + // Update discovery manager with the actual bound address and + // attach the UPnP read handle so port-mapped candidates flow + // through local-phase scans. { // parking_lot::Mutex doesn't poison - no need for map_err let mut discovery = discovery_manager.lock(); discovery.set_bound_address(local_addr); + discovery.set_upnp_state_rx(upnp_state_rx); info!( "Updated discovery manager with bound address: {}", local_addr @@ -1821,6 +1871,7 @@ impl NatTraversalEndpoint { handshake_tx: hs_tx, handshake_rx: TokioMutex::new(hs_rx), closed_at: dashmap::DashMap::new(), + upnp_service: parking_lot::Mutex::new(Some(upnp_service)), }; // Multi-transport listening: Spawn receive tasks for all online transports @@ -4635,6 +4686,17 @@ impl NatTraversalEndpoint { self.incoming_notify.notify_waiters(); self.shutdown_notify.notify_waiters(); + // Best-effort UPnP teardown. The endpoint is the sole owner of + // the service (the discovery manager only holds a read-only + // `UpnpStateRx`), so we can move it out and call its async + // shutdown directly. Failures are swallowed inside the service — + // the lease is the ultimate safety net. The mutex guard is + // dropped before the await so the resulting future stays `Send`. + let upnp_service = self.upnp_service.lock().take(); + if let Some(service) = upnp_service { + service.shutdown().await; + } + // Close all active connections // DashMap: collect addresses then remove them one by one let addrs: Vec = self.connections.iter().map(|e| *e.key()).collect(); @@ -4863,13 +4925,19 @@ impl NatTraversalEndpoint { // Create candidate pairs with priorities (ICE-like pairing) let mut candidate_pairs = Vec::new(); + // Both `Local` and `PortMapped` describe the local endpoint's + // reachability — the former from a host interface, the latter from + // a router-side port mapping. Treat both as the local side. + let is_local_side = |source: CandidateSource| { + matches!(source, CandidateSource::Local | CandidateSource::PortMapped) + }; let local_candidates = discovery_candidates .iter() - .filter(|c| matches!(c.source, CandidateSource::Local)) + .filter(|c| is_local_side(c.source)) .collect::>(); let remote_candidates = discovery_candidates .iter() - .filter(|c| !matches!(c.source, CandidateSource::Local)) + .filter(|c| !is_local_side(c.source)) .collect::>(); // Pair each local candidate with each remote candidate @@ -4916,8 +4984,16 @@ impl NatTraversalEndpoint { // ICE candidate pair priority formula: min(G,D) * 2^32 + max(G,D) * 2 + (G>D ? 1 : 0) // Where G is controlling agent priority, D is controlled agent priority + // ICE-style type preference for router-guaranteed port mappings. + // Slotted between ServerReflexive (100) and Host (126): port-mapped + // addresses are reflexive (the gateway sees us through NAT) but + // come with an explicit forwarding commitment for the lease + // duration, so they outrank ordinary OBSERVED_ADDRESS reports. + const PORT_MAPPED_TYPE_PREFERENCE: u32 = 110; + let local_type_preference = match local.source { CandidateSource::Local => 126, + CandidateSource::PortMapped => PORT_MAPPED_TYPE_PREFERENCE, CandidateSource::Observed { .. } => 100, CandidateSource::Predicted => 75, CandidateSource::Peer => 50, @@ -4925,6 +5001,7 @@ impl NatTraversalEndpoint { let remote_type_preference = match remote.source { CandidateSource::Local => 126, + CandidateSource::PortMapped => PORT_MAPPED_TYPE_PREFERENCE, CandidateSource::Observed { .. } => 100, CandidateSource::Predicted => 75, CandidateSource::Peer => 50, @@ -6058,10 +6135,13 @@ impl NatTraversalEndpoint { let target_wire_id = target_peer_id.unwrap_or_else(|| Self::wire_id_from_addr(target_addr)); info!( "Sending PUNCH_ME_NOW coordination request for {} to coordinator {} (wire_id={}, from_peer_id={}, from_addr={})", - target_addr, coordinator, + target_addr, + coordinator, hex::encode(&target_wire_id[..8]), - target_peer_id.map(|p| hex::encode(&p[..8])).unwrap_or_else(|| "none".to_string()), - !target_peer_id.is_some(), + target_peer_id + .map(|p| hex::encode(&p[..8])) + .unwrap_or_else(|| "none".to_string()), + target_peer_id.is_none(), ); // Get our external address - this is where the target peer should punch to diff --git a/src/p2p_endpoint.rs b/src/p2p_endpoint.rs index a3d7afc9..9c45749b 100644 --- a/src/p2p_endpoint.rs +++ b/src/p2p_endpoint.rs @@ -1822,7 +1822,10 @@ impl P2pEndpoint { if let Some(ref pid) = target_peer_id { info!( "try_hole_punch: calling initiate_nat_traversal({}, {}) with peer ID {} (dashmap key={})", - target, coordinator, hex::encode(&pid[..8]), target + target, + coordinator, + hex::encode(&pid[..8]), + target ); } else { info!( diff --git a/src/unified_config.rs b/src/unified_config.rs index 63f9b7c2..3886b737 100644 --- a/src/unified_config.rs +++ b/src/unified_config.rs @@ -144,6 +144,12 @@ pub struct NatConfig { /// /// Default: `false` pub allow_loopback: bool, + + /// Best-effort UPnP IGD port mapping configuration. When enabled + /// (default), the endpoint asks the local router to forward its UDP + /// port and surfaces the resulting public address as a high-priority + /// NAT traversal candidate. Failure is silent and non-fatal. + pub upnp: crate::upnp::UpnpConfig, } impl Default for NatConfig { @@ -157,6 +163,7 @@ impl Default for NatConfig { max_concurrent_attempts: 3, prefer_rfc_nat_traversal: true, allow_loopback: false, + upnp: crate::upnp::UpnpConfig::default(), } } } @@ -310,6 +317,7 @@ impl P2pConfig { transport_registry: Some(Arc::new(self.transport_registry.clone())), max_message_size: self.max_message_size, allow_loopback: self.nat.allow_loopback, + upnp: self.nat.upnp.clone(), } } diff --git a/src/upnp.rs b/src/upnp.rs new file mode 100644 index 00000000..5f93d54b --- /dev/null +++ b/src/upnp.rs @@ -0,0 +1,708 @@ +// Copyright 2024 Saorsa Labs Ltd. +// +// This Saorsa Network Software is licensed under the General Public License (GPL), version 3. +// Please see the file LICENSE-GPL, or visit for the full text. + +//! Best-effort UPnP IGD port mapping. +//! +//! This module asks the local Internet Gateway Device (typically a home +//! router) to forward a single UDP port to our endpoint. When successful, +//! the gateway provides a deterministic public `ip:port` reachable from +//! the open internet, which is then surfaced as a high-priority NAT +//! traversal candidate alongside locally-discovered and peer-observed +//! addresses. +//! +//! # Best-effort contract +//! +//! Everything in this module is **strictly additive**. The endpoint must +//! behave identically to a non-UPnP build when the gateway: +//! +//! * does not exist (no router on the LAN, or it does not speak SSDP), +//! * has UPnP IGD disabled in its administrative settings, +//! * supports UPnP but refuses the mapping request, +//! * accepts the request but later forgets it / reboots / changes IPs. +//! +//! Concretely this means: +//! +//! 1. [`UpnpMappingService::start`] never returns an error and never blocks +//! on network I/O — it spawns a background task and returns immediately. +//! 2. All failures are swallowed and logged at `debug` level. The only +//! `info` log line is the success path. +//! 3. Discovery is single-shot per service lifetime. A router that did not +//! answer once is left alone for the rest of the session — there is no +//! periodic re-probe. +//! 4. The lease is finite (one hour by default), so a crashed process +//! cannot leak a permanent mapping on the gateway. +//! +//! Callers consume the service by polling [`UpnpMappingService::current`] +//! when they want the most recent state. The poll is a lock-free atomic +//! load on the underlying `tokio::sync::watch` channel, so it is cheap to +//! call from the candidate discovery hot path. + +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use serde::{Deserialize, Serialize}; +use tokio::sync::watch; +use tokio::task::JoinHandle; +#[cfg(feature = "upnp")] +use tracing::{debug, info, warn}; + +/// Default lease duration requested from the gateway. +/// +/// One hour balances two concerns: short enough that a crashed process +/// cannot leak a permanent mapping on the router, long enough that the +/// refresh task does not generate noticeable network churn. +const DEFAULT_LEASE: Duration = Duration::from_secs(3600); + +/// Default budget for the initial gateway discovery probe. +/// +/// SSDP M-SEARCH multicasts and waits for responses; without a hard +/// deadline a non-UPnP LAN would force the background task to wait the +/// full SSDP timeout (~10s) before giving up. Two seconds is enough for +/// any cooperating gateway on the same broadcast domain. +const DEFAULT_DISCOVERY_TIMEOUT: Duration = Duration::from_secs(2); + +/// Best-effort budget for the cleanup `DeletePortMapping` request issued +/// during graceful shutdown. The lease is the ultimate safety net, so +/// blocking shutdown waiting for an unresponsive router would be wrong. +#[cfg(feature = "upnp")] +const SHUTDOWN_UNMAP_BUDGET: Duration = Duration::from_millis(500); + +/// Configuration for [`UpnpMappingService`]. +/// +/// Defaults are tuned for the common case (residential broadband + a +/// consumer router) and should rarely need to be overridden. Use +/// [`UpnpConfig::disabled`] to explicitly opt out at runtime. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct UpnpConfig { + /// Master switch. When `false`, [`UpnpMappingService::start`] returns + /// a service that is permanently in [`UpnpState::Unavailable`] and + /// performs no network I/O. + pub enabled: bool, + + /// Lease duration to request from the gateway. The refresh task will + /// renew at half this interval. + #[serde(with = "duration_secs")] + pub lease_duration: Duration, + + /// Maximum time to wait for the initial gateway discovery probe. + /// After this deadline elapses with no gateway response, the service + /// transitions to [`UpnpState::Unavailable`] and stops trying. + #[serde(with = "duration_millis")] + pub discovery_timeout: Duration, +} + +impl Default for UpnpConfig { + fn default() -> Self { + Self { + enabled: true, + lease_duration: DEFAULT_LEASE, + discovery_timeout: DEFAULT_DISCOVERY_TIMEOUT, + } + } +} + +impl UpnpConfig { + /// Construct a configuration that permanently disables UPnP. + pub const fn disabled() -> Self { + Self { + enabled: false, + lease_duration: DEFAULT_LEASE, + discovery_timeout: DEFAULT_DISCOVERY_TIMEOUT, + } + } +} + +/// Snapshot of the UPnP mapping state at a point in time. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum UpnpState { + /// Initial discovery is still in flight or has not yet started. + Probing, + /// No usable gateway is available for this session. This is a sticky + /// state — once entered, the service stays here until shut down. + /// Reached when SSDP discovery times out, the gateway refuses the + /// mapping, returns a non-public external IP, or otherwise fails. + Unavailable, + /// Gateway is forwarding `external` to our local UDP port. + Mapped { + /// Public address that remote peers can dial to reach this + /// endpoint via the gateway-managed mapping. + external: SocketAddr, + /// Wall-clock instant at which the current lease expires. The + /// background refresh task renews the lease before this point; + /// callers should treat the value as informational. + lease_expires_at: Instant, + }, +} + +/// Background service that maintains a single UDP UPnP mapping for the +/// endpoint's local port. +/// +/// Construct with [`UpnpMappingService::start`]. Read state with +/// [`UpnpMappingService::current`] or hand a [`UpnpStateRx`] to consumers +/// via [`UpnpMappingService::subscribe`]. Tear down with +/// [`UpnpMappingService::shutdown`] (the implementation also has a +/// best-effort `Drop` fallback for the panic path). +pub struct UpnpMappingService { + state: watch::Receiver, + inner: Arc, +} + +/// Read-only handle to the current [`UpnpState`]. +/// +/// Cloneable, lock-free, and decoupled from service ownership: callers +/// that only need to observe the mapping (for example, the candidate +/// discovery manager) take a `UpnpStateRx` instead of an +/// `Arc`, leaving the endpoint as the sole owner of +/// the service so graceful shutdown can reclaim and unmap it. +#[derive(Clone)] +pub struct UpnpStateRx { + inner: watch::Receiver, +} + +impl UpnpStateRx { + /// Lock-free snapshot of the most recent state. + pub fn current(&self) -> UpnpState { + self.inner.borrow().clone() + } + + /// Test-only constructor that pins the receiver to a fixed state. + #[cfg(test)] + pub(crate) fn for_test(state: UpnpState) -> Self { + let (_tx, rx) = watch::channel(state); + Self { inner: rx } + } +} + +struct ServiceInner { + shutdown: tokio::sync::Notify, + /// Once the background task observes the shutdown notification it + /// stores the active mapping (if any) here so [`UpnpMappingService::shutdown`] + /// can issue the final `DeletePortMapping` from the caller's task. + /// We deliberately keep the cleanup off the background task so that + /// dropping the runtime in tests does not block on the unmap RPC. + last_mapping: parking_lot::Mutex>, + handle: parking_lot::Mutex>>, +} + +#[derive(Debug, Clone)] +#[cfg_attr(not(feature = "upnp"), allow(dead_code))] +struct ActiveMapping { + external_port: u16, + gateway: GatewayHandle, +} + +impl UpnpMappingService { + /// Spawn the UPnP service for `local_udp_port`. + /// + /// This is infallible by design — even when UPnP is unsupported on + /// the host, this returns a service stuck in [`UpnpState::Unavailable`]. + /// The returned service starts in [`UpnpState::Probing`] when enabled. + pub fn start(local_udp_port: u16, config: UpnpConfig) -> Self { + let (tx, rx) = watch::channel(UpnpState::Probing); + let inner = Arc::new(ServiceInner { + shutdown: tokio::sync::Notify::new(), + last_mapping: parking_lot::Mutex::new(None), + handle: parking_lot::Mutex::new(None), + }); + + if !config.enabled { + // Permanently unavailable — never touches the network. + let _ = tx.send(UpnpState::Unavailable); + return Self { state: rx, inner }; + } + + let handle = spawn_background_task(local_udp_port, config, tx, Arc::clone(&inner)); + *inner.handle.lock() = handle; + Self { state: rx, inner } + } + + /// Lock-free snapshot of the most recent state. + /// + /// Cheap enough to call from a discovery hot path on every poll. + pub fn current(&self) -> UpnpState { + self.state.borrow().clone() + } + + /// Clone the watch receiver so callers can poll state without owning + /// a reference to the service itself. + /// + /// Use this when the consumer only needs to read the current mapping + /// (for example, the candidate discovery manager) — it keeps service + /// lifetime cleanly owned by the endpoint and lets graceful shutdown + /// reclaim the unique `Arc` for `try_unwrap`. + pub fn subscribe(&self) -> UpnpStateRx { + UpnpStateRx { + inner: self.state.clone(), + } + } + + /// Best-effort graceful teardown. + /// + /// Signals the background task to stop, then attempts a single + /// `DeletePortMapping` against the gateway with a 500ms budget. + /// All errors are swallowed — if the router has gone away, the lease + /// expires naturally. Mutex guards are released before the awaits so + /// the resulting future stays `Send` for callers running on a + /// multi-threaded tokio runtime. + pub async fn shutdown(self) { + self.inner.shutdown.notify_waiters(); + + let handle = self.inner.handle.lock().take(); + if let Some(handle) = handle { + handle.abort(); + let _ = handle.await; + } + + let active = self.inner.last_mapping.lock().take(); + if let Some(active) = active { + best_effort_unmap(active).await; + } + } +} + +impl Drop for UpnpMappingService { + fn drop(&mut self) { + // Crash-path safety: notify any background task and abort it. + // We deliberately do *not* attempt async unmap here — the lease + // is the ultimate safety net. + self.inner.shutdown.notify_waiters(); + if let Some(handle) = self.inner.handle.lock().take() { + handle.abort(); + } + } +} + +/// Returns true if `addr` looks like a publicly routable IP address. +/// +/// We require this check because misbehaving routers will sometimes return +/// their LAN-side address as the "external" IP via `GetExternalIP`. Trusting +/// such a value would poison NAT traversal candidate selection — the +/// endpoint would advertise an unreachable RFC1918 address as if it were +/// public. +#[cfg_attr(not(feature = "upnp"), allow(dead_code))] +pub(crate) fn is_plausibly_public(addr: IpAddr) -> bool { + match addr { + IpAddr::V4(v4) => is_plausibly_public_v4(v4), + IpAddr::V6(v6) => { + // Reject loopback, unspecified, multicast, link-local. Anything + // else (global unicast, ULA) is acceptable — ULAs are not + // routable but a misconfigured gateway returning a ULA is rare + // enough that we let the candidate validator catch it later. + !(v6.is_loopback() + || v6.is_unspecified() + || v6.is_multicast() + || is_ipv6_link_local(v6)) + } + } +} + +#[cfg_attr(not(feature = "upnp"), allow(dead_code))] +fn is_plausibly_public_v4(addr: Ipv4Addr) -> bool { + if addr.is_loopback() + || addr.is_unspecified() + || addr.is_broadcast() + || addr.is_multicast() + || addr.is_link_local() + || addr.is_documentation() + { + return false; + } + if addr.is_private() { + return false; + } + // CGNAT range (RFC 6598) — addresses here are NAT'd by the carrier and + // are not directly reachable from the public internet, so a UPnP + // mapping against a 100.64/10 "external" IP is useless. + let octets = addr.octets(); + if octets[0] == 100 && (64..=127).contains(&octets[1]) { + return false; + } + true +} + +#[cfg_attr(not(feature = "upnp"), allow(dead_code))] +fn is_ipv6_link_local(addr: std::net::Ipv6Addr) -> bool { + let segments = addr.segments(); + segments[0] & 0xFFC0 == 0xFE80 +} + +// --------------------------------------------------------------------------- +// Backend selection: real `igd-next` implementation when the `upnp` feature +// is enabled, no-op stub otherwise. Both backends share the public types +// above so call sites do not need to be feature-gated. +// --------------------------------------------------------------------------- + +#[cfg(feature = "upnp")] +mod backend { + use super::*; + use igd_next::PortMappingProtocol; + use igd_next::SearchOptions; + use igd_next::aio::Gateway as GenericGateway; + use igd_next::aio::tokio::{Tokio, search_gateway}; + + pub(super) type GatewayHandle = Arc>; + + /// Description sent to the gateway. Most consumer routers expose this + /// in the admin UI's port-forwarding table. + const MAPPING_DESCRIPTION: &str = concat!("saorsa-transport/", env!("CARGO_PKG_VERSION")); + + pub(super) fn spawn_background_task( + local_port: u16, + config: UpnpConfig, + tx: watch::Sender, + inner: Arc, + ) -> Option> { + let handle = tokio::spawn(async move { + run_service(local_port, config, tx, inner).await; + }); + Some(handle) + } + + async fn run_service( + local_port: u16, + config: UpnpConfig, + tx: watch::Sender, + inner: Arc, + ) { + let gateway = match discover_gateway(config.discovery_timeout).await { + Some(gw) => Arc::new(gw), + None => { + let _ = tx.send(UpnpState::Unavailable); + return; + } + }; + + // Validate the gateway's claimed external IP before trusting any + // mapping it offers. A router that returns its LAN address here is + // misconfigured and unsafe to use — surfacing such an "external" + // address as a NAT traversal candidate would actively break peers. + let external_ip = match gateway.get_external_ip().await { + Ok(ip) => ip, + Err(err) => { + debug!(error = %err, "upnp: get_external_ip failed"); + let _ = tx.send(UpnpState::Unavailable); + return; + } + }; + if !is_plausibly_public(external_ip) { + warn!( + external_ip = %external_ip, + "upnp: gateway returned a non-public external IP, refusing to use" + ); + let _ = tx.send(UpnpState::Unavailable); + return; + } + + let local_addr = local_socket_for_mapping(local_port); + let mapped_port = + match request_mapping(&gateway, local_addr, local_port, config.lease_duration).await { + Some(port) => port, + None => { + let _ = tx.send(UpnpState::Unavailable); + return; + } + }; + + let external = SocketAddr::new(external_ip, mapped_port); + let mut lease_expires_at = Instant::now() + config.lease_duration; + info!( + external = %external, + lease_secs = config.lease_duration.as_secs(), + "upnp: gateway mapping active" + ); + + // Record the active mapping so the shutdown path can clean it up. + *inner.last_mapping.lock() = Some(ActiveMapping { + external_port: mapped_port, + gateway: Arc::clone(&gateway), + }); + + let _ = tx.send(UpnpState::Mapped { + external, + lease_expires_at, + }); + + // Refresh loop: re-request the mapping at half the lease interval. + // Failure here is not fatal — we transition to Unavailable, leave + // the existing mapping to expire on its own, and exit the task. + loop { + let refresh_in = (config.lease_duration / 2).max(Duration::from_secs(30)); + tokio::select! { + () = inner.shutdown.notified() => { + return; + } + () = tokio::time::sleep(refresh_in) => {} + } + + match request_mapping(&gateway, local_addr, mapped_port, config.lease_duration).await { + Some(port) if port == mapped_port => { + lease_expires_at = Instant::now() + config.lease_duration; + let _ = tx.send(UpnpState::Mapped { + external, + lease_expires_at, + }); + } + _ => { + debug!("upnp: lease refresh failed, marking unavailable"); + *inner.last_mapping.lock() = None; + let _ = tx.send(UpnpState::Unavailable); + return; + } + } + } + } + + async fn discover_gateway(timeout: Duration) -> Option> { + let opts = SearchOptions { + timeout: Some(timeout), + ..Default::default() + }; + match tokio::time::timeout(timeout, search_gateway(opts)).await { + Ok(Ok(gateway)) => Some(gateway), + Ok(Err(err)) => { + debug!(error = %err, "upnp: gateway discovery failed"); + None + } + Err(_) => { + debug!("upnp: gateway discovery timed out"); + None + } + } + } + + /// Request a UDP mapping for `local_addr`, preferring port preservation. + /// + /// Tries `add_port(preferred_external)` first because matching the + /// internal port keeps the mapped candidate aligned with what peers + /// will see via OBSERVED_ADDRESS. Falls back to `add_any_port` so the + /// gateway can pick a free port if the preferred one is taken. + async fn request_mapping( + gateway: &GenericGateway, + local_addr: SocketAddr, + preferred_external: u16, + lease: Duration, + ) -> Option { + let lease_secs = u32::try_from(lease.as_secs()).unwrap_or(u32::MAX); + + match gateway + .add_port( + PortMappingProtocol::UDP, + preferred_external, + local_addr, + lease_secs, + MAPPING_DESCRIPTION, + ) + .await + { + Ok(()) => return Some(preferred_external), + Err(err) => { + debug!( + preferred_external, + error = %err, + "upnp: add_port for preferred external failed, falling back to add_any_port" + ); + } + } + + match gateway + .add_any_port( + PortMappingProtocol::UDP, + local_addr, + lease_secs, + MAPPING_DESCRIPTION, + ) + .await + { + Ok(port) => Some(port), + Err(err) => { + debug!(error = %err, "upnp: add_any_port failed"); + None + } + } + } + + /// Build a `SocketAddr` for the gateway to forward traffic to. + /// + /// `igd-next` requires an explicit local IP rather than `0.0.0.0`, + /// because the gateway needs to know which LAN host owns the mapping. + /// We pick the first IPv4 address that matches the egress route to the + /// gateway by relying on the OS-default outbound socket trick: connect + /// a UDP socket to a public address and read its local IP. The remote + /// address is never actually contacted. + fn local_socket_for_mapping(local_port: u16) -> SocketAddr { + // 192.0.2.1 (TEST-NET-1) is RFC 5737 documentation space — packets + // are not routed but the kernel will still pick the correct + // outbound interface for the route lookup. + let probe = std::net::UdpSocket::bind("0.0.0.0:0") + .and_then(|sock| { + sock.connect("192.0.2.1:9")?; + sock.local_addr() + }) + .map(|addr| addr.ip()); + + let local_ip = match probe { + Ok(IpAddr::V4(v4)) if !v4.is_unspecified() => IpAddr::V4(v4), + // UPnP IGD v1 only deals in IPv4 mappings; if the egress route + // resolved to IPv6 (or failed entirely) we fall back to the + // unspecified address and let `add_port` reject it. The error + // is logged at `debug` and surfaces as `Unavailable`. + _ => IpAddr::V4(Ipv4Addr::UNSPECIFIED), + }; + SocketAddr::new(local_ip, local_port) + } + + pub(super) async fn best_effort_unmap(active: ActiveMapping) { + let unmap = active + .gateway + .remove_port(PortMappingProtocol::UDP, active.external_port); + match tokio::time::timeout(SHUTDOWN_UNMAP_BUDGET, unmap).await { + Ok(Ok(())) => debug!("upnp: deleted port mapping on shutdown"), + Ok(Err(err)) => debug!(error = %err, "upnp: delete_port_mapping failed on shutdown"), + Err(_) => debug!("upnp: delete_port_mapping timed out on shutdown"), + } + } +} + +#[cfg(not(feature = "upnp"))] +mod backend { + use super::*; + + /// Stub gateway handle used when the `upnp` feature is disabled. + /// Carries no state and is never instantiated at runtime. + pub(super) type GatewayHandle = (); + + pub(super) fn spawn_background_task( + _local_port: u16, + _config: UpnpConfig, + tx: watch::Sender, + _inner: Arc, + ) -> Option> { + // Without the feature we cannot probe a gateway, so transition + // straight to Unavailable and skip spawning a task entirely. + let _ = tx.send(UpnpState::Unavailable); + None + } + + pub(super) async fn best_effort_unmap(_active: ActiveMapping) { + // No backend → nothing to release. + } +} + +use backend::{GatewayHandle, best_effort_unmap, spawn_background_task}; + +// --------------------------------------------------------------------------- +// Serde helpers — keep human-readable units in serialized config files +// without inflicting them on the public API. +// --------------------------------------------------------------------------- + +mod duration_secs { + use serde::{Deserialize, Deserializer, Serializer}; + use std::time::Duration; + + pub fn serialize(value: &Duration, ser: S) -> Result { + ser.serialize_u64(value.as_secs()) + } + + pub fn deserialize<'de, D: Deserializer<'de>>(de: D) -> Result { + let secs = u64::deserialize(de)?; + Ok(Duration::from_secs(secs)) + } +} + +mod duration_millis { + use serde::{Deserialize, Deserializer, Serializer}; + use std::time::Duration; + + pub fn serialize(value: &Duration, ser: S) -> Result { + ser.serialize_u64(value.as_millis() as u64) + } + + pub fn deserialize<'de, D: Deserializer<'de>>(de: D) -> Result { + let ms = u64::deserialize(de)?; + Ok(Duration::from_millis(ms)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::net::Ipv6Addr; + + #[test] + fn disabled_service_reports_unavailable_immediately() { + let service = UpnpMappingService::start(0, UpnpConfig::disabled()); + assert_eq!(service.current(), UpnpState::Unavailable); + } + + #[test] + fn default_config_is_enabled_with_one_hour_lease() { + let cfg = UpnpConfig::default(); + assert!(cfg.enabled); + assert_eq!(cfg.lease_duration, DEFAULT_LEASE); + assert_eq!(cfg.discovery_timeout, DEFAULT_DISCOVERY_TIMEOUT); + } + + #[test] + fn rejects_rfc1918_addresses_as_external_ip() { + for blocked in [ + Ipv4Addr::new(10, 0, 0, 1), + Ipv4Addr::new(172, 16, 5, 9), + Ipv4Addr::new(192, 168, 1, 254), + ] { + assert!( + !is_plausibly_public(IpAddr::V4(blocked)), + "{blocked} should be rejected as non-public" + ); + } + } + + #[test] + fn rejects_loopback_link_local_and_cgnat() { + assert!(!is_plausibly_public(IpAddr::V4(Ipv4Addr::LOCALHOST))); + assert!(!is_plausibly_public(IpAddr::V4(Ipv4Addr::UNSPECIFIED))); + assert!(!is_plausibly_public(IpAddr::V4(Ipv4Addr::BROADCAST))); + assert!(!is_plausibly_public(IpAddr::V4(Ipv4Addr::new( + 169, 254, 1, 1 + )))); + assert!(!is_plausibly_public(IpAddr::V4(Ipv4Addr::new( + 100, 64, 0, 1 + )))); + assert!(!is_plausibly_public(IpAddr::V4(Ipv4Addr::new( + 100, 127, 255, 254 + )))); + } + + #[test] + fn accepts_public_ipv4_outside_special_ranges() { + assert!(is_plausibly_public(IpAddr::V4(Ipv4Addr::new(8, 8, 8, 8)))); + assert!(is_plausibly_public(IpAddr::V4(Ipv4Addr::new(1, 1, 1, 1)))); + } + + #[test] + fn rejects_documentation_ranges() { + // RFC 5737 documentation prefixes — must never be advertised as + // a real external IP, regardless of what a misbehaving gateway + // might claim. + assert!(!is_plausibly_public(IpAddr::V4(Ipv4Addr::new( + 192, 0, 2, 1 + )))); + assert!(!is_plausibly_public(IpAddr::V4(Ipv4Addr::new( + 198, 51, 100, 1 + )))); + assert!(!is_plausibly_public(IpAddr::V4(Ipv4Addr::new( + 203, 0, 113, 1 + )))); + } + + #[test] + fn accepts_global_unicast_ipv6_and_rejects_link_local() { + let global = Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 1); + let link_local = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + assert!(is_plausibly_public(IpAddr::V6(global))); + assert!(!is_plausibly_public(IpAddr::V6(link_local))); + assert!(!is_plausibly_public(IpAddr::V6(Ipv6Addr::LOCALHOST))); + } +} diff --git a/tests/relay_queue_tests.rs b/tests/relay_queue_tests.rs index be5cb402..20375f1e 100644 --- a/tests/relay_queue_tests.rs +++ b/tests/relay_queue_tests.rs @@ -55,6 +55,7 @@ mod nat_traversal_api_tests { transport_registry: None, max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: false, + upnp: Default::default(), }; assert_eq!(config.known_peers.len(), 1); @@ -191,6 +192,7 @@ mod functional_tests { transport_registry: None, max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: false, + upnp: Default::default(), }; // May fail due to zero values or other validation @@ -217,6 +219,7 @@ mod functional_tests { transport_registry: None, max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: false, + upnp: Default::default(), }; let result = NatTraversalEndpoint::new(valid_config, None, None).await; @@ -409,6 +412,7 @@ mod performance_tests { transport_registry: None, max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: false, + upnp: Default::default(), }; // Use the config to prevent optimization @@ -479,6 +483,7 @@ mod relay_functionality_tests { transport_registry: None, max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: false, + upnp: Default::default(), }; // This might be accepted or rejected depending on implementation diff --git a/tests/security_regression_tests.rs b/tests/security_regression_tests.rs index 48a66271..3a8b450c 100644 --- a/tests/security_regression_tests.rs +++ b/tests/security_regression_tests.rs @@ -37,6 +37,7 @@ fn test_peer_config() -> NatTraversalConfig { transport_registry: None, max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: true, + upnp: Default::default(), } } @@ -61,6 +62,7 @@ fn test_server_config() -> NatTraversalConfig { transport_registry: None, max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: true, + upnp: Default::default(), } } @@ -108,6 +110,7 @@ async fn test_error_handling_no_panic() { transport_registry: None, max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: true, + upnp: Default::default(), }; let result1 = NatTraversalEndpoint::new(config1, None, None).await; @@ -136,6 +139,7 @@ async fn test_error_handling_no_panic() { transport_registry: None, max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: true, + upnp: Default::default(), }; let result2 = NatTraversalEndpoint::new(config2, None, None).await; @@ -227,6 +231,7 @@ async fn test_malformed_config_handling() { transport_registry: None, max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: true, + upnp: Default::default(), }; let result = NatTraversalEndpoint::new(no_peers_config, None, None).await; @@ -256,6 +261,7 @@ async fn test_malformed_config_handling() { transport_registry: None, max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: true, + upnp: Default::default(), }; let result2 = NatTraversalEndpoint::new(extreme_config, None, None).await; @@ -292,6 +298,7 @@ async fn test_input_sanitization() { transport_registry: None, max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: true, + upnp: Default::default(), }; // This should either work or fail gracefully, not exhaust memory or panic @@ -363,6 +370,7 @@ mod specific_regression_tests { transport_registry: None, max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: true, + upnp: Default::default(), }; // Should not panic and should handle random port selection @@ -414,6 +422,7 @@ mod specific_regression_tests { transport_registry: None, max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: true, + upnp: Default::default(), }; // Should not panic, even if configuration is inconsistent From 090a45cc62b998e1ad65675734a6e8448d7dae77 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Mon, 6 Apr 2026 15:59:12 +0200 Subject: [PATCH 16/43] refactor!: remove orphaned CandidatePair public types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removes `pub struct CandidatePair` and `pub enum CandidatePairState` from `nat_traversal_api.rs`. They were only ever used by the dead hole-punch chain deleted in 5280123d, and `connection/nat_traversal.rs` defines its own unrelated `pub(super) CandidatePair` for the live coordinator path. With the chain gone, both types are orphans in the public API surface. Also tightens the two tombstone comments — drops the box-drawing borders, fixes the misleading "implemented elsewhere in this file" phrasing (the production flow spans `p2p_endpoint.rs` and the PUNCH_ME_NOW helpers later in this file), and cross-links them. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/nat_traversal_api.rs | 87 ++++++++++------------------------------ 1 file changed, 22 insertions(+), 65 deletions(-) diff --git a/src/nat_traversal_api.rs b/src/nat_traversal_api.rs index 89ff67a4..f039af35 100644 --- a/src/nat_traversal_api.rs +++ b/src/nat_traversal_api.rs @@ -567,34 +567,6 @@ impl BootstrapNode { } } -/// A candidate pair for hole punching (ICE-like) -#[derive(Debug, Clone)] -pub struct CandidatePair { - /// Local candidate address - pub local_candidate: CandidateAddress, - /// Remote candidate address - pub remote_candidate: CandidateAddress, - /// Combined priority for this pair - pub priority: u64, - /// Current state of this candidate pair - pub state: CandidatePairState, -} - -/// State of a candidate pair during hole punching -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum CandidatePairState { - /// Waiting to be checked - Waiting, - /// Currently being checked - InProgress, - /// Check succeeded - Succeeded, - /// Check failed - Failed, - /// Cancelled due to higher priority success - Cancelled, -} - /// Active NAT traversal session state #[derive(Debug)] struct NatTraversalSession { @@ -3236,20 +3208,12 @@ impl NatTraversalEndpoint { Ok(connection) } - // ───────────────────────────────────────────────────────────────────── - // Note: the historical `NatTraversalEndpoint::connect_with_fallback` - // method that lived here has been removed. It was an unused duplicate - // of `P2pEndpoint::connect_with_fallback` (in `p2p_endpoint.rs`), which - // is the actual production entry point reached through `LinkTransport:: - // dial_addr` and the `saorsa-transport` example binary. The removed - // copy delegated to `attempt_hole_punching` (also removed below), an - // implementation that crafted a hand-rolled "PATH_CHALLENGE" UDP - // datagram on a freshly bound socket — both unworkable in practice - // (the bind raced Quinn for the port; the bytes were not a valid - // QUIC packet so the receiver dropped them) and misleading during - // debugging because the surrounding `#[allow(dead_code)]` markers - // disguised that nothing in the path could ever succeed. - // ───────────────────────────────────────────────────────────────────── + // Removed: the duplicate `NatTraversalEndpoint::connect_with_fallback`. + // Production hole-punch fallback lives in + // `crate::p2p_endpoint::P2pEndpoint::connect_with_fallback`, reached via + // `LinkTransport::dial_addr` and the `saorsa-transport` example binary. + // See the tombstone further down this file for the deleted helpers and + // why they could never have worked. /// Get the relay manager for advanced relay operations /// @@ -4671,32 +4635,25 @@ impl NatTraversalEndpoint { Ok(frame) } - // ───────────────────────────────────────────────────────────────────── - // Removed: the dead `attempt_hole_punching` / - // `attempt_quic_hole_punching` / `get_candidate_pairs_for_addr` / - // `calculate_candidate_pair_priority` / `create_path_challenge_packet` - // / `store_successful_candidate_pair` / `get_successful_candidate_address` - // chain. These were only ever called from the duplicate - // `NatTraversalEndpoint::connect_with_fallback` (also removed above) - // and could not have worked in production: - // - // 1. `attempt_quic_hole_punching` bound a fresh `std::net::UdpSocket` - // to the local candidate address, which always fails on a real - // node because Quinn already owns the port — UDP binds are - // exclusive. - // 2. The "QUIC packet" it sent was a hand-rolled byte sequence - // (`0x40 [0,0,0,1] 0x1a <8 random>`) that is not a valid - // encrypted QUIC packet, so any receiving Quinn endpoint - // silently dropped it. - // 3. The success branch then waited 100 ms on a blocking - // `recv_from` for a "response" that no compliant peer would - // ever send. + // Removed: the dead `attempt_hole_punching` chain + // (`attempt_quic_hole_punching`, `get_candidate_pairs_for_addr`, + // `calculate_candidate_pair_priority`, `create_path_challenge_packet`, + // `store_successful_candidate_pair`, `get_successful_candidate_address`). + // Only ever called from the duplicate + // `NatTraversalEndpoint::connect_with_fallback` (also removed). Could + // not have worked in production: it bound a fresh `std::net::UdpSocket` + // to a port Quinn already owned (UDP binds are exclusive), then sent a + // hand-rolled `0x40 [0,0,0,1] 0x1a <8 random>` byte sequence that is + // not a valid encrypted QUIC packet (any receiver drops it), then + // blocked the async runtime in a 100 ms `recv_from` for a response no + // compliant peer would ever send. The `#[allow(dead_code)]` markers on + // every function disguised this from grep-driven debugging. // // Production hole-punch coordination lives in // `crate::p2p_endpoint::P2pEndpoint::connect_with_fallback_inner`, - // which uses the proper coordinator-mediated PUNCH_ME_NOW flow - // implemented elsewhere in this file. - // ───────────────────────────────────────────────────────────────────── + // which drives the coordinator-mediated PUNCH_ME_NOW flow whose + // server-side helpers (`send_coordination_request_with_peer_id`, etc.) + // are defined later in this file. /// Attempt connection to a specific candidate address fn attempt_connection_to_candidate( From b906e65f76e2c839dc039e0c9fe369ad0b641c37 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Mon, 6 Apr 2026 15:59:17 +0200 Subject: [PATCH 17/43] style: apply rustfmt to two unrelated info! call sites MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pre-existing formatting drift caught by `cargo fmt --all -- --check` on this branch but unrelated to the hole-punch chain removal: - `src/endpoint.rs:387` — collapse a `.as_ref().map(...)` chain that rustfmt now wants on one line. - `src/p2p_endpoint.rs:1822` — split a multi-arg `info!` so each positional argument sits on its own line. No semantic change. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/endpoint.rs | 4 +--- src/p2p_endpoint.rs | 5 ++++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/endpoint.rs b/src/endpoint.rs index 1291d4f2..99cf2417 100644 --- a/src/endpoint.rs +++ b/src/endpoint.rs @@ -385,9 +385,7 @@ impl Endpoint { .connections .iter() .filter_map(|(_, meta)| { - meta.peer_id - .as_ref() - .map(|pid| hex::encode(&pid.0[..8])) + meta.peer_id.as_ref().map(|pid| hex::encode(&pid.0[..8])) }) .collect(); tracing::warn!( diff --git a/src/p2p_endpoint.rs b/src/p2p_endpoint.rs index a3d7afc9..9c45749b 100644 --- a/src/p2p_endpoint.rs +++ b/src/p2p_endpoint.rs @@ -1822,7 +1822,10 @@ impl P2pEndpoint { if let Some(ref pid) = target_peer_id { info!( "try_hole_punch: calling initiate_nat_traversal({}, {}) with peer ID {} (dashmap key={})", - target, coordinator, hex::encode(&pid[..8]), target + target, + coordinator, + hex::encode(&pid[..8]), + target ); } else { info!( From 21bda712ee687be885b0921c99845ea8f9edec3f Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Mon, 6 Apr 2026 16:00:49 +0200 Subject: [PATCH 18/43] fix: reject IPv6 documentation prefix in UPnP external IP classifier MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The IPv4 plausibility check rejects RFC 5737 documentation space (192.0.2/24, 198.51.100/24, 203.0.113/24) so a misbehaving router cannot poison NAT traversal candidate discovery by returning a documentation address as its "external" IP. The IPv6 side of the classifier was missing the equivalent rejection of RFC 3849 `2001:db8::/32`, leaving a small but real asymmetry in the stated threat model. Split `is_plausibly_public` into symmetric v4/v6 helpers, add an `is_ipv6_documentation` check backed by named constants for the prefix bytes, and replace the hand-rolled `is_ipv6_link_local` with stdlib's `Ipv6Addr::is_unicast_link_local()` (stable since 1.76 — well under the project MSRV of 1.88). Fixes up the test that would have caught this: the positive assertion in `accepts_global_unicast_ipv6_and_rejects_link_local` was using `2001:db8::1` as its "global unicast" example, which accidentally hid the gap. Swap it for `2606:4700:4700::1111` (Cloudflare DNS), add `rejects_ipv6_documentation_range` with a neighbouring `/32` as a positive control, and add `rejects_ipv6_multicast_and_unspecified` for basic coverage completeness. Drive-bys from the deep review of PR #40: * `CandidateDiscoveryManager::set_upnp_state_rx` downgraded from `pub` to `pub(crate)` — it's an internal wiring hook used only by the endpoint constructor, not a public API. * Clarifying comment on `local_socket_for_mapping` explaining why the sync `std::net::UdpSocket::bind`/`connect` pair is safe to call inside the tokio task (pure kernel route lookup, no wire I/O, runs once per session). Tests: 1470/1470 lib tests pass with `--features upnp`, and the UPnP module now has 11 passing tests (up from 9) across both the real and stub backends. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/candidate_discovery.rs | 5 ++- src/upnp.rs | 85 ++++++++++++++++++++++++++++++++------ 2 files changed, 76 insertions(+), 14 deletions(-) diff --git a/src/candidate_discovery.rs b/src/candidate_discovery.rs index 393e3d2c..17d4e087 100644 --- a/src/candidate_discovery.rs +++ b/src/candidate_discovery.rs @@ -719,7 +719,10 @@ impl CandidateDiscoveryManager { /// Calling this is optional and best-effort — if the handle never /// reaches [`crate::upnp::UpnpState::Mapped`], discovery behaves /// identically to a manager without UPnP attached. - pub fn set_upnp_state_rx(&mut self, state_rx: crate::upnp::UpnpStateRx) { + /// + /// Internal plumbing hook for the endpoint constructor; not exposed + /// on the public API surface. + pub(crate) fn set_upnp_state_rx(&mut self, state_rx: crate::upnp::UpnpStateRx) { self.upnp = Some(state_rx); } diff --git a/src/upnp.rs b/src/upnp.rs index 5f93d54b..0e356221 100644 --- a/src/upnp.rs +++ b/src/upnp.rs @@ -286,16 +286,7 @@ impl Drop for UpnpMappingService { pub(crate) fn is_plausibly_public(addr: IpAddr) -> bool { match addr { IpAddr::V4(v4) => is_plausibly_public_v4(v4), - IpAddr::V6(v6) => { - // Reject loopback, unspecified, multicast, link-local. Anything - // else (global unicast, ULA) is acceptable — ULAs are not - // routable but a misconfigured gateway returning a ULA is rare - // enough that we let the candidate validator catch it later. - !(v6.is_loopback() - || v6.is_unspecified() - || v6.is_multicast() - || is_ipv6_link_local(v6)) - } + IpAddr::V6(v6) => is_plausibly_public_v6(v6), } } @@ -324,9 +315,39 @@ fn is_plausibly_public_v4(addr: Ipv4Addr) -> bool { } #[cfg_attr(not(feature = "upnp"), allow(dead_code))] -fn is_ipv6_link_local(addr: std::net::Ipv6Addr) -> bool { +fn is_plausibly_public_v6(addr: std::net::Ipv6Addr) -> bool { + // Reject the standard garbage: loopback, unspecified, multicast, + // link-local unicast, documentation. Anything else (global unicast, + // ULA) is acceptable — ULAs are not routable but a misconfigured + // gateway returning a ULA is rare enough that we let the candidate + // validator catch it later. + // + // Mirrors the IPv4 classifier's rejection of RFC 5737 documentation + // space so a misbehaving router cannot poison candidate discovery by + // returning an RFC 3849 `2001:db8::/32` address as its "external" IP. + !(addr.is_loopback() + || addr.is_unspecified() + || addr.is_multicast() + || addr.is_unicast_link_local() + || is_ipv6_documentation(addr)) +} + +/// First 16-bit group of the RFC 3849 IPv6 documentation prefix +/// `2001:db8::/32`. +const IPV6_DOCUMENTATION_PREFIX_HI: u16 = 0x2001; +/// Second 16-bit group of the RFC 3849 IPv6 documentation prefix +/// `2001:db8::/32`. +const IPV6_DOCUMENTATION_PREFIX_LO: u16 = 0x0db8; + +/// RFC 3849 documentation prefix — `2001:db8::/32`. +/// +/// Stdlib does not expose an `is_documentation` helper for `Ipv6Addr`, so +/// we match the prefix manually. Kept separate to mirror the v4 +/// `Ipv4Addr::is_documentation` call path at the classifier site. +#[cfg_attr(not(feature = "upnp"), allow(dead_code))] +fn is_ipv6_documentation(addr: std::net::Ipv6Addr) -> bool { let segments = addr.segments(); - segments[0] & 0xFFC0 == 0xFE80 + segments[0] == IPV6_DOCUMENTATION_PREFIX_HI && segments[1] == IPV6_DOCUMENTATION_PREFIX_LO } // --------------------------------------------------------------------------- @@ -532,6 +553,12 @@ mod backend { /// gateway by relying on the OS-default outbound socket trick: connect /// a UDP socket to a public address and read its local IP. The remote /// address is never actually contacted. + /// + /// This uses `std::net::UdpSocket` rather than `tokio::net::UdpSocket` + /// because both `bind` and `connect` on UDP are pure kernel route + /// lookups — there is no wire I/O, so the executor thread is not + /// actually blocked. Called once per session at the top of the + /// background task, before the real SSDP discovery begins. fn local_socket_for_mapping(local_port: u16) -> SocketAddr { // 192.0.2.1 (TEST-NET-1) is RFC 5737 documentation space — packets // are not routed but the kernel will still pick the correct @@ -699,10 +726,42 @@ mod tests { #[test] fn accepts_global_unicast_ipv6_and_rejects_link_local() { - let global = Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 1); + // 2606:4700:4700::1111 is Cloudflare DNS, a real global unicast + // address. Explicitly chosen over 2001:db8::/32 so this test + // exercises the happy path rather than accidentally landing in + // documentation space. + let global = Ipv6Addr::new(0x2606, 0x4700, 0x4700, 0, 0, 0, 0, 0x1111); let link_local = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); assert!(is_plausibly_public(IpAddr::V6(global))); assert!(!is_plausibly_public(IpAddr::V6(link_local))); assert!(!is_plausibly_public(IpAddr::V6(Ipv6Addr::LOCALHOST))); } + + #[test] + fn rejects_ipv6_documentation_range() { + // RFC 3849 `2001:db8::/32` is the IPv6 counterpart of the RFC + // 5737 documentation prefixes. A misbehaving router returning an + // address from this range must never be accepted as an external + // IP, matching the IPv4 `is_documentation()` rejection. + assert!(!is_plausibly_public(IpAddr::V6(Ipv6Addr::new( + 0x2001, 0x0db8, 0, 0, 0, 0, 0, 1 + )))); + assert!(!is_plausibly_public(IpAddr::V6(Ipv6Addr::new( + 0x2001, 0x0db8, 0xdead, 0xbeef, 0, 0, 0, 0x42 + )))); + // A neighbouring /32 (2001:0db9::) is not documentation space + // and must still be accepted. + assert!(is_plausibly_public(IpAddr::V6(Ipv6Addr::new( + 0x2001, 0x0db9, 0, 0, 0, 0, 0, 1 + )))); + } + + #[test] + fn rejects_ipv6_multicast_and_unspecified() { + assert!(!is_plausibly_public(IpAddr::V6(Ipv6Addr::UNSPECIFIED))); + // ff00::/8 — multicast. + assert!(!is_plausibly_public(IpAddr::V6(Ipv6Addr::new( + 0xff02, 0, 0, 0, 0, 0, 0, 1 + )))); + } } From 3c847aae5215e98d462fe2bbb163be6083930baa Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Mon, 6 Apr 2026 19:13:32 +0200 Subject: [PATCH 19/43] perf!: hold read lock on router during send via atomic selection stats `P2pEndpoint::send()` previously took a write lock on the `ConnectionRouter` just to call `select_engine_for_addr`, which only needed to bump a stats counter. At high node counts (1000-node testnet) this serialised every outbound send through a single exclusive lock and was a dominant contention point. Convert `RouterStats::{quic,constrained,fallback}_selections` to `AtomicU64` so the `select_engine*` family can take `&self`, then switch the send path to `router.read().await`. Other counters stay plain `u64` because they are only mutated on `&mut self` connect/accept/event-loop paths. Hand-implement `Clone` for `RouterStats` since `AtomicU64` isn't `Clone`, loading each counter with `Relaxed` ordering (stats are monotonic and don't synchronise other state). BREAKING: `RouterStats::{quic,constrained,fallback}_selections` are now `AtomicU64` instead of `u64`. External readers must use `.load(Ordering::Relaxed)`. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/connection_router.rs | 184 ++++++++++++++++++++++--------- src/p2p_endpoint.rs | 17 ++- tests/constrained_integration.rs | 19 ++-- 3 files changed, 152 insertions(+), 68 deletions(-) diff --git a/src/connection_router.rs b/src/connection_router.rs index 353624d8..d667f498 100644 --- a/src/connection_router.rs +++ b/src/connection_router.rs @@ -64,6 +64,7 @@ use std::fmt; use std::net::SocketAddr; use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; use crate::constrained::{ AdapterEvent, ConnectionId as ConstrainedConnId, ConstrainedError, ConstrainedHandle, @@ -786,8 +787,18 @@ impl RouterEvent { } } -/// Router statistics -#[derive(Debug, Clone, Default)] +/// Router statistics. +/// +/// Selection counters (`quic_selections`, `constrained_selections`, +/// `fallback_selections`) are stored as [`AtomicU64`] so the selection +/// methods can take `&self` and be called under a `RwLock::read()` guard. +/// Without this, every outbound send on a `P2pEndpoint` had to acquire an +/// exclusive write lock on the router just to bump a counter, serialising +/// all sends through a single lock (see p2p_endpoint.rs `send()`). +/// +/// Other counters are still plain `u64` because they are only touched on +/// `&mut self` paths (connect/accept/event-loop). +#[derive(Debug, Default)] pub struct RouterStats { /// Total connections routed through QUIC pub quic_connections: u64, @@ -810,19 +821,42 @@ pub struct RouterStats { /// Connection failures pub connection_failures: u64, - /// Engine selection decisions (QUIC chosen) - pub quic_selections: u64, + /// Engine selection decisions (QUIC chosen). Atomic so the selection + /// path does not require exclusive access. + pub quic_selections: AtomicU64, - /// Engine selection decisions (Constrained chosen) - pub constrained_selections: u64, + /// Engine selection decisions (Constrained chosen). Atomic so the + /// selection path does not require exclusive access. + pub constrained_selections: AtomicU64, - /// Fallback selections (when preferred engine unavailable) - pub fallback_selections: u64, + /// Fallback selections (when preferred engine unavailable). Atomic so + /// the selection path does not require exclusive access. + pub fallback_selections: AtomicU64, /// Total events processed pub events_processed: u64, } +impl Clone for RouterStats { + fn clone(&self) -> Self { + Self { + quic_connections: self.quic_connections, + constrained_connections: self.constrained_connections, + quic_bytes_sent: self.quic_bytes_sent, + constrained_bytes_sent: self.constrained_bytes_sent, + quic_bytes_received: self.quic_bytes_received, + constrained_bytes_received: self.constrained_bytes_received, + connection_failures: self.connection_failures, + quic_selections: AtomicU64::new(self.quic_selections.load(Ordering::Relaxed)), + constrained_selections: AtomicU64::new( + self.constrained_selections.load(Ordering::Relaxed), + ), + fallback_selections: AtomicU64::new(self.fallback_selections.load(Ordering::Relaxed)), + events_processed: self.events_processed, + } + } +} + /// Connection router for automatic protocol engine selection /// /// The router examines transport capabilities and routes connections @@ -913,17 +947,19 @@ impl ConnectionRouter { self.quic_endpoint.is_some() } - /// Select the appropriate protocol engine for a transport - pub fn select_engine(&mut self, capabilities: &TransportCapabilities) -> ProtocolEngine { + /// Select the appropriate protocol engine for a transport. + /// + /// Takes `&self` — selection counters are atomic so this can run under + /// a read-lock guard without serialising all callers. + pub fn select_engine(&self, capabilities: &TransportCapabilities) -> ProtocolEngine { let result = self.select_engine_detailed(capabilities); result.engine } - /// Select engine with detailed selection result - pub fn select_engine_detailed( - &mut self, - capabilities: &TransportCapabilities, - ) -> SelectionResult { + /// Select engine with detailed selection result. + /// + /// Takes `&self` — selection counters are atomic. + pub fn select_engine_detailed(&self, capabilities: &TransportCapabilities) -> SelectionResult { let supports_quic = capabilities.supports_full_quic(); let (engine, reason) = if supports_quic { @@ -939,10 +975,16 @@ impl ConnectionRouter { (ProtocolEngine::Constrained, SelectionReason::TooConstrained) }; - // Update selection stats + // Update selection stats via atomic counters. match engine { - ProtocolEngine::Quic => self.stats.quic_selections += 1, - ProtocolEngine::Constrained => self.stats.constrained_selections += 1, + ProtocolEngine::Quic => { + self.stats.quic_selections.fetch_add(1, Ordering::Relaxed); + } + ProtocolEngine::Constrained => { + self.stats + .constrained_selections + .fetch_add(1, Ordering::Relaxed); + } } tracing::debug!( @@ -962,12 +1004,13 @@ impl ConnectionRouter { } } - /// Select engine with fallback support + /// Select engine with fallback support. /// - /// If the preferred engine is unavailable (e.g., QUIC endpoint not initialized), - /// this method will attempt to use the fallback engine. + /// If the preferred engine is unavailable (e.g., QUIC endpoint not + /// initialized), this method will attempt to use the fallback engine. + /// Takes `&self` — all mutations are via atomic counters. pub fn select_engine_with_fallback( - &mut self, + &self, capabilities: &TransportCapabilities, quic_available: bool, constrained_available: bool, @@ -979,7 +1022,9 @@ impl ConnectionRouter { ProtocolEngine::Quic if quic_available => (ProtocolEngine::Quic, preferred), ProtocolEngine::Quic if constrained_available => { // Fall back to constrained - self.stats.fallback_selections += 1; + self.stats + .fallback_selections + .fetch_add(1, Ordering::Relaxed); tracing::warn!( preferred = "QUIC", fallback = "Constrained", @@ -1000,7 +1045,9 @@ impl ConnectionRouter { } ProtocolEngine::Constrained if quic_available && capabilities.supports_full_quic() => { // Fall back to QUIC (only if transport supports it) - self.stats.fallback_selections += 1; + self.stats + .fallback_selections + .fetch_add(1, Ordering::Relaxed); tracing::warn!( preferred = "Constrained", fallback = "QUIC", @@ -1033,17 +1080,27 @@ impl ConnectionRouter { } }; - // Adjust stats for fallback + // Adjust stats for fallback: the inner select_engine_detailed call + // incremented the *preferred* counter, so when we actually fell + // back we need to decrement it and increment the one we chose. if result.is_fallback { match engine { ProtocolEngine::Quic => { - self.stats.quic_selections += 1; - self.stats.constrained_selections = - self.stats.constrained_selections.saturating_sub(1); + self.stats.quic_selections.fetch_add(1, Ordering::Relaxed); + // saturating_sub via CAS loop on relaxed load/store + let cur = self.stats.constrained_selections.load(Ordering::Relaxed); + self.stats + .constrained_selections + .store(cur.saturating_sub(1), Ordering::Relaxed); } ProtocolEngine::Constrained => { - self.stats.constrained_selections += 1; - self.stats.quic_selections = self.stats.quic_selections.saturating_sub(1); + self.stats + .constrained_selections + .fetch_add(1, Ordering::Relaxed); + let cur = self.stats.quic_selections.load(Ordering::Relaxed); + self.stats + .quic_selections + .store(cur.saturating_sub(1), Ordering::Relaxed); } } } @@ -1051,13 +1108,17 @@ impl ConnectionRouter { Ok(result) } - /// Select engine based on destination address - pub fn select_engine_for_addr(&mut self, addr: &TransportAddr) -> ProtocolEngine { + /// Select engine based on destination address. + /// + /// Takes `&self` so the hot send path can hold only a read lock. + pub fn select_engine_for_addr(&self, addr: &TransportAddr) -> ProtocolEngine { self.select_engine_for_addr_detailed(addr).engine } - /// Select engine based on destination address with detailed result - pub fn select_engine_for_addr_detailed(&mut self, addr: &TransportAddr) -> SelectionResult { + /// Select engine based on destination address with detailed result. + /// + /// Takes `&self` so the hot send path can hold only a read lock. + pub fn select_engine_for_addr_detailed(&self, addr: &TransportAddr) -> SelectionResult { // Determine capabilities based on address type let capabilities = Self::capabilities_for_addr(addr); self.select_engine_detailed(&capabilities) @@ -1416,17 +1477,17 @@ mod tests { #[test] fn test_engine_selection_for_quic() { - let mut router = ConnectionRouter::new(RouterConfig::default()); + let router = ConnectionRouter::new(RouterConfig::default()); let addr = TransportAddr::Quic("127.0.0.1:9000".parse().unwrap()); let engine = router.select_engine_for_addr(&addr); assert_eq!(engine, ProtocolEngine::Quic); - assert_eq!(router.stats().quic_selections, 1); + assert_eq!(router.stats().quic_selections.load(Ordering::Relaxed), 1); } #[test] fn test_engine_selection_for_ble() { - let mut router = ConnectionRouter::new(RouterConfig::default()); + let router = ConnectionRouter::new(RouterConfig::default()); let addr = TransportAddr::Ble { mac: [0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF], psm: 128, @@ -1434,12 +1495,18 @@ mod tests { let engine = router.select_engine_for_addr(&addr); assert_eq!(engine, ProtocolEngine::Constrained); - assert_eq!(router.stats().constrained_selections, 1); + assert_eq!( + router + .stats() + .constrained_selections + .load(Ordering::Relaxed), + 1 + ); } #[test] fn test_engine_selection_for_lora() { - let mut router = ConnectionRouter::new(RouterConfig::default()); + let router = ConnectionRouter::new(RouterConfig::default()); let addr = TransportAddr::LoRa { dev_addr: [0x12, 0x34, 0x56, 0x78], freq_hz: 868_000_000, @@ -1549,7 +1616,7 @@ mod tests { #[test] fn test_router_stats() { - let mut router = ConnectionRouter::new(RouterConfig::default()); + let router = ConnectionRouter::new(RouterConfig::default()); // Make some selections let quic_addr = TransportAddr::Quic("127.0.0.1:9000".parse().unwrap()); @@ -1563,8 +1630,8 @@ mod tests { let _ = router.select_engine_for_addr(&ble_addr); let stats = router.stats(); - assert_eq!(stats.quic_selections, 2); - assert_eq!(stats.constrained_selections, 1); + assert_eq!(stats.quic_selections.load(Ordering::Relaxed), 2); + assert_eq!(stats.constrained_selections.load(Ordering::Relaxed), 1); } #[test] @@ -1608,7 +1675,7 @@ mod tests { #[test] fn test_select_engine_detailed_udp() { - let mut router = ConnectionRouter::new(RouterConfig::default()); + let router = ConnectionRouter::new(RouterConfig::default()); let capabilities = TransportCapabilities::broadband(); let result = router.select_engine_detailed(&capabilities); @@ -1620,7 +1687,7 @@ mod tests { #[test] fn test_select_engine_detailed_ble() { - let mut router = ConnectionRouter::new(RouterConfig::default()); + let router = ConnectionRouter::new(RouterConfig::default()); let capabilities = TransportCapabilities::ble(); let result = router.select_engine_detailed(&capabilities); @@ -1634,7 +1701,7 @@ mod tests { // Configure router to prefer constrained even for broadband let mut config = RouterConfig::default(); config.prefer_quic = false; - let mut router = ConnectionRouter::new(config); + let router = ConnectionRouter::new(config); let capabilities = TransportCapabilities::broadband(); let result = router.select_engine_detailed(&capabilities); @@ -1644,7 +1711,7 @@ mod tests { #[test] fn test_select_engine_with_fallback_quic_available() { - let mut router = ConnectionRouter::new(RouterConfig::default()); + let router = ConnectionRouter::new(RouterConfig::default()); let capabilities = TransportCapabilities::broadband(); let result = router @@ -1656,7 +1723,7 @@ mod tests { #[test] fn test_select_engine_with_fallback_to_constrained() { - let mut router = ConnectionRouter::new(RouterConfig::default()); + let router = ConnectionRouter::new(RouterConfig::default()); let capabilities = TransportCapabilities::broadband(); // QUIC unavailable, constrained available @@ -1666,13 +1733,16 @@ mod tests { assert_eq!(result.engine, ProtocolEngine::Constrained); assert!(result.is_fallback); assert_eq!(result.reason, SelectionReason::QuicUnavailableFallback); - assert_eq!(router.stats().fallback_selections, 1); + assert_eq!( + router.stats().fallback_selections.load(Ordering::Relaxed), + 1 + ); } #[test] fn test_select_engine_with_fallback_constrained_preferred() { let config = RouterConfig::for_ble_focus(); - let mut router = ConnectionRouter::new(config); + let router = ConnectionRouter::new(config); let capabilities = TransportCapabilities::broadband(); // Constrained preferred but unavailable, QUIC available @@ -1690,7 +1760,7 @@ mod tests { #[test] fn test_select_engine_with_fallback_no_engines() { - let mut router = ConnectionRouter::new(RouterConfig::default()); + let router = ConnectionRouter::new(RouterConfig::default()); let capabilities = TransportCapabilities::broadband(); // Neither engine available @@ -1766,16 +1836,22 @@ mod tests { #[test] fn test_fallback_stats_tracking() { - let mut router = ConnectionRouter::new(RouterConfig::default()); + let router = ConnectionRouter::new(RouterConfig::default()); let capabilities = TransportCapabilities::broadband(); // Normal selection - no fallback let _ = router.select_engine_with_fallback(&capabilities, true, true); - assert_eq!(router.stats().fallback_selections, 0); + assert_eq!( + router.stats().fallback_selections.load(Ordering::Relaxed), + 0 + ); // Fallback selection let _ = router.select_engine_with_fallback(&capabilities, false, true); - assert_eq!(router.stats().fallback_selections, 1); + assert_eq!( + router.stats().fallback_selections.load(Ordering::Relaxed), + 1 + ); } // ======================================================================== @@ -1892,7 +1968,7 @@ mod tests { fn test_router_with_fallback_quic_unavailable_but_transport_supports() { // When QUIC is unavailable but transport supports QUIC, // should fall back to constrained - let mut router = ConnectionRouter::new(RouterConfig::default()); + let router = ConnectionRouter::new(RouterConfig::default()); let capabilities = TransportCapabilities::broadband(); let result = router diff --git a/src/p2p_endpoint.rs b/src/p2p_endpoint.rs index 9c45749b..1cc52dfa 100644 --- a/src/p2p_endpoint.rs +++ b/src/p2p_endpoint.rs @@ -2264,11 +2264,18 @@ impl P2pEndpoint { } }; - // Select protocol engine based on transport address - let engine = { - let mut router = self.router.write().await; - router.select_engine_for_addr(&transport_addr) - }; + // Select protocol engine based on transport address. + // + // Uses a read lock: `select_engine_for_addr` takes `&self` and + // updates its stats counters via atomics. The previous code held + // an exclusive write lock here, which serialised every outbound + // send on the endpoint through a single lock and was a dominant + // contention point at high node counts (1000-node testnet). + let engine = self + .router + .read() + .await + .select_engine_for_addr(&transport_addr); match engine { crate::transport::ProtocolEngine::Quic => { diff --git a/tests/constrained_integration.rs b/tests/constrained_integration.rs index 2b30b6a1..f6ce0aac 100644 --- a/tests/constrained_integration.rs +++ b/tests/constrained_integration.rs @@ -268,11 +268,12 @@ fn test_connection_close() { use saorsa_transport::connection_router::{ConnectionRouter, RouterConfig}; use saorsa_transport::transport::ProtocolEngine; +use std::sync::atomic::Ordering; /// Test that ConnectionRouter correctly selects Constrained engine for BLE addresses #[test] fn test_router_selects_constrained_for_ble() { - let mut router = ConnectionRouter::new(RouterConfig::default()); + let router = ConnectionRouter::new(RouterConfig::default()); let ble_addr = TransportAddr::Ble { mac: [0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF], @@ -288,14 +289,14 @@ fn test_router_selects_constrained_for_ble() { // Verify stats tracking let stats = router.stats(); - assert_eq!(stats.constrained_selections, 1); - assert_eq!(stats.quic_selections, 0); + assert_eq!(stats.constrained_selections.load(Ordering::Relaxed), 1); + assert_eq!(stats.quic_selections.load(Ordering::Relaxed), 0); } /// Test that ConnectionRouter correctly selects QUIC engine for UDP addresses #[test] fn test_router_selects_quic_for_udp() { - let mut router = ConnectionRouter::new(RouterConfig::default()); + let router = ConnectionRouter::new(RouterConfig::default()); let udp_addr = TransportAddr::Udp("127.0.0.1:9000".parse().unwrap()); @@ -304,14 +305,14 @@ fn test_router_selects_quic_for_udp() { // Verify stats tracking let stats = router.stats(); - assert_eq!(stats.quic_selections, 1); - assert_eq!(stats.constrained_selections, 0); + assert_eq!(stats.quic_selections.load(Ordering::Relaxed), 1); + assert_eq!(stats.constrained_selections.load(Ordering::Relaxed), 0); } /// Test mixed transport selection (UDP and BLE peers) #[test] fn test_mixed_transport_selection() { - let mut router = ConnectionRouter::new(RouterConfig::default()); + let router = ConnectionRouter::new(RouterConfig::default()); let udp_addr = TransportAddr::Udp("192.168.1.100:8080".parse().unwrap()); let ble_addr = TransportAddr::Ble { @@ -339,8 +340,8 @@ fn test_mixed_transport_selection() { // Verify cumulative stats let stats = router.stats(); - assert_eq!(stats.quic_selections, 1); - assert_eq!(stats.constrained_selections, 2); + assert_eq!(stats.quic_selections.load(Ordering::Relaxed), 1); + assert_eq!(stats.constrained_selections.load(Ordering::Relaxed), 2); } /// Test synthetic socket address generation for BLE From 45bf73803d146ba878425a95a94d4c571e319764 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Mon, 6 Apr 2026 19:46:12 +0200 Subject: [PATCH 20/43] =?UTF-8?q?fix:=20address=20PR=20#42=20review=20?= =?UTF-8?q?=E2=80=94=20atomic=20fallback=20decrement,=20read-lock=20connec?= =?UTF-8?q?t=5Ftransport,=20bench=20cleanup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three follow-ups from the PR #42 review: 1. `connection_router.rs`: the fallback-adjustment path decremented `{quic,constrained}_selections` via a plain `load` + `store` — not the "CAS loop" the comment claimed. Under concurrent callers (which this PR is designed to enable) that pattern silently loses updates, including increments from other concurrent `select_engine_detailed` callers. Replace with `fetch_update(Relaxed, Relaxed, saturating_sub)`. 2. `p2p_endpoint.rs`: `connect_transport` held an exclusive write lock on the router just to call `select_engine_for_addr`, even though that method is now `&self`. Downgrade engine selection to a read lock; only the Constrained branch re-acquires a write lock for `router.connect(addr)`. The QUIC branch now holds no router lock. 3. `benches/connection_router.rs`: the previous PR updated test `let mut router`s but missed the bench, leaving 9 unused_mut warnings that broke `clippy --all-targets -D warnings` on CI. Line 187 (`bench_constrained_connect`) correctly retains `mut` for its `router.connect` call. Verified: fmt clean, clippy clean, connection_router (45) + constrained_integration (24) + p2p_endpoint (20) tests all pass. Co-Authored-By: Claude Opus 4.6 (1M context) --- benches/connection_router.rs | 18 +++++++++--------- src/connection_router.rs | 21 ++++++++++++--------- src/p2p_endpoint.rs | 15 ++++++++++----- 3 files changed, 31 insertions(+), 23 deletions(-) diff --git a/benches/connection_router.rs b/benches/connection_router.rs index f9c429b2..d8db9e5d 100644 --- a/benches/connection_router.rs +++ b/benches/connection_router.rs @@ -38,7 +38,7 @@ fn bench_engine_selection(c: &mut Criterion) { // Benchmark UDP address selection group.bench_function("udp_address", |b| { - let mut router = ConnectionRouter::new(RouterConfig::default()); + let router = ConnectionRouter::new(RouterConfig::default()); b.iter(|| { let engine = router.select_engine_for_addr(black_box(&udp_transport)); black_box(engine) @@ -47,7 +47,7 @@ fn bench_engine_selection(c: &mut Criterion) { // Benchmark BLE address selection group.bench_function("ble_address", |b| { - let mut router = ConnectionRouter::new(RouterConfig::default()); + let router = ConnectionRouter::new(RouterConfig::default()); b.iter(|| { let engine = router.select_engine_for_addr(black_box(&ble_transport)); black_box(engine) @@ -56,7 +56,7 @@ fn bench_engine_selection(c: &mut Criterion) { // Benchmark LoRa address selection group.bench_function("lora_address", |b| { - let mut router = ConnectionRouter::new(RouterConfig::default()); + let router = ConnectionRouter::new(RouterConfig::default()); b.iter(|| { let engine = router.select_engine_for_addr(black_box(&lora_transport)); black_box(engine) @@ -76,7 +76,7 @@ fn bench_engine_selection_detailed(c: &mut Criterion) { // Benchmark broadband selection group.bench_function("broadband_detailed", |b| { - let mut router = ConnectionRouter::new(RouterConfig::default()); + let router = ConnectionRouter::new(RouterConfig::default()); b.iter(|| { let result = router.select_engine_detailed(black_box(&broadband_caps)); black_box(result) @@ -85,7 +85,7 @@ fn bench_engine_selection_detailed(c: &mut Criterion) { // Benchmark BLE selection group.bench_function("ble_detailed", |b| { - let mut router = ConnectionRouter::new(RouterConfig::default()); + let router = ConnectionRouter::new(RouterConfig::default()); b.iter(|| { let result = router.select_engine_detailed(black_box(&ble_caps)); black_box(result) @@ -94,7 +94,7 @@ fn bench_engine_selection_detailed(c: &mut Criterion) { // Benchmark LoRa selection group.bench_function("lora_detailed", |b| { - let mut router = ConnectionRouter::new(RouterConfig::default()); + let router = ConnectionRouter::new(RouterConfig::default()); b.iter(|| { let result = router.select_engine_detailed(black_box(&lora_caps)); black_box(result) @@ -112,7 +112,7 @@ fn bench_fallback_selection(c: &mut Criterion) { // Benchmark with QUIC available group.bench_function("quic_available", |b| { - let mut router = ConnectionRouter::new(RouterConfig::default()); + let router = ConnectionRouter::new(RouterConfig::default()); b.iter(|| { let result = router.select_engine_with_fallback( black_box(&broadband_caps), @@ -125,7 +125,7 @@ fn bench_fallback_selection(c: &mut Criterion) { // Benchmark with fallback needed group.bench_function("quic_fallback", |b| { - let mut router = ConnectionRouter::new(RouterConfig::default()); + let router = ConnectionRouter::new(RouterConfig::default()); b.iter(|| { let result = router.select_engine_with_fallback( black_box(&broadband_caps), @@ -215,7 +215,7 @@ fn bench_stats_tracking(c: &mut Criterion) { // Benchmark selection with stats update group.bench_function("selection_with_stats", |b| { - let mut router = ConnectionRouter::new(RouterConfig::default()); + let router = ConnectionRouter::new(RouterConfig::default()); b.iter(|| { let _ = router.select_engine_for_addr(black_box(&udp_addr)); let _ = router.select_engine_for_addr(black_box(&ble_addr)); diff --git a/src/connection_router.rs b/src/connection_router.rs index d667f498..0827a024 100644 --- a/src/connection_router.rs +++ b/src/connection_router.rs @@ -1083,24 +1083,27 @@ impl ConnectionRouter { // Adjust stats for fallback: the inner select_engine_detailed call // incremented the *preferred* counter, so when we actually fell // back we need to decrement it and increment the one we chose. + // Both operations must be atomic so concurrent callers (now allowed + // because the function takes `&self`) cannot lose updates. if result.is_fallback { match engine { ProtocolEngine::Quic => { self.stats.quic_selections.fetch_add(1, Ordering::Relaxed); - // saturating_sub via CAS loop on relaxed load/store - let cur = self.stats.constrained_selections.load(Ordering::Relaxed); - self.stats - .constrained_selections - .store(cur.saturating_sub(1), Ordering::Relaxed); + let _ = self.stats.constrained_selections.fetch_update( + Ordering::Relaxed, + Ordering::Relaxed, + |v| Some(v.saturating_sub(1)), + ); } ProtocolEngine::Constrained => { self.stats .constrained_selections .fetch_add(1, Ordering::Relaxed); - let cur = self.stats.quic_selections.load(Ordering::Relaxed); - self.stats - .quic_selections - .store(cur.saturating_sub(1), Ordering::Relaxed); + let _ = self.stats.quic_selections.fetch_update( + Ordering::Relaxed, + Ordering::Relaxed, + |v| Some(v.saturating_sub(1)), + ); } } } diff --git a/src/p2p_endpoint.rs b/src/p2p_endpoint.rs index 1cc52dfa..e2a60ef5 100644 --- a/src/p2p_endpoint.rs +++ b/src/p2p_endpoint.rs @@ -1014,9 +1014,14 @@ impl P2pEndpoint { return Err(EndpointError::ShuttingDown); } - // Use the router to determine the appropriate engine - let mut router = self.router.write().await; - let engine = router.select_engine_for_addr(addr); + // Use the router to determine the appropriate engine. + // + // Engine selection only needs a read lock now that + // `select_engine_for_addr` takes `&self` and uses atomic counters. + // The Constrained branch then re-acquires a write lock for + // `router.connect(addr)`, which is still `&mut self`. The QUIC + // branch never needs a write lock at all. + let engine = self.router.read().await.select_engine_for_addr(addr); info!("Connecting to {} via {:?} engine", addr, engine); @@ -1029,11 +1034,11 @@ impl P2pEndpoint { addr )) })?; - drop(router); // Release lock before async operation self.connect(socket_addr).await } ProtocolEngine::Constrained => { - // For constrained transports, use the router's constrained connection + // For constrained transports, use the router's constrained connection. + let mut router = self.router.write().await; let _routed = router.connect(addr).map_err(|e| { EndpointError::Connection(format!("Constrained connection failed: {}", e)) })?; From 386c6e84602da2f1b4e871ac78cadf92f4dde725 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Mon, 6 Apr 2026 20:42:47 +0200 Subject: [PATCH 21/43] refactor!: make ConnectionRouter fully lock-free and interior-mutable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to PR #42 review feedback. The previous commit dropped the write-lock from the `send` hot path but kept `Arc>` in `P2pEndpoint` and `pub AtomicU64` fields on `RouterStats`. This finishes the job: 1. `RouterStats` fields are now private. External consumers read via accessor methods (`quic_selections() -> u64`, etc.) or capture a plain-`u64` point-in-time view via the new `RouterStatsSnapshot` value type returned from `RouterStats::snapshot()`. The manual `Clone` impl is gone — `RouterStatsSnapshot` replaces it for the "take a copy" use-case. `P2pEndpoint::routing_stats()` now returns `RouterStatsSnapshot` and is no longer `async`. 2. Every counter in `RouterStats` is an `AtomicU64` (not just the selection counters). This lets every mutating method on `ConnectionRouter` run under `&self`. 3. `ConnectionRouter` is now fully interior-mutable: - `constrained_transport: Option` → `OnceLock` (lazy init under `&self`, once set never revoked). - `next_quic_id: u64` → `AtomicU64` (`fetch_add`). - `set_quic_endpoint` deleted along with its redundant caller in `P2pEndpoint::new` — `with_full_config` already installs it. - `connect`, `connect_async`, `connect_peer`, `connect_quic_async`, `connect_constrained`, `accept_quic`, `poll_events`, and `process_constrained_incoming` all take `&self`. 4. `P2pEndpoint::router` is now `Arc` instead of `Arc>`. `send()` and `connect_transport()` no longer `.read().await`/`.write().await` on the router at all — the previously-remaining write lock in `connect_transport`'s constrained branch is gone. `P2pEndpoint::router()` now returns `&Arc` synchronously. 5. Added a compile-time `assert_send_sync::()` so any future change that accidentally adds a non-`Sync` field fails the build immediately instead of surfacing a confusing error at the `P2pEndpoint` clone site. 6. Documented the TOCTOU assumption in `connect_transport`: engine selection and engine-use are two separate `&self` calls, but the race is closed by construction (QUIC endpoint fixed at construction, constrained transport lazy-init via `OnceLock` and never torn down). Comment explains what a future change must do if that invariant is relaxed. BREAKING: - `RouterStats::{quic_connections,constrained_connections,*_bytes_*, connection_failures,quic_selections,constrained_selections, fallback_selections,events_processed}` are no longer `pub` fields. Use the same-named accessor methods (returning `u64`), or `RouterStats::snapshot()` for a plain-`u64` struct. - `ConnectionRouter::set_quic_endpoint` has been removed. - `P2pEndpoint::router()` is no longer `async` and returns `&Arc` instead of an `RwLockReadGuard`. - `P2pEndpoint::routing_stats()` is no longer `async` and returns `RouterStatsSnapshot` instead of `RouterStats`. - Every method on `ConnectionRouter` now takes `&self` — external callers holding a `&mut ConnectionRouter` should switch to `&self` (or `Arc`). Verified: fmt clean, clippy --all-targets -D warnings clean, connection_router (45) + constrained_integration (24) + p2p_endpoint (20) tests all pass. Co-Authored-By: Claude Opus 4.6 (1M context) --- benches/connection_router.rs | 4 +- src/connection_router.rs | 380 +++++++++++++++++++------------ src/p2p_endpoint.rs | 78 ++++--- tests/constrained_integration.rs | 13 +- 4 files changed, 284 insertions(+), 191 deletions(-) diff --git a/benches/connection_router.rs b/benches/connection_router.rs index d8db9e5d..f059b106 100644 --- a/benches/connection_router.rs +++ b/benches/connection_router.rs @@ -184,7 +184,7 @@ fn bench_constrained_connect(c: &mut Criterion) { // Benchmark constrained connection creation group.bench_function("ble_connect", |b| { - let mut router = ConnectionRouter::new(RouterConfig::default()); + let router = ConnectionRouter::new(RouterConfig::default()); b.iter(|| { let result = router.connect(black_box(&ble_addr)); black_box(result) @@ -219,7 +219,7 @@ fn bench_stats_tracking(c: &mut Criterion) { b.iter(|| { let _ = router.select_engine_for_addr(black_box(&udp_addr)); let _ = router.select_engine_for_addr(black_box(&ble_addr)); - let stats = router.stats().clone(); + let stats = router.stats().snapshot(); black_box(stats) }); }); diff --git a/src/connection_router.rs b/src/connection_router.rs index 0827a024..68ff26f2 100644 --- a/src/connection_router.rs +++ b/src/connection_router.rs @@ -63,8 +63,8 @@ use std::fmt; use std::net::SocketAddr; -use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, OnceLock}; use crate::constrained::{ AdapterEvent, ConnectionId as ConstrainedConnId, ConstrainedError, ConstrainedHandle, @@ -789,96 +789,202 @@ impl RouterEvent { /// Router statistics. /// -/// Selection counters (`quic_selections`, `constrained_selections`, -/// `fallback_selections`) are stored as [`AtomicU64`] so the selection -/// methods can take `&self` and be called under a `RwLock::read()` guard. -/// Without this, every outbound send on a `P2pEndpoint` had to acquire an -/// exclusive write lock on the router just to bump a counter, serialising -/// all sends through a single lock (see p2p_endpoint.rs `send()`). +/// All counters are lock-free [`AtomicU64`]s. Fields are private; external +/// code reads them via the accessor methods or captures a consistent-ish +/// point-in-time view via [`RouterStats::snapshot`]. /// -/// Other counters are still plain `u64` because they are only touched on -/// `&mut self` paths (connect/accept/event-loop). +/// Making every counter atomic lets every mutating method on +/// [`ConnectionRouter`] take `&self`. Combined with lazy-init of the +/// constrained transport via [`OnceLock`], this removes the need to wrap +/// the router in a `RwLock` at all — concurrent sends do not block each +/// other on stat updates. +/// +/// Ordering: all operations use [`Ordering::Relaxed`]. These counters are +/// purely diagnostic; they do not synchronise any other state. #[derive(Debug, Default)] pub struct RouterStats { /// Total connections routed through QUIC - pub quic_connections: u64, + quic_connections: AtomicU64, /// Total connections routed through Constrained - pub constrained_connections: u64, + constrained_connections: AtomicU64, /// Total bytes sent via QUIC - pub quic_bytes_sent: u64, + quic_bytes_sent: AtomicU64, /// Total bytes sent via Constrained - pub constrained_bytes_sent: u64, + constrained_bytes_sent: AtomicU64, /// Total bytes received via QUIC - pub quic_bytes_received: u64, + quic_bytes_received: AtomicU64, /// Total bytes received via Constrained - pub constrained_bytes_received: u64, + constrained_bytes_received: AtomicU64, /// Connection failures - pub connection_failures: u64, + connection_failures: AtomicU64, - /// Engine selection decisions (QUIC chosen). Atomic so the selection - /// path does not require exclusive access. - pub quic_selections: AtomicU64, + /// Engine selection decisions (QUIC chosen) + quic_selections: AtomicU64, - /// Engine selection decisions (Constrained chosen). Atomic so the - /// selection path does not require exclusive access. - pub constrained_selections: AtomicU64, + /// Engine selection decisions (Constrained chosen) + constrained_selections: AtomicU64, - /// Fallback selections (when preferred engine unavailable). Atomic so - /// the selection path does not require exclusive access. - pub fallback_selections: AtomicU64, + /// Fallback selections (when preferred engine unavailable) + fallback_selections: AtomicU64, /// Total events processed - pub events_processed: u64, + events_processed: AtomicU64, } -impl Clone for RouterStats { - fn clone(&self) -> Self { - Self { - quic_connections: self.quic_connections, - constrained_connections: self.constrained_connections, - quic_bytes_sent: self.quic_bytes_sent, - constrained_bytes_sent: self.constrained_bytes_sent, - quic_bytes_received: self.quic_bytes_received, - constrained_bytes_received: self.constrained_bytes_received, - connection_failures: self.connection_failures, - quic_selections: AtomicU64::new(self.quic_selections.load(Ordering::Relaxed)), - constrained_selections: AtomicU64::new( - self.constrained_selections.load(Ordering::Relaxed), - ), - fallback_selections: AtomicU64::new(self.fallback_selections.load(Ordering::Relaxed)), - events_processed: self.events_processed, +impl RouterStats { + /// Total connections routed through QUIC. + pub fn quic_connections(&self) -> u64 { + self.quic_connections.load(Ordering::Relaxed) + } + + /// Total connections routed through Constrained. + pub fn constrained_connections(&self) -> u64 { + self.constrained_connections.load(Ordering::Relaxed) + } + + /// Total bytes sent via QUIC. + pub fn quic_bytes_sent(&self) -> u64 { + self.quic_bytes_sent.load(Ordering::Relaxed) + } + + /// Total bytes sent via Constrained. + pub fn constrained_bytes_sent(&self) -> u64 { + self.constrained_bytes_sent.load(Ordering::Relaxed) + } + + /// Total bytes received via QUIC. + pub fn quic_bytes_received(&self) -> u64 { + self.quic_bytes_received.load(Ordering::Relaxed) + } + + /// Total bytes received via Constrained. + pub fn constrained_bytes_received(&self) -> u64 { + self.constrained_bytes_received.load(Ordering::Relaxed) + } + + /// Connection failures. + pub fn connection_failures(&self) -> u64 { + self.connection_failures.load(Ordering::Relaxed) + } + + /// Engine-selection decisions where QUIC was chosen. + pub fn quic_selections(&self) -> u64 { + self.quic_selections.load(Ordering::Relaxed) + } + + /// Engine-selection decisions where Constrained was chosen. + pub fn constrained_selections(&self) -> u64 { + self.constrained_selections.load(Ordering::Relaxed) + } + + /// Fallback selections (preferred engine unavailable, alternate used). + pub fn fallback_selections(&self) -> u64 { + self.fallback_selections.load(Ordering::Relaxed) + } + + /// Total router events processed. + pub fn events_processed(&self) -> u64 { + self.events_processed.load(Ordering::Relaxed) + } + + /// Capture a plain-`u64` snapshot of all counters. + /// + /// The snapshot is *not* a globally consistent point-in-time view: + /// because each field is loaded independently, a concurrent update can + /// land between two loads. Selection counters in particular can + /// transiently disagree because the fallback path increments one + /// counter and decrements another non-atomically across fields. For + /// rate-calculation and monitoring this is fine; callers that need + /// per-field accuracy should use the individual accessors and accept + /// that they too are only eventually consistent. + pub fn snapshot(&self) -> RouterStatsSnapshot { + RouterStatsSnapshot { + quic_connections: self.quic_connections(), + constrained_connections: self.constrained_connections(), + quic_bytes_sent: self.quic_bytes_sent(), + constrained_bytes_sent: self.constrained_bytes_sent(), + quic_bytes_received: self.quic_bytes_received(), + constrained_bytes_received: self.constrained_bytes_received(), + connection_failures: self.connection_failures(), + quic_selections: self.quic_selections(), + constrained_selections: self.constrained_selections(), + fallback_selections: self.fallback_selections(), + events_processed: self.events_processed(), } } } +/// Plain-`u64` snapshot of [`RouterStats`]. +/// +/// Value type with no atomics, safe to pass across threads, serialise, or +/// diff against a later snapshot for rate calculations. Produced via +/// [`RouterStats::snapshot`]. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct RouterStatsSnapshot { + /// Total connections routed through QUIC. + pub quic_connections: u64, + /// Total connections routed through Constrained. + pub constrained_connections: u64, + /// Total bytes sent via QUIC. + pub quic_bytes_sent: u64, + /// Total bytes sent via Constrained. + pub constrained_bytes_sent: u64, + /// Total bytes received via QUIC. + pub quic_bytes_received: u64, + /// Total bytes received via Constrained. + pub constrained_bytes_received: u64, + /// Connection failures. + pub connection_failures: u64, + /// Engine-selection decisions where QUIC was chosen. + pub quic_selections: u64, + /// Engine-selection decisions where Constrained was chosen. + pub constrained_selections: u64, + /// Fallback selections (preferred engine unavailable). + pub fallback_selections: u64, + /// Total router events processed. + pub events_processed: u64, +} + /// Connection router for automatic protocol engine selection /// /// The router examines transport capabilities and routes connections /// through either QUIC or the Constrained engine as appropriate. +/// +/// # Thread safety +/// +/// All methods take `&self` — there is no interior `RwLock`. Stats are +/// atomic, the constrained transport is lazy-initialised via [`OnceLock`], +/// the QUIC endpoint is set at construction time only, and the next +/// QUIC-connection ID is an [`AtomicU64`]. Wrap the router in [`Arc`] and +/// call it from any number of concurrent tasks. pub struct ConnectionRouter { /// Router configuration config: RouterConfig, - /// Constrained transport (created lazily when needed) - constrained_transport: Option, + /// Constrained transport (initialised on first constrained connect). + /// `OnceLock` allows lazy init under `&self`; once set, never revoked. + constrained_transport: OnceLock, /// Transport registry for capability lookups registry: Option>, - /// NAT traversal endpoint for QUIC connections + /// NAT traversal endpoint for QUIC connections. Set at construction + /// time only — there is no API to revoke or replace it, which is what + /// lets the hot send path read it under `&self` without locking. quic_endpoint: Option>, - /// Router statistics + /// Router statistics (all counters atomic, `&self` mutable) stats: RouterStats, - /// Next QUIC connection ID (for tracking) - next_quic_id: u64, + /// Next QUIC connection ID (for tracking). Atomic so + /// `connect_quic_async` / `accept_quic` can run under `&self`. + next_quic_id: AtomicU64, } impl ConnectionRouter { @@ -886,11 +992,11 @@ impl ConnectionRouter { pub fn new(config: RouterConfig) -> Self { Self { config, - constrained_transport: None, + constrained_transport: OnceLock::new(), registry: None, quic_endpoint: None, stats: RouterStats::default(), - next_quic_id: 1, + next_quic_id: AtomicU64::new(1), } } @@ -898,11 +1004,11 @@ impl ConnectionRouter { pub fn with_registry(config: RouterConfig, registry: Arc) -> Self { Self { config, - constrained_transport: None, + constrained_transport: OnceLock::new(), registry: Some(registry), quic_endpoint: None, stats: RouterStats::default(), - next_quic_id: 1, + next_quic_id: AtomicU64::new(1), } } @@ -913,11 +1019,11 @@ impl ConnectionRouter { ) -> Self { Self { config, - constrained_transport: None, + constrained_transport: OnceLock::new(), registry: None, quic_endpoint: Some(quic_endpoint), stats: RouterStats::default(), - next_quic_id: 1, + next_quic_id: AtomicU64::new(1), } } @@ -929,19 +1035,14 @@ impl ConnectionRouter { ) -> Self { Self { config, - constrained_transport: None, + constrained_transport: OnceLock::new(), registry: Some(registry), quic_endpoint: Some(quic_endpoint), stats: RouterStats::default(), - next_quic_id: 1, + next_quic_id: AtomicU64::new(1), } } - /// Set the QUIC endpoint after construction - pub fn set_quic_endpoint(&mut self, endpoint: Arc) { - self.quic_endpoint = Some(endpoint); - } - /// Check if QUIC endpoint is available pub fn is_quic_available(&self) -> bool { self.quic_endpoint.is_some() @@ -1150,7 +1251,7 @@ impl ConnectionRouter { /// /// This method only works for constrained connections. For QUIC connections, /// use `connect_async()` instead. - pub fn connect(&mut self, remote: &TransportAddr) -> Result { + pub fn connect(&self, remote: &TransportAddr) -> Result { let engine = self.select_engine_for_addr(remote); match engine { @@ -1164,7 +1265,7 @@ impl ConnectionRouter { /// This method handles both QUIC and constrained connections. For QUIC connections, /// it requires a peer ID and server name. pub async fn connect_async( - &mut self, + &self, remote: &TransportAddr, server_name: Option<&str>, ) -> Result { @@ -1190,7 +1291,7 @@ impl ConnectionRouter { /// Convenience method for QUIC connections that doesn't require engine selection /// (assumes QUIC is appropriate for the given address). pub async fn connect_peer( - &mut self, + &self, remote_addr: SocketAddr, server_name: &str, ) -> Result { @@ -1202,7 +1303,7 @@ impl ConnectionRouter { /// /// This method returns an error indicating async is required for QUIC connections. /// Use `connect_quic_async` instead for actual QUIC connections. - fn connect_quic(&mut self, remote: &TransportAddr) -> Result { + fn connect_quic(&self, remote: &TransportAddr) -> Result { // QUIC connections require async - this sync version returns an error // directing users to use the async method Err(RouterError::Quic { @@ -1217,7 +1318,7 @@ impl ConnectionRouter { /// /// This method initiates a QUIC connection through the NatTraversalEndpoint. pub async fn connect_quic_async( - &mut self, + &self, remote: &TransportAddr, server_name: &str, ) -> Result { @@ -1234,10 +1335,9 @@ impl ConnectionRouter { // Connect through the NAT traversal endpoint let connection = endpoint.connect_to(server_name, socket_addr).await?; - // Assign connection ID and update stats - let connection_id = self.next_quic_id; - self.next_quic_id += 1; - self.stats.quic_connections += 1; + // Assign connection ID and update stats atomically. + let connection_id = self.next_quic_id.fetch_add(1, Ordering::Relaxed); + self.stats.quic_connections.fetch_add(1, Ordering::Relaxed); tracing::info!( connection_id, @@ -1253,27 +1353,20 @@ impl ConnectionRouter { } /// Connect using the Constrained engine - fn connect_constrained( - &mut self, - remote: &TransportAddr, - ) -> Result { - // Initialize constrained transport if needed - if self.constrained_transport.is_none() { - let transport = ConstrainedTransport::new(self.config.constrained_config.clone()); - self.constrained_transport = Some(transport); - } - - let transport = - self.constrained_transport - .as_ref() - .ok_or(RouterError::NoTransportAvailable { - addr: remote.clone(), - })?; + fn connect_constrained(&self, remote: &TransportAddr) -> Result { + // Lazy-initialise the constrained transport on first use. + // `OnceLock::get_or_init` runs the closure at most once even under + // concurrent callers; all callers then see the same transport. + let transport = self + .constrained_transport + .get_or_init(|| ConstrainedTransport::new(self.config.constrained_config.clone())); let handle = transport.handle(); let connection_id = handle.connect(remote)?; - self.stats.constrained_connections += 1; + self.stats + .constrained_connections + .fetch_add(1, Ordering::Relaxed); Ok(RoutedConnection::Constrained { remote: remote.clone(), @@ -1284,7 +1377,7 @@ impl ConnectionRouter { /// Get the constrained transport handle (for direct access if needed) pub fn constrained_handle(&self) -> Option { - self.constrained_transport.as_ref().map(|t| t.handle()) + self.constrained_transport.get().map(|t| t.handle()) } /// Check if a transport supports QUIC @@ -1295,7 +1388,7 @@ impl ConnectionRouter { /// Check if constrained engine is initialized pub fn is_constrained_initialized(&self) -> bool { - self.constrained_transport.is_some() + self.constrained_transport.get().is_some() } /// Get router statistics @@ -1329,7 +1422,7 @@ impl ConnectionRouter { /// Note: This is a sync method that only polls constrained events. /// For QUIC events, use `poll_events_async()` or the event callback /// mechanism on the NatTraversalEndpoint. - pub fn poll_events(&mut self) -> Vec { + pub fn poll_events(&self) -> Vec { let mut events = Vec::new(); // Collect constrained events and convert to unified format @@ -1337,12 +1430,14 @@ impl ConnectionRouter { while let Some(adapter_event) = handle.next_event() { let router_event = RouterEvent::from_adapter_event(adapter_event, None); - // Update stats based on event type + // Update stats based on event type (atomic RMW, no lock). if let RouterEvent::DataReceived { data, .. } = &router_event { - self.stats.constrained_bytes_received += data.len() as u64; + self.stats + .constrained_bytes_received + .fetch_add(data.len() as u64, Ordering::Relaxed); } - self.stats.events_processed += 1; + self.stats.events_processed.fetch_add(1, Ordering::Relaxed); events.push(router_event); } } @@ -1354,7 +1449,7 @@ impl ConnectionRouter { /// /// This method waits for an incoming connection on the QUIC endpoint /// and returns it wrapped as a RoutedConnection. - pub async fn accept_quic(&mut self) -> Result { + pub async fn accept_quic(&self) -> Result { let endpoint = self .quic_endpoint .as_ref() @@ -1364,10 +1459,9 @@ impl ConnectionRouter { let transport_addr = TransportAddr::Udp(remote_addr); - // Assign connection ID and update stats - let connection_id = self.next_quic_id; - self.next_quic_id += 1; - self.stats.quic_connections += 1; + // Assign connection ID and update stats atomically. + let connection_id = self.next_quic_id.fetch_add(1, Ordering::Relaxed); + self.stats.quic_connections.fetch_add(1, Ordering::Relaxed); tracing::info!( connection_id, @@ -1392,7 +1486,7 @@ impl ConnectionRouter { /// This should be called when data is received from the underlying /// transport (e.g., BLE characteristic notification, LoRa packet). pub fn process_constrained_incoming( - &mut self, + &self, remote: &TransportAddr, data: &[u8], ) -> Result, RouterError> { @@ -1411,10 +1505,12 @@ impl ConnectionRouter { let router_event = RouterEvent::from_adapter_event(adapter_event, Some(remote)); if let RouterEvent::DataReceived { data, .. } = &router_event { - self.stats.constrained_bytes_received += data.len() as u64; + self.stats + .constrained_bytes_received + .fetch_add(data.len() as u64, Ordering::Relaxed); } - self.stats.events_processed += 1; + self.stats.events_processed.fetch_add(1, Ordering::Relaxed); events.push(router_event); } @@ -1444,13 +1540,25 @@ impl fmt::Debug for ConnectionRouter { .field("config", &self.config) .field( "constrained_initialized", - &self.constrained_transport.is_some(), + &self.constrained_transport.get().is_some(), ) .field("stats", &self.stats) .finish() } } +// Compile-time check: `Arc` must be safe to share +// across tasks, so `ConnectionRouter` needs to be both `Send` and `Sync`. +// This static assertion fails the build early if a future change (e.g. +// adding a non-`Sync` field) breaks that invariant, instead of surfacing +// a confusing error deep inside the `P2pEndpoint` clone path. +const _: fn() = || { + fn assert_send_sync() {} + assert_send_sync::(); + assert_send_sync::(); + assert_send_sync::(); +}; + #[cfg(test)] mod tests { use super::*; @@ -1485,7 +1593,7 @@ mod tests { let engine = router.select_engine_for_addr(&addr); assert_eq!(engine, ProtocolEngine::Quic); - assert_eq!(router.stats().quic_selections.load(Ordering::Relaxed), 1); + assert_eq!(router.stats().quic_selections(), 1); } #[test] @@ -1498,13 +1606,7 @@ mod tests { let engine = router.select_engine_for_addr(&addr); assert_eq!(engine, ProtocolEngine::Constrained); - assert_eq!( - router - .stats() - .constrained_selections - .load(Ordering::Relaxed), - 1 - ); + assert_eq!(router.stats().constrained_selections(), 1); } #[test] @@ -1535,7 +1637,7 @@ mod tests { #[test] fn test_connect_constrained() { - let mut router = ConnectionRouter::new(RouterConfig::default()); + let router = ConnectionRouter::new(RouterConfig::default()); let addr = TransportAddr::Ble { mac: [0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF], psm: 128, @@ -1548,14 +1650,14 @@ mod tests { assert!(conn.is_constrained()); assert_eq!(conn.engine(), ProtocolEngine::Constrained); assert_eq!(conn.remote_addr(), &addr); - assert_eq!(router.stats().constrained_connections, 1); + assert_eq!(router.stats().constrained_connections(), 1); } #[test] fn test_connect_quic_requires_async() { // QUIC connections require async - the sync connect() method // should return an error for QUIC addresses - let mut router = ConnectionRouter::new(RouterConfig::default()); + let router = ConnectionRouter::new(RouterConfig::default()); let addr = TransportAddr::Quic("127.0.0.1:9000".parse().unwrap()); let result = router.connect(&addr); @@ -1588,7 +1690,7 @@ mod tests { #[test] fn test_routed_connection_send_constrained() { - let mut router = ConnectionRouter::new(RouterConfig::default()); + let router = ConnectionRouter::new(RouterConfig::default()); let addr = TransportAddr::Ble { mac: [0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF], psm: 128, @@ -1606,7 +1708,7 @@ mod tests { #[test] fn test_routed_connection_close() { - let mut router = ConnectionRouter::new(RouterConfig::default()); + let router = ConnectionRouter::new(RouterConfig::default()); let addr = TransportAddr::Ble { mac: [0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF], psm: 128, @@ -1633,13 +1735,13 @@ mod tests { let _ = router.select_engine_for_addr(&ble_addr); let stats = router.stats(); - assert_eq!(stats.quic_selections.load(Ordering::Relaxed), 2); - assert_eq!(stats.constrained_selections.load(Ordering::Relaxed), 1); + assert_eq!(stats.quic_selections(), 2); + assert_eq!(stats.constrained_selections(), 1); } #[test] fn test_constrained_handle_access() { - let mut router = ConnectionRouter::new(RouterConfig::default()); + let router = ConnectionRouter::new(RouterConfig::default()); // Initially no handle assert!(router.constrained_handle().is_none()); @@ -1736,10 +1838,7 @@ mod tests { assert_eq!(result.engine, ProtocolEngine::Constrained); assert!(result.is_fallback); assert_eq!(result.reason, SelectionReason::QuicUnavailableFallback); - assert_eq!( - router.stats().fallback_selections.load(Ordering::Relaxed), - 1 - ); + assert_eq!(router.stats().fallback_selections(), 1); } #[test] @@ -1824,7 +1923,7 @@ mod tests { #[test] fn test_is_constrained_initialized() { - let mut router = ConnectionRouter::new(RouterConfig::default()); + let router = ConnectionRouter::new(RouterConfig::default()); assert!(!router.is_constrained_initialized()); // Initialize by connecting to BLE @@ -1844,17 +1943,11 @@ mod tests { // Normal selection - no fallback let _ = router.select_engine_with_fallback(&capabilities, true, true); - assert_eq!( - router.stats().fallback_selections.load(Ordering::Relaxed), - 0 - ); + assert_eq!(router.stats().fallback_selections(), 0); // Fallback selection let _ = router.select_engine_with_fallback(&capabilities, false, true); - assert_eq!( - router.stats().fallback_selections.load(Ordering::Relaxed), - 1 - ); + assert_eq!(router.stats().fallback_selections(), 1); } // ======================================================================== @@ -1881,7 +1974,7 @@ mod tests { #[test] fn test_routed_connection_accessors_constrained() { - let mut router = ConnectionRouter::new(RouterConfig::default()); + let router = ConnectionRouter::new(RouterConfig::default()); let addr = TransportAddr::Ble { mac: [0x11, 0x22, 0x33, 0x44, 0x55, 0x66], psm: 128, @@ -1902,13 +1995,10 @@ mod tests { } #[test] - fn test_set_quic_endpoint() { + fn test_quic_endpoint_unset_by_default() { let router = ConnectionRouter::new(RouterConfig::default()); assert!(!router.is_quic_available()); assert!(router.quic_endpoint().is_none()); - - // We can't easily construct a NatTraversalEndpoint in a unit test, - // but we verify the setter method exists and the state tracking works } #[test] @@ -1922,7 +2012,7 @@ mod tests { #[test] fn test_routed_connection_debug_constrained() { - let mut router = ConnectionRouter::new(RouterConfig::default()); + let router = ConnectionRouter::new(RouterConfig::default()); let addr = TransportAddr::Ble { mac: [0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF], psm: 128, @@ -1984,14 +2074,14 @@ mod tests { #[test] fn test_poll_events_empty() { - let mut router = ConnectionRouter::new(RouterConfig::default()); + let router = ConnectionRouter::new(RouterConfig::default()); let events = router.poll_events(); assert!(events.is_empty()); } #[test] fn test_poll_events_after_constrained_connect() { - let mut router = ConnectionRouter::new(RouterConfig::default()); + let router = ConnectionRouter::new(RouterConfig::default()); let addr = TransportAddr::Ble { mac: [0x11, 0x22, 0x33, 0x44, 0x55, 0x66], psm: 128, @@ -2012,7 +2102,7 @@ mod tests { #[test] fn test_connection_mtu() { - let mut router = ConnectionRouter::new(RouterConfig::default()); + let router = ConnectionRouter::new(RouterConfig::default()); let addr = TransportAddr::Ble { mac: [0x11, 0x22, 0x33, 0x44, 0x55, 0x66], psm: 128, @@ -2027,7 +2117,7 @@ mod tests { #[test] fn test_connection_stats_constrained() { - let mut router = ConnectionRouter::new(RouterConfig::default()); + let router = ConnectionRouter::new(RouterConfig::default()); let addr = TransportAddr::Ble { mac: [0x11, 0x22, 0x33, 0x44, 0x55, 0x66], psm: 128, @@ -2055,7 +2145,7 @@ mod tests { #[test] fn test_close_with_reason_constrained() { - let mut router = ConnectionRouter::new(RouterConfig::default()); + let router = ConnectionRouter::new(RouterConfig::default()); let addr = TransportAddr::Ble { mac: [0x11, 0x22, 0x33, 0x44, 0x55, 0x66], psm: 128, @@ -2068,7 +2158,7 @@ mod tests { #[test] fn test_is_open_after_close() { - let mut router = ConnectionRouter::new(RouterConfig::default()); + let router = ConnectionRouter::new(RouterConfig::default()); let addr = TransportAddr::Ble { mac: [0x11, 0x22, 0x33, 0x44, 0x55, 0x66], psm: 128, @@ -2085,7 +2175,7 @@ mod tests { #[tokio::test] async fn test_send_async_constrained() { - let mut router = ConnectionRouter::new(RouterConfig::default()); + let router = ConnectionRouter::new(RouterConfig::default()); let addr = TransportAddr::Ble { mac: [0x11, 0x22, 0x33, 0x44, 0x55, 0x66], psm: 128, @@ -2102,7 +2192,7 @@ mod tests { #[tokio::test] async fn test_recv_async_constrained_no_data() { - let mut router = ConnectionRouter::new(RouterConfig::default()); + let router = ConnectionRouter::new(RouterConfig::default()); let addr = TransportAddr::Ble { mac: [0x11, 0x22, 0x33, 0x44, 0x55, 0x66], psm: 128, diff --git a/src/p2p_endpoint.rs b/src/p2p_endpoint.rs index e2a60ef5..6a0725df 100644 --- a/src/p2p_endpoint.rs +++ b/src/p2p_endpoint.rs @@ -152,8 +152,10 @@ pub struct P2pEndpoint { /// Connection router for automatic protocol engine selection /// /// Routes connections through either QUIC (for broadband) or Constrained - /// engine (for BLE/LoRa) based on transport capabilities. - router: Arc>, + /// engine (for BLE/LoRa) based on transport capabilities. The router is + /// fully interior-mutable — all methods take `&self` and stat/state + /// mutations are lock-free — so no `RwLock` is needed. + router: Arc, /// Mapping from TransportAddr to ConnectionId for constrained connections /// @@ -736,15 +738,14 @@ impl P2pEndpoint { enable_metrics: true, max_connections: 256, }; - let mut router = ConnectionRouter::with_full_config( + // `with_full_config` already installs the QUIC endpoint; no + // post-construction setter is needed. + let router = ConnectionRouter::with_full_config( router_config, Arc::clone(&transport_registry), Arc::clone(&inner_arc), ); - // Set QUIC endpoint on the router - router.set_quic_endpoint(Arc::clone(&inner_arc)); - // Create channel for data received from background reader tasks let (data_tx, data_rx) = mpsc::channel(config.data_channel_capacity); let reader_tasks = Arc::new(tokio::sync::Mutex::new(tokio::task::JoinSet::new())); @@ -766,7 +767,7 @@ impl P2pEndpoint { pending_data: Arc::new(RwLock::new(BoundedPendingBuffer::default())), bootstrap_cache, transport_registry, - router: Arc::new(RwLock::new(router)), + router: Arc::new(router), constrained_connections: Arc::new(RwLock::new(HashMap::new())), constrained_peer_addrs: Arc::new(RwLock::new(HashMap::new())), hole_punch_target_peer_ids: Arc::new(dashmap::DashMap::new()), @@ -1016,12 +1017,18 @@ impl P2pEndpoint { // Use the router to determine the appropriate engine. // - // Engine selection only needs a read lock now that - // `select_engine_for_addr` takes `&self` and uses atomic counters. - // The Constrained branch then re-acquires a write lock for - // `router.connect(addr)`, which is still `&mut self`. The QUIC - // branch never needs a write lock at all. - let engine = self.router.read().await.select_engine_for_addr(addr); + // Both `select_engine_for_addr` and `connect` take `&self` on + // `ConnectionRouter`, so there is no locking at all on the hot + // path. Selection and connect are two separate calls — there is a + // theoretical TOCTOU window where the engine picked here could + // become unavailable before the connect runs. In practice the + // router has no API to revoke or replace an engine once installed + // (the QUIC endpoint is set at construction time, the constrained + // transport is lazy-initialised and never torn down), so the race + // is closed by construction. If that invariant is ever relaxed, + // this call site needs to handle an engine-unavailable error from + // `connect()` explicitly. + let engine = self.router.select_engine_for_addr(addr); info!("Connecting to {} via {:?} engine", addr, engine); @@ -1037,9 +1044,9 @@ impl P2pEndpoint { self.connect(socket_addr).await } ProtocolEngine::Constrained => { - // For constrained transports, use the router's constrained connection. - let mut router = self.router.write().await; - let _routed = router.connect(addr).map_err(|e| { + // For constrained transports, use the router's connect + // path. No lock needed — `connect` takes `&self`. + let _routed = self.router.connect(addr).map_err(|e| { EndpointError::Connection(format!("Constrained connection failed: {}", e)) })?; @@ -1054,8 +1061,6 @@ impl P2pEndpoint { last_activity: Instant::now(), }; - // Store peer keyed by synthetic address - drop(router); // Release lock before acquiring connected_peers lock self.connected_peers .write() .await @@ -1082,17 +1087,18 @@ impl P2pEndpoint { /// Get the connection router for advanced routing control /// - /// Returns a reference to the connection router which can be used to: - /// - Query engine selection for addresses - /// - Get routing statistics - /// - Configure routing behavior - pub async fn router(&self) -> tokio::sync::RwLockReadGuard<'_, ConnectionRouter> { - self.router.read().await + /// Returns a shared reference to the connection router which can be + /// used to query engine selection for addresses, read routing stats, + /// or drive connects/accepts directly. All router methods take + /// `&self`, so multiple callers can use the returned handle + /// concurrently. + pub fn router(&self) -> &Arc { + &self.router } - /// Get routing statistics - pub async fn routing_stats(&self) -> crate::connection_router::RouterStats { - self.router.read().await.stats().clone() + /// Get a point-in-time snapshot of router statistics. + pub fn routing_stats(&self) -> crate::connection_router::RouterStatsSnapshot { + self.router.stats().snapshot() } /// Register a constrained connection for a transport address @@ -2271,16 +2277,14 @@ impl P2pEndpoint { // Select protocol engine based on transport address. // - // Uses a read lock: `select_engine_for_addr` takes `&self` and - // updates its stats counters via atomics. The previous code held - // an exclusive write lock here, which serialised every outbound - // send on the endpoint through a single lock and was a dominant - // contention point at high node counts (1000-node testnet). - let engine = self - .router - .read() - .await - .select_engine_for_addr(&transport_addr); + // No lock: `select_engine_for_addr` takes `&self` on + // `ConnectionRouter` and bumps its stats counters via atomics, so + // concurrent sends can run fully in parallel. The previous + // implementation held an exclusive write lock on the router here, + // which serialised every outbound send on the endpoint through a + // single lock and was a dominant contention point at high node + // counts (1000-node testnet). + let engine = self.router.select_engine_for_addr(&transport_addr); match engine { crate::transport::ProtocolEngine::Quic => { diff --git a/tests/constrained_integration.rs b/tests/constrained_integration.rs index f6ce0aac..e65d5617 100644 --- a/tests/constrained_integration.rs +++ b/tests/constrained_integration.rs @@ -268,7 +268,6 @@ fn test_connection_close() { use saorsa_transport::connection_router::{ConnectionRouter, RouterConfig}; use saorsa_transport::transport::ProtocolEngine; -use std::sync::atomic::Ordering; /// Test that ConnectionRouter correctly selects Constrained engine for BLE addresses #[test] @@ -289,8 +288,8 @@ fn test_router_selects_constrained_for_ble() { // Verify stats tracking let stats = router.stats(); - assert_eq!(stats.constrained_selections.load(Ordering::Relaxed), 1); - assert_eq!(stats.quic_selections.load(Ordering::Relaxed), 0); + assert_eq!(stats.constrained_selections(), 1); + assert_eq!(stats.quic_selections(), 0); } /// Test that ConnectionRouter correctly selects QUIC engine for UDP addresses @@ -305,8 +304,8 @@ fn test_router_selects_quic_for_udp() { // Verify stats tracking let stats = router.stats(); - assert_eq!(stats.quic_selections.load(Ordering::Relaxed), 1); - assert_eq!(stats.constrained_selections.load(Ordering::Relaxed), 0); + assert_eq!(stats.quic_selections(), 1); + assert_eq!(stats.constrained_selections(), 0); } /// Test mixed transport selection (UDP and BLE peers) @@ -340,8 +339,8 @@ fn test_mixed_transport_selection() { // Verify cumulative stats let stats = router.stats(); - assert_eq!(stats.quic_selections.load(Ordering::Relaxed), 1); - assert_eq!(stats.constrained_selections.load(Ordering::Relaxed), 2); + assert_eq!(stats.quic_selections(), 1); + assert_eq!(stats.constrained_selections(), 2); } /// Test synthetic socket address generation for BLE From 496721bee0e5b535d5aec07513ec3ced29d20644 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Mon, 6 Apr 2026 21:06:11 +0100 Subject: [PATCH 22/43] fix: raise coordinator PUNCH_ME_NOW rate limit from 5 to 50 per minute The per-connection rate limit of 5 coordination requests per 60-second window is too restrictive for bootstrap nodes acting as coordinators. On a network with 40+ NAT-restricted nodes, a client needs to send ~80 PUNCH_ME_NOW frames (2 rounds per target) during initial connection establishment. The low limit causes 50% of relay requests to be silently dropped, preventing hole punches from succeeding. Raise to 50 per minute to allow clients to hole-punch to all targets in a single burst while still providing protection against amplification attacks. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/connection/nat_traversal.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/connection/nat_traversal.rs b/src/connection/nat_traversal.rs index 5067ed10..b388b833 100644 --- a/src/connection/nat_traversal.rs +++ b/src/connection/nat_traversal.rs @@ -480,7 +480,7 @@ impl SecurityValidationState { max_candidates_per_window: 20, // Max 20 candidates per 60 seconds rate_window: Duration::from_secs(60), coordination_requests: VecDeque::new(), - max_coordination_per_window: 5, // Max 5 coordination requests per 60 seconds + max_coordination_per_window: 50, // Max 50 coordination requests per 60 seconds address_validation_cache: HashMap::new(), validation_cache_timeout: Duration::from_secs(300), // 5 minute cache allow_loopback, From 70ab29b15c52a75881750c29d9388eb9b2123d00 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Mon, 6 Apr 2026 21:06:11 +0100 Subject: [PATCH 23/43] fix: raise coordinator PUNCH_ME_NOW rate limit from 5 to 50 per minute MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-connection rate limit of 5 coordination requests per 60-second window is too restrictive for nodes acting as hole-punch coordinators. Any node can be selected as a coordinator — either a bootstrap node during initial discovery, or a regular network node chosen as a DHT referrer. On a network with many NAT-restricted nodes, a peer may need to send dozens of PUNCH_ME_NOW frames through a single coordinator during connection establishment. The low limit causes relay requests to be silently dropped, preventing hole punches from succeeding and forcing expensive MASQUE relay fallback. Observed 50% rejection rate on a 52-node testnet. Raise to 50 per minute to allow burst hole-punching while still providing protection against amplification attacks. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/connection/nat_traversal.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/connection/nat_traversal.rs b/src/connection/nat_traversal.rs index 5067ed10..b388b833 100644 --- a/src/connection/nat_traversal.rs +++ b/src/connection/nat_traversal.rs @@ -480,7 +480,7 @@ impl SecurityValidationState { max_candidates_per_window: 20, // Max 20 candidates per 60 seconds rate_window: Duration::from_secs(60), coordination_requests: VecDeque::new(), - max_coordination_per_window: 5, // Max 5 coordination requests per 60 seconds + max_coordination_per_window: 50, // Max 50 coordination requests per 60 seconds address_validation_cache: HashMap::new(), validation_cache_timeout: Duration::from_secs(300), // 5 minute cache allow_loopback, From b58e671feb649e6804166b5ad3b44cdb47164146 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Tue, 7 Apr 2026 00:16:31 +0200 Subject: [PATCH 24/43] feat(nat): rotate hole-punch coordinators with capped per-attempt timeout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds two coordinated changes that move coordinator load off the small set of bootstrap nodes that today serve as the de-facto hole-punch coordinator for every cold-starting peer. Tier 2 — list-form preferred coordinators with rotation: - hole_punch_preferred_coordinators changes from DashMap to DashMap>. Callers (saorsa-core's DHT layer) supply a ranked list best-first; the existing single-coordinator setter remains as a thin wrapper. - A new merge_preferred_coordinators helper inserts the ranked list at the front of coordinator_candidates, deduping any pre-existing copies. Pure function, unit-tested. - The hole-punch loop in connect_with_fallback_inner rotates through the first N candidates (where N = preferred list length). All but the final attempt use a new short timeout (PER_COORDINATOR_QUICK_HOLEPUNCH_TIMEOUT = 1.5s) so a busy or unreachable coordinator is abandoned quickly; the final attempt uses the strategy's full hole-punch timeout to give it time to complete the punch. When no preferred list is set, the legacy single-coordinator retry behaviour is preserved. - Bounds-safe rotation via .get() instead of indexing. Tier 4 (lite) — coordinator-side back-pressure with silent refusal: - New NatTraversalConfig fields coordinator_max_active_relays (default 32) and coordinator_relay_slot_timeout (default 5s), validated in 1..=256 and 100ms..=60s respectively. - Plumbed through TransportConfig via setters mirroring allow_loopback, into NatTraversalState::new and BootstrapCoordinator::new. - BootstrapCoordinator gains a relay_slots HashMap keyed by (initiator_peer_id, target_peer_id) recording arrival Instant, plus a backpressure_refusals stat. - process_punch_me_now_frame inline-sweeps stale slots before the back-pressure check (so ghost slots from crashed peers cannot leak the counter), then either accepts and inserts/refreshes the slot or silently returns Ok(None) and increments the refusal stat. Re-arming the same (initiator, target) pair refreshes the slot timestamp without consuming additional capacity. - Only the relay branch (frames carrying target_peer_id) consumes a slot — non-relay echo frames are unchanged. The two tiers compose: a coordinator at capacity silently drops the relay; the initiator's per-attempt timeout (Tier 2) drives it to its next preferred coordinator. No new wire frame is needed, so this is backwards-compatible with older peers. Tests added: - 5 unit tests for merge_preferred_coordinators (front insertion, ordering preservation, dedup, empty handling) - 5 unit tests for BootstrapCoordinator back-pressure (under-cap accept, at-cap silent refuse, slot re-arm without capacity consumption, sweep reclaims stale slots, non-relay frames don't consume slots) Test files (security_regression_tests.rs, relay_queue_tests.rs) updated to include the two new fields in their NatTraversalConfig struct literals. This is the second of three PRs landing smarter hole-punch coordinator selection. The first (saorsa-core) added round-aware referrer ranking in the DHT layer; the third will wire saorsa-core to call the new list-form set_hole_punch_preferred_coordinators API. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/config/transport.rs | 43 ++++ src/connection/mod.rs | 4 + src/connection/nat_traversal.rs | 312 +++++++++++++++++++++++++- src/nat_traversal_api.rs | 77 +++++++ src/p2p_endpoint.rs | 345 ++++++++++++++++++++++++----- src/unified_config.rs | 4 + tests/relay_queue_tests.rs | 10 + tests/security_regression_tests.rs | 18 ++ 8 files changed, 759 insertions(+), 54 deletions(-) diff --git a/src/config/transport.rs b/src/config/transport.rs index 26c4e1b9..813247d3 100644 --- a/src/config/transport.rs +++ b/src/config/transport.rs @@ -65,6 +65,19 @@ pub struct TransportConfig { /// Allow loopback addresses as valid NAT traversal candidates pub(crate) allow_loopback: bool, + + /// Maximum number of concurrent hole-punch relay slots this node will + /// service when acting as a coordinator (Tier 4 lite back-pressure). + /// When the active count reaches this cap, incoming `PUNCH_ME_NOW` + /// frames that would otherwise be relayed are silently refused so the + /// initiator advances to its next preferred coordinator. + pub(crate) coordinator_max_active_relays: usize, + + /// Reclamation timeout for stale coordinator relay slots. Acts as a + /// safety net so a relay slot cannot leak forever if a peer crashes + /// mid-coordination, a NAT rebind silently drops the session, or a + /// follow-up signal is lost. + pub(crate) coordinator_relay_slot_timeout: Duration, } impl TransportConfig { @@ -470,6 +483,21 @@ impl TransportConfig { self.allow_loopback = allow; self } + + /// Cap on simultaneous hole-punch relay slots when this node acts as a + /// coordinator. Higher = more concurrency capacity, lower = stricter + /// back-pressure shedding under storm load. Default: 32. + pub fn coordinator_max_active_relays(&mut self, max: usize) -> &mut Self { + self.coordinator_max_active_relays = max; + self + } + + /// Maximum lifetime of a coordinator relay slot before it is reclaimed + /// by the inline garbage-collection sweep. Default: 5 seconds. + pub fn coordinator_relay_slot_timeout(&mut self, timeout: Duration) -> &mut Self { + self.coordinator_relay_slot_timeout = timeout; + self + } } impl Default for TransportConfig { @@ -523,6 +551,11 @@ impl Default for TransportConfig { ml_dsa_65: false, }), allow_loopback: false, + // Tier 4 (lite) coordinator back-pressure defaults. See + // `coordinator_max_active_relays`/`coordinator_relay_slot_timeout` + // setters for the rationale behind these numbers. + coordinator_max_active_relays: 32, + coordinator_relay_slot_timeout: Duration::from_secs(5), } } } @@ -559,6 +592,8 @@ impl fmt::Debug for TransportConfig { address_discovery_config, pqc_algorithms, allow_loopback, + coordinator_max_active_relays, + coordinator_relay_slot_timeout, } = self; fmt.debug_struct("TransportConfig") .field("max_concurrent_bidi_streams", max_concurrent_bidi_streams) @@ -591,6 +626,14 @@ impl fmt::Debug for TransportConfig { .field("address_discovery_config", address_discovery_config) .field("pqc_algorithms", pqc_algorithms) .field("allow_loopback", allow_loopback) + .field( + "coordinator_max_active_relays", + coordinator_max_active_relays, + ) + .field( + "coordinator_relay_slot_timeout", + coordinator_relay_slot_timeout, + ) .finish_non_exhaustive() } } diff --git a/src/connection/mod.rs b/src/connection/mod.rs index 3b05685f..b8bad703 100644 --- a/src/connection/mod.rs +++ b/src/connection/mod.rs @@ -4535,6 +4535,8 @@ impl Connection { max_candidates, coordination_timeout, self.config.allow_loopback, + self.config.coordinator_max_active_relays, + self.config.coordinator_relay_slot_timeout, )); trace!("NAT traversal initialized for symmetric P2P node"); @@ -4733,6 +4735,8 @@ impl Connection { 8, Duration::from_secs(10), self.config.allow_loopback, + self.config.coordinator_max_active_relays, + self.config.coordinator_relay_slot_timeout, )); } diff --git a/src/connection/nat_traversal.rs b/src/connection/nat_traversal.rs index b388b833..a0ae3e48 100644 --- a/src/connection/nat_traversal.rs +++ b/src/connection/nat_traversal.rs @@ -1820,15 +1820,25 @@ impl NatTraversalState { /// /// v0.13.0: Role parameter removed - all nodes are symmetric P2P nodes. /// Every node can initiate, accept, and coordinate NAT traversal. + /// + /// `coordinator_max_active_relays` and `coordinator_relay_slot_timeout` + /// configure Tier 4 (lite) coordinator-side back-pressure: when this + /// node acts as a hole-punch coordinator, it will silently refuse new + /// `PUNCH_ME_NOW` relays once `coordinator_max_active_relays` slots are + /// in flight, and reclaim stale slots that exceed the timeout. pub(super) fn new( max_candidates: u32, coordination_timeout: Duration, allow_loopback: bool, + coordinator_max_active_relays: usize, + coordinator_relay_slot_timeout: Duration, ) -> Self { // v0.13.0: All nodes can coordinate - always create coordinator let bootstrap_coordinator = Some(BootstrapCoordinator::new( BootstrapConfig::default(), allow_loopback, + coordinator_max_active_relays, + coordinator_relay_slot_timeout, )); Self { @@ -3556,6 +3566,22 @@ pub(crate) struct BootstrapCoordinator { security_validator: SecurityValidationState, /// Statistics for bootstrap operations stats: BootstrapStats, + /// Active hole-punch relay slots indexed by `(initiator_peer_id, + /// target_peer_id)`. Each entry records the wall-clock arrival time of + /// the `PUNCH_ME_NOW` frame that opened the slot. Slots are reclaimed + /// either implicitly (when a follow-up frame from the same pair lands) + /// or by the inline garbage-collection sweep run on every incoming + /// frame: any slot older than `coordinator_relay_slot_timeout` is + /// evicted before the back-pressure check, which keeps the active + /// count from leaking on ghost sessions where the initiator crashed + /// or the target became unreachable mid-coordination. + relay_slots: HashMap<(PeerId, PeerId), Instant>, + /// Cap on simultaneous relay slots (Tier 4 lite back-pressure). + /// Plumbed from [`crate::config::TransportConfig::coordinator_max_active_relays`]. + relay_slot_capacity: usize, + /// Reclamation timeout for stale relay slots. Plumbed from + /// [`crate::config::TransportConfig::coordinator_relay_slot_timeout`]. + relay_slot_timeout: Duration, } // Removed legacy CoordinationSessionId type /// Peer identifier for bootstrap coordination @@ -3641,19 +3667,47 @@ pub(crate) struct BootstrapStats { successful_coordinations: u64, /// Security rejections security_rejections: u64, + /// Refusals due to active-relay back-pressure (Tier 4 lite). Tracks the + /// number of `PUNCH_ME_NOW` frames silently dropped because the + /// coordinator was at `relay_slot_capacity` when they arrived. + backpressure_refusals: u64, } // Removed session state machine enums and recovery actions impl BootstrapCoordinator { - /// Create a new bootstrap coordinator - pub(crate) fn new(_config: BootstrapConfig, allow_loopback: bool) -> Self { + /// Create a new bootstrap coordinator. + /// + /// `relay_slot_capacity` and `relay_slot_timeout` configure the + /// Tier 4 (lite) back-pressure: incoming `PUNCH_ME_NOW` relay frames + /// are silently refused once `relay_slot_capacity` slots are in + /// flight, and any slot older than `relay_slot_timeout` is reclaimed + /// by the inline garbage-collection sweep on every incoming frame. + pub(crate) fn new( + _config: BootstrapConfig, + allow_loopback: bool, + relay_slot_capacity: usize, + relay_slot_timeout: Duration, + ) -> Self { Self { address_observations: HashMap::new(), peer_index: HashMap::new(), coordination_table: HashMap::new(), security_validator: SecurityValidationState::new(allow_loopback), stats: BootstrapStats::default(), + relay_slots: HashMap::new(), + relay_slot_capacity, + relay_slot_timeout, } } + + /// Reclaim relay slots whose arrival timestamp is older than the + /// configured `relay_slot_timeout`. Called inline at the start of + /// every `PUNCH_ME_NOW` frame so that ghost slots from crashed peers + /// or dropped sessions cannot leak the active-relay counter. + fn sweep_stale_relay_slots(&mut self, now: Instant) { + let timeout = self.relay_slot_timeout; + self.relay_slots + .retain(|_, &mut arrived_at| now.duration_since(arrived_at) < timeout); + } /// Observe a peer's address from an incoming connection /// /// This is called when a peer connects to this bootstrap node, @@ -3789,6 +3843,37 @@ impl BootstrapCoordinator { ); })?; + // Tier 4 (lite) back-pressure: only the relay branch (where the + // frame carries an explicit `target_peer_id`) consumes a slot. If + // we are at capacity for active relays, silently drop the frame — + // the initiator's per-attempt timeout (Tier 2 rotation) will move + // it on to its next preferred coordinator. + // + // Inline garbage-collection sweep first so any stale slots from + // crashed peers are reclaimed before the cap check. + if let Some(target_peer_id) = frame.target_peer_id { + self.sweep_stale_relay_slots(now); + let slot_key = (from_peer, target_peer_id); + // Re-arming an existing slot for the same (initiator, target) + // pair does not consume an additional slot — common during + // multi-round coordination where the same pair re-sends. + let already_active = self.relay_slots.contains_key(&slot_key); + if !already_active && self.relay_slots.len() >= self.relay_slot_capacity { + self.stats.backpressure_refusals = + self.stats.backpressure_refusals.saturating_add(1); + debug!( + "PUNCH_ME_NOW relay refused: coordinator at capacity ({}/{}) — initiator {:?} → target {:?}", + self.relay_slots.len(), + self.relay_slot_capacity, + hex::encode(&from_peer[..8]), + hex::encode(&target_peer_id[..8]) + ); + return Ok(None); + } + // Accept: insert/refresh the slot timestamp. + self.relay_slots.insert(slot_key, now); + } + // Track coordination entry minimally let _entry = self .coordination_table @@ -3907,6 +3992,8 @@ mod tests { 10, // max_candidates Duration::from_secs(30), // coordination_timeout true, // allow_loopback for tests + 32, // coordinator_max_active_relays + Duration::from_secs(5), // coordinator_relay_slot_timeout ) } @@ -4289,6 +4376,8 @@ mod tests { 100, // max_candidates (high enough to not limit) Duration::from_secs(30), true, // allow_loopback for tests + 32, + Duration::from_secs(5), ); let now = Instant::now(); @@ -4319,7 +4408,13 @@ mod tests { #[test] fn test_add_pairs_at_exact_limit() { // Test behavior when exactly at the limit - let mut state = NatTraversalState::new(100, Duration::from_secs(30), true); + let mut state = NatTraversalState::new( + 100, + Duration::from_secs(30), + true, + 32, + Duration::from_secs(5), + ); let now = Instant::now(); // Add candidates to get close to limit (14 × 14 = 196 pairs) @@ -4399,7 +4494,13 @@ mod tests { #[test] fn test_incremental_add_with_zero_remaining_capacity() { // Test that incremental add gracefully handles zero capacity - let mut state = NatTraversalState::new(100, Duration::from_secs(30), true); + let mut state = NatTraversalState::new( + 100, + Duration::from_secs(30), + true, + 32, + Duration::from_secs(5), + ); let now = Instant::now(); // Fill up to the limit @@ -4431,4 +4532,207 @@ mod tests { "Should handle limit gracefully without panic" ); } + + // ---- Tier 4 (lite): coordinator-side back-pressure ---- + + /// Helper: build a `BootstrapCoordinator` with controllable cap and + /// timeout for back-pressure tests. + fn make_bp_coordinator(capacity: usize, timeout: Duration) -> BootstrapCoordinator { + BootstrapCoordinator::new( + BootstrapConfig::default(), + true, // allow_loopback so test addrs aren't rejected + capacity, + timeout, + ) + } + + /// Helper: build a `PunchMeNow` frame for the relay path (with target). + fn make_relay_frame(round: u64, target_peer_id: [u8; 32]) -> crate::frame::PunchMeNow { + crate::frame::PunchMeNow { + round: VarInt::from_u64(round).unwrap_or(VarInt::from_u32(0)), + paired_with_sequence_number: VarInt::from_u32(0), + address: SocketAddr::from(([127, 0, 0, 1], 9000)), + target_peer_id: Some(target_peer_id), + } + } + + fn peer_id_with_byte(byte: u8) -> [u8; 32] { + let mut id = [0u8; 32]; + id[0] = byte; + id + } + + #[test] + fn coordinator_accepts_relay_under_capacity() { + let mut coord = make_bp_coordinator(4, Duration::from_secs(5)); + let now = Instant::now(); + let from = peer_id_with_byte(0x01); + let target = peer_id_with_byte(0x02); + let frame = make_relay_frame(1, target); + let source_addr = SocketAddr::from(([127, 0, 0, 1], 5000)); + + let result = coord + .process_punch_me_now_frame(from, source_addr, &frame, now) + .expect("relay under cap should not error"); + + assert!( + result.is_some(), + "relay under capacity should produce a coordination frame" + ); + assert_eq!(coord.relay_slots.len(), 1, "one slot should be allocated"); + assert_eq!( + coord.stats.backpressure_refusals, 0, + "no refusal stat increment expected when under cap" + ); + } + + #[test] + fn coordinator_refuses_silently_at_capacity() { + let capacity = 2; + let mut coord = make_bp_coordinator(capacity, Duration::from_secs(5)); + let now = Instant::now(); + let source_addr = SocketAddr::from(([127, 0, 0, 1], 5000)); + + // Fill capacity with two distinct (initiator, target) pairs. + for i in 0..capacity { + let from = peer_id_with_byte(0x10 + i as u8); + let target = peer_id_with_byte(0x20 + i as u8); + let frame = make_relay_frame(i as u64 + 1, target); + let result = coord + .process_punch_me_now_frame(from, source_addr, &frame, now) + .expect("under-cap relay should not error"); + assert!(result.is_some(), "slot {} should be accepted", i); + } + assert_eq!(coord.relay_slots.len(), capacity); + + // The next distinct pair must be silently refused: Ok(None). + let overflow_from = peer_id_with_byte(0xFE); + let overflow_target = peer_id_with_byte(0xFD); + let overflow_frame = make_relay_frame(99, overflow_target); + let result = coord + .process_punch_me_now_frame(overflow_from, source_addr, &overflow_frame, now) + .expect("at-cap refusal must be silent (no error)"); + + assert!( + result.is_none(), + "at-cap refusal must produce no coordination frame" + ); + assert_eq!( + coord.relay_slots.len(), + capacity, + "refused frame must not consume a slot" + ); + assert_eq!( + coord.stats.backpressure_refusals, 1, + "back-pressure stat must increment on refusal" + ); + } + + #[test] + fn coordinator_re_arms_existing_slot_without_consuming_capacity() { + let mut coord = make_bp_coordinator(2, Duration::from_secs(5)); + let now = Instant::now(); + let source_addr = SocketAddr::from(([127, 0, 0, 1], 5000)); + let from = peer_id_with_byte(0x01); + let target = peer_id_with_byte(0x02); + let frame = make_relay_frame(1, target); + + // Fill the first slot. + coord + .process_punch_me_now_frame(from, source_addr, &frame, now) + .expect("first frame ok"); + assert_eq!(coord.relay_slots.len(), 1); + + // Re-send for the same (from, target) pair: must not consume an + // additional slot, must still be accepted. + let later = now + Duration::from_millis(500); + let result = coord + .process_punch_me_now_frame(from, source_addr, &frame, later) + .expect("re-arm ok"); + assert!(result.is_some(), "re-armed slot should still be relayed"); + assert_eq!( + coord.relay_slots.len(), + 1, + "re-arming the same pair must not allocate a second slot" + ); + // Slot timestamp should refresh to the later instant. + let slot_arrived = coord + .relay_slots + .get(&(from, target)) + .copied() + .expect("slot present"); + assert_eq!( + slot_arrived, later, + "re-arm must refresh the slot timestamp" + ); + } + + #[test] + fn coordinator_sweep_reclaims_stale_slots() { + let timeout = Duration::from_secs(5); + let mut coord = make_bp_coordinator(2, timeout); + let now = Instant::now(); + let source_addr = SocketAddr::from(([127, 0, 0, 1], 5000)); + + // Fill capacity. + for i in 0..2u8 { + let from = peer_id_with_byte(0x10 + i); + let target = peer_id_with_byte(0x20 + i); + let frame = make_relay_frame(i as u64 + 1, target); + coord + .process_punch_me_now_frame(from, source_addr, &frame, now) + .expect("under-cap ok"); + } + assert_eq!(coord.relay_slots.len(), 2); + + // Advance well past the slot timeout. The next incoming frame's + // inline sweep must reclaim both stale slots before applying the + // back-pressure check. + let much_later = now + timeout + Duration::from_secs(1); + let new_from = peer_id_with_byte(0xAA); + let new_target = peer_id_with_byte(0xBB); + let new_frame = make_relay_frame(42, new_target); + let result = coord + .process_punch_me_now_frame(new_from, source_addr, &new_frame, much_later) + .expect("post-sweep frame should succeed"); + + assert!( + result.is_some(), + "frame after sweep must be accepted (capacity reclaimed)" + ); + assert_eq!( + coord.relay_slots.len(), + 1, + "stale slots must be reclaimed; only the new one remains" + ); + assert_eq!( + coord.stats.backpressure_refusals, 0, + "no refusal expected because sweep freed capacity in time" + ); + } + + #[test] + fn coordinator_non_relay_frame_does_not_consume_slot() { + // PUNCH_ME_NOW without a target_peer_id is the response/echo path, + // not a relay request — it must NOT consume a back-pressure slot. + let mut coord = make_bp_coordinator(1, Duration::from_secs(5)); + let now = Instant::now(); + let source_addr = SocketAddr::from(([127, 0, 0, 1], 5000)); + let from = peer_id_with_byte(0x01); + + let frame = crate::frame::PunchMeNow { + round: VarInt::from_u32(1), + paired_with_sequence_number: VarInt::from_u32(0), + address: SocketAddr::from(([127, 0, 0, 1], 9000)), + target_peer_id: None, + }; + let _ = coord + .process_punch_me_now_frame(from, source_addr, &frame, now) + .expect("non-relay frame ok"); + assert_eq!( + coord.relay_slots.len(), + 0, + "non-relay frame must not consume a slot" + ); + } } diff --git a/src/nat_traversal_api.rs b/src/nat_traversal_api.rs index d1858c18..f757c082 100644 --- a/src/nat_traversal_api.rs +++ b/src/nat_traversal_api.rs @@ -492,6 +492,45 @@ pub struct NatTraversalConfig { #[serde(default)] pub allow_loopback: bool, + /// Maximum number of concurrent hole-punch relay slots this node will + /// service as a coordinator. + /// + /// When the active relay count reaches this cap, incoming `PUNCH_ME_NOW` + /// frames that would otherwise be relayed are *silently refused*: the + /// coordinator drops the relay without notifying the initiator. The + /// initiator's per-attempt timeout (Tier 2 rotation) drives it to + /// advance to the next preferred coordinator in its list. + /// + /// Defaults to [`NatTraversalConfig::DEFAULT_COORDINATOR_MAX_ACTIVE_RELAYS`] + /// (32). Picked as roughly 32% of saorsa-core's 100-connection cap so + /// the coordinator still has headroom for its own peer traffic, while + /// being low enough that a cold-start storm is shed and pushed onto + /// alternates. + /// + /// Each "active relay" represents one in-flight initiator→target + /// coordination, indexed by `(initiator_peer_id, target_peer_id)`. The + /// counter is decremented when the relay completes or after + /// [`Self::coordinator_relay_slot_timeout`] elapses (whichever first). + #[serde(default = "default_coordinator_max_active_relays")] + pub coordinator_max_active_relays: usize, + + /// Maximum lifetime of a coordinator relay slot before it is reclaimed + /// by the inline garbage-collection sweep. + /// + /// A successful hole-punch typically completes in 1-3 seconds; this + /// timeout exists purely as a safety net so that a relay slot cannot + /// leak forever if a peer crashes mid-coordination, a NAT rebind + /// silently drops the session, or a follow-up signal is lost. Without + /// it the active-relay counter would drift upward over time and the + /// coordinator would eventually refuse legitimate traffic. + /// + /// Defaults to [`NatTraversalConfig::DEFAULT_COORDINATOR_RELAY_SLOT_TIMEOUT`] + /// (5 seconds): comfortably above the worst-case successful punch + /// latency on high-RTT links, but short enough to keep ghost slots + /// from impacting steady-state burst capacity. + #[serde(default = "default_coordinator_relay_slot_timeout")] + pub coordinator_relay_slot_timeout: Duration, + /// Best-effort UPnP IGD port mapping configuration. /// /// When enabled, the endpoint asks the local Internet Gateway Device @@ -509,6 +548,24 @@ fn default_max_message_size() -> usize { crate::unified_config::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE } +fn default_coordinator_max_active_relays() -> usize { + NatTraversalConfig::DEFAULT_COORDINATOR_MAX_ACTIVE_RELAYS +} + +fn default_coordinator_relay_slot_timeout() -> Duration { + NatTraversalConfig::DEFAULT_COORDINATOR_RELAY_SLOT_TIMEOUT +} + +impl NatTraversalConfig { + /// Default cap on simultaneous coordinator relay slots. + /// See [`Self::coordinator_max_active_relays`] for rationale. + pub const DEFAULT_COORDINATOR_MAX_ACTIVE_RELAYS: usize = 32; + + /// Default reclamation timeout for stale coordinator relay slots. + /// See [`Self::coordinator_relay_slot_timeout`] for rationale. + pub const DEFAULT_COORDINATOR_RELAY_SLOT_TIMEOUT: Duration = Duration::from_secs(5); +} + /// Convert `max_message_size` to a QUIC `VarInt` for stream/send window configuration. /// /// Clamps to `VarInt::MAX` if the value exceeds the QUIC variable-length integer range. @@ -1088,6 +1145,8 @@ impl Default for NatTraversalConfig { transport_registry: None, // Use direct UDP binding by default max_message_size: crate::unified_config::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: false, + coordinator_max_active_relays: Self::DEFAULT_COORDINATOR_MAX_ACTIVE_RELAYS, + coordinator_relay_slot_timeout: Self::DEFAULT_COORDINATOR_RELAY_SLOT_TIMEOUT, upnp: crate::upnp::UpnpConfig::default(), } } @@ -1138,6 +1197,20 @@ impl ConfigValidator for NatTraversalConfig { )); } + // Validate coordinator back-pressure limits (Tier 4 lite). + validate_range( + self.coordinator_max_active_relays, + 1, + 256, + "coordinator_max_active_relays", + )?; + validate_duration( + self.coordinator_relay_slot_timeout, + Duration::from_millis(100), + Duration::from_secs(60), + "coordinator_relay_slot_timeout", + )?; + Ok(()) } } @@ -2613,6 +2686,8 @@ impl NatTraversalEndpoint { }; transport_config.nat_traversal_config(Some(nat_config)); transport_config.allow_loopback(config.allow_loopback); + transport_config.coordinator_max_active_relays(config.coordinator_max_active_relays); + transport_config.coordinator_relay_slot_timeout(config.coordinator_relay_slot_timeout); server_config.transport_config(Arc::new(transport_config)); @@ -2684,6 +2759,8 @@ impl NatTraversalEndpoint { }; transport_config.nat_traversal_config(Some(nat_config)); transport_config.allow_loopback(config.allow_loopback); + transport_config.coordinator_max_active_relays(config.coordinator_max_active_relays); + transport_config.coordinator_relay_slot_timeout(config.coordinator_relay_slot_timeout); client_config.transport_config(Arc::new(transport_config)); diff --git a/src/p2p_endpoint.rs b/src/p2p_endpoint.rs index 6a0725df..6d09425e 100644 --- a/src/p2p_endpoint.rs +++ b/src/p2p_endpoint.rs @@ -92,6 +92,19 @@ const STALE_REAPER_INTERVAL: Duration = Duration::from_secs(10); /// through the pinhole needs only 1-2 RTTs (~600ms at 300ms worst-case RTT). const POST_HOLEPUNCH_DIRECT_RETRY_TIMEOUT: Duration = Duration::from_secs(1); +/// Per-attempt hole-punch timeout used when rotating through a list of +/// preferred coordinators. Kept short so a busy or unreachable coordinator +/// is abandoned quickly and the next one in the list is tried; the *last* +/// coordinator in the rotation falls back to the strategy's full +/// hole-punch timeout to give it time to actually complete the punch. +/// +/// Tuned for the Tier 2 + Tier 4 (lite) coordinator-rotation flow: 1.5s +/// is comfortably above one round-trip on most internet links but well +/// below the strategy default (~8s), so the worst-case wait for K +/// preferred coordinators is roughly `(K-1) * 1.5s + 8s` instead of +/// `K * 8s`. +const PER_COORDINATOR_QUICK_HOLEPUNCH_TIMEOUT: Duration = Duration::from_millis(1500); + use crate::SHUTDOWN_DRAIN_TIMEOUT; /// Extract the raw SPKI (SubjectPublicKeyInfo) bytes from a QUIC connection's @@ -176,12 +189,17 @@ pub struct P2pEndpoint { /// address so concurrent dials don't race on shared state. hole_punch_target_peer_ids: Arc>, - /// Per-target preferred coordinator for hole-punch relay. When the DHT - /// lookup discovers a peer via a FindNode response from another node, that - /// responding node (the "referrer") has a connection to the discovered peer - /// and is an ideal coordinator for PUNCH_ME_NOW relay. Keyed by target - /// address, value is the referrer's socket address. - hole_punch_preferred_coordinators: Arc>, + /// Per-target preferred coordinators for hole-punch relay. When the DHT + /// lookup discovers a peer via FindNode responses from one or more peers, + /// those responding nodes (the "referrers") all have a connection to the + /// discovered peer and are good coordinator candidates. Keyed by target + /// address, value is an ordered list of referrer socket addresses ranked + /// best-first by the caller (e.g. by DHT lookup round, trust score). + /// During hole-punching the list is iterated front to back: the first + /// candidates get a short per-attempt timeout so we rotate quickly past + /// busy or unreachable coordinators; the last candidate gets the full + /// hole-punch timeout to give it time to actually complete the punch. + hole_punch_preferred_coordinators: Arc>>, /// Channel sender for data received from QUIC reader tasks and constrained poller data_tx: mpsc::Sender<(SocketAddr, Vec)>, @@ -1227,16 +1245,44 @@ impl P2pEndpoint { self.hole_punch_target_peer_ids.insert(target, peer_id); } - /// Set a preferred coordinator for hole-punching to a specific target. - /// The preferred coordinator is a peer that referred us to the target - /// during a DHT lookup, so it has a connection to the target. + /// Set an ordered list of preferred coordinators for hole-punching to a + /// specific target. + /// + /// The caller (typically saorsa-core's DHT layer) is expected to rank + /// the list best-first using its own quality signals — e.g. DHT lookup + /// round, trust score, observed latency. During hole-punching the list + /// is iterated front to back: the first `coordinators.len() - 1` get a + /// short per-attempt timeout so a busy or unreachable coordinator is + /// abandoned quickly; the last coordinator gets the full strategy + /// hole-punch timeout to give it time to complete the punch. + /// + /// Empty `coordinators` removes any preferred coordinators for `target`. + pub async fn set_hole_punch_preferred_coordinators( + &self, + target: SocketAddr, + coordinators: Vec, + ) { + if coordinators.is_empty() { + self.hole_punch_preferred_coordinators.remove(&target); + } else { + self.hole_punch_preferred_coordinators + .insert(target, coordinators); + } + } + + /// Set a single preferred coordinator for hole-punching to a specific + /// target. + /// + /// Thin wrapper around [`Self::set_hole_punch_preferred_coordinators`] + /// retained for callers that have only one coordinator candidate. New + /// callers should prefer the list form. pub async fn set_hole_punch_preferred_coordinator( &self, target: SocketAddr, coordinator: SocketAddr, ) { - self.hole_punch_preferred_coordinators - .insert(target, coordinator); + self.set_hole_punch_preferred_coordinators(target, vec![coordinator]) + .await; } /// Connect with automatic fallback: Direct → HolePunch → Relay. @@ -1312,6 +1358,36 @@ impl P2pEndpoint { result } + /// Merge a ranked list of preferred hole-punch coordinators into the + /// front of `coordinator_candidates`, preserving the relative order of + /// `preferred` and removing any pre-existing duplicates from the + /// candidate list. + /// + /// After this call returns, `coordinator_candidates[0..preferred.len()]` + /// equals `preferred` (in order). The hole-punch loop uses + /// `preferred.len()` directly to decide which attempts get the short + /// rotation timeout vs. the strategy's full hole-punch timeout. + /// + /// Pure function (no `&self`, no I/O) — extracted from + /// `connect_with_fallback_inner` so the front-insertion behaviour can + /// be unit-tested without spinning up a full endpoint. + fn merge_preferred_coordinators( + coordinator_candidates: &mut Vec, + preferred: &[SocketAddr], + ) { + if preferred.is_empty() { + return; + } + // Drop any pre-existing copies of the preferred entries from the + // tail so we don't end up with duplicates after the front-insert. + coordinator_candidates.retain(|a| !preferred.contains(a)); + // Insert in reverse so `preferred[0]` ends up at index 0, + // `preferred[1]` at index 1, etc. + for preferred_addr in preferred.iter().rev() { + coordinator_candidates.insert(0, *preferred_addr); + } + } + /// Inner implementation of connect_with_fallback (separated for dedup wrapper). async fn connect_with_fallback_inner( &self, @@ -1343,17 +1419,32 @@ impl P2pEndpoint { } } - // If the DHT referrer set a preferred coordinator for this target, - // move it to the front of the candidate list so round 1 uses it. + // If the DHT layer set preferred coordinators for this target, move + // them to the front of the candidate list in order so the hole-punch + // loop tries them first. Each preferred coordinator is removed from + // its existing position (if any) before being inserted at the front + // so the relative ordering of the preferred list is preserved. + // + // `preferred_coordinator_count` is captured for the hole-punch loop: + // when > 0 the loop rotates through `coordinator_candidates[0..count]` + // with `PER_COORDINATOR_QUICK_HOLEPUNCH_TIMEOUT` per non-final attempt, + // and the strategy's full timeout for the last attempt. When 0 the + // loop falls back to the existing single-coordinator retry behaviour. + let mut preferred_coordinator_count: usize = 0; if let Some(target_addr) = target { if let Some(preferred) = self.hole_punch_preferred_coordinators.get(&target_addr) { - let preferred_addr = *preferred; - coordinator_candidates.retain(|a| *a != preferred_addr); - coordinator_candidates.insert(0, preferred_addr); - info!( - "Using preferred coordinator {} for target {} (DHT referrer)", - preferred_addr, target_addr - ); + let preferred_list: Vec = preferred.clone(); + drop(preferred); // Release the DashMap entry guard before mutating coordinator_candidates. + Self::merge_preferred_coordinators(&mut coordinator_candidates, &preferred_list); + preferred_coordinator_count = preferred_list.len(); + if preferred_coordinator_count > 0 { + info!( + "Using {} preferred coordinator(s) for target {} (DHT referrers): {:?}", + preferred_list.len(), + target_addr, + preferred_list + ); + } } else { info!( "No preferred coordinator for target {} (not discovered via DHT referral)", @@ -1438,6 +1529,14 @@ impl P2pEndpoint { direct_addresses.push(v4); } + // Index of the preferred coordinator currently being attempted (when + // `preferred_coordinator_count > 0`). The hole-punch loop advances + // this on each failed round and uses it together with + // `preferred_coordinator_count` to decide whether the *next* attempt + // is the final one (full strategy timeout) or an interim rotation + // attempt (`PER_COORDINATOR_QUICK_HOLEPUNCH_TIMEOUT`). + let mut current_preferred_coordinator_idx: usize = 0; + loop { // Check if a previous hole-punch attempt established the connection // asynchronously (e.g. the target connected to us after receiving @@ -1578,44 +1677,94 @@ impl P2pEndpoint { .or(target_ipv6) .ok_or(EndpointError::NoAddress)?; + // Coordinator-rotation policy (Tier 2): + // + // When `preferred_coordinator_count > 0` we have a ranked + // list of DHT-supplied coordinators at + // `coordinator_candidates[0..preferred_coordinator_count]` + // and we rotate through them on each failed round. The + // first `count - 1` attempts use a short timeout + // (`PER_COORDINATOR_QUICK_HOLEPUNCH_TIMEOUT`) so a busy or + // unreachable coordinator is abandoned quickly; the final + // attempt uses the strategy's full hole-punch timeout to + // give it time to actually complete. + // + // When `preferred_coordinator_count == 0` (no DHT + // referrers — first contact, or non-DHT dial) we fall + // back to the legacy single-coordinator behaviour: + // strategy timeout per round, retry the same coordinator + // until `should_retry_holepunch` is exhausted. + let is_rotating = preferred_coordinator_count > 0; + let is_final_rotation_attempt = is_rotating + && current_preferred_coordinator_idx + 1 >= preferred_coordinator_count; + let attempt_timeout = if is_rotating && !is_final_rotation_attempt { + PER_COORDINATOR_QUICK_HOLEPUNCH_TIMEOUT + } else { + strategy.holepunch_timeout() + }; + info!( - "Trying hole-punch to {} via {} (round {})", - target, coordinator, round + "Trying hole-punch to {} via {} (round {}, attempt timeout {:?}, rotating={})", + target, coordinator, round, attempt_timeout, is_rotating ); // Use our existing NAT traversal infrastructure - match timeout( - strategy.holepunch_timeout(), - self.try_hole_punch(target, coordinator), - ) - .await - { + let attempt_result = + timeout(attempt_timeout, self.try_hole_punch(target, coordinator)).await; + + // Common post-attempt step: try a quick direct connect. + // The NAT binding may have been created by the target's + // outgoing packets even though our try_hole_punch didn't + // detect the connection. + let post_direct = async { + if let Ok(Ok(peer_conn)) = + timeout(POST_HOLEPUNCH_DIRECT_RETRY_TIMEOUT, self.connect(target)).await + { + info!("✓ Post-hole-punch direct connect succeeded to {}", target); + Some(peer_conn) + } else { + None + } + }; + + match attempt_result { Ok(Ok(conn)) => { info!("✓ Hole-punch succeeded to {} via {}", target, coordinator); return Ok((conn, ConnectionMethod::HolePunched { coordinator })); } Ok(Err(e)) => { - // After a failed hole-punch round, try a quick direct - // connect — the NAT binding may have been created by - // the target's outgoing packets even though our - // try_hole_punch didn't detect the connection. - if let Ok(Ok(peer_conn)) = - timeout(POST_HOLEPUNCH_DIRECT_RETRY_TIMEOUT, self.connect(target)) - .await - { - info!("✓ Post-hole-punch direct connect succeeded to {}", target); + if let Some(peer_conn) = post_direct.await { return Ok(( peer_conn, ConnectionMethod::HolePunched { coordinator }, )); } strategy.record_holepunch_error(round, e.to_string()); - if strategy.should_retry_holepunch() { - // Keep the same coordinator for retries. The preferred - // coordinator (index 0) was chosen because it has a - // known connection to the target. Switching to a random - // fallback wastes another round on a coordinator that - // likely can't relay to the target. + // Bounds-safe rotation: bail out of rotation and + // fall back to relay if for any reason the index + // would go out of bounds (defensive — by + // construction the bound holds while + // `current_preferred_coordinator_idx + 1 < preferred_coordinator_count`). + let next_coord = if is_rotating && !is_final_rotation_attempt { + coordinator_candidates + .get(current_preferred_coordinator_idx + 1) + .copied() + } else { + None + }; + if let Some(next_coord) = next_coord { + current_preferred_coordinator_idx += 1; + info!( + "Hole-punch via {} failed ({}), rotating to preferred coordinator {}/{}: {}", + coordinator, + e, + current_preferred_coordinator_idx + 1, + preferred_coordinator_count, + next_coord + ); + strategy.set_coordinator(next_coord); + strategy.increment_round(); + } else if strategy.should_retry_holepunch() { info!( "Hole-punch round {} failed, retrying with same coordinator", round @@ -1627,19 +1776,33 @@ impl P2pEndpoint { } } Err(_) => { - // Same: try a quick direct connect after timeout - if let Ok(Ok(peer_conn)) = - timeout(POST_HOLEPUNCH_DIRECT_RETRY_TIMEOUT, self.connect(target)) - .await - { - info!("✓ Post-hole-punch direct connect succeeded to {}", target); + if let Some(peer_conn) = post_direct.await { return Ok(( peer_conn, ConnectionMethod::HolePunched { coordinator }, )); } strategy.record_holepunch_error(round, "Timeout".to_string()); - if strategy.should_retry_holepunch() { + let next_coord = if is_rotating && !is_final_rotation_attempt { + coordinator_candidates + .get(current_preferred_coordinator_idx + 1) + .copied() + } else { + None + }; + if let Some(next_coord) = next_coord { + current_preferred_coordinator_idx += 1; + info!( + "Hole-punch via {} timed out after {:?}, rotating to preferred coordinator {}/{}: {}", + coordinator, + attempt_timeout, + current_preferred_coordinator_idx + 1, + preferred_coordinator_count, + next_coord + ); + strategy.set_coordinator(next_coord); + strategy.increment_round(); + } else if strategy.should_retry_holepunch() { info!( "Hole-punch round {} timed out, retrying with same coordinator", round @@ -3819,4 +3982,86 @@ mod tests { assert_eq!(conn.remote_addr, TransportAddr::Quic(socket_addr)); assert!(conn.authenticated); } + + // ---- Tier 2: preferred-coordinator front-merge ---- + + fn make_addr(octet: u8) -> SocketAddr { + SocketAddr::from(([10, 0, 0, octet], 9000)) + } + + #[test] + fn merge_preferred_coordinators_empty_preferred_is_no_op() { + let mut candidates = vec![make_addr(1), make_addr(2)]; + let original = candidates.clone(); + P2pEndpoint::merge_preferred_coordinators(&mut candidates, &[]); + assert_eq!( + candidates, original, + "empty preferred must not mutate the candidate list" + ); + } + + #[test] + fn merge_preferred_coordinators_inserts_at_front_in_order() { + let mut candidates = vec![make_addr(10), make_addr(11)]; + let preferred = vec![make_addr(1), make_addr(2), make_addr(3)]; + P2pEndpoint::merge_preferred_coordinators(&mut candidates, &preferred); + + assert_eq!( + candidates, + vec![ + make_addr(1), + make_addr(2), + make_addr(3), + make_addr(10), + make_addr(11), + ], + "preferred entries must occupy [0..preferred.len()] in order" + ); + } + + #[test] + fn merge_preferred_coordinators_dedupes_existing_entries() { + // make_addr(2) is BOTH a pre-existing candidate AND in the preferred + // list. After the merge it should appear exactly once, at its + // preferred-list position (index 1), not at its original tail spot. + let mut candidates = vec![make_addr(2), make_addr(10), make_addr(11)]; + let preferred = vec![make_addr(1), make_addr(2)]; + P2pEndpoint::merge_preferred_coordinators(&mut candidates, &preferred); + + assert_eq!( + candidates, + vec![make_addr(1), make_addr(2), make_addr(10), make_addr(11),], + "duplicate preferred entries must end up in the preferred slot, not the tail" + ); + // No accidental duplication. + assert_eq!( + candidates.iter().filter(|a| **a == make_addr(2)).count(), + 1, + "make_addr(2) must appear exactly once after dedup" + ); + } + + #[test] + fn merge_preferred_coordinators_only_dedupes_preferred_entries() { + // Pre-existing candidates that are NOT in the preferred list must + // remain in their original tail order. + let mut candidates = vec![make_addr(10), make_addr(11), make_addr(12)]; + let preferred = vec![make_addr(1)]; + P2pEndpoint::merge_preferred_coordinators(&mut candidates, &preferred); + + assert_eq!( + candidates, + vec![make_addr(1), make_addr(10), make_addr(11), make_addr(12),], + "non-preferred candidates must keep their original relative order" + ); + } + + #[test] + fn merge_preferred_coordinators_works_on_empty_candidate_list() { + let mut candidates: Vec = Vec::new(); + let preferred = vec![make_addr(1), make_addr(2)]; + P2pEndpoint::merge_preferred_coordinators(&mut candidates, &preferred); + + assert_eq!(candidates, vec![make_addr(1), make_addr(2)]); + } } diff --git a/src/unified_config.rs b/src/unified_config.rs index 3886b737..1fae6ad2 100644 --- a/src/unified_config.rs +++ b/src/unified_config.rs @@ -317,6 +317,10 @@ impl P2pConfig { transport_registry: Some(Arc::new(self.transport_registry.clone())), max_message_size: self.max_message_size, allow_loopback: self.nat.allow_loopback, + coordinator_max_active_relays: + crate::nat_traversal_api::NatTraversalConfig::DEFAULT_COORDINATOR_MAX_ACTIVE_RELAYS, + coordinator_relay_slot_timeout: + crate::nat_traversal_api::NatTraversalConfig::DEFAULT_COORDINATOR_RELAY_SLOT_TIMEOUT, upnp: self.nat.upnp.clone(), } } diff --git a/tests/relay_queue_tests.rs b/tests/relay_queue_tests.rs index 20375f1e..c2d43d76 100644 --- a/tests/relay_queue_tests.rs +++ b/tests/relay_queue_tests.rs @@ -55,6 +55,8 @@ mod nat_traversal_api_tests { transport_registry: None, max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: false, + coordinator_max_active_relays: 32, + coordinator_relay_slot_timeout: Duration::from_secs(5), upnp: Default::default(), }; @@ -192,6 +194,8 @@ mod functional_tests { transport_registry: None, max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: false, + coordinator_max_active_relays: 32, + coordinator_relay_slot_timeout: Duration::from_secs(5), upnp: Default::default(), }; @@ -219,6 +223,8 @@ mod functional_tests { transport_registry: None, max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: false, + coordinator_max_active_relays: 32, + coordinator_relay_slot_timeout: Duration::from_secs(5), upnp: Default::default(), }; @@ -412,6 +418,8 @@ mod performance_tests { transport_registry: None, max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: false, + coordinator_max_active_relays: 32, + coordinator_relay_slot_timeout: Duration::from_secs(5), upnp: Default::default(), }; @@ -483,6 +491,8 @@ mod relay_functionality_tests { transport_registry: None, max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: false, + coordinator_max_active_relays: 32, + coordinator_relay_slot_timeout: Duration::from_secs(5), upnp: Default::default(), }; diff --git a/tests/security_regression_tests.rs b/tests/security_regression_tests.rs index 3a8b450c..a9536dd1 100644 --- a/tests/security_regression_tests.rs +++ b/tests/security_regression_tests.rs @@ -37,6 +37,8 @@ fn test_peer_config() -> NatTraversalConfig { transport_registry: None, max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: true, + coordinator_max_active_relays: 32, + coordinator_relay_slot_timeout: Duration::from_secs(5), upnp: Default::default(), } } @@ -62,6 +64,8 @@ fn test_server_config() -> NatTraversalConfig { transport_registry: None, max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: true, + coordinator_max_active_relays: 32, + coordinator_relay_slot_timeout: Duration::from_secs(5), upnp: Default::default(), } } @@ -110,6 +114,8 @@ async fn test_error_handling_no_panic() { transport_registry: None, max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: true, + coordinator_max_active_relays: 32, + coordinator_relay_slot_timeout: Duration::from_secs(5), upnp: Default::default(), }; @@ -139,6 +145,8 @@ async fn test_error_handling_no_panic() { transport_registry: None, max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: true, + coordinator_max_active_relays: 32, + coordinator_relay_slot_timeout: Duration::from_secs(5), upnp: Default::default(), }; @@ -231,6 +239,8 @@ async fn test_malformed_config_handling() { transport_registry: None, max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: true, + coordinator_max_active_relays: 32, + coordinator_relay_slot_timeout: Duration::from_secs(5), upnp: Default::default(), }; @@ -261,6 +271,8 @@ async fn test_malformed_config_handling() { transport_registry: None, max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: true, + coordinator_max_active_relays: 32, + coordinator_relay_slot_timeout: Duration::from_secs(5), upnp: Default::default(), }; @@ -298,6 +310,8 @@ async fn test_input_sanitization() { transport_registry: None, max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: true, + coordinator_max_active_relays: 32, + coordinator_relay_slot_timeout: Duration::from_secs(5), upnp: Default::default(), }; @@ -370,6 +384,8 @@ mod specific_regression_tests { transport_registry: None, max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: true, + coordinator_max_active_relays: 32, + coordinator_relay_slot_timeout: Duration::from_secs(5), upnp: Default::default(), }; @@ -422,6 +438,8 @@ mod specific_regression_tests { transport_registry: None, max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: true, + coordinator_max_active_relays: 32, + coordinator_relay_slot_timeout: Duration::from_secs(5), upnp: Default::default(), }; From 7f2887e8a667ba5b8e60babdd58d0f71da6631ea Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Tue, 7 Apr 2026 01:12:49 +0200 Subject: [PATCH 25/43] refactor!(nat): node-wide coordinator back-pressure via shared RelaySlotTable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses PR #43 deep-review findings H1, H2, H3, M1, M2, L1–L6. ## H1 — node-wide scope (breaking) Back-pressure state moves out of BootstrapCoordinator into a new shared `RelaySlotTable` (`src/relay_slot_table.rs`) that every QUIC connection spawned by a NatTraversalEndpoint references via Arc. The 32-slot cap is now enforced *across all connections at the node*, not per-connection — which is what the PR description originally promised but the code didn't deliver. One table is instantiated in `create_inner_endpoint` and injected into both server-side and client-side TransportConfig. The old `TransportConfig::{coordinator_max_active_relays, coordinator_relay_slot_timeout}` fields are gone; they are replaced by `TransportConfig::relay_slot_table: Option>`. BREAKING: `TransportConfig::coordinator_max_active_relays` and `coordinator_relay_slot_timeout` setters removed. Callers using `NatTraversalConfig` should switch to `coordinator_max_active_relays` (unchanged) and the renamed `coordinator_relay_slot_idle_timeout`. ## H2 — explicit release on connection close `impl Drop for BootstrapCoordinator` calls `RelaySlotTable::release_for_initiator(addr)`, reclaiming every slot owned by the closed connection immediately. The idle timeout is now an honest safety net for crashes/NAT rebinds, not the only release path. The coordinator cannot directly observe hole-punch outcomes (traffic flows initiator↔target, bypassing the coordinator), so the three release mechanisms are: connection close (fast), re-arm refresh, and idle sweep. All three are documented on the config field. ## H3 — drop dead `from_peer` from slot key Slot key is now `(SocketAddr, [u8; 32])` — `(initiator_addr, target_peer_id)`. The old `from_peer = derive_peer_id_from_connection()` was a connection-id hash that was constant per BootstrapCoordinator instance, so keying on it added zero discrimination in production. The socket address is stable across rounds within a session and distinct across initiators, which is exactly what dedup needs. ## M1 — rotation / max_holepunch_rounds interaction documented New paragraph on `set_hole_punch_preferred_coordinators` explains how the rotation index and strategy round counter advance together, and the edge case where a caller raising `max_holepunch_rounds` above K gives the final coordinator extra retries. ## M2 — fields plumbed through `P2pConfig::nat` `NatConfig` gains `coordinator_max_active_relays` and `coordinator_relay_slot_idle_timeout`; `to_nat_config()` now reads from `self.nat.*` instead of hardcoding defaults, so downstream P2pConfig callers can tune the cap. ## Minor fixes - L1: misleading "Coordination completed" trace on the refusal path is replaced with an accurate "refused by node-wide back-pressure" trace. - L2: RelaySlotTable warns at the first refusal and every 16 thereafter so operators see a log line at the start of a storm. - L3: debug_assert! in the connect loop proves `coordinator_candidates[idx] == strategy_coordinator` while rotating. - L4: sweep is amortized — `sweep_if_due` runs the `retain` only if the previous sweep was at least 100ms ago, bounding per-frame cost. - L5: `merge_preferred_coordinators` builds the merged list with one allocation instead of `Vec::insert(0, ..)` in a loop (O(N+M) not O(N·M)). - L6: test helpers use `VarInt::from_u32` directly; dead `unwrap_or` fallback gone. ## Tests - 6 new `RelaySlotTable` unit tests (under-cap, at-cap refuse, re-arm, idle sweep, release_for_initiator, refusal counter). - 4 new `BootstrapCoordinator`-integration tests verifying the shared table is consulted for the relay branch, the non-relay branch doesn't consume a slot, capacity refusal is silent, and — new for H2 — that dropping the coordinator releases exactly the slots it owned while leaving other initiators' slots alone. - Old 5 per-coordinator back-pressure tests are gone (the data structure they exercised no longer exists). - Existing relay_queue_tests.rs and security_regression_tests.rs struct literals updated for the renamed field. 1485 lib tests + 24 integration tests pass; cargo fmt clean; cargo clippy --all-targets -- -D warnings -D clippy::unwrap_used -D clippy::expect_used clean. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/config/transport.rs | 68 +++-- src/connection/mod.rs | 14 +- src/connection/nat_traversal.rs | 386 +++++++++++++---------------- src/lib.rs | 3 + src/nat_traversal_api.rs | 104 ++++---- src/p2p_endpoint.rs | 48 +++- src/relay_slot_table.rs | 325 ++++++++++++++++++++++++ src/unified_config.rs | 33 ++- tests/relay_queue_tests.rs | 10 +- tests/security_regression_tests.rs | 18 +- 10 files changed, 686 insertions(+), 323 deletions(-) create mode 100644 src/relay_slot_table.rs diff --git a/src/config/transport.rs b/src/config/transport.rs index 813247d3..7f75eb12 100644 --- a/src/config/transport.rs +++ b/src/config/transport.rs @@ -66,18 +66,17 @@ pub struct TransportConfig { /// Allow loopback addresses as valid NAT traversal candidates pub(crate) allow_loopback: bool, - /// Maximum number of concurrent hole-punch relay slots this node will - /// service when acting as a coordinator (Tier 4 lite back-pressure). - /// When the active count reaches this cap, incoming `PUNCH_ME_NOW` - /// frames that would otherwise be relayed are silently refused so the - /// initiator advances to its next preferred coordinator. - pub(crate) coordinator_max_active_relays: usize, - - /// Reclamation timeout for stale coordinator relay slots. Acts as a - /// safety net so a relay slot cannot leak forever if a peer crashes - /// mid-coordination, a NAT rebind silently drops the session, or a - /// follow-up signal is lost. - pub(crate) coordinator_relay_slot_timeout: Duration, + /// Shared, node-wide hole-punch coordinator back-pressure table + /// (Tier 4 lite). When this is `Some`, every connection that lands + /// at this node and acts as a coordinator gates incoming + /// `PUNCH_ME_NOW` relay frames against the shared table — the cap + /// is enforced *across* connections, not per-connection. When `None` + /// (low-level test fixtures, internal Quinn-style use), back-pressure + /// is disabled and the coordinator behaves as in pre-Tier-4 builds. + /// + /// Owned and instantiated by `P2pEndpoint::new`; injected into + /// `TransportConfig` before the config is frozen behind `Arc`. + pub(crate) relay_slot_table: Option>, } impl TransportConfig { @@ -484,18 +483,16 @@ impl TransportConfig { self } - /// Cap on simultaneous hole-punch relay slots when this node acts as a - /// coordinator. Higher = more concurrency capacity, lower = stricter - /// back-pressure shedding under storm load. Default: 32. - pub fn coordinator_max_active_relays(&mut self, max: usize) -> &mut Self { - self.coordinator_max_active_relays = max; - self - } - - /// Maximum lifetime of a coordinator relay slot before it is reclaimed - /// by the inline garbage-collection sweep. Default: 5 seconds. - pub fn coordinator_relay_slot_timeout(&mut self, timeout: Duration) -> &mut Self { - self.coordinator_relay_slot_timeout = timeout; + /// Inject the node-wide hole-punch coordinator back-pressure table + /// (Tier 4 lite). Called from `P2pEndpoint::new` so that every QUIC + /// connection spawned from this transport config shares one table. + /// `None` disables back-pressure (used by Quinn-style low-level + /// fixtures that do not run a coordinator). + pub fn relay_slot_table( + &mut self, + table: Option>, + ) -> &mut Self { + self.relay_slot_table = table; self } } @@ -551,11 +548,12 @@ impl Default for TransportConfig { ml_dsa_65: false, }), allow_loopback: false, - // Tier 4 (lite) coordinator back-pressure defaults. See - // `coordinator_max_active_relays`/`coordinator_relay_slot_timeout` - // setters for the rationale behind these numbers. - coordinator_max_active_relays: 32, - coordinator_relay_slot_timeout: Duration::from_secs(5), + // No back-pressure table by default — `P2pEndpoint::new` + // injects one before connections are spawned. Quinn-style + // fixtures that bypass `P2pEndpoint` opt out of coordinator + // back-pressure entirely, which matches the pre-Tier-4 + // behaviour they were originally written against. + relay_slot_table: None, } } } @@ -592,8 +590,7 @@ impl fmt::Debug for TransportConfig { address_discovery_config, pqc_algorithms, allow_loopback, - coordinator_max_active_relays, - coordinator_relay_slot_timeout, + relay_slot_table, } = self; fmt.debug_struct("TransportConfig") .field("max_concurrent_bidi_streams", max_concurrent_bidi_streams) @@ -626,14 +623,7 @@ impl fmt::Debug for TransportConfig { .field("address_discovery_config", address_discovery_config) .field("pqc_algorithms", pqc_algorithms) .field("allow_loopback", allow_loopback) - .field( - "coordinator_max_active_relays", - coordinator_max_active_relays, - ) - .field( - "coordinator_relay_slot_timeout", - coordinator_relay_slot_timeout, - ) + .field("relay_slot_table", relay_slot_table) .finish_non_exhaustive() } } diff --git a/src/connection/mod.rs b/src/connection/mod.rs index b8bad703..eb9c660a 100644 --- a/src/connection/mod.rs +++ b/src/connection/mod.rs @@ -4535,8 +4535,7 @@ impl Connection { max_candidates, coordination_timeout, self.config.allow_loopback, - self.config.coordinator_max_active_relays, - self.config.coordinator_relay_slot_timeout, + self.config.relay_slot_table.clone(), )); trace!("NAT traversal initialized for symmetric P2P node"); @@ -4735,8 +4734,7 @@ impl Connection { 8, Duration::from_secs(10), self.config.allow_loopback, - self.config.coordinator_max_active_relays, - self.config.coordinator_relay_slot_timeout, + self.config.relay_slot_table.clone(), )); } @@ -4870,7 +4868,13 @@ impl Connection { return Ok(()); } Ok(None) => { - trace!("Coordination completed or no action needed"); + // Reaching this branch with `target_peer_id.is_some()` + // (the only branch that calls this) means the + // shared back-pressure table refused the relay. + // The table itself logs and counts the refusal — + // we drop silently so the initiator falls back + // to the per-attempt timeout (Tier 2 rotation). + trace!("PUNCH_ME_NOW relay refused by node-wide back-pressure"); return Ok(()); } Err(e) => { diff --git a/src/connection/nat_traversal.rs b/src/connection/nat_traversal.rs index a0ae3e48..ea02aa0d 100644 --- a/src/connection/nat_traversal.rs +++ b/src/connection/nat_traversal.rs @@ -8,6 +8,7 @@ use std::{ collections::{HashMap, VecDeque}, net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}, + sync::Arc, time::Duration, }; @@ -1821,24 +1822,23 @@ impl NatTraversalState { /// v0.13.0: Role parameter removed - all nodes are symmetric P2P nodes. /// Every node can initiate, accept, and coordinate NAT traversal. /// - /// `coordinator_max_active_relays` and `coordinator_relay_slot_timeout` - /// configure Tier 4 (lite) coordinator-side back-pressure: when this - /// node acts as a hole-punch coordinator, it will silently refuse new - /// `PUNCH_ME_NOW` relays once `coordinator_max_active_relays` slots are - /// in flight, and reclaim stale slots that exceed the timeout. + /// `relay_slot_table` is the shared, node-wide back-pressure table + /// (Tier 4 lite). When `Some`, the bootstrap coordinator embedded in + /// this state gates incoming `PUNCH_ME_NOW` relay frames against the + /// shared table — the cap is enforced across *all* connections at + /// this node, not per-connection. Pass `None` in low-level fixtures + /// that do not run a coordinator. pub(super) fn new( max_candidates: u32, coordination_timeout: Duration, allow_loopback: bool, - coordinator_max_active_relays: usize, - coordinator_relay_slot_timeout: Duration, + relay_slot_table: Option>, ) -> Self { // v0.13.0: All nodes can coordinate - always create coordinator let bootstrap_coordinator = Some(BootstrapCoordinator::new( BootstrapConfig::default(), allow_loopback, - coordinator_max_active_relays, - coordinator_relay_slot_timeout, + relay_slot_table, )); Self { @@ -3566,22 +3566,33 @@ pub(crate) struct BootstrapCoordinator { security_validator: SecurityValidationState, /// Statistics for bootstrap operations stats: BootstrapStats, - /// Active hole-punch relay slots indexed by `(initiator_peer_id, - /// target_peer_id)`. Each entry records the wall-clock arrival time of - /// the `PUNCH_ME_NOW` frame that opened the slot. Slots are reclaimed - /// either implicitly (when a follow-up frame from the same pair lands) - /// or by the inline garbage-collection sweep run on every incoming - /// frame: any slot older than `coordinator_relay_slot_timeout` is - /// evicted before the back-pressure check, which keeps the active - /// count from leaking on ghost sessions where the initiator crashed - /// or the target became unreachable mid-coordination. - relay_slots: HashMap<(PeerId, PeerId), Instant>, - /// Cap on simultaneous relay slots (Tier 4 lite back-pressure). - /// Plumbed from [`crate::config::TransportConfig::coordinator_max_active_relays`]. - relay_slot_capacity: usize, - /// Reclamation timeout for stale relay slots. Plumbed from - /// [`crate::config::TransportConfig::coordinator_relay_slot_timeout`]. - relay_slot_timeout: Duration, + /// Shared, node-wide back-pressure table (Tier 4 lite). When `Some`, + /// every incoming `PUNCH_ME_NOW` relay frame must acquire a slot in + /// this table before being relayed; the cap is enforced *across all* + /// connections at this node, not per-connection. + /// + /// On `Drop` (i.e. when the connection that hosts this coordinator + /// closes) all slots whose initiator address matches the connection's + /// remote address are released — the explicit-completion path that + /// reclaims capacity ahead of the idle-timeout safety net. + relay_slot_table: Option>, + /// Remote address of the connection that owns this coordinator. + /// Captured the first time we relay a frame; used as the slot key's + /// initiator-side identifier and as the argument to + /// `release_for_initiator` in [`Drop`]. `None` until the first + /// `PUNCH_ME_NOW` arrives. + relay_initiator_addr: Option, +} + +impl Drop for BootstrapCoordinator { + fn drop(&mut self) { + // Explicitly release every slot we opened so the shared table + // doesn't have to wait out the idle timeout for a connection + // that has just closed. + if let (Some(table), Some(addr)) = (&self.relay_slot_table, self.relay_initiator_addr) { + table.release_for_initiator(addr); + } + } } // Removed legacy CoordinationSessionId type /// Peer identifier for bootstrap coordination @@ -3667,25 +3678,21 @@ pub(crate) struct BootstrapStats { successful_coordinations: u64, /// Security rejections security_rejections: u64, - /// Refusals due to active-relay back-pressure (Tier 4 lite). Tracks the - /// number of `PUNCH_ME_NOW` frames silently dropped because the - /// coordinator was at `relay_slot_capacity` when they arrived. - backpressure_refusals: u64, } // Removed session state machine enums and recovery actions impl BootstrapCoordinator { /// Create a new bootstrap coordinator. /// - /// `relay_slot_capacity` and `relay_slot_timeout` configure the - /// Tier 4 (lite) back-pressure: incoming `PUNCH_ME_NOW` relay frames - /// are silently refused once `relay_slot_capacity` slots are in - /// flight, and any slot older than `relay_slot_timeout` is reclaimed - /// by the inline garbage-collection sweep on every incoming frame. + /// `relay_slot_table` is the shared, node-wide back-pressure table + /// (Tier 4 lite). When `Some`, incoming `PUNCH_ME_NOW` relay frames + /// must acquire a slot from the table before being relayed; the cap + /// is enforced across all connections at this node. Pass `None` in + /// low-level test fixtures that exercise the connection state machine + /// without a P2pEndpoint. pub(crate) fn new( _config: BootstrapConfig, allow_loopback: bool, - relay_slot_capacity: usize, - relay_slot_timeout: Duration, + relay_slot_table: Option>, ) -> Self { Self { address_observations: HashMap::new(), @@ -3693,21 +3700,11 @@ impl BootstrapCoordinator { coordination_table: HashMap::new(), security_validator: SecurityValidationState::new(allow_loopback), stats: BootstrapStats::default(), - relay_slots: HashMap::new(), - relay_slot_capacity, - relay_slot_timeout, + relay_slot_table, + relay_initiator_addr: None, } } - /// Reclaim relay slots whose arrival timestamp is older than the - /// configured `relay_slot_timeout`. Called inline at the start of - /// every `PUNCH_ME_NOW` frame so that ghost slots from crashed peers - /// or dropped sessions cannot leak the active-relay counter. - fn sweep_stale_relay_slots(&mut self, now: Instant) { - let timeout = self.relay_slot_timeout; - self.relay_slots - .retain(|_, &mut arrived_at| now.duration_since(arrived_at) < timeout); - } /// Observe a peer's address from an incoming connection /// /// This is called when a peer connects to this bootstrap node, @@ -3844,34 +3841,34 @@ impl BootstrapCoordinator { })?; // Tier 4 (lite) back-pressure: only the relay branch (where the - // frame carries an explicit `target_peer_id`) consumes a slot. If - // we are at capacity for active relays, silently drop the frame — - // the initiator's per-attempt timeout (Tier 2 rotation) will move - // it on to its next preferred coordinator. + // frame carries an explicit `target_peer_id`) consumes a slot. + // The shared `RelaySlotTable` enforces the cap *across all + // connections* at this node — when full, the relay is silently + // refused and the initiator's per-attempt timeout (Tier 2 + // rotation) drives it to its next preferred coordinator. // - // Inline garbage-collection sweep first so any stale slots from - // crashed peers are reclaimed before the cap check. + // Slots are keyed by `(initiator_addr, target_peer_id)`. The + // initiator address is the connection's remote socket address + // (constant for the lifetime of this BootstrapCoordinator), so + // multi-round coordination from the same peer naturally re-arms + // the same slot without consuming additional capacity. if let Some(target_peer_id) = frame.target_peer_id { - self.sweep_stale_relay_slots(now); - let slot_key = (from_peer, target_peer_id); - // Re-arming an existing slot for the same (initiator, target) - // pair does not consume an additional slot — common during - // multi-round coordination where the same pair re-sends. - let already_active = self.relay_slots.contains_key(&slot_key); - if !already_active && self.relay_slots.len() >= self.relay_slot_capacity { - self.stats.backpressure_refusals = - self.stats.backpressure_refusals.saturating_add(1); - debug!( - "PUNCH_ME_NOW relay refused: coordinator at capacity ({}/{}) — initiator {:?} → target {:?}", - self.relay_slots.len(), - self.relay_slot_capacity, - hex::encode(&from_peer[..8]), - hex::encode(&target_peer_id[..8]) - ); + // Cache the initiator addr the first time we see it so + // `Drop` can release every slot we opened, even if the + // connection closes mid-session. + if self.relay_initiator_addr.is_none() { + self.relay_initiator_addr = Some(source_addr); + } + if let Some(table) = &self.relay_slot_table + && !table.try_acquire(source_addr, target_peer_id, now) + { + // Refused. The table itself logs/counts the event; + // returning `Ok(None)` means "no coordination frame + // produced" and is dispatched at the call site as a + // silent drop, surfacing to the initiator only as a + // per-attempt timeout. return Ok(None); } - // Accept: insert/refresh the slot timestamp. - self.relay_slots.insert(slot_key, now); } // Track coordination entry minimally @@ -3986,14 +3983,23 @@ impl BootstrapCoordinator { mod tests { use super::*; + /// Build a test fixture `RelaySlotTable` so the BootstrapCoordinator + /// embedded in `NatTraversalState` can exercise the back-pressure + /// path. Production code obtains the table from `P2pEndpoint`. + fn make_test_relay_slot_table() -> Arc { + Arc::new(crate::relay_slot_table::RelaySlotTable::new( + 32, + Duration::from_secs(5), + )) + } + // v0.13.0: Role parameter removed - all nodes are symmetric P2P nodes fn create_test_state() -> NatTraversalState { NatTraversalState::new( 10, // max_candidates Duration::from_secs(30), // coordination_timeout true, // allow_loopback for tests - 32, // coordinator_max_active_relays - Duration::from_secs(5), // coordinator_relay_slot_timeout + Some(make_test_relay_slot_table()), ) } @@ -4376,8 +4382,7 @@ mod tests { 100, // max_candidates (high enough to not limit) Duration::from_secs(30), true, // allow_loopback for tests - 32, - Duration::from_secs(5), + Some(make_test_relay_slot_table()), ); let now = Instant::now(); @@ -4412,8 +4417,7 @@ mod tests { 100, Duration::from_secs(30), true, - 32, - Duration::from_secs(5), + Some(make_test_relay_slot_table()), ); let now = Instant::now(); @@ -4498,8 +4502,7 @@ mod tests { 100, Duration::from_secs(30), true, - 32, - Duration::from_secs(5), + Some(make_test_relay_slot_table()), ); let now = Instant::now(); @@ -4534,22 +4537,40 @@ mod tests { } // ---- Tier 4 (lite): coordinator-side back-pressure ---- - - /// Helper: build a `BootstrapCoordinator` with controllable cap and - /// timeout for back-pressure tests. - fn make_bp_coordinator(capacity: usize, timeout: Duration) -> BootstrapCoordinator { - BootstrapCoordinator::new( + // + // The pure data-structure tests live next to the table itself in + // `crate::relay_slot_table::tests`. The tests below verify the + // *integration* between `BootstrapCoordinator` and the shared + // `RelaySlotTable`: that the relay branch consumes a slot, the + // non-relay (echo) branch does not, and that the coordinator + // releases its slots in `Drop` so a closed connection reclaims + // capacity ahead of the idle-timeout safety net. + + /// Build a `BootstrapCoordinator` wired to a fresh shared + /// `RelaySlotTable` with the given capacity. Returns both so tests + /// can inspect the table directly. + fn make_coord_with_table( + capacity: usize, + timeout: Duration, + ) -> ( + BootstrapCoordinator, + Arc, + ) { + let table = Arc::new(crate::relay_slot_table::RelaySlotTable::new( + capacity, timeout, + )); + let coord = BootstrapCoordinator::new( BootstrapConfig::default(), - true, // allow_loopback so test addrs aren't rejected - capacity, - timeout, - ) + true, // allow_loopback for test addrs + Some(Arc::clone(&table)), + ); + (coord, table) } - /// Helper: build a `PunchMeNow` frame for the relay path (with target). - fn make_relay_frame(round: u64, target_peer_id: [u8; 32]) -> crate::frame::PunchMeNow { + /// `PunchMeNow` frame for the relay path (with target). + fn relay_frame(round: u32, target_peer_id: [u8; 32]) -> crate::frame::PunchMeNow { crate::frame::PunchMeNow { - round: VarInt::from_u64(round).unwrap_or(VarInt::from_u32(0)), + round: VarInt::from_u32(round), paired_with_sequence_number: VarInt::from_u32(0), address: SocketAddr::from(([127, 0, 0, 1], 9000)), target_peer_id: Some(target_peer_id), @@ -4563,151 +4584,52 @@ mod tests { } #[test] - fn coordinator_accepts_relay_under_capacity() { - let mut coord = make_bp_coordinator(4, Duration::from_secs(5)); + fn coordinator_relay_consumes_shared_slot() { + let (mut coord, table) = make_coord_with_table(4, Duration::from_secs(5)); let now = Instant::now(); let from = peer_id_with_byte(0x01); let target = peer_id_with_byte(0x02); - let frame = make_relay_frame(1, target); let source_addr = SocketAddr::from(([127, 0, 0, 1], 5000)); let result = coord - .process_punch_me_now_frame(from, source_addr, &frame, now) + .process_punch_me_now_frame(from, source_addr, &relay_frame(1, target), now) .expect("relay under cap should not error"); assert!( result.is_some(), "relay under capacity should produce a coordination frame" ); - assert_eq!(coord.relay_slots.len(), 1, "one slot should be allocated"); - assert_eq!( - coord.stats.backpressure_refusals, 0, - "no refusal stat increment expected when under cap" - ); + assert_eq!(table.active_count(), 1); + assert_eq!(table.backpressure_refusals(), 0); } #[test] - fn coordinator_refuses_silently_at_capacity() { - let capacity = 2; - let mut coord = make_bp_coordinator(capacity, Duration::from_secs(5)); + fn coordinator_refuses_silently_when_table_at_capacity() { + // Pre-fill the shared table from outside the coordinator. The + // coordinator's relay attempt then sees the cap and silently + // refuses, returning Ok(None). + let (mut coord, table) = make_coord_with_table(1, Duration::from_secs(5)); let now = Instant::now(); - let source_addr = SocketAddr::from(([127, 0, 0, 1], 5000)); - - // Fill capacity with two distinct (initiator, target) pairs. - for i in 0..capacity { - let from = peer_id_with_byte(0x10 + i as u8); - let target = peer_id_with_byte(0x20 + i as u8); - let frame = make_relay_frame(i as u64 + 1, target); - let result = coord - .process_punch_me_now_frame(from, source_addr, &frame, now) - .expect("under-cap relay should not error"); - assert!(result.is_some(), "slot {} should be accepted", i); - } - assert_eq!(coord.relay_slots.len(), capacity); - - // The next distinct pair must be silently refused: Ok(None). - let overflow_from = peer_id_with_byte(0xFE); - let overflow_target = peer_id_with_byte(0xFD); - let overflow_frame = make_relay_frame(99, overflow_target); - let result = coord - .process_punch_me_now_frame(overflow_from, source_addr, &overflow_frame, now) - .expect("at-cap refusal must be silent (no error)"); - - assert!( - result.is_none(), - "at-cap refusal must produce no coordination frame" - ); - assert_eq!( - coord.relay_slots.len(), - capacity, - "refused frame must not consume a slot" - ); - assert_eq!( - coord.stats.backpressure_refusals, 1, - "back-pressure stat must increment on refusal" - ); - } + let other_initiator = SocketAddr::from(([127, 0, 0, 1], 9999)); + assert!(table.try_acquire(other_initiator, peer_id_with_byte(0xAB), now)); + assert_eq!(table.active_count(), 1); - #[test] - fn coordinator_re_arms_existing_slot_without_consuming_capacity() { - let mut coord = make_bp_coordinator(2, Duration::from_secs(5)); - let now = Instant::now(); - let source_addr = SocketAddr::from(([127, 0, 0, 1], 5000)); let from = peer_id_with_byte(0x01); let target = peer_id_with_byte(0x02); - let frame = make_relay_frame(1, target); - - // Fill the first slot. - coord - .process_punch_me_now_frame(from, source_addr, &frame, now) - .expect("first frame ok"); - assert_eq!(coord.relay_slots.len(), 1); - - // Re-send for the same (from, target) pair: must not consume an - // additional slot, must still be accepted. - let later = now + Duration::from_millis(500); - let result = coord - .process_punch_me_now_frame(from, source_addr, &frame, later) - .expect("re-arm ok"); - assert!(result.is_some(), "re-armed slot should still be relayed"); - assert_eq!( - coord.relay_slots.len(), - 1, - "re-arming the same pair must not allocate a second slot" - ); - // Slot timestamp should refresh to the later instant. - let slot_arrived = coord - .relay_slots - .get(&(from, target)) - .copied() - .expect("slot present"); - assert_eq!( - slot_arrived, later, - "re-arm must refresh the slot timestamp" - ); - } - - #[test] - fn coordinator_sweep_reclaims_stale_slots() { - let timeout = Duration::from_secs(5); - let mut coord = make_bp_coordinator(2, timeout); - let now = Instant::now(); let source_addr = SocketAddr::from(([127, 0, 0, 1], 5000)); - - // Fill capacity. - for i in 0..2u8 { - let from = peer_id_with_byte(0x10 + i); - let target = peer_id_with_byte(0x20 + i); - let frame = make_relay_frame(i as u64 + 1, target); - coord - .process_punch_me_now_frame(from, source_addr, &frame, now) - .expect("under-cap ok"); - } - assert_eq!(coord.relay_slots.len(), 2); - - // Advance well past the slot timeout. The next incoming frame's - // inline sweep must reclaim both stale slots before applying the - // back-pressure check. - let much_later = now + timeout + Duration::from_secs(1); - let new_from = peer_id_with_byte(0xAA); - let new_target = peer_id_with_byte(0xBB); - let new_frame = make_relay_frame(42, new_target); let result = coord - .process_punch_me_now_frame(new_from, source_addr, &new_frame, much_later) - .expect("post-sweep frame should succeed"); + .process_punch_me_now_frame(from, source_addr, &relay_frame(1, target), now) + .expect("refusal must be silent (Ok)"); assert!( - result.is_some(), - "frame after sweep must be accepted (capacity reclaimed)" + result.is_none(), + "at-cap refusal must produce no coordination frame" ); + assert_eq!(table.active_count(), 1, "refused frame must not insert"); assert_eq!( - coord.relay_slots.len(), + table.backpressure_refusals(), 1, - "stale slots must be reclaimed; only the new one remains" - ); - assert_eq!( - coord.stats.backpressure_refusals, 0, - "no refusal expected because sweep freed capacity in time" + "table refusal stat must increment" ); } @@ -4715,7 +4637,7 @@ mod tests { fn coordinator_non_relay_frame_does_not_consume_slot() { // PUNCH_ME_NOW without a target_peer_id is the response/echo path, // not a relay request — it must NOT consume a back-pressure slot. - let mut coord = make_bp_coordinator(1, Duration::from_secs(5)); + let (mut coord, table) = make_coord_with_table(1, Duration::from_secs(5)); let now = Instant::now(); let source_addr = SocketAddr::from(([127, 0, 0, 1], 5000)); let from = peer_id_with_byte(0x01); @@ -4730,9 +4652,47 @@ mod tests { .process_punch_me_now_frame(from, source_addr, &frame, now) .expect("non-relay frame ok"); assert_eq!( - coord.relay_slots.len(), + table.active_count(), 0, "non-relay frame must not consume a slot" ); } + + #[test] + fn coordinator_drop_releases_owned_slots() { + // This is the "explicit completion" path that fixes H2 — when + // the connection that hosts a coordinator drops, every slot it + // opened must be reclaimed without waiting out the idle timeout. + let (mut coord, table) = make_coord_with_table(8, Duration::from_secs(5)); + let now = Instant::now(); + let from = peer_id_with_byte(0x01); + let source_addr = SocketAddr::from(([127, 0, 0, 1], 5000)); + + // Open three slots from this coordinator (three distinct targets). + for t in [0xAA, 0xBB, 0xCC] { + let _ = coord + .process_punch_me_now_frame( + from, + source_addr, + &relay_frame(1, peer_id_with_byte(t)), + now, + ) + .expect("relay under cap ok"); + } + // And one slot from a *different* initiator (a different + // BootstrapCoordinator instance would normally own this; we + // simulate by acquiring directly). + let other_initiator = SocketAddr::from(([10, 0, 0, 1], 7777)); + assert!(table.try_acquire(other_initiator, peer_id_with_byte(0xDD), now)); + assert_eq!(table.active_count(), 4); + + // Drop the coordinator. Its three slots must be released; the + // other initiator's slot must remain. + drop(coord); + assert_eq!( + table.active_count(), + 1, + "Drop must release every slot owned by this initiator address" + ); + } } diff --git a/src/lib.rs b/src/lib.rs index 3f98961a..1e2ed47d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -259,6 +259,9 @@ pub mod metrics; /// TURN-style relay protocol for NAT traversal fallback pub mod relay; +/// Node-wide hole-punch coordinator back-pressure (Tier 4 lite). +pub mod relay_slot_table; + /// MASQUE CONNECT-UDP Bind protocol for fully connectable P2P nodes pub mod masque; diff --git a/src/nat_traversal_api.rs b/src/nat_traversal_api.rs index f757c082..8b1a05be 100644 --- a/src/nat_traversal_api.rs +++ b/src/nat_traversal_api.rs @@ -492,44 +492,51 @@ pub struct NatTraversalConfig { #[serde(default)] pub allow_loopback: bool, - /// Maximum number of concurrent hole-punch relay slots this node will - /// service as a coordinator. + /// Cap on simultaneous in-flight hole-punch coordinator sessions + /// **across the entire node** (Tier 4 lite back-pressure). /// - /// When the active relay count reaches this cap, incoming `PUNCH_ME_NOW` - /// frames that would otherwise be relayed are *silently refused*: the - /// coordinator drops the relay without notifying the initiator. The - /// initiator's per-attempt timeout (Tier 2 rotation) drives it to - /// advance to the next preferred coordinator in its list. + /// When the shared `RelaySlotTable` is full, additional `PUNCH_ME_NOW` + /// relay frames are *silently refused*: the coordinator drops them + /// without notifying the initiator, and the initiator's per-attempt + /// timeout (Tier 2 rotation) advances to the next preferred + /// coordinator in its list. /// - /// Defaults to [`NatTraversalConfig::DEFAULT_COORDINATOR_MAX_ACTIVE_RELAYS`] - /// (32). Picked as roughly 32% of saorsa-core's 100-connection cap so - /// the coordinator still has headroom for its own peer traffic, while - /// being low enough that a cold-start storm is shed and pushed onto - /// alternates. + /// A "session" is one `(initiator_addr, target_peer_id)` pair. The + /// same pair re-sending across rounds re-arms one slot rather than + /// allocating new ones. Slots are released either by the explicit + /// connection-close path (when the initiator's connection drops, the + /// `BootstrapCoordinator::Drop` releases every slot it owned) or by + /// the [`Self::coordinator_relay_slot_idle_timeout`] safety net for + /// peers that vanish without an orderly close. /// - /// Each "active relay" represents one in-flight initiator→target - /// coordination, indexed by `(initiator_peer_id, target_peer_id)`. The - /// counter is decremented when the relay completes or after - /// [`Self::coordinator_relay_slot_timeout`] elapses (whichever first). + /// Defaults to [`NatTraversalConfig::DEFAULT_COORDINATOR_MAX_ACTIVE_RELAYS`] + /// (32). Sized to keep a coordinator's worst-case in-flight + /// coordination work bounded under a cold-start storm of peers all + /// converging on the same bootstrap, while still leaving headroom + /// for steady-state per-peer traffic. #[serde(default = "default_coordinator_max_active_relays")] pub coordinator_max_active_relays: usize, - /// Maximum lifetime of a coordinator relay slot before it is reclaimed - /// by the inline garbage-collection sweep. + /// Idle-release timeout for an in-flight coordinator relay session. /// - /// A successful hole-punch typically completes in 1-3 seconds; this - /// timeout exists purely as a safety net so that a relay slot cannot - /// leak forever if a peer crashes mid-coordination, a NAT rebind - /// silently drops the session, or a follow-up signal is lost. Without - /// it the active-relay counter would drift upward over time and the - /// coordinator would eventually refuse legitimate traffic. + /// A slot lasts from the first `PUNCH_ME_NOW` arrival until either + /// (a) the connection that owns it closes — in which case + /// `BootstrapCoordinator::Drop` releases all of that connection's + /// slots immediately, or (b) no new round arrives for the same + /// `(initiator_addr, target_peer_id)` pair within this idle window — + /// the *safety net* for peers that crash, get NAT-rebound, or stop + /// rotating without an orderly close. The coordinator cannot + /// directly observe whether the punch ultimately succeeded (the + /// punch traffic flows initiator↔target, bypassing the coordinator), + /// so the idle timeout is the only signal available for "vanished" + /// sessions. /// - /// Defaults to [`NatTraversalConfig::DEFAULT_COORDINATOR_RELAY_SLOT_TIMEOUT`] + /// Defaults to [`NatTraversalConfig::DEFAULT_COORDINATOR_RELAY_SLOT_IDLE_TIMEOUT`] /// (5 seconds): comfortably above the worst-case successful punch - /// latency on high-RTT links, but short enough to keep ghost slots - /// from impacting steady-state burst capacity. - #[serde(default = "default_coordinator_relay_slot_timeout")] - pub coordinator_relay_slot_timeout: Duration, + /// latency on high-RTT links, short enough to keep capacity from + /// being held by ghost sessions. + #[serde(default = "default_coordinator_relay_slot_idle_timeout")] + pub coordinator_relay_slot_idle_timeout: Duration, /// Best-effort UPnP IGD port mapping configuration. /// @@ -552,18 +559,19 @@ fn default_coordinator_max_active_relays() -> usize { NatTraversalConfig::DEFAULT_COORDINATOR_MAX_ACTIVE_RELAYS } -fn default_coordinator_relay_slot_timeout() -> Duration { - NatTraversalConfig::DEFAULT_COORDINATOR_RELAY_SLOT_TIMEOUT +fn default_coordinator_relay_slot_idle_timeout() -> Duration { + NatTraversalConfig::DEFAULT_COORDINATOR_RELAY_SLOT_IDLE_TIMEOUT } impl NatTraversalConfig { - /// Default cap on simultaneous coordinator relay slots. + /// Default cap on simultaneous coordinator relay sessions. /// See [`Self::coordinator_max_active_relays`] for rationale. pub const DEFAULT_COORDINATOR_MAX_ACTIVE_RELAYS: usize = 32; - /// Default reclamation timeout for stale coordinator relay slots. - /// See [`Self::coordinator_relay_slot_timeout`] for rationale. - pub const DEFAULT_COORDINATOR_RELAY_SLOT_TIMEOUT: Duration = Duration::from_secs(5); + /// Default idle-release timeout for in-flight coordinator relay + /// sessions. See [`Self::coordinator_relay_slot_idle_timeout`] for + /// rationale. + pub const DEFAULT_COORDINATOR_RELAY_SLOT_IDLE_TIMEOUT: Duration = Duration::from_secs(5); } /// Convert `max_message_size` to a QUIC `VarInt` for stream/send window configuration. @@ -1146,7 +1154,7 @@ impl Default for NatTraversalConfig { max_message_size: crate::unified_config::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: false, coordinator_max_active_relays: Self::DEFAULT_COORDINATOR_MAX_ACTIVE_RELAYS, - coordinator_relay_slot_timeout: Self::DEFAULT_COORDINATOR_RELAY_SLOT_TIMEOUT, + coordinator_relay_slot_idle_timeout: Self::DEFAULT_COORDINATOR_RELAY_SLOT_IDLE_TIMEOUT, upnp: crate::upnp::UpnpConfig::default(), } } @@ -1201,14 +1209,14 @@ impl ConfigValidator for NatTraversalConfig { validate_range( self.coordinator_max_active_relays, 1, - 256, + 1024, "coordinator_max_active_relays", )?; validate_duration( - self.coordinator_relay_slot_timeout, + self.coordinator_relay_slot_idle_timeout, Duration::from_millis(100), Duration::from_secs(60), - "coordinator_relay_slot_timeout", + "coordinator_relay_slot_idle_timeout", )?; Ok(()) @@ -2623,6 +2631,18 @@ impl NatTraversalEndpoint { > { use std::sync::Arc; + // Tier 4 (lite) coordinator back-pressure: every connection + // spawned by this endpoint shares ONE node-wide + // `RelaySlotTable`. Both the server-side `TransportConfig` and + // the client-side `TransportConfig` get a clone of the same + // `Arc`, so a relay arriving on a server-accepted connection + // and a relay arriving on a client-initiated connection both + // count against the same cap. + let relay_slot_table = Arc::new(crate::relay_slot_table::RelaySlotTable::new( + config.coordinator_max_active_relays, + config.coordinator_relay_slot_idle_timeout, + )); + // v0.13.0+: All nodes are symmetric P2P nodes - always create server config let server_config = { info!("Creating server config using Raw Public Keys (RFC 7250) for symmetric P2P node"); @@ -2686,8 +2706,7 @@ impl NatTraversalEndpoint { }; transport_config.nat_traversal_config(Some(nat_config)); transport_config.allow_loopback(config.allow_loopback); - transport_config.coordinator_max_active_relays(config.coordinator_max_active_relays); - transport_config.coordinator_relay_slot_timeout(config.coordinator_relay_slot_timeout); + transport_config.relay_slot_table(Some(Arc::clone(&relay_slot_table))); server_config.transport_config(Arc::new(transport_config)); @@ -2759,8 +2778,7 @@ impl NatTraversalEndpoint { }; transport_config.nat_traversal_config(Some(nat_config)); transport_config.allow_loopback(config.allow_loopback); - transport_config.coordinator_max_active_relays(config.coordinator_max_active_relays); - transport_config.coordinator_relay_slot_timeout(config.coordinator_relay_slot_timeout); + transport_config.relay_slot_table(Some(Arc::clone(&relay_slot_table))); client_config.transport_config(Arc::new(transport_config)); diff --git a/src/p2p_endpoint.rs b/src/p2p_endpoint.rs index 6d09425e..b15fcf88 100644 --- a/src/p2p_endpoint.rs +++ b/src/p2p_endpoint.rs @@ -1257,6 +1257,23 @@ impl P2pEndpoint { /// hole-punch timeout to give it time to complete the punch. /// /// Empty `coordinators` removes any preferred coordinators for `target`. + /// + /// ## Interaction with `StrategyConfig::max_holepunch_rounds` + /// + /// Each rotation step in the connect loop calls + /// `ConnectionStrategy::increment_round`, so the strategy's per-round + /// counter and the rotation index advance together. With the default + /// `max_holepunch_rounds = 2`, supplying `K ≥ 2` preferred coordinators + /// gives each coordinator (including the final one) exactly one + /// attempt — the rotation fully replaces the legacy retry loop and the + /// worst-case dial time is `(K-1) * 1.5s + 8s`. + /// + /// If a caller has explicitly raised `max_holepunch_rounds` (e.g. + /// `with_max_holepunch_rounds(5)`) **and** also supplies a preferred + /// list, the *final* coordinator inherits the leftover round budget + /// — it will be retried `max_rounds - K + 1` times at the full + /// hole-punch timeout. This is usually fine but worth knowing if you + /// were expecting the rotation to be the only retry mechanism. pub async fn set_hole_punch_preferred_coordinators( &self, target: SocketAddr, @@ -1381,11 +1398,13 @@ impl P2pEndpoint { // Drop any pre-existing copies of the preferred entries from the // tail so we don't end up with duplicates after the front-insert. coordinator_candidates.retain(|a| !preferred.contains(a)); - // Insert in reverse so `preferred[0]` ends up at index 0, - // `preferred[1]` at index 1, etc. - for preferred_addr in preferred.iter().rev() { - coordinator_candidates.insert(0, *preferred_addr); - } + // Build the merged list in one allocation rather than calling + // `Vec::insert(0, ..)` in a loop (which shifts the entire tail + // on every iteration — O(N·M) instead of O(N+M)). + let mut merged = Vec::with_capacity(preferred.len() + coordinator_candidates.len()); + merged.extend_from_slice(preferred); + merged.append(coordinator_candidates); + *coordinator_candidates = merged; } /// Inner implementation of connect_with_fallback (separated for dedup wrapper). @@ -1703,6 +1722,25 @@ impl P2pEndpoint { strategy.holepunch_timeout() }; + // Invariant: while rotating, the strategy's current + // coordinator must equal `coordinator_candidates[idx]`. + // This is maintained by `set_coordinator()` on every + // rotation step; the assert catches any future + // regression where a caller sets the strategy's + // coordinator out of band without updating the + // candidate list. + debug_assert!( + !is_rotating + || coordinator_candidates + .get(current_preferred_coordinator_idx) + .copied() + == Some(coordinator), + "rotation index out of sync with strategy coordinator: idx={}, coord={}, candidates={:?}", + current_preferred_coordinator_idx, + coordinator, + coordinator_candidates, + ); + info!( "Trying hole-punch to {} via {} (round {}, attempt timeout {:?}, rotating={})", target, coordinator, round, attempt_timeout, is_rotating diff --git a/src/relay_slot_table.rs b/src/relay_slot_table.rs new file mode 100644 index 00000000..a03b860d --- /dev/null +++ b/src/relay_slot_table.rs @@ -0,0 +1,325 @@ +// Copyright 2024 Saorsa Labs Ltd. +// +// This Saorsa Network Software is licensed under the General Public License (GPL), version 3. +// Please see the file LICENSE-GPL, or visit for the full text. +// +// Full details available at https://saorsalabs.com/licenses + +//! Node-wide hole-punch coordinator back-pressure (Tier 4 lite). +//! +//! Every connection that lands at a node and acts as a hole-punch coordinator +//! shares one [`RelaySlotTable`]. The table caps the number of in-flight +//! `(initiator, target)` relay sessions across the entire node, so a storm +//! of cold-starting peers cannot pile up unbounded coordination work on a +//! single bootstrap. When the cap is reached, additional `PUNCH_ME_NOW` +//! relay frames are silently refused — the initiator's per-attempt timeout +//! drives it to its next preferred coordinator (Tier 2 rotation). +//! +//! ## Lifetime model +//! +//! A "slot" represents an active coordination *session*: the same +//! `(initiator_addr, target_peer_id)` pair sending one or more +//! `PUNCH_ME_NOW` frames over the lifetime of a hole-punch attempt. The +//! coordinator cannot directly observe whether a punch ultimately succeeded +//! (the punch traffic flows initiator↔target, bypassing the coordinator), +//! so slot release happens via three mechanisms: +//! +//! 1. **Inactivity timeout** ([`RelaySlotTable::idle_timeout`]). If no new +//! rounds for the same key arrive within this window the session is +//! considered done — either the punch succeeded (no more rounds needed) +//! or it definitively failed (the initiator rotated away). Default 5s. +//! +//! 2. **Connection close** via [`RelaySlotTable::release_for_initiator`]. +//! When the initiator's connection drops, every slot it owned is +//! reclaimed immediately rather than waiting for the inactivity timeout. +//! Called from `BootstrapCoordinator::Drop`. +//! +//! 3. **Explicit re-arm refresh**. A re-sent frame for the same key +//! refreshes the timestamp without consuming additional capacity. +//! +//! ## Key choice +//! +//! Slots are keyed by `(initiator_addr, target_peer_id)` rather than +//! `(initiator_peer_id, target_peer_id)` because the cryptographic PeerId +//! is not available inside the QUIC connection state machine where the +//! `PUNCH_ME_NOW` frame is processed (PQC auth state lives one layer up +//! in `P2pEndpoint`). The remote socket address is constant across rounds +//! within a session and unique enough across distinct initiators to give +//! correct dedup behaviour for the back-pressure cap. + +use std::collections::HashMap; +use std::net::SocketAddr; +use std::sync::Mutex; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, Instant}; + +use tracing::{debug, warn}; + +/// Cryptographic peer identifier — BLAKE3 hash of an ML-DSA-65 public key. +/// Local alias to keep the table independent of the connection layer. +pub(crate) type RelayTargetId = [u8; 32]; + +/// Minimum interval between consecutive amortized sweeps. Sweeping less +/// often than this on a hot path keeps the per-frame overhead bounded +/// without letting expired entries pile up. +const SWEEP_AMORTIZATION_INTERVAL: Duration = Duration::from_millis(100); + +/// One refusal warning every this many refusals, so an operator gets a +/// log line at the start of a storm and periodically thereafter without +/// flooding logs at line-rate. +const REFUSAL_WARN_INTERVAL: u64 = 16; + +/// Node-wide table of in-flight hole-punch coordinator relay slots. +/// +/// Cheap to clone via `Arc`. Internal state is guarded by a single +/// `Mutex`; contention is bounded because each acquire/release holds the +/// lock for a short critical section (a HashMap lookup plus optional +/// amortized retain). +pub struct RelaySlotTable { + inner: Mutex, + capacity: usize, + idle_timeout: Duration, + backpressure_refusals: AtomicU64, +} + +struct RelaySlotTableInner { + slots: HashMap<(SocketAddr, RelayTargetId), Instant>, + last_swept: Instant, +} + +impl RelaySlotTable { + /// Create a new shared table with the given capacity and idle timeout. + /// + /// `capacity` caps the number of distinct simultaneous in-flight + /// `(initiator_addr, target_peer_id)` sessions across the node. + /// `idle_timeout` is how long a slot lingers after its last refresh + /// before being reclaimed by the inline sweep — picks up the slack + /// when an initiator stops sending without explicitly releasing + /// (e.g. NAT rebind or process crash). + pub fn new(capacity: usize, idle_timeout: Duration) -> Self { + Self { + inner: Mutex::new(RelaySlotTableInner { + slots: HashMap::new(), + last_swept: Instant::now(), + }), + capacity, + idle_timeout, + backpressure_refusals: AtomicU64::new(0), + } + } + + /// Try to acquire a slot for `(initiator_addr, target_peer_id)`. + /// + /// Returns `true` if the relay should proceed, `false` if the table + /// is at capacity. A re-acquisition for an already-held key always + /// succeeds and refreshes the timestamp without consuming additional + /// capacity — exactly what multi-round coordination needs. + pub(crate) fn try_acquire( + &self, + initiator_addr: SocketAddr, + target_peer_id: RelayTargetId, + now: Instant, + ) -> bool { + let mut inner = match self.inner.lock() { + Ok(g) => g, + Err(poisoned) => poisoned.into_inner(), + }; + Self::sweep_if_due(&mut inner, self.idle_timeout, now); + + let key = (initiator_addr, target_peer_id); + let already_active = inner.slots.contains_key(&key); + if !already_active && inner.slots.len() >= self.capacity { + // Drop the lock before logging so the warn! call cannot + // back-pressure the lock holder under contention. + let active = inner.slots.len(); + drop(inner); + let prior = self.backpressure_refusals.fetch_add(1, Ordering::Relaxed); + // Log once at first refusal, then periodically. + if prior == 0 || (prior + 1).is_multiple_of(REFUSAL_WARN_INTERVAL) { + warn!( + "hole-punch coordinator at capacity: refused relay #{} ({}/{} slots in use, initiator={})", + prior + 1, + active, + self.capacity, + initiator_addr, + ); + } else { + debug!( + "hole-punch relay refused (back-pressure): initiator={} target={}", + initiator_addr, + hex::encode(&target_peer_id[..8]) + ); + } + return false; + } + inner.slots.insert(key, now); + true + } + + /// Explicitly release every slot owned by `initiator_addr`. Called + /// from `BootstrapCoordinator::Drop` when the initiator's connection + /// closes, so the table doesn't have to wait out the idle timeout to + /// reclaim capacity for a known-dead session. + pub(crate) fn release_for_initiator(&self, initiator_addr: SocketAddr) { + let mut inner = match self.inner.lock() { + Ok(g) => g, + Err(poisoned) => poisoned.into_inner(), + }; + inner.slots.retain(|(addr, _), _| *addr != initiator_addr); + } + + /// Total number of relay frames refused since the table was created. + pub fn backpressure_refusals(&self) -> u64 { + self.backpressure_refusals.load(Ordering::Relaxed) + } + + /// Configured capacity (maximum simultaneous active slots). + pub fn capacity(&self) -> usize { + self.capacity + } + + /// Configured idle-release timeout for inactive slots. + pub fn idle_timeout(&self) -> Duration { + self.idle_timeout + } + + /// Snapshot of the current active slot count. Test/diagnostic only; + /// callers must treat the value as advisory because the table may + /// change between calls. + pub fn active_count(&self) -> usize { + match self.inner.lock() { + Ok(g) => g.slots.len(), + Err(poisoned) => poisoned.into_inner().slots.len(), + } + } + + /// Amortized sweep: prune slots whose last refresh is older than the + /// idle timeout, but only if the previous sweep was at least + /// [`SWEEP_AMORTIZATION_INTERVAL`] ago. This bounds the per-frame + /// retain cost on hot paths while still draining stale entries + /// promptly enough to free capacity ahead of the next storm. + fn sweep_if_due(inner: &mut RelaySlotTableInner, idle_timeout: Duration, now: Instant) { + if now.duration_since(inner.last_swept) < SWEEP_AMORTIZATION_INTERVAL { + return; + } + inner + .slots + .retain(|_, arrived_at| now.duration_since(*arrived_at) < idle_timeout); + inner.last_swept = now; + } +} + +impl std::fmt::Debug for RelaySlotTable { + fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fmt.debug_struct("RelaySlotTable") + .field("capacity", &self.capacity) + .field("idle_timeout", &self.idle_timeout) + .field( + "backpressure_refusals", + &self.backpressure_refusals.load(Ordering::Relaxed), + ) + .finish_non_exhaustive() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn target(byte: u8) -> RelayTargetId { + let mut id = [0u8; 32]; + id[0] = byte; + id + } + + fn addr(port: u16) -> SocketAddr { + SocketAddr::from(([127, 0, 0, 1], port)) + } + + #[test] + fn under_capacity_acquires() { + let table = RelaySlotTable::new(4, Duration::from_secs(5)); + let now = Instant::now(); + assert!(table.try_acquire(addr(5000), target(0x01), now)); + assert_eq!(table.active_count(), 1); + assert_eq!(table.backpressure_refusals(), 0); + } + + #[test] + fn at_capacity_refuses_silently() { + let table = RelaySlotTable::new(2, Duration::from_secs(5)); + let now = Instant::now(); + assert!(table.try_acquire(addr(5000), target(0x01), now)); + assert!(table.try_acquire(addr(5001), target(0x02), now)); + assert!(!table.try_acquire(addr(5002), target(0x03), now)); + assert_eq!(table.active_count(), 2); + assert_eq!(table.backpressure_refusals(), 1); + } + + #[test] + fn re_arm_refreshes_without_consuming_capacity() { + let table = RelaySlotTable::new(2, Duration::from_secs(5)); + let now = Instant::now(); + assert!(table.try_acquire(addr(5000), target(0x01), now)); + let later = now + Duration::from_millis(500); + assert!(table.try_acquire(addr(5000), target(0x01), later)); + assert_eq!( + table.active_count(), + 1, + "re-arm must not allocate a second slot" + ); + } + + #[test] + fn idle_sweep_reclaims_stale_slots() { + let timeout = Duration::from_secs(5); + let table = RelaySlotTable::new(2, timeout); + let now = Instant::now(); + assert!(table.try_acquire(addr(5000), target(0x01), now)); + assert!(table.try_acquire(addr(5001), target(0x02), now)); + // Past idle timeout AND past sweep amortization interval. + let much_later = now + timeout + Duration::from_secs(1); + assert!(table.try_acquire(addr(5002), target(0x03), much_later)); + assert_eq!( + table.active_count(), + 1, + "stale slots reclaimed by inline sweep before the cap check" + ); + assert_eq!(table.backpressure_refusals(), 0); + } + + #[test] + fn release_for_initiator_drops_owned_slots_only() { + let table = RelaySlotTable::new(8, Duration::from_secs(5)); + let now = Instant::now(); + // Two distinct sessions for initiator A. + assert!(table.try_acquire(addr(5000), target(0x01), now)); + assert!(table.try_acquire(addr(5000), target(0x02), now)); + // One session for a different initiator B. + assert!(table.try_acquire(addr(5999), target(0x03), now)); + assert_eq!(table.active_count(), 3); + + table.release_for_initiator(addr(5000)); + assert_eq!( + table.active_count(), + 1, + "release must drop slots for the named initiator only" + ); + // The B slot is still there. + let later = now + Duration::from_millis(50); + assert!(table.try_acquire(addr(5999), target(0x03), later)); + assert_eq!(table.active_count(), 1); + } + + #[test] + fn refusal_count_accumulates_across_distinct_targets() { + let table = RelaySlotTable::new(1, Duration::from_secs(5)); + let now = Instant::now(); + assert!(table.try_acquire(addr(5000), target(0x01), now)); + // Three distinct refusals at the same instant — sweep won't fire. + assert!(!table.try_acquire(addr(5001), target(0x02), now)); + assert!(!table.try_acquire(addr(5002), target(0x03), now)); + assert!(!table.try_acquire(addr(5003), target(0x04), now)); + assert_eq!(table.backpressure_refusals(), 3); + } +} diff --git a/src/unified_config.rs b/src/unified_config.rs index 1fae6ad2..1d79a695 100644 --- a/src/unified_config.rs +++ b/src/unified_config.rs @@ -145,6 +145,29 @@ pub struct NatConfig { /// Default: `false` pub allow_loopback: bool, + /// Cap on simultaneous in-flight hole-punch coordinator sessions + /// **across the entire node** (Tier 4 lite back-pressure). When the + /// shared `RelaySlotTable` is at capacity, additional `PUNCH_ME_NOW` + /// relay frames are silently refused so the initiator's per-attempt + /// timeout (Tier 2 rotation) can advance to its next preferred + /// coordinator. See + /// [`crate::nat_traversal_api::NatTraversalConfig::coordinator_max_active_relays`] + /// for the full rationale. + /// + /// Default: 32. + pub coordinator_max_active_relays: usize, + + /// Idle-release timeout for an in-flight coordinator relay session. + /// A slot lasts from the first `PUNCH_ME_NOW` until either the + /// owning connection closes (immediate release) or this many + /// seconds with no further rounds for the same + /// `(initiator_addr, target_peer_id)` pair. See + /// [`crate::nat_traversal_api::NatTraversalConfig::coordinator_relay_slot_idle_timeout`] + /// for the full rationale. + /// + /// Default: 5 seconds. + pub coordinator_relay_slot_idle_timeout: Duration, + /// Best-effort UPnP IGD port mapping configuration. When enabled /// (default), the endpoint asks the local router to forward its UDP /// port and surfaces the resulting public address as a high-priority @@ -163,6 +186,10 @@ impl Default for NatConfig { max_concurrent_attempts: 3, prefer_rfc_nat_traversal: true, allow_loopback: false, + coordinator_max_active_relays: + crate::nat_traversal_api::NatTraversalConfig::DEFAULT_COORDINATOR_MAX_ACTIVE_RELAYS, + coordinator_relay_slot_idle_timeout: + crate::nat_traversal_api::NatTraversalConfig::DEFAULT_COORDINATOR_RELAY_SLOT_IDLE_TIMEOUT, upnp: crate::upnp::UpnpConfig::default(), } } @@ -317,10 +344,8 @@ impl P2pConfig { transport_registry: Some(Arc::new(self.transport_registry.clone())), max_message_size: self.max_message_size, allow_loopback: self.nat.allow_loopback, - coordinator_max_active_relays: - crate::nat_traversal_api::NatTraversalConfig::DEFAULT_COORDINATOR_MAX_ACTIVE_RELAYS, - coordinator_relay_slot_timeout: - crate::nat_traversal_api::NatTraversalConfig::DEFAULT_COORDINATOR_RELAY_SLOT_TIMEOUT, + coordinator_max_active_relays: self.nat.coordinator_max_active_relays, + coordinator_relay_slot_idle_timeout: self.nat.coordinator_relay_slot_idle_timeout, upnp: self.nat.upnp.clone(), } } diff --git a/tests/relay_queue_tests.rs b/tests/relay_queue_tests.rs index c2d43d76..1e9848b5 100644 --- a/tests/relay_queue_tests.rs +++ b/tests/relay_queue_tests.rs @@ -56,7 +56,7 @@ mod nat_traversal_api_tests { max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: false, coordinator_max_active_relays: 32, - coordinator_relay_slot_timeout: Duration::from_secs(5), + coordinator_relay_slot_idle_timeout: Duration::from_secs(5), upnp: Default::default(), }; @@ -195,7 +195,7 @@ mod functional_tests { max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: false, coordinator_max_active_relays: 32, - coordinator_relay_slot_timeout: Duration::from_secs(5), + coordinator_relay_slot_idle_timeout: Duration::from_secs(5), upnp: Default::default(), }; @@ -224,7 +224,7 @@ mod functional_tests { max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: false, coordinator_max_active_relays: 32, - coordinator_relay_slot_timeout: Duration::from_secs(5), + coordinator_relay_slot_idle_timeout: Duration::from_secs(5), upnp: Default::default(), }; @@ -419,7 +419,7 @@ mod performance_tests { max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: false, coordinator_max_active_relays: 32, - coordinator_relay_slot_timeout: Duration::from_secs(5), + coordinator_relay_slot_idle_timeout: Duration::from_secs(5), upnp: Default::default(), }; @@ -492,7 +492,7 @@ mod relay_functionality_tests { max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: false, coordinator_max_active_relays: 32, - coordinator_relay_slot_timeout: Duration::from_secs(5), + coordinator_relay_slot_idle_timeout: Duration::from_secs(5), upnp: Default::default(), }; diff --git a/tests/security_regression_tests.rs b/tests/security_regression_tests.rs index a9536dd1..e3f6a427 100644 --- a/tests/security_regression_tests.rs +++ b/tests/security_regression_tests.rs @@ -38,7 +38,7 @@ fn test_peer_config() -> NatTraversalConfig { max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: true, coordinator_max_active_relays: 32, - coordinator_relay_slot_timeout: Duration::from_secs(5), + coordinator_relay_slot_idle_timeout: Duration::from_secs(5), upnp: Default::default(), } } @@ -65,7 +65,7 @@ fn test_server_config() -> NatTraversalConfig { max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: true, coordinator_max_active_relays: 32, - coordinator_relay_slot_timeout: Duration::from_secs(5), + coordinator_relay_slot_idle_timeout: Duration::from_secs(5), upnp: Default::default(), } } @@ -115,7 +115,7 @@ async fn test_error_handling_no_panic() { max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: true, coordinator_max_active_relays: 32, - coordinator_relay_slot_timeout: Duration::from_secs(5), + coordinator_relay_slot_idle_timeout: Duration::from_secs(5), upnp: Default::default(), }; @@ -146,7 +146,7 @@ async fn test_error_handling_no_panic() { max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: true, coordinator_max_active_relays: 32, - coordinator_relay_slot_timeout: Duration::from_secs(5), + coordinator_relay_slot_idle_timeout: Duration::from_secs(5), upnp: Default::default(), }; @@ -240,7 +240,7 @@ async fn test_malformed_config_handling() { max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: true, coordinator_max_active_relays: 32, - coordinator_relay_slot_timeout: Duration::from_secs(5), + coordinator_relay_slot_idle_timeout: Duration::from_secs(5), upnp: Default::default(), }; @@ -272,7 +272,7 @@ async fn test_malformed_config_handling() { max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: true, coordinator_max_active_relays: 32, - coordinator_relay_slot_timeout: Duration::from_secs(5), + coordinator_relay_slot_idle_timeout: Duration::from_secs(5), upnp: Default::default(), }; @@ -311,7 +311,7 @@ async fn test_input_sanitization() { max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: true, coordinator_max_active_relays: 32, - coordinator_relay_slot_timeout: Duration::from_secs(5), + coordinator_relay_slot_idle_timeout: Duration::from_secs(5), upnp: Default::default(), }; @@ -385,7 +385,7 @@ mod specific_regression_tests { max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: true, coordinator_max_active_relays: 32, - coordinator_relay_slot_timeout: Duration::from_secs(5), + coordinator_relay_slot_idle_timeout: Duration::from_secs(5), upnp: Default::default(), }; @@ -439,7 +439,7 @@ mod specific_regression_tests { max_message_size: saorsa_transport::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE, allow_loopback: true, coordinator_max_active_relays: 32, - coordinator_relay_slot_timeout: Duration::from_secs(5), + coordinator_relay_slot_idle_timeout: Duration::from_secs(5), upnp: Default::default(), }; From 6e85dd904d33071c82c8629b504ceeffd5af8655 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Tue, 7 Apr 2026 14:47:40 +0100 Subject: [PATCH 26/43] fix: raise coordinator PUNCH_ME_NOW rate limit from 50 to 300 per minute On larger networks with many NAT-restricted nodes, 50 per minute is still too low during the initial bootstrap burst when all nodes are hole-punching to each other simultaneously. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/connection/nat_traversal.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/connection/nat_traversal.rs b/src/connection/nat_traversal.rs index b388b833..4b664377 100644 --- a/src/connection/nat_traversal.rs +++ b/src/connection/nat_traversal.rs @@ -480,7 +480,7 @@ impl SecurityValidationState { max_candidates_per_window: 20, // Max 20 candidates per 60 seconds rate_window: Duration::from_secs(60), coordination_requests: VecDeque::new(), - max_coordination_per_window: 50, // Max 50 coordination requests per 60 seconds + max_coordination_per_window: 300, // Max 300 coordination requests per 60 seconds address_validation_cache: HashMap::new(), validation_cache_timeout: Duration::from_secs(300), // 5 minute cache allow_loopback, From a6c82d0e05c56f4aa308567faa1d68e851cc9bc4 Mon Sep 17 00:00:00 2001 From: grumbach Date: Wed, 8 Apr 2026 10:58:14 +0900 Subject: [PATCH 27/43] fix: raise send_ack_timeout from 500ms to 5s for cross-region connections 500ms was too tight for hole-punched and cross-region connections where the path includes 3+ RTTs (open_uni + data + peer ACK). Cross-continent RTTs of 200-300ms meant the identity announce frequently failed silently (fire-and-forget, never retried), leaving both peers stuck in a 15s identity exchange timeout. 5 seconds is generous enough for any real connection path while still detecting dead connections quickly (well under the 15s identity timeout). Fast profile raised proportionally (250ms -> 2.5s). --- src/config/nat_timeouts.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/config/nat_timeouts.rs b/src/config/nat_timeouts.rs index ceba2347..d8f3be84 100644 --- a/src/config/nat_timeouts.rs +++ b/src/config/nat_timeouts.rs @@ -132,10 +132,18 @@ impl Default for RelayTimeouts { } /// Default time to wait for the peer to acknowledge stream data after a send. -const DEFAULT_SEND_ACK_TIMEOUT: Duration = Duration::from_millis(500); +/// +/// 500 ms was too tight for cross-region and hole-punched connections where +/// the path includes 3 RTTs (open_uni + data + peer ACK). Cross-continent +/// RTTs of 200-300 ms meant the identity announce frequently failed silently, +/// leaving both peers stuck in a 15-second identity exchange timeout. +/// +/// 5 seconds is generous enough for any real connection path while still +/// detecting dead connections quickly (well under the 15s identity timeout). +const DEFAULT_SEND_ACK_TIMEOUT: Duration = Duration::from_secs(5); /// Fast-network send ACK timeout (halved from default, matching the fast profile pattern). -const FAST_SEND_ACK_TIMEOUT: Duration = Duration::from_millis(250); +const FAST_SEND_ACK_TIMEOUT: Duration = Duration::from_millis(2500); /// Master timeout configuration #[derive(Debug, Clone, Serialize, Deserialize)] From 55ffca78df200f61145234203b779c4bbd136964 Mon Sep 17 00:00:00 2001 From: grumbach Date: Wed, 8 Apr 2026 12:23:58 +0900 Subject: [PATCH 28/43] fix(nat): accept loop keeps newer connection on dedup instead of closing it The accept loop was closing the NEWER incoming connection when a live connection to the same address existed, keeping the older one. This is backwards: the newer connection is the one the remote peer just completed a handshake on and is actively using. Closing it kills their identity exchange, causing 15s timeouts. This was the root cause of the regression from PR #43 (coordinator rotation): when the rotation opened a new connection to a coordinator that already had an old connection, the acceptor closed the new one, leaving neither side with a working connection. Fix: replace old connection with new one (consistent with add_connection which also always overwrites with the newer connection). Also reset the emitted set so the replacement gets a reader task and PeerConnected event. Defense in depth: try_hole_punch now checks the DashMap in addition to connected_peers before opening a new coordinator connection, preventing unnecessary duplicates from being created in the first place. --- src/nat_traversal_api.rs | 36 +++++++++++++++++++++++------------- src/p2p_endpoint.rs | 7 +++++-- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/src/nat_traversal_api.rs b/src/nat_traversal_api.rs index 8b1a05be..6cd19dd1 100644 --- a/src/nat_traversal_api.rs +++ b/src/nat_traversal_api.rs @@ -3768,12 +3768,15 @@ impl NatTraversalEndpoint { let remote_address = connection.remote_address(); info!("Accepted connection from {} (unified path)", remote_address); - // Only insert if no existing LIVE connection to this address. - // Unconditionally overwriting would replace a working connection - // with a duplicate that may die shortly, leaving the DashMap - // pointing at a dead connection while the original's reader - // task still runs. - // Check both raw and normalized forms (IPv4-mapped IPv6). + // If an existing live connection to this address exists, + // replace it with the newer one. The remote peer just + // completed a fresh TLS handshake on this connection, so + // this is the one they are actively using. Closing the + // newer connection (the old behavior) kills the remote's + // active connection and breaks identity exchange. + // + // This is consistent with `add_connection` which also + // always overwrites with the newer connection. let normalized_remote = crate::shared::normalize_socket_addr(remote_address); let has_live = |addr: &std::net::SocketAddr| -> bool { connections2 @@ -3782,19 +3785,26 @@ impl NatTraversalEndpoint { }; if has_live(&remote_address) || has_live(&normalized_remote) { info!( - "accept_loop: {} already has a live connection, keeping existing", + "accept_loop: {} replacing existing connection with newer", remote_address ); - connection.close(0u32.into(), b"duplicate"); - return; // exit this handshake task + // Close the OLD connection, not the new one + if let Some(old) = connections2.get(&remote_address) { + old.value().close(0u32.into(), b"superseded"); + } else if let Some(old) = connections2.get(&normalized_remote) { + old.value().close(0u32.into(), b"superseded"); + } + // Allow re-emission so the new connection gets a + // reader task and PeerConnected event + emitted2.remove(&remote_address); + emitted2.remove(&normalized_remote); } connections2.insert(remote_address, connection.clone()); // Only forward to handshake_tx if this is the first time - // we've seen this address. Without this guard, a - // simultaneous-open (both sides connect at the same time) - // sends two entries to handshake_tx, causing duplicate - // reader tasks for the same connection address. + // (or first time since replacement) we've seen this address. + // Without this guard, simultaneous-open sends two entries + // to handshake_tx, causing duplicate reader tasks. if emitted2.insert(remote_address) { if let Some(ref server) = relay_server2 { let conn_clone = connection.clone(); diff --git a/src/p2p_endpoint.rs b/src/p2p_endpoint.rs index b15fcf88..a8f9596e 100644 --- a/src/p2p_endpoint.rs +++ b/src/p2p_endpoint.rs @@ -2012,8 +2012,11 @@ impl P2pEndpoint { target, coordinator ); - // First ensure we're connected to the coordinator - if !self.is_connected_to_addr(coordinator).await { + // First ensure we're connected to the coordinator. + // Check both connected_peers (app-level) and the DashMap (transport-level) + // to avoid creating unnecessary duplicate connections when the stale reaper + // has cleaned connected_peers but the DashMap still has a live connection. + if !self.is_connected_to_addr(coordinator).await && !self.inner.is_connected(&coordinator) { info!( "try_hole_punch: connecting to coordinator {} first", coordinator From 1478a1b8d3458b784497553aa26b8a917661c2ee Mon Sep 17 00:00:00 2001 From: grumbach Date: Tue, 7 Apr 2026 14:36:41 +0900 Subject: [PATCH 29/43] fix(reachability): replace has_public_ip with scope-aware peer-verified direct reachability Port of David Irvine reachability fix (a5b9db46 from ant-quic). Key changes: - New src/reachability.rs module with ReachabilityScope, TraversalMethod, socket_addr_scope() classifier, and DIRECT_REACHABILITY_TTL (15min) - NodeStatus: has_public_ip -> has_global_address (address property, not reachability proof), added direct_reachability_scope field - can_receive_direct now requires peer-verified evidence (active direct incoming connections or fresh scope-aware timestamps) - can_help_traversal() simplified to just can_receive_direct - TraversalMethod moved from node_event to reachability module - PeerConnection gains traversal_method and side fields - P2pEvent::PeerConnected gains traversal_method field - EndpointStats tracks active_direct_incoming_connections and last_direct_{loopback,local,global}_at timestamps - detect_nat_type simplified to soft debug hint only - Removed get_nat_stats() / nat_stats() placeholder methods - link_transport Capabilities gains direct_reachability_scope --- examples/simple_p2p.rs | 1 + src/bin/e2e-test-node.rs | 1 + src/lib.rs | 6 +- src/link_transport.rs | 4 + src/link_transport_impl.rs | 12 +- src/nat_traversal_api.rs | 30 ++--- src/node.rs | 127 +++++++++----------- src/node_event.rs | 31 +---- src/node_status.rs | 63 +++++++--- src/p2p_endpoint.rs | 168 ++++++++++++++++++++++---- src/reachability.rs | 176 ++++++++++++++++++++++++++++ tests/connection_lifecycle_tests.rs | 9 +- tests/constrained_integration.rs | 4 + tests/event_migration.rs | 11 ++ 14 files changed, 476 insertions(+), 167 deletions(-) create mode 100644 src/reachability.rs diff --git a/examples/simple_p2p.rs b/examples/simple_p2p.rs index 2a6f18ab..f10fe87f 100644 --- a/examples/simple_p2p.rs +++ b/examples/simple_p2p.rs @@ -39,6 +39,7 @@ async fn main() -> anyhow::Result<()> { addr, public_key, side, + .. } => { println!( "Connected to peer at {} (side: {:?}, has key: {})", diff --git a/src/bin/e2e-test-node.rs b/src/bin/e2e-test-node.rs index 29d3b665..72ca1379 100644 --- a/src/bin/e2e-test-node.rs +++ b/src/bin/e2e-test-node.rs @@ -713,6 +713,7 @@ async fn handle_event( addr, public_key: _, side, + traversal_method: _, } => { let direction = if side.is_client() { "outbound" diff --git a/src/lib.rs b/src/lib.rs index 1e2ed47d..b62ba757 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -168,6 +168,9 @@ pub mod node_status; /// Unified events for P2P nodes pub mod node_event; +/// Reachability scope and traversal metadata shared across APIs +pub mod reachability; + // Core implementation modules /// Configuration structures and validation pub mod config; @@ -333,6 +336,7 @@ pub use nat_traversal_api::{ BootstrapNode, CandidateAddress, NatTraversalConfig, NatTraversalEndpoint, NatTraversalError, NatTraversalEvent, NatTraversalStatistics, }; +pub use reachability::{ReachabilityScope, TraversalMethod}; // ============================================================================ // SIMPLE API EXPORTS - Zero Configuration P2P (RECOMMENDED) @@ -348,7 +352,7 @@ pub use node_config::{NodeConfig, NodeConfigBuilder}; pub use node_status::{NatType, NodeStatus}; /// Unified events for P2P nodes -pub use node_event::{DisconnectReason as NodeDisconnectReason, NodeEvent, TraversalMethod}; +pub use node_event::{DisconnectReason as NodeDisconnectReason, NodeEvent}; // ============================================================================ // P2P API EXPORTS (for advanced use) diff --git a/src/link_transport.rs b/src/link_transport.rs index 7c1670ce..43ce24c0 100644 --- a/src/link_transport.rs +++ b/src/link_transport.rs @@ -624,6 +624,9 @@ pub struct Capabilities { /// Observed external addresses for this peer. pub observed_addrs: Vec, + /// Broadest direct reachability scope verified for this connected peer. + pub direct_reachability_scope: Option, + /// Protocols this peer advertises support for. pub protocols: Vec, @@ -661,6 +664,7 @@ impl Default for Capabilities { supports_relay: false, supports_coordination: false, observed_addrs: Vec::new(), + direct_reachability_scope: None, protocols: Vec::new(), last_seen: SystemTime::UNIX_EPOCH, rtt_ms_p50: 0, diff --git a/src/link_transport_impl.rs b/src/link_transport_impl.rs index 98482e69..37aae558 100644 --- a/src/link_transport_impl.rs +++ b/src/link_transport_impl.rs @@ -490,13 +490,20 @@ impl P2pLinkTransport { addr, public_key, side: _, + traversal_method, } => { // Extract SocketAddr (currently UDP-only) let socket_addr = addr.as_socket_addr().unwrap_or_else(|| { // Fallback for non-UDP transports - use unspecified address SocketAddr::from(([0, 0, 0, 0], 0)) }); - let caps = Capabilities::new_connected(socket_addr); + let mut caps = Capabilities::new_connected(socket_addr); + if traversal_method.is_direct() { + caps.supports_relay = true; + caps.supports_coordination = true; + caps.direct_reachability_scope = + crate::reachability::socket_addr_scope(socket_addr); + } // Update capabilities cache keyed by address if let Ok(mut state) = state.write() { state.capabilities.insert(socket_addr, caps.clone()); @@ -537,6 +544,9 @@ impl P2pLinkTransport { if let Ok(mut state) = state.write() { if let Some(caps) = state.capabilities.get_mut(&socket_addr) { caps.is_connected = false; + caps.supports_relay = false; + caps.supports_coordination = false; + caps.direct_reachability_scope = None; } } Some(LinkEvent::PeerDisconnected { diff --git a/src/nat_traversal_api.rs b/src/nat_traversal_api.rs index 6cd19dd1..c7aa92be 100644 --- a/src/nat_traversal_api.rs +++ b/src/nat_traversal_api.rs @@ -15,6 +15,7 @@ use std::{fmt, net::SocketAddr, sync::Arc, time::Duration}; use crate::constrained::{ConstrainedEngine, EngineConfig, EngineEvent}; +use crate::reachability::TraversalMethod; use crate::transport::TransportRegistry; use crate::SHUTDOWN_DRAIN_TIMEOUT; @@ -1042,6 +1043,8 @@ pub enum NatTraversalEvent { remote_address: SocketAddr, /// Who initiated the connection (Client = we connected, Server = they connected) side: Side, + /// Whether the connection was direct, hole-punched, or relayed. + traversal_method: TraversalMethod, /// ML-DSA-65 public key extracted from the TLS identity, if available public_key: Option>, }, @@ -2976,6 +2979,7 @@ impl NatTraversalEndpoint { event_tx.send(NatTraversalEvent::ConnectionEstablished { remote_address, side: Side::Server, + traversal_method: TraversalMethod::Direct, public_key, }); incoming_notify.notify_one(); @@ -3346,6 +3350,7 @@ impl NatTraversalEndpoint { let _ = event_tx.send(NatTraversalEvent::ConnectionEstablished { remote_address: remote_addr, side: Side::Client, + traversal_method: TraversalMethod::Direct, public_key, }); self.incoming_notify.notify_one(); @@ -3971,11 +3976,13 @@ impl NatTraversalEndpoint { /// * `addr` - The remote address of the connection /// * `connection` - The established QUIC connection /// * `side` - Who initiated the connection (Client = we connected, Server = they connected) + /// * `traversal_method` - Whether the path is direct, hole-punched, or relayed pub fn spawn_connection_handler( &self, addr: SocketAddr, connection: InnerConnection, side: Side, + traversal_method: TraversalMethod, ) -> Result<(), NatTraversalError> { let event_tx = self.event_tx.as_ref().cloned().ok_or_else(|| { NatTraversalError::ConfigError("NAT traversal event channel not configured".to_string()) @@ -3992,6 +3999,7 @@ impl NatTraversalEndpoint { let _ = event_tx.send(NatTraversalEvent::ConnectionEstablished { remote_address, side, + traversal_method, public_key, }); self.incoming_notify.notify_one(); @@ -4927,6 +4935,7 @@ impl NatTraversalEndpoint { event_tx.send(NatTraversalEvent::ConnectionEstablished { remote_address: remote, side: Side::Client, + traversal_method: TraversalMethod::HolePunch, public_key, }); incoming_notify.notify_one(); @@ -6641,6 +6650,7 @@ impl NatTraversalEndpoint { callback(NatTraversalEvent::ConnectionEstablished { remote_address: candidate_address, side: Side::Client, + traversal_method: TraversalMethod::HolePunch, public_key, }); } @@ -6728,26 +6738,6 @@ impl NatTraversalEndpoint { Err(NatTraversalError::PeerNotConnected) } } - - /// Get NAT traversal statistics - #[allow(clippy::panic)] - pub fn get_nat_stats( - &self, - ) -> Result> { - // Return default statistics for now - // In a real implementation, this would collect actual stats from the endpoint - Ok(NatTraversalStatistics { - active_sessions: self.active_sessions.len(), - // parking_lot::RwLock doesn't poison - always succeeds - total_bootstrap_nodes: self.bootstrap_nodes.read().len(), - successful_coordinations: 7, - average_coordination_time: self.timeout_config.nat_traversal.retry_interval, - total_attempts: 10, - successful_connections: 7, - direct_connections: 5, - relayed_connections: 2, - }) - } } impl fmt::Debug for NatTraversalEndpoint { diff --git a/src/node.rs b/src/node.rs index 2594939c..f8be60aa 100644 --- a/src/node.rs +++ b/src/node.rs @@ -56,6 +56,7 @@ use crate::node_config::NodeConfig; use crate::node_event::NodeEvent; use crate::node_status::{NatType, NodeStatus}; use crate::p2p_endpoint::{EndpointError, P2pEndpoint, P2pEvent, PeerConnection}; +use crate::reachability::{DIRECT_REACHABILITY_TTL, socket_addr_scope}; use crate::unified_config::P2pConfig; use crate::unified_config::load_or_generate_endpoint_keypair; @@ -328,10 +329,12 @@ impl Node { addr, public_key, side: _, + traversal_method, } => Some(NodeEvent::PeerConnected { addr, public_key, - direct: true, // P2pEvent doesn't distinguish, assume direct + method: traversal_method, + direct: traversal_method.is_direct(), }), P2pEvent::PeerDisconnected { addr, reason } => Some(NodeEvent::PeerDisconnected { addr: addr.to_synthetic_socket_addr(), @@ -502,22 +505,15 @@ impl Node { /// ``` pub async fn status(&self) -> NodeStatus { let stats = self.inner.stats().await; - let nat_stats = self.inner.nat_stats().ok(); let connected_peers = self.inner.connected_peers().await; - // Determine NAT type from stats - let nat_type = self.detect_nat_type(&stats, nat_stats.as_ref()); + // Determine NAT type from observed connection outcomes only. + let nat_type = self.detect_nat_type(&stats); - // Check if we have public IP + // Address knowledge and reachability are separate concepts. + // A global address is not proof of direct reachability. let local_addr = self.local_addr(); let external_addr = self.external_addr(); - let has_public_ip = match (local_addr, external_addr) { - (Some(local), Some(external)) => { - // Public if external matches local (ignoring port differences) - local.ip() == external.ip() - } - _ => false, - }; // Collect external addresses let mut external_addrs = Vec::new(); @@ -532,34 +528,48 @@ impl Node { 0.0 }; - // Determine if we can help with traversal - let can_receive_direct = - has_public_ip || nat_type == NatType::FullCone || nat_type == NatType::None; - - // Check relay status from NAT stats - // Currently, relay status is indicated by having relayed_connections > 0 - // and active sessions that may be acting as relays - let (is_relaying, relay_sessions, relay_bytes_forwarded) = if let Some(ref nat) = nat_stats - { - // If we have any active sessions and are accepting connections, - // we're potentially relaying - let relaying = nat.relayed_connections > 0 && can_receive_direct; + let has_global_address = external_addrs + .iter() + .copied() + .chain(local_addr) + .any(|addr| { + socket_addr_scope(addr) + .is_some_and(|scope| scope == crate::ReachabilityScope::Global) + }); + + // A node is directly reachable only after fresh, peer-verified direct + // inbound evidence. Scope is freshness-aware too, so an old global + // observation cannot keep inflating current reachability. + let fresh_scope = [ ( - relaying, - if relaying { nat.active_sessions } else { 0 }, - 0u64, // Not tracked yet - future enhancement - ) - } else { - (false, 0, 0) - }; + crate::ReachabilityScope::Global, + stats.last_direct_global_at, + ), + ( + crate::ReachabilityScope::LocalNetwork, + stats.last_direct_local_at, + ), + ( + crate::ReachabilityScope::Loopback, + stats.last_direct_loopback_at, + ), + ] + .into_iter() + .find_map(|(scope, seen)| { + seen.filter(|instant| instant.elapsed() <= DIRECT_REACHABILITY_TTL) + .map(|_| scope) + }); + let can_receive_direct = + stats.active_direct_incoming_connections > 0 || fresh_scope.is_some(); + let direct_reachability_scope = fresh_scope; - // Check coordination status - // Any node with active sessions is acting as a coordinator - let (is_coordinating, coordination_sessions) = if let Some(ref nat) = nat_stats { - (nat.active_sessions > 0, nat.active_sessions) - } else { - (false, 0) - }; + // Relay/coordinator activity must be backed by real runtime metrics. + // The NAT stats path is still placeholder-ish, so stay conservative here. + let is_relaying = false; + let relay_sessions = 0; + let relay_bytes_forwarded = 0u64; + let is_coordinating = false; + let coordination_sessions = 0; // Calculate average RTT from connected peers let mut total_rtt = Duration::ZERO; @@ -589,7 +599,8 @@ impl Node { external_addrs, nat_type, can_receive_direct, - has_public_ip, + direct_reachability_scope, + has_global_address, connected_peers: connected_peers.len(), active_connections: stats.active_connections, pending_connections: 0, // Not tracked yet @@ -655,51 +666,21 @@ impl Node { // === Private Helpers === /// Detect NAT type from statistics - fn detect_nat_type( - &self, - stats: &crate::p2p_endpoint::EndpointStats, - nat_stats: Option<&crate::nat_traversal_api::NatTraversalStatistics>, - ) -> NatType { - // If we have lots of direct connections and no relayed, likely no/easy NAT + fn detect_nat_type(&self, stats: &crate::p2p_endpoint::EndpointStats) -> NatType { + // This remains a soft debug hint only. Do not treat it as direct + // reachability evidence. if stats.direct_connections > 0 && stats.relayed_connections == 0 { - if let Some(nat) = nat_stats { - // Calculate direct connection rate - let total = nat.direct_connections + nat.relayed_connections; - if total > 0 { - let direct_rate = nat.direct_connections as f64 / total as f64; - if direct_rate > 0.9 { - return NatType::FullCone; - } - } - } - return NatType::FullCone; // Assume easy NAT if all direct + return NatType::FullCone; } - // If we have mixed connections, harder NAT if stats.direct_connections > 0 && stats.relayed_connections > 0 { - if let Some(nat) = nat_stats { - // Calculate success rate from total attempts vs successful connections - let success_rate = if nat.total_attempts > 0 { - nat.successful_connections as f64 / nat.total_attempts as f64 - } else { - 0.0 - }; - - if success_rate > 0.7 { - return NatType::PortRestricted; - } else if success_rate > 0.3 { - return NatType::AddressRestricted; - } - } return NatType::PortRestricted; } - // If mostly relayed, likely symmetric NAT if stats.relayed_connections > stats.direct_connections { return NatType::Symmetric; } - // Not enough data yet NatType::Unknown } } diff --git a/src/node_event.rs b/src/node_event.rs index ebc24141..b0fab658 100644 --- a/src/node_event.rs +++ b/src/node_event.rs @@ -37,6 +37,7 @@ use std::net::SocketAddr; use crate::node_status::NatType; +pub use crate::reachability::TraversalMethod; use crate::transport::TransportAddr; /// Reason for peer disconnection @@ -85,7 +86,9 @@ pub enum NodeEvent { addr: TransportAddr, /// The peer's public key bytes (ML-DSA-65 SPKI), if available from TLS handshake public_key: Option>, - /// Whether this is a direct connection (vs relayed) + /// How the connection was established. + method: TraversalMethod, + /// Whether this is a direct connection (vs relayed or assisted) direct: bool, }, @@ -186,30 +189,6 @@ pub enum NodeEvent { }, } -/// Method used for NAT traversal -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum TraversalMethod { - /// Direct connection (no NAT or easy NAT) - Direct, - /// Hole punching succeeded - HolePunch, - /// Connection via relay - Relay, - /// Port prediction for symmetric NAT - PortPrediction, -} - -impl std::fmt::Display for TraversalMethod { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Direct => write!(f, "direct"), - Self::HolePunch => write!(f, "hole punch"), - Self::Relay => write!(f, "relay"), - Self::PortPrediction => write!(f, "port prediction"), - } - } -} - impl NodeEvent { /// Check if this is a connection event pub fn is_connection_event(&self) -> bool { @@ -305,6 +284,7 @@ mod tests { let event = NodeEvent::PeerConnected { addr: TransportAddr::Udp(test_addr()), public_key: None, + method: TraversalMethod::Direct, direct: true, }; @@ -417,6 +397,7 @@ mod tests { let event = NodeEvent::PeerConnected { addr: TransportAddr::Udp(test_addr()), public_key: None, + method: TraversalMethod::Direct, direct: true, }; diff --git a/src/node_status.rs b/src/node_status.rs index 23bc709f..42edc711 100644 --- a/src/node_status.rs +++ b/src/node_status.rs @@ -28,15 +28,18 @@ use std::net::SocketAddr; use std::time::Duration; +pub use crate::reachability::ReachabilityScope; + /// Detected NAT type for the node /// /// NAT type affects connectivity - some types are easier to traverse than others. /// The node automatically detects its NAT type and adjusts traversal strategies. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] pub enum NatType { - /// No NAT detected - direct public connectivity + /// No NAT detected. /// - /// The node has a public IP address and can accept connections directly. + /// This indicates the observed path did not require NAT traversal. It does + /// not, by itself, prove current direct reachability to other peers. None, /// Full cone NAT - easiest to traverse @@ -73,7 +76,7 @@ pub enum NatType { impl std::fmt::Display for NatType { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - Self::None => write!(f, "None (Public IP)"), + Self::None => write!(f, "None (No NAT detected)"), Self::FullCone => write!(f, "Full Cone"), Self::AddressRestricted => write!(f, "Address Restricted"), Self::PortRestricted => write!(f, "Port Restricted"), @@ -91,7 +94,7 @@ impl std::fmt::Display for NatType { /// # Status Categories /// /// - **Identity**: peer_id, local_addr, external_addrs -/// - **NAT Status**: nat_type, can_receive_direct, has_public_ip +/// - **NAT Status**: nat_type, can_receive_direct, direct_reachability_scope, has_global_address /// - **Connections**: connected_peers, active_connections, pending_connections /// - **NAT Traversal**: direct_connections, relayed_connections, hole_punch_success_rate /// - **Relay**: is_relaying, relay_sessions, relay_bytes_forwarded @@ -118,13 +121,17 @@ pub struct NodeStatus { /// Whether this node can receive direct connections /// - /// `true` if the node has a public IP or is behind a traversable NAT. + /// `true` only after this node has peer-verified evidence that another + /// node reached it directly without coordinator or relay assistance. pub can_receive_direct: bool, - /// Whether this node has a public IP + /// Broadest scope in which direct inbound reachability has been verified. + pub direct_reachability_scope: Option, + + /// Whether this node has a globally routable address candidate. /// - /// `true` if local_addr matches an external_addr (no NAT). - pub has_public_ip: bool, + /// This is an address property, not proof of reachability. + pub has_global_address: bool, // --- Connections --- /// Number of connected peers @@ -151,8 +158,8 @@ pub struct NodeStatus { // --- Relay Status (NEW - key visibility) --- /// Whether this node is currently acting as a relay for others /// - /// `true` if this node has public connectivity and is forwarding - /// traffic for peers behind restrictive NATs. + /// `true` if this node has fresh peer-verified direct reachability and is + /// forwarding traffic for peers behind restrictive NATs. pub is_relaying: bool, /// Number of active relay sessions @@ -165,7 +172,8 @@ pub struct NodeStatus { /// Whether this node is coordinating NAT traversal /// /// `true` if this node is helping peers coordinate hole punching. - /// All nodes with public connectivity act as coordinators. + /// Fresh peer-verified direct reachability is the signal other peers should + /// use when deciding whether this node is a viable coordinator. pub is_coordinating: bool, /// Number of active coordination sessions @@ -189,7 +197,8 @@ impl Default for NodeStatus { external_addrs: Vec::new(), nat_type: NatType::Unknown, can_receive_direct: false, - has_public_ip: false, + direct_reachability_scope: None, + has_global_address: false, connected_peers: 0, active_connections: 0, pending_connections: 0, @@ -215,10 +224,10 @@ impl NodeStatus { /// Check if node can help with NAT traversal /// - /// Returns true if the node has public connectivity and can + /// Returns true if the node has peer-verified direct reachability and can /// act as coordinator/relay for other peers. pub fn can_help_traversal(&self) -> bool { - self.has_public_ip || self.can_receive_direct + self.can_receive_direct } /// Get the total number of connections (direct + relayed) @@ -245,7 +254,7 @@ mod tests { #[test] fn test_nat_type_display() { - assert_eq!(format!("{}", NatType::None), "None (Public IP)"); + assert_eq!(format!("{}", NatType::None), "None (No NAT detected)"); assert_eq!(format!("{}", NatType::FullCone), "Full Cone"); assert_eq!( format!("{}", NatType::AddressRestricted), @@ -266,7 +275,8 @@ mod tests { let status = NodeStatus::default(); assert_eq!(status.nat_type, NatType::Unknown); assert!(!status.can_receive_direct); - assert!(!status.has_public_ip); + assert_eq!(status.direct_reachability_scope, None); + assert!(!status.has_global_address); assert_eq!(status.connected_peers, 0); assert!(!status.is_relaying); assert!(!status.is_coordinating); @@ -286,14 +296,29 @@ mod tests { let mut status = NodeStatus::default(); assert!(!status.can_help_traversal()); - status.has_public_ip = true; - assert!(status.can_help_traversal()); + status.has_global_address = true; + assert!( + !status.can_help_traversal(), + "Global address alone must not imply direct reachability" + ); - status.has_public_ip = false; status.can_receive_direct = true; + status.direct_reachability_scope = Some(ReachabilityScope::Global); assert!(status.can_help_traversal()); } + #[test] + fn test_direct_reachability_scope_tracks_observer_scope() { + let mut status = NodeStatus::default(); + status.can_receive_direct = true; + status.direct_reachability_scope = Some(ReachabilityScope::LocalNetwork); + + assert_eq!( + status.direct_reachability_scope, + Some(ReachabilityScope::LocalNetwork) + ); + } + #[test] fn test_total_connections() { let mut status = NodeStatus::default(); diff --git a/src/p2p_endpoint.rs b/src/p2p_endpoint.rs index a8f9596e..fa571652 100644 --- a/src/p2p_endpoint.rs +++ b/src/p2p_endpoint.rs @@ -71,9 +71,8 @@ use crate::constrained::EngineEvent; use crate::crypto::raw_public_keys::key_utils::generate_ml_dsa_keypair; use crate::happy_eyeballs::{self, HappyEyeballsConfig}; pub use crate::nat_traversal_api::TraversalPhase; -use crate::nat_traversal_api::{ - NatTraversalEndpoint, NatTraversalError, NatTraversalEvent, NatTraversalStatistics, -}; +use crate::nat_traversal_api::{NatTraversalEndpoint, NatTraversalError, NatTraversalEvent}; +use crate::reachability::{ReachabilityScope, TraversalMethod, socket_addr_scope}; use crate::transport::{ProtocolEngine, TransportAddr, TransportRegistry}; use crate::unified_config::P2pConfig; use rustls; @@ -251,6 +250,12 @@ pub struct PeerConnection { /// Remote address (supports all transport types) pub remote_addr: TransportAddr, + /// How this connection was established. + pub traversal_method: TraversalMethod, + + /// Who initiated the connection. + pub side: Side, + /// Whether peer is authenticated pub authenticated: bool, @@ -298,9 +303,21 @@ pub struct EndpointStats { /// Successful NAT traversals pub nat_traversal_successes: u64, - /// Direct connections (no NAT traversal needed) + /// Direct connections (no coordinator or relay needed) pub direct_connections: u64, + /// Currently active direct inbound connections from peers. + pub active_direct_incoming_connections: u64, + + /// Most recent loopback-scoped direct inbound observation. + pub last_direct_loopback_at: Option, + + /// Most recent LAN-scoped direct inbound observation. + pub last_direct_local_at: Option, + + /// Most recent globally scoped direct inbound observation. + pub last_direct_global_at: Option, + /// Relayed connections pub relayed_connections: u64, @@ -326,6 +343,10 @@ impl Default for EndpointStats { nat_traversal_attempts: 0, nat_traversal_successes: 0, direct_connections: 0, + active_direct_incoming_connections: 0, + last_direct_loopback_at: None, + last_direct_local_at: None, + last_direct_global_at: None, relayed_connections: 0, total_bootstrap_nodes: 0, connected_bootstrap_nodes: 0, @@ -349,7 +370,7 @@ impl Default for EndpointStats { /// /// while let Ok(event) = events.recv().await { /// match event { -/// P2pEvent::PeerConnected { peer_id, addr, side } => { +/// P2pEvent::PeerConnected { addr, public_key, side, traversal_method } => { /// // Handle different transport types /// match addr { /// TransportAddr::Quic(socket_addr) => { @@ -403,6 +424,8 @@ pub enum P2pEvent { public_key: Option>, /// Who initiated the connection (Client = we connected, Server = they connected) side: Side, + /// Whether the connection was direct, hole-punched, or relayed. + traversal_method: TraversalMethod, }, /// A peer has disconnected. @@ -586,6 +609,10 @@ async fn do_cleanup_connection( { let mut s = stats.write().await; s.active_connections = s.active_connections.saturating_sub(1); + if peer_conn.traversal_method.is_direct() && peer_conn.side.is_server() { + s.active_direct_incoming_connections = + s.active_direct_incoming_connections.saturating_sub(1); + } } let _ = event_tx.send(P2pEvent::PeerDisconnected { @@ -650,17 +677,45 @@ impl P2pEndpoint { NatTraversalEvent::ConnectionEstablished { remote_address, side, + traversal_method, public_key, } => { stats_guard.nat_traversal_successes += 1; stats_guard.active_connections += 1; stats_guard.successful_connections += 1; + match traversal_method { + TraversalMethod::Direct => { + stats_guard.direct_connections += 1; + if side.is_server() { + stats_guard.active_direct_incoming_connections += 1; + let now = Instant::now(); + match socket_addr_scope(*remote_address) { + Some(ReachabilityScope::Loopback) => { + stats_guard.last_direct_loopback_at = Some(now); + } + Some(ReachabilityScope::LocalNetwork) => { + stats_guard.last_direct_local_at = Some(now); + } + Some(ReachabilityScope::Global) => { + stats_guard.last_direct_global_at = Some(now); + } + None => {} + } + } + } + TraversalMethod::Relay => { + stats_guard.relayed_connections += 1; + } + TraversalMethod::HolePunch | TraversalMethod::PortPrediction => {} + } + // Broadcast event with connection direction let _ = event_tx.send(P2pEvent::PeerConnected { addr: TransportAddr::Quic(*remote_address), public_key: public_key.clone(), side: *side, + traversal_method: *traversal_method, }); } NatTraversalEvent::TraversalFailed { remote_address, .. } => { @@ -960,7 +1015,7 @@ impl P2pEndpoint { // Spawn handler (we initiated the connection = Client side) self.inner - .spawn_connection_handler(addr, connection, Side::Client) + .spawn_connection_handler(addr, connection, Side::Client, TraversalMethod::Direct) .map_err(EndpointError::NatTraversal)?; // Create peer connection record @@ -968,6 +1023,8 @@ impl P2pEndpoint { let peer_conn = PeerConnection { public_key: remote_public_key.clone(), remote_addr: TransportAddr::Quic(addr), + traversal_method: TraversalMethod::Direct, + side: Side::Client, authenticated: true, // TLS handles authentication connected_at: Instant::now(), last_activity: Instant::now(), @@ -999,6 +1056,7 @@ impl P2pEndpoint { addr: TransportAddr::Quic(addr), public_key: remote_public_key, side: Side::Client, + traversal_method: TraversalMethod::Direct, }); Ok(peer_conn) @@ -1074,6 +1132,8 @@ impl P2pEndpoint { let peer_conn = PeerConnection { public_key: None, // Constrained connections don't have TLS auth yet remote_addr: addr.clone(), + traversal_method: TraversalMethod::Direct, + side: Side::Client, authenticated: false, connected_at: Instant::now(), last_activity: Instant::now(), @@ -1096,6 +1156,7 @@ impl P2pEndpoint { addr: addr.clone(), public_key: None, side: Side::Client, + traversal_method: TraversalMethod::Direct, }); Ok(peer_conn) @@ -1570,6 +1631,8 @@ impl P2pEndpoint { let peer_conn = PeerConnection { public_key: None, remote_addr: TransportAddr::Quic(target_addr), + traversal_method: TraversalMethod::HolePunch, + side: Side::Client, authenticated: true, connected_at: Instant::now(), last_activity: Instant::now(), @@ -1589,6 +1652,7 @@ impl P2pEndpoint { addr: TransportAddr::Quic(target_addr), public_key: peer_conn.public_key.clone(), side: Side::Client, + traversal_method: TraversalMethod::HolePunch, }); return Ok(( @@ -1964,12 +2028,14 @@ impl P2pEndpoint { // Spawn connection handler (Client side - we initiated) self.inner - .spawn_connection_handler(addr, connection, Side::Client) + .spawn_connection_handler(addr, connection, Side::Client, TraversalMethod::Direct) .map_err(EndpointError::NatTraversal)?; let peer_conn = PeerConnection { public_key: remote_public_key.clone(), remote_addr: TransportAddr::Quic(addr), + traversal_method: TraversalMethod::Direct, + side: Side::Client, authenticated: true, connected_at: Instant::now(), last_activity: Instant::now(), @@ -1996,6 +2062,7 @@ impl P2pEndpoint { addr: TransportAddr::Quic(addr), public_key: remote_public_key, side: Side::Client, + traversal_method: TraversalMethod::Direct, }); Ok(peer_conn) @@ -2107,6 +2174,8 @@ impl P2pEndpoint { let peer_conn = PeerConnection { public_key: None, remote_addr: TransportAddr::Quic(actual_addr), + traversal_method: TraversalMethod::HolePunch, + side: Side::Client, authenticated: true, connected_at: Instant::now(), last_activity: Instant::now(), @@ -2231,12 +2300,14 @@ impl P2pEndpoint { .map_err(EndpointError::NatTraversal)?; self.inner - .spawn_connection_handler(target, connection, Side::Client) + .spawn_connection_handler(target, connection, Side::Client, TraversalMethod::Relay) .map_err(EndpointError::NatTraversal)?; let peer_conn = PeerConnection { public_key: remote_public_key, remote_addr: TransportAddr::Quic(target), + traversal_method: TraversalMethod::Relay, + side: Side::Client, authenticated: true, connected_at: Instant::now(), last_activity: Instant::now(), @@ -2262,6 +2333,28 @@ impl P2pEndpoint { } /// Check if we're connected to a specific address + async fn record_direct_incoming_stats(&self, remote_addr: SocketAddr) { + let mut stats = self.stats.write().await; + stats.active_connections += 1; + stats.successful_connections += 1; + stats.direct_connections += 1; + stats.active_direct_incoming_connections += 1; + let now = Instant::now(); + + match socket_addr_scope(remote_addr) { + Some(ReachabilityScope::Loopback) => { + stats.last_direct_loopback_at = Some(now); + } + Some(ReachabilityScope::LocalNetwork) => { + stats.last_direct_local_at = Some(now); + } + Some(ReachabilityScope::Global) => { + stats.last_direct_global_at = Some(now); + } + None => {} + } + } + async fn is_connected_to_addr(&self, addr: SocketAddr) -> bool { let transport_addr = TransportAddr::Quic(addr); let peers = self.connected_peers.read().await; @@ -2289,10 +2382,12 @@ impl P2pEndpoint { let remote_public_key = extract_public_key_bytes_from_connection(&connection); // They initiated the connection to us = Server side - if let Err(e) = - self.inner - .spawn_connection_handler(remote_addr, connection, Side::Server) - { + if let Err(e) = self.inner.spawn_connection_handler( + remote_addr, + connection, + Side::Server, + TraversalMethod::Direct, + ) { error!("Failed to spawn connection handler: {}", e); return None; } @@ -2301,6 +2396,8 @@ impl P2pEndpoint { let peer_conn = PeerConnection { public_key: remote_public_key.clone(), remote_addr: TransportAddr::Quic(remote_addr), + traversal_method: TraversalMethod::Direct, + side: Side::Server, authenticated: true, // TLS handles authentication connected_at: Instant::now(), last_activity: Instant::now(), @@ -2332,17 +2429,14 @@ impl P2pEndpoint { .await .insert(remote_addr, peer_conn.clone()); - { - let mut stats = self.stats.write().await; - stats.active_connections += 1; - stats.successful_connections += 1; - } + self.record_direct_incoming_stats(remote_addr).await; // They initiated the connection to us = Server side let _ = self.event_tx.send(P2pEvent::PeerConnected { addr: TransportAddr::Quic(remote_addr), public_key: remote_public_key, side: Side::Server, + traversal_method: TraversalMethod::Direct, }); Some(peer_conn) @@ -2462,6 +2556,8 @@ impl P2pEndpoint { let peer_conn = PeerConnection { public_key: None, remote_addr: TransportAddr::Quic(*addr), + traversal_method: TraversalMethod::HolePunch, + side: Side::Server, authenticated: true, connected_at: Instant::now(), last_activity: Instant::now(), @@ -2471,6 +2567,7 @@ impl P2pEndpoint { addr: TransportAddr::Quic(*addr), public_key: None, side: Side::Server, + traversal_method: TraversalMethod::HolePunch, }); (TransportAddr::Quic(*addr), Some(conn)) } else { @@ -2683,13 +2780,6 @@ impl P2pEndpoint { }) } - /// Get NAT traversal statistics - pub fn nat_stats(&self) -> Result { - self.inner - .get_nat_stats() - .map_err(|e| EndpointError::Connection(e.to_string())) - } - // === Known Peers === /// Connect to configured known peers @@ -3015,6 +3105,8 @@ impl P2pEndpoint { PeerConnection { public_key: None, remote_addr: addr.clone(), + traversal_method: TraversalMethod::Direct, + side, authenticated: false, connected_at: Instant::now(), last_activity: Instant::now(), @@ -3024,6 +3116,7 @@ impl P2pEndpoint { addr: addr.clone(), public_key: None, side, + traversal_method: TraversalMethod::Direct, }); } @@ -3390,6 +3483,8 @@ impl P2pEndpoint { let peer_conn = PeerConnection { public_key: None, remote_addr: TransportAddr::Quic(addr), + traversal_method: TraversalMethod::HolePunch, + side: Side::Server, authenticated: true, connected_at: Instant::now(), last_activity: Instant::now(), @@ -3460,6 +3555,7 @@ impl P2pEndpoint { addr: TransportAddr::Quic(addr), public_key: None, side: Side::Server, + traversal_method: TraversalMethod::HolePunch, }); // Spawn a reader task for the connection so incoming streams @@ -3588,6 +3684,8 @@ mod tests { let conn = PeerConnection { public_key: None, remote_addr: TransportAddr::Quic(socket_addr), + traversal_method: TraversalMethod::Direct, + side: Side::Client, authenticated: false, connected_at: Instant::now(), last_activity: Instant::now(), @@ -3699,6 +3797,7 @@ mod tests { addr: TransportAddr::Quic(socket_addr), public_key: None, side: Side::Client, + traversal_method: TraversalMethod::Direct, }; // Verify event fields @@ -3706,11 +3805,13 @@ mod tests { addr, public_key, side, + traversal_method, } = event { assert!(public_key.is_none()); assert_eq!(addr, TransportAddr::Quic(socket_addr)); assert!(side.is_client()); + assert_eq!(traversal_method, TraversalMethod::Direct); // Verify as_socket_addr() works let extracted = addr.as_socket_addr(); @@ -3731,6 +3832,7 @@ mod tests { }, public_key: None, side: Side::Server, + traversal_method: TraversalMethod::Direct, }; // Verify event fields @@ -3738,10 +3840,12 @@ mod tests { addr, public_key, side, + traversal_method, } = event { assert!(public_key.is_none()); assert!(side.is_server()); + assert_eq!(traversal_method, TraversalMethod::Direct); // Verify as_socket_addr() returns None for BLE assert!(addr.as_socket_addr().is_none()); @@ -3778,6 +3882,7 @@ mod tests { addr: TransportAddr::Quic(socket_addr), public_key: Some(vec![0x11; 32]), side: Side::Client, + traversal_method: TraversalMethod::Direct, }; // Verify events are Clone @@ -3807,6 +3912,8 @@ mod tests { let udp_conn = PeerConnection { public_key: None, remote_addr: TransportAddr::Quic(udp_addr), + traversal_method: TraversalMethod::Direct, + side: Side::Client, authenticated: true, connected_at: Instant::now(), last_activity: Instant::now(), @@ -3825,6 +3932,8 @@ mod tests { mac: mac_addr, psm: 128, }, + traversal_method: TraversalMethod::Direct, + side: Side::Client, authenticated: true, connected_at: Instant::now(), last_activity: Instant::now(), @@ -3842,6 +3951,7 @@ mod tests { addr: TransportAddr::Quic(socket_addr), public_key: None, side: Side::Client, + traversal_method: TraversalMethod::Direct, }; // Verify display formatting works for logging @@ -3871,6 +3981,8 @@ mod tests { let conn = PeerConnection { public_key: None, remote_addr: TransportAddr::Quic(socket_addr), + traversal_method: TraversalMethod::Direct, + side: Side::Client, authenticated: true, connected_at: Instant::now(), last_activity: Instant::now(), @@ -3901,6 +4013,8 @@ mod tests { PeerConnection { public_key: None, remote_addr: TransportAddr::Quic(udp_addr), + traversal_method: TraversalMethod::Direct, + side: Side::Client, authenticated: true, connected_at: Instant::now(), last_activity: Instant::now(), @@ -3919,6 +4033,8 @@ mod tests { PeerConnection { public_key: None, remote_addr: ble_addr, + traversal_method: TraversalMethod::Direct, + side: Side::Client, authenticated: true, connected_at: Instant::now(), last_activity: Instant::now(), @@ -3961,6 +4077,8 @@ mod tests { PeerConnection { public_key: None, remote_addr: TransportAddr::Quic(socket_addr), + traversal_method: TraversalMethod::Direct, + side: Side::Client, authenticated: true, connected_at: Instant::now(), last_activity: Instant::now(), @@ -4010,6 +4128,8 @@ mod tests { let mut conn = PeerConnection { public_key: None, remote_addr: TransportAddr::Quic(socket_addr), + traversal_method: TraversalMethod::Direct, + side: Side::Client, authenticated: false, connected_at: Instant::now(), last_activity: Instant::now(), diff --git a/src/reachability.rs b/src/reachability.rs new file mode 100644 index 00000000..00f4cf65 --- /dev/null +++ b/src/reachability.rs @@ -0,0 +1,176 @@ +// Copyright 2024 Saorsa Labs Ltd. +// +// This Saorsa Network Software is licensed under the General Public License (GPL), version 3. +// Please see the file LICENSE-GPL, or visit for the full text. +// +// Full details available at https://saorsalabs.com/licenses + +//! Reachability and connection path helpers. +//! +//! This module separates address classification from actual reachability. +//! A node may know that an address is globally routable without knowing whether +//! other peers can reach it directly. Direct reachability is only learned from +//! successful peer-observed direct connections. + +use std::net::{IpAddr, SocketAddr}; +use std::time::Duration; + +use serde::{Deserialize, Serialize}; + +/// Default freshness window for peer-verified direct reachability. +/// +/// Direct reachability is inherently time-sensitive, especially for NAT-backed +/// addresses whose mappings may expire. Evidence older than this should no +/// longer be treated as current relay/coordinator capability. +pub const DIRECT_REACHABILITY_TTL: Duration = Duration::from_secs(15 * 60); + +/// Scope in which a socket address is directly reachable. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +pub enum ReachabilityScope { + /// Reachable only from the same host. + Loopback, + /// Reachable on the local network, including RFC1918/ULA/link-local space. + LocalNetwork, + /// Reachable using a globally routable address. + Global, +} + +impl std::fmt::Display for ReachabilityScope { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Loopback => write!(f, "loopback"), + Self::LocalNetwork => write!(f, "local-network"), + Self::Global => write!(f, "global"), + } + } +} + +impl ReachabilityScope { + /// Returns the broader of two scopes. + pub fn broaden(self, other: Self) -> Self { + self.max(other) + } +} + +/// Method used to establish a connection. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum TraversalMethod { + /// Direct connection, no coordinator or relay involved. + Direct, + /// Coordinated hole punching. + HolePunch, + /// Connection established via relay. + Relay, + /// Port prediction for symmetric NATs. + PortPrediction, +} + +impl TraversalMethod { + /// Whether this connection path is directly reachable without assistance. + pub const fn is_direct(self) -> bool { + matches!(self, Self::Direct) + } +} + +impl std::fmt::Display for TraversalMethod { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Direct => write!(f, "direct"), + Self::HolePunch => write!(f, "hole punch"), + Self::Relay => write!(f, "relay"), + Self::PortPrediction => write!(f, "port prediction"), + } + } +} + +/// Classify the reachability scope implied by an address. +/// +/// Returns `None` for unspecified or multicast addresses, which are not useful +/// as direct reachability evidence. +pub fn socket_addr_scope(addr: SocketAddr) -> Option { + match addr.ip() { + IpAddr::V4(ipv4) => { + if ipv4.is_unspecified() || ipv4.is_multicast() { + None + } else if ipv4.is_loopback() { + Some(ReachabilityScope::Loopback) + } else if ipv4.is_private() || ipv4.is_link_local() { + Some(ReachabilityScope::LocalNetwork) + } else { + Some(ReachabilityScope::Global) + } + } + IpAddr::V6(ipv6) => { + if ipv6.is_unspecified() || ipv6.is_multicast() { + None + } else if ipv6.is_loopback() { + Some(ReachabilityScope::Loopback) + } else if ipv6.is_unique_local() || ipv6.is_unicast_link_local() { + Some(ReachabilityScope::LocalNetwork) + } else { + Some(ReachabilityScope::Global) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::net::{Ipv4Addr, Ipv6Addr}; + + #[test] + fn test_socket_addr_scope_ipv4() { + assert_eq!( + socket_addr_scope(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9000)), + Some(ReachabilityScope::Loopback) + ); + assert_eq!( + socket_addr_scope(SocketAddr::new( + IpAddr::V4(Ipv4Addr::new(192, 168, 1, 10)), + 9000 + )), + Some(ReachabilityScope::LocalNetwork) + ); + assert_eq!( + socket_addr_scope(SocketAddr::new( + IpAddr::V4(Ipv4Addr::new(203, 0, 113, 10)), + 9000 + )), + Some(ReachabilityScope::Global) + ); + assert_eq!( + socket_addr_scope(SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 9000)), + None + ); + } + + #[test] + fn test_socket_addr_scope_ipv6() { + assert_eq!( + socket_addr_scope(SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 9000)), + Some(ReachabilityScope::Loopback) + ); + assert_eq!( + socket_addr_scope(SocketAddr::new( + IpAddr::V6("fd00::1".parse::().expect("valid ULA")), + 9000, + )), + Some(ReachabilityScope::LocalNetwork) + ); + assert_eq!( + socket_addr_scope(SocketAddr::new( + IpAddr::V6("2001:db8::1".parse::().expect("valid global v6")), + 9000, + )), + Some(ReachabilityScope::Global) + ); + } + + #[test] + fn test_traversal_method_direct_flag() { + assert!(TraversalMethod::Direct.is_direct()); + assert!(!TraversalMethod::HolePunch.is_direct()); + assert!(!TraversalMethod::Relay.is_direct()); + } +} diff --git a/tests/connection_lifecycle_tests.rs b/tests/connection_lifecycle_tests.rs index 55061ec6..1a161ca5 100644 --- a/tests/connection_lifecycle_tests.rs +++ b/tests/connection_lifecycle_tests.rs @@ -283,15 +283,16 @@ mod connection_lifecycle { /// Test NAT statistics #[tokio::test] - async fn test_nat_statistics() { + async fn test_endpoint_stats() { let config = test_node_config(vec![]); let node = P2pEndpoint::new(config) .await .expect("Failed to create node"); - // Get NAT stats - synchronous call - let _nat_stats = node.nat_stats(); - println!("NAT stats received"); + // Get endpoint stats + let stats = node.stats().await; + assert_eq!(stats.active_connections, 0); + println!("Endpoint stats received"); shutdown_with_timeout(node).await; } diff --git a/tests/constrained_integration.rs b/tests/constrained_integration.rs index e65d5617..6b7f0322 100644 --- a/tests/constrained_integration.rs +++ b/tests/constrained_integration.rs @@ -753,6 +753,8 @@ fn test_peer_connection_transport_addr() { let peer_conn_udp = PeerConnection { public_key: Some(vec![0x11; 32]), remote_addr: udp_addr.clone(), + traversal_method: saorsa_transport::TraversalMethod::Direct, + side: saorsa_transport::Side::Client, authenticated: true, connected_at: Instant::now(), last_activity: Instant::now(), @@ -771,6 +773,8 @@ fn test_peer_connection_transport_addr() { let peer_conn_ble = PeerConnection { public_key: None, remote_addr: ble_addr.clone(), + traversal_method: saorsa_transport::TraversalMethod::Direct, + side: saorsa_transport::Side::Client, authenticated: false, connected_at: Instant::now(), last_activity: Instant::now(), diff --git a/tests/event_migration.rs b/tests/event_migration.rs index d0724bcf..6d43b23d 100644 --- a/tests/event_migration.rs +++ b/tests/event_migration.rs @@ -76,6 +76,7 @@ fn test_peer_connected_event_construction_udp() { addr: TransportAddr::Udp(socket_addr), public_key: Some(test_public_key.clone()), side: saorsa_transport::Side::Client, + traversal_method: saorsa_transport::TraversalMethod::Direct, }; // Verify we can destructure it correctly @@ -83,6 +84,7 @@ fn test_peer_connected_event_construction_udp() { addr, public_key, side, + .. } = event { assert_eq!(public_key.unwrap(), test_public_key); @@ -128,6 +130,7 @@ fn test_event_clone_for_broadcast() { addr: TransportAddr::Udp(socket_addr), public_key: Some(vec![0xaa; 32]), side: saorsa_transport::Side::Server, + traversal_method: saorsa_transport::TraversalMethod::Direct, }; // Clone is required for broadcast channel @@ -140,11 +143,13 @@ fn test_event_clone_for_broadcast() { addr: a1, public_key: pk1, side: s1, + .. }, P2pEvent::PeerConnected { addr: a2, public_key: pk2, side: s2, + .. }, ) => { assert_eq!(pk1, pk2); @@ -166,6 +171,7 @@ fn test_multi_transport_events() { addr: TransportAddr::Udp(udp_addr), public_key: Some(vec![0x01; 32]), side: saorsa_transport::Side::Client, + traversal_method: saorsa_transport::TraversalMethod::Direct, }; // BLE event @@ -176,6 +182,7 @@ fn test_multi_transport_events() { }, public_key: Some(vec![0x02; 32]), side: saorsa_transport::Side::Server, + traversal_method: saorsa_transport::TraversalMethod::Direct, }; // Verify we can distinguish between them @@ -202,6 +209,7 @@ fn test_transport_aware_event_handling() { addr: TransportAddr::Udp("10.0.0.1:8080".parse().expect("valid")), public_key: Some(vec![0x01; 32]), side: saorsa_transport::Side::Client, + traversal_method: saorsa_transport::TraversalMethod::Direct, }, P2pEvent::PeerConnected { addr: TransportAddr::Ble { @@ -210,6 +218,7 @@ fn test_transport_aware_event_handling() { }, public_key: Some(vec![0x02; 32]), side: saorsa_transport::Side::Server, + traversal_method: saorsa_transport::TraversalMethod::Direct, }, P2pEvent::ExternalAddressDiscovered { addr: TransportAddr::Udp("203.0.113.1:9000".parse().expect("valid")), @@ -248,6 +257,7 @@ fn test_backward_compatibility_with_as_socket_addr() { addr: TransportAddr::Udp(socket_addr), public_key: Some(vec![0xff; 32]), side: saorsa_transport::Side::Client, + traversal_method: saorsa_transport::TraversalMethod::Direct, }; // Simulate legacy code that expects SocketAddr @@ -291,6 +301,7 @@ fn test_event_debug_formatting() { addr: TransportAddr::Udp("192.168.0.100:9001".parse().expect("valid")), public_key: Some(vec![0x55; 32]), side: saorsa_transport::Side::Client, + traversal_method: saorsa_transport::TraversalMethod::Direct, }; let debug = format!("{:?}", event); From d12d86c67826c7361a3c17196a5fdf62acbd4cfa Mon Sep 17 00:00:00 2001 From: grumbach Date: Tue, 7 Apr 2026 16:18:49 +0900 Subject: [PATCH 30/43] =?UTF-8?q?fix:=20address=20review=20feedback=20?= =?UTF-8?q?=E2=80=94=20double-count=20bug,=20relay=20capability,=20rustdoc?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove record_direct_incoming_stats() which double-counted active_direct_incoming_connections (event_callback already handles it via spawn_connection_handler -> ConnectionEstablished) - Only set supports_relay/supports_coordination when side.is_client() (we connected to them, proving they accept inbound). A peer that connected to us only proves they can make outbound connections. - Fix all pre-existing broken rustdoc links (LinkTransport::dial -> dial_addr, UpnpMappingService/RelaySlotTable scope issues, escaped brackets in relay_server.rs) - Add missing PortPrediction test coverage --- src/lib.rs | 4 ++-- src/link_transport.rs | 2 +- src/link_transport_impl.rs | 9 +++++++-- src/masque/relay_server.rs | 2 +- src/nat_traversal_api.rs | 2 +- src/p2p_endpoint.rs | 30 +++++++----------------------- src/reachability.rs | 1 + src/relay_slot_table.rs | 6 +++--- src/upnp.rs | 4 ++-- 9 files changed, 25 insertions(+), 35 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index b62ba757..4bd4fa0e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -203,8 +203,8 @@ pub mod tracing; /// Best-effort UPnP IGD port mapping for NAT traversal assistance. /// /// This module is feature-gated behind `upnp` (enabled by default). When -/// disabled, [`UpnpMappingService`] is still present but is a no-op stub -/// that always reports [`UpnpState::Unavailable`]. +/// disabled, [`upnp::UpnpMappingService`] is still present but is a no-op stub +/// that always reports [`upnp::UpnpState::Unavailable`]. pub mod upnp; // Public modules with new structure diff --git a/src/link_transport.rs b/src/link_transport.rs index 43ce24c0..8786a168 100644 --- a/src/link_transport.rs +++ b/src/link_transport.rs @@ -1030,7 +1030,7 @@ pub type BoxStream<'a, T> = Pin + Send + /// /// This trait abstracts a single QUIC connection, providing methods to /// open streams and send/receive datagrams. Connections are obtained via -/// [`LinkTransport::dial`] or [`LinkTransport::accept`]. +/// [`LinkTransport::dial_addr`] or [`LinkTransport::accept`]. /// /// # Stream Types /// diff --git a/src/link_transport_impl.rs b/src/link_transport_impl.rs index 37aae558..e32c2fd9 100644 --- a/src/link_transport_impl.rs +++ b/src/link_transport_impl.rs @@ -489,7 +489,7 @@ impl P2pLinkTransport { P2pEvent::PeerConnected { addr, public_key, - side: _, + side, traversal_method, } => { // Extract SocketAddr (currently UDP-only) @@ -498,7 +498,12 @@ impl P2pLinkTransport { SocketAddr::from(([0, 0, 0, 0], 0)) }); let mut caps = Capabilities::new_connected(socket_addr); - if traversal_method.is_direct() { + // Only promote relay/coordinator when we connected to + // them directly (Client side), proving they accept + // inbound connections. A peer that connected to us + // (Server side) only proves they can make outbound + // connections, not that they are reachable by others. + if traversal_method.is_direct() && side.is_client() { caps.supports_relay = true; caps.supports_coordination = true; caps.direct_reachability_scope = diff --git a/src/masque/relay_server.rs b/src/masque/relay_server.rs index 942bcb29..bb27d649 100644 --- a/src/masque/relay_server.rs +++ b/src/masque/relay_server.rs @@ -868,7 +868,7 @@ impl MasqueRelayServer { /// of unreliable QUIC datagrams. This avoids the MTU limitation that causes /// "datagram too large" errors for QUIC Initial packets (1200+ bytes). /// - /// Protocol: each forwarded packet is framed as [4-byte BE length][payload]. + /// Protocol: each forwarded packet is framed as \[4-byte BE length\]\[payload\]. pub async fn run_stream_forwarding_loop( self: &Arc, session_id: u64, diff --git a/src/nat_traversal_api.rs b/src/nat_traversal_api.rs index c7aa92be..066d9a80 100644 --- a/src/nat_traversal_api.rs +++ b/src/nat_traversal_api.rs @@ -479,7 +479,7 @@ pub struct NatTraversalConfig { /// Internally tunes the QUIC per-stream receive window so that a single /// message of this size can be transmitted without flow-control rejection. /// - /// Default: [`P2pConfig::DEFAULT_MAX_MESSAGE_SIZE`] (1 MiB). + /// Default: [`crate::P2pConfig::DEFAULT_MAX_MESSAGE_SIZE`] (1 MiB). #[serde(default = "default_max_message_size")] pub max_message_size: usize, diff --git a/src/p2p_endpoint.rs b/src/p2p_endpoint.rs index fa571652..33f47eed 100644 --- a/src/p2p_endpoint.rs +++ b/src/p2p_endpoint.rs @@ -2332,28 +2332,11 @@ impl P2pEndpoint { Ok(peer_conn) } - /// Check if we're connected to a specific address - async fn record_direct_incoming_stats(&self, remote_addr: SocketAddr) { - let mut stats = self.stats.write().await; - stats.active_connections += 1; - stats.successful_connections += 1; - stats.direct_connections += 1; - stats.active_direct_incoming_connections += 1; - let now = Instant::now(); - - match socket_addr_scope(remote_addr) { - Some(ReachabilityScope::Loopback) => { - stats.last_direct_loopback_at = Some(now); - } - Some(ReachabilityScope::LocalNetwork) => { - stats.last_direct_local_at = Some(now); - } - Some(ReachabilityScope::Global) => { - stats.last_direct_global_at = Some(now); - } - None => {} - } - } + // NOTE: direct incoming stats (active_direct_incoming_connections and + // scope timestamps) are recorded exclusively in the event_callback + // closure when NatTraversalEvent::ConnectionEstablished is emitted by + // spawn_connection_handler. No separate increment here to avoid + // double-counting. async fn is_connected_to_addr(&self, addr: SocketAddr) -> bool { let transport_addr = TransportAddr::Quic(addr); @@ -2429,7 +2412,8 @@ impl P2pEndpoint { .await .insert(remote_addr, peer_conn.clone()); - self.record_direct_incoming_stats(remote_addr).await; + // Stats are recorded by the event_callback when + // spawn_connection_handler emits ConnectionEstablished. // They initiated the connection to us = Server side let _ = self.event_tx.send(P2pEvent::PeerConnected { diff --git a/src/reachability.rs b/src/reachability.rs index 00f4cf65..45adfb8b 100644 --- a/src/reachability.rs +++ b/src/reachability.rs @@ -172,5 +172,6 @@ mod tests { assert!(TraversalMethod::Direct.is_direct()); assert!(!TraversalMethod::HolePunch.is_direct()); assert!(!TraversalMethod::Relay.is_direct()); + assert!(!TraversalMethod::PortPrediction.is_direct()); } } diff --git a/src/relay_slot_table.rs b/src/relay_slot_table.rs index a03b860d..e750eca3 100644 --- a/src/relay_slot_table.rs +++ b/src/relay_slot_table.rs @@ -8,7 +8,7 @@ //! Node-wide hole-punch coordinator back-pressure (Tier 4 lite). //! //! Every connection that lands at a node and acts as a hole-punch coordinator -//! shares one [`RelaySlotTable`]. The table caps the number of in-flight +//! shares one [`RelaySlotTable`](crate::relay_slot_table::RelaySlotTable). The table caps the number of in-flight //! `(initiator, target)` relay sessions across the entire node, so a storm //! of cold-starting peers cannot pile up unbounded coordination work on a //! single bootstrap. When the cap is reached, additional `PUNCH_ME_NOW` @@ -24,12 +24,12 @@ //! (the punch traffic flows initiator↔target, bypassing the coordinator), //! so slot release happens via three mechanisms: //! -//! 1. **Inactivity timeout** ([`RelaySlotTable::idle_timeout`]). If no new +//! 1. **Inactivity timeout** ([`RelaySlotTable::idle_timeout`](crate::relay_slot_table::RelaySlotTable::idle_timeout)). If no new //! rounds for the same key arrive within this window the session is //! considered done — either the punch succeeded (no more rounds needed) //! or it definitively failed (the initiator rotated away). Default 5s. //! -//! 2. **Connection close** via [`RelaySlotTable::release_for_initiator`]. +//! 2. **Connection close** via `RelaySlotTable::release_for_initiator`. //! When the initiator's connection drops, every slot it owned is //! reclaimed immediately rather than waiting for the inactivity timeout. //! Called from `BootstrapCoordinator::Drop`. diff --git a/src/upnp.rs b/src/upnp.rs index 0e356221..6e846022 100644 --- a/src/upnp.rs +++ b/src/upnp.rs @@ -24,7 +24,7 @@ //! //! Concretely this means: //! -//! 1. [`UpnpMappingService::start`] never returns an error and never blocks +//! 1. [`UpnpMappingService::start`](crate::upnp::UpnpMappingService::start) never returns an error and never blocks //! on network I/O — it spawns a background task and returns immediately. //! 2. All failures are swallowed and logged at `debug` level. The only //! `info` log line is the success path. @@ -34,7 +34,7 @@ //! 4. The lease is finite (one hour by default), so a crashed process //! cannot leak a permanent mapping on the gateway. //! -//! Callers consume the service by polling [`UpnpMappingService::current`] +//! Callers consume the service by polling [`UpnpMappingService::current`](crate::upnp::UpnpMappingService::current) //! when they want the most recent state. The poll is a lock-free atomic //! load on the underlying `tokio::sync::watch` channel, so it is cheap to //! call from the candidate discovery hot path. From 6f139c74316c2cae10b41d1892fb54f1e3fa6451 Mon Sep 17 00:00:00 2001 From: grumbach Date: Tue, 7 Apr 2026 16:27:17 +0900 Subject: [PATCH 31/43] fix(relay): rotate through all relay candidates instead of failing on first --- src/p2p_endpoint.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/p2p_endpoint.rs b/src/p2p_endpoint.rs index 33f47eed..215fced2 100644 --- a/src/p2p_endpoint.rs +++ b/src/p2p_endpoint.rs @@ -1939,12 +1939,12 @@ impl P2pEndpoint { return Ok((conn, ConnectionMethod::Relayed { relay: relay_addr })); } Ok(Err(e)) => { - debug!("Relay connection failed: {}", e); - strategy.transition_to_failed(e.to_string()); + debug!("Relay connection failed: {e}"); + strategy.transition_to_next_relay(e.to_string()); } Err(_) => { debug!("Relay connection timed out"); - strategy.transition_to_failed("Timeout"); + strategy.transition_to_next_relay("Timeout"); } } } From e44b25e9973ff09489a651d5065f809c92e1c1a5 Mon Sep 17 00:00:00 2001 From: grumbach Date: Tue, 7 Apr 2026 16:32:37 +0900 Subject: [PATCH 32/43] fix(relay): return socket on session reuse instead of None --- src/nat_traversal_api.rs | 206 +++++++++++++++++++++++++++++++-------- 1 file changed, 164 insertions(+), 42 deletions(-) diff --git a/src/nat_traversal_api.rs b/src/nat_traversal_api.rs index 066d9a80..e43e55b7 100644 --- a/src/nat_traversal_api.rs +++ b/src/nat_traversal_api.rs @@ -1424,11 +1424,11 @@ impl NatTraversalEndpoint { }; // Use the local address as the public address (will be updated when external address is discovered) let server = MasqueRelayServer::new(relay_config, local_addr); - info!( - "Created MASQUE relay server on {} (symmetric P2P node)", - local_addr - ); - Some(Arc::new(server)) + info!("Created MASQUE relay server on {local_addr} (symmetric P2P node)"); + let server = Arc::new(server); + // Spawn periodic cleanup so expired sessions are reaped automatically + let _cleanup_handle = MasqueRelayServer::spawn_cleanup_task(&server); + Some(server) }; // Clone the callback for background tasks before moving into endpoint @@ -1848,11 +1848,11 @@ impl NatTraversalEndpoint { }; // Use the local address as the public address (will be updated when external address is discovered) let server = MasqueRelayServer::new(relay_config, local_addr); - info!( - "Created MASQUE relay server on {} (symmetric P2P node)", - local_addr - ); - Some(Arc::new(server)) + info!("Created MASQUE relay server on {local_addr} (symmetric P2P node)"); + let server = Arc::new(server); + // Spawn periodic cleanup so expired sessions are reaped automatically + let _cleanup_handle = MasqueRelayServer::spawn_cleanup_task(&server); + Some(server) }; // Clone the callback for background tasks before moving into endpoint @@ -3450,16 +3450,29 @@ impl NatTraversalEndpoint { ), NatTraversalError, > { - // Check if we already have an active session to this relay + // Check if we already have an active session to this relay. + // If so, open a new bidi stream on the existing connection and perform + // a fresh CONNECT-UDP handshake so the caller gets a usable socket. // DashMap provides lock-free .get() that returns Option> if let Some(session) = self.relay_sessions.get(&relay_addr) { if session.is_active() { - debug!("Reusing existing relay session to {}", relay_addr); - return Ok((session.public_address, None)); + debug!("Reusing existing relay session to {relay_addr}"); + let connection = session.connection.clone(); + let existing_public_address = session.public_address; + // Drop the DashMap ref before awaiting to avoid holding it across await + drop(session); + + return self + .open_relay_stream_and_handshake( + connection, + relay_addr, + existing_public_address, + ) + .await; } } - info!("Establishing relay session to {}", relay_addr); + info!("Establishing relay session to {relay_addr}"); // Prefer reusing an existing peer connection to the relay. // The relay server's handle_relay_requests is spawned for each ACCEPTED @@ -3467,7 +3480,7 @@ impl NatTraversalEndpoint { // already listening for bidi streams. let connection = if let Some(existing) = self.connections.get(&relay_addr) { if existing.close_reason().is_none() { - info!("Reusing existing peer connection to relay {}", relay_addr); + info!("Reusing existing peer connection to relay {relay_addr}"); existing.clone() } else { // Existing connection is dead — fall back to creating a new one @@ -3479,16 +3492,53 @@ impl NatTraversalEndpoint { self.connect_new_to_relay(relay_addr).await? }; + let (public_address, relay_socket) = self + .open_relay_stream_and_handshake(connection.clone(), relay_addr, None) + .await?; + + // Store the session + let session = RelaySession { + connection, + public_address, + established_at: std::time::Instant::now(), + relay_addr, + }; + + // DashMap provides lock-free .insert() + self.relay_sessions.insert(relay_addr, session); + + Ok((public_address, relay_socket)) + } + + /// Open a new bidi stream on `connection`, perform the CONNECT-UDP + /// handshake, and return the public address together with a relay socket. + /// + /// When `existing_public_address` is `Some`, it is used as a fallback if + /// the relay response does not include a proxy address (session-reuse + /// path). When `None`, the address comes solely from the response + /// (new-session path). + async fn open_relay_stream_and_handshake( + &self, + connection: InnerConnection, + relay_addr: SocketAddr, + existing_public_address: Option, + ) -> Result< + ( + Option, + Option>, + ), + NatTraversalError, + > { // Open a bidirectional stream for the CONNECT-UDP handshake let (mut send_stream, mut recv_stream) = connection.open_bi().await.map_err(|e| { - NatTraversalError::ConnectionFailed(format!("Failed to open relay stream: {}", e)) + NatTraversalError::ConnectionFailed(format!("Failed to open relay stream: {e}")) })?; // Send CONNECT-UDP Bind request with length prefix (stream stays open for data) let request = ConnectUdpRequest::bind_any(); let request_bytes = request.encode(); - debug!("Sending CONNECT-UDP Bind request to relay: {:?}", request); + debug!("Sending CONNECT-UDP Bind request to relay: {request:?}"); // Length-prefixed framing: [4-byte BE length][payload] let req_len = request_bytes.len() as u32; @@ -3496,12 +3546,12 @@ impl NatTraversalEndpoint { .write_all(&req_len.to_be_bytes()) .await .map_err(|e| { - NatTraversalError::ConnectionFailed(format!("Failed to send request length: {}", e)) + NatTraversalError::ConnectionFailed(format!("Failed to send request length: {e}")) })?; send_stream.write_all(&request_bytes).await.map_err(|e| { - NatTraversalError::ConnectionFailed(format!("Failed to send relay request: {}", e)) + NatTraversalError::ConnectionFailed(format!("Failed to send relay request: {e}")) })?; - // Do NOT call finish() — stream stays open for data forwarding + // Do NOT call finish() -- stream stays open for data forwarding // Read length-prefixed response let mut resp_len_buf = [0u8; 4]; @@ -3510,8 +3560,7 @@ impl NatTraversalEndpoint { .await .map_err(|e| { NatTraversalError::ConnectionFailed(format!( - "Failed to read relay response length: {}", - e + "Failed to read relay response length: {e}" )) })?; let resp_len = u32::from_be_bytes(resp_len_buf) as usize; @@ -3520,44 +3569,32 @@ impl NatTraversalEndpoint { .read_exact(&mut response_bytes) .await .map_err(|e| { - NatTraversalError::ConnectionFailed(format!("Failed to read relay response: {}", e)) + NatTraversalError::ConnectionFailed(format!("Failed to read relay response: {e}")) })?; let response = ConnectUdpResponse::decode(&mut bytes::Bytes::from(response_bytes)) .map_err(|e| { - NatTraversalError::ProtocolError(format!("Invalid relay response: {}", e)) + NatTraversalError::ProtocolError(format!("Invalid relay response: {e}")) })?; if !response.is_success() { let reason = response.reason.unwrap_or_else(|| "unknown".to_string()); return Err(NatTraversalError::ConnectionFailed(format!( - "Relay rejected request: {} (status {})", - reason, response.status + "Relay rejected request: {reason} (status {})", + response.status ))); } - let public_address = response.proxy_public_address; + // Use the address from the response, falling back to the stored one + // when reusing an existing session. + let public_address = response.proxy_public_address.or(existing_public_address); - info!( - "Relay session established with public address: {:?}", - public_address - ); + info!("Relay session established with public address: {public_address:?}"); // Create the MasqueRelaySocket from the open streams let relay_socket = public_address .map(|addr| crate::masque::MasqueRelaySocket::new(send_stream, recv_stream, addr)); - // Store the session - let session = RelaySession { - connection, - public_address, - established_at: std::time::Instant::now(), - relay_addr, - }; - - // DashMap provides lock-free .insert() - self.relay_sessions.insert(relay_addr, session); - // Notify the relay manager if let Some(ref manager) = self.relay_manager { if let Ok(resp) = @@ -6997,6 +7034,91 @@ mod tests { // let endpoint = NatTraversalEndpoint::new(config, None).unwrap(); } + #[test] + fn test_bootstrap_node_defaults_can_coordinate_false() { + let addr: SocketAddr = "1.2.3.4:5000".parse().unwrap(); + let node = BootstrapNode::new(addr); + assert!( + !node.can_coordinate, + "New bootstrap nodes should default to can_coordinate=false" + ); + } + + /// Verify that `select_coordinator` filters by `can_coordinate` and + /// weights by RTT and coordination_count. + #[tokio::test] + async fn test_select_coordinator_quality_weighted() { + let config = NatTraversalConfig { + known_peers: Vec::new(), + bind_addr: Some("127.0.0.1:0".parse().unwrap()), + ..Default::default() + }; + + let endpoint = NatTraversalEndpoint::new(config, None, None) + .await + .expect("Endpoint creation should succeed"); + + // Initially no coordinators available (no known_peers, no connections) + assert!( + endpoint.select_coordinator().is_none(), + "No coordinators should be available initially" + ); + + // Add some bootstrap nodes with varying quality + { + let mut nodes = endpoint.bootstrap_nodes.write(); + nodes.push(BootstrapNode { + address: "1.2.3.4:5000".parse().unwrap(), + last_seen: std::time::Instant::now(), + can_coordinate: false, // NOT eligible + rtt: Some(Duration::from_millis(10)), + coordination_count: 0, + }); + nodes.push(BootstrapNode { + address: "5.6.7.8:6000".parse().unwrap(), + last_seen: std::time::Instant::now(), + can_coordinate: true, // eligible - low RTT + rtt: Some(Duration::from_millis(20)), + coordination_count: 0, + }); + nodes.push(BootstrapNode { + address: "9.10.11.12:7000".parse().unwrap(), + last_seen: std::time::Instant::now(), + can_coordinate: true, // eligible - high RTT + rtt: Some(Duration::from_millis(500)), + coordination_count: 10, + }); + } + + // select_coordinator must never return the non-coordinator node + let non_coord: SocketAddr = "1.2.3.4:5000".parse().unwrap(); + for _ in 0..100 { + let selected = endpoint.select_coordinator(); + assert!(selected.is_some(), "Should find at least one coordinator"); + assert_ne!( + selected.unwrap(), + non_coord, + "Should never select a node with can_coordinate=false" + ); + } + + // With many trials, the low-RTT node should be selected more often + let mut low_rtt_count = 0u32; + let trials = 1000; + let low_rtt_addr: SocketAddr = "5.6.7.8:6000".parse().unwrap(); + for _ in 0..trials { + if endpoint.select_coordinator() == Some(low_rtt_addr) { + low_rtt_count += 1; + } + } + // The low-RTT node should be selected significantly more often + // (at least 60% of the time given the 25x RTT advantage) + assert!( + low_rtt_count > trials * 60 / 100, + "Low-RTT node should be preferred, got {low_rtt_count}/{trials}" + ); + } + #[test] fn test_candidate_address_validation() { use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; From f475bd06324e00ead189013fdf20971f1e876bd3 Mon Sep 17 00:00:00 2001 From: grumbach Date: Tue, 7 Apr 2026 16:35:56 +0900 Subject: [PATCH 33/43] fix(relay): schedule periodic cleanup of expired relay sessions The cleanup_expired_sessions() method existed but was never called periodically. Add spawn_cleanup_task() which uses a Weak reference and tokio::time::interval to reap timed-out sessions at the configured cleanup_interval (default 60s). The task stops automatically when the server Arc is dropped. Wire up the cleanup task at both relay server creation sites in nat_traversal_api.rs so it starts as soon as the node boots. --- src/masque/relay_server.rs | 88 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/src/masque/relay_server.rs b/src/masque/relay_server.rs index bb27d649..f6384b08 100644 --- a/src/masque/relay_server.rs +++ b/src/masque/relay_server.rs @@ -39,6 +39,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{Duration, Instant}; use tokio::net::UdpSocket; use tokio::sync::RwLock; +use tokio::task::JoinHandle; use crate::VarInt; use crate::high_level::Connection as QuicConnection; @@ -1116,6 +1117,45 @@ impl MasqueRelayServer { .map(|(id, _)| *id) .collect() } + + /// Spawn a background task that periodically cleans up expired relay sessions. + /// + /// Uses [`MasqueRelayConfig::cleanup_interval`] to determine how often the + /// cleanup runs. The task holds a [`std::sync::Weak`] reference to the server, + /// so it will stop automatically once the last [`Arc`] is + /// dropped. + /// + /// Returns a [`JoinHandle`] that can be used to abort the task if needed. + pub fn spawn_cleanup_task(server: &Arc) -> JoinHandle<()> { + let weak = Arc::downgrade(server); + let interval_duration = server.config.cleanup_interval; + + tokio::spawn(async move { + let mut interval = tokio::time::interval(interval_duration); + // The first tick completes immediately; skip it so we don't + // run cleanup right at startup before any sessions exist. + interval.tick().await; + + loop { + interval.tick().await; + + let Some(server) = weak.upgrade() else { + tracing::debug!("Relay server dropped, stopping cleanup task"); + break; + }; + + let cleaned = server.cleanup_expired_sessions().await; + if cleaned > 0 { + let remaining = server.session_count().await; + tracing::info!( + cleaned, + remaining, + "Periodic relay session cleanup completed" + ); + } + } + }) + } } /// Summary information about a session @@ -1506,4 +1546,52 @@ mod tests { // bind_any has no target, so no bridging assert!(!info.is_bridging); } + + #[tokio::test] + async fn test_cleanup_task_stops_when_server_dropped() { + let config = MasqueRelayConfig { + cleanup_interval: Duration::from_millis(50), + ..Default::default() + }; + let server = Arc::new(MasqueRelayServer::new(config, test_addr(9000))); + let handle = MasqueRelayServer::spawn_cleanup_task(&server); + + // Let the cleanup task run at least one tick + tokio::time::sleep(Duration::from_millis(80)).await; + assert!(!handle.is_finished()); + + // Drop the server; the Weak reference will fail to upgrade + drop(server); + tokio::time::sleep(Duration::from_millis(80)).await; + assert!(handle.is_finished()); + } + + #[tokio::test] + async fn test_cleanup_task_reaps_expired_sessions() { + let config = MasqueRelayConfig { + cleanup_interval: Duration::from_millis(50), + session_config: RelaySessionConfig { + session_timeout: Duration::from_millis(10), + ..Default::default() + }, + ..Default::default() + }; + let server = Arc::new(MasqueRelayServer::new(config, test_addr(9000))); + let _handle = MasqueRelayServer::spawn_cleanup_task(&server); + + // Create a session + let request = ConnectUdpRequest::bind_any(); + let response = server + .handle_connect_request(&request, client_addr(1)) + .await + .unwrap(); + assert_eq!(response.status, 200); + assert_eq!(server.session_count().await, 1); + + // Wait for the session to expire AND for the cleanup tick + tokio::time::sleep(Duration::from_millis(150)).await; + + // The periodic cleanup should have reaped the expired session + assert_eq!(server.session_count().await, 0); + } } From 44b46620ee6561a69d0e738680bbc08199ec0491 Mon Sep 17 00:00:00 2001 From: grumbach Date: Tue, 7 Apr 2026 16:38:54 +0900 Subject: [PATCH 34/43] fix(nat): quality-aware coordinator selection with RTT weighting and load balancing --- src/nat_traversal_api.rs | 96 +++++++++++++++++++++++++++++++++------- src/p2p_endpoint.rs | 7 +++ 2 files changed, 88 insertions(+), 15 deletions(-) diff --git a/src/nat_traversal_api.rs b/src/nat_traversal_api.rs index e43e55b7..0691b941 100644 --- a/src/nat_traversal_api.rs +++ b/src/nat_traversal_api.rs @@ -641,12 +641,16 @@ pub struct BootstrapNode { } impl BootstrapNode { - /// Create a new bootstrap node + /// Create a new bootstrap node. + /// + /// Defaults `can_coordinate` to `false`. Callers must explicitly set it + /// to `true` via [`NatTraversalEndpoint::set_can_coordinate`] once they + /// have evidence the peer is directly reachable. pub fn new(address: SocketAddr) -> Self { Self { address, last_seen: std::time::Instant::now(), - can_coordinate: true, + can_coordinate: false, rtt: None, coordination_count: 0, } @@ -2594,7 +2598,7 @@ impl NatTraversalEndpoint { bootstrap_nodes.push(BootstrapNode { address, last_seen: std::time::Instant::now(), - can_coordinate: true, + can_coordinate: false, rtt: None, coordination_count: 0, }); @@ -3979,22 +3983,22 @@ impl NatTraversalEndpoint { self.connections.len() ); - // Register connected peer as a potential coordinator for NAT traversal. - // In the symmetric P2P architecture (v0.13.0+), any connected node can - // coordinate hole-punching for us. + // Register connected peer as a potential bootstrap node for NAT traversal. + // Coordination capability is NOT assumed here -- callers must explicitly + // mark the node via `set_can_coordinate()` once they have evidence the + // peer is directly reachable (e.g., we successfully connected outbound). { let mut nodes = self.bootstrap_nodes.write(); if !nodes.iter().any(|n| n.address == addr) { nodes.push(BootstrapNode { address: addr, last_seen: std::time::Instant::now(), - can_coordinate: true, + can_coordinate: false, rtt: None, coordination_count: 0, }); info!( - "add_connection: registered {} as NAT traversal coordinator ({} total)", - addr, + "add_connection: registered {addr} as bootstrap node ({} total, can_coordinate=false)", nodes.len() ); } @@ -4007,6 +4011,20 @@ impl NatTraversalEndpoint { Ok(()) } + /// Mark (or unmark) a bootstrap node as capable of coordinating NAT traversal. + /// + /// Call this after evidence that the peer is directly reachable -- e.g., + /// after a successful outbound connection. Nodes connected via relay or + /// hole-punch should NOT be marked as coordinators since they may be + /// behind restrictive NAT themselves. + pub fn set_can_coordinate(&self, addr: &SocketAddr, can_coordinate: bool) { + let mut nodes = self.bootstrap_nodes.write(); + if let Some(node) = nodes.iter_mut().find(|n| &n.address == addr) { + node.can_coordinate = can_coordinate; + info!("set_can_coordinate: {addr} -> can_coordinate={can_coordinate}"); + } + } + /// Spawn the NAT traversal handler loop for an existing connection referenced by the endpoint. /// /// # Arguments @@ -5791,16 +5809,64 @@ impl NatTraversalEndpoint { } } - /// Select a coordinator from available bootstrap nodes + /// Select a coordinator from available bootstrap nodes. + /// + /// Filters to nodes that can actually coordinate (directly reachable, not + /// behind restrictive NAT) and weights selection by RTT (lower is better) + /// and `coordination_count` (lower is better) for load balancing. fn select_coordinator(&self) -> Option { // parking_lot::RwLock doesn't poison - always succeeds let nodes = self.bootstrap_nodes.read(); - // Simple round-robin or random selection - if !nodes.is_empty() { - let idx = rand::random::() % nodes.len(); - return Some(nodes[idx].address); + + // Only consider nodes that have been verified as coordinators + let eligible: Vec<&BootstrapNode> = nodes.iter().filter(|n| n.can_coordinate).collect(); + + if eligible.is_empty() { + return None; } - None + + // Compute a quality score for each eligible node. + // Higher score = better candidate. We use inverse RTT and inverse + // coordination_count so that lower values produce higher scores. + let scores: Vec = eligible + .iter() + .map(|node| { + // RTT component: prefer lower RTT. Use 500ms as the default + // when RTT is unknown (conservative but not disqualifying). + let rtt_ms = node + .rtt + .map(|d| d.as_millis() as f64) + .unwrap_or(500.0) + .max(1.0); // avoid division by zero + let rtt_score = 1000.0 / rtt_ms; + + // Load-balancing component: prefer less-loaded coordinators. + // Add 1 to avoid division by zero for fresh nodes. + let load_score = 1.0 / (node.coordination_count as f64 + 1.0); + + // Combined score: RTT matters more (weight 3) than load (weight 1) + rtt_score * 3.0 + load_score + }) + .collect(); + + let total_score: f64 = scores.iter().sum(); + if total_score <= 0.0 { + // Fallback: pick the first eligible node + return eligible.first().map(|n| n.address); + } + + // Weighted random selection: pick a node proportional to its score + let roll = rand::random::() * total_score; + let mut cumulative = 0.0; + for (node, score) in eligible.iter().zip(scores.iter()) { + cumulative += score; + if roll < cumulative { + return Some(node.address); + } + } + + // Floating-point edge case: return the last eligible node + eligible.last().map(|n| n.address) } /// Send coordination request to bootstrap node diff --git a/src/p2p_endpoint.rs b/src/p2p_endpoint.rs index 215fced2..99255f41 100644 --- a/src/p2p_endpoint.rs +++ b/src/p2p_endpoint.rs @@ -1013,6 +1013,10 @@ impl P2pEndpoint { .add_connection(addr, connection.clone()) .map_err(EndpointError::NatTraversal)?; + // We connected directly to this peer, so it is publicly reachable + // and can serve as a NAT traversal coordinator. + self.inner.set_can_coordinate(&addr, true); + // Spawn handler (we initiated the connection = Client side) self.inner .spawn_connection_handler(addr, connection, Side::Client, TraversalMethod::Direct) @@ -2026,6 +2030,9 @@ impl P2pEndpoint { .add_connection(addr, connection.clone()) .map_err(EndpointError::NatTraversal)?; + // Direct outbound connection succeeded -- peer is publicly reachable + self.inner.set_can_coordinate(&addr, true); + // Spawn connection handler (Client side - we initiated) self.inner .spawn_connection_handler(addr, connection, Side::Client, TraversalMethod::Direct) From 664ce201a6debdb01c8e7b01c159c1ee3cce8849 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Wed, 8 Apr 2026 16:33:17 +0100 Subject: [PATCH 35/43] fix(nat): defer old connection close on dedup to avoid disrupting in-flight DHT queries The accept loop was closing the old connection immediately with "superseded" when replacing it with a newer connection to the same address. This caused the remote peer to tear down all state on that connection, including in-flight DHT FindNode queries and quote requests. The lost responses forced the DHT to re-walk, triggering more connections, more supersedes, and a cascading slowdown. On a 60-node testnet with 80% port-restricted NAT, this caused: - 264 superseded connections per upload - Quote collection taking 8+ minutes (vs expected ~1.5 minutes) - 212 hole-punch timeout events Fix: instead of closing the old connection synchronously, spawn a task that waits 5 seconds before closing it. This gives in-flight operations time to complete on the old connection while new operations use the replacement. The DashMap is updated immediately so sends go on the new connection. Tested on 60-node testnet (48 NAT / 12 standard): - 0 supersede-related disruptions - Upload times back to ~1m 45s steady state - 8 consecutive 50MB uploads with 0 failures Co-Authored-By: Claude Opus 4.6 (1M context) --- src/nat_traversal_api.rs | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/src/nat_traversal_api.rs b/src/nat_traversal_api.rs index 0691b941..41612331 100644 --- a/src/nat_traversal_api.rs +++ b/src/nat_traversal_api.rs @@ -3831,14 +3831,25 @@ impl NatTraversalEndpoint { }; if has_live(&remote_address) || has_live(&normalized_remote) { info!( - "accept_loop: {} replacing existing connection with newer", + "accept_loop: {} replacing existing connection with newer (deferred close in 5s)", remote_address ); - // Close the OLD connection, not the new one - if let Some(old) = connections2.get(&remote_address) { - old.value().close(0u32.into(), b"superseded"); - } else if let Some(old) = connections2.get(&normalized_remote) { - old.value().close(0u32.into(), b"superseded"); + // Close the old connection after a grace period so + // in-flight DHT operations can complete. Closing + // immediately causes the remote to tear down all + // state (including pending queries). The 5s delay + // allows responses to arrive before the connection + // is torn down. + let old_conn = if let Some(old) = connections2.get(&remote_address) { + Some(old.value().clone()) + } else { + connections2.get(&normalized_remote).map(|old| old.value().clone()) + }; + if let Some(old) = old_conn { + tokio::spawn(async move { + tokio::time::sleep(std::time::Duration::from_secs(5)).await; + old.close(0u32.into(), b"superseded"); + }); } // Allow re-emission so the new connection gets a // reader task and PeerConnected event From 2cabf2b5e00d00650283265a915ab690d1529183 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Wed, 8 Apr 2026 19:35:58 +0100 Subject: [PATCH 36/43] fix(nat): normalize IPv4-mapped IPv6 addresses in hole-punch DashMap lookups MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit saorsa-core stores hole_punch_target_peer_ids and hole_punch_preferred_coordinators under IPv4 keys (e.g. 63.177.242.27:10008), but connect_with_fallback receives IPv6-mapped addresses ([::ffff:63.177.242.27]:10008) from dual-stack sockets. The DashMap lookup misses, causing: 1. target_peer_id falls back to wire_id_from_addr (address-based ID) 2. The coordinator can't match the address-based ID against its peer connection table (which stores real ML-DSA peer IDs) 3. PUNCH_ME_NOW relay fails with "No connection found" 4. Hole-punch times out after 2 rounds → MASQUE relay fallback At 60 nodes this was masked because bootstrap coordinators (public IP) had connections to most peers and the address fallback happened to work. At 990 nodes, NAT nodes are selected as coordinators more often and the address-based wire ID never matches. Fix: normalize all addresses to plain IPv4 when inserting into and looking up from both hole-punch DashMaps, using the existing normalize_socket_addr() helper. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/p2p_endpoint.rs | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/p2p_endpoint.rs b/src/p2p_endpoint.rs index 99255f41..f8595967 100644 --- a/src/p2p_endpoint.rs +++ b/src/p2p_endpoint.rs @@ -74,6 +74,7 @@ pub use crate::nat_traversal_api::TraversalPhase; use crate::nat_traversal_api::{NatTraversalEndpoint, NatTraversalError, NatTraversalEvent}; use crate::reachability::{ReachabilityScope, TraversalMethod, socket_addr_scope}; use crate::transport::{ProtocolEngine, TransportAddr, TransportRegistry}; +use crate::shared::normalize_socket_addr; use crate::unified_config::P2pConfig; use rustls; @@ -1307,6 +1308,7 @@ impl P2pEndpoint { /// Keyed by target address so concurrent dials to different peers each /// get their own peer ID without racing on shared state. pub async fn set_hole_punch_target_peer_id(&self, target: SocketAddr, peer_id: [u8; 32]) { + let target = normalize_socket_addr(target); self.hole_punch_target_peer_ids.insert(target, peer_id); } @@ -1344,6 +1346,7 @@ impl P2pEndpoint { target: SocketAddr, coordinators: Vec, ) { + let target = normalize_socket_addr(target); if coordinators.is_empty() { self.hole_punch_preferred_coordinators.remove(&target); } else { @@ -1516,7 +1519,11 @@ impl P2pEndpoint { // loop falls back to the existing single-coordinator retry behaviour. let mut preferred_coordinator_count: usize = 0; if let Some(target_addr) = target { - if let Some(preferred) = self.hole_punch_preferred_coordinators.get(&target_addr) { + let normalized_target_addr = normalize_socket_addr(target_addr); + if let Some(preferred) = self + .hole_punch_preferred_coordinators + .get(&normalized_target_addr) + { let preferred_list: Vec = preferred.clone(); drop(preferred); // Release the DashMap entry guard before mutating coordinator_candidates. Self::merge_preferred_coordinators(&mut coordinator_candidates, &preferred_list); @@ -2107,7 +2114,19 @@ impl P2pEndpoint { // Initiate NAT traversal — sends PUNCH_ME_NOW to coordinator. // Look up the target peer ID from the per-target map. This avoids // races when multiple concurrent connections share the same P2pEndpoint. - let target_peer_id = self.hole_punch_target_peer_ids.get(&target).map(|v| *v); + // Normalize the key to handle IPv4-mapped IPv6 addresses (::ffff:x.x.x.x) + // that may differ from the IPv4 key used by saorsa-core when storing. + let normalized_target = normalize_socket_addr(target); + if normalized_target != target { + info!( + "try_hole_punch: normalized target {} -> {} for peer ID lookup", + target, normalized_target + ); + } + let target_peer_id = self + .hole_punch_target_peer_ids + .get(&normalized_target) + .map(|v| *v); if let Some(ref pid) = target_peer_id { info!( "try_hole_punch: calling initiate_nat_traversal({}, {}) with peer ID {} (dashmap key={})", From b2e6aaafce5425c22b3ee716b51c8bbfc19731b0 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Thu, 9 Apr 2026 17:58:27 +0100 Subject: [PATCH 37/43] fix: NAT traversal fixes for large-scale testnets (990+ nodes) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six fixes for issues discovered on a 990-node testnet with 20% NAT simulation: 1. False symmetric NAT detection: is_symmetric_nat() now only checks observed addresses from known peers (bootstrap nodes), preventing relay-allocated ports from triggering false positives. 2. Relay address pollution: poll_discovery filters observations whose port doesn't match the local listen port, preventing MASQUE relay ports from being published in the DHT as the node's own address. 3. Simultaneous-connect tie-breaking: accept loop uses deterministic peer ID comparison (lower ID keeps outbound) to resolve duplicate connections from hole-punch races, preventing in-flight request loss. 4. Connection cleanup race: do_cleanup_connection checks whether the connection at an address is still live before removing, preventing a dying reader task from nuking a replacement connection. 5. Timeout reductions: hole-punch per-round 8s→3s, relay 10s→5s, MASQUE relay session establishment hard-capped at 5s. 6. Coordinator accumulation: set_hole_punch_preferred_coordinator appends referrers instead of overwriting, giving the rotation loop multiple coordinator options. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/connection_strategy.rs | 4 +- src/nat_traversal_api.rs | 162 ++++++++++++++++++++++++++++++++----- src/p2p_endpoint.rs | 67 ++++++++++++--- 3 files changed, 198 insertions(+), 35 deletions(-) diff --git a/src/connection_strategy.rs b/src/connection_strategy.rs index a3e9953d..63eac0ce 100644 --- a/src/connection_strategy.rs +++ b/src/connection_strategy.rs @@ -180,8 +180,8 @@ impl Default for StrategyConfig { Self { ipv4_timeout: Duration::from_secs(3), ipv6_timeout: Duration::from_secs(3), - holepunch_timeout: Duration::from_secs(8), - relay_timeout: Duration::from_secs(10), + holepunch_timeout: Duration::from_secs(3), + relay_timeout: Duration::from_secs(5), max_holepunch_rounds: 2, ipv6_enabled: true, relay_enabled: true, diff --git a/src/nat_traversal_api.rs b/src/nat_traversal_api.rs index 41612331..7f984890 100644 --- a/src/nat_traversal_api.rs +++ b/src/nat_traversal_api.rs @@ -252,6 +252,10 @@ pub struct NatTraversalEndpoint { /// NAT traversal configuration config: NatTraversalConfig, + /// BLAKE3 hash of our local SPKI public key, used for deterministic + /// simultaneous-connect tie-breaking. Both sides keep the connection + /// initiated by the peer with the lexicographically lower peer ID. + local_peer_id: [u8; 32], /// Known bootstrap/coordinator nodes /// Uses parking_lot::RwLock for faster, non-poisoning reads bootstrap_nodes: Arc>>, @@ -1464,9 +1468,18 @@ impl NatTraversalEndpoint { // Channel for background handshake completion (persistent across accept calls) let (hs_tx, hs_rx) = mpsc::channel(32); + // Compute local peer ID = BLAKE3(public_key_spki) for + // deterministic simultaneous-connect tie-breaking. + let local_peer_id: [u8; 32] = config + .identity_key + .as_ref() + .map(|(pub_key, _)| *blake3::hash(pub_key.0.as_ref()).as_bytes()) + .unwrap_or([0u8; 32]); + let endpoint = Self { inner_endpoint: Some(inner_endpoint.clone()), config: config.clone(), + local_peer_id, bootstrap_nodes, active_sessions: Arc::new(dashmap::DashMap::new()), discovery_manager, @@ -1673,6 +1686,13 @@ impl NatTraversalEndpoint { let local_session_id = DiscoverySessionId::Local; let relay_setup_attempted_clone = endpoint.relay_setup_attempted.clone(); + let known_peers_for_poll: std::collections::HashSet = + endpoint.config.known_peers.iter().copied().collect(); + let local_port_for_poll: u16 = endpoint + .config + .bind_addr + .map(|a| a.port()) + .unwrap_or(0); tokio::spawn(async move { Self::poll_discovery( discovery_manager_clone, @@ -1682,6 +1702,8 @@ impl NatTraversalEndpoint { event_callback_for_poll, local_session_id, relay_setup_attempted_clone, + known_peers_for_poll, + local_port_for_poll, ) .await; }); @@ -1888,9 +1910,18 @@ impl NatTraversalEndpoint { // Channel for background handshake completion (persistent across accept calls) let (hs_tx, hs_rx) = mpsc::channel(32); + // Compute local peer ID = BLAKE3(public_key_spki) for + // deterministic simultaneous-connect tie-breaking. + let local_peer_id: [u8; 32] = config + .identity_key + .as_ref() + .map(|(pub_key, _)| *blake3::hash(pub_key.0.as_ref()).as_bytes()) + .unwrap_or([0u8; 32]); + let endpoint = Self { inner_endpoint: Some(inner_endpoint.clone()), config: config.clone(), + local_peer_id, bootstrap_nodes, active_sessions: Arc::new(dashmap::DashMap::new()), discovery_manager, @@ -2097,6 +2128,13 @@ impl NatTraversalEndpoint { let local_session_id = DiscoverySessionId::Local; let relay_setup_attempted_clone = endpoint.relay_setup_attempted.clone(); + let known_peers_for_poll: std::collections::HashSet = + endpoint.config.known_peers.iter().copied().collect(); + let local_port_for_poll: u16 = endpoint + .config + .bind_addr + .map(|a| a.port()) + .unwrap_or(0); tokio::spawn(async move { Self::poll_discovery( discovery_manager_clone, @@ -2106,6 +2144,8 @@ impl NatTraversalEndpoint { event_callback_for_poll, local_session_id, relay_setup_attempted_clone, + known_peers_for_poll, + local_port_for_poll, ) .await; }); @@ -3165,6 +3205,8 @@ impl NatTraversalEndpoint { event_callback: Option>, local_session_id: DiscoverySessionId, relay_setup_attempted: Arc, + _known_peers: std::collections::HashSet, + local_listen_port: u16, ) { use tokio::time::{Duration, interval}; @@ -3195,6 +3237,30 @@ impl NatTraversalEndpoint { observed ); if let Some(observed_addr) = observed { + // Port filter: if we know our listen port (non-zero), + // only accept observations that report the same port. + // MASQUE relay sessions allocate ephemeral ports on the + // relay server (same IP, different port). These relay + // ports are NOT our listen address and must not be + // published in the DHT or used for symmetric-NAT + // detection. This single check replaces the previous + // IP-baseline approach and catches all relay-port + // pollution regardless of whether the relay runs on a + // known peer, a regular peer, or our own node. + if local_listen_port != 0 + && observed_addr.port() != local_listen_port + { + tracing::debug!( + "poll_discovery_task: SKIPPING observation {} from {} \ + (port {} != listen port {})", + observed_addr, + remote_addr, + observed_addr.port(), + local_listen_port, + ); + continue; + } + // Emit event if this is the first time this remote reported this address if emitted_discovery.insert((remote_addr, observed_addr)) { info!( @@ -3757,6 +3823,7 @@ impl NatTraversalEndpoint { let event_tx_opt = self.event_tx.clone(); let shutdown = self.shutdown.clone(); let incoming_notify = self.incoming_notify.clone(); + let local_peer_id = self.local_peer_id; tokio::spawn(async move { loop { @@ -3814,15 +3881,20 @@ impl NatTraversalEndpoint { let remote_address = connection.remote_address(); info!("Accepted connection from {} (unified path)", remote_address); - // If an existing live connection to this address exists, - // replace it with the newer one. The remote peer just - // completed a fresh TLS handshake on this connection, so - // this is the one they are actively using. Closing the - // newer connection (the old behavior) kills the remote's - // active connection and breaks identity exchange. + // Simultaneous-connect dedup with deterministic + // tie-breaking. When two nodes connect to each other at + // the same time, both end up with two QUIC connections. + // We resolve this deterministically: the node with the + // lexicographically *lower* peer ID keeps its outbound + // connection (the one it initiated). Because both sides + // apply the same rule, they converge on keeping exactly + // one connection without any signalling. // - // This is consistent with `add_connection` which also - // always overwrites with the newer connection. + // For an *accepted* (inbound) connection, "we are the + // higher ID" means the remote initiated it and we + // should keep it (close ours). "We are the lower ID" + // means we should keep our outbound instead, so we + // close this incoming one. let normalized_remote = crate::shared::normalize_socket_addr(remote_address); let has_live = |addr: &std::net::SocketAddr| -> bool { connections2 @@ -3830,16 +3902,40 @@ impl NatTraversalEndpoint { .is_some_and(|e| e.value().close_reason().is_none()) }; if has_live(&remote_address) || has_live(&normalized_remote) { + // Extract the remote peer's public key to compute + // their peer ID for tie-breaking. + let remote_peer_id: Option<[u8; 32]> = + Self::extract_public_key_from_connection(&connection) + .map(|spki| *blake3::hash(&spki).as_bytes()); + + if let Some(remote_id) = remote_peer_id { + // Deterministic rule: the peer with the lower + // ID keeps its *outbound* connection. This is + // an inbound connection, so: + // - If local < remote: we keep our outbound + // → close this inbound, keep existing. + // - If local > remote: remote keeps their + // outbound (this conn) → close existing. + // - If equal (self-connect): close incoming. + if local_peer_id <= remote_id { + info!( + "accept_loop: {} simultaneous-connect tie-break: \ + keeping existing outbound (local_id < remote_id)", + remote_address + ); + connection.close(0u32.into(), b"simultaneous-open-tiebreak"); + return; + } + } + + // Remote has lower ID, so they keep their outbound + // (this connection). Close our existing outbound + // with a grace period for in-flight operations. info!( - "accept_loop: {} replacing existing connection with newer (deferred close in 5s)", + "accept_loop: {} simultaneous-connect tie-break: \ + replacing existing with inbound (remote_id < local_id)", remote_address ); - // Close the old connection after a grace period so - // in-flight DHT operations can complete. Closing - // immediately causes the remote to tear down all - // state (including pending queries). The 5s delay - // allows responses to arrive before the connection - // is torn down. let old_conn = if let Some(old) = connections2.get(&remote_address) { Some(old.value().clone()) } else { @@ -4169,16 +4265,40 @@ impl NatTraversalEndpoint { Ok(None) } - /// Detect symmetric NAT by checking port diversity across peer connections. + /// Detect symmetric NAT by checking port diversity across **known-peer** + /// (bootstrap) connections only. + /// + /// Returns `true` if at least 2 different external ports are observed + /// from known peers, indicating that the NAT assigns a different port + /// per destination (symmetric NAT behaviour). /// - /// Returns `true` if at least 2 different external ports are observed from - /// different peers, indicating that the NAT assigns a different port per - /// destination (symmetric NAT behaviour). + /// Only known-peer connections are consulted because: + /// 1. They are public, long-lived, and use our primary QUIC socket. + /// 2. Relay connections (MASQUE) allocate a *separate* port on the + /// relay server, which is not evidence of symmetric NAT on our + /// local gateway. Including relay ports would false-positive on + /// every public node that happens to set up a relay fallback. + /// 3. Ephemeral peer connections may transit different NAT paths + /// (multi-homed hosts, VPNs) that are not representative. pub fn is_symmetric_nat(&self) -> bool { + let known_peer_addrs: std::collections::HashSet<_> = + self.config.known_peers.iter().copied().collect(); + + if known_peer_addrs.is_empty() { + // Without known peers we have no reliable baseline — be + // conservative and assume we are NOT behind symmetric NAT. + return false; + } + let mut observed_ports = std::collections::HashSet::new(); for entry in self.connections.iter() { - if let Some(addr) = entry.value().observed_address() { + let conn = entry.value(); + // Only consider connections to known (bootstrap) peers. + if !known_peer_addrs.contains(&conn.remote_address()) { + continue; + } + if let Some(addr) = conn.observed_address() { observed_ports.insert(addr.port()); } } @@ -4186,7 +4306,7 @@ impl NatTraversalEndpoint { let is_symmetric = observed_ports.len() >= 2; if is_symmetric { info!( - "Symmetric NAT detected: {} different external ports observed ({:?})", + "Symmetric NAT detected: {} different external ports observed from known peers ({:?})", observed_ports.len(), observed_ports ); diff --git a/src/p2p_endpoint.rs b/src/p2p_endpoint.rs index f8595967..e5d63e0c 100644 --- a/src/p2p_endpoint.rs +++ b/src/p2p_endpoint.rs @@ -593,11 +593,27 @@ async fn do_cleanup_connection( addr: &SocketAddr, reason: DisconnectReason, ) -> bool { - // Step 1: Remove from connected_peers (canonical lock #1) - let removed = connected_peers.write().await.remove(addr); + // Step 1: Check if the NAT traversal layer still has a *live* + // connection at this address. When a simultaneous-connect + // tie-break or hole-punch race replaces the old connection with + // a newer one, the old reader task exits and triggers cleanup. + // If the newer connection is already live in the DashMap, + // removing the address would kill the replacement — causing + // "Peer not found" errors on subsequent sends. Only proceed + // with full cleanup when no live connection remains. + if let Ok(None) = inner.remove_connection(addr) { + // remove_connection returned Ok(None) — a live connection + // exists at this address; the dead reader's cleanup must + // not disturb it. + debug!( + "do_cleanup_connection: {} still has a live replacement connection, skipping cleanup", + addr + ); + return false; + } - // Step 2: Remove from NAT traversal layer (lock-free DashMap) - let _ = inner.remove_connection(addr); + // Step 2: Remove from connected_peers (canonical lock #1) + let removed = connected_peers.write().await.remove(addr); // Step 3: Remove and abort reader task (canonical lock #2) let abort_handle = reader_handles.write().await.remove(addr); @@ -1366,8 +1382,22 @@ impl P2pEndpoint { target: SocketAddr, coordinator: SocketAddr, ) { - self.set_hole_punch_preferred_coordinators(target, vec![coordinator]) - .await; + let target = normalize_socket_addr(target); + // Append to the existing list instead of overwriting. + // During an iterative DHT lookup, multiple peers may refer us + // to the same target. Each referrer is a potential coordinator. + // Accumulating them gives the hole-punch rotation loop several + // options, so a coordinator that lacks a connection to the + // target can be skipped quickly (1.5s) in favour of one that + // does. + self.hole_punch_preferred_coordinators + .entry(target) + .and_modify(|v| { + if !v.contains(&coordinator) { + v.push(coordinator); + } + }) + .or_insert_with(|| vec![coordinator]); } /// Connect with automatic fallback: Direct → HolePunch → Relay. @@ -2248,12 +2278,25 @@ impl P2pEndpoint { target, relay_addr ); - // Step 1: Establish relay session (control plane handshake) - let (public_addr, relay_socket) = self - .inner - .establish_relay_session(relay_addr) - .await - .map_err(EndpointError::NatTraversal)?; + // Step 1: Establish relay session (control plane handshake). + // Hard 5-second timeout: the CONNECT-UDP handshake should + // complete in well under 1 second on any healthy relay. If + // the relay is unreachable or overloaded, fail fast so the + // strategy can try the next relay or give up. + let (public_addr, relay_socket) = match timeout( + Duration::from_secs(5), + self.inner.establish_relay_session(relay_addr), + ) + .await + { + Ok(result) => result.map_err(EndpointError::NatTraversal)?, + Err(_) => { + return Err(EndpointError::Connection(format!( + "MASQUE relay session establishment timed out (5s) for {}", + relay_addr + ))); + } + }; info!( "MASQUE relay session established via {} (public addr: {:?})", From 9a8ffa256ce0f07b4d5b22d97b0010ac1be8ad1d Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Thu, 9 Apr 2026 19:02:38 +0100 Subject: [PATCH 38/43] =?UTF-8?q?feat:=20PUNCH=5FME=5FNOW=5FNACK=20?= =?UTF-8?q?=E2=80=94=20coordinator=20signals=20target=20not=20found?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a coordinator receives a PUNCH_ME_NOW relay request but cannot find the target peer among its connections, it now sends a NACK frame back to the requester instead of silently dropping the request. This allows the requester to immediately rotate to the next coordinator (sub-RTT) instead of waiting for the full 3-second hole-punch timeout. On a 990-node testnet with 20% NAT, this eliminates ~5 minutes of dead time during DHT bootstrap where the client was waiting for timeouts on coordinators that didn't have the target. Implementation: - New QUIC extension frame PUNCH_ME_NOW_NACK (type 0x3d7e99) - Coordinator sends NACK via pending_relay_events back to requester - NACK flows: endpoint → high-level endpoint → channel → DashSet + Notify - try_hole_punch poll loop wakes on NACK and returns error immediately - Backward compatible: old nodes ignore unknown frame type per RFC 9000 Co-Authored-By: Claude Opus 4.6 (1M context) --- src/connection/mod.rs | 34 ++++++++++++ src/connection/spaces.rs | 5 ++ src/connection/stats.rs | 1 + src/endpoint.rs | 44 ++++++++++++---- src/frame.rs | 9 +++- src/frame/nat_traversal_unified.rs | 52 +++++++++++++++++++ src/high_level/endpoint.rs | 19 +++++++ src/nat_traversal_api.rs | 83 ++++++++++++++++++++++++++++++ src/p2p_endpoint.rs | 14 +++++ src/shared.rs | 8 +++ 10 files changed, 259 insertions(+), 10 deletions(-) diff --git a/src/connection/mod.rs b/src/connection/mod.rs index eb9c660a..3612d9b3 100644 --- a/src/connection/mod.rs +++ b/src/connection/mod.rs @@ -1633,6 +1633,13 @@ impl Connection { // Enqueue PunchMeNow frame for transmission self.spaces[SpaceId::Data].pending.punch_me_now.push(punch); } + QueuePunchMeNowNack(nack) => { + // Enqueue NACK for transmission (coordinator → requester) + self.spaces[SpaceId::Data] + .pending + .punch_me_now_nack + .push(nack); + } } } @@ -3626,6 +3633,18 @@ impl Connection { Frame::TryConnectToResponse(response) => { self.handle_try_connect_to_response(&response)?; } + Frame::PunchMeNowNack(nack) => { + tracing::info!( + "Received PUNCH_ME_NOW_NACK: round={}, target={}", + nack.round, + hex::encode(&nack.target_peer_id[..8]) + ); + self.endpoint_events.push_back( + EndpointEventInner::PunchMeNowNacked { + target_peer_id: nack.target_peer_id, + }, + ); + } } } @@ -4195,6 +4214,21 @@ impl Connection { self.stats.frame_tx.punch_me_now += 1; } + // NAT traversal frames - PunchMeNowNack + while buf.len() + frame::PunchMeNowNack::SIZE_BOUND < max_size + && space_id == SpaceId::Data + { + let nack = match space.pending.punch_me_now_nack.pop() { + Some(x) => x, + None => break, + }; + encode_or_close!(nack.try_encode(buf), "PUNCH_ME_NOW_NACK"); + sent.retransmits + .get_or_create() + .punch_me_now_nack + .push(nack); + } + // NAT traversal frames - RemoveAddress while buf.len() + frame::RemoveAddress::SIZE_BOUND < max_size && space_id == SpaceId::Data { let remove_address = match space.pending.remove_addresses.pop() { diff --git a/src/connection/spaces.rs b/src/connection/spaces.rs index a1f64b5c..9ef6ad0c 100644 --- a/src/connection/spaces.rs +++ b/src/connection/spaces.rs @@ -340,6 +340,8 @@ pub struct Retransmits { pub(super) add_addresses: Vec, /// NAT traversal PunchMeNow frames to be sent pub(super) punch_me_now: Vec, + /// PUNCH_ME_NOW_NACK frames to be sent (coordinator → requester) + pub(super) punch_me_now_nack: Vec, /// NAT traversal RemoveAddress frames to be sent pub(super) remove_addresses: Vec, /// OBSERVED_ADDRESS frames to be sent @@ -368,6 +370,7 @@ impl Retransmits { && self.new_tokens.is_empty() && self.add_addresses.is_empty() && self.punch_me_now.is_empty() + && self.punch_me_now_nack.is_empty() && self.remove_addresses.is_empty() && self.outbound_observations.is_empty() && self.try_connect_to.is_empty() @@ -396,6 +399,8 @@ impl ::std::ops::BitOrAssign for Retransmits { self.new_tokens.extend_from_slice(&rhs.new_tokens); self.add_addresses.extend_from_slice(&rhs.add_addresses); self.punch_me_now.extend_from_slice(&rhs.punch_me_now); + self.punch_me_now_nack + .extend_from_slice(&rhs.punch_me_now_nack); self.remove_addresses .extend_from_slice(&rhs.remove_addresses); self.outbound_observations diff --git a/src/connection/stats.rs b/src/connection/stats.rs index dcd808bd..2fcef390 100644 --- a/src/connection/stats.rs +++ b/src/connection/stats.rs @@ -112,6 +112,7 @@ impl FrameStats { Frame::ObservedAddress(_) => self.observed_address += 1, Frame::TryConnectTo(_) => self.try_connect_to += 1, Frame::TryConnectToResponse(_) => self.try_connect_to_response += 1, + Frame::PunchMeNowNack(_) => {} // Counted via relay stats, not per-frame } } } diff --git a/src/endpoint.rs b/src/endpoint.rs index 99cf2417..24babd50 100644 --- a/src/endpoint.rs +++ b/src/endpoint.rs @@ -116,6 +116,9 @@ pub struct Endpoint { /// Pending peer address updates from ADD_ADDRESS frames. /// Each entry is (peer_connection_addr, new_advertised_addr). pending_peer_address_updates: Vec<(SocketAddr, SocketAddr)>, + /// Pending PUNCH_ME_NOW NACKs received from coordinators. + /// Drained by the high-level layer to notify try_hole_punch poll loops. + pending_nacks: Vec<[u8; 32]>, } /// Deterministic 32-byte wire ID from a SocketAddr, used to correlate @@ -162,6 +165,7 @@ impl Endpoint { pending_relay_events: Vec::new(), pending_hole_punch_addrs: Vec::new(), pending_peer_address_updates: Vec::new(), + pending_nacks: Vec::new(), } } @@ -244,6 +248,13 @@ impl Endpoint { self.pending_peer_address_updates.drain(..) } + /// Drain pending PUNCH_ME_NOW NACKs from coordinators. + /// Returns target_peer_id values for which the coordinator could not find + /// a connection. The high-level layer should notify waiting hole-punch loops. + pub fn drain_nacks(&mut self) -> impl Iterator + '_ { + self.pending_nacks.drain(..) + } + /// Set the peer ID for an existing connection pub fn set_connection_peer_id(&mut self, connection_handle: ConnectionHandle, peer_id: PeerId) { if let Some(connection) = self.connections.get_mut(connection_handle.0) { @@ -381,18 +392,24 @@ impl Endpoint { ); } } else { - let known_peers: Vec = self - .connections - .iter() - .filter_map(|(_, meta)| { - meta.peer_id.as_ref().map(|pid| hex::encode(&pid.0[..8])) - }) - .collect(); tracing::warn!( - "No connection found for PUNCH_ME_NOW relay target peer_id={}, checked {} connections. Known peers: [{}]", + "No connection found for PUNCH_ME_NOW relay target peer_id={}, checked {} connections — sending NACK", hex::encode(&target_peer_id[..8]), self.connections.len(), - known_peers.join(", ") + ); + // Send NACK back to the requester so it can immediately + // rotate to another coordinator instead of waiting for timeout. + let nack = crate::frame::PunchMeNowNack { + round: punch_me_now.round, + target_peer_id, + }; + self.pending_relay_events.push(( + ch, + ConnectionEvent(ConnectionEventInner::QueuePunchMeNowNack(nack)), + )); + tracing::info!( + "Sent PUNCH_ME_NOW_NACK for target {} back to requester", + hex::encode(&target_peer_id[..8]) ); } } @@ -465,6 +482,15 @@ impl Endpoint { // connection attempt and send back the TryConnectToResponse. // For now, this event is acknowledged but not acted upon at the endpoint level. } + EndpointEventInner::PunchMeNowNacked { target_peer_id } => { + // Store NACK for the high-level layer to drain and propagate + // to try_hole_punch poll loops. + self.pending_nacks.push(target_peer_id); + tracing::info!( + "NACK received for target {}, queued for high-level layer", + hex::encode(&target_peer_id[..8]) + ); + } Drained => { if let Some(conn) = self.connections.try_remove(ch.0) { self.index.remove(&conn); diff --git a/src/frame.rs b/src/frame.rs index f879e193..1318461a 100644 --- a/src/frame.rs +++ b/src/frame.rs @@ -173,6 +173,7 @@ frame_types! { TRY_CONNECT_TO_IPV6 = 0x3d7e96, TRY_CONNECT_TO_RESPONSE_IPV4 = 0x3d7e97, TRY_CONNECT_TO_RESPONSE_IPV6 = 0x3d7e98, + PUNCH_ME_NOW_NACK = 0x3d7e99, // DATAGRAM } @@ -211,6 +212,7 @@ pub(crate) enum Frame { ObservedAddress(ObservedAddress), TryConnectTo(TryConnectTo), TryConnectToResponse(TryConnectToResponse), + PunchMeNowNack(PunchMeNowNack), } impl Frame { @@ -273,6 +275,7 @@ impl Frame { SocketAddr::V4(_) => FrameType::TRY_CONNECT_TO_RESPONSE_IPV4, SocketAddr::V6(_) => FrameType::TRY_CONNECT_TO_RESPONSE_IPV6, }, + PunchMeNowNack(_) => FrameType::PUNCH_ME_NOW_NACK, } } @@ -900,6 +903,9 @@ impl Iter { FrameType::TRY_CONNECT_TO_RESPONSE_IPV6 => { Frame::TryConnectToResponse(TryConnectToResponse::decode(&mut self.bytes, true)?) } + FrameType::PUNCH_ME_NOW_NACK => { + Frame::PunchMeNowNack(PunchMeNowNack::decode(&mut self.bytes)?) + } _ => { if let Some(s) = ty.stream() { Frame::Stream(Stream { @@ -1186,7 +1192,8 @@ impl AckFrequency { // Re-export unified NAT traversal frames pub(crate) use nat_traversal_unified::{ - AddAddress, PunchMeNow, RemoveAddress, TryConnectError, TryConnectTo, TryConnectToResponse, + AddAddress, PunchMeNow, PunchMeNowNack, RemoveAddress, TryConnectError, TryConnectTo, + TryConnectToResponse, }; /// Address Discovery frame for informing peers of their observed address diff --git a/src/frame/nat_traversal_unified.rs b/src/frame/nat_traversal_unified.rs index 8eed190a..c228031e 100644 --- a/src/frame/nat_traversal_unified.rs +++ b/src/frame/nat_traversal_unified.rs @@ -853,6 +853,58 @@ pub fn peer_supports_rfc_nat(transport_params: &[u8]) -> bool { }) } +/// PUNCH_ME_NOW_NACK — sent by a coordinator back to the requester when +/// it cannot relay a PUNCH_ME_NOW because the target peer is not among +/// its connections. This allows the requester to immediately rotate to +/// the next coordinator instead of waiting for a timeout. +/// +/// Wire format (single frame type — no IPv4/IPv6 variants): +/// ```text +/// +-------------------+------------------+ +/// | Frame Type | Round | Target | +/// | (VarInt: 0x3d7e99)| (VarInt)| Peer ID| +/// | | | (32 B) | +/// +-------------------+------------------+ +/// ``` +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PunchMeNowNack { + /// Round number echoed from the original PUNCH_ME_NOW + pub round: VarInt, + /// The peer ID the coordinator could not find + pub target_peer_id: [u8; 32], +} + +impl PunchMeNowNack { + /// Upper bound on encoded size: frame type (4) + round (9) + peer_id (32) + pub const SIZE_BOUND: usize = 4 + 9 + 32; + + pub fn encode(&self, buf: &mut W) { + if self.try_encode(buf).is_err() { + log_encode_overflow("PunchMeNowNack"); + } + } + + pub fn try_encode(&self, buf: &mut W) -> Result<(), VarIntBoundsExceeded> { + buf.write_var(FrameType::PUNCH_ME_NOW_NACK.0)?; + buf.write_var(self.round.into_inner())?; + buf.put_slice(&self.target_peer_id); + Ok(()) + } + + pub fn decode(r: &mut R) -> Result { + let round = r.get()?; + if r.remaining() < 32 { + return Err(UnexpectedEnd); + } + let mut target_peer_id = [0u8; 32]; + r.copy_to_slice(&mut target_peer_id); + Ok(Self { + round, + target_peer_id, + }) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/high_level/endpoint.rs b/src/high_level/endpoint.rs index 8f94e76b..8a7ae4b3 100644 --- a/src/high_level/endpoint.rs +++ b/src/high_level/endpoint.rs @@ -313,6 +313,13 @@ impl Endpoint { } } + /// Set the channel for forwarding PUNCH_ME_NOW NACKs to the NatTraversalEndpoint. + pub fn set_nack_tx(&self, tx: mpsc::UnboundedSender<[u8; 32]>) { + if let Ok(mut state) = self.inner.0.state.lock() { + state.nack_tx = Some(tx); + } + } + /// Connect to a remote endpoint /// /// `server_name` must be covered by the certificate presented by the server. This prevents a @@ -699,6 +706,8 @@ pub(crate) struct State { /// for full connection tracking instead of fire-and-forget. hole_punch_tx: Option>, peer_address_update_tx: Option>, + /// Channel for forwarding PUNCH_ME_NOW NACKs to the NatTraversalEndpoint + nack_tx: Option>, } #[derive(Debug)] @@ -843,6 +852,15 @@ impl State { } } + // Drain PUNCH_ME_NOW NACKs from coordinators and forward to NatTraversalEndpoint + let nacks: Vec<[u8; 32]> = self.inner.drain_nacks().collect(); + for target_peer_id in nacks { + did_work = true; + if let Some(ref tx) = self.nack_tx { + let _ = tx.send(target_peer_id); + } + } + did_work } } @@ -1003,6 +1021,7 @@ impl EndpointRef { default_client_config: None, hole_punch_tx: None, peer_address_update_tx: None, + nack_tx: None, }), })) } diff --git a/src/nat_traversal_api.rs b/src/nat_traversal_api.rs index 7f984890..1a83a7fd 100644 --- a/src/nat_traversal_api.rs +++ b/src/nat_traversal_api.rs @@ -344,6 +344,11 @@ pub struct NatTraversalEndpoint { /// receives them. Persistent across calls so no connections are lost. handshake_tx: mpsc::Sender>, handshake_rx: TokioMutex>>, + /// PUNCH_ME_NOW NACKs received from coordinators, keyed by target_peer_id. + /// Written by the high-level endpoint poll loop, read by try_hole_punch. + nack_set: Arc>, + /// Notification for new NACK arrivals — wakes try_hole_punch poll loops. + nack_notify: Arc, /// Tracks when each connection was first observed as closed. /// Used to enforce a grace period before removing dead connections. closed_at: dashmap::DashMap, @@ -1465,6 +1470,10 @@ impl NatTraversalEndpoint { let (peer_addr_tx, peer_addr_rx) = mpsc::unbounded_channel(); inner_endpoint.set_peer_address_update_tx(peer_addr_tx); + // Channel for PUNCH_ME_NOW NACK forwarding + let (nack_tx, nack_rx) = mpsc::unbounded_channel(); + inner_endpoint.set_nack_tx(nack_tx); + // Channel for background handshake completion (persistent across accept calls) let (hs_tx, hs_rx) = mpsc::channel(32); @@ -1513,6 +1522,8 @@ impl NatTraversalEndpoint { hole_punch_rx: TokioMutex::new(hole_punch_rx), handshake_tx: hs_tx, handshake_rx: TokioMutex::new(hs_rx), + nack_set: Arc::new(dashmap::DashSet::new()), + nack_notify: Arc::new(tokio::sync::Notify::new()), closed_at: dashmap::DashMap::new(), upnp_service: parking_lot::Mutex::new(Some(upnp_service)), }; @@ -1678,6 +1689,30 @@ impl NatTraversalEndpoint { endpoint.spawn_accept_loop(); info!("Accept loop spawned (unified path, parallel handshakes)"); + // Start NACK forwarder: drains NACK channel from high-level endpoint + // and records into DashSet for try_hole_punch to consume. + { + let nack_set = endpoint.nack_set.clone(); + let nack_notify = endpoint.nack_notify.clone(); + let shutdown = endpoint.shutdown.clone(); + tokio::spawn(async move { + let mut nack_rx = nack_rx; + while !shutdown.load(std::sync::atomic::Ordering::Relaxed) { + match nack_rx.recv().await { + Some(target_peer_id) => { + tracing::info!( + "NACK received for target {}, notifying hole-punch loops", + hex::encode(&target_peer_id[..8]) + ); + nack_set.insert(target_peer_id); + nack_notify.notify_waiters(); + } + None => break, // Channel closed + } + } + }); + } + // Start background discovery polling task let discovery_manager_clone = endpoint.discovery_manager.clone(); let shutdown_clone = endpoint.shutdown.clone(); @@ -1907,6 +1942,10 @@ impl NatTraversalEndpoint { let (peer_addr_tx, peer_addr_rx) = mpsc::unbounded_channel(); inner_endpoint.set_peer_address_update_tx(peer_addr_tx); + // Channel for PUNCH_ME_NOW NACK forwarding + let (nack_tx, nack_rx) = mpsc::unbounded_channel(); + inner_endpoint.set_nack_tx(nack_tx); + // Channel for background handshake completion (persistent across accept calls) let (hs_tx, hs_rx) = mpsc::channel(32); @@ -1955,6 +1994,8 @@ impl NatTraversalEndpoint { hole_punch_rx: TokioMutex::new(hole_punch_rx), handshake_tx: hs_tx, handshake_rx: TokioMutex::new(hs_rx), + nack_set: Arc::new(dashmap::DashSet::new()), + nack_notify: Arc::new(tokio::sync::Notify::new()), closed_at: dashmap::DashMap::new(), upnp_service: parking_lot::Mutex::new(Some(upnp_service)), }; @@ -2120,6 +2161,30 @@ impl NatTraversalEndpoint { endpoint.spawn_accept_loop(); info!("Accept loop spawned (unified path, parallel handshakes)"); + // Start NACK forwarder: drains NACK channel from high-level endpoint + // and records into DashSet for try_hole_punch to consume. + { + let nack_set = endpoint.nack_set.clone(); + let nack_notify = endpoint.nack_notify.clone(); + let shutdown = endpoint.shutdown.clone(); + tokio::spawn(async move { + let mut nack_rx = nack_rx; + while !shutdown.load(std::sync::atomic::Ordering::Relaxed) { + match nack_rx.recv().await { + Some(target_peer_id) => { + tracing::info!( + "NACK received for target {}, notifying hole-punch loops", + hex::encode(&target_peer_id[..8]) + ); + nack_set.insert(target_peer_id); + nack_notify.notify_waiters(); + } + None => break, // Channel closed + } + } + }); + } + // Start background discovery polling task let discovery_manager_clone = endpoint.discovery_manager.clone(); let shutdown_clone = endpoint.shutdown.clone(); @@ -2181,6 +2246,24 @@ impl NatTraversalEndpoint { } } + /// Record a PUNCH_ME_NOW NACK for a target peer ID. + /// Called by the high-level endpoint when it drains NACKs from the low-level. + pub fn record_nack(&self, target_peer_id: [u8; 32]) { + self.nack_set.insert(target_peer_id); + self.nack_notify.notify_waiters(); + } + + /// Check and consume a NACK for a specific target peer ID. + /// Returns true if a NACK was pending for this target (and removes it). + pub fn consume_nack(&self, target_peer_id: &[u8; 32]) -> bool { + self.nack_set.remove(target_peer_id).is_some() + } + + /// Get a reference to the NACK notification handle. + pub fn nack_notify(&self) -> &tokio::sync::Notify { + &self.nack_notify + } + /// Get the event callback pub fn get_event_callback(&self) -> Option<&Arc> { self.event_callback.as_ref() diff --git a/src/p2p_endpoint.rs b/src/p2p_endpoint.rs index e5d63e0c..aade30a6 100644 --- a/src/p2p_endpoint.rs +++ b/src/p2p_endpoint.rs @@ -2259,6 +2259,20 @@ impl P2pEndpoint { _ = self.inner.connection_notify().notified() => { debug!("try_hole_punch: connection_notify fired for {}", target); } + _ = self.inner.nack_notify().notified() => { + // Check if the NACK is for our target + if let Some(ref pid) = target_peer_id { + if self.inner.consume_nack(pid) { + info!( + "try_hole_punch: NACK received from coordinator for target {} — aborting immediately", + target + ); + return Err(EndpointError::Connection( + format!("Coordinator NACK: target {} not found", target), + )); + } + } + } _ = self.shutdown.cancelled() => { return Err(EndpointError::ShuttingDown); } diff --git a/src/shared.rs b/src/shared.rs index 44e47530..cbed76b1 100644 --- a/src/shared.rs +++ b/src/shared.rs @@ -28,6 +28,8 @@ pub(crate) enum ConnectionEventInner { QueueAddAddress(crate::frame::AddAddress), /// Queue a PUNCH_ME_NOW frame for transmission QueuePunchMeNow(crate::frame::PunchMeNow), + /// Queue a PUNCH_ME_NOW_NACK frame for transmission (coordinator → requester) + QueuePunchMeNowNack(crate::frame::PunchMeNowNack), } /// Variant of [`ConnectionEventInner`]. @@ -105,6 +107,12 @@ pub(crate) enum EndpointEventInner { requester_connection: SocketAddr, requested_at: crate::Instant, }, + /// A coordinator could not relay PUNCH_ME_NOW — target peer not found + /// among its connections. The requesting node should try another coordinator. + PunchMeNowNacked { + /// The target peer ID that the coordinator could not find + target_peer_id: [u8; 32], + }, } /// Protocol-level identifier for a connection. From 35a43c70b7b2e0a8848aa9e528506354790b3ccf Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Sun, 12 Apr 2026 16:58:46 +0100 Subject: [PATCH 39/43] feat: increase ACK timeout size budget from 1ms/KB to 4ms/KB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-stream ACK timeout after finish() used a size-proportional budget of 1 ms per KB on top of the base send_ack_timeout. For large payloads (e.g. 4 MB chunks) on bandwidth-constrained residential connections, this budget was insufficient — a 2.8 MB chunk only got ~2.8 s of budget, totalling ~7.8 s with the 5 s base, while the actual transfer can take 10-20 s on a slow uplink. Increase the coefficient to ~4 ms/KB (~250 KB/s floor). A 2.8 MB chunk now gets ~11 s of budget, totalling ~16 s with the base timeout. This eliminates chunk PUT timeouts on slow connections while having no effect on fast connections (where the ACK arrives well before the timeout). Co-Authored-By: Claude Opus 4.6 (1M context) --- src/p2p_endpoint.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/p2p_endpoint.rs b/src/p2p_endpoint.rs index aade30a6..6eaf9029 100644 --- a/src/p2p_endpoint.rs +++ b/src/p2p_endpoint.rs @@ -2701,12 +2701,11 @@ impl P2pEndpoint { // // The base timeout handles small messages and dead-connection // detection. For large payloads we add time proportional to - // size: QUIC slow-start over a high-RTT path needs multiple - // round trips to ramp the congestion window, so a 4 MB chunk - // over a 250 ms RTT link can take 2-3 s just to transmit. + // size at ~4 ms/KB (~250 KB/s floor): a 4 MB chunk gets ~16 s + // of budget on top of the base timeout. let base_timeout = self.config.timeouts.send_ack_timeout; let size_budget = - std::time::Duration::from_millis((data.len() as u64).saturating_div(1024)); + std::time::Duration::from_millis((data.len() as u64).saturating_div(256)); let ack_timeout = base_timeout + size_budget; match timeout(ack_timeout, send_stream.stopped()).await { Ok(Ok(None)) => {} From 8f1dd140f6e4c05109525823b7ec02b6c37128d8 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Tue, 14 Apr 2026 14:52:52 +0100 Subject: [PATCH 40/43] fix: increase handshake channel capacity from 32 to 1024 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bounded mpsc channel that carries completed connections from the accept loop (nat_traversal_api) to the reader task spawn path (p2p_endpoint) had a capacity of 32. When the consumer stalled briefly under write-lock contention in saorsa-core's accept loop, the channel filled and all subsequent connection handoffs blocked indefinitely — the send result was silently discarded with `let _ =`. After 15+ hours in a 1000-node testnet, this caused 1,079 connections to be accepted at the QUIC level but never forwarded to spawn reader tasks, stalling identity exchange and degrading upload times from ~175s to 358s+. Increase to 1024 to provide sufficient buffer for transient stalls. The primary fix is in saorsa-core (spawning registration work off the accept loop), but the larger buffer provides defense in depth. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/nat_traversal_api.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/nat_traversal_api.rs b/src/nat_traversal_api.rs index 1a83a7fd..43677bec 100644 --- a/src/nat_traversal_api.rs +++ b/src/nat_traversal_api.rs @@ -1474,8 +1474,12 @@ impl NatTraversalEndpoint { let (nack_tx, nack_rx) = mpsc::unbounded_channel(); inner_endpoint.set_nack_tx(nack_tx); - // Channel for background handshake completion (persistent across accept calls) - let (hs_tx, hs_rx) = mpsc::channel(32); + // Channel for background handshake completion (persistent across accept calls). + // Capacity 1024: the consumer (accept_connection_direct → P2pEndpoint::accept) + // can stall briefly under write-lock contention in saorsa-core's accept loop. + // A small buffer (32) caused the pipeline to back up after 15+ hours in a + // 1000-node testnet, blocking all new connection handoffs. + let (hs_tx, hs_rx) = mpsc::channel(1024); // Compute local peer ID = BLAKE3(public_key_spki) for // deterministic simultaneous-connect tie-breaking. @@ -1946,8 +1950,12 @@ impl NatTraversalEndpoint { let (nack_tx, nack_rx) = mpsc::unbounded_channel(); inner_endpoint.set_nack_tx(nack_tx); - // Channel for background handshake completion (persistent across accept calls) - let (hs_tx, hs_rx) = mpsc::channel(32); + // Channel for background handshake completion (persistent across accept calls). + // Capacity 1024: the consumer (accept_connection_direct → P2pEndpoint::accept) + // can stall briefly under write-lock contention in saorsa-core's accept loop. + // A small buffer (32) caused the pipeline to back up after 15+ hours in a + // 1000-node testnet, blocking all new connection handoffs. + let (hs_tx, hs_rx) = mpsc::channel(1024); // Compute local peer ID = BLAKE3(public_key_spki) for // deterministic simultaneous-connect tie-breaking. From 66a1c6f3e2f96b46e86dce5fadeeade6e2a2e7d9 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Tue, 14 Apr 2026 22:32:18 +0100 Subject: [PATCH 41/43] Revert "Merge pull request #59 from saorsa-labs/fix/handshake-channel-capacity" This reverts commit 7986b01604b745fc9e7c3714d46486485d0771cf, reversing changes made to f2b30ad1fa6f90a0a2d51d8b271893e01ec6aec7. --- src/nat_traversal_api.rs | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/src/nat_traversal_api.rs b/src/nat_traversal_api.rs index 43677bec..1a83a7fd 100644 --- a/src/nat_traversal_api.rs +++ b/src/nat_traversal_api.rs @@ -1474,12 +1474,8 @@ impl NatTraversalEndpoint { let (nack_tx, nack_rx) = mpsc::unbounded_channel(); inner_endpoint.set_nack_tx(nack_tx); - // Channel for background handshake completion (persistent across accept calls). - // Capacity 1024: the consumer (accept_connection_direct → P2pEndpoint::accept) - // can stall briefly under write-lock contention in saorsa-core's accept loop. - // A small buffer (32) caused the pipeline to back up after 15+ hours in a - // 1000-node testnet, blocking all new connection handoffs. - let (hs_tx, hs_rx) = mpsc::channel(1024); + // Channel for background handshake completion (persistent across accept calls) + let (hs_tx, hs_rx) = mpsc::channel(32); // Compute local peer ID = BLAKE3(public_key_spki) for // deterministic simultaneous-connect tie-breaking. @@ -1950,12 +1946,8 @@ impl NatTraversalEndpoint { let (nack_tx, nack_rx) = mpsc::unbounded_channel(); inner_endpoint.set_nack_tx(nack_tx); - // Channel for background handshake completion (persistent across accept calls). - // Capacity 1024: the consumer (accept_connection_direct → P2pEndpoint::accept) - // can stall briefly under write-lock contention in saorsa-core's accept loop. - // A small buffer (32) caused the pipeline to back up after 15+ hours in a - // 1000-node testnet, blocking all new connection handoffs. - let (hs_tx, hs_rx) = mpsc::channel(1024); + // Channel for background handshake completion (persistent across accept calls) + let (hs_tx, hs_rx) = mpsc::channel(32); // Compute local peer ID = BLAKE3(public_key_spki) for // deterministic simultaneous-connect tie-breaking. From 49a13c4ffba40f38a5ad392ace0ca5cc6f85a662 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Tue, 14 Apr 2026 22:39:14 +0100 Subject: [PATCH 42/43] chore: bump version to 0.32.0 Co-Authored-By: Claude Opus 4.6 (1M context) --- Cargo.lock | 213 ++++++++++++++++++++++++++--------------------------- Cargo.toml | 2 +- 2 files changed, 104 insertions(+), 111 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6ce39384..3cecf204 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -284,22 +284,22 @@ checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] name = "bitflags" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" [[package]] name = "blake3" -version = "1.8.3" +version = "1.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2468ef7d57b3fb7e16b576e8377cdbde2320c60e1491e961d11da40fc4f02a2d" +checksum = "4d2d5991425dfd0785aed03aedcf0b321d61975c9b5b3689c774a2610ae0b51e" dependencies = [ "arrayref", "arrayvec", "cc", "cfg-if", "constant_time_eq", - "cpufeatures 0.2.17", + "cpufeatures 0.3.0", ] [[package]] @@ -409,9 +409,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.58" +version = "1.2.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1e928d4b69e3077709075a938a05ffbedfa53a84c8f766efbf8220bb1ff60e1" +checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20" dependencies = [ "find-msvc-tools", "jobserver", @@ -456,7 +456,7 @@ checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" dependencies = [ "cfg-if", "cpufeatures 0.3.0", - "rand_core 0.10.0", + "rand_core 0.10.1", ] [[package]] @@ -1070,9 +1070,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.3.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" [[package]] name = "ff" @@ -1294,7 +1294,7 @@ dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", - "rand_core 0.10.0", + "rand_core 0.10.1", "wasip2", "wasip3", ] @@ -1376,9 +1376,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.16.1" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" [[package]] name = "heapless" @@ -1498,9 +1498,9 @@ checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" [[package]] name = "hyper" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" dependencies = [ "atomic-waker", "bytes", @@ -1512,7 +1512,6 @@ dependencies = [ "httparse", "itoa", "pin-project-lite", - "pin-utils", "smallvec", "tokio", "want", @@ -1520,15 +1519,14 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.7" +version = "0.27.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ "http", "hyper", "hyper-util", "rustls", - "rustls-pki-types", "tokio", "tokio-rustls", "tower-service", @@ -1583,12 +1581,13 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" dependencies = [ "displaydoc", "potential_utf", + "utf8_iter", "yoke", "zerofrom", "zerovec", @@ -1596,9 +1595,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" dependencies = [ "displaydoc", "litemap", @@ -1609,9 +1608,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -1623,15 +1622,15 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" [[package]] name = "icu_properties" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" dependencies = [ "icu_collections", "icu_locale_core", @@ -1643,15 +1642,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" [[package]] name = "icu_provider" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" dependencies = [ "displaydoc", "icu_locale_core", @@ -1709,7 +1708,7 @@ dependencies = [ "hyper", "hyper-util", "log", - "rand 0.10.0", + "rand 0.10.1", "tokio", "url", "xmltree", @@ -1717,12 +1716,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.13.0" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.16.1", + "hashbrown 0.17.0", "serde", "serde_core", ] @@ -1744,9 +1743,9 @@ checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" [[package]] name = "iri-string" -version = "0.7.11" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8e7418f59cc01c88316161279a7f665217ae316b388e58a0d10e29f54f1e5eb" +checksum = "25e659a4bb38e810ebc252e53b5814ff908a8c58c2a9ce2fae1bbec24cbf4e20" dependencies = [ "memchr", "serde", @@ -1878,9 +1877,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.92" +version = "0.3.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc4c90f45aa2e6eacbe8645f77fdea542ac97a494bcd117a67df9ff4d611f995" +checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca" dependencies = [ "cfg-if", "futures-util", @@ -1921,9 +1920,9 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "libc" -version = "0.2.183" +version = "0.2.185" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" +checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" [[package]] name = "libdbus-sys" @@ -1936,9 +1935,9 @@ dependencies = [ [[package]] name = "libredox" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ddbf48fd451246b1f8c2610bd3b4ac0cc6e149d89832867093ab69a17194f08" +checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c" dependencies = [ "libc", ] @@ -1951,9 +1950,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" [[package]] name = "lock_api" @@ -2260,12 +2259,6 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - [[package]] name = "pkcs8" version = "0.10.2" @@ -2278,9 +2271,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" [[package]] name = "plotters" @@ -2348,9 +2341,9 @@ dependencies = [ [[package]] name = "potential_utf" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" dependencies = [ "zerovec", ] @@ -2408,7 +2401,7 @@ dependencies = [ "bit-vec", "bitflags", "num-traits", - "rand 0.9.2", + "rand 0.9.4", "rand_chacha 0.9.0", "rand_xorshift", "regex-syntax", @@ -2455,7 +2448,7 @@ checksum = "95c589f335db0f6aaa168a7cd27b1fc6920f5e1470c804f814d9cd6e62a0f70b" dependencies = [ "env_logger", "log", - "rand 0.10.0", + "rand 0.10.1", ] [[package]] @@ -2499,7 +2492,7 @@ dependencies = [ "bytes", "getrandom 0.3.4", "lru-slab", - "rand 0.9.2", + "rand 0.9.4", "ring", "rustc-hash", "rustls", @@ -2572,9 +2565,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", @@ -2582,13 +2575,13 @@ dependencies = [ [[package]] name = "rand" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc266eb313df6c5c09c1c7b1fbe2510961e5bcd3add930c1e31f7ed9da0feff8" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" dependencies = [ "chacha20 0.10.0", "getrandom 0.4.2", - "rand_core 0.10.0", + "rand_core 0.10.1", ] [[package]] @@ -2631,9 +2624,9 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" [[package]] name = "rand_pcg" @@ -2655,9 +2648,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" dependencies = [ "either", "rayon-core", @@ -2826,9 +2819,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.37" +version = "0.23.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" +checksum = "69f9466fb2c14ea04357e91413efb882e2a6d4a406e625449bc0a5d360d53a21" dependencies = [ "aws-lc-rs", "log", @@ -2910,9 +2903,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.10" +version = "0.103.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" +checksum = "8279bb85272c9f10811ae6a6c547ff594d6a7f3c6c6b02ee9726d1d0dcfcdd06" dependencies = [ "aws-lc-rs", "ring", @@ -3007,7 +3000,7 @@ dependencies = [ [[package]] name = "saorsa-transport" -version = "0.31.0" +version = "0.32.0" dependencies = [ "anyhow", "arbitrary", @@ -3152,9 +3145,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "serde" @@ -3529,9 +3522,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" dependencies = [ "displaydoc", "zerovec", @@ -3564,9 +3557,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.50.0" +version = "1.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" +checksum = "a91135f59b1cbf38c91e73cf3386fca9bb77915c45ce2771460c9d92f0f3d776" dependencies = [ "bytes", "libc", @@ -3581,9 +3574,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.6.1" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", @@ -3893,9 +3886,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.115" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6523d69017b7633e396a89c5efab138161ed5aafcbc8d3e5c5a42ae38f50495a" +checksum = "0bf938a0bacb0469e83c1e148908bd7d5a6010354cf4fb73279b7447422e3a89" dependencies = [ "cfg-if", "once_cell", @@ -3906,9 +3899,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.65" +version = "0.4.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d1faf851e778dfa54db7cd438b70758eba9755cb47403f3496edd7c8fc212f0" +checksum = "f371d383f2fb139252e0bfac3b81b265689bf45b6874af544ffa4c975ac1ebf8" dependencies = [ "js-sys", "wasm-bindgen", @@ -3916,9 +3909,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.115" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e3a6c758eb2f701ed3d052ff5737f5bfe6614326ea7f3bbac7156192dc32e67" +checksum = "eeff24f84126c0ec2db7a449f0c2ec963c6a49efe0698c4242929da037ca28ed" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3926,9 +3919,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.115" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "921de2737904886b52bcbb237301552d05969a6f9c40d261eb0533c8b055fedf" +checksum = "9d08065faf983b2b80a79fd87d8254c409281cf7de75fc4b773019824196c904" dependencies = [ "bumpalo", "proc-macro2", @@ -3939,9 +3932,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.115" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a93e946af942b58934c604527337bad9ae33ba1d5c6900bbb41c2c07c2364a93" +checksum = "5fd04d9e306f1907bd13c6361b5c6bfc7b3b3c095ed3f8a9246390f8dbdee129" dependencies = [ "unicode-ident", ] @@ -3982,9 +3975,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.92" +version = "0.3.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84cde8507f4d7cfcb1185b8cb5890c494ffea65edbe1ba82cfd63661c805ed94" +checksum = "4f2dfbb17949fa2088e5d39408c48368947b86f7834484e87b73de55bc14d97d" dependencies = [ "js-sys", "wasm-bindgen", @@ -4636,9 +4629,9 @@ dependencies = [ [[package]] name = "writeable" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" [[package]] name = "x25519-dalek" @@ -4702,9 +4695,9 @@ dependencies = [ [[package]] name = "yoke" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -4713,9 +4706,9 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", @@ -4745,18 +4738,18 @@ dependencies = [ [[package]] name = "zerofrom" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", @@ -4786,9 +4779,9 @@ dependencies = [ [[package]] name = "zerotrie" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" dependencies = [ "displaydoc", "yoke", @@ -4797,9 +4790,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.5" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" dependencies = [ "yoke", "zerofrom", @@ -4808,9 +4801,9 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 75326e6a..03d54bfb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,7 @@ members = [ [package] name = "saorsa-transport" -version = "0.31.0" +version = "0.32.0" edition = "2024" rust-version = "1.88.0" license = "MIT OR Apache-2.0" From a6cc9fb6279f96c053c7f3fd7cd7418909f5524d Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Tue, 14 Apr 2026 22:44:16 +0100 Subject: [PATCH 43/43] fix: apply rustfmt formatting and fix test_default_config assertion Co-Authored-By: Claude Opus 4.6 (1M context) --- src/connection/mod.rs | 10 ++++------ src/connection_strategy.rs | 4 ++-- src/nat_traversal_api.rs | 20 ++++++-------------- src/p2p_endpoint.rs | 2 +- 4 files changed, 13 insertions(+), 23 deletions(-) diff --git a/src/connection/mod.rs b/src/connection/mod.rs index 3612d9b3..0fba7f76 100644 --- a/src/connection/mod.rs +++ b/src/connection/mod.rs @@ -3639,11 +3639,10 @@ impl Connection { nack.round, hex::encode(&nack.target_peer_id[..8]) ); - self.endpoint_events.push_back( - EndpointEventInner::PunchMeNowNacked { + self.endpoint_events + .push_back(EndpointEventInner::PunchMeNowNacked { target_peer_id: nack.target_peer_id, - }, - ); + }); } } } @@ -4215,8 +4214,7 @@ impl Connection { } // NAT traversal frames - PunchMeNowNack - while buf.len() + frame::PunchMeNowNack::SIZE_BOUND < max_size - && space_id == SpaceId::Data + while buf.len() + frame::PunchMeNowNack::SIZE_BOUND < max_size && space_id == SpaceId::Data { let nack = match space.pending.punch_me_now_nack.pop() { Some(x) => x, diff --git a/src/connection_strategy.rs b/src/connection_strategy.rs index 63eac0ce..1e598746 100644 --- a/src/connection_strategy.rs +++ b/src/connection_strategy.rs @@ -486,8 +486,8 @@ mod tests { let config = StrategyConfig::default(); assert_eq!(config.ipv4_timeout, Duration::from_secs(3)); assert_eq!(config.ipv6_timeout, Duration::from_secs(3)); - assert_eq!(config.holepunch_timeout, Duration::from_secs(8)); - assert_eq!(config.relay_timeout, Duration::from_secs(10)); + assert_eq!(config.holepunch_timeout, Duration::from_secs(3)); + assert_eq!(config.relay_timeout, Duration::from_secs(5)); assert_eq!(config.max_holepunch_rounds, 2); assert!(config.ipv6_enabled); assert!(config.relay_enabled); diff --git a/src/nat_traversal_api.rs b/src/nat_traversal_api.rs index 1a83a7fd..0e5152a9 100644 --- a/src/nat_traversal_api.rs +++ b/src/nat_traversal_api.rs @@ -1723,11 +1723,7 @@ impl NatTraversalEndpoint { let relay_setup_attempted_clone = endpoint.relay_setup_attempted.clone(); let known_peers_for_poll: std::collections::HashSet = endpoint.config.known_peers.iter().copied().collect(); - let local_port_for_poll: u16 = endpoint - .config - .bind_addr - .map(|a| a.port()) - .unwrap_or(0); + let local_port_for_poll: u16 = endpoint.config.bind_addr.map(|a| a.port()).unwrap_or(0); tokio::spawn(async move { Self::poll_discovery( discovery_manager_clone, @@ -2195,11 +2191,7 @@ impl NatTraversalEndpoint { let relay_setup_attempted_clone = endpoint.relay_setup_attempted.clone(); let known_peers_for_poll: std::collections::HashSet = endpoint.config.known_peers.iter().copied().collect(); - let local_port_for_poll: u16 = endpoint - .config - .bind_addr - .map(|a| a.port()) - .unwrap_or(0); + let local_port_for_poll: u16 = endpoint.config.bind_addr.map(|a| a.port()).unwrap_or(0); tokio::spawn(async move { Self::poll_discovery( discovery_manager_clone, @@ -3330,9 +3322,7 @@ impl NatTraversalEndpoint { // IP-baseline approach and catches all relay-port // pollution regardless of whether the relay runs on a // known peer, a regular peer, or our own node. - if local_listen_port != 0 - && observed_addr.port() != local_listen_port - { + if local_listen_port != 0 && observed_addr.port() != local_listen_port { tracing::debug!( "poll_discovery_task: SKIPPING observation {} from {} \ (port {} != listen port {})", @@ -4022,7 +4012,9 @@ impl NatTraversalEndpoint { let old_conn = if let Some(old) = connections2.get(&remote_address) { Some(old.value().clone()) } else { - connections2.get(&normalized_remote).map(|old| old.value().clone()) + connections2 + .get(&normalized_remote) + .map(|old| old.value().clone()) }; if let Some(old) = old_conn { tokio::spawn(async move { diff --git a/src/p2p_endpoint.rs b/src/p2p_endpoint.rs index 6eaf9029..511ee88f 100644 --- a/src/p2p_endpoint.rs +++ b/src/p2p_endpoint.rs @@ -73,8 +73,8 @@ use crate::happy_eyeballs::{self, HappyEyeballsConfig}; pub use crate::nat_traversal_api::TraversalPhase; use crate::nat_traversal_api::{NatTraversalEndpoint, NatTraversalError, NatTraversalEvent}; use crate::reachability::{ReachabilityScope, TraversalMethod, socket_addr_scope}; -use crate::transport::{ProtocolEngine, TransportAddr, TransportRegistry}; use crate::shared::normalize_socket_addr; +use crate::transport::{ProtocolEngine, TransportAddr, TransportRegistry}; use crate::unified_config::P2pConfig; use rustls;