From 60f0c823d69a26ad783bf14bbe737926404570c1 Mon Sep 17 00:00:00 2001 From: Michael Neale Date: Wed, 20 May 2026 15:51:41 +1000 Subject: [PATCH 1/2] fix(mesh): skip filtered peers in gossip dial loop to unwedge `--auto` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #576 added two gossip-ingest filters (version floor + idle-transitive client). They correctly reject ghost peers at ingest, but their reject path `state.peers.remove(&id) + return` interacts badly with the dial loop later in the same gossip exchange: ```text attempt_run_auto_join → join_with_retry → connect_to_peer(invite_peer) → initiate_gossip → gossip_round_trip (30s timeout wraps the WHOLE continuation including the dial loop below) → apply_gossip_announcements → apply_announced_peers (PR #576 filters remove ghosts here) → for each ann in their_announcements: maybe_connect_discovered_peer(addr) → connect_to_peer(addr) → if state.peers.contains_key(&peer_id) { return Ok(()) } → else: 30s connect timeout per unreachable host ``` `connect_to_peer`'s fast-path keys on `state.peers.contains_key`. PR #576's filter actively removes filtered peers from `state.peers` before the dial loop, so the fast-path no longer fires — and each unreachable ghost address gets a real 30s connect timeout, sequentially. With ~445 v0.57.x ghosts in a payload, that's hours of being stuck inside `initiate_gossip`, which means `attempt_run_auto_join` never returns and `run_auto` never reaches `LaunchPlan` / model load. ## Fix Apply the same filter inside `maybe_connect_discovered_peer`, before the dial. If a peer announcement would be rejected by the version-floor or idle-transitive-client filter at ingest, do not dial it from the discovery loop either. This is a single 18-line skip at the call site — no change to the ingest filters, no shared state with ingest beyond the predicates that are already public to the module. ## Validation Local 4-run reliability test on this branch (`serve --auto --model Qwen/Qwen2.5-3B-Instruct-GGUF:qwen2.5-3b-instruct-q4_k_m`): | Run | loaded | ready | peers | inference | |----:|--------|-------|------:|------------------| | 1 | ✅ | ✅ | 37 | "Surething" | | 2 | ✅ | ✅ | 43 | "Sure thing." | | 3 | ✅ | ✅ | 44 | "Sure thing." | | 4 | ✅ | ✅ | 43 | "Surething." | Peer table stays clean (~40 peers, all v0.60+, no v0.57.x ghosts) — the PR #576 ingest filters do their job. Last CKPT reached: `CKPT 9: after setup_run_auto_console_state` (full startup sequence completes). `cargo test -p mesh-llm-host-runtime --lib` — 1424/1424 pass, including PR #576's existing filter tests plus a new regression test that exercises the skip path: * `maybe_connect_discovered_peer_skips_filtered_announcements` — calls the dial entry with a below-floor announcement and an idle-transitive announcement, asserts both return well under the 30 s connect timeout and neither produces an entry in `state.connections` or `state.peers`. ## Supersedes This is the proper fix for the wedge described in PR #601, which disabled the ingest filters entirely as a short-term workaround. With this PR the filters stay on, the local peer table stays clean, and `--auto` is reliable. PR #601 can be closed once this lands. --- .../mesh-llm-host-runtime/src/mesh/gossip.rs | 87 +++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/crates/mesh-llm-host-runtime/src/mesh/gossip.rs b/crates/mesh-llm-host-runtime/src/mesh/gossip.rs index 8c6925836a..f672f4400e 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/gossip.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/gossip.rs @@ -365,6 +365,24 @@ impl Node { { return; } + // Skip dialing peers we would reject at ingest. Without this skip, + // `update_transitive_peer` removes the peer from `state.peers` (so + // `connect_to_peer`'s already-known fast-path does not fire), and we + // would spend 30 s per host walking through an entire payload of + // unreachable ghost addresses sequentially, wedging the surrounding + // gossip exchange. The same gates that filter the local table also + // gate the outbound dial here. + if !version_allowed_for_rebroadcast(ann.version.as_deref()) + || peer_is_idle_transitive_client(ann) + { + tracing::debug!( + "Skipping discovered peer {} (filtered: version={:?} role={:?})", + peer_id.fmt_short(), + ann.version, + ann.role + ); + return; + } if let Err(error) = Box::pin(self.connect_to_peer(addr)).await { if log_discovery_failure_as_warning { tracing::warn!("Failed to discover peer: {error}"); @@ -1642,4 +1660,73 @@ mod tests { "direct add of v0.57.0 peer must be rejected (no local state entry)" ); } + + /// Regression test for the `--auto` startup wedge: when a transitive + /// gossip payload includes peers that would be rejected at ingest + /// (version-floor or idle-transitive-client), `maybe_connect_discovered_peer` + /// must skip the dial. Otherwise each unreachable ghost address triggers + /// a 30 s `connect_to_peer` timeout sequentially in the dial loop, + /// wedging the surrounding gossip exchange (and the `attempt_run_auto_join` + /// that initiated it) for tens of minutes. + /// + /// The function returns without panicking and without dialing within a + /// generous time bound — a real dial to a fake address would block on + /// the 30 s `PEER_CONNECT_AND_GOSSIP_TIMEOUT`. We assert the result is + /// reached well under that bound and that no connection entry was created. + #[tokio::test] + async fn maybe_connect_discovered_peer_skips_filtered_announcements() { + let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); + let my_role = NodeRole::Worker; + + // Below-floor version — must be skipped without dialing. + let old_addr = test_addr(0x57); + let old_id = old_addr.id; + let mut old_ann = test_announcement(None); + old_ann.addr = old_addr.clone(); + old_ann.role = NodeRole::Client; + old_ann.version = Some("0.57.0".to_string()); + + // Idle transitive client (matching version, but no hostname / no + // direct measurement / no model interests) — must also be skipped. + let idle_addr = test_addr(0xC1); + let idle_id = idle_addr.id; + let mut idle_ann = test_announcement(None); + idle_ann.addr = idle_addr.clone(); + idle_ann.role = NodeRole::Client; + idle_ann.version = Some("0.65.1".to_string()); + + // Both calls together must return well under the 30 s connect + // timeout. If the dial-loop skip is missing, each call will block + // on PEER_CONNECT_AND_GOSSIP_TIMEOUT (30 s) attempting to dial the + // fake test address. + tokio::time::timeout(std::time::Duration::from_secs(5), async { + node.maybe_connect_discovered_peer(&my_role, old_addr, &old_ann, true, false) + .await; + node.maybe_connect_discovered_peer(&my_role, idle_addr, &idle_ann, true, false) + .await; + }) + .await + .expect("filtered peers must be skipped quickly, not dialed"); + + // No connection was attempted (no entry in state.connections), and + // no peer was added (the filtered announcements never reach add_peer + // or update_transitive_peer through this path). + let state = node.state.lock().await; + assert!( + !state.connections.contains_key(&old_id), + "below-floor peer must not be dialed" + ); + assert!( + !state.connections.contains_key(&idle_id), + "idle transitive client must not be dialed" + ); + assert!( + !state.peers.contains_key(&old_id), + "below-floor peer must not be added (this path is dial-only)" + ); + assert!( + !state.peers.contains_key(&idle_id), + "idle transitive client must not be added (this path is dial-only)" + ); + } } From eaba33d125dede8d9e74926e5499c3eceae0621d Mon Sep 17 00:00:00 2001 From: Michael Neale Date: Wed, 20 May 2026 15:57:51 +1000 Subject: [PATCH 2/2] refactor(mesh): extract dial-loop filter into helper for cognitive complexity The previous commit pushed `maybe_connect_discovered_peer` to a cognitive complexity of 27 (limit 20). Extract the filter check into a small associated helper `discovered_peer_is_filtered` so the call site stays compact and clippy is happy with `-D warnings`. No behavior change. `cargo test -p mesh-llm-host-runtime --lib gossip::` \u2014 24/24 pass. `cargo clippy -p mesh-llm-host-runtime --all-targets -- -D warnings` clean. --- .../mesh-llm-host-runtime/src/mesh/gossip.rs | 39 ++++++++++--------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/crates/mesh-llm-host-runtime/src/mesh/gossip.rs b/crates/mesh-llm-host-runtime/src/mesh/gossip.rs index f672f4400e..3f72e87611 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/gossip.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/gossip.rs @@ -362,27 +362,10 @@ impl Node { || self .discovered_peer_already_known(peer_id, known_peer_check_uses_connections) .await + || Self::discovered_peer_is_filtered(peer_id, ann) { return; } - // Skip dialing peers we would reject at ingest. Without this skip, - // `update_transitive_peer` removes the peer from `state.peers` (so - // `connect_to_peer`'s already-known fast-path does not fire), and we - // would spend 30 s per host walking through an entire payload of - // unreachable ghost addresses sequentially, wedging the surrounding - // gossip exchange. The same gates that filter the local table also - // gate the outbound dial here. - if !version_allowed_for_rebroadcast(ann.version.as_deref()) - || peer_is_idle_transitive_client(ann) - { - tracing::debug!( - "Skipping discovered peer {} (filtered: version={:?} role={:?})", - peer_id.fmt_short(), - ann.version, - ann.role - ); - return; - } if let Err(error) = Box::pin(self.connect_to_peer(addr)).await { if log_discovery_failure_as_warning { tracing::warn!("Failed to discover peer: {error}"); @@ -395,6 +378,26 @@ impl Node { } } + /// Returns `true` if the announcement would be rejected by the same + /// gates that filter ingest. Skipping the dial here avoids spending + /// 30s per host walking through unreachable ghost addresses + /// sequentially in the gossip exchange dial loop — the wedge that + /// caused `--auto` startup to hang. + fn discovered_peer_is_filtered(peer_id: EndpointId, ann: &PeerAnnouncement) -> bool { + if !version_allowed_for_rebroadcast(ann.version.as_deref()) + || peer_is_idle_transitive_client(ann) + { + tracing::debug!( + "Skipping discovered peer {} (filtered: version={:?} role={:?})", + peer_id.fmt_short(), + ann.version, + ann.role + ); + return true; + } + false + } + fn should_skip_discovered_peer( &self, my_role: &super::NodeRole,