diff --git a/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/mod.rs b/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/mod.rs index c2d0c389a3..d7a7086618 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/mod.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/mod.rs @@ -458,6 +458,15 @@ pub async fn build_moa_config( // is the dominant chat path, so a tighter default is the right // default. first_answer_grace: std::time::Duration::from_secs(3), + // Tier-gate patience: how long small-tier-only answers/consensus + // are held when a big-tier strong worker (e.g. MiniMax) is still + // running. 20s covers the strong worker's typical first-token + // latency on agent-sized prompts over the public mesh without + // approaching worker_timeout (60s). Hard-bounded: at expiry all + // decision rules revert to ungated behavior. Same-tier pools are + // unaffected, so "many small models lift each other" keeps its + // current latency profile. + strong_patience: std::time::Duration::from_secs(20), // Defaults to leaving each model's thinking behavior alone. // `try_handle_moa` overrides this from the inbound request body // when the caller has expressed a preference diff --git a/crates/mesh-mixture-of-agents/src/arbiter.rs b/crates/mesh-mixture-of-agents/src/arbiter.rs index 498704dbb6..70c26d9e82 100644 --- a/crates/mesh-mixture-of-agents/src/arbiter.rs +++ b/crates/mesh-mixture-of-agents/src/arbiter.rs @@ -11,6 +11,7 @@ //! 5. All uncertainty → escalate to reducer use crate::normalize::{OutputKind, WorkerOutput}; +use crate::worker::WorkerRole; use serde_json::Value; /// Pick the best tool proposal: prefer proposals that have arguments, @@ -176,6 +177,25 @@ pub fn arbitrate(outputs: &[WorkerOutput], has_tools: bool) -> Decision { } } +/// Tier-gap gating for early decisions. +/// +/// When the worker pool mixes a big-tier Strong worker with small-tier +/// workers (the "MiniMax + small Qwens" shape), small-tier-only +/// consensus must not finalize *against* the strong worker while it is +/// still running — two fast small models agreeing on a wrong answer +/// would otherwise outvote the model most likely to be right. The +/// fan-out loop bounds how long the gate can hold via `strong_patience`, +/// so this never becomes an unbounded wait. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum StrongGate { + /// No quality gap, patience disabled, or patience expired: decide + /// on count and confidence alone (previous behavior). + Off, + /// Quality gap active. `strong_pending` is true while the Strong + /// worker has not yet finished (succeeded or failed). + Active { strong_pending: bool }, +} + /// Try to decide early with a partial set of worker outputs. /// /// Returns `Some(decision)` if we can confidently resolve without waiting @@ -189,6 +209,7 @@ pub fn try_early_decision( total_workers: usize, total_finished: usize, has_tools: bool, + strong_gate: StrongGate, ) -> Option { if outputs.is_empty() { // All workers finished but none succeeded @@ -199,6 +220,12 @@ pub fn try_early_decision( } let remaining = total_workers.saturating_sub(total_finished); + let strong_pending = matches!( + strong_gate, + StrongGate::Active { + strong_pending: true + } + ); // ── Only one worker will ever respond ─────────────────────────── // If we have 1 successful output and no more workers are coming, @@ -217,6 +244,17 @@ pub fn try_early_decision( if !majority_failed { return None; } + // Tier gate: a small-tier sole-survivor *answer* must not + // finalize while the strong worker is still running. The + // fan-out loop bounds this wait via `strong_patience`. Tool + // proposals are exempt — they are schema-verified and holding + // them would slow agent loops. + if strong_pending + && outputs[0].role != WorkerRole::Strong + && outputs[0].kind == OutputKind::Answer + { + return None; + } // Majority failed — return the sole survivor tracing::info!( "moa: early exit — sole survivor, {failed_count}/{total_workers} workers failed", @@ -244,15 +282,14 @@ pub fn try_early_decision( }; if let Some((cluster_size, best)) = agreeing_cluster { let majority = cluster_size * 2 >= answers.len(); - if majority && best.confidence >= 0.5 { - tracing::info!( - "moa: early exit — {}/{} workers agree on answer (conf={:.2}), {} still pending", - cluster_size, - answers.len(), - best.confidence, - remaining, - ); - return Some(Decision::Answer(best.payload.clone())); + let qualifies = majority && best.confidence >= 0.5; + if let Some(decision) = qualifies + .then(|| { + tier_aware_consensus_decision(&answers, cluster_size, best, strong_gate, remaining) + }) + .flatten() + { + return Some(decision); } } @@ -302,6 +339,75 @@ pub fn try_early_decision( None } +/// Resolve a majority answer cluster against the strong-worker tier gate. +/// +/// Consensus that includes the strong worker's answer passes through — +/// that is agreement *with* the strong model, not against it. Small-tier- +/// only consensus is held back (returns `None`) while the strong worker +/// is still pending so the model most likely to be right gets to weigh +/// in. The fan-out loop bounds how long the hold can last. +fn tier_aware_consensus_decision( + answers: &[&WorkerOutput], + cluster_size: usize, + best: &WorkerOutput, + strong_gate: StrongGate, + remaining: usize, +) -> Option { + let strong_pending = matches!( + strong_gate, + StrongGate::Active { + strong_pending: true + } + ); + let strong_answer = answers + .iter() + .find(|a| a.role == WorkerRole::Strong && is_usable_answer(a)); + if strong_pending && strong_answer.is_none() { + tracing::info!( + "moa: consensus held — {}/{} small-tier workers agree but strong worker \ + still pending (patience window active)", + cluster_size, + answers.len(), + ); + return None; + } + + // Strong landed but does NOT share the small-tier cluster: prefer the + // strong worker's answer over small-model consensus. Holding for the + // strong worker only buys it a seat; this is what makes its answer + // actually win on disagreement. When the strong worker IS in the + // cluster (its content agrees with the representative under the same + // bidirectional-subset rule the clusterer uses), the cluster + // representative already reflects its content, so ship that. + if let Some(strong) = strong_answer { + let strong_tokens = content_tokens(&strong.payload); + let best_tokens = content_tokens(&best.payload); + let in_cluster = !strong_tokens.is_empty() + && !best_tokens.is_empty() + && !has_negation_mismatch(&strong_tokens, &best_tokens) + && (strong_tokens.is_subset(&best_tokens) || best_tokens.is_subset(&strong_tokens)); + if !in_cluster { + tracing::info!( + "moa: strong worker dissents from {}/{} small-tier consensus — preferring \ + strong answer (conf={:.2})", + cluster_size, + answers.len(), + strong.confidence, + ); + return Some(Decision::Answer(strong.payload.clone())); + } + } + + tracing::info!( + "moa: early exit — {}/{} workers agree on answer (conf={:.2}), {} still pending", + cluster_size, + answers.len(), + best.confidence, + remaining, + ); + Some(Decision::Answer(best.payload.clone())) +} + /// Tokens that flip meaning. Always preserved as content (even if they'd /// fail the length / stopword filters), AND a leftover negation in either /// side of a comparison blocks clustering regardless of subset relation. @@ -482,7 +588,6 @@ fn is_usable_answer(output: &WorkerOutput) -> bool { #[cfg(test)] mod tests { use super::*; - use crate::worker::WorkerRole; fn make_output(kind: OutputKind, confidence: f32, payload: &str) -> WorkerOutput { WorkerOutput { @@ -564,7 +669,7 @@ mod tests { fn early_decision_none_with_one_of_three() { let outputs = vec![make_output(OutputKind::Answer, 0.9, "Paris")]; // 1 of 3 — too early to decide - assert!(try_early_decision(&outputs, 3, outputs.len(), false).is_none()); + assert!(try_early_decision(&outputs, 3, outputs.len(), false, StrongGate::Off).is_none()); } #[test] @@ -574,7 +679,7 @@ mod tests { make_output(OutputKind::Answer, 0.9, "Paris is the capital"), ]; // 2 of 3 agree — early exit - match try_early_decision(&outputs, 3, outputs.len(), false) { + match try_early_decision(&outputs, 3, outputs.len(), false, StrongGate::Off) { Some(Decision::Answer(text)) => assert!(text.contains("Paris")), other => panic!("expected early Answer, got {other:?}"), } @@ -586,7 +691,7 @@ mod tests { make_tool_output(0.8, "read_file", serde_json::json!({"path": "a.rs"})), make_tool_output(0.7, "read_file", serde_json::json!({"path": "a.rs"})), ]; - match try_early_decision(&outputs, 3, outputs.len(), true) { + match try_early_decision(&outputs, 3, outputs.len(), true, StrongGate::Off) { Some(Decision::ToolCall { name, .. }) => assert_eq!(name, "read_file"), other => panic!("expected early ToolCall, got {other:?}"), } @@ -598,7 +703,7 @@ mod tests { make_tool_output(0.7, "read_file", serde_json::json!({})), make_output(OutputKind::Answer, 0.8, "I know the answer"), ]; - match try_early_decision(&outputs, 3, outputs.len(), true) { + match try_early_decision(&outputs, 3, outputs.len(), true, StrongGate::Off) { Some(Decision::NeedsReducer { .. }) => {} other => panic!("expected early NeedsReducer, got {other:?}"), } @@ -614,7 +719,7 @@ mod tests { make_output(OutputKind::Answer, 0.9, "The capital of France is Paris"), make_output(OutputKind::Answer, 0.8, "The capital of France is Berlin"), ]; - let res = try_early_decision(&outputs, 3, outputs.len(), false); + let res = try_early_decision(&outputs, 3, outputs.len(), false, StrongGate::Off); assert!( res.is_none(), "disagreeing answers should not trigger early-exit, got {res:?}" @@ -630,7 +735,7 @@ mod tests { make_output(OutputKind::Answer, 0.7, "Paris"), make_output(OutputKind::Answer, 0.9, "Paris is the capital of France"), ]; - match try_early_decision(&outputs, 3, outputs.len(), false) { + match try_early_decision(&outputs, 3, outputs.len(), false, StrongGate::Off) { // Representative picks the most complete (most tokens) member. Some(Decision::Answer(text)) => { let lower = text.to_lowercase(); @@ -653,7 +758,7 @@ mod tests { make_output(OutputKind::Answer, 0.8, "Paris is the capital"), make_output(OutputKind::Answer, 0.6, "I think it's Lyon"), ]; - match try_early_decision(&outputs, 4, outputs.len(), false) { + match try_early_decision(&outputs, 4, outputs.len(), false, StrongGate::Off) { Some(Decision::Answer(text)) => assert!( text.to_lowercase().contains("paris"), "should pick from the agreeing cluster, got {text:?}" @@ -673,7 +778,7 @@ mod tests { make_output(OutputKind::Answer, 0.9, "The capital of France is Berlin"), make_output(OutputKind::Answer, 0.5, "The capital of France is Madrid"), ]; - let res = try_early_decision(&outputs, 4, outputs.len(), false); + let res = try_early_decision(&outputs, 4, outputs.len(), false, StrongGate::Off); assert!( res.is_none(), "three disagreeing answers should not early-exit, got {res:?}" @@ -689,7 +794,7 @@ mod tests { make_output(OutputKind::Answer, 0.9, "You should use grep"), make_output(OutputKind::Answer, 0.8, "You should not use grep"), ]; - let res = try_early_decision(&outputs, 3, outputs.len(), false); + let res = try_early_decision(&outputs, 3, outputs.len(), false, StrongGate::Off); assert!( res.is_none(), "affirmative vs negated answer should not cluster, got {res:?}" @@ -704,7 +809,7 @@ mod tests { make_output(OutputKind::Answer, 0.9, "Do that"), make_output(OutputKind::Answer, 0.8, "Don't do that"), ]; - let res = try_early_decision(&outputs, 3, outputs.len(), false); + let res = try_early_decision(&outputs, 3, outputs.len(), false, StrongGate::Off); assert!( res.is_none(), "affirmative vs negated should not cluster, got {res:?}" @@ -718,7 +823,7 @@ mod tests { make_output(OutputKind::Answer, 0.9, "42"), make_output(OutputKind::Answer, 0.8, "The answer is 42"), ]; - match try_early_decision(&outputs, 3, outputs.len(), false) { + match try_early_decision(&outputs, 3, outputs.len(), false, StrongGate::Off) { Some(Decision::Answer(text)) => assert!(text.contains("42")), other => panic!("expected numeric agreement, got {other:?}"), } @@ -729,7 +834,7 @@ mod tests { // 1 success out of 3, other 2 failed — should return the single answer let outputs = vec![make_output(OutputKind::Answer, 0.8, "Paris")]; // total_workers=3, total_finished=3 (1 success + 2 failures), remaining=0 - match try_early_decision(&outputs, 3, 3, false) { + match try_early_decision(&outputs, 3, 3, false, StrongGate::Off) { Some(Decision::Answer(text)) => assert!(text.contains("Paris")), other => panic!("expected early Answer for sole survivor, got {other:?}"), } @@ -742,7 +847,7 @@ mod tests { make_output(OutputKind::Answer, 0.4, "could be Paris"), ]; // Both answers but low confidence — should wait for more - assert!(try_early_decision(&outputs, 3, outputs.len(), false).is_none()); + assert!(try_early_decision(&outputs, 3, outputs.len(), false, StrongGate::Off).is_none()); } #[test] @@ -785,7 +890,7 @@ mod tests { // 1 success, 3 failures, 1 still pending — majority failed, return sole survivor let outputs = vec![make_output(OutputKind::Answer, 0.8, "Paris")]; // total_workers=5, total_finished=4 (1 success + 3 failures), remaining=1 - match try_early_decision(&outputs, 5, 4, false) { + match try_early_decision(&outputs, 5, 4, false, StrongGate::Off) { Some(Decision::Answer(text)) => assert!(text.contains("Paris")), other => { panic!("expected early Answer for sole survivor (majority failed), got {other:?}") @@ -798,7 +903,7 @@ mod tests { // 1 success, 1 failure, 3 still pending — minority failed, wait for more let outputs = vec![make_output(OutputKind::Answer, 0.8, "Paris")]; // total_workers=5, total_finished=2 (1 success + 1 failure), remaining=3 - assert!(try_early_decision(&outputs, 5, 2, false).is_none()); + assert!(try_early_decision(&outputs, 5, 2, false, StrongGate::Off).is_none()); } #[test] @@ -856,4 +961,146 @@ mod tests { // Both have args, so confidence wins assert_eq!(best.model, "model-b"); } + + // ── Tier gate (StrongGate) ─────────────────────────────────────── + + fn make_role_output( + kind: OutputKind, + confidence: f32, + payload: &str, + role: WorkerRole, + ) -> WorkerOutput { + WorkerOutput { + role, + ..make_output(kind, confidence, payload) + } + } + + const GATE_PENDING: StrongGate = StrongGate::Active { + strong_pending: true, + }; + + #[test] + fn gate_holds_small_tier_consensus_while_strong_pending() { + // Two small-tier workers agree — without the gate this would + // early-exit. With the strong worker still running, hold. + let outputs = vec![ + make_role_output(OutputKind::Answer, 0.8, "Paris", WorkerRole::Fast), + make_role_output( + OutputKind::Answer, + 0.9, + "Paris is the capital", + WorkerRole::Specialist, + ), + ]; + assert!( + try_early_decision(&outputs, 3, outputs.len(), false, GATE_PENDING).is_none(), + "small-tier consensus must be held while the strong worker is pending" + ); + // Same outputs, gate off → previous behavior (early exit). + assert!( + try_early_decision(&outputs, 3, outputs.len(), false, StrongGate::Off).is_some(), + "gate off must preserve pre-gate early-exit behavior" + ); + } + + #[test] + fn gate_passes_consensus_that_includes_strong_worker() { + // Strong worker has answered and agrees — that's agreement WITH + // the strong model. Ship it even though another worker is pending. + let outputs = vec![ + make_role_output(OutputKind::Answer, 0.8, "Paris", WorkerRole::Fast), + make_role_output( + OutputKind::Answer, + 0.9, + "Paris is the capital", + WorkerRole::Strong, + ), + ]; + let gate = StrongGate::Active { + strong_pending: false, + }; + match try_early_decision(&outputs, 3, outputs.len(), false, gate) { + Some(Decision::Answer(text)) => assert!(text.contains("Paris")), + other => panic!("expected early Answer with strong agreement, got {other:?}"), + } + } + + #[test] + fn gate_holds_small_tier_sole_survivor_answer() { + // Majority failed, sole survivor is a small-tier Answer, strong + // still pending → hold (the fan-out patience timer bounds this). + let outputs = vec![make_role_output( + OutputKind::Answer, + 0.9, + "Tokyo", + WorkerRole::Fast, + )]; + // 3 dispatched, 2 finished (1 ok + 1 failed) → majority_failed + assert!( + try_early_decision(&outputs, 3, 2, false, GATE_PENDING).is_none(), + "small-tier sole-survivor answer must be held while strong is pending" + ); + // Gate off → pre-gate behavior: sole survivor ships. + assert!(try_early_decision(&outputs, 3, 2, false, StrongGate::Off).is_some()); + } + + #[test] + fn gate_does_not_hold_tool_proposals() { + // Tool proposals are schema-verified and exempt from the gate — + // agent loops must stay snappy. + let outputs = vec![WorkerOutput { + role: WorkerRole::Fast, + ..make_tool_output(0.9, "read_file", serde_json::json!({"path": "x"})) + }]; + // Majority failed → sole-survivor path, but it's a ToolProposal. + match try_early_decision(&outputs, 3, 2, true, GATE_PENDING) { + Some(Decision::ToolCall { name, .. }) => assert_eq!(name, "read_file"), + other => panic!("tool proposals must not be gated, got {other:?}"), + } + } + + #[test] + fn strong_dissent_wins_over_small_consensus() { + // Two small workers agree on "Sydney"; the strong worker landed + // with a different answer ("Canberra"). Gate no longer pending. + // The strong worker's answer must win, not the small consensus. + let outputs = vec![ + make_role_output(OutputKind::Answer, 0.9, "Sydney", WorkerRole::Fast), + make_role_output(OutputKind::Answer, 0.9, "Sydney", WorkerRole::Specialist), + make_role_output(OutputKind::Answer, 0.7, "Canberra", WorkerRole::Strong), + ]; + let gate = StrongGate::Active { + strong_pending: false, + }; + match try_early_decision(&outputs, 3, outputs.len(), false, gate) { + Some(Decision::Answer(text)) => assert!( + text.contains("Canberra"), + "strong dissent must win over small consensus, got {text:?}" + ), + other => panic!("expected strong's Answer, got {other:?}"), + } + } + + #[test] + fn gate_releases_when_strong_finished() { + // Strong worker finished (failed or succeeded without usable + // answer) — gate no longer pending, consensus ships. + let outputs = vec![ + make_role_output(OutputKind::Answer, 0.8, "Paris", WorkerRole::Fast), + make_role_output( + OutputKind::Answer, + 0.9, + "Paris is the capital", + WorkerRole::Specialist, + ), + ]; + let gate = StrongGate::Active { + strong_pending: false, + }; + assert!( + try_early_decision(&outputs, 4, outputs.len(), false, gate).is_some(), + "consensus must ship once the strong worker has finished" + ); + } } diff --git a/crates/mesh-mixture-of-agents/src/fanout.rs b/crates/mesh-mixture-of-agents/src/fanout.rs index d9bfb5fba4..32251934fc 100644 --- a/crates/mesh-mixture-of-agents/src/fanout.rs +++ b/crates/mesh-mixture-of-agents/src/fanout.rs @@ -26,6 +26,18 @@ pub(crate) enum GraceMode { Tool, } +/// Time-based decision policy for a single fan-out: how long to wait +/// before shipping a partial result, and how long to hold small-tier +/// decisions for a pending big-tier strong worker. +#[derive(Clone, Copy, Debug)] +pub(crate) struct GatherPolicy { + /// See [`crate::GatewayConfig::first_answer_grace`]. + pub first_answer_grace: Duration, + pub grace_mode: GraceMode, + /// See [`crate::GatewayConfig::strong_patience`]. + pub strong_patience: Duration, +} + /// Identifier for a worker we dispatched. Used to reconcile the /// per-worker accounting at the end of fan-out so the — possibly /// aborted or panicked — task's existence still shows up in @@ -41,13 +53,17 @@ pub(crate) async fn gather_workers_incremental( has_tools: bool, allowed_tools: &[String], tools: Option<&Value>, - first_answer_grace: Duration, - grace_mode: GraceMode, + policy: GatherPolicy, ) -> ( Vec, Vec, Option, ) { + let GatherPolicy { + first_answer_grace, + grace_mode, + strong_patience, + } = policy; let total_workers = dispatched.len(); let mut outputs = Vec::new(); let mut summaries = Vec::new(); @@ -55,6 +71,25 @@ pub(crate) async fn gather_workers_incremental( let dispatched_at = Instant::now(); let grace_enabled = grace_mode != GraceMode::Disabled && !first_answer_grace.is_zero(); + // Tier gate: when the pool mixes a big-tier Strong worker with + // small-tier workers, small-tier-only consensus is held until the + // strong worker finishes OR the patience window expires. The window + // is a hard bound — when it lapses the gate switches off entirely + // and every decision rule reverts to pre-gate behavior, so a stuck + // strong worker can never hold the turn hostage (the failure mode + // that sank PR #820). + let mut strong_finished = false; + let gate_enabled = !strong_patience.is_zero() + && crate::worker::has_quality_gap(dispatched.iter().map(|d| (d.model.as_str(), d.role))); + let strong_gate = |strong_finished: bool, elapsed: Duration| -> arbiter::StrongGate { + if !gate_enabled || elapsed >= strong_patience { + return arbiter::StrongGate::Off; + } + arbiter::StrongGate::Active { + strong_pending: !strong_finished, + } + }; + // Grace eligibility: once the grace window has elapsed and a qualifying // partial decision exists, ship it instead of waiting for the slow tail. // Answer grace handles ordinary chat, including tool-enabled clients that @@ -83,7 +118,23 @@ pub(crate) async fn gather_workers_incremental( } else { Duration::from_secs(60 * 60) }; - let armed = grace_eligible(&outputs); + // While the tier gate is actively holding (strong worker pending, + // patience not yet expired), arm a wake-up at patience expiry so a + // held consensus decision is re-evaluated even if no further worker + // event arrives. Without this, a stuck strong worker would mean the + // next re-check only happens at worker_timeout. + let gate_holding = gate_enabled + && !strong_finished + && dispatched_at.elapsed() < strong_patience + && !outputs.is_empty(); + let patience_remaining = strong_patience.saturating_sub(dispatched_at.elapsed()); + + // The answer grace is also tier-gated: a single small-tier answer + // must not ship at grace expiry while the strong worker is still + // inside its patience window — that was the dominant path where a + // small model's answer pre-empted the strong one. Tool grace is + // exempt (schema-verified proposals; agent loops stay snappy). + let armed = grace_eligible(&outputs) && !(gate_holding && grace_mode == GraceMode::Answer); let join_result = tokio::select! { biased; @@ -104,6 +155,26 @@ pub(crate) async fn gather_workers_incremental( reconcile_dispatched(dispatched, &mut summaries); return (outputs, summaries, Some(decision)); } + _ = tokio::time::sleep(patience_remaining), if gate_holding => { + tracing::info!( + "moa: strong patience expired after {}ms — re-evaluating held outputs", + dispatched_at.elapsed().as_millis(), + ); + // Gate is now Off (elapsed >= strong_patience); re-run the + // early-decision check over what we already have. + if let Some(decision) = arbiter::try_early_decision( + &outputs, + total_workers, + total_finished, + has_tools, + arbiter::StrongGate::Off, + ) { + drain_after_early_exit(join_set, &mut summaries).await; + reconcile_dispatched(dispatched, &mut summaries); + return (outputs, summaries, Some(decision)); + } + continue; + } }; let Some(join_result) = join_result else { @@ -113,6 +184,9 @@ pub(crate) async fn gather_workers_incremental( match join_result { Ok((model, role, Ok(text), elapsed)) => { total_finished += 1; + if role == WorkerRole::Strong { + strong_finished = true; + } let mut normalized = normalize::normalize_worker_output(&text, &model, role, elapsed); enforce_tool_call_contract(&mut normalized, allowed_tools, tools, &model); @@ -135,9 +209,13 @@ pub(crate) async fn gather_workers_incremental( }); outputs.push(normalized); - if let Some(decision) = - arbiter::try_early_decision(&outputs, total_workers, total_finished, has_tools) - { + if let Some(decision) = arbiter::try_early_decision( + &outputs, + total_workers, + total_finished, + has_tools, + strong_gate(strong_finished, dispatched_at.elapsed()), + ) { drain_after_early_exit(join_set, &mut summaries).await; reconcile_dispatched(dispatched, &mut summaries); return (outputs, summaries, Some(decision)); @@ -145,6 +223,9 @@ pub(crate) async fn gather_workers_incremental( } Ok((model, role, Err(e), elapsed)) => { total_finished += 1; + if role == WorkerRole::Strong { + strong_finished = true; + } tracing::warn!( "moa: worker {} ({}) failed after {}ms: {}", model, @@ -161,9 +242,13 @@ pub(crate) async fn gather_workers_incremental( confidence: None, }); - if let Some(decision) = - arbiter::try_early_decision(&outputs, total_workers, total_finished, has_tools) - { + if let Some(decision) = arbiter::try_early_decision( + &outputs, + total_workers, + total_finished, + has_tools, + strong_gate(strong_finished, dispatched_at.elapsed()), + ) { drain_after_early_exit(join_set, &mut summaries).await; reconcile_dispatched(dispatched, &mut summaries); return (outputs, summaries, Some(decision)); @@ -173,9 +258,15 @@ pub(crate) async fn gather_workers_incremental( total_finished += 1; tracing::warn!("moa: worker task panicked or was cancelled: {e}"); // No (model, role) payload available from a JoinError, so - // we cannot attribute this slot here. `reconcile_dispatched` - // at the end picks up any dispatched worker that has not - // produced a summary by name. + // we cannot attribute this slot here — including whether it + // was the Strong worker. If a panicking Strong leaves + // `strong_finished` false, the tier gate simply holds until + // `strong_patience` expires (its bounded fallback) rather + // than releasing immediately. Panicking workers are rare and + // the worst case is one extra patience window of latency, so + // we don't add fragile JoinError↔slot correlation to shave + // it. `reconcile_dispatched` still attributes the slot by + // name at the end for accounting. } } } @@ -185,15 +276,27 @@ pub(crate) async fn gather_workers_incremental( } fn grace_answer_decision(outputs: &[WorkerOutput]) -> arbiter::Decision { - let answer = outputs - .iter() - .filter(|o| o.kind == normalize::OutputKind::Answer) - .max_by(|a, b| { - a.confidence - .partial_cmp(&b.confidence) - .unwrap_or(std::cmp::Ordering::Equal) - }) - .expect("answer grace requires at least one Answer"); + // Prefer the Strong worker's qualifying answer when it has landed: + // if the biggest model already answered, shipping a smaller model's + // marginally-higher-confidence answer instead would defeat the point + // of waiting for it. Confidence is self-reported and not comparable + // across models, so role is the better tie-breaker here. + let strong = outputs.iter().find(|o| { + o.kind == normalize::OutputKind::Answer + && o.role == WorkerRole::Strong + && o.confidence >= GRACE_MIN_CONFIDENCE + }); + let answer = strong.unwrap_or_else(|| { + outputs + .iter() + .filter(|o| o.kind == normalize::OutputKind::Answer) + .max_by(|a, b| { + a.confidence + .partial_cmp(&b.confidence) + .unwrap_or(std::cmp::Ordering::Equal) + }) + .expect("answer grace requires at least one Answer") + }); let answer_count = outputs .iter() .filter(|o| o.kind == normalize::OutputKind::Answer) @@ -370,8 +473,11 @@ mod tests { false, // has_tools &[], None, - Duration::from_millis(50), - GraceMode::Answer, + GatherPolicy { + first_answer_grace: Duration::from_millis(50), + grace_mode: GraceMode::Answer, + strong_patience: Duration::ZERO, + }, ) .await; let elapsed = started.elapsed(); @@ -423,8 +529,11 @@ mod tests { true, // has_tools &[], None, - Duration::from_millis(50), - GraceMode::Answer, + GatherPolicy { + first_answer_grace: Duration::from_millis(50), + grace_mode: GraceMode::Answer, + strong_patience: Duration::ZERO, + }, ) .await; let elapsed = started.elapsed(); @@ -471,8 +580,11 @@ mod tests { true, &[], None, - Duration::from_millis(50), - GraceMode::Disabled, + GatherPolicy { + first_answer_grace: Duration::from_millis(50), + grace_mode: GraceMode::Disabled, + strong_patience: Duration::ZERO, + }, ) .await; let elapsed = started.elapsed(); @@ -511,8 +623,11 @@ mod tests { true, &["read".to_string()], None, - Duration::from_millis(50), - GraceMode::Tool, + GatherPolicy { + first_answer_grace: Duration::from_millis(50), + grace_mode: GraceMode::Tool, + strong_patience: Duration::ZERO, + }, ) .await; let elapsed = started.elapsed(); @@ -566,8 +681,11 @@ mod tests { false, // has_tools &[], None, - Duration::ZERO, - GraceMode::Answer, + GatherPolicy { + first_answer_grace: Duration::ZERO, + grace_mode: GraceMode::Answer, + strong_patience: Duration::ZERO, + }, ) .await; let elapsed = started.elapsed(); @@ -614,8 +732,11 @@ mod tests { false, &[], None, - Duration::from_millis(50), - GraceMode::Answer, + GatherPolicy { + first_answer_grace: Duration::from_millis(50), + grace_mode: GraceMode::Answer, + strong_patience: Duration::ZERO, + }, ) .await; let elapsed = started.elapsed(); @@ -667,8 +788,11 @@ mod tests { false, &[], None, - Duration::from_millis(50), - GraceMode::Answer, + GatherPolicy { + first_answer_grace: Duration::from_millis(50), + grace_mode: GraceMode::Answer, + strong_patience: Duration::ZERO, + }, ) .await; let elapsed = started.elapsed(); @@ -728,8 +852,11 @@ mod tests { false, &[], None, - Duration::from_millis(100), - GraceMode::Answer, + GatherPolicy { + first_answer_grace: Duration::from_millis(100), + grace_mode: GraceMode::Answer, + strong_patience: Duration::ZERO, + }, ) .await; let decision = decision.expect("grace must yield a Decision"); diff --git a/crates/mesh-mixture-of-agents/src/lib.rs b/crates/mesh-mixture-of-agents/src/lib.rs index 96bbd6fad0..ec1ef3a2b0 100644 --- a/crates/mesh-mixture-of-agents/src/lib.rs +++ b/crates/mesh-mixture-of-agents/src/lib.rs @@ -86,6 +86,15 @@ pub struct GatewayConfig { /// (conf >= 0.5) is in, accept it instead of waiting for consensus. /// Disabled for tool turns. Zero disables entirely. pub first_answer_grace: Duration, + /// Tier-gate patience: when the worker pool mixes a big-tier Strong + /// worker with small-tier workers, small-tier-only answers and + /// consensus are held for up to this long after dispatch to give the + /// strong worker a chance to weigh in. A hard bound — once it lapses, + /// all decision rules revert to ungated behavior, so a stuck strong + /// worker can never hold the turn hostage. Zero disables the gate. + /// Has no effect when all workers are the same tier. Tool proposals + /// are never held. + pub strong_patience: Duration, /// Override for whether reasoning workers should think. Propagated to /// every worker and the reducer as `chat_template_kwargs.enable_thinking` /// (and `reasoning_effort: "none"` when disabled). @@ -297,8 +306,11 @@ async fn handle_query( query_uses_tools, allowed_tools, session.tools(), - config.first_answer_grace, - grace_mode, + fanout::GatherPolicy { + first_answer_grace: config.first_answer_grace, + grace_mode, + strong_patience: config.strong_patience, + }, ) .await; @@ -1531,6 +1543,7 @@ mod response_builder_tests { reducer_timeout: Duration::from_secs(1), hedge_delay: Duration::from_millis(10), first_answer_grace: Duration::from_millis(10), + strong_patience: Duration::ZERO, enable_thinking: Some(false), }; let forced_tool = ForcedToolChoice { diff --git a/crates/mesh-mixture-of-agents/src/worker.rs b/crates/mesh-mixture-of-agents/src/worker.rs index d9bf78c481..28ed566e0c 100644 --- a/crates/mesh-mixture-of-agents/src/worker.rs +++ b/crates/mesh-mixture-of-agents/src/worker.rs @@ -99,6 +99,28 @@ pub fn assign_roles(models: &[ModelEntry]) -> Vec { assignments } +/// Does this worker pool have a real quality gap? +/// +/// Takes `(model_name, role)` pairs. True when the Strong-role worker +/// is big-tier (multi-digit-B or no advertised size) AND at least one +/// other worker is small-tier (single-digit-B). This is the "MiniMax + +/// small Qwens" shape where mixing tiers can pull answers down. When +/// all workers are the same tier (e.g. several small models lifting +/// each other via consensus) there is no gap and tier-aware patience +/// stays disabled. +pub fn has_quality_gap<'a>(workers: impl IntoIterator) -> bool { + let mut strong_is_big = false; + let mut any_small_non_strong = false; + for (name, role) in workers { + if role == WorkerRole::Strong { + strong_is_big = !is_single_digit_b_name(name); + } else if is_single_digit_b_name(name) { + any_small_non_strong = true; + } + } + strong_is_big && any_small_non_strong +} + /// Return true if `name` advertises a single-digit billion-parameter /// count, e.g. "Qwen3.5-2B-Q4_K_M" or "llama-3-7b-instruct". /// @@ -389,4 +411,42 @@ mod tests { assert_eq!(strip_thinking("思答案"), "答案"); assert_eq!(strip_thinking("前置中间隐"), "前置中间"); } + + #[test] + fn quality_gap_minimax_plus_small_qwens() { + // The motivating shape: big-tier strong + small-tier workers. + let workers = [ + ("Qwen2.5-3B-Instruct", WorkerRole::Fast), + ("Qwen3-8B", WorkerRole::Specialist), + ("MiniMax-M2.5", WorkerRole::Strong), + ]; + assert!(has_quality_gap(workers.iter().copied())); + } + + #[test] + fn no_quality_gap_when_all_small_tier() { + // "Many small models lift each other" — gate must stay off so + // same-tier consensus keeps its current latency profile. + let workers = [ + ("Qwen2.5-3B-Instruct", WorkerRole::Fast), + ("llama-3-7b-instruct", WorkerRole::Specialist), + ("Qwen3-8B", WorkerRole::Strong), + ]; + assert!(!has_quality_gap(workers.iter().copied())); + } + + #[test] + fn no_quality_gap_when_all_big_tier() { + let workers = [ + ("Qwen3-32B", WorkerRole::Fast), + ("MiniMax-M2.5", WorkerRole::Strong), + ]; + assert!(!has_quality_gap(workers.iter().copied())); + } + + #[test] + fn no_quality_gap_without_strong_role() { + let workers = [("Qwen2.5-3B-Instruct", WorkerRole::Generalist)]; + assert!(!has_quality_gap(workers.iter().copied())); + } } diff --git a/crates/mesh-mixture-of-agents/tests/sim_all_workers_fail.rs b/crates/mesh-mixture-of-agents/tests/sim_all_workers_fail.rs index 59f3f97877..69fd455011 100644 --- a/crates/mesh-mixture-of-agents/tests/sim_all_workers_fail.rs +++ b/crates/mesh-mixture-of-agents/tests/sim_all_workers_fail.rs @@ -92,6 +92,7 @@ fn three_failing_backends() -> moa::GatewayConfig { hedge_delay: Duration::from_millis(200), reducer_timeout: Duration::from_secs(2), first_answer_grace: Duration::ZERO, + strong_patience: Duration::ZERO, enable_thinking: None, } } diff --git a/crates/mesh-mixture-of-agents/tests/sim_enable_thinking_propagation.rs b/crates/mesh-mixture-of-agents/tests/sim_enable_thinking_propagation.rs index 28884a7349..58809be365 100644 --- a/crates/mesh-mixture-of-agents/tests/sim_enable_thinking_propagation.rs +++ b/crates/mesh-mixture-of-agents/tests/sim_enable_thinking_propagation.rs @@ -100,6 +100,7 @@ fn build_config( reducer_timeout: Duration::from_secs(5), hedge_delay: Duration::from_secs(1), first_answer_grace: Duration::ZERO, + strong_patience: Duration::ZERO, enable_thinking, } } diff --git a/crates/mesh-mixture-of-agents/tests/sim_strong_patience.rs b/crates/mesh-mixture-of-agents/tests/sim_strong_patience.rs new file mode 100644 index 0000000000..56a70eaca6 --- /dev/null +++ b/crates/mesh-mixture-of-agents/tests/sim_strong_patience.rs @@ -0,0 +1,236 @@ +//! Pin the tier-gate (strong patience) contract. +//! +//! Background: with a mixed pool (big-tier Strong worker + small-tier +//! workers, e.g. MiniMax + small Qwens), two fast small workers agreeing +//! could finalize the turn before the strong worker produced anything — +//! "dumbing down" the mesh answer to small-model consensus. Self-MoA +//! (arXiv:2502.00674) shows MoA output quality tracks proposer quality +//! far more than diversity, so the strong worker's answer must get a +//! bounded chance to land. +//! +//! Three contracts pinned here: +//! +//! 1. **Held consensus**: small-tier consensus does not finalize while +//! the strong worker is still running inside the patience window; +//! when the strong worker lands with a usable answer, it wins. +//! 2. **Hard bound (anti-#820)**: when the strong worker hangs past the +//! patience window, the turn still completes promptly with the held +//! small-tier consensus — never blocked until worker_timeout. +//! 3. **Same-tier pools unaffected**: an all-small pool keeps the +//! pre-gate early-exit behavior (gate disabled by tier analysis). + +use async_trait::async_trait; +use mesh_mixture_of_agents as moa; +use serde_json::{Value, json}; +use std::sync::Arc; +use std::time::Duration; + +/// Backend that returns a fixed answer after a configurable delay. +struct DelayedAnswerBackend { + text: String, + delay: Duration, +} + +impl DelayedAnswerBackend { + fn new(text: impl Into, delay: Duration) -> Arc { + Arc::new(Self { + text: text.into(), + delay, + }) + } +} + +#[async_trait] +impl moa::ModelBackend for DelayedAnswerBackend { + async fn chat_completion( + &self, + _model: &str, + _messages: &[Value], + _tools: Option<&Value>, + _max_tokens: u32, + _timeout: Duration, + _sampling: moa::SamplingParams, + ) -> Result { + tokio::time::sleep(self.delay).await; + Ok(json!({ + "choices": [{"message": {"content": self.text}}], + })) + } +} + +fn user_turn(content: &str) -> Value { + json!({ + "model": "mesh", + "messages": [{"role": "user", "content": content}], + "max_tokens": 32, + }) +} + +fn response_text(result: &moa::TurnResult) -> String { + result + .response_body + .pointer("/choices/0/message/content") + .and_then(Value::as_str) + .unwrap_or_default() + .to_string() +} + +/// Mixed-tier pool: two fast small workers agree on a (wrong) answer, +/// the big-tier strong worker lands later with a different one. +fn mixed_pool( + small_delay: Duration, + strong_delay: Duration, + strong_patience: Duration, +) -> moa::GatewayConfig { + let small_a = DelayedAnswerBackend::new("Sydney", small_delay); + let small_b = DelayedAnswerBackend::new("Sydney", small_delay + Duration::from_millis(20)); + let strong = DelayedAnswerBackend::new("Canberra", strong_delay); + + let backends: Vec> = vec![small_a, small_b, strong]; + let models = vec![ + moa::ModelEntry { + name: "qwen-3b".into(), + backend_index: 0, + }, + moa::ModelEntry { + name: "qwen-8b".into(), + backend_index: 1, + }, + moa::ModelEntry { + name: "MiniMax-M2.5".into(), + backend_index: 2, + }, + ]; + + moa::GatewayConfig { + backends, + models, + worker_timeout: Duration::from_secs(30), + hedge_delay: Duration::from_millis(200), + reducer_timeout: Duration::from_secs(2), + // Grace disabled so this test isolates the consensus path. + first_answer_grace: Duration::ZERO, + strong_patience, + enable_thinking: None, + } +} + +#[tokio::test] +async fn small_consensus_is_held_until_strong_lands() { + // Small workers agree at ~50ms; strong lands at 300ms — well inside + // the patience window. The strong answer must get to participate + // instead of the turn finalizing at 50ms on small consensus alone. + let config = mixed_pool( + Duration::from_millis(50), + Duration::from_millis(300), + Duration::from_secs(10), + ); + let result = moa::handle_turn(&config, &user_turn("Capital of Australia? One word.")).await; + + let strong_summary = result + .worker_summaries + .iter() + .find(|w| w.model == "MiniMax-M2.5") + .expect("strong worker must be accounted for"); + assert!( + strong_summary.succeeded, + "strong worker must have been allowed to finish (not aborted by \ + small-tier consensus); summaries: {:?}", + result + .worker_summaries + .iter() + .map(|w| (&w.model, w.succeeded)) + .collect::>() + ); + // The contract is "when the strong worker lands with a usable answer, + // it wins" — not merely "it was allowed to finish". The strong worker + // disagrees with the small-tier consensus ("Sydney"), so its answer + // ("Canberra") must be the one that ships. + assert!( + response_text(&result).contains("Canberra"), + "strong worker's answer must win over small-tier consensus once it lands; \ + got {:?}", + response_text(&result) + ); +} + +#[tokio::test] +async fn patience_expiry_releases_held_consensus() { + // Anti-#820: the strong worker effectively hangs (10s) while patience + // is 500ms. The held small-tier consensus must ship at patience + // expiry — NOT wait for the strong worker or worker_timeout. + let config = mixed_pool( + Duration::from_millis(50), + Duration::from_secs(10), + Duration::from_millis(500), + ); + let started = std::time::Instant::now(); + let result = moa::handle_turn(&config, &user_turn("Capital of Australia? One word.")).await; + let elapsed = started.elapsed(); + + // Pin the bound near the configured patience window (500ms) with + // scheduler slack, so a regression that pushes release out toward + // worker_timeout (30s) is actually caught — a loose 5s bound would + // silently pass several-second regressions. + assert!( + elapsed < Duration::from_millis(1500), + "held consensus must be released promptly at patience expiry (~500ms), \ + not block on the hung strong worker; took {elapsed:?}" + ); + assert!( + response_text(&result).contains("Sydney"), + "after patience expiry the small-tier consensus answer ships; got {:?}", + response_text(&result) + ); +} + +#[tokio::test] +async fn same_tier_pool_keeps_early_exit() { + // All-small pool: gate must be disabled by tier analysis, so two + // agreeing fast workers early-exit without waiting for the third. + let fast_a = DelayedAnswerBackend::new("Tokyo", Duration::from_millis(20)); + let fast_b = DelayedAnswerBackend::new("Tokyo", Duration::from_millis(40)); + let slow = DelayedAnswerBackend::new("Tokyo", Duration::from_secs(10)); + + let backends: Vec> = vec![fast_a, fast_b, slow]; + let models = vec![ + moa::ModelEntry { + name: "qwen-3b".into(), + backend_index: 0, + }, + moa::ModelEntry { + name: "llama-7b".into(), + backend_index: 1, + }, + moa::ModelEntry { + name: "qwen-8b".into(), + backend_index: 2, + }, + ]; + let config = moa::GatewayConfig { + backends, + models, + worker_timeout: Duration::from_secs(30), + hedge_delay: Duration::from_millis(200), + reducer_timeout: Duration::from_secs(2), + first_answer_grace: Duration::ZERO, + // Patience configured ON — but the pool is same-tier, so the + // tier analysis must disable the gate anyway. + strong_patience: Duration::from_secs(10), + enable_thinking: None, + }; + + let started = std::time::Instant::now(); + let result = moa::handle_turn(&config, &user_turn("Capital of Japan? One word.")).await; + let elapsed = started.elapsed(); + + assert_eq!( + result.turn_kind, + moa::TurnKind::EarlyExit, + "same-tier consensus must keep the early-exit path" + ); + assert!( + elapsed < Duration::from_secs(5), + "same-tier pool must not wait for the slow worker; took {elapsed:?}" + ); +} diff --git a/crates/mesh-mixture-of-agents/tests/sim_tool_call_text_not_passed_as_content.rs b/crates/mesh-mixture-of-agents/tests/sim_tool_call_text_not_passed_as_content.rs index 8a29d6f2ea..c6a8ff65e7 100644 --- a/crates/mesh-mixture-of-agents/tests/sim_tool_call_text_not_passed_as_content.rs +++ b/crates/mesh-mixture-of-agents/tests/sim_tool_call_text_not_passed_as_content.rs @@ -92,6 +92,7 @@ fn config_with_three_workers_returning(text: &str) -> moa::GatewayConfig { hedge_delay: Duration::from_millis(50), reducer_timeout: Duration::from_secs(2), first_answer_grace: Duration::ZERO, + strong_patience: Duration::ZERO, enable_thinking: None, } } diff --git a/crates/mesh-mixture-of-agents/tests/sim_tool_result_routes_to_reducer.rs b/crates/mesh-mixture-of-agents/tests/sim_tool_result_routes_to_reducer.rs index e0e232d67c..654fdda7f5 100644 --- a/crates/mesh-mixture-of-agents/tests/sim_tool_result_routes_to_reducer.rs +++ b/crates/mesh-mixture-of-agents/tests/sim_tool_result_routes_to_reducer.rs @@ -114,6 +114,7 @@ fn config_with_three_recording_workers() -> ( hedge_delay: Duration::from_millis(50), reducer_timeout: Duration::from_secs(2), first_answer_grace: Duration::ZERO, + strong_patience: Duration::ZERO, enable_thinking: None, }; (config, fast, mid, strong) diff --git a/crates/mesh-mixture-of-agents/tests/sim_worker_accounting.rs b/crates/mesh-mixture-of-agents/tests/sim_worker_accounting.rs index bc290264cc..c9f79a2d44 100644 --- a/crates/mesh-mixture-of-agents/tests/sim_worker_accounting.rs +++ b/crates/mesh-mixture-of-agents/tests/sim_worker_accounting.rs @@ -113,6 +113,7 @@ fn four_workers_two_fast_consensus() -> moa::GatewayConfig { hedge_delay: Duration::from_millis(200), reducer_timeout: Duration::from_secs(2), first_answer_grace: Duration::ZERO, + strong_patience: Duration::ZERO, enable_thinking: None, } }