Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -458,6 +458,15 @@ pub async fn build_moa_config(
// is the dominant chat path, so a tighter default is the right
// default.
first_answer_grace: std::time::Duration::from_secs(3),
// Tier-gate patience: how long small-tier-only answers/consensus
// are held when a big-tier strong worker (e.g. MiniMax) is still
// running. 20s covers the strong worker's typical first-token
// latency on agent-sized prompts over the public mesh without
// approaching worker_timeout (60s). Hard-bounded: at expiry all
// decision rules revert to ungated behavior. Same-tier pools are
// unaffected, so "many small models lift each other" keeps its
// current latency profile.
strong_patience: std::time::Duration::from_secs(20),
// Defaults to leaving each model's thinking behavior alone.
// `try_handle_moa` overrides this from the inbound request body
// when the caller has expressed a preference
Expand Down
246 changes: 221 additions & 25 deletions crates/mesh-mixture-of-agents/src/arbiter.rs

Large diffs are not rendered by default.

189 changes: 155 additions & 34 deletions crates/mesh-mixture-of-agents/src/fanout.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,18 @@ pub(crate) enum GraceMode {
Tool,
}

/// Time-based decision policy for a single fan-out: how long to wait
/// before shipping a partial result, and how long to hold small-tier
/// decisions for a pending big-tier strong worker.
#[derive(Clone, Copy, Debug)]
pub(crate) struct GatherPolicy {
/// See [`crate::GatewayConfig::first_answer_grace`].
pub first_answer_grace: Duration,
pub grace_mode: GraceMode,
/// See [`crate::GatewayConfig::strong_patience`].
pub strong_patience: Duration,
}

/// Identifier for a worker we dispatched. Used to reconcile the
/// per-worker accounting at the end of fan-out so the — possibly
/// aborted or panicked — task's existence still shows up in
Expand All @@ -41,20 +53,43 @@ pub(crate) async fn gather_workers_incremental(
has_tools: bool,
allowed_tools: &[String],
tools: Option<&Value>,
first_answer_grace: Duration,
grace_mode: GraceMode,
policy: GatherPolicy,
) -> (
Vec<WorkerOutput>,
Vec<WorkerSummary>,
Option<arbiter::Decision>,
) {
let GatherPolicy {
first_answer_grace,
grace_mode,
strong_patience,
} = policy;
let total_workers = dispatched.len();
let mut outputs = Vec::new();
let mut summaries = Vec::new();
let mut total_finished: usize = 0;
let dispatched_at = Instant::now();
let grace_enabled = grace_mode != GraceMode::Disabled && !first_answer_grace.is_zero();

// Tier gate: when the pool mixes a big-tier Strong worker with
// small-tier workers, small-tier-only consensus is held until the
// strong worker finishes OR the patience window expires. The window
// is a hard bound — when it lapses the gate switches off entirely
// and every decision rule reverts to pre-gate behavior, so a stuck
// strong worker can never hold the turn hostage (the failure mode
// that sank PR #820).
let mut strong_finished = false;
let gate_enabled = !strong_patience.is_zero()
&& crate::worker::has_quality_gap(dispatched.iter().map(|d| (d.model.as_str(), d.role)));
let strong_gate = |strong_finished: bool, elapsed: Duration| -> arbiter::StrongGate {
if !gate_enabled || elapsed >= strong_patience {
return arbiter::StrongGate::Off;
}
arbiter::StrongGate::Active {
strong_pending: !strong_finished,
}
};

// Grace eligibility: once the grace window has elapsed and a qualifying
// partial decision exists, ship it instead of waiting for the slow tail.
// Answer grace handles ordinary chat, including tool-enabled clients that
Expand Down Expand Up @@ -83,7 +118,23 @@ pub(crate) async fn gather_workers_incremental(
} else {
Duration::from_secs(60 * 60)
};
let armed = grace_eligible(&outputs);
// While the tier gate is actively holding (strong worker pending,
// patience not yet expired), arm a wake-up at patience expiry so a
// held consensus decision is re-evaluated even if no further worker
// event arrives. Without this, a stuck strong worker would mean the
// next re-check only happens at worker_timeout.
let gate_holding = gate_enabled
&& !strong_finished
&& dispatched_at.elapsed() < strong_patience
&& !outputs.is_empty();
let patience_remaining = strong_patience.saturating_sub(dispatched_at.elapsed());

// The answer grace is also tier-gated: a single small-tier answer
// must not ship at grace expiry while the strong worker is still
// inside its patience window — that was the dominant path where a
// small model's answer pre-empted the strong one. Tool grace is
// exempt (schema-verified proposals; agent loops stay snappy).
let armed = grace_eligible(&outputs) && !(gate_holding && grace_mode == GraceMode::Answer);

let join_result = tokio::select! {
biased;
Expand All @@ -104,6 +155,26 @@ pub(crate) async fn gather_workers_incremental(
reconcile_dispatched(dispatched, &mut summaries);
return (outputs, summaries, Some(decision));
}
_ = tokio::time::sleep(patience_remaining), if gate_holding => {
tracing::info!(
"moa: strong patience expired after {}ms — re-evaluating held outputs",
dispatched_at.elapsed().as_millis(),
);
// Gate is now Off (elapsed >= strong_patience); re-run the
// early-decision check over what we already have.
if let Some(decision) = arbiter::try_early_decision(
&outputs,
total_workers,
total_finished,
has_tools,
arbiter::StrongGate::Off,
) {
drain_after_early_exit(join_set, &mut summaries).await;
reconcile_dispatched(dispatched, &mut summaries);
return (outputs, summaries, Some(decision));
}
continue;
}
};

let Some(join_result) = join_result else {
Expand All @@ -113,6 +184,9 @@ pub(crate) async fn gather_workers_incremental(
match join_result {
Ok((model, role, Ok(text), elapsed)) => {
total_finished += 1;
if role == WorkerRole::Strong {
strong_finished = true;
}
let mut normalized =
normalize::normalize_worker_output(&text, &model, role, elapsed);
enforce_tool_call_contract(&mut normalized, allowed_tools, tools, &model);
Expand All @@ -135,16 +209,23 @@ pub(crate) async fn gather_workers_incremental(
});
outputs.push(normalized);

if let Some(decision) =
arbiter::try_early_decision(&outputs, total_workers, total_finished, has_tools)
{
if let Some(decision) = arbiter::try_early_decision(
&outputs,
total_workers,
total_finished,
has_tools,
strong_gate(strong_finished, dispatched_at.elapsed()),
) {
drain_after_early_exit(join_set, &mut summaries).await;
reconcile_dispatched(dispatched, &mut summaries);
return (outputs, summaries, Some(decision));
}
}
Ok((model, role, Err(e), elapsed)) => {
total_finished += 1;
if role == WorkerRole::Strong {
strong_finished = true;
}
tracing::warn!(
"moa: worker {} ({}) failed after {}ms: {}",
model,
Expand All @@ -161,9 +242,13 @@ pub(crate) async fn gather_workers_incremental(
confidence: None,
});

if let Some(decision) =
arbiter::try_early_decision(&outputs, total_workers, total_finished, has_tools)
{
if let Some(decision) = arbiter::try_early_decision(
&outputs,
total_workers,
total_finished,
has_tools,
strong_gate(strong_finished, dispatched_at.elapsed()),
) {
drain_after_early_exit(join_set, &mut summaries).await;
reconcile_dispatched(dispatched, &mut summaries);
return (outputs, summaries, Some(decision));
Expand All @@ -185,15 +270,27 @@ pub(crate) async fn gather_workers_incremental(
}

fn grace_answer_decision(outputs: &[WorkerOutput]) -> arbiter::Decision {
let answer = outputs
.iter()
.filter(|o| o.kind == normalize::OutputKind::Answer)
.max_by(|a, b| {
a.confidence
.partial_cmp(&b.confidence)
.unwrap_or(std::cmp::Ordering::Equal)
})
.expect("answer grace requires at least one Answer");
// Prefer the Strong worker's qualifying answer when it has landed:
// if the biggest model already answered, shipping a smaller model's
// marginally-higher-confidence answer instead would defeat the point
// of waiting for it. Confidence is self-reported and not comparable
// across models, so role is the better tie-breaker here.
let strong = outputs.iter().find(|o| {
o.kind == normalize::OutputKind::Answer
&& o.role == WorkerRole::Strong
&& o.confidence >= GRACE_MIN_CONFIDENCE
});
let answer = strong.unwrap_or_else(|| {
outputs
.iter()
.filter(|o| o.kind == normalize::OutputKind::Answer)
.max_by(|a, b| {
a.confidence
.partial_cmp(&b.confidence)
.unwrap_or(std::cmp::Ordering::Equal)
})
.expect("answer grace requires at least one Answer")
});
let answer_count = outputs
.iter()
.filter(|o| o.kind == normalize::OutputKind::Answer)
Expand Down Expand Up @@ -370,8 +467,11 @@ mod tests {
false, // has_tools
&[],
None,
Duration::from_millis(50),
GraceMode::Answer,
GatherPolicy {
first_answer_grace: Duration::from_millis(50),
grace_mode: GraceMode::Answer,
strong_patience: Duration::ZERO,
},
)
.await;
let elapsed = started.elapsed();
Expand Down Expand Up @@ -423,8 +523,11 @@ mod tests {
true, // has_tools
&[],
None,
Duration::from_millis(50),
GraceMode::Answer,
GatherPolicy {
first_answer_grace: Duration::from_millis(50),
grace_mode: GraceMode::Answer,
strong_patience: Duration::ZERO,
},
)
.await;
let elapsed = started.elapsed();
Expand Down Expand Up @@ -471,8 +574,11 @@ mod tests {
true,
&[],
None,
Duration::from_millis(50),
GraceMode::Disabled,
GatherPolicy {
first_answer_grace: Duration::from_millis(50),
grace_mode: GraceMode::Disabled,
strong_patience: Duration::ZERO,
},
)
.await;
let elapsed = started.elapsed();
Expand Down Expand Up @@ -511,8 +617,11 @@ mod tests {
true,
&["read".to_string()],
None,
Duration::from_millis(50),
GraceMode::Tool,
GatherPolicy {
first_answer_grace: Duration::from_millis(50),
grace_mode: GraceMode::Tool,
strong_patience: Duration::ZERO,
},
)
.await;
let elapsed = started.elapsed();
Expand Down Expand Up @@ -566,8 +675,11 @@ mod tests {
false, // has_tools
&[],
None,
Duration::ZERO,
GraceMode::Answer,
GatherPolicy {
first_answer_grace: Duration::ZERO,
grace_mode: GraceMode::Answer,
strong_patience: Duration::ZERO,
},
)
.await;
let elapsed = started.elapsed();
Expand Down Expand Up @@ -614,8 +726,11 @@ mod tests {
false,
&[],
None,
Duration::from_millis(50),
GraceMode::Answer,
GatherPolicy {
first_answer_grace: Duration::from_millis(50),
grace_mode: GraceMode::Answer,
strong_patience: Duration::ZERO,
},
)
.await;
let elapsed = started.elapsed();
Expand Down Expand Up @@ -667,8 +782,11 @@ mod tests {
false,
&[],
None,
Duration::from_millis(50),
GraceMode::Answer,
GatherPolicy {
first_answer_grace: Duration::from_millis(50),
grace_mode: GraceMode::Answer,
strong_patience: Duration::ZERO,
},
)
.await;
let elapsed = started.elapsed();
Expand Down Expand Up @@ -728,8 +846,11 @@ mod tests {
false,
&[],
None,
Duration::from_millis(100),
GraceMode::Answer,
GatherPolicy {
first_answer_grace: Duration::from_millis(100),
grace_mode: GraceMode::Answer,
strong_patience: Duration::ZERO,
},
)
.await;
let decision = decision.expect("grace must yield a Decision");
Expand Down
17 changes: 15 additions & 2 deletions crates/mesh-mixture-of-agents/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,15 @@ pub struct GatewayConfig {
/// (conf >= 0.5) is in, accept it instead of waiting for consensus.
/// Disabled for tool turns. Zero disables entirely.
pub first_answer_grace: Duration,
/// Tier-gate patience: when the worker pool mixes a big-tier Strong
/// worker with small-tier workers, small-tier-only answers and
/// consensus are held for up to this long after dispatch to give the
/// strong worker a chance to weigh in. A hard bound — once it lapses,
/// all decision rules revert to ungated behavior, so a stuck strong
/// worker can never hold the turn hostage. Zero disables the gate.
/// Has no effect when all workers are the same tier. Tool proposals
/// are never held.
pub strong_patience: Duration,
/// Override for whether reasoning workers should think. Propagated to
/// every worker and the reducer as `chat_template_kwargs.enable_thinking`
/// (and `reasoning_effort: "none"` when disabled).
Expand Down Expand Up @@ -297,8 +306,11 @@ async fn handle_query(
query_uses_tools,
allowed_tools,
session.tools(),
config.first_answer_grace,
grace_mode,
fanout::GatherPolicy {
first_answer_grace: config.first_answer_grace,
grace_mode,
strong_patience: config.strong_patience,
},
)
.await;

Expand Down Expand Up @@ -1531,6 +1543,7 @@ mod response_builder_tests {
reducer_timeout: Duration::from_secs(1),
hedge_delay: Duration::from_millis(10),
first_answer_grace: Duration::from_millis(10),
strong_patience: Duration::ZERO,
enable_thinking: Some(false),
};
let forced_tool = ForcedToolChoice {
Expand Down
Loading
Loading