From b828ebb99bcf7528db8edfa1def2bf67b99c6f29 Mon Sep 17 00:00:00 2001 From: Anderson Leal Date: Fri, 10 Jul 2026 12:33:06 -0300 Subject: [PATCH 01/11] (MOT-3949) feat(harness): durable trigger_fired records + spawn provenance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every real fire now appends a model-invisible `trigger_fired` custom entry to the owner session (subscriptions/fired.rs): notify fires, react simple spawns, join arrivals, and join completions — with entry ids idempotent against engine redelivery. One durable artifact drives both the console's chat fire notices and the panel's fired-trigger ghosts. Child sessions are stamped `spawned_by: trigger|agent` and direct spawn seed entries get `e_spawn_` ids + a spawn origin, so the console can tell trigger-fired work from agent-spawned work. --- harness/src/functions/react.rs | 136 +++++++++++++++- harness/src/ids.rs | 7 + harness/src/subagent.rs | 23 ++- harness/src/subscriptions/fired.rs | 186 ++++++++++++++++++++++ harness/src/subscriptions/mod.rs | 1 + harness/src/subscriptions/notify_agent.rs | 35 ++++ 6 files changed, 379 insertions(+), 9 deletions(-) create mode 100644 harness/src/subscriptions/fired.rs diff --git a/harness/src/functions/react.rs b/harness/src/functions/react.rs index 9e250153b..35400cd82 100644 --- a/harness/src/functions/react.rs +++ b/harness/src/functions/react.rs @@ -384,8 +384,38 @@ pub async fn handle( spawn_depth, ) .await; - if spec.once && matches!(&res, Ok(r) if r.spawned) { - once_unregister(deps, &spec).await; + if let Ok(r) = &res { + if r.spawned { + let sub = spec.subscription_id.as_deref().unwrap_or("sub"); + let entry_id = if spec.once { + format!("e_trigfired_{sub}") + } else { + // Recurring: the child session id is unique per spawn. + // ponytail: falls back to `spawn` when the spawn returned + // no child id, so two child-less recurring fires dedup to + // one record — acceptable, bounded by the fire-rate gate. + format!( + "e_trigfired_{sub}_{}", + r.child_session_id.as_deref().unwrap_or("spawn") + ) + }; + // Record with the binding still live (so its engine trigger + // id resolves) BEFORE the once teardown below. + emit_fired( + deps, + &spec, + &event, + &entry_id, + r.child_session_id.as_deref(), + spec.once, + None, + None, + ) + .await; + if spec.once { + once_unregister(deps, &spec).await; + } + } } res } @@ -424,6 +454,28 @@ async fn join_edge( let arrived = arrived_count(&rec); let expected = join.expect.len(); if arrived < expected { + let note = format!("{arrived}/{expected} arrived"); + // ponytail: these join entry ids are cycle-invariant, so a re-armed + // join's cycle ≥2 records dedup away (append_custom is idempotent on + // entry_id — the same property that absorbs engine redelivery). Key in + // a cycle counter if later cycles ever need their own notices. + emit_fired( + deps, + spec, + &event, + &format!("e_trigfired_join_{}_{}", join.id, join.key), + None, // nothing spawned yet + false, // predecessor stays registered until the join completes + Some(crate::subscriptions::fired::JoinProgress { + id: &join.id, + key: &join.key, + arrived, + expected, + completed: false, + }), + Some(¬e), + ) + .await; return Ok(ReactResult::note(format!( "join {}: {arrived}/{expected} arrived", join.id @@ -465,6 +517,36 @@ async fn join_edge( // is already resolved by the caller. Joins fire once, so this cannot spam. let task = gather_inputs_task(&spec.task, &rec); let res = spawn_reaction(deps, task, spec, parent, spawn_depth).await; + + // The join committed: the downstream spawned and (unless re-armed) every + // predecessor was just auto-unregistered above. One completion record lets + // the console mark the whole join fired + retired and post the notice. + // Gated on `spawned` like the simple edge — spawn_reaction swallows + // dispatch errors into `spawned: false`, and a record claiming "spawned" + // for a spawn that never happened would mislead the chat. + if let Ok(r) = &res { + if r.spawned { + let note = format!("{expected}/{expected} arrived — spawned"); + emit_fired( + deps, + spec, + &event, + &format!("e_trigfired_join_{}_done", join.id), + r.child_session_id.as_deref(), + !join.rearm, + Some(crate::subscriptions::fired::JoinProgress { + id: &join.id, + key: &join.key, + arrived: expected, + expected, + completed: true, + }), + Some(¬e), + ) + .await; + } + } + // The delete is the cycle reset: a stale record (fire=1, all keys arrived) // makes the next cycle's fire-guard land on 2 and refuse forever — for a // rearmed join that is a permanent, silent wedge. Retry transient state @@ -650,6 +732,56 @@ async fn once_unregister(deps: &Deps, spec: &ReactSpec) { } } +/// Append a durable `trigger_fired` record into the owner (registering) chat so +/// the console renders a turn-less notice and keeps a fired binding visible in +/// the panel after teardown. Best-effort; owner-less raw registrations (no chat +/// to surface into) are skipped. Read the engine trigger id BEFORE any +/// retirement so a still-live binding resolves. +#[allow(clippy::too_many_arguments)] +async fn emit_fired( + deps: &Deps, + spec: &ReactSpec, + event: &Value, + entry_id: &str, + child_session_id: Option<&str>, + retired: bool, + join: Option>, + note: Option<&str>, +) { + use crate::subscriptions::fired; + let Some(owner) = spec.owner_session_id.as_deref() else { + return; + }; + let sub = spec.subscription_id.as_deref().unwrap_or(""); + let trigger_id = spec + .subscription_id + .as_deref() + .and_then(|s| deps.subscriptions.trigger_id_of(s)); + let (scope, key) = fired::event_state_watch(event); + let session = deps.session().await; + fired::emit( + &session, + owner, + entry_id, + fired::TriggerFired { + subscription_id: sub, + trigger_id: trigger_id.as_deref(), + target: "spawn", + label: None, + model: Some(&spec.model), + once: spec.once, + retired, + scope, + key, + child_session_id, + join, + note, + fired_at: fired::now_ms(), + }, + ) + .await; +} + async fn unregister_subscription(deps: &Deps, id: &str) -> Result<(), HarnessError> { deps.iii .trigger(TriggerRequest { diff --git a/harness/src/ids.rs b/harness/src/ids.rs index 8eb447e2b..017ca48f6 100644 --- a/harness/src/ids.rs +++ b/harness/src/ids.rs @@ -47,6 +47,13 @@ pub fn react_entry_id() -> String { format!("e_react_{}", short_uuid()) } +/// The opening task entry of a direct (agent-called) spawn (`e_spawn_`). +/// Same pattern as `e_react_`: the prefix survives transcript reads so the +/// console renders the task as machine-sent, not something the human typed. +pub fn spawn_entry_id() -> String { + format!("e_spawn_{}", short_uuid()) +} + /// The assistant message of a generate step: `e___assistant`. /// A resumed step streams into this same entry rather than appending a second. pub fn assistant_entry_id(turn_id: &str, step: u64) -> String { diff --git a/harness/src/subagent.rs b/harness/src/subagent.rs index 64813ebb2..a19d9ad10 100644 --- a/harness/src/subagent.rs +++ b/harness/src/subagent.rs @@ -159,17 +159,27 @@ async fn seed_child( // trigger-fired spawn has no parent turn, but a caller-supplied // `parent_session_id` still writes a display-only link so the console nests // the child (no policy inheritance, no parent-call resolution). + // `spawned_by` tells the console tree WHO created the child — a trigger + // reaction (`reactive_depth` is stamped only by `harness::react`) or an + // agent's direct `harness::spawn` — so the sidebar can differentiate them. + let spawned_by = if req.reactive_depth.is_some() { + "trigger" + } else { + "agent" + }; let linkage = match parent { Some(p) => Some(json!({ "parent_session_id": p.session_id, "parent_turn_id": p.turn_id, "function_call_id": p.function_call_id, "depth": depth, + "spawned_by": spawned_by, })), None => req.parent_session_id.as_ref().map(|psid| { json!({ "parent_session_id": psid, "depth": depth, + "spawned_by": spawned_by, }) }), }; @@ -191,12 +201,11 @@ async fn seed_child( None => session.create(None, linkage.as_ref()).await?, }; - // The task is the child's opening user message. React-fired spawns - // (`reactive_depth` is stamped only by `harness::react`) mark the entry — - // `{ reaction: true }` origin plus an `e_react_` id, the notify pattern — - // so clients render the task as a trigger reaction, not as something the - // human typed (a reaction delivered into a chat looks user-authored - // otherwise). + // The task is the child's opening user message — machine-sent either way, + // so every spawn marks the entry (origin + a readable id prefix, the + // notify pattern) or clients would render it as something the human typed. + // React-fired spawns mark `{ reaction: true }` / `e_react_`; direct + // agent spawns mark `{ spawn: true }` / `e_spawn_`. let task = normalize_message(req.task.clone())?; let (entry_id, origin) = if req.reactive_depth.is_some() { let mut origin = json!({ "reaction": true }); @@ -205,7 +214,7 @@ async fn seed_child( } (Some(ids::react_entry_id()), Some(origin)) } else { - (None, None) + (Some(ids::spawn_entry_id()), Some(json!({ "spawn": true }))) }; session .append( diff --git a/harness/src/subscriptions/fired.rs b/harness/src/subscriptions/fired.rs new file mode 100644 index 000000000..888cd555b --- /dev/null +++ b/harness/src/subscriptions/fired.rs @@ -0,0 +1,186 @@ +//! Durable `trigger_fired` bookkeeping entries. +//! +//! Every real subscription fire — a notify wake, a react spawn, or a join edge — +//! appends a `kind: "custom"` `trigger_fired` entry into the OWNER session's +//! transcript (the chat that registered the trigger). Custom entries are +//! model-invisible (excluded from default `session::messages` reads and the +//! model context), so this is a pure UI signal with two uses on the console: +//! * render a turn-less "trigger fired" notice in the timeline, and +//! * keep a fired `once` trigger visible in the panel after the engine +//! unregisters it (the 5s poll can no longer see it, but this durable +//! record can). +//! +//! Best-effort: a failed append logs and returns; it never blocks the fire's +//! real work (the notification wake or the sub-agent spawn). + +use serde::Serialize; +use serde_json::{json, Value}; + +use crate::clients::session::SessionClient; +use crate::types::message::AgentMessage; + +/// custom_type stamped on the transcript entry (mirrored by the console mapper). +pub const CUSTOM_TYPE: &str = "trigger_fired"; + +/// Join-barrier progress for a react join edge fire. +#[derive(Debug, Serialize)] +pub struct JoinProgress<'a> { + pub id: &'a str, + pub key: &'a str, + pub arrived: usize, + pub expected: usize, + /// The last predecessor arrived and the downstream spawned this call. + pub completed: bool, +} + +/// The `data` payload of a `trigger_fired` custom entry. Carries enough for the +/// console to render both the chat notice and a standalone fired panel row after +/// a reload (label / target / model / state watch / join progress). +#[derive(Debug, Serialize)] +pub struct TriggerFired<'a> { + pub subscription_id: &'a str, + /// Engine trigger id — lets the console dedup against a still-registered + /// (recurring) panel row. Absent when the local slot no longer maps one. + #[serde(skip_serializing_if = "Option::is_none")] + pub trigger_id: Option<&'a str>, + /// `"notify"` (wakes this chat) or `"spawn"` (react sub-agent). + pub target: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + pub label: Option<&'a str>, + /// Reacting sub-agent model (react only). + #[serde(skip_serializing_if = "Option::is_none")] + pub model: Option<&'a str>, + pub once: bool, + /// This fire unregistered the binding (once teardown, or join predecessor GC). + pub retired: bool, + /// state-trigger watch, extracted from the fired event when present. + #[serde(skip_serializing_if = "Option::is_none")] + pub scope: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + pub key: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + pub child_session_id: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + pub join: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub note: Option<&'a str>, + pub fired_at: i64, +} + +/// Append the fired record into the owner session. Best-effort — logs and +/// returns on error so a transcript hiccup never blocks the fire. +pub async fn emit( + session: &SessionClient, + owner_session_id: &str, + entry_id: &str, + rec: TriggerFired<'_>, +) { + let data = match serde_json::to_value(&rec) { + Ok(v) => v, + Err(e) => { + tracing::warn!(error = %e, "trigger_fired record serialize failed; dropping"); + return; + } + }; + if let Err(e) = session + .append_custom( + owner_session_id, + CUSTOM_TYPE, + data, + entry_id, + Some(&json!({ "trigger_fired": true })), + ) + .await + { + tracing::warn!( + error = %e, + session_id = %owner_session_id, + entry_id = %entry_id, + "trigger_fired record append failed (non-fatal)" + ); + } +} + +/// Current wall-clock ms for the record's `fired_at`. +pub fn now_ms() -> i64 { + AgentMessage::now_ms() +} + +/// A state fire delivers `{scope?, key}` in its event; other trigger types +/// (cron/stream/turn) carry no watch. Best-effort — returns `(None, None)` +/// when absent. +pub fn event_state_watch(event: &Value) -> (Option<&str>, Option<&str>) { + ( + event.get("scope").and_then(Value::as_str), + event.get("key").and_then(Value::as_str), + ) +} + +/// `e_notify_…` → `e_trigfired_…`, reusing the notify fire's monotonic suffix so +/// a redelivered engine fire dedups on the same entry id. +pub fn entry_id_from_notify(notify_entry_id: &str) -> String { + notify_entry_id.replacen("e_notify_", "e_trigfired_", 1) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn entry_id_swaps_only_the_notify_prefix() { + assert_eq!(entry_id_from_notify("e_notify_sub_1"), "e_trigfired_sub_1"); + assert_eq!( + entry_id_from_notify("e_notify_sub_1_7"), + "e_trigfired_sub_1_7" + ); + // Only the leading occurrence is swapped. + assert_eq!( + entry_id_from_notify("e_notify_e_notify_x"), + "e_trigfired_e_notify_x" + ); + } + + #[test] + fn state_watch_reads_scope_and_key_from_event() { + let ev = json!({ "scope": "cache-repl-pipeline", "key": "facts", "value": 1 }); + assert_eq!( + event_state_watch(&ev), + (Some("cache-repl-pipeline"), Some("facts")) + ); + // Turn/cron events carry no watch. + assert_eq!( + event_state_watch(&json!({ "session_id": "s" })), + (None, None) + ); + assert_eq!(event_state_watch(&Value::Null), (None, None)); + } + + #[test] + fn record_omits_empty_optionals() { + let rec = TriggerFired { + subscription_id: "sub_1", + trigger_id: None, + target: "notify", + label: None, + model: None, + once: true, + retired: true, + scope: None, + key: None, + child_session_id: None, + join: None, + note: None, + fired_at: 42, + }; + let v = serde_json::to_value(&rec).unwrap(); + assert_eq!(v["subscription_id"], "sub_1"); + assert_eq!(v["target"], "notify"); + assert_eq!(v["once"], true); + assert_eq!(v["retired"], true); + assert_eq!(v["fired_at"], 42); + // Skipped optionals must not appear. + assert!(v.get("trigger_id").is_none()); + assert!(v.get("model").is_none()); + assert!(v.get("join").is_none()); + } +} diff --git a/harness/src/subscriptions/mod.rs b/harness/src/subscriptions/mod.rs index d9f81502f..e0ee355b2 100644 --- a/harness/src/subscriptions/mod.rs +++ b/harness/src/subscriptions/mod.rs @@ -24,6 +24,7 @@ //! intercepts the agent's `engine::register_trigger` call, never trusted from //! model arguments. +pub mod fired; pub mod notify_agent; pub mod reconcile; pub mod registry; diff --git a/harness/src/subscriptions/notify_agent.rs b/harness/src/subscriptions/notify_agent.rs index a6400ca3a..5e386c85b 100644 --- a/harness/src/subscriptions/notify_agent.rs +++ b/harness/src/subscriptions/notify_agent.rs @@ -112,6 +112,10 @@ async fn on_fire(deps: &Deps, event: Value, metadata: Option) { } }; + // Read the engine trigger id BEFORE claim_fire removes a once entry, so the + // fired record can dedup against a still-registered recurring panel row. + let engine_trigger_id = deps.subscriptions.trigger_id_of(&meta.subscription_id); + let Some(claim) = deps.subscriptions .claim_fire(&meta.subscription_id, &meta.session_id, meta.once) @@ -122,6 +126,37 @@ async fn on_fire(deps: &Deps, event: Value, metadata: Option) { crate::functions::subscribe::unregister_engine_trigger(deps, trigger_id).await; } + // Durable, turn-less UI signal: the console renders a "trigger fired" notice + // and keeps a fired `once` binding visible after the engine unregisters it. + // Written before the notification inject so the fire is recorded even if the + // wake path errors. + { + use crate::subscriptions::fired; + let (scope, key) = fired::event_state_watch(&event); + let session = deps.session().await; + fired::emit( + &session, + &meta.session_id, + &fired::entry_id_from_notify(&claim.entry_id), + fired::TriggerFired { + subscription_id: &meta.subscription_id, + trigger_id: engine_trigger_id.as_deref(), + target: "notify", + label: meta.label.as_deref(), + model: None, + once: meta.once, + retired: claim.trigger_id.is_some(), + scope, + key, + child_session_id: None, + join: None, + note: None, + fired_at: fired::now_ms(), + }, + ) + .await; + } + let (message, origin) = notification_message(&meta, &event); if let Err(e) = crate::functions::send::inject( From 009f6599d3358b6cf21e688c58333a50dddc26ee Mon Sep 17 00:00:00 2001 From: Anderson Leal Date: Fri, 10 Jul 2026 12:33:26 -0300 Subject: [PATCH 02/11] (MOT-3952, MOT-3953) fix(harness): chat dispatch-failure race, streaming args tail, agent_trigger no-target error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - clients/router.rs: the frame loop races the held-open router::chat trigger (750ms grace drain) so a dispatch failure before a writer attaches fails the turn instead of hanging it forever; synthesized error kind Permanent/Transient by response error. - clients/router.rs: accumulate raw FunctioncallDelta text and ride a bounded `_streaming` tail on coalesced partials while a call's arguments are still forming — UIs can show the command being written; disappears once args parse. - turn_loop/trigger.rs: a call still targeting `agent_trigger` at dispatch (arguments empty/null/unparseable) fails locally with a teachable error instead of a doomed engine dispatch (`function_not_found: agent_trigger`); wrapper sentinel pinned in policy.rs. --- harness/src/clients/router.rs | 186 ++++++++++++++++++++++++++++++++-- harness/src/policy.rs | 17 ++++ harness/src/trigger.rs | 19 ++++ harness/src/turn_loop.rs | 22 ++++ 4 files changed, 237 insertions(+), 7 deletions(-) diff --git a/harness/src/clients/router.rs b/harness/src/clients/router.rs index 8eb182d51..1bde5faa9 100644 --- a/harness/src/clients/router.rs +++ b/harness/src/clients/router.rs @@ -143,7 +143,7 @@ impl RouterClient { let iii = self.iii.clone(); let timeout_ms = self.timeout_ms; let parent_cx = iii_helpers::observability::opentelemetry::Context::current(); - let trigger = tokio::spawn( + let mut trigger = tokio::spawn( async move { iii.trigger(TriggerRequest { function_id: "router::chat".into(), @@ -162,8 +162,40 @@ impl RouterClient { .unwrap_or_else(Instant::now); let mut final_message: Option = None; let mut terminal_error: Option = None; + // Raw in-flight tool-call arguments per call id: providers degrade + // incomplete args to a placeholder object (replay safety), so the + // delta text accumulated here is the only live view of a long + // arguments stream — injected into coalesced partials as + // `_streaming` so UIs can show the command being formed. + let mut args_acc: std::collections::HashMap = + std::collections::HashMap::new(); - while let Some(frame) = rx.recv().await { + // Consume frames until reader EOF — but ALSO watch the held-open + // trigger: when it resolves, the router is done (ack) or the dispatch + // failed before a writer ever attached (e.g. provider unavailable). + // In the failure case the channel never EOFs, so waiting on frames + // alone would hang this turn forever with the session stuck "working". + // After the trigger resolves, drain already-buffered frames briefly, + // then fall through to the outcome handling below. + let mut response: Option> = None; + loop { + let frame = if response.is_some() { + // timeout elapsed → grace drain over, no writer is coming + tokio::time::timeout(Duration::from_millis(750), rx.recv()) + .await + .unwrap_or_default() + } else { + tokio::select! { + f = rx.recv() => f, + r = &mut trigger => { + response = Some(r.map_err(|e| { + HarnessError::Internal(format!("router::chat task: {e}")) + })?); + continue; + } + } + }; + let Some(frame) = frame else { break }; let Ok(event) = serde_json::from_str::(&frame) else { continue; }; @@ -186,9 +218,17 @@ impl RouterClient { terminal_error.get_or_insert(msg); } other => { + if let AssistantMessageEvent::FunctioncallDelta { partial, delta } = &other { + if let Some(id) = open_call_id(partial) { + args_acc.entry(id.to_string()).or_default().push_str(delta); + } + } if let Some(partial) = partial_of(&other) { if last_emit.elapsed() >= coalesce { - sink.on_update(partial).await; + match enrich_streaming_args(partial, &args_acc) { + Some(enriched) => sink.on_update(&enriched).await, + None => sink.on_update(partial).await, + } last_emit = Instant::now(); } } @@ -199,9 +239,12 @@ impl RouterClient { // Stream drained; stop the pump and collect the trigger ack. cancel.notify_waiters(); let _ = pump.await; - let response = trigger - .await - .map_err(|e| HarnessError::Internal(format!("router::chat task: {e}")))?; + let response = match response { + Some(r) => r, + None => trigger + .await + .map_err(|e| HarnessError::Internal(format!("router::chat task: {e}")))?, + }; let (ok, response_error) = match &response { Ok(v) => { @@ -223,7 +266,14 @@ impl RouterClient { .clone() .or_else(|| terminal_error.clone()) .or_else(|| Some("router produced no terminal frame".to_string())); - m.error_kind = Some(crate::types::event::ErrorKind::Transient); + // A dispatch/ack rejection (e.g. provider unavailable) is + // authoritative — retrying in-turn cannot help. Only a + // frame-less-but-acked stream stays classified transient. + m.error_kind = Some(if response_error.is_some() { + crate::types::event::ErrorKind::Permanent + } else { + crate::types::event::ErrorKind::Transient + }); m }); @@ -366,6 +416,65 @@ impl StreamSink for CapturingSink { } } +/// The call currently receiving argument deltas: blocks stream in order, so +/// it is the last function_call block of the partial. +fn open_call_id(partial: &AssistantMessage) -> Option<&str> { + partial.content.iter().rev().find_map(|b| match b { + crate::types::content::ContentBlock::FunctionCall { id, .. } => Some(id.as_str()), + _ => None, + }) +} + +/// Inject the in-flight raw-arguments tail into a coalesced partial as a +/// `_streaming` field beside the provider's placeholder, so UIs can render +/// the command being formed. Injected only while the accumulated text does +/// not yet parse — once it parses, the provider partial already carries the +/// real arguments. Returns None when nothing was injected. +fn enrich_streaming_args( + partial: &AssistantMessage, + acc: &std::collections::HashMap, +) -> Option { + use crate::types::content::ContentBlock; + if acc.is_empty() { + return None; + } + let mut out: Option = None; + for (i, block) in partial.content.iter().enumerate() { + let ContentBlock::FunctionCall { id, arguments, .. } = block else { + continue; + }; + let Some(raw) = acc.get(id) else { continue }; + if raw.is_empty() || serde_json::from_str::(raw).is_ok() { + continue; + } + // Degraded placeholders are always objects; anything else is final. + let Value::Object(map) = arguments else { + continue; + }; + let mut map = map.clone(); + map.insert( + "_streaming".into(), + Value::String(utf8_tail(raw, 1500).to_string()), + ); + let msg = out.get_or_insert_with(|| partial.clone()); + if let ContentBlock::FunctionCall { arguments, .. } = &mut msg.content[i] { + *arguments = Value::Object(map); + } + } + out +} + +fn utf8_tail(s: &str, max: usize) -> &str { + if s.len() <= max { + return s; + } + let mut start = s.len() - max; + while !s.is_char_boundary(start) { + start += 1; + } + &s[start..] +} + fn partial_of(event: &AssistantMessageEvent) -> Option<&AssistantMessage> { match event { AssistantMessageEvent::Start { partial } @@ -381,3 +490,66 @@ fn partial_of(event: &AssistantMessageEvent) -> Option<&AssistantMessage> { _ => None, } } + +#[cfg(test)] +mod streaming_args_tests { + use super::*; + use crate::types::content::ContentBlock; + use std::collections::HashMap; + + #[test] + fn enrich_injects_tail_only_while_args_are_unparsed() { + let mut m = empty_assistant("p", "m"); + m.content = vec![ContentBlock::FunctionCall { + id: "c1".into(), + function_id: "agent_trigger".into(), + arguments: json!({ "function": "state::set" }), + }]; + + // No accumulated raw args → forwarded untouched. + assert!(enrich_streaming_args(&m, &HashMap::new()).is_none()); + + // Incomplete raw args → the live tail rides beside the salvaged + // fields so the UI can show the command being formed. + let mut acc = HashMap::new(); + acc.insert( + "c1".to_string(), + r#"{"function":"state::set","payload":{"value":"grow"#.to_string(), + ); + let enriched = enrich_streaming_args(&m, &acc).unwrap(); + let ContentBlock::FunctionCall { arguments, .. } = &enriched.content[0] else { + panic!("want function_call"); + }; + assert_eq!(arguments["function"], "state::set"); + assert!(arguments["_streaming"].as_str().unwrap().ends_with("grow")); + + // Complete raw args → the provider partial already carries them. + acc.insert( + "c1".to_string(), + r#"{"function":"state::set","payload":{}}"#.to_string(), + ); + assert!(enrich_streaming_args(&m, &acc).is_none()); + } + + #[test] + fn open_call_id_is_the_last_call_block_and_tail_is_char_safe() { + let mut m = empty_assistant("p", "m"); + m.content = vec![ + ContentBlock::FunctionCall { + id: "c1".into(), + function_id: "a".into(), + arguments: json!({}), + }, + ContentBlock::FunctionCall { + id: "c2".into(), + function_id: "b".into(), + arguments: json!({}), + }, + ]; + assert_eq!(open_call_id(&m), Some("c2")); + // Tail never slices mid-codepoint. + let s = format!("{}é", "x".repeat(1499)); + assert!(utf8_tail(&s, 1500).len() <= 1500); + assert!(utf8_tail(&s, 1500).ends_with('é')); + } +} diff --git a/harness/src/policy.rs b/harness/src/policy.rs index dedba3de4..e8a751c30 100644 --- a/harness/src/policy.rs +++ b/harness/src/policy.rs @@ -294,6 +294,23 @@ mod tests { assert_eq!(calls[0].arguments, json!({ "cmd": "ls" })); } + // A wrapper call whose arguments carry no resolvable `function` (null — + // e.g. a local model emitted unparseable args the provider degraded) + // keeps the wrapper name as its target: the dispatch loop matches that + // sentinel and fails locally instead of triggering `agent_trigger` on + // the engine (a guaranteed function_not_found). + #[test] + fn wrapper_call_without_target_keeps_wrapper_sentinel() { + let msg = assistant_with(vec![ContentBlock::FunctionCall { + id: "fc_1".into(), + function_id: AGENT_TRIGGER_NAME.into(), + arguments: Value::Null, + }]); + let calls = plan_calls(&msg, ExposeMode::AgentTrigger); + assert_eq!(calls[0].function_id, AGENT_TRIGGER_NAME); + assert_eq!(calls[0].kind, CallKind::Trigger); + } + #[test] fn flattened_agent_trigger_args_are_hoisted_into_payload() { // The model put the target's arguments beside `function` instead of diff --git a/harness/src/trigger.rs b/harness/src/trigger.rs index 717f94e4b..4025de479 100644 --- a/harness/src/trigger.rs +++ b/harness/src/trigger.rs @@ -135,6 +135,25 @@ pub fn denied_result(function_id: &str) -> ResultData { } } +/// The `is_error` result for an `agent_trigger` call with no resolvable +/// target — arguments were empty, null, or unparseable (local models emit +/// malformed JSON args). Dispatching the wrapper name to the engine would +/// only return a cryptic `function_not_found: agent_trigger`. +pub fn wrapper_without_target_result(arguments: &Value) -> ResultData { + let mut got = arguments.to_string(); + got.truncate(200); + let msg = format!( + "agent_trigger was called without a usable target (arguments were {got}); expected \ + {{\"function\": \"\", \"payload\": {{...}}}}. Re-issue the call with the \ + target function id." + ); + ResultData { + content: vec![ContentBlock::text(msg.clone())], + is_error: true, + details: json!({ "error": "agent_trigger_no_target", "message": msg }), + } +} + /// Normalise an arbitrary function return into content blocks. `details` /// always carries the raw value; content is a string render, an explicit /// `content` block array, or a compact JSON fallback. diff --git a/harness/src/turn_loop.rs b/harness/src/turn_loop.rs index 1c17e8278..f76ecddb5 100644 --- a/harness/src/turn_loop.rs +++ b/harness/src/turn_loop.rs @@ -435,6 +435,28 @@ pub async fn run_step( _ => {} } + // A call still named `agent_trigger` here means plan_calls found + // no resolvable `function` in its arguments (empty/null/ + // unparseable — local models flub JSON args). The wrapper is not + // an engine function; fail locally with a teachable error instead + // of the engine's cryptic function_not_found. + if call.function_id == policy::AGENT_TRIGGER_NAME { + let data = trigger::wrapper_without_target_result(&call.arguments); + let entry_id = ids::function_result_entry_id(&record.turn_id, &call.id); + append_function_result( + &session, + &record, + call, + &data, + &entry_id, + &origin(&record.turn_id), + ) + .await?; + mark_done(&mut record, &call.id, &entry_id); + crate::state::put_turn(&deps.iii, &record, cfg.session_timeout_ms).await?; + continue; + } + // Fail-closed glob policy first — structural and final. Hooks run // only after it passes (a denial never reaches a hook). if !policy.allows(&call.function_id) { From bfdb3a671d2e43287345f291a665691af7d8c501 Mon Sep 17 00:00:00 2001 From: Anderson Leal Date: Fri, 10 Jul 2026 12:33:26 -0300 Subject: [PATCH 03/11] (MOT-3950, MOT-3951, MOT-3952, MOT-3953) fix(llm-router): availability lifecycle, id-only models::get, ping/idle guard, args salvage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - registry/store.rs: restore resets every record to available:false (pessimistic — availability is proven by a send/refresh, not history); chat.rs heals it back on RelayResult::Done + emits provider::changed; register.rs boot-nudges every known provider's on_router_ready after READY (the engine drops router::ready bindings when the router disconnects, so re-declaration can't depend on them). (MOT-3950) - catalog: models::get treats an empty wire provider as unset and resolves id-only lookups when the id is unambiguous across slices — react turns (no provider on the spec) no longer fall to the 8k fallback window and compact every step. (MOT-3951) - chat/relay.rs: once content has started, only non-ping frames reset the idle budget — a provider pinging past a dead upstream trips Idle at idle_timeout_ms instead of zombie-ing to the engine's 300s stream_timeout ("stream ended without a terminal frame"). Pre-content pings still reset (slow first token behind keepalives stays legitimate). (MOT-3952) - types/messages.rs: degraded_arguments — shared salvage for unparseable tool-call args: recover the complete leading fields of a partial object, else carry the malformed text as {"_raw": …}; always an object (replay-safe), evidence preserved. (MOT-3953) --- llm-router/src/catalog/handlers.rs | 5 +- llm-router/src/catalog/queries.rs | 47 +++++++++++++- llm-router/src/chat/chat.rs | 12 ++++ llm-router/src/chat/relay.rs | 83 ++++++++++++++++++++++-- llm-router/src/register.rs | 28 +++++++- llm-router/src/registry/store.rs | 11 ++++ llm-router/src/types/messages.rs | 101 +++++++++++++++++++++++++++++ llm-router/tests/integration.rs | 18 +++++ 8 files changed, 297 insertions(+), 8 deletions(-) diff --git a/llm-router/src/catalog/handlers.rs b/llm-router/src/catalog/handlers.rs index 2c6a53307..39a45f582 100644 --- a/llm-router/src/catalog/handlers.rs +++ b/llm-router/src/catalog/handlers.rs @@ -39,7 +39,10 @@ pub fn make_models_get( move |req: ModelGetRequest| { let catalog = catalog.clone(); Box::pin(async move { - let model = models_get(&catalog, &req.provider, &req.id).await; + // provider defaults to "" on the wire — treat empty as unset so + // id-only lookups resolve when the id is unambiguous. + let provider = (!req.provider.is_empty()).then_some(req.provider.as_str()); + let model = models_get(&catalog, provider, &req.id).await; // null when unregistered (the cold-window signal) Ok(model.map(|model| ModelGetResponse { model })) }) diff --git a/llm-router/src/catalog/queries.rs b/llm-router/src/catalog/queries.rs index c73ef2b9b..8ef8bf4ca 100644 --- a/llm-router/src/catalog/queries.rs +++ b/llm-router/src/catalog/queries.rs @@ -33,8 +33,27 @@ pub async fn models_list( models } -pub async fn models_get(store: &CatalogStore, provider: &str, id: &str) -> Option { - store.get(provider, id).await +pub async fn models_get(store: &CatalogStore, provider: Option<&str>, id: &str) -> Option { + match provider { + Some(p) => store.get(p, id).await, + // Provider-less lookup: `router::chat` resolves these via routing, but + // metadata consumers (context-manager budgets a turn's context window + // through models::get) often only know the model id — e.g. a react + // spec carries no provider. Without this they silently fell to the + // conservative 8k fallback and compacted tiny sessions every step. + None => find_by_id(store.all().await, id), + } +} + +/// The unique catalog model with `id`, if exactly one provider serves it. +/// Ambiguous ids stay `None` (fail-open cold-window rule) — full precedence +/// resolution lives in routing, not here. +/// ponytail: unique-match only; thread routing::decide through if two +/// providers ever serve the same model id in practice. +fn find_by_id(models: Vec, id: &str) -> Option { + let mut matches = models.into_iter().filter(|m| m.id == id); + let first = matches.next()?; + matches.next().is_none().then_some(first) } /// Unknown model → false; request-shaping callers use models::get → null for @@ -75,6 +94,30 @@ mod tests { } } + // Provider-less models::get: unique ids resolve (context-manager budgets + // react turns that only know the model id — a null here silently throttles + // them to the 8k fallback window); ambiguous/unknown ids stay None. + #[test] + fn find_by_id_resolves_only_unambiguous_ids() { + let a = sonnet(); + let mut b = sonnet(); + b.provider = "other".into(); + let mut c = sonnet(); + c.id = "glm-5.2".into(); + c.provider = "zai".into(); + + // Unique id → resolves regardless of provider. + assert_eq!( + find_by_id(vec![a.clone(), c.clone()], "glm-5.2").map(|m| m.provider), + Some("zai".to_string()) + ); + // Two providers serving the same id → ambiguous → None. + assert_eq!(find_by_id(vec![a.clone(), b], "claude-sonnet-4"), None); + // Unknown id → None. + assert_eq!(find_by_id(vec![a], "nope"), None); + assert_eq!(find_by_id(vec![], "claude-sonnet-4"), None); + } + // Store-backed list/get/supports flows are exercised against a real engine // in tests/integration.rs; the capability mapping is pure and pinned here. #[test] diff --git a/llm-router/src/chat/chat.rs b/llm-router/src/chat/chat.rs index a22bc4b6f..1ba78cc3b 100644 --- a/llm-router/src/chat/chat.rs +++ b/llm-router/src/chat/chat.rs @@ -329,6 +329,18 @@ impl ChatPipeline { let AssistantMessageEvent::Done { message } = terminal else { unreachable!() }; + // A completed stream is definitive proof the provider is + // serving — heal a stale "down" flag (e.g. the boot-time + // reset in `RegistryStore::load`, or a past transient + // function_not_found) without waiting for a re-register. + if self.registry.set_availability(provider, true).await { + self.events + .emit( + triggers::PROVIDER_CHANGED, + json!({ "provider": provider, "op": "available" }), + ) + .await; + } return Ok(ChatResponse { ok: true, provider: provider.to_string(), diff --git a/llm-router/src/chat/relay.rs b/llm-router/src/chat/relay.rs index 6fcda416f..ca6a4a9e9 100644 --- a/llm-router/src/chat/relay.rs +++ b/llm-router/src/chat/relay.rs @@ -110,8 +110,15 @@ fn partial_of(ev: &AssistantMessageEvent) -> Option<&AssistantMessage> { } /// One provider attempt (design § chat flow step 5). Reads provider frames, -/// enforces the idle budget (any frame incl. ping resets it), fills cost_usd, -/// forwards in order, tracks the partial, classifies how the stream ended. +/// enforces the idle budget, fills cost_usd, forwards in order, tracks the +/// partial, classifies how the stream ended. +/// +/// Idle semantics: before any content, every frame (incl. ping) resets the +/// budget — a slow first token behind provider keepalives is legitimate +/// (local llama.cpp prompt eval). Once content has started, only non-ping +/// frames reset it: a provider pinging past a dead upstream must trip the +/// idle guard, not zombie on until the engine's stream_timeout kills the +/// call ("stream ended without a terminal frame"). pub async fn relay_frames( reader: &mut Box, sink: &dyn FrameSink, @@ -120,9 +127,20 @@ pub async fn relay_frames( let mut forwarded = false; // a non-ping frame reached the caller (gates retry) let mut partial: Option = None; let mut usage: Option = None; + let mut content_started = false; + let mut last_progress = std::time::Instant::now(); loop { - let read = reader.next(opts.idle).await; + let budget = if content_started { + opts.idle.saturating_sub(last_progress.elapsed()) + } else { + opts.idle + }; + let read = if budget.is_zero() { + ReadEvent::Timeout + } else { + reader.next(budget).await + }; if opts.aborted.load(Ordering::SeqCst) { reader.close(); @@ -154,6 +172,16 @@ pub async fn relay_frames( let Ok(ev) = serde_json::from_str::(&msg) else { continue; // malformed frame: skip, never fatal }; + if !matches!(ev, AssistantMessageEvent::Ping) { + last_progress = std::time::Instant::now(); + // Start carries an empty partial pre-content; it must not + // arm the strict budget or slow-first-token providers trip. + if partial_of(&ev).is_some() + && !matches!(ev, AssistantMessageEvent::Start { .. }) + { + content_started = true; + } + } if let Some(p) = partial_of(&ev) { partial = Some(p.clone()); } @@ -424,7 +452,8 @@ mod loop_tests { assert_eq!(p.unwrap().content.len(), 1); assert_eq!(usage.unwrap().input, Some(9)); - // idle: pings reset the clock and are forwarded but never count as content + // idle: pre-content pings reset the clock and are forwarded but never + // count as content (slow first token behind keepalives is legitimate) let provider = FakeChannel::new(); let caller = FakeChannel::new(); let (_, o) = opts(80); @@ -447,6 +476,52 @@ mod loop_tests { assert!(!forwarded); } + #[tokio::test] + async fn pings_do_not_extend_idle_once_content_started() { + // A stalled upstream behind a pinging provider must trip the idle + // guard — not zombie on until the engine's stream_timeout kills the + // call as "stream ended without a terminal frame". + let provider = FakeChannel::new(); + let caller = FakeChannel::new(); + send( + &provider, + &AssistantMessageEvent::Start { + partial: partial(""), + }, + ); + send( + &provider, + &AssistantMessageEvent::TextDelta { + partial: partial("hi"), + delta: "hi".into(), + }, + ); + let writer = provider.writer.clone(); + let feeder = tokio::spawn(async move { + loop { + tokio::time::sleep(Duration::from_millis(20)).await; + if writer + .send(&serde_json::to_string(&AssistantMessageEvent::Ping).unwrap()) + .is_err() + { + break; + } + } + }); + let (_, o) = opts(100); + let mut reader: Box = Box::new(provider.reader); + let result = relay_frames(&mut reader, &caller.writer.clone(), &o).await; + feeder.abort(); + let RelayResult::NoTerminal { + reason, forwarded, .. + } = result + else { + panic!("want no-terminal, got {result:?}") + }; + assert!(matches!(reason, NoTerminalReason::Idle)); + assert!(forwarded); + } + #[tokio::test] async fn caller_gone_closes_the_provider_reader_and_abort_is_observed() { let provider = FakeChannel::new(); diff --git a/llm-router/src/register.rs b/llm-router/src/register.rs index dcdec6990..ee223efa9 100644 --- a/llm-router/src/register.rs +++ b/llm-router/src/register.rs @@ -11,7 +11,7 @@ use std::collections::BTreeMap; use std::sync::{Arc, RwLock}; use iii_sdk::errors::Error; -use iii_sdk::protocol::RegisterTriggerInput; +use iii_sdk::protocol::{RegisterTriggerInput, TriggerRequest}; use iii_sdk::{IIIClient, RegisterFunction}; use serde_json::{json, Value}; @@ -218,6 +218,32 @@ pub async fn register_router(iii: IIIClient) -> Result { // 7. ready — providers re-declare on this events.emit(crate::triggers::READY, json!({})).await; + // The ready fan-out only reaches providers whose `router::ready` binding + // still exists — and the engine drops those bindings when THIS worker + // (the trigger type's owner) disconnects, so providers that outlived a + // router restart never hear it. Nudge every restored provider directly: + // `provider::::on_router_ready` is the deterministic per-provider + // handler (same one the fan-out targets), a live provider re-declares — + // flipping the boot-reset availability back up — and a dead one is + // function_not_found, which is exactly the right answer. Detached: boot + // must not block on provider round-trips. + { + let iii = iii.clone(); + let ids = registry.ids().await; + tokio::spawn(async move { + for id in ids { + let _ = iii + .trigger(TriggerRequest { + function_id: format!("provider::{id}::on_router_ready"), + payload: json!({}), + action: None, + timeout_ms: Some(10_000), + }) + .await; + } + }); + } + Ok(RouterRefs { registry, catalog, diff --git a/llm-router/src/registry/store.rs b/llm-router/src/registry/store.rs index 6390c8832..0f38b4322 100644 --- a/llm-router/src/registry/store.rs +++ b/llm-router/src/registry/store.rs @@ -107,6 +107,17 @@ impl RegistryStore { }; let mut records = self.records.lock().await; *records = serde_json::from_value(stored).unwrap_or_default(); + // Persisted availability is stale across a router restart: a provider + // that died while the router was down would be restored as "up" and + // stay wrongly listed until a dispatch burned on it (F9: topology + // events can't be resolved to providers). Restore every record DOWN + // and let the sources of truth flip it up: providers re-declare on + // `router::ready` (upsert emits the op:"available" recovery), and a + // successful dispatch heals it too (chat.rs `Done` arm). Not persisted + // here — flags re-persist on their next real change. + for rec in records.values_mut() { + rec.available = false; + } Ok(()) } diff --git a/llm-router/src/types/messages.rs b/llm-router/src/types/messages.rs index 464773902..fe56f40bd 100644 --- a/llm-router/src/types/messages.rs +++ b/llm-router/src/types/messages.rs @@ -139,9 +139,110 @@ pub fn reorder_displaced_results(messages: &[AgentMessage]) -> Vec<&AgentMessage out } +/// Degrade unparseable tool-call arguments to a safe OBJECT placeholder. +/// +/// Providers must never emit null or a bare string for a call's arguments — +/// a non-object `tool_use.input` is a hard Anthropic 400 once the turn is +/// replayed (sessions switch providers). Two-step recovery: +/// +/// 1. Salvage the complete leading fields of a partial object — a call cut +/// mid-stream (`{"function":"state::set","payload":{"key":`) keeps its +/// known prefix (`{"function":"state::set"}`), so long-streaming calls +/// stay identifiable in UIs instead of rendering as an anonymous `{}`. +/// 2. Otherwise carry the malformed text as `{"_raw": }` so the +/// evidence of what the model actually sent survives for rendering and +/// for the harness's teachable no-target error. +pub fn degraded_arguments(args_json: &str) -> serde_json::Value { + if let Some(map) = salvage_leading_object_fields(args_json) { + return serde_json::Value::Object(map); + } + serde_json::json!({ "_raw": utf8_head(args_json, 2048) }) +} + +/// The complete leading fields of a partial JSON object string, if any. +/// Scans only the first 4KB (a partial is rebuilt per stream event, so this +/// must stay O(head); leading fields live in the prefix), cutting at +/// top-level commas and taking the longest prefix that parses once closed. +pub fn salvage_leading_object_fields( + args: &str, +) -> Option> { + let head = utf8_head(args, 4096); + let bytes = head.as_bytes(); + let (mut depth, mut in_str, mut esc, mut started) = (0usize, false, false, false); + let mut cuts: Vec = Vec::new(); + for (i, &b) in bytes.iter().enumerate() { + if esc { + esc = false; + continue; + } + match b { + b'\\' if in_str => esc = true, + b'"' => in_str = !in_str, + _ if in_str => {} + b'{' | b'[' => { + depth += 1; + started = true; + } + b'}' | b']' => depth = depth.saturating_sub(1), + b',' if depth == 1 => cuts.push(i), + _ => {} + } + } + if !started { + return None; + } + // Longest salvageable prefix wins; candidates are few (top-level commas). + for &cut in cuts.iter().rev() { + let candidate = format!("{}}}", &head[..cut]); + if let Ok(serde_json::Value::Object(map)) = serde_json::from_str(&candidate) { + if !map.is_empty() { + return Some(map); + } + } + } + None +} + +fn utf8_head(s: &str, max: usize) -> &str { + if s.len() <= max { + return s; + } + let mut end = max; + while !s.is_char_boundary(end) { + end -= 1; + } + &s[..end] +} + #[cfg(test)] mod tests { use super::*; + + #[test] + fn degraded_arguments_salvages_leading_fields_or_keeps_raw() { + // Mid-stream cut: the known prefix survives as a real object. + assert_eq!( + degraded_arguments(r#"{"function":"state::set","payload":{"key":"art"#), + serde_json::json!({ "function": "state::set" }) + ); + // Longest complete prefix wins, nested commas/strings don't cut. + assert_eq!( + degraded_arguments( + r#"{"function":"a::b","payload":{"x":"1,2","y":[3,4]},"extra":{"cut":"# + ), + serde_json::json!({ "function": "a::b", "payload": { "x": "1,2", "y": [3, 4] } }) + ); + // Nothing salvageable (no complete top-level field yet, or not JSON): + // the raw text survives as evidence, always inside an object. + assert_eq!( + degraded_arguments(r#"{"function":"state::se"#), + serde_json::json!({ "_raw": r#"{"function":"state::se"# }) + ); + assert_eq!( + degraded_arguments("{'key': 'v'}"), + serde_json::json!({ "_raw": "{'key': 'v'}" }) + ); + } use crate::types::events::StopReason; fn assistant(content: Vec) -> AgentMessage { diff --git a/llm-router/tests/integration.rs b/llm-router/tests/integration.rs index cf1d18274..36ec5ce7e 100644 --- a/llm-router/tests/integration.rs +++ b/llm-router/tests/integration.rs @@ -549,6 +549,14 @@ async fn registry_survives_a_router_restart_and_token_stays_bound() { .await .expect("provider list"); assert_eq!(list["providers"][0]["id"], "real", "list: {list}"); + // Availability is NOT trusted across a restart: the persisted "up" flag + // could belong to a provider that died while the router was down. It is + // restored DOWN and only re-declaration / a successful dispatch flip it up. + assert_eq!( + list["providers"][0]["available"], + json!(false), + "restored availability must be pessimistic: {list}" + ); // re-declare with the original token: idempotent, same token accepted let again = call( @@ -560,6 +568,16 @@ async fn registry_survives_a_router_restart_and_token_stays_bound() { .expect("re-declare accepted"); assert_eq!(again["registration_token"], json!(provider.token.clone())); + // …and the re-declaration is what brings the provider back up. + let list = call(&second, "router::provider::list", json!({})) + .await + .expect("provider list after re-declare"); + assert_eq!( + list["providers"][0]["available"], + json!(true), + "re-declare restores availability: {list}" + ); + second.shutdown(); } From 6206176e76d14802cf392bb26eb4553a4c4fe2d3 Mon Sep 17 00:00:00 2001 From: Anderson Leal Date: Fri, 10 Jul 2026 12:33:47 -0300 Subject: [PATCH 04/11] (MOT-3952, MOT-3953) fix(providers): read_timeout on upstream clients + salvage degraded tool args MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - read_timeout(120s) on the upstream reqwest client in the five cloud providers: a stalled connection otherwise pings the router past its idle guard until the engine kills the call at stream_timeout. Bounds silence between reads, not stream length (healthy streams emit SSE pings). llamacpp deliberately skipped — no keepalives and legitimately slow prompt eval. (MOT-3952) - all six providers degrade unparseable tool-call arguments through the shared llm_router degraded_arguments: mid-stream partials keep their known leading fields (a long-streaming call stays identifiable as ƒ state::set instead of an anonymous {}), malformed finals keep the raw text as {"_raw": …} evidence — always an object, so cross-provider replay can't 400 the way null/bare-string did. (MOT-3953) --- provider-anthropic/src/register.rs | 8 +++-- provider-anthropic/src/sse.rs | 15 +++++---- provider-llamacpp/src/sse.rs | 44 ++++++++++++++++++++++++++- provider-openai-codex/src/register.rs | 3 ++ provider-openai-codex/src/sse.rs | 6 +++- provider-openai/src/register.rs | 6 ++-- provider-openai/src/sse.rs | 6 +++- provider-xai/src/register.rs | 6 ++-- provider-xai/src/sse.rs | 6 +++- provider-zai/src/register.rs | 6 ++-- provider-zai/src/sse.rs | 6 +++- 11 files changed, 93 insertions(+), 19 deletions(-) diff --git a/provider-anthropic/src/register.rs b/provider-anthropic/src/register.rs index 72034a1c9..46c9320c5 100644 --- a/provider-anthropic/src/register.rs +++ b/provider-anthropic/src/register.rs @@ -107,10 +107,14 @@ pub async fn declare_and_refresh(iii: IIIClient, http: reqwest::Client) { } pub async fn register_provider(iii: IIIClient) -> Result<(), Error> { - // Streaming uses no total timeout (the router owns stream budgets); - // connect failures surface fast. + // Streaming uses no total timeout (the router owns stream budgets), but + // reads are silence-bounded: a stalled upstream otherwise pings the router + // past its idle guard until the engine kills the call at stream_timeout + // ("stream ended without a terminal frame"). Healthy streams emit SSE + // pings, so 120s of socket silence means a dead connection. let http = reqwest::Client::builder() .connect_timeout(Duration::from_secs(10)) + .read_timeout(Duration::from_secs(120)) .build() .expect("reqwest client"); diff --git a/provider-anthropic/src/sse.rs b/provider-anthropic/src/sse.rs index ee2f589f8..652aa53f8 100644 --- a/provider-anthropic/src/sse.rs +++ b/provider-anthropic/src/sse.rs @@ -109,18 +109,21 @@ fn push_block_content( } BlockKind::ToolUse => { if let Some(tc) = state.function_calls.get(idx) { - // An interrupted stream leaves `args_json` as a partial, - // unparseable blob (e.g. `{"cmd":`). A tool_use input must be a - // JSON object, so degrade anything that fails to parse to `{}` - // rather than null — a null input is a hard Anthropic 400 once - // the aborted turn is replayed. + // An interrupted/in-flight stream leaves `args_json` as a + // partial, unparseable blob (e.g. `{"cmd":`). A tool_use + // input must be a JSON object (null/string is a hard 400 on + // replay), so degrade to the salvaged leading fields — long- + // streaming calls keep their known prefix (`function` target) + // instead of an anonymous `{}` — else `{"_raw": …}`. let arguments = if tc.args_json.is_empty() { serde_json::json!({}) } else { serde_json::from_str(&tc.args_json) .ok() .filter(Value::is_object) - .unwrap_or_else(|| serde_json::json!({})) + .unwrap_or_else(|| { + llm_router::types::messages::degraded_arguments(&tc.args_json) + }) }; out.push(ContentBlock::FunctionCall { id: tc.id.clone(), diff --git a/provider-llamacpp/src/sse.rs b/provider-llamacpp/src/sse.rs index 6d339bb48..50b7e5be4 100644 --- a/provider-llamacpp/src/sse.rs +++ b/provider-llamacpp/src/sse.rs @@ -95,7 +95,11 @@ fn build_content(state: &PartialState) -> Vec { let arguments = if fc.args_json.is_empty() { serde_json::json!({}) } else { - serde_json::from_str(&fc.args_json).unwrap_or(Value::Null) + // Unparseable args (mid-stream partials, local models misquoting + // JSON) degrade to the salvaged leading fields or `{"_raw": …}` — + // always an object (replay-safe) that preserves the evidence. + serde_json::from_str(&fc.args_json) + .unwrap_or_else(|_| llm_router::types::messages::degraded_arguments(&fc.args_json)) }; out.push(ContentBlock::FunctionCall { id: fc.id.clone(), @@ -461,6 +465,44 @@ mod tests { } } + // Local models misquote JSON args; the malformed text must survive as + // `{"_raw": …}` evidence (a null would erase what the model actually + // sent, leaving an undiagnosable empty call — and the harness turns + // null wrapper args into a literal `agent_trigger` dispatch). + #[test] + fn malformed_args_survive_as_raw_evidence() { + let (state, _) = run(&[ + json!({"choices":[{"index":0,"delta":{"tool_calls":[ + {"index":0,"id":"call_1","type":"function","function":{"name":"state__set","arguments":"{'key':"}}]}}]}), + json!({"choices":[{"index":0,"delta":{"tool_calls":[ + {"index":0,"function":{"arguments":"'v'}"}}]}}]}), + json!({"choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}]}), + ]); + let final_msg = build_final(&state, "llama-test"); + match &final_msg.content[0] { + ContentBlock::FunctionCall { arguments, .. } => { + assert_eq!(arguments, &json!({ "_raw": "{'key':'v'}" })); + } + other => panic!("want function_call, got {other:?}"), + } + } + + // A wrapper call cut mid-payload keeps its already-known leading fields — + // the console can show `ƒ state::set` while a huge payload streams. + #[test] + fn partial_args_salvage_leading_fields() { + let (state, _) = run(&[json!({"choices":[{"index":0,"delta":{"tool_calls":[ + {"index":0,"id":"call_1","type":"function","function":{"name":"agent_trigger", + "arguments":"{\"function\":\"state::set\",\"payload\":{\"key\":\"article\",\"value\":\"long"}}]}}]})]); + let partial = build_partial(&state, "llama-test"); + match &partial.content[0] { + ContentBlock::FunctionCall { arguments, .. } => { + assert_eq!(arguments["function"], "state::set"); + } + other => panic!("want function_call, got {other:?}"), + } + } + #[test] fn text_then_tool_calls_closes_text_block_first() { let (state, events) = run(&[ diff --git a/provider-openai-codex/src/register.rs b/provider-openai-codex/src/register.rs index 60427dd0b..178f7ea39 100644 --- a/provider-openai-codex/src/register.rs +++ b/provider-openai-codex/src/register.rs @@ -104,8 +104,11 @@ pub async fn declare_and_refresh(iii: IIIClient, http: reqwest::Client) { } pub async fn register_provider(iii: IIIClient) -> Result<(), Error> { + // Reads are silence-bounded: a stalled upstream otherwise pings the router + // past its idle guard until the engine kills the call at stream_timeout. let http = reqwest::Client::builder() .connect_timeout(Duration::from_secs(10)) + .read_timeout(Duration::from_secs(120)) .build() .expect("reqwest client"); diff --git a/provider-openai-codex/src/sse.rs b/provider-openai-codex/src/sse.rs index 0b7c2e50b..46b5a9039 100644 --- a/provider-openai-codex/src/sse.rs +++ b/provider-openai-codex/src/sse.rs @@ -98,7 +98,11 @@ fn build_content(state: &PartialState) -> Vec { let arguments = if tc.args_json.is_empty() { serde_json::json!({}) } else { - serde_json::from_str(&tc.args_json).unwrap_or(Value::Null) + // Unparseable args (mid-stream partials, malformed JSON) degrade + // to the salvaged leading fields or `{"_raw": …}` — always an + // object (replay-safe) that preserves the evidence. + serde_json::from_str(&tc.args_json) + .unwrap_or_else(|_| llm_router::types::messages::degraded_arguments(&tc.args_json)) }; out.push(ContentBlock::FunctionCall { id: tc.id.clone(), diff --git a/provider-openai/src/register.rs b/provider-openai/src/register.rs index 02361dc45..4583a749d 100644 --- a/provider-openai/src/register.rs +++ b/provider-openai/src/register.rs @@ -107,10 +107,12 @@ pub async fn declare_and_refresh(iii: IIIClient, http: reqwest::Client) { } pub async fn register_provider(iii: IIIClient) -> Result<(), Error> { - // Streaming uses no total timeout (the router owns stream budgets); - // connect failures surface fast. + // Streaming uses no total timeout (the router owns stream budgets), but + // reads are silence-bounded: a stalled upstream otherwise pings the router + // past its idle guard until the engine kills the call at stream_timeout. let http = reqwest::Client::builder() .connect_timeout(Duration::from_secs(10)) + .read_timeout(Duration::from_secs(120)) .build() .expect("reqwest client"); diff --git a/provider-openai/src/sse.rs b/provider-openai/src/sse.rs index bc203b842..d222dc2b5 100644 --- a/provider-openai/src/sse.rs +++ b/provider-openai/src/sse.rs @@ -84,7 +84,11 @@ fn build_content(state: &PartialState) -> Vec { let arguments = if fc.args_json.is_empty() { serde_json::json!({}) } else { - serde_json::from_str(&fc.args_json).unwrap_or(Value::Null) + // Unparseable args (mid-stream partials, malformed JSON) degrade + // to the salvaged leading fields or `{"_raw": …}` — always an + // object (replay-safe) that preserves the evidence. + serde_json::from_str(&fc.args_json) + .unwrap_or_else(|_| llm_router::types::messages::degraded_arguments(&fc.args_json)) }; out.push(ContentBlock::FunctionCall { id: fc.id.clone(), diff --git a/provider-xai/src/register.rs b/provider-xai/src/register.rs index d3a36598b..b23d2d1c4 100644 --- a/provider-xai/src/register.rs +++ b/provider-xai/src/register.rs @@ -107,10 +107,12 @@ pub async fn declare_and_refresh(iii: IIIClient, http: reqwest::Client) { } pub async fn register_provider(iii: IIIClient) -> Result<(), Error> { - // Streaming uses no total timeout (the router owns stream budgets); - // connect failures surface fast. + // Streaming uses no total timeout (the router owns stream budgets), but + // reads are silence-bounded: a stalled upstream otherwise pings the router + // past its idle guard until the engine kills the call at stream_timeout. let http = reqwest::Client::builder() .connect_timeout(Duration::from_secs(10)) + .read_timeout(Duration::from_secs(120)) .build() .expect("reqwest client"); diff --git a/provider-xai/src/sse.rs b/provider-xai/src/sse.rs index e3d8047d2..7bb3dab11 100644 --- a/provider-xai/src/sse.rs +++ b/provider-xai/src/sse.rs @@ -94,7 +94,11 @@ fn build_content(state: &PartialState) -> Vec { let arguments = if fc.args_json.is_empty() { serde_json::json!({}) } else { - serde_json::from_str(&fc.args_json).unwrap_or(Value::Null) + // Unparseable args (mid-stream partials, malformed JSON) degrade + // to the salvaged leading fields or `{"_raw": …}` — always an + // object (replay-safe) that preserves the evidence. + serde_json::from_str(&fc.args_json) + .unwrap_or_else(|_| llm_router::types::messages::degraded_arguments(&fc.args_json)) }; out.push(ContentBlock::FunctionCall { id: fc.id.clone(), diff --git a/provider-zai/src/register.rs b/provider-zai/src/register.rs index 3f524e0c2..200f25ce1 100644 --- a/provider-zai/src/register.rs +++ b/provider-zai/src/register.rs @@ -110,10 +110,12 @@ pub async fn declare_and_refresh(iii: IIIClient) { } pub async fn register_provider(iii: IIIClient) -> Result<(), Error> { - // Streaming uses no total timeout (the router owns stream budgets); - // connect failures surface fast. + // Streaming uses no total timeout (the router owns stream budgets), but + // reads are silence-bounded: a stalled upstream otherwise pings the router + // past its idle guard until the engine kills the call at stream_timeout. let http = reqwest::Client::builder() .connect_timeout(Duration::from_secs(10)) + .read_timeout(Duration::from_secs(120)) .build() .expect("reqwest client"); diff --git a/provider-zai/src/sse.rs b/provider-zai/src/sse.rs index 067b53297..0d876c3dc 100644 --- a/provider-zai/src/sse.rs +++ b/provider-zai/src/sse.rs @@ -94,7 +94,11 @@ fn build_content(state: &PartialState) -> Vec { let arguments = if fc.args_json.is_empty() { serde_json::json!({}) } else { - serde_json::from_str(&fc.args_json).unwrap_or(Value::Null) + // Unparseable args (mid-stream partials, malformed JSON) degrade + // to the salvaged leading fields or `{"_raw": …}` — always an + // object (replay-safe) that preserves the evidence. + serde_json::from_str(&fc.args_json) + .unwrap_or_else(|_| llm_router::types::messages::degraded_arguments(&fc.args_json)) }; out.push(ContentBlock::FunctionCall { id: fc.id.clone(), From 3b5d67770e437ab473181627bd1996ca75b6d1f9 Mon Sep 17 00:00:00 2001 From: Anderson Leal Date: Fri, 10 Jul 2026 12:33:47 -0300 Subject: [PATCH 05/11] (MOT-3949, MOT-3950, MOT-3953, MOT-3954) feat(console): fired-trigger visibility, error surfacing, streaming args pane, re-hydration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fired triggers stay visible: trigger_fired chat notices (no new turn), panel ghosts "fired · unregistered" with dismiss + counts, full-row retention so the workflow DAG survives retirement; spawn seed tasks render like reaction tasks; sidebar Zap/Bot provenance icons from spawned_by. (MOT-3949) - Error surfacing: session::messages read-backs deliver custom entries as role:"custom" — the mapper handles both wire shapes via one dispatcher, so error/turn-failed/compaction/trigger-fired entries actually render; stream-rescue aborts the spinner when the session errors; the waiting shimmer shows the under-the-hood detail; model picker disables models of unavailable providers ("not loaded") and refreshes on router::provider::changed. (MOT-3950) - Live tool args: unwrap the harness `_streaming` tail and render a "request · streaming…" pane in FunctionCallCard while a call's arguments form. (MOT-3953) - markBackgroundedStale: backgrounded sessions re-hydrate on activation, so entries frozen mid-snapshot during the away-gap self-repair from durable truth. (MOT-3954) --- console/web/src/components/chat/ChatView.tsx | 59 +++++- console/web/src/components/chat/Message.tsx | 42 ++++ .../web/src/components/chat/MessageList.tsx | 7 +- .../web/src/components/chat/ModelPicker.tsx | 18 +- .../src/components/chat/SessionTriggers.tsx | 121 ++++++++--- .../function-call/FunctionCallCard.tsx | 39 +++- .../components/sidebar/ConversationRow.tsx | 20 +- .../web/src/hooks/use-conversations.test.ts | 47 +++++ console/web/src/hooks/use-conversations.ts | 39 ++++ .../web/src/hooks/use-model-picker-source.ts | 24 ++- console/web/src/lib/backend/triggers.test.ts | 135 ++++++++++++ console/web/src/lib/backend/triggers.ts | 80 +++++++ console/web/src/lib/models-catalog.ts | 69 ++++-- .../web/src/lib/sessions/entry-mapper.test.ts | 196 ++++++++++++++++++ console/web/src/lib/sessions/entry-mapper.ts | 148 ++++++++++++- console/web/src/types/chat.ts | 42 +++- 16 files changed, 1015 insertions(+), 71 deletions(-) create mode 100644 console/web/src/lib/backend/triggers.test.ts diff --git a/console/web/src/components/chat/ChatView.tsx b/console/web/src/components/chat/ChatView.tsx index aa426658d..4cda63f86 100644 --- a/console/web/src/components/chat/ChatView.tsx +++ b/console/web/src/components/chat/ChatView.tsx @@ -19,7 +19,10 @@ import { useWorktreeBinding } from '@/hooks/use-worktree-binding' import { useWorktreeEvents } from '@/hooks/use-worktree-events' import type { ChatBackend } from '@/lib/backend' import { predictedUserEntryId } from '@/lib/backend/harness-send' -import type { SessionTriggerInfo } from '@/lib/backend/triggers' +import { + mergeFiredTriggers, + type SessionTriggerInfo, +} from '@/lib/backend/triggers' import type { CompactResult, QueuedMessagePreview } from '@/lib/backend/types' import { useConversationsCtxOptional } from '@/lib/conversations-context' import { expandFileMentions, parseFileMentions } from '@/lib/file-mentions' @@ -54,6 +57,7 @@ import { type SystemMessage, type ThinkingLevel, type ThoughtMessage, + type TriggerFiredData, type UserMessage, } from '@/types/chat' import { Composer, type ComposerSubmitPayload } from './Composer' @@ -232,15 +236,24 @@ export function ChatView({ const [sessionTriggers, setSessionTriggers] = useState( [], ) + // Every full row this tab has EVER polled, by engine trigger id. When a + // once/join binding fires and retires, the poll drops it — this cache lets + // the fired ghost keep its full metadata (join grouping, spawn pin, task) + // so the workflow strip and flow DAG survive the pipeline completing. + const seenTriggersRef = useRef>(new Map()) const refreshTriggers = useCallback(() => { const listTriggers = backend.listTriggers if (!listTriggers) return listTriggers(conversation.id) - .then(setSessionTriggers) + .then((rows) => { + for (const row of rows) seenTriggersRef.current.set(row.id, row) + setSessionTriggers(rows) + }) .catch(() => {}) }, [backend.listTriggers, conversation.id]) useEffect(() => { if (!backend.listTriggers) return + seenTriggersRef.current = new Map() setSessionTriggers([]) refreshTriggers() const timer = window.setInterval(refreshTriggers, 5000) @@ -295,6 +308,28 @@ export function ChatView({ refreshTriggers, ]) + // Fired-trigger history: durable `trigger_fired` transcript entries (mapped to + // system messages). Drives the panel's fired/unregistered ghost rows so a + // once-trigger stays visible after the engine drops it from the poll. + const firedTriggers = useMemo(() => { + const out: TriggerFiredData[] = [] + for (const m of conversation.messages) { + if (m.role === 'system' && m.kind === 'trigger-fired' && m.trigger) { + out.push(m.trigger) + } + } + return out + }, [conversation.messages]) + const mergedTriggers = useMemo( + () => + mergeFiredTriggers( + sessionTriggers, + firedTriggers, + seenTriggersRef.current, + ), + [sessionTriggers, firedTriggers], + ) + // The strip's rows: this tab's drafts first, then server-queued rows not // already covered by a draft or an arrived transcript row (a stale poll // must not re-show a message that just drained into the chat). @@ -1088,6 +1123,17 @@ export function ChatView({ void backend.abortRun?.(sessionId).catch(() => {}) }, [backend, sessionId]) + // Rescue a parked stream loop: the session hit a terminal error server-side + // (status-changed arrives on the session-directory subscription) but the + // local `for await` is still waiting on a turn-completed that may never + // come. Abort locally — the generator returns silently on abort, and the + // red notice renders from the transcript's durable `error` entry. + useEffect(() => { + if (isStreaming && conversation.status === 'error') { + abortRef.current?.abort() + } + }, [isStreaming, conversation.status]) + // Covers the gap between submit / fcall-end and the next streamed content, // where the assistant/thought shimmer hasn't yet rendered. const isThinking = @@ -1315,6 +1361,13 @@ export function ChatView({ ) : null} ) : message.reaction ? ( + ) : message.spawn ? ( + ) : ( ) @@ -92,6 +94,8 @@ export function Message({ case 'system': return message.kind === 'compaction' ? ( + ) : message.kind === 'trigger-fired' ? ( + ) : ( ) @@ -161,6 +165,26 @@ function NotificationMessage({ message }: { message: UserMessageType }) { ) } +/** + * A subscription fire (`kind: 'trigger-fired'`): a turn-less notice that a + * registered trigger fired — a state/cron spawn, a notify wake, or a join + * edge. `message.content` is the pre-rendered one-liner (name · action). + */ +function TriggerFiredNotice({ message }: { message: SystemMessageType }) { + return ( +
+ + + + trigger fired + + {' · '} + {message.content} + +
+ ) +} + /** * The one-line hint for a reaction's collapsed payload: the firing session * and status for an event, the predecessor keys for a join's inputs. @@ -221,6 +245,24 @@ function ReactionTaskMessage({ message }: { message: UserMessageType }) { ) } +/** + * A direct `harness::spawn` seed task (`spawn: true`): the sub-agent's opening + * input, but sent by the PARENT agent — labeled and left-aligned like a + * reaction task so it never reads as something the human typed. + */ +function SpawnTaskMessage({ message }: { message: UserMessageType }) { + return ( +
+
+ spawn · sub-agent task +
+
+ {message.content} +
+
+ ) +} + function UserMessage({ message }: { message: UserMessageType }) { return (
diff --git a/console/web/src/components/chat/MessageList.tsx b/console/web/src/components/chat/MessageList.tsx index c03f45c2f..4a203eba1 100644 --- a/console/web/src/components/chat/MessageList.tsx +++ b/console/web/src/components/chat/MessageList.tsx @@ -16,6 +16,10 @@ interface MessageListProps { visible outputs (after submit, or between fcall-end and the next turn's first token). */ isThinking?: boolean + /** Under-the-hood context shown as the waiting shimmer (e.g. "dispatching + zai::glm-5.2" or the session's status_reason). Falls back to "thinking…" + when absent. */ + thinkingDetail?: string density?: 'route' | 'dock' onResolveApproval?: ( sessionId: string, @@ -84,6 +88,7 @@ function groupConsecutiveFcalls(messages: MessageType[]): RenderItem[] { export function MessageList({ messages, isThinking, + thinkingDetail, density = 'route', onResolveApproval, onAlwaysAllow, @@ -187,7 +192,7 @@ export function MessageList({ )} {isThinking ? (
- thinking… + {thinkingDetail ?? 'thinking…'}
) : null}
diff --git a/console/web/src/components/chat/ModelPicker.tsx b/console/web/src/components/chat/ModelPicker.tsx index 77c2da55d..7653d6124 100644 --- a/console/web/src/components/chat/ModelPicker.tsx +++ b/console/web/src/components/chat/ModelPicker.tsx @@ -82,6 +82,13 @@ export function ModelPicker({ const presentIds = ctx?.presentProviders.map((p) => p.id) ?? [] const presentSet = new Set(presentIds) + // Providers the router declares but whose worker is not loaded — their + // catalog models would only fail with `provider_unavailable` at dispatch. + const unavailableSet = new Set( + (ctx?.presentProviders ?? []) + .filter((p) => p.available === false) + .map((p) => p.id), + ) const optionsById = useMemo( () => new Map(options.map((o) => [o.id, o])), @@ -163,7 +170,8 @@ export function ModelPicker({ > {groups.map((g) => { - const unconfigured = g.options.length === 0 + const unavailable = unavailableSet.has(g.label) + const unconfigured = !unavailable && g.options.length === 0 return (
@@ -171,7 +179,11 @@ export function ModelPicker({ {g.label} - {unconfigured ? ( + {unavailable ? ( + + not loaded + + ) : unconfigured ? ( not configured @@ -202,10 +214,12 @@ export function ModelPicker({
diff --git a/console/web/src/components/chat/SessionTriggers.tsx b/console/web/src/components/chat/SessionTriggers.tsx index 94e045736..f03276f43 100644 --- a/console/web/src/components/chat/SessionTriggers.tsx +++ b/console/web/src/components/chat/SessionTriggers.tsx @@ -180,7 +180,9 @@ function TriggerRow({ const task = reactTask(trigger) const name = memberKey ?? trigger.label ?? null return ( -
+
{connector ? ( {connector} @@ -208,7 +210,11 @@ function TriggerRow({ ) : null} {stateNote ? ` · ${stateNote}` : ''} - {trigger.once ? ' · once' : ''} + {trigger.fired + ? ' · fired · unregistered' + : trigger.once + ? ' · once' + : ''} @@ -245,6 +252,22 @@ export function SessionTriggers({ const [flowOpen, setFlowOpen] = useState(false) const [clearArming, setClearArming] = useState(false) const [clearing, setClearing] = useState(false) + // Fired ghost rows the user dismissed — local per-tab view state; they + // resurrect from the transcript on reload, so no persistence needed. + const [dismissed, setDismissed] = useState>(() => new Set()) + + const visibleTriggers = useMemo( + () => triggers.filter((t) => !dismissed.has(t.id)), + [triggers, dismissed], + ) + // Still-registered bindings only — the flow DAG and the counts must not + // include fired ghosts, which are history, not pipeline structure. + const liveTriggers = useMemo( + () => visibleTriggers.filter((t) => !t.fired), + [visibleTriggers], + ) + const registeredCount = liveTriggers.length + const firedCount = visibleTriggers.length - registeredCount // The DAG probes presence for every state binding, not just the visible // rows, so the flow view can color unwritten roots even while collapsed. @@ -257,7 +280,7 @@ export function SessionTriggers({ useEffect(() => { if (!probeKeys || !checkStateKey) return let alive = true - for (const trigger of triggers) { + for (const trigger of visibleTriggers) { const watch = stateWatch(trigger) if (!watch) continue void checkStateKey(watch.scope, watch.key).then((present) => { @@ -268,7 +291,7 @@ export function SessionTriggers({ return () => { alive = false } - }, [probeKeys, checkStateKey, triggers]) + }, [probeKeys, checkStateKey, visibleTriggers]) const stateNote = (trigger: SessionTriggerInfo): string | null => { const watch = stateWatch(trigger) @@ -278,8 +301,11 @@ export function SessionTriggers({ if (present === undefined) return `on ${label}` return present ? `on ${label} — written` : `on ${label} — not written yet` } - const workflow = useMemo(() => buildTriggerWorkflow(triggers), [triggers]) - const selected = triggers.find((t) => t.id === selectedId) ?? null + const workflow = useMemo( + () => buildTriggerWorkflow(visibleTriggers), + [visibleTriggers], + ) + const selected = visibleTriggers.find((t) => t.id === selectedId) ?? null const selectedIsReact = selected?.functionId === 'harness::react' const selectedMetadata = selected ? remainingMetadata(selected.metadata, selectedIsReact) @@ -294,7 +320,7 @@ export function SessionTriggers({ ? selected.metadata.provider : null - if (triggers.length === 0) return null + if (visibleTriggers.length === 0) return null const unregister = async (id: string) => { setBusyId(id) @@ -306,10 +332,27 @@ export function SessionTriggers({ } } + const dismiss = (id: string) => { + setDismissed((prev) => new Set(prev).add(id)) + setSelectedId((current) => (current === id ? null : current)) + } + + // A fired ghost row has no engine handle — its ✕ dismisses locally; a live + // row's ✕ unregisters the engine trigger. + const rowAction = (t: SessionTriggerInfo) => + t.fired ? dismiss(t.id) : void unregister(t.id) + const clearAll = async () => { setClearing(true) try { await onClearAll?.() + // Live bindings are unregistered by onClearAll; fired ghosts have no + // engine handle, so sweep them from view here too. + setDismissed((prev) => { + const next = new Set(prev) + for (const t of visibleTriggers) if (t.fired) next.add(t.id) + return next + }) setSelectedId(null) } finally { setClearing(false) @@ -327,7 +370,7 @@ export function SessionTriggers({
- unregister all {triggers.length} triggers? + unregister all {registeredCount} triggers? {' '} this tears down the pipeline. @@ -360,12 +403,13 @@ export function SessionTriggers({ > - {triggers.length} trigger{triggers.length === 1 ? '' : 's'}{' '} + {registeredCount} trigger{registeredCount === 1 ? '' : 's'}{' '} registered {workflow.hasStructure && workflow.levels.length > 1 ? ` · ${workflow.levels.length} stages` : ''} + {firedCount > 0 ? ` · ${firedCount} fired` : ''} @@ -460,7 +504,7 @@ export function SessionTriggers({ memberKey={joinMeta(member)?.key} busy={busyId === member.id} onOpen={() => setSelectedId(member.id)} - onUnregister={() => void unregister(member.id)} + onUnregister={() => rowAction(member)} /> ))}
@@ -472,23 +516,21 @@ export function SessionTriggers({ stateNote={stateNote(unit.members[0])} busy={busyId === unit.members[0].id} onOpen={() => setSelectedId(unit.members[0].id)} - onUnregister={() => - void unregister(unit.members[0].id) - } + onUnregister={() => rowAction(unit.members[0])} /> ), )}
) }) - : triggers.map((trigger) => ( + : visibleTriggers.map((trigger) => ( setSelectedId(trigger.id)} - onUnregister={() => void unregister(trigger.id)} + onUnregister={() => rowAction(trigger)} /> ))}
@@ -571,9 +613,11 @@ export function SessionTriggers({ ) : null}
lifetime
- {selected.once - ? 'once — retires after first fire' - : 'until unregistered'} + {selected.fired + ? 'fired — already unregistered' + : selected.once + ? 'once — retires after first fire' + : 'until unregistered'}
{selectedSubscription ? ( <> @@ -608,15 +652,28 @@ export function SessionTriggers({ /> ) : null}
- + {/* A fired row has no engine binding left to unregister — + offering it would only produce a guaranteed error. */} + {selected.fired ? ( + + ) : ( + + )}
) : null} @@ -635,12 +692,12 @@ export function SessionTriggers({ pipeline flow - the reactive graph these {triggers.length} bindings form — state - writes and completions on the left, the sub-agents they spawn - flowing right. + the reactive graph these {visibleTriggers.length} bindings form — + state writes and completions on the left, the sub-agents they spawn + flowing right. fired bindings stay in the graph as pipeline history.
- +
diff --git a/console/web/src/components/function-call/FunctionCallCard.tsx b/console/web/src/components/function-call/FunctionCallCard.tsx index 80f5b39a0..54b3d0285 100644 --- a/console/web/src/components/function-call/FunctionCallCard.tsx +++ b/console/web/src/components/function-call/FunctionCallCard.tsx @@ -252,6 +252,15 @@ export function FunctionCallCard({ }: FunctionCallCardProps) { const pending = !!message.pendingApproval const running = !!message.running + // Raw in-flight arguments tail (`_streaming`, injected by the harness + // while a call's arguments are still forming) — rendered as a live pane. + const streamingTail = + running && + message.input && + typeof message.input === 'object' && + typeof (message.input as { _streaming?: unknown })._streaming === 'string' + ? (message.input as { _streaming: string })._streaming + : undefined const filesystemAccess = pending ? message.filesystemAccess : undefined const [open, setOpen] = useState(!!defaultOpen || pending) const [tab, setTab] = useState<'terminal' | 'json'>('terminal') @@ -294,7 +303,7 @@ export function FunctionCallCard({ const showRequestPaneAbove = pending ? !customPreview : running - ? !hasCustomTerminal + ? !hasCustomTerminal && streamingTail === undefined : false const runResolve = async (kind: 'approve' | 'deny' | 'always_allow') => { @@ -401,7 +410,9 @@ export function FunctionCallCard({ ) : null} {running && !pending ? ( - hasCustomTerminal ? ( + streamingTail !== undefined ? ( + + ) : hasCustomTerminal ? (
{customTerminal}
) : ( @@ -591,6 +602,30 @@ function PaneShell({ ) } +/** + * Live view of a call's arguments while they stream — the harness rides the + * raw tail on `_streaming` (providers degrade the incomplete JSON itself). + * Column-reverse pins the newest text to the bottom, terminal-style. + */ +function StreamingArgsPane({ text }: { text: string }) { + return ( +
+
+ request + + {' '} + · streaming… + +
+
+ 
+          {text}
+
+
+
+ ) +} + function ValuePane({ label, value, bordered }: ValuePaneProps) { const empty = isEmptyValue(value) const primitive = !empty && isPrimitive(value) diff --git a/console/web/src/components/sidebar/ConversationRow.tsx b/console/web/src/components/sidebar/ConversationRow.tsx index 21d514309..2d2aceee5 100644 --- a/console/web/src/components/sidebar/ConversationRow.tsx +++ b/console/web/src/components/sidebar/ConversationRow.tsx @@ -1,4 +1,4 @@ -import { ChevronDown, ChevronRight } from 'lucide-react' +import { Bot, ChevronDown, ChevronRight, Zap } from 'lucide-react' import { useEffect, useRef, useState } from 'react' import { StatusDot } from '@/components/ui/StatusDot' import { cn } from '@/lib/utils' @@ -125,6 +125,24 @@ export function ConversationRow({ ) ) : null} + {/* Sub-agent origin: ⚡ a trigger reaction spawned it, 🤖 an agent's + direct harness::spawn did. Absent on roots and pre-stamp sessions. */} + {depth > 0 && conversation.spawnedBy ? ( + + {conversation.spawnedBy === 'trigger' ? ( + + ) : ( + + )} + + ) : null}
{editing ? ( { expect(next.messages).toBe(existing.messages) expect(next.hydrated).toBe(true) }) + + it('maps metadata.spawned_by to the sidebar origin discriminant', () => { + const spawned = (v: unknown) => + mergeConversationMeta( + undefined, + sessionMeta({ + metadata: { parent_session_id: 'console-parent', spawned_by: v }, + }), + ).spawnedBy + expect(spawned('trigger')).toBe('trigger') + expect(spawned('agent')).toBe('agent') + // Unknown/absent values (pre-stamp sessions) stay undefined. + expect(spawned('something-else')).toBeUndefined() + expect(spawned(undefined)).toBeUndefined() + }) +}) + +describe('markBackgroundedStale', () => { + // Regression: transcript events subscribe for the ACTIVE session only, so + // a session backgrounded mid-turn misses entry updates (a function call + // freezes as `ƒ …` with empty request/response). Staling it on switch + // makes re-activation re-hydrate from durable truth. + it('marks hydrated backgrounded sessions stale, leaves the active one', () => { + const sessions = [ + conversation({ id: 'active', hydrated: true }), + conversation({ id: 'backgrounded', hydrated: true }), + conversation({ id: 'draft', draft: true, hydrated: true }), + conversation({ id: 'never-opened', hydrated: false }), + ] + + const next = markBackgroundedStale(sessions, 'active') + + expect(next.find((c) => c.id === 'active')?.hydrated).toBe(true) + expect(next.find((c) => c.id === 'backgrounded')?.hydrated).toBe(false) + // Drafts are local-only (no server transcript to refetch). + expect(next.find((c) => c.id === 'draft')?.hydrated).toBe(true) + expect(next.find((c) => c.id === 'never-opened')?.hydrated).toBe(false) + }) + + it('returns the same array when nothing needs staling', () => { + const sessions = [ + conversation({ id: 'active', hydrated: true }), + conversation({ id: 'never-opened', hydrated: false }), + ] + expect(markBackgroundedStale(sessions, 'active')).toBe(sessions) + }) }) describe('appendMessageToConversation', () => { diff --git a/console/web/src/hooks/use-conversations.ts b/console/web/src/hooks/use-conversations.ts index cf041ea29..938da6e99 100644 --- a/console/web/src/hooks/use-conversations.ts +++ b/console/web/src/hooks/use-conversations.ts @@ -143,6 +143,10 @@ function conversationFromMeta(meta: SessionMeta): Conversation { ? md.parent_session_id : undefined, depth: typeof md.depth === 'number' ? md.depth : undefined, + spawnedBy: + md.spawned_by === 'trigger' || md.spawned_by === 'agent' + ? md.spawned_by + : undefined, messages: [], status: meta.status, statusReason: meta.status_reason, @@ -171,6 +175,27 @@ export function applyCatalogModelFallback( return changed ? next : conversations } +/** + * Mark every backgrounded server-backed conversation stale so the next + * activation re-hydrates it. A transcript subscription exists only for the + * ACTIVE session, so entry events emitted while a session is backgrounded + * are lost — a function call caught mid-snapshot freezes as `ƒ …` with an + * empty request/response until durable truth is re-fetched. Returns the + * same array when nothing changed. + */ +export function markBackgroundedStale( + conversations: Conversation[], + activeId: string | null, +): Conversation[] { + let changed = false + const next = conversations.map((c) => { + if (c.id === activeId || c.draft || !c.hydrated) return c + changed = true + return { ...c, hydrated: false } + }) + return changed ? next : conversations +} + export function mergeConversationMeta( existing: Conversation | undefined, meta: SessionMeta, @@ -365,6 +390,10 @@ export function useConversations( ? md.parent_session_id : c.parentId, depth: typeof md.depth === 'number' ? md.depth : c.depth, + spawnedBy: + md.spawned_by === 'trigger' || md.spawned_by === 'agent' + ? md.spawned_by + : c.spawnedBy, updatedAt: event.timestamp, } }) @@ -512,6 +541,16 @@ export function useConversations( } }, [activeIsServerBacked, activeId, conversations, patchConversation]) + /* Backgrounded sessions receive no transcript events (the subscription + above is active-only), so anything that changed while away is missing + from their in-memory messages. Mark them stale on every activation + switch; the hydration effect above then refetches on return, folding + durable truth over frozen mid-stream snapshots. */ + useEffect(() => { + if (!serverEnabled) return + setConversations((prev) => markBackgroundedStale(prev, activeId)) + }, [serverEnabled, activeId]) + /* Migrate model ids once catalog-backed keys are known (local-only; the server metadata is rewritten on the next explicit model change). Gated on catalogReady so a stale placeholder catalog can't clobber picks. */ diff --git a/console/web/src/hooks/use-model-picker-source.ts b/console/web/src/hooks/use-model-picker-source.ts index 6866b8c61..345178fa1 100644 --- a/console/web/src/hooks/use-model-picker-source.ts +++ b/console/web/src/hooks/use-model-picker-source.ts @@ -6,6 +6,7 @@ import { fetchProviderList, type ProviderListEntry, subscribeModelChanges, + subscribeProviderChanges, } from '@/lib/models-catalog' import type { ModelOption } from '@/types/chat' @@ -85,7 +86,7 @@ export function useModelPickerSource( useEffect(() => { if (backendId !== 'real' || !harnessAvailable) return let disposed = false - let dispose: (() => void) | undefined + const disposers: (() => void)[] = [] let timer: ReturnType | null = null const onChange = () => { @@ -96,18 +97,23 @@ export function useModelPickerSource( }, 150) } - void subscribeModelChanges(onChange).then((d) => { - if (disposed) { - d() - return - } - dispose = d - }) + // Catalog changes AND provider availability flips — a provider going + // down/up keeps its catalog models, so only provider::changed re-greys + // or re-enables its picker group without a manual refresh. + for (const subscribe of [subscribeModelChanges, subscribeProviderChanges]) { + void subscribe(onChange).then((d) => { + if (disposed) { + d() + return + } + disposers.push(d) + }) + } return () => { disposed = true if (timer !== null) clearTimeout(timer) - dispose?.() + for (const d of disposers) d() } }, [backendId, harnessAvailable, refresh]) diff --git a/console/web/src/lib/backend/triggers.test.ts b/console/web/src/lib/backend/triggers.test.ts new file mode 100644 index 000000000..9a2f1cbb1 --- /dev/null +++ b/console/web/src/lib/backend/triggers.test.ts @@ -0,0 +1,135 @@ +import { describe, expect, it } from 'vitest' +import type { TriggerFiredData } from '@/types/chat' +import { mergeFiredTriggers, type SessionTriggerInfo } from './triggers' + +const live = ( + id: string, + over: Partial = {}, +): SessionTriggerInfo => ({ + id, + triggerType: 'state', + functionId: 'harness::react', + config: {}, + configSummary: '', + ...over, +}) + +const rec = (over: Partial = {}): TriggerFiredData => ({ + subscription_id: 'sub_1', + target: 'spawn', + once: true, + retired: true, + fired_at: 1, + ...over, +}) + +describe('mergeFiredTriggers', () => { + it('appends a ghost row for a retired fire absent from the poll', () => { + const merged = mergeFiredTriggers( + [], + [rec({ trigger_id: 't-1', model: 'm', scope: 'sc', key: 'k' })], + ) + expect(merged).toHaveLength(1) + expect(merged[0]).toMatchObject({ + id: 't-1', + fired: true, + once: true, + triggerType: 'state', + functionId: 'harness::react', + config: { scope: 'sc', key: 'k' }, + }) + }) + + it('annotates a still-polled retired trigger in place instead of ghosting', () => { + const polled = [live('t-1', { label: 'facts', once: true })] + const merged = mergeFiredTriggers(polled, [ + rec({ trigger_id: 't-1', fired_at: 7 }), + ]) + expect(merged).toHaveLength(1) + // Same row (full config/metadata retained), just marked fired. + expect(merged[0]).toMatchObject({ + id: 't-1', + label: 'facts', + fired: true, + firedAt: 7, + }) + }) + + it('ignores non-retired fires (binding still live)', () => { + expect(mergeFiredTriggers([], [rec({ retired: false })])).toEqual([]) + }) + + it('collapses repeat fires of the same trigger to one newest ghost', () => { + const merged = mergeFiredTriggers( + [], + [ + rec({ trigger_id: 't-1', fired_at: 1 }), + rec({ trigger_id: 't-1', fired_at: 2 }), + ], + ) + expect(merged).toHaveLength(1) + expect(merged[0]).toMatchObject({ id: 't-1', firedAt: 2 }) + }) + + it('prefers the full last-seen row for a ghost so workflow structure survives', () => { + const full = live('t-1', { + label: 'insights', + once: true, + metadata: { + join: { id: 'J1', expect: ['insights', 'glossary'], key: 'insights' }, + model: 'm', + task: 'merge everything', + }, + }) + const seen = new Map([[full.id, full]]) + const merged = mergeFiredTriggers([], [rec({ trigger_id: 't-1' })], seen) + expect(merged).toHaveLength(1) + // Full metadata retained (join grouping / DAG structure), fired flagged. + expect(merged[0]).toMatchObject({ + id: 't-1', + fired: true, + firedAt: 1, + label: 'insights', + metadata: { join: { id: 'J1' }, task: 'merge everything' }, + }) + // Without the cache (e.g. after a reload) the thin record ghost stands. + const thin = mergeFiredTriggers([], [rec({ trigger_id: 't-1' })]) + expect(thin[0].metadata?.join).toBeUndefined() + }) + + it('falls back to a synthetic id when the record has no trigger id', () => { + const merged = mergeFiredTriggers( + [], + [rec({ subscription_id: 'sub_9', trigger_id: undefined })], + ) + expect(merged[0]).toMatchObject({ id: 'fired:sub_9', fired: true }) + }) + + it('never renders an empty ghost title: label-less non-state falls back to "trigger"', () => { + const merged = mergeFiredTriggers( + [], + [rec({ trigger_id: 't-3', key: undefined, label: undefined })], + ) + expect(merged[0]).toMatchObject({ id: 't-3', triggerType: 'trigger' }) + }) + + it('marks a notify fire as a notify-target ghost', () => { + const merged = mergeFiredTriggers( + [], + [ + rec({ + trigger_id: 't-2', + target: 'notify', + label: 'ping', + key: undefined, + }), + ], + ) + expect(merged[0]).toMatchObject({ + id: 't-2', + fired: true, + functionId: 'harness::notify_agent', + label: 'ping', + }) + }) +}) diff --git a/console/web/src/lib/backend/triggers.ts b/console/web/src/lib/backend/triggers.ts index 5dcf282a1..c7c41a362 100644 --- a/console/web/src/lib/backend/triggers.ts +++ b/console/web/src/lib/backend/triggers.ts @@ -10,6 +10,7 @@ */ import type { IiiClient } from '@/lib/iii-client' +import type { TriggerFiredData } from '@/types/chat' export interface SessionTriggerInfo { /** Engine trigger id — the unregister handle. */ @@ -23,6 +24,15 @@ export interface SessionTriggerInfo { label?: string once?: boolean metadata?: Record + /** + * This trigger already fired and was unregistered (per a durable + * `trigger_fired` transcript entry — see `mergeFiredTriggers`). Either a + * still-polled row annotated ahead of the next poll, or a synthesized + * "ghost" row reconstructed after the poll dropped it. No engine handle to + * unregister — the ✕ dismisses locally instead. + */ + fired?: boolean + firedAt?: number } const NOTIFY_TARGET = 'harness::notify_agent' @@ -103,3 +113,73 @@ export async function unregisterTrigger( ): Promise { await client.trigger('engine::unregister_trigger', { id: triggerId }) } + +/** Reconstruct a fired-and-unregistered trigger's panel row from its record. */ +function firedGhostRow(t: TriggerFiredData): SessionTriggerInfo { + const isState = typeof t.key === 'string' + return { + id: t.trigger_id ?? `fired:${t.subscription_id}`, + // The record carries no trigger_type; infer state from the watch and fall + // back to a generic name so a label-less ghost never renders an empty row. + triggerType: isState ? 'state' : t.join ? 'join' : 'trigger', + functionId: t.target === 'spawn' ? REACT_TARGET : NOTIFY_TARGET, + config: isState ? { scope: t.scope, key: t.key } : undefined, + configSummary: '', + label: t.label ?? (t.join ? `join ${t.join.id}` : undefined), + once: t.once, + metadata: t.model ? { model: t.model } : undefined, + fired: true, + firedAt: t.fired_at, + } +} + +/** + * Merge the live poll with fired-trigger history. A *retired* fire means the + * binding was unregistered engine-side: if the (≤5s stale) poll still lists + * it, annotate that row as fired in place; once the poll drops it, append a + * greyed "ghost" row. Non-retired fires leave their live row untouched; + * repeat fires collapse to one record (newest wins). + * + * Ghost fidelity is two-tier: prefer the FULL last-seen polled row (from + * `seenRows`) so join grouping and the workflow/DAG structure survive the + * binding's retirement — the fired record alone carries no `metadata.join` / + * spawn pin / task, and a pipeline of fired thin ghosts would collapse to a + * flat list. The thin record-only ghost remains the post-reload fallback. + * + * ponytail: a completed join collapses to a single fired row (`join `) + * rather than resurrecting each predecessor row — enough to show it fired + + * retired. Per-predecessor ghosts if that granularity is ever needed. + */ +export function mergeFiredTriggers( + polled: SessionTriggerInfo[], + fired: TriggerFiredData[], + seenRows?: ReadonlyMap, +): SessionTriggerInfo[] { + const liveIds = new Set(polled.map((t) => t.id)) + const retiredLive = new Map() + const ghosts: SessionTriggerInfo[] = [] + const seen = new Set() + for (let i = fired.length - 1; i >= 0; i--) { + const t = fired[i] + if (!t.retired) continue + const key = t.trigger_id ?? t.subscription_id + if (seen.has(key)) continue + seen.add(key) + if (t.trigger_id && liveIds.has(t.trigger_id)) { + retiredLive.set(t.trigger_id, t) // stale poll row — mark, don't ghost + } else { + const remembered = t.trigger_id ? seenRows?.get(t.trigger_id) : undefined + ghosts.push( + remembered + ? { ...remembered, fired: true, firedAt: t.fired_at } + : firedGhostRow(t), + ) + } + } + if (retiredLive.size === 0 && ghosts.length === 0) return polled + const rows = polled.map((row) => { + const t = retiredLive.get(row.id) + return t ? { ...row, fired: true, firedAt: t.fired_at } : row + }) + return [...rows, ...ghosts] +} diff --git a/console/web/src/lib/models-catalog.ts b/console/web/src/lib/models-catalog.ts index 6479f122f..ef75fe1fc 100644 --- a/console/web/src/lib/models-catalog.ts +++ b/console/web/src/lib/models-catalog.ts @@ -76,18 +76,13 @@ export async function refreshProviderModels( const MODELS_CHANGED_FN = 'iii::console::models_changed' /** llm-router custom trigger type (worker-owned fan-out, not pubsub). */ const MODELS_CHANGED_TRIGGER = 'router::models::changed' +const PROVIDERS_CHANGED_FN = 'iii::console::providers_changed' +/** Fired on provider availability flips (worker registered / dispatch found it gone). */ +const PROVIDERS_CHANGED_TRIGGER = 'router::provider::changed' -/** - * Subscribe to the llm-router `router::models::changed` trigger so the picker - * re-pulls the catalog when a provider reconciles — credential added/removed, - * `refresh_models`, or a provider worker added/removed. - * - * Registers a browser-local handler plus the router-owned trigger type - * (`llm-router/README.md` § Events). Returns a disposer; on failure (e.g. - * llm-router absent) the binding is dropped silently and callers rely on - * manual refresh / config-save hooks. - */ -export async function subscribeModelChanges( +async function subscribeRouterTrigger( + fnId: string, + triggerType: string, onChange: () => void, ): Promise<() => void> { const client = await getIiiClient() @@ -95,12 +90,12 @@ export async function subscribeModelChanges( let offTrigger: (() => void) | undefined try { // `on()` registers `::`; the trigger targets the same id. - offHandler = client.on(MODELS_CHANGED_FN, () => { + offHandler = client.on(fnId, () => { onChange() }) offTrigger = client.registerTrigger({ - type: MODELS_CHANGED_TRIGGER, - function_id: `${MODELS_CHANGED_FN}::${client.browserId}`, + type: triggerType, + function_id: `${fnId}::${client.browserId}`, config: {}, }) } catch { @@ -121,11 +116,55 @@ export async function subscribeModelChanges( } } +/** + * Subscribe to the llm-router `router::models::changed` trigger so the picker + * re-pulls the catalog when a provider reconciles — credential added/removed, + * `refresh_models`, or a provider worker added/removed. + * + * Registers a browser-local handler plus the router-owned trigger type + * (`llm-router/README.md` § Events). Returns a disposer; on failure (e.g. + * llm-router absent) the binding is dropped silently and callers rely on + * manual refresh / config-save hooks. + */ +export async function subscribeModelChanges( + onChange: () => void, +): Promise<() => void> { + return subscribeRouterTrigger( + MODELS_CHANGED_FN, + MODELS_CHANGED_TRIGGER, + onChange, + ) +} + +/** + * Subscribe to `router::provider::changed` — availability flips (a provider + * worker (re)registered, or a dispatch discovered it gone). Model catalogs + * survive a provider going down, so ONLY this trigger tells the picker to + * re-read `router::provider::list` and grey/un-grey the group without a + * manual refresh. + */ +export async function subscribeProviderChanges( + onChange: () => void, +): Promise<() => void> { + return subscribeRouterTrigger( + PROVIDERS_CHANGED_FN, + PROVIDERS_CHANGED_TRIGGER, + onChange, + ) +} + /** A provider declared to the router, from `router::provider::list`. */ export interface ProviderListEntry { id: string display_name: string supports_model_listing: boolean + /** + * The provider WORKER is loaded/connected. `false` = the router knows the + * provider (its models may still sit in the catalog) but dispatching to it + * fails with `router/provider_unavailable` — the picker renders the group + * as "not loaded" and disables its models. + */ + available: boolean } /** @@ -151,6 +190,8 @@ export async function fetchProviderList(): Promise { id, display_name: typeof o.display_name === 'string' ? o.display_name : id, supports_model_listing: o.supports_model_listing === true, + // Absent on older routers — treat as available (previous behavior). + available: o.available !== false, }) } return out diff --git a/console/web/src/lib/sessions/entry-mapper.test.ts b/console/web/src/lib/sessions/entry-mapper.test.ts index 2d3ad1535..0a2f58143 100644 --- a/console/web/src/lib/sessions/entry-mapper.test.ts +++ b/console/web/src/lib/sessions/entry-mapper.test.ts @@ -7,6 +7,7 @@ import { entrySegments, splitReactionTask, transcriptToMessages, + triggerFiredSummary, } from './entry-mapper' import type { AgentMessage, TranscriptItem } from './types' @@ -177,6 +178,18 @@ describe('entrySegments', () => { ).not.toHaveProperty('reaction') }) + it('marks direct-spawn seed tasks (origin on events, prefix on reads)', () => { + expect( + entrySegments(userItem('e-1', 'build the report', { spawn: true }))[0], + ).toMatchObject({ spawn: true }) + expect( + entrySegments(userItem('e_spawn_ab12', 'build it'))[0], + ).toMatchObject({ spawn: true }) + expect( + entrySegments(userItem('e-2', 'typed by hand'))[0], + ).not.toHaveProperty('spawn') + }) + it('splits an assistant entry into thought/text/function-call segments by block', () => { const segments = entrySegments( assistantItem('e-a', [ @@ -208,6 +221,45 @@ describe('entrySegments', () => { expect(entrySegments(assistantItem('e-a', []))).toEqual([]) }) + // Mid-stream, the harness rides the raw in-flight args tail on + // `_streaming` beside the salvaged fields — the request pane shows the + // command forming instead of `empty` for the whole stream. + it('surfaces the streaming arguments tail while wrapper args form', () => { + const withTarget = entrySegments( + assistantItem('e-a', [ + { + type: 'function_call', + id: 'fc-1', + function_id: 'agent_trigger', + arguments: { + function: 'state::set', + _streaming: '"payload":{"value":"grow', + }, + }, + ]), + )[0] + expect(withTarget).toMatchObject({ + functionId: 'state::set', + input: { _streaming: '"payload":{"value":"grow' }, + }) + expect(withTarget).not.toMatchObject({ unresolvedTarget: true }) + + const noTarget = entrySegments( + assistantItem('e-b', [ + { + type: 'function_call', + id: 'fc-2', + function_id: 'agent_trigger', + arguments: { _streaming: '{"fun' }, + }, + ]), + )[0] + expect(noTarget).toMatchObject({ + unresolvedTarget: true, + input: { _streaming: '{"fun' }, + }) + }) + it('maps a compaction custom entry to the compaction marker', () => { const [marker] = entrySegments({ entry_id: 'e-c', @@ -225,6 +277,150 @@ describe('entrySegments', () => { }) }) + it('maps role:custom read-back messages through the same typed dispatch', () => { + // The exact `session::messages` wire shape: kind:custom entries come back + // as a `role: 'custom'` MESSAGE (custom_type + details), not `item.custom`. + const [err] = entrySegments({ + entry_id: 'e_t_faa6_error', + message: { + role: 'custom', + custom_type: 'error', + content: [], + details: { + reason: + 'remote error (router/provider_unavailable): provider zai unavailable', + }, + timestamp: 9, + }, + }) + expect(err).toMatchObject({ + role: 'system', + tone: 'error', + content: + 'turn failed — remote error (router/provider_unavailable): provider zai unavailable', + createdAt: 9, + }) + const [fired] = entrySegments({ + entry_id: 'e_trigfired_sub_9', + message: { + role: 'custom', + custom_type: 'trigger_fired', + content: [], + details: { + subscription_id: 'sub_9', + target: 'spawn', + once: true, + retired: true, + fired_at: 4, + }, + timestamp: 4, + }, + }) + expect(fired).toMatchObject({ role: 'system', kind: 'trigger-fired' }) + // Unknown custom types still fall back to their display text. + const [note] = entrySegments({ + entry_id: 'e-x', + message: { + role: 'custom', + custom_type: 'someother', + content: [], + display: 'hello from a worker', + timestamp: 5, + }, + }) + expect(note).toMatchObject({ + role: 'system', + content: 'hello from a worker', + }) + }) + + it('maps harness error/notice custom entries to visible system notices', () => { + const [err] = entrySegments({ + entry_id: 'e_t1_error', + custom: { + custom_type: 'error', + data: { reason: 'provider zai unavailable' }, + }, + }) + expect(err).toMatchObject({ + role: 'system', + kind: 'notice', + tone: 'error', + content: 'turn failed — provider zai unavailable', + }) + const [notice] = entrySegments({ + entry_id: 'e_t1_max_turns', + custom: { + custom_type: 'notice', + data: { reason: 'max_turns', message: 'max_turns (8) reached' }, + }, + }) + expect(notice).toMatchObject({ + role: 'system', + tone: 'info', + content: 'max_turns (8) reached', + }) + }) + + it('maps a trigger_fired custom entry to a turn-less notice carrying its record', () => { + const [notice] = entrySegments({ + entry_id: 'e_trigfired_sub_1', + custom: { + custom_type: 'trigger_fired', + data: { + subscription_id: 'sub_1', + trigger_id: 't-1', + target: 'spawn', + model: 'claude-sonnet-4-6', + once: true, + retired: true, + scope: 'cache-repl-pipeline', + key: 'facts', + fired_at: 42, + }, + }, + }) + expect(notice).toMatchObject({ + id: 'e_trigfired_sub_1', + role: 'system', + kind: 'trigger-fired', + createdAt: 42, + trigger: { subscription_id: 'sub_1', target: 'spawn', retired: true }, + }) + expect((notice as { content: string }).content).toBe( + 'cache-repl-pipeline/facts · spawned claude-sonnet-4-6 · unregistered', + ) + }) + + it('triggerFiredSummary reads join progress and notify targets', () => { + expect( + triggerFiredSummary({ + subscription_id: 's', + target: 'spawn', + once: false, + retired: false, + join: { + id: 'J1', + key: 'insights', + arrived: 1, + expected: 2, + completed: false, + }, + fired_at: 0, + }), + ).toBe('join J1 · 1/2 arrived') + expect( + triggerFiredSummary({ + subscription_id: 's', + target: 'notify', + label: 'ping', + once: true, + retired: true, + fired_at: 0, + }), + ).toBe('ping · notified this chat · unregistered') + }) + it('flags an agent_trigger whose target is not resolvable yet', () => { // Providers degrade partial/streaming JSON arguments to `{}`, so the // wrapped target function is unknown until the stream finishes. diff --git a/console/web/src/lib/sessions/entry-mapper.ts b/console/web/src/lib/sessions/entry-mapper.ts index 1193b4fd0..27f5278f1 100644 --- a/console/web/src/lib/sessions/entry-mapper.ts +++ b/console/web/src/lib/sessions/entry-mapper.ts @@ -28,11 +28,108 @@ import type { FunctionCallMessage, Message, SystemMessage, + TriggerFiredData, UserMessage, } from '@/types/chat' import type { AgentMessage, ContentBlock, TranscriptItem } from './types' export const COMPACTION_CUSTOM_TYPE = 'compaction' +export const TRIGGER_FIRED_CUSTOM_TYPE = 'trigger_fired' +/** Harness turn-failure record (`{ reason }`) — `finalize_failed`. */ +export const ERROR_CUSTOM_TYPE = 'error' +/** Harness informational record (`{ reason, message }`) — e.g. max_turns. */ +export const NOTICE_CUSTOM_TYPE = 'notice' + +/** + * Map one typed custom record (however it arrived — `item.custom` on events, + * a `role: 'custom'` message on `session::messages` read-backs) to its UI + * segments. `null` means "not a typed record" so the caller can fall back. + */ +function customSegments( + entryId: string, + customType: string, + data: unknown, + timestamp: number, +): Message[] | null { + switch (customType) { + case COMPACTION_CUSTOM_TYPE: + return [compactionMarker(entryId, data, timestamp)] + case TRIGGER_FIRED_CUSTOM_TYPE: + return [triggerFiredMessage(entryId, data, timestamp)] + // Harness failure/notice records: without these the turn can end in an + // error the chat never shows (the session flips to "error" silently). + case ERROR_CUSTOM_TYPE: + return [customNotice(entryId, data, 'error', 'turn failed', timestamp)] + case NOTICE_CUSTOM_TYPE: + return [customNotice(entryId, data, 'info', 'notice', timestamp)] + default: + return null + } +} + +function customNotice( + entryId: string, + data: unknown, + tone: 'info' | 'error', + fallback: string, + timestamp: number, +): SystemMessage { + const d = (data ?? {}) as { reason?: unknown; message?: unknown } + const content = + typeof d.message === 'string' + ? d.message + : typeof d.reason === 'string' + ? tone === 'error' + ? `turn failed — ${d.reason}` + : d.reason + : fallback + return { + id: entryId, + role: 'system', + kind: 'notice', + content, + tone, + createdAt: timestamp, + } +} + +/** The trigger's display name: label, else join id, else state scope/key. */ +export function triggerFiredName(t: TriggerFiredData): string { + if (t.label) return t.label + if (t.join) return `join ${t.join.id}` + if (t.key) return t.scope ? `${t.scope}/${t.key}` : t.key + return 'trigger' +} + +/** Plain one-liner for the fired notice (also the a11y/fallback content). */ +export function triggerFiredSummary(t: TriggerFiredData): string { + const name = triggerFiredName(t) + if (t.join && !t.join.completed) { + return `${name} · ${t.join.arrived}/${t.join.expected} arrived` + } + const action = + t.target === 'spawn' + ? `spawned${t.model ? ` ${t.model}` : ''}` + : 'notified this chat' + return `${name} · ${action}${t.retired ? ' · unregistered' : ''}` +} + +function triggerFiredMessage( + entryId: string, + data: unknown, + timestamp: number, +): SystemMessage { + const t = (data ?? {}) as TriggerFiredData + return { + id: entryId, + role: 'system', + kind: 'trigger-fired', + content: triggerFiredSummary(t), + tone: 'info', + trigger: t, + createdAt: typeof t.fired_at === 'number' ? t.fired_at : timestamp, + } +} /** The harness wraps every tool call in agent_trigger; unwrap for display. */ function unwrapFunctionCall( @@ -44,10 +141,29 @@ function unwrapFunctionCall( } { if (block.function_id === 'agent_trigger') { if (block.arguments && typeof block.arguments === 'object') { - const args = block.arguments as { function?: unknown; payload?: unknown } + const args = block.arguments as { + function?: unknown + payload?: unknown + _streaming?: unknown + } + // Mid-stream, the harness rides the raw in-flight arguments tail on + // `_streaming` (providers degrade the incomplete JSON itself), so the + // command can be watched forming in the request pane. + const streaming = + typeof args._streaming === 'string' ? args._streaming : undefined if (typeof args.function === 'string' && args.function.length > 0) { + if (streaming !== undefined) { + return { functionId: args.function, input: { _streaming: streaming } } + } return { functionId: args.function, input: args.payload ?? {} } } + if (streaming !== undefined) { + return { + functionId: block.function_id, + input: { _streaming: streaming }, + unresolvedTarget: true, + } + } } // The target is unknown while the wrapper's arguments are still // streaming (providers degrade partial JSON to `{}`). Flag it so the UI @@ -163,10 +279,14 @@ export function entrySegments( sessionId?: string, ): Message[] { if (item.custom) { - if (item.custom.custom_type === COMPACTION_CUSTOM_TYPE) { - return [compactionMarker(item.entry_id, item.custom.data, Date.now())] - } - return [] + return ( + customSegments( + item.entry_id, + item.custom.custom_type, + item.custom.data, + Date.now(), + ) ?? [] + ) } const message = item.message if (!message) return [] @@ -174,7 +294,7 @@ export function entrySegments( switch (message.role) { case 'user': { const origin = item.origin as - | { notification?: unknown; reaction?: unknown } + | { notification?: unknown; reaction?: unknown; spawn?: unknown } | undefined const isNotif = origin?.notification === true || item.entry_id.startsWith('e_notify_') @@ -182,6 +302,9 @@ export function entrySegments( // `e_react_` prefix on reads — session::messages carries no origin). const isReaction = origin?.reaction === true || item.entry_id.startsWith('e_react_') + // A direct `harness::spawn` seed task — same pattern, `e_spawn_` prefix. + const isSpawn = + origin?.spawn === true || item.entry_id.startsWith('e_spawn_') const { text, attachments } = splitUserContent(message.content) const split = isReaction ? splitReactionTask(text) : { task: text } const msg: UserMessage = { @@ -192,6 +315,7 @@ export function entrySegments( ...(attachments.length > 0 ? { attachments } : {}), ...(isNotif ? { notification: true } : {}), ...(isReaction ? { reaction: true } : {}), + ...(isSpawn ? { spawn: true } : {}), ...(split.appendix ? { reactionEvent: split.appendix } : {}), } return [msg] @@ -201,6 +325,18 @@ export function entrySegments( case 'function_result': return [] case 'custom': { + // `session::messages` read-backs surface kind:custom entries as a + // `role: 'custom'` message (`custom_type` + `details`) — the same + // records some paths deliver as `item.custom`. Dispatch typed records + // through the shared mapper first; anything unrecognized falls back to + // its display text. + const typed = customSegments( + item.entry_id, + message.custom_type, + message.details, + message.timestamp, + ) + if (typed) return typed const content = message.display ?? textOf(message.content) if (!content) return [] const msg: SystemMessage = { diff --git a/console/web/src/types/chat.ts b/console/web/src/types/chat.ts index e440e4700..35baeaaf5 100644 --- a/console/web/src/types/chat.ts +++ b/console/web/src/types/chat.ts @@ -63,6 +63,8 @@ export interface UserMessage extends BaseMessage { notification?: boolean /** A react-fired task delivered into this session — machine-sent, not typed. */ reaction?: boolean + /** A direct `harness::spawn` task seeding this session — machine-sent, not typed. */ + spawn?: boolean /** * The firing event (or join inputs) `harness::react` appended to the task, * split off by the entry mapper: rendered as collapsible JSON, not prose. @@ -116,18 +118,50 @@ export interface FunctionCallMessage extends BaseMessage { } } +/** + * A subscription fire, mirrored from the harness `trigger_fired` custom entry + * (`subscriptions/fired.rs`). Drives the turn-less "trigger fired" chat notice + * and lets the panel keep a fired `once` trigger visible after it unregisters. + */ +export interface TriggerFiredData { + subscription_id: string + /** Engine trigger id — correlates to a live panel row's `id`. */ + trigger_id?: string + target: 'notify' | 'spawn' + label?: string + model?: string + once: boolean + /** This fire unregistered the binding (once teardown / join predecessor GC). */ + retired: boolean + scope?: string + key?: string + child_session_id?: string + join?: { + id: string + key: string + arrived: number + expected: number + completed: boolean + } + note?: string + fired_at: number +} + /** * `kind: 'compaction'` renders the collapsed-history marker in the * transcript. The session-manager transcript is the single source of truth * for what the provider sees, so this marker is purely presentational. + * `kind: 'trigger-fired'` renders a turn-less subscription-fire notice. */ export interface SystemMessage extends BaseMessage { role: 'system' content: string tone?: 'info' | 'warn' | 'error' - kind?: 'notice' | 'compaction' + kind?: 'notice' | 'compaction' | 'trigger-fired' summaryText?: string tokensBefore?: number + /** Present on `kind: 'trigger-fired'`. */ + trigger?: TriggerFiredData } export type Message = @@ -197,6 +231,12 @@ export interface Conversation { parentId?: string /** Spawn depth: 0 = root orchestrator (from `metadata.depth`). */ depth?: number + /** + * Who created this child session (from `metadata.spawned_by`, stamped by the + * harness): a trigger reaction or an agent's direct `harness::spawn`. + * Absent on root chats and pre-existing sessions. Drives the sidebar icon. + */ + spawnedBy?: 'trigger' | 'agent' /** Driver-owned session status (spinner + sidebar indicator). */ status?: ConversationStatus statusReason?: string From 4203b8a648a241134e75b96329cd87c77fc94d4d Mon Sep 17 00:00:00 2001 From: Anderson Leal Date: Fri, 10 Jul 2026 13:23:34 -0300 Subject: [PATCH 06/11] fix(provider-anthropic): enable fine-grained tool streaming (MOT-3952) Without the fine-grained-tool-streaming-2025-05-14 beta, the Messages API buffers tool_use input server-side: measured one ~108B leading delta, then 119.4s of ping-only silence, then all 3434 deltas (376KB) in a 0.6s burst for a ~4k-token input. Large state::set-style args therefore exceed the router's 120s post-content idle budget and a healthy stream gets killed mid-args (the "stream idle past 120000ms" failures). With the beta header the same probe streams continuously (max 3.6s event gap), the idle guard only fires on real stalls, and the console's streaming-args pane gets live deltas. The beta's early-stop partial-JSON risk is already handled by degraded_arguments salvage. --- provider-anthropic/src/request.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/provider-anthropic/src/request.rs b/provider-anthropic/src/request.rs index fbdb078c2..10a174942 100644 --- a/provider-anthropic/src/request.rs +++ b/provider-anthropic/src/request.rs @@ -13,6 +13,16 @@ use serde_json::{json, Value}; pub const ANTHROPIC_VERSION: &str = "2023-06-01"; +/// Stream `tool_use.input` incrementally instead of the default server-side +/// buffering, which goes ping-only silent for the whole generation of a large +/// input (measured: ~120s of silence for a ~4k-token input, one leading +/// ~100B delta then a single burst) — long enough to trip the router's +/// 120s post-content idle guard and kill a healthy stream. Trade-off per the +/// API docs: on an early stop the accumulated input may be partial/invalid +/// JSON, which `llm_router::types::messages::degraded_arguments` already +/// salvages. +pub const ANTHROPIC_BETA: &str = "fine-grained-tool-streaming-2025-05-14"; + pub struct BodyArgs { pub model: String, pub max_tokens: u64, @@ -87,6 +97,7 @@ pub fn build_headers(cfg: &AnthropicConfig) -> Vec<(&'static str, String)> { vec![ auth_header(cfg.auth_mode, &cfg.credential_value), ("anthropic-version", ANTHROPIC_VERSION.to_string()), + ("anthropic-beta", ANTHROPIC_BETA.to_string()), ("content-type", "application/json".to_string()), ] } @@ -231,7 +242,10 @@ mod tests { let h = build_headers(&cfg(AuthMode::ApiKey)); assert!(h.contains(&("x-api-key", "sk-test".to_string()))); assert!(h.contains(&("anthropic-version", ANTHROPIC_VERSION.to_string()))); - assert!(!h.iter().any(|(k, _)| *k == "anthropic-beta")); + // Fine-grained tool streaming: without it, large tool inputs are + // server-buffered into a ping-only silence that trips the router's + // idle guard. + assert!(h.contains(&("anthropic-beta", ANTHROPIC_BETA.to_string()))); let h = build_headers(&cfg(AuthMode::OauthBearer)); assert!(h.contains(&("authorization", "Bearer sk-test".to_string()))); From f5ef51147f0197ab7efb023e0e733192e123c86d Mon Sep 17 00:00:00 2001 From: Anderson Leal Date: Fri, 10 Jul 2026 13:54:50 -0300 Subject: [PATCH 07/11] fix(args): never execute provider-degraded tool arguments (MOT-3952, MOT-3953) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Salvaged partial arguments are now stamped "_partial": true, and the turn loop refuses to dispatch any call whose arguments carry _partial/_raw, failing locally with a teachable arguments_truncated result instead. With fine-grained tool streaming a max_tokens cutoff mid-args ends the stream successfully, so the salvaged prefix would otherwise have executed with whatever fields happened to complete. Also closes the scalar bypass in five providers (openai, xai, zai, llamacpp, openai-codex): parseable non-object args (null, strings, arrays) now degrade like unparseable ones — anthropic already had the Value::is_object filter — keeping the object-only replay contract. And the teachable-error previews truncate on char boundaries; the byte-indexed String::truncate panicked on CJK/emoji model output. Found by codex review and coderabbit on PR #468. --- harness/src/trigger.rs | 48 ++++++++++++++++++++++++++++++-- harness/src/turn_loop.rs | 23 +++++++++++++++ llm-router/src/types/messages.rs | 13 ++++++--- provider-llamacpp/src/sse.rs | 7 ++++- provider-openai-codex/src/sse.rs | 4 ++- provider-openai/src/sse.rs | 4 ++- provider-xai/src/sse.rs | 4 ++- provider-zai/src/sse.rs | 4 ++- 8 files changed, 96 insertions(+), 11 deletions(-) diff --git a/harness/src/trigger.rs b/harness/src/trigger.rs index 4025de479..78429c6f0 100644 --- a/harness/src/trigger.rs +++ b/harness/src/trigger.rs @@ -139,9 +139,19 @@ pub fn denied_result(function_id: &str) -> ResultData { /// target — arguments were empty, null, or unparseable (local models emit /// malformed JSON args). Dispatching the wrapper name to the engine would /// only return a cryptic `function_not_found: agent_trigger`. +/// Char-safe ≤200-char preview of raw arguments for error messages — +/// model-emitted JSON serializes with literal UTF-8, and a byte-indexed +/// `String::truncate` panics mid-char on CJK/emoji payloads. +fn arguments_preview(arguments: &Value) -> String { + let s = arguments.to_string(); + match s.char_indices().nth(200) { + Some((i, _)) => s[..i].to_string(), + None => s, + } +} + pub fn wrapper_without_target_result(arguments: &Value) -> ResultData { - let mut got = arguments.to_string(); - got.truncate(200); + let got = arguments_preview(arguments); let msg = format!( "agent_trigger was called without a usable target (arguments were {got}); expected \ {{\"function\": \"\", \"payload\": {{...}}}}. Re-issue the call with the \ @@ -154,6 +164,25 @@ pub fn wrapper_without_target_result(arguments: &Value) -> ResultData { } } +/// Provider-degraded arguments (a stream that died or hit max_tokens +/// mid-args, salvaged to a `"_partial": true` prefix or a raw `{"_raw": …}` +/// evidence object) must never execute: the salvage preserves evidence for +/// the transcript, not intent. Teachable local failure, mirroring +/// [`wrapper_without_target_result`]. +pub fn truncated_arguments_result(function_id: &str, arguments: &Value) -> ResultData { + let got = arguments_preview(arguments); + let msg = format!( + "the arguments for {function_id} arrived truncated (the model stream ended \ + mid-arguments; received {got}). The call was NOT executed — re-issue it with \ + complete arguments." + ); + ResultData { + content: vec![ContentBlock::text(msg.clone())], + is_error: true, + details: json!({ "error": "arguments_truncated", "message": msg }), + } +} + /// Normalise an arbitrary function return into content blocks. `details` /// always carries the raw value; content is a string render, an explicit /// `content` block array, or a compact JSON fallback. @@ -208,6 +237,21 @@ mod tests { })) } + #[test] + fn arguments_preview_is_char_safe_on_multibyte_payloads() { + // Byte 200 lands mid-emoji: the old byte-indexed String::truncate + // panicked here on model-emitted CJK/emoji args. + let args = json!({ "text": "🎉".repeat(100) }); + let preview = arguments_preview(&args); + assert!(preview.chars().count() <= 200); + assert!(args.to_string().starts_with(&preview)); + // Both teachable results render without panicking. + assert!(wrapper_without_target_result(&args).is_error); + assert!(truncated_arguments_result("state::set", &args).is_error); + // Short args pass through whole. + assert_eq!(arguments_preview(&json!({"a": 1})), r#"{"a":1}"#); + } + #[test] fn normalize_string_value() { let (content, is_error) = normalize(&json!("hello")); diff --git a/harness/src/turn_loop.rs b/harness/src/turn_loop.rs index f76ecddb5..8e2eab7c6 100644 --- a/harness/src/turn_loop.rs +++ b/harness/src/turn_loop.rs @@ -457,6 +457,29 @@ pub async fn run_step( continue; } + // Provider-degraded arguments: a stream that died or was cut by + // max_tokens mid-args arrives as a salvaged `"_partial": true` + // prefix or a raw `{"_raw": …}` evidence object (the router's + // degraded_arguments). Executing partial intent is worse than + // failing — the complete-looking leading fields may be missing + // the constraints the model was still writing. + if call.arguments.get("_partial").is_some() || call.arguments.get("_raw").is_some() { + let data = trigger::truncated_arguments_result(&call.function_id, &call.arguments); + let entry_id = ids::function_result_entry_id(&record.turn_id, &call.id); + append_function_result( + &session, + &record, + call, + &data, + &entry_id, + &origin(&record.turn_id), + ) + .await?; + mark_done(&mut record, &call.id, &entry_id); + crate::state::put_turn(&deps.iii, &record, cfg.session_timeout_ms).await?; + continue; + } + // Fail-closed glob policy first — structural and final. Hooks run // only after it passes (a denial never reaches a hook). if !policy.allows(&call.function_id) { diff --git a/llm-router/src/types/messages.rs b/llm-router/src/types/messages.rs index fe56f40bd..e5375fc2f 100644 --- a/llm-router/src/types/messages.rs +++ b/llm-router/src/types/messages.rs @@ -149,11 +149,15 @@ pub fn reorder_displaced_results(messages: &[AgentMessage]) -> Vec<&AgentMessage /// mid-stream (`{"function":"state::set","payload":{"key":`) keeps its /// known prefix (`{"function":"state::set"}`), so long-streaming calls /// stay identifiable in UIs instead of rendering as an anonymous `{}`. +/// The salvaged object is stamped `"_partial": true` so dispatch layers +/// can refuse to execute partial intent (a truncated call must surface a +/// teachable error, not run with whatever fields happened to complete). /// 2. Otherwise carry the malformed text as `{"_raw": }` so the /// evidence of what the model actually sent survives for rendering and /// for the harness's teachable no-target error. pub fn degraded_arguments(args_json: &str) -> serde_json::Value { - if let Some(map) = salvage_leading_object_fields(args_json) { + if let Some(mut map) = salvage_leading_object_fields(args_json) { + map.insert("_partial".to_string(), serde_json::Value::Bool(true)); return serde_json::Value::Object(map); } serde_json::json!({ "_raw": utf8_head(args_json, 2048) }) @@ -220,17 +224,18 @@ mod tests { #[test] fn degraded_arguments_salvages_leading_fields_or_keeps_raw() { - // Mid-stream cut: the known prefix survives as a real object. + // Mid-stream cut: the known prefix survives as a real object, marked + // partial so it renders but never executes. assert_eq!( degraded_arguments(r#"{"function":"state::set","payload":{"key":"art"#), - serde_json::json!({ "function": "state::set" }) + serde_json::json!({ "function": "state::set", "_partial": true }) ); // Longest complete prefix wins, nested commas/strings don't cut. assert_eq!( degraded_arguments( r#"{"function":"a::b","payload":{"x":"1,2","y":[3,4]},"extra":{"cut":"# ), - serde_json::json!({ "function": "a::b", "payload": { "x": "1,2", "y": [3, 4] } }) + serde_json::json!({ "function": "a::b", "payload": { "x": "1,2", "y": [3, 4] }, "_partial": true }) ); // Nothing salvageable (no complete top-level field yet, or not JSON): // the raw text survives as evidence, always inside an object. diff --git a/provider-llamacpp/src/sse.rs b/provider-llamacpp/src/sse.rs index 50b7e5be4..c34d7e814 100644 --- a/provider-llamacpp/src/sse.rs +++ b/provider-llamacpp/src/sse.rs @@ -99,7 +99,9 @@ fn build_content(state: &PartialState) -> Vec { // JSON) degrade to the salvaged leading fields or `{"_raw": …}` — // always an object (replay-safe) that preserves the evidence. serde_json::from_str(&fc.args_json) - .unwrap_or_else(|_| llm_router::types::messages::degraded_arguments(&fc.args_json)) + .ok() + .filter(Value::is_object) + .unwrap_or_else(|| llm_router::types::messages::degraded_arguments(&fc.args_json)) }; out.push(ContentBlock::FunctionCall { id: fc.id.clone(), @@ -498,6 +500,9 @@ mod tests { match &partial.content[0] { ContentBlock::FunctionCall { arguments, .. } => { assert_eq!(arguments["function"], "state::set"); + // Salvage marker: the harness refuses to execute partial + // intent, so it must survive the provider boundary. + assert_eq!(arguments["_partial"], true); } other => panic!("want function_call, got {other:?}"), } diff --git a/provider-openai-codex/src/sse.rs b/provider-openai-codex/src/sse.rs index 46b5a9039..02b0fd0ee 100644 --- a/provider-openai-codex/src/sse.rs +++ b/provider-openai-codex/src/sse.rs @@ -102,7 +102,9 @@ fn build_content(state: &PartialState) -> Vec { // to the salvaged leading fields or `{"_raw": …}` — always an // object (replay-safe) that preserves the evidence. serde_json::from_str(&tc.args_json) - .unwrap_or_else(|_| llm_router::types::messages::degraded_arguments(&tc.args_json)) + .ok() + .filter(Value::is_object) + .unwrap_or_else(|| llm_router::types::messages::degraded_arguments(&tc.args_json)) }; out.push(ContentBlock::FunctionCall { id: tc.id.clone(), diff --git a/provider-openai/src/sse.rs b/provider-openai/src/sse.rs index d222dc2b5..a9fc0042b 100644 --- a/provider-openai/src/sse.rs +++ b/provider-openai/src/sse.rs @@ -88,7 +88,9 @@ fn build_content(state: &PartialState) -> Vec { // to the salvaged leading fields or `{"_raw": …}` — always an // object (replay-safe) that preserves the evidence. serde_json::from_str(&fc.args_json) - .unwrap_or_else(|_| llm_router::types::messages::degraded_arguments(&fc.args_json)) + .ok() + .filter(Value::is_object) + .unwrap_or_else(|| llm_router::types::messages::degraded_arguments(&fc.args_json)) }; out.push(ContentBlock::FunctionCall { id: fc.id.clone(), diff --git a/provider-xai/src/sse.rs b/provider-xai/src/sse.rs index 7bb3dab11..4d883732e 100644 --- a/provider-xai/src/sse.rs +++ b/provider-xai/src/sse.rs @@ -98,7 +98,9 @@ fn build_content(state: &PartialState) -> Vec { // to the salvaged leading fields or `{"_raw": …}` — always an // object (replay-safe) that preserves the evidence. serde_json::from_str(&fc.args_json) - .unwrap_or_else(|_| llm_router::types::messages::degraded_arguments(&fc.args_json)) + .ok() + .filter(Value::is_object) + .unwrap_or_else(|| llm_router::types::messages::degraded_arguments(&fc.args_json)) }; out.push(ContentBlock::FunctionCall { id: fc.id.clone(), diff --git a/provider-zai/src/sse.rs b/provider-zai/src/sse.rs index 0d876c3dc..92bf775c7 100644 --- a/provider-zai/src/sse.rs +++ b/provider-zai/src/sse.rs @@ -98,7 +98,9 @@ fn build_content(state: &PartialState) -> Vec { // to the salvaged leading fields or `{"_raw": …}` — always an // object (replay-safe) that preserves the evidence. serde_json::from_str(&fc.args_json) - .unwrap_or_else(|_| llm_router::types::messages::degraded_arguments(&fc.args_json)) + .ok() + .filter(Value::is_object) + .unwrap_or_else(|| llm_router::types::messages::degraded_arguments(&fc.args_json)) }; out.push(ContentBlock::FunctionCall { id: fc.id.clone(), From 30b15310cef857838f5133b0e1e9aeeff9a65d35 Mon Sep 17 00:00:00 2001 From: Anderson Leal Date: Fri, 10 Jul 2026 13:55:05 -0300 Subject: [PATCH 08/11] fix(harness,console): accurate fired records, ok-stream drain, console guards (MOT-3949, MOT-3952, MOT-3954) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - react: fired records use a per-fire entry id (spawned turn id) so recurring owner-delivered reactions stop deduping every fire after the first into one notice; the engine trigger id is resolved while the binding is live, teardown runs first, and retired reflects the actual unregister outcome — a failed teardown no longer renders a live, still-firing row as unregistered with only a local dismiss. - router client: after the ack, a 750ms lull only means EOF for failed dispatches; an ok response keeps draining for its terminal frame (10s cap) instead of synthesizing a no-terminal error over a healthy stream. A frame-less-but-acked stream now fails the outcome instead of completing ok around an empty error-stopped message. - console: transcript hydration is keyed on the active conversation's hydrated flag, so live entry events no longer cancel and restart the paginated fetch mid-stream; the thinking-level buttons are gated on provider availability, closing the side door that selected an unavailable model past its disabled row. Found by codex review and coderabbit on PR #468. --- .../web/src/components/chat/ModelPicker.tsx | 6 +- console/web/src/hooks/use-conversations.ts | 19 +++- harness/src/clients/router.rs | 37 +++++++- harness/src/functions/react.rs | 92 +++++++++++++------ 4 files changed, 113 insertions(+), 41 deletions(-) diff --git a/console/web/src/components/chat/ModelPicker.tsx b/console/web/src/components/chat/ModelPicker.tsx index 7653d6124..471e1413b 100644 --- a/console/web/src/components/chat/ModelPicker.tsx +++ b/console/web/src/components/chat/ModelPicker.tsx @@ -208,7 +208,11 @@ export function ModelPicker({
{g.options.map((opt) => { const expanded = expandedModelId === opt.id - const showThinking = opt.supportsThinking === true + // Gates the edit button AND the level panel: picking a + // level calls onChange(opt.id), which must not select a + // model whose row is disabled as unavailable. + const showThinking = + opt.supportsThinking === true && !unavailable return (
diff --git a/console/web/src/hooks/use-conversations.ts b/console/web/src/hooks/use-conversations.ts index 938da6e99..5ab8b510a 100644 --- a/console/web/src/hooks/use-conversations.ts +++ b/console/web/src/hooks/use-conversations.ts @@ -504,11 +504,20 @@ export function useConversations( /* Hydrate the active conversation's transcript once (read-back, then the live subscription above keeps it current). Folding through applyEntryUpsert makes the read idempotent against events that raced in - while the fetch was in flight. */ + while the fetch was in flight. + + Keyed on the active conversation's `hydrated` FLAG, not the + conversations array: every live message-added/updated rebuilds the + array, and an array dep would cancel + restart the in-flight fetch on + each event — under a fast stream the paginated read never lands and + hammers the backend. The flag stays false for the whole fetch, so live + events don't disturb it. */ + const activeNeedsHydration = + activeIsServerBacked && + !!activeId && + conversations.some((c) => c.id === activeId && !c.hydrated) useEffect(() => { - if (!activeIsServerBacked || !activeId) return - const conv = conversations.find((c) => c.id === activeId) - if (!conv || conv.hydrated) return + if (!activeNeedsHydration || !activeId) return const sessionId = activeId let cancelled = false void fetchTranscript(sessionId) @@ -539,7 +548,7 @@ export function useConversations( return () => { cancelled = true } - }, [activeIsServerBacked, activeId, conversations, patchConversation]) + }, [activeNeedsHydration, activeId, patchConversation]) /* Backgrounded sessions receive no transcript events (the subscription above is active-only), so anything that changed while away is missing diff --git a/harness/src/clients/router.rs b/harness/src/clients/router.rs index 1bde5faa9..df53bf0a4 100644 --- a/harness/src/clients/router.rs +++ b/harness/src/clients/router.rs @@ -178,12 +178,26 @@ impl RouterClient { // After the trigger resolves, drain already-buffered frames briefly, // then fall through to the outcome handling below. let mut response: Option> = None; + let mut drain_deadline = Instant::now(); loop { let frame = if response.is_some() { - // timeout elapsed → grace drain over, no writer is coming - tokio::time::timeout(Duration::from_millis(750), rx.recv()) - .await - .unwrap_or_default() + match tokio::time::timeout(Duration::from_millis(750), rx.recv()).await { + Ok(f) => f, + // A lull after the ack means "no writer is coming" only + // for failed dispatches. A successful response is always + // followed by a terminal frame, so keep draining for it — + // treating the lull as EOF would synthesize a no-terminal + // error over an ok stream (the forwarder can lag the ack). + // ponytail: 10s cap; a writer lagging past it is a wedge. + Err(_) => { + let ok_ack = matches!(&response, Some(Ok(v)) + if v.get("ok").and_then(Value::as_bool).unwrap_or(true)); + if ok_ack && final_message.is_none() && Instant::now() < drain_deadline { + continue; + } + None + } + } } else { tokio::select! { f = rx.recv() => f, @@ -191,6 +205,7 @@ impl RouterClient { response = Some(r.map_err(|e| { HarnessError::Internal(format!("router::chat task: {e}")) })?); + drain_deadline = Instant::now() + Duration::from_secs(10); continue; } } @@ -278,11 +293,23 @@ impl RouterClient { }); let stop_reason = Some(message.stop_reason); + // A frame-less-but-acked stream synthesizes an Error message above + // with no terminal_error/response_error set — derive the outcome + // error from it so the turn fails instead of completing "ok" around + // an empty error-stopped message. let error = terminal_error .or(response_error) + .or_else(|| { + (message.stop_reason == StopReason::Error).then(|| { + message + .error_message + .clone() + .unwrap_or_else(|| "router produced no terminal frame".to_string()) + }) + }) .filter(|_| !ok || message.stop_reason == StopReason::Error); Ok(ChatOutcome { - ok: ok && error.is_none(), + ok: ok && message.stop_reason != StopReason::Error && error.is_none(), message, stop_reason, error, diff --git a/harness/src/functions/react.rs b/harness/src/functions/react.rs index 35400cd82..8438c528b 100644 --- a/harness/src/functions/react.rs +++ b/harness/src/functions/react.rs @@ -214,14 +214,21 @@ pub struct ReactResult { /// error). Present iff `!spawned`. #[serde(skip_serializing_if = "Option::is_none")] pub note: Option, + /// The spawned turn id — per-fire unique even when delivery reuses the + /// pinned/owner session (the default). Local bookkeeping for fired-record + /// entry ids only; kept off the wire. + #[serde(skip)] + #[schemars(skip)] + pub child_turn_id: Option, } impl ReactResult { - fn spawned(child: Option) -> Self { + fn spawned(child: Option, turn: Option) -> Self { Self { spawned: true, child_session_id: child, note: None, + child_turn_id: turn, } } fn note(msg: impl Into) -> Self { @@ -229,6 +236,7 @@ impl ReactResult { spawned: false, child_session_id: None, note: Some(msg.into()), + child_turn_id: None, } } } @@ -387,34 +395,42 @@ pub async fn handle( if let Ok(r) = &res { if r.spawned { let sub = spec.subscription_id.as_deref().unwrap_or("sub"); - let entry_id = if spec.once { - format!("e_trigfired_{sub}") - } else { - // Recurring: the child session id is unique per spawn. - // ponytail: falls back to `spawn` when the spawn returned - // no child id, so two child-less recurring fires dedup to - // one record — acceptable, bounded by the fire-rate gate. - format!( - "e_trigfired_{sub}_{}", - r.child_session_id.as_deref().unwrap_or("spawn") - ) - }; - // Record with the binding still live (so its engine trigger - // id resolves) BEFORE the once teardown below. + // Per-fire suffix: the spawned turn id is unique even when + // delivery reuses the pinned/owner session (the default), + // where the child session id repeats and would dedup every + // recurring fire after the first into one record. + // ponytail: `spawn` fallback when the spawn returned no + // ids — such fires dedup to one record, bounded by the + // fire-rate gate. + let fire_key = r + .child_turn_id + .as_deref() + .or(r.child_session_id.as_deref()) + .unwrap_or("spawn"); + let entry_id = format!("e_trigfired_{sub}_{fire_key}"); + // Resolve the engine trigger id while the binding is live, + // tear the once-binding down, then record what actually + // happened: a failed unregister must not claim `retired` — + // the row is still live in the panel and must keep its + // real unregister action (the retained mapping retries on + // the next fire, whose record then carries retired:true). + let trigger_id = spec + .subscription_id + .as_deref() + .and_then(|s| deps.subscriptions.trigger_id_of(s)); + let retired = spec.once && once_unregister(deps, &spec).await; emit_fired( deps, &spec, &event, &entry_id, r.child_session_id.as_deref(), - spec.once, + retired, + trigger_id, None, None, ) .await; - if spec.once { - once_unregister(deps, &spec).await; - } } } res @@ -466,6 +482,7 @@ async fn join_edge( &format!("e_trigfired_join_{}_{}", join.id, join.key), None, // nothing spawned yet false, // predecessor stays registered until the join completes + None, // binding live — resolve inside Some(crate::subscriptions::fired::JoinProgress { id: &join.id, key: &join.key, @@ -534,6 +551,7 @@ async fn join_edge( &format!("e_trigfired_join_{}_done", join.id), r.child_session_id.as_deref(), !join.rearm, + None, // predecessors already retired; sub-keyed ghost is right Some(crate::subscriptions::fired::JoinProgress { id: &join.id, key: &join.key, @@ -597,6 +615,10 @@ async fn spawn_reaction( .get("child_session_id") .and_then(Value::as_str) .map(str::to_string); + let turn = v + .get("child_turn_id") + .and_then(Value::as_str) + .map(str::to_string); tracing::info!( child_session_id = child.as_deref(), model = %spec.model, @@ -604,7 +626,7 @@ async fn spawn_reaction( reactive_depth, "harness::react: reaction spawned" ); - Ok(ReactResult::spawned(child)) + Ok(ReactResult::spawned(child, turn)) } Err(e) => { tracing::warn!(error = %e, "harness::react: harness::spawn dispatch failed"); @@ -720,23 +742,31 @@ async fn retire_binding(deps: &Deps, id: &str) -> Result<(), HarnessError> { /// A `once: true` simple edge spawned: retire its binding so it never refires. /// Best-effort — a failed unregister only risks an extra fire, never the /// spawn, and the retained mapping lets the next fire retry the retirement. -async fn once_unregister(deps: &Deps, spec: &ReactSpec) { +/// Returns whether the binding was actually retired, so the fired record's +/// `retired` flag reflects reality (a live binding mislabeled retired would +/// render dismiss-only in the console while it keeps firing). +async fn once_unregister(deps: &Deps, spec: &ReactSpec) -> bool { let Some(id) = spec.subscription_id.as_deref() else { tracing::warn!( "harness::react: once-binding fired without a subscription id; cannot auto-unregister" ); - return; + return false; }; - if let Err(e) = retire_binding(deps, id).await { - tracing::warn!(error = %e, subscription = %id, "harness::react: once-binding auto-unregister failed; retrying on the next fire"); + match retire_binding(deps, id).await { + Ok(()) => true, + Err(e) => { + tracing::warn!(error = %e, subscription = %id, "harness::react: once-binding auto-unregister failed; retrying on the next fire"); + false + } } } /// Append a durable `trigger_fired` record into the owner (registering) chat so /// the console renders a turn-less notice and keeps a fired binding visible in /// the panel after teardown. Best-effort; owner-less raw registrations (no chat -/// to surface into) are skipped. Read the engine trigger id BEFORE any -/// retirement so a still-live binding resolves. +/// to surface into) are skipped. Callers that tear a binding down pass the +/// pre-resolved `trigger_id` (read while the binding was live); `None` falls +/// back to resolving the still-live binding here. #[allow(clippy::too_many_arguments)] async fn emit_fired( deps: &Deps, @@ -745,6 +775,7 @@ async fn emit_fired( entry_id: &str, child_session_id: Option<&str>, retired: bool, + trigger_id: Option, join: Option>, note: Option<&str>, ) { @@ -753,10 +784,11 @@ async fn emit_fired( return; }; let sub = spec.subscription_id.as_deref().unwrap_or(""); - let trigger_id = spec - .subscription_id - .as_deref() - .and_then(|s| deps.subscriptions.trigger_id_of(s)); + let trigger_id = trigger_id.or_else(|| { + spec.subscription_id + .as_deref() + .and_then(|s| deps.subscriptions.trigger_id_of(s)) + }); let (scope, key) = fired::event_state_watch(event); let session = deps.session().await; fired::emit( From 98eb53c909499ff39a25caff99146c94481d8982 Mon Sep 17 00:00:00 2001 From: Anderson Leal Date: Fri, 10 Jul 2026 14:07:15 -0300 Subject: [PATCH 09/11] fix(provider-anthropic): eager_input_streaming replaces retired beta header (MOT-3952) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fine-grained-tool-streaming-2025-05-14 went GA as the per-tool eager_input_streaming flag, and stale anthropic-beta values get rejected by some gateways (e.g. Bedrock). Probe-verified on api.anthropic.com: with the flag and no header a ~1k-word tool input streams with a max 0.4s non-ping gap; the same request without it buffers server-side (40s ping-only silence, then one 134KB burst) — the signature that was tripping the router's 120s idle guard on large state::set args. Found by coderabbit on PR #468. --- provider-anthropic/src/request.rs | 24 ++++++++---------------- provider-anthropic/src/wire/tools.rs | 15 +++++++++++++++ 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/provider-anthropic/src/request.rs b/provider-anthropic/src/request.rs index 10a174942..27bfa3b76 100644 --- a/provider-anthropic/src/request.rs +++ b/provider-anthropic/src/request.rs @@ -13,16 +13,6 @@ use serde_json::{json, Value}; pub const ANTHROPIC_VERSION: &str = "2023-06-01"; -/// Stream `tool_use.input` incrementally instead of the default server-side -/// buffering, which goes ping-only silent for the whole generation of a large -/// input (measured: ~120s of silence for a ~4k-token input, one leading -/// ~100B delta then a single burst) — long enough to trip the router's -/// 120s post-content idle guard and kill a healthy stream. Trade-off per the -/// API docs: on an early stop the accumulated input may be partial/invalid -/// JSON, which `llm_router::types::messages::degraded_arguments` already -/// salvages. -pub const ANTHROPIC_BETA: &str = "fine-grained-tool-streaming-2025-05-14"; - pub struct BodyArgs { pub model: String, pub max_tokens: u64, @@ -92,12 +82,14 @@ pub fn auth_header(auth_mode: AuthMode, credential_value: &str) -> (&'static str } } -/// No thinking beta header: adaptive thinking interleaves natively. +/// No beta headers: adaptive thinking interleaves natively, and incremental +/// tool-input streaming is the GA per-tool `eager_input_streaming` flag +/// (stamped in `wire::tools`), not the retired fine-grained-tool-streaming +/// beta header some gateways now reject. pub fn build_headers(cfg: &AnthropicConfig) -> Vec<(&'static str, String)> { vec![ auth_header(cfg.auth_mode, &cfg.credential_value), ("anthropic-version", ANTHROPIC_VERSION.to_string()), - ("anthropic-beta", ANTHROPIC_BETA.to_string()), ("content-type", "application/json".to_string()), ] } @@ -242,10 +234,10 @@ mod tests { let h = build_headers(&cfg(AuthMode::ApiKey)); assert!(h.contains(&("x-api-key", "sk-test".to_string()))); assert!(h.contains(&("anthropic-version", ANTHROPIC_VERSION.to_string()))); - // Fine-grained tool streaming: without it, large tool inputs are - // server-buffered into a ping-only silence that trips the router's - // idle guard. - assert!(h.contains(&("anthropic-beta", ANTHROPIC_BETA.to_string()))); + // Tool-input streaming is the per-tool eager_input_streaming flag + // (wire::tools), NOT a beta header — stale beta values get rejected + // by some gateways. + assert!(!h.iter().any(|(k, _)| *k == "anthropic-beta")); let h = build_headers(&cfg(AuthMode::OauthBearer)); assert!(h.contains(&("authorization", "Bearer sk-test".to_string()))); diff --git a/provider-anthropic/src/wire/tools.rs b/provider-anthropic/src/wire/tools.rs index 828c4fbb8..ae57d14df 100644 --- a/provider-anthropic/src/wire/tools.rs +++ b/provider-anthropic/src/wire/tools.rs @@ -11,6 +11,17 @@ pub fn functions_to_wire(tools: &[AgentFunction]) -> Vec { "name": encode_tool_name(&t.name), "description": t.description, "input_schema": t.parameters, + // Stream tool input incrementally instead of the default + // server-side buffering, which goes ping-only silent for the + // whole generation of a large input (measured: ~120s for a + // ~4k-token input, one leading ~100B delta then a single + // burst) — long enough to trip the router's 120s post-content + // idle guard and kill a healthy stream. GA successor of the + // fine-grained-tool-streaming-2025-05-14 beta header (which + // some gateways now reject). Trade-off: on an early stop the + // input may be partial/invalid JSON, which degraded_arguments + // salvages and the harness refuses to execute. + "eager_input_streaming": true, }) }) .collect() @@ -34,6 +45,10 @@ mod tests { assert_eq!(wire[0]["name"], "agent__trigger"); assert_eq!(wire[0]["description"], "Invoke an iii function"); assert_eq!(wire[0]["input_schema"]["type"], "object"); + // Incremental tool-input streaming: without it, large inputs are + // server-buffered into a ping-only silence that trips the router's + // idle guard. + assert_eq!(wire[0]["eager_input_streaming"], true); assert!( wire[0].get("label").is_none(), "label/execution_mode are iii-side only" From baf61a04db1840a3ec7fdd963a89c1b7d45b3f76 Mon Sep 17 00:00:00 2001 From: Anderson Leal Date: Fri, 10 Jul 2026 14:16:24 -0300 Subject: [PATCH 10/11] chore(console): drop redundant borrow in assert format arg (clippy 1.97) Stable clippy 1.97 added useless_borrows_in_formatting, failing the console rust lint CI job on a pre-existing test assertion. --- console/tests/integration.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/console/tests/integration.rs b/console/tests/integration.rs index 75c8b50ff..86df053a9 100644 --- a/console/tests/integration.rs +++ b/console/tests/integration.rs @@ -114,7 +114,7 @@ async fn end_to_end_http_and_ws_proxy() { assert!( body.contains("id=\"root\""), "GET / did not include the SPA mount point; body starts with: {}", - &body.chars().take(200).collect::(), + body.chars().take(200).collect::(), ); // 2. Pluck the first asset href out of index.html and GET it. From 269bf27b6551a01471910f095e73103e522cf78c Mon Sep 17 00:00:00 2001 From: Anderson Leal Date: Fri, 10 Jul 2026 14:59:12 -0300 Subject: [PATCH 11/11] fix(harness,providers,console): close degraded-args unwrap bypass, honest retirement, hydration replay (MOT-3952, MOT-3954) Codex round-3 findings: - plan_calls propagates _partial/_raw from the agent_trigger wrapper into the unwrapped payload: a max_tokens cut landing after a complete payload salvaged to {function, payload, _partial} and the unwrap shed the marker, bypassing the turn loop's refusal to execute provider-degraded arguments. - unregister_engine_trigger returns the real outcome; notify once-fires and join completions record retired only when teardown actually succeeded, instead of a dismiss-only console ghost for a trigger that can still fire. - Provider upstream read_timeout (120s) is overridable via PROVIDER_READ_TIMEOUT_SECS so a fixed cap can't undercut router idle/stream budgets raised for slow self-hosted endpoints. - Console hydration buffers live upserts that land while the transcript fetch is in flight and replays them over the snapshot (mergeHydratedTranscript), so the older read can't clobber a newer revision that hydrated: true would then pin stale. --- .../web/src/hooks/use-conversations.test.ts | 48 ++++++- console/web/src/hooks/use-conversations.ts | 121 +++++++++++++----- harness/src/functions/react.rs | 17 ++- harness/src/functions/subscribe.rs | 12 +- harness/src/policy.rs | 36 +++++- harness/src/subscriptions/notify_agent.rs | 8 +- provider-anthropic/src/register.rs | 15 ++- provider-openai-codex/src/register.rs | 13 +- provider-openai/src/register.rs | 13 +- provider-xai/src/register.rs | 13 +- provider-zai/src/register.rs | 13 +- 11 files changed, 256 insertions(+), 53 deletions(-) diff --git a/console/web/src/hooks/use-conversations.test.ts b/console/web/src/hooks/use-conversations.test.ts index f15d96cc3..85a651c15 100644 --- a/console/web/src/hooks/use-conversations.test.ts +++ b/console/web/src/hooks/use-conversations.test.ts @@ -1,11 +1,13 @@ import { describe, expect, it } from 'vitest' -import type { SessionMeta } from '@/lib/sessions/types' +import { transcriptToMessages } from '@/lib/sessions/entry-mapper' +import type { SessionMeta, TranscriptItem } from '@/lib/sessions/types' import type { Conversation } from '@/types/chat' import { appendMessageToConversation, applyCatalogModelFallback, markBackgroundedStale, mergeConversationMeta, + mergeHydratedTranscript, } from './use-conversations' function conversation(overrides: Partial): Conversation { @@ -170,6 +172,50 @@ describe('markBackgroundedStale', () => { }) }) +describe('mergeHydratedTranscript', () => { + const opts = { sessionId: 'console-1', working: false } + + function assistantItem(entryId: string, text: string): TranscriptItem { + return { + entry_id: entryId, + message: { + role: 'assistant', + content: [{ type: 'text', text }], + stop_reason: 'end', + model: 'm', + provider: 'p', + timestamp: 2, + }, + } + } + const toMessages = (items: TranscriptItem[]) => + transcriptToMessages(items, 'console-1', { working: false }) + + // Regression: an update landing while the hydration fetch was in flight is + // newer than the snapshot; without the replay the older read wins and + // `hydrated: true` pins the stale text until the next session switch. + it('replays a mid-fetch upsert over the older fetched snapshot', () => { + const merged = mergeHydratedTranscript( + toMessages([assistantItem('e1', 'old partial')]), + [], + [{ item: assistantItem('e1', 'final text'), updated: true }], + opts, + ) + expect(merged).toHaveLength(1) + expect(merged[0]).toMatchObject({ id: 'e1:0', content: 'final text' }) + }) + + it('keeps live-only messages the read did not return', () => { + const merged = mergeHydratedTranscript( + toMessages([assistantItem('e1', 'a')]), + toMessages([assistantItem('local-1', 'pending')]), + [], + opts, + ) + expect(merged.map((m) => m.id)).toEqual(['e1:0', 'local-1:0']) + }) +}) + describe('appendMessageToConversation', () => { it('marks a session working as soon as the user send is appended', () => { const next = appendMessageToConversation( diff --git a/console/web/src/hooks/use-conversations.ts b/console/web/src/hooks/use-conversations.ts index 5ab8b510a..102c24c8d 100644 --- a/console/web/src/hooks/use-conversations.ts +++ b/console/web/src/hooks/use-conversations.ts @@ -44,7 +44,7 @@ import { subscribeSessionDirectory, subscribeSessionTranscript, } from '@/lib/sessions/events' -import type { SessionMeta } from '@/lib/sessions/types' +import type { SessionMeta, TranscriptItem } from '@/lib/sessions/types' import { loadActiveId, loadLastModel, @@ -269,6 +269,36 @@ export interface ConversationsApi { * `catalogReady` gates it so a stale placeholder catalog can't clobber picks. * @param serverEnabled Wire the store to session-manager (real backend only). */ +/** Live entry upsert captured while a hydration fetch was in flight. */ +export type HydrationUpsert = { item: TranscriptItem; updated: boolean } + +/** Fold a hydration read together with what the live feed did meanwhile: + replay the buffered upserts on top (same entry id → live wins, the + fetched snapshot predates them), then re-append live-only messages the + read didn't return. Without the replay, an update landing mid-fetch is + clobbered by the older snapshot and `hydrated: true` pins it stale. */ +export function mergeHydratedTranscript( + fetched: Message[], + live: Message[], + upserts: HydrationUpsert[], + opts: { sessionId: string; working: boolean }, +): Message[] { + let messages = fetched + for (const u of upserts) { + messages = applyEntryUpsert(messages, u.item, { + sessionId: opts.sessionId, + streaming: u.updated ? opts.working : undefined, + working: opts.working, + }) + } + for (const m of live) { + if (!messages.some((existing) => existing.id === m.id)) { + messages = [...messages, m] + } + } + return messages +} + export function useConversations( catalogKeysForValidation?: readonly string[], catalogReady?: boolean, @@ -290,6 +320,13 @@ export function useConversations( /** Highest seen `message-updated` revision per (session, entry). */ const revisionsRef = useRef(new Map>()) + /** Upserts received while a hydration fetch is in flight; replayed over + the fetched snapshot so the older read can't clobber a newer entry. */ + const hydrationBufferRef = useRef<{ + sessionId: string + upserts: HydrationUpsert[] + } | null>(null) + const patchConversation = useCallback( (id: string, patch: (c: Conversation) => Conversation) => { setConversations((list) => list.map((c) => (c.id === id ? patch(c) : c))) @@ -454,18 +491,22 @@ export function useConversations( if (cancelled) return off = subscribeSessionTranscript(client, sessionId, { onMessageAdded: (event) => { + const item = { + entry_id: event.entry_id, + message: event.message, + custom: event.custom, + origin: event.origin, + } + const buf = hydrationBufferRef.current + if (buf && buf.sessionId === sessionId) { + buf.upserts.push({ item, updated: false }) + } patchConversation(sessionId, (c) => ({ ...c, - messages: applyEntryUpsert( - c.messages, - { - entry_id: event.entry_id, - message: event.message, - custom: event.custom, - origin: event.origin, - }, - { sessionId, working: c.status === 'working' }, - ), + messages: applyEntryUpsert(c.messages, item, { + sessionId, + working: c.status === 'working', + }), updatedAt: event.timestamp, })) }, @@ -474,21 +515,22 @@ export function useConversations( const prev = revs.get(event.entry_id) ?? -1 if (event.revision <= prev) return revs.set(event.entry_id, event.revision) + const item = { + entry_id: event.entry_id, + message: event.message, + origin: event.origin, + } + const buf = hydrationBufferRef.current + if (buf && buf.sessionId === sessionId) { + buf.upserts.push({ item, updated: true }) + } patchConversation(sessionId, (c) => ({ ...c, - messages: applyEntryUpsert( - c.messages, - { - entry_id: event.entry_id, - message: event.message, - origin: event.origin, - }, - { - sessionId, - streaming: c.status === 'working', - working: c.status === 'working', - }, - ), + messages: applyEntryUpsert(c.messages, item, { + sessionId, + streaming: c.status === 'working', + working: c.status === 'working', + }), updatedAt: event.timestamp, })) }, @@ -520,20 +562,31 @@ export function useConversations( if (!activeNeedsHydration || !activeId) return const sessionId = activeId let cancelled = false + /* Buffer live upserts for the fetch window: for the same entry id they + are newer than the snapshot, and replaying them after the merge keeps + the read from clobbering a revision that landed mid-flight. */ + const upserts: HydrationUpsert[] = [] + hydrationBufferRef.current = { sessionId, upserts } + const releaseBuffer = () => { + if (hydrationBufferRef.current?.upserts === upserts) { + hydrationBufferRef.current = null + } + } void fetchTranscript(sessionId) .then((items) => { if (cancelled) return patchConversation(sessionId, (c) => { - let messages = transcriptToMessages(items, sessionId, { - working: c.status === 'working', - }) - // Re-apply anything the live feed already reconciled on top. - for (const m of c.messages) { - if (!messages.some((existing) => existing.id === m.id)) { - messages = [...messages, m] - } + const working = c.status === 'working' + return { + ...c, + messages: mergeHydratedTranscript( + transcriptToMessages(items, sessionId, { working }), + c.messages, + upserts, + { sessionId, working }, + ), + hydrated: true, } - return { ...c, messages, hydrated: true } }) }) .catch((err) => { @@ -545,8 +598,10 @@ export function useConversations( ) } }) + .finally(releaseBuffer) return () => { cancelled = true + releaseBuffer() } }, [activeNeedsHydration, activeId, patchConversation]) diff --git a/harness/src/functions/react.rs b/harness/src/functions/react.rs index 8438c528b..4c47f442f 100644 --- a/harness/src/functions/react.rs +++ b/harness/src/functions/react.rs @@ -519,11 +519,13 @@ async fn join_edge( // when the agent registered through the engine::register_trigger // interceptor) so fired joins don't leak the session's subscription cap. // Best-effort — a failed unregister never blocks the downstream spawn. + let mut retired = !join.rearm; if join.rearm { tracing::info!(join = %join.id, "harness::react: join re-armed; predecessor subscriptions stay registered"); } else { for id in join_binding_ids(&rec) { if let Err(e) = retire_binding(deps, &id).await { + retired = false; tracing::warn!(error = %e, join = %join.id, subscription = %id, "harness::react: join predecessor auto-unregister failed"); } } @@ -535,12 +537,13 @@ async fn join_edge( let task = gather_inputs_task(&spec.task, &rec); let res = spawn_reaction(deps, task, spec, parent, spawn_depth).await; - // The join committed: the downstream spawned and (unless re-armed) every - // predecessor was just auto-unregistered above. One completion record lets - // the console mark the whole join fired + retired and post the notice. - // Gated on `spawned` like the simple edge — spawn_reaction swallows - // dispatch errors into `spawned: false`, and a record claiming "spawned" - // for a spawn that never happened would mislead the chat. + // The join committed: the downstream spawned and (unless re-armed) the + // predecessors were torn down above — `retired` carries the real outcome, + // so a failed unregister is never reported as gone. One completion record + // lets the console mark the whole join fired + retired and post the + // notice. Gated on `spawned` like the simple edge — spawn_reaction + // swallows dispatch errors into `spawned: false`, and a record claiming + // "spawned" for a spawn that never happened would mislead the chat. if let Ok(r) = &res { if r.spawned { let note = format!("{expected}/{expected} arrived — spawned"); @@ -550,7 +553,7 @@ async fn join_edge( &event, &format!("e_trigfired_join_{}_done", join.id), r.child_session_id.as_deref(), - !join.rearm, + retired, None, // predecessors already retired; sub-keyed ghost is right Some(crate::subscriptions::fired::JoinProgress { id: &join.id, diff --git a/harness/src/functions/subscribe.rs b/harness/src/functions/subscribe.rs index bbb3f250d..7f30fd28f 100644 --- a/harness/src/functions/subscribe.rs +++ b/harness/src/functions/subscribe.rs @@ -619,8 +619,10 @@ async fn handle_react( }) } -pub async fn unregister_engine_trigger(deps: &Deps, trigger_id: &str) { - if let Err(e) = deps +/// Best-effort engine-side teardown; `true` when the engine accepted it, so +/// callers recording a fired-trigger `retired` flag report the real outcome. +pub async fn unregister_engine_trigger(deps: &Deps, trigger_id: &str) -> bool { + match deps .iii .trigger(TriggerRequest { function_id: UNREGISTER_TRIGGER_ID.to_string(), @@ -630,7 +632,11 @@ pub async fn unregister_engine_trigger(deps: &Deps, trigger_id: &str) { }) .await { - tracing::warn!(trigger_id, error = %e, "subscription trigger unregister failed"); + Ok(_) => true, + Err(e) => { + tracing::warn!(trigger_id, error = %e, "subscription trigger unregister failed"); + false + } } } diff --git a/harness/src/policy.rs b/harness/src/policy.rs index e8a751c30..52822db5a 100644 --- a/harness/src/policy.rs +++ b/harness/src/policy.rs @@ -181,7 +181,7 @@ pub fn plan_calls(message: &AssistantMessage, expose: ExposeMode) -> Vec Vec) { else { return; }; + // `retired` reflects the actual teardown outcome: a failed unregister + // leaves the trigger live engine-side, and recording `true` would give + // the console a dismiss-only ghost for a trigger that can still fire. + let mut retired = false; if let Some(trigger_id) = claim.trigger_id.as_deref() { - crate::functions::subscribe::unregister_engine_trigger(deps, trigger_id).await; + retired = crate::functions::subscribe::unregister_engine_trigger(deps, trigger_id).await; } // Durable, turn-less UI signal: the console renders a "trigger fired" notice @@ -145,7 +149,7 @@ async fn on_fire(deps: &Deps, event: Value, metadata: Option) { label: meta.label.as_deref(), model: None, once: meta.once, - retired: claim.trigger_id.is_some(), + retired, scope, key, child_session_id: None, diff --git a/provider-anthropic/src/register.rs b/provider-anthropic/src/register.rs index 46c9320c5..d49da60d2 100644 --- a/provider-anthropic/src/register.rs +++ b/provider-anthropic/src/register.rs @@ -106,15 +106,26 @@ pub async fn declare_and_refresh(iii: IIIClient, http: reqwest::Client) { } } +/// Upstream read-silence bound, overridable via `PROVIDER_READ_TIMEOUT_SECS`: +/// a fixed 120s cap must not undercut router idle/stream budgets deliberately +/// raised for slow endpoints (long prompt eval on self-hosted gateways). +fn read_timeout() -> Duration { + std::env::var("PROVIDER_READ_TIMEOUT_SECS") + .ok() + .and_then(|s| s.parse().ok()) + .map(Duration::from_secs) + .unwrap_or(Duration::from_secs(120)) +} + pub async fn register_provider(iii: IIIClient) -> Result<(), Error> { // Streaming uses no total timeout (the router owns stream budgets), but // reads are silence-bounded: a stalled upstream otherwise pings the router // past its idle guard until the engine kills the call at stream_timeout // ("stream ended without a terminal frame"). Healthy streams emit SSE - // pings, so 120s of socket silence means a dead connection. + // pings, so prolonged socket silence means a dead connection. let http = reqwest::Client::builder() .connect_timeout(Duration::from_secs(10)) - .read_timeout(Duration::from_secs(120)) + .read_timeout(read_timeout()) .build() .expect("reqwest client"); diff --git a/provider-openai-codex/src/register.rs b/provider-openai-codex/src/register.rs index 178f7ea39..7470f6166 100644 --- a/provider-openai-codex/src/register.rs +++ b/provider-openai-codex/src/register.rs @@ -103,12 +103,23 @@ pub async fn declare_and_refresh(iii: IIIClient, http: reqwest::Client) { } } +/// Upstream read-silence bound, overridable via `PROVIDER_READ_TIMEOUT_SECS`: +/// a fixed 120s cap must not undercut router idle/stream budgets deliberately +/// raised for slow endpoints (long prompt eval on self-hosted gateways). +fn read_timeout() -> Duration { + std::env::var("PROVIDER_READ_TIMEOUT_SECS") + .ok() + .and_then(|s| s.parse().ok()) + .map(Duration::from_secs) + .unwrap_or(Duration::from_secs(120)) +} + pub async fn register_provider(iii: IIIClient) -> Result<(), Error> { // Reads are silence-bounded: a stalled upstream otherwise pings the router // past its idle guard until the engine kills the call at stream_timeout. let http = reqwest::Client::builder() .connect_timeout(Duration::from_secs(10)) - .read_timeout(Duration::from_secs(120)) + .read_timeout(read_timeout()) .build() .expect("reqwest client"); diff --git a/provider-openai/src/register.rs b/provider-openai/src/register.rs index 4583a749d..e791ea164 100644 --- a/provider-openai/src/register.rs +++ b/provider-openai/src/register.rs @@ -106,13 +106,24 @@ pub async fn declare_and_refresh(iii: IIIClient, http: reqwest::Client) { } } +/// Upstream read-silence bound, overridable via `PROVIDER_READ_TIMEOUT_SECS`: +/// a fixed 120s cap must not undercut router idle/stream budgets deliberately +/// raised for slow endpoints (long prompt eval on self-hosted gateways). +fn read_timeout() -> Duration { + std::env::var("PROVIDER_READ_TIMEOUT_SECS") + .ok() + .and_then(|s| s.parse().ok()) + .map(Duration::from_secs) + .unwrap_or(Duration::from_secs(120)) +} + pub async fn register_provider(iii: IIIClient) -> Result<(), Error> { // Streaming uses no total timeout (the router owns stream budgets), but // reads are silence-bounded: a stalled upstream otherwise pings the router // past its idle guard until the engine kills the call at stream_timeout. let http = reqwest::Client::builder() .connect_timeout(Duration::from_secs(10)) - .read_timeout(Duration::from_secs(120)) + .read_timeout(read_timeout()) .build() .expect("reqwest client"); diff --git a/provider-xai/src/register.rs b/provider-xai/src/register.rs index b23d2d1c4..9cc13b9ab 100644 --- a/provider-xai/src/register.rs +++ b/provider-xai/src/register.rs @@ -106,13 +106,24 @@ pub async fn declare_and_refresh(iii: IIIClient, http: reqwest::Client) { } } +/// Upstream read-silence bound, overridable via `PROVIDER_READ_TIMEOUT_SECS`: +/// a fixed 120s cap must not undercut router idle/stream budgets deliberately +/// raised for slow endpoints (long prompt eval on self-hosted gateways). +fn read_timeout() -> Duration { + std::env::var("PROVIDER_READ_TIMEOUT_SECS") + .ok() + .and_then(|s| s.parse().ok()) + .map(Duration::from_secs) + .unwrap_or(Duration::from_secs(120)) +} + pub async fn register_provider(iii: IIIClient) -> Result<(), Error> { // Streaming uses no total timeout (the router owns stream budgets), but // reads are silence-bounded: a stalled upstream otherwise pings the router // past its idle guard until the engine kills the call at stream_timeout. let http = reqwest::Client::builder() .connect_timeout(Duration::from_secs(10)) - .read_timeout(Duration::from_secs(120)) + .read_timeout(read_timeout()) .build() .expect("reqwest client"); diff --git a/provider-zai/src/register.rs b/provider-zai/src/register.rs index 200f25ce1..6f03836a9 100644 --- a/provider-zai/src/register.rs +++ b/provider-zai/src/register.rs @@ -109,13 +109,24 @@ pub async fn declare_and_refresh(iii: IIIClient) { } } +/// Upstream read-silence bound, overridable via `PROVIDER_READ_TIMEOUT_SECS`: +/// a fixed 120s cap must not undercut router idle/stream budgets deliberately +/// raised for slow endpoints (long prompt eval on self-hosted gateways). +fn read_timeout() -> Duration { + std::env::var("PROVIDER_READ_TIMEOUT_SECS") + .ok() + .and_then(|s| s.parse().ok()) + .map(Duration::from_secs) + .unwrap_or(Duration::from_secs(120)) +} + pub async fn register_provider(iii: IIIClient) -> Result<(), Error> { // Streaming uses no total timeout (the router owns stream budgets), but // reads are silence-bounded: a stalled upstream otherwise pings the router // past its idle guard until the engine kills the call at stream_timeout. let http = reqwest::Client::builder() .connect_timeout(Duration::from_secs(10)) - .read_timeout(Duration::from_secs(120)) + .read_timeout(read_timeout()) .build() .expect("reqwest client");