From 04bb234c9fcf312a49715b551f8a6e57b217ae19 Mon Sep 17 00:00:00 2001 From: serrrfirat Date: Wed, 12 Aug 2026 20:09:15 +0300 Subject: [PATCH 1/4] fix(loop-host): resolve deferred capabilities before guard --- .../ironclaw_loop_host/src/model_gateway.rs | 35 +++++++++++- .../ironclaw_loop_host/tests/llm_gateway.rs | 54 +++++++++++++++++++ 2 files changed, 88 insertions(+), 1 deletion(-) diff --git a/crates/loop/ironclaw_loop_host/src/model_gateway.rs b/crates/loop/ironclaw_loop_host/src/model_gateway.rs index a3269dea222..f16870edd0c 100644 --- a/crates/loop/ironclaw_loop_host/src/model_gateway.rs +++ b/crates/loop/ironclaw_loop_host/src/model_gateway.rs @@ -1424,7 +1424,10 @@ where } if !tool_definitions.is_empty() { let unavailable_capability_guard = - unavailable_requested_capability_guard(&completion.messages, &tool_definitions); + unavailable_requested_capability_guard(&completion.messages, &tool_definitions) + .filter(|guard| { + !requested_capability_is_resolvable(capabilities.as_ref(), guard) + }); let mut recovery_tool_names = Vec::with_capacity(tool_definitions.len()); let mut llm_tool_definitions = tool_definitions .into_iter() @@ -1998,6 +2001,36 @@ impl UnavailableCapabilityGuard { } } +fn requested_capability_is_resolvable( + capabilities: &dyn ironclaw_loop_contracts::LoopCapabilityPort, + guard: &UnavailableCapabilityGuard, +) -> bool { + // The advertised surface may be only a token-bounded subset. Resolve the + // exact provider name through the fully decorated port before classifying + // it as unavailable; the port applies disclosure policy and remains the + // authoritative fail-closed gate for deferred direct calls. + let provider_name = guard.capability_id.as_str().replace('.', "__"); + let Ok(probe) = ProviderToolCall::from_parts( + "capability-availability-guard", + "capability-availability-guard", + None, + "capability-availability-guard", + provider_name, + serde_json::json!({}), + ) else { + return false; + }; + capabilities + .provider_tool_call_capability_ids(&probe) + .is_ok_and(|ids| { + ids.provider_capability_id == guard.capability_id + || ids + .effective_capability_ids + .iter() + .any(|id| id == &guard.capability_id) + }) +} + fn unavailable_requested_capability_guard( messages: &[ChatMessage], tool_definitions: &[ProviderToolDefinition], diff --git a/crates/loop/ironclaw_loop_host/tests/llm_gateway.rs b/crates/loop/ironclaw_loop_host/tests/llm_gateway.rs index 6d597e2da96..b3a02a253c1 100644 --- a/crates/loop/ironclaw_loop_host/tests/llm_gateway.rs +++ b/crates/loop/ironclaw_loop_host/tests/llm_gateway.rs @@ -992,6 +992,51 @@ async fn gateway_allows_policy_filtered_discovery_for_named_deferred_capability( assert_eq!(capabilities.registered.lock().unwrap().len(), 1); } +#[tokio::test] +async fn gateway_allows_prerequisite_and_discovery_for_named_deferred_capability() { + let provider = Arc::new(ToolAwareProvider::tool_calls(vec![ + ToolCall { + id: "call_prerequisite".to_string(), + name: "demo__echo".to_string(), + arguments: serde_json::json!({"message": "inspect before using hidden tool"}), + reasoning: None, + signature: None, + arguments_parse_error: None, + }, + ToolCall { + id: "call_search".to_string(), + name: "tool_search".to_string(), + arguments: serde_json::json!({"query": "demo.hidden"}), + reasoning: None, + signature: None, + arguments_parse_error: None, + }, + ])); + let gateway = LlmProviderModelGateway::with_provider_identity( + STATIC_PROVIDER_ID, + provider, + LlmModelProfilePolicy::new() + .allow_model_profile(interactive_model(), Some("host-selected-model".to_string())), + ); + let capabilities = Arc::new(GatewayCapabilityPort::with_deferred_prerequisite_surface()); + let mut request = model_request(interactive_model()); + request.messages[1].content = + "Use the demo.hidden capability after inspecting its input.".to_string(); + + let response = gateway + .stream_model_with_capabilities(request, capabilities.clone()) + .await + .unwrap(); + + let ParentLoopOutput::CapabilityCalls(calls) = response.output else { + panic!("expected prerequisite and discovery calls"); + }; + assert_eq!(calls.len(), 2); + assert_eq!(calls[0].capability_id.as_str(), "demo.echo"); + assert_eq!(calls[1].capability_id.as_str(), "ironclaw.tool_search"); + assert_eq!(capabilities.registered.lock().unwrap().len(), 2); +} + #[tokio::test] async fn gateway_allows_exact_named_deferred_capability_for_policy_resolution() { let provider = Arc::new(ToolAwareProvider::tool_calls(vec![ToolCall { @@ -5241,6 +5286,15 @@ impl GatewayCapabilityPort { } } + fn with_deferred_prerequisite_surface() -> Self { + let mut port = Self::with_hidden_resolvable_tool_surface(); + let bridge = Self::with_discovery_bridge_surface(); + port.definitions.extend(bridge.definitions); + port.resolvable_definitions + .extend(bridge.resolvable_definitions); + port + } + fn with_builtin_shell_surface() -> Self { let definitions = vec![ProviderToolDefinition { capability_id: CapabilityId::new("builtin.shell").unwrap(), From d1d7d5f06aadaf4b0f62dcf3a6240531bb160faf Mon Sep 17 00:00:00 2001 From: serrrfirat Date: Wed, 12 Aug 2026 20:37:02 +0300 Subject: [PATCH 2/4] fix(loop-host): retain mixed-request suppression (#7551) --- .../ironclaw_loop_host/src/model_gateway.rs | 23 ++++++++++--- .../ironclaw_loop_host/tests/llm_gateway.rs | 33 +++++++++++++++++++ 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/crates/loop/ironclaw_loop_host/src/model_gateway.rs b/crates/loop/ironclaw_loop_host/src/model_gateway.rs index f16870edd0c..fd202e026ef 100644 --- a/crates/loop/ironclaw_loop_host/src/model_gateway.rs +++ b/crates/loop/ironclaw_loop_host/src/model_gateway.rs @@ -1424,8 +1424,9 @@ where } if !tool_definitions.is_empty() { let unavailable_capability_guard = - unavailable_requested_capability_guard(&completion.messages, &tool_definitions) - .filter(|guard| { + requested_unadvertised_capability_guards(&completion.messages, &tool_definitions) + .into_iter() + .find(|guard| { !requested_capability_is_resolvable(capabilities.as_ref(), guard) }); let mut recovery_tool_names = Vec::with_capacity(tool_definitions.len()); @@ -2031,14 +2032,27 @@ fn requested_capability_is_resolvable( }) } +#[cfg(test)] fn unavailable_requested_capability_guard( messages: &[ChatMessage], tool_definitions: &[ProviderToolDefinition], ) -> Option { + requested_unadvertised_capability_guards(messages, tool_definitions) + .into_iter() + .next() +} + +fn requested_unadvertised_capability_guards( + messages: &[ChatMessage], + tool_definitions: &[ProviderToolDefinition], +) -> Vec { let latest_user = messages .iter() .rev() - .find(|message| message.role == Role::User)?; + .find(|message| message.role == Role::User); + let Some(latest_user) = latest_user else { + return Vec::new(); + }; let visible_capability_ids = tool_definitions .iter() .map(|definition| definition.capability_id.as_str()) @@ -2055,8 +2069,9 @@ fn unavailable_requested_capability_guard( extract_explicit_capability_request_ids(&latest_user.content, &visible_namespaces) .into_iter() - .find(|capability_id| !visible_capability_ids.contains(capability_id.as_str())) + .filter(|capability_id| !visible_capability_ids.contains(capability_id.as_str())) .map(|capability_id| UnavailableCapabilityGuard { capability_id }) + .collect() } fn extract_explicit_capability_request_ids( diff --git a/crates/loop/ironclaw_loop_host/tests/llm_gateway.rs b/crates/loop/ironclaw_loop_host/tests/llm_gateway.rs index b3a02a253c1..734e22f03a9 100644 --- a/crates/loop/ironclaw_loop_host/tests/llm_gateway.rs +++ b/crates/loop/ironclaw_loop_host/tests/llm_gateway.rs @@ -1037,6 +1037,39 @@ async fn gateway_allows_prerequisite_and_discovery_for_named_deferred_capability assert_eq!(capabilities.registered.lock().unwrap().len(), 2); } +#[tokio::test] +async fn gateway_suppresses_substitute_when_later_named_capability_is_unavailable() { + let provider = Arc::new(ToolAwareProvider::tool_calls(vec![ToolCall { + id: "call_substitute".to_string(), + name: "demo__echo".to_string(), + arguments: serde_json::json!({"message": "substitute"}), + reasoning: None, + signature: None, + arguments_parse_error: None, + }])); + let gateway = LlmProviderModelGateway::with_provider_identity( + STATIC_PROVIDER_ID, + provider, + LlmModelProfilePolicy::new() + .allow_model_profile(interactive_model(), Some("host-selected-model".to_string())), + ); + let capabilities = Arc::new(GatewayCapabilityPort::with_deferred_prerequisite_surface()); + let mut request = model_request(interactive_model()); + request.messages[1].content = + "Use the demo.hidden capability, then use the builtin.disabled capability.".to_string(); + + let response = gateway + .stream_model_with_capabilities(request, capabilities.clone()) + .await + .unwrap(); + + assert!(capabilities.registered.lock().unwrap().is_empty()); + let ParentLoopOutput::AssistantReply(reply) = response.output else { + panic!("expected unavailable capability reply"); + }; + assert!(reply.content.contains("unavailable or disabled")); +} + #[tokio::test] async fn gateway_allows_exact_named_deferred_capability_for_policy_resolution() { let provider = Arc::new(ToolAwareProvider::tool_calls(vec![ToolCall { From 93ae624e80e6310451c4f186c4eae821f7d4f63d Mon Sep 17 00:00:00 2001 From: serrrfirat Date: Thu, 13 Aug 2026 00:26:01 +0300 Subject: [PATCH 3/4] fix(loop-host): repair unavailable capability calls --- .../ironclaw_loop_host/src/model_gateway.rs | 403 +----------------- .../ironclaw_loop_host/tests/llm_gateway.rs | 219 +++------- 2 files changed, 71 insertions(+), 551 deletions(-) diff --git a/crates/loop/ironclaw_loop_host/src/model_gateway.rs b/crates/loop/ironclaw_loop_host/src/model_gateway.rs index fd202e026ef..e3d24993b8b 100644 --- a/crates/loop/ironclaw_loop_host/src/model_gateway.rs +++ b/crates/loop/ironclaw_loop_host/src/model_gateway.rs @@ -31,10 +31,7 @@ use crate::{ }; use async_trait::async_trait; use ironclaw_common::llm_costs::{default_cost, model_cost}; -use ironclaw_host_api::{ - approval::sha256_digest_token, - ids::{CapabilityId, ProviderToolName}, -}; +use ironclaw_host_api::{approval::sha256_digest_token, ids::ProviderToolName}; use ironclaw_llm::{ ChatMessage, CompletionRequest, CompletionResponse, CompletionStreamSink, ContentPart, FinishReason, ImageUrl, LlmError, LlmProvider, Role, ToolCall, ToolCompletionRequest, @@ -90,7 +87,6 @@ const PROVIDER_TOOL_ARGUMENTS_OMITTED_MARKER: &str = const PROVIDER_TOOL_ARGUMENTS_INVALID_MARKER: &str = "arguments omitted because the provider emitted malformed tool-call JSON"; const CONTEXT_SHADOW_TARGET: &str = "ironclaw::reborn::context_shadow"; -const UNAVAILABLE_CAPABILITY_REPLY: &str = "That capability is unavailable or disabled for this request, so I will not route it through another tool."; fn trace_model_latency_ok( operation: &'static str, @@ -1423,12 +1419,6 @@ where ); } if !tool_definitions.is_empty() { - let unavailable_capability_guard = - requested_unadvertised_capability_guards(&completion.messages, &tool_definitions) - .into_iter() - .find(|guard| { - !requested_capability_is_resolvable(capabilities.as_ref(), guard) - }); let mut recovery_tool_names = Vec::with_capacity(tool_definitions.len()); let mut llm_tool_definitions = tool_definitions .into_iter() @@ -1506,7 +1496,6 @@ where .as_deref() .unwrap_or("model_call=unknown"), &replay_identity, - unavailable_capability_guard.as_ref(), ) .await { @@ -1589,7 +1578,6 @@ where .as_deref() .unwrap_or("model_call=unknown"), &replay_identity, - unavailable_capability_guard.as_ref(), ) .await; match &result { @@ -1768,7 +1756,6 @@ async fn tool_response_to_host( capabilities: Arc, provider_turn_scope: &str, replay_identity: &ProviderReplayIdentity, - unavailable_capability_guard: Option<&UnavailableCapabilityGuard>, ) -> Result { if tracing::enabled!(tracing::Level::DEBUG) { let tool_call_name_sample = response @@ -1791,28 +1778,6 @@ async fn tool_response_to_host( FinishReason::ToolUse | FinishReason::Stop ) { - if let Some(guard) = unavailable_capability_guard - && response - .tool_calls - .iter() - .any(|call| !guard.permits_policy_checked_call(call)) - { - debug!( - requested_capability_id = %guard.capability_id, - tool_call_count = response.tool_calls.len(), - "reborn model gateway suppressed provider tool calls after unavailable named capability request" - ); - return Ok(HostManagedModelResponse::assistant_reply_with_reasoning( - UNAVAILABLE_CAPABILITY_REPLY, - response.reasoning, - ) - .with_usage(LoopModelUsage { - input_tokens: response.input_tokens, - output_tokens: response.output_tokens, - cache_read_input_tokens: response.cache_read_input_tokens, - cache_creation_input_tokens: response.cache_creation_input_tokens, - })); - } let advertised_tool_names = capabilities .tool_definitions() .map_err(map_capability_host_error)? @@ -1978,253 +1943,6 @@ fn provider_calls_are_advertised_or_resolvable( true } -#[derive(Debug, Clone, PartialEq, Eq)] -struct UnavailableCapabilityGuard { - capability_id: CapabilityId, -} - -impl UnavailableCapabilityGuard { - fn permits_policy_checked_call(&self, call: &ToolCall) -> bool { - if matches!(call.name.as_str(), "tool_search" | "tool_describe") { - return true; - } - let canonical = self.capability_id.as_str(); - let encoded = canonical.replace('.', "__"); - if call.name == canonical || call.name == encoded { - return true; - } - call.name == "tool_call" - && call - .arguments - .get("name") - .and_then(serde_json::Value::as_str) - .is_some_and(|name| name == canonical || name == encoded) - } -} - -fn requested_capability_is_resolvable( - capabilities: &dyn ironclaw_loop_contracts::LoopCapabilityPort, - guard: &UnavailableCapabilityGuard, -) -> bool { - // The advertised surface may be only a token-bounded subset. Resolve the - // exact provider name through the fully decorated port before classifying - // it as unavailable; the port applies disclosure policy and remains the - // authoritative fail-closed gate for deferred direct calls. - let provider_name = guard.capability_id.as_str().replace('.', "__"); - let Ok(probe) = ProviderToolCall::from_parts( - "capability-availability-guard", - "capability-availability-guard", - None, - "capability-availability-guard", - provider_name, - serde_json::json!({}), - ) else { - return false; - }; - capabilities - .provider_tool_call_capability_ids(&probe) - .is_ok_and(|ids| { - ids.provider_capability_id == guard.capability_id - || ids - .effective_capability_ids - .iter() - .any(|id| id == &guard.capability_id) - }) -} - -#[cfg(test)] -fn unavailable_requested_capability_guard( - messages: &[ChatMessage], - tool_definitions: &[ProviderToolDefinition], -) -> Option { - requested_unadvertised_capability_guards(messages, tool_definitions) - .into_iter() - .next() -} - -fn requested_unadvertised_capability_guards( - messages: &[ChatMessage], - tool_definitions: &[ProviderToolDefinition], -) -> Vec { - let latest_user = messages - .iter() - .rev() - .find(|message| message.role == Role::User); - let Some(latest_user) = latest_user else { - return Vec::new(); - }; - let visible_capability_ids = tool_definitions - .iter() - .map(|definition| definition.capability_id.as_str()) - .collect::>(); - // Namespaces the agent actually has (a visible capability shares the prefix, - // e.g. `builtin`). Used only to rescue backticked references to REAL - // capability namespaces from the inline-code skip — a backticked `builtin.echo` - // is still a request, whereas a backticked `playwright.sync_api` (a library - // whose namespace this agent doesn't have) is a code reference. - let visible_namespaces = visible_capability_ids - .iter() - .filter_map(|id| id.split('.').next()) - .collect::>(); - - extract_explicit_capability_request_ids(&latest_user.content, &visible_namespaces) - .into_iter() - .filter(|capability_id| !visible_capability_ids.contains(capability_id.as_str())) - .map(|capability_id| UnavailableCapabilityGuard { capability_id }) - .collect() -} - -fn extract_explicit_capability_request_ids( - content: &str, - visible_namespaces: &HashSet<&str>, -) -> Vec { - let mut ids = Vec::new(); - let mut token_start = None; - // Track Markdown inline-code parity (per line) in this same single pass so we - // never rescan the line for each token — one long user line with many - // capability-shaped tokens would otherwise be O(n^2). - let mut in_inline_code = false; - let mut token_in_code = false; - for (index, character) in content.char_indices() { - if is_capability_token_char(character) { - if token_start.is_none() { - token_start = Some(index); - token_in_code = in_inline_code; - } - continue; - } - if let Some(start) = token_start.take() { - push_explicit_capability_request_token( - content, - start, - index, - token_in_code, - visible_namespaces, - &mut ids, - ); - } - match character { - '\n' => in_inline_code = false, - '`' => in_inline_code = !in_inline_code, - _ => {} - } - } - if let Some(start) = token_start { - push_explicit_capability_request_token( - content, - start, - content.len(), - token_in_code, - visible_namespaces, - &mut ids, - ); - } - ids -} - -fn is_capability_token_char(character: char) -> bool { - character.is_ascii_lowercase() - || character.is_ascii_digit() - || matches!(character, '_' | '-' | '.') -} - -fn push_explicit_capability_request_token( - content: &str, - start: usize, - end: usize, - in_inline_code: bool, - visible_namespaces: &HashSet<&str>, - ids: &mut Vec, -) { - let token = &content[start..end]; - if !is_likely_capability_reference(token) - || !is_explicit_capability_request_token(content, start, end) - { - return; - } - // Tokens written in Markdown inline code (e.g. "use `playwright.sync_api`", a - // Python module) are code references, not capability requests — ignore them. - // Two exceptions keep genuine requests covered even when backticked: - // - the prompt explicitly labels the token a tool/capability - // ("use the `builtin.http` capability"), or - // - the token names a real capability namespace this agent has - // (`builtin.echo` — `builtin` is a live namespace, unlike `playwright`). - if in_inline_code - && !has_capability_noun_context(content, start, end) - && !token_namespace_is_visible(token, visible_namespaces) - { - return; - } - if let Ok(capability_id) = CapabilityId::new(token) - && !ids.iter().any(|existing| existing == &capability_id) - { - ids.push(capability_id); - } -} - -fn is_likely_capability_reference(token: &str) -> bool { - // A decimal number lifted from prose (e.g. "use 0.95 in formulas") tokenizes - // as `digits.digits`, which satisfies the `namespace.name` shape below and is - // otherwise mistaken for an explicitly requested capability id. That trips the - // unavailable-capability guard and suppresses the entire turn's tool calls, - // stranding the model ("…will not route it through another tool"). A real - // capability id is never a bare number, so reject anything that parses as one. - if token.parse::().is_ok() { - return false; - } - token.starts_with("builtin.") || token.split('.').count() == 2 -} - -/// True when the token's namespace (its first dotted segment) is one the agent -/// actually has — a backticked reference to a real capability namespace is still -/// a request, unlike a library reference (`playwright.sync_api`). -fn token_namespace_is_visible(token: &str, visible_namespaces: &HashSet<&str>) -> bool { - token - .split('.') - .next() - .is_some_and(|namespace| visible_namespaces.contains(namespace)) -} - -/// The request-word immediately before `start` (alphanumeric/`_`/`-` run). -fn previous_request_word(content: &str, start: usize) -> Option<&str> { - content[..start] - .trim_end() - .rsplit(|character: char| !is_capability_request_word_char(character)) - .find(|word| !word.is_empty()) -} - -/// True when the word right before or after the token is an explicit "tool" / -/// "capability" noun — the prompt is calling the token out as a capability, so -/// it's a genuine request even when written in backticks. -fn has_capability_noun_context(content: &str, start: usize, end: usize) -> bool { - let next_word = content[end..] - .trim_start() - .split(|character: char| !is_capability_request_word_char(character)) - .find(|word| !word.is_empty()); - previous_request_word(content, start).is_some_and(is_capability_request_noun) - || next_word.is_some_and(is_capability_request_noun) -} - -fn is_explicit_capability_request_token(content: &str, start: usize, end: usize) -> bool { - previous_request_word(content, start).is_some_and(is_capability_request_verb) - || has_capability_noun_context(content, start, end) -} - -fn is_capability_request_word_char(character: char) -> bool { - character.is_ascii_alphanumeric() || matches!(character, '_' | '-') -} - -fn is_capability_request_verb(word: &str) -> bool { - matches!( - word.to_ascii_lowercase().as_str(), - "use" | "using" | "call" | "run" | "execute" | "invoke" - ) -} - -fn is_capability_request_noun(word: &str) -> bool { - matches!(word.to_ascii_lowercase().as_str(), "tool" | "capability") -} - fn provider_tool_call_from_llm( tool_call: ToolCall, response_reasoning: Option, @@ -2407,7 +2125,8 @@ fn map_provider_tool_output_error(error: AgentLoopHostError) -> HostManagedModel fn is_repairable_provider_tool_output_error(error: &HostManagedModelError) -> bool { error.kind == HostManagedModelErrorKind::InvalidOutput && (is_provider_arguments_too_large_summary(&error.safe_summary) - || is_provider_tool_arguments_parse_error_summary(&error.safe_summary)) + || is_provider_tool_arguments_parse_error_summary(&error.safe_summary) + || error.safe_summary == InvalidOutputReason::OutsideCapabilitySurface.safe_summary()) } fn is_provider_tool_arguments_parse_error_summary(safe_summary: &str) -> bool { @@ -2451,7 +2170,7 @@ fn provider_tool_repair_result_content(tool_call: &ToolCall, safe_summary: &str) content.push_str(parse_error); } content.push_str( - "\n\nNone of this response's tool calls were executed. Retry with valid JSON arguments or answer directly without this tool if it is not needed.", + "\n\nNone of this response's tool calls were executed. Retry with an available capability and valid arguments, or answer directly without the rejected tool.", ); content } @@ -3172,84 +2891,6 @@ mod tests { assert_eq!(auth.next_fallback_index, None); } - fn tool_def(capability_id: &str, name: &str) -> ProviderToolDefinition { - ProviderToolDefinition { - capability_id: CapabilityId::new(capability_id).unwrap(), - name: ProviderToolName::new(name).unwrap(), - description: String::new(), - description_trust: Default::default(), - parameters: serde_json::json!({}), - } - } - - #[test] - fn guard_ignores_incidental_code_references() { - // The playwright/browser tasks literally instruct: "use `playwright.sync_api`" - // — a Python module named right after a request verb. That is NOT a - // capability request; the guard must not fire and suppress the model's - // legitimate write_file calls. - let messages = vec![ChatMessage::user( - "Read form.html, then use `playwright.sync_api` (Python sync API) to \ - write an end-to-end test saved as test_form.py.", - )]; - let tools = vec![ - tool_def("builtin.write_file", "builtin__write_file"), - tool_def("builtin.read_file", "builtin__read_file"), - ]; - assert!( - unavailable_requested_capability_guard(&messages, &tools).is_none(), - "guard must not misfire on the code reference `playwright.sync_api`" - ); - } - - #[test] - fn guard_still_fires_on_real_disabled_capability() { - // A genuine, un-backticked request for a capability that isn't visible must - // still fire (`builtin.http` is gated off here). - let messages = vec![ChatMessage::user( - "Fetch the page using the builtin.http capability.", - )]; - let tools = vec![tool_def("builtin.write_file", "builtin__write_file")]; - let guard = unavailable_requested_capability_guard(&messages, &tools); - assert!( - guard.is_some(), - "guard should still fire for a real builtin capability that is disabled" - ); - assert_eq!(guard.unwrap().capability_id.as_str(), "builtin.http"); - } - - #[test] - fn guard_fires_on_backticked_capability_with_explicit_noun() { - // Backticks alone don't excuse a request the prompt explicitly labels a - // capability/tool — the inline-code skip must not swallow a genuine - // request. Here `builtin.http` is backticked but called a "capability". - let messages = vec![ChatMessage::user( - "Fetch the page using the `builtin.http` capability.", - )]; - let tools = vec![tool_def("builtin.write_file", "builtin__write_file")]; - let guard = unavailable_requested_capability_guard(&messages, &tools); - assert!( - guard.is_some(), - "explicitly-labeled capability must still fire even when backticked" - ); - assert_eq!(guard.unwrap().capability_id.as_str(), "builtin.http"); - } - - #[test] - fn guard_fires_on_backticked_known_namespace_capability() { - // A backticked reference to a REAL capability namespace this agent has - // (`builtin`) is still a request, even with only a request verb and no - // tool/capability noun — unlike a library ref such as `playwright.sync_api`. - let messages = vec![ChatMessage::user("Use `builtin.echo` to print the banner.")]; - let tools = vec![tool_def("builtin.write_file", "builtin__write_file")]; - let guard = unavailable_requested_capability_guard(&messages, &tools); - assert!( - guard.is_some(), - "backticked known-namespace capability must still fire" - ); - assert_eq!(guard.unwrap().capability_id.as_str(), "builtin.echo"); - } - #[test] fn unconfigured_provider_error_maps_to_credential_unavailable_not_availability() { // A placeholder "no LLM configured" failure must not be classified as @@ -3843,40 +3484,4 @@ mod tests { "repaired assistant message must preserve typed reasoning_details" ); } - - #[test] - fn is_likely_capability_reference_rejects_decimal_numbers() { - // Decimals from prose ("use 0.95 in formulas") tokenize as `digits.digits` - // and must NOT be treated as capability references — that false positive - // trips the unavailable-capability guard and suppresses the whole turn. - for token in ["0.95", "1.5", "95.0", "3.524", "0.0158"] { - assert!( - !is_likely_capability_reference(token), - "decimal {token} must not look like a capability reference" - ); - } - // Real capability ids are still recognized. - for token in ["builtin.shell", "builtin.read_file", "gmail.send"] { - assert!( - is_likely_capability_reference(token), - "{token} should be a capability reference" - ); - } - } - - #[test] - fn guard_ignores_decimal_in_prose() { - // Regression for OfficeQA UID0242: the prompt "compute the - // correlation-adjusted 95% = 0.95 (use 0.95 in formulas)" previously had - // "0.95" extracted as an explicitly requested (but unavailable) capability, - // suppressing every tool call so the model gave up. - let messages = vec![ChatMessage::user( - "compute the correlation-adjusted 95% = 0.95 (use 0.95 in formulas)", - )]; - let tools = vec![tool_def("builtin.shell", "builtin__shell")]; - assert!( - unavailable_requested_capability_guard(&messages, &tools).is_none(), - "guard must not misfire on the decimal `0.95`" - ); - } } diff --git a/crates/loop/ironclaw_loop_host/tests/llm_gateway.rs b/crates/loop/ironclaw_loop_host/tests/llm_gateway.rs index 734e22f03a9..35706965d32 100644 --- a/crates/loop/ironclaw_loop_host/tests/llm_gateway.rs +++ b/crates/loop/ironclaw_loop_host/tests/llm_gateway.rs @@ -907,56 +907,6 @@ async fn gateway_allows_unadvertised_tool_call_when_capability_port_resolves_it( assert_eq!(registered[0].name.as_str(), "demo__hidden"); } -#[tokio::test] -async fn gateway_suppresses_tool_calls_when_user_names_unavailable_capability() { - let provider = Arc::new(ToolAwareProvider::tool_calls(vec![ToolCall { - id: "call_shell".to_string(), - name: "builtin_shell".to_string(), - arguments: serde_json::json!({ - "command": "echo \"disabled-test\"", - "workdir": "/workspace" - }), - reasoning: None, - signature: None, - arguments_parse_error: None, - }])); - let gateway = LlmProviderModelGateway::with_provider_identity( - STATIC_PROVIDER_ID, - provider.clone(), - LlmModelProfilePolicy::new() - .allow_model_profile(interactive_model(), Some("host-selected-model".to_string())), - ); - let capabilities = Arc::new(GatewayCapabilityPort::with_builtin_shell_surface()); - let mut request = model_request(interactive_model()); - request.messages[1].content = "Use builtin.echo to print:\ndisabled-test".to_string(); - - let response = gateway - .stream_model_with_capabilities(request, capabilities.clone()) - .await - .unwrap(); - - assert_eq!(provider.tool_requests.lock().unwrap().len(), 1); - assert!( - capabilities.registered.lock().unwrap().is_empty(), - "suppressed substitute tool call must not be registered as a capability activity" - ); - let ParentLoopOutput::AssistantReply(reply) = response.output else { - panic!("expected assistant reply"); - }; - assert!( - reply.content.contains("unavailable or disabled"), - "expected unavailable capability reply, got {:?}", - reply.content - ); - assert!( - reply - .content - .contains("will not route it through another tool"), - "expected no-workaround reply, got {:?}", - reply.content - ); -} - #[tokio::test] async fn gateway_allows_policy_filtered_discovery_for_named_deferred_capability() { let provider = Arc::new(ToolAwareProvider::tool_calls(vec![ToolCall { @@ -1038,7 +988,7 @@ async fn gateway_allows_prerequisite_and_discovery_for_named_deferred_capability } #[tokio::test] -async fn gateway_suppresses_substitute_when_later_named_capability_is_unavailable() { +async fn gateway_allows_valid_call_when_user_also_names_unavailable_capability() { let provider = Arc::new(ToolAwareProvider::tool_calls(vec![ToolCall { id: "call_substitute".to_string(), name: "demo__echo".to_string(), @@ -1063,11 +1013,12 @@ async fn gateway_suppresses_substitute_when_later_named_capability_is_unavailabl .await .unwrap(); - assert!(capabilities.registered.lock().unwrap().is_empty()); - let ParentLoopOutput::AssistantReply(reply) = response.output else { - panic!("expected unavailable capability reply"); + let ParentLoopOutput::CapabilityCalls(calls) = response.output else { + panic!("expected valid capability call"); }; - assert!(reply.content.contains("unavailable or disabled")); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].capability_id.as_str(), "demo.echo"); + assert_eq!(capabilities.registered.lock().unwrap().len(), 1); } #[tokio::test] @@ -1148,84 +1099,6 @@ async fn gateway_allows_describe_and_wrapped_exact_deferred_capability() { assert_eq!(capabilities.registered.lock().unwrap().len(), 2); } -#[tokio::test] -async fn gateway_suppresses_wrapped_unrelated_deferred_capability() { - let provider = Arc::new(ToolAwareProvider::tool_calls(vec![ToolCall { - id: "call_wrapped".to_string(), - name: "tool_call".to_string(), - arguments: serde_json::json!({ - "name": "demo__other", - "arguments": "{}", - }), - reasoning: None, - signature: None, - arguments_parse_error: None, - }])); - let gateway = LlmProviderModelGateway::with_provider_identity( - STATIC_PROVIDER_ID, - provider, - LlmModelProfilePolicy::new() - .allow_model_profile(interactive_model(), Some("host-selected-model".to_string())), - ); - let capabilities = Arc::new(GatewayCapabilityPort::with_discovery_bridge_surface()); - let mut request = model_request(interactive_model()); - request.messages[1].content = "Use the demo.hidden capability.".to_string(); - - let response = gateway - .stream_model_with_capabilities(request, capabilities.clone()) - .await - .unwrap(); - - assert!(capabilities.registered.lock().unwrap().is_empty()); - assert!(matches!( - response.output, - ParentLoopOutput::AssistantReply(_) - )); -} - -#[tokio::test] -async fn gateway_suppresses_tool_calls_when_user_names_unavailable_hidden_namespace_capability() { - let provider = Arc::new(ToolAwareProvider::tool_calls(vec![ToolCall { - id: "call_shell".to_string(), - name: "builtin_shell".to_string(), - arguments: serde_json::json!({ - "command": "echo \"gmail workaround\"", - "workdir": "/workspace" - }), - reasoning: None, - signature: None, - arguments_parse_error: None, - }])); - let gateway = LlmProviderModelGateway::with_provider_identity( - STATIC_PROVIDER_ID, - provider.clone(), - LlmModelProfilePolicy::new() - .allow_model_profile(interactive_model(), Some("host-selected-model".to_string())), - ); - let capabilities = Arc::new(GatewayCapabilityPort::with_builtin_shell_surface()); - let mut request = model_request(interactive_model()); - request.messages[1].content = "Use gmail.send to email the report".to_string(); - - let response = gateway - .stream_model_with_capabilities(request, capabilities.clone()) - .await - .unwrap(); - - assert_eq!(provider.tool_requests.lock().unwrap().len(), 1); - assert!( - capabilities.registered.lock().unwrap().is_empty(), - "suppressed substitute tool call must not be registered as a capability activity" - ); - let ParentLoopOutput::AssistantReply(reply) = response.output else { - panic!("expected assistant reply"); - }; - assert!( - reply.content.contains("unavailable or disabled"), - "expected unavailable capability reply, got {:?}", - reply.content - ); -} - #[tokio::test] async fn gateway_does_not_treat_plain_dotted_domain_as_unavailable_capability() { let provider = Arc::new(ToolAwareProvider::tool_calls(vec![ToolCall { @@ -1448,40 +1321,82 @@ async fn gateway_preserves_structured_tool_calls_when_content_has_legacy_marker( } #[tokio::test] -async fn gateway_rejects_unknown_provider_tool_call_before_registration() { - let provider = Arc::new(ToolAwareProvider::tool_calls(vec![ - ToolCall { - id: "call_1".to_string(), - name: "demo__echo".to_string(), - arguments: serde_json::json!({"message":"one"}), +async fn gateway_repairs_unknown_provider_tool_call_before_registration() { + let provider = Arc::new(ToolAwareProvider::tool_response_sequence(vec![ + ToolCompletionResponse { + content: None, + tool_calls: vec![ + ToolCall { + id: "call_1".to_string(), + name: "demo__echo".to_string(), + arguments: serde_json::json!({"message":"one"}), + reasoning: None, + signature: None, + arguments_parse_error: None, + }, + ToolCall { + id: "call_2".to_string(), + name: "hidden__tool".to_string(), + arguments: serde_json::json!({"message":"two"}), + reasoning: None, + signature: None, + arguments_parse_error: None, + }, + ], + input_tokens: 1, + output_tokens: 1, + finish_reason: FinishReason::ToolUse, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, reasoning: None, - signature: None, - arguments_parse_error: None, + reasoning_details: None, }, - ToolCall { - id: "call_2".to_string(), - name: "hidden__tool".to_string(), - arguments: serde_json::json!({"message":"two"}), + ToolCompletionResponse { + content: None, + tool_calls: vec![ToolCall { + id: "call_retry".to_string(), + name: "demo__echo".to_string(), + arguments: serde_json::json!({"message":"recovered"}), + reasoning: None, + signature: None, + arguments_parse_error: None, + }], + input_tokens: 1, + output_tokens: 1, + finish_reason: FinishReason::ToolUse, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, reasoning: None, - signature: None, - arguments_parse_error: None, + reasoning_details: None, }, ])); let gateway = LlmProviderModelGateway::with_provider_identity( STATIC_PROVIDER_ID, - provider, + provider.clone(), LlmModelProfilePolicy::new() .allow_model_profile(interactive_model(), Some("host-selected-model".to_string())), ); let capabilities = Arc::new(GatewayCapabilityPort::with_tool_surface()); - let error = gateway + let response = gateway .stream_model_with_capabilities(model_request(interactive_model()), capabilities.clone()) .await - .unwrap_err(); + .unwrap(); - assert_eq!(error.kind, HostManagedModelErrorKind::InvalidOutput); - assert!(capabilities.registered.lock().unwrap().is_empty()); + let ParentLoopOutput::CapabilityCalls(calls) = response.output else { + panic!("expected repaired capability call"); + }; + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].capability_id.as_str(), "demo.echo"); + assert_eq!(capabilities.registered.lock().unwrap().len(), 1); + let requests = provider.tool_requests.lock().unwrap(); + assert_eq!(requests.len(), 2); + assert!(requests[1].messages.iter().any(|message| { + message.role == Role::Tool + && message + .content + .contains("outside the advertised capability surface") + })); } #[tokio::test] From 96eeab79cba017a9d40d6bf054d11c40388cdf2c Mon Sep 17 00:00:00 2001 From: serrrfirat Date: Thu, 13 Aug 2026 14:33:21 +0300 Subject: [PATCH 4/4] test(loop-host): assert gateway repair feedback --- tests/integration/tool_call.rs | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/tests/integration/tool_call.rs b/tests/integration/tool_call.rs index c0f7baa133e..851fb06336a 100644 --- a/tests/integration/tool_call.rs +++ b/tests/integration/tool_call.rs @@ -786,10 +786,10 @@ async fn disabled_spawn_subagent_capability_is_stripped_from_model_surface() { /// A model that calls the disabled `builtin.spawn_subagent` anyway is rejected /// at the gateway (`CapabilitySurfacePolicyFilter`, before -/// `register_provider_tool_call` ever stages an invocation). The loop must -/// surface the precise `outside_capability_surface` observation to the model, -/// let it repair the response on the next call, and complete without ever -/// dispatching or reporting the rejected call as successful. +/// `register_provider_tool_call` ever stages an invocation). The gateway must +/// return precise batch-rejection feedback to the model, let it repair the +/// response on the next call, and complete without ever dispatching or +/// reporting the rejected call as successful. #[tokio::test] async fn disabled_spawn_subagent_capability_call_recovers_without_dispatch() { let h = RebornIntegrationHarness::test_default() @@ -810,12 +810,14 @@ async fn disabled_spawn_subagent_capability_call_recovers_without_dispatch() { h.assert_reply_contains("continue without it") .await .expect("repaired reply is finalized"); - h.assert_model_request_contains( - "model error observation: invalid_output reason=outside_capability_surface; \ - repair the response and continue", - ) + h.assert_model_message_content_in_order(&[ + "Tool call batch rejected by host:", + "model returned a tool call outside the advertised capability surface", + "None of this response's tool calls were executed.", + "Retry with an available capability", + ]) .await - .expect("the retry tells the model precisely why its tool call was rejected"); + .expect("the gateway tells the model precisely why its tool-call batch was rejected"); h.assert_tool_not_invoked("builtin.spawn_subagent") .await