Skip to content

fix(responses): preserve text instead of inferring tool calls #14846

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rmccorm4 merged 6 commits into from
Sep 17, 2026
Merged

fix(responses): preserve text instead of inferring tool calls #14846

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
30b7adb
fix(responses): preserve text instead of inferring tool calls
xianlubird Sep 15, 2026
f41e5dc
test(responses): remove duplicate named-call conversion assertion
xianlubird Sep 15, 2026
b59bb7e
Merge branch 'main' into codex/remove-responses-text-tool-fallback
xianlubird Sep 15, 2026
afc3468
Merge branch 'main' into codex/remove-responses-text-tool-fallback
xianlubird Sep 16, 2026
8f4d139
Merge branch 'main' into codex/remove-responses-text-tool-fallback
xianlubird Sep 17, 2026
bb6491c
Merge branch 'main' into codex/remove-responses-text-tool-fallback
rmccorm4 Sep 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
276 changes: 26 additions & 250 deletions lib/llm/src/protocols/openai/responses/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -947,76 +947,6 @@ fn convert_top_logprobs(input: Option<u8>) -> Option<u8> {
input.map(|x| x.min(20))
}

/// Parse `<tool_call>` blocks from model text output.
/// Returns a list of (name, arguments_json) tuples.
/// Returns an empty vec immediately if no `<tool_call>` tag is present.
fn parse_tool_call_text(text: &str) -> Vec<(String, String)> {
if !text.contains("<tool_call>") {
return Vec::new();
}
let mut results = Vec::new();
let mut search_start = 0;
while let Some(start) = text[search_start..].find("<tool_call>") {
let abs_start = search_start + start + "<tool_call>".len();
if let Some(end) = text[abs_start..].find("</tool_call>") {
let block = text[abs_start..abs_start + end].trim();
if let Ok(parsed) = serde_json::from_str::<serde_json::Value>(block) {
let name = parsed
.get("name")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string();
let arguments = if let Some(args) = parsed.get("arguments") {
if args.is_string() {
args.as_str().unwrap_or("{}").to_string()
} else {
serde_json::to_string(args).unwrap_or_else(|_| "{}".to_string())
}
} else {
"{}".to_string()
};
if !name.is_empty() {
results.push((name, arguments));
}
}
search_start = abs_start + end + "</tool_call>".len();
} else {
break;
}
}
results
}

/// Strip `<tool_call>...</tool_call>` blocks and any `<think>...</think>` blocks from text.
/// Returns the original string (no allocation) if no tags are present.
fn strip_tool_call_text(text: &str) -> std::borrow::Cow<'_, str> {
let has_tool = text.contains("<tool_call>");
let has_think = text.contains("<think>");
if !has_tool && !has_think {
return std::borrow::Cow::Borrowed(text);
}

fn strip_tag(input: &mut String, open: &str, close: &str) {
while let Some(start) = input.find(open) {
if let Some(end_offset) = input[start..].find(close) {
input.replace_range(start..start + end_offset + close.len(), "");
} else {
input.truncate(start);
break;
}
}
}

let mut result = text.to_string();
if has_tool {
strip_tag(&mut result, "<tool_call>", "</tool_call>");
}
if has_think {
strip_tag(&mut result, "<think>", "</think>");
}
std::borrow::Cow::Owned(result)
}

// ---------------------------------------------------------------------------
// Chat Completions -> Responses API response conversion
// ---------------------------------------------------------------------------
Expand Down Expand Up @@ -1151,18 +1081,6 @@ fn make_text_message(id: String, text: String) -> OutputItem {
})
}

/// Build a function call output item with generated IDs.
fn make_function_call(name: String, arguments: String, namespace: Option<String>) -> OutputItem {
OutputItem::FunctionCall(FunctionToolCall {
arguments,
call_id: format!("call_{}", Uuid::new_v4().simple()),
namespace,
name,
id: Some(format!("fc_{}", Uuid::new_v4().simple())),
status: Some(OutputStatus::Completed),
})
}

/// Convert a ChatCompletion response into a Responses API response object,
/// echoing back the actual request parameters from `params`.
pub fn chat_completion_to_response(
Expand Down Expand Up @@ -1213,8 +1131,6 @@ pub fn chat_completion_to_response(
}));
}
}
// Handle text content -- also parse <tool_call> blocks from models
// that emit tool calls as text (e.g. Qwen3)
let content_text = match choice.message.content {
Some(dynamo_protocols::types::ChatCompletionMessageContent::Text(text)) => Some(text),
Some(dynamo_protocols::types::ChatCompletionMessageContent::Parts(_)) => {
Expand All @@ -1228,39 +1144,7 @@ pub fn chat_completion_to_response(
if let Some(content_text) = content_text
&& !content_text.is_empty()
{
let has_enabled_tools = params.tools.as_ref().is_some_and(|tools| {
tools.iter().any(|tool| match tool {
Tool::Function(_) => true,
Tool::Namespace(namespace) => namespace
.tools
.iter()
.any(|tool| matches!(tool, NamespaceToolParamTool::Function(_))),
_ => false,
})
}) && !matches!(
params.tool_choice,
Some(ToolChoiceParam::Mode(ToolChoiceOptions::None))
);
let parsed_calls = if has_enabled_tools {
parse_tool_call_text(&content_text)
} else {
Vec::new()
};
if !parsed_calls.is_empty() {
for (name, arguments) in parsed_calls {
let namespace = params.namespace_for_function(&name);
output.push(make_function_call(name, arguments, namespace));
}
let remaining = strip_tool_call_text(&content_text);
if !remaining.trim().is_empty() {
output.push(make_text_message(
message_id.clone(),
remaining.into_owned(),
));
}
} else {
output.push(make_text_message(message_id.clone(), content_text));
}
output.push(make_text_message(message_id.clone(), content_text));
}

if output.is_empty() {
Expand Down Expand Up @@ -3028,153 +2912,45 @@ mod tests {
}

#[test]
fn test_parse_tool_call_text() {
// Standard Qwen3 format
let text = r#"<think>
Let me check the weather.
</think>

<tool_call>
{"name": "get_weather", "arguments": {"location": "San Francisco"}}
</tool_call>"#;
let calls = parse_tool_call_text(text);
assert_eq!(calls.len(), 1);
assert_eq!(calls[0].0, "get_weather");
let args: serde_json::Value = serde_json::from_str(&calls[0].1).unwrap();
assert_eq!(args["location"], "San Francisco");
}

#[test]
fn test_parse_tool_call_text_multiple() {
let text = r#"<tool_call>
{"name": "func_a", "arguments": {"x": 1}}
</tool_call>
<tool_call>
{"name": "func_b", "arguments": {"y": 2}}
</tool_call>"#;
let calls = parse_tool_call_text(text);
assert_eq!(calls.len(), 2);
assert_eq!(calls[0].0, "func_a");
assert_eq!(calls[1].0, "func_b");
}

#[test]
fn test_parse_tool_call_text_no_calls() {
let text = "Just a regular message with no tool calls.";
let calls = parse_tool_call_text(text);
assert!(calls.is_empty());
}

#[test]
fn test_text_tool_calls_preserved_when_tools_are_disabled() {
let text = r#"Example: <think>reasoning</think><tool_call>{"name":"get_weather","arguments":{"city":"Beijing"}}</tool_call>"#;
let tools = vec![
serde_json::from_value(serde_json::json!({
"type": "function", "name": "get_weather", "parameters": {"type": "object"}
}))
.unwrap(),
];
for (tools, tool_choice) in [
(
Some(tools),
Some(ToolChoiceParam::Mode(ToolChoiceOptions::None)),
),
(None, None),
(
Some(serde_json::from_value(serde_json::json!([{
"type": "namespace", "name": "weather", "description": "Weather tools", "tools": []
}])).unwrap()),
Some(ToolChoiceParam::Mode(ToolChoiceOptions::Auto)),
),
] {
let params = ResponseParams {
tools,
tool_choice,
..Default::default()
};
let response =
chat_completion_to_response(make_chat_resp_with_text(text), &params, None).unwrap();
assert_eq!(response.inner.output.len(), 1);
let OutputItem::Message(message) = &response.inner.output[0] else {
panic!("tool-disabled request must preserve literal tool-call text");
};
let OutputMessageContent::OutputText(content) = &message.content[0] else {
panic!("expected output text");
};
assert_eq!(content.text, text);
}
}

#[test]
fn test_text_tool_calls_parsed_when_tools_are_enabled() {
let text =
r#"<tool_call>{"name":"get_weather","arguments":{"city":"Beijing"}}</tool_call>"#;
let tools = serde_json::from_value(serde_json::json!([{
"type": "function", "name": "get_weather", "parameters": {"type": "object"}
}]))
.unwrap();
let params = ResponseParams {
tools: Some(tools),
tool_choice: Some(ToolChoiceParam::Mode(ToolChoiceOptions::Auto)),
..Default::default()
};
let response =
chat_completion_to_response(make_chat_resp_with_text(text), &params, None).unwrap();
assert_eq!(response.inner.output.len(), 1);
let OutputItem::FunctionCall(call) = &response.inner.output[0] else {
panic!("expected a function call for an enabled tool request");
};
assert_eq!(call.name, "get_weather");
assert_eq!(
serde_json::from_str::<serde_json::Value>(&call.arguments).unwrap(),
serde_json::json!({"city":"Beijing"})
);
}

#[test]
fn test_text_tool_call_preserves_namespace() {
fn test_response_preserves_literal_tool_markup_alongside_structured_call() {
let text = r#"Example: <think>reasoning</think><tool_call>{"name":"get_weather","arguments":{}}</tool_call>"#;
let params = ResponseParams {
tools: Some(
serde_json::from_value(serde_json::json!([{
"type": "namespace",
"name": "weather",
"description": "Weather tools",
"type": "namespace", "name": "weather", "description": "Weather tools",
"tools": [{"type": "function", "name": "get_weather"}]
}]))
.unwrap(),
),
tool_choice: Some(ToolChoiceParam::Mode(ToolChoiceOptions::Auto)),
..Default::default()
};
let response = chat_completion_to_response(
make_chat_resp_with_text(
r#"<tool_call>{"name":"get_weather","arguments":{}}</tool_call>"#,
),
&params,
None,
)
.unwrap();
assert_eq!(response.inner.output.len(), 1);
let mut chat = make_chat_resp_with_text(text);
chat.inner.choices[0].message.tool_calls = Some(vec![ChatCompletionMessageToolCall {
id: "call_original".into(),
r#type: FunctionType::Function,
function: dynamo_protocols::types::FunctionCall {
name: "get_weather".into(),
arguments: r#"{"city":"Paris"}"#.into(),
},
}]);
chat.inner.choices[0].finish_reason = Some(FinishReason::ToolCalls);
let response = chat_completion_to_response(chat, &params, None).unwrap();
assert_eq!(response.inner.output.len(), 2);
let OutputItem::FunctionCall(call) = &response.inner.output[0] else {
panic!("expected a namespaced function call");
panic!("expected the structured function call");
};
assert_eq!(call.call_id, "call_original");
assert_eq!(call.name, "get_weather");
assert_eq!(call.namespace.as_deref(), Some("weather"));
assert_eq!(call.arguments, "{}");
}

#[test]
fn test_strip_tool_call_text() {
let text = r#"<think>
thinking
</think>

<tool_call>
{"name": "f", "arguments": {}}
</tool_call>"#;
let stripped = strip_tool_call_text(text);
assert!(!stripped.contains("<tool_call>"));
assert!(!stripped.contains("<think>"));
assert_eq!(call.arguments, r#"{"city":"Paris"}"#);
let OutputItem::Message(message) = &response.inner.output[1] else {
panic!("expected the original text");
};
let OutputMessageContent::OutputText(content) = &message.content[0] else {
panic!("expected output text");
};
assert_eq!(content.text, text);
}

// ── PR1: reasoning / text.format / service_tier pass-through tests ──
Expand Down
46 changes: 46 additions & 0 deletions lib/llm/tests/aggregators.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1060,3 +1060,49 @@ async fn test_muse_unified_batch_native_fallback_zero_calls_still_strips_markup(
guided-JSON synthetic placeholder"
);
}

#[tokio::test]
async fn test_responses_tool_parsing_is_owned_by_the_aggregator() {
use dynamo_llm::protocols::openai::responses::{ResponseParams, chat_completion_to_response};
use dynamo_protocols::types::responses::{OutputItem, OutputMessageContent};

let raw = r#"<tool_call>{"name":"get_weather","arguments":{"city":"Paris"}}</tool_call>"#;
let params = ResponseParams {
tools: Some(
serde_json::from_value(serde_json::json!([{
"type": "function", "name": "get_weather", "parameters": {"type": "object"}
}]))
.unwrap(),
),
..Default::default()
};
for parser in [None, Some("hermes".to_string())] {
let parses_tools = parser.is_some();
let chat = NvCreateChatCompletionResponse::from_annotated_stream(
futures::stream::iter([make_stream_delta(Some(raw), None)]),
ParsingOptions::new(parser, None),
)
.await
.unwrap();
let response = chat_completion_to_response(chat, &params, None).unwrap();
assert_eq!(response.inner.output.len(), 1);
if parses_tools {
let OutputItem::FunctionCall(call) = &response.inner.output[0] else {
panic!("expected an upstream-parsed call");
};
assert_eq!(call.name, "get_weather");
assert_eq!(
serde_json::from_str::<serde_json::Value>(&call.arguments).unwrap(),
serde_json::json!({"city":"Paris"})
);
} else {
let OutputItem::Message(message) = &response.inner.output[0] else {
panic!("no-parser output must remain text");
};
let OutputMessageContent::OutputText(content) = &message.content[0] else {
panic!("expected output text");
};
assert_eq!(content.text, raw);
}
}
}
Loading
Loading