Skip to content

Improve tagged tool parsing with reasoning #23773

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
bartdeboer wants to merge 2 commits into from
Closed

Improve tagged tool parsing with reasoning #23773

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f78ec03
Fix tagged tool calls inside reasoning
bartdeboer May 26, 2026
ce979f0
Accept required tagged tool args in any order
bartdeboer May 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
108 changes: 95 additions & 13 deletions common/chat-auto-parser-generator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
#include "nlohmann/json.hpp"
#include "peg-parser.h"

#include <algorithm>
#include <numeric>
#include <stdexcept>
#include <string>

Expand Down Expand Up @@ -405,22 +407,42 @@ common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_conte
}
}

// Build required arg sequence in definition order
common_peg_parser args_seq = p.eps();
for (size_t i = 0; i < required_parsers.size(); i++) {
if (i > 0) {
args_seq = args_seq + p.space();
}
args_seq = args_seq + required_parsers[i];
}

// Build optional args with flexible ordering
common_peg_parser optional_args = p.eps();
if (!optional_parsers.empty()) {
common_peg_parser any_opt = p.choice();
for (const auto & opt : optional_parsers) {
any_opt |= opt;
}
args_seq = args_seq + p.repeat(p.space() + any_opt, 0, -1);
optional_args = p.repeat(any_opt + p.space(), 0, -1);
}

// Build required args in any order. Tagged arguments carry their
// parameter name in-band, so the model should not have to emit required
// arguments in schema/property order.
common_peg_parser args_seq = p.eps();
constexpr size_t max_required_permutations = 6;
if (required_parsers.empty()) {
args_seq = optional_args;
} else if (required_parsers.size() <= max_required_permutations) {
std::vector<size_t> order(required_parsers.size());
std::iota(order.begin(), order.end(), 0);

common_peg_parser required_orders = p.choice();
do {
common_peg_parser seq = optional_args;
for (size_t idx : order) {
seq = seq + required_parsers[idx] + p.space() + optional_args;
}
required_orders |= seq;
} while (std::next_permutation(order.begin(), order.end()));

args_seq = required_orders;
} else {
// Avoid factorial grammar growth for unusually large schemas.
args_seq = optional_args;
for (const auto & required : required_parsers) {
args_seq = args_seq + required + p.space() + optional_args;
}
}

if (!arguments.start.empty()) {
Expand Down Expand Up @@ -454,13 +476,24 @@ common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_conte
auto require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;

common_peg_parser tool_calls = p.eps();
common_peg_parser tool_call_item = p.eps();

if (!format.per_call_start.empty()) {
auto wrapped_call = format.per_call_start + p.space() + tool_choice + p.space() + format.per_call_end;
if (format.section_start.empty()) {
// Lazy grammars are activated from the first tool-call marker in the
// generated suffix. The trigger grammar must validate the tool call,
// but it should not require the tool call to be the last segment in
// the completion. Tagged tool calls can appear inside reasoning
// blocks, followed by </think>, content, or more segments that the
// full parser will handle afterwards.
p.trigger_rule("tool-call", wrapped_call + p.rest());
}
tool_call_item = wrapped_call + p.space();
if (inputs.parallel_tool_calls) {
tool_calls = p.trigger_rule("tool-call", wrapped_call + p.zero_or_more(p.space() + wrapped_call) + p.space());
tool_calls = wrapped_call + p.zero_or_more(p.space() + wrapped_call) + p.space();
} else {
tool_calls = p.trigger_rule("tool-call", wrapped_call + p.space());
tool_calls = wrapped_call + p.space();
}
if (!format.section_start.empty()) {
tool_calls = p.trigger_rule("tool-calls",
Expand All @@ -486,6 +519,55 @@ common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_conte

std::string trigger_marker = !format.section_start.empty() ? format.section_start : format.per_call_start;
auto content_before_tools = trigger_marker.empty() ? p.eps() : p.until(trigger_marker);

// Treat tag-based reasoning markers as mode switches rather than containers that
// hide tools. Text inside the reasoning markers is emitted as reasoning_content,
// text outside is emitted as content, and valid tool calls can appear in either
// mode. This path applies to tagged-argument tool formats, such as:
//
// <tool_call>
// <function=name>
// <parameter=arg>value</parameter>
// </function>
// </tool_call>
//
// Use a specific tool boundary that includes the function-name prefix when
// possible, so prose such as "I might call <tool_call> later" remains text.
if (ctx.extracting_reasoning && ctx.reasoning &&
!trim_whitespace(ctx.reasoning->start).empty() &&
!trim_whitespace(ctx.reasoning->end).empty() &&
format.section_start.empty() &&
!trim_whitespace(format.per_call_start).empty() &&
!trim_whitespace(format.per_call_end).empty()) {
const std::string think_start = trim_whitespace(ctx.reasoning->start);
const std::string think_end = trim_whitespace(ctx.reasoning->end);
const std::string tool_start = trim_whitespace(format.per_call_start) +
(function.name_prefix.empty() ? "" : "\n" + function.name_prefix);

auto in_think_boundary = p.choice({
p.literal(tool_start),
p.literal(think_end),
});
auto root_boundary = p.choice({
p.literal(think_start),
p.literal(think_end),
p.literal(tool_start),
});

auto reasoning_chunk = p.reasoning(
p.negate(in_think_boundary) + p.any() + p.until_one_of({ tool_start, think_end }));
auto content_chunk = p.content(
p.negate(root_boundary) + p.any() + p.until_one_of({ think_start, think_end, tool_start }));

auto stale_think_end = p.literal(think_end) + p.space();
auto think_block =
p.literal(think_start) + p.space() +
p.zero_or_more(p.choice({ tool_call_item, reasoning_chunk })) +
p.optional(stale_think_end);

return p.zero_or_more(p.choice({ think_block, tool_call_item, stale_think_end, content_chunk })) + p.end();
}

return ctx.reasoning_parser + p.optional(p.content(content_before_tools)) + tool_calls + p.end();
}

Expand Down
148 changes: 125 additions & 23 deletions tests/test-chat.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -477,6 +477,50 @@ static common_chat_tool python_tool{
"required": ["code"]
})",
};
static common_chat_tool write_tool{
/* .name = */ "write",
/* .description = */ "write a file",
/* .parameters = */ R"({
"type": "object",
"properties": {
"file": {
"type": "string",
"description": "File path."
},
"content": {
"type": "string",
"description": "Full replacement file content."
}
},
"required": ["file", "content"]
})",
};
static common_chat_tool edit_snake_tool{
/* .name = */ "edit",
/* .description = */ "edit a file",
/* .parameters = */ R"({
"type": "object",
"properties": {
"file": {
"type": "string",
"description": "File path."
},
"new_string": {
"type": "string",
"description": "Replacement text."
},
"old_string": {
"type": "string",
"description": "Exact text to replace."
},
"replace_all": {
"type": "boolean",
"description": "Replace every occurrence."
}
},
"required": ["file", "old_string", "new_string"]
})",
};

static common_chat_tool html_tool{
/* .name = */ "html",
Expand Down Expand Up @@ -1908,6 +1952,47 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
})
.run();

tst.test(
"<tool_call>\n"
"<function=edit>\n"
"<parameter=file>\n"
"/workspace/jsonlfilter/test_parse.go\n"
"</parameter>\n"
"<parameter=old_string>\n"
"package main\n"
"\n"
"import \"testing\"\n"
"\n"
"func TestDebug(t *testing.T) {\n"
"\tline := `{name:Jette,status:sleepy}`\n"
"\tt.Logf(\"Status: %q\\n\", result[\"status\"])\n"
"}\n"
"</parameter>\n"
"<parameter=new_string>\n"
"package main\n"
"\n"
"import (\n"
"\t\"strings\"\n"
"\t\"testing\"\n"
")\n"
"\n"
"func TestDebug(t *testing.T) {\n"
"\tline := `{name:Jette,status:sleepy}`\n"
"\tt.Logf(\"Status: %q\\n\", result[\"status\"])\n"
"}\n"
"</parameter>\n"
"</function>\n"
"</tool_call>")
.enable_thinking(false)
.reasoning_format(COMMON_REASONING_FORMAT_AUTO)
.tools({
edit_snake_tool
})
.expect_tool_calls({
{ "edit", R"({"file":"/workspace/jsonlfilter/test_parse.go","old_string":"package main\n\nimport \"testing\"\n\nfunc TestDebug(t *testing.T) {\n\tline := `{name:Jette,status:sleepy}`\n\tt.Logf(\"Status: %q\\n\", result[\"status\"])\n}","new_string":"package main\n\nimport (\n\t\"strings\"\n\t\"testing\"\n)\n\nfunc TestDebug(t *testing.T) {\n\tline := `{name:Jette,status:sleepy}`\n\tt.Logf(\"Status: %q\\n\", result[\"status\"])\n}"})", {} },
})
.run();

tst.test(
"I need to output the invoice details in JSON\n"
"</think>\n\n"
Expand All @@ -1919,7 +2004,8 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
.expect_content(R"({"amount": 123.45, "date": "2025-12-03"})")
.run();

// tool call segment in reasoning
// Qwen3.5 can emit tool calls inside <think>. Treat <think> as a
// reasoning/content mode switch, not as a container that hides tools.
tst.test(
"Let's call a tool: <tool_call>\n"
"<function=python>\n"
Expand All @@ -1946,22 +2032,45 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
.tools({
python_tool
})
.expect_reasoning(
"Let's call a tool: <tool_call>\n"
"<function=python>\n"
"<parameter=code>\n"
"def hello():\n"
" print(\"Not the real call!\")\n"
"\n"
"hello()\n"
"</parameter>\n"
"</function>\n"
"</tool_call>")
.expect_reasoning("Let's call a tool:")
.expect_tool_calls({
{ "python", "{\"code\": \"def hello():\\n print(\\\"Not the real call!\\\")\\n\\nhello()\"}", {} },
{ "python", "{\"code\": \"def hello():\\n print(\\\"Hello, world!\\\")\\n\\nhello()\"}", {} },
})
.run();

tst.test(
"Need to write a debug file.\n"
"<tool_call>\n"
"<function=write>\n"
"<parameter=file>\n"
"/workspace/jsonlfilter/debug_test.go\n"
"</parameter>\n"
"<parameter=content>\n"
"package main\n"
"\n"
"func main() {\n"
" line := `{name:Jette,status:sleepy}`\n"
" fmt.Printf(\"Status: %q\\n\", result[\"status\"])\n"
"}\n"
"\n"
"</parameter>\n"
"</function>\n"
"</tool_call>\n"
"</think>\n"
"Done.")
.enable_thinking(true)
.reasoning_format(COMMON_REASONING_FORMAT_AUTO)
.tools({
write_tool
})
.expect_reasoning("Need to write a debug file.")
.expect_content("Done.")
.expect_tool_calls({
{ "write", R"({"file":"/workspace/jsonlfilter/debug_test.go","content":"package main\n\nfunc main() {\n line := `{name:Jette,status:sleepy}`\n fmt.Printf(\"Status: %q\\n\", result[\"status\"])\n}\n"})", {} },
})
.run();

// No args tool
tst.test(
"<tool_call>\n"
Expand Down Expand Up @@ -2417,7 +2526,8 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
.expect_content(R"({"amount": 123.45, "date": "2025-12-03"})")
.run();

// tool call segment in reasoning
// Qwen3.5 can emit tool calls inside <think>. Treat <think> as a
// reasoning/content mode switch, not as a container that hides tools.
tst.test(
"Let's call a tool: <tool_call>\n"
"<function=python>\n"
Expand Down Expand Up @@ -2445,17 +2555,9 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
.tools({
python_tool
})
.expect_reasoning("Let's call a tool: <tool_call>\n"
"<function=python>\n"
"<parameter=code>\n"
"def hello():\n"
" print(\"Not the real call!\")\n"
"\n"
"hello()\n"
"</parameter>\n"
"</function>\n"
"</tool_call>\n")
.expect_reasoning("Let's call a tool:")
.expect_tool_calls({
{ "python", "{\"code\": \"def hello():\\n print(\\\"Not the real call!\\\")\\n\\nhello()\"}", {} },
{ "python", "{\"code\": \"def hello():\\n print(\\\"Hello, world!\\\")\\n\\nhello()\"}", {} },
})
.run();
Expand Down