From 52ff665c50ac124f4e8d68378637a777621228ea Mon Sep 17 00:00:00 2001 From: Bart de Boer Date: Fri, 5 Jun 2026 17:26:06 +0000 Subject: [PATCH 1/6] Add tagged thinking tool parser --- common/chat.cpp | 170 ++++++++++++++++++++++++++++++++++++++ common/chat.h | 9 +- tests/test-chat.cpp | 196 +++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 371 insertions(+), 4 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index 7740f35c0edc..5032893615c7 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1110,6 +1110,153 @@ static common_chat_params common_chat_params_init_ministral_3(const common_chat_ return data; } +static common_chat_params common_chat_params_init_tagged_thinking_tools(const common_chat_template & tmpl, + const autoparser::generation_params & inputs) { + common_chat_params data; + + const bool has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + const bool extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; + + data.supports_thinking = true; + data.thinking_start_tag = ""; + data.thinking_end_tag = ""; + data.prompt = common_chat_template_direct_apply(tmpl, inputs); + data.generation_prompt = common_chat_template_generation_prompt(tmpl, inputs); + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + data.strict_eof_on_complete = true; + data.preserved_tokens = { + "", + "", + "", + "", + }; + + if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) { + return data; + } + + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + auto generation_prompt = p.literal(data.generation_prompt); + auto tool_choices = p.choice(); + + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + const auto name = function.at("name").get(); + + auto arg_choices = p.choice(); + foreach_parameter(function, [&](const std::string & prop_name, const json & prop_schema, bool) { + const bool is_string = prop_schema.value("type", "") == "string"; + + auto value_until = [&](const std::string & marker) { + return is_string ? + p.tool_arg_string_value(p.until(marker)) : + p.tool_arg_value(p.until(marker)); + }; + + auto arg_open = p.tool_arg_open( + p.literal("")); + + auto arg = p.tool_arg(arg_open + p.choice({ + p.literal("\n") + value_until("\n") + p.tool_arg_close(p.literal("\n")), + value_until("") + p.tool_arg_close(p.literal("")), + })); + arg_choices |= p.rule("tool-" + name + "-arg-" + prop_name, arg); + }); + + auto args = p.tool_args(p.zero_or_more(arg_choices + p.space())); + auto func = p.tool( + p.tool_open(p.literal("")) + + p.space() + args + p.space() + + p.tool_close(p.literal(""))); + + tool_choices |= p.rule("tool-" + name, func); + }); + + auto tool_call = + p.literal("") + p.space() + tool_choices + p.space() + p.literal(""); + + const int max_calls = inputs.parallel_tool_calls ? -1 : 1; + auto tool_calls = p.repeat(tool_call + p.space(), 1, max_calls); + + p.rule("tool-tail", [&]() { + auto content_until_boundary = p.content(p.until_one_of({"", ""})); + return p.choice({ + p.literal("") + p.space() + p.content(p.rest()), + content_until_boundary + p.choice({ + p.ref("tool-call"), + p.literal("") + p.space() + p.content(p.rest()), + p.content(p.rest()), + }), + }); + }); + auto tool_suffix = p.trigger_rule("tool-call", tool_calls + p.ref("tool-tail")); + + auto outside = p.choice({ + p.content(p.until("")) + tool_suffix, + p.content(p.rest()), + }); + + if (!extract_reasoning || !inputs.enable_thinking) { + return generation_prompt + outside; + } + + // Classify the emitted completion rather than the generation prompt. Some + // templates prefill "\n", but Qwen-style models may omit the + // matching close tag for tool-call and final-answer turns. In practice: + // - emitted closes reasoning; + // - emitted makes any preceding text reasoning; + // - no emitted reasoning/tool tags means final assistant content. + auto reasoning_before_boundary = + p.reasoning(p.until_one_of({"", "\n", "\n"})) + + p.optional(p.literal("\n")); + + auto reasoning_until_tool = + p.peek(p.until_one_of({"", "\n", "\n"}) + p.literal("\n")) + + reasoning_before_boundary + + tool_suffix; + + auto reasoning_until_close = + p.peek(p.until_one_of({"", "\n", "\n"}) + p.choice({ + p.literal(""), + p.literal("\n"), + })) + + reasoning_before_boundary + + p.literal("") + p.space() + outside; + + auto reasoning_then_boundary = p.choice({ + tool_suffix, + reasoning_until_tool, + reasoning_until_close, + p.content(p.rest()), + }); + + auto generated_reasoning = p.literal("") + p.space() + reasoning_then_boundary; + + return generation_prompt + p.choice({ + generated_reasoning, + reasoning_then_boundary, + }); + }); + + data.parser = parser.save(); + data.grammar_lazy = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; + data.grammar = build_grammar([&](const common_grammar_builder & builder) { + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + auto schema = function.at("parameters"); + builder.resolve_refs(schema); + }); + parser.build_grammar(builder, data.grammar_lazy); + }); + if (data.grammar_lazy) { + data.grammar_triggers = { + { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "" }, + }; + } + + return data; +} + static common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & tmpl, const autoparser::generation_params & inputs) { common_chat_params data; @@ -2917,6 +3064,20 @@ std::optional common_chat_try_specialized_template( return common_chat_params_init_ministral_3(tmpl, params); } + // Tagged thinking/tool protocol: reasoning uses ... and tools use + // .... + if (params.tools.is_array() && !params.tools.empty() && + params.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE && + src.find("") != std::string::npos && + src.find("") != std::string::npos && + src.find("") != std::string::npos && + src.find("") != std::string::npos && + src.find("") != std::string::npos) { LOG_DBG("Using specialized template: GPT-OSS\n"); @@ -3257,6 +3418,15 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & src_pars common_peg_parse_context ctx(effective_input, flags); auto result = parser.parse(ctx); + // Streaming parses are lenient so an unterminated prefix can still produce deltas. + // Some complete parses intentionally use EOF as a boundary; retry those strictly + // when requested so "until"/"rest" parsers can commit at the actual end. + if (!is_partial && params.strict_eof_on_complete && result.need_more_input()) { + flags = common_peg_parse_flags(flags & ~COMMON_PEG_PARSE_FLAG_LENIENT); + ctx = common_peg_parse_context(effective_input, flags); + result = parser.parse(ctx); + } + if (result.fail()) { // During partial parsing, return partial results if any AST nodes were captured // This allows streaming to work correctly for formats like FUNC_MARKDOWN_CODE_BLOCK diff --git a/common/chat.h b/common/chat.h index 6d5b220aebb5..93d5c313804f 100644 --- a/common/chat.h +++ b/common/chat.h @@ -280,6 +280,9 @@ struct common_chat_params { std::vector preserved_tokens; std::vector additional_stops; std::string parser; + // If a complete parse reports NEED_MORE in lenient streaming mode, retry without + // leniency so EOF can be used as a real boundary by parsers that opt in. + bool strict_eof_on_complete = false; common_chat_msg_delimiters message_delimiters; }; @@ -295,11 +298,13 @@ struct common_chat_parser_params { bool is_continuation = false; bool echo = false; // Include assistant prefilled msg in output bool debug = false; // Enable debug output for PEG parser + bool strict_eof_on_complete = false; common_peg_arena parser = {}; common_chat_parser_params() = default; common_chat_parser_params(const common_chat_params & chat_params) { - format = chat_params.format; - generation_prompt = chat_params.generation_prompt; + format = chat_params.format; + generation_prompt = chat_params.generation_prompt; + strict_eof_on_complete = chat_params.strict_eof_on_complete; } }; diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index 01b07953a627..c0cb7108d2b3 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -1451,6 +1451,11 @@ class peg_tester { template_path_(template_path), detailed_debug_(detailed_debug) {} + explicit peg_tester(common_chat_templates_ptr tmpls, const std::string & template_path, const bool detailed_debug = false) : + tmpls_(std::move(tmpls)), + template_path_(template_path), + detailed_debug_(detailed_debug) {} + const std::string & template_path() const { return template_path_; } peg_test_builder test(const std::string & input); @@ -2247,7 +2252,6 @@ static void test_template_output_peg_parsers(bool detailed_debug) { }) .run(); - // test code that starts with indent tst.test( "\n" @@ -2267,6 +2271,100 @@ static void test_template_output_peg_parsers(bool detailed_debug) { }) .run(); + tst.test( + "Need to inspect the current directory.\n" + "\n" + "\n" + "\n" + "pwd\n" + "\n" + "\n" + "\n" + "\n" + "Done.") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ run_in_terminal_tool }) + .expect_reasoning("Need to inspect the current directory.") + .expect_content("Done.") + .expect_tool_calls({ + { "run_in_terminal", R"({"command": "pwd"})", {} }, + }) + .run(); + + tst.test( + "Need to inspect the current directory.\n" + "\n" + "\n" + "\n" + "pwd\n" + "\n" + "\n" + "\n" + "Done.") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ run_in_terminal_tool }) + .expect_reasoning("Need to inspect the current directory.") + .expect_content("Done.") + .expect_tool_calls({ + { "run_in_terminal", R"({"command": "pwd"})", {} }, + }) + .run(); + + tst.test( + "\n" + "\n" + "\n" + "new text\n" + "\n" + "\n" + "old text\n" + "\n" + "\n" + "foo.cpp\n" + "\n" + "\n" + "") + .enable_thinking(false) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ edit_tool }) + .expect_tool_calls({ + { "edit", R"({"newString": "new text", "oldString": "old text", "filename": "foo.cpp"})", {} }, + }) + .run(); + + tst.test( + "Need the directory.\n" + "\n" + "\n" + "\n" + "pwd\n" + "\n" + "\n" + "\n" + "Need the files.\n" + "\n" + "\n" + "\n" + "ls\n" + "\n" + "\n" + "\n" + "\n" + "Done.") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .parallel_tool_calls(true) + .tools({ run_in_terminal_tool }) + .expect_reasoning("Need the directory.") + .expect_content("Need the files.\nDone.") + .expect_tool_calls({ + { "run_in_terminal", R"({"command": "pwd"})", {} }, + { "run_in_terminal", R"({"command": "ls"})", {} }, + }) + .run(); + tst.test( "I need to output the invoice details in JSON\n" "\n\n" @@ -2357,7 +2455,7 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .tools({ special_function_tool }) - .expect_reasoning("\n\n") + .expect_reasoning("") .expect_tool_calls({ { "special_function", "{\"arg1\": 1}", "" } }) .run(); @@ -2472,6 +2570,13 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .expect_content("Final answer without tools.") .run(); + tst.test("Final answer without thinking.") + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .enable_thinking(true) + .tools({ run_in_terminal_tool }) + .expect_content("Final answer without thinking.") + .run(); + // Continuation tests tst.test("world!\nWhat's up?") .reasoning_format(COMMON_REASONING_FORMAT_AUTO) @@ -2578,6 +2683,93 @@ static void test_template_output_peg_parsers(bool detailed_debug) { }) .run(); } + + { + auto src = read_file("models/templates/Qwen3.5-4B.jinja"); + string_replace_all(src, + "{%- if add_generation_prompt %}\n" + " {{- '<|im_start|>assistant\\n' }}\n" + " {%- if enable_thinking is defined and enable_thinking is false %}\n" + " {{- '\\n\\n\\n\\n' }}\n" + " {%- else %}\n" + " {{- '\\n' }}\n" + " {%- endif %}\n" + "{%- endif %}", + "{%- if add_generation_prompt %}\n" + " {{- '<|im_start|>assistant\\n' }}\n" + "{%- endif %}"); + + auto no_prefill = peg_tester( + common_chat_templates_ptr(common_chat_templates_init(/* model= */ nullptr, src)), + "models/templates/Qwen3.5-4B-no-prefill-think.jinja", + detailed_debug); + + no_prefill.test( + "\n" + "I need to run a terminal command.\n" + "\n\n" + "\n" + "\n" + "\n" + "pwd\n" + "\n" + "\n" + "") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ run_in_terminal_tool }) + .expect_reasoning("I need to run a terminal command.") + .expect_tool_calls({ + { "run_in_terminal", R"({"command": "pwd"})", {} }, + }) + .run(); + + no_prefill.test( + "\n" + "I need to run a terminal command.\n" + "\n" + "\n" + "\n" + "pwd\n" + "\n" + "\n" + "\n" + "Done.") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ run_in_terminal_tool }) + .expect_reasoning("I need to run a terminal command.") + .expect_content("Done.") + .expect_tool_calls({ + { "run_in_terminal", R"({"command": "pwd"})", {} }, + }) + .run(); + + no_prefill.test( + "I need to run a terminal command.\n" + "\n" + "\n" + "\n" + "pwd\n" + "\n" + "\n" + "") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ run_in_terminal_tool }) + .expect_reasoning("I need to run a terminal command.") + .expect_tool_calls({ + { "run_in_terminal", R"({"command": "pwd"})", {} }, + }) + .run(); + + no_prefill.test("Final answer without thinking.") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ run_in_terminal_tool }) + .expect_content("Final answer without thinking.") + .run(); + } } { From 7c6ebd837a10d9993588be1247e5b1ebf065bad4 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Tue, 28 Jul 2026 16:53:48 -0500 Subject: [PATCH 2/6] chat : refactor and add permute helper --- common/chat-peg-parser.cpp | 40 +++++ common/chat-peg-parser.h | 5 + common/chat.cpp | 276 ++++++++++++++--------------- common/chat.h | 9 +- tests/test-chat-peg-parser.cpp | 103 +++++++++++ tests/test-chat.cpp | 314 +++++---------------------------- 6 files changed, 331 insertions(+), 416 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index f786f5ff2314..1910b4f1e135 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -6,6 +6,9 @@ #include +#include +#include + using ordered_json = nlohmann::ordered_json; static std::string_view trim_trailing_space(std::string_view sv, int max = -1) { @@ -235,6 +238,43 @@ common_peg_parser common_chat_peg_builder::tag_with_safe_content(const std::stri return zero_or_more(choice({ p, content_chunk })); } +common_peg_parser common_chat_peg_builder::permute(const std::string & rule_prefix, + const std::vector & parsers) { + if (parsers.empty()) { + return eps(); + } + + if (parsers.size() == 1 || parsers.size() > COMMON_CHAT_MAX_PERMUTE) { + return sequence(parsers); + } + + std::map rules; + std::function remaining_of; + + remaining_of = [&](uint32_t remaining) -> common_peg_parser { + if (remaining == 0) { + return eps(); + } + + auto cached = rules.find(remaining); + if (cached != rules.end()) { + return cached->second; + } + + auto alternatives = choice(); + for (size_t i = 0; i < parsers.size(); i++) { + const uint32_t bit = 1u << i; + if (remaining & bit) { + alternatives |= parsers[i] + remaining_of(remaining & ~bit); + } + } + + return rules.emplace(remaining, rule(rule_prefix + "-" + std::to_string(remaining), alternatives)).first->second; + }; + + return remaining_of((1u << parsers.size()) - 1); +} + std::string & common_chat_peg_mapper::args_target() { return (current_tool && !current_tool->name.empty()) ? current_tool->arguments : args_buffer; } diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.h index cd14f2c11750..5d764dbaa0ec 100644 --- a/common/chat-peg-parser.h +++ b/common/chat-peg-parser.h @@ -55,6 +55,8 @@ class common_chat_peg_minimax_m3_mapper : public common_chat_peg_mapper { struct content_structure; struct tool_call_structure; +constexpr size_t COMMON_CHAT_MAX_PERMUTE = 6; + class common_chat_peg_builder : public common_peg_parser_builder { public: // Tag constants (from former common_chat_peg_base_builder) @@ -105,6 +107,9 @@ class common_chat_peg_builder : public common_peg_parser_builder { common_peg_parser tool_arg_json_value(const common_peg_parser & p) { return tag(TOOL_ARG_VALUE, p); } + // Matches every parser exactly once, in any order. + common_peg_parser permute(const std::string & rule_prefix, const std::vector & parsers); + // Return a parser that parses the prefix of a string, up to a given delimiter. common_peg_parser prefix(const std::string & s, const std::string & delimiter = {}); diff --git a/common/chat.cpp b/common/chat.cpp index 5032893615c7..ba7d3f63c18d 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1110,148 +1110,157 @@ static common_chat_params common_chat_params_init_ministral_3(const common_chat_ return data; } -static common_chat_params common_chat_params_init_tagged_thinking_tools(const common_chat_template & tmpl, - const autoparser::generation_params & inputs) { +static common_chat_params common_chat_params_init_qwen3_coder(const common_chat_template & tmpl, + const autoparser::generation_params & inputs) { common_chat_params data; - const bool has_tools = inputs.tools.is_array() && !inputs.tools.empty(); - const bool extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; - - data.supports_thinking = true; - data.thinking_start_tag = ""; - data.thinking_end_tag = ""; - data.prompt = common_chat_template_direct_apply(tmpl, inputs); - data.generation_prompt = common_chat_template_generation_prompt(tmpl, inputs); - data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; - data.strict_eof_on_complete = true; - data.preserved_tokens = { - "", - "", + const std::string GEN_PREFIX = "<|im_start|>assistant\n"; + + data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); + data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + + auto supports_reasoning = tmpl.source().find("") != std::string::npos; + + data.supports_thinking = supports_reasoning; + data.preserved_tokens = { "", "", }; - if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) { - return data; + if (supports_reasoning) { + data.thinking_start_tag = ""; + // Support both and as reasoning end sequences. + data.thinking_end_tags = { "", "" }; + data.preserved_tokens.insert(data.preserved_tokens.end(), { "", "" }); + } + + data.message_delimiters = { + { COMMON_CHAT_ROLE_ASSISTANT, "<|im_start|>assistant" }, + { COMMON_CHAT_ROLE_TOOL, "<|im_start|>user\n" }, + { COMMON_CHAT_ROLE_TOOL, "<|im_start|>tool" }, + { COMMON_CHAT_ROLE_USER, "<|im_start|>user" }, + { COMMON_CHAT_ROLE_SYSTEM, "<|im_start|>system" }, + }; + + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty(); + auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; + auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); + + if (inputs.has_continuation()) { + const auto & msg = inputs.continue_msg; + + data.generation_prompt = GEN_PREFIX; + if (supports_reasoning) { + data.generation_prompt += "\n" + msg.reasoning_content; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += "\n\n\n"; + } + } + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += msg.render_content(); + } + + data.prompt += data.generation_prompt; } auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { - auto generation_prompt = p.literal(data.generation_prompt); - auto tool_choices = p.choice(); + auto generation_prompt = p.literal(GEN_PREFIX); - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - const auto name = function.at("name").get(); + auto reasoning = p.eps(); + if (supports_reasoning && extract_reasoning) { + reasoning = p.optional("" + p.space() + + p.reasoning(p.until_one_of({ "", "" })) + + (p.literal("") | p.peek(p.literal("")))); + } - auto arg_choices = p.choice(); - foreach_parameter(function, [&](const std::string & prop_name, const json & prop_schema, bool) { - const bool is_string = prop_schema.value("type", "") == "string"; + // Response format parser + if (has_response_format) { + return generation_prompt + (reasoning << p.content(p.schema(p.json(), "response-format", inputs.json_schema))); + } - auto value_until = [&](const std::string & marker) { - return is_string ? - p.tool_arg_string_value(p.until(marker)) : - p.tool_arg_value(p.until(marker)); - }; + // Tool call parser + if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { + auto arg_close = p.tool_arg_close(p.literal("\n\n")); + auto arg_string = p.rule("xml-arg-string", + p.ac(p.tool_arg_string_value(p.until("\n\n")) + arg_close, "\n\n")); - auto arg_open = p.tool_arg_open( - p.literal("")); + auto tool_choice = p.choice(); + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + std::string name = function.at("name"); + auto parameters = function.contains("parameters") ? function.at("parameters") : json::object(); - auto arg = p.tool_arg(arg_open + p.choice({ - p.literal("\n") + value_until("\n") + p.tool_arg_close(p.literal("\n")), - value_until("") + p.tool_arg_close(p.literal("")), - })); - arg_choices |= p.rule("tool-" + name + "-arg-" + prop_name, arg); - }); + auto schema_info = common_schema_info(); + schema_info.resolve_refs(parameters); - auto args = p.tool_args(p.zero_or_more(arg_choices + p.space())); - auto func = p.tool( - p.tool_open(p.literal("")) + - p.space() + args + p.space() + - p.tool_close(p.literal(""))); + std::vector required_args; + std::vector optional_args; - tool_choices |= p.rule("tool-" + name, func); - }); + foreach_parameter(function, [&](const std::string & param_name, const json & param_schema, bool is_required) { + auto rule_name = "tool-" + name + "-arg-" + param_name; - auto tool_call = - p.literal("") + p.space() + tool_choices + p.space() + p.literal(""); - - const int max_calls = inputs.parallel_tool_calls ? -1 : 1; - auto tool_calls = p.repeat(tool_call + p.space(), 1, max_calls); - - p.rule("tool-tail", [&]() { - auto content_until_boundary = p.content(p.until_one_of({"", ""})); - return p.choice({ - p.literal("") + p.space() + p.content(p.rest()), - content_until_boundary + p.choice({ - p.ref("tool-call"), - p.literal("") + p.space() + p.content(p.rest()), - p.content(p.rest()), - }), - }); - }); - auto tool_suffix = p.trigger_rule("tool-call", tool_calls + p.ref("tool-tail")); + auto arg_open = p.tool_arg_open("\n"); - auto outside = p.choice({ - p.content(p.until("")) + tool_suffix, - p.content(p.rest()), - }); + auto arg_value = schema_info.resolves_to_string(param_schema) ? + arg_string : + p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", param_schema)) + arg_close; - if (!extract_reasoning || !inputs.enable_thinking) { - return generation_prompt + outside; - } - - // Classify the emitted completion rather than the generation prompt. Some - // templates prefill "\n", but Qwen-style models may omit the - // matching close tag for tool-call and final-answer turns. In practice: - // - emitted closes reasoning; - // - emitted makes any preceding text reasoning; - // - no emitted reasoning/tool tags means final assistant content. - auto reasoning_before_boundary = - p.reasoning(p.until_one_of({"", "\n", "\n"})) + - p.optional(p.literal("\n")); - - auto reasoning_until_tool = - p.peek(p.until_one_of({"", "\n", "\n"}) + p.literal("\n")) + - reasoning_before_boundary + - tool_suffix; - - auto reasoning_until_close = - p.peek(p.until_one_of({"", "\n", "\n"}) + p.choice({ - p.literal(""), - p.literal("\n"), - })) + - reasoning_before_boundary + - p.literal("") + p.space() + outside; - - auto reasoning_then_boundary = p.choice({ - tool_suffix, - reasoning_until_tool, - reasoning_until_close, - p.content(p.rest()), - }); + auto arg_rule = p.rule(rule_name, p.tool_arg(arg_open + arg_value)); - auto generated_reasoning = p.literal("") + p.space() + reasoning_then_boundary; + (is_required ? required_args : optional_args).push_back(arg_rule); + }); - return generation_prompt + p.choice({ - generated_reasoning, - reasoning_then_boundary, - }); + // Accept required arguments in any order, as Qwen does not always adhere to the + // order provided. + auto args = p.permute("tool-" + name + "-args", required_args); + if (!optional_args.empty()) { + args = args + p.zero_or_more(p.choice(optional_args)); + } + + auto func = p.tool(p.tool_open("\n") + + p.tool_args(args) + + p.tool_close(p.literal("\n"))); + + tool_choice |= p.rule("tool-" + name, func); + }); + + auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0; + auto max_calls = inputs.parallel_tool_calls ? -1 : 1; + auto tool_call = p.rule("tool-call", "\n" + tool_choice + "" + p.space()); + auto tool_calls = p.trigger_rule("tool-call-root", p.repeat(tool_call, min_calls, max_calls)); + + return generation_prompt + (reasoning << p.content(p.until("")) << tool_calls); + } + + // Content only parser + return generation_prompt + (reasoning << p.content(p.rest())); }); - data.parser = parser.save(); - data.grammar_lazy = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; - data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.at("parameters"); - builder.resolve_refs(schema); + data.parser = parser.save(); + + if (include_grammar) { + data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; + + data.grammar = build_grammar([&](const common_grammar_builder & builder) { + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + auto schema = function.contains("parameters") ? function.at("parameters") : json::object(); + builder.resolve_refs(schema); + }); + if (has_response_format) { + auto schema = inputs.json_schema; + builder.resolve_refs(schema); + } + parser.build_grammar(builder, data.grammar_lazy); }); - parser.build_grammar(builder, data.grammar_lazy); - }); - if (data.grammar_lazy) { - data.grammar_triggers = { - { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "" }, - }; + + if (data.grammar_lazy) { + data.grammar_triggers = { + { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "" } + }; + } } return data; @@ -3064,20 +3073,6 @@ std::optional common_chat_try_specialized_template( return common_chat_params_init_ministral_3(tmpl, params); } - // Tagged thinking/tool protocol: reasoning uses ... and tools use - // .... - if (params.tools.is_array() && !params.tools.empty() && - params.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE && - src.find("") != std::string::npos && - src.find("") != std::string::npos && - src.find("") != std::string::npos && - src.find("") != std::string::npos && - src.find("") != std::string::npos) { LOG_DBG("Using specialized template: GPT-OSS\n"); @@ -3167,6 +3162,14 @@ std::optional common_chat_try_specialized_template( return common_chat_params_init_minicpm5(tmpl, params); } + // Qwen3-Coder XML tool calls, also used by Nemotron Nano 3, Qwen3.5 and StepFun-3.5-Flash + if (src.find("") != std::string::npos && + src.find(" preserved_tokens; std::vector additional_stops; std::string parser; - // If a complete parse reports NEED_MORE in lenient streaming mode, retry without - // leniency so EOF can be used as a real boundary by parsers that opt in. - bool strict_eof_on_complete = false; common_chat_msg_delimiters message_delimiters; }; @@ -298,13 +295,11 @@ struct common_chat_parser_params { bool is_continuation = false; bool echo = false; // Include assistant prefilled msg in output bool debug = false; // Enable debug output for PEG parser - bool strict_eof_on_complete = false; common_peg_arena parser = {}; common_chat_parser_params() = default; common_chat_parser_params(const common_chat_params & chat_params) { - format = chat_params.format; - generation_prompt = chat_params.generation_prompt; - strict_eof_on_complete = chat_params.strict_eof_on_complete; + format = chat_params.format; + generation_prompt = chat_params.generation_prompt; } }; diff --git a/tests/test-chat-peg-parser.cpp b/tests/test-chat-peg-parser.cpp index 908b13fd0ca7..3ab7a67b6a82 100644 --- a/tests/test-chat-peg-parser.cpp +++ b/tests/test-chat-peg-parser.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include "nlohmann/json.hpp" @@ -21,6 +22,7 @@ static void test_example_qwen3_non_coder(testing & t); static void test_command7_parser_compare(testing & t); static void test_prefix_tool_names(testing & t); static void test_tagged_peg_parser(testing & t); +static void test_permute(testing & t); int main(int argc, char * argv[]) { testing t(std::cout); @@ -39,6 +41,7 @@ int main(int argc, char * argv[]) { t.test("comparison", test_command7_parser_compare); t.test("prefix tool names", test_prefix_tool_names); t.test("tagged peg parser", test_tagged_peg_parser); + t.test("permute", test_permute); return t.summary(); } @@ -981,3 +984,103 @@ static void test_tagged_peg_parser(testing & t) { t.assert_equal("fun_post should be '>'", ">", result.tags["fun_post"]); }); } + +static void test_permute(testing & t) { + auto accepts = [](const common_peg_arena & parser, const std::string & input) { + common_peg_parse_context ctx(input); + return parser.parse(ctx).success(); + }; + + auto gbnf_of = [](const common_peg_arena & parser) { + return build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); + }; + + auto assert_gbnf_equal = [](testing & t, const std::string & expected, const std::string & actual) { + static const std::regex leading_ws_re = std::regex(R"((^|\n)\s+)"); + t.assert_equal("gbnf are equal", std::regex_replace(expected, leading_ws_re, "$1"), actual); + }; + + auto count_rules = [](const std::string & gbnf, const std::string & prefix) { + size_t count = 0; + for (const auto & line : string_split(gbnf, '\n')) { + if (line.rfind(prefix, 0) == 0) { + count++; + } + } + return count; + }; + + t.test("accepts every ordering", [&](testing & t) { + auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) { + return p.permute("abc", { p.literal("a"), p.literal("b"), p.literal("c") }) + p.end(); + }); + + for (const std::string input : { "abc", "acb", "bac", "bca", "cab", "cba" }) { + t.assert_true("accepts " + input, accepts(parser, input)); + } + }); + + t.test("single element", [&](testing & t) { + auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) { + return p.permute("a", { p.literal("a") }) + p.end(); + }); + + t.assert_true("accepts a", accepts(parser, "a")); + t.assert_true("rejects aa", !accepts(parser, "aa")); + }); + + t.test("grammar left-factorizes shared tails", [&](testing & t) { + auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) { + return p.permute("abc", { p.literal("a"), p.literal("b"), p.literal("c") }) + p.end(); + }); + + // Every rule is one remaining subset, keyed by bitmask: abc-3 is {a,b}, abc-7 is {a,b,c}. + // Each subset is emitted once and shared by every branch that leads into it. + assert_gbnf_equal(t, R"""( + abc-1 ::= "a" + abc-2 ::= "b" + abc-3 ::= "a" abc-2 | "b" abc-1 + abc-4 ::= "c" + abc-5 ::= "a" abc-4 | "c" abc-1 + abc-6 ::= "b" abc-4 | "c" abc-2 + abc-7 ::= "a" abc-6 | "b" abc-5 | "c" abc-3 + root ::= abc-7 + space ::= | " " | "\n"{1,2} [ \t]{0,20} + )""", gbnf_of(parser)); + }); + + t.test("grammar emits one rule per remaining subset", [&](testing & t) { + auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) { + return p.permute("abcd", { p.literal("a"), p.literal("b"), p.literal("c"), p.literal("d") }) + p.end(); + }); + + // 2^4 - 1 non-empty subsets, one rule each - not the 4! = 24 orderings. + t.assert_equal("permute rule count", 15u, count_rules(gbnf_of(parser), "abcd-")); + }); + + t.test("grammar emits no rules for a single element", [&](testing & t) { + auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) { + return p.permute("a", { p.literal("a") }) + p.end(); + }); + + assert_gbnf_equal(t, R"""( + root ::= "a" + space ::= | " " | "\n"{1,2} [ \t]{0,20} + )""", gbnf_of(parser)); + }); + + t.test("grammar falls back to the given order when too large", [&](testing & t) { + auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) { + std::vector parsers; + for (size_t i = 0; i <= COMMON_CHAT_MAX_PERMUTE; i++) { + parsers.push_back(p.literal(std::string(1, (char) ('a' + i)))); + } + return p.permute("big", parsers) + p.end(); + }); + + assert_gbnf_equal(t, R"""( + root ::= "a" "b" "c" "d" "e" "f" "g" + space ::= | " " | "\n"{1,2} [ \t]{0,20} + )""", gbnf_of(parser)); + }); +} diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index c0cb7108d2b3..019eb6921830 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -1451,11 +1451,6 @@ class peg_tester { template_path_(template_path), detailed_debug_(detailed_debug) {} - explicit peg_tester(common_chat_templates_ptr tmpls, const std::string & template_path, const bool detailed_debug = false) : - tmpls_(std::move(tmpls)), - template_path_(template_path), - detailed_debug_(detailed_debug) {} - const std::string & template_path() const { return template_path_; } peg_test_builder test(const std::string & input); @@ -2252,6 +2247,7 @@ static void test_template_output_peg_parsers(bool detailed_debug) { }) .run(); + // test code that starts with indent tst.test( "\n" @@ -2272,150 +2268,49 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .run(); tst.test( - "Need to inspect the current directory.\n" - "\n" - "\n" - "\n" - "pwd\n" - "\n" - "\n" - "\n" - "\n" - "Done.") - .enable_thinking(true) + "I need to output the invoice details in JSON\n" + "\n\n" + R"({"amount": 123.45, "date": "2025-12-03"})") .reasoning_format(COMMON_REASONING_FORMAT_AUTO) - .tools({ run_in_terminal_tool }) - .expect_reasoning("Need to inspect the current directory.") - .expect_content("Done.") - .expect_tool_calls({ - { "run_in_terminal", R"({"command": "pwd"})", {} }, - }) + .enable_thinking(true) + .json_schema(invoice_schema) + .expect_reasoning("I need to output the invoice details in JSON") + .expect_content(R"({"amount": 123.45, "date": "2025-12-03"})") .run(); + // a tool call ends the prefilled thinking block, with or without a closing tst.test( - "Need to inspect the current directory.\n" "\n" "\n" "\n" "pwd\n" "\n" "\n" - "\n" - "Done.") + "") .enable_thinking(true) .reasoning_format(COMMON_REASONING_FORMAT_AUTO) .tools({ run_in_terminal_tool }) - .expect_reasoning("Need to inspect the current directory.") - .expect_content("Done.") .expect_tool_calls({ { "run_in_terminal", R"({"command": "pwd"})", {} }, }) .run(); + // ...including after the model has thought about it tst.test( - "\n" - "\n" - "\n" - "new text\n" - "\n" - "\n" - "old text\n" - "\n" - "\n" - "foo.cpp\n" - "\n" - "\n" - "") - .enable_thinking(false) - .reasoning_format(COMMON_REASONING_FORMAT_AUTO) - .tools({ edit_tool }) - .expect_tool_calls({ - { "edit", R"({"newString": "new text", "oldString": "old text", "filename": "foo.cpp"})", {} }, - }) - .run(); - - tst.test( - "Need the directory.\n" + "Need to inspect the current directory.\n" "\n" "\n" "\n" "pwd\n" "\n" "\n" - "\n" - "Need the files.\n" - "\n" - "\n" - "\n" - "ls\n" - "\n" - "\n" - "\n" - "\n" - "Done.") + "") .enable_thinking(true) .reasoning_format(COMMON_REASONING_FORMAT_AUTO) - .parallel_tool_calls(true) .tools({ run_in_terminal_tool }) - .expect_reasoning("Need the directory.") - .expect_content("Need the files.\nDone.") + .expect_reasoning("Need to inspect the current directory.") .expect_tool_calls({ { "run_in_terminal", R"({"command": "pwd"})", {} }, - { "run_in_terminal", R"({"command": "ls"})", {} }, - }) - .run(); - - tst.test( - "I need to output the invoice details in JSON\n" - "\n\n" - R"({"amount": 123.45, "date": "2025-12-03"})") - .reasoning_format(COMMON_REASONING_FORMAT_AUTO) - .enable_thinking(true) - .json_schema(invoice_schema) - .expect_reasoning("I need to output the invoice details in JSON") - .expect_content(R"({"amount": 123.45, "date": "2025-12-03"})") - .run(); - - // tool call segment in reasoning - tst.test( - "Let's call a tool: \n" - "\n" - "\n" - "def hello():\n" - " print(\"Not the real call!\")\n" - "\n" - "hello()\n" - "\n" - "\n" - "\n\n\n" - "\n" - "\n" - "\n" - "def hello():\n" - " print(\"Hello, world!\")\n" - "\n" - "hello()\n" - "\n" - "\n" - "") - .enable_thinking(true) - .reasoning_format(COMMON_REASONING_FORMAT_AUTO) - .tools({ - python_tool - }) - .expect_reasoning( - "Let's call a tool: \n" - "\n" - "\n" - "def hello():\n" - " print(\"Not the real call!\")\n" - "\n" - "hello()\n" - "\n" - "\n" - "") - .expect_tool_calls({ - { "python", "{\"code\": \"def hello():\\n print(\\\"Hello, world!\\\")\\n\\nhello()\"}", {} }, }) .run(); @@ -2455,7 +2350,7 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .tools({ special_function_tool }) - .expect_reasoning("") + .expect_reasoning("\n\n") .expect_tool_calls({ { "special_function", "{\"arg1\": 1}", "" } }) .run(); @@ -2559,24 +2454,6 @@ static void test_template_output_peg_parsers(bool detailed_debug) { }) .run(); - tst.test( - "I might call later, but I am still thinking.\n" - "\n\n" - "Final answer without tools.") - .reasoning_format(COMMON_REASONING_FORMAT_AUTO) - .enable_thinking(true) - .tools({ run_in_terminal_tool }) - .expect_reasoning("I might call later, but I am still thinking.") - .expect_content("Final answer without tools.") - .run(); - - tst.test("Final answer without thinking.") - .reasoning_format(COMMON_REASONING_FORMAT_AUTO) - .enable_thinking(true) - .tools({ run_in_terminal_tool }) - .expect_content("Final answer without thinking.") - .run(); - // Continuation tests tst.test("world!\nWhat's up?") .reasoning_format(COMMON_REASONING_FORMAT_AUTO) @@ -2683,93 +2560,6 @@ static void test_template_output_peg_parsers(bool detailed_debug) { }) .run(); } - - { - auto src = read_file("models/templates/Qwen3.5-4B.jinja"); - string_replace_all(src, - "{%- if add_generation_prompt %}\n" - " {{- '<|im_start|>assistant\\n' }}\n" - " {%- if enable_thinking is defined and enable_thinking is false %}\n" - " {{- '\\n\\n\\n\\n' }}\n" - " {%- else %}\n" - " {{- '\\n' }}\n" - " {%- endif %}\n" - "{%- endif %}", - "{%- if add_generation_prompt %}\n" - " {{- '<|im_start|>assistant\\n' }}\n" - "{%- endif %}"); - - auto no_prefill = peg_tester( - common_chat_templates_ptr(common_chat_templates_init(/* model= */ nullptr, src)), - "models/templates/Qwen3.5-4B-no-prefill-think.jinja", - detailed_debug); - - no_prefill.test( - "\n" - "I need to run a terminal command.\n" - "\n\n" - "\n" - "\n" - "\n" - "pwd\n" - "\n" - "\n" - "") - .enable_thinking(true) - .reasoning_format(COMMON_REASONING_FORMAT_AUTO) - .tools({ run_in_terminal_tool }) - .expect_reasoning("I need to run a terminal command.") - .expect_tool_calls({ - { "run_in_terminal", R"({"command": "pwd"})", {} }, - }) - .run(); - - no_prefill.test( - "\n" - "I need to run a terminal command.\n" - "\n" - "\n" - "\n" - "pwd\n" - "\n" - "\n" - "\n" - "Done.") - .enable_thinking(true) - .reasoning_format(COMMON_REASONING_FORMAT_AUTO) - .tools({ run_in_terminal_tool }) - .expect_reasoning("I need to run a terminal command.") - .expect_content("Done.") - .expect_tool_calls({ - { "run_in_terminal", R"({"command": "pwd"})", {} }, - }) - .run(); - - no_prefill.test( - "I need to run a terminal command.\n" - "\n" - "\n" - "\n" - "pwd\n" - "\n" - "\n" - "") - .enable_thinking(true) - .reasoning_format(COMMON_REASONING_FORMAT_AUTO) - .tools({ run_in_terminal_tool }) - .expect_reasoning("I need to run a terminal command.") - .expect_tool_calls({ - { "run_in_terminal", R"({"command": "pwd"})", {} }, - }) - .run(); - - no_prefill.test("Final answer without thinking.") - .enable_thinking(true) - .reasoning_format(COMMON_REASONING_FORMAT_AUTO) - .tools({ run_in_terminal_tool }) - .expect_content("Final answer without thinking.") - .run(); - } } { @@ -2968,49 +2758,6 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .expect_content(R"({"amount": 123.45, "date": "2025-12-03"})") .run(); - // tool call segment in reasoning - tst.test( - "Let's call a tool: \n" - "\n" - "\n" - "def hello():\n" - " print(\"Not the real call!\")\n" - "\n" - "hello()\n" - "\n" - "\n" - "\n\n" - "\n" - "\n" - "\n" - "def hello():\n" - " print(\"Hello, world!\")\n" - "\n" - "hello()\n" - "\n" - "\n" - "\n" - ) - .enable_thinking(true) - .reasoning_format(COMMON_REASONING_FORMAT_AUTO) - .tools({ - python_tool - }) - .expect_reasoning("Let's call a tool: \n" - "\n" - "\n" - "def hello():\n" - " print(\"Not the real call!\")\n" - "\n" - "hello()\n" - "\n" - "\n" - "\n") - .expect_tool_calls({ - { "python", "{\"code\": \"def hello():\\n print(\\\"Hello, world!\\\")\\n\\nhello()\"}", {} }, - }) - .run(); - // Continuation tests tst.test("world!\nWhat's up?") .reasoning_format(COMMON_REASONING_FORMAT_AUTO) @@ -3872,6 +3619,37 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .expect_reconstruction() .run(); + // Test flexible required argument ordering (required args still come first, in any order) + tst.test( + "\n" + "\n" + "\n#include\n\n" + "\nfoo.c\n\n" + "\n#iclunde\n\n" + "\n" + "") + .tools({ edit_tool }) + .expect_tool_calls({ + { "edit", R"({"newString": "#include", "filename": "foo.c", "oldString": "#iclunde"})", {} }, + }) + .expect_reconstruction() + .run(); + + tst.test( + "\n" + "\n" + "\n42\n\n" + "\nhello\n\n" + "\n200\n\n" + "\n" + "") + .tools({ tool_2req_4opt }) + .expect_tool_calls({ + { "tool_2req_4opt", R"({"req2": 42, "req1": "hello", "opt2": 200})", {} }, + }) + .expect_reconstruction() + .run(); + // Test flexible optional argument ordering (2 required + 4 optional, reversed optional order) tst.test( "\n" From 4bc63c901036ab2eb97866860d683b722e56bb76 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Tue, 28 Jul 2026 17:23:19 -0500 Subject: [PATCH 3/6] cont : add support for omission --- common/chat.cpp | 19 +++++++++++----- tests/test-chat.cpp | 55 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 6 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index ba7d3f63c18d..c0d2dacda5d3 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1226,12 +1226,18 @@ static common_chat_params common_chat_params_init_qwen3_coder(const common_chat_ tool_choice |= p.rule("tool-" + name, func); }); - auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0; - auto max_calls = inputs.parallel_tool_calls ? -1 : 1; - auto tool_call = p.rule("tool-call", "\n" + tool_choice + "" + p.space()); - auto tool_calls = p.trigger_rule("tool-call-root", p.repeat(tool_call, min_calls, max_calls)); + auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0; + + // Qwen3-Coder models may occasionally omit the token. + auto tool_call_body = tool_choice + "" + p.space(); + auto tool_call_first = p.rule("tool-call-first", p.optional(p.literal("\n")) + tool_call_body); + auto tool_call = p.rule("tool-call", "\n" + tool_call_body); + + auto calls = inputs.parallel_tool_calls ? tool_call_first + p.zero_or_more(tool_call) : tool_call_first; + auto tool_calls = p.trigger_rule("tool-call-root", p.repeat(calls, min_calls, 1)); - return generation_prompt + (reasoning << p.content(p.until("")) << tool_calls); + return generation_prompt + + (reasoning << p.content(p.until_one_of({ "", "" } + { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "" }, + { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, " and go straight to + tst.test( + "\n" + "\n" + "1\n" + "\n" + "\n" + "") + .tools({ special_function_tool }) + .expect(message_assist_call) + .run(); + + tst.test( + "Let me call it.\n" + "\n" + "\n" + "1\n" + "\n" + "\n" + "") + .tools({ special_function_tool }) + .expect_content("Let me call it.\n") + .expect_tool_calls({ + { "special_function", R"({"arg1": 1})", {} }, + }) + .run(); + + // Only the first call may omit it, the rest keep the \n separator + tst.test( + "\n" + "\n" + "1\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "1\n" + "\n" + "\n" + "2\n" + "\n" + "\n" + "") + .parallel_tool_calls(true) + .tools({ + special_function_tool, special_function_tool_with_optional_param + }) + .expect_tool_calls({ + { "special_function", R"({"arg1": 1})", {} }, + { "special_function_with_opt", R"({"arg1": 1, "arg2": 2})", {} }, + }) + .run(); + tst.test( "\n" "\n" From 93776cfff04b26fb6021b4c37cc3fdc42144206d Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Wed, 29 Jul 2026 13:19:36 -0500 Subject: [PATCH 4/6] cont : update tool delimiters --- common/chat.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index c0d2dacda5d3..f508d1d1ef66 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1137,8 +1137,8 @@ static common_chat_params common_chat_params_init_qwen3_coder(const common_chat_ data.message_delimiters = { { COMMON_CHAT_ROLE_ASSISTANT, "<|im_start|>assistant" }, - { COMMON_CHAT_ROLE_TOOL, "<|im_start|>user\n" }, - { COMMON_CHAT_ROLE_TOOL, "<|im_start|>tool" }, + { COMMON_CHAT_ROLE_TOOL, "<|im_start|>user\n" }, // Qwen3-Coder, Qwen3.5, Nemotron Nano 3 + { COMMON_CHAT_ROLE_TOOL, "<|im_start|>tool_response" }, // StepFun-3.5-Flash { COMMON_CHAT_ROLE_USER, "<|im_start|>user" }, { COMMON_CHAT_ROLE_SYSTEM, "<|im_start|>system" }, }; From 1f76cec73afee2894f2ea82156c087d1448f4491 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Wed, 29 Jul 2026 16:18:13 -0500 Subject: [PATCH 5/6] cont : add comment for qwen3-coder --- common/chat.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/common/chat.cpp b/common/chat.cpp index f508d1d1ef66..2019629ad540 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1131,6 +1131,7 @@ static common_chat_params common_chat_params_init_qwen3_coder(const common_chat_ if (supports_reasoning) { data.thinking_start_tag = ""; // Support both and as reasoning end sequences. + // ", "" }; data.preserved_tokens.insert(data.preserved_tokens.end(), { "", "" }); } From d2a2f64e9e18ced67bc2056438eed7af9925a412 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Fri, 31 Jul 2026 22:20:53 -0500 Subject: [PATCH 6/6] cont : fix trigger pattern for " }, - { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "" }, + // Trigger on "