diff --git a/common/chat.cpp b/common/chat.cpp index 0cee80434ece..6daa4c3c62b0 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1102,6 +1102,153 @@ static common_chat_params common_chat_params_init_ministral_3(const common_chat_ return data; } +static common_chat_params common_chat_params_init_tagged_thinking_tools(const common_chat_template & tmpl, + const autoparser::generation_params & inputs) { + common_chat_params data; + + const bool has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + const bool extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; + + data.supports_thinking = true; + data.thinking_start_tag = ""; + data.thinking_end_tag = ""; + data.prompt = common_chat_template_direct_apply(tmpl, inputs); + data.generation_prompt = common_chat_template_generation_prompt(tmpl, inputs); + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + data.strict_eof_on_complete = true; + data.preserved_tokens = { + "", + "", + "", + "", + }; + + if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) { + return data; + } + + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + auto generation_prompt = p.literal(data.generation_prompt); + auto tool_choices = p.choice(); + + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + const auto name = function.at("name").get(); + + auto arg_choices = p.choice(); + foreach_parameter(function, [&](const std::string & prop_name, const json & prop_schema, bool) { + const bool is_string = prop_schema.value("type", "") == "string"; + + auto value_until = [&](const std::string & marker) { + return is_string ? + p.tool_arg_string_value(p.until(marker)) : + p.tool_arg_value(p.until(marker)); + }; + + auto arg_open = p.tool_arg_open( + p.literal("")); + + auto arg = p.tool_arg(arg_open + p.choice({ + p.literal("\n") + value_until("\n") + p.tool_arg_close(p.literal("\n")), + value_until("") + p.tool_arg_close(p.literal("")), + })); + arg_choices |= p.rule("tool-" + name + "-arg-" + prop_name, arg); + }); + + auto args = p.tool_args(p.zero_or_more(arg_choices + p.space())); + auto func = p.tool( + p.tool_open(p.literal("")) + + p.space() + args + p.space() + + p.tool_close(p.literal(""))); + + tool_choices |= p.rule("tool-" + name, func); + }); + + auto tool_call = + p.literal("") + p.space() + tool_choices + p.space() + p.literal(""); + + const int max_calls = inputs.parallel_tool_calls ? -1 : 1; + auto tool_calls = p.repeat(tool_call + p.space(), 1, max_calls); + + p.rule("tool-tail", [&]() { + auto content_until_boundary = p.content(p.until_one_of({"", ""})); + return p.choice({ + p.literal("") + p.space() + p.content(p.rest()), + content_until_boundary + p.choice({ + p.ref("tool-call"), + p.literal("") + p.space() + p.content(p.rest()), + p.content(p.rest()), + }), + }); + }); + auto tool_suffix = p.trigger_rule("tool-call", tool_calls + p.ref("tool-tail")); + + auto outside = p.choice({ + p.content(p.until("")) + tool_suffix, + p.content(p.rest()), + }); + + if (!extract_reasoning || !inputs.enable_thinking) { + return generation_prompt + outside; + } + + // Classify the emitted completion rather than the generation prompt. Some + // templates prefill "\n", but Qwen-style models may omit the + // matching close tag for tool-call and final-answer turns. In practice: + // - emitted closes reasoning; + // - emitted makes any preceding text reasoning; + // - no emitted reasoning/tool tags means final assistant content. + auto reasoning_before_boundary = + p.reasoning(p.until_one_of({"", "\n", "\n"})) + + p.optional(p.literal("\n")); + + auto reasoning_until_tool = + p.peek(p.until_one_of({"", "\n", "\n"}) + p.literal("\n")) + + reasoning_before_boundary + + tool_suffix; + + auto reasoning_until_close = + p.peek(p.until_one_of({"", "\n", "\n"}) + p.choice({ + p.literal(""), + p.literal("\n"), + })) + + reasoning_before_boundary + + p.literal("") + p.space() + outside; + + auto reasoning_then_boundary = p.choice({ + tool_suffix, + reasoning_until_tool, + reasoning_until_close, + p.content(p.rest()), + }); + + auto generated_reasoning = p.literal("") + p.space() + reasoning_then_boundary; + + return generation_prompt + p.choice({ + generated_reasoning, + reasoning_then_boundary, + }); + }); + + data.parser = parser.save(); + data.grammar_lazy = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; + data.grammar = build_grammar([&](const common_grammar_builder & builder) { + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + auto schema = function.at("parameters"); + builder.resolve_refs(schema); + }); + parser.build_grammar(builder, data.grammar_lazy); + }); + if (data.grammar_lazy) { + data.grammar_triggers = { + { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "" }, + }; + } + + return data; +} + static common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & tmpl, const autoparser::generation_params & inputs) { common_chat_params data; @@ -2398,6 +2545,20 @@ std::optional common_chat_try_specialized_template( return common_chat_params_init_ministral_3(tmpl, params); } + // Tagged thinking/tool protocol: reasoning uses ... and tools use + // .... + if (params.tools.is_array() && !params.tools.empty() && + params.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE && + src.find("") != std::string::npos && + src.find("") != std::string::npos && + src.find("") != std::string::npos && + src.find("") != std::string::npos && + src.find("") != std::string::npos) { LOG_DBG("Using specialized template: GPT-OSS\n"); @@ -2707,6 +2868,15 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & src_pars common_peg_parse_context ctx(effective_input, flags); auto result = parser.parse(ctx); + // Streaming parses are lenient so an unterminated prefix can still produce deltas. + // Some complete parses intentionally use EOF as a boundary; retry those strictly + // when requested so "until"/"rest" parsers can commit at the actual end. + if (!is_partial && params.strict_eof_on_complete && result.need_more_input()) { + flags = common_peg_parse_flags(flags & ~COMMON_PEG_PARSE_FLAG_LENIENT); + ctx = common_peg_parse_context(effective_input, flags); + result = parser.parse(ctx); + } + if (result.fail()) { // During partial parsing, return partial results if any AST nodes were captured // This allows streaming to work correctly for formats like FUNC_MARKDOWN_CODE_BLOCK diff --git a/common/chat.h b/common/chat.h index 7898f1623f54..b4b07de2e2c4 100644 --- a/common/chat.h +++ b/common/chat.h @@ -279,6 +279,9 @@ struct common_chat_params { std::vector preserved_tokens; std::vector additional_stops; std::string parser; + // If a complete parse reports NEED_MORE in lenient streaming mode, retry without + // leniency so EOF can be used as a real boundary by parsers that opt in. + bool strict_eof_on_complete = false; common_chat_msg_delimiters message_delimiters; }; @@ -294,11 +297,13 @@ struct common_chat_parser_params { bool is_continuation = false; bool echo = false; // Include assistant prefilled msg in output bool debug = false; // Enable debug output for PEG parser + bool strict_eof_on_complete = false; common_peg_arena parser = {}; common_chat_parser_params() = default; common_chat_parser_params(const common_chat_params & chat_params) { - format = chat_params.format; - generation_prompt = chat_params.generation_prompt; + format = chat_params.format; + generation_prompt = chat_params.generation_prompt; + strict_eof_on_complete = chat_params.strict_eof_on_complete; } }; diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index c38aed8cfeb5..13fcbaa138b7 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -1375,6 +1375,11 @@ class peg_tester { template_path_(template_path), detailed_debug_(detailed_debug) {} + explicit peg_tester(common_chat_templates_ptr tmpls, const std::string & template_path, const bool detailed_debug = false) : + tmpls_(std::move(tmpls)), + template_path_(template_path), + detailed_debug_(detailed_debug) {} + const std::string & template_path() const { return template_path_; } peg_test_builder test(const std::string & input); @@ -2171,7 +2176,6 @@ static void test_template_output_peg_parsers(bool detailed_debug) { }) .run(); - // test code that starts with indent tst.test( "\n" @@ -2191,6 +2195,100 @@ static void test_template_output_peg_parsers(bool detailed_debug) { }) .run(); + tst.test( + "Need to inspect the current directory.\n" + "\n" + "\n" + "\n" + "pwd\n" + "\n" + "\n" + "\n" + "\n" + "Done.") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ run_in_terminal_tool }) + .expect_reasoning("Need to inspect the current directory.") + .expect_content("Done.") + .expect_tool_calls({ + { "run_in_terminal", R"({"command": "pwd"})", {} }, + }) + .run(); + + tst.test( + "Need to inspect the current directory.\n" + "\n" + "\n" + "\n" + "pwd\n" + "\n" + "\n" + "\n" + "Done.") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ run_in_terminal_tool }) + .expect_reasoning("Need to inspect the current directory.") + .expect_content("Done.") + .expect_tool_calls({ + { "run_in_terminal", R"({"command": "pwd"})", {} }, + }) + .run(); + + tst.test( + "\n" + "\n" + "\n" + "new text\n" + "\n" + "\n" + "old text\n" + "\n" + "\n" + "foo.cpp\n" + "\n" + "\n" + "") + .enable_thinking(false) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ edit_tool }) + .expect_tool_calls({ + { "edit", R"({"newString": "new text", "oldString": "old text", "filename": "foo.cpp"})", {} }, + }) + .run(); + + tst.test( + "Need the directory.\n" + "\n" + "\n" + "\n" + "pwd\n" + "\n" + "\n" + "\n" + "Need the files.\n" + "\n" + "\n" + "\n" + "ls\n" + "\n" + "\n" + "\n" + "\n" + "Done.") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .parallel_tool_calls(true) + .tools({ run_in_terminal_tool }) + .expect_reasoning("Need the directory.") + .expect_content("Need the files.\nDone.") + .expect_tool_calls({ + { "run_in_terminal", R"({"command": "pwd"})", {} }, + { "run_in_terminal", R"({"command": "ls"})", {} }, + }) + .run(); + tst.test( "I need to output the invoice details in JSON\n" "\n\n" @@ -2281,7 +2379,7 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .tools({ special_function_tool }) - .expect_reasoning("\n\n") + .expect_reasoning("") .expect_tool_calls({ { "special_function", "{\"arg1\": 1}", "" } }) .run(); @@ -2396,6 +2494,13 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .expect_content("Final answer without tools.") .run(); + tst.test("Final answer without thinking.") + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .enable_thinking(true) + .tools({ run_in_terminal_tool }) + .expect_content("Final answer without thinking.") + .run(); + // Continuation tests tst.test("world!\nWhat's up?") .reasoning_format(COMMON_REASONING_FORMAT_AUTO) @@ -2502,6 +2607,93 @@ static void test_template_output_peg_parsers(bool detailed_debug) { }) .run(); } + + { + auto src = read_file("models/templates/Qwen3.5-4B.jinja"); + string_replace_all(src, + "{%- if add_generation_prompt %}\n" + " {{- '<|im_start|>assistant\\n' }}\n" + " {%- if enable_thinking is defined and enable_thinking is false %}\n" + " {{- '\\n\\n\\n\\n' }}\n" + " {%- else %}\n" + " {{- '\\n' }}\n" + " {%- endif %}\n" + "{%- endif %}", + "{%- if add_generation_prompt %}\n" + " {{- '<|im_start|>assistant\\n' }}\n" + "{%- endif %}"); + + auto no_prefill = peg_tester( + common_chat_templates_ptr(common_chat_templates_init(/* model= */ nullptr, src)), + "models/templates/Qwen3.5-4B-no-prefill-think.jinja", + detailed_debug); + + no_prefill.test( + "\n" + "I need to run a terminal command.\n" + "\n\n" + "\n" + "\n" + "\n" + "pwd\n" + "\n" + "\n" + "") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ run_in_terminal_tool }) + .expect_reasoning("I need to run a terminal command.") + .expect_tool_calls({ + { "run_in_terminal", R"({"command": "pwd"})", {} }, + }) + .run(); + + no_prefill.test( + "\n" + "I need to run a terminal command.\n" + "\n" + "\n" + "\n" + "pwd\n" + "\n" + "\n" + "\n" + "Done.") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ run_in_terminal_tool }) + .expect_reasoning("I need to run a terminal command.") + .expect_content("Done.") + .expect_tool_calls({ + { "run_in_terminal", R"({"command": "pwd"})", {} }, + }) + .run(); + + no_prefill.test( + "I need to run a terminal command.\n" + "\n" + "\n" + "\n" + "pwd\n" + "\n" + "\n" + "") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ run_in_terminal_tool }) + .expect_reasoning("I need to run a terminal command.") + .expect_tool_calls({ + { "run_in_terminal", R"({"command": "pwd"})", {} }, + }) + .run(); + + no_prefill.test("Final answer without thinking.") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ run_in_terminal_tool }) + .expect_content("Final answer without thinking.") + .run(); + } } {