diff --git a/common/chat-auto-parser-generator.cpp b/common/chat-auto-parser-generator.cpp index af84ff323daf..a440de6a5cbb 100644 --- a/common/chat-auto-parser-generator.cpp +++ b/common/chat-auto-parser-generator.cpp @@ -13,6 +13,12 @@ using json = nlohmann::ordered_json; +// Tool-call markers arrive with whatever whitespace the template renders after them +// ("\n"), but models vary it (" \n"). Markers are used as grammar +// triggers, parser literals and until() needles, so demanding the exact bytes silently +// turns a tool call into plain content. Trimming is safe: every marker is followed by +// p.space(), which takes any whitespace run (build_parser() trims reasoning markers too). + // Helper to iterate over tools/functions static void foreach_function(const json & tools, const std::function & fn) { for (const auto & tool : tools) { @@ -77,8 +83,11 @@ common_chat_params peg_generator::generate_parser(const common_chat_template & // Build grammar if tools are present bool has_tools = autoparser.tools.format.mode != tool_format::NONE && inputs.tools.is_array() && !inputs.tools.empty(); - std::string trigger_marker = !autoparser.tools.format.section_start.empty() ? autoparser.tools.format.section_start : - autoparser.tools.format.per_call_start; + // A lazy grammar only engages once its trigger is complete, so it can never + // constrain whitespace that is part of the trigger. + std::string trigger_marker = trim_trailing_whitespace( + !autoparser.tools.format.section_start.empty() ? autoparser.tools.format.section_start : + autoparser.tools.format.per_call_start); bool has_response_format = !inputs.json_schema.empty() && inputs.json_schema.is_object(); bool include_grammar = has_response_format || (has_tools && @@ -370,7 +379,9 @@ common_peg_parser analyze_tools::build_tool_parser_tag_json(parser_build_context tool_calls = p.optional(tool_calls); } - std::string trigger_marker = !format.section_start.empty() ? format.section_start : format.per_call_start; + // A needle that misses swallows the whole tool call as content. + std::string trigger_marker = trim_trailing_whitespace( + !format.section_start.empty() ? format.section_start : format.per_call_start); auto content_before_tools = trigger_marker.empty() ? p.eps() : p.until(trigger_marker); return ctx.reasoning_parser + p.optional(p.content(content_before_tools)) + tool_calls + p.end(); } @@ -473,7 +484,9 @@ common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_conte common_peg_parser tool_calls = p.eps(); if (!format.per_call_start.empty()) { - auto wrapped_call = format.per_call_start + p.space() + tool_choice + p.space() + format.per_call_end; + // p.space() below already absorbs the marker's trailing whitespace. + auto wrapped_call = trim_trailing_whitespace(format.per_call_start) + p.space() + tool_choice + + p.space() + format.per_call_end; if (inputs.parallel_tool_calls) { tool_calls = p.trigger_rule("tool-call", wrapped_call + p.zero_or_more(p.space() + wrapped_call) + p.space()); } else { @@ -501,7 +514,9 @@ common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_conte tool_calls = p.optional(tool_calls); } - std::string trigger_marker = !format.section_start.empty() ? format.section_start : format.per_call_start; + // A needle that misses swallows the whole tool call as content. + std::string trigger_marker = trim_trailing_whitespace( + !format.section_start.empty() ? format.section_start : format.per_call_start); auto content_before_tools = trigger_marker.empty() ? p.eps() : p.until(trigger_marker); return ctx.reasoning_parser + p.optional(p.content(content_before_tools)) + tool_calls + p.end(); } diff --git a/models/templates/Nanbeige4.2-3B.jinja b/models/templates/Nanbeige4.2-3B.jinja new file mode 100644 index 000000000000..f2e394026e4e --- /dev/null +++ b/models/templates/Nanbeige4.2-3B.jinja @@ -0,0 +1,195 @@ + + +{%- macro visible_text(content) -%} + {%- if content is string -%} + {{- content }} + {%- elif content is iterable and content is not mapping -%} + {%- for item in content -%} + {%- if item is mapping and item.type == 'text' -%} + {{- item.text }} + {%- elif item is string -%} + {{- item }} + {%- elif item is mapping and item.type in ['image', 'image_url', 'video', 'video_url', 'audio', 'audio_url', 'input_audio'] -%} + {%- set media_type = item.type | replace('_url', '') | replace('input_', '') -%} + {{- "You are unable to process this " ~ media_type ~ " because you don't have multi-modal input ability. Try different methods." }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{- content }} + {%- endif -%} +{%- endmacro -%} + + +{%- set tool_call_format = tool_call_format if tool_call_format is defined else 'xml' %} +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages|length > 0 and messages[0].get('role', '') == 'system' %} + {{- visible_text(messages[0].content) + '\n\n' }} + {%- else %} + {{- '你是一位工具函数调用专家,你会得到一个问题和一组可能的工具函数。根据问题,你需要进行一个或多个函数/工具调用以实现目的,请尽量尝试探索通过工具解决问题。\n如果没有一个函数可以使用,请直接使用自然语言回复用户。\n如果给定的问题缺少函数所需的参数,请使用自然语言进行提问,向用户询问必要信息。\n如果调用结果已经足够回答用户问题,请对历史结果进行总结,使用自然语言回复用户。' }} + {%- endif %} + + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + + {%- if tool_call_format == 'json' %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n" }} + {{- '\n{\"name\": , \"arguments\": }\n<|im_end|>\n' }} + {%- else %} + {{- "\n\n\nFor each function call, output the function name and arguments within the following XML format:\n" }} + {{- ' + + +value_1 + + +This is the value for the second parameter +that can span +multiple lines + + +<|im_end|>\n' }} + {%- endif %} + +{%- else %} + {%- if messages|length > 0 and messages[0].get('role', '') == 'system' %} + {{- '<|im_start|>system\n' + visible_text(messages[0].content) + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\n你是南北阁,一款由BOSS直聘自主研发并训练的专业大语言模型。<|im_end|>\n' }} + {%- endif %} +{%- endif %} + +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.get('role', '') == "user" and visible_text(message.content) is string and not(visible_text(message.content).startswith('') and visible_text(message.content).endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} + +{%- for message in messages %} + {%- if visible_text(message.content) is string %} + {%- set content = visible_text(message.content) %} + {%- else %} + {%- set content = '' %} + {%- endif %} + + {%- if message.get('role', '') == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + + {%- elif message.get('role', '') == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n').rstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + + {%- if (preserve_thinking is defined and preserve_thinking is false) and (loop.index0 < ns.last_query_index) %} + {{- '<|im_start|>' + message.get('role', '') + '\n\n\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.get('role', '') + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- endif %} + + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- if tool_call_format == 'json' %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- else %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + + {%- if loop.first %} + {%- if content|trim %} + {{- ' + + + +' }} + {%- else %} + {{- ' + +' }} + {%- endif %} + {%- else %} + {{- ' + + +' }} + {%- endif %} + + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- ' +' }} + {%- set args_value = args_value | string if args_value is string else args_value | tojson | safe %} + {{- args_value }} + {{- ' + +' }} + {%- endfor %} + {%- endif %} + {{- ' +' }} + {%- endfor %} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} + + {%- elif message.get('role', '') == "tool" %} + {%- if loop.previtem and loop.previtem.get('role', '') != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or loop.nextitem.get('role', '') != "tool" %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- elif message.get('role', '') != '' %} + {{- '<|im_start|>' + message.get('role', '') + '\n' + content + '<|im_end|>' + '\n' }} + {%- endif %} +{%- endfor %} + +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + + {%- if enable_thinking is defined and enable_thinking is false %} + {{- ' + + + +' }} + {%- else %} + {{- ' +' }} + {%- endif %} + +{%- endif %} diff --git a/tests/test-chat-auto-parser.cpp b/tests/test-chat-auto-parser.cpp index 4218f8d5747d..ca01bdf32cec 100644 --- a/tests/test-chat-auto-parser.cpp +++ b/tests/test-chat-auto-parser.cpp @@ -90,6 +90,10 @@ static void test_normalize_quotes_with_embedded_quotes(testing & t); // TAG_WITH_TAGGED argument parsing tests static void test_tagged_args_with_embedded_quotes(testing & t); +// Tool-call marker whitespace tolerance tests +static void test_marker_whitespace_tolerance(testing & t); +static void test_trim_trailing_whitespace(testing & t); + static void test_role_markers_all_templates(testing & t); int main(int argc, char * argv[]) { @@ -117,6 +121,8 @@ int main(int argc, char * argv[]) { t.test("standard_json_tools", test_standard_json_tools_formats); t.test("normalize_quotes_to_json", test_normalize_quotes_to_json); t.test("tagged_args_embedded_quotes", test_tagged_args_with_embedded_quotes); + t.test("trim_trailing_whitespace", test_trim_trailing_whitespace); + t.test("marker_whitespace_tolerance", test_marker_whitespace_tolerance); t.test("role_markers_all_templates", test_role_markers_all_templates); return t.summary(); @@ -1304,7 +1310,7 @@ static common_chat_template load_template(testing & t, const std::string & templ } std::string template_source = buf.str(); common_chat_template tmpl(template_source, "", ""); - t.assert_true("Nemotron template loaded successfully", template_source.length() > 0); + t.assert_true("template loaded successfully: " + template_path, template_source.length() > 0); return tmpl; } @@ -1978,6 +1984,7 @@ static void test_role_markers_all_templates(testing & t) { { "Bielik-11B-v3.0-Instruct.jinja", "<|im_start|>user", "<|im_start|>assistant" }, { "HuggingFaceTB-SmolLM3-3B.jinja", "<|im_start|>user", "<|im_start|>assistant" }, { "MiMo-VL.jinja", "<|im_start|>user", "<|im_start|>assistant" }, + { "Nanbeige4.2-3B.jinja", "<|im_start|>user", "<|im_start|>assistant" }, { "NousResearch-Hermes-2-Pro-Llama-3-8B-tool_use.jinja", "<|im_start|>user", "<|im_start|>assistant" }, { "NousResearch-Hermes-3-Llama-3.1-8B-tool_use.jinja", "<|im_start|>user", "<|im_start|>assistant" }, { "NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.jinja", "<|im_start|>user", "<|im_start|>assistant" }, @@ -2075,6 +2082,140 @@ static void test_role_markers_all_templates(testing & t) { } } +// ============================================================================ +// Tool-call marker whitespace tolerance +// +// Models vary the whitespace after a marker (" \n" vs "\n"). If any +// use of the marker demands the exact bytes, the call is silently parsed as content. +// ============================================================================ +static json marker_test_tools() { + return json::parse(R"([ + {"type": "function", "function": { + "name": "read_file", + "description": "Read a file from disk", + "parameters": {"type": "object", + "properties": {"path": {"type": "string"}}, + "required": ["path"]}}} + ])"); +} + +static common_chat_params marker_test_params(testing & t, const std::string & template_path) { + common_chat_template tmpl = load_template(t, template_path); + autoparser::generation_params inputs; + inputs.messages = json::parse(R"([{"role": "user", "content": "Read /etc/hostname."}])"); + inputs.tools = marker_test_tools(); + inputs.tool_choice = COMMON_CHAT_TOOL_CHOICE_AUTO; + inputs.parallel_tool_calls = true; + inputs.reasoning_format = COMMON_REASONING_FORMAT_AUTO; + inputs.add_generation_prompt = true; + inputs.enable_thinking = true; + return autoparser::peg_generator::generate_parser(tmpl, inputs); +} + +static void test_marker_whitespace_tolerance(testing & t) { + struct tmpl_case { + const char * path; + const char * before_call; // model output preceding the first marker + }; + + // Both render "\n...", so both derive a marker with a + // trailing newline. Nanbeige pre-opens , so its output starts in reasoning. + const std::vector cases = { + { "models/templates/Nanbeige4.2-3B.jinja", "thinking\n\n\n" }, + { "models/templates/Qwen3-Coder.jinja", "" }, + }; + + // Whitespace runs a model may emit between the marker and "> whitespace = { + { "newline (canonical)", "\n" }, + { "space then newline", " \n" }, + { "single space", " " }, + { "two spaces", " " }, + { "two newlines", "\n\n" }, + { "tab then newline", "\t\n" }, + { "newline then spaces", "\n " }, + }; + + const std::string call_body = + "\n" + "\n/etc/hostname\n\n" + "\n"; + + for (const auto & tc : cases) { + t.test(tc.path, [&](testing & t) { + auto params = marker_test_params(t, tc.path); + + t.assert_true("lazy grammar is used", params.grammar_lazy); + t.assert_true("a grammar trigger was produced", !params.grammar_triggers.empty()); + for (const auto & trigger : params.grammar_triggers) { + // Whitespace in the trigger is sampled before the grammar can fire. + t.assert_equal("trigger carries no trailing whitespace", + trim_trailing_whitespace(trigger.value), trigger.value); + } + + common_chat_parser_params pp(params); + pp.reasoning_format = COMMON_REASONING_FORMAT_AUTO; + pp.parser.load(params.parser); + + for (const auto & ws : whitespace) { + const std::string & label = ws.first; + const std::string input = + std::string(tc.before_call) + "" + ws.second + call_body; + const auto msg = common_chat_parse(input, /* is_partial = */ false, pp); + + t.assert_equal("one tool call parsed after " + label, (size_t) 1, msg.tool_calls.size()); + if (msg.tool_calls.size() == 1) { + t.assert_equal("tool name after " + label, + std::string("read_file"), msg.tool_calls[0].name); + t.assert_equal("tool arguments after " + label, + std::string("{\"path\":\"/etc/hostname\"}"), msg.tool_calls[0].arguments); + } + // The regression signature: the call surviving as plain content. + t.assert_true("marker did not leak into content after " + label, + msg.content.find("") == std::string::npos); + } + + // Exercises the until() needle: prose kept, call still parsed. + { + const std::string input = std::string(tc.before_call) + + "Let me read it.\n\n \n" + call_body; + const auto msg = common_chat_parse(input, /* is_partial = */ false, pp); + t.assert_equal("call after prose parses", (size_t) 1, msg.tool_calls.size()); + t.assert_true("prose retained as content", + msg.content.find("Let me read it.") != std::string::npos); + t.assert_true("marker did not leak into content after prose", + msg.content.find("") == std::string::npos); + } + + // Parallel calls where every marker is malformed. + { + const std::string input = std::string(tc.before_call) + + " " + call_body + "\n \n" + call_body; + const auto msg = common_chat_parse(input, /* is_partial = */ false, pp); + t.assert_equal("both malformed parallel calls parse", (size_t) 2, msg.tool_calls.size()); + t.assert_true("marker did not leak into content for parallel calls", + msg.content.find("") == std::string::npos); + } + }); + } +} + +static void test_trim_trailing_whitespace(testing & t) { + t.assert_equal("trailing newline removed", + std::string(""), trim_trailing_whitespace("\n")); + t.assert_equal("mixed trailing run removed", + std::string(""), trim_trailing_whitespace(" \n\t ")); + t.assert_equal("nothing to remove", + std::string(""), trim_trailing_whitespace("")); + t.assert_equal("interior whitespace preserved", + std::string("a b"), trim_trailing_whitespace("a b ")); + t.assert_equal("leading whitespace preserved", + std::string("\n"), trim_trailing_whitespace("\n\n")); + t.assert_equal("all whitespace collapses to empty", + std::string(""), trim_trailing_whitespace(" \n")); + t.assert_equal("empty stays empty", std::string(""), trim_trailing_whitespace("")); +} + // Test that reproduces the Seed-OSS template issue with embedded quotes static void test_tagged_args_with_embedded_quotes(testing & t) { json tools = build_edit_tool(); diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index 01b07953a627..fa15aaa1fbe9 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -3553,6 +3553,82 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .run(); } + { + // Nanbeige4.2 -- ChatML, XML-style tool calls, pre-opened by the + // generation prompt. It emits " " rather than "\n" for + // ~25% of calls; the grammar permits that (space ::= " ") so the parser must too. + auto tst = peg_tester("models/templates/Nanbeige4.2-3B.jinja", detailed_debug); + + tst.test("I'm\nthinking\n\n\nHello, world!\nWhat's up?") + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .enable_thinking(true) + .expect(message_assist_thoughts) + .run(); + + // canonical marker + tst.test( + "\n" + "\n" + "\n1\n\n" + "\n" + "") + .enable_thinking(false) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ special_function_tool }) + .expect(message_assist_call) + .expect_reconstruction() + .run(); + + // marker followed by a single space instead of a newline + tst.test( + " " + "\n" + "\n1\n\n" + "\n" + "") + .enable_thinking(false) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ special_function_tool }) + .expect(message_assist_call) + .run(); + + // same, after a reasoning block + tst.test( + "I'm\nthinking\n\n\n" + " " + "\n" + "\n1\n\n" + "\n" + "") + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ special_function_tool }) + .expect(message_assist_call_thoughts) + .run(); + + // parallel calls, both markers followed by a space + tst.test( + " " + "\n" + "\n1\n\n" + "\n" + "\n" + " " + "\n" + "\n1\n\n" + "\n2\n\n" + "\n" + "") + .enable_thinking(false) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .parallel_tool_calls(true) + .tools({ special_function_tool, special_function_tool_with_optional_param }) + .expect_tool_calls({ + { "special_function", R"({"arg1": 1})", {} }, + { "special_function_with_opt", R"({"arg1": 1, "arg2": 2})", {} }, + }) + .run(); + } + { // Qwen3-Coder (tool calling with XML-style format) auto tst = peg_tester("models/templates/Qwen3-Coder.jinja", detailed_debug);