diff --git a/common/chat-auto-parser.h b/common/chat-auto-parser.h index d47b09dcf6fd..2ea2fd67c975 100644 --- a/common/chat-auto-parser.h +++ b/common/chat-auto-parser.h @@ -381,6 +381,7 @@ struct autoparser { jinja::caps jinja_caps; std::string user_start; std::string assistant_start; + std::string tool_response_start; analyze_reasoning reasoning; analyze_content content; analyze_tools tools; @@ -394,6 +395,7 @@ struct autoparser { // Find the starting marker for the user message and assistant message std::string detect_user_start_marker(const common_chat_template & tmpl); std::string detect_assistant_start_marker(const common_chat_template & tmpl); + std::string detect_tool_response_start_marker(const common_chat_template & tmpl); // Run full differential analysis on a template void analyze_template(const common_chat_template & tmpl); diff --git a/common/chat-diff-analyzer.cpp b/common/chat-diff-analyzer.cpp index 127278dfb2ac..3fcb2eeb422e 100644 --- a/common/chat-diff-analyzer.cpp +++ b/common/chat-diff-analyzer.cpp @@ -233,6 +233,7 @@ void autoparser::analyze_template(const common_chat_template & tmpl) { tools = analyze_tools(jinja_caps.supports_tool_calls ? analyze_tools(tmpl, jinja_caps, reasoning) : analyze_tools()); assistant_start = detect_assistant_start_marker(tmpl); user_start = detect_user_start_marker(tmpl); + tool_response_start = detect_tool_response_start_marker(tmpl); collect_preserved_tokens(); for (auto & workaround : workarounds) { @@ -242,6 +243,7 @@ void autoparser::analyze_template(const common_chat_template & tmpl) { LOG_DBG("\n--- Reasoning & Content Structure ---\n"); LOG_DBG("user_msg_start: %s\n", user_start.c_str()); LOG_DBG("assistant_msg_start: %s\n", assistant_start.c_str()); + LOG_DBG("tool_response_start: %s\n", tool_response_start.c_str()); LOG_DBG("reasoning_mode: %s\n", mode_to_str(reasoning.mode).c_str()); LOG_DBG("reasoning_start: '%s'\n", reasoning.start.c_str()); LOG_DBG("reasoning_end: '%s'\n", reasoning.end.c_str()); @@ -430,6 +432,48 @@ std::string autoparser::detect_user_start_marker(const common_chat_template & tm return trim_whitespace(result.str()); } +std::string autoparser::detect_tool_response_start_marker(const common_chat_template & tmpl) { + json tool_msg = json{ + { "role", "tool" }, + { "content", "T_TOOL_MSG" } + }; + + json assistant_no_reasoning = json{ + { "role", "assistant" }, + { "content", "A_ASST_MSG" } + }; + + template_params params; + params.messages = json::array({ assistant_no_reasoning }); + params.add_generation_prompt = false; + params.enable_thinking = true; + + auto comparison = compare_variants( + tmpl, params, [&](template_params & p) { + p.messages = json::array({ assistant_no_reasoning, tool_msg }); + } + ); + + if (!comparison) { + LOG_WRN(ANSI_ORANGE "%s: Template application failed, skipping tool start detection\n" ANSI_RESET, __func__); + return ""; + } + + auto usermsg = comparison->diff.right; + if (usermsg.find("T_TOOL_MSG") == std::string::npos) { + LOG_WRN(ANSI_ORANGE "%s: Did not find tool message in tool message block, skipping detection\n" ANSI_RESET, __func__); + } + + auto ast_prefix = usermsg.substr(0, usermsg.find("T_TOOL_MSG")); + if (!reasoning.start.empty() && ast_prefix.find(trim_whitespace(reasoning.start)) != std::string::npos) { + ast_prefix = ast_prefix.substr(0, ast_prefix.find(trim_whitespace(reasoning.start))); + } + if (!reasoning.end.empty() && ast_prefix.find(trim_whitespace(reasoning.end)) != std::string::npos) { + ast_prefix = ast_prefix.substr(0, ast_prefix.find(trim_whitespace(reasoning.end))); + } + return trim_whitespace(ast_prefix); +} + analyze_reasoning::analyze_reasoning(const common_chat_template & tmpl, bool supports_tools) : analyze_base(tmpl) { LOG_DBG(ANSI_PURPLE "=== Starting differential analysis ===\n" ANSI_RESET); diff --git a/common/chat.cpp b/common/chat.cpp index 22d2ee4a2a11..39224e146c8f 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -15,6 +15,7 @@ #include "nlohmann/json.hpp" +#include #include #include #include @@ -138,6 +139,11 @@ common_chat_msg_delimiters common_chat_msg_delimiters_parse(const json & delimit }); } + // Process delimiters from longest to shortest, so token groups matching multiple delimiters can be supported (specialization) + std::sort(result.delimiters.begin(), result.delimiters.end(), [](const common_chat_msg_delimiter & a, const common_chat_msg_delimiter & b) { + return a.delimiter.length() > b.delimiter.length(); + }); + return result; } @@ -2766,6 +2772,9 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_ if (!autoparser.user_start.empty()) { delimiters.add(COMMON_CHAT_ROLE_USER, autoparser.user_start); } + if (!autoparser.tool_response_start.empty()) { + delimiters.add(COMMON_CHAT_ROLE_TOOL, autoparser.tool_response_start); + } auto_params.message_delimiters = std::move(delimiters); diff --git a/common/chat.h b/common/chat.h index 7898f1623f54..ec47e6bf4c43 100644 --- a/common/chat.h +++ b/common/chat.h @@ -179,15 +179,6 @@ struct common_chat_msg_spans { } return false; } - - int32_t last_user_message_pos() const { - for (auto it = spans.rbegin(); it != spans.rend(); ++it) { - if (it->role == COMMON_CHAT_ROLE_USER) { - return (int32_t) it->pos; - } - } - return -1; - } }; struct common_chat_msg_delimiter { diff --git a/common/common.h b/common/common.h index bffc1767a7d1..b1fa7f96072b 100644 --- a/common/common.h +++ b/common/common.h @@ -623,7 +623,7 @@ struct common_params { bool cache_prompt = true; // whether to enable prompt caching bool cache_idle_slots = true; // save and clear idle slots upon starting a new task int32_t n_ctx_checkpoints = 32; // max number of context checkpoints per slot - int32_t checkpoint_min_step = 8192; // minimum spacing between context checkpoints + int32_t checkpoint_min_step = 256; // minimum spacing between context checkpoints int32_t cache_ram_mib = 8192; // -1 = no limit, 0 - disable, 1 = 1 MiB, etc. std::string hostname = "127.0.0.1"; diff --git a/tests/test-chat-auto-parser.cpp b/tests/test-chat-auto-parser.cpp index 78e42c65a50c..d9a669861f2f 100644 --- a/tests/test-chat-auto-parser.cpp +++ b/tests/test-chat-auto-parser.cpp @@ -1865,6 +1865,7 @@ struct role_marker_case { std::string template_file; std::string expected_user_start; std::string expected_assistant_start; + std::string expected_tool_response_start; }; static void test_role_markers_all_templates(testing & t) { @@ -1875,93 +1876,93 @@ static void test_role_markers_all_templates(testing & t) { // markers it detected first. const std::vector cases = { // ChatML family: <|im_start|>{role} ... <|im_end|> - { "Bielik-11B-v3.0-Instruct.jinja", "<|im_start|>user", "<|im_start|>assistant" }, - { "HuggingFaceTB-SmolLM3-3B.jinja", "<|im_start|>user", "<|im_start|>assistant" }, - { "MiMo-VL.jinja", "<|im_start|>user", "<|im_start|>assistant" }, - { "NousResearch-Hermes-2-Pro-Llama-3-8B-tool_use.jinja", "<|im_start|>user", "<|im_start|>assistant" }, - { "NousResearch-Hermes-3-Llama-3.1-8B-tool_use.jinja", "<|im_start|>user", "<|im_start|>assistant" }, - { "NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.jinja", "<|im_start|>user", "<|im_start|>assistant" }, - { "Qwen3.5-4B.jinja", "<|im_start|>user", "<|im_start|>assistant" }, - { "Qwen3-Coder.jinja", "<|im_start|>user", "<|im_start|>assistant" }, - { "Qwen-Qwen2.5-7B-Instruct.jinja", "<|im_start|>user", "<|im_start|>assistant" }, - { "Qwen-Qwen3-0.6B.jinja", "<|im_start|>user", "<|im_start|>assistant" }, - { "Qwen-QwQ-32B.jinja", "<|im_start|>user", "<|im_start|>assistant" }, - { "StepFun3.5-Flash.jinja", "<|im_start|>user", "<|im_start|>assistant" }, + { "Bielik-11B-v3.0-Instruct.jinja", "<|im_start|>user", "<|im_start|>assistant", "<|im_start|>user\n<|function_output|>" }, + { "HuggingFaceTB-SmolLM3-3B.jinja", "<|im_start|>user", "<|im_start|>assistant", "<|im_start|>user" }, + { "MiMo-VL.jinja", "<|im_start|>user", "<|im_start|>assistant", "<|im_start|>user\n" }, + { "NousResearch-Hermes-2-Pro-Llama-3-8B-tool_use.jinja", "<|im_start|>user", "<|im_start|>assistant", "<|im_start|>tool\n" }, + { "NousResearch-Hermes-3-Llama-3.1-8B-tool_use.jinja", "<|im_start|>user", "<|im_start|>assistant", "<|im_start|>tool\n" }, + { "NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.jinja", "<|im_start|>user", "<|im_start|>assistant", "<|im_start|>user\n" }, + { "Qwen3.5-4B.jinja", "<|im_start|>user", "<|im_start|>assistant", "" }, + { "Qwen3-Coder.jinja", "<|im_start|>user", "<|im_start|>assistant", "<|im_start|>user\n" }, + { "Qwen-Qwen2.5-7B-Instruct.jinja", "<|im_start|>user", "<|im_start|>assistant", "<|im_start|>user\n" }, + { "Qwen-Qwen3-0.6B.jinja", "<|im_start|>user", "<|im_start|>assistant", "<|im_start|>user\n" }, + { "Qwen-QwQ-32B.jinja", "<|im_start|>user", "<|im_start|>assistant", "<|im_start|>user\n" }, + { "StepFun3.5-Flash.jinja", "<|im_start|>user", "<|im_start|>assistant", "<|im_start|>tool_response\n" }, // DeepSeek family - { "deepseek-ai-DeepSeek-R1-Distill-Llama-8B.jinja", "<｜User｜>", "<｜Assistant｜>" }, - { "deepseek-ai-DeepSeek-R1-Distill-Qwen-32B.jinja", "<｜User｜>", "<｜Assistant｜>" }, - { "deepseek-ai-DeepSeek-V3.1.jinja", "<｜User｜>", "<｜Assistant｜>" }, - { "llama-cpp-deepseek-r1.jinja", "<｜User｜>", "<｜Assistant｜>" }, + { "deepseek-ai-DeepSeek-R1-Distill-Llama-8B.jinja", "<｜User｜>", "<｜Assistant｜>", "<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>" }, + { "deepseek-ai-DeepSeek-R1-Distill-Qwen-32B.jinja", "<｜User｜>", "<｜Assistant｜>", "<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>" }, + { "deepseek-ai-DeepSeek-V3.1.jinja", "<｜User｜>", "<｜Assistant｜>", "<｜tool▁output▁begin｜>" }, + { "llama-cpp-deepseek-r1.jinja", "<｜User｜>", "<｜Assistant｜>", "<｜tool▁outputs▁begin｜>\n<｜tool▁output▁begin｜>" }, // Llama 3 header family - { "meetkai-functionary-medium-v3.1.jinja", "<|start_header_id|>user<|end_header_id|>", "<|start_header_id|>assistant<|end_header_id|>" }, - { "meta-llama-Llama-3.1-8B-Instruct.jinja", "<|start_header_id|>user<|end_header_id|>", "<|start_header_id|>assistant<|end_header_id|>" }, - { "meta-llama-Llama-3.2-3B-Instruct.jinja", "<|start_header_id|>user<|end_header_id|>", "<|start_header_id|>assistant<|end_header_id|>" }, - { "meta-llama-Llama-3.3-70B-Instruct.jinja", "<|start_header_id|>user<|end_header_id|>", "<|start_header_id|>assistant<|end_header_id|>" }, + { "meetkai-functionary-medium-v3.1.jinja", "<|start_header_id|>user<|end_header_id|>", "<|start_header_id|>assistant<|end_header_id|>", "<|start_header_id|>ipython<|end_header_id|>" }, + { "meta-llama-Llama-3.1-8B-Instruct.jinja", "<|start_header_id|>user<|end_header_id|>", "<|start_header_id|>assistant<|end_header_id|>", "<|start_header_id|>ipython<|end_header_id|>\n\n\"" }, + { "meta-llama-Llama-3.2-3B-Instruct.jinja", "<|start_header_id|>user<|end_header_id|>", "<|start_header_id|>assistant<|end_header_id|>", "<|start_header_id|>ipython<|end_header_id|>\n\n\"" }, + { "meta-llama-Llama-3.3-70B-Instruct.jinja", "<|start_header_id|>user<|end_header_id|>", "<|start_header_id|>assistant<|end_header_id|>", "<|start_header_id|>ipython<|end_header_id|>\n\n\"" }, // fireworks-ai forces a trailing assistant header even without add_generation_prompt, // so the marker is absorbed into the common suffix and assistant_start is detected as empty. - { "fireworks-ai-llama-3-firefunction-v2.jinja", "<|start_header_id|>user<|end_header_id|>", "<|start_header_id|>assistant<|end_header_id|>" }, + { "fireworks-ai-llama-3-firefunction-v2.jinja", "<|start_header_id|>user<|end_header_id|>", "<|start_header_id|>assistant<|end_header_id|>", "<|eot_id|><|start_header_id|>tool<|end_header_id|>" }, // Phi/GLM/Apriel-style: <|user|> / <|assistant|> - { "microsoft-Phi-3.5-mini-instruct.jinja", "<|user|>", "<|assistant|>" }, - { "GLM-4.6.jinja", "<|user|>", "<|assistant|>" }, - { "unsloth-Apriel-1.5.jinja", "<|user|>", "<|assistant|>" }, - { "GLM-4.7-Flash.jinja", "<|user|>", "<|assistant|>" }, + { "microsoft-Phi-3.5-mini-instruct.jinja", "<|user|>", "<|assistant|>", "" }, + { "GLM-4.6.jinja", "<|user|>", "<|assistant|>", "<|observation|>\n" }, + { "unsloth-Apriel-1.5.jinja", "<|user|>", "<|assistant|>", "<|tool_result|>" }, + { "GLM-4.7-Flash.jinja", "<|user|>", "<|assistant|>", "<|observation|>" }, // Gemma 2: {user|model} - { "google-gemma-2-2b-it.jinja", "user", "model" }, + { "google-gemma-2-2b-it.jinja", "user", "model", "" }, // IBM Granite - { "ibm-granite-granite-3.3-2B-Instruct.jinja", "<|start_of_role|>user<|end_of_role|>", "<|start_of_role|>assistant<|end_of_role|>" }, - { "ibm-granite-granite-4.0.jinja", "<|start_of_role|>user<|end_of_role|>", "<|start_of_role|>assistant<|end_of_role|>" }, + { "ibm-granite-granite-3.3-2B-Instruct.jinja", "<|start_of_role|>user<|end_of_role|>", "<|start_of_role|>assistant<|end_of_role|>", "<|start_of_role|>tool<|end_of_role|>" }, + { "ibm-granite-granite-4.0.jinja", "<|start_of_role|>user<|end_of_role|>", "<|start_of_role|>assistant<|end_of_role|>", "<|start_of_role|>user<|end_of_role|>\n" }, // Cohere R-series { "CohereForAI-c4ai-command-r7b-12-2024-tool_use.jinja", - "<|START_OF_TURN_TOKEN|><|USER_TOKEN|>", "<|START_RESPONSE|>" }, + "<|START_OF_TURN_TOKEN|><|USER_TOKEN|>", "<|START_RESPONSE|>" , "" }, { "CohereForAI-c4ai-command-r-plus-tool_use.jinja", - "<|START_OF_TURN_TOKEN|><|USER_TOKEN|>", "<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>" }, + "<|START_OF_TURN_TOKEN|><|USER_TOKEN|>", "<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>" , "<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>" }, // Mistral: assistant content follows [/INST] immediately, no header - { "mistralai-Mistral-Nemo-Instruct-2407.jinja", "[INST]", "" }, - { "Mistral-Small-3.2-24B-Instruct-2506.jinja", "[INST]", "" }, + { "mistralai-Mistral-Nemo-Instruct-2407.jinja", "[INST]", "", "" }, + { "Mistral-Small-3.2-24B-Instruct-2506.jinja", "[INST]", "", "" }, // Apertus uses <|user_start|> / <|assistant_start|> but the user diff // carries the preceding <|assistant_end|> from the previous turn. - { "Apertus-8B-Instruct.jinja", "<|user_start|>", "<|assistant_start|>" }, + { "Apertus-8B-Instruct.jinja", "<|user_start|>", "<|assistant_start|>", "[" }, // Apriel 1.6 wraps the assistant body with <|begin_assistant|>, but // <|begin_assistant|> is also the detected reasoning start, so the // assistant_start is trimmed back to the preceding newline. - { "Apriel-1.6-15b-Thinker-fixed.jinja", "<|begin_user|>", "<|begin_assistant|>" }, + { "Apriel-1.6-15b-Thinker-fixed.jinja", "<|begin_user|>", "<|begin_assistant|>", "<|end|>\n<|begin_tool_result|>" }, // ByteDance Seed-OSS: {role} - { "ByteDance-Seed-OSS.jinja", "user", "assistant" }, + { "ByteDance-Seed-OSS.jinja", "user", "assistant", "tool" }, // GigaChat 3.1: {role}<|role_sep|> - { "GigaChat3.1-10B-A1.8B.jinja", "user<|role_sep|>", "assistant<|role_sep|>" }, + { "GigaChat3.1-10B-A1.8B.jinja", "user<|role_sep|>", "assistant<|role_sep|>", "function result<|role_sep|>" }, // MiniMax M2: ]~b]{user|ai} - { "MiniMax-M2.jinja", "]~b]user", "]~b]ai" }, + { "MiniMax-M2.jinja", "]~b]user", "]~b]ai", "" }, // HunYuan V3: <｜hy_User:opensource｜> / <｜hy_Assistant:opensource｜> { "tencent-Hy3.jinja", "<｜hy_User:opensource｜>", "<｜hy_Assistant:opensource｜>" }, // Nemotron Nano v2: {User|Assistant}; assistant marker // is followed by a prefilled block that gets included. - { "NVIDIA-Nemotron-Nano-v2.jinja", "User", "Assistant" }, + { "NVIDIA-Nemotron-Nano-v2.jinja", "User", "Assistant", "A_ASST_MSG\n\nUser\n[" }, // Reka Edge: "human: " / "assistant: " — but the rendered preamble // depends on enable_thinking, which currently confuses the user-start // diff and trims the marker down. Lock in the observed value. - { "Reka-Edge.jinja", "human:", "assistant:" }, + { "Reka-Edge.jinja", "human:", "assistant:", "human: " }, // RWKV-world chat preset: "User: " / "Assistant: " - { "llama-cpp-rwkv-world.jinja", "User:", "Assistant:" }, + { "llama-cpp-rwkv-world.jinja", "User:", "Assistant:", "" }, // Upstage Solar 100B: <|begin|>{role}... but reasoning marker absorbs // the "<|begin|>assistant" prefix from assistant_start. - { "upstage-Solar-Open-100B.jinja", "<|begin|>user<|content|>", "<|begin|>assistant" }, + { "upstage-Solar-Open-100B.jinja", "<|begin|>user<|content|>", "<|begin|>assistant", "<|end|><|begin|>tool<|tool_response|><|tool_response:begin|><|tool_response:name|><|tool_response:result|>" }, }; for (const auto & c : cases) { @@ -1971,6 +1972,7 @@ static void test_role_markers_all_templates(testing & t) { ap.analyze_template(tmpl); t.assert_equal("user_start", c.expected_user_start, ap.user_start); t.assert_equal("assistant_start", c.expected_assistant_start, ap.assistant_start); + t.assert_equal("tool_response_start", c.expected_tool_response_start, ap.tool_response_start); }); } }