diff --git a/common/chat-auto-parser.h b/common/chat-auto-parser.h index 9e8113f2442..e893b3024d5 100644 --- a/common/chat-auto-parser.h +++ b/common/chat-auto-parser.h @@ -380,6 +380,7 @@ struct autoparser { jinja::caps jinja_caps; std::string user_start; std::string assistant_start; + std::string tool_response_start; analyze_reasoning reasoning; analyze_content content; analyze_tools tools; @@ -393,6 +394,7 @@ struct autoparser { // Find the starting marker for the user message and assistant message std::string detect_user_start_marker(const common_chat_template & tmpl); std::string detect_assistant_start_marker(const common_chat_template & tmpl); + std::string detect_tool_response_start_marker(const common_chat_template & tmpl); // Run full differential analysis on a template void analyze_template(const common_chat_template & tmpl); diff --git a/common/chat-diff-analyzer.cpp b/common/chat-diff-analyzer.cpp index b166ee5a18f..7539cafc001 100644 --- a/common/chat-diff-analyzer.cpp +++ b/common/chat-diff-analyzer.cpp @@ -27,6 +27,7 @@ static const std::string ARG_FIRST = "AA_ARG_FST_AA"; static const std::string ARG_SECOND = "BB_ARG_SND_BB"; static const std::string USER_MSG = "U_USER_MSG Hello END_U"; static const std::string USER_MSG_TWO = "V_USER_MSG Hello END_V"; +static const std::string TOOL_MSG = "T_TOOL_MSG Tool response END_T"; static const std::string ASSISTANT_MSG = "A_ASST_MSG I can help END_A"; static const std::string THINKING_CONTENT = "REASON_PART I am thinking END_R"; static const std::string CALL_ID_001 = "call00001"; @@ -233,6 +234,7 @@ void autoparser::analyze_template(const common_chat_template & tmpl) { tools = analyze_tools(jinja_caps.supports_tool_calls ? analyze_tools(tmpl, jinja_caps, reasoning) : analyze_tools()); assistant_start = detect_assistant_start_marker(tmpl); user_start = detect_user_start_marker(tmpl); + tool_response_start = detect_tool_response_start_marker(tmpl); collect_preserved_tokens(); for (auto & workaround : workarounds) { @@ -242,6 +244,7 @@ void autoparser::analyze_template(const common_chat_template & tmpl) { LOG_DBG("\n--- Reasoning & Content Structure ---\n"); LOG_DBG("user_msg_start: %s\n", user_start.c_str()); LOG_DBG("assistant_msg_start: %s\n", assistant_start.c_str()); + LOG_DBG("tool_response_start: %s\n", tool_response_start.c_str()); LOG_DBG("reasoning_mode: %s\n", mode_to_str(reasoning.mode).c_str()); LOG_DBG("reasoning_start: '%s'\n", reasoning.start.c_str()); LOG_DBG("reasoning_end: '%s'\n", reasoning.end.c_str()); @@ -428,6 +431,42 @@ std::string autoparser::detect_user_start_marker(const common_chat_template & tm return trim_whitespace(result.str()); } +std::string autoparser::detect_tool_response_start_marker(const common_chat_template & tmpl) { + json tool_msg = json{ + { "role", "tool" }, + { "content", TOOL_MSG } + }; + + json assistant_no_reasoning = json{ + { "role", "assistant" }, + { "content", ASSISTANT_MSG } + }; + + template_params params; + params.messages = json::array({ assistant_no_reasoning }); + params.add_generation_prompt = false; + params.enable_thinking = true; + + auto comparison = compare_variants( + tmpl, params, [&](template_params & p) { + p.messages = json::array({ assistant_no_reasoning, tool_msg }); + } + ); + + if (!comparison) { + LOG_WRN(ANSI_ORANGE "%s: Template application failed, skipping tool start detection\n" ANSI_RESET, __func__); + return ""; + } + + auto usermsg = comparison->diff.right; + if (usermsg.find(TOOL_MSG) == std::string::npos) { + LOG_WRN(ANSI_ORANGE "%s: Did not find tool message in tool message block, skipping detection\n" ANSI_RESET, __func__); + } + + auto ast_prefix = usermsg.substr(0, usermsg.find(TOOL_MSG)); + return trim_whitespace(ast_prefix); +} + analyze_reasoning::analyze_reasoning(const common_chat_template & tmpl, bool supports_tools) : analyze_base(tmpl) { LOG_DBG(ANSI_PURPLE "=== Starting differential analysis ===\n" ANSI_RESET); diff --git a/common/chat.cpp b/common/chat.cpp index 22d2ee4a2a1..b8d93a90d15 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -138,6 +138,11 @@ common_chat_msg_delimiters common_chat_msg_delimiters_parse(const json & delimit }); } + // Process delimiters from longest to shortest, so token groups matching multiple delimiters can be supported (specialization) + sort(result.delimiters.begin(), result.delimiters.end(), [](common_chat_msg_delimiter a, common_chat_msg_delimiter b) { + return ( a.delimiter.length() > b.delimiter.length()); + }); + return result; } @@ -2766,6 +2771,9 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_ if (!autoparser.user_start.empty()) { delimiters.add(COMMON_CHAT_ROLE_USER, autoparser.user_start); } + if (!autoparser.tool_response_start.empty()) { + delimiters.add(COMMON_CHAT_ROLE_TOOL, autoparser.tool_response_start); + } auto_params.message_delimiters = std::move(delimiters); diff --git a/common/chat.h b/common/chat.h index 7898f1623f5..ec47e6bf4c4 100644 --- a/common/chat.h +++ b/common/chat.h @@ -179,15 +179,6 @@ struct common_chat_msg_spans { } return false; } - - int32_t last_user_message_pos() const { - for (auto it = spans.rbegin(); it != spans.rend(); ++it) { - if (it->role == COMMON_CHAT_ROLE_USER) { - return (int32_t) it->pos; - } - } - return -1; - } }; struct common_chat_msg_delimiter { diff --git a/common/common.h b/common/common.h index b7d85b5dd1d..b04154db01a 100644 --- a/common/common.h +++ b/common/common.h @@ -622,7 +622,7 @@ struct common_params { bool cache_prompt = true; // whether to enable prompt caching bool cache_idle_slots = true; // save and clear idle slots upon starting a new task int32_t n_ctx_checkpoints = 32; // max number of context checkpoints per slot - int32_t checkpoint_min_step = 8192; // minimum spacing between context checkpoints + int32_t checkpoint_min_step = 256; // minimum spacing between context checkpoints int32_t cache_ram_mib = 8192; // -1 = no limit, 0 - disable, 1 = 1 MiB, etc. std::string hostname = "127.0.0.1"; diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index df566658154..8288e98f003 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -3406,7 +3406,6 @@ struct server_context_impl { } const auto & spans = slot.task->params.message_spans; - const auto last_user_pos = spans.last_user_message_pos(); // add prompt tokens for processing in the current batch while (slot.prompt.n_tokens() < slot.task->n_tokens() && batch.size() < n_batch) { @@ -3470,7 +3469,6 @@ struct server_context_impl { const bool near_prompt_end = slot.task->n_tokens() < slot.prompt.n_tokens() + n_ubatch; const bool is_user_start = spans.is_user_start(n_tokens_start); - const bool is_last_user_message = n_tokens_start == last_user_pos; // entire prompt has been processed if (slot.prompt.n_tokens() == slot.task->n_tokens()) { @@ -3505,8 +3503,8 @@ struct server_context_impl { // do not checkpoint after mtmd chunks do_checkpoint = do_checkpoint && !has_mtmd; - // no need to create checkpoints that are too close together, unless it's the last user message - do_checkpoint = do_checkpoint && (slot.prompt.checkpoints.empty() || is_last_user_message || n_tokens_start > slot.prompt.checkpoints.back().n_tokens + params_base.checkpoint_min_step); + // no need to create checkpoints that are too close together + do_checkpoint = do_checkpoint && (slot.prompt.checkpoints.empty() || n_tokens_start > slot.prompt.checkpoints.back().n_tokens + params_base.checkpoint_min_step); SLT_DBG(slot, "main/do_checkpoint = %s, pos_min = %d, pos_max = %d\n", do_checkpoint ? "yes" : "no", pos_min, pos_max); // note: we create the checkpoint before calling llama_decode(), so the current batch is not