server: tool role detection and create checkpoints only on user messages

common/chat-auto-parser.h

@@ -380,6 +380,7 @@ struct autoparser {
     jinja::caps          jinja_caps;
     std::string          user_start;
     std::string          assistant_start;
+    std::string          tool_response_start;
     analyze_reasoning    reasoning;
     analyze_content      content;
     analyze_tools        tools;
@@ -393,6 +394,7 @@ struct autoparser {
     // Find the starting marker for the user message and assistant message
     std::string detect_user_start_marker(const common_chat_template & tmpl);
     std::string detect_assistant_start_marker(const common_chat_template & tmpl);
+    std::string detect_tool_response_start_marker(const common_chat_template & tmpl);
 
     // Run full differential analysis on a template
     void analyze_template(const common_chat_template & tmpl);

common/chat-diff-analyzer.cpp

@@ -27,6 +27,7 @@ static const std::string ARG_FIRST = "AA_ARG_FST_AA";
 static const std::string ARG_SECOND = "BB_ARG_SND_BB";
 static const std::string USER_MSG = "U_USER_MSG Hello END_U";
 static const std::string USER_MSG_TWO = "V_USER_MSG Hello END_V";
+static const std::string TOOL_MSG = "T_TOOL_MSG Tool response END_T";
 static const std::string ASSISTANT_MSG = "A_ASST_MSG I can help END_A";
 static const std::string THINKING_CONTENT = "REASON_PART I am thinking END_R";
 static const std::string CALL_ID_001 = "call00001";
@@ -233,6 +234,7 @@ void autoparser::analyze_template(const common_chat_template & tmpl) {
     tools = analyze_tools(jinja_caps.supports_tool_calls ? analyze_tools(tmpl, jinja_caps, reasoning) : analyze_tools());
     assistant_start = detect_assistant_start_marker(tmpl);
     user_start = detect_user_start_marker(tmpl);
+    tool_response_start = detect_tool_response_start_marker(tmpl);
     collect_preserved_tokens();
 
     for (auto & workaround : workarounds) {
@@ -242,6 +244,7 @@ void autoparser::analyze_template(const common_chat_template & tmpl) {
     LOG_DBG("\n--- Reasoning & Content Structure ---\n");
     LOG_DBG("user_msg_start: %s\n", user_start.c_str());
     LOG_DBG("assistant_msg_start: %s\n", assistant_start.c_str());
+    LOG_DBG("tool_response_start: %s\n", tool_response_start.c_str());
     LOG_DBG("reasoning_mode: %s\n", mode_to_str(reasoning.mode).c_str());
     LOG_DBG("reasoning_start: '%s'\n", reasoning.start.c_str());
     LOG_DBG("reasoning_end: '%s'\n", reasoning.end.c_str());
@@ -428,6 +431,42 @@ std::string autoparser::detect_user_start_marker(const common_chat_template & tm
     return trim_whitespace(result.str());
 }
 
+std::string autoparser::detect_tool_response_start_marker(const common_chat_template & tmpl) {
+    json tool_msg = json{
+        { "role",    "tool"   },
+        { "content", TOOL_MSG }
+    };
+
+    json assistant_no_reasoning = json{
+        { "role",    "assistant"   },
+        { "content", ASSISTANT_MSG }
+    };
+
+    template_params params;
+    params.messages              = json::array({ assistant_no_reasoning });
+    params.add_generation_prompt = false;
+    params.enable_thinking       = true;
+
+    auto comparison = compare_variants(
+        tmpl, params, [&](template_params & p) {
+            p.messages = json::array({ assistant_no_reasoning, tool_msg });
+        }
+    );
+
+    if (!comparison) {
+        LOG_WRN(ANSI_ORANGE "%s: Template application failed, skipping tool start detection\n" ANSI_RESET, __func__);
+        return "";
+    }
+
+    auto usermsg = comparison->diff.right;
+    if (usermsg.find(TOOL_MSG) == std::string::npos) {
+        LOG_WRN(ANSI_ORANGE "%s: Did not find tool message in tool message block, skipping detection\n" ANSI_RESET, __func__);
+    }
+
+    auto ast_prefix = usermsg.substr(0, usermsg.find(TOOL_MSG));
+    return trim_whitespace(ast_prefix);
+}
+
 analyze_reasoning::analyze_reasoning(const common_chat_template & tmpl, bool supports_tools)
     : analyze_base(tmpl) {
     LOG_DBG(ANSI_PURPLE "=== Starting differential analysis ===\n" ANSI_RESET);

common/chat.cpp

@@ -138,6 +138,11 @@ common_chat_msg_delimiters common_chat_msg_delimiters_parse(const json & delimit
         });
     }
 
+    // Process delimiters from longest to shortest, so token groups matching multiple delimiters can be supported (specialization)
+    sort(result.delimiters.begin(), result.delimiters.end(), [](common_chat_msg_delimiter a, common_chat_msg_delimiter b) {
+        return ( a.delimiter.length() > b.delimiter.length());
+    });
+
     return result;
 }
 
@@ -2766,6 +2771,9 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_
         if (!autoparser.user_start.empty()) {
             delimiters.add(COMMON_CHAT_ROLE_USER, autoparser.user_start);
         }
+        if (!autoparser.tool_response_start.empty()) {
+            delimiters.add(COMMON_CHAT_ROLE_TOOL, autoparser.tool_response_start);
+        }
 
         auto_params.message_delimiters = std::move(delimiters);
 

common/chat.h

@@ -179,15 +179,6 @@ struct common_chat_msg_spans {
         }
         return false;
     }
-
-    int32_t last_user_message_pos() const {
-        for (auto it = spans.rbegin(); it != spans.rend(); ++it) {
-            if (it->role == COMMON_CHAT_ROLE_USER) {
-                return (int32_t) it->pos;
-            }
-        }
-        return -1;
-    }
 };
 
 struct common_chat_msg_delimiter {

common/common.h

@@ -622,7 +622,7 @@ struct common_params {
     bool    cache_prompt        = true;  // whether to enable prompt caching
     bool    cache_idle_slots    = true;  // save and clear idle slots upon starting a new task
     int32_t n_ctx_checkpoints   = 32;    // max number of context checkpoints per slot
-    int32_t checkpoint_min_step = 8192;  // minimum spacing between context checkpoints
+    int32_t checkpoint_min_step = 256;  // minimum spacing between context checkpoints
     int32_t cache_ram_mib       = 8192;  // -1 = no limit, 0 - disable, 1 = 1 MiB, etc.
 
     std::string hostname      = "127.0.0.1";

tools/server/server-context.cpp

@@ -3406,7 +3406,6 @@ struct server_context_impl {
                     }
 
                     const auto & spans = slot.task->params.message_spans;
-                    const auto last_user_pos = spans.last_user_message_pos();
 
                     // add prompt tokens for processing in the current batch
                     while (slot.prompt.n_tokens() < slot.task->n_tokens() && batch.size() < n_batch) {
@@ -3470,7 +3469,6 @@ struct server_context_impl {
                     const bool near_prompt_end = slot.task->n_tokens() < slot.prompt.n_tokens() + n_ubatch;
 
                     const bool is_user_start = spans.is_user_start(n_tokens_start);
-                    const bool is_last_user_message = n_tokens_start == last_user_pos;
 
                     // entire prompt has been processed
                     if (slot.prompt.n_tokens() == slot.task->n_tokens()) {
@@ -3505,8 +3503,8 @@ struct server_context_impl {
                     // do not checkpoint after mtmd chunks
                     do_checkpoint = do_checkpoint && !has_mtmd;
 
-                    // no need to create checkpoints that are too close together, unless it's the last user message
-                    do_checkpoint = do_checkpoint && (slot.prompt.checkpoints.empty() || is_last_user_message || n_tokens_start > slot.prompt.checkpoints.back().n_tokens + params_base.checkpoint_min_step);
+                    // no need to create checkpoints that are too close together
+                    do_checkpoint = do_checkpoint && (slot.prompt.checkpoints.empty() || n_tokens_start > slot.prompt.checkpoints.back().n_tokens + params_base.checkpoint_min_step);
                     SLT_DBG(slot, "main/do_checkpoint = %s, pos_min = %d, pos_max = %d\n", do_checkpoint ? "yes" : "no", pos_min, pos_max);
 
                     // note: we create the checkpoint before calling llama_decode(), so the current batch is not