Skip to content

server: tool role detection and create checkpoints only on user messages #25416

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Kononnable wants to merge 3 commits into from
Closed

server: tool role detection and create checkpoints only on user messages #25416

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
049c94c
recognize tool response messages
Kononnable Jul 7, 2026
f23745b
revert: cont : increase min-step to 8192 and always produce a chkpt f…
Kononnable Jul 7, 2026
e3e9abf
remove redundant checks from detect_tool_response_start_marker
Kononnable Jul 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions common/chat-auto-parser.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -380,6 +380,7 @@ struct autoparser {
jinja::caps jinja_caps;
std::string user_start;
std::string assistant_start;
std::string tool_response_start;
analyze_reasoning reasoning;
analyze_content content;
analyze_tools tools;
Expand All @@ -393,6 +394,7 @@ struct autoparser {
// Find the starting marker for the user message and assistant message
std::string detect_user_start_marker(const common_chat_template & tmpl);
std::string detect_assistant_start_marker(const common_chat_template & tmpl);
std::string detect_tool_response_start_marker(const common_chat_template & tmpl);

// Run full differential analysis on a template
void analyze_template(const common_chat_template & tmpl);
Expand Down
45 changes: 45 additions & 0 deletions common/chat-diff-analyzer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ static const std::string ARG_FIRST = "AA_ARG_FST_AA";
static const std::string ARG_SECOND = "BB_ARG_SND_BB";
static const std::string USER_MSG = "U_USER_MSG Hello END_U";
static const std::string USER_MSG_TWO = "V_USER_MSG Hello END_V";
static const std::string TOOL_MSG = "T_TOOL_MSG Tool response END_T";
static const std::string ASSISTANT_MSG = "A_ASST_MSG I can help END_A";
static const std::string THINKING_CONTENT = "REASON_PART I am thinking END_R";
static const std::string CALL_ID_001 = "call00001";
Expand Down Expand Up @@ -233,6 +234,7 @@ void autoparser::analyze_template(const common_chat_template & tmpl) {
tools = analyze_tools(jinja_caps.supports_tool_calls ? analyze_tools(tmpl, jinja_caps, reasoning) : analyze_tools());
assistant_start = detect_assistant_start_marker(tmpl);
user_start = detect_user_start_marker(tmpl);
tool_response_start = detect_tool_response_start_marker(tmpl);
collect_preserved_tokens();

for (auto & workaround : workarounds) {
Expand All @@ -242,6 +244,7 @@ void autoparser::analyze_template(const common_chat_template & tmpl) {
LOG_DBG("\n--- Reasoning & Content Structure ---\n");
LOG_DBG("user_msg_start: %s\n", user_start.c_str());
LOG_DBG("assistant_msg_start: %s\n", assistant_start.c_str());
LOG_DBG("tool_response_start: %s\n", tool_response_start.c_str());
LOG_DBG("reasoning_mode: %s\n", mode_to_str(reasoning.mode).c_str());
LOG_DBG("reasoning_start: '%s'\n", reasoning.start.c_str());
LOG_DBG("reasoning_end: '%s'\n", reasoning.end.c_str());
Expand Down Expand Up @@ -428,6 +431,48 @@ std::string autoparser::detect_user_start_marker(const common_chat_template & tm
return trim_whitespace(result.str());
}

std::string autoparser::detect_tool_response_start_marker(const common_chat_template & tmpl) {
json tool_msg = json{
{ "role", "tool" },
{ "content", TOOL_MSG }
};

json assistant_no_reasoning = json{
{ "role", "assistant" },
{ "content", ASSISTANT_MSG }
};

template_params params;
params.messages = json::array({ assistant_no_reasoning });
params.add_generation_prompt = false;
params.enable_thinking = true;

auto comparison = compare_variants(
tmpl, params, [&](template_params & p) {
p.messages = json::array({ assistant_no_reasoning, tool_msg });
}
);

if (!comparison) {
LOG_WRN(ANSI_ORANGE "%s: Template application failed, skipping tool start detection\n" ANSI_RESET, __func__);
return "";
}

auto usermsg = comparison->diff.right;
if (usermsg.find(TOOL_MSG) == std::string::npos) {
LOG_WRN(ANSI_ORANGE "%s: Did not find tool message in tool message block, skipping detection\n" ANSI_RESET, __func__);
}

auto ast_prefix = usermsg.substr(0, usermsg.find(TOOL_MSG));
if (!reasoning.start.empty() && ast_prefix.find(trim_whitespace(reasoning.start)) != std::string::npos) {
ast_prefix = ast_prefix.substr(0, ast_prefix.find(trim_whitespace(reasoning.start)));
Comment thread
Kononnable marked this conversation as resolved.
Outdated
}
if (!reasoning.end.empty() && ast_prefix.find(trim_whitespace(reasoning.end)) != std::string::npos) {
ast_prefix = ast_prefix.substr(0, ast_prefix.find(trim_whitespace(reasoning.end)));
Comment thread
Kononnable marked this conversation as resolved.
Outdated
}
return trim_whitespace(ast_prefix);
}

analyze_reasoning::analyze_reasoning(const common_chat_template & tmpl, bool supports_tools)
: analyze_base(tmpl) {
LOG_DBG(ANSI_PURPLE "=== Starting differential analysis ===\n" ANSI_RESET);
Expand Down
8 changes: 8 additions & 0 deletions common/chat.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,11 @@ common_chat_msg_delimiters common_chat_msg_delimiters_parse(const json & delimit
});
}

// Process delimiters from longest to shortest, so token groups matching multiple delimiters can be supported (specialization)
sort(result.delimiters.begin(), result.delimiters.end(), [](common_chat_msg_delimiter a, common_chat_msg_delimiter b) {
return ( a.delimiter.length() > b.delimiter.length());
});

return result;
}

Expand Down Expand Up @@ -2766,6 +2771,9 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_
if (!autoparser.user_start.empty()) {
delimiters.add(COMMON_CHAT_ROLE_USER, autoparser.user_start);
}
if (!autoparser.tool_response_start.empty()) {
delimiters.add(COMMON_CHAT_ROLE_TOOL, autoparser.tool_response_start);
}

auto_params.message_delimiters = std::move(delimiters);

Expand Down
9 changes: 0 additions & 9 deletions common/chat.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -179,15 +179,6 @@ struct common_chat_msg_spans {
}
return false;
}

int32_t last_user_message_pos() const {
for (auto it = spans.rbegin(); it != spans.rend(); ++it) {
if (it->role == COMMON_CHAT_ROLE_USER) {
return (int32_t) it->pos;
}
}
return -1;
}
};

struct common_chat_msg_delimiter {
Expand Down
2 changes: 1 addition & 1 deletion common/common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -622,7 +622,7 @@ struct common_params {
bool cache_prompt = true; // whether to enable prompt caching
bool cache_idle_slots = true; // save and clear idle slots upon starting a new task
int32_t n_ctx_checkpoints = 32; // max number of context checkpoints per slot
int32_t checkpoint_min_step = 8192; // minimum spacing between context checkpoints
int32_t checkpoint_min_step = 256; // minimum spacing between context checkpoints
int32_t cache_ram_mib = 8192; // -1 = no limit, 0 - disable, 1 = 1 MiB, etc.

std::string hostname = "127.0.0.1";
Expand Down
6 changes: 2 additions & 4 deletions tools/server/server-context.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3406,7 +3406,6 @@ struct server_context_impl {
}

const auto & spans = slot.task->params.message_spans;
const auto last_user_pos = spans.last_user_message_pos();

// add prompt tokens for processing in the current batch
while (slot.prompt.n_tokens() < slot.task->n_tokens() && batch.size() < n_batch) {
Expand Down Expand Up @@ -3470,7 +3469,6 @@ struct server_context_impl {
const bool near_prompt_end = slot.task->n_tokens() < slot.prompt.n_tokens() + n_ubatch;

const bool is_user_start = spans.is_user_start(n_tokens_start);
const bool is_last_user_message = n_tokens_start == last_user_pos;

// entire prompt has been processed
if (slot.prompt.n_tokens() == slot.task->n_tokens()) {
Expand Down Expand Up @@ -3505,8 +3503,8 @@ struct server_context_impl {
// do not checkpoint after mtmd chunks
do_checkpoint = do_checkpoint && !has_mtmd;

// no need to create checkpoints that are too close together, unless it's the last user message
do_checkpoint = do_checkpoint && (slot.prompt.checkpoints.empty() || is_last_user_message || n_tokens_start > slot.prompt.checkpoints.back().n_tokens + params_base.checkpoint_min_step);
// no need to create checkpoints that are too close together
do_checkpoint = do_checkpoint && (slot.prompt.checkpoints.empty() || n_tokens_start > slot.prompt.checkpoints.back().n_tokens + params_base.checkpoint_min_step);
SLT_DBG(slot, "main/do_checkpoint = %s, pos_min = %d, pos_max = %d\n", do_checkpoint ? "yes" : "no", pos_min, pos_max);

// note: we create the checkpoint before calling llama_decode(), so the current batch is not
Expand Down