Skip to content

server: fix prefill speed regression on session resumption #25417

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
SakshamKapoor2911 wants to merge 1 commit into from
Closed

server: fix prefill speed regression on session resumption #25417

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions common/chat-auto-parser.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -381,6 +381,7 @@ struct autoparser {
jinja::caps jinja_caps;
std::string user_start;
std::string assistant_start;
std::string tool_response_start;
analyze_reasoning reasoning;
analyze_content content;
analyze_tools tools;
Expand All @@ -394,6 +395,7 @@ struct autoparser {
// Find the starting marker for the user message and assistant message
std::string detect_user_start_marker(const common_chat_template & tmpl);
std::string detect_assistant_start_marker(const common_chat_template & tmpl);
std::string detect_tool_response_start_marker(const common_chat_template & tmpl);

// Run full differential analysis on a template
void analyze_template(const common_chat_template & tmpl);
Expand Down
44 changes: 44 additions & 0 deletions common/chat-diff-analyzer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -233,6 +233,7 @@ void autoparser::analyze_template(const common_chat_template & tmpl) {
tools = analyze_tools(jinja_caps.supports_tool_calls ? analyze_tools(tmpl, jinja_caps, reasoning) : analyze_tools());
assistant_start = detect_assistant_start_marker(tmpl);
user_start = detect_user_start_marker(tmpl);
tool_response_start = detect_tool_response_start_marker(tmpl);
collect_preserved_tokens();

for (auto & workaround : workarounds) {
Expand All @@ -242,6 +243,7 @@ void autoparser::analyze_template(const common_chat_template & tmpl) {
LOG_DBG("\n--- Reasoning & Content Structure ---\n");
LOG_DBG("user_msg_start: %s\n", user_start.c_str());
LOG_DBG("assistant_msg_start: %s\n", assistant_start.c_str());
LOG_DBG("tool_response_start: %s\n", tool_response_start.c_str());
LOG_DBG("reasoning_mode: %s\n", mode_to_str(reasoning.mode).c_str());
LOG_DBG("reasoning_start: '%s'\n", reasoning.start.c_str());
LOG_DBG("reasoning_end: '%s'\n", reasoning.end.c_str());
Expand Down Expand Up @@ -430,6 +432,48 @@ std::string autoparser::detect_user_start_marker(const common_chat_template & tm
return trim_whitespace(result.str());
}

std::string autoparser::detect_tool_response_start_marker(const common_chat_template & tmpl) {
json tool_msg = json{
{ "role", "tool" },
{ "content", "T_TOOL_MSG" }
};

json assistant_no_reasoning = json{
{ "role", "assistant" },
{ "content", "A_ASST_MSG" }
};

template_params params;
params.messages = json::array({ assistant_no_reasoning });
params.add_generation_prompt = false;
params.enable_thinking = true;

auto comparison = compare_variants(
tmpl, params, [&](template_params & p) {
p.messages = json::array({ assistant_no_reasoning, tool_msg });
}
);

if (!comparison) {
LOG_WRN(ANSI_ORANGE "%s: Template application failed, skipping tool start detection\n" ANSI_RESET, __func__);
return "";
}

auto usermsg = comparison->diff.right;
if (usermsg.find("T_TOOL_MSG") == std::string::npos) {
LOG_WRN(ANSI_ORANGE "%s: Did not find tool message in tool message block, skipping detection\n" ANSI_RESET, __func__);
}

auto ast_prefix = usermsg.substr(0, usermsg.find("T_TOOL_MSG"));
if (!reasoning.start.empty() && ast_prefix.find(trim_whitespace(reasoning.start)) != std::string::npos) {
ast_prefix = ast_prefix.substr(0, ast_prefix.find(trim_whitespace(reasoning.start)));
}
if (!reasoning.end.empty() && ast_prefix.find(trim_whitespace(reasoning.end)) != std::string::npos) {
ast_prefix = ast_prefix.substr(0, ast_prefix.find(trim_whitespace(reasoning.end)));
}
return trim_whitespace(ast_prefix);
}

analyze_reasoning::analyze_reasoning(const common_chat_template & tmpl, bool supports_tools)
: analyze_base(tmpl) {
LOG_DBG(ANSI_PURPLE "=== Starting differential analysis ===\n" ANSI_RESET);
Expand Down
9 changes: 9 additions & 0 deletions common/chat.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

#include "nlohmann/json.hpp"

#include <algorithm>
#include <cstdio>
#include <cstdlib>
#include <ctime>
Expand Down Expand Up @@ -138,6 +139,11 @@ common_chat_msg_delimiters common_chat_msg_delimiters_parse(const json & delimit
});
}

// Process delimiters from longest to shortest, so token groups matching multiple delimiters can be supported (specialization)
std::sort(result.delimiters.begin(), result.delimiters.end(), [](const common_chat_msg_delimiter & a, const common_chat_msg_delimiter & b) {
return a.delimiter.length() > b.delimiter.length();
});

return result;
}

Expand Down Expand Up @@ -2766,6 +2772,9 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_
if (!autoparser.user_start.empty()) {
delimiters.add(COMMON_CHAT_ROLE_USER, autoparser.user_start);
}
if (!autoparser.tool_response_start.empty()) {
delimiters.add(COMMON_CHAT_ROLE_TOOL, autoparser.tool_response_start);
}

auto_params.message_delimiters = std::move(delimiters);

Expand Down
9 changes: 0 additions & 9 deletions common/chat.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -179,15 +179,6 @@ struct common_chat_msg_spans {
}
return false;
}

int32_t last_user_message_pos() const {
for (auto it = spans.rbegin(); it != spans.rend(); ++it) {
if (it->role == COMMON_CHAT_ROLE_USER) {
return (int32_t) it->pos;
}
}
return -1;
}
};

struct common_chat_msg_delimiter {
Expand Down
2 changes: 1 addition & 1 deletion common/common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -623,7 +623,7 @@ struct common_params {
bool cache_prompt = true; // whether to enable prompt caching
bool cache_idle_slots = true; // save and clear idle slots upon starting a new task
int32_t n_ctx_checkpoints = 32; // max number of context checkpoints per slot
int32_t checkpoint_min_step = 8192; // minimum spacing between context checkpoints
int32_t checkpoint_min_step = 256; // minimum spacing between context checkpoints
int32_t cache_ram_mib = 8192; // -1 = no limit, 0 - disable, 1 = 1 MiB, etc.

std::string hostname = "127.0.0.1";
Expand Down
84 changes: 43 additions & 41 deletions tests/test-chat-auto-parser.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1865,6 +1865,7 @@ struct role_marker_case {
std::string template_file;
std::string expected_user_start;
std::string expected_assistant_start;
std::string expected_tool_response_start;
};

static void test_role_markers_all_templates(testing & t) {
Expand All @@ -1875,93 +1876,93 @@ static void test_role_markers_all_templates(testing & t) {
// markers it detected first.
const std::vector<role_marker_case> cases = {
// ChatML family: <|im_start|>{role} ... <|im_end|>
{ "Bielik-11B-v3.0-Instruct.jinja", "<|im_start|>user", "<|im_start|>assistant" },
{ "HuggingFaceTB-SmolLM3-3B.jinja", "<|im_start|>user", "<|im_start|>assistant" },
{ "MiMo-VL.jinja", "<|im_start|>user", "<|im_start|>assistant" },
{ "NousResearch-Hermes-2-Pro-Llama-3-8B-tool_use.jinja", "<|im_start|>user", "<|im_start|>assistant" },
{ "NousResearch-Hermes-3-Llama-3.1-8B-tool_use.jinja", "<|im_start|>user", "<|im_start|>assistant" },
{ "NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.jinja", "<|im_start|>user", "<|im_start|>assistant" },
{ "Qwen3.5-4B.jinja", "<|im_start|>user", "<|im_start|>assistant" },
{ "Qwen3-Coder.jinja", "<|im_start|>user", "<|im_start|>assistant" },
{ "Qwen-Qwen2.5-7B-Instruct.jinja", "<|im_start|>user", "<|im_start|>assistant" },
{ "Qwen-Qwen3-0.6B.jinja", "<|im_start|>user", "<|im_start|>assistant" },
{ "Qwen-QwQ-32B.jinja", "<|im_start|>user", "<|im_start|>assistant" },
{ "StepFun3.5-Flash.jinja", "<|im_start|>user", "<|im_start|>assistant" },
{ "Bielik-11B-v3.0-Instruct.jinja", "<|im_start|>user", "<|im_start|>assistant", "<|im_start|>user\n<|function_output|>" },
{ "HuggingFaceTB-SmolLM3-3B.jinja", "<|im_start|>user", "<|im_start|>assistant", "<|im_start|>user" },
{ "MiMo-VL.jinja", "<|im_start|>user", "<|im_start|>assistant", "<|im_start|>user\n<tool_response>" },
{ "NousResearch-Hermes-2-Pro-Llama-3-8B-tool_use.jinja", "<|im_start|>user", "<|im_start|>assistant", "<|im_start|>tool\n<tool_response>" },
{ "NousResearch-Hermes-3-Llama-3.1-8B-tool_use.jinja", "<|im_start|>user", "<|im_start|>assistant", "<|im_start|>tool\n<tool_response>" },
{ "NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.jinja", "<|im_start|>user", "<|im_start|>assistant", "<|im_start|>user\n<tool_response>" },
{ "Qwen3.5-4B.jinja", "<|im_start|>user", "<|im_start|>assistant", "" },
{ "Qwen3-Coder.jinja", "<|im_start|>user", "<|im_start|>assistant", "<|im_start|>user\n<tool_response>" },
{ "Qwen-Qwen2.5-7B-Instruct.jinja", "<|im_start|>user", "<|im_start|>assistant", "<|im_start|>user\n<tool_response>" },
{ "Qwen-Qwen3-0.6B.jinja", "<|im_start|>user", "<|im_start|>assistant", "<|im_start|>user\n<tool_response>" },
{ "Qwen-QwQ-32B.jinja", "<|im_start|>user", "<|im_start|>assistant", "<|im_start|>user\n<tool_response>" },
{ "StepFun3.5-Flash.jinja", "<|im_start|>user", "<|im_start|>assistant", "<|im_start|>tool_response\n<tool_response>" },

// DeepSeek family
{ "deepseek-ai-DeepSeek-R1-Distill-Llama-8B.jinja", "<｜User｜>", "<｜Assistant｜>" },
{ "deepseek-ai-DeepSeek-R1-Distill-Qwen-32B.jinja", "<｜User｜>", "<｜Assistant｜>" },
{ "deepseek-ai-DeepSeek-V3.1.jinja", "<｜User｜>", "<｜Assistant｜>" },
{ "llama-cpp-deepseek-r1.jinja", "<｜User｜>", "<｜Assistant｜>" },
{ "deepseek-ai-DeepSeek-R1-Distill-Llama-8B.jinja", "<｜User｜>", "<｜Assistant｜>", "<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>" },
{ "deepseek-ai-DeepSeek-R1-Distill-Qwen-32B.jinja", "<｜User｜>", "<｜Assistant｜>", "<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>" },
{ "deepseek-ai-DeepSeek-V3.1.jinja", "<｜User｜>", "<｜Assistant｜>", "<｜tool▁output▁begin｜>" },
{ "llama-cpp-deepseek-r1.jinja", "<｜User｜>", "<｜Assistant｜>", "<｜tool▁outputs▁begin｜>\n<｜tool▁output▁begin｜>" },

// Llama 3 header family
{ "meetkai-functionary-medium-v3.1.jinja", "<|start_header_id|>user<|end_header_id|>", "<|start_header_id|>assistant<|end_header_id|>" },
{ "meta-llama-Llama-3.1-8B-Instruct.jinja", "<|start_header_id|>user<|end_header_id|>", "<|start_header_id|>assistant<|end_header_id|>" },
{ "meta-llama-Llama-3.2-3B-Instruct.jinja", "<|start_header_id|>user<|end_header_id|>", "<|start_header_id|>assistant<|end_header_id|>" },
{ "meta-llama-Llama-3.3-70B-Instruct.jinja", "<|start_header_id|>user<|end_header_id|>", "<|start_header_id|>assistant<|end_header_id|>" },
{ "meetkai-functionary-medium-v3.1.jinja", "<|start_header_id|>user<|end_header_id|>", "<|start_header_id|>assistant<|end_header_id|>", "<|start_header_id|>ipython<|end_header_id|>" },
{ "meta-llama-Llama-3.1-8B-Instruct.jinja", "<|start_header_id|>user<|end_header_id|>", "<|start_header_id|>assistant<|end_header_id|>", "<|start_header_id|>ipython<|end_header_id|>\n\n\"" },
{ "meta-llama-Llama-3.2-3B-Instruct.jinja", "<|start_header_id|>user<|end_header_id|>", "<|start_header_id|>assistant<|end_header_id|>", "<|start_header_id|>ipython<|end_header_id|>\n\n\"" },
{ "meta-llama-Llama-3.3-70B-Instruct.jinja", "<|start_header_id|>user<|end_header_id|>", "<|start_header_id|>assistant<|end_header_id|>", "<|start_header_id|>ipython<|end_header_id|>\n\n\"" },
// fireworks-ai forces a trailing assistant header even without add_generation_prompt,
// so the marker is absorbed into the common suffix and assistant_start is detected as empty.
{ "fireworks-ai-llama-3-firefunction-v2.jinja", "<|start_header_id|>user<|end_header_id|>", "<|start_header_id|>assistant<|end_header_id|>" },
{ "fireworks-ai-llama-3-firefunction-v2.jinja", "<|start_header_id|>user<|end_header_id|>", "<|start_header_id|>assistant<|end_header_id|>", "<|eot_id|><|start_header_id|>tool<|end_header_id|>" },

// Phi/GLM/Apriel-style: <|user|> / <|assistant|>
{ "microsoft-Phi-3.5-mini-instruct.jinja", "<|user|>", "<|assistant|>" },
{ "GLM-4.6.jinja", "<|user|>", "<|assistant|>" },
{ "unsloth-Apriel-1.5.jinja", "<|user|>", "<|assistant|>" },
{ "GLM-4.7-Flash.jinja", "<|user|>", "<|assistant|>" },
{ "microsoft-Phi-3.5-mini-instruct.jinja", "<|user|>", "<|assistant|>", "" },
{ "GLM-4.6.jinja", "<|user|>", "<|assistant|>", "<|observation|>\n<tool_response>" },
{ "unsloth-Apriel-1.5.jinja", "<|user|>", "<|assistant|>", "<|tool_result|>" },
{ "GLM-4.7-Flash.jinja", "<|user|>", "<|assistant|>", "<|observation|><tool_response>" },

// Gemma 2: <start_of_turn>{user|model}
{ "google-gemma-2-2b-it.jinja", "<start_of_turn>user", "<start_of_turn>model" },
{ "google-gemma-2-2b-it.jinja", "<start_of_turn>user", "<start_of_turn>model", "" },

// IBM Granite
{ "ibm-granite-granite-3.3-2B-Instruct.jinja", "<|start_of_role|>user<|end_of_role|>", "<|start_of_role|>assistant<|end_of_role|>" },
{ "ibm-granite-granite-4.0.jinja", "<|start_of_role|>user<|end_of_role|>", "<|start_of_role|>assistant<|end_of_role|>" },
{ "ibm-granite-granite-3.3-2B-Instruct.jinja", "<|start_of_role|>user<|end_of_role|>", "<|start_of_role|>assistant<|end_of_role|>", "<|start_of_role|>tool<|end_of_role|>" },
{ "ibm-granite-granite-4.0.jinja", "<|start_of_role|>user<|end_of_role|>", "<|start_of_role|>assistant<|end_of_role|>", "<|start_of_role|>user<|end_of_role|>\n<tool_response>" },

// Cohere R-series
{ "CohereForAI-c4ai-command-r7b-12-2024-tool_use.jinja",
"<|START_OF_TURN_TOKEN|><|USER_TOKEN|>", "<|START_RESPONSE|>" },
"<|START_OF_TURN_TOKEN|><|USER_TOKEN|>", "<|START_RESPONSE|>" , "" },
{ "CohereForAI-c4ai-command-r-plus-tool_use.jinja",
"<|START_OF_TURN_TOKEN|><|USER_TOKEN|>", "<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>" },
"<|START_OF_TURN_TOKEN|><|USER_TOKEN|>", "<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>" , "<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|><results>" },

// Mistral: assistant content follows [/INST] immediately, no header
{ "mistralai-Mistral-Nemo-Instruct-2407.jinja", "[INST]", "" },
{ "Mistral-Small-3.2-24B-Instruct-2506.jinja", "[INST]", "" },
{ "mistralai-Mistral-Nemo-Instruct-2407.jinja", "[INST]", "", "" },
{ "Mistral-Small-3.2-24B-Instruct-2506.jinja", "[INST]", "", "" },

// Apertus uses <|user_start|> / <|assistant_start|> but the user diff
// carries the preceding <|assistant_end|> from the previous turn.
{ "Apertus-8B-Instruct.jinja", "<|user_start|>", "<|assistant_start|>" },
{ "Apertus-8B-Instruct.jinja", "<|user_start|>", "<|assistant_start|>", "[" },

// Apriel 1.6 wraps the assistant body with <|begin_assistant|>, but
// <|begin_assistant|> is also the detected reasoning start, so the
// assistant_start is trimmed back to the preceding newline.
{ "Apriel-1.6-15b-Thinker-fixed.jinja", "<|begin_user|>", "<|begin_assistant|>" },
{ "Apriel-1.6-15b-Thinker-fixed.jinja", "<|begin_user|>", "<|begin_assistant|>", "<|end|>\n<|begin_tool_result|>" },

// ByteDance Seed-OSS: <seed:bos>{role}
{ "ByteDance-Seed-OSS.jinja", "<seed:bos>user", "<seed:bos>assistant" },
{ "ByteDance-Seed-OSS.jinja", "<seed:bos>user", "<seed:bos>assistant", "<seed:bos>tool" },

// GigaChat 3.1: {role}<|role_sep|>
{ "GigaChat3.1-10B-A1.8B.jinja", "user<|role_sep|>", "assistant<|role_sep|>" },
{ "GigaChat3.1-10B-A1.8B.jinja", "user<|role_sep|>", "assistant<|role_sep|>", "function result<|role_sep|>" },

// MiniMax M2: ]~b]{user|ai}
{ "MiniMax-M2.jinja", "]~b]user", "]~b]ai" },
{ "MiniMax-M2.jinja", "]~b]user", "]~b]ai", "" },

// HunYuan V3: <｜hy_User:opensource｜> / <｜hy_Assistant:opensource｜>
{ "tencent-Hy3.jinja", "<｜hy_User:opensource｜>", "<｜hy_Assistant:opensource｜>" },

// Nemotron Nano v2: <SPECIAL_11>{User|Assistant}; assistant marker
// is followed by a prefilled <think> block that gets included.
{ "NVIDIA-Nemotron-Nano-v2.jinja", "<SPECIAL_11>User", "<SPECIAL_11>Assistant" },
{ "NVIDIA-Nemotron-Nano-v2.jinja", "<SPECIAL_11>User", "<SPECIAL_11>Assistant", "A_ASST_MSG\n<SPECIAL_12>\n<SPECIAL_11>User\n<TOOL_RESPONSE>[" },

// Reka Edge: "human: " / "assistant: " — but the rendered preamble
// depends on enable_thinking, which currently confuses the user-start
// diff and trims the marker down. Lock in the observed value.
{ "Reka-Edge.jinja", "human:", "assistant:" },
{ "Reka-Edge.jinja", "human:", "assistant:", "human: <tool_response>" },

// RWKV-world chat preset: "User: " / "Assistant: "
{ "llama-cpp-rwkv-world.jinja", "User:", "Assistant:" },
{ "llama-cpp-rwkv-world.jinja", "User:", "Assistant:", "" },

// Upstage Solar 100B: <|begin|>{role}... but reasoning marker absorbs
// the "<|begin|>assistant" prefix from assistant_start.
{ "upstage-Solar-Open-100B.jinja", "<|begin|>user<|content|>", "<|begin|>assistant" },
{ "upstage-Solar-Open-100B.jinja", "<|begin|>user<|content|>", "<|begin|>assistant", "<|end|><|begin|>tool<|tool_response|><|tool_response:begin|><|tool_response:name|><|tool_response:result|>" },
};

for (const auto & c : cases) {
Expand All @@ -1971,6 +1972,7 @@ static void test_role_markers_all_templates(testing & t) {
ap.analyze_template(tmpl);
t.assert_equal("user_start", c.expected_user_start, ap.user_start);
t.assert_equal("assistant_start", c.expected_assistant_start, ap.assistant_start);
t.assert_equal("tool_response_start", c.expected_tool_response_start, ap.tool_response_start);
});
}
}
Expand Down