From e33ae790b90b79bc82dda948481bc52d31d1bf90 Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Thu, 2 Jul 2026 12:59:44 +0200 Subject: [PATCH 1/3] chat: trim messages sent to StepFun parser (fixes long reasoning loops) --- common/chat.cpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/common/chat.cpp b/common/chat.cpp index 6da59f4dbd2c..9e345d3ad840 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -2378,6 +2378,18 @@ static void func_args_not_string(json & messages) { } } +static void trim_all_content(json & messages) { + GGML_ASSERT(messages.is_array()); + for (auto & message : messages) { + if (message.contains("reasoning_content")) { + message["reasoning_content"] = trim_whitespace(message["reasoning_content"]); + } + if (message.contains("content")) { + message["content"] = trim_whitespace(message["content"]); + } + } +} + } // MiniCPM5 format: @@ -2685,6 +2697,11 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_ workaround::func_args_not_string(params.messages); } + if (tmpl.src.find("You have access to the following functions in JSONSchema format") != std::string::npos) { + // StepFun: we need to trim all contents and reasoning contents before passing them to the template + workaround::trim_all_content(params.messages); + } + params.extra_context = common_chat_extra_context(); for (auto el : inputs.chat_template_kwargs) { params.extra_context[el.first] = json::parse(el.second); From 31b9a9b6fa0a8298315414ac63190e36c0cfba0c Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Thu, 2 Jul 2026 14:42:04 +0200 Subject: [PATCH 2/3] add regression test; remove duplicate template --- .../templates/stepfun-ai-Step-3.5-Flash.jinja | 80 ------------------- tests/test-chat-auto-parser.cpp | 1 - tests/test-chat.cpp | 24 ++++++ 3 files changed, 24 insertions(+), 81 deletions(-) delete mode 100644 models/templates/stepfun-ai-Step-3.5-Flash.jinja diff --git a/models/templates/stepfun-ai-Step-3.5-Flash.jinja b/models/templates/stepfun-ai-Step-3.5-Flash.jinja deleted file mode 100644 index c09ea497dada..000000000000 --- a/models/templates/stepfun-ai-Step-3.5-Flash.jinja +++ /dev/null @@ -1,80 +0,0 @@ -{% macro render_content(content) %}{% if content is none %}{{- '' }}{% elif content is string %}{{- content }}{% elif content is mapping %}{{- content['value'] if 'value' in content else content['text'] }}{% elif content is iterable %}{% for item in content %}{% if item.type == 'text' %}{{- item['value'] if 'value' in item else item['text'] }}{% elif item.type == 'image' %}{% endif %}{% endfor %}{% endif %}{% endmacro %} -{{bos_token}}{%- if tools %} - {{- '<|im_start|>system\n' }} - {%- if messages[0].role == 'system' %} - {{- render_content(messages[0].content) + '\n\n' }} - {%- endif %} - {{- "# Tools\n\nYou have access to the following functions in JSONSchema format:\n\n" }} - {%- for tool in tools %} - {{- "\n" }} - {{- tool | tojson(ensure_ascii=False) }} - {%- endfor %} - {{- "\n\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner \n...\n block must be nested within \n...\n XML tags\n- Required parameters MUST be specified\n<|im_end|>\n" }} -{%- else %} - {%- if messages[0].role == 'system' %} - {{- '<|im_start|>system\n' + render_content(messages[0].content) + '<|im_end|>\n' }} - {%- endif %} -{%- endif %} -{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} -{%- for message in messages[::-1] %} - {%- set index = (messages|length - 1) - loop.index0 %} - {%- if ns.multi_step_tool and message.role == "user" and render_content(message.content) is string and not(render_content(message.content).startswith('') and render_content(message.content).endswith('')) %} - {%- set ns.multi_step_tool = false %} - {%- set ns.last_query_index = index %} - {%- endif %} -{%- endfor %} -{%- for message in messages %} - {%- set content = render_content(message.content) %} - {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} - {%- set role_name = 'observation' if (message.role == "system" and not loop.first and message.name == 'observation') else message.role %} - {{- '<|im_start|>' + role_name + '\n' + content + '<|im_end|>' + '\n' }} - {%- elif message.role == "assistant" %} - {%- if message.reasoning_content is string %} - {%- set reasoning_content = render_content(message.reasoning_content) %} - {%- else %} - {%- if '' in content %} - {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} - {%- set content = content.split('')[-1].lstrip('\n') %} - {%- else %} - {%- set reasoning_content = '' %} - {%- endif %} - {%- endif %} - {%- if loop.index0 > ns.last_query_index %} - {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n' + content }} - {%- else %} - {{- '<|im_start|>' + message.role + '\n' + content }} - {%- endif %} - {%- if message.tool_calls %} - {%- for tool_call in message.tool_calls %} - {%- if tool_call.function is defined %} - {%- set tool_call = tool_call.function %} - {%- endif %} - {{- '\n\n' }} - {%- if tool_call.arguments is defined %} - {%- set arguments = tool_call.arguments %} - {%- for args_name, args_value in arguments|items %} - {{- '\n' }} - {%- set args_value = args_value | tojson(ensure_ascii=False) | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} - {{- args_value }} - {{- '\n\n' }} - {%- endfor %} - {%- endif %} - {{- '\n' }} - {%- endfor %} - {%- endif %} - {{- '<|im_end|>\n' }} - {%- elif message.role == "tool" %} - {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} - {{- '<|im_start|>tool_response\n' }} - {%- endif %} - {{- '' }} - {{- content }} - {{- '' }} - {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} - {{- '<|im_end|>\n' }} - {%- endif %} - {%- endif %} -{%- endfor %} -{%- if add_generation_prompt %} - {{- '<|im_start|>assistant\n\n' }} -{%- endif %} diff --git a/tests/test-chat-auto-parser.cpp b/tests/test-chat-auto-parser.cpp index 5cc1057532c3..d15fdd2c022a 100644 --- a/tests/test-chat-auto-parser.cpp +++ b/tests/test-chat-auto-parser.cpp @@ -1887,7 +1887,6 @@ static void test_role_markers_all_templates(testing & t) { { "Qwen-Qwen3-0.6B.jinja", "<|im_start|>user", "<|im_start|>assistant" }, { "Qwen-QwQ-32B.jinja", "<|im_start|>user", "<|im_start|>assistant" }, { "StepFun3.5-Flash.jinja", "<|im_start|>user", "<|im_start|>assistant" }, - { "stepfun-ai-Step-3.5-Flash.jinja", "<|im_start|>user", "<|im_start|>assistant" }, // DeepSeek family { "deepseek-ai-DeepSeek-R1-Distill-Llama-8B.jinja", "<｜User｜>", "<｜Assistant｜>" }, diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index 5f71e5da6e39..844bf5d876a2 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -3155,6 +3155,30 @@ static void test_template_output_peg_parsers(bool detailed_debug) { } } } + + { + // StepFun trimming regression test (see https://github.com/ggml-org/llama.cpp/pull/25238) + auto tmpls = read_templates("models/templates/StepFun3.5-Flash.jinja"); + + common_chat_msg message_chatbot = simple_assist_msg("Let me check.\n\n", "I am thinking.\n\n"); + + { + common_chat_templates_inputs inputs; + inputs.messages = { message_chatbot }; + inputs.add_generation_prompt = true; + + auto params = common_chat_templates_apply(tmpls.get(), inputs); + + if (params.prompt.find("Let me check.\n\n") != std::string::npos) { + throw std::runtime_error("StepFun 3.5: content not trimmed"); + } + + if (params.prompt.find("I am thinking.\n\n") != std::string::npos) { + throw std::runtime_error("StepFun 3.5: reasoning_content not trimmed"); + } + } + } + } { From 4417cb1e926ea26e9a148572879ce17be399fea8 Mon Sep 17 00:00:00 2001 From: tarruda Date: Fri, 3 Jul 2026 14:28:58 +0200 Subject: [PATCH 3/3] chat: trim StepFun content parts before rendering The StepFun trim workaround ran on the already-rendered messages, where typed content parts have been concatenated into a single string, so the per-part whitespace could no longer be reached. Move the trim ahead of rendering and apply it to content_parts text as well as the string content and reasoning_content. Adds a content-parts regression test. Co-Authored-By: Piotr Wilkin Assisted-By: Claude Fable 5 --- common/chat.cpp | 35 ++++++++++++++++++++++------------- tests/test-chat.cpp | 29 +++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 13 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index 9e345d3ad840..22d2ee4a2a11 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -2378,14 +2378,19 @@ static void func_args_not_string(json & messages) { } } -static void trim_all_content(json & messages) { - GGML_ASSERT(messages.is_array()); +// Trim leading/trailing whitespace from message contents before rendering. This +// has to run on the messages (not on the rendered JSON) because templates with +// string-only content caps concatenate typed content parts into a single string +// during rendering, after which the per-part whitespace can no longer be reached. +// Both the plain string content and the text of typed content parts are trimmed. +static void trim_all_content(std::vector & messages) { for (auto & message : messages) { - if (message.contains("reasoning_content")) { - message["reasoning_content"] = trim_whitespace(message["reasoning_content"]); - } - if (message.contains("content")) { - message["content"] = trim_whitespace(message["content"]); + message.content = trim_whitespace(message.content); + message.reasoning_content = trim_whitespace(message.reasoning_content); + for (auto & part : message.content_parts) { + if (part.type == "text") { + part.text = trim_whitespace(part.text); + } } } } @@ -2646,7 +2651,16 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_ params.tools.is_array() && tmpls->template_tool_use ? *tmpls->template_tool_use : *tmpls->template_default; const auto & src = tmpl.source(); const auto & caps = tmpl.original_caps(); - params.messages = render_message_to_json(inputs.messages, tmpl.original_caps()); + std::vector trimmed_messages; + const std::vector * messages_to_render = &inputs.messages; + if (src.find("You have access to the following functions in JSONSchema format") != std::string::npos) { + // StepFun: trim message contents (including typed content parts) before rendering, + // otherwise leftover whitespace drives the model into reasoning loops (issue #24181) + trimmed_messages = inputs.messages; + workaround::trim_all_content(trimmed_messages); + messages_to_render = &trimmed_messages; + } + params.messages = render_message_to_json(*messages_to_render, tmpl.original_caps()); params.tool_choice = inputs.tool_choice; params.reasoning_format = inputs.reasoning_format; params.enable_thinking = inputs.enable_thinking; @@ -2697,11 +2711,6 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_ workaround::func_args_not_string(params.messages); } - if (tmpl.src.find("You have access to the following functions in JSONSchema format") != std::string::npos) { - // StepFun: we need to trim all contents and reasoning contents before passing them to the template - workaround::trim_all_content(params.messages); - } - params.extra_context = common_chat_extra_context(); for (auto el : inputs.chat_template_kwargs) { params.extra_context[el.first] = json::parse(el.second); diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index 844bf5d876a2..e1e0a59e6de6 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -3177,6 +3177,35 @@ static void test_template_output_peg_parsers(bool detailed_debug) { throw std::runtime_error("StepFun 3.5: reasoning_content not trimmed"); } } + + { + // Trimming must also reach typed (text) content parts, not just string content + // (see https://github.com/ggml-org/llama.cpp/pull/25238) + common_chat_msg message_parts; + message_parts.role = "user"; + message_parts.content_parts = { + { /* .type = */ "text", /* .text = */ "First part.\n\n" }, + { /* .type = */ "media_marker", /* .text = */ "<__media__>" }, + { /* .type = */ "text", /* .text = */ "Second part.\n\n" }, + }; + + common_chat_templates_inputs inputs; + inputs.messages = { message_parts }; + inputs.add_generation_prompt = true; + + auto params = common_chat_templates_apply(tmpls.get(), inputs); + + if (params.prompt.find("First part.\n\n") != std::string::npos || + params.prompt.find("Second part.\n\n") != std::string::npos) { + throw std::runtime_error("StepFun 3.5: text content parts not trimmed"); + } + + // the trimmed text itself must still be present + if (params.prompt.find("First part.") == std::string::npos || + params.prompt.find("Second part.") == std::string::npos) { + throw std::runtime_error("StepFun 3.5: text content parts missing after trim"); + } + } } }