Skip to content

[Frontend] Extract parse and assembly out of batch chat derender #60073

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sfeng33 merged 2 commits into from
Oct 5, 2026
Merged

[Frontend] Extract parse and assembly out of batch chat derender #60073

Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d8204dd
[Frontend] Extract parse and assembly out of batch chat derender
hickeyma Oct 5, 2026
f2cdd21
Merge branch 'main' into move-parse-assembly
sfeng33 Oct 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
184 changes: 101 additions & 83 deletions vllm/renderers/online_derenderer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,12 @@ def _derender_chat(
skip_special_tokens=skip_special,
)

chat_template_kwargs = (
self._resolve_chat_template_kwargs(chat_request)
if has_parser and chat_request is not None
else {}
)

for choice in generate_response.choices:
if not choice.token_ids:
raise ValueError(f"choice {choice.index} has empty or null token_ids")
Expand All @@ -161,83 +167,29 @@ def _derender_chat(
else None
)

if has_parser:
assert self.parser is not None and chat_request is not None
# Parser path: decode with special tokens preserved
# so the parser can see markers like </think>,
# <tool_call>, or Harmony channel tokens.
decoded_text, _ = self._detokenize_delta(
tokenizer,
choice.token_ids,
seed_state,
skip_special_tokens=skip_special,
spaces_between_special_tokens=spaces_between,
)

chat_template_kwargs: dict[str, Any] = {}
if not self.use_harmony:
chat_template_kwargs = (
chat_request.build_chat_params(
self.chat_template,
self.chat_template_content_format,
)
.with_defaults(self.default_chat_template_kwargs)
.chat_template_kwargs
)
# With a parser, special tokens are preserved so it can see
# markers like </think>, <tool_call>, or Harmony channel tokens.
# Without one, the request's skip_special_tokens is honoured
# (default True when no request was given).
decoded_text, _ = self._detokenize_delta(
tokenizer,
choice.token_ids,
seed_state,
skip_special_tokens=skip_special,
spaces_between_special_tokens=spaces_between,
)

parser = self.parser(
if has_parser:
assert chat_request is not None
message = self._parse_and_assemble(
tokenizer,
chat_request.tools,
chat_template_kwargs=chat_template_kwargs,
model_config=self.model_config,
)
if generate_response.prompt_token_ids is not None:
parser.set_prompt_token_ids(generate_response.prompt_token_ids)
reasoning, content, tool_calls = parser.parse(
decoded_text,
choice.token_ids,
chat_request,
enable_auto_tools=self.enable_auto_tools,
model_output_token_ids=choice.token_ids,
)

if not getattr(chat_request, "include_reasoning", True):
reasoning = None

tc_items = (
[
ToolCall(
id=random_uuid(),
function=tc,
)
for tc in tool_calls
]
if tool_calls
else []
)

is_named_tool_choice = (
type(chat_request.tool_choice) is ChatCompletionNamedToolChoiceParam
)
is_required_tool_choice = chat_request.tool_choice == "required"
if is_named_tool_choice or is_required_tool_choice:
content = content or ""

message = ChatMessage(
role="assistant",
reasoning=reasoning,
content=content,
tool_calls=tc_items,
chat_template_kwargs,
generate_response.prompt_token_ids,
)
else:
# No parser: plain detokenization honouring the request's
# skip_special_tokens (default True when no request was given).
decoded_text, _ = self._detokenize_delta(
tokenizer,
choice.token_ids,
seed_state,
skip_special_tokens=skip_special,
spaces_between_special_tokens=spaces_between,
)
message = ChatMessage(role="assistant", content=decoded_text)

choices.append(
Expand All @@ -251,6 +203,83 @@ def _derender_chat(

return choices

def _resolve_chat_template_kwargs(
self, chat_request: ChatCompletionRequest
) -> dict[str, Any]:
if self.use_harmony:
return {}
return (
chat_request.build_chat_params(
self.chat_template,
self.chat_template_content_format,
)
.with_defaults(self.default_chat_template_kwargs)
.chat_template_kwargs
)

def _parse_and_assemble(
self,
tokenizer: TokenizerLike,
text: str,
token_ids: Sequence[int],
chat_request: ChatCompletionRequest,
chat_template_kwargs: dict[str, Any],
prompt_token_ids: list[int] | None,
) -> ChatMessage:
"""Parse decoded output text into an assistant `ChatMessage`.

Args:
tokenizer: Tokenizer handed to the parser.
text: Decoded output text to parse.
token_ids: Output token IDs the text was decoded from.
chat_request: Request supplying tools, `tool_choice` and
`include_reasoning`.
chat_template_kwargs: Already resolved template kwargs.
prompt_token_ids: Prompt token IDs, if known.

Returns:
The assembled assistant message.

"""
assert self.parser is not None
parser = self.parser(
tokenizer,
chat_request.tools,
chat_template_kwargs=chat_template_kwargs,
model_config=self.model_config,
)
if prompt_token_ids is not None:
parser.set_prompt_token_ids(prompt_token_ids)
reasoning, content, tool_calls = parser.parse(
text,
chat_request,
enable_auto_tools=self.enable_auto_tools,
model_output_token_ids=token_ids,
)

if not getattr(chat_request, "include_reasoning", True):
reasoning = None

tc_items = (
[ToolCall(id=random_uuid(), function=tc) for tc in tool_calls]
if tool_calls
else []
)

is_named_tool_choice = (
type(chat_request.tool_choice) is ChatCompletionNamedToolChoiceParam
)
is_required_tool_choice = chat_request.tool_choice == "required"
if is_named_tool_choice or is_required_tool_choice:
content = content or ""

return ChatMessage(
role="assistant",
reasoning=reasoning,
content=content,
tool_calls=tc_items,
)

def _detokenize_delta(
self,
tokenizer: TokenizerLike,
Expand Down Expand Up @@ -497,21 +526,10 @@ def _derender_chat_stream_parsed(
"""
tokenizer = self.renderer.get_tokenizer()

chat_template_kwargs: dict[str, Any] = {}
if not self.use_harmony:
chat_template_kwargs = (
chat_request.build_chat_params(
self.chat_template,
self.chat_template_content_format,
)
.with_defaults(self.default_chat_template_kwargs)
.chat_template_kwargs
)

parser = parser_cls(
tokenizer,
chat_request.tools,
chat_template_kwargs=chat_template_kwargs,
chat_template_kwargs=self._resolve_chat_template_kwargs(chat_request),
model_config=self.model_config,
)

Expand Down
Loading