Skip to content

[Frontend] [Bugfix] respect server-level default chat template kwargs in reasoning parser #31581

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
chaunceyjiang merged 4 commits into from
Jan 5, 2026
Merged

[Frontend] [Bugfix] respect server-level default chat template kwargs in reasoning parser #31581

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
8332c67
respect server-level default chat template kwargs in reasoning parser
cjackal Dec 31, 2025
310e1c4
refactor following gemini comment
cjackal Jan 1, 2026
9a72ac9
Update vllm/entrypoints/openai/serving_engine.py
cjackal Jan 4, 2026
e88515d
Merge branch 'main' into respec-default-chat-template-kwargs
chaunceyjiang Jan 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 12 additions & 2 deletions vllm/entrypoints/openai/serving_chat.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -659,9 +659,14 @@ async def chat_completion_stream_generator(
"Tokenizer not available when `skip_tokenizer_init=True`"
)

# Pass the same chat template kwargs as used in tokenization
chat_template_kwargs = self._prepare_extra_chat_template_kwargs(
request.chat_template_kwargs,
self.default_chat_template_kwargs,
)
reasoning_parser = self.reasoning_parser(
tokenizer,
chat_template_kwargs=request.chat_template_kwargs, # type: ignore
chat_template_kwargs=chat_template_kwargs, # type: ignore[call-arg]
)
except RuntimeError as e:
logger.exception("Error in reasoning parser creation.")
Expand Down Expand Up @@ -1437,9 +1442,14 @@ async def chat_completion_full_generator(
"Tokenizer not available when `skip_tokenizer_init=True`"
)

# Pass the same chat template kwargs as used in tokenization
chat_template_kwargs = self._prepare_extra_chat_template_kwargs(
request.chat_template_kwargs,
self.default_chat_template_kwargs,
)
reasoning_parser = self.reasoning_parser(
tokenizer,
chat_template_kwargs=request.chat_template_kwargs, # type: ignore
chat_template_kwargs=chat_template_kwargs, # type: ignore[call-arg]
)
except RuntimeError as e:
logger.exception("Error in reasoning parser creation.")
Expand Down
19 changes: 16 additions & 3 deletions vllm/entrypoints/openai/serving_engine.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1146,6 +1146,18 @@ def _validate_chat_template(
)
return None

@staticmethod
def _prepare_extra_chat_template_kwargs(
request_chat_template_kwargs: dict[str, Any] | None = None,
default_chat_template_kwargs: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""Helper to merge server-default and request-specific chat template kwargs."""
request_chat_template_kwargs = request_chat_template_kwargs or {}
if default_chat_template_kwargs is None:
return request_chat_template_kwargs
# Apply server defaults first, then request kwargs override.
return default_chat_template_kwargs | request_chat_template_kwargs

async def _preprocess_chat(
self,
request: ChatLikeRequest | ResponsesRequest,
Expand Down Expand Up @@ -1184,9 +1196,10 @@ async def _preprocess_chat(
tools=tool_dicts,
documents=documents,
)
if default_chat_template_kwargs:
_chat_template_kwargs.update(default_chat_template_kwargs)
_chat_template_kwargs.update(chat_template_kwargs or {})
_chat_template_kwargs |= self._prepare_extra_chat_template_kwargs(
chat_template_kwargs,
default_chat_template_kwargs,
)

request_prompt: str | list[int]

Expand Down