From 379f70ab4cce521bd21c5ea3401b2dfccbb9d1b2 Mon Sep 17 00:00:00 2001 From: Krishnan Prashanth Date: Wed, 15 Jul 2026 13:04:18 -0700 Subject: [PATCH 1/3] fix(frontend): forward request chat_template_kwargs to vLLM template Signed-off-by: Krishnan Prashanth --- components/src/dynamo/frontend/prepost.py | 36 +++++++--- .../tests/test_vllm_processor_unit.py | 70 +++++++++++++++++++ 2 files changed, 97 insertions(+), 9 deletions(-) diff --git a/components/src/dynamo/frontend/prepost.py b/components/src/dynamo/frontend/prepost.py index 003813e99ce6..e5ad46cedcae 100644 --- a/components/src/dynamo/frontend/prepost.py +++ b/components/src/dynamo/frontend/prepost.py @@ -136,8 +136,23 @@ def _prepare_request( ) else None ) - chat_template_kwargs = dict(request_for_sampling.chat_template_kwargs or {}) - chat_template_kwargs["reasoning_effort"] = request_for_sampling.reasoning_effort + # The Rust bindings serialize this field as `chat_template_args` (serde + # `alias = "chat_template_kwargs"` applies on deserialize only), so a request + # arriving as a raw dict carries the client's kwargs there rather than under + # the vLLM-native `chat_template_kwargs`. Fall back to it so per-request + # template control is not silently dropped, mirroring sglang_prepost.py. + raw_template_args = ( + request.get("chat_template_args") if isinstance(request, dict) else None + ) + chat_template_kwargs = dict( + request_for_sampling.chat_template_kwargs or raw_template_args or {} + ) + # A top-level reasoning_effort wins; otherwise keep a nested one and fall back + # to None so the key is always present for templates that consult it. + if request_for_sampling.reasoning_effort is not None: + chat_template_kwargs["reasoning_effort"] = request_for_sampling.reasoning_effort + else: + chat_template_kwargs.setdefault("reasoning_effort", None) # Mistral warns that tokenize=False is unsafe for chat templates. is_mistral_tokenizer = ( @@ -154,14 +169,17 @@ def _prepare_request( chat_params = ChatParams( chat_template=request_for_sampling.chat_template, chat_template_content_format="auto", - chat_template_kwargs=dict( - add_generation_prompt=request_for_sampling.add_generation_prompt, - continue_final_message=request_for_sampling.continue_final_message, - tools=tool_dicts, - documents=request_for_sampling.documents, - tokenize=tokenize_in_template, + # Renderer-managed keys are authoritative and applied last so a client + # that nests one of them in chat_template_kwargs can't trigger a + # duplicate-keyword TypeError. + chat_template_kwargs={ **chat_template_kwargs, - ), + "add_generation_prompt": request_for_sampling.add_generation_prompt, + "continue_final_message": request_for_sampling.continue_final_message, + "tools": tool_dicts, + "documents": request_for_sampling.documents, + "tokenize": tokenize_in_template, + }, ) return ( diff --git a/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py b/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py index ed54cbe82d9b..c312104021a8 100644 --- a/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py +++ b/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py @@ -155,6 +155,76 @@ def test_no_tools_in_request(self, tokenizer): ), "No tools in request should produce None tools in template" +class TestChatTemplateArgsPassthrough: # regression for #11704 — client template kwargs must reach the template + """Per-request chat template kwargs must survive into the rendered template. + + The Rust bindings serialize the request field as ``chat_template_args`` + (serde ``alias = "chat_template_kwargs"`` only applies on deserialize), so a + request arriving as a raw dict carries the client's kwargs under + ``chat_template_args``. The vLLM processor previously read only + ``chat_template_kwargs`` and silently dropped it. + """ + + def test_chat_template_args_reaches_template(self, tokenizer): + """Kwargs keyed as chat_template_args (the pythonize'd key) reach the template.""" + _, _, _, _, chat_params = _prepare_request( + { + "model": MODEL, + "messages": [{"role": "user", "content": "Hello"}], + "chat_template_args": {"enable_thinking": False}, + }, + tokenizer=tokenizer, + tool_parser_class=None, + ) + assert ( + chat_params.chat_template_kwargs.get("enable_thinking") is False + ), "chat_template_args must be forwarded to the chat template" + + def test_chat_template_kwargs_native_key_still_works(self, tokenizer): + """The vLLM-native chat_template_kwargs key keeps working.""" + _, _, _, _, chat_params = _prepare_request( + { + "model": MODEL, + "messages": [{"role": "user", "content": "Hello"}], + "chat_template_kwargs": {"enable_thinking": False}, + }, + tokenizer=tokenizer, + tool_parser_class=None, + ) + assert ( + chat_params.chat_template_kwargs.get("enable_thinking") is False + ), "native chat_template_kwargs must be forwarded to the chat template" + + def test_nested_reasoning_effort_is_not_clobbered(self, tokenizer): + """A reasoning_effort nested in template kwargs survives the top-level default.""" + _, _, _, _, chat_params = _prepare_request( + { + "model": MODEL, + "messages": [{"role": "user", "content": "Hello"}], + "chat_template_args": {"reasoning_effort": "high"}, + }, + tokenizer=tokenizer, + tool_parser_class=None, + ) + assert ( + chat_params.chat_template_kwargs.get("reasoning_effort") == "high" + ), "nested reasoning_effort must not be overwritten by an absent top-level field" + + def test_reserved_render_key_in_template_args_does_not_crash(self, tokenizer): + """A renderer-reserved key nested in template kwargs must not raise TypeError.""" + _, _, _, _, chat_params = _prepare_request( + { + "model": MODEL, + "messages": [{"role": "user", "content": "Hello"}], + "chat_template_args": {"documents": [{"text": "doc"}]}, + }, + tokenizer=tokenizer, + tool_parser_class=None, + ) + # The renderer-managed value wins; the point is that merging does not raise. + assert "documents" in chat_params.chat_template_kwargs + + class TestMultimodalFeatureMetadata: def _feature( self, modality, mm_hash, offset, length, data=_DEFAULT_MM_DATA, is_embed=None From 0659872c3180047c0c504331117a7c2664343398 Mon Sep 17 00:00:00 2001 From: Krishnan Prashanth Date: Wed, 15 Jul 2026 14:11:12 -0700 Subject: [PATCH 2/3] chore(frontend): trim comments to succinct why-only Signed-off-by: Krishnan Prashanth --- components/src/dynamo/frontend/prepost.py | 14 ++++---------- .../frontend/tests/test_vllm_processor_unit.py | 10 ++++------ 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/components/src/dynamo/frontend/prepost.py b/components/src/dynamo/frontend/prepost.py index e5ad46cedcae..0b92fdcc6cb0 100644 --- a/components/src/dynamo/frontend/prepost.py +++ b/components/src/dynamo/frontend/prepost.py @@ -136,19 +136,15 @@ def _prepare_request( ) else None ) - # The Rust bindings serialize this field as `chat_template_args` (serde - # `alias = "chat_template_kwargs"` applies on deserialize only), so a request - # arriving as a raw dict carries the client's kwargs there rather than under - # the vLLM-native `chat_template_kwargs`. Fall back to it so per-request - # template control is not silently dropped, mirroring sglang_prepost.py. + # serde's `alias` is deserialize-only, so pythonize emits the Rust field + # name `chat_template_args`; read it too or client kwargs are dropped. raw_template_args = ( request.get("chat_template_args") if isinstance(request, dict) else None ) chat_template_kwargs = dict( request_for_sampling.chat_template_kwargs or raw_template_args or {} ) - # A top-level reasoning_effort wins; otherwise keep a nested one and fall back - # to None so the key is always present for templates that consult it. + # Don't let an absent top-level field clobber a nested reasoning_effort. if request_for_sampling.reasoning_effort is not None: chat_template_kwargs["reasoning_effort"] = request_for_sampling.reasoning_effort else: @@ -169,9 +165,7 @@ def _prepare_request( chat_params = ChatParams( chat_template=request_for_sampling.chat_template, chat_template_content_format="auto", - # Renderer-managed keys are authoritative and applied last so a client - # that nests one of them in chat_template_kwargs can't trigger a - # duplicate-keyword TypeError. + # Renderer-managed keys last so a nested duplicate can't raise TypeError. chat_template_kwargs={ **chat_template_kwargs, "add_generation_prompt": request_for_sampling.add_generation_prompt, diff --git a/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py b/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py index c312104021a8..1dba0ef75134 100644 --- a/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py +++ b/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py @@ -155,14 +155,12 @@ def test_no_tools_in_request(self, tokenizer): ), "No tools in request should produce None tools in template" -class TestChatTemplateArgsPassthrough: # regression for #11704 — client template kwargs must reach the template +class TestChatTemplateArgsPassthrough: """Per-request chat template kwargs must survive into the rendered template. - The Rust bindings serialize the request field as ``chat_template_args`` - (serde ``alias = "chat_template_kwargs"`` only applies on deserialize), so a - request arriving as a raw dict carries the client's kwargs under - ``chat_template_args``. The vLLM processor previously read only - ``chat_template_kwargs`` and silently dropped it. + pythonize serializes the request field under its Rust name + ``chat_template_args`` (serde ``alias`` is deserialize-only), so the vLLM + processor must read that key, not only vLLM's native ``chat_template_kwargs``. """ def test_chat_template_args_reaches_template(self, tokenizer): From 5ac652515e6a854b55b8703895241b347d18bf31 Mon Sep 17 00:00:00 2001 From: Krishnan Prashanth Date: Wed, 15 Jul 2026 16:32:08 -0700 Subject: [PATCH 3/3] test(frontend): strengthen chat_template_kwargs precedence assertions Signed-off-by: Krishnan Prashanth --- .../frontend/tests/test_vllm_processor_unit.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py b/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py index 1dba0ef75134..5ebdc355edb7 100644 --- a/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py +++ b/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py @@ -208,6 +208,20 @@ def test_nested_reasoning_effort_is_not_clobbered(self, tokenizer): chat_params.chat_template_kwargs.get("reasoning_effort") == "high" ), "nested reasoning_effort must not be overwritten by an absent top-level field" + def test_top_level_reasoning_effort_wins_over_nested(self, tokenizer): + """An explicit top-level reasoning_effort overrides a nested one.""" + _, _, _, _, chat_params = _prepare_request( + { + "model": MODEL, + "messages": [{"role": "user", "content": "Hello"}], + "reasoning_effort": "low", + "chat_template_args": {"reasoning_effort": "high"}, + }, + tokenizer=tokenizer, + tool_parser_class=None, + ) + assert chat_params.chat_template_kwargs.get("reasoning_effort") == "low" + def test_reserved_render_key_in_template_args_does_not_crash(self, tokenizer): """A renderer-reserved key nested in template kwargs must not raise TypeError.""" _, _, _, _, chat_params = _prepare_request( @@ -219,8 +233,8 @@ def test_reserved_render_key_in_template_args_does_not_crash(self, tokenizer): tokenizer=tokenizer, tool_parser_class=None, ) - # The renderer-managed value wins; the point is that merging does not raise. - assert "documents" in chat_params.chat_template_kwargs + # Renderer-managed value wins over the client's nested key (no crash either). + assert chat_params.chat_template_kwargs["documents"] is None class TestMultimodalFeatureMetadata: