From 2f4995c7f8371bad5326e1303a06610a9582ec81 Mon Sep 17 00:00:00 2001 From: Martin Hickey Date: Mon, 5 Oct 2026 17:50:14 +0100 Subject: [PATCH 01/11] [Frontend] Add derender output mode to generate Add output_mode="derender" to /inference/v1/generate so the generate server returns the parsed assistant message (content, reasoning and tool calls) itself, with no separate /derender call. It parses the engine's own text, so it sees the stop string cut and the decode flags set by adjust_request. That's why it matches coupled chat where /derender doesn't. /render now returns a parse_context with every chat request. It holds the tools, tool_choice, parallel_tool_calls, include_reasoning, the resolved chat_template_kwargs and the history tool call count. Other output modes ignore it, so prefill legs can take the rendered request as is. Streaming isn't supported yet and returns a 400. Batch /derender now honours parallel_tool_calls=false like coupled chat. Template kwargs resolution is shared between /render and the derenderer. Part of #56851. Signed-off-by: Martin Hickey --- docs/serving/online_serving/derenderer.md | 2 + .../online_serving/token_in_token_out.md | 16 ++- .../derender/test_derender_parity.py | 132 ++++++++++++++++++ .../derender/test_derender_stream.py | 48 +++++++ .../token_in_token_out/test_protocol.py | 66 ++++++++- .../token_in_token_out/test_serving_tokens.py | 73 +++++++++- vllm/entrypoints/scale_out/factories.py | 1 + vllm/entrypoints/scale_out/render/serving.py | 25 +++- .../scale_out/token_in_token_out/protocol.py | 56 +++++++- .../scale_out/token_in_token_out/serving.py | 66 ++++++++- vllm/parser/abstract_parser.py | 6 + vllm/renderers/online_derenderer.py | 83 +++++++++-- vllm/renderers/online_renderer.py | 21 +++ 13 files changed, 570 insertions(+), 25 deletions(-) diff --git a/docs/serving/online_serving/derenderer.md b/docs/serving/online_serving/derenderer.md index 743b56a888c2..c46cd383b9a0 100644 --- a/docs/serving/online_serving/derenderer.md +++ b/docs/serving/online_serving/derenderer.md @@ -89,6 +89,8 @@ The derenderer builds its tool and reasoning parsers from its own server flags p - Start the render server with the same `--tool-call-parser`, `--reasoning-parser`, `--enable-auto-tool-choice`, `--chat-template` and `--default-chat-template-kwargs` you'd give `vllm serve` for this model. If `/render` and `/derender` run on different servers, give both the same values. - Send the full `chat_request` that went to `/render`, not just `messages` and `tools`. Fields like `chat_template_kwargs`, `reasoning_effort`, `tool_choice` and `include_reasoning` change how the output is parsed. +`/derender` decodes with the flags in `chat_request` which can differ from what the engine ran with. For parsers whose `adjust_request` changes decoding, use [`output_mode: "derender"`](token_in_token_out.md#derender) on `/inference/v1/generate`. It parses the engine's own text and matches coupled chat. + A mismatch doesn't fail. The parser just splits `reasoning`, `content` and `tool_calls` differently from what `vllm serve` would return. `/v1/completions/derender` only detokenizes. It never runs tool or reasoning parsers, the same as `/v1/completions` on `vllm serve`. From `completion_request` it reads `skip_special_tokens`, and `logprobs` and `logprob_token_ids` to cut `top_logprobs` the way `/v1/completions` does. diff --git a/docs/serving/online_serving/token_in_token_out.md b/docs/serving/online_serving/token_in_token_out.md index 6279fbd78b18..8f56eff1a836 100644 --- a/docs/serving/online_serving/token_in_token_out.md +++ b/docs/serving/online_serving/token_in_token_out.md @@ -54,6 +54,7 @@ The request field `output_mode` selects how much of the postprocessing the serve | --- | --- | --- | | `tokens` (default) | nothing, token IDs only | nothing | | `text` | `text` on every choice and decoded tokens with `bytes` in `logprobs` | a tokenizer | +| `derender` | the parsed assistant `message` (`content`, `reasoning`, `tool_calls`) on every choice, plus the `text` level's `logprobs` | a tokenizer and the parser flags you'd give `vllm serve` | `text` returns text and token IDs in a single call, without a separate [derender](derenderer.md) hop per chunk. This suits latency sensitive streaming, where the derender hop lands on every output token. @@ -83,6 +84,17 @@ print(response["choices"][0]["token_ids"]) - `/derender` only accepts `output_mode: "tokens"` responses and returns a 400 for text responses which are already detokenized. - With `stream: true`, each choice's `text` is the delta since the previous chunk. A chunk is sent whenever the engine output carries new text or a `finish_reason`, even with no new token IDs. That covers text held back for stop string matching and the final output after an abort. +### Derender + +`derender` returns the assistant `message` a coupled `/v1/chat/completions` would, without a separate [derender](derenderer.md) call. `/v1/chat/completions/render` and `/v1/messages/render` return `parse_context` on the `GenerateRequest`. Send it back with `output_mode: "derender"`. It carries the tools, `tool_choice`, `include_reasoning`, resolved `chat_template_kwargs` and the history tool call count. + +- The parser reads the engine's `text`, so it sees the stop string cut and the post `adjust_request` decode flags. That makes it match coupled chat for parsers whose `adjust_request` changes decoding which `/derender` can't. +- A server with a tool or reasoning parser needs `parse_context`. Without a parser, `message.content` is the plain text. +- `parse_context` is ignored at other levels, so a rendered request can go to a prefill leg unchanged. +- `logprobs` are dropped when `include_reasoning` is false, so they don't expose the hidden reasoning. Coupled chat still returns them until [#59590](https://github.com/vllm-project/vllm/pull/59590) lands. +- `finish_reason` is the engine's value. Coupled chat reports `tool_calls` when a tool is called which this level doesn't do yet. +- `stream: true` isn't supported yet. + ### Logprobs With `output_mode: "tokens"`, logprobs are `GenerateLogProbs` with an integer `token_id` and `rank` per entry and no `token` or `bytes` (see [Generate Output Logprobs](renderer.md#generate-output-logprobs)). With `output_mode: "text"`, they are `ChatCompletionLogProbs` carrying the decoded token strings and their UTF-8 `bytes` for the sampled token and every entry in `top_logprobs`; a server started with `--return-tokens-as-token-ids` returns `token_id:N` placeholders there instead, as `/v1/completions` does. @@ -91,8 +103,10 @@ With `output_mode: "tokens"`, logprobs are `GenerateLogProbs` with an integer `t The server returns a 400 for: -- `output_mode: "text"` on a server without a tokenizer (`--tokens-only` or `--skip-tokenizer-init`). +- `output_mode: "text"` or `"derender"` on a server without a tokenizer (`--tokens-only` or `--skip-tokenizer-init`). - `output_mode: "text"` with `sampling_params.detokenize: false`. +- `output_mode: "derender"` with `stream: true`. +- `output_mode: "derender"` without `parse_context` on a server with a tool or reasoning parser. - An unsupported `output_mode` value. For using `output_mode` with separate prefill and decode pools, see [Disaggregated Prefilling](../../features/disagg_prefill.md#generate-api-output-modes). diff --git a/tests/entrypoints/scale_out/derender/test_derender_parity.py b/tests/entrypoints/scale_out/derender/test_derender_parity.py index af9bf1894a47..4f1726ef7f1c 100644 --- a/tests/entrypoints/scale_out/derender/test_derender_parity.py +++ b/tests/entrypoints/scale_out/derender/test_derender_parity.py @@ -878,3 +878,135 @@ async def test_text_parity_stop_string(client, stream): if coupled["stop_reason"] != stop: pytest.skip("Model did not emit the stop string") assert derendered[len(inline_text) :].startswith(stop) + + +# --------------------------------------------------------------------------- +# Inline derender level: generate(output_mode="derender") == chat completions +# --------------------------------------------------------------------------- + + +async def _inline_derender( + client: httpx.AsyncClient, messages: list[dict], **extra +) -> dict: + """Render a chat request, then generate it at the derender level with + the `parse_context` that `/render` returned.""" + render = await client.post( + "/v1/chat/completions/render", + json={ + "model": MODEL, + "messages": messages, + "temperature": 0, + "max_tokens": 128, + **extra, + }, + ) + assert render.status_code == 200, render.text + generate_request = render.json() + assert generate_request["parse_context"] is not None + + resp = await client.post( + "/inference/v1/generate", + json={**generate_request, "output_mode": "derender"}, + ) + assert resp.status_code == 200, resp.text + data = resp.json() + assert data["output_mode"] == "derender" + return data + + +def _assert_inline_parity(coupled: dict, inline: dict) -> None: + assert len(inline["choices"]) == len(coupled["choices"]) + for c, i in zip(coupled["choices"], inline["choices"]): + assert i["token_ids"] == c["token_ids"], ( + "greedy (temperature=0) generation was expected to be deterministic " + "across the coupled and inline calls" + ) + assert i["message"]["content"] == c["message"]["content"] + assert i["message"].get("reasoning") == c["message"].get("reasoning") + assert _tool_sig(i) == _tool_sig(c) + # TODO(#59324): compare tool call finish_reason once it is mapped. + if not _tool_sig(c): + assert i["finish_reason"] == c["finish_reason"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("content", "extra"), + [ + pytest.param(TEXT_MESSAGES[0]["content"], {}, id="plain"), + pytest.param( + "What is 17 times 23? Think it through.", + {"include_reasoning": True, "max_tokens": 256}, + id="reasoning", + ), + pytest.param( + "What is 17 times 23? Think it through.", + {"include_reasoning": False, "max_tokens": 256}, + id="reasoning-hidden", + ), + pytest.param( + "What's the weather in Paris?", + {"tools": TOOLS, "tool_choice": "auto", "max_tokens": 1024}, + id="tool-auto", + ), + pytest.param( + "What's the weather in Paris?", + {"tools": TOOLS, "tool_choice": "required", "max_tokens": 1024}, + id="tool-required", + ), + pytest.param( + "What's the weather in Paris?", + {"tools": TOOLS, "tool_choice": FORCE_WEATHER_TOOL, "max_tokens": 1024}, + id="tool-named", + ), + pytest.param( + "What's the weather in Paris and in Rome?", + { + "tools": TOOLS, + "tool_choice": "required", + "parallel_tool_calls": False, + "max_tokens": 1024, + }, + id="tool-no-parallel", + ), + pytest.param( + "Write two short lines about the sea.", {"stop": ["\n"]}, id="stop" + ), + pytest.param(TEXT_MESSAGES[0]["content"], {"n": 2}, id="n2"), + ], +) +async def test_inline_derender_parity(client, content, extra): + """The derender level parses the engine's own text, so it matches coupled + chat, including for hermes with tools whose `adjust_request` changes the + decode flags.""" + messages = [{"role": "user", "content": content}] + coupled = await _coupled(client, messages, **extra) + inline = await _inline_derender(client, messages, **extra) + _assert_inline_parity(coupled, inline) + + if extra.get("parallel_tool_calls") is False: + assert len(_tool_sig(inline["choices"][0])) <= 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("include_reasoning", [True, False]) +async def test_inline_derender_logprobs(client, include_reasoning): + """Logprobs match coupled chat and are dropped when reasoning is hidden.""" + messages = [{"role": "user", "content": "What is 2+2?"}] + extra = { + "logprobs": True, + "top_logprobs": 3, + "include_reasoning": include_reasoning, + } + inline = (await _inline_derender(client, messages, **extra))["choices"][0] + + if not include_reasoning: + assert inline["logprobs"] is None + return + + coupled = (await _coupled(client, messages, **extra))["choices"][0] + i_content = inline["logprobs"]["content"] + c_content = coupled["logprobs"]["content"] + assert [(e["token"], e["bytes"]) for e in i_content] == [ + (e["token"], e["bytes"]) for e in c_content + ] diff --git a/tests/entrypoints/scale_out/derender/test_derender_stream.py b/tests/entrypoints/scale_out/derender/test_derender_stream.py index 2044ee981e8e..c4fae33d844e 100644 --- a/tests/entrypoints/scale_out/derender/test_derender_stream.py +++ b/tests/entrypoints/scale_out/derender/test_derender_stream.py @@ -35,6 +35,7 @@ DeltaFunctionCall, DeltaMessage, DeltaToolCall, + FunctionCall, PerRequestMetrics, SpeculativeDecodingMetrics, ) @@ -55,6 +56,7 @@ GenerateTokensResponse, GenerateTokensStreamChoice, GenerateTokensStreamResponse, + ParseContext, ) from vllm.entrypoints.serve.engine.protocol import ErrorResponse, UsageInfo from vllm.parser import Parser @@ -427,6 +429,52 @@ def batch_content(token_ids: list[int], finish_reason: str) -> str: assert stream.choices[0].delta.content == expected +_WEATHER_TOOLS = [ + { + "type": "function", + "function": {"name": "get_weather", "parameters": {"type": "object"}}, + } +] + + +@pytest.mark.parametrize( + "tool_choice", + ["auto", "required", {"type": "function", "function": {"name": "get_weather"}}], +) +def test_derender_text_choices_builds_request_from_parse_context( + derenderer, monkeypatch, tool_choice +): + """Every tool_choice survives the synthetic request validators, the + history count is seeded and parallel_tool_calls=false keeps one call.""" + parser = MagicMock() + parser.parse.return_value = ( + None, + None, + [ + FunctionCall(name="get_weather", arguments="{}"), + FunctionCall(name="get_weather", arguments="{}"), + ], + ) + monkeypatch.setattr(derenderer, "parser", MagicMock(return_value=parser)) + parse_context = ParseContext.model_validate( + { + "tools": _WEATHER_TOOLS, + "tool_choice": tool_choice, + "parallel_tool_calls": False, + "history_tool_call_cnt": 3, + } + ) + output = MagicMock(text="raw", token_ids=[1, 2]) + + (message,) = derenderer._derender_text_choices([output], parse_context, [11]) + + chat_request = parser.parse.call_args.args[1] + assert chat_request.tool_choice == parse_context.tool_choice + parser.set_prompt_token_ids.assert_called_once_with([11]) + parser.set_history_tool_call_count.assert_called_once_with(3) + assert len(message.tool_calls) == 1 + + class TestDetokenizeDelta: """_detokenize_delta: chunked decode must equal one shot decode.""" diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_protocol.py b/tests/entrypoints/scale_out/token_in_token_out/test_protocol.py index 842dded1c76c..41ac3d0adf44 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_protocol.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_protocol.py @@ -17,6 +17,7 @@ DerenderChatStreamRequest, DerenderCompletionRequest, DerenderCompletionStreamRequest, + GenerateDerenderResponse, GenerateRequest, GenerateResponse, GenerateStreamResponse, @@ -158,7 +159,36 @@ def test_output_mode_text_is_accepted(): assert GenerateRequest.model_validate(payload).output_mode == "text" -@pytest.mark.parametrize("output_mode", ["derender", "bogus", None]) +def test_output_mode_derender_is_accepted_with_parse_context(): + payload = { + **_base_payload(), + "output_mode": "derender", + "parse_context": {"include_reasoning": False, "history_tool_call_cnt": 2}, + } + request = GenerateRequest.model_validate(payload) + assert request.output_mode == "derender" + assert request.parse_context is not None + assert request.parse_context.include_reasoning is False + assert request.parse_context.history_tool_call_cnt == 2 + + +def test_parse_context_is_accepted_at_other_levels(): + """/render always sets it, so a prefill leg must accept it unchanged.""" + payload = {**_base_payload(), "parse_context": {}} + assert GenerateRequest.model_validate(payload).output_mode == "tokens" + + +def test_derender_mode_rejects_detokenize_false(): + payload = { + "token_ids": [1, 2, 3], + "sampling_params": {"detokenize": False}, + "output_mode": "derender", + } + with pytest.raises(ValidationError, match="detokenize"): + GenerateRequest.model_validate(payload) + + +@pytest.mark.parametrize("output_mode", ["bogus", None]) def test_output_mode_rejects_unsupported_levels(output_mode): """Levels the server does not implement fail instead of falling back to tokens which would answer 200 with the wrong response shape.""" @@ -220,7 +250,24 @@ def test_text_response_requires_text_on_every_choice(): ) -@pytest.mark.parametrize("output_mode", ["derender", None, 5]) +def test_derender_response_dispatches_on_output_mode(): + parsed = TypeAdapter(GenerateResponse).validate_python( + { + "output_mode": "derender", + "choices": [ + { + "index": 0, + "token_ids": [1], + "message": {"role": "assistant", "content": "hi"}, + } + ], + } + ) + assert isinstance(parsed, GenerateDerenderResponse) + assert parsed.choices[0].message.content == "hi" + + +@pytest.mark.parametrize("output_mode", [None, 5]) def test_response_rejects_unknown_output_mode(output_mode): with pytest.raises(ValidationError): TypeAdapter(GenerateResponse).validate_python( @@ -240,6 +287,21 @@ def test_derender_requests_accept_responses_without_output_mode(): ) +def test_derender_chat_request_rejects_derender_responses(): + derendered = { + "output_mode": "derender", + "choices": [ + { + "index": 0, + "token_ids": [1], + "message": {"role": "assistant", "content": "hi"}, + } + ], + } + with pytest.raises(ValidationError, match="output_mode"): + DerenderChatRequest.model_validate({"generate_response": derendered}) + + def test_derender_requests_reject_text_responses(): """Text responses are already detokenized with stop strings applied, so re-decoding their token_ids would silently undo the truncation.""" diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py b/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py index 48b9dd509977..7e735ae15d18 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py @@ -743,7 +743,10 @@ async def test_text_mode_stream_matches_non_stream(client, tokenizer, messages): pytest.param( {"sampling_params": {"detokenize": False}}, "detokenize", id="no-detok" ), - pytest.param({"output_mode": "derender"}, "output_mode", id="unsupported"), + pytest.param({"output_mode": "bogus"}, "output_mode", id="unsupported"), + pytest.param( + {"output_mode": "derender", "stream": True}, "stream", id="derender-stream" + ), ], ) async def test_text_mode_invalid_requests_return_400(client, overrides, expected): @@ -757,6 +760,74 @@ async def test_text_mode_invalid_requests_return_400(client, overrides, expected assert expected in resp.json()["error"]["message"] +@pytest.mark.asyncio +@pytest.mark.skipif( + envs.VLLM_USE_RUST_FRONTEND, + reason="output_mode is not supported by the Rust frontend", +) +async def test_derender_mode_without_parser_returns_text_message( + client, tokenizer, messages +): + """With no parser the message content is the engine's detokenized text.""" + token_ids = _chat_prompt_token_ids(tokenizer, messages) + payload = _text_mode_payload(token_ids) + + text = (await client.post(GEN_ENDPOINT, json=payload)).json() + resp = await client.post(GEN_ENDPOINT, json={**payload, "output_mode": "derender"}) + resp.raise_for_status() + data = resp.json() + + assert data["output_mode"] == "derender" + assert data["choices"][0]["message"]["content"] == text["choices"][0]["text"] + assert data["choices"][0]["token_ids"] == text["choices"][0]["token_ids"] + + +@pytest.mark.asyncio +@pytest.mark.skipif( + envs.VLLM_USE_RUST_FRONTEND, + reason="output_mode is not supported by the Rust frontend", +) +@pytest.mark.parametrize( + "server", + [["--tool-call-parser", "hermes", "--enable-auto-tool-choice"]], + indirect=True, +) +async def test_derender_mode_with_parser_matches_chat_completions(client, messages): + chat = { + "model": MODEL_NAME, + "messages": messages, + "max_tokens": 24, + "temperature": 0.0, + "chat_template_kwargs": {"enable_thinking": False}, + } + render = await client.post("/v1/chat/completions/render", json=chat) + render.raise_for_status() + generate_request = render.json() + assert generate_request["parse_context"] is not None + + missing = await client.post( + GEN_ENDPOINT, + json={ + **{k: v for k, v in generate_request.items() if k != "parse_context"}, + "output_mode": "derender", + }, + ) + assert missing.status_code == 400 + assert "parse_context" in missing.json()["error"]["message"] + + resp = await client.post( + GEN_ENDPOINT, json={**generate_request, "output_mode": "derender"} + ) + resp.raise_for_status() + chat_resp = await client.post("/v1/chat/completions", json=chat) + + expected = chat_resp.json()["choices"][0] + got = resp.json()["choices"][0] + assert got["message"]["content"] == expected["message"]["content"] + assert got["message"]["tool_calls"] == expected["message"]["tool_calls"] + assert got["finish_reason"] == expected["finish_reason"] + + @pytest.mark.asyncio @pytest.mark.skipif( envs.VLLM_USE_RUST_FRONTEND, diff --git a/vllm/entrypoints/scale_out/factories.py b/vllm/entrypoints/scale_out/factories.py index 45bb9dbcde6b..feba9d275970 100644 --- a/vllm/entrypoints/scale_out/factories.py +++ b/vllm/entrypoints/scale_out/factories.py @@ -60,6 +60,7 @@ def init_scale_out_state( engine_client, state.openai_serving_models, state.online_renderer, + online_derenderer=state.online_derenderer, request_logger=request_logger, return_tokens_as_token_ids=args.return_tokens_as_token_ids, enable_prompt_tokens_details=args.enable_prompt_tokens_details, diff --git a/vllm/entrypoints/scale_out/render/serving.py b/vllm/entrypoints/scale_out/render/serving.py index 260402413515..80afbf91f732 100644 --- a/vllm/entrypoints/scale_out/render/serving.py +++ b/vllm/entrypoints/scale_out/render/serving.py @@ -20,6 +20,7 @@ from vllm.entrypoints.scale_out.token_in_token_out.protocol import ( GenerateRequest, MultiModalFeatures, + ParseContext, ReasoningParserKwargs, ) from vllm.entrypoints.serve.engine.protocol import ErrorResponse @@ -29,11 +30,15 @@ from vllm.inputs import EngineInput from vllm.logger import init_logger from vllm.multimodal.parse import MultiModalDataParser +from vllm.parser.utils import count_history_tool_calls from vllm.renderers.inputs.preprocess import ( extract_prompt_components, extract_prompt_len, ) -from vllm.renderers.online_renderer import OnlineRenderer +from vllm.renderers.online_renderer import ( + OnlineRenderer, + resolve_chat_template_kwargs, +) from vllm.sampling_params import SamplingParams logger = init_logger(__name__) @@ -184,6 +189,24 @@ async def render_chat_request( cache_salt=request.cache_salt, priority=request.priority, token_offsets=engine_input.get("prompt_token_offsets"), + parse_context=self._build_parse_context(request), + ) + + def _build_parse_context(self, request: ChatCompletionRequest) -> ParseContext: + renderer = self.online_renderer + return ParseContext( + tools=request.tools, + tool_choice=request.tool_choice, + parallel_tool_calls=request.parallel_tool_calls, + include_reasoning=request.include_reasoning, + chat_template_kwargs=resolve_chat_template_kwargs( + request, + renderer.chat_template, + renderer.chat_template_content_format, + renderer.default_chat_template_kwargs, + use_harmony=renderer.use_harmony, + ), + history_tool_call_cnt=count_history_tool_calls(request), ) async def render_messages_request( diff --git a/vllm/entrypoints/scale_out/token_in_token_out/protocol.py b/vllm/entrypoints/scale_out/token_in_token_out/protocol.py index c8d82eeb5a23..3a1f97ce8eda 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/protocol.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/protocol.py @@ -22,8 +22,11 @@ ) from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionLogProbs, + ChatCompletionNamedToolChoiceParam, ChatCompletionRequest, ChatCompletionStreamResponse, + ChatCompletionToolsParam, + ChatMessage, ) from vllm.entrypoints.openai.completion.protocol import ( CompletionRequest, @@ -38,7 +41,26 @@ ####### Tokens IN <> Tokens OUT ####### -OutputMode: TypeAlias = Literal["tokens", "text"] +OutputMode: TypeAlias = Literal["tokens", "text", "derender"] + + +class ParseContext(BaseModel): + """Chat request context the generate server needs to parse output. + + Built by `/render` from the post-`adjust_request` chat request and read + only when `output_mode="derender"`. + """ + + tools: list[ChatCompletionToolsParam] | None = None + tool_choice: ( + Literal["none", "auto", "required"] | ChatCompletionNamedToolChoiceParam | None + ) = "none" + parallel_tool_calls: bool | None = True + include_reasoning: bool = True + chat_template_kwargs: dict[str, Any] = Field(default_factory=dict) + """Template kwargs with the render server's defaults already applied.""" + history_tool_call_cnt: int = Field(default=0, ge=0) + """Tool calls already in the rendered conversation, for sequential IDs.""" class PlaceholderRangeInfo(BaseModel): @@ -247,8 +269,19 @@ def _require_ec_for_metadata_only(self) -> "GenerateRequest": description=( "Response level. 'tokens' returns token IDs only. 'text' also " "returns the detokenized text and logprobs with decoded token " - "strings. 'text' needs a server that loads a tokenizer and " - "sampling_params.detokenize to be true." + "strings. 'derender' returns the parsed assistant message and " + "needs `parse_context` when the server has a parser. 'text' and " + "'derender' need a server that loads a tokenizer and " + "sampling_params.detokenize to be true. 'derender' does not " + "support `stream`." + ), + ) + parse_context: ParseContext | None = Field( + default=None, + description=( + "Chat context from `/render`. Read only when " + "output_mode='derender' and ignored at other levels, so a " + "rendered request can be sent to a prefill leg unchanged." ), ) @@ -492,6 +525,15 @@ class GenerateTextStreamResponse(GenerateStreamResponseBase): choices: list[GenerateTextStreamChoice] +class GenerateDerenderChoice(GenerateChoiceBase): + message: ChatMessage + """Assistant message parsed from the output text.""" + + logprobs: ChatCompletionLogProbs | None = None + """Logprobs with decoded token strings and `bytes`. None when reasoning + is hidden by `include_reasoning=false`.""" + + class GenerateResponseBase(BaseModel): request_id: str = Field( default_factory=lambda: f"{random_uuid()}", @@ -532,6 +574,11 @@ class GenerateTextResponse(GenerateResponseBase): choices: list[GenerateTextChoice] +class GenerateDerenderResponse(GenerateResponseBase): + output_mode: Literal["derender"] = "derender" + choices: list[GenerateDerenderChoice] + + def output_mode_or_tokens(value: Any) -> Any: """Discriminator for `GenerateResponse` and `GenerateStreamResponse`. @@ -548,7 +595,8 @@ def output_mode_or_tokens(value: Any) -> Any: # `Generate...` classes to construct. GenerateResponse: TypeAlias = Annotated[ Annotated[GenerateTokensResponse, Tag("tokens")] - | Annotated[GenerateTextResponse, Tag("text")], + | Annotated[GenerateTextResponse, Tag("text")] + | Annotated[GenerateDerenderResponse, Tag("derender")], Discriminator(output_mode_or_tokens), ] GenerateStreamResponse: TypeAlias = Annotated[ diff --git a/vllm/entrypoints/scale_out/token_in_token_out/serving.py b/vllm/entrypoints/scale_out/token_in_token_out/serving.py index 02a68108662a..d1a9028b2148 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/serving.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/serving.py @@ -36,7 +36,7 @@ ) from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage from vllm.entrypoints.serve.utils.request_logger import RequestLogger -from vllm.exceptions import GenerationError +from vllm.exceptions import GenerationError, VLLMValidationError from vllm.inputs import EngineInput, TokensPrompt, mm_input from vllm.logger import init_logger from vllm.logprobs import Logprob @@ -47,7 +47,7 @@ ) from vllm.outputs import RequestOutput from vllm.renderers.chat_utils import AsyncMultiModalItemTracker -from vllm.renderers.online_renderer import OnlineRenderer +from vllm.renderers.online_derenderer import OnlineDerenderer from vllm.sampling_params import RequestOutputKind, SamplingParams from vllm.tokenizers import TokenizerLike from vllm.utils.collection_utils import as_list @@ -61,6 +61,8 @@ GenerateLogProb, GenerateLogProbs, GenerateLogProbsContent, + GenerateDerenderChoice, + GenerateDerenderResponse, GenerateRequest, GenerateResponse, GenerateTextChoice, @@ -120,6 +122,7 @@ def __init__( models: OpenAIServingModels, online_renderer: OnlineRenderer, *, + online_derenderer: OnlineDerenderer | None = None, request_logger: RequestLogger | None, force_no_detokenize: bool = False, return_tokens_as_token_ids: bool = False, @@ -133,6 +136,7 @@ def __init__( return_tokens_as_token_ids=return_tokens_as_token_ids, ) self.online_renderer = online_renderer + self.online_derenderer = online_derenderer self.enable_prompt_tokens_details = enable_prompt_tokens_details self.enable_log_outputs = enable_log_outputs self.force_no_detokenize = force_no_detokenize @@ -230,6 +234,24 @@ async def serve_tokens( "--skip-tokenizer-init). Send the request to a server that " "loads a tokenizer, or use output_mode='tokens'." ) + if request.output_mode == "derender": + # TODO(#57442): reject return_token_logprobs once it lands. + if request.stream: + return self.create_error_response( + "output_mode='derender' does not support stream=true." + ) + if self.online_derenderer is None: + return self.create_error_response( + "output_mode='derender' is not available on this server." + ) + if self.online_derenderer.parser is not None and ( + request.parse_context is None + ): + return self.create_error_response( + "output_mode='derender' requires parse_context on a server " + "with a tool or reasoning parser. Use the one returned by " + "/render." + ) try: msgspec.msgpack.encode( ( @@ -396,7 +418,8 @@ async def serve_tokens_full_generator( created_time = int(time.time()) final_res: RequestOutput | None = None sampling_params: SamplingParams = request.sampling_params - text_mode = request.output_mode == "text" + derender_mode = request.output_mode == "derender" + text_mode = request.output_mode == "text" or derender_mode tokenizer = self._logprobs_tokenizer(text_mode) try: @@ -409,6 +432,7 @@ async def serve_tokens_full_generator( tokens_choices: list[GenerateTokensChoice] = [] text_choices: list[GenerateTextChoice] = [] + derender_choice_fields: list[dict[str, Any]] = [] num_generated_tokens = 0 for output in final_res.outputs: self._raise_if_error(output.finish_reason, request_id) @@ -454,7 +478,9 @@ async def serve_tokens_full_generator( sampling_mask=sampling_mask, weight_versions=output.weight_versions, ) - if text_mode: + if derender_mode: + derender_choice_fields.append(choice_fields) + elif text_mode: text_choices.append( GenerateTextChoice(text=output.text, **choice_fields) ) @@ -462,6 +488,28 @@ async def serve_tokens_full_generator( tokens_choices.append(GenerateTokensChoice(**choice_fields)) num_generated_tokens += len(output.token_ids) + derender_choices: list[GenerateDerenderChoice] = [] + if derender_mode: + assert self.online_derenderer is not None + parse_context = request.parse_context + try: + messages = await self.online_derenderer.derender_text_choices( + final_res.outputs, parse_context, final_res.prompt_token_ids + ) + except (ValueError, VLLMValidationError) as e: + return self.create_error_response(e) + hide_logprobs = ( + parse_context is not None and not parse_context.include_reasoning + ) + # TODO(#59324): map finish_reason with resolve_finish_reason so + # auto tool calls report "tool_calls" like coupled chat. + for message, fields in zip(messages, derender_choice_fields, strict=True): + if hide_logprobs: + fields["logprobs"] = None + derender_choices.append( + GenerateDerenderChoice(message=message, **fields) + ) + assert final_res.prompt_token_ids is not None num_prompt_tokens = len(final_res.prompt_token_ids) if final_res.encoder_prompt_token_ids is not None: @@ -507,8 +555,14 @@ async def serve_tokens_full_generator( kv_transfer_params=final_res.kv_transfer_params, ec_transfer_params=final_res.ec_transfer_params, ) - response: GenerateTokensResponse | GenerateTextResponse - if text_mode: + response: ( + GenerateTokensResponse | GenerateTextResponse | GenerateDerenderResponse + ) + if derender_mode: + response = GenerateDerenderResponse( + choices=derender_choices, **response_fields + ) + elif text_mode: response = GenerateTextResponse(choices=text_choices, **response_fields) else: response = GenerateTokensResponse(choices=tokens_choices, **response_fields) diff --git a/vllm/parser/abstract_parser.py b/vllm/parser/abstract_parser.py index 287cad5c24c2..6b145e1d27d7 100644 --- a/vllm/parser/abstract_parser.py +++ b/vllm/parser/abstract_parser.py @@ -196,6 +196,12 @@ def _initialize_history_tool_call_cnt( state.history_tool_call_cnt = count_history_tool_calls(request) state.history_tool_call_cnt_initialized = True + def set_history_tool_call_count(self, count: int) -> None: + """Seed the history count when `request.messages` is not available.""" + state = self._stream_state + state.history_tool_call_cnt = count + state.history_tool_call_cnt_initialized = True + def adjust_request( self, request: ChatCompletionRequest | ResponsesRequest ) -> ChatCompletionRequest | ResponsesRequest: diff --git a/vllm/renderers/online_derenderer.py b/vllm/renderers/online_derenderer.py index c8d4a07585c5..b8cd7f1c105e 100644 --- a/vllm/renderers/online_derenderer.py +++ b/vllm/renderers/online_derenderer.py @@ -40,17 +40,20 @@ GenerateLogProbs, GenerateTokensResponse, GenerateTokensStreamResponse, + ParseContext, ) from vllm.entrypoints.serve.engine.protocol import UsageInfo from vllm.entrypoints.serve.utils.request_logger import RequestLogger from vllm.entrypoints.serve.utils.tool_calls_utils import resolve_finish_reason from vllm.logger import init_logger +from vllm.outputs import CompletionOutput from vllm.parser import Parser, ParserManager from vllm.renderers import BaseRenderer from vllm.renderers.chat_utils import ( ChatTemplateContentFormatOption, make_tool_call_id, ) +from vllm.renderers.online_renderer import resolve_chat_template_kwargs from vllm.tokenizers import TokenizerLike from vllm.tokenizers.detokenizer_utils import ( convert_prompt_ids_to_tokens, @@ -116,6 +119,9 @@ def __init__( self._derender_chat_async = make_async( self._derender_chat, executor=renderer._executor ) + self._derender_text_choices_async = make_async( + self._derender_text_choices, executor=renderer._executor + ) self._derender_completion_async = make_async( self._derender_completion, executor=renderer._executor ) @@ -278,18 +284,69 @@ def _derender_chat( return choices + async def derender_text_choices( + self, + outputs: Sequence[CompletionOutput], + parse_context: ParseContext | None, + prompt_token_ids: list[int] | None, + ) -> list[ChatMessage]: + """Parse the engine's decoded output text into assistant messages. + + Args: + outputs: Final engine outputs, already detokenized. + parse_context: Chat context from `/render`. None is only valid + on a server without a parser. + prompt_token_ids: Prompt token IDs, if known. + + Returns: + One message per output, in order. + + """ + return await self._derender_text_choices_async( + outputs, parse_context, prompt_token_ids + ) + + def _derender_text_choices( + self, + outputs: Sequence[CompletionOutput], + parse_context: ParseContext | None, + prompt_token_ids: list[int] | None, + ) -> list[ChatMessage]: + if self.parser is None: + return [ChatMessage(role="assistant", content=o.text) for o in outputs] + + assert parse_context is not None + tokenizer = self.renderer.get_tokenizer() + # Validate from plain data: the request validators only accept a + # named tool_choice as a dict. + chat_request = ChatCompletionRequest.model_validate( + { + "messages": [], + **parse_context.model_dump(exclude={"history_tool_call_cnt"}), + } + ) + return [ + self._parse_and_assemble( + tokenizer, + output.text, + output.token_ids, + chat_request, + parse_context.chat_template_kwargs, + prompt_token_ids, + history_tool_call_count=parse_context.history_tool_call_cnt, + ) + for output in outputs + ] + def _resolve_chat_template_kwargs( self, chat_request: ChatCompletionRequest ) -> dict[str, Any]: - if self.use_harmony: - return {} - return ( - chat_request.build_chat_params( - self.chat_template, - self.chat_template_content_format, - ) - .with_defaults(self.default_chat_template_kwargs) - .chat_template_kwargs + return resolve_chat_template_kwargs( + chat_request, + self.chat_template, + self.chat_template_content_format, + self.default_chat_template_kwargs, + use_harmony=self.use_harmony, ) def _parse_and_assemble( @@ -300,6 +357,7 @@ def _parse_and_assemble( chat_request: ChatCompletionRequest, chat_template_kwargs: dict[str, Any], prompt_token_ids: list[int] | None, + history_tool_call_count: int | None = None, ) -> ChatMessage: """Parse decoded output text into an assistant `ChatMessage`. @@ -311,6 +369,9 @@ def _parse_and_assemble( `include_reasoning`. chat_template_kwargs: Already resolved template kwargs. prompt_token_ids: Prompt token IDs, if known. + history_tool_call_count: Tool calls in the conversation, for + requests that carry no messages. None derives it from + `chat_request`. Returns: The assembled assistant message. @@ -325,6 +386,8 @@ def _parse_and_assemble( ) if prompt_token_ids is not None: parser.set_prompt_token_ids(prompt_token_ids) + if history_tool_call_count is not None: + parser.set_history_tool_call_count(history_tool_call_count) reasoning, content, tool_calls = parser.parse( text, chat_request, @@ -338,7 +401,7 @@ def _parse_and_assemble( tc_items = ( [ ToolCall(id=tc.id or make_tool_call_id(), function=tc) - for tc in tool_calls + for tc in maybe_filter_parallel_tool_calls(tool_calls, chat_request) ] if tool_calls else [] diff --git a/vllm/renderers/online_renderer.py b/vllm/renderers/online_renderer.py index 65771211d0a5..5b109e1f51c7 100644 --- a/vllm/renderers/online_renderer.py +++ b/vllm/renderers/online_renderer.py @@ -151,6 +151,27 @@ def _reused_prompt_token_ids( return ids +def resolve_chat_template_kwargs( + request: ChatCompletionRequest, + chat_template: str | None, + chat_template_content_format: ChatTemplateContentFormatOption, + default_chat_template_kwargs: dict[str, Any], + *, + use_harmony: bool, +) -> dict[str, Any]: + """Template kwargs for `request` with the server defaults applied. + + Harmony models don't render through a chat template, so they get none. + """ + if use_harmony: + return {} + return ( + request.build_chat_params(chat_template, chat_template_content_format) + .with_defaults(default_chat_template_kwargs) + .chat_template_kwargs + ) + + class OnlineRenderer: def __init__( self, From 675f502bd9afa94b9853569721a416a263e36d86 Mon Sep 17 00:00:00 2001 From: Martin Hickey Date: Fri, 16 Oct 2026 12:20:52 +0100 Subject: [PATCH 02/11] Fix after rebase Signed-off-by: Martin Hickey --- vllm/entrypoints/scale_out/token_in_token_out/serving.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/vllm/entrypoints/scale_out/token_in_token_out/serving.py b/vllm/entrypoints/scale_out/token_in_token_out/serving.py index d1a9028b2148..e7985f675a3f 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/serving.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/serving.py @@ -48,6 +48,7 @@ from vllm.outputs import RequestOutput from vllm.renderers.chat_utils import AsyncMultiModalItemTracker from vllm.renderers.online_derenderer import OnlineDerenderer +from vllm.renderers.online_renderer import OnlineRenderer from vllm.sampling_params import RequestOutputKind, SamplingParams from vllm.tokenizers import TokenizerLike from vllm.utils.collection_utils import as_list @@ -58,11 +59,11 @@ placeholder_ranges_from_engine_input, ) from .protocol import ( + GenerateDerenderChoice, + GenerateDerenderResponse, GenerateLogProb, GenerateLogProbs, GenerateLogProbsContent, - GenerateDerenderChoice, - GenerateDerenderResponse, GenerateRequest, GenerateResponse, GenerateTextChoice, From abe5a845b2f01d18599dddd4c88abf7853252876 Mon Sep 17 00:00:00 2001 From: Martin Hickey Date: Fri, 16 Oct 2026 14:04:46 +0100 Subject: [PATCH 03/11] [Frontend] Map finish_reason and cut logprobs in generate derender mode The derender level now reports finish_reason the way coupled chat does via resolve_finish_reason. This means auto and required tool calls come back as "tool_calls" and a named tool_choice as "stop". Logprobs at this level now follow chat too. They turn on whenever num_logprobs is set. logprob_token_ids therefore works and top_logprobs is cut with chat_top_logprobs_limit. The tokens and text levels don't change. The parity tests now check finish_reason and tool call ID shape on every case and there's a new logprob_token_ids case. Also updates to the docs and adds #60915 to the logprob flag TODO. Signed-off-by: Martin Hickey --- .../online_serving/token_in_token_out.md | 6 +-- .../derender/test_derender_parity.py | 37 +++++++++++----- .../derender/test_derender_stream.py | 8 +++- .../scale_out/token_in_token_out/serving.py | 44 +++++++++++++------ vllm/renderers/online_derenderer.py | 26 +++++++---- 5 files changed, 84 insertions(+), 37 deletions(-) diff --git a/docs/serving/online_serving/token_in_token_out.md b/docs/serving/online_serving/token_in_token_out.md index 8f56eff1a836..0830e143a95a 100644 --- a/docs/serving/online_serving/token_in_token_out.md +++ b/docs/serving/online_serving/token_in_token_out.md @@ -54,7 +54,7 @@ The request field `output_mode` selects how much of the postprocessing the serve | --- | --- | --- | | `tokens` (default) | nothing, token IDs only | nothing | | `text` | `text` on every choice and decoded tokens with `bytes` in `logprobs` | a tokenizer | -| `derender` | the parsed assistant `message` (`content`, `reasoning`, `tool_calls`) on every choice, plus the `text` level's `logprobs` | a tokenizer and the parser flags you'd give `vllm serve` | +| `derender` | the parsed assistant `message` (`content`, `reasoning`, `tool_calls`) on every choice, plus `logprobs` shaped like coupled chat's | a tokenizer and the parser flags you'd give `vllm serve` | `text` returns text and token IDs in a single call, without a separate [derender](derenderer.md) hop per chunk. This suits latency sensitive streaming, where the derender hop lands on every output token. @@ -91,8 +91,8 @@ print(response["choices"][0]["token_ids"]) - The parser reads the engine's `text`, so it sees the stop string cut and the post `adjust_request` decode flags. That makes it match coupled chat for parsers whose `adjust_request` changes decoding which `/derender` can't. - A server with a tool or reasoning parser needs `parse_context`. Without a parser, `message.content` is the plain text. - `parse_context` is ignored at other levels, so a rendered request can go to a prefill leg unchanged. -- `logprobs` are dropped when `include_reasoning` is false, so they don't expose the hidden reasoning. Coupled chat still returns them until [#59590](https://github.com/vllm-project/vllm/pull/59590) lands. -- `finish_reason` is the engine's value. Coupled chat reports `tool_calls` when a tool is called which this level doesn't do yet. +- `logprobs` follow coupled chat. `top_logprobs` is cut to `sampling_params.logprobs`, `logprob_token_ids` is honored and `logprobs` are dropped when `include_reasoning` is false so they don't expose the hidden reasoning. +- `finish_reason` follows coupled chat too. It's `tool_calls` when the message has tool calls except with a named `tool_choice` or a `length` cut. - `stream: true` isn't supported yet. ### Logprobs diff --git a/tests/entrypoints/scale_out/derender/test_derender_parity.py b/tests/entrypoints/scale_out/derender/test_derender_parity.py index 4f1726ef7f1c..36e75f496e34 100644 --- a/tests/entrypoints/scale_out/derender/test_derender_parity.py +++ b/tests/entrypoints/scale_out/derender/test_derender_parity.py @@ -924,9 +924,8 @@ def _assert_inline_parity(coupled: dict, inline: dict) -> None: assert i["message"]["content"] == c["message"]["content"] assert i["message"].get("reasoning") == c["message"].get("reasoning") assert _tool_sig(i) == _tool_sig(c) - # TODO(#59324): compare tool call finish_reason once it is mapped. - if not _tool_sig(c): - assert i["finish_reason"] == c["finish_reason"] + assert _tool_id_shapes(i) == _tool_id_shapes(c) + assert i["finish_reason"] == c["finish_reason"] @pytest.mark.asyncio @@ -988,16 +987,24 @@ async def test_inline_derender_parity(client, content, extra): assert len(_tool_sig(inline["choices"][0])) <= 1 +def _top_tokens(entry: dict) -> list[str]: + return [top["token"] for top in entry["top_logprobs"]] + + @pytest.mark.asyncio -@pytest.mark.parametrize("include_reasoning", [True, False]) -async def test_inline_derender_logprobs(client, include_reasoning): - """Logprobs match coupled chat and are dropped when reasoning is hidden.""" +@pytest.mark.parametrize( + ("include_reasoning", "extra"), + [ + pytest.param(True, {"top_logprobs": 3}, id="top3"), + pytest.param(False, {"top_logprobs": 3}, id="top3-reasoning-hidden"), + pytest.param(True, {"logprob_token_ids": [15, 16, 17]}, id="token-ids"), + ], +) +async def test_inline_derender_logprobs(client, include_reasoning, extra): + """Logprobs match coupled chat, including the `top_logprobs` cut and + `logprob_token_ids`, and are dropped when reasoning is hidden.""" messages = [{"role": "user", "content": "What is 2+2?"}] - extra = { - "logprobs": True, - "top_logprobs": 3, - "include_reasoning": include_reasoning, - } + extra = {"logprobs": True, "include_reasoning": include_reasoning, **extra} inline = (await _inline_derender(client, messages, **extra))["choices"][0] if not include_reasoning: @@ -1010,3 +1017,11 @@ async def test_inline_derender_logprobs(client, include_reasoning): assert [(e["token"], e["bytes"]) for e in i_content] == [ (e["token"], e["bytes"]) for e in c_content ] + if "top_logprobs" in extra: + # Near ties can reorder candidates across the two generations, so + # compare the cut, not the candidates. + assert all(len(e["top_logprobs"]) == 3 for e in i_content) + else: + assert [sorted(_top_tokens(e)) for e in i_content] == [ + sorted(_top_tokens(e)) for e in c_content + ] diff --git a/tests/entrypoints/scale_out/derender/test_derender_stream.py b/tests/entrypoints/scale_out/derender/test_derender_stream.py index c4fae33d844e..820ce1c3ffc3 100644 --- a/tests/entrypoints/scale_out/derender/test_derender_stream.py +++ b/tests/entrypoints/scale_out/derender/test_derender_stream.py @@ -464,15 +464,19 @@ def test_derender_text_choices_builds_request_from_parse_context( "history_tool_call_cnt": 3, } ) - output = MagicMock(text="raw", token_ids=[1, 2]) + output = MagicMock(text="raw", token_ids=[1, 2], finish_reason="stop") - (message,) = derenderer._derender_text_choices([output], parse_context, [11]) + ((message, finish_reason),) = derenderer._derender_text_choices( + [output], parse_context, [11] + ) chat_request = parser.parse.call_args.args[1] assert chat_request.tool_choice == parse_context.tool_choice parser.set_prompt_token_ids.assert_called_once_with([11]) parser.set_history_tool_call_count.assert_called_once_with(3) assert len(message.tool_calls) == 1 + # A forced named tool reports "stop", like coupled chat. + assert finish_reason == ("stop" if isinstance(tool_choice, dict) else "tool_calls") class TestDetokenizeDelta: diff --git a/vllm/entrypoints/scale_out/token_in_token_out/serving.py b/vllm/entrypoints/scale_out/token_in_token_out/serving.py index e7985f675a3f..9b943c009d5f 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/serving.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/serving.py @@ -3,6 +3,7 @@ import asyncio +import itertools import math import time from collections.abc import AsyncGenerator @@ -20,6 +21,7 @@ from vllm.entrypoints.generate.base.serving import ( GenerateBaseServing, build_spec_decoding_metrics, + chat_top_logprobs_limit, clamp_prompt_logprobs, format_token_id_placeholder, ) @@ -236,7 +238,8 @@ async def serve_tokens( "loads a tokenizer, or use output_mode='tokens'." ) if request.output_mode == "derender": - # TODO(#57442): reject return_token_logprobs once it lands. + # TODO(#57442, #60915): reject the sampled-only and top-k logprob + # flags here once either lands. if request.stream: return self.create_error_response( "output_mode='derender' does not support stream=true." @@ -422,6 +425,16 @@ async def serve_tokens_full_generator( derender_mode = request.output_mode == "derender" text_mode = request.output_mode == "text" or derender_mode tokenizer = self._logprobs_tokenizer(text_mode) + # The derender level returns chat logprobs, so it honors + # logprob_token_ids and applies the chat top_logprobs cut. + if derender_mode: + num_logprobs = sampling_params.num_logprobs + top_limit = chat_top_logprobs_limit( + sampling_params.logprobs, bool(sampling_params.logprob_token_ids) + ) + else: + num_logprobs = sampling_params.logprobs + top_limit = None try: async for res in result_generator: @@ -442,7 +455,7 @@ async def serve_tokens_full_generator( out_logprobs = output.logprobs # This is top_logprobs in completions API - if sampling_params.logprobs is not None: + if num_logprobs is not None: assert out_logprobs is not None, "Did not output logprobs" logprobs: GenerateLogProbs | ChatCompletionLogProbs | None if text_mode: @@ -450,6 +463,7 @@ async def serve_tokens_full_generator( token_ids=token_ids, top_logprobs=out_logprobs, tokenizer=tokenizer, + top_limit=top_limit, ) else: logprobs = self._create_tokens_logprobs( @@ -494,7 +508,7 @@ async def serve_tokens_full_generator( assert self.online_derenderer is not None parse_context = request.parse_context try: - messages = await self.online_derenderer.derender_text_choices( + parsed = await self.online_derenderer.derender_text_choices( final_res.outputs, parse_context, final_res.prompt_token_ids ) except (ValueError, VLLMValidationError) as e: @@ -502,9 +516,10 @@ async def serve_tokens_full_generator( hide_logprobs = ( parse_context is not None and not parse_context.include_reasoning ) - # TODO(#59324): map finish_reason with resolve_finish_reason so - # auto tool calls report "tool_calls" like coupled chat. - for message, fields in zip(messages, derender_choice_fields, strict=True): + for (message, finish_reason), fields in zip( + parsed, derender_choice_fields, strict=True + ): + fields["finish_reason"] = finish_reason if hide_logprobs: fields["logprobs"] = None derender_choices.append( @@ -780,14 +795,15 @@ def _create_text_logprobs( token_ids: GenericSequence[int], top_logprobs: GenericSequence[dict[int, Logprob] | None], tokenizer: TokenizerLike | None = None, + top_limit: int | None = None, ) -> ChatCompletionLogProbs: - """Create OpenAI-style logprobs for ``output_mode="text"``. + """Create OpenAI style logprobs for `output_mode="text"`. - With a ``tokenizer`` the tokens are decoded strings and carry ``bytes``; - without one (``--return-tokens-as-token-ids``) they are ``token_id:N`` - placeholders. Like tokens mode, ``top_logprobs`` carries every candidate - the engine returned (the sampled token first, then the top k), not cut - to the requested count. + With a `tokenizer` the tokens are decoded strings and carry `bytes`. + Without one (`--return-tokens-as-token-ids`) they are `token_id:N` + placeholders. `top_logprobs` keeps the first `top_limit` candidates + the engine returned (the sampled token first, then the top k) or all + of them when `top_limit` is None, as in tokens mode. """ logprobs_content: list[ChatCompletionLogProbsContent] = [] @@ -809,7 +825,9 @@ def _create_text_logprobs( bytes=token_bytes, top_logprobs=[ _top_logprob(top_id, logprob, tokenizer) - for top_id, logprob in step_top_logprobs.items() + for top_id, logprob in itertools.islice( + step_top_logprobs.items(), top_limit + ) ], ) ) diff --git a/vllm/renderers/online_derenderer.py b/vllm/renderers/online_derenderer.py index b8cd7f1c105e..a5693228fb45 100644 --- a/vllm/renderers/online_derenderer.py +++ b/vllm/renderers/online_derenderer.py @@ -289,7 +289,7 @@ async def derender_text_choices( outputs: Sequence[CompletionOutput], parse_context: ParseContext | None, prompt_token_ids: list[int] | None, - ) -> list[ChatMessage]: + ) -> list[tuple[ChatMessage, str]]: """Parse the engine's decoded output text into assistant messages. Args: @@ -299,7 +299,7 @@ async def derender_text_choices( prompt_token_ids: Prompt token IDs, if known. Returns: - One message per output, in order. + One message and chat `finish_reason` per output in order. """ return await self._derender_text_choices_async( @@ -311,9 +311,15 @@ def _derender_text_choices( outputs: Sequence[CompletionOutput], parse_context: ParseContext | None, prompt_token_ids: list[int] | None, - ) -> list[ChatMessage]: + ) -> list[tuple[ChatMessage, str]]: if self.parser is None: - return [ChatMessage(role="assistant", content=o.text) for o in outputs] + return [ + ( + ChatMessage(role="assistant", content=o.text), + resolve_finish_reason(o.finish_reason, None, False), + ) + for o in outputs + ] assert parse_context is not None tokenizer = self.renderer.get_tokenizer() @@ -325,8 +331,9 @@ def _derender_text_choices( **parse_context.model_dump(exclude={"history_tool_call_cnt"}), } ) - return [ - self._parse_and_assemble( + results: list[tuple[ChatMessage, str]] = [] + for output in outputs: + message = self._parse_and_assemble( tokenizer, output.text, output.token_ids, @@ -335,8 +342,11 @@ def _derender_text_choices( prompt_token_ids, history_tool_call_count=parse_context.history_tool_call_cnt, ) - for output in outputs - ] + finish_reason = resolve_finish_reason( + output.finish_reason, chat_request, bool(message.tool_calls) + ) + results.append((message, finish_reason)) + return results def _resolve_chat_template_kwargs( self, chat_request: ChatCompletionRequest From d2cc530bf0d7590880318a0fbfd654b7d1803a1c Mon Sep 17 00:00:00 2001 From: Martin Hickey Date: Fri, 16 Oct 2026 15:10:56 +0100 Subject: [PATCH 04/11] Validate parse_context before generating in derender mode Build the chat request from parse_context in serve_tokens which makes a bad context gets a 400 before the engine runs. The validated request is passed down instead of the Optional context which drops the asserts flagged in review. online_derenderer is now required on ServingTokens. When a parser hides the reasoning, drop token_ids along with logprobs like coupled chat does and only do it when the server has a parser. ParseContext now rejects unknown fields. Removed the duplicate template kwargs helper and resolve the kwargs once in /render. Signed-off-by: Martin Hickey --- .../online_serving/token_in_token_out.md | 6 +- pyproject.toml | 2 + .../derender/test_derender_parity.py | 1 + .../derender/test_derender_stream.py | 2 +- .../test_generate_stream.py | 2 + .../token_in_token_out/test_protocol.py | 7 ++ .../token_in_token_out/test_serving_tokens.py | 11 +++ vllm/entrypoints/scale_out/render/serving.py | 39 +++++------ .../scale_out/token_in_token_out/protocol.py | 20 ++++++ .../scale_out/token_in_token_out/serving.py | 59 ++++++++++------ vllm/renderers/online_derenderer.py | 69 ++++++++----------- vllm/renderers/online_renderer.py | 21 ------ 12 files changed, 132 insertions(+), 107 deletions(-) diff --git a/docs/serving/online_serving/token_in_token_out.md b/docs/serving/online_serving/token_in_token_out.md index 0830e143a95a..a14324caf7c2 100644 --- a/docs/serving/online_serving/token_in_token_out.md +++ b/docs/serving/online_serving/token_in_token_out.md @@ -91,7 +91,7 @@ print(response["choices"][0]["token_ids"]) - The parser reads the engine's `text`, so it sees the stop string cut and the post `adjust_request` decode flags. That makes it match coupled chat for parsers whose `adjust_request` changes decoding which `/derender` can't. - A server with a tool or reasoning parser needs `parse_context`. Without a parser, `message.content` is the plain text. - `parse_context` is ignored at other levels, so a rendered request can go to a prefill leg unchanged. -- `logprobs` follow coupled chat. `top_logprobs` is cut to `sampling_params.logprobs`, `logprob_token_ids` is honored and `logprobs` are dropped when `include_reasoning` is false so they don't expose the hidden reasoning. +- `logprobs` follow coupled chat. `top_logprobs` is cut to `sampling_params.logprobs`, `logprob_token_ids` is honored. When a parser hides the reasoning with `include_reasoning: false`, `logprobs` and `token_ids` are dropped so they don't expose it. - `finish_reason` follows coupled chat too. It's `tool_calls` when the message has tool calls except with a named `tool_choice` or a `length` cut. - `stream: true` isn't supported yet. @@ -104,9 +104,9 @@ With `output_mode: "tokens"`, logprobs are `GenerateLogProbs` with an integer `t The server returns a 400 for: - `output_mode: "text"` or `"derender"` on a server without a tokenizer (`--tokens-only` or `--skip-tokenizer-init`). -- `output_mode: "text"` with `sampling_params.detokenize: false`. +- `output_mode: "text"` or `"derender"` with `sampling_params.detokenize: false`. - `output_mode: "derender"` with `stream: true`. -- `output_mode: "derender"` without `parse_context` on a server with a tool or reasoning parser. +- `output_mode: "derender"` without `parse_context` on a server with a tool or reasoning parser or with a `parse_context` that a chat request would reject, such as `tool_choice: "required"` without `tools`. This is checked before generating. - An unsupported `output_mode` value. For using `output_mode` with separate prefill and decode pools, see [Disaggregated Prefilling](../../features/disagg_prefill.md#generate-api-output-modes). diff --git a/pyproject.toml b/pyproject.toml index 3f09b75e9f6d..0aba92b38166 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -225,6 +225,8 @@ FoPE = "FoPE" k_ot = "k_ot" view_seperator = "view_seperator" inverse_std_variences = "inverse_std_variences" +# deliberate misspelling in a test that unknown fields are rejected +tool_choise = "tool_choise" [tool.typos.default.extend-words] MoNE = "MoNE" diff --git a/tests/entrypoints/scale_out/derender/test_derender_parity.py b/tests/entrypoints/scale_out/derender/test_derender_parity.py index 36e75f496e34..a7a495566817 100644 --- a/tests/entrypoints/scale_out/derender/test_derender_parity.py +++ b/tests/entrypoints/scale_out/derender/test_derender_parity.py @@ -1009,6 +1009,7 @@ async def test_inline_derender_logprobs(client, include_reasoning, extra): if not include_reasoning: assert inline["logprobs"] is None + assert inline["token_ids"] is None return coupled = (await _coupled(client, messages, **extra))["choices"][0] diff --git a/tests/entrypoints/scale_out/derender/test_derender_stream.py b/tests/entrypoints/scale_out/derender/test_derender_stream.py index 820ce1c3ffc3..fc0664f9bc56 100644 --- a/tests/entrypoints/scale_out/derender/test_derender_stream.py +++ b/tests/entrypoints/scale_out/derender/test_derender_stream.py @@ -467,7 +467,7 @@ def test_derender_text_choices_builds_request_from_parse_context( output = MagicMock(text="raw", token_ids=[1, 2], finish_reason="stop") ((message, finish_reason),) = derenderer._derender_text_choices( - [output], parse_context, [11] + [output], parse_context.to_chat_request(), 3, [11] ) chat_request = parser.parse.call_args.args[1] diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py b/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py index c30732f3150f..a85355b29611 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py @@ -37,6 +37,7 @@ ) from vllm.outputs import CompletionOutput, RequestOutput, SamplingMask from vllm.renderers import renderer_from_config +from vllm.renderers.online_derenderer import OnlineDerenderer from vllm.renderers.online_renderer import OnlineRenderer from vllm.sampling_params import SamplingParams from vllm.v1.engine.async_llm import AsyncLLM @@ -122,6 +123,7 @@ def _build_serving_tokens(engine: AsyncLLM, **kwargs) -> ServingTokens: engine, models, online_renderer=online_renderer, + online_derenderer=MagicMock(spec=OnlineDerenderer), request_logger=None, **kwargs, ) diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_protocol.py b/tests/entrypoints/scale_out/token_in_token_out/test_protocol.py index 41ac3d0adf44..f55ed2ff2a31 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_protocol.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_protocol.py @@ -178,6 +178,13 @@ def test_parse_context_is_accepted_at_other_levels(): assert GenerateRequest.model_validate(payload).output_mode == "tokens" +def test_parse_context_rejects_unknown_fields(): + """A misspelled field would otherwise be dropped and parse silently wrong.""" + payload = {**_base_payload(), "parse_context": {"tool_choise": "auto"}} + with pytest.raises(ValidationError, match="tool_choise"): + GenerateRequest.model_validate(payload) + + def test_derender_mode_rejects_detokenize_false(): payload = { "token_ids": [1, 2, 3], diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py b/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py index 7e735ae15d18..139d4c3ebc4e 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py @@ -815,6 +815,17 @@ async def test_derender_mode_with_parser_matches_chat_completions(client, messag assert missing.status_code == 400 assert "parse_context" in missing.json()["error"]["message"] + invalid = await client.post( + GEN_ENDPOINT, + json={ + **generate_request, + "output_mode": "derender", + "parse_context": {"tool_choice": "required"}, + }, + ) + assert invalid.status_code == 400 + assert "tools" in invalid.json()["error"]["message"] + resp = await client.post( GEN_ENDPOINT, json={**generate_request, "output_mode": "derender"} ) diff --git a/vllm/entrypoints/scale_out/render/serving.py b/vllm/entrypoints/scale_out/render/serving.py index 80afbf91f732..33d2cbbce457 100644 --- a/vllm/entrypoints/scale_out/render/serving.py +++ b/vllm/entrypoints/scale_out/render/serving.py @@ -35,10 +35,7 @@ extract_prompt_components, extract_prompt_len, ) -from vllm.renderers.online_renderer import ( - OnlineRenderer, - resolve_chat_template_kwargs, -) +from vllm.renderers.online_renderer import OnlineRenderer from vllm.sampling_params import SamplingParams logger = init_logger(__name__) @@ -155,7 +152,10 @@ async def render_chat_request( # Resolve here what OpenAIServingChat passes to the engine: # `_grammar_from_parser` (set by `adjust_request`) is not serialized. - reasoning_parser_kwargs = self._reasoning_parser_kwargs(request) + chat_template_kwargs = self.online_renderer.effective_chat_template_kwargs( + request + ) + reasoning_parser_kwargs = self._reasoning_parser_kwargs(chat_template_kwargs) parser = None if reasoning_parser_kwargs is not None: assert self.online_renderer.parser is not None @@ -189,22 +189,21 @@ async def render_chat_request( cache_salt=request.cache_salt, priority=request.priority, token_offsets=engine_input.get("prompt_token_offsets"), - parse_context=self._build_parse_context(request), + parse_context=self._build_parse_context(request, chat_template_kwargs), ) - def _build_parse_context(self, request: ChatCompletionRequest) -> ParseContext: - renderer = self.online_renderer + def _build_parse_context( + self, request: ChatCompletionRequest, chat_template_kwargs: dict[str, Any] + ) -> ParseContext: return ParseContext( tools=request.tools, tool_choice=request.tool_choice, parallel_tool_calls=request.parallel_tool_calls, include_reasoning=request.include_reasoning, - chat_template_kwargs=resolve_chat_template_kwargs( - request, - renderer.chat_template, - renderer.chat_template_content_format, - renderer.default_chat_template_kwargs, - use_harmony=renderer.use_harmony, + # Like the derenderer, Harmony output is parsed without template + # kwargs. + chat_template_kwargs=( + {} if self.online_renderer.use_harmony else chat_template_kwargs ), history_tool_call_cnt=count_history_tool_calls(request), ) @@ -352,7 +351,9 @@ async def render_responses_request( ), sampling_params=params, model=request.model, - reasoning_parser_kwargs=self._reasoning_parser_kwargs(request), + reasoning_parser_kwargs=self._reasoning_parser_kwargs( + self.online_renderer.effective_chat_template_kwargs(request) + ), stream=bool(request.stream), cache_salt=request.cache_salt, priority=request.priority, @@ -362,16 +363,12 @@ async def render_responses_request( ) def _reasoning_parser_kwargs( - self, request: ChatCompletionRequest | ResponsesRequest + self, chat_template_kwargs: dict[str, Any] ) -> ReasoningParserKwargs | None: parser_cls = self.online_renderer.parser if parser_cls is None or parser_cls.reasoning_parser_cls is None: return None - return ReasoningParserKwargs( - chat_template_kwargs=self.online_renderer.effective_chat_template_kwargs( - request - ) - ) + return ReasoningParserKwargs(chat_template_kwargs=chat_template_kwargs) def _placeholder_metadata_fields(self, modality: str) -> set[str]: """Fields the EC consumer still requires after embeddings transfer.""" diff --git a/vllm/entrypoints/scale_out/token_in_token_out/protocol.py b/vllm/entrypoints/scale_out/token_in_token_out/protocol.py index 3a1f97ce8eda..5d75b4248e70 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/protocol.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/protocol.py @@ -51,6 +51,8 @@ class ParseContext(BaseModel): only when `output_mode="derender"`. """ + model_config = ConfigDict(extra="forbid") + tools: list[ChatCompletionToolsParam] | None = None tool_choice: ( Literal["none", "auto", "required"] | ChatCompletionNamedToolChoiceParam | None @@ -62,6 +64,24 @@ class ParseContext(BaseModel): history_tool_call_cnt: int = Field(default=0, ge=0) """Tool calls already in the rendered conversation, for sequential IDs.""" + def to_chat_request(self) -> ChatCompletionRequest: + """Build the message less chat request the parser reads. + + Validated from plain data because the request validators only accept + a named `tool_choice` as a dict. + + Raises: + ValueError: If the context fails the chat request validators. + VLLMValidationError: Same for the tool usage validators. + + """ + return ChatCompletionRequest.model_validate( + { + "messages": [], + **self.model_dump(exclude={"history_tool_call_cnt"}), + } + ) + class PlaceholderRangeInfo(BaseModel): """Serializable placeholder location for a single multi-modal item.""" diff --git a/vllm/entrypoints/scale_out/token_in_token_out/serving.py b/vllm/entrypoints/scale_out/token_in_token_out/serving.py index 9b943c009d5f..2a96549685f8 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/serving.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/serving.py @@ -29,6 +29,7 @@ ChatCompletionLogProb, ChatCompletionLogProbs, ChatCompletionLogProbsContent, + ChatCompletionRequest, ) from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.serve.engine.protocol import ( @@ -125,7 +126,7 @@ def __init__( models: OpenAIServingModels, online_renderer: OnlineRenderer, *, - online_derenderer: OnlineDerenderer | None = None, + online_derenderer: OnlineDerenderer, request_logger: RequestLogger | None, force_no_detokenize: bool = False, return_tokens_as_token_ids: bool = False, @@ -237,6 +238,8 @@ async def serve_tokens( "--skip-tokenizer-init). Send the request to a server that " "loads a tokenizer, or use output_mode='tokens'." ) + # Built before generating so an invalid parse_context fails fast. + derender_request: ChatCompletionRequest | None = None if request.output_mode == "derender": # TODO(#57442, #60915): reject the sampled-only and top-k logprob # flags here once either lands. @@ -244,18 +247,17 @@ async def serve_tokens( return self.create_error_response( "output_mode='derender' does not support stream=true." ) - if self.online_derenderer is None: - return self.create_error_response( - "output_mode='derender' is not available on this server." - ) - if self.online_derenderer.parser is not None and ( - request.parse_context is None - ): - return self.create_error_response( - "output_mode='derender' requires parse_context on a server " - "with a tool or reasoning parser. Use the one returned by " - "/render." - ) + if self.online_derenderer.parser is not None: + if request.parse_context is None: + return self.create_error_response( + "output_mode='derender' requires parse_context on a " + "server with a tool or reasoning parser. Use the one " + "returned by /render." + ) + try: + derender_request = request.parse_context.to_chat_request() + except (ValueError, VLLMValidationError) as e: + return self.create_error_response(e) try: msgspec.msgpack.encode( ( @@ -408,7 +410,12 @@ async def serve_tokens( ) return await self.serve_tokens_full_generator( - request, result_generator, request_id, model_name, request_metadata + request, + result_generator, + request_id, + model_name, + request_metadata, + derender_request=derender_request, ) async def serve_tokens_full_generator( @@ -418,6 +425,7 @@ async def serve_tokens_full_generator( request_id: str, model_name: str, request_metadata: RequestResponseMetadata, + derender_request: ChatCompletionRequest | None = None, ) -> ErrorResponse | GenerateResponse: created_time = int(time.time()) final_res: RequestOutput | None = None @@ -505,23 +513,32 @@ async def serve_tokens_full_generator( derender_choices: list[GenerateDerenderChoice] = [] if derender_mode: - assert self.online_derenderer is not None - parse_context = request.parse_context + history_tool_call_cnt = ( + request.parse_context.history_tool_call_cnt + if request.parse_context is not None + else 0 + ) try: parsed = await self.online_derenderer.derender_text_choices( - final_res.outputs, parse_context, final_res.prompt_token_ids + final_res.outputs, + derender_request, + history_tool_call_cnt, + final_res.prompt_token_ids, ) except (ValueError, VLLMValidationError) as e: return self.create_error_response(e) - hide_logprobs = ( - parse_context is not None and not parse_context.include_reasoning + # Like coupled chat, hide per-token metadata when a parser hides + # the reasoning: logprobs or token IDs would leak it. + hide_metadata = ( + derender_request is not None and not derender_request.include_reasoning ) for (message, finish_reason), fields in zip( parsed, derender_choice_fields, strict=True ): fields["finish_reason"] = finish_reason - if hide_logprobs: + if hide_metadata: fields["logprobs"] = None + fields["token_ids"] = None derender_choices.append( GenerateDerenderChoice(message=message, **fields) ) @@ -797,7 +814,7 @@ def _create_text_logprobs( tokenizer: TokenizerLike | None = None, top_limit: int | None = None, ) -> ChatCompletionLogProbs: - """Create OpenAI style logprobs for `output_mode="text"`. + """Create OpenAI style logprobs for the `text` and `derender` levels. With a `tokenizer` the tokens are decoded strings and carry `bytes`. Without one (`--return-tokens-as-token-ids`) they are `token_id:N` diff --git a/vllm/renderers/online_derenderer.py b/vllm/renderers/online_derenderer.py index a5693228fb45..7620119f4e19 100644 --- a/vllm/renderers/online_derenderer.py +++ b/vllm/renderers/online_derenderer.py @@ -40,7 +40,6 @@ GenerateLogProbs, GenerateTokensResponse, GenerateTokensStreamResponse, - ParseContext, ) from vllm.entrypoints.serve.engine.protocol import UsageInfo from vllm.entrypoints.serve.utils.request_logger import RequestLogger @@ -53,7 +52,6 @@ ChatTemplateContentFormatOption, make_tool_call_id, ) -from vllm.renderers.online_renderer import resolve_chat_template_kwargs from vllm.tokenizers import TokenizerLike from vllm.tokenizers.detokenizer_utils import ( convert_prompt_ids_to_tokens, @@ -287,15 +285,17 @@ def _derender_chat( async def derender_text_choices( self, outputs: Sequence[CompletionOutput], - parse_context: ParseContext | None, + chat_request: ChatCompletionRequest | None, + history_tool_call_cnt: int, prompt_token_ids: list[int] | None, ) -> list[tuple[ChatMessage, str]]: """Parse the engine's decoded output text into assistant messages. Args: outputs: Final engine outputs, already detokenized. - parse_context: Chat context from `/render`. None is only valid - on a server without a parser. + chat_request: Chat request built from `parse_context`. With None + or no parser on this server, the message content is the text. + history_tool_call_cnt: Tool calls already in the conversation. prompt_token_ids: Prompt token IDs, if known. Returns: @@ -303,45 +303,31 @@ async def derender_text_choices( """ return await self._derender_text_choices_async( - outputs, parse_context, prompt_token_ids + outputs, chat_request, history_tool_call_cnt, prompt_token_ids ) def _derender_text_choices( self, outputs: Sequence[CompletionOutput], - parse_context: ParseContext | None, + chat_request: ChatCompletionRequest | None, + history_tool_call_cnt: int, prompt_token_ids: list[int] | None, ) -> list[tuple[ChatMessage, str]]: - if self.parser is None: - return [ - ( - ChatMessage(role="assistant", content=o.text), - resolve_finish_reason(o.finish_reason, None, False), - ) - for o in outputs - ] - - assert parse_context is not None tokenizer = self.renderer.get_tokenizer() - # Validate from plain data: the request validators only accept a - # named tool_choice as a dict. - chat_request = ChatCompletionRequest.model_validate( - { - "messages": [], - **parse_context.model_dump(exclude={"history_tool_call_cnt"}), - } - ) results: list[tuple[ChatMessage, str]] = [] for output in outputs: - message = self._parse_and_assemble( - tokenizer, - output.text, - output.token_ids, - chat_request, - parse_context.chat_template_kwargs, - prompt_token_ids, - history_tool_call_count=parse_context.history_tool_call_cnt, - ) + if self.parser is None or chat_request is None: + message = ChatMessage(role="assistant", content=output.text) + else: + message = self._parse_and_assemble( + tokenizer, + output.text, + output.token_ids, + chat_request, + chat_request.chat_template_kwargs or {}, + prompt_token_ids, + history_tool_call_count=history_tool_call_cnt, + ) finish_reason = resolve_finish_reason( output.finish_reason, chat_request, bool(message.tool_calls) ) @@ -351,12 +337,15 @@ def _derender_text_choices( def _resolve_chat_template_kwargs( self, chat_request: ChatCompletionRequest ) -> dict[str, Any]: - return resolve_chat_template_kwargs( - chat_request, - self.chat_template, - self.chat_template_content_format, - self.default_chat_template_kwargs, - use_harmony=self.use_harmony, + if self.use_harmony: + return {} + return ( + chat_request.build_chat_params( + self.chat_template, + self.chat_template_content_format, + ) + .with_defaults(self.default_chat_template_kwargs) + .chat_template_kwargs ) def _parse_and_assemble( diff --git a/vllm/renderers/online_renderer.py b/vllm/renderers/online_renderer.py index 5b109e1f51c7..65771211d0a5 100644 --- a/vllm/renderers/online_renderer.py +++ b/vllm/renderers/online_renderer.py @@ -151,27 +151,6 @@ def _reused_prompt_token_ids( return ids -def resolve_chat_template_kwargs( - request: ChatCompletionRequest, - chat_template: str | None, - chat_template_content_format: ChatTemplateContentFormatOption, - default_chat_template_kwargs: dict[str, Any], - *, - use_harmony: bool, -) -> dict[str, Any]: - """Template kwargs for `request` with the server defaults applied. - - Harmony models don't render through a chat template, so they get none. - """ - if use_harmony: - return {} - return ( - request.build_chat_params(chat_template, chat_template_content_format) - .with_defaults(default_chat_template_kwargs) - .chat_template_kwargs - ) - - class OnlineRenderer: def __init__( self, From 1e3af7596d67c64e04aff45bc0fe4f19e47225a2 Mon Sep 17 00:00:00 2001 From: Martin Hickey Date: Tue, 20 Oct 2026 10:50:02 +0100 Subject: [PATCH 05/11] Update after review Ignore test_protocol.py in typos instead of allowlisting tool_choise for the whole repo. Move the usage, metrics and shared response fields out of serve_tokens_full_generator into _build_response_fields. Signed-off-by: Martin Hickey --- pyproject.toml | 3 +- .../scale_out/token_in_token_out/serving.py | 87 ++++++++++++------- 2 files changed, 55 insertions(+), 35 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0aba92b38166..4f25ce30618e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -199,6 +199,7 @@ extend-exclude = ["tests/models/fixtures/*", "tests/prompts/*", "tests/tokenizer "tests/entrypoints/speech_to_text/transcription/test_transcription_validation.py", "docs/governance/process.md", "docs/community/reviewers.md", "docs/assets/contributing/vllm_bench_serve_timeline.html", "tests/v1/engine/test_fast_incdec_prefix_err.py", ".git/*", "csrc/cpu/sgl-kernels/*", + "tests/entrypoints/scale_out/token_in_token_out/test_protocol.py", "rust/src/chat/src/renderer/deepseek_v32/fixtures/*", "rust/src/parser/**", "rust/src/text/src/output/decoded.rs", "rust/src/tokenizer/src/incremental.rs", @@ -225,8 +226,6 @@ FoPE = "FoPE" k_ot = "k_ot" view_seperator = "view_seperator" inverse_std_variences = "inverse_std_variences" -# deliberate misspelling in a test that unknown fields are rejected -tool_choise = "tool_choise" [tool.typos.default.extend-words] MoNE = "MoNE" diff --git a/vllm/entrypoints/scale_out/token_in_token_out/serving.py b/vllm/entrypoints/scale_out/token_in_token_out/serving.py index 2a96549685f8..068f14abe853 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/serving.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/serving.py @@ -543,6 +543,59 @@ async def serve_tokens_full_generator( GenerateDerenderChoice(message=message, **fields) ) + response_fields = self._build_response_fields( + request, + final_res, + request_id, + model_name, + created_time, + num_generated_tokens, + request_metadata, + ) + response: ( + GenerateTokensResponse | GenerateTextResponse | GenerateDerenderResponse + ) + if derender_mode: + response = GenerateDerenderResponse( + choices=derender_choices, **response_fields + ) + elif text_mode: + response = GenerateTextResponse(choices=text_choices, **response_fields) + else: + response = GenerateTokensResponse(choices=tokens_choices, **response_fields) + + # Log complete response if output logging is enabled + if self.enable_log_outputs and self.request_logger: + for choice in response.choices: + # Get the corresponding output token IDs + output_token_ids = None + if choice.index < len(final_res.outputs): + output_token_ids = final_res.outputs[choice.index].token_ids + + if output_token_ids: + # Log token_ids only. + self.request_logger.log_outputs( + request_id=request_id, + outputs="", + output_token_ids=output_token_ids, + finish_reason=choice.finish_reason, + is_streaming=False, + delta=False, + ) + + return response + + def _build_response_fields( + self, + request: GenerateRequest, + final_res: RequestOutput, + request_id: str, + model_name: str, + created_time: int, + num_generated_tokens: int, + request_metadata: RequestResponseMetadata, + ) -> dict[str, Any]: + """Build the fields shared by every non-streaming response level.""" assert final_res.prompt_token_ids is not None num_prompt_tokens = len(final_res.prompt_token_ids) if final_res.encoder_prompt_token_ids is not None: @@ -569,7 +622,7 @@ async def serve_tokens_full_generator( spec_stats = build_spec_decoding_metrics(final_res) if spec_stats is not None: per_request_metrics = PerRequestMetrics(speculative_decoding=spec_stats) - response_fields: dict[str, Any] = dict( + return dict( request_id=request_id, created=created_time, model=model_name, @@ -588,38 +641,6 @@ async def serve_tokens_full_generator( kv_transfer_params=final_res.kv_transfer_params, ec_transfer_params=final_res.ec_transfer_params, ) - response: ( - GenerateTokensResponse | GenerateTextResponse | GenerateDerenderResponse - ) - if derender_mode: - response = GenerateDerenderResponse( - choices=derender_choices, **response_fields - ) - elif text_mode: - response = GenerateTextResponse(choices=text_choices, **response_fields) - else: - response = GenerateTokensResponse(choices=tokens_choices, **response_fields) - - # Log complete response if output logging is enabled - if self.enable_log_outputs and self.request_logger: - for choice in response.choices: - # Get the corresponding output token IDs - output_token_ids = None - if choice.index < len(final_res.outputs): - output_token_ids = final_res.outputs[choice.index].token_ids - - if output_token_ids: - # Log token_ids only. - self.request_logger.log_outputs( - request_id=request_id, - outputs="", - output_token_ids=output_token_ids, - finish_reason=choice.finish_reason, - is_streaming=False, - delta=False, - ) - - return response async def serve_tokens_stream_generator( self, From 213239b8c1d52729f0b828c6ed33c0a4d2e6e302 Mon Sep 17 00:00:00 2001 From: Martin Hickey Date: Tue, 20 Oct 2026 11:08:40 +0100 Subject: [PATCH 06/11] Fix derender tests after rebase Empty tool_calls are now left out of chat messages, so look the key up with .get(). The n=2 parity case can't run greedy, so give it a temperature and seed instead. Signed-off-by: Martin Hickey --- .../scale_out/derender/test_derender_parity.py | 8 ++++++-- .../scale_out/derender/test_derender_stream.py | 2 +- .../scale_out/token_in_token_out/test_serving_tokens.py | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/entrypoints/scale_out/derender/test_derender_parity.py b/tests/entrypoints/scale_out/derender/test_derender_parity.py index a7a495566817..121a79853697 100644 --- a/tests/entrypoints/scale_out/derender/test_derender_parity.py +++ b/tests/entrypoints/scale_out/derender/test_derender_parity.py @@ -918,7 +918,7 @@ def _assert_inline_parity(coupled: dict, inline: dict) -> None: assert len(inline["choices"]) == len(coupled["choices"]) for c, i in zip(coupled["choices"], inline["choices"]): assert i["token_ids"] == c["token_ids"], ( - "greedy (temperature=0) generation was expected to be deterministic " + "greedy or seeded generation was expected to be deterministic " "across the coupled and inline calls" ) assert i["message"]["content"] == c["message"]["content"] @@ -971,7 +971,11 @@ def _assert_inline_parity(coupled: dict, inline: dict) -> None: pytest.param( "Write two short lines about the sea.", {"stop": ["\n"]}, id="stop" ), - pytest.param(TEXT_MESSAGES[0]["content"], {"n": 2}, id="n2"), + pytest.param( + TEXT_MESSAGES[0]["content"], + {"n": 2, "temperature": 0.8, "seed": 42}, + id="n2", + ), ], ) async def test_inline_derender_parity(client, content, extra): diff --git a/tests/entrypoints/scale_out/derender/test_derender_stream.py b/tests/entrypoints/scale_out/derender/test_derender_stream.py index fc0664f9bc56..8c516f0a2603 100644 --- a/tests/entrypoints/scale_out/derender/test_derender_stream.py +++ b/tests/entrypoints/scale_out/derender/test_derender_stream.py @@ -2491,7 +2491,7 @@ async def test_stream_parsed_matches_batch_tool_call(parser_client, parser_token ) assert batch_resp.status_code == 200, batch_resp.text batch_choice = batch_resp.json()["choices"][0] - if not batch_choice["message"]["tool_calls"]: + if not batch_choice["message"].get("tool_calls"): pytest.skip("Model did not emit a tool call") for chunk_sizes in ([len(output_ids)], [1] * len(output_ids)): diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py b/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py index 139d4c3ebc4e..63e1cf4180d4 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py @@ -835,7 +835,7 @@ async def test_derender_mode_with_parser_matches_chat_completions(client, messag expected = chat_resp.json()["choices"][0] got = resp.json()["choices"][0] assert got["message"]["content"] == expected["message"]["content"] - assert got["message"]["tool_calls"] == expected["message"]["tool_calls"] + assert got["message"].get("tool_calls") == expected["message"].get("tool_calls") assert got["finish_reason"] == expected["finish_reason"] From 8d4eba068082a3447eb312119bd4e7fdc83f34b0 Mon Sep 17 00:00:00 2001 From: Martin Hickey Date: Tue, 20 Oct 2026 12:09:37 +0100 Subject: [PATCH 07/11] Fix Cohere render crash and tool checks in derender mode The Cohere render route copied its API server only template kwargs into parse_context. The position-to-source map has tuple keys, so JSON encoding failed and requests with documents or tool results got a 500. Strip those keys the same way reasoning_parser_kwargs already does. As a result, derender citations come back without document sources. This is now documented. to_chat_request re-ran the full chat validators. The Anthropic converter sets tool fields directly and can produce tool_choice "auto" with no tools or tools: []. Coupled /v1/messages serves these but derender returned a 400. Build the request with model_construct and keep only the check that a named tool_choice matches one of the tools. Signed-off-by: Martin Hickey --- .../online_serving/token_in_token_out.md | 3 +- tests/entrypoints/cohere/test_api_router.py | 19 ++++++++---- .../token_in_token_out/test_protocol.py | 18 ++++++++++++ .../token_in_token_out/test_serving_tokens.py | 6 ++-- vllm/entrypoints/cohere/api_router.py | 6 ++++ .../scale_out/token_in_token_out/protocol.py | 29 +++++++++++++------ 6 files changed, 63 insertions(+), 18 deletions(-) diff --git a/docs/serving/online_serving/token_in_token_out.md b/docs/serving/online_serving/token_in_token_out.md index a14324caf7c2..c8b287ab1196 100644 --- a/docs/serving/online_serving/token_in_token_out.md +++ b/docs/serving/online_serving/token_in_token_out.md @@ -94,6 +94,7 @@ print(response["choices"][0]["token_ids"]) - `logprobs` follow coupled chat. `top_logprobs` is cut to `sampling_params.logprobs`, `logprob_token_ids` is honored. When a parser hides the reasoning with `include_reasoning: false`, `logprobs` and `token_ids` are dropped so they don't expose it. - `finish_reason` follows coupled chat too. It's `tool_calls` when the message has tool calls except with a named `tool_choice` or a `length` cut. - `stream: true` isn't supported yet. +- `/cohere/v2/chat/render` drops the Cohere document map from `parse_context`, so citations come back without their document sources. ### Logprobs @@ -106,7 +107,7 @@ The server returns a 400 for: - `output_mode: "text"` or `"derender"` on a server without a tokenizer (`--tokens-only` or `--skip-tokenizer-init`). - `output_mode: "text"` or `"derender"` with `sampling_params.detokenize: false`. - `output_mode: "derender"` with `stream: true`. -- `output_mode: "derender"` without `parse_context` on a server with a tool or reasoning parser or with a `parse_context` that a chat request would reject, such as `tool_choice: "required"` without `tools`. This is checked before generating. +- `output_mode: "derender"` without `parse_context` on a server with a tool or reasoning parser or with a named `tool_choice` in `parse_context` that matches none of its `tools`. This is checked before generating. - An unsupported `output_mode` value. For using `output_mode` with separate prefill and decode pools, see [Disaggregated Prefilling](../../features/disagg_prefill.md#generate-api-output-modes). diff --git a/tests/entrypoints/cohere/test_api_router.py b/tests/entrypoints/cohere/test_api_router.py index 2c2fce190232..b735241803ea 100644 --- a/tests/entrypoints/cohere/test_api_router.py +++ b/tests/entrypoints/cohere/test_api_router.py @@ -38,6 +38,7 @@ ) from vllm.entrypoints.scale_out.token_in_token_out.protocol import ( GenerateRequest, + ParseContext, ReasoningParserKwargs, ) from vllm.entrypoints.serve.engine.protocol import ErrorInfo, ErrorResponse @@ -488,15 +489,18 @@ def test_cohere_request_surface_survives_conversion(self): assert kwargs["safety_mode"] == "strict" def test_api_server_only_kwargs_not_forwarded(self): - """As on ``/cohere/v2/chat``, the engine's reasoning parser must not - receive the template kwargs that only the API server reads.""" + """As on ``/cohere/v2/chat``, the engine's reasoning parser and the + derender ``parse_context`` must not receive the template kwargs that + only the API server reads. Their tuple keys can't be JSON encoded.""" + kwargs = { + "safety_mode": "strict", + **dict.fromkeys(_API_SERVER_ONLY_TEMPLATE_KWARGS, {(0, 0): "x"}), + } result = _generate_request() result.reasoning_parser_kwargs = ReasoningParserKwargs( - chat_template_kwargs={ - "safety_mode": "strict", - **dict.fromkeys(_API_SERVER_ONLY_TEMPLATE_KWARGS, "x"), - } + chat_template_kwargs=kwargs ) + result.parse_context = ParseContext(chat_template_kwargs=kwargs) app = _build_render_app(_RenderChatHandler(), _RenderHandler(result)) with TestClient(app) as client: r = client.post("/cohere/v2/chat/render", json=_minimal_request_body()) @@ -504,6 +508,9 @@ def test_api_server_only_kwargs_not_forwarded(self): assert r.json()["reasoning_parser_kwargs"] == { "chat_template_kwargs": {"safety_mode": "strict"} } + assert r.json()["parse_context"]["chat_template_kwargs"] == { + "safety_mode": "strict" + } def test_501_when_chat_handler_missing(self): app = _build_render_app(None, _RenderHandler(_generate_request())) diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_protocol.py b/tests/entrypoints/scale_out/token_in_token_out/test_protocol.py index f55ed2ff2a31..54c629908ab8 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_protocol.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_protocol.py @@ -27,8 +27,10 @@ GenerateTokensResponse, GenerateTokensStreamResponse, MultiModalFeatures, + ParseContext, PlaceholderRangeInfo, ) +from vllm.exceptions import VLLMValidationError from vllm.sampling_params import SamplingParams @@ -185,6 +187,22 @@ def test_parse_context_rejects_unknown_fields(): GenerateRequest.model_validate(payload) +@pytest.mark.parametrize("tools", [None, []]) +def test_parse_context_to_chat_request_accepts_converter_tool_fields(tools): + """The Anthropic converter emits these and coupled /v1/messages serves them.""" + chat_request = ParseContext(tools=tools, tool_choice="auto").to_chat_request() + assert chat_request.tools == tools + assert chat_request.tool_choice == "auto" + + +def test_parse_context_to_chat_request_rejects_unknown_named_tool(): + parse_context = ParseContext.model_validate( + {"tool_choice": {"type": "function", "function": {"name": "nope"}}} + ) + with pytest.raises(VLLMValidationError, match="tool_choice"): + parse_context.to_chat_request() + + def test_derender_mode_rejects_detokenize_false(): payload = { "token_ids": [1, 2, 3], diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py b/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py index 63e1cf4180d4..70c3abcb7df2 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py @@ -820,11 +820,13 @@ async def test_derender_mode_with_parser_matches_chat_completions(client, messag json={ **generate_request, "output_mode": "derender", - "parse_context": {"tool_choice": "required"}, + "parse_context": { + "tool_choice": {"type": "function", "function": {"name": "nope"}} + }, }, ) assert invalid.status_code == 400 - assert "tools" in invalid.json()["error"]["message"] + assert "tool_choice" in invalid.json()["error"]["message"] resp = await client.post( GEN_ENDPOINT, json={**generate_request, "output_mode": "derender"} diff --git a/vllm/entrypoints/cohere/api_router.py b/vllm/entrypoints/cohere/api_router.py index 5362b1d9e95d..db63c30f4fa4 100644 --- a/vllm/entrypoints/cohere/api_router.py +++ b/vllm/entrypoints/cohere/api_router.py @@ -202,6 +202,12 @@ async def chat_v2_render(request: CohereChatV2Request, raw_request: Request): kwargs.chat_template_kwargs = handler._engine_chat_template_kwargs( kwargs.chat_template_kwargs ) + # Citations from derender mode lack document sources: the + # position-to-source map can't go over JSON. + if (parse_context := result.parse_context) is not None: + parse_context.chat_template_kwargs = handler._engine_chat_template_kwargs( + parse_context.chat_template_kwargs + ) return JSONResponse(content=result.model_dump()) diff --git a/vllm/entrypoints/scale_out/token_in_token_out/protocol.py b/vllm/entrypoints/scale_out/token_in_token_out/protocol.py index 5d75b4248e70..6312950c01ab 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/protocol.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/protocol.py @@ -33,6 +33,7 @@ CompletionStreamResponse, ) from vllm.entrypoints.serve.engine.protocol import UsageInfo +from vllm.exceptions import VLLMValidationError from vllm.logprobs import Logprob from vllm.outputs import WeightVersionSpan from vllm.renderers import TokenizeParams @@ -67,19 +68,29 @@ class ParseContext(BaseModel): def to_chat_request(self) -> ChatCompletionRequest: """Build the message less chat request the parser reads. - Validated from plain data because the request validators only accept - a named `tool_choice` as a dict. + Built without the chat request validators: the Anthropic converter + assigns tool fields that they would reject (e.g. `tools: []`) and the + coupled path serves those as is. Raises: - ValueError: If the context fails the chat request validators. - VLLMValidationError: Same for the tool usage validators. + VLLMValidationError: If a named `tool_choice` matches no tool. """ - return ChatCompletionRequest.model_validate( - { - "messages": [], - **self.model_dump(exclude={"history_tool_call_cnt"}), - } + if isinstance(self.tool_choice, ChatCompletionNamedToolChoiceParam): + name = self.tool_choice.function.name + if not any(tool.function.name == name for tool in self.tools or ()): + raise VLLMValidationError( + "The tool specified in `tool_choice` does not match any" + " of the specified `tools`", + parameter="tool_choice", + ) + return ChatCompletionRequest.model_construct( + messages=[], + tools=self.tools, + tool_choice=self.tool_choice, + parallel_tool_calls=self.parallel_tool_calls, + include_reasoning=self.include_reasoning, + chat_template_kwargs=self.chat_template_kwargs, ) From d0edf768f2e9e071be580355deb9384e5aee81fd Mon Sep 17 00:00:00 2001 From: Martin Hickey Date: Tue, 20 Oct 2026 12:44:26 +0100 Subject: [PATCH 08/11] Loosen parse_context checks and keep its template kwargs JSON-only parse_context goes out on every rendered chat reques but its fields are only read in derender mode. It used extra="forbid", so a field added by a newer render would make an older generate node reject every request. Prefill legs included. Unknown fields now fail only when output_mode is "derender". At the other levels they're logged once and ignored. chat_template_kwargs is now typed as JSON values. Render strips the API server only kwargs before it builds the context, using the handler's _engine_chat_template_kwargs, so Cohere's _position_to_source never gets into the wire shape. This replaces the Cohere router's cleanup after render which the stricter type would now reject. Signed-off-by: Martin Hickey --- .../online_serving/token_in_token_out.md | 2 +- tests/entrypoints/cohere/test_api_router.py | 14 ++++---- .../token_in_token_out/test_protocol.py | 14 ++++++-- vllm/entrypoints/cohere/api_router.py | 13 ++++---- vllm/entrypoints/scale_out/render/serving.py | 19 ++++++++++- .../scale_out/token_in_token_out/protocol.py | 33 ++++++++++++++++--- 6 files changed, 72 insertions(+), 23 deletions(-) diff --git a/docs/serving/online_serving/token_in_token_out.md b/docs/serving/online_serving/token_in_token_out.md index c8b287ab1196..de5f9ee50fd3 100644 --- a/docs/serving/online_serving/token_in_token_out.md +++ b/docs/serving/online_serving/token_in_token_out.md @@ -86,7 +86,7 @@ print(response["choices"][0]["token_ids"]) ### Derender -`derender` returns the assistant `message` a coupled `/v1/chat/completions` would, without a separate [derender](derenderer.md) call. `/v1/chat/completions/render` and `/v1/messages/render` return `parse_context` on the `GenerateRequest`. Send it back with `output_mode: "derender"`. It carries the tools, `tool_choice`, `include_reasoning`, resolved `chat_template_kwargs` and the history tool call count. +`derender` returns the assistant `message` a coupled `/v1/chat/completions` would, without a separate [derender](derenderer.md) call. `/v1/chat/completions/render` and `/v1/messages/render` return `parse_context` on the `GenerateRequest`. Send it back with `output_mode: "derender"`. It carries the tools, `tool_choice`, `include_reasoning`, resolved JSON only `chat_template_kwargs` and the history tool call count. A `parse_context` field this server doesn't know fails a `derender` request and is ignored at the other levels. - The parser reads the engine's `text`, so it sees the stop string cut and the post `adjust_request` decode flags. That makes it match coupled chat for parsers whose `adjust_request` changes decoding which `/derender` can't. - A server with a tool or reasoning parser needs `parse_context`. Without a parser, `message.content` is the plain text. diff --git a/tests/entrypoints/cohere/test_api_router.py b/tests/entrypoints/cohere/test_api_router.py index b735241803ea..59628203eef4 100644 --- a/tests/entrypoints/cohere/test_api_router.py +++ b/tests/entrypoints/cohere/test_api_router.py @@ -38,7 +38,6 @@ ) from vllm.entrypoints.scale_out.token_in_token_out.protocol import ( GenerateRequest, - ParseContext, ReasoningParserKwargs, ) from vllm.entrypoints.serve.engine.protocol import ErrorInfo, ErrorResponse @@ -123,9 +122,11 @@ class _RenderHandler: def __init__(self, result): self.result = result self.seen_request = None + self.seen_kwargs_filter = None - async def render_chat_request(self, request): + async def render_chat_request(self, request, engine_chat_template_kwargs=None): self.seen_request = request + self.seen_kwargs_filter = engine_chat_template_kwargs if isinstance(self.result, Exception): raise self.result return self.result @@ -500,17 +501,16 @@ def test_api_server_only_kwargs_not_forwarded(self): result.reasoning_parser_kwargs = ReasoningParserKwargs( chat_template_kwargs=kwargs ) - result.parse_context = ParseContext(chat_template_kwargs=kwargs) - app = _build_render_app(_RenderChatHandler(), _RenderHandler(result)) + render_handler = _RenderHandler(result) + app = _build_render_app(_RenderChatHandler(), render_handler) with TestClient(app) as client: r = client.post("/cohere/v2/chat/render", json=_minimal_request_body()) assert r.status_code == HTTPStatus.OK assert r.json()["reasoning_parser_kwargs"] == { "chat_template_kwargs": {"safety_mode": "strict"} } - assert r.json()["parse_context"]["chat_template_kwargs"] == { - "safety_mode": "strict" - } + assert render_handler.seen_kwargs_filter is not None + assert render_handler.seen_kwargs_filter(kwargs) == {"safety_mode": "strict"} def test_501_when_chat_handler_missing(self): app = _build_render_app(None, _RenderHandler(_generate_request())) diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_protocol.py b/tests/entrypoints/scale_out/token_in_token_out/test_protocol.py index 54c629908ab8..159125115a69 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_protocol.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_protocol.py @@ -180,11 +180,19 @@ def test_parse_context_is_accepted_at_other_levels(): assert GenerateRequest.model_validate(payload).output_mode == "tokens" -def test_parse_context_rejects_unknown_fields(): - """A misspelled field would otherwise be dropped and parse silently wrong.""" +def test_parse_context_unknown_fields_rejected_only_in_derender_mode(): + """An unknown field from a newer /render must fail derender which would + parse without it but not the levels that never read the context.""" payload = {**_base_payload(), "parse_context": {"tool_choise": "auto"}} + assert GenerateRequest.model_validate(payload).output_mode == "tokens" with pytest.raises(ValidationError, match="tool_choise"): - GenerateRequest.model_validate(payload) + GenerateRequest.model_validate({**payload, "output_mode": "derender"}) + + +def test_parse_context_rejects_non_json_template_kwargs(): + """Render internal state must not become part of the wire shape.""" + with pytest.raises(ValidationError, match="chat_template_kwargs"): + ParseContext(chat_template_kwargs={"_position_to_source": {(0, 0): "x"}}) @pytest.mark.parametrize("tools", [None, []]) diff --git a/vllm/entrypoints/cohere/api_router.py b/vllm/entrypoints/cohere/api_router.py index db63c30f4fa4..44968842adf1 100644 --- a/vllm/entrypoints/cohere/api_router.py +++ b/vllm/entrypoints/cohere/api_router.py @@ -190,7 +190,12 @@ async def chat_v2_render(request: CohereChatV2Request, raw_request: Request): try: chat_request = handler.to_chat_completion_request(request) - result = await render_handler.render_chat_request(chat_request) + # Citations from derender mode lack document sources: the + # position-to-source map can't go over JSON. + result = await render_handler.render_chat_request( + chat_request, + engine_chat_template_kwargs=handler._engine_chat_template_kwargs, + ) except Exception as e: logger.exception("Error in /cohere/v2/chat/render: %s", e) return _error_response(create_error_response(e), raw_request) @@ -202,12 +207,6 @@ async def chat_v2_render(request: CohereChatV2Request, raw_request: Request): kwargs.chat_template_kwargs = handler._engine_chat_template_kwargs( kwargs.chat_template_kwargs ) - # Citations from derender mode lack document sources: the - # position-to-source map can't go over JSON. - if (parse_context := result.parse_context) is not None: - parse_context.chat_template_kwargs = handler._engine_chat_template_kwargs( - parse_context.chat_template_kwargs - ) return JSONResponse(content=result.model_dump()) diff --git a/vllm/entrypoints/scale_out/render/serving.py b/vllm/entrypoints/scale_out/render/serving.py index 33d2cbbce457..6d68e75ee64c 100644 --- a/vllm/entrypoints/scale_out/render/serving.py +++ b/vllm/entrypoints/scale_out/render/serving.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from collections.abc import Callable from typing import TYPE_CHECKING, Any from fastapi import Request @@ -101,11 +102,22 @@ async def render_chat_request( self, request: ChatCompletionRequest, raw_request: Request | None = None, + engine_chat_template_kwargs: ( + Callable[[dict[str, Any]], dict[str, Any]] | None + ) = None, ) -> GenerateRequest | ErrorResponse: """Validate the model and preprocess a chat completion request. This is the authoritative implementation used directly by the GPU-less render server and delegated to by OpenAIServingChat. + + Args: + request: The chat completion request. + raw_request: The incoming HTTP request, if any. + engine_chat_template_kwargs: The calling handler's + `_engine_chat_template_kwargs`, applied to the kwargs put in + `parse_context` to drop API server only state. + """ error_check_ret = await self._check_model(request) if error_check_ret is not None: @@ -189,7 +201,12 @@ async def render_chat_request( cache_salt=request.cache_salt, priority=request.priority, token_offsets=engine_input.get("prompt_token_offsets"), - parse_context=self._build_parse_context(request, chat_template_kwargs), + parse_context=self._build_parse_context( + request, + engine_chat_template_kwargs(chat_template_kwargs) + if engine_chat_template_kwargs is not None + else chat_template_kwargs, + ), ) def _build_parse_context( diff --git a/vllm/entrypoints/scale_out/token_in_token_out/protocol.py b/vllm/entrypoints/scale_out/token_in_token_out/protocol.py index 6312950c01ab..ae71640288e3 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/protocol.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/protocol.py @@ -7,6 +7,7 @@ ConfigDict, Discriminator, Field, + JsonValue, NonNegativeInt, PrivateAttr, Tag, @@ -34,12 +35,15 @@ ) from vllm.entrypoints.serve.engine.protocol import UsageInfo from vllm.exceptions import VLLMValidationError +from vllm.logger import init_logger from vllm.logprobs import Logprob from vllm.outputs import WeightVersionSpan from vllm.renderers import TokenizeParams from vllm.sampling_params import SamplingParams from vllm.utils import random_uuid +logger = init_logger(__name__) + ####### Tokens IN <> Tokens OUT ####### OutputMode: TypeAlias = Literal["tokens", "text", "derender"] @@ -49,10 +53,11 @@ class ParseContext(BaseModel): """Chat request context the generate server needs to parse output. Built by `/render` from the post-`adjust_request` chat request and read - only when `output_mode="derender"`. + only when `output_mode="derender"`. Unknown fields from a newer `/render` + are kept so `GenerateRequest` can reject them only in that mode. """ - model_config = ConfigDict(extra="forbid") + model_config = ConfigDict(extra="allow") tools: list[ChatCompletionToolsParam] | None = None tool_choice: ( @@ -60,8 +65,9 @@ class ParseContext(BaseModel): ) = "none" parallel_tool_calls: bool | None = True include_reasoning: bool = True - chat_template_kwargs: dict[str, Any] = Field(default_factory=dict) - """Template kwargs with the render server's defaults already applied.""" + chat_template_kwargs: dict[str, JsonValue] = Field(default_factory=dict) + """Template kwargs with the render server's defaults already applied, + without the API server only entries a handler keeps out of the engine.""" history_tool_call_cnt: int = Field(default=0, ge=0) """Tool calls already in the rendered conversation, for sequential IDs.""" @@ -392,6 +398,25 @@ def _check_output_mode_detokenizes(self) -> "GenerateRequest": ) return self + @model_validator(mode="after") + def _check_parse_context_fields(self) -> "GenerateRequest": + """Fail on fields from a newer `/render` only where they are read.""" + if self.parse_context is None or not ( + unknown := sorted(self.parse_context.model_extra or ()) + ): + return self + if self.output_mode == "derender": + raise ValueError( + f"Unknown parse_context fields {unknown}; the render server " + "is newer than this generate server" + ) + logger.warning_once( + "Ignoring unknown parse_context fields %s at output_mode=%r", + ", ".join(unknown), + self.output_mode, + ) + return self + @model_validator(mode="after") def _validate_multimodal_feature_bounds(self) -> "GenerateRequest": if self.features is None: From cba37b4f6140136dbb5b82312b969dedc0aca022 Mon Sep 17 00:00:00 2001 From: Martin Hickey Date: Tue, 20 Oct 2026 21:19:42 +0100 Subject: [PATCH 09/11] Hide sampling_mask with hidden reasoning and fix citation docs When include_reasoning is false, derender mode now drops sampling_mask along with logprobs and token_ids. Otherwise the mask would give away the hidden reasoning tokens. Derender mode doesn't return Cohere citations at all, so the comment and docs no longer say it returns them without sources. Signed-off-by: Martin Hickey --- docs/serving/online_serving/token_in_token_out.md | 4 ++-- vllm/entrypoints/cohere/api_router.py | 4 ++-- vllm/entrypoints/scale_out/token_in_token_out/serving.py | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/serving/online_serving/token_in_token_out.md b/docs/serving/online_serving/token_in_token_out.md index de5f9ee50fd3..f71c5e78fb83 100644 --- a/docs/serving/online_serving/token_in_token_out.md +++ b/docs/serving/online_serving/token_in_token_out.md @@ -91,10 +91,10 @@ print(response["choices"][0]["token_ids"]) - The parser reads the engine's `text`, so it sees the stop string cut and the post `adjust_request` decode flags. That makes it match coupled chat for parsers whose `adjust_request` changes decoding which `/derender` can't. - A server with a tool or reasoning parser needs `parse_context`. Without a parser, `message.content` is the plain text. - `parse_context` is ignored at other levels, so a rendered request can go to a prefill leg unchanged. -- `logprobs` follow coupled chat. `top_logprobs` is cut to `sampling_params.logprobs`, `logprob_token_ids` is honored. When a parser hides the reasoning with `include_reasoning: false`, `logprobs` and `token_ids` are dropped so they don't expose it. +- `logprobs` follow coupled chat. `top_logprobs` is cut to `sampling_params.logprobs`, `logprob_token_ids` is honored. When a parser hides the reasoning with `include_reasoning: false`, `logprobs`, `token_ids` and `sampling_mask` are dropped so they don't expose it. - `finish_reason` follows coupled chat too. It's `tool_calls` when the message has tool calls except with a named `tool_choice` or a `length` cut. - `stream: true` isn't supported yet. -- `/cohere/v2/chat/render` drops the Cohere document map from `parse_context`, so citations come back without their document sources. +- Cohere grounding citations aren't returned. The parser strips the citation spans from `content`, and the document map can't go over JSON in `parse_context`. ### Logprobs diff --git a/vllm/entrypoints/cohere/api_router.py b/vllm/entrypoints/cohere/api_router.py index 44968842adf1..1d1f825d4880 100644 --- a/vllm/entrypoints/cohere/api_router.py +++ b/vllm/entrypoints/cohere/api_router.py @@ -190,8 +190,8 @@ async def chat_v2_render(request: CohereChatV2Request, raw_request: Request): try: chat_request = handler.to_chat_completion_request(request) - # Citations from derender mode lack document sources: the - # position-to-source map can't go over JSON. + # Derender mode returns no citations: the position-to-source + # map can't go over JSON. result = await render_handler.render_chat_request( chat_request, engine_chat_template_kwargs=handler._engine_chat_template_kwargs, diff --git a/vllm/entrypoints/scale_out/token_in_token_out/serving.py b/vllm/entrypoints/scale_out/token_in_token_out/serving.py index 068f14abe853..b46720e8a29d 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/serving.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/serving.py @@ -528,7 +528,7 @@ async def serve_tokens_full_generator( except (ValueError, VLLMValidationError) as e: return self.create_error_response(e) # Like coupled chat, hide per-token metadata when a parser hides - # the reasoning: logprobs or token IDs would leak it. + # the reasoning: logprobs, token IDs or sampling masks would leak it. hide_metadata = ( derender_request is not None and not derender_request.include_reasoning ) @@ -539,6 +539,7 @@ async def serve_tokens_full_generator( if hide_metadata: fields["logprobs"] = None fields["token_ids"] = None + fields["sampling_mask"] = None derender_choices.append( GenerateDerenderChoice(message=message, **fields) ) From 32c85c363a40e2f252c27ccd26961cd8bde9c8ab Mon Sep 17 00:00:00 2001 From: Martin Hickey Date: Wed, 21 Oct 2026 10:46:32 +0100 Subject: [PATCH 10/11] Fix ruff error after rebase Signed-off-by: Martin Hickey --- vllm/renderers/online_derenderer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/vllm/renderers/online_derenderer.py b/vllm/renderers/online_derenderer.py index 7620119f4e19..8f801f180c28 100644 --- a/vllm/renderers/online_derenderer.py +++ b/vllm/renderers/online_derenderer.py @@ -43,7 +43,10 @@ ) from vllm.entrypoints.serve.engine.protocol import UsageInfo from vllm.entrypoints.serve.utils.request_logger import RequestLogger -from vllm.entrypoints.serve.utils.tool_calls_utils import resolve_finish_reason +from vllm.entrypoints.serve.utils.tool_calls_utils import ( + maybe_filter_parallel_tool_calls, + resolve_finish_reason, +) from vllm.logger import init_logger from vllm.outputs import CompletionOutput from vllm.parser import Parser, ParserManager From caba43414e18ac522c83b31df7364ff60134f707 Mon Sep 17 00:00:00 2001 From: Martin Hickey Date: Wed, 21 Oct 2026 13:04:45 +0100 Subject: [PATCH 11/11] Fix flaky reasoning hidden derender parity test With include_reasoning=False, both the coupled and inline paths hide token_ids which means the determinism check was comparing None to None. The two greedy runs can still produce different text (seen on ROCm) and the content check then failed. When token ids are hidden, only check that token_ids and reasoning are hidden on both sides and skip the text comparison. The reasoning case still covers content parity for the same prompt. Signed-off-by: Martin Hickey --- .../entrypoints/scale_out/derender/test_derender_parity.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/entrypoints/scale_out/derender/test_derender_parity.py b/tests/entrypoints/scale_out/derender/test_derender_parity.py index 121a79853697..a31cfc69f32a 100644 --- a/tests/entrypoints/scale_out/derender/test_derender_parity.py +++ b/tests/entrypoints/scale_out/derender/test_derender_parity.py @@ -917,6 +917,13 @@ async def _inline_derender( def _assert_inline_parity(coupled: dict, inline: dict) -> None: assert len(inline["choices"]) == len(coupled["choices"]) for c, i in zip(coupled["choices"], inline["choices"]): + if c["token_ids"] is None: + # Hidden reasoning suppresses token ids on both paths, so the two + # generations can't be shown to match and their text may differ. + assert i["token_ids"] is None + assert i["message"].get("reasoning") is None + assert c["message"].get("reasoning") is None + continue assert i["token_ids"] == c["token_ids"], ( "greedy or seeded generation was expected to be deterministic " "across the coupled and inline calls"