Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/serving/online_serving/derenderer.md
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ The server keeps no state between calls. Everything the next call needs is in `s
- **Send every chunk through, including the last ones.** The finish reason usually arrives with the final tokens. A chunk with a `finish_reason` and no tokens is accepted too and flushes any buffered tool call arguments. With `stream_options: {"include_usage": true}`, generate ends with a usage only chunk (`choices: []`). Send it through derender like any other chunk to get the usage chunk for the response. `usage.prompt_tokens` is the request's `prompt_tokens` if set, otherwise the generate chunk's.
- **Send the same context on every call.** `chat_request` and `prompt_token_ids` aren't kept between calls, so they go with every chunk including the usage chunk. Both are required when a tool or reasoning parser is configured. `prompt_token_ids` is the `token_ids` of the `GenerateRequest` returned by `/render`.
- **Don't forward `[DONE]`.** It marks the end of the generate stream and isn't a chunk.
- **`stop` strings are rejected.** A streaming chat or completions request that carries stop strings returns `400`. The engine holds that text back with `output_mode=text`. Batch `/derender` still applies stop strings. A known stop token on the final chunk is still dropped, and that check does not add fields to `stream_state`.

Streaming derender resolves `logprobs` on the plain detokenization path, including `token_id:N` placeholders, same as the non-streaming endpoints (see [Streaming state and logprobs](#streaming-state-and-logprobs)). The parser path doesn't resolve them yet and drops them.

Expand Down
268 changes: 268 additions & 0 deletions tests/entrypoints/scale_out/derender/test_derender_stop.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,268 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

"""Batch and streaming derender must hide stop text the coupled server hides."""

import asyncio
from types import SimpleNamespace

from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
GenerateResponse,
GenerateResponseChoice,
GenerateResponseStreamChoice,
GenerateStreamResponse,
)
from vllm.renderers.online_derenderer import (
OnlineDerenderer,
decode_with_stop,
normalize_stop_strings,
truncate_at_stop_string,
)

WORDS = {1: "one", 2: "two", 3: "three", 4: "number"}


class _Tokenizer:
def __init__(self, eos_token_id=None):
self.eos_token_id = eos_token_id

def decode(self, token_ids, skip_special_tokens=True):
return " ".join(WORDS[token_id] for token_id in token_ids)


def _derenderer(eos_token_id=None, generation_eos=None):
model_config = None
if generation_eos is not None:
model_config = SimpleNamespace(
try_get_generation_config=lambda: {"eos_token_id": generation_eos}
)
return SimpleNamespace(
parser=None,
model_config=model_config,
renderer=SimpleNamespace(
get_tokenizer=lambda: _Tokenizer(eos_token_id=eos_token_id)
),
)


def _chat(token_ids, *, eos_token_id=None, generation_eos=None, **kwargs):
response = GenerateResponse(
request_id="chatcmpl-stop",
choices=[
GenerateResponseChoice(
index=0,
token_ids=token_ids,
finish_reason=kwargs.pop("finish_reason", "stop"),
)
],
)
request = ChatCompletionRequest(
model="tiny",
messages=[{"role": "user", "content": "hi"}],
**kwargs,
)
choices = OnlineDerenderer._derender_chat(
_derenderer(eos_token_id=eos_token_id, generation_eos=generation_eos),
response,
request,
)
return choices[0]


def test_chat_stop_string_is_left_out_of_the_text():
choice = _chat([1, 2, 3], stop=["three"])
# Same cut as the engine: the stop word goes, the space before it stays.
assert choice.message.content == "one two "
assert choice.stop_reason == "three"


def test_chat_keeps_the_stop_string_when_asked():
choice = _chat([1, 2, 3], stop=["three"], include_stop_str_in_output=True)
assert choice.message.content == "one two three"
assert choice.stop_reason == "three"


def test_chat_keeps_the_last_token_when_it_is_not_a_stop_token():
# finish_reason stop alone is not enough. Round-trip fixtures encode
# real text and do not append a stop id.
choice = _chat([1, 2, 3])
assert choice.message.content == "one two three"
assert choice.stop_reason is None


def test_chat_client_stop_token_drops_the_last_id():
choice = _chat([1, 2, 3], stop_token_ids=[3])
assert choice.message.content == "one two"
assert choice.stop_reason == 3


def test_chat_primary_eos_drops_the_last_id_and_leaves_stop_reason_empty():
choice = _chat([1, 2, 3], eos_token_id=3)
assert choice.message.content == "one two"
assert choice.stop_reason is None


def test_chat_generation_config_eos_drops_the_last_id():
choice = _chat([1, 2, 3], generation_eos=[3, 9])
assert choice.message.content == "one two"
assert choice.stop_reason == 3


def test_chat_keeps_the_stop_token_when_asked():
choice = _chat([1, 2, 3], stop_token_ids=[3], include_stop_str_in_output=True)
assert choice.message.content == "one two three"
assert choice.stop_reason == 3


def test_chat_token_stop_runs_before_the_string_scan():
# The engine drops the stop token, then looks for a stop string in what remains.
choice = _chat([1, 2, 3], eos_token_id=3, stop=["two"])
assert choice.message.content == "one "
assert choice.stop_reason == "two"


def test_length_finish_keeps_the_last_token():
choice = _chat([1, 2, 3], finish_reason="length")
assert choice.message.content == "one two three"
assert choice.stop_reason is None


def test_completion_stop_string_is_left_out_of_the_text():
response = GenerateResponse(
request_id="cmpl-stop",
choices=[
GenerateResponseChoice(
index=0,
token_ids=[1, 4],
finish_reason="stop",
)
],
)
request = CompletionRequest(model="tiny", prompt="hi", stop=["number"])
choices, _, _ = OnlineDerenderer._derender_completion(
_derenderer(),
[response],
completion_request=request,
)
assert choices[0].text == "one "
assert choices[0].stop_reason == "number"


def test_streaming_stop_strings_are_rejected_by_the_helper():
assert normalize_stop_strings(["three"]) == ["three"]
assert normalize_stop_strings([]) == []
assert normalize_stop_strings(None) == []


def test_stop_helpers_match_the_coupled_rule():
text, ids, reason = decode_with_stop(
_Tokenizer(),
[1, 2, 3],
skip_special_tokens=True,
finish_reason="stop",
stop=None,
include_stop_str_in_output=False,
stop_token_ids=[3],
)
assert text == "one two"
assert ids == [1, 2]
assert reason == 3
kept, kept_ids, kept_reason = decode_with_stop(
_Tokenizer(),
[1, 2, 3],
skip_special_tokens=True,
finish_reason="stop",
stop=["three"],
include_stop_str_in_output=False,
)
assert kept == "one two "
assert kept_ids == [1, 2, 3]
assert kept_reason == "three"
text, reason = truncate_at_stop_string("one two three", ["three"], False)
assert text == "one two "
assert reason == "three"


def _stream_chunk(token_ids, finish_reason=None):
return GenerateStreamResponse(
request_id="cmpl-stream-stop",
choices=[
GenerateResponseStreamChoice(
index=0,
token_ids=token_ids,
finish_reason=finish_reason,
)
],
)


def test_completion_stream_drops_a_known_stop_token_on_the_final_chunk():
seen: list[list[int]] = []

async def detok(tokenizer, delta_token_ids, stream_state, skip_special_tokens=True):
seen.append(list(delta_token_ids))
text = " ".join(WORDS[token_id] for token_id in delta_token_ids)
return text, stream_state

derenderer = SimpleNamespace(
_detokenize_delta_async=detok,
renderer=SimpleNamespace(get_tokenizer=lambda: _Tokenizer(eos_token_id=9)),
model_config=None,
)
request = CompletionRequest(model="tiny", prompt="hi", stop_token_ids=[3])

async def run():
first, stream_state = await OnlineDerenderer.derender_completion_stream(
derenderer,
"tiny",
_stream_chunk([1, 2]),
completion_request=request,
)
second, stream_state = await OnlineDerenderer.derender_completion_stream(
derenderer,
"tiny",
_stream_chunk([3], finish_reason="stop"),
state=stream_state,
completion_request=request,
)
return first, second, stream_state

first, second, stream_state = asyncio.run(run())
assert first.choices[0].text == "one two"
assert first.choices[0].stop_reason is None
assert second.choices[0].text == ""
assert second.choices[0].stop_reason == 3
assert not hasattr(stream_state, "held_text")
assert seen == [[1, 2], []]


def test_completion_stream_keeps_the_stop_token_when_asked():
async def detok(tokenizer, delta_token_ids, stream_state, skip_special_tokens=True):
text = " ".join(WORDS[token_id] for token_id in delta_token_ids)
return text, stream_state

derenderer = SimpleNamespace(
_detokenize_delta_async=detok,
renderer=SimpleNamespace(get_tokenizer=lambda: _Tokenizer()),
model_config=None,
)
request = CompletionRequest(
model="tiny",
prompt="hi",
stop_token_ids=[3],
include_stop_str_in_output=True,
)

async def run():
return await OnlineDerenderer.derender_completion_stream(
derenderer,
"tiny",
_stream_chunk([1, 3], finish_reason="stop"),
completion_request=request,
)

chunk, _ = asyncio.run(run())
assert chunk.choices[0].text == "one three"
assert chunk.choices[0].stop_reason == 3
63 changes: 63 additions & 0 deletions tests/entrypoints/scale_out/derender/test_derender_stream.py
Original file line number Diff line number Diff line change
Expand Up @@ -1345,6 +1345,69 @@ async def test_missing_chat_request_without_parser_ok(self):
result = await serving.derender_chat_stream_response(request)
assert not isinstance(result, ErrorResponse)

@pytest.mark.asyncio
async def test_plain_stream_rejects_stop_strings(self):
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
DerenderChatStreamRequest,
)

serving = self._make_serving(parser_configured=False)
request = DerenderChatStreamRequest(
stream=True,
model=MODEL_NAME,
generate_chunk=_make_stream_chunk([1, 2]),
chat_request=_chat_request(stop=["three"]),
)
result = await serving.derender_chat_stream_response(request)
assert isinstance(result, ErrorResponse)
assert result.error.code == 400
assert "stop strings" in result.error.message
assert "output_mode=text" in result.error.message
serving.online_derenderer.derender_chat_stream.assert_not_called()

@pytest.mark.asyncio
async def test_completion_stream_rejects_stop_strings(self):
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
DerenderCompletionStreamRequest,
)

serving = self._make_serving(parser_configured=False)
request = DerenderCompletionStreamRequest(
stream=True,
model=MODEL_NAME,
generate_chunk=_make_stream_chunk([1, 2]),
completion_request=CompletionRequest(
model=MODEL_NAME, prompt="hi", stop=["three"]
),
)
result = await serving.derender_completion_stream_response(request)
assert isinstance(result, ErrorResponse)
assert result.error.code == 400
assert "stop strings" in result.error.message
assert "output_mode=text" in result.error.message
serving.online_derenderer.derender_completion_stream.assert_not_called()

@pytest.mark.asyncio
async def test_parser_stream_rejects_stop_strings(self):
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
DerenderChatStreamRequest,
)

serving = self._make_serving(parser_configured=True)
request = DerenderChatStreamRequest(
stream=True,
model=MODEL_NAME,
generate_chunk=_make_stream_chunk([1, 2]),
chat_request=_chat_request(stop=["three"]),
prompt_token_ids=[1],
)
result = await serving.derender_chat_stream_response(request)
assert isinstance(result, ErrorResponse)
assert result.error.code == 400
assert "stop strings" in result.error.message
serving.online_derenderer.derender_chat_stream.assert_not_called()

@pytest.mark.asyncio
async def test_missing_prompt_token_ids_with_parser_rejected(self):
"""A parser configured model must reject a missing prompt_token_ids
Expand Down
25 changes: 24 additions & 1 deletion vllm/entrypoints/scale_out/derender/serving.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,10 @@
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.inputs import EngineInput
from vllm.logger import init_logger
from vllm.renderers.online_derenderer import OnlineDerenderer
from vllm.renderers.online_derenderer import (
OnlineDerenderer,
normalize_stop_strings,
)

from ..token_in_token_out.mm_features import extract_mm_features
from ..token_in_token_out.protocol import (
Expand All @@ -35,6 +38,16 @@

logger = init_logger(__name__)

# Streaming derender does not hold a stop string across chunks. The engine
# already does that hold-back when the caller asks for output_mode=text.
_STREAMING_STOP_STRINGS = (
"stop strings are not supported on streaming /derender. "
"The engine holds them back with output_mode=text "
"(https://github.com/vllm-project/vllm/issues/56851). "
"Send the finished generate response to the non-streaming "
"derender endpoint."
)


class ServingDerender(BaseServing):
def __init__(
Expand Down Expand Up @@ -271,6 +284,11 @@ async def derender_chat_stream_response(
"markup into content."
)

if request.chat_request is not None and normalize_stop_strings(
request.chat_request.stop
):
return self.create_error_response(_STREAMING_STOP_STRINGS)

if (
self.online_derenderer.parser is not None
and request.prompt_token_ids is None
Expand Down Expand Up @@ -367,6 +385,11 @@ async def derender_completion_stream_response(
if error_check_ret is not None:
return error_check_ret

if request.completion_request is not None and normalize_stop_strings(
request.completion_request.stop
):
return self.create_error_response(_STREAMING_STOP_STRINGS)

model_name = request.model or self.models.model_name()
try:
(
Expand Down
Loading
Loading