diff --git a/docs/serving/online_serving/derenderer.md b/docs/serving/online_serving/derenderer.md index e6d1221fd4bc..82600a9559bc 100644 --- a/docs/serving/online_serving/derenderer.md +++ b/docs/serving/online_serving/derenderer.md @@ -97,6 +97,7 @@ The server keeps no state between calls. Everything the next call needs is in `s - **Send every chunk through, including the last ones.** The finish reason usually arrives with the final tokens. A chunk with a `finish_reason` and no tokens is accepted too and flushes any buffered tool call arguments. With `stream_options: {"include_usage": true}`, generate ends with a usage only chunk (`choices: []`). Send it through derender like any other chunk to get the usage chunk for the response. `usage.prompt_tokens` is the request's `prompt_tokens` if set, otherwise the generate chunk's. - **Send the same context on every call.** `chat_request` and `prompt_token_ids` aren't kept between calls, so they go with every chunk including the usage chunk. Both are required when a tool or reasoning parser is configured. `prompt_token_ids` is the `token_ids` of the `GenerateRequest` returned by `/render`. - **Don't forward `[DONE]`.** It marks the end of the generate stream and isn't a chunk. +- **`stop` strings are rejected.** A streaming chat or completions request that carries stop strings returns `400`. The engine holds that text back with `output_mode=text`. Batch `/derender` still applies stop strings. A known stop token on the final chunk is still dropped, and that check does not add fields to `stream_state`. Streaming derender resolves `logprobs` on the plain detokenization path, including `token_id:N` placeholders, same as the non-streaming endpoints (see [Streaming state and logprobs](#streaming-state-and-logprobs)). The parser path doesn't resolve them yet and drops them. diff --git a/tests/entrypoints/scale_out/derender/test_derender_stop.py b/tests/entrypoints/scale_out/derender/test_derender_stop.py new file mode 100644 index 000000000000..e89f931f4ad8 --- /dev/null +++ b/tests/entrypoints/scale_out/derender/test_derender_stop.py @@ -0,0 +1,268 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Batch and streaming derender must hide stop text the coupled server hides.""" + +import asyncio +from types import SimpleNamespace + +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest +from vllm.entrypoints.openai.completion.protocol import CompletionRequest +from vllm.entrypoints.scale_out.token_in_token_out.protocol import ( + GenerateResponse, + GenerateResponseChoice, + GenerateResponseStreamChoice, + GenerateStreamResponse, +) +from vllm.renderers.online_derenderer import ( + OnlineDerenderer, + decode_with_stop, + normalize_stop_strings, + truncate_at_stop_string, +) + +WORDS = {1: "one", 2: "two", 3: "three", 4: "number"} + + +class _Tokenizer: + def __init__(self, eos_token_id=None): + self.eos_token_id = eos_token_id + + def decode(self, token_ids, skip_special_tokens=True): + return " ".join(WORDS[token_id] for token_id in token_ids) + + +def _derenderer(eos_token_id=None, generation_eos=None): + model_config = None + if generation_eos is not None: + model_config = SimpleNamespace( + try_get_generation_config=lambda: {"eos_token_id": generation_eos} + ) + return SimpleNamespace( + parser=None, + model_config=model_config, + renderer=SimpleNamespace( + get_tokenizer=lambda: _Tokenizer(eos_token_id=eos_token_id) + ), + ) + + +def _chat(token_ids, *, eos_token_id=None, generation_eos=None, **kwargs): + response = GenerateResponse( + request_id="chatcmpl-stop", + choices=[ + GenerateResponseChoice( + index=0, + token_ids=token_ids, + finish_reason=kwargs.pop("finish_reason", "stop"), + ) + ], + ) + request = ChatCompletionRequest( + model="tiny", + messages=[{"role": "user", "content": "hi"}], + **kwargs, + ) + choices = OnlineDerenderer._derender_chat( + _derenderer(eos_token_id=eos_token_id, generation_eos=generation_eos), + response, + request, + ) + return choices[0] + + +def test_chat_stop_string_is_left_out_of_the_text(): + choice = _chat([1, 2, 3], stop=["three"]) + # Same cut as the engine: the stop word goes, the space before it stays. + assert choice.message.content == "one two " + assert choice.stop_reason == "three" + + +def test_chat_keeps_the_stop_string_when_asked(): + choice = _chat([1, 2, 3], stop=["three"], include_stop_str_in_output=True) + assert choice.message.content == "one two three" + assert choice.stop_reason == "three" + + +def test_chat_keeps_the_last_token_when_it_is_not_a_stop_token(): + # finish_reason stop alone is not enough. Round-trip fixtures encode + # real text and do not append a stop id. + choice = _chat([1, 2, 3]) + assert choice.message.content == "one two three" + assert choice.stop_reason is None + + +def test_chat_client_stop_token_drops_the_last_id(): + choice = _chat([1, 2, 3], stop_token_ids=[3]) + assert choice.message.content == "one two" + assert choice.stop_reason == 3 + + +def test_chat_primary_eos_drops_the_last_id_and_leaves_stop_reason_empty(): + choice = _chat([1, 2, 3], eos_token_id=3) + assert choice.message.content == "one two" + assert choice.stop_reason is None + + +def test_chat_generation_config_eos_drops_the_last_id(): + choice = _chat([1, 2, 3], generation_eos=[3, 9]) + assert choice.message.content == "one two" + assert choice.stop_reason == 3 + + +def test_chat_keeps_the_stop_token_when_asked(): + choice = _chat([1, 2, 3], stop_token_ids=[3], include_stop_str_in_output=True) + assert choice.message.content == "one two three" + assert choice.stop_reason == 3 + + +def test_chat_token_stop_runs_before_the_string_scan(): + # The engine drops the stop token, then looks for a stop string in what remains. + choice = _chat([1, 2, 3], eos_token_id=3, stop=["two"]) + assert choice.message.content == "one " + assert choice.stop_reason == "two" + + +def test_length_finish_keeps_the_last_token(): + choice = _chat([1, 2, 3], finish_reason="length") + assert choice.message.content == "one two three" + assert choice.stop_reason is None + + +def test_completion_stop_string_is_left_out_of_the_text(): + response = GenerateResponse( + request_id="cmpl-stop", + choices=[ + GenerateResponseChoice( + index=0, + token_ids=[1, 4], + finish_reason="stop", + ) + ], + ) + request = CompletionRequest(model="tiny", prompt="hi", stop=["number"]) + choices, _, _ = OnlineDerenderer._derender_completion( + _derenderer(), + [response], + completion_request=request, + ) + assert choices[0].text == "one " + assert choices[0].stop_reason == "number" + + +def test_streaming_stop_strings_are_rejected_by_the_helper(): + assert normalize_stop_strings(["three"]) == ["three"] + assert normalize_stop_strings([]) == [] + assert normalize_stop_strings(None) == [] + + +def test_stop_helpers_match_the_coupled_rule(): + text, ids, reason = decode_with_stop( + _Tokenizer(), + [1, 2, 3], + skip_special_tokens=True, + finish_reason="stop", + stop=None, + include_stop_str_in_output=False, + stop_token_ids=[3], + ) + assert text == "one two" + assert ids == [1, 2] + assert reason == 3 + kept, kept_ids, kept_reason = decode_with_stop( + _Tokenizer(), + [1, 2, 3], + skip_special_tokens=True, + finish_reason="stop", + stop=["three"], + include_stop_str_in_output=False, + ) + assert kept == "one two " + assert kept_ids == [1, 2, 3] + assert kept_reason == "three" + text, reason = truncate_at_stop_string("one two three", ["three"], False) + assert text == "one two " + assert reason == "three" + + +def _stream_chunk(token_ids, finish_reason=None): + return GenerateStreamResponse( + request_id="cmpl-stream-stop", + choices=[ + GenerateResponseStreamChoice( + index=0, + token_ids=token_ids, + finish_reason=finish_reason, + ) + ], + ) + + +def test_completion_stream_drops_a_known_stop_token_on_the_final_chunk(): + seen: list[list[int]] = [] + + async def detok(tokenizer, delta_token_ids, stream_state, skip_special_tokens=True): + seen.append(list(delta_token_ids)) + text = " ".join(WORDS[token_id] for token_id in delta_token_ids) + return text, stream_state + + derenderer = SimpleNamespace( + _detokenize_delta_async=detok, + renderer=SimpleNamespace(get_tokenizer=lambda: _Tokenizer(eos_token_id=9)), + model_config=None, + ) + request = CompletionRequest(model="tiny", prompt="hi", stop_token_ids=[3]) + + async def run(): + first, stream_state = await OnlineDerenderer.derender_completion_stream( + derenderer, + "tiny", + _stream_chunk([1, 2]), + completion_request=request, + ) + second, stream_state = await OnlineDerenderer.derender_completion_stream( + derenderer, + "tiny", + _stream_chunk([3], finish_reason="stop"), + state=stream_state, + completion_request=request, + ) + return first, second, stream_state + + first, second, stream_state = asyncio.run(run()) + assert first.choices[0].text == "one two" + assert first.choices[0].stop_reason is None + assert second.choices[0].text == "" + assert second.choices[0].stop_reason == 3 + assert not hasattr(stream_state, "held_text") + assert seen == [[1, 2], []] + + +def test_completion_stream_keeps_the_stop_token_when_asked(): + async def detok(tokenizer, delta_token_ids, stream_state, skip_special_tokens=True): + text = " ".join(WORDS[token_id] for token_id in delta_token_ids) + return text, stream_state + + derenderer = SimpleNamespace( + _detokenize_delta_async=detok, + renderer=SimpleNamespace(get_tokenizer=lambda: _Tokenizer()), + model_config=None, + ) + request = CompletionRequest( + model="tiny", + prompt="hi", + stop_token_ids=[3], + include_stop_str_in_output=True, + ) + + async def run(): + return await OnlineDerenderer.derender_completion_stream( + derenderer, + "tiny", + _stream_chunk([1, 3], finish_reason="stop"), + completion_request=request, + ) + + chunk, _ = asyncio.run(run()) + assert chunk.choices[0].text == "one three" + assert chunk.choices[0].stop_reason == 3 diff --git a/tests/entrypoints/scale_out/derender/test_derender_stream.py b/tests/entrypoints/scale_out/derender/test_derender_stream.py index 63ee9bcad8a4..f29fdbdcf7e2 100644 --- a/tests/entrypoints/scale_out/derender/test_derender_stream.py +++ b/tests/entrypoints/scale_out/derender/test_derender_stream.py @@ -1345,6 +1345,69 @@ async def test_missing_chat_request_without_parser_ok(self): result = await serving.derender_chat_stream_response(request) assert not isinstance(result, ErrorResponse) + @pytest.mark.asyncio + async def test_plain_stream_rejects_stop_strings(self): + from vllm.entrypoints.scale_out.token_in_token_out.protocol import ( + DerenderChatStreamRequest, + ) + + serving = self._make_serving(parser_configured=False) + request = DerenderChatStreamRequest( + stream=True, + model=MODEL_NAME, + generate_chunk=_make_stream_chunk([1, 2]), + chat_request=_chat_request(stop=["three"]), + ) + result = await serving.derender_chat_stream_response(request) + assert isinstance(result, ErrorResponse) + assert result.error.code == 400 + assert "stop strings" in result.error.message + assert "output_mode=text" in result.error.message + serving.online_derenderer.derender_chat_stream.assert_not_called() + + @pytest.mark.asyncio + async def test_completion_stream_rejects_stop_strings(self): + from vllm.entrypoints.openai.completion.protocol import CompletionRequest + from vllm.entrypoints.scale_out.token_in_token_out.protocol import ( + DerenderCompletionStreamRequest, + ) + + serving = self._make_serving(parser_configured=False) + request = DerenderCompletionStreamRequest( + stream=True, + model=MODEL_NAME, + generate_chunk=_make_stream_chunk([1, 2]), + completion_request=CompletionRequest( + model=MODEL_NAME, prompt="hi", stop=["three"] + ), + ) + result = await serving.derender_completion_stream_response(request) + assert isinstance(result, ErrorResponse) + assert result.error.code == 400 + assert "stop strings" in result.error.message + assert "output_mode=text" in result.error.message + serving.online_derenderer.derender_completion_stream.assert_not_called() + + @pytest.mark.asyncio + async def test_parser_stream_rejects_stop_strings(self): + from vllm.entrypoints.scale_out.token_in_token_out.protocol import ( + DerenderChatStreamRequest, + ) + + serving = self._make_serving(parser_configured=True) + request = DerenderChatStreamRequest( + stream=True, + model=MODEL_NAME, + generate_chunk=_make_stream_chunk([1, 2]), + chat_request=_chat_request(stop=["three"]), + prompt_token_ids=[1], + ) + result = await serving.derender_chat_stream_response(request) + assert isinstance(result, ErrorResponse) + assert result.error.code == 400 + assert "stop strings" in result.error.message + serving.online_derenderer.derender_chat_stream.assert_not_called() + @pytest.mark.asyncio async def test_missing_prompt_token_ids_with_parser_rejected(self): """A parser configured model must reject a missing prompt_token_ids diff --git a/vllm/entrypoints/scale_out/derender/serving.py b/vllm/entrypoints/scale_out/derender/serving.py index 5da736640373..8c95b8a46a83 100644 --- a/vllm/entrypoints/scale_out/derender/serving.py +++ b/vllm/entrypoints/scale_out/derender/serving.py @@ -20,7 +20,10 @@ from vllm.entrypoints.serve.utils.request_logger import RequestLogger from vllm.inputs import EngineInput from vllm.logger import init_logger -from vllm.renderers.online_derenderer import OnlineDerenderer +from vllm.renderers.online_derenderer import ( + OnlineDerenderer, + normalize_stop_strings, +) from ..token_in_token_out.mm_features import extract_mm_features from ..token_in_token_out.protocol import ( @@ -35,6 +38,16 @@ logger = init_logger(__name__) +# Streaming derender does not hold a stop string across chunks. The engine +# already does that hold-back when the caller asks for output_mode=text. +_STREAMING_STOP_STRINGS = ( + "stop strings are not supported on streaming /derender. " + "The engine holds them back with output_mode=text " + "(https://github.com/vllm-project/vllm/issues/56851). " + "Send the finished generate response to the non-streaming " + "derender endpoint." +) + class ServingDerender(BaseServing): def __init__( @@ -271,6 +284,11 @@ async def derender_chat_stream_response( "markup into content." ) + if request.chat_request is not None and normalize_stop_strings( + request.chat_request.stop + ): + return self.create_error_response(_STREAMING_STOP_STRINGS) + if ( self.online_derenderer.parser is not None and request.prompt_token_ids is None @@ -367,6 +385,11 @@ async def derender_completion_stream_response( if error_check_ret is not None: return error_check_ret + if request.completion_request is not None and normalize_stop_strings( + request.completion_request.stop + ): + return self.create_error_response(_STREAMING_STOP_STRINGS) + model_name = request.model or self.models.model_name() try: ( diff --git a/vllm/renderers/online_derenderer.py b/vllm/renderers/online_derenderer.py index e1f67397ce94..bbd74a52e19d 100644 --- a/vllm/renderers/online_derenderer.py +++ b/vllm/renderers/online_derenderer.py @@ -43,10 +43,144 @@ from vllm.tokenizers.detokenizer_utils import detokenize_incrementally from vllm.utils import random_uuid from vllm.utils.async_utils import make_async +from vllm.v1.engine.detokenizer import check_stop_strings logger = init_logger(__name__) +def normalize_stop_strings(stop: str | Sequence[str] | None) -> list[str]: + """Return the request stop strings, dropping empty values.""" + if stop is None: + return [] + if isinstance(stop, str): + return [stop] if stop else [] + return [item for item in stop if item] + + +def generation_eos_ids(model_config: Any) -> set[int]: + """EOS ids from the model's generation config, if that config is loaded.""" + getter = getattr(model_config, "try_get_generation_config", None) + if getter is None: + return set() + config = getter() or {} + eos_ids = config.get("eos_token_id") + if eos_ids is None: + return set() + if isinstance(eos_ids, int): + return {eos_ids} + return {int(token_id) for token_id in eos_ids} + + +def is_stop_token( + token_id: int, + stop_token_ids: Sequence[int] | None, + eos_token_id: int | None, + generation_eos: set[int], +) -> bool: + """True when this id is one the engine would treat as a token stop.""" + if stop_token_ids and token_id in stop_token_ids: + return True + if eos_token_id is not None and token_id == eos_token_id: + return True + return token_id in generation_eos + + +def decode_with_stop( + tokenizer: TokenizerLike, + token_ids: Sequence[int], + *, + skip_special_tokens: bool, + finish_reason: str | None, + stop: str | Sequence[str] | None, + include_stop_str_in_output: bool, + stop_token_ids: Sequence[int] | None = None, + eos_token_id: int | None = None, + generation_eos: set[int] | None = None, +) -> tuple[str, list[int], int | str | None]: + """Decode output text the way the engine's detokenizer does. + + A token stop is checked first, and only when the last id is a known stop + token. The string check then runs on whatever text is left. A matched + stop string wins. The primary EOS leaves stop_reason empty. + """ + ids = list(token_ids) + known_eos = generation_eos or set() + token_reason: int | None = None + if ( + finish_reason == "stop" + and ids + and is_stop_token(ids[-1], stop_token_ids, eos_token_id, known_eos) + ): + stopped_token = ids[-1] + token_reason = None if stopped_token == eos_token_id else stopped_token + if not include_stop_str_in_output: + ids = ids[:-1] + + text = tokenizer.decode(ids, skip_special_tokens=skip_special_tokens) + text, stopped_str = truncate_at_stop_string(text, stop, include_stop_str_in_output) + if stopped_str is not None: + return text, ids, stopped_str + return text, ids, token_reason + + +def truncate_at_stop_string( + text: str, + stop: str | Sequence[str] | None, + include_in_output: bool, +) -> tuple[str, str | None]: + """Cut decoded text at the earliest stop string. + + Returns the text the client should see and the matched stop string. + """ + strings = normalize_stop_strings(stop) + if not strings or not text: + return text, None + matched = check_stop_strings( + text, + len(text), + strings, + include_in_output, + ) + if matched is None: + return text, None + stop_str, offset = matched + if offset >= 0: + text = text[:offset] + return text, stop_str + + +def strip_finished_stop_token( + token_ids: Sequence[int], + *, + finish_reason: str | None, + include_in_output: bool, + stop_token_ids: Sequence[int] | None, + eos_token_id: int | None, + generation_eos: set[int], +) -> tuple[list[int], int | None]: + """Drop a trailing stop id on a finished chunk, matching the engine. + + Returns the ids to detokenize and the stop_reason for a non-primary + stop id. The primary EOS leaves the reason empty. + ``include_in_output`` keeps the id and still reports the reason. + This looks only at the current chunk, so streaming stores nothing for it. + """ + ids = list(token_ids) + if ( + finish_reason != "stop" + or not ids + or not is_stop_token(ids[-1], stop_token_ids, eos_token_id, generation_eos) + ): + return ids, None + token_id = ids[-1] + reason = ( + None if eos_token_id is not None and token_id == eos_token_id else token_id + ) + if include_in_output: + return ids, reason + return ids[:-1], reason + + class OnlineDerenderer: def __init__( self, @@ -129,6 +263,36 @@ def _derender_chat( if not choice.token_ids: raise ValueError(f"choice {choice.index} has empty or null token_ids") + include_stop = ( + chat_request.include_stop_str_in_output + if chat_request is not None + else False + ) + stop = chat_request.stop if chat_request is not None else None + use_parser = self.parser is not None and chat_request is not None + skip_special = ( + False + if use_parser + else ( + chat_request.skip_special_tokens + if chat_request is not None + else True + ) + ) + decoded_text, token_ids, stop_reason = decode_with_stop( + tokenizer, + choice.token_ids, + skip_special_tokens=skip_special, + finish_reason=choice.finish_reason, + stop=stop, + include_stop_str_in_output=include_stop, + stop_token_ids=( + chat_request.stop_token_ids if chat_request is not None else None + ), + eos_token_id=getattr(tokenizer, "eos_token_id", None), + generation_eos=generation_eos_ids(getattr(self, "model_config", None)), + ) + resolved_logprobs = ( _resolve_logprobs(choice.logprobs, tokenizer) if choice.logprobs is not None @@ -136,13 +300,8 @@ def _derender_chat( ) if self.parser is not None and chat_request is not None: - # Parser path: decode with special tokens preserved - # so the parser can see markers like , - # , or Harmony channel tokens. - decoded_text = tokenizer.decode( - choice.token_ids, skip_special_tokens=False - ) - + # Parser path keeps special-token markers so the parser can + # see , , or Harmony channel tokens. chat_template_kwargs: dict[str, Any] = {} if not self.use_harmony: chat_template_kwargs = ( @@ -164,7 +323,7 @@ def _derender_chat( decoded_text, chat_request, enable_auto_tools=self.enable_auto_tools, - model_output_token_ids=choice.token_ids, + model_output_token_ids=token_ids, ) if not getattr(chat_request, "include_reasoning", True): @@ -196,16 +355,6 @@ def _derender_chat( tool_calls=tc_items, ) else: - # No parser: plain detokenization honouring the request's - # skip_special_tokens (default True when no request was given). - skip_special = ( - chat_request.skip_special_tokens - if chat_request is not None - else True - ) - decoded_text = tokenizer.decode( - choice.token_ids, skip_special_tokens=skip_special - ) message = ChatMessage(role="assistant", content=decoded_text) choices.append( @@ -214,6 +363,7 @@ def _derender_chat( message=message, logprobs=resolved_logprobs, finish_reason=choice.finish_reason, + stop_reason=stop_reason, ) ) @@ -360,13 +510,34 @@ async def derender_chat_stream( skip_special = ( chat_request.skip_special_tokens if chat_request is not None else True ) + include_stop = ( + chat_request.include_stop_str_in_output + if chat_request is not None + else False + ) + stop_token_ids = ( + chat_request.stop_token_ids if chat_request is not None else None + ) + eos_token_id = getattr(tokenizer, "eos_token_id", None) + generation_eos = generation_eos_ids(getattr(self, "model_config", None)) stream_choices: list[ChatCompletionResponseStreamChoice] = [] updated_state = state for choice in generate_chunk.choices: - delta_tids = choice.token_ids or [] + raw_tids = list(choice.token_ids or []) + decode_tids, stop_reason = strip_finished_stop_token( + raw_tids, + finish_reason=choice.finish_reason, + include_in_output=include_stop, + stop_token_ids=stop_token_ids, + eos_token_id=eos_token_id, + generation_eos=generation_eos, + ) new_text, updated_state = await self._detokenize_delta_async( - tokenizer, delta_tids, updated_state, skip_special_tokens=skip_special + tokenizer, + decode_tids, + updated_state, + skip_special_tokens=skip_special, ) # NOTE: parser-configured servers dispatch to @@ -389,7 +560,7 @@ async def derender_chat_stream( update={ "role_sent": True, "logprob_context_token_ids": _logprob_context_tail( - state.logprob_context_token_ids, delta_tids + state.logprob_context_token_ids, raw_tids ), } ) @@ -404,6 +575,7 @@ async def derender_chat_stream( delta=delta, logprobs=resolved_logprobs, finish_reason=choice.finish_reason, + stop_reason=stop_reason, ) ) @@ -665,8 +837,30 @@ def _derender_completion( "has empty or null token_ids" ) - decoded_text = tokenizer.decode( - choice.token_ids, skip_special_tokens=skip_special + include_stop = ( + completion_request.include_stop_str_in_output + if completion_request is not None + else False + ) + stop = ( + completion_request.stop if completion_request is not None else None + ) + decoded_text, _, stop_reason = decode_with_stop( + tokenizer, + choice.token_ids, + skip_special_tokens=skip_special, + finish_reason=choice.finish_reason, + stop=stop, + include_stop_str_in_output=include_stop, + stop_token_ids=( + completion_request.stop_token_ids + if completion_request is not None + else None + ), + eos_token_id=getattr(tokenizer, "eos_token_id", None), + generation_eos=generation_eos_ids( + getattr(self, "model_config", None) + ), ) completion_logprobs = None if choice.logprobs is not None: @@ -679,6 +873,7 @@ def _derender_completion( index=index, text=decoded_text, finish_reason=choice.finish_reason, + stop_reason=stop_reason, logprobs=completion_logprobs, ) ) @@ -736,13 +931,36 @@ async def derender_completion_stream( if completion_request is not None else True ) + include_stop = ( + completion_request.include_stop_str_in_output + if completion_request is not None + else False + ) + stop_token_ids = ( + completion_request.stop_token_ids + if completion_request is not None + else None + ) + eos_token_id = getattr(tokenizer, "eos_token_id", None) + generation_eos = generation_eos_ids(getattr(self, "model_config", None)) stream_choices: list[CompletionResponseStreamChoice] = [] updated_state = state for choice in generate_chunk.choices: - delta_tids = choice.token_ids or [] + raw_tids = list(choice.token_ids or []) + decode_tids, stop_reason = strip_finished_stop_token( + raw_tids, + finish_reason=choice.finish_reason, + include_in_output=include_stop, + stop_token_ids=stop_token_ids, + eos_token_id=eos_token_id, + generation_eos=generation_eos, + ) new_text, updated_state = await self._detokenize_delta_async( - tokenizer, delta_tids, updated_state, skip_special_tokens=skip_special + tokenizer, + decode_tids, + updated_state, + skip_special_tokens=skip_special, ) completion_logprobs = None @@ -759,7 +977,7 @@ async def derender_completion_stream( updated_state = updated_state.model_copy( update={ "logprob_context_token_ids": _logprob_context_tail( - state.logprob_context_token_ids, delta_tids + state.logprob_context_token_ids, raw_tids ), "logprob_text_offset": state.logprob_text_offset + len(new_text), } @@ -771,6 +989,7 @@ async def derender_completion_stream( text=new_text, logprobs=completion_logprobs, finish_reason=choice.finish_reason, + stop_reason=stop_reason, ) )