From 4d1ab630ed4fe51f86e6b114b1e2e89ff9e9a575 Mon Sep 17 00:00:00 2001 From: Simon Veitner Date: Thu, 1 Oct 2026 11:23:14 -0400 Subject: [PATCH 1/2] [Bugfix][Structured Output] Feed the token that implicitly ends reasoning to the grammar Some reasoning parsers end reasoning on the tool-call start token when no is emitted (e.g. GLM's ). The structured-output manager started the grammar one token after that token, so the grammar never saw it and forced a second ; required/named tool calls were lost. ParserEngine.find_reasoning_end now also says whether the end token is content (an implicit end), and StructuredOutputManager._get_constraint_bounds returns both the grammar start and the constraint start. The grammar consumes an implicit end token that it accepts, while masking still starts after it. The fix is limited to engine-based parsers; ReasoningParser is unchanged. Fixes #59608 Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Simon Veitner --- tests/parser/engine/test_deepseek_v4.py | 4 +- tests/parser/engine/test_parser_engine.py | 45 ++++++++-- .../spec_decode/test_mtp_structured_output.py | 7 +- .../test_structured_output_manager.py | 87 +++++++++++++++++++ vllm/parser/engine/adapters.py | 6 +- vllm/parser/engine/parser_engine.py | 16 +++- vllm/v1/structured_output/__init__.py | 78 +++++++++++------ 7 files changed, 194 insertions(+), 49 deletions(-) diff --git a/tests/parser/engine/test_deepseek_v4.py b/tests/parser/engine/test_deepseek_v4.py index 2cf3a8bb85f3..f5b66d9dc1cb 100644 --- a/tests/parser/engine/test_deepseek_v4.py +++ b/tests/parser/engine/test_deepseek_v4.py @@ -643,8 +643,8 @@ def test_reasoning_end_token_ids_drop_multi_token_tool_opener( # token-ID fast path; the implicit tool-call exit still goes through # the text lexer. assert thinking_parser.reasoning_end_token_ids == {_THINK_END_ID} - assert thinking_parser.find_reasoning_end_offset([7, _THINK_END_ID, 8]) == 1 - assert thinking_parser.find_reasoning_end_offset([7, 8]) == 2 + assert thinking_parser.find_reasoning_end([7, _THINK_END_ID, 8]) == (1, False) + assert thinking_parser.find_reasoning_end([7, 8]) == (2, False) def test_streaming_reasoning_implicit_end(self, thinking_parser): chunks = [ diff --git a/tests/parser/engine/test_parser_engine.py b/tests/parser/engine/test_parser_engine.py index 8fd7c4d2623f..dc91cef74028 100644 --- a/tests/parser/engine/test_parser_engine.py +++ b/tests/parser/engine/test_parser_engine.py @@ -28,12 +28,13 @@ from vllm.parser.abstract_parser import DelegatingParser from vllm.parser.engine.adapters import make_adapters from vllm.parser.engine.events import EventType, SemanticEvent -from vllm.parser.engine.parser_engine import ParserEngine +from vllm.parser.engine.parser_engine import ParserEngine, ReasoningEnd from vllm.parser.engine.parser_engine_config import ( ParserEngineConfig, ParserState, Transition, ) +from vllm.parser.glm47_moe import glm47_moe_config from vllm.parser.parser_manager import ParserManager # ── Shared test configs ────────────────────────────────────────────── @@ -158,7 +159,10 @@ def test_every_reasoning_end_exit_contributes(self): (EventType.REASONING_END, EventType.TOOL_CALL_START), ), ) - assert _make_engine(cfg).reasoning_end_token_ids == {201, 202} + engine = _make_engine(cfg) + assert engine.reasoning_end_token_ids == {201, 202} + assert engine.find_reasoning_end([5, 201]) == ReasoningEnd(1, False) + assert engine.find_reasoning_end([5, 202]) == ReasoningEnd(1, True) def test_transition_staying_in_reasoning_is_ignored(self): cfg = _with_reasoning_exits(("THINK_START", ParserState.REASONING, ())) @@ -182,17 +186,40 @@ def test_unresolved_think_end_fails_closed(self): def test_config_without_reasoning_has_empty_set(self): assert _make_engine(_hermes_config()).reasoning_end_token_ids == frozenset() - def test_find_reasoning_end_offset_returns_first_match(self): + def test_find_reasoning_end_returns_first_match(self): engine = _make_engine() - assert engine.find_reasoning_end_offset([5, 201, 6, 201]) == 1 - assert engine.find_reasoning_end_offset([5, 6]) == 2 - assert engine.find_reasoning_end_offset([]) == 0 + assert engine.find_reasoning_end([5, 201, 6, 201]) == ReasoningEnd(1, False) + assert engine.find_reasoning_end([5, 6]) == ReasoningEnd(2, False) + assert engine.find_reasoning_end([]) == ReasoningEnd(0, False) # Rejected-draft placeholders never match. - assert engine.find_reasoning_end_offset([-1, 201]) == 1 + assert engine.find_reasoning_end([-1, 201]) == ReasoningEnd(1, False) - def test_find_reasoning_end_offset_with_empty_set_returns_none(self): + def test_find_reasoning_end_with_empty_set_returns_none(self): engine = _make_engine(_hermes_config()) - assert engine.find_reasoning_end_offset([201]) is None + assert engine.find_reasoning_end([201]) is None + + +@pytest.mark.parametrize( + ("config", "implicit_token"), + [ + pytest.param(glm47_moe_config(), "", id="glm47_moe"), + ], +) +def test_implicit_reasoning_end_of_parsers( + config: ParserEngineConfig, implicit_token: str +): + literals = sorted(set(config.token_id_terminals.values())) + vocab = {literal: 1000 + i for i, literal in enumerate(literals)} + engine = _make_engine(config, vocab=vocab) + implicit_ids = set() + for token_id in engine.reasoning_end_token_ids: + end = engine.find_reasoning_end([5, token_id, 6]) + assert end is not None and end.offset == 1 + kept = engine.extract_content_ids([5, token_id, 6]) == [token_id, 6] + assert end.implicit == kept + if end.implicit: + implicit_ids.add(token_id) + assert implicit_ids == {vocab[implicit_token]} # ── TestEventsToDelta ──────────────────────────────────────────────── diff --git a/tests/v1/spec_decode/test_mtp_structured_output.py b/tests/v1/spec_decode/test_mtp_structured_output.py index 5e4312227d7e..8a72765db150 100644 --- a/tests/v1/spec_decode/test_mtp_structured_output.py +++ b/tests/v1/spec_decode/test_mtp_structured_output.py @@ -11,6 +11,7 @@ from vllm.config.model import ModelConfig from vllm.config.speculative import SpeculativeConfig from vllm.parser.engine.adapters import ParserEngineReasoningAdapter +from vllm.parser.engine.parser_engine import ReasoningEnd from vllm.sampling_params import SamplingParams, StructuredOutputsParams from vllm.v1.request import Request from vllm.v1.structured_output import StructuredOutputManager @@ -113,12 +114,12 @@ def __init__(self, end_token_ids): def reasoning_end_token_ids(self): return self._end_token_ids - def find_reasoning_end_offset(self, token_ids): + def find_reasoning_end(self, token_ids): self.windows.append(list(token_ids)) for offset, token in enumerate(token_ids): if token in self._end_token_ids: - return offset - return len(token_ids) + return ReasoningEnd(offset, False) + return ReasoningEnd(len(token_ids), False) def is_reasoning_end(self, input_ids): return any(token in self._end_token_ids for token in input_ids) diff --git a/tests/v1/structured_output/test_structured_output_manager.py b/tests/v1/structured_output/test_structured_output_manager.py index 6082bcd6a7b1..3497fadc97d1 100644 --- a/tests/v1/structured_output/test_structured_output_manager.py +++ b/tests/v1/structured_output/test_structured_output_manager.py @@ -11,6 +11,8 @@ from vllm.config.model import ModelConfig from vllm.config.speculative import SpeculativeConfig from vllm.config.structured_outputs import StructuredOutputsBackend +from vllm.parser.engine.adapters import ParserEngineReasoningAdapter +from vllm.parser.engine.parser_engine import ReasoningEnd from vllm.sampling_params import SamplingParams, StructuredOutputsParams from vllm.v1.request import Request from vllm.v1.structured_output import StructuredOutputManager @@ -514,3 +516,88 @@ def test_outlines_termination(tokenizer): assert grammar.validate_tokens([eos]) == [] assert grammar.accept_tokens(request.request_id, [one, eos, one]) assert grammar.is_terminated() + + +class MockEngineReasoner(ParserEngineReasoningAdapter): + """Ends reasoning implicitly on `marker`, which is also content.""" + + def __init__(self, tokenizer, marker: int): + self.marker = marker + + @property + def reasoning_end_token_ids(self): + return frozenset({self.marker}) + + def find_reasoning_end(self, token_ids): + ids = list(token_ids) + if self.marker in ids: + return ReasoningEnd(ids.index(self.marker), True) + return ReasoningEnd(len(ids), False) + + +@pytest.mark.parametrize("backend", BACKENDS) +@pytest.mark.parametrize( + ("marker", "drafts", "row_pattern"), + [ + pytest.param("{", (" ", "{", "}", EOS), "UUCCU", id="inclusive"), + pytest.param("{", ("{", "}"), "UCC", id="inclusive_first"), + pytest.param("{", (" ", " ", "{"), "UUUC", id="inclusive_last"), + pytest.param("z", (" ", "z", "{", "}"), "UUCCC", id="not_accepted"), + ], +) +def test_inclusive_reasoning_end_token( + tokenizer, + backend: StructuredOutputsBackend, + marker: str, + drafts: tuple[str, ...], + row_pattern: str, +): + """An implicit end token ("{", like GLM's ``) is sampled + unconstrained but fed to the grammar. One it rejects ("z") is not fed to it.""" + manager, request = _build_harness( + tokenizer, + backend, + reasoning_ended=False, + reasoning_parser_kwargs={"marker": _single_token(tokenizer, marker)}, + ) + manager.reasoner_cls = MockEngineReasoner + token_ids = _to_token_ids(tokenizer, drafts) + _run_real_flow( + manager, + request, + raw_drafts=token_ids, + expected_validated=token_ids, + expected_row_pattern=row_pattern, + expected_reasoning=True, + expect_terminated=EOS in drafts, + ) + grammar = request.structured_output_request.grammar # type: ignore[union-attr] + assert grammar is not None and not isinstance(grammar, Exception) + assert grammar.validate_tokens([_single_token(tokenizer, "{")]) == [] + + +@pytest.mark.parametrize("backend", BACKENDS) +@pytest.mark.parametrize( + ("reasoner_cls", "consumed"), + [ + pytest.param(MockEngineReasoner, True, id="no_drafts"), + pytest.param(MockReasoner, False, id="non_engine"), + ], +) +def test_end_token_committed_alone( + tokenizer, backend: StructuredOutputsBackend, reasoner_cls, consumed: bool +): + """Only an engine-based reasoner feeds the end token "{" to the grammar.""" + brace = _single_token(tokenizer, "{") + manager, request = _build_harness( + tokenizer, + backend, + reasoning_ended=False, + reasoning_parser_kwargs={"marker": brace}, + ) + manager.reasoner_cls = reasoner_cls + request.append_output_token_ids([brace]) + assert manager.accept_tokens(request, [brace]) + grammar = request.structured_output_request.grammar # type: ignore[union-attr] + assert grammar is not None and not isinstance(grammar, Exception) + assert (grammar.validate_tokens([brace]) == []) is consumed diff --git a/vllm/parser/engine/adapters.py b/vllm/parser/engine/adapters.py index 4f4aa9f5a1e0..cd7aaa729785 100644 --- a/vllm/parser/engine/adapters.py +++ b/vllm/parser/engine/adapters.py @@ -27,7 +27,7 @@ ChatCompletionRequest, ) from vllm.entrypoints.openai.responses.protocol import ResponsesRequest - from vllm.parser.engine.parser_engine import ParserEngine + from vllm.parser.engine.parser_engine import ParserEngine, ReasoningEnd from vllm.tokenizers import TokenizerLike from vllm.tool_parsers.utils import Tool @@ -65,8 +65,8 @@ def _skip_tool_parsing(self) -> Iterator[None]: def is_reasoning_end(self, input_ids: Sequence[int]) -> bool: return self._parser_engine.is_reasoning_end(list(input_ids)) - def find_reasoning_end_offset(self, token_ids: Sequence[int]) -> int | None: - return self._parser_engine.find_reasoning_end_offset(token_ids) + def find_reasoning_end(self, token_ids: Sequence[int]) -> ReasoningEnd | None: + return self._parser_engine.find_reasoning_end(token_ids) def adjust_initial_state_from_prompt(self, prompt_token_ids: Sequence[int]) -> None: self._parser_engine.adjust_initial_state_from_prompt(prompt_token_ids) diff --git a/vllm/parser/engine/parser_engine.py b/vllm/parser/engine/parser_engine.py index 527c310c1485..b04fdaa7944d 100644 --- a/vllm/parser/engine/parser_engine.py +++ b/vllm/parser/engine/parser_engine.py @@ -9,7 +9,7 @@ import json from collections.abc import Sequence from functools import cached_property -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, NamedTuple import regex as re @@ -45,6 +45,13 @@ logger = init_logger(__name__) +class ReasoningEnd(NamedTuple): + """Where reasoning ends in a token sequence.""" + + offset: int # index of the first end token, or the sequence length + implicit: bool # the end token is also content, e.g. a tool-call start + + class ToolCallSlot: __slots__ = ( "id", @@ -638,14 +645,15 @@ def _derive_reasoning_end_token_ids( def reasoning_end_token_ids(self) -> frozenset[int]: return self._reasoning_end_token_ids - def find_reasoning_end_offset(self, token_ids: Sequence[int]) -> int | None: + def find_reasoning_end(self, token_ids: Sequence[int]) -> ReasoningEnd | None: + """First reasoning end in `token_ids`, and if its token is content.""" end_ids = self._reasoning_end_token_ids if not end_ids: return None for offset, token_id in enumerate(token_ids): if token_id in end_ids: - return offset - return len(token_ids) + return ReasoningEnd(offset, token_id != self._reasoning_end_token_id) + return ReasoningEnd(len(token_ids), False) def is_reasoning_end(self, input_ids: list[int]) -> bool: config = self.parser_engine_config diff --git a/vllm/v1/structured_output/__init__.py b/vllm/v1/structured_output/__init__.py index 3b35d39d1447..7a9a374b0d53 100644 --- a/vllm/v1/structured_output/__init__.py +++ b/vllm/v1/structured_output/__init__.py @@ -3,7 +3,7 @@ import multiprocessing from collections.abc import Iterable, Sequence from concurrent.futures import Future, ThreadPoolExecutor -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, NamedTuple from vllm.config import VllmConfig from vllm.logger import init_logger @@ -33,6 +33,17 @@ logger = init_logger(__name__) +class ConstraintBounds(NamedTuple): + """Where tokens start being fed to the grammar and being masked.""" + + grammar_start: int + constraint_start: int + + +_CONSTRAINED_FROM_START = ConstraintBounds(0, 0) +_NO_SPEC_TOKENS = ConstraintBounds(1, 1) + + class StructuredOutputManager: """Engine-level manager for structured output requests.""" @@ -217,32 +228,34 @@ def _async_submit_fill_bitmask( ) -> Future: return self.executor_for_fillmask.submit(self._fill_bitmasks, batch) - def _get_constraint_start( + def _get_constraint_bounds( self, request: "Request", spec_tokens: Sequence[int], spec_tokens_committed: bool = False, - ) -> int: + ) -> ConstraintBounds: """Return the index into `spec_tokens` where tokens should start being constrained by the grammar. Assumes `spec_tokens` are stripped of -1 padding tokens. Returns `len(spec_tokens) + 1` if no token should be constrained after accepting all `spec_tokens`. + This is `constraint_start`; `grammar_start` is one less if reasoning ends + implicitly on a content token (e.g. a tool-call start) the grammar accepts. `spec_tokens_committed` is True if spec_tokens is already in `request.all_token_ids`. """ if self.enable_in_reasoning: - return 0 + return _CONSTRAINED_FROM_START structured_req = request.structured_output_request assert structured_req is not None if structured_req.reasoning_ended: - return 0 + return _CONSTRAINED_FROM_START reasoner = self._get_reasoner(request) if reasoner is None: - return 0 + return _CONSTRAINED_FROM_START if structured_req.reasoning_ended is None: # This should be removed here, but since `openai_gptoss` @@ -250,30 +263,37 @@ def _get_constraint_start( # After unifying the `openai_gptoss` and non-`openai_gptoss` styles, # it can be removed. if reasoner.is_reasoning_end(request.prompt_token_ids or []): - return 0 + return _CONSTRAINED_FROM_START structured_req.reasoning_ended = False num_spec_tokens = len(spec_tokens) if num_spec_tokens <= 0: - return num_spec_tokens + 1 + return _NO_SPEC_TOKENS - # Use `find_reasoning_end_offset` to find constraint start if supported + # Engine-based parsers also report whether the end token is content. if ( isinstance(reasoner, ParserEngineReasoningAdapter) and reasoner.reasoning_end_token_ids ): - offset = reasoner.find_reasoning_end_offset(spec_tokens) - if offset is not None: - return offset + 1 - - # Fallback to `find_reasoning_end_offset` + end = reasoner.find_reasoning_end(spec_tokens) + if end is not None: + constraint_start = end.offset + 1 + if end.implicit: + grammar = structured_req.grammar + if TYPE_CHECKING: + assert isinstance(grammar, StructuredOutputGrammar) + if grammar.validate_tokens([spec_tokens[end.offset]]): + return ConstraintBounds(end.offset, constraint_start) + return ConstraintBounds(constraint_start, constraint_start) + + # Fallback: the grammar starts after the end token. # TODO: Build a read-only Sequence view over all_token_ids # instead of copying the entire all_token_ids into input_ids. if spec_tokens_committed: if not spec_tokens or not reasoner.is_reasoning_end_streaming( request.all_token_ids, spec_tokens ): - return num_spec_tokens + 1 + return ConstraintBounds(num_spec_tokens + 1, num_spec_tokens + 1) input_ids = request.all_token_ids.copy() delta_ids = list(spec_tokens) @@ -281,15 +301,15 @@ def _get_constraint_start( input_ids = request.all_token_ids.copy() input_ids.extend(spec_tokens) if not reasoner.is_reasoning_end_streaming(input_ids, spec_tokens): - return num_spec_tokens + 1 + return ConstraintBounds(num_spec_tokens + 1, num_spec_tokens + 1) delta_ids = list(spec_tokens) for i in range(num_spec_tokens - 1, 0, -1): input_ids.pop() delta_ids.pop() if not reasoner.is_reasoning_end_streaming(input_ids, delta_ids): - return i + 1 - return 1 + return ConstraintBounds(i + 1, i + 1) + return ConstraintBounds(1, 1) def validate_tokens(self, request: "Request", spec_tokens: list[int]) -> list[int]: """Return the longest unconstrained or grammar-valid prefix of `spec_tokens`.""" @@ -297,7 +317,9 @@ def validate_tokens(self, request: "Request", spec_tokens: list[int]) -> list[in return spec_tokens spec_tokens = strip_speculative_padding(spec_tokens) - constraint_start = self._get_constraint_start(request, spec_tokens) + grammar_start, constraint_start = self._get_constraint_bounds( + request, spec_tokens + ) if constraint_start >= len(spec_tokens): return spec_tokens @@ -307,8 +329,8 @@ def validate_tokens(self, request: "Request", spec_tokens: list[int]) -> list[in grammar = structured_req.grammar if TYPE_CHECKING: assert isinstance(grammar, StructuredOutputGrammar) - prefix = spec_tokens[:constraint_start] - validated = grammar.validate_tokens(spec_tokens[constraint_start:]) + prefix = spec_tokens[:grammar_start] + validated = grammar.validate_tokens(spec_tokens[grammar_start:]) return prefix + validated def grammar_bitmask( @@ -358,7 +380,9 @@ def grammar_bitmask( if TYPE_CHECKING: assert isinstance(grammar, StructuredOutputGrammar) - apply_bitmask = self._get_constraint_start(request, ()) == 0 + apply_bitmask = ( + self._get_constraint_bounds(request, ()).constraint_start == 0 + ) batch.append((grammar, cumulative_index, apply_bitmask)) if len(batch) == self.fill_bitmask_parallel_batch_size: promises.append(self._async_submit_fill_bitmask(batch)) @@ -384,7 +408,7 @@ def grammar_bitmask( assert isinstance(grammar, StructuredOutputGrammar) req_tokens = scheduled_spec_decode_tokens.get(req_id, list()) - constraint_start = self._get_constraint_start( + grammar_start, constraint_start = self._get_constraint_bounds( request, strip_speculative_padding(req_tokens) ) state_advancements = 0 @@ -402,7 +426,7 @@ def grammar_bitmask( self._fill_bitmasks(((grammar, cumulative_index, apply_bitmask),)) if token == -1: seen_padding = True - elif apply_bitmask: + elif not seen_padding and i >= grammar_start: if not grammar.is_terminated() and grammar.accept_tokens( req_id, [token] ): @@ -453,7 +477,7 @@ def accept_tokens(self, request: "Request", new_token_ids: list[int]) -> bool: if TYPE_CHECKING: assert isinstance(grammar, StructuredOutputGrammar) - constraint_start = self._get_constraint_start( + grammar_start, constraint_start = self._get_constraint_bounds( request, new_token_ids, spec_tokens_committed=True ) # Early return only when the constraint hasn't started. @@ -461,9 +485,7 @@ def accept_tokens(self, request: "Request", new_token_ids: list[int]) -> bool: if constraint_start > len(new_token_ids): return True structured_req.reasoning_ended = True - return grammar.accept_tokens( - request.request_id, new_token_ids[constraint_start:] - ) + return grammar.accept_tokens(request.request_id, new_token_ids[grammar_start:]) def clear_backend(self) -> None: if self.backend is not None: From f4ca6088fd37a9422ba3cddd9d3a1abb864da487 Mon Sep 17 00:00:00 2001 From: Simon Veitner Date: Thu, 8 Oct 2026 17:40:51 +0000 Subject: [PATCH 2/2] Apply review suggestion from @yzong-rh Let constraint_start only control where grammar masking begins in grammar_bitmask. validate_tokens and accept_tokens now work from grammar_start, so _get_constraint_bounds always sets grammar_start to an implicit reasoning-end token and no longer probes the grammar with validate_tokens. The grammar rejects invalid implicit markers through validate_tokens, and accepted markers are fed to it normally. Fold the implicit-end coverage into FlowCase (implicit_end_accepted, implicit_end_rejected) and drop the separate inclusive-end tests. Suggested in https://github.com/vllm-project/vllm/pull/59627#issuecomment-6064189804 Co-authored-by: Yifan Zong <263738792+yzong-rh@users.noreply.github.com> Signed-off-by: Simon Veitner --- .../test_structured_output_manager.py | 131 ++++++------------ vllm/v1/structured_output/__init__.py | 20 +-- 2 files changed, 52 insertions(+), 99 deletions(-) diff --git a/tests/v1/structured_output/test_structured_output_manager.py b/tests/v1/structured_output/test_structured_output_manager.py index 3497fadc97d1..fbceb3bd07e4 100644 --- a/tests/v1/structured_output/test_structured_output_manager.py +++ b/tests/v1/structured_output/test_structured_output_manager.py @@ -20,6 +20,7 @@ TOKENIZER = "gpt2" THINK_END = "\n" # reasoning-end marker (single GPT-2 token) +IMPLICIT_THINK_END = "z" # accepted in strings, rejected at the object start EOS = "<|eos|>" # resolved to tokenizer.eos_token_id JSON_SCHEMA = '{"type": "object"}' BACKENDS = ("xgrammar", "guidance") @@ -45,6 +46,7 @@ class FlowCase: prefix: str = "" expected_validated: tuple[str, ...] | None = None reasoning_ended: bool | None = False + engine_reasoner: bool = False xfail_guidance: str | None = None @@ -61,6 +63,23 @@ def is_reasoning_end_streaming(self, input_ids, delta_ids): return self.is_reasoning_end(delta_ids) +class MockEngineReasoner(ParserEngineReasoningAdapter): + """Ends reasoning implicitly on ``marker``, which is also content.""" + + def __init__(self, tokenizer, marker: int): + self.marker = marker + + @property + def reasoning_end_token_ids(self): + return frozenset({self.marker}) + + def find_reasoning_end(self, token_ids): + ids = list(token_ids) + if self.marker in ids: + return ReasoningEnd(ids.index(self.marker), True) + return ReasoningEnd(len(ids), False) + + def _single_token(tokenizer, text: str) -> int: token_ids = tokenizer.encode(text) assert len(token_ids) == 1, (text, token_ids) @@ -333,6 +352,28 @@ def _run_real_flow( ), id="becomes_active_terminates", ), + pytest.param( + FlowCase( + prefix='{"a": "', + raw_drafts=(IMPLICIT_THINK_END,), + expected_row_pattern="UC", + expected_reasoning=True, + expect_terminated=False, + engine_reasoner=True, + ), + id="implicit_end_accepted", + ), + pytest.param( + FlowCase( + raw_drafts=(" ", IMPLICIT_THINK_END, "{", "}"), + expected_validated=(" ",), + expected_row_pattern="UUUUU", + expected_reasoning=False, + expect_terminated=False, + engine_reasoner=True, + ), + id="implicit_end_rejected", + ), ] @@ -346,7 +387,8 @@ def test_real_flow( if backend == "guidance" and case.xfail_guidance: pytest.xfail(case.xfail_guidance) - reasoner_kwargs = {"marker": _single_token(tokenizer, THINK_END)} + marker = IMPLICIT_THINK_END if case.engine_reasoner else THINK_END + reasoner_kwargs = {"marker": _single_token(tokenizer, marker)} manager, request = _build_harness( tokenizer, backend, @@ -354,6 +396,8 @@ def test_real_flow( reasoning_ended=case.reasoning_ended, reasoning_parser_kwargs=reasoner_kwargs, ) + if case.engine_reasoner: + manager.reasoner_cls = MockEngineReasoner raw_drafts = _to_token_ids(tokenizer, case.raw_drafts) expected_texts = ( @@ -516,88 +560,3 @@ def test_outlines_termination(tokenizer): assert grammar.validate_tokens([eos]) == [] assert grammar.accept_tokens(request.request_id, [one, eos, one]) assert grammar.is_terminated() - - -class MockEngineReasoner(ParserEngineReasoningAdapter): - """Ends reasoning implicitly on `marker`, which is also content.""" - - def __init__(self, tokenizer, marker: int): - self.marker = marker - - @property - def reasoning_end_token_ids(self): - return frozenset({self.marker}) - - def find_reasoning_end(self, token_ids): - ids = list(token_ids) - if self.marker in ids: - return ReasoningEnd(ids.index(self.marker), True) - return ReasoningEnd(len(ids), False) - - -@pytest.mark.parametrize("backend", BACKENDS) -@pytest.mark.parametrize( - ("marker", "drafts", "row_pattern"), - [ - pytest.param("{", (" ", "{", "}", EOS), "UUCCU", id="inclusive"), - pytest.param("{", ("{", "}"), "UCC", id="inclusive_first"), - pytest.param("{", (" ", " ", "{"), "UUUC", id="inclusive_last"), - pytest.param("z", (" ", "z", "{", "}"), "UUCCC", id="not_accepted"), - ], -) -def test_inclusive_reasoning_end_token( - tokenizer, - backend: StructuredOutputsBackend, - marker: str, - drafts: tuple[str, ...], - row_pattern: str, -): - """An implicit end token ("{", like GLM's ``) is sampled - unconstrained but fed to the grammar. One it rejects ("z") is not fed to it.""" - manager, request = _build_harness( - tokenizer, - backend, - reasoning_ended=False, - reasoning_parser_kwargs={"marker": _single_token(tokenizer, marker)}, - ) - manager.reasoner_cls = MockEngineReasoner - token_ids = _to_token_ids(tokenizer, drafts) - _run_real_flow( - manager, - request, - raw_drafts=token_ids, - expected_validated=token_ids, - expected_row_pattern=row_pattern, - expected_reasoning=True, - expect_terminated=EOS in drafts, - ) - grammar = request.structured_output_request.grammar # type: ignore[union-attr] - assert grammar is not None and not isinstance(grammar, Exception) - assert grammar.validate_tokens([_single_token(tokenizer, "{")]) == [] - - -@pytest.mark.parametrize("backend", BACKENDS) -@pytest.mark.parametrize( - ("reasoner_cls", "consumed"), - [ - pytest.param(MockEngineReasoner, True, id="no_drafts"), - pytest.param(MockReasoner, False, id="non_engine"), - ], -) -def test_end_token_committed_alone( - tokenizer, backend: StructuredOutputsBackend, reasoner_cls, consumed: bool -): - """Only an engine-based reasoner feeds the end token "{" to the grammar.""" - brace = _single_token(tokenizer, "{") - manager, request = _build_harness( - tokenizer, - backend, - reasoning_ended=False, - reasoning_parser_kwargs={"marker": brace}, - ) - manager.reasoner_cls = reasoner_cls - request.append_output_token_ids([brace]) - assert manager.accept_tokens(request, [brace]) - grammar = request.structured_output_request.grammar # type: ignore[union-attr] - assert grammar is not None and not isinstance(grammar, Exception) - assert (grammar.validate_tokens([brace]) == []) is consumed diff --git a/vllm/v1/structured_output/__init__.py b/vllm/v1/structured_output/__init__.py index 7a9a374b0d53..2163a800a563 100644 --- a/vllm/v1/structured_output/__init__.py +++ b/vllm/v1/structured_output/__init__.py @@ -240,7 +240,7 @@ def _get_constraint_bounds( Returns `len(spec_tokens) + 1` if no token should be constrained after accepting all `spec_tokens`. This is `constraint_start`; `grammar_start` is one less if reasoning ends - implicitly on a content token (e.g. a tool-call start) the grammar accepts. + implicitly on a content token (e.g. a tool-call start). `spec_tokens_committed` is True if spec_tokens is already in `request.all_token_ids`. @@ -279,11 +279,7 @@ def _get_constraint_bounds( if end is not None: constraint_start = end.offset + 1 if end.implicit: - grammar = structured_req.grammar - if TYPE_CHECKING: - assert isinstance(grammar, StructuredOutputGrammar) - if grammar.validate_tokens([spec_tokens[end.offset]]): - return ConstraintBounds(end.offset, constraint_start) + return ConstraintBounds(end.offset, constraint_start) return ConstraintBounds(constraint_start, constraint_start) # Fallback: the grammar starts after the end token. @@ -317,10 +313,8 @@ def validate_tokens(self, request: "Request", spec_tokens: list[int]) -> list[in return spec_tokens spec_tokens = strip_speculative_padding(spec_tokens) - grammar_start, constraint_start = self._get_constraint_bounds( - request, spec_tokens - ) - if constraint_start >= len(spec_tokens): + grammar_start, _ = self._get_constraint_bounds(request, spec_tokens) + if grammar_start >= len(spec_tokens): return spec_tokens structured_req = request.structured_output_request @@ -477,12 +471,12 @@ def accept_tokens(self, request: "Request", new_token_ids: list[int]) -> bool: if TYPE_CHECKING: assert isinstance(grammar, StructuredOutputGrammar) - grammar_start, constraint_start = self._get_constraint_bounds( + grammar_start, _ = self._get_constraint_bounds( request, new_token_ids, spec_tokens_committed=True ) - # Early return only when the constraint hasn't started. + # Early return only when no reasoning-end token is present. # Otherwise, latch structured_req.reasoning_ended. - if constraint_start > len(new_token_ids): + if grammar_start > len(new_token_ids): return True structured_req.reasoning_ended = True return grammar.accept_tokens(request.request_id, new_token_ids[grammar_start:])