From e99e9276b5d4985e3288835a542ba064b8b175ae Mon Sep 17 00:00:00 2001 From: logprobz <321553542+logprobz@users.noreply.github.com> Date: Mon, 31 Aug 2026 01:17:39 +0000 Subject: [PATCH 01/12] fix(glm47): preserve tool delimiters in argument values Keep tool-call delimiters as argument data while the declarative parser is inside an argument value. Retain progressive argument deltas and add non-streaming and streaming regressions. Co-authored-by: OpenAI Codex Signed-off-by: logprobz <321553542+logprobz@users.noreply.github.com> --- .../test_glm47_moe_tool_parser.py | 67 +++++++++++++++++++ vllm/parser/glm47_moe.py | 39 +++++++---- 2 files changed, 93 insertions(+), 13 deletions(-) diff --git a/tests/tool_parsers/test_glm47_moe_tool_parser.py b/tests/tool_parsers/test_glm47_moe_tool_parser.py index 224fea08d741..c6540812b32d 100644 --- a/tests/tool_parsers/test_glm47_moe_tool_parser.py +++ b/tests/tool_parsers/test_glm47_moe_tool_parser.py @@ -187,6 +187,24 @@ def test_whitespace_content_none(self, glm47_tool_parser, mock_request): r = glm47_tool_parser.extract_tool_calls(out, request=mock_request) assert r.content is None + def test_tool_delimiters_in_arg_value(self, glm47_tool_parser, mock_request): + value = "close then open " + out = ( + "get_weather" + "city" + f"{value}" + "" + ) + + result = glm47_tool_parser.extract_tool_calls(out, request=mock_request) + + assert result.tools_called + assert len(result.tool_calls) == 1 + assert result.content is None + function = result.tool_calls[0].function + assert function.name == "get_weather" + assert json.loads(function.arguments) == {"city": value} + def _reset(parser): parser.current_tool_name_sent = False @@ -266,3 +284,52 @@ def test_with_args(self, glm47_tool_parser, mock_request): ] args = json.loads("".join(arguments)) assert args["city"] == "Beijing" + + def test_tool_delimiters_in_arg_value(self, glm47_tool_parser, mock_request): + _reset(glm47_tool_parser) + value = "close then open " + chunks = [ + "", + "get_weather", + "city", + "close ", + "", + " then open ", + "", + "", + "", + ] + current_text = "" + deltas = [] + for chunk in chunks: + current_text += chunk + delta = glm47_tool_parser.extract_tool_calls_streaming( + previous_text="", + current_text=current_text, + delta_text=chunk, + previous_token_ids=[], + current_token_ids=[], + delta_token_ids=[], + request=mock_request, + ) + if delta: + deltas.append(delta) + finish = glm47_tool_parser.finish_streaming() + if finish: + deltas.append(finish) + + calls = [call for delta in deltas for call in (delta.tool_calls or [])] + names = [ + call.function.name for call in calls if call.function and call.function.name + ] + arguments = [ + call.function.arguments + for call in calls + if call.function and call.function.arguments + ] + content = "".join(delta.content or "" for delta in deltas) + + assert {call.index for call in calls} == {0} + assert names == ["get_weather"] + assert content == "" + assert json.loads("".join(arguments)) == {"city": value} diff --git a/vllm/parser/glm47_moe.py b/vllm/parser/glm47_moe.py index 8aa4feef2599..789d459f2ff6 100644 --- a/vllm/parser/glm47_moe.py +++ b/vllm/parser/glm47_moe.py @@ -73,16 +73,26 @@ def _glm47_arg_converter(raw_args: str, partial: bool) -> str: @functools.cache def glm47_moe_config(thinking: bool = True) -> ParserEngineConfig: arg_tag_transitions = { - (ParserState.TOOL_ARGS, terminal): Transition( + (ParserState.TOOL_NAME, "ARG_KEY_START"): Transition( + ParserState.TOOL_BETWEEN, + (EventType.ARG_VALUE_CHUNK,), + ), + (ParserState.TOOL_BETWEEN, "ARG_KEY_START"): Transition( + ParserState.TOOL_BETWEEN, + (EventType.ARG_VALUE_CHUNK,), + ), + (ParserState.TOOL_BETWEEN, "ARG_KEY_END"): Transition( + ParserState.TOOL_BETWEEN, + (EventType.ARG_VALUE_CHUNK,), + ), + (ParserState.TOOL_BETWEEN, "ARG_VALUE_START"): Transition( ParserState.TOOL_ARGS, (EventType.ARG_VALUE_CHUNK,), - ) - for terminal in ( - "ARG_KEY_START", - "ARG_KEY_END", - "ARG_VALUE_START", - "ARG_VALUE_END", - ) + ), + (ParserState.TOOL_ARGS, "ARG_VALUE_END"): Transition( + ParserState.TOOL_BETWEEN, + (EventType.ARG_VALUE_CHUNK,), + ), } reasoning_terminals = ( @@ -151,20 +161,23 @@ def glm47_moe_config(thinking: bool = True) -> ParserEngineConfig: ParserState.TOOL_NAME, (EventType.TOOL_CALL_START,), ), - (ParserState.TOOL_NAME, "ARG_KEY_START"): Transition( - ParserState.TOOL_ARGS, - (EventType.ARG_VALUE_CHUNK,), - ), (ParserState.TOOL_NAME, "TOOL_END"): Transition( ParserState.CONTENT, (EventType.TOOL_CALL_END,), ), - (ParserState.TOOL_ARGS, "TOOL_END"): Transition( + (ParserState.TOOL_BETWEEN, "TOOL_END"): Transition( ParserState.CONTENT, (EventType.TOOL_CALL_END,), ), **arg_tag_transitions, }, + content_events={ + ParserState.CONTENT: EventType.TEXT_CHUNK, + ParserState.REASONING: EventType.REASONING_CHUNK, + ParserState.TOOL_NAME: EventType.TOOL_NAME, + ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK, + ParserState.TOOL_BETWEEN: EventType.ARG_VALUE_CHUNK, + }, arg_converter=_glm47_arg_converter, stream_arg_deltas=True, tool_args_json=False, From 868351da799fe34b5d7309b417bea86dc92f4618 Mon Sep 17 00:00:00 2001 From: logprobz <321553542+logprobz@users.noreply.github.com> Date: Mon, 31 Aug 2026 05:36:14 +0000 Subject: [PATCH 02/12] fix(parser): preserve anchors before stripped trailing drops Defer a trailing DROP terminal when serving retains its token ID but strips its text. Preserve exact structural anchors and cover scanner, full chat, and streaming behavior. Co-authored-by: OpenAI Codex Signed-off-by: logprobz <321553542+logprobz@users.noreply.github.com> --- .../chat_completion/test_serving_chat.py | 268 ++++++++++++++++++ tests/parser/engine/test_token_id_scanner.py | 135 +++++++++ vllm/parser/engine/token_id_scanner.py | 22 ++ 3 files changed, 425 insertions(+) diff --git a/tests/entrypoints/openai/chat_completion/test_serving_chat.py b/tests/entrypoints/openai/chat_completion/test_serving_chat.py index debf547593fa..4561c88dc05d 100644 --- a/tests/entrypoints/openai/chat_completion/test_serving_chat.py +++ b/tests/entrypoints/openai/chat_completion/test_serving_chat.py @@ -17,6 +17,7 @@ verify_chat_response, verify_harmony_messages, ) +from tests.parser.engine.replay_harness import MockTokenizer from tests.utils import RemoteOpenAIServer from vllm._aiter_ops import is_aiter_found_and_supported from vllm.config import MultiModalConfig @@ -47,6 +48,7 @@ from vllm.multimodal.inputs import PlaceholderRange from vllm.outputs import CompletionOutput, RequestOutput from vllm.parser import HarmonyParser +from vllm.parser.glm47_moe import Glm47MoeParser from vllm.renderers.hf import HfRenderer from vllm.renderers.mistral import MistralRenderer from vllm.renderers.online_renderer import OnlineRenderer @@ -849,6 +851,272 @@ class MockEngine: errored: bool = False +_GLM47_THINK_END = "" +_GLM47_TOOL_START = "" +_GLM47_TOOL_END = "" +_GLM47_ARG_KEY_START = "" +_GLM47_ARG_KEY_END = "" +_GLM47_ARG_VALUE_START = "" +_GLM47_ARG_VALUE_END = "" +_GLM47_OBSERVATION = "<|observation|>" +_GLM47_THINK_END_ID = 154842 +_GLM47_TOOL_START_ID = 154843 +_GLM47_TOOL_END_ID = 154844 +_GLM47_ARG_KEY_START_ID = 154847 +_GLM47_ARG_KEY_END_ID = 154848 +_GLM47_ARG_VALUE_START_ID = 154849 +_GLM47_ARG_VALUE_END_ID = 154850 +_GLM47_OBSERVATION_ID = 154829 +_GLM47_REASONING_ID = 301 +_GLM47_NAME_ID = 302 +_GLM47_KEY_ID = 303 +_GLM47_VALUE_PREFIX_ID = 304 +_GLM47_VALUE_SUFFIX_ID = 305 +_GLM47_REASONING = "Reasoning describes the requested value." +_GLM47_GENERATED_TOKENS = [ + (_GLM47_REASONING_ID, _GLM47_REASONING), + (_GLM47_THINK_END_ID, _GLM47_THINK_END), + (_GLM47_TOOL_START_ID, _GLM47_TOOL_START), + (_GLM47_NAME_ID, "record_value"), + (_GLM47_ARG_KEY_START_ID, _GLM47_ARG_KEY_START), + (_GLM47_KEY_ID, "value"), + (_GLM47_ARG_KEY_END_ID, _GLM47_ARG_KEY_END), + (_GLM47_ARG_VALUE_START_ID, _GLM47_ARG_VALUE_START), + (_GLM47_VALUE_PREFIX_ID, " and "), + (_GLM47_VALUE_SUFFIX_ID, ""), + (_GLM47_ARG_VALUE_END_ID, _GLM47_ARG_VALUE_END), + (_GLM47_TOOL_END_ID, _GLM47_TOOL_END), +] +_GLM47_OUTPUT_TEXT = "".join(text for _, text in _GLM47_GENERATED_TOKENS) +_GLM47_OUTPUT_IDS = [ + *(token_id for token_id, _ in _GLM47_GENERATED_TOKENS), + _GLM47_OBSERVATION_ID, +] +_GLM47_TOOLS = [ + { + "type": "function", + "function": { + "name": "record_value", + "description": "Record an exact value.", + "parameters": { + "type": "object", + "properties": {"value": {"type": "string"}}, + "required": ["value"], + "additionalProperties": False, + }, + }, + } +] + + +def _glm47_stop_tokenizer() -> MockTokenizer: + vocab = { + "": 154841, + _GLM47_THINK_END: _GLM47_THINK_END_ID, + _GLM47_TOOL_START: _GLM47_TOOL_START_ID, + _GLM47_TOOL_END: _GLM47_TOOL_END_ID, + _GLM47_ARG_KEY_START: _GLM47_ARG_KEY_START_ID, + _GLM47_ARG_KEY_END: _GLM47_ARG_KEY_END_ID, + _GLM47_ARG_VALUE_START: _GLM47_ARG_VALUE_START_ID, + _GLM47_ARG_VALUE_END: _GLM47_ARG_VALUE_END_ID, + _GLM47_OBSERVATION: _GLM47_OBSERVATION_ID, + } + tokens = [ + *_GLM47_GENERATED_TOKENS, + (154841, ""), + (_GLM47_OBSERVATION_ID, _GLM47_OBSERVATION), + ] + return MockTokenizer(vocab=vocab, tokens=tokens) + + +def _glm47_request(*, stream: bool) -> ChatCompletionRequest: + return ChatCompletionRequest( + model=MODEL_NAME, + messages=[{"role": "user", "content": "Record the exact value."}], + tools=_GLM47_TOOLS, + tool_choice="auto", + parallel_tool_calls=False, + include_reasoning=True, + return_token_ids=True, + stream=stream, + ) + + +def _glm47_request_output( + *, + text: str, + token_ids: list[int], + finished: bool, +) -> RequestOutput: + return RequestOutput( + request_id="glm47-stop-test", + prompt="test", + prompt_token_ids=[1, 2, 3], + prompt_logprobs=None, + outputs=[ + CompletionOutput( + index=0, + text=text, + token_ids=token_ids, + cumulative_logprob=0.0, + logprobs=None, + finish_reason="stop" if finished else None, + stop_reason=_GLM47_OBSERVATION_ID if finished else None, + ) + ], + finished=finished, + ) + + +def _glm47_serving_chat() -> OpenAIServingChat: + serving = _build_serving_chat( + MockEngine(), + reasoning_parser="glm45", + tool_parser="glm47", + enable_auto_tools=True, + ) + assert serving.parser_cls is Glm47MoeParser + return serving + + +@pytest.mark.asyncio +async def test_glm47_full_response_with_stripped_trailing_drop(): + tokenizer = _glm47_stop_tokenizer() + request = _glm47_request(stream=False) + normalized_messages = {} + + for mode, text in { + "matching": _GLM47_OUTPUT_TEXT + _GLM47_OBSERVATION, + "stop_stripped": _GLM47_OUTPUT_TEXT, + }.items(): + serving = _glm47_serving_chat() + assert serving.parser_cls is not None + parser = serving.parser_cls(tokenizer, request.tools) + response = await serving.chat_completion_full_generator( + request=request, + result_generator=_single_request_output( + _glm47_request_output( + text=text, + token_ids=list(_GLM47_OUTPUT_IDS), + finished=True, + ) + ), + request_id="chatcmpl-glm47-stop-test", + model_name=MODEL_NAME, + conversation=[], + tokenizer=tokenizer, + request_metadata=RequestResponseMetadata( + request_id="chatcmpl-glm47-stop-test", + model_name=MODEL_NAME, + ), + parser=parser, + ) + + assert isinstance(response, ChatCompletionResponse) + choice = response.choices[0] + message = choice.message + assert message.content in (None, "") + assert len(message.tool_calls or []) == 1 + call = message.tool_calls[0].function + assert call.name == "record_value" + assert json.loads(call.arguments) == {"value": " and "} + assert choice.finish_reason == "tool_calls" + assert choice.stop_reason == _GLM47_OBSERVATION_ID + assert choice.token_ids is not None + assert choice.token_ids[-1] == _GLM47_OBSERVATION_ID + normalized_messages[mode] = { + "reasoning": message.reasoning, + "content": message.content, + "name": call.name, + "arguments": json.loads(call.arguments), + "finish_reason": choice.finish_reason, + "stop_reason": choice.stop_reason, + "token_ids": choice.token_ids, + } + + assert normalized_messages["matching"] == normalized_messages["stop_stripped"] + + +@pytest.mark.asyncio +async def test_glm47_stream_response_with_empty_trailing_drop(): + tokenizer = _glm47_stop_tokenizer() + request = _glm47_request(stream=True) + serving = _glm47_serving_chat() + + async def result_generator(): + for token_id, text in _GLM47_GENERATED_TOKENS: + yield _glm47_request_output( + text=text, + token_ids=[token_id], + finished=False, + ) + yield _glm47_request_output( + text="", + token_ids=[_GLM47_OBSERVATION_ID], + finished=True, + ) + + call_ids = set() + indexes = set() + names = [] + argument_fragments = [] + content_fragments = [] + finish_reasons = [] + stop_reasons = [] + token_id_deltas = [] + done = False + + async for line in serving.chat_completion_stream_generator( + request=request, + result_generator=result_generator(), + request_id="chatcmpl-glm47-stop-test", + model_name=MODEL_NAME, + conversation=[], + tokenizer=tokenizer, + request_metadata=RequestResponseMetadata( + request_id="chatcmpl-glm47-stop-test", + model_name=MODEL_NAME, + ), + ): + payload = line.removeprefix("data: ").strip() + if payload == "[DONE]": + done = True + continue + document = json.loads(payload) + for choice in document.get("choices", []): + if choice.get("finish_reason"): + finish_reasons.append(choice["finish_reason"]) + if choice.get("stop_reason") is not None: + stop_reasons.append(choice["stop_reason"]) + if choice.get("token_ids") is not None: + token_id_deltas.append(choice["token_ids"]) + delta = choice.get("delta") or {} + if isinstance(delta.get("content"), str): + content_fragments.append(delta["content"]) + for call in delta.get("tool_calls") or []: + indexes.add(call["index"]) + if isinstance(call.get("id"), str): + call_ids.add(call["id"]) + function = call.get("function") or {} + if isinstance(function.get("name"), str): + names.append(function["name"]) + if isinstance(function.get("arguments"), str): + argument_fragments.append(function["arguments"]) + + assert done + assert indexes == {0} + assert len(call_ids) == 1 + assert "".join(names) == "record_value" + assert len(argument_fragments) > 1 + assert json.loads("".join(argument_fragments)) == { + "value": " and " + } + assert "".join(content_fragments) == "" + assert finish_reasons == ["tool_calls"] + assert stop_reasons == [_GLM47_OBSERVATION_ID] + assert token_id_deltas[-1] == [_GLM47_OBSERVATION_ID] + + async def _async_serving_chat_init(): engine = MockEngine() diff --git a/tests/parser/engine/test_token_id_scanner.py b/tests/parser/engine/test_token_id_scanner.py index c37a46793aaa..77e244b99192 100644 --- a/tests/parser/engine/test_token_id_scanner.py +++ b/tests/parser/engine/test_token_id_scanner.py @@ -9,6 +9,7 @@ from vllm.parser.engine.events import EventType from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine from vllm.parser.engine.token_id_scanner import ( + DROP_TERMINAL, PreLexedTerminal, TextChunk, TokenIDScanner, @@ -1015,3 +1016,137 @@ def test_last_anchor_missing_preceding_still_emitted(self, bare_scanner): assert bare_scanner._deferred_post_text == "more" assert len(bare_scanner._deferred_terminals) == 1 assert bare_scanner._deferred_terminals[0].terminal == "THINK_END" + + +class TestTrailingDropAnchorRecovery: + DROP_TEXT = "<|observation|>" + DROP_TEXT_2 = "<|user|>" + DROP_ID = 112 + DROP_ID_2 = 113 + TEXT_ID = 201 + BODY_ID = 202 + AFTER_ID = 203 + BODY = ( + "record_valuevalue" + " and " + ) + + def _scanner(self) -> TokenIDScanner: + tokenizer = MagicMock() + tokenizer.decode.side_effect = lambda ids: { + self.TEXT_ID: "prefix", + self.BODY_ID: self.BODY, + self.AFTER_ID: "after", + TOOL_START_ID: TOOL_START, + TOOL_END_ID: TOOL_END, + self.DROP_ID: self.DROP_TEXT, + self.DROP_ID_2: self.DROP_TEXT_2, + }[ids[0]] + return TokenIDScanner( + { + TOOL_START_ID: "TOOL_START", + TOOL_END_ID: "TOOL_END", + self.DROP_ID: DROP_TERMINAL, + self.DROP_ID_2: DROP_TERMINAL, + }, + tokenizer, + ) + + @staticmethod + def _signature(items, *, without_drop: bool = False): + result: list[tuple[object, ...]] = [] + for item in items: + if isinstance(item, TextChunk): + result.append(("text", item.text, item.token_count)) + elif not without_drop or item.terminal != DROP_TERMINAL: + result.append(("terminal", item.terminal)) + return result + + def _texts_and_ids(self, *drop_ids: int): + ids = [ + self.TEXT_ID, + TOOL_START_ID, + self.BODY_ID, + TOOL_END_ID, + *drop_ids, + ] + matching = f"prefix{TOOL_START}{self.BODY}{TOOL_END}" + for drop_id in drop_ids: + matching += { + self.DROP_ID: self.DROP_TEXT, + self.DROP_ID_2: self.DROP_TEXT_2, + }[drop_id] + stripped = f"prefix{TOOL_START}{self.BODY}{TOOL_END}" + return matching, stripped, ids + + def test_stop_stripped_trailing_drop_preserves_structural_anchors(self): + matching, stripped, ids = self._texts_and_ids(self.DROP_ID) + matching_scanner = self._scanner() + stripped_scanner = self._scanner() + + matching_items = matching_scanner.scan(matching, ids) + stripped_items = stripped_scanner.scan(stripped, ids) + + expected = [ + ("text", "prefix", 1), + ("terminal", "TOOL_START"), + ("text", self.BODY, 1), + ("terminal", "TOOL_END"), + ] + assert self._signature(matching_items, without_drop=True) == expected + assert self._signature(stripped_items) == expected + assert [item.terminal for item in stripped_scanner._deferred_terminals] == [ + DROP_TERMINAL + ] + assert stripped_scanner._deferred_prefix_token_counts == [0] + assert stripped_scanner._deferred_post_text == "" + + def test_stop_stripped_drop_preserves_text_token_count(self): + matching_scanner = self._scanner() + stripped_scanner = self._scanner() + + matching = matching_scanner.scan( + "prefix" + self.DROP_TEXT, + [self.TEXT_ID, self.DROP_ID], + ) + stripped = stripped_scanner.scan( + "prefix", + [self.TEXT_ID, self.DROP_ID], + ) + + matching_count = sum( + item.token_count for item in matching if isinstance(item, TextChunk) + ) + stripped_count = sum( + item.token_count for item in stripped if isinstance(item, TextChunk) + ) + assert matching_count == stripped_count == 1 + + def test_deferred_trailing_drop_resolves_on_next_delta(self): + _, stripped, ids = self._texts_and_ids(self.DROP_ID) + scanner = self._scanner() + scanner.scan(stripped, ids) + + items = scanner.scan(self.DROP_TEXT + "after", [self.AFTER_ID]) + + assert self._signature(items) == [ + ("terminal", DROP_TERMINAL), + ("text", "after", 1), + ] + assert scanner._deferred_terminals == [] + + def test_absent_trailing_drop_is_flushed_in_order(self): + _, stripped, ids = self._texts_and_ids(self.DROP_ID, self.DROP_ID_2) + scanner = self._scanner() + + stripped_items = scanner.scan(stripped, ids) + pending = scanner.flush_pending() + + assert self._signature(stripped_items) == [ + ("text", "prefix", 1), + ("terminal", "TOOL_START"), + ("text", self.BODY, 1), + ("terminal", "TOOL_END"), + ] + assert [item.token_id for item in pending] == [self.DROP_ID, self.DROP_ID_2] + assert all(item.terminal == DROP_TERMINAL for item in pending) diff --git a/vllm/parser/engine/token_id_scanner.py b/vllm/parser/engine/token_id_scanner.py index 60a64dcae281..49555b622d27 100644 --- a/vllm/parser/engine/token_id_scanner.py +++ b/vllm/parser/engine/token_id_scanner.py @@ -282,6 +282,8 @@ def _recover_holdback_text( ``delta_text`` that has no corresponding token ID in ``delta_token_ids``. This hold-back text always appears as a prefix of ``delta_text``. + + Serving can strip a trailing DROP terminal's text while retaining its ID. """ if not results: return [TextChunk(delta_text)] @@ -297,6 +299,26 @@ def _recover_holdback_text( if pos == 0: return results + trailing_drop_index = len(results) + while trailing_drop_index > 0: + trailing_drop = results[trailing_drop_index - 1] + if ( + not isinstance(trailing_drop, PreLexedTerminal) + or trailing_drop.terminal != DROP_TERMINAL + ): + break + trailing_drop_index -= 1 + + retained = results[:trailing_drop_index] + retained_text = self._join_decoded_text(retained) + if retained_text and retained_text == delta_text: + trailing_drops = results[trailing_drop_index:] + for trailing_drop in trailing_drops: + assert isinstance(trailing_drop, PreLexedTerminal) + self._deferred_terminals.append(trailing_drop) + self._deferred_prefix_token_counts.append(0) + return retained + # Fallback: SentencePiece context-dependent decoding mismatch. # Rebuild from delta_text using PreLexedTerminals as split anchors. return self._rebuild_from_anchors(delta_text, results) From 4b07713fe2f36a7ae6cfb9d287c283dfc19cb9c9 Mon Sep 17 00:00:00 2001 From: logprobz <321553542+logprobz@users.noreply.github.com> Date: Mon, 31 Aug 2026 08:39:53 +0000 Subject: [PATCH 03/12] feat(parser): support pending argument boundaries Add a default-inert parser state and non-whitespace transition hook so model grammars can delay ambiguous argument boundaries without bypassing fast paths, skip-tool handling, or finish. Co-authored-by: OpenAI Codex Signed-off-by: logprobz <321553542+logprobz@users.noreply.github.com> --- tests/parser/engine/test_engine.py | 107 ++++++++++++++++++ vllm/parser/engine/parser_engine_config.py | 7 ++ vllm/parser/engine/streaming_parser_engine.py | 5 + 3 files changed, 119 insertions(+) diff --git a/tests/parser/engine/test_engine.py b/tests/parser/engine/test_engine.py index 07f545571885..870932546c9f 100644 --- a/tests/parser/engine/test_engine.py +++ b/tests/parser/engine/test_engine.py @@ -780,6 +780,113 @@ def test_finish_no_tool_call_end_without_tool_index(self): assert engine.state == ParserState.CONTENT +class TestPendingArgValueEnd: + @staticmethod + def _config() -> ParserEngineConfig: + arg_chunk = (EventType.ARG_VALUE_CHUNK,) + return ParserEngineConfig( + name="pending_arg_value_end", + terminals={ + "TOOL_START": "", + "TOOL_END": "", + "ARG_KEY_START": "", + "ARG_KEY_END": "", + "ARG_VALUE_START": "", + "ARG_VALUE_END": "", + }, + transitions={ + (ParserState.CONTENT, "TOOL_START"): Transition( + ParserState.TOOL_NAME, + (EventType.TOOL_CALL_START,), + ), + (ParserState.TOOL_NAME, "ARG_KEY_START"): Transition( + ParserState.TOOL_BETWEEN, + arg_chunk, + ), + (ParserState.TOOL_BETWEEN, "ARG_KEY_END"): Transition( + ParserState.TOOL_BETWEEN, + arg_chunk, + ), + (ParserState.TOOL_BETWEEN, "ARG_VALUE_START"): Transition( + ParserState.TOOL_ARGS, + arg_chunk, + ), + (ParserState.TOOL_ARGS, "ARG_VALUE_END"): Transition( + ParserState.TOOL_ARG_END_PENDING, + arg_chunk, + ), + (ParserState.TOOL_ARG_END_PENDING, "ARG_VALUE_END"): Transition( + ParserState.TOOL_ARG_END_PENDING, + arg_chunk, + ), + (ParserState.TOOL_ARG_END_PENDING, "ARG_KEY_START"): Transition( + ParserState.TOOL_BETWEEN, + arg_chunk, + ), + (ParserState.TOOL_ARG_END_PENDING, "TOOL_END"): Transition( + ParserState.CONTENT, + (EventType.TOOL_CALL_END,), + ), + }, + non_whitespace_transitions={ + ParserState.TOOL_ARG_END_PENDING: Transition( + ParserState.TOOL_ARGS, + arg_chunk, + ) + }, + content_events={ + ParserState.CONTENT: EventType.TEXT_CHUNK, + ParserState.TOOL_NAME: EventType.TOOL_NAME, + ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK, + ParserState.TOOL_BETWEEN: EventType.ARG_VALUE_CHUNK, + ParserState.TOOL_ARG_END_PENDING: EventType.ARG_VALUE_CHUNK, + }, + tool_args_json=False, + ) + + def test_pending_arg_value_end_rejects_candidate_on_content(self): + engine = StreamingParserEngine( + self._config(), + tokenizer=None, + ) + events = engine.feed( + "record_valuevalueleft ", + [], + ) + assert engine.state == ParserState.TOOL_ARGS + + events.extend(engine.feed("", [])) + events.extend(engine.feed(" \n", [])) + assert engine.state == ParserState.TOOL_ARG_END_PENDING + events.extend(engine.feed("then ", [])) + assert engine.state == ParserState.TOOL_ARGS + + literal_end_events = engine.feed("", []) + assert all( + event.type != EventType.TOOL_CALL_END for event in literal_end_events + ) + events.extend(literal_end_events) + events.extend(engine.feed(" as data more", [])) + events.extend(engine.feed("", [])) + events.extend(engine.feed("after", [])) + events.extend(engine.finish()) + + assert sum(event.type == EventType.TOOL_CALL_START for event in events) == 1 + assert sum(event.type == EventType.TOOL_CALL_END for event in events) == 1 + raw_args = "".join( + event.value for event in events if event.type == EventType.ARG_VALUE_CHUNK + ) + assert raw_args == ( + "value" + "left \nthen as data more" + "" + ) + content = "".join( + event.value for event in events if event.type == EventType.TEXT_CHUNK + ) + assert content == "after" + + class TestRegexTerminalInfraRemoved: """TerminalDef.priority, LexerShape.regex_terminals, and the regex matching loop were removed.""" diff --git a/vllm/parser/engine/parser_engine_config.py b/vllm/parser/engine/parser_engine_config.py index ad83e331490a..4b9205a6a9a7 100644 --- a/vllm/parser/engine/parser_engine_config.py +++ b/vllm/parser/engine/parser_engine_config.py @@ -11,6 +11,8 @@ * **transitions** – a state machine mapping ``(state, terminal) → (new_state, events_to_emit)`` that drives semantic event generation during streaming. +* **non_whitespace_transitions** – optional state changes triggered by plain + content that contains a non-whitespace character. * **content_events** – what :class:`EventType` to emit for plain content (non-terminal text) in each state. """ @@ -33,6 +35,7 @@ class ParserState(Enum): TOOL_NAME = auto() TOOL_ARGS = auto() TOOL_BETWEEN = auto() + TOOL_ARG_END_PENDING = auto() @dataclass(frozen=True, slots=True) @@ -62,6 +65,10 @@ class ParserEngineConfig: default_factory=dict, ) + non_whitespace_transitions: dict[ParserState, Transition] = field( + default_factory=dict, + ) + content_events: dict[ParserState, EventType] = field( default_factory=lambda: { ParserState.CONTENT: EventType.TEXT_CHUNK, diff --git a/vllm/parser/engine/streaming_parser_engine.py b/vllm/parser/engine/streaming_parser_engine.py index ec21b05f4d3d..8f51e2ce3645 100644 --- a/vllm/parser/engine/streaming_parser_engine.py +++ b/vllm/parser/engine/streaming_parser_engine.py @@ -300,6 +300,7 @@ def finish(self) -> list[SemanticEvent]: ParserState.TOOL_ARGS, ParserState.TOOL_NAME, ParserState.TOOL_BETWEEN, + ParserState.TOOL_ARG_END_PENDING, ): if self.tool_index >= 0: events.append( @@ -355,6 +356,7 @@ def _process_lex_tokens(self, tokens: list[LexToken]) -> list[SemanticEvent]: ParserState.TOOL_NAME, ParserState.TOOL_ARGS, ParserState.TOOL_BETWEEN, + ParserState.TOOL_ARG_END_PENDING, } ) @@ -430,6 +432,9 @@ def _on_terminal( return self._apply_transition(transition, value, token_count) def _emit_for_state(self, text: str, token_count: int = 0) -> list[SemanticEvent]: + transition = self.config.non_whitespace_transitions.get(self.state) + if transition is not None and text.strip(): + return self._apply_transition(transition, text, token_count) if self.state == ParserState.MESSAGE_HEADER: self._message_header_buffer += text self._message_header_token_count += token_count From c9365fcd4e9f620be5bf95a47001baa0458ef917 Mon Sep 17 00:00:00 2001 From: logprobz <321553542+logprobz@users.noreply.github.com> Date: Mon, 31 Aug 2026 09:15:38 +0000 Subject: [PATCH 04/12] fix(glm47): delay ambiguous argument value ends Treat literal argument end tags as data until a next argument or tool close confirms the boundary. Replace regex extraction with deterministic lookahead and flush deferred post-tool content in single-pass parsing. Co-authored-by: OpenAI Codex Signed-off-by: logprobz <321553542+logprobz@users.noreply.github.com> --- .../chat_completion/test_serving_chat.py | 187 ++++++++++++ tests/parser/engine/test_parser_engine.py | 12 + .../test_glm47_moe_tool_parser.py | 281 ++++++++++++++++++ vllm/parser/engine/parser_engine.py | 20 +- vllm/parser/glm47_moe.py | 105 +++++-- 5 files changed, 576 insertions(+), 29 deletions(-) diff --git a/tests/entrypoints/openai/chat_completion/test_serving_chat.py b/tests/entrypoints/openai/chat_completion/test_serving_chat.py index 4561c88dc05d..559392f5f4d4 100644 --- a/tests/entrypoints/openai/chat_completion/test_serving_chat.py +++ b/tests/entrypoints/openai/chat_completion/test_serving_chat.py @@ -892,6 +892,30 @@ class MockEngine: *(token_id for token_id, _ in _GLM47_GENERATED_TOKENS), _GLM47_OBSERVATION_ID, ] +_GLM47_LITERAL_ARG_END_VALUE = "left then as data more" +_GLM47_LITERAL_ARG_END_TOKENS = [ + (_GLM47_REASONING_ID, _GLM47_REASONING), + (_GLM47_THINK_END_ID, _GLM47_THINK_END), + (_GLM47_TOOL_START_ID, _GLM47_TOOL_START), + (_GLM47_NAME_ID, "record_value"), + (_GLM47_ARG_KEY_START_ID, _GLM47_ARG_KEY_START), + (_GLM47_KEY_ID, "value"), + (_GLM47_ARG_KEY_END_ID, _GLM47_ARG_KEY_END), + (_GLM47_ARG_VALUE_START_ID, _GLM47_ARG_VALUE_START), + (306, "left "), + (_GLM47_ARG_VALUE_END_ID, _GLM47_ARG_VALUE_END), + (307, " then "), + (_GLM47_TOOL_END_ID, _GLM47_TOOL_END), + (308, " as data more"), + (_GLM47_ARG_VALUE_END_ID, _GLM47_ARG_VALUE_END), + (_GLM47_TOOL_END_ID, _GLM47_TOOL_END), + (309, "after"), +] +_GLM47_LITERAL_ARG_END_TEXT = "".join(text for _, text in _GLM47_LITERAL_ARG_END_TOKENS) +_GLM47_LITERAL_ARG_END_IDS = [ + *(token_id for token_id, _ in _GLM47_LITERAL_ARG_END_TOKENS), + _GLM47_OBSERVATION_ID, +] _GLM47_TOOLS = [ { "type": "function", @@ -923,6 +947,7 @@ def _glm47_stop_tokenizer() -> MockTokenizer: } tokens = [ *_GLM47_GENERATED_TOKENS, + *_GLM47_LITERAL_ARG_END_TOKENS, (154841, ""), (_GLM47_OBSERVATION_ID, _GLM47_OBSERVATION), ] @@ -979,6 +1004,168 @@ def _glm47_serving_chat() -> OpenAIServingChat: return serving +async def _collect_glm47_stream(tokens, *, holdback_arg_value_end: bool = False): + tokenizer = _glm47_stop_tokenizer() + request = _glm47_request(stream=True) + serving = _glm47_serving_chat() + + async def result_generator(): + held_arg_value_end = False + for token_id, text in tokens: + if ( + holdback_arg_value_end + and not held_arg_value_end + and token_id == _GLM47_ARG_VALUE_END_ID + ): + held_arg_value_end = True + yield _glm47_request_output( + text="", + token_ids=[token_id], + finished=False, + ) + continue + if held_arg_value_end: + text = _GLM47_ARG_VALUE_END + text + held_arg_value_end = False + yield _glm47_request_output( + text=text, + token_ids=[token_id], + finished=False, + ) + yield _glm47_request_output( + text="", + token_ids=[_GLM47_OBSERVATION_ID], + finished=True, + ) + + result: dict[str, Any] = { + "call_ids": set(), + "indexes": set(), + "names": [], + "arguments": [], + "content": [], + "finish_reasons": [], + "stop_reasons": [], + "token_ids": [], + "done": False, + } + async for line in serving.chat_completion_stream_generator( + request=request, + result_generator=result_generator(), + request_id="chatcmpl-glm47-arg-end-test", + model_name=MODEL_NAME, + conversation=[], + tokenizer=tokenizer, + request_metadata=RequestResponseMetadata( + request_id="chatcmpl-glm47-arg-end-test", + model_name=MODEL_NAME, + ), + ): + payload = line.removeprefix("data: ").strip() + if payload == "[DONE]": + result["done"] = True + continue + document = json.loads(payload) + for choice in document.get("choices", []): + if choice.get("finish_reason"): + result["finish_reasons"].append(choice["finish_reason"]) + if choice.get("stop_reason") is not None: + result["stop_reasons"].append(choice["stop_reason"]) + if choice.get("token_ids") is not None: + result["token_ids"].append(choice["token_ids"]) + delta = choice.get("delta") or {} + if isinstance(delta.get("content"), str): + result["content"].append(delta["content"]) + for call in delta.get("tool_calls") or []: + result["indexes"].add(call["index"]) + if isinstance(call.get("id"), str): + result["call_ids"].add(call["id"]) + function = call.get("function") or {} + if isinstance(function.get("name"), str): + result["names"].append(function["name"]) + if isinstance(function.get("arguments"), str): + result["arguments"].append(function["arguments"]) + return result + + +@pytest.mark.asyncio +async def test_glm47_full_response_with_literal_arg_value_end(): + tokenizer = _glm47_stop_tokenizer() + request = _glm47_request(stream=False) + serving = _glm47_serving_chat() + assert serving.parser_cls is not None + parser = serving.parser_cls(tokenizer, request.tools) + + response = await serving.chat_completion_full_generator( + request=request, + result_generator=_single_request_output( + _glm47_request_output( + text=_GLM47_LITERAL_ARG_END_TEXT, + token_ids=list(_GLM47_LITERAL_ARG_END_IDS), + finished=True, + ) + ), + request_id="chatcmpl-glm47-arg-end-test", + model_name=MODEL_NAME, + conversation=[], + tokenizer=tokenizer, + request_metadata=RequestResponseMetadata( + request_id="chatcmpl-glm47-arg-end-test", + model_name=MODEL_NAME, + ), + parser=parser, + ) + + assert isinstance(response, ChatCompletionResponse) + choice = response.choices[0] + message = choice.message + assert message.content == "after" + assert len(message.tool_calls or []) == 1 + call = message.tool_calls[0].function + assert call.name == "record_value" + assert json.loads(call.arguments) == {"value": _GLM47_LITERAL_ARG_END_VALUE} + assert choice.finish_reason == "tool_calls" + assert choice.stop_reason == _GLM47_OBSERVATION_ID + assert choice.token_ids is not None + assert choice.token_ids[-1] == _GLM47_OBSERVATION_ID + + +@pytest.mark.asyncio +async def test_glm47_stream_response_with_literal_arg_value_end(): + result = await _collect_glm47_stream(_GLM47_LITERAL_ARG_END_TOKENS) + + assert result["done"] + assert result["indexes"] == {0} + assert len(result["call_ids"]) == 1 + assert "".join(result["names"]) == "record_value" + assert len(result["arguments"]) > 1 + assert json.loads("".join(result["arguments"])) == { + "value": _GLM47_LITERAL_ARG_END_VALUE + } + assert "".join(result["content"]) == "after" + assert result["finish_reasons"] == ["tool_calls"] + assert result["stop_reasons"] == [_GLM47_OBSERVATION_ID] + assert result["token_ids"][-1] == [_GLM47_OBSERVATION_ID] + + +@pytest.mark.asyncio +async def test_glm47_stream_literal_arg_value_end_with_detokenizer_holdback(): + result = await _collect_glm47_stream( + _GLM47_LITERAL_ARG_END_TOKENS, + holdback_arg_value_end=True, + ) + + assert result["done"] + assert result["indexes"] == {0} + assert len(result["call_ids"]) == 1 + assert "".join(result["names"]) == "record_value" + assert json.loads("".join(result["arguments"])) == { + "value": _GLM47_LITERAL_ARG_END_VALUE + } + assert "".join(result["content"]) == "after" + assert result["finish_reasons"] == ["tool_calls"] + + @pytest.mark.asyncio async def test_glm47_full_response_with_stripped_trailing_drop(): tokenizer = _glm47_stop_tokenizer() diff --git a/tests/parser/engine/test_parser_engine.py b/tests/parser/engine/test_parser_engine.py index 37c9e9734fee..c178074a5ec5 100644 --- a/tests/parser/engine/test_parser_engine.py +++ b/tests/parser/engine/test_parser_engine.py @@ -523,6 +523,18 @@ def test_text_after_tool_deferred_even_when_finished(self): assert delta.tool_calls assert delta.content is None + def test_single_pass_flushes_deferred_post_tool_content(self): + engine = _make_engine(config=_hermes_config()) + + _, content, result = engine._single_pass_parse( + '{"name":"f","arguments":{}}after', + [], + ) + + assert result.tools_called + assert content == "after" + assert result.content == "after" + def test_text_before_tool_not_deferred(self): engine = _make_engine() engine._content_has_nonws = True diff --git a/tests/tool_parsers/test_glm47_moe_tool_parser.py b/tests/tool_parsers/test_glm47_moe_tool_parser.py index c6540812b32d..122e517a0bb6 100644 --- a/tests/tool_parsers/test_glm47_moe_tool_parser.py +++ b/tests/tool_parsers/test_glm47_moe_tool_parser.py @@ -4,6 +4,7 @@ """Tests for the GLM-4.7 tool call parser.""" import json +from types import SimpleNamespace from unittest.mock import Mock import pytest @@ -21,6 +22,25 @@ from vllm.tool_parsers.glm47_moe_tool_parser import Glm47MoeModelToolParser MODEL = "zai-org/GLM-4.7" +VALUE = "left then as data more" +OUTPUT = ( + "record_value" + "value" + f"{VALUE}after" +) +RECORD_VALUE_TOOL = ChatCompletionToolsParam( + function=FunctionDefinition( + name="record_value", + parameters={ + "type": "object", + "properties": { + "value": {"type": "string"}, + "path": {"type": "string"}, + }, + "required": ["value"], + }, + ) +) @pytest.fixture(scope="module") @@ -46,6 +66,7 @@ def sample_tools(): }, ), ), + RECORD_VALUE_TOOL, ] @@ -62,6 +83,111 @@ def mock_request(sample_tools) -> ChatCompletionRequest: return request +@pytest.fixture +def glm_parser(glm47_tool_parser, mock_request): + return glm47_tool_parser, mock_request + + +def _protocol_chunks(output: str) -> list[str]: + terminals = ( + "", + "", + "", + "", + "", + "", + "", + ) + chunks = [] + offset = 0 + while offset < len(output): + terminal = next( + (item for item in terminals if output.startswith(item, offset)), + None, + ) + if terminal is not None: + chunks.append(terminal) + offset += len(terminal) + continue + next_offsets = [ + position + for item in terminals + if (position := output.find(item, offset + 1)) >= 0 + ] + next_offset = min(next_offsets, default=len(output)) + chunks.append(output[offset:next_offset]) + offset = next_offset + return chunks + + +def run_glm_output(glm_parser, output: str, *, mode: str): + parser, request = glm_parser + if mode == "whole": + result = parser.extract_tool_calls(output, request=request) + calls = [ + SimpleNamespace( + index=index, + name=call.function.name, + arguments=call.function.arguments, + ) + for index, call in enumerate(result.tool_calls) + ] + return SimpleNamespace(calls=calls, content=result.content or "") + + _reset(parser) + chunks = list(output) if mode == "character" else _protocol_chunks(output) + current_text = "" + deltas = [] + for chunk in chunks: + previous_text = current_text + current_text += chunk + delta = parser.extract_tool_calls_streaming( + previous_text=previous_text, + current_text=current_text, + delta_text=chunk, + previous_token_ids=[], + current_token_ids=[], + delta_token_ids=[], + request=request, + ) + if delta: + deltas.append(delta) + finish = parser.finish_streaming() + if finish: + deltas.append(finish) + + call_parts: dict[int, dict[str, str]] = {} + for delta in deltas: + for call in delta.tool_calls or []: + parts = call_parts.setdefault(call.index, {"name": "", "arguments": ""}) + if call.function: + parts["name"] += call.function.name or "" + parts["arguments"] += call.function.arguments or "" + calls = [ + SimpleNamespace(index=index, **call_parts[index]) + for index in sorted(call_parts) + ] + content = "".join(delta.content or "" for delta in deltas) + return SimpleNamespace(calls=calls, content=content) + + +def collect_calls(result): + return result.calls + + +def collect_names(calls) -> list[str]: + return [call.name for call in calls] + + +def collect_arguments(calls) -> str: + assert len(calls) == 1 + return calls[0].arguments + + +def collect_content(result) -> str: + return result.content + + @pytest.fixture def namespace_tool_request() -> ResponsesRequest: return ResponsesRequest.model_validate( @@ -205,6 +331,161 @@ def test_tool_delimiters_in_arg_value(self, glm47_tool_parser, mock_request): assert function.name == "get_weather" assert json.loads(function.arguments) == {"city": value} + @pytest.mark.parametrize("mode", ["whole", "character", "token"]) + def test_literal_arg_value_end_keeps_later_tool_end_as_data( + self, + mode, + glm_parser, + ): + deltas = run_glm_output(glm_parser, OUTPUT, mode=mode) + calls = collect_calls(deltas) + assert {call.index for call in calls} == {0} + assert collect_names(calls) == ["record_value"] + assert json.loads(collect_arguments(calls)) == {"value": VALUE} + assert collect_content(deltas) == "after" + + @pytest.mark.parametrize("mode", ["whole", "character", "token"]) + @pytest.mark.parametrize( + "value", + [ + pytest.param("starts", id="start"), + pytest.param("middle continues", id="middle"), + pytest.param("ends", id="adjacent-end"), + pytest.param( + "one two three", + id="repeated", + ), + pytest.param( + "left text " + " right", + id="later-terminals", + ), + pytest.param("雪 \\tmp\\file & bold", id="data-bytes"), + pytest.param("", id="empty"), + ], + ) + def test_literal_arg_value_end_edge_values( + self, + mode, + value, + glm_parser, + ): + output = ( + "record_value" + "value" + f"{value}" + "after" + ) + + result = run_glm_output(glm_parser, output, mode=mode) + calls = collect_calls(result) + + assert {call.index for call in calls} == {0} + assert collect_names(calls) == ["record_value"] + assert json.loads(collect_arguments(calls)) == {"value": value} + assert collect_content(result) == "after" + + @pytest.mark.parametrize("mode", ["whole", "character", "token"]) + def test_literal_arg_key_start_before_real_second_argument( + self, + mode, + glm_parser, + ): + value = "see tag" + output = ( + "record_value" + "value" + f"{value}" + "path/tmp/x" + "after" + ) + + result = run_glm_output(glm_parser, output, mode=mode) + calls = collect_calls(result) + + assert json.loads(collect_arguments(calls)) == { + "value": value, + "path": "/tmp/x", + } + assert collect_content(result) == "after" + + def test_exact_arg_value_end_arg_key_boundary_is_structural(self, glm_parser): + output = ( + "record_value" + "valueleft " + "path/tmp/x" + "" + ) + + result = run_glm_output(glm_parser, output, mode="whole") + + assert json.loads(collect_arguments(collect_calls(result))) == { + "value": "left ", + "path": "/tmp/x", + } + + def test_exact_arg_value_end_tool_end_boundary_is_structural(self, glm_parser): + output = ( + "record_value" + "valueleft " + "tail" + ) + + result = run_glm_output(glm_parser, output, mode="whole") + + assert json.loads(collect_arguments(collect_calls(result))) == { + "value": "left " + } + assert collect_content(result) == "tail" + + def test_stray_text_after_real_closer_uses_malformed_finish_contract( + self, + glm_parser, + ): + output = ( + "record_value" + "valueBeijing" + " ok trailing text" + ) + + result = run_glm_output(glm_parser, output, mode="whole") + + assert json.loads(collect_arguments(collect_calls(result))) == {} + assert collect_content(result) == "" + + def test_missing_real_arg_value_end_remains_malformed(self, glm_parser): + output = ( + "record_value" + "valueunterminated" + "after" + ) + + result = run_glm_output(glm_parser, output, mode="whole") + + assert json.loads(collect_arguments(collect_calls(result))) == {} + assert collect_content(result) == "" + + def test_literal_arg_value_end_round_trips_to_responses_output( + self, + glm_parser, + ): + parser, request = glm_parser + result = parser.extract_tool_calls(OUTPUT, request=request) + function = result.tool_calls[0].function + + output_items = build_response_output_items( + reasoning=None, + content=result.content, + tool_calls=[function], + tools=request.tools, + ) + response_call = next( + item for item in output_items if isinstance(item, ResponseFunctionToolCall) + ) + + assert response_call.name == "record_value" + assert json.loads(response_call.arguments) == {"value": VALUE} + def _reset(parser): parser.current_tool_name_sent = False diff --git a/vllm/parser/engine/parser_engine.py b/vllm/parser/engine/parser_engine.py index 5ed837d22df1..60cc78f27864 100644 --- a/vllm/parser/engine/parser_engine.py +++ b/vllm/parser/engine/parser_engine.py @@ -646,17 +646,21 @@ def _single_pass_parse( events.extend(self._engine.finish()) delta = self._events_to_delta(events, finished=True) - tool_call_info = self._build_extracted_result() - - reasoning = delta.reasoning if delta else None + deferred_delta = self._events_to_delta([], finished=True) + tool_call_info = self._build_extracted_result(delta, deferred_delta) + + reasoning = ( + "".join( + item.reasoning or "" + for item in (delta, deferred_delta) + if item is not None + ) + or None + ) if reasoning and self._strip_trailing_reasoning_ws: reasoning = reasoning.rstrip() or None - content = delta.content if delta else None - if content: - content = self._strip_content_whitespace( - content, tool_call_info.tools_called - ) + content = tool_call_info.content return reasoning, content, tool_call_info diff --git a/vllm/parser/glm47_moe.py b/vllm/parser/glm47_moe.py index 789d459f2ff6..b7d4b79dc6fe 100644 --- a/vllm/parser/glm47_moe.py +++ b/vllm/parser/glm47_moe.py @@ -16,8 +16,6 @@ import json from typing import TYPE_CHECKING -import regex as re - from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest from vllm.entrypoints.openai.responses.protocol import ResponsesRequest from vllm.parser.engine.events import EventType @@ -41,31 +39,77 @@ ARG_VALUE_START = "" ARG_VALUE_END = "" -_ARG_RE = re.compile( - r"(?P.*?)\s*" - r"(?P.*?)", - re.DOTALL, -) -_PARTIAL_ARG_RE = re.compile( - r"(?P.*?)\s*" - r"(?P.*)$", - re.DOTALL, -) + +def _skip_whitespace(text: str, offset: int) -> int: + while offset < len(text) and text[offset].isspace(): + offset += 1 + return offset def _glm47_arg_converter(raw_args: str, partial: bool) -> str: params: dict[str, object] = {} + cursor = 0 + + while True: + key_start = raw_args.find(ARG_KEY_START, cursor) + if key_start < 0: + break + key_value_start = key_start + len(ARG_KEY_START) + + key_end_search = key_value_start + key_end = -1 + value_start = -1 + while True: + candidate = raw_args.find(ARG_KEY_END, key_end_search) + if candidate < 0: + break + after_key = _skip_whitespace(raw_args, candidate + len(ARG_KEY_END)) + if raw_args.startswith(ARG_VALUE_START, after_key): + key_end = candidate + value_start = after_key + len(ARG_VALUE_START) + break + key_end_search = candidate + 1 + + if key_end < 0: + break + + key = raw_args[key_value_start:key_end].strip() + value_end_search = value_start + value_end = -1 + next_cursor = -1 + pending_value_end = -1 + while True: + candidate = raw_args.find(ARG_VALUE_END, value_end_search) + if candidate < 0: + break + after_value = _skip_whitespace( + raw_args, + candidate + len(ARG_VALUE_END), + ) + if raw_args.startswith(ARG_KEY_START, after_value): + value_end = candidate + next_cursor = after_value + break + if after_value == len(raw_args): + if partial: + pending_value_end = candidate + else: + value_end = candidate + next_cursor = after_value + break + value_end_search = candidate + 1 - for match in _ARG_RE.finditer(raw_args): - params[match.group("key").strip()] = match.group("value") + if value_end >= 0: + params[key] = raw_args[value_start:value_end] + cursor = next_cursor + if cursor >= len(raw_args): + break + continue - if partial: - remaining = _ARG_RE.sub("", raw_args) - match = _PARTIAL_ARG_RE.search(remaining) - if match: - key = match.group("key").strip() - if key: - params[key] = match.group("value") + if partial and key: + partial_end = pending_value_end if pending_value_end >= 0 else len(raw_args) + params[key] = raw_args[value_start:partial_end] + break return json.dumps(params, ensure_ascii=False) @@ -90,6 +134,14 @@ def glm47_moe_config(thinking: bool = True) -> ParserEngineConfig: (EventType.ARG_VALUE_CHUNK,), ), (ParserState.TOOL_ARGS, "ARG_VALUE_END"): Transition( + ParserState.TOOL_ARG_END_PENDING, + (EventType.ARG_VALUE_CHUNK,), + ), + (ParserState.TOOL_ARG_END_PENDING, "ARG_VALUE_END"): Transition( + ParserState.TOOL_ARG_END_PENDING, + (EventType.ARG_VALUE_CHUNK,), + ), + (ParserState.TOOL_ARG_END_PENDING, "ARG_KEY_START"): Transition( ParserState.TOOL_BETWEEN, (EventType.ARG_VALUE_CHUNK,), ), @@ -169,14 +221,25 @@ def glm47_moe_config(thinking: bool = True) -> ParserEngineConfig: ParserState.CONTENT, (EventType.TOOL_CALL_END,), ), + (ParserState.TOOL_ARG_END_PENDING, "TOOL_END"): Transition( + ParserState.CONTENT, + (EventType.TOOL_CALL_END,), + ), **arg_tag_transitions, }, + non_whitespace_transitions={ + ParserState.TOOL_ARG_END_PENDING: Transition( + ParserState.TOOL_ARGS, + (EventType.ARG_VALUE_CHUNK,), + ) + }, content_events={ ParserState.CONTENT: EventType.TEXT_CHUNK, ParserState.REASONING: EventType.REASONING_CHUNK, ParserState.TOOL_NAME: EventType.TOOL_NAME, ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK, ParserState.TOOL_BETWEEN: EventType.ARG_VALUE_CHUNK, + ParserState.TOOL_ARG_END_PENDING: EventType.ARG_VALUE_CHUNK, }, arg_converter=_glm47_arg_converter, stream_arg_deltas=True, From e326335fb69dc8e369f800b23daafe606dd9076b Mon Sep 17 00:00:00 2001 From: logprobz <321553542+logprobz@users.noreply.github.com> Date: Mon, 31 Aug 2026 10:24:26 +0000 Subject: [PATCH 05/12] fix(parser): preserve scanner holdback ordering Bind reconstructed token text at the delta suffix, retain ordered token-count carriers across deferred terminals, and prevent automatic DROP aliases from replacing configured token IDs. Keep alias text fallback unchanged. Co-authored-by: OpenAI Codex Signed-off-by: logprobz <321553542+logprobz@users.noreply.github.com> (cherry picked from commit 1518c03eb1dbfcc3f3fc0cc959d442f2a2e5bc0b) --- tests/parser/engine/test_token_id_scanner.py | 152 ++++++++++++++++++ vllm/parser/engine/streaming_parser_engine.py | 10 +- vllm/parser/engine/token_id_scanner.py | 40 +++-- 3 files changed, 184 insertions(+), 18 deletions(-) diff --git a/tests/parser/engine/test_token_id_scanner.py b/tests/parser/engine/test_token_id_scanner.py index 77e244b99192..b31d7cdf7e5b 100644 --- a/tests/parser/engine/test_token_id_scanner.py +++ b/tests/parser/engine/test_token_id_scanner.py @@ -7,6 +7,11 @@ import pytest from vllm.parser.engine.events import EventType +from vllm.parser.engine.parser_engine_config import ( + ParserEngineConfig, + ParserState, + Transition, +) from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine from vllm.parser.engine.token_id_scanner import ( DROP_TERMINAL, @@ -1150,3 +1155,150 @@ def test_absent_trailing_drop_is_flushed_in_order(self): ] assert [item.token_id for item in pending] == [self.DROP_ID, self.DROP_ID_2] assert all(item.terminal == DROP_TERMINAL for item in pending) + + +def _cleanup_tokenizer(decoded: dict[int, str]) -> MagicMock: + tokenizer = MagicMock() + tokenizer.decode.side_effect = lambda ids: decoded[ids[0]] + return tokenizer + + +def test_reconstructed_current_text_binds_to_suffix(): + body_id = 10 + end_id = 11 + scanner = TokenIDScanner( + {end_id: "END"}, + _cleanup_tokenizer({body_id: "tail", end_id: ""}), + ) + + items = scanner.scan("tail held tail", [body_id, end_id]) + + assert items == [ + TextChunk("tail held "), + TextChunk("tail", ("tail",), 1), + PreLexedTerminal("END", end_id, ""), + ] + + +def test_stripped_trailing_drop_preserves_holdback_and_token_count(): + body_id = 10 + drop_id = 11 + scanner = TokenIDScanner( + {drop_id: DROP_TERMINAL}, + _cleanup_tokenizer({body_id: "prefix", drop_id: ""}), + ) + + items = scanner.scan("holdback prefix", [body_id, drop_id]) + + assert items == [ + TextChunk("holdback "), + TextChunk("prefix", ("prefix",), 1), + ] + assert scanner.flush_pending() == [ + PreLexedTerminal(DROP_TERMINAL, drop_id, "") + ] + + +def test_resolved_terminal_preserves_deferred_trailing_count(): + body_id = 10 + end_id = 11 + scanner = TokenIDScanner( + {end_id: "END"}, + _cleanup_tokenizer({body_id: "body", end_id: ""}), + ) + + assert scanner.scan("", [end_id, body_id]) == [] + assert scanner.scan("", []) == [ + PreLexedTerminal("END", end_id, ""), + TextChunk("", token_count=1), + ] + + +@pytest.mark.parametrize("finish", [False, True], ids=["resolve", "flush"]) +def test_inter_terminal_count_carrier_stays_ordered(finish: bool): + start_id = 10 + body_id = 11 + end_id = 12 + scanner = TokenIDScanner( + {start_id: "START", end_id: "END"}, + _cleanup_tokenizer( + { + start_id: "", + body_id: "body", + end_id: "", + } + ), + ) + + assert scanner.scan("", [start_id, body_id, end_id]) == [] + items = scanner.flush_pending() if finish else scanner.scan("", []) + + assert items == [ + PreLexedTerminal("START", start_id, ""), + TextChunk("", token_count=1), + PreLexedTerminal("END", end_id, ""), + ] + + +@pytest.mark.parametrize( + "protection", + ["terminals", "token_id_terminals", "preserve_tokens"], +) +def test_auto_drop_alias_does_not_override_protected_id(protection: str): + protected = "" + alias = "" + dropped = "" + protected_id = 10 + dropped_id = 11 + tokenizer = _cleanup_tokenizer({protected_id: protected, dropped_id: dropped}) + tokenizer.get_vocab.return_value = { + protected: protected_id, + alias: protected_id, + dropped: dropped_id, + } + tokenizer.all_special_tokens = [protected, alias, dropped] + tokenizer.all_special_ids = [protected_id, protected_id, dropped_id] + + terminals = {"PROTECTED": protected} if protection == "terminals" else {} + token_id_terminals = ( + {"PROTECTED": protected} if protection == "token_id_terminals" else {} + ) + preserve_tokens = ( + frozenset({protected}) if protection == "preserve_tokens" else frozenset() + ) + transitions = ( + { + (ParserState.CONTENT, "PROTECTED"): Transition( + ParserState.CONTENT, + (EventType.TEXT_CHUNK,), + ) + } + if protection != "preserve_tokens" + else {} + ) + config = ParserEngineConfig( + name=f"protected_{protection}", + terminals=terminals, + token_id_terminals=token_id_terminals, + transitions=transitions, + content_events={ParserState.CONTENT: EventType.TEXT_CHUNK}, + initial_state=ParserState.CONTENT, + preserve_tokens=preserve_tokens, + ) + engine = StreamingParserEngine(config, tokenizer) + + events = engine.feed(protected, [protected_id]) + events.extend(engine.finish()) + + assert ( + "".join(event.value for event in events if event.type == EventType.TEXT_CHUNK) + == protected + ) + + alias_events = StreamingParserEngine(config, tokenizer).parse_complete(alias) + assert ( + "".join( + event.value for event in alias_events if event.type == EventType.TEXT_CHUNK + ) + == "" + ) diff --git a/vllm/parser/engine/streaming_parser_engine.py b/vllm/parser/engine/streaming_parser_engine.py index 8f51e2ce3645..a26c4865891a 100644 --- a/vllm/parser/engine/streaming_parser_engine.py +++ b/vllm/parser/engine/streaming_parser_engine.py @@ -39,6 +39,7 @@ class _DropInfo: def _build_drop_info( config: ParserEngineConfig, tokenizer, + vocab: dict[str, int], ) -> _DropInfo | None: try: special_tokens: list[str] = list(tokenizer.all_special_tokens) @@ -54,12 +55,16 @@ def _build_drop_info( | set(config.terminals.values()) | config.preserve_tokens ) + protected_token_ids = { + tid for text in configured_texts if (tid := vocab.get(text)) is not None + } extra_token_ids: dict[int, str] = {} drop_texts: set[str] = set() for text, tid in zip(special_tokens, special_ids): if text not in configured_texts: - extra_token_ids[tid] = DROP_TERMINAL + if tid not in protected_token_ids: + extra_token_ids[tid] = DROP_TERMINAL drop_texts.add(text) if not drop_texts: @@ -130,7 +135,8 @@ def __init__( drop_info: _DropInfo | None = None if tokenizer is not None: - drop_info = _build_drop_info(config, tokenizer) + assert vocab is not None + drop_info = _build_drop_info(config, tokenizer, vocab) lexer_shape = config.lexer_shape if drop_info is not None: diff --git a/vllm/parser/engine/token_id_scanner.py b/vllm/parser/engine/token_id_scanner.py index 49555b622d27..260d1412edf5 100644 --- a/vllm/parser/engine/token_id_scanner.py +++ b/vllm/parser/engine/token_id_scanner.py @@ -123,6 +123,8 @@ def scan( deferred_trailing_count + len(token_texts), ) ) + elif deferred_trailing_count: + prefix_items.append(TextChunk("", token_count=deferred_trailing_count)) return prefix_items or self._EMPTY decoded_token_texts = [self._decode_token(tid) for tid in delta_token_ids] @@ -189,17 +191,17 @@ def flush_pending(self) -> list[LexerInput]: if not self._deferred_terminals and not self._deferred_post_text: return [] results: list[LexerInput] = [] - if self._deferred_post_text: - prefix_count = ( - self._deferred_prefix_token_counts[0] - if self._deferred_prefix_token_counts - else 0 - ) - results.append( - TextChunk(self._deferred_post_text, token_count=prefix_count) - ) - self._deferred_post_text = "" - results.extend(self._deferred_terminals) + deferred_post_text = self._deferred_post_text + deferred = self._deferred_terminals + prefix_token_counts = self._deferred_prefix_token_counts + if not deferred and deferred_post_text: + results.append(TextChunk(deferred_post_text)) + for idx, terminal in enumerate(deferred): + prefix_text = deferred_post_text if idx == 0 else "" + prefix_count = prefix_token_counts[idx] + if prefix_text or prefix_count: + results.append(TextChunk(prefix_text, token_count=prefix_count)) + results.append(terminal) if self._deferred_trailing_token_count: results.append( TextChunk("", token_count=self._deferred_trailing_token_count) @@ -207,6 +209,7 @@ def flush_pending(self) -> list[LexerInput]: self._deferred_terminals.clear() self._deferred_prefix_token_counts.clear() self._deferred_trailing_token_count = 0 + self._deferred_post_text = "" return results def _resolve_deferred( @@ -253,6 +256,8 @@ def _resolve_deferred( results.append(terminal) remaining = remaining[pos + len(terminal.text) :] elif pos == 0: + if prefix_token_count: + results.append(TextChunk("", token_count=prefix_token_count)) results.append(terminal) remaining = remaining[len(terminal.text) :] else: @@ -293,10 +298,10 @@ def _recover_holdback_text( if not reconstructed: return [TextChunk(delta_text)] + results - pos = delta_text.find(reconstructed) - if pos > 0: - return [TextChunk(delta_text[:pos])] + results - if pos == 0: + if delta_text.endswith(reconstructed): + holdback_text = delta_text[: -len(reconstructed)] + if holdback_text: + return [TextChunk(holdback_text)] + results return results trailing_drop_index = len(results) @@ -311,12 +316,15 @@ def _recover_holdback_text( retained = results[:trailing_drop_index] retained_text = self._join_decoded_text(retained) - if retained_text and retained_text == delta_text: + if retained_text and delta_text.endswith(retained_text): + holdback_text = delta_text[: -len(retained_text)] trailing_drops = results[trailing_drop_index:] for trailing_drop in trailing_drops: assert isinstance(trailing_drop, PreLexedTerminal) self._deferred_terminals.append(trailing_drop) self._deferred_prefix_token_counts.append(0) + if holdback_text: + return [TextChunk(holdback_text)] + retained return retained # Fallback: SentencePiece context-dependent decoding mismatch. From bf92e33cc4b1b688a7317fe1217cba14d6af4d25 Mon Sep 17 00:00:00 2001 From: logprobz <321553542+logprobz@users.noreply.github.com> Date: Mon, 31 Aug 2026 19:36:04 +0000 Subject: [PATCH 06/12] fix(parser): resolve protected aliases and deferred anchors Treat DROP lexer definitions as strict whenever token IDs are present, and anchor complete deferred terminal sequences before the exact current-token suffix. Preserve text-only drops and the incremental fallback. Co-authored-by: OpenAI Codex Signed-off-by: logprobz <321553542+logprobz@users.noreply.github.com> --- tests/parser/engine/test_token_id_scanner.py | 109 ++++++++++++++++++ vllm/parser/engine/streaming_parser_engine.py | 7 +- vllm/parser/engine/token_id_scanner.py | 52 ++++++++- 3 files changed, 164 insertions(+), 4 deletions(-) diff --git a/tests/parser/engine/test_token_id_scanner.py b/tests/parser/engine/test_token_id_scanner.py index b31d7cdf7e5b..dc7cd70889ee 100644 --- a/tests/parser/engine/test_token_id_scanner.py +++ b/tests/parser/engine/test_token_id_scanner.py @@ -1302,3 +1302,112 @@ def test_auto_drop_alias_does_not_override_protected_id(protection: str): ) == "" ) + + +@pytest.mark.parametrize( + "protection", + ["terminals", "token_id_terminals", "preserve_tokens"], +) +def test_alias_spelling_survives_when_its_only_id_is_protected( + protection: str, +): + protected = "" + alias = "" + protected_id = 10 + tokenizer = _cleanup_tokenizer({protected_id: alias}) + tokenizer.get_vocab.return_value = { + protected: protected_id, + alias: protected_id, + } + tokenizer.all_special_tokens = [protected, alias] + tokenizer.all_special_ids = [protected_id, protected_id] + + terminals = {"PROTECTED": protected} if protection == "terminals" else {} + token_id_terminals = ( + {"PROTECTED": protected} if protection == "token_id_terminals" else {} + ) + preserve_tokens = ( + frozenset({protected}) if protection == "preserve_tokens" else frozenset() + ) + transitions = ( + { + (ParserState.CONTENT, "PROTECTED"): Transition( + ParserState.CONTENT, + (EventType.TEXT_CHUNK,), + ) + } + if protection != "preserve_tokens" + else {} + ) + config = ParserEngineConfig( + name=f"alias_only_{protection}", + terminals=terminals, + token_id_terminals=token_id_terminals, + transitions=transitions, + content_events={ParserState.CONTENT: EventType.TEXT_CHUNK}, + initial_state=ParserState.CONTENT, + preserve_tokens=preserve_tokens, + ) + + events = StreamingParserEngine(config, tokenizer).feed(alias, [protected_id]) + assert ( + "".join(event.value for event in events if event.type == EventType.TEXT_CHUNK) + == alias + ) + + text_only = StreamingParserEngine(config, tokenizer).parse_complete(alias) + assert not any( + event.value == alias + for event in text_only + if event.type == EventType.TEXT_CHUNK + ) + + +def test_deferred_terminal_binds_after_literal_lookalike(): + body_id = 10 + end_id = 11 + scanner = TokenIDScanner( + {end_id: "END"}, + _cleanup_tokenizer({body_id: "after", end_id: ""}), + ) + + assert scanner.scan("", [end_id]) == [] + items = scanner.scan("literal real after", [body_id]) + + assert items == [ + TextChunk("literal real "), + PreLexedTerminal("END", end_id, ""), + TextChunk("after", ("after",), 1), + ] + + +def test_deferred_terminal_sequence_uses_current_suffix_boundary(): + start_id = 10 + body_id = 11 + end_id = 12 + after_id = 13 + scanner = TokenIDScanner( + {start_id: "START", end_id: "END"}, + _cleanup_tokenizer( + { + start_id: "", + body_id: "between", + end_id: "", + after_id: "after", + } + ), + ) + + assert scanner.scan("", [start_id, body_id, end_id]) == [] + items = scanner.scan( + "literal real after", + [after_id], + ) + + assert items == [ + TextChunk("literal real "), + PreLexedTerminal("START", start_id, ""), + TextChunk("", token_count=1), + PreLexedTerminal("END", end_id, ""), + TextChunk("after", ("after",), 1), + ] diff --git a/vllm/parser/engine/streaming_parser_engine.py b/vllm/parser/engine/streaming_parser_engine.py index a26c4865891a..6adafdb79838 100644 --- a/vllm/parser/engine/streaming_parser_engine.py +++ b/vllm/parser/engine/streaming_parser_engine.py @@ -151,9 +151,10 @@ def __init__( tokenizer, ) - self._token_id_terminal_names: frozenset[str] = frozenset( - resolved_token_ids.values() - ) + token_id_terminal_names = set(resolved_token_ids.values()) + if drop_info is not None: + token_id_terminal_names.add(DROP_TERMINAL) + self._token_id_terminal_names = frozenset(token_id_terminal_names) self._lexer = IncrementalLexer(lexer_shape, content_terminal=CONTENT_TERMINAL) diff --git a/vllm/parser/engine/token_id_scanner.py b/vllm/parser/engine/token_id_scanner.py index 260d1412edf5..85ac64352c71 100644 --- a/vllm/parser/engine/token_id_scanner.py +++ b/vllm/parser/engine/token_id_scanner.py @@ -87,8 +87,13 @@ def scan( deferred_trailing_count = 0 if self._deferred_terminals: + current_reconstruction = "".join(self._decode_tokens(delta_token_ids)) prefix_items, effective_text, deferred_trailing_count = ( - self._resolve_deferred(delta_text, len(delta_token_ids)) + self._resolve_deferred( + delta_text, + len(delta_token_ids), + current_reconstruction, + ) ) if not self.token_id_to_terminal: @@ -216,6 +221,7 @@ def _resolve_deferred( self, delta_text: str, current_token_count: int = 0, + current_reconstruction: str = "", ) -> tuple[list[LexerInput], str, int]: """Resolve deferred terminals against new delta_text. @@ -244,6 +250,16 @@ def _resolve_deferred( remaining = self._deferred_post_text + remaining self._deferred_post_text = "" + suffix_resolution = self._resolve_deferred_before_suffix( + remaining, + current_reconstruction, + deferred, + prefix_token_counts, + ) + if suffix_resolution is not None: + results, remaining = suffix_resolution + return results, remaining, trailing_token_count + # Duplicate-text deferred terminals resolve left-to-right via # find(); correct when each terminal text appears once in sequence. for idx, terminal in enumerate(deferred): @@ -276,6 +292,40 @@ def _resolve_deferred( return results, remaining, trailing_token_count + @staticmethod + def _resolve_deferred_before_suffix( + text: str, + current_reconstruction: str, + deferred: list[PreLexedTerminal], + prefix_token_counts: list[int], + ) -> tuple[list[LexerInput], str] | None: + if not current_reconstruction or not text.endswith(current_reconstruction): + return None + + search_end = len(text) - len(current_reconstruction) + positions = [-1] * len(deferred) + for idx in range(len(deferred) - 1, -1, -1): + position = text.rfind(deferred[idx].text, 0, search_end) + if position < 0: + return None + positions[idx] = position + search_end = position + + results: list[LexerInput] = [] + consumed = 0 + for terminal, prefix_token_count, position in zip( + deferred, + prefix_token_counts, + positions, + ): + prefix_text = text[consumed:position] + if prefix_text or prefix_token_count: + results.append(TextChunk(prefix_text, token_count=prefix_token_count)) + results.append(terminal) + consumed = position + len(terminal.text) + + return results, text[consumed:] + def _recover_holdback_text( self, delta_text: str, From 1c0f1b527cf413d650dfb083e5755c3e2e0c75c4 Mon Sep 17 00:00:00 2001 From: logprobz <321553542+logprobz@users.noreply.github.com> Date: Mon, 31 Aug 2026 20:52:33 +0000 Subject: [PATCH 07/12] fix(glm47): finalize malformed argument streams Co-authored-by: OpenAI Codex Signed-off-by: logprobz <321553542+logprobz@users.noreply.github.com> --- .../test_glm47_moe_tool_parser.py | 42 ++++++++++++++++--- vllm/parser/glm47_moe.py | 20 ++++++--- 2 files changed, 52 insertions(+), 10 deletions(-) diff --git a/tests/tool_parsers/test_glm47_moe_tool_parser.py b/tests/tool_parsers/test_glm47_moe_tool_parser.py index 122e517a0bb6..b0c9fc104ee7 100644 --- a/tests/tool_parsers/test_glm47_moe_tool_parser.py +++ b/tests/tool_parsers/test_glm47_moe_tool_parser.py @@ -438,8 +438,10 @@ def test_exact_arg_value_end_tool_end_boundary_is_structural(self, glm_parser): } assert collect_content(result) == "tail" + @pytest.mark.parametrize("mode", ["whole", "character", "token"]) def test_stray_text_after_real_closer_uses_malformed_finish_contract( self, + mode, glm_parser, ): output = ( @@ -448,21 +450,51 @@ def test_stray_text_after_real_closer_uses_malformed_finish_contract( " ok trailing text" ) - result = run_glm_output(glm_parser, output, mode="whole") + result = run_glm_output(glm_parser, output, mode=mode) - assert json.loads(collect_arguments(collect_calls(result))) == {} + assert json.loads(collect_arguments(collect_calls(result))) == { + "value": "Beijing" + } assert collect_content(result) == "" - def test_missing_real_arg_value_end_remains_malformed(self, glm_parser): + @pytest.mark.parametrize("mode", ["whole", "character", "token"]) + def test_missing_real_arg_value_end_remains_malformed( + self, + mode, + glm_parser, + ): output = ( "record_value" "valueunterminated" "after" ) - result = run_glm_output(glm_parser, output, mode="whole") + result = run_glm_output(glm_parser, output, mode=mode) + + assert json.loads(collect_arguments(collect_calls(result))) == { + "value": "unterminatedafter" + } + assert collect_content(result) == "" + + @pytest.mark.parametrize("mode", ["whole", "character", "token"]) + def test_malformed_last_argument_keeps_prior_arguments_valid( + self, + mode, + glm_parser, + ): + output = ( + "record_value" + "valuefirst" + "path/tmp/x" + " junk trailing text" + ) + + result = run_glm_output(glm_parser, output, mode=mode) - assert json.loads(collect_arguments(collect_calls(result))) == {} + assert json.loads(collect_arguments(collect_calls(result))) == { + "value": "first", + "path": "/tmp/x", + } assert collect_content(result) == "" def test_literal_arg_value_end_round_trips_to_responses_output( diff --git a/vllm/parser/glm47_moe.py b/vllm/parser/glm47_moe.py index b7d4b79dc6fe..bb8d0f730ddd 100644 --- a/vllm/parser/glm47_moe.py +++ b/vllm/parser/glm47_moe.py @@ -68,7 +68,7 @@ def _glm47_arg_converter(raw_args: str, partial: bool) -> str: key_end = candidate value_start = after_key + len(ARG_VALUE_START) break - key_end_search = candidate + 1 + key_end_search = candidate + len(ARG_KEY_END) if key_end < 0: break @@ -78,10 +78,12 @@ def _glm47_arg_converter(raw_args: str, partial: bool) -> str: value_end = -1 next_cursor = -1 pending_value_end = -1 + last_value_end = -1 while True: candidate = raw_args.find(ARG_VALUE_END, value_end_search) if candidate < 0: break + last_value_end = candidate after_value = _skip_whitespace( raw_args, candidate + len(ARG_VALUE_END), @@ -97,7 +99,13 @@ def _glm47_arg_converter(raw_args: str, partial: bool) -> str: value_end = candidate next_cursor = after_value break - value_end_search = candidate + 1 + if partial: + pending_value_end = candidate + value_end_search = candidate + len(ARG_VALUE_END) + + if not partial and value_end < 0 and last_value_end >= 0: + value_end = last_value_end + next_cursor = last_value_end + len(ARG_VALUE_END) if value_end >= 0: params[key] = raw_args[value_start:value_end] @@ -106,9 +114,11 @@ def _glm47_arg_converter(raw_args: str, partial: bool) -> str: break continue - if partial and key: - partial_end = pending_value_end if pending_value_end >= 0 else len(raw_args) - params[key] = raw_args[value_start:partial_end] + if key: + value_end = pending_value_end if partial else len(raw_args) + if value_end < 0: + value_end = len(raw_args) + params[key] = raw_args[value_start:value_end] break return json.dumps(params, ensure_ascii=False) From 6f3e747ddbd3f91cc4d4667a34e38e7570be3a68 Mon Sep 17 00:00:00 2001 From: logprobz <321553542+logprobz@users.noreply.github.com> Date: Mon, 31 Aug 2026 20:54:17 +0000 Subject: [PATCH 08/12] refactor(parser): simplify resolved parser state Co-authored-by: OpenAI Codex Signed-off-by: logprobz <321553542+logprobz@users.noreply.github.com> --- vllm/parser/engine/parser_engine.py | 8 +------- vllm/parser/engine/streaming_parser_engine.py | 12 ++++-------- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/vllm/parser/engine/parser_engine.py b/vllm/parser/engine/parser_engine.py index 60cc78f27864..ff8ec41be78a 100644 --- a/vllm/parser/engine/parser_engine.py +++ b/vllm/parser/engine/parser_engine.py @@ -565,17 +565,11 @@ def extract_tool_calls_from_content( so it can parse content that has already had reasoning stripped. """ self._check_skip_tool_parsing(request) - _, parsed_content, tool_call_info = self._single_pass_parse( + _, _, tool_call_info = self._single_pass_parse( content, [], initial_state=ParserState.CONTENT, ) - if parsed_content is not None and tool_call_info.content is None: - tool_call_info = ExtractedToolCallInformation( - tools_called=tool_call_info.tools_called, - tool_calls=tool_call_info.tool_calls, - content=parsed_content, - ) return tool_call_info def extract_tool_calls_streaming( diff --git a/vllm/parser/engine/streaming_parser_engine.py b/vllm/parser/engine/streaming_parser_engine.py index 6adafdb79838..369bdf0ddd16 100644 --- a/vllm/parser/engine/streaming_parser_engine.py +++ b/vllm/parser/engine/streaming_parser_engine.py @@ -124,19 +124,15 @@ def __init__( self.config = config resolved_token_ids: dict[int, str] = {} + drop_info: _DropInfo | None = None if tokenizer is not None: - if vocab is None: - vocab = tokenizer.get_vocab() + resolved_vocab = vocab if vocab is not None else tokenizer.get_vocab() if config.token_id_terminals: for terminal_name, token_text in config.token_id_terminals.items(): - tid = vocab.get(token_text) + tid = resolved_vocab.get(token_text) if tid is not None: resolved_token_ids[tid] = terminal_name - - drop_info: _DropInfo | None = None - if tokenizer is not None: - assert vocab is not None - drop_info = _build_drop_info(config, tokenizer, vocab) + drop_info = _build_drop_info(config, tokenizer, resolved_vocab) lexer_shape = config.lexer_shape if drop_info is not None: From 96cecb9e4cdafaa77639dbf9da5f8471f2efe578 Mon Sep 17 00:00:00 2001 From: logprobz <321553542+logprobz@users.noreply.github.com> Date: Mon, 31 Aug 2026 21:15:04 +0000 Subject: [PATCH 09/12] test(glm47): exercise real token id streams Co-authored-by: OpenAI Codex Signed-off-by: logprobz <321553542+logprobz@users.noreply.github.com> --- .../test_glm47_moe_tool_parser.py | 47 ++++++++++++++----- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/tests/tool_parsers/test_glm47_moe_tool_parser.py b/tests/tool_parsers/test_glm47_moe_tool_parser.py index b0c9fc104ee7..99061b0cc331 100644 --- a/tests/tool_parsers/test_glm47_moe_tool_parser.py +++ b/tests/tool_parsers/test_glm47_moe_tool_parser.py @@ -22,6 +22,7 @@ from vllm.tool_parsers.glm47_moe_tool_parser import Glm47MoeModelToolParser MODEL = "zai-org/GLM-4.7" +GLM_OUTPUT_MODES = ("whole", "character", "protocol", "token") VALUE = "left then as data more" OUTPUT = ( "record_value" @@ -135,19 +136,42 @@ def run_glm_output(glm_parser, output: str, *, mode: str): return SimpleNamespace(calls=calls, content=result.content or "") _reset(parser) - chunks = list(output) if mode == "character" else _protocol_chunks(output) + stream_steps: list[tuple[str, list[int]]] + if mode == "character": + stream_steps = [(chunk, []) for chunk in output] + elif mode == "protocol": + stream_steps = [(chunk, []) for chunk in _protocol_chunks(output)] + elif mode == "token": + tokenizer = parser.model_tokenizer + token_ids = tokenizer.encode(output, add_special_tokens=False) + stream_steps = [] + previous_decoded = "" + for index, token_id in enumerate(token_ids): + decoded = tokenizer.decode( + token_ids[: index + 1], + skip_special_tokens=False, + ) + assert decoded.startswith(previous_decoded) + stream_steps.append((decoded[len(previous_decoded) :], [token_id])) + previous_decoded = decoded + else: + raise ValueError(f"unknown GLM output mode: {mode}") + current_text = "" + current_token_ids: list[int] = [] deltas = [] - for chunk in chunks: + for chunk, delta_token_ids in stream_steps: previous_text = current_text + previous_token_ids = current_token_ids current_text += chunk + current_token_ids = [*previous_token_ids, *delta_token_ids] delta = parser.extract_tool_calls_streaming( previous_text=previous_text, current_text=current_text, delta_text=chunk, - previous_token_ids=[], - current_token_ids=[], - delta_token_ids=[], + previous_token_ids=previous_token_ids, + current_token_ids=current_token_ids, + delta_token_ids=delta_token_ids, request=request, ) if delta: @@ -331,7 +355,7 @@ def test_tool_delimiters_in_arg_value(self, glm47_tool_parser, mock_request): assert function.name == "get_weather" assert json.loads(function.arguments) == {"city": value} - @pytest.mark.parametrize("mode", ["whole", "character", "token"]) + @pytest.mark.parametrize("mode", GLM_OUTPUT_MODES) def test_literal_arg_value_end_keeps_later_tool_end_as_data( self, mode, @@ -344,7 +368,7 @@ def test_literal_arg_value_end_keeps_later_tool_end_as_data( assert json.loads(collect_arguments(calls)) == {"value": VALUE} assert collect_content(deltas) == "after" - @pytest.mark.parametrize("mode", ["whole", "character", "token"]) + @pytest.mark.parametrize("mode", GLM_OUTPUT_MODES) @pytest.mark.parametrize( "value", [ @@ -385,7 +409,7 @@ def test_literal_arg_value_end_edge_values( assert json.loads(collect_arguments(calls)) == {"value": value} assert collect_content(result) == "after" - @pytest.mark.parametrize("mode", ["whole", "character", "token"]) + @pytest.mark.parametrize("mode", GLM_OUTPUT_MODES) def test_literal_arg_key_start_before_real_second_argument( self, mode, @@ -438,7 +462,7 @@ def test_exact_arg_value_end_tool_end_boundary_is_structural(self, glm_parser): } assert collect_content(result) == "tail" - @pytest.mark.parametrize("mode", ["whole", "character", "token"]) + @pytest.mark.parametrize("mode", GLM_OUTPUT_MODES) def test_stray_text_after_real_closer_uses_malformed_finish_contract( self, mode, @@ -455,9 +479,10 @@ def test_stray_text_after_real_closer_uses_malformed_finish_contract( assert json.loads(collect_arguments(collect_calls(result))) == { "value": "Beijing" } + # Bytes after a malformed boundary remain inside the open call. assert collect_content(result) == "" - @pytest.mark.parametrize("mode", ["whole", "character", "token"]) + @pytest.mark.parametrize("mode", GLM_OUTPUT_MODES) def test_missing_real_arg_value_end_remains_malformed( self, mode, @@ -476,7 +501,7 @@ def test_missing_real_arg_value_end_remains_malformed( } assert collect_content(result) == "" - @pytest.mark.parametrize("mode", ["whole", "character", "token"]) + @pytest.mark.parametrize("mode", GLM_OUTPUT_MODES) def test_malformed_last_argument_keeps_prior_arguments_valid( self, mode, From a9526fe837bfc73da19c1fda99accf1e6073d78c Mon Sep 17 00:00:00 2001 From: logprobz <321553542+logprobz@users.noreply.github.com> Date: Mon, 31 Aug 2026 21:22:54 +0000 Subject: [PATCH 10/12] test(glm47): cover multibyte token holdback Co-authored-by: OpenAI Codex Signed-off-by: logprobz <321553542+logprobz@users.noreply.github.com> --- .../test_glm47_moe_tool_parser.py | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/tests/tool_parsers/test_glm47_moe_tool_parser.py b/tests/tool_parsers/test_glm47_moe_tool_parser.py index 99061b0cc331..969d96a5e0c2 100644 --- a/tests/tool_parsers/test_glm47_moe_tool_parser.py +++ b/tests/tool_parsers/test_glm47_moe_tool_parser.py @@ -19,6 +19,7 @@ from vllm.entrypoints.openai.responses.protocol import ResponsesRequest from vllm.entrypoints.openai.responses.utils import build_response_output_items from vllm.tokenizers import get_tokenizer +from vllm.tokenizers.detokenizer_utils import detokenize_incrementally from vllm.tool_parsers.glm47_moe_tool_parser import Glm47MoeModelToolParser MODEL = "zai-org/GLM-4.7" @@ -145,15 +146,25 @@ def run_glm_output(glm_parser, output: str, *, mode: str): tokenizer = parser.model_tokenizer token_ids = tokenizer.encode(output, add_special_tokens=False) stream_steps = [] - previous_decoded = "" + previous_tokens: list[str] | None = None + prefix_offset = 0 + read_offset = 0 for index, token_id in enumerate(token_ids): - decoded = tokenizer.decode( - token_ids[: index + 1], - skip_special_tokens=False, + (new_tokens, delta_text, prefix_offset, read_offset) = ( + detokenize_incrementally( + tokenizer=tokenizer, + all_input_ids=token_ids[: index + 1], + prev_tokens=previous_tokens, + prefix_offset=prefix_offset, + read_offset=read_offset, + skip_special_tokens=False, + spaces_between_special_tokens=True, + ) + ) + stream_steps.append((delta_text, [token_id])) + previous_tokens = ( + previous_tokens + new_tokens if previous_tokens else new_tokens ) - assert decoded.startswith(previous_decoded) - stream_steps.append((decoded[len(previous_decoded) :], [token_id])) - previous_decoded = decoded else: raise ValueError(f"unknown GLM output mode: {mode}") @@ -385,6 +396,7 @@ def test_literal_arg_value_end_keeps_later_tool_end_as_data( id="later-terminals", ), pytest.param("雪 \\tmp\\file & bold", id="data-bytes"), + pytest.param("ok 🎉 ∮ 𠀋 done", id="byte-fallback"), pytest.param("", id="empty"), ], ) From 594c8dc5b7599b89e3133ecd990c083b80606427 Mon Sep 17 00:00:00 2001 From: logprobz <321553542+logprobz@users.noreply.github.com> Date: Tue, 1 Sep 2026 05:27:53 +0000 Subject: [PATCH 11/12] test(glm47): lock audited serving edge cases Co-authored-by: OpenAI Codex Signed-off-by: logprobz <321553542+logprobz@users.noreply.github.com> --- .../chat_completion/test_serving_chat.py | 41 ++++++++++++++++++ .../test_glm47_moe_tool_parser.py | 42 ++++++++++++++++++- 2 files changed, 81 insertions(+), 2 deletions(-) diff --git a/tests/entrypoints/openai/chat_completion/test_serving_chat.py b/tests/entrypoints/openai/chat_completion/test_serving_chat.py index 559392f5f4d4..9d08204e4c71 100644 --- a/tests/entrypoints/openai/chat_completion/test_serving_chat.py +++ b/tests/entrypoints/openai/chat_completion/test_serving_chat.py @@ -1043,6 +1043,7 @@ async def result_generator(): "indexes": set(), "names": [], "arguments": [], + "reasoning": [], "content": [], "finish_reasons": [], "stop_reasons": [], @@ -1074,6 +1075,8 @@ async def result_generator(): if choice.get("token_ids") is not None: result["token_ids"].append(choice["token_ids"]) delta = choice.get("delta") or {} + if isinstance(delta.get("reasoning"), str): + result["reasoning"].append(delta["reasoning"]) if isinstance(delta.get("content"), str): result["content"].append(delta["content"]) for call in delta.get("tool_calls") or []: @@ -1119,6 +1122,7 @@ async def test_glm47_full_response_with_literal_arg_value_end(): assert isinstance(response, ChatCompletionResponse) choice = response.choices[0] message = choice.message + assert message.reasoning == _GLM47_REASONING assert message.content == "after" assert len(message.tool_calls or []) == 1 call = message.tool_calls[0].function @@ -1138,6 +1142,7 @@ async def test_glm47_stream_response_with_literal_arg_value_end(): assert result["indexes"] == {0} assert len(result["call_ids"]) == 1 assert "".join(result["names"]) == "record_value" + assert "".join(result["reasoning"]) == _GLM47_REASONING assert len(result["arguments"]) > 1 assert json.loads("".join(result["arguments"])) == { "value": _GLM47_LITERAL_ARG_END_VALUE @@ -1159,6 +1164,7 @@ async def test_glm47_stream_literal_arg_value_end_with_detokenizer_holdback(): assert result["indexes"] == {0} assert len(result["call_ids"]) == 1 assert "".join(result["names"]) == "record_value" + assert "".join(result["reasoning"]) == _GLM47_REASONING assert json.loads("".join(result["arguments"])) == { "value": _GLM47_LITERAL_ARG_END_VALUE } @@ -1166,6 +1172,40 @@ async def test_glm47_stream_literal_arg_value_end_with_detokenizer_holdback(): assert result["finish_reasons"] == ["tool_calls"] +@pytest.mark.asyncio +async def test_glm47_stream_actual_delimiter_ids_with_stripped_stop(): + tokens = [ + (_GLM47_REASONING_ID, _GLM47_REASONING), + (_GLM47_THINK_END_ID, _GLM47_THINK_END), + (_GLM47_TOOL_START_ID, _GLM47_TOOL_START), + (_GLM47_NAME_ID, "record_value"), + (_GLM47_ARG_KEY_START_ID, _GLM47_ARG_KEY_START), + (_GLM47_KEY_ID, "value"), + (_GLM47_ARG_KEY_END_ID, _GLM47_ARG_KEY_END), + (_GLM47_ARG_VALUE_START_ID, _GLM47_ARG_VALUE_START), + (_GLM47_TOOL_END_ID, _GLM47_TOOL_END), + (307, " then "), + (_GLM47_TOOL_START_ID, _GLM47_TOOL_START), + (_GLM47_ARG_VALUE_END_ID, _GLM47_ARG_VALUE_END), + (_GLM47_TOOL_END_ID, _GLM47_TOOL_END), + (309, "after"), + ] + result = await _collect_glm47_stream(tokens) + assert result["done"] + assert result["indexes"] == {0} + assert len(result["call_ids"]) == 1 + assert "".join(result["names"]) == "record_value" + assert len(result["arguments"]) > 1 + assert json.loads("".join(result["arguments"])) == { + "value": " then " + } + assert "".join(result["reasoning"]) == _GLM47_REASONING + assert "".join(result["content"]) == "after" + assert result["finish_reasons"] == ["tool_calls"] + assert result["stop_reasons"] == [_GLM47_OBSERVATION_ID] + assert result["token_ids"][-1] == [_GLM47_OBSERVATION_ID] + + @pytest.mark.asyncio async def test_glm47_full_response_with_stripped_trailing_drop(): tokenizer = _glm47_stop_tokenizer() @@ -1202,6 +1242,7 @@ async def test_glm47_full_response_with_stripped_trailing_drop(): assert isinstance(response, ChatCompletionResponse) choice = response.choices[0] message = choice.message + assert message.reasoning == _GLM47_REASONING assert message.content in (None, "") assert len(message.tool_calls or []) == 1 call = message.tool_calls[0].function diff --git a/tests/tool_parsers/test_glm47_moe_tool_parser.py b/tests/tool_parsers/test_glm47_moe_tool_parser.py index 969d96a5e0c2..9b0ab4fe8787 100644 --- a/tests/tool_parsers/test_glm47_moe_tool_parser.py +++ b/tests/tool_parsers/test_glm47_moe_tool_parser.py @@ -134,9 +134,14 @@ def run_glm_output(glm_parser, output: str, *, mode: str): ) for index, call in enumerate(result.tool_calls) ] - return SimpleNamespace(calls=calls, content=result.content or "") + return SimpleNamespace( + calls=calls, + content=result.content or "", + empty_token_deltas=0, + ) _reset(parser) + empty_token_deltas = 0 stream_steps: list[tuple[str, list[int]]] if mode == "character": stream_steps = [(chunk, []) for chunk in output] @@ -161,6 +166,8 @@ def run_glm_output(glm_parser, output: str, *, mode: str): spaces_between_special_tokens=True, ) ) + if not delta_text: + empty_token_deltas += 1 stream_steps.append((delta_text, [token_id])) previous_tokens = ( previous_tokens + new_tokens if previous_tokens else new_tokens @@ -203,7 +210,11 @@ def run_glm_output(glm_parser, output: str, *, mode: str): for index in sorted(call_parts) ] content = "".join(delta.content or "" for delta in deltas) - return SimpleNamespace(calls=calls, content=content) + return SimpleNamespace( + calls=calls, + content=content, + empty_token_deltas=empty_token_deltas, + ) def collect_calls(result): @@ -534,6 +545,33 @@ def test_malformed_last_argument_keeps_prior_arguments_valid( } assert collect_content(result) == "" + @pytest.mark.parametrize("mode", GLM_OUTPUT_MODES) + def test_unterminated_second_argument_remains_valid(self, mode, glm_parser): + output = ( + "record_value" + "valuefirst" + "path/tmp/x" + "tail" + ) + result = run_glm_output(glm_parser, output, mode=mode) + assert json.loads(collect_arguments(collect_calls(result))) == { + "value": "first", + "path": "/tmp/xtail", + } + assert collect_content(result) == "" + + def test_token_mode_exercises_multibyte_holdback(self, glm_parser): + value = "ok 🎉 ∮ 𠀋 done" + output = ( + "record_value" + "value" + f"{value}after" + ) + result = run_glm_output(glm_parser, output, mode="token") + assert result.empty_token_deltas > 0 + assert json.loads(collect_arguments(collect_calls(result))) == {"value": value} + assert collect_content(result) == "after" + def test_literal_arg_value_end_round_trips_to_responses_output( self, glm_parser, From 6f2214ccb780671bb8bb4a8e51fcfbcfce6d4253 Mon Sep 17 00:00:00 2001 From: logprobz <321553542+logprobz@users.noreply.github.com> Date: Tue, 1 Sep 2026 05:29:50 +0000 Subject: [PATCH 12/12] refactor(parser): preserve config constructor compatibility Co-authored-by: OpenAI Codex Signed-off-by: logprobz <321553542+logprobz@users.noreply.github.com> --- tests/parser/engine/test_engine.py | 7 ++++++ vllm/parser/engine/parser_engine_config.py | 8 +++--- vllm/parser/inkling.py | 29 ---------------------- 3 files changed, 11 insertions(+), 33 deletions(-) diff --git a/tests/parser/engine/test_engine.py b/tests/parser/engine/test_engine.py index 870932546c9f..3358ac99567e 100644 --- a/tests/parser/engine/test_engine.py +++ b/tests/parser/engine/test_engine.py @@ -22,6 +22,13 @@ from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine +def test_parser_engine_config_positional_content_events_compatibility(): + content_events = {ParserState.CONTENT: EventType.TEXT_CHUNK} + config = ParserEngineConfig("positional", {}, {}, {}, content_events) + assert config.content_events is content_events + assert config.non_whitespace_transitions == {} + + def _hermes_config() -> ParserEngineConfig: """Simple Hermes-style config: JSON.""" return ParserEngineConfig( diff --git a/vllm/parser/engine/parser_engine_config.py b/vllm/parser/engine/parser_engine_config.py index 4b9205a6a9a7..c725ecdc680c 100644 --- a/vllm/parser/engine/parser_engine_config.py +++ b/vllm/parser/engine/parser_engine_config.py @@ -65,10 +65,6 @@ class ParserEngineConfig: default_factory=dict, ) - non_whitespace_transitions: dict[ParserState, Transition] = field( - default_factory=dict, - ) - content_events: dict[ParserState, EventType] = field( default_factory=lambda: { ParserState.CONTENT: EventType.TEXT_CHUNK, @@ -103,6 +99,10 @@ class ParserEngineConfig: # Reject tool calls whose names are absent from the request tools. validate_tool_names: bool = False + non_whitespace_transitions: dict[ParserState, Transition] = field( + default_factory=dict, + ) + @cached_property def terminal_defs(self): from vllm.parser.engine.incremental_lexer import terminals_from_literals diff --git a/vllm/parser/inkling.py b/vllm/parser/inkling.py index bd2109950022..90c8279abe5a 100644 --- a/vllm/parser/inkling.py +++ b/vllm/parser/inkling.py @@ -29,9 +29,6 @@ from collections.abc import Sequence from typing import TYPE_CHECKING -from vllm.entrypoints.openai.engine.protocol import ( - ExtractedToolCallInformation, -) from vllm.parser.engine.events import EventType from vllm.parser.engine.parser_engine import ParserEngine from vllm.parser.engine.parser_engine_config import ( @@ -380,32 +377,6 @@ def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int: count += 1 return count - def _single_pass_parse( - self, - text: str, - token_ids: Sequence[int], - initial_state: ParserState | None = None, - ) -> tuple[str | None, str | None, ExtractedToolCallInformation]: - reasoning, content, tool_call_info = super()._single_pass_parse( - text, token_ids, initial_state=initial_state - ) - # The engine defers content that follows tool-call events within a - # single pass; Inkling allows text blocks after tool-call blocks, so - # flush the trailing text (matching the Rust unified parser). - if self._deferred_content: - trailing = self._deferred_content - self._deferred_content = "" - content = self._strip_content_whitespace( - (content or "") + trailing, - tool_call_info.tools_called, - ) - tool_call_info = ExtractedToolCallInformation( - tools_called=tool_call_info.tools_called, - tool_calls=tool_call_info.tool_calls, - content=content, - ) - return reasoning, content, tool_call_info - @staticmethod def _extract_args_value(parsed: dict) -> str | None: # Inkling wraps arguments under "args" rather than "arguments".