From 7222ff823bf88ab52fddb60698df444c39b4fdd9 Mon Sep 17 00:00:00 2001 From: Sohom Chakraborty Date: Wed, 23 Sep 2026 15:42:07 +0000 Subject: [PATCH] [Bugfix][Frontend] Materialize tool_calls in /tokenize and pooling chat requests `ChatCompletionRequest` normalizes `messages` (tool_calls iterators -> lists, `reasoning_content` -> `reasoning`) and re-materializes the lazy `ValidatorIterator` pydantic wraps `tool_calls` in. `TokenizeChatRequest` and the pooling `ChatRequestMixin` carry the same `messages` field but skipped both passes, so an assistant message with `content: null` + `tool_calls` sent to `/tokenize` reached the tokenizer as a `ValidatorIterator` and failed on `copy.deepcopy` with "cannot pickle 'ValidatorIterator' object" (HTTP 400). Move the two passes into shared helpers in `chat_utils` and apply them from all three request models. Regression tests parametrize over the three models. Fixes #57730 Signed-off-by: Sohom Chakraborty Co-authored-by: Claude --- .../openai/test_tool_calls_serialization.py | 66 +++++++++++++++++++ vllm/entrypoints/chat_utils.py | 52 +++++++++++++++ .../openai/chat_completion/protocol.py | 41 ++---------- vllm/entrypoints/pooling/base/protocol.py | 12 ++++ vllm/entrypoints/serve/tokenize/protocol.py | 12 ++++ 5 files changed, 146 insertions(+), 37 deletions(-) diff --git a/tests/entrypoints/openai/test_tool_calls_serialization.py b/tests/entrypoints/openai/test_tool_calls_serialization.py index cedc2575b80e..21bf0a2faca5 100644 --- a/tests/entrypoints/openai/test_tool_calls_serialization.py +++ b/tests/entrypoints/openai/test_tool_calls_serialization.py @@ -15,9 +15,13 @@ "ValueError: Unexpected tool call id ...". """ +import copy + import pytest from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest +from vllm.entrypoints.pooling.embed.protocol import EmbeddingChatRequest +from vllm.entrypoints.serve.tokenize.protocol import TokenizeChatRequest def _make_tool_call(tc_id: str, name: str, args: str) -> dict: @@ -148,3 +152,65 @@ def test_multiple_tool_calls_materialised(num_tool_calls: int): # Verify after model_dump_json too _ = req.model_dump_json() assert len(assistant_msg.get("tool_calls", [])) == num_tool_calls + + +def _tool_call_conversation() -> list: + tool_call = _make_tool_call("call_1", "lookup_weather", '{"city": "Shanghai"}') + return [ + {"role": "user", "content": "What is the weather in Shanghai tomorrow?"}, + {"role": "assistant", "content": None, "tool_calls": [tool_call]}, + { + "role": "tool", + "tool_call_id": "call_1", + "content": '{"temp_c": 24, "condition": "cloudy"}', + }, + {"role": "user", "content": "Do I need an umbrella?"}, + ] + + +@pytest.mark.parametrize( + "request_cls", + [ChatCompletionRequest, TokenizeChatRequest, EmbeddingChatRequest], +) +def test_tool_calls_materialised_for_all_chat_request_models(request_cls): + """Every request model carrying ``messages`` must materialise tool_calls. + + Regression test for https://github.com/vllm-project/vllm/issues/57730: + ``/tokenize`` (and pooling chat endpoints) validated ``messages`` without + the materialisation pass used by ``/v1/chat/completions``, so an assistant + message with ``content: null`` + ``tool_calls`` kept a lazy + ``ValidatorIterator`` that blew up on ``copy.deepcopy`` inside tokenizers + ("cannot pickle 'ValidatorIterator' object"). + """ + req = request_cls(model="test-model", messages=_tool_call_conversation()) + + assistant_msg = req.messages[1] + assert isinstance(assistant_msg, dict) + assert isinstance(assistant_msg.get("tool_calls"), list) + + # Tokenizers/renderers deep-copy messages; this must not raise. + copied = copy.deepcopy(req.messages) + assert copied[1]["tool_calls"][0]["function"]["name"] == "lookup_weather" + + # Reading twice must not consume anything. + assert len(list(assistant_msg["tool_calls"])) == 1 + assert len(list(assistant_msg["tool_calls"])) == 1 + + +@pytest.mark.parametrize( + "request_cls", + [ChatCompletionRequest, TokenizeChatRequest, EmbeddingChatRequest], +) +def test_reasoning_content_normalised_for_all_chat_request_models(request_cls): + """Deprecated ``reasoning_content`` is renamed to ``reasoning`` everywhere.""" + messages = [ + {"role": "user", "content": "Hi"}, + {"role": "assistant", "content": "Hello", "reasoning_content": "think"}, + ] + + req = request_cls(model="test-model", messages=messages) + + assistant_msg = req.messages[1] + assert isinstance(assistant_msg, dict) + assert assistant_msg.get("reasoning") == "think" + assert "reasoning_content" not in assistant_msg diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py index e3cc744ba70d..9d8463512142 100644 --- a/vllm/entrypoints/chat_utils.py +++ b/vllm/entrypoints/chat_utils.py @@ -396,6 +396,58 @@ class CustomChatCompletionMessageParam(TypedDict, total=False): ) +def normalize_chat_messages_before_validation(data: Any) -> Any: + """Pre-process raw ``messages`` dicts before Pydantic field validation. + + Intended to be called from a ``mode="before"`` model validator of any + request model that carries a ``messages: list[ChatCompletionMessageParam]`` + field (chat completions, tokenize, pooling chat requests, ...). + + Performs two normalizations in a single pass: + + - Converts ``tool_calls`` generators/iterators to lists so one-shot + generators are not consumed during union type matching. + - Renames the deprecated ``reasoning_content`` field to ``reasoning`` so + downstream code only needs to check one field. + """ + if not isinstance(data, dict): + return data + messages = data.get("messages") + if not isinstance(messages, list): + return data + for msg in messages: + if not isinstance(msg, dict): + continue + tool_calls = msg.get("tool_calls") + if tool_calls is not None and not isinstance(tool_calls, list): + msg["tool_calls"] = list(tool_calls) + reasoning_content = msg.pop("reasoning_content", None) + if reasoning_content is not None and msg.get("reasoning") is None: + msg["reasoning"] = reasoning_content + return data + + +def materialize_tool_calls_in_messages( + messages: Iterable[ChatCompletionMessageParam], +) -> None: + """Convert Pydantic ``ValidatorIterator`` wrappers back to plain lists. + + Even after :func:`normalize_chat_messages_before_validation` converts + iterables to lists, Pydantic re-wraps ``tool_calls`` in a lazy + ``ValidatorIterator`` when validating against the ``Iterable[...]`` type + of ``ChatCompletionAssistantMessageParam``. Such wrappers cannot be + pickled/deep-copied and are consumed on first iteration, so downstream + code (tokenizers, renderers, ``model_dump_json``) must always see plain + lists. Intended to be called from a ``mode="after"`` model validator. + """ + for msg in messages: + if not isinstance(msg, dict): + continue + tool_calls = msg.get("tool_calls") + if tool_calls is not None and not isinstance(tool_calls, list): + msg["tool_calls"] = list(tool_calls) + + # TODO: Make fields ReadOnly once mypy supports it class ConversationMessage(TypedDict, total=False): role: Required[str] diff --git a/vllm/entrypoints/openai/chat_completion/protocol.py b/vllm/entrypoints/openai/chat_completion/protocol.py index 225e99244141..605d78594435 100644 --- a/vllm/entrypoints/openai/chat_completion/protocol.py +++ b/vllm/entrypoints/openai/chat_completion/protocol.py @@ -22,6 +22,8 @@ from vllm.entrypoints.chat_utils import ( ChatCompletionMessageParam, ChatTemplateContentFormatOption, + materialize_tool_calls_in_messages, + normalize_chat_messages_before_validation, ) from vllm.entrypoints.generate.base.protocol import ( AnyResponseFormat, @@ -520,46 +522,11 @@ def check_cache_salt_support(cls, data: Any) -> Any: @model_validator(mode="before") @classmethod def _normalize_messages_before(cls, data: Any) -> Any: - """Pre-process message dicts before Pydantic field validation. - - Performs two normalizations in a single pass: - - Converts tool_calls generators/iterators to lists so one-shot - generators are not consumed during union type matching. - - Renames the deprecated ``reasoning_content`` field to - ``reasoning`` so downstream code only needs to check one field. - """ - if not isinstance(data, dict): - return data - messages = data.get("messages") - if not isinstance(messages, list): - return data - for msg in messages: - if not isinstance(msg, dict): - continue - tool_calls = msg.get("tool_calls") - if tool_calls is not None and not isinstance(tool_calls, list): - msg["tool_calls"] = list(tool_calls) - reasoning_content = msg.pop("reasoning_content", None) - if reasoning_content is not None and msg.get("reasoning") is None: - msg["reasoning"] = reasoning_content - return data + return normalize_chat_messages_before_validation(data) @model_validator(mode="after") def _materialize_tool_calls_after(self) -> "ChatCompletionRequest": - """Convert Pydantic ValidatorIterator wrappers back to lists. - - Even after the "before" validator converts iterables to lists, - Pydantic re-wraps them in a ValidatorIterator when validating - against ChatCompletionAssistantMessageParam's Iterable[...] type. - This "after" pass materialises those wrappers so downstream code - (tokenizers, model_dump_json) always sees plain lists. - """ - for msg in self.messages: - if not isinstance(msg, dict): - continue - tool_calls = msg.get("tool_calls") - if tool_calls is not None and not isinstance(tool_calls, list): - msg["tool_calls"] = list(tool_calls) + materialize_tool_calls_in_messages(self.messages) return self _grammar_from_parser: bool = PrivateAttr(default=False) diff --git a/vllm/entrypoints/pooling/base/protocol.py b/vllm/entrypoints/pooling/base/protocol.py index c1366734f7c1..6856e1e093bf 100644 --- a/vllm/entrypoints/pooling/base/protocol.py +++ b/vllm/entrypoints/pooling/base/protocol.py @@ -10,6 +10,8 @@ from vllm.entrypoints.chat_utils import ( ChatCompletionMessageParam, ChatTemplateContentFormatOption, + materialize_tool_calls_in_messages, + normalize_chat_messages_before_validation, ) from vllm.entrypoints.generate.base.protocol import validate_cache_salt from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel @@ -316,6 +318,16 @@ class ChatRequestMixin(ChatRequestOptionsMixin): messages: list[ChatCompletionMessageParam] # --8<-- [end:chat-params] + @model_validator(mode="before") + @classmethod + def _normalize_messages_before(cls, data: Any) -> Any: + return normalize_chat_messages_before_validation(data) + + @model_validator(mode="after") + def _materialize_tool_calls_after(self) -> "ChatRequestMixin": + materialize_tool_calls_in_messages(self.messages) + return self + class EncodingRequestMixin(OpenAIBaseModel): # --8<-- [start:encoding-params] diff --git a/vllm/entrypoints/serve/tokenize/protocol.py b/vllm/entrypoints/serve/tokenize/protocol.py index 130a1b4b45c4..6d909cb1ad44 100644 --- a/vllm/entrypoints/serve/tokenize/protocol.py +++ b/vllm/entrypoints/serve/tokenize/protocol.py @@ -10,6 +10,8 @@ from vllm.entrypoints.chat_utils import ( ChatCompletionMessageParam, ChatTemplateContentFormatOption, + materialize_tool_calls_in_messages, + normalize_chat_messages_before_validation, ) from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionToolsParam from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel @@ -125,6 +127,16 @@ def check_generation_prompt(cls, data): ) return data + @model_validator(mode="before") + @classmethod + def _normalize_messages_before(cls, data: Any) -> Any: + return normalize_chat_messages_before_validation(data) + + @model_validator(mode="after") + def _materialize_tool_calls_after(self) -> "TokenizeChatRequest": + materialize_tool_calls_in_messages(self.messages) + return self + def build_chat_params( self, default_template: str | None,