diff --git a/tests/entrypoints/openai/test_tool_choice_content_none.py b/tests/entrypoints/openai/test_tool_choice_content_none.py index c1da5918697c..75a5c578cca4 100644 --- a/tests/entrypoints/openai/test_tool_choice_content_none.py +++ b/tests/entrypoints/openai/test_tool_choice_content_none.py @@ -4,7 +4,6 @@ import pytest from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest -from vllm.entrypoints.openai.engine.serving import OpenAIServing from vllm.entrypoints.openai.responses.protocol import ResponsesRequest from vllm.parser.abstract_parser import DelegatingParser @@ -32,11 +31,8 @@ def extract_reasoning_streaming( ): return None - def extract_tool_calls(self, model_output: str, request): - return None - -def test_parse_tool_calls_from_content_allows_named_tool_choice_with_none_content(): +def test_chat_completion_named_tool_choice_with_none_content(): request = ChatCompletionRequest.model_validate( { "model": "test-model", @@ -53,17 +49,15 @@ def test_parse_tool_calls_from_content_allows_named_tool_choice_with_none_conten "tool_choice": {"type": "function", "function": {"name": "get_weather"}}, } ) + parser = _DummyDelegatingParser(tokenizer=None) - tool_calls, content = OpenAIServing._parse_tool_calls_from_content( + tool_calls, content = parser._extract_tool_calls( + content=None, request=request, - tokenizer=None, enable_auto_tools=True, - tool_parser_cls=None, - content=None, ) assert content is None - assert tool_calls is not None assert tool_calls == [] diff --git a/tests/parser/test_parse.py b/tests/parser/test_parse.py new file mode 100644 index 000000000000..bac8b64bcbd6 --- /dev/null +++ b/tests/parser/test_parse.py @@ -0,0 +1,262 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import json + +import pytest + +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest +from vllm.parser.abstract_parser import _WrappedParser +from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser +from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser + + +class ThinkReasoningParser(BaseThinkingReasoningParser): + @property + def start_token(self) -> str: + return "" + + @property + def end_token(self) -> str: + return "" + + +MODEL_OUTPUT = ( + "let me think about this" + '\n{"name": "get_weather", ' + '"arguments": {"city": "Dallas"}}\n' +) + +PLAIN_TEXT = "The weather in Dallas is sunny and 75°F." + +TOOL_CALL_ONLY = ( + '\n{"name": "get_weather", ' + '"arguments": {"city": "Dallas"}}\n' +) + +TOOL_ARGUMENTS = '{"city": "Dallas"}' + + +@pytest.fixture(scope="module") +def tokenizer(): + from vllm.tokenizers import get_tokenizer + + return get_tokenizer("Qwen/Qwen3-32B") + + +def make_request(**overrides): + base = { + "model": "test-model", + "messages": [{"role": "user", "content": "hi"}], + } + base.update(overrides) + return ChatCompletionRequest.model_validate(base) + + +TOOLS = [ + { + "type": "function", + "function": { + "name": "get_weather", + "parameters": {"type": "object", "properties": {}}, + }, + } +] + + +def make_parser(tokenizer, reasoning=False, tool=False): + _WrappedParser.reasoning_parser_cls = ThinkReasoningParser if reasoning else None + _WrappedParser.tool_parser_cls = Hermes2ProToolParser if tool else None + return _WrappedParser(tokenizer) + + +@pytest.mark.parametrize( + "reasoning,tool", + [(False, False), (False, True)], + ids=["neither", "tool-only"], +) +def test_parse_plain_text_no_reasoning_parser(tokenizer, reasoning, tool): + parser = make_parser(tokenizer, reasoning=reasoning, tool=tool) + request = make_request() + r, content, tool_calls = parser.parse(PLAIN_TEXT, request) + + assert r is None + assert content == PLAIN_TEXT + assert tool_calls is not None + assert len(tool_calls) == 0 + + +@pytest.mark.parametrize( + "reasoning,tool", + [(True, False), (True, True)], + ids=["reasoning-only", "both"], +) +def test_parse_plain_text_with_reasoning_parser(tokenizer, reasoning, tool): + parser = make_parser(tokenizer, reasoning=reasoning, tool=tool) + request = make_request() + r, content, tool_calls = parser.parse(PLAIN_TEXT, request) + + assert r == PLAIN_TEXT + assert content is None + assert tool_calls is not None + assert len(tool_calls) == 0 + + +def test_parse_both_parsers(tokenizer): + parser = make_parser(tokenizer, reasoning=True, tool=True) + request = make_request(tools=TOOLS) + reasoning, content, tool_calls = parser.parse( + MODEL_OUTPUT, request, enable_auto_tools=True + ) + + assert reasoning is not None + assert "let me think about this" in reasoning + assert tool_calls is not None + assert len(tool_calls) == 1 + assert tool_calls[0].name == "get_weather" + assert json.loads(tool_calls[0].arguments) == {"city": "Dallas"} + assert not content or content.strip() == "" + + +def test_parse_reasoning_only(tokenizer): + parser = make_parser(tokenizer, reasoning=True, tool=False) + request = make_request() + reasoning, content, tool_calls = parser.parse(MODEL_OUTPUT, request) + + assert reasoning is not None + assert "let me think about this" in reasoning + assert content is not None + assert "" in content + assert "get_weather" in content + assert tool_calls is not None + assert len(tool_calls) == 0 + + +def test_parse_tool_only(tokenizer): + parser = make_parser(tokenizer, reasoning=False, tool=True) + request = make_request(tools=TOOLS) + reasoning, content, tool_calls = parser.parse( + MODEL_OUTPUT, request, enable_auto_tools=True + ) + + assert reasoning is None + assert tool_calls is not None + assert len(tool_calls) == 1 + assert tool_calls[0].name == "get_weather" + assert json.loads(tool_calls[0].arguments) == {"city": "Dallas"} + + +def test_parse_named_tool_choice(tokenizer): + parser = make_parser(tokenizer, reasoning=False, tool=True) + request = make_request( + tools=TOOLS, + tool_choice={ + "type": "function", + "function": {"name": "get_weather"}, + }, + ) + reasoning, content, tool_calls = parser.parse( + TOOL_ARGUMENTS, request, enable_auto_tools=True + ) + + assert reasoning is None + assert content is None + assert tool_calls is not None + assert len(tool_calls) == 1 + assert tool_calls[0].name == "get_weather" + assert tool_calls[0].arguments == TOOL_ARGUMENTS + + +def test_parse_named_tool_choice_with_reasoning(tokenizer): + parser = make_parser(tokenizer, reasoning=True, tool=True) + model_output = f"thinking{TOOL_ARGUMENTS}" + request = make_request( + tools=TOOLS, + tool_choice={ + "type": "function", + "function": {"name": "get_weather"}, + }, + ) + reasoning, content, tool_calls = parser.parse( + model_output, request, enable_auto_tools=True + ) + + assert reasoning is not None + assert "thinking" in reasoning + assert content is None + assert len(tool_calls) == 1 + assert tool_calls[0].name == "get_weather" + assert tool_calls[0].arguments == TOOL_ARGUMENTS + + +def test_parse_required_tool_choice(tokenizer): + parser = make_parser(tokenizer, reasoning=False, tool=True) + functions_json = json.dumps( + [ + {"name": "get_weather", "parameters": {"city": "Dallas"}}, + {"name": "get_time", "parameters": {"timezone": "UTC"}}, + ] + ) + request = make_request(tools=TOOLS, tool_choice="required") + reasoning, content, tool_calls = parser.parse( + functions_json, request, enable_auto_tools=True + ) + + assert reasoning is None + assert content is None + assert tool_calls is not None + assert len(tool_calls) == 2 + assert tool_calls[0].name == "get_weather" + assert json.loads(tool_calls[0].arguments) == {"city": "Dallas"} + assert tool_calls[1].name == "get_time" + assert json.loads(tool_calls[1].arguments) == {"timezone": "UTC"} + + +def test_parse_named_tool_choice_content_none(tokenizer): + parser = make_parser(tokenizer, reasoning=False, tool=True) + request = make_request( + tools=TOOLS, + tool_choice={ + "type": "function", + "function": {"name": "get_weather"}, + }, + ) + reasoning, content, tool_calls = parser.parse("", request, enable_auto_tools=True) + assert reasoning is None + assert content is None + assert tool_calls is not None + + +def test_parse_required_tool_choice_content_none(tokenizer): + parser = make_parser(tokenizer, reasoning=False, tool=True) + request = make_request(tools=TOOLS, tool_choice="required") + reasoning, content, tool_calls = parser.parse("", request, enable_auto_tools=True) + assert reasoning is None + assert content is None + assert tool_calls is not None + assert len(tool_calls) == 0 + + +def test_parse_auto_tools_no_parser(tokenizer): + parser = make_parser(tokenizer, reasoning=False, tool=False) + request = make_request() + reasoning, content, tool_calls = parser.parse( + TOOL_CALL_ONLY, request, enable_auto_tools=True + ) + + assert reasoning is None + assert content == TOOL_CALL_ONLY + assert tool_calls is not None + assert len(tool_calls) == 0 + + +def test_parse_auto_tools_no_calls_returns_none(tokenizer): + parser = make_parser(tokenizer, reasoning=False, tool=True) + request = make_request(tools=TOOLS) + reasoning, content, tool_calls = parser.parse( + PLAIN_TEXT, request, enable_auto_tools=True + ) + + assert reasoning is None + assert content == PLAIN_TEXT + assert tool_calls is None diff --git a/vllm/entrypoints/openai/chat_completion/serving.py b/vllm/entrypoints/openai/chat_completion/serving.py index 412583f8b65a..a378fb79d3bc 100644 --- a/vllm/entrypoints/openai/chat_completion/serving.py +++ b/vllm/entrypoints/openai/chat_completion/serving.py @@ -357,6 +357,14 @@ async def _create_chat_completion( assert len(generators) == 1 (result_generator,) = generators + parser: Parser | None = None + if self.parser_cls is not None: + parser = self.parser_cls( + tokenizer, + request.tools, + chat_template_kwargs=chat_template_kwargs, + ) + if request.stream: return self.chat_completion_stream_generator( request, @@ -378,7 +386,7 @@ async def _create_chat_completion( conversation, tokenizer, request_metadata, - reasoning_parser, + parser, ) def get_chat_request_role(self, request: ChatCompletionRequest) -> str: @@ -934,7 +942,7 @@ async def chat_completion_full_generator( conversation: list[ConversationMessage], tokenizer: TokenizerLike, request_metadata: RequestResponseMetadata, - reasoning_parser: ReasoningParser | None = None, + parser: Parser | None = None, ) -> ErrorResponse | ChatCompletionResponse: created_time = int(time.time()) final_res: RequestOutput | None = None @@ -1043,28 +1051,20 @@ async def chat_completion_full_generator( choices.append(choice_data) continue - if reasoning_parser: - # If the reasoning parser is enabled, - # tool calls are extracted exclusively from the content. - reasoning, content = reasoning_parser.extract_reasoning( - output.text, request=request + if parser is not None: + reasoning, content, tool_calls = parser.parse( + output.text, + request, + enable_auto_tools=self.enable_auto_tools, ) if not request.include_reasoning: reasoning = None else: reasoning = None content = output.text + tool_calls = [] auto_tools_called = False - # if auto tools are not enabled, and a named tool choice using - # outlines is not being used - tool_calls, content = self._parse_tool_calls_from_content( - request=request, - tokenizer=tokenizer, - content=content, - enable_auto_tools=self.enable_auto_tools, - tool_parser_cls=self.tool_parser, - ) if is_mistral_tokenizer(tokenizer): from vllm.tool_parsers.mistral_tool_parser import MistralToolCall diff --git a/vllm/entrypoints/openai/engine/serving.py b/vllm/entrypoints/openai/engine/serving.py index ff67575fcc6c..61b2656bac0f 100644 --- a/vllm/entrypoints/openai/engine/serving.py +++ b/vllm/entrypoints/openai/engine/serving.py @@ -1,6 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import contextlib import json import time from collections.abc import Awaitable, Mapping @@ -9,8 +8,7 @@ from typing import Any, ClassVar, Generic, Protocol, TypeAlias, TypeVar from fastapi import Request -from openai.types.responses import ToolChoiceFunction -from pydantic import ConfigDict, TypeAdapter, ValidationError +from pydantic import ConfigDict from starlette.datastructures import Headers import vllm.envs as envs @@ -21,7 +19,6 @@ from vllm.entrypoints.logger import RequestLogger from vllm.entrypoints.openai.chat_completion.protocol import ( BatchChatCompletionRequest, - ChatCompletionNamedToolChoiceParam, ChatCompletionRequest, ChatCompletionResponse, ) @@ -31,8 +28,6 @@ ) from vllm.entrypoints.openai.engine.protocol import ( ErrorResponse, - FunctionCall, - FunctionDefinition, GenerationError, ) from vllm.entrypoints.openai.models.serving import OpenAIServingModels @@ -61,14 +56,12 @@ ) from vllm.sampling_params import BeamSearchParams, SamplingParams from vllm.tokenizers import TokenizerLike -from vllm.tool_parsers import ToolParser from vllm.tracing import ( contains_trace_headers, extract_trace_headers, log_tracing_disabled_warning, ) from vllm.utils import random_uuid -from vllm.utils.mistral import is_mistral_tool_parser logger = init_logger(__name__) @@ -451,124 +444,6 @@ async def _with_kv_transfer_rejection_cleanup( exc_info=True, ) - @staticmethod - def _parse_tool_calls_from_content( - request: ResponsesRequest | ChatCompletionRequest, - tokenizer: TokenizerLike | None, - enable_auto_tools: bool, - tool_parser_cls: type[ToolParser] | None, - content: str | None = None, - ) -> tuple[list[FunctionCall] | None, str | None]: - # When the Mistral grammar factory injected structured outputs, - # let the parser handle the output. - use_mistral_tool_parser = ( - isinstance(request, ChatCompletionRequest) - and is_mistral_tool_parser(tool_parser_cls) - and request._grammar_from_tool_parser - ) - - function_calls = list[FunctionCall]() - if ( - not use_mistral_tool_parser - and request.tool_choice - and isinstance(request.tool_choice, ToolChoiceFunction) - ): - # Forced Function Call (Responses API) - if content is None: - return [], None - function_calls.append( - FunctionCall(name=request.tool_choice.name, arguments=content) - ) - content = None # Clear content since tool is called. - elif ( - not use_mistral_tool_parser - and request.tool_choice - and isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam) - and (tool_parser_cls is None or tool_parser_cls.supports_required_and_named) - ): - # Named function with standard JSON-based parsing - if content is None: - return [], None - function_calls.append( - FunctionCall(name=request.tool_choice.function.name, arguments=content) - ) - content = None # Clear content since tool is called. - elif ( - not use_mistral_tool_parser - and request.tool_choice == "required" - and (tool_parser_cls is None or tool_parser_cls.supports_required_and_named) - ): - # "required" with standard JSON-based parsing - tool_calls = [] - with contextlib.suppress(ValidationError): - content = content or "" - tool_calls = TypeAdapter(list[FunctionDefinition]).validate_json( - content - ) - for tool_call in tool_calls: - function_calls.append( - FunctionCall( - name=tool_call.name, - arguments=json.dumps(tool_call.parameters, ensure_ascii=False), - ) - ) - content = None # Clear content since tool is called. - elif tool_parser_cls and ( - use_mistral_tool_parser - or ( - enable_auto_tools - and ( - request.tool_choice == "auto" - or request.tool_choice is None - or ( - not tool_parser_cls.supports_required_and_named - and request.tools - and ( - request.tool_choice == "required" - or isinstance( - request.tool_choice, - ChatCompletionNamedToolChoiceParam, - ) - ) - ) - ) - ) - ): - # Automatic Tool Call Parsing (also used as fallback for - # required/named when supports_required_and_named=False) - if tokenizer is None: - raise ValueError( - "Tokenizer not available when `skip_tokenizer_init=True`" - ) - - try: - tool_parser = tool_parser_cls(tokenizer, request.tools) - except RuntimeError as e: - logger.exception("Error in tool parser creation.") - raise e - tool_call_info = tool_parser.extract_tool_calls( - content if content is not None else "", - request=request, # type: ignore - ) - if tool_call_info is not None and tool_call_info.tools_called: - # extract_tool_calls() returns a list of tool calls. - function_calls.extend( - FunctionCall( - id=tool_call.id, - name=tool_call.function.name, - arguments=tool_call.function.arguments, - ) - for tool_call in tool_call_info.tool_calls - ) - content = tool_call_info.content - if content and content.strip() == "": - content = None - else: - # No tool calls. - return None, content - - return function_calls, content - @staticmethod def _get_decoded_token( logprob: Logprob, diff --git a/vllm/parser/abstract_parser.py b/vllm/parser/abstract_parser.py index 96c04805bcda..6a7193413f7f 100644 --- a/vllm/parser/abstract_parser.py +++ b/vllm/parser/abstract_parser.py @@ -44,6 +44,7 @@ ) from vllm.tool_parsers.utils import Tool from vllm.utils import random_uuid +from vllm.utils.mistral import is_mistral_tool_parser logger = init_logger(__name__) @@ -313,6 +314,24 @@ def extract_tool_calls_streaming( A DeltaMessage with tool_calls field, or None. """ + @abstractmethod + def parse( + self, + model_output: str, + request: ChatCompletionRequest | ResponsesRequest, + enable_auto_tools: bool = False, + ) -> tuple[str | None, str | None, list[FunctionCall] | None]: + """Parse a complete model output, extracting reasoning and tool calls. + + Args: + model_output: The complete model-generated string. + request: The request object used to generate the output. + enable_auto_tools: Whether to enable automatic tool call parsing. + + Returns: + A tuple of (reasoning, content, tool_calls). + """ + @abstractmethod def parse_delta( self, @@ -511,6 +530,99 @@ def _parse_tool_calls( # No tool calls return [], content + def _extract_tool_calls( + self, + content: str | None, + request: ChatCompletionRequest | ResponsesRequest, + enable_auto_tools: bool = False, + ) -> tuple[list[FunctionCall] | None, str | None]: + tool_parser = self._tool_parser + if tool_parser is None: + return [], content + + # When the Mistral grammar factory injected structured outputs, + # let the parser handle the output. + use_mistral_tool_parser = ( + is_mistral_tool_parser(type(tool_parser)) + and isinstance(request, ChatCompletionRequest) + and request._grammar_from_tool_parser + ) + + supports_required_and_named = tool_parser.supports_required_and_named + is_named_tool_choice = request.tool_choice and isinstance( + request.tool_choice, + (ToolChoiceFunction, ChatCompletionNamedToolChoiceParam), + ) + is_required_tool_choice = request.tool_choice == "required" + is_auto_tool_choice = enable_auto_tools and ( + request.tool_choice == "auto" + or request.tool_choice is None + or ( + not supports_required_and_named + and (is_named_tool_choice or is_required_tool_choice) + ) + ) + + tool_calls = list[FunctionCall]() + if ( + is_named_tool_choice + and supports_required_and_named + and not use_mistral_tool_parser + ): + if content is None: + return [], None + tool_calls.append( + FunctionCall( + name=self._get_function_name(request), + arguments=content, + ) + ) + content = None + elif ( + is_required_tool_choice + and supports_required_and_named + and not use_mistral_tool_parser + ): + # "required" with standard JSON-based parsing + parsed_calls = [] + with contextlib.suppress(ValidationError): + content = content or "" + parsed_calls = TypeAdapter(list[FunctionDefinition]).validate_json( + content + ) + for tc in parsed_calls: + tool_calls.append( + FunctionCall( + name=tc.name, + arguments=json.dumps(tc.parameters, ensure_ascii=False), + ) + ) + content = None + elif is_auto_tool_choice or use_mistral_tool_parser: + # Automatic Tool Call Parsing (also used as fallback for + # required/named when supports_required_and_named=False) + tool_call_info = tool_parser.extract_tool_calls( + content if content is not None else "", + request=request, # type: ignore + ) + if tool_call_info is not None and tool_call_info.tools_called: + tool_calls.extend( + FunctionCall( + id=tc.id, + name=tc.function.name, + arguments=tc.function.arguments, + ) + for tc in tool_call_info.tool_calls + ) + content = tool_call_info.content + if content and content.strip() == "": + content = None + else: + # No tool calls. + return None, content + + return tool_calls, content + def adjust_request( self, request: ChatCompletionRequest | ResponsesRequest ) -> ChatCompletionRequest | ResponsesRequest: @@ -672,6 +784,20 @@ def _append_unstreamed_tool_args( last_tc.function.arguments or "" ) + self._tool_parser.get_remaining_unstreamed_args() + def parse( + self, + model_output: str, + request: ChatCompletionRequest | ResponsesRequest, + enable_auto_tools: bool = False, + ) -> tuple[str | None, str | None, list[FunctionCall] | None]: + reasoning, content = self.extract_reasoning(model_output, request) + tool_calls, content = self._extract_tool_calls( + content=content, + request=request, + enable_auto_tools=enable_auto_tools, + ) + return reasoning, content, tool_calls + def parse_delta( self, delta_text: str,