diff --git a/tests/entrypoints/openai/test_tool_choice_content_none.py b/tests/entrypoints/openai/test_tool_choice_content_none.py
index c1da5918697c..75a5c578cca4 100644
--- a/tests/entrypoints/openai/test_tool_choice_content_none.py
+++ b/tests/entrypoints/openai/test_tool_choice_content_none.py
@@ -4,7 +4,6 @@
import pytest
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
-from vllm.entrypoints.openai.engine.serving import OpenAIServing
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.parser.abstract_parser import DelegatingParser
@@ -32,11 +31,8 @@ def extract_reasoning_streaming(
):
return None
- def extract_tool_calls(self, model_output: str, request):
- return None
-
-def test_parse_tool_calls_from_content_allows_named_tool_choice_with_none_content():
+def test_chat_completion_named_tool_choice_with_none_content():
request = ChatCompletionRequest.model_validate(
{
"model": "test-model",
@@ -53,17 +49,15 @@ def test_parse_tool_calls_from_content_allows_named_tool_choice_with_none_conten
"tool_choice": {"type": "function", "function": {"name": "get_weather"}},
}
)
+ parser = _DummyDelegatingParser(tokenizer=None)
- tool_calls, content = OpenAIServing._parse_tool_calls_from_content(
+ tool_calls, content = parser._extract_tool_calls(
+ content=None,
request=request,
- tokenizer=None,
enable_auto_tools=True,
- tool_parser_cls=None,
- content=None,
)
assert content is None
- assert tool_calls is not None
assert tool_calls == []
diff --git a/tests/parser/test_parse.py b/tests/parser/test_parse.py
new file mode 100644
index 000000000000..bac8b64bcbd6
--- /dev/null
+++ b/tests/parser/test_parse.py
@@ -0,0 +1,262 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import json
+
+import pytest
+
+from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
+from vllm.parser.abstract_parser import _WrappedParser
+from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
+from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser
+
+
+class ThinkReasoningParser(BaseThinkingReasoningParser):
+ @property
+ def start_token(self) -> str:
+ return ""
+
+ @property
+ def end_token(self) -> str:
+ return ""
+
+
+MODEL_OUTPUT = (
+ "let me think about this"
+ '\n{"name": "get_weather", '
+ '"arguments": {"city": "Dallas"}}\n'
+)
+
+PLAIN_TEXT = "The weather in Dallas is sunny and 75°F."
+
+TOOL_CALL_ONLY = (
+ '\n{"name": "get_weather", '
+ '"arguments": {"city": "Dallas"}}\n'
+)
+
+TOOL_ARGUMENTS = '{"city": "Dallas"}'
+
+
+@pytest.fixture(scope="module")
+def tokenizer():
+ from vllm.tokenizers import get_tokenizer
+
+ return get_tokenizer("Qwen/Qwen3-32B")
+
+
+def make_request(**overrides):
+ base = {
+ "model": "test-model",
+ "messages": [{"role": "user", "content": "hi"}],
+ }
+ base.update(overrides)
+ return ChatCompletionRequest.model_validate(base)
+
+
+TOOLS = [
+ {
+ "type": "function",
+ "function": {
+ "name": "get_weather",
+ "parameters": {"type": "object", "properties": {}},
+ },
+ }
+]
+
+
+def make_parser(tokenizer, reasoning=False, tool=False):
+ _WrappedParser.reasoning_parser_cls = ThinkReasoningParser if reasoning else None
+ _WrappedParser.tool_parser_cls = Hermes2ProToolParser if tool else None
+ return _WrappedParser(tokenizer)
+
+
+@pytest.mark.parametrize(
+ "reasoning,tool",
+ [(False, False), (False, True)],
+ ids=["neither", "tool-only"],
+)
+def test_parse_plain_text_no_reasoning_parser(tokenizer, reasoning, tool):
+ parser = make_parser(tokenizer, reasoning=reasoning, tool=tool)
+ request = make_request()
+ r, content, tool_calls = parser.parse(PLAIN_TEXT, request)
+
+ assert r is None
+ assert content == PLAIN_TEXT
+ assert tool_calls is not None
+ assert len(tool_calls) == 0
+
+
+@pytest.mark.parametrize(
+ "reasoning,tool",
+ [(True, False), (True, True)],
+ ids=["reasoning-only", "both"],
+)
+def test_parse_plain_text_with_reasoning_parser(tokenizer, reasoning, tool):
+ parser = make_parser(tokenizer, reasoning=reasoning, tool=tool)
+ request = make_request()
+ r, content, tool_calls = parser.parse(PLAIN_TEXT, request)
+
+ assert r == PLAIN_TEXT
+ assert content is None
+ assert tool_calls is not None
+ assert len(tool_calls) == 0
+
+
+def test_parse_both_parsers(tokenizer):
+ parser = make_parser(tokenizer, reasoning=True, tool=True)
+ request = make_request(tools=TOOLS)
+ reasoning, content, tool_calls = parser.parse(
+ MODEL_OUTPUT, request, enable_auto_tools=True
+ )
+
+ assert reasoning is not None
+ assert "let me think about this" in reasoning
+ assert tool_calls is not None
+ assert len(tool_calls) == 1
+ assert tool_calls[0].name == "get_weather"
+ assert json.loads(tool_calls[0].arguments) == {"city": "Dallas"}
+ assert not content or content.strip() == ""
+
+
+def test_parse_reasoning_only(tokenizer):
+ parser = make_parser(tokenizer, reasoning=True, tool=False)
+ request = make_request()
+ reasoning, content, tool_calls = parser.parse(MODEL_OUTPUT, request)
+
+ assert reasoning is not None
+ assert "let me think about this" in reasoning
+ assert content is not None
+ assert "" in content
+ assert "get_weather" in content
+ assert tool_calls is not None
+ assert len(tool_calls) == 0
+
+
+def test_parse_tool_only(tokenizer):
+ parser = make_parser(tokenizer, reasoning=False, tool=True)
+ request = make_request(tools=TOOLS)
+ reasoning, content, tool_calls = parser.parse(
+ MODEL_OUTPUT, request, enable_auto_tools=True
+ )
+
+ assert reasoning is None
+ assert tool_calls is not None
+ assert len(tool_calls) == 1
+ assert tool_calls[0].name == "get_weather"
+ assert json.loads(tool_calls[0].arguments) == {"city": "Dallas"}
+
+
+def test_parse_named_tool_choice(tokenizer):
+ parser = make_parser(tokenizer, reasoning=False, tool=True)
+ request = make_request(
+ tools=TOOLS,
+ tool_choice={
+ "type": "function",
+ "function": {"name": "get_weather"},
+ },
+ )
+ reasoning, content, tool_calls = parser.parse(
+ TOOL_ARGUMENTS, request, enable_auto_tools=True
+ )
+
+ assert reasoning is None
+ assert content is None
+ assert tool_calls is not None
+ assert len(tool_calls) == 1
+ assert tool_calls[0].name == "get_weather"
+ assert tool_calls[0].arguments == TOOL_ARGUMENTS
+
+
+def test_parse_named_tool_choice_with_reasoning(tokenizer):
+ parser = make_parser(tokenizer, reasoning=True, tool=True)
+ model_output = f"thinking{TOOL_ARGUMENTS}"
+ request = make_request(
+ tools=TOOLS,
+ tool_choice={
+ "type": "function",
+ "function": {"name": "get_weather"},
+ },
+ )
+ reasoning, content, tool_calls = parser.parse(
+ model_output, request, enable_auto_tools=True
+ )
+
+ assert reasoning is not None
+ assert "thinking" in reasoning
+ assert content is None
+ assert len(tool_calls) == 1
+ assert tool_calls[0].name == "get_weather"
+ assert tool_calls[0].arguments == TOOL_ARGUMENTS
+
+
+def test_parse_required_tool_choice(tokenizer):
+ parser = make_parser(tokenizer, reasoning=False, tool=True)
+ functions_json = json.dumps(
+ [
+ {"name": "get_weather", "parameters": {"city": "Dallas"}},
+ {"name": "get_time", "parameters": {"timezone": "UTC"}},
+ ]
+ )
+ request = make_request(tools=TOOLS, tool_choice="required")
+ reasoning, content, tool_calls = parser.parse(
+ functions_json, request, enable_auto_tools=True
+ )
+
+ assert reasoning is None
+ assert content is None
+ assert tool_calls is not None
+ assert len(tool_calls) == 2
+ assert tool_calls[0].name == "get_weather"
+ assert json.loads(tool_calls[0].arguments) == {"city": "Dallas"}
+ assert tool_calls[1].name == "get_time"
+ assert json.loads(tool_calls[1].arguments) == {"timezone": "UTC"}
+
+
+def test_parse_named_tool_choice_content_none(tokenizer):
+ parser = make_parser(tokenizer, reasoning=False, tool=True)
+ request = make_request(
+ tools=TOOLS,
+ tool_choice={
+ "type": "function",
+ "function": {"name": "get_weather"},
+ },
+ )
+ reasoning, content, tool_calls = parser.parse("", request, enable_auto_tools=True)
+ assert reasoning is None
+ assert content is None
+ assert tool_calls is not None
+
+
+def test_parse_required_tool_choice_content_none(tokenizer):
+ parser = make_parser(tokenizer, reasoning=False, tool=True)
+ request = make_request(tools=TOOLS, tool_choice="required")
+ reasoning, content, tool_calls = parser.parse("", request, enable_auto_tools=True)
+ assert reasoning is None
+ assert content is None
+ assert tool_calls is not None
+ assert len(tool_calls) == 0
+
+
+def test_parse_auto_tools_no_parser(tokenizer):
+ parser = make_parser(tokenizer, reasoning=False, tool=False)
+ request = make_request()
+ reasoning, content, tool_calls = parser.parse(
+ TOOL_CALL_ONLY, request, enable_auto_tools=True
+ )
+
+ assert reasoning is None
+ assert content == TOOL_CALL_ONLY
+ assert tool_calls is not None
+ assert len(tool_calls) == 0
+
+
+def test_parse_auto_tools_no_calls_returns_none(tokenizer):
+ parser = make_parser(tokenizer, reasoning=False, tool=True)
+ request = make_request(tools=TOOLS)
+ reasoning, content, tool_calls = parser.parse(
+ PLAIN_TEXT, request, enable_auto_tools=True
+ )
+
+ assert reasoning is None
+ assert content == PLAIN_TEXT
+ assert tool_calls is None
diff --git a/vllm/entrypoints/openai/chat_completion/serving.py b/vllm/entrypoints/openai/chat_completion/serving.py
index 412583f8b65a..a378fb79d3bc 100644
--- a/vllm/entrypoints/openai/chat_completion/serving.py
+++ b/vllm/entrypoints/openai/chat_completion/serving.py
@@ -357,6 +357,14 @@ async def _create_chat_completion(
assert len(generators) == 1
(result_generator,) = generators
+ parser: Parser | None = None
+ if self.parser_cls is not None:
+ parser = self.parser_cls(
+ tokenizer,
+ request.tools,
+ chat_template_kwargs=chat_template_kwargs,
+ )
+
if request.stream:
return self.chat_completion_stream_generator(
request,
@@ -378,7 +386,7 @@ async def _create_chat_completion(
conversation,
tokenizer,
request_metadata,
- reasoning_parser,
+ parser,
)
def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
@@ -934,7 +942,7 @@ async def chat_completion_full_generator(
conversation: list[ConversationMessage],
tokenizer: TokenizerLike,
request_metadata: RequestResponseMetadata,
- reasoning_parser: ReasoningParser | None = None,
+ parser: Parser | None = None,
) -> ErrorResponse | ChatCompletionResponse:
created_time = int(time.time())
final_res: RequestOutput | None = None
@@ -1043,28 +1051,20 @@ async def chat_completion_full_generator(
choices.append(choice_data)
continue
- if reasoning_parser:
- # If the reasoning parser is enabled,
- # tool calls are extracted exclusively from the content.
- reasoning, content = reasoning_parser.extract_reasoning(
- output.text, request=request
+ if parser is not None:
+ reasoning, content, tool_calls = parser.parse(
+ output.text,
+ request,
+ enable_auto_tools=self.enable_auto_tools,
)
if not request.include_reasoning:
reasoning = None
else:
reasoning = None
content = output.text
+ tool_calls = []
auto_tools_called = False
- # if auto tools are not enabled, and a named tool choice using
- # outlines is not being used
- tool_calls, content = self._parse_tool_calls_from_content(
- request=request,
- tokenizer=tokenizer,
- content=content,
- enable_auto_tools=self.enable_auto_tools,
- tool_parser_cls=self.tool_parser,
- )
if is_mistral_tokenizer(tokenizer):
from vllm.tool_parsers.mistral_tool_parser import MistralToolCall
diff --git a/vllm/entrypoints/openai/engine/serving.py b/vllm/entrypoints/openai/engine/serving.py
index ff67575fcc6c..61b2656bac0f 100644
--- a/vllm/entrypoints/openai/engine/serving.py
+++ b/vllm/entrypoints/openai/engine/serving.py
@@ -1,6 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-import contextlib
import json
import time
from collections.abc import Awaitable, Mapping
@@ -9,8 +8,7 @@
from typing import Any, ClassVar, Generic, Protocol, TypeAlias, TypeVar
from fastapi import Request
-from openai.types.responses import ToolChoiceFunction
-from pydantic import ConfigDict, TypeAdapter, ValidationError
+from pydantic import ConfigDict
from starlette.datastructures import Headers
import vllm.envs as envs
@@ -21,7 +19,6 @@
from vllm.entrypoints.logger import RequestLogger
from vllm.entrypoints.openai.chat_completion.protocol import (
BatchChatCompletionRequest,
- ChatCompletionNamedToolChoiceParam,
ChatCompletionRequest,
ChatCompletionResponse,
)
@@ -31,8 +28,6 @@
)
from vllm.entrypoints.openai.engine.protocol import (
ErrorResponse,
- FunctionCall,
- FunctionDefinition,
GenerationError,
)
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
@@ -61,14 +56,12 @@
)
from vllm.sampling_params import BeamSearchParams, SamplingParams
from vllm.tokenizers import TokenizerLike
-from vllm.tool_parsers import ToolParser
from vllm.tracing import (
contains_trace_headers,
extract_trace_headers,
log_tracing_disabled_warning,
)
from vllm.utils import random_uuid
-from vllm.utils.mistral import is_mistral_tool_parser
logger = init_logger(__name__)
@@ -451,124 +444,6 @@ async def _with_kv_transfer_rejection_cleanup(
exc_info=True,
)
- @staticmethod
- def _parse_tool_calls_from_content(
- request: ResponsesRequest | ChatCompletionRequest,
- tokenizer: TokenizerLike | None,
- enable_auto_tools: bool,
- tool_parser_cls: type[ToolParser] | None,
- content: str | None = None,
- ) -> tuple[list[FunctionCall] | None, str | None]:
- # When the Mistral grammar factory injected structured outputs,
- # let the parser handle the output.
- use_mistral_tool_parser = (
- isinstance(request, ChatCompletionRequest)
- and is_mistral_tool_parser(tool_parser_cls)
- and request._grammar_from_tool_parser
- )
-
- function_calls = list[FunctionCall]()
- if (
- not use_mistral_tool_parser
- and request.tool_choice
- and isinstance(request.tool_choice, ToolChoiceFunction)
- ):
- # Forced Function Call (Responses API)
- if content is None:
- return [], None
- function_calls.append(
- FunctionCall(name=request.tool_choice.name, arguments=content)
- )
- content = None # Clear content since tool is called.
- elif (
- not use_mistral_tool_parser
- and request.tool_choice
- and isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)
- and (tool_parser_cls is None or tool_parser_cls.supports_required_and_named)
- ):
- # Named function with standard JSON-based parsing
- if content is None:
- return [], None
- function_calls.append(
- FunctionCall(name=request.tool_choice.function.name, arguments=content)
- )
- content = None # Clear content since tool is called.
- elif (
- not use_mistral_tool_parser
- and request.tool_choice == "required"
- and (tool_parser_cls is None or tool_parser_cls.supports_required_and_named)
- ):
- # "required" with standard JSON-based parsing
- tool_calls = []
- with contextlib.suppress(ValidationError):
- content = content or ""
- tool_calls = TypeAdapter(list[FunctionDefinition]).validate_json(
- content
- )
- for tool_call in tool_calls:
- function_calls.append(
- FunctionCall(
- name=tool_call.name,
- arguments=json.dumps(tool_call.parameters, ensure_ascii=False),
- )
- )
- content = None # Clear content since tool is called.
- elif tool_parser_cls and (
- use_mistral_tool_parser
- or (
- enable_auto_tools
- and (
- request.tool_choice == "auto"
- or request.tool_choice is None
- or (
- not tool_parser_cls.supports_required_and_named
- and request.tools
- and (
- request.tool_choice == "required"
- or isinstance(
- request.tool_choice,
- ChatCompletionNamedToolChoiceParam,
- )
- )
- )
- )
- )
- ):
- # Automatic Tool Call Parsing (also used as fallback for
- # required/named when supports_required_and_named=False)
- if tokenizer is None:
- raise ValueError(
- "Tokenizer not available when `skip_tokenizer_init=True`"
- )
-
- try:
- tool_parser = tool_parser_cls(tokenizer, request.tools)
- except RuntimeError as e:
- logger.exception("Error in tool parser creation.")
- raise e
- tool_call_info = tool_parser.extract_tool_calls(
- content if content is not None else "",
- request=request, # type: ignore
- )
- if tool_call_info is not None and tool_call_info.tools_called:
- # extract_tool_calls() returns a list of tool calls.
- function_calls.extend(
- FunctionCall(
- id=tool_call.id,
- name=tool_call.function.name,
- arguments=tool_call.function.arguments,
- )
- for tool_call in tool_call_info.tool_calls
- )
- content = tool_call_info.content
- if content and content.strip() == "":
- content = None
- else:
- # No tool calls.
- return None, content
-
- return function_calls, content
-
@staticmethod
def _get_decoded_token(
logprob: Logprob,
diff --git a/vllm/parser/abstract_parser.py b/vllm/parser/abstract_parser.py
index 96c04805bcda..6a7193413f7f 100644
--- a/vllm/parser/abstract_parser.py
+++ b/vllm/parser/abstract_parser.py
@@ -44,6 +44,7 @@
)
from vllm.tool_parsers.utils import Tool
from vllm.utils import random_uuid
+from vllm.utils.mistral import is_mistral_tool_parser
logger = init_logger(__name__)
@@ -313,6 +314,24 @@ def extract_tool_calls_streaming(
A DeltaMessage with tool_calls field, or None.
"""
+ @abstractmethod
+ def parse(
+ self,
+ model_output: str,
+ request: ChatCompletionRequest | ResponsesRequest,
+ enable_auto_tools: bool = False,
+ ) -> tuple[str | None, str | None, list[FunctionCall] | None]:
+ """Parse a complete model output, extracting reasoning and tool calls.
+
+ Args:
+ model_output: The complete model-generated string.
+ request: The request object used to generate the output.
+ enable_auto_tools: Whether to enable automatic tool call parsing.
+
+ Returns:
+ A tuple of (reasoning, content, tool_calls).
+ """
+
@abstractmethod
def parse_delta(
self,
@@ -511,6 +530,99 @@ def _parse_tool_calls(
# No tool calls
return [], content
+ def _extract_tool_calls(
+ self,
+ content: str | None,
+ request: ChatCompletionRequest | ResponsesRequest,
+ enable_auto_tools: bool = False,
+ ) -> tuple[list[FunctionCall] | None, str | None]:
+ tool_parser = self._tool_parser
+ if tool_parser is None:
+ return [], content
+
+ # When the Mistral grammar factory injected structured outputs,
+ # let the parser handle the output.
+ use_mistral_tool_parser = (
+ is_mistral_tool_parser(type(tool_parser))
+ and isinstance(request, ChatCompletionRequest)
+ and request._grammar_from_tool_parser
+ )
+
+ supports_required_and_named = tool_parser.supports_required_and_named
+ is_named_tool_choice = request.tool_choice and isinstance(
+ request.tool_choice,
+ (ToolChoiceFunction, ChatCompletionNamedToolChoiceParam),
+ )
+ is_required_tool_choice = request.tool_choice == "required"
+ is_auto_tool_choice = enable_auto_tools and (
+ request.tool_choice == "auto"
+ or request.tool_choice is None
+ or (
+ not supports_required_and_named
+ and (is_named_tool_choice or is_required_tool_choice)
+ )
+ )
+
+ tool_calls = list[FunctionCall]()
+ if (
+ is_named_tool_choice
+ and supports_required_and_named
+ and not use_mistral_tool_parser
+ ):
+ if content is None:
+ return [], None
+ tool_calls.append(
+ FunctionCall(
+ name=self._get_function_name(request),
+ arguments=content,
+ )
+ )
+ content = None
+ elif (
+ is_required_tool_choice
+ and supports_required_and_named
+ and not use_mistral_tool_parser
+ ):
+ # "required" with standard JSON-based parsing
+ parsed_calls = []
+ with contextlib.suppress(ValidationError):
+ content = content or ""
+ parsed_calls = TypeAdapter(list[FunctionDefinition]).validate_json(
+ content
+ )
+ for tc in parsed_calls:
+ tool_calls.append(
+ FunctionCall(
+ name=tc.name,
+ arguments=json.dumps(tc.parameters, ensure_ascii=False),
+ )
+ )
+ content = None
+ elif is_auto_tool_choice or use_mistral_tool_parser:
+ # Automatic Tool Call Parsing (also used as fallback for
+ # required/named when supports_required_and_named=False)
+ tool_call_info = tool_parser.extract_tool_calls(
+ content if content is not None else "",
+ request=request, # type: ignore
+ )
+ if tool_call_info is not None and tool_call_info.tools_called:
+ tool_calls.extend(
+ FunctionCall(
+ id=tc.id,
+ name=tc.function.name,
+ arguments=tc.function.arguments,
+ )
+ for tc in tool_call_info.tool_calls
+ )
+ content = tool_call_info.content
+ if content and content.strip() == "":
+ content = None
+ else:
+ # No tool calls.
+ return None, content
+
+ return tool_calls, content
+
def adjust_request(
self, request: ChatCompletionRequest | ResponsesRequest
) -> ChatCompletionRequest | ResponsesRequest:
@@ -672,6 +784,20 @@ def _append_unstreamed_tool_args(
last_tc.function.arguments or ""
) + self._tool_parser.get_remaining_unstreamed_args()
+ def parse(
+ self,
+ model_output: str,
+ request: ChatCompletionRequest | ResponsesRequest,
+ enable_auto_tools: bool = False,
+ ) -> tuple[str | None, str | None, list[FunctionCall] | None]:
+ reasoning, content = self.extract_reasoning(model_output, request)
+ tool_calls, content = self._extract_tool_calls(
+ content=content,
+ request=request,
+ enable_auto_tools=enable_auto_tools,
+ )
+ return reasoning, content, tool_calls
+
def parse_delta(
self,
delta_text: str,