Skip to content

Commit e3e9bd9

Browse files
K-Mistelegarg-amit
K-Mistele
authored and
garg-amit
committed
[Bugfix] Streamed tool calls now more strictly follow OpenAI's format; ensures Vercel AI SDK compatibility (vllm-project#8272)
Signed-off-by: Amit Garg <[email protected]>
1 parent 0963eac commit e3e9bd9

File tree

6 files changed

+19
-44
lines changed

6 files changed

+19
-44
lines changed

‎tests/tool_use/utils.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ class ServerConfig(TypedDict):
1919
CONFIGS: Dict[str, ServerConfig] = {
2020
"hermes": {
2121
"model":
22-
"NousResearch/Hermes-2-Pro-Llama-3-8B",
22+
"NousResearch/Hermes-3-Llama-3.1-8B",
2323
"arguments": [
2424
"--tool-call-parser", "hermes", "--chat-template",
2525
str(VLLM_PATH / "examples/tool_chat_template_hermes.jinja")

‎vllm/entrypoints/openai/protocol.py

-7
Original file line numberDiff line numberDiff line change
@@ -713,13 +713,6 @@ class DeltaToolCall(OpenAIBaseModel):
713713
function: Optional[DeltaFunctionCall] = None
714714

715715

716-
# the initial delta that gets sent once a new tool call is started;
717-
class InitialDeltaToolCall(DeltaToolCall):
718-
id: str = Field(default_factory=lambda: f"chatcmpl-tool-{random_uuid()}")
719-
type: Literal["function"] = "function"
720-
index: int
721-
722-
723716
class ExtractedToolCallInformation(BaseModel):
724717
# indicate if tools were called
725718
tools_called: bool

‎vllm/entrypoints/openai/serving_chat.py

+5-1
Original file line numberDiff line numberDiff line change
@@ -271,9 +271,13 @@ async def chat_completion_stream_generator(
271271
# NOTE num_choices defaults to 1 so this usually executes
272272
# once per request
273273
for i in range(num_choices):
274+
274275
choice_data = ChatCompletionResponseStreamChoice(
275276
index=i,
276-
delta=DeltaMessage(role=role),
277+
delta=DeltaMessage(
278+
role=role,
279+
content="",
280+
),
277281
logprobs=None,
278282
finish_reason=None)
279283
chunk = ChatCompletionStreamResponse(

‎vllm/entrypoints/openai/tool_parsers/abstract_tool_parser.py

-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ def __init__(self, tokenizer: AnyTokenizer):
2020
# the index of the tool call that is currently being parsed
2121
self.current_tool_id: int = -1
2222
self.current_tool_name_sent: bool = False
23-
self.current_tool_initial_sent: bool = False
2423
self.streamed_args_for_tool: List[str] = []
2524

2625
self.model_tokenizer = tokenizer

‎vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py

+5-15
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,14 @@
88
from vllm.entrypoints.openai.protocol import (DeltaFunctionCall, DeltaMessage,
99
DeltaToolCall,
1010
ExtractedToolCallInformation,
11-
FunctionCall,
12-
InitialDeltaToolCall, ToolCall)
11+
FunctionCall, ToolCall)
1312
from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (
1413
ToolParser)
1514
from vllm.entrypoints.openai.tool_parsers.utils import (
1615
extract_intermediate_diff)
1716
from vllm.logger import init_logger
1817
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
18+
from vllm.utils import random_uuid
1919

2020
logger = init_logger(__name__)
2121

@@ -34,7 +34,6 @@ def __init__(self, tokenizer: AnyTokenizer):
3434
self.prev_tool_call_arr: List[Dict] = []
3535
self.current_tool_id: int = -1
3636
self.current_tool_name_sent = False
37-
self.current_tool_initial_sent: bool = False
3837
self.streamed_args_for_tool: List[str] = [
3938
] # map what has been streamed for each tool so far to a list
4039

@@ -168,7 +167,6 @@ def extract_tool_calls_streaming(
168167
# set cursors and state appropriately
169168
self.current_tool_id += 1
170169
self.current_tool_name_sent = False
171-
self.current_tool_initial_sent = False
172170
self.streamed_args_for_tool.append("")
173171
logger.debug("Starting on a new tool %s", self.current_tool_id)
174172

@@ -218,24 +216,16 @@ def extract_tool_calls_streaming(
218216
logger.debug('not enough tokens to parse into JSON yet')
219217
return None
220218

221-
# case - we haven't sent the initial delta with the tool call ID
222-
# (it will be sent)
223-
if not self.current_tool_initial_sent:
224-
self.current_tool_initial_sent = True
225-
return DeltaMessage(tool_calls=[
226-
InitialDeltaToolCall(
227-
index=self.current_tool_id).model_dump(
228-
exclude_none=True)
229-
])
230-
231219
# case - we haven't sent the tool name yet. If it's available, send
232220
# it. otherwise, wait until it's available.
233-
elif not self.current_tool_name_sent:
221+
if not self.current_tool_name_sent:
234222
function_name: Union[str, None] = current_tool_call.get("name")
235223
if function_name:
236224
self.current_tool_name_sent = True
237225
return DeltaMessage(tool_calls=[
238226
DeltaToolCall(index=self.current_tool_id,
227+
type="function",
228+
id=f"chatcmpl-tool-{random_uuid()}",
239229
function=DeltaFunctionCall(
240230
name=function_name).model_dump(
241231
exclude_none=True))

‎vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py

+8-19
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,14 @@
88
from vllm.entrypoints.openai.protocol import (DeltaFunctionCall, DeltaMessage,
99
DeltaToolCall,
1010
ExtractedToolCallInformation,
11-
FunctionCall,
12-
InitialDeltaToolCall, ToolCall)
11+
FunctionCall, ToolCall)
1312
from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (
1413
ToolParser)
1514
from vllm.entrypoints.openai.tool_parsers.utils import (
1615
extract_intermediate_diff)
1716
from vllm.logger import init_logger
1817
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
18+
from vllm.utils import random_uuid
1919

2020
logger = init_logger(__name__)
2121

@@ -25,7 +25,7 @@ class MistralToolParser(ToolParser):
2525
Tool call parser for Mistral 7B Instruct v0.3, intended for use with the
2626
examples/tool_chat_template_mistral.jinja template.
2727
28-
Used when --enable-auto-tool-choice --tool-call-parser gmistral are all set
28+
Used when --enable-auto-tool-choice --tool-call-parser mistral are all set
2929
"""
3030

3131
def __init__(self, tokenizer: AnyTokenizer):
@@ -42,7 +42,6 @@ def __init__(self, tokenizer: AnyTokenizer):
4242
self.prev_tool_call_arr: List[Dict] = []
4343
self.current_tool_id: int = -1
4444
self.current_tool_name_sent: bool = False
45-
self.current_tool_initial_sent: bool = False
4645
self.streamed_args_for_tool: List[str] = [
4746
] # map what has been streamed for each tool so far to a list
4847
self.bot_token = "[TOOL_CALLS]"
@@ -91,7 +90,6 @@ def extract_tool_calls(self,
9190

9291
except Exception as e:
9392
logger.error("Error in extracting tool call from response: %s", e)
94-
print("ERROR", e)
9593
# return information to just treat the tool call as regular JSON
9694
return ExtractedToolCallInformation(tools_called=False,
9795
tool_calls=[],
@@ -109,7 +107,7 @@ def extract_tool_calls_streaming(
109107

110108
# if the tool call token is not in the tokens generated so far, append
111109
# output to contents since it's not a tool
112-
if self.bot_token_id not in current_token_ids:
110+
if self.bot_token not in current_text:
113111
return DeltaMessage(content=delta_text)
114112

115113
# if the tool call token ID IS in the tokens generated so far, that
@@ -134,7 +132,7 @@ def extract_tool_calls_streaming(
134132
# replace BOT token with empty string, and convert single quotes
135133
# to double to allow parsing as JSON since mistral uses single
136134
# quotes instead of double for tool calls
137-
parsable_arr = current_text.split(self.bot_token)[1]
135+
parsable_arr = current_text.split(self.bot_token)[-1]
138136

139137
# tool calls are generated in an array, so do partial JSON
140138
# parsing on the entire array
@@ -186,31 +184,22 @@ def extract_tool_calls_streaming(
186184
# re-set stuff pertaining to progress in the current tool
187185
self.current_tool_id = len(tool_call_arr) - 1
188186
self.current_tool_name_sent = False
189-
self.current_tool_initial_sent = False
190187
self.streamed_args_for_tool.append("")
191188
logger.debug("starting on new tool %d", self.current_tool_id)
192189
return delta
193190

194191
# case: update an existing tool - this is handled below
195192

196-
# if the current tool initial data incl. the id, type=function
197-
# and idx not sent, send that
198-
if not self.current_tool_initial_sent:
199-
self.current_tool_initial_sent = True
200-
delta = DeltaMessage(tool_calls=[
201-
InitialDeltaToolCall(
202-
index=self.current_tool_id).model_dump(
203-
exclude_none=True)
204-
])
205-
206193
# if the current tool name hasn't been sent, send if available
207194
# - otherwise send nothing
208-
elif not self.current_tool_name_sent:
195+
if not self.current_tool_name_sent:
209196
function_name = current_tool_call.get("name")
210197
if function_name:
211198

212199
delta = DeltaMessage(tool_calls=[
213200
DeltaToolCall(index=self.current_tool_id,
201+
type="function",
202+
id=f"chatcmpl-tool-{random_uuid()}",
214203
function=DeltaFunctionCall(
215204
name=function_name).model_dump(
216205
exclude_none=True))

0 commit comments

Comments
 (0)