Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
117 changes: 116 additions & 1 deletion tests/entrypoints/openai/parser/test_harmony_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

import pytest
from openai.types.responses import FunctionTool
from openai.types.responses import FunctionTool, NamespaceTool
from openai_harmony import DeveloperContent, Message, Role

from tests.entrypoints.openai.utils import verify_harmony_messages
Expand All @@ -11,6 +11,7 @@
auto_drop_analysis_messages,
create_tool_definition,
extract_function_from_recipient,
get_developer_message,
get_system_message,
has_custom_tools,
is_function_recipient,
Expand Down Expand Up @@ -1065,3 +1066,117 @@ def test_reasoning_with_empty_content_returns_none(self):
msg = response_input_to_harmony(item, prev_responses=[])

assert msg is None


class TestGetDeveloperMessageNamespaceTools:
"""Tests for get_developer_message handling of namespace tools."""

def test_namespace_tool_accepted(self):
"""Namespace tools should not raise ValueError."""
namespace_tool = NamespaceTool(
type="namespace",
name="mcp__computer_use",
description="Computer control tools.",
tools=[
FunctionTool(
type="function",
name="get_app_state",
description="Get the current state of an app.",
parameters={
"type": "object",
"properties": {
"app": {"type": "string"},
},
"required": ["app"],
},
),
],
)
msg = get_developer_message(tools=[namespace_tool])
assert msg.author.role == Role.DEVELOPER
content = msg.content[0]
assert isinstance(content, DeveloperContent)
tool_names = [t.name for t in content.function_tools]
assert "mcp__computer_use__get_app_state" in tool_names

def test_namespace_tool_with_multiple_functions(self):
"""Namespace tool with multiple inner functions should flatten all."""
namespace_tool = NamespaceTool(
type="namespace",
name="codex",
description="Codex tools.",
tools=[
FunctionTool(
type="function",
name="shell",
description="Run a shell command.",
parameters={
"type": "object",
"properties": {"cmd": {"type": "string"}},
"required": ["cmd"],
},
),
FunctionTool(
type="function",
name="read_file",
description="Read a file.",
parameters={
"type": "object",
"properties": {"path": {"type": "string"}},
"required": ["path"],
},
),
],
)
msg = get_developer_message(tools=[namespace_tool])
content = msg.content[0]
tool_names = [t.name for t in content.function_tools]
assert "codex__shell" in tool_names
assert "codex__read_file" in tool_names

def test_mixed_function_and_namespace_tools(self):
"""Mix of function and namespace tools should all be included."""
function_tool = FunctionTool(
type="function",
name="get_weather",
description="Get weather.",
parameters=_TOOL_PARAMETERS,
)
namespace_tool = NamespaceTool(
type="namespace",
name="mcp",
description="MCP tools.",
tools=[
FunctionTool(
type="function",
name="search",
description="Search.",
parameters=_TOOL_PARAMETERS,
),
],
)
msg = get_developer_message(tools=[function_tool, namespace_tool])
content = msg.content[0]
tool_names = [t.name for t in content.function_tools]
assert "get_weather" in tool_names
assert "mcp__search" in tool_names

def test_namespace_tool_none_description(self):
"""Namespace inner tool with None description defaults to empty."""
namespace_tool = NamespaceTool(
type="namespace",
name="ns",
description="Namespace.",
tools=[
FunctionTool(
type="function",
name="tool_a",
description=None,
parameters=_TOOL_PARAMETERS,
),
],
)
msg = get_developer_message(tools=[namespace_tool])
content = msg.content[0]
tool_descs = {t.name: t for t in content.function_tools}
assert tool_descs["ns__tool_a"].description == ""
77 changes: 77 additions & 0 deletions tests/entrypoints/openai/responses/test_harmony.py
Original file line number Diff line number Diff line change
Expand Up @@ -1246,3 +1246,80 @@ async def test_system_prompt_structured_content(client: OpenAI, model_name: str)
assert response is not None
assert response.status == "completed"
assert response.output_text is not None


NAMESPACE_TOOL_SCHEMA = {
"type": "namespace",
"name": "codex",
"description": "Codex development tools.",
"tools": [
{
"type": "function",
"name": "shell",

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It looks like this is not actually a namespace tool, right?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is a namespace tool, the outer dict has "type": "namespace" (line 1252) with "name": "codex" as the namespace. The inner entry at line 1257 has "type": "function" because that's how the OpenAI Responses API NamespaceTool schema works: namespace tools are wrappers that group function tools under a common namespace.

The test verifies that:

  1. The request is accepted without ValueError: tool type namespace not supported
  2. The output correctly resolves the flat name codex__shell back to name="shell" + namespace="codex"

This matches the existing test in test_namespace_tool_separator.py which uses the same structure for non-harmony models.

"description": "Run a shell command and return stdout/stderr.",
"parameters": {
"type": "object",
"properties": {
"cmd": {
"type": "string",
"description": "The shell command to execute.",
}
},
"required": ["cmd"],
"additionalProperties": False,
},
}
],
}


@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_namespace_tool_calling(client: OpenAI, model_name: str):
"""Namespace tools should be accepted and flattened for harmony models."""
tools = [NAMESPACE_TOOL_SCHEMA]

response = await retry_for_tool_call(
client,
model=model_name,
expected_tool_type="function_call",
input="Run 'echo hello' in the shell.",
tools=tools,
temperature=0.0,
)
assert response.status == "completed"
assert has_output_type(response, "function_call"), (
f"Expected function_call in output, got: "
f"{[getattr(o, 'type', None) for o in response.output]}"
)

tool_call = next(o for o in response.output if o.type == "function_call")
assert tool_call.name == "shell"
assert tool_call.namespace == "codex"
assert json.loads(tool_call.arguments).get("cmd") is not None


@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_namespace_tool_calling_streaming(client: OpenAI, model_name: str):
"""Namespace tools in streaming mode should emit proper namespace fields."""
tools = [NAMESPACE_TOOL_SCHEMA]

response = await retry_streaming_for(
client,
model=model_name,
expected_event_type="response.output_item.done",
input="Run 'echo hello' in the shell.",
tools=tools,
temperature=0.0,
)
done_events = [
e
for e in response
if e.type == "response.output_item.done"
and getattr(e.item, "type", None) == "function_call"
]
assert len(done_events) >= 1
tool_call = done_events[0].item
assert tool_call.name == "shell"
assert tool_call.namespace == "codex"
98 changes: 98 additions & 0 deletions tests/entrypoints/openai/responses/test_serving_responses.py
Original file line number Diff line number Diff line change
Expand Up @@ -680,6 +680,104 @@ def test_zero_delta_items_should_preserve_streaming_lifecycle(
assert "response.output_item.done" in type_names


class TestHarmonyNamespaceToolStreaming:
"""Tests for namespace tool name resolution in Harmony streaming events."""

@staticmethod
def _make_segment(*, channel, recipient, delta="{}"):
return Segment(channel=channel, recipient=recipient, delta=delta)

@staticmethod
def _make_previous_item(*, channel, recipient, text='{"cmd": "echo hi"}'):
content_part = MagicMock()
content_part.text = text
item = MagicMock()
item.channel = channel
item.recipient = recipient
item.content = [content_part]
return item

def test_delta_events_resolve_namespace(self) -> None:
"""Streaming delta should emit resolved name + namespace fields."""
from vllm.entrypoints.openai.responses.streaming_events import (
emit_content_delta_events,
)
from vllm.tool_parsers.utils import ResponsesToolCallName

name_map = {
"codex__shell": ResponsesToolCallName(name="shell", namespace="codex"),
}
fn_names = frozenset({"codex__shell"})
segment = self._make_segment(
channel="commentary",
recipient="functions.codex__shell",
delta='{"cmd":',
)
state = StreamingState()

events = emit_content_delta_events(
segment, state, fn_names, tool_call_name_map=name_map
)

added_events = [e for e in events if e.type == "response.output_item.added"]
assert len(added_events) == 1
item = added_events[0].item
assert item.name == "shell"
assert item.namespace == "codex"

def test_done_events_resolve_namespace(self) -> None:
"""Streaming done should emit resolved name + namespace fields."""
from vllm.entrypoints.openai.responses.streaming_events import (
emit_previous_item_done_events,
)
from vllm.tool_parsers.utils import ResponsesToolCallName

name_map = {
"codex__shell": ResponsesToolCallName(name="shell", namespace="codex"),
}
fn_names = frozenset({"codex__shell"})
previous = self._make_previous_item(
channel="commentary", recipient="functions.codex__shell"
)
state = StreamingState()
state.is_first_function_call_delta = True
state.current_item_id = "fc_test"
state.current_call_id = "call_test"
state.current_output_index = 0

events = emit_previous_item_done_events(
previous, state, fn_names, tool_call_name_map=name_map
)

done_events = [e for e in events if e.type == "response.output_item.done"]
assert len(done_events) == 1
item = done_events[0].item
assert item.name == "shell"
assert item.namespace == "codex"

def test_delta_events_no_namespace_passthrough(self) -> None:
"""Without name_map, function name passes through unchanged."""
from vllm.entrypoints.openai.responses.streaming_events import (
emit_content_delta_events,
)

fn_names = frozenset({"get_weather"})
segment = self._make_segment(
channel="commentary",
recipient="functions.get_weather",
delta='{"lat":',
)
state = StreamingState()

events = emit_content_delta_events(segment, state, fn_names)

added_events = [e for e in events if e.type == "response.output_item.added"]
assert len(added_events) == 1
item = added_events[0].item
assert item.name == "get_weather"
assert item.namespace is None


def _make_simple_context_with_output(text, token_ids, response_parser=None):
"""Create a SimpleContext with a RequestOutput containing the given text."""
ctx = SimpleContext(response_parser=response_parser)
Expand Down
25 changes: 18 additions & 7 deletions vllm/entrypoints/openai/parser/harmony_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
from vllm import envs
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionToolsParam
from vllm.logger import init_logger
from vllm.tool_parsers.utils import flat_namespace_tool_name

logger = init_logger(__name__)

Expand Down Expand Up @@ -169,23 +170,33 @@ def get_developer_message(
if instructions is not None and not envs.VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS:
dev_msg_content = dev_msg_content.with_instructions(instructions)
if tools is not None:
function_tools: list[Tool | ChatCompletionToolsParam] = []
function_tool_descriptions: list[ToolDescription] = []
for tool in tools:
if tool.type in (
"web_search_preview",
"code_interpreter",
"container",
):
pass

elif tool.type == "function":
function_tools.append(tool)
function_tool_descriptions.append(create_tool_definition(tool))
elif tool.type == "namespace":
for namespaced_tool in tool.tools:
if namespaced_tool.type != "function":
continue
flat_name = flat_namespace_tool_name(
tool.name, namespaced_tool.name
)
function_tool_descriptions.append(
ToolDescription.new(
name=flat_name,
description=namespaced_tool.description or "",
parameters=namespaced_tool.parameters,
)
)
else:
raise ValueError(f"tool type {tool.type} not supported")
if function_tools:
function_tool_descriptions = [
create_tool_definition(tool) for tool in function_tools
]
if function_tool_descriptions:
dev_msg_content = dev_msg_content.with_function_tools(
function_tool_descriptions
)
Expand Down
Loading
Loading