Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 15 additions & 9 deletions tests/entrypoints/openai/parser/test_harmony_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -938,6 +938,9 @@ def test_drops_non_assistant_analysis_messages(self) -> None:
assert cleaned_messages == messages[1:]


pytestmark = pytest.mark.skip_global_cleanup


def test_has_custom_tools() -> None:
assert not has_custom_tools(set())
assert not has_custom_tools({"web_search_preview", "code_interpreter", "container"})
Expand Down Expand Up @@ -1029,8 +1032,8 @@ def test_reasoning_with_multiple_content_items(self):
)
assert msg.channel == "analysis"

def test_reasoning_without_content_returns_none(self):
"""Test reasoning item without content field returns None."""
def test_reasoning_without_content_returns_empty_message(self):
"""Test reasoning item without content field returns empty message."""
item = {
"type": "reasoning",
"id": "rs_123",
Expand All @@ -1039,10 +1042,11 @@ def test_reasoning_without_content_returns_none(self):

msg = response_input_to_harmony(item, prev_responses=[])

assert msg is None
assert msg is not None
assert msg.content[0].text == ""

def test_reasoning_with_none_content_returns_none(self):
"""Test reasoning item with content=None returns None."""
def test_reasoning_with_none_content_returns_empty_message(self):
"""Test reasoning item with content=None returns empty message."""
item = {
"type": "reasoning",
"id": "rs_123",
Expand All @@ -1052,10 +1056,11 @@ def test_reasoning_with_none_content_returns_none(self):

msg = response_input_to_harmony(item, prev_responses=[])

assert msg is None
assert msg is not None
assert msg.content[0].text == ""

def test_reasoning_with_empty_content_returns_none(self):
"""Test reasoning item with empty content list returns None."""
def test_reasoning_with_empty_content_returns_empty_message(self):
"""Test reasoning item with empty content list returns empty message."""
item = {
"type": "reasoning",
"id": "rs_123",
Expand All @@ -1064,4 +1069,5 @@ def test_reasoning_with_empty_content_returns_none(self):

msg = response_input_to_harmony(item, prev_responses=[])

assert msg is None
assert msg is not None
assert msg.content[0].text == ""
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@

from vllm.entrypoints.openai.responses.protocol import ResponsesRequest

pytestmark = pytest.mark.skip_global_cleanup


def test_function_call_dict_converted_to_object():
"""Test that function_call dictionaries are correctly parsed into
Expand Down Expand Up @@ -375,3 +377,80 @@ def test_assistant_output_style_content_coerced():
assert item.content[0].annotations == []
assert item.status == "completed"
assert item.id.startswith("msg_")


def _item_attr(item, key):
if isinstance(item, dict):
return item.get(key)
return getattr(item, key)


def _as_list(value):
return value if isinstance(value, list) else list(value)


def test_assistant_and_reasoning_items_without_id_status_are_accepted():
"""Assistant/reasoning history should accept missing id/status fields."""
request_data = {
"model": "gpt-oss",
"input": [
{
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": "hello"}],
},
{
"type": "message",
"role": "assistant",
"content": [{"type": "output_text", "text": "hi"}],
},
{
"type": "reasoning",
"summary": [],
"encrypted_content": None,
},
],
}

request = ResponsesRequest(**request_data)

assert len(request.input) == 3
assistant_item = request.input[1]
reasoning_item = request.input[2]

assert _item_attr(assistant_item, "id")
assert _item_attr(assistant_item, "status") == "completed"
assert _item_attr(reasoning_item, "id")
assert _item_attr(reasoning_item, "status") == "completed"
assert _as_list(_item_attr(reasoning_item, "summary")) == []


def test_assistant_and_reasoning_keep_existing_id_status():
"""Existing id/status should not be overwritten by normalization."""
request_data = {
"model": "gpt-oss",
"input": [
{
"type": "message",
"role": "assistant",
"id": "msg_existing",
"status": "in_progress",
"content": [{"type": "output_text", "text": "partial"}],
},
{
"type": "reasoning",
"id": "rs_existing",
"status": "incomplete",
"summary": [{"type": "summary_text", "text": "thinking"}],
},
],
}

request = ResponsesRequest(**request_data)

assistant_item = request.input[0]
reasoning_item = request.input[1]
assert _item_attr(assistant_item, "id") == "msg_existing"
assert _item_attr(assistant_item, "status") == "in_progress"
assert _item_attr(reasoning_item, "id") == "rs_existing"
assert _item_attr(reasoning_item, "status") == "incomplete"
68 changes: 68 additions & 0 deletions tests/entrypoints/openai/responses/test_protocol.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from openai.types.responses.response_output_message import ResponseOutputMessage
from openai_harmony import (
Message,
)
Expand All @@ -13,6 +14,8 @@
)
from vllm.exceptions import VLLMValidationError

pytestmark = pytest.mark.skip_global_cleanup


def test_serialize_message() -> None:
dict_value = {"a": 1, "b": "2"}
Expand Down Expand Up @@ -43,6 +46,71 @@ def test_serialize_messages() -> None:
assert serialize_messages([msg, dict_value]) == [msg_value, dict_value]


def test_normalize_openresponses_input_item_stamps_type() -> None:
"""A typeless assistant item must keep its role and content.

``type`` is optional in the OpenResponses schema, but filling in
``id``/``status`` without it makes the item match ``ItemReference`` in the
input union, which collapses it to ``{"id": ...}`` and silently drops the
assistant turn from the conversation. Regression test for that.
"""
item = {
"role": "assistant",
"content": [{"type": "output_text", "text": "hi"}],
}

normalized = ResponsesRequest._normalize_openresponses_input_item(item)

assert normalized["type"] == "message"
assert normalized["role"] == "assistant"
assert normalized["id"].startswith("msg_")
assert normalized["status"] == "completed"
# annotations are filled in for output_text content
assert normalized["content"][0]["annotations"] == []
# the caller's dict is not mutated
assert "type" not in item


def test_normalize_openresponses_input_item_reasoning_type() -> None:
"""An explicit reasoning item keeps its type and gets a summary default."""
normalized = ResponsesRequest._normalize_openresponses_input_item(
{"type": "reasoning", "content": [{"type": "reasoning_text", "text": "why"}]}
)

assert normalized["type"] == "reasoning"
assert normalized["id"].startswith("rs_")
assert normalized["status"] == "completed"
assert normalized["summary"] == []


def test_input_item_parsing_typeless_assistant_message() -> None:
"""End to end: a typeless assistant item survives as a real message."""
request = ResponsesRequest.model_validate(
{
"input": [
{
"role": "assistant",
"content": [{"type": "output_text", "text": "hi"}],
}
]
}
)

assert isinstance(request.input, list)
(parsed,) = request.input
assert isinstance(parsed, ResponseOutputMessage)
assert parsed.role == "assistant"
assert parsed.content[0].text == "hi"


def test_input_item_parsing_leaves_non_assistant_items_alone() -> None:
"""User messages are untouched — normalization only targets assistant items."""
item = {"role": "user", "content": "hello"}
request = ResponsesRequest.model_validate({"input": [item]})

assert request.input == [item]


def test_custom_tool_grammar_format_rejected() -> None:
grammar = {"type": "grammar", "syntax": "lark", "definition": 'start: "pwd"'}
with pytest.raises(VLLMValidationError) as exc_info:
Expand Down
15 changes: 15 additions & 0 deletions tests/entrypoints/openai/responses/test_responses_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@
)
from vllm.exceptions import VLLMValidationError

pytestmark = pytest.mark.skip_global_cleanup


def _single_chat_message(item):
message = _construct_message_from_response_item(item)
Expand Down Expand Up @@ -253,6 +255,19 @@ def test_construct_chat_messages_preserves_single_item_conversions(self):
assert formatted_item["role"] == "assistant"
assert formatted_item["content"] == "dongyi"

# Assistant messages with empty content (e.g. tool-only turns) should
# not crash; they map to empty string content.
empty_output_item = ResponseOutputMessage(
id="msg_empty_content",
content=[],
role="assistant",
status="completed",
type="message",
)
formatted_item = _single_chat_message(empty_output_item)
assert formatted_item["role"] == "assistant"
assert formatted_item["content"] == ""


class TestCustomTools:
def test_custom_tool_call_output_item(self):
Expand Down
120 changes: 120 additions & 0 deletions tests/entrypoints/openai/responses/test_simple.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,49 @@
MODEL_NAME = "Qwen/Qwen3-8B"


def _input_text_message(text: str) -> dict:
return {
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": text}],
}


def _assistant_output_text_message(
text: str,
*,
item_id: str | None = None,
status: str | None = None,
) -> dict:
message = {
"type": "message",
"role": "assistant",
"content": [{"type": "output_text", "text": text}],
}
if item_id is not None:
message["id"] = item_id
if status is not None:
message["status"] = status
return message


def _reasoning_item(
*,
item_id: str | None = None,
status: str | None = None,
) -> dict:
item: dict[str, object] = {
"type": "reasoning",
"summary": [],
"encrypted_content": None,
}
if item_id is not None:
item["id"] = item_id
if status is not None:
item["status"] = status
return item


@pytest.fixture(scope="module")
def server():
from .conftest import BASE_TEST_ENV
Expand Down Expand Up @@ -94,6 +137,83 @@ async def test_reasoning_item(client: OpenAI, model_name: str):
assert type(response.output[1].content[0].text) is str


@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_input_history_openai_shape_e2e(client: OpenAI, model_name: str):
"""E2E: OpenAI-style history with assistant/reasoning id+status is accepted."""
response = await client.responses.create(
model=model_name,
input=[
_input_text_message("You are a concise assistant."),
_assistant_output_text_message(
"Acknowledged.",
item_id="msg_hist_openai_1",
status="completed",
),
_reasoning_item(item_id="rs_hist_openai_1", status="completed"),
_input_text_message("Reply with one short sentence."),
],
temperature=0.0,
)

assert response is not None
assert response.status == "completed"
assert response.output_text is not None


@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_input_history_openresponses_shape_e2e(client: OpenAI, model_name: str):
"""E2E: OpenResponses-style history allows optional assistant/reasoning ids."""
response = await client.responses.create(
model=model_name,
input=[
_input_text_message("You are a concise assistant."),
_assistant_output_text_message("Acknowledged."),
_reasoning_item(),
_input_text_message("Reply with one short sentence."),
],
temperature=0.0,
)

assert response is not None
assert response.status == "completed"
assert response.output_text is not None


@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_multiturn_history_replay_without_ids_e2e(
client: OpenAI, model_name: str
):
"""E2E: second turn succeeds when replayed assistant/reasoning omit id/status."""
first_turn_prompt = "Remember this marker for the next turn: MARKER-729."
first_response = await client.responses.create(
model=model_name,
input=first_turn_prompt,
temperature=0.0,
)

assert first_response is not None
assert first_response.status == "completed"
assert first_response.output_text is not None

second_response = await client.responses.create(
model=model_name,
input=[
_input_text_message(first_turn_prompt),
_assistant_output_text_message(first_response.output_text),
_reasoning_item(),
_input_text_message("What marker did I ask you to remember?"),
],
temperature=0.0,
)

assert second_response is not None
assert second_response.status == "completed"
assert second_response.output_text is not None


@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_streaming_output_consistency(client: OpenAI, model_name: str):
Expand Down
Loading
Loading