Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 38 additions & 0 deletions tests/entrypoints/multimodal/openai/responses/test_image.py
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,44 @@ async def test_single_chat_session_image_base64encoded(
assert len(response.output_text) > 0


@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize("part_type", ["input_image", "image_url"])
@pytest.mark.parametrize("raw_image_url", TEST_IMAGE_ASSETS)
async def test_single_chat_session_image_chat_completions_format(
client: openai.AsyncOpenAI,
model_name: str,
part_type: str,
raw_image_url: str,
url_encoded_image: dict[str, str],
):
# #46631: accept chat-completions image shapes (image_url type or
# input_image with nested image_url, no detail)
content_text = "What's in this image?"
messages = [
{
"role": "user",
"content": [
{
"type": part_type,
"image_url": {"url": url_encoded_image[raw_image_url]},

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can see that the E2E test above already supports "type": "input_image".

Could you look into how it is supported? I don’t think handling this in protocol.py is very clear.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have attempted to trace it to the best of my abilities. input_image is parsed in chat_utils.py (MM_PARSER_MAP["input_image"] → _ResponsesInputImageParser → mm_parser.parse_image). That parser is TypeAdapter(ResponseInputImageParam), and the type is a TypedDict with detail: Required and image_url: Optional[str] (flat string).

my understanding is that the chat-completions shape (nested image_url={"url": ...}, no detail) is rejected at ResponsesRequest validation, before chat_utils runs (_ResponsesInputImageParser rejects it too). That's why the normalization has to sit at the request-validation layer rather than in the image parser.

to make image normalization more clear, I pulled it out into a named _normalize_chat_completions_image_parts helper so the intent is explicit.

},
{"type": "input_text", "text": content_text},
],
}
]
response = await client.responses.create(
model=model_name,
input=messages,
)
assert response.status == "completed"
assert len(response.output) >= 1
message = next((out for out in response.output if out.type == "message"), None)
assert message is not None
assert message.role == "assistant"
assert len(response.output_text) > 0


@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize(
Expand Down
22 changes: 22 additions & 0 deletions tests/tool_use/test_responses_request_validations.py
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,28 @@ def test_input_image_detail_defaults_or_preserves(detail: str | None) -> None:
)


@pytest.mark.parametrize("part_type", ["input_image", "image_url"])
def test_input_image_accepts_chat_completions_format(part_type: str) -> None:
# #46631: chat-completions image parts must validate on /v1/responses.
item = {
"role": "user",
"content": [
{
"type": part_type,
"image_url": {"url": "https://example.com/image.png"},
}
],
}

request = ResponsesRequest.model_validate({"input": [item]})

assert request.input[0]["content"][0] == {
"type": "input_image",
"image_url": "https://example.com/image.png",
"detail": "auto",
}


# Regression tests for parallel_tool_calls=null crash in Responses API
# (from_request() passed None to ResponsesResponse.parallel_tool_calls,
# a non-optional bool field, causing a Pydantic ValidationError during
Expand Down
24 changes: 14 additions & 10 deletions vllm/entrypoints/openai/responses/protocol.py
Original file line number Diff line number Diff line change
Expand Up @@ -136,8 +136,12 @@ class ResponseRawMessageAndToken(OpenAIBaseModel):
ResponseInputOutputItem: TypeAlias = ResponseInputItemParam | ResponseOutputItem


def _default_input_image_details(value: Any) -> Any:
"""Set the API default for input images before SDK type validation."""
def _normalize_input_images(value: Any) -> Any:
"""Normalize input images before SDK type validation.

Sets the API default ``detail`` and accepts the chat-completions shapes
(``image_url`` part type, nested ``image_url: {"url": ...}``).
"""
if not isinstance(value, dict):
return value

Expand All @@ -149,13 +153,13 @@ def _default_input_image_details(value: Any) -> Any:
changed = False
for part in content:
new_part = part
if (
isinstance(part, dict)
and part.get("type") == "input_image"
and "detail" not in part
):
new_part = {**part, "detail": "auto"}
changed = True
if isinstance(part, dict) and part.get("type") in ("input_image", "image_url"):
new_part = {**part, "type": "input_image"}
image_url = new_part.get("image_url")
if isinstance(image_url, dict) and "url" in image_url:
new_part["image_url"] = image_url["url"]
new_part.setdefault("detail", "auto")
changed = changed or new_part != part
new_content.append(new_part)

return {**value, "content": new_content} if changed else value
Expand Down Expand Up @@ -183,7 +187,7 @@ class ResponsesRequest(OpenAIBaseModel):
| list[
Annotated[
ResponseInputOutputItem,
BeforeValidator(_default_input_image_details),
BeforeValidator(_normalize_input_images),
]
]
)
Expand Down
Loading