Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions docs/serving/online_serving/renderer.md
Original file line number Diff line number Diff line change
Expand Up @@ -154,11 +154,15 @@ should split those fields:
continue to work.

Callers that need only the token layout and item hashes, such as cache-aware
routers, can set `"return_mm_kwargs": false` on the render request. The response
routers, can set `"mm_payload": "none"` on the render request. The response
then keeps `mm_hashes` and `mm_placeholders` and sets `kwargs_data` and
`mm_metadata` to null, so the processed tensors are neither serialized nor sent.
Do not forward such a response to `/inference/v1/generate`, which reads a null
`kwargs_data` as every item being cached.
`kwargs_data` as every item being cached. The default, `"tensors"`, returns
both fields.

`"return_mm_kwargs": false` is a deprecated alias for `"mm_payload": "none"`
and is ignored when `mm_payload` is set.

## Example

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@
import torch
from pydantic import ValidationError

from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.entrypoints.scale_out.token_in_token_out.mm_features import (
extract_mm_features,
merge_mm_kwargs_items,
Expand Down Expand Up @@ -173,7 +177,7 @@ def test_extract_without_kwargs_keeps_layout():
engine_input, _pixel_values, _image_grid_thw = _image_engine_input()

full = extract_mm_features(engine_input)
layout = extract_mm_features(engine_input, include_mm_kwargs=False)
layout = extract_mm_features(engine_input, mm_payload="none")

assert full is not None and layout is not None
assert layout.kwargs_data is None
Expand All @@ -182,6 +186,40 @@ def test_extract_without_kwargs_keeps_layout():
assert layout.mm_placeholders == full.mm_placeholders


_MESSAGES = [{"role": "user", "content": "hi"}]


@pytest.mark.parametrize(
("request_cls", "body"),
[
(ChatCompletionRequest, {"model": "m", "messages": _MESSAGES}),
(CompletionRequest, {"model": "m", "prompt": "hi"}),
(ResponsesRequest, {"model": "m", "input": "hi"}),
(
AnthropicMessagesRequest,
{"model": "m", "max_tokens": 8, "messages": _MESSAGES},
),
],
)
def test_return_mm_kwargs_is_alias_for_mm_payload(request_cls, body):
"""Clients of the deprecated bool keep getting layout-only responses."""
assert request_cls.model_validate(body).mm_payload == "tensors"

for return_mm_kwargs, expected in [(False, "none"), (True, "tensors")]:
request = request_cls.model_validate(
{**body, "return_mm_kwargs": return_mm_kwargs}
)
assert request.mm_payload == expected

explicit = request_cls.model_validate(
{**body, "return_mm_kwargs": False, "mm_payload": "tensors"}
)
assert explicit.mm_payload == "tensors"

with pytest.raises(ValidationError):
request_cls.model_validate({**body, "mm_payload": "source"})


def test_extract_includes_declared_placeholder_metadata_fields():
"""EC placeholder metadata is kept even when keep_on_cpu is unset."""
engine_input, _pixel_values, image_grid_thw = _image_engine_input(
Expand Down
35 changes: 27 additions & 8 deletions vllm/entrypoints/anthropic/protocol.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@
from pydantic import BaseModel, Field, field_validator, model_validator

import vllm.envs as envs
from vllm.entrypoints.generate.base.protocol import (
MMPayload,
resolve_return_mm_kwargs_alias,
)


class AnthropicError(BaseModel):
Expand Down Expand Up @@ -229,15 +233,25 @@ class AnthropicMessagesRequest(BaseModel):
top_p: float | None = None

# vLLM-specific fields that are not in Anthropic spec
return_mm_kwargs: bool = Field(
default=True,
mm_payload: MMPayload = Field(
default="tensors",
description=(
"If false, the render response's `features` set `kwargs_data` "
"and `mm_metadata` to null, for callers that need only the token "
"layout and item hashes, such as cache-aware routers. Do not send "
"such a response to `/inference/v1/generate`, which reads a null "
"`kwargs_data` as every item being cached. Only supported on the "
"render endpoints; ignored on regular generation endpoints."
"What the render response's `features` carry for multimodal "
"items. 'tensors' serializes the processed tensors into "
"`kwargs_data` and `mm_metadata`. 'none' sets both to null for "
"callers that need only the token layout and item hashes, such as "
"cache-aware routers. Do not send a 'none' response to "
"`/inference/v1/generate` which reads a null `kwargs_data` as "
"every item being cached. Only supported on the render endpoints. "
"Ignored on regular generation endpoints."
),
)
return_mm_kwargs: bool | None = Field(
default=None,
deprecated="return_mm_kwargs is deprecated in favor of the mm_payload field",
description=(
"Deprecated alias: false is the same as `mm_payload='none'`. "
"Ignored when `mm_payload` is set."
),
)
cache_salt: str | None = Field(
Expand Down Expand Up @@ -304,6 +318,11 @@ def validate_system(cls, v):
_reject_tool_changes(v)
return v

@model_validator(mode="before")
@classmethod
def apply_return_mm_kwargs_alias(cls, data: Any) -> Any:
return resolve_return_mm_kwargs_alias(data)

@model_validator(mode="after")
def validate_thinking_budget(self) -> "AnthropicMessagesRequest":
# P/D prefill legs are sent with max_tokens=1 and never decode; the
Expand Down
13 changes: 13 additions & 0 deletions vllm/entrypoints/generate/base/protocol.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@
BeforeValidator(lambda v: 0 if v is None else v, json_schema_input_type=int | None),
]

MMPayload: TypeAlias = Literal["tensors", "none"]

_CACHE_SALT_FORBIDDEN_CHARS = frozenset("@/\\\x00")
_MAX_CACHE_SALT_LENGTH = 128

Expand All @@ -56,6 +58,17 @@ def validate_cache_salt(cache_salt: object) -> None:
)


def resolve_return_mm_kwargs_alias(data: Any) -> Any:
"""Map the deprecated `return_mm_kwargs=False` onto `mm_payload="none"`."""
if (
isinstance(data, dict)
and data.get("return_mm_kwargs") is False
and "mm_payload" not in data
):
data = {**data, "mm_payload": "none"}
return data


def validate_request_mm_kwargs(
*,
mm_processor_kwargs: dict[str, Any] | None,
Expand Down
33 changes: 25 additions & 8 deletions vllm/entrypoints/openai/chat_completion/protocol.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,13 @@
DeltaMessage,
FunctionCall,
FunctionDefinition,
MMPayload,
PerRequestMetrics,
StopParam,
StreamOptions,
ToolCall,
TopLogprobsParam,
resolve_return_mm_kwargs_alias,
structured_outputs_from_response_format,
validate_cache_salt,
validate_structural_tag_response_format,
Expand Down Expand Up @@ -458,15 +460,25 @@ class ChatCompletionRequest(OpenAIBaseModel):
),
)

return_mm_kwargs: bool = Field(
default=True,
mm_payload: MMPayload = Field(
default="tensors",
description=(
"What the render response's `features` carry for multimodal "
"items. 'tensors' serializes the processed tensors into "
"`kwargs_data` and `mm_metadata`. 'none' sets both to null for "
"callers that need only the token layout and item hashes, such as "
"cache-aware routers. Do not send a 'none' response to "
"`/inference/v1/generate` which reads a null `kwargs_data` as "
"every item being cached. Only supported on the render endpoints. "
"Ignored on regular generation endpoints."
),
)
return_mm_kwargs: bool | None = Field(
default=None,
deprecated="return_mm_kwargs is deprecated in favor of the mm_payload field",
description=(
"If false, the render response's `features` set `kwargs_data` "
"and `mm_metadata` to null, for callers that need only the token "
"layout and item hashes, such as cache-aware routers. Do not send "
"such a response to `/inference/v1/generate`, which reads a null "
"`kwargs_data` as every item being cached. Only supported on the "
"render endpoints; ignored on regular generation endpoints."
"Deprecated alias: false is the same as `mm_payload='none'`. "
"Ignored when `mm_payload` is set."
),
)

Expand Down Expand Up @@ -526,6 +538,11 @@ class ChatCompletionRequest(OpenAIBaseModel):

# --8<-- [end:chat-completion-extra-params]

@model_validator(mode="before")
@classmethod
def apply_return_mm_kwargs_alias(cls, data: Any) -> Any:
return resolve_return_mm_kwargs_alias(data)

@model_validator(mode="before")
@classmethod
def check_cache_salt_support(cls, data: Any) -> Any:
Expand Down
33 changes: 25 additions & 8 deletions vllm/entrypoints/openai/completion/protocol.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,11 @@
from vllm.config import ModelConfig
from vllm.entrypoints.generate.base.protocol import (
AnyResponseFormat,
MMPayload,
PerRequestMetrics,
StopParam,
StreamOptions,
resolve_return_mm_kwargs_alias,
structured_outputs_from_response_format,
validate_cache_salt,
validate_structural_tag_response_format,
Expand Down Expand Up @@ -197,15 +199,25 @@ class CompletionRequest(OpenAIBaseModel):
),
)

return_mm_kwargs: bool = Field(
default=True,
mm_payload: MMPayload = Field(
default="tensors",
description=(
"What the render response's `features` carry for multimodal "
"items. 'tensors' serializes the processed tensors into "
"`kwargs_data` and `mm_metadata`. 'none' sets both to null for "
"callers that need only the token layout and item hashes, such as "
"cache-aware routers. Do not send a 'none' response to "
"`/inference/v1/generate` which reads a null `kwargs_data` as "
"every item being cached. Only supported on the render endpoints. "
"Ignored on regular generation endpoints."
),
)
return_mm_kwargs: bool | None = Field(
default=None,
deprecated="return_mm_kwargs is deprecated in favor of the mm_payload field",
description=(
"If false, the render response's `features` set `kwargs_data` "
"and `mm_metadata` to null, for callers that need only the token "
"layout and item hashes, such as cache-aware routers. Do not send "
"such a response to `/inference/v1/generate`, which reads a null "
"`kwargs_data` as every item being cached. Only supported on the "
"render endpoints; ignored on regular generation endpoints."
"Deprecated alias: false is the same as `mm_payload='none'`. "
"Ignored when `mm_payload` is set."
),
)

Expand Down Expand Up @@ -402,6 +414,11 @@ def to_sampling_params(
routed_experts_prompt_start=self.routed_experts_prompt_start,
)

@model_validator(mode="before")
@classmethod
def apply_return_mm_kwargs_alias(cls, data: Any) -> Any:
return resolve_return_mm_kwargs_alias(data)

@model_validator(mode="before")
@classmethod
def check_cache_salt_support(cls, data: Any) -> Any:
Expand Down
33 changes: 25 additions & 8 deletions vllm/entrypoints/openai/responses/protocol.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,9 +57,11 @@

from vllm.config import ModelConfig
from vllm.entrypoints.generate.base.protocol import (
MMPayload,
PerRequestMetrics,
StopParam,
TopLogprobsParam,
resolve_return_mm_kwargs_alias,
validate_cache_salt,
)
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
Expand Down Expand Up @@ -332,15 +334,25 @@ class ResponsesRequest(OpenAIBaseModel):
"if the served model does not use priority scheduling."
),
)
return_mm_kwargs: bool = Field(
default=True,
mm_payload: MMPayload = Field(
default="tensors",
description=(
"What the render response's `features` carry for multimodal "
"items. 'tensors' serializes the processed tensors into "
"`kwargs_data` and `mm_metadata`. 'none' sets both to null for "
"callers that need only the token layout and item hashes, such as "
"cache-aware routers. Do not send a 'none' response to "
"`/inference/v1/generate` which reads a null `kwargs_data` as "
"every item being cached. Only supported on the render endpoints. "
"Ignored on regular generation endpoints."
),
)
return_mm_kwargs: bool | None = Field(
default=None,
deprecated="return_mm_kwargs is deprecated in favor of the mm_payload field",
description=(
"If false, the render response's `features` set `kwargs_data` "
"and `mm_metadata` to null, for callers that need only the token "
"layout and item hashes, such as cache-aware routers. Do not send "
"such a response to `/inference/v1/generate`, which reads a null "
"`kwargs_data` as every item being cached. Only supported on the "
"render endpoints; ignored on regular generation endpoints."
"Deprecated alias: false is the same as `mm_payload='none'`. "
"Ignored when `mm_payload` is set."
),
)
cache_salt: str | None = Field(
Expand Down Expand Up @@ -568,6 +580,11 @@ def is_include_output_logprobs(self) -> bool:
and "message.output_text.logprobs" in self.include
)

@model_validator(mode="before")
@classmethod
def apply_return_mm_kwargs_alias(cls, data: Any) -> Any:
return resolve_return_mm_kwargs_alias(data)

@model_validator(mode="before")
@classmethod
def check_cache_salt_support(cls, data: Any) -> Any:
Expand Down
13 changes: 7 additions & 6 deletions vllm/entrypoints/scale_out/render/serving.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
from vllm.entrypoints.generate.base.protocol import MMPayload
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.openai.models.serving import (
Expand Down Expand Up @@ -149,7 +150,7 @@ async def render_chat_request(
request_id=request_id,
token_ids=token_ids,
features=self._extract_mm_features(
engine_input, include_mm_kwargs=request.return_mm_kwargs
engine_input, mm_payload=request.mm_payload
),
sampling_params=params,
model=request.model,
Expand All @@ -176,7 +177,7 @@ async def render_messages_request(
chat_req = AnthropicServingMessages.to_chat_completion_request(
request, merge_inline_system=self._merge_inline_system
)
chat_req.return_mm_kwargs = request.return_mm_kwargs
chat_req.mm_payload = request.mm_payload
return await self.render_chat_request(chat_req, raw_request)

async def render_completion_request(
Expand Down Expand Up @@ -234,7 +235,7 @@ async def render_completion_request(
request_id=request_id,
token_ids=token_ids,
features=self._extract_mm_features(
engine_input, include_mm_kwargs=request.return_mm_kwargs
engine_input, mm_payload=request.mm_payload
),
sampling_params=params,
model=request.model,
Expand Down Expand Up @@ -296,7 +297,7 @@ async def render_responses_request(
request_id=request.request_id,
token_ids=list(token_ids),
features=self._extract_mm_features(
engine_input, include_mm_kwargs=request.return_mm_kwargs
engine_input, mm_payload=request.mm_payload
),
sampling_params=params,
model=request.model,
Expand Down Expand Up @@ -349,10 +350,10 @@ def _extract_mm_features(
self,
engine_input: EngineInput,
*,
include_mm_kwargs: bool = True,
mm_payload: MMPayload = "tensors",
) -> MultiModalFeatures | None:
return extract_mm_features(
engine_input,
metadata_fields_for=self._placeholder_metadata_fields,
include_mm_kwargs=include_mm_kwargs,
mm_payload=mm_payload,
)
Loading
Loading