[Security] Reject unresolved multimodal inputs before execution

docs/serving/online_serving/renderer.md

@@ -170,8 +170,8 @@ Callers that need only the token layout and item hashes, such as cache-aware
 routers, can set `"mm_payload": "none"` on the render request. The response
 then keeps `mm_hashes` and `mm_placeholders` and sets `kwargs_data` and
 `mm_metadata` to null, so the processed tensors are neither serialized nor sent.
-Do not forward such a response to `/inference/v1/generate`, which reads a null
-`kwargs_data` as every item being cached. The default, `"tensors"`, returns
+Forwarding such a response to `/inference/v1/generate` returns HTTP 400 because
+each item requires payload data or metadata. The default, `"tensors"`, returns
 both fields.
 
 `"return_mm_kwargs": false` is a deprecated alias for `"mm_payload": "none"`

docs/serving/online_serving/token_in_token_out.md

@@ -109,6 +109,9 @@ The server returns a 400 for:
 - `output_mode: "derender"` with `stream: true`.
 - `output_mode: "derender"` without `parse_context` on a server with a tool or reasoning parser or with a named `tool_choice` in `parse_context` that matches none of its `tools`. This is checked before generating.
 - An unsupported `output_mode` value.
+- A multimodal feature with neither payload data nor metadata. Render with
+  `mm_payload: "tensors"` to include the payload. Metadata-only inputs require
+  `ec_transfer_params` and the model's position metadata.
 
 For using `output_mode` with separate prefill and decode pools, see [Disaggregated Prefilling](../../features/disagg_prefill.md#generate-api-output-modes).
 

tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py

@@ -6,12 +6,15 @@
 from typing import Any
 from unittest.mock import AsyncMock, MagicMock
 
+import httpx
 import pytest
+from fastapi import FastAPI
 
 from vllm.config.multimodal import MultiModalConfig
 from vllm.entrypoints.generate.base.protocol import StreamOptions
 from vllm.entrypoints.openai.models.protocol import BaseModelPath
 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
+from vllm.entrypoints.scale_out.token_in_token_out.api_router import router
 from vllm.entrypoints.scale_out.token_in_token_out.mm_features import (
     placeholder_ranges_from_engine_input,
 )
@@ -342,6 +345,39 @@ async def test_serve_tokens_rejects_invalid_shm_handle_before_engine_handoff():
     engine.generate.assert_not_called()
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("stream", [False, True])
+@pytest.mark.parametrize("kwargs_data", [None, {"image": [None]}])
+async def test_generate_rejects_unresolved_features_before_engine(stream, kwargs_data):
+    engine = _mock_engine()
+    serving = _build_serving_tokens(engine)
+    request = GenerateRequest(
+        token_ids=[1, 2, 3],
+        sampling_params={"max_tokens": 1},
+        model=MODEL_NAME,
+        stream=stream,
+        features=MultiModalFeatures(
+            mm_hashes={"image": ["missing-image"]},
+            mm_placeholders={"image": [PlaceholderRangeInfo(offset=1, length=1)]},
+            kwargs_data=kwargs_data,
+        ),
+    )
+
+    app = FastAPI()
+    app.include_router(router)
+    app.state.serving_tokens = serving
+    async with httpx.AsyncClient(
+        transport=httpx.ASGITransport(app=app), base_url="http://test"
+    ) as client:
+        response = await client.post(
+            "/inference/v1/generate", json=request.model_dump(mode="json")
+        )
+
+    assert response.status_code == 400
+    assert "payload data or metadata" in response.json()["error"]["message"]
+    engine.generate.assert_not_called()
+
+
 @pytest.mark.asyncio
 async def test_stream_basic():
     """Streaming returns SSE chunks with correct token_ids and ends with [DONE]."""

tests/entrypoints/scale_out/token_in_token_out/test_serving_multimodal_tokens.py

@@ -57,6 +57,61 @@ async def client(server: RemoteOpenAIServer):
         yield c
 
 
+@pytest.mark.asyncio
+async def test_layout_only_features_return_bad_request_and_server_recovers(
+    client, test_image
+):
+    """Layout-only render output must fail before reaching the model runner."""
+    render_payload = {
+        "model": MODEL_NAME,
+        "messages": [
+            {
+                "role": "user",
+                "content": [
+                    {
+                        "type": "image_url",
+                        "image_url": {
+                            "url": encode_image_url(test_image, format="PNG")
+                        },
+                    },
+                    {"type": "text", "text": "What color is this image?"},
+                ],
+            }
+        ],
+        "mm_payload": "none",
+    }
+    rendered = await client.post(RENDER_ENDPOINT, json=render_payload)
+    rendered.raise_for_status()
+    data = rendered.json()
+    response = await client.post(
+        GEN_ENDPOINT,
+        json={
+            "model": MODEL_NAME,
+            "token_ids": data["token_ids"],
+            "features": data["features"],
+            "sampling_params": {"max_tokens": 1},
+        },
+    )
+    assert response.status_code == 400
+    assert "payload data or metadata" in response.json()["error"]["message"]
+
+    render_payload["mm_payload"] = "tensors"
+    rendered = await client.post(RENDER_ENDPOINT, json=render_payload)
+    rendered.raise_for_status()
+    data = rendered.json()
+    response = await client.post(
+        GEN_ENDPOINT,
+        json={
+            "model": MODEL_NAME,
+            "token_ids": data["token_ids"],
+            "features": data["features"],
+            "sampling_params": {"max_tokens": 1},
+        },
+    )
+    response.raise_for_status()
+    assert response.json()["choices"][0]["token_ids"]
+
+
 @pytest.mark.asyncio
 async def test_render_to_generate_roundtrip(client, test_image):
     """End-to-end: render a multimodal chat -> feed into generate -> decode.

tests/v1/core/test_scheduler.py

@@ -33,6 +33,10 @@
 from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats
 from vllm.distributed.kv_transfer.kv_connector.v1.multi_connector import MultiConnector
 from vllm.lora.request import LoRARequest
+from vllm.multimodal.cache import (
+    LruKeyReplicatedReceiverCache,
+    MultiModalCacheMissError,
+)
 from vllm.multimodal.inputs import (
     MultiModalBatchedField,
     MultiModalFeatureSpec,
@@ -57,7 +61,7 @@
 from vllm.v1.core.sched.output import CachedRequestData, SchedulerOutput
 from vllm.v1.core.sched.scheduler import Scheduler
 from vllm.v1.core.single_type_kv_cache_manager import register_all_kvcache_specs
-from vllm.v1.engine import FinishReason
+from vllm.v1.engine import EngineCoreRequest, FinishReason
 from vllm.v1.engine.core import EngineCore
 from vllm.v1.executor.uniproc_executor import UniProcExecutor
 from vllm.v1.hisparse.coordinator import get_hisparse_coordinator
@@ -7232,6 +7236,85 @@ def test_encoder_cache_accepts_matching_embed_count():
     assert compatible.request_id not in scheduler_output.scheduled_encoder_inputs
 
 
+@pytest.mark.parametrize(
+    "model", ["llava-hf/llava-1.5-7b-hf", "Qwen/Qwen2-VL-2B-Instruct"]
+)
+@pytest.mark.parametrize(
+    "cache_kind", ["none", "chunked", "encoder", "external", "prefix", "receiver"]
+)
+def test_admission_requires_resolved_multimodal_data(model, cache_kind):
+    """Only a receiver-cache hit supplies data needed at worker admission."""
+    scheduler = create_scheduler(
+        model=model,
+        enable_prefix_caching=True,
+        max_num_batched_tokens=16 if cache_kind == "chunked" else 8192,
+    )
+    assert scheduler.vllm_config.model_config.uses_mrope == model.startswith("Qwen/")
+    owner, candidate = create_requests(
+        num_requests=2,
+        num_tokens=33,
+        same_prompt=True,
+        mm_hashes_list=[["shared-image"], ["shared-image"]],
+        mm_positions=[
+            [PlaceholderRange(offset=20, length=4)],
+            [PlaceholderRange(offset=20, length=4)],
+        ],
+    )
+    core = object.__new__(EngineCore)
+    core.scheduler = scheduler
+    core.mm_receiver_cache = None
+    core.request_block_hasher = None
+    if cache_kind == "encoder":
+        scheduler.encoder_cache_manager.allocate(owner, 0)
+        assert scheduler.encoder_cache_manager.check_and_update_cache(candidate, 0)
+    elif cache_kind == "external":
+        scheduler.ec_connector = Mock()
+        scheduler.ec_connector.has_cache_item.return_value = True
+    elif cache_kind == "prefix":
+        scheduler.add_request(owner)
+        _model_output(scheduler, scheduler.schedule(), [[100]])
+        _, num_cached_tokens, _ = scheduler.kv_cache_manager.get_computed_blocks(
+            candidate
+        )
+        assert num_cached_tokens >= 24
+    elif cache_kind == "receiver":
+        core.mm_receiver_cache = LruKeyReplicatedReceiverCache(
+            scheduler.vllm_config.model_config
+        )
+        core.mm_receiver_cache.get_and_update_features(owner.mm_features)
+
+    candidate.mm_features[0].data = None
+    request = EngineCoreRequest(
+        request_id=candidate.request_id,
+        external_req_id=candidate.request_id,
+        prompt_token_ids=candidate.prompt_token_ids,
+        mm_features=candidate.mm_features,
+        sampling_params=candidate.sampling_params,
+        pooling_params=None,
+        arrival_time=0,
+        lora_request=None,
+        cache_salt=None,
+        data_parallel_rank=None,
+    )
+    if cache_kind == "receiver":
+        admitted, _ = core.preprocess_add_request(request)
+        assert admitted.mm_features[0].data is not None
+    else:
+        with pytest.raises(MultiModalCacheMissError) as exc_info:
+            core.preprocess_add_request(request)
+        assert exc_info.value.mm_hashes == ["shared-image"]
+        assert candidate.request_id not in scheduler.requests
+        if cache_kind == "external":
+            scheduler.ec_connector.has_cache_item.assert_not_called()
+            scheduler.ec_connector = None
+
+    healthy_request = create_requests(
+        num_requests=1, num_tokens=8, req_ids=["healthy-request"]
+    )[0]
+    scheduler.add_request(healthy_request)
+    assert healthy_request.request_id in scheduler.schedule().num_scheduled_tokens
+
+
 def test_encoder_cache_rejects_mismatched_embed_count(caplog_vllm):
     """A completed request's freeable entry rejects an incompatible reuse."""
     scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf")

vllm/entrypoints/scale_out/token_in_token_out/protocol.py

@@ -141,8 +141,9 @@ class MultiModalFeatures(BaseModel):
 
     Each value is a list parallel to `mm_hashes[modality]`.  A `str`
     entry is a base64-encoded `MultiModalKwargsItem`; `None` means
-    the item should be resolved from cache.  The entire field is
-    `None` for metadata-only (cache-hit) responses.
+    no full payload is serialized for the item. Generation requires
+    either payload data or `mm_metadata` for each item. The entire field
+    is `None` for layout-only or metadata-only responses.
     """
 
     mm_metadata: dict[str, list[str | None]] | None = None

vllm/entrypoints/scale_out/token_in_token_out/serving.py

@@ -306,6 +306,11 @@ async def serve_tokens(
             # Deserialize full tensor data and optional metadata-only data.
             # Metadata-only items are valid when ec_transfer_params is set.
             mm_kwargs = mm_kwargs_from_features(features)
+            if any(item is None for items in mm_kwargs.values() for item in items):
+                return self.create_error_response(
+                    "Multimodal features require payload data or metadata. "
+                    "Render with mm_payload='tensors' to include the payload."
+                )
             if error := self._validate_mm_cache_handles(mm_kwargs, features.mm_hashes):
                 return error
 

vllm/v1/engine/core.py

@@ -1180,6 +1180,15 @@ def preprocess_add_request(self, request: EngineCoreRequest) -> tuple[Request, i
                 request.mm_features
             )
 
+        if request.mm_features:
+            missing_mm_hashes = [
+                feature.mm_hash or feature.identifier
+                for feature in request.mm_features
+                if feature.data is None
+            ]
+            if missing_mm_hashes:
+                raise MultiModalCacheMissError(missing_mm_hashes)
+
         req = Request.from_engine_core_request(request, self.request_block_hasher)
         if req.use_structured_output:
             # Note on thread safety: no race condition.