WIP: Reduce multimodal payload in token-in/token-out by deferring pixel_va…

docs/serving/online_serving/renderer.md

@@ -13,4 +13,29 @@ Our renderer API is designed to disaggregate the render phase(preprocessing) and
 - [Chat Completions Render API](renderer.md) (`/v1/chat/completions/render`)
     - Render chat completions
 
+## Shipping source image bytes instead of tensors
+
+By default a multimodal render response carries the processed tensors inline: `features.kwargs_data` holds a base64-encoded `pixel_values` (and friends) per item, which is typically hundreds of KB to several MB per image. Setting `skip_pixel_values: true` on a render request returns the **original encoded image bytes** in `features.raw_images` instead, and leaves `features.kwargs_data` null. The two fields are mutually exclusive.
+
+Everything else is unchanged: the renderer still runs the HF processor, because `mm_placeholders` lengths are derived from the processed tensor shapes. What the flag saves is serialization and wire bytes, not frontend CPU.
+
+```bash
+curl http://localhost:8000/v1/chat/completions/render -H 'Content-Type: application/json' -d '{
+  "model": "Qwen/Qwen3-VL-2B-Instruct",
+  "messages": [{"role": "user", "content": [
+    {"type": "image_url", "image_url": {"url": "https://example.com/cat.jpg"}},
+    {"type": "text", "text": "What is in this image?"}
+  ]}],
+  "skip_pixel_values": true
+}'
+```
+
+The response can be posted to `/inference/v1/generate` unchanged. That tier reloads each image from its bytes and re-runs the processor, consulting the multimodal cache first, so repeated images cost nothing.
+
+Constraints:
+
+- **Images only.** Video hashes fold in frame-sampling metadata, and audio and prompt embeds never retain source bytes. If any item in the request lacks them, the whole response falls back to `kwargs_data` and logs a warning.
+- **Both tiers must run the same model and the same `--media-io-kwargs`.** The generate tier recomputes `mm_hashes` from the bytes and rejects the request with a 400 if they disagree with the render tier's.
+- **`mm_processor_kwargs` are not carried over** to `/inference/v1/generate`; a render request that used them is caught by the same hash check.
+
 For the post processing counterpart that turns generated token IDs back into OpenAI compatible responses, see the [Derenderer APIs](derenderer.md).

tests/entrypoints/scale_out/render/test_raw_images.py

@@ -0,0 +1,77 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+"""Unit tests for the ``skip_pixel_values`` raw-byte payload.
+
+``raw_images`` is positional against ``mm_hashes``, and the consumer has no
+way to mix it with serialized tensors, so the producer must either cover
+every item or decline the whole request. These tests pin that rule without
+needing a model.
+"""
+
+import logging
+
+import pybase64
+import pytest
+
+from vllm.entrypoints.scale_out.render import serving as render_serving
+from vllm.entrypoints.scale_out.render.serving import _encode_raw_images
+
+
+@pytest.fixture
+def warnings(caplog):
+    """Capture this module's warnings; the `vllm` logger does not propagate."""
+    logger = logging.getLogger(render_serving.__name__)
+    logger.addHandler(caplog.handler)
+    try:
+        yield caplog
+    finally:
+        logger.removeHandler(caplog.handler)
+
+
+def test_encodes_every_item_when_all_bytes_are_retained():
+    encoded = _encode_raw_images(
+        {"image": ["h0", "h1"]},
+        {"mm_raw_bytes": {"image": [b"\x89PNG-a", b"\x89PNG-b"]}},  # type: ignore[typeddict-item]
+    )
+
+    assert encoded is not None
+    assert [pybase64.b64decode(e, validate=True) for e in encoded["image"]] == [
+        b"\x89PNG-a",
+        b"\x89PNG-b",
+    ]
+
+
+def test_declines_when_a_modality_never_retained_bytes(warnings):
+    """Audio and video carry no source bytes, so a mixed request must fall
+    back to shipping tensors for everything rather than half a payload — and
+    say so, since the caller asked for the opposite."""
+    encoded = _encode_raw_images(
+        {"image": ["h0"], "audio": ["h1"]},
+        {"mm_raw_bytes": {"image": [b"\x89PNG"]}},  # type: ignore[typeddict-item]
+    )
+
+    assert encoded is None
+    assert "audio" in warnings.text
+
+
+def test_declines_on_partial_coverage_within_a_modality():
+    assert (
+        _encode_raw_images(
+            {"image": ["h0", "h1"]},
+            {"mm_raw_bytes": {"image": [b"\x89PNG", None]}},  # type: ignore[typeddict-item]
+        )
+        is None
+    )
+
+    assert (
+        _encode_raw_images(
+            {"image": ["h0", "h1"]},
+            {"mm_raw_bytes": {"image": [b"\x89PNG"]}},  # type: ignore[typeddict-item]
+        )
+        is None
+    )
+
+
+def test_declines_when_the_renderer_reported_nothing():
+    assert _encode_raw_images({"image": ["h0"]}, {}) is None  # type: ignore[arg-type]

tests/entrypoints/scale_out/render/test_render_multimodal.py

@@ -4,6 +4,7 @@
 """Multimodal tests for the /render endpoints that expose prompt preprocessing."""
 
 import httpx
+import pybase64
 import pytest
 import pytest_asyncio
 
@@ -153,3 +154,58 @@ async def test_tokenize_matches_render_for_multimodal_input(
 
     assert tokenize_data["tokens"] == render_data["token_ids"]
     assert tokenize_data["count"] == len(render_data["token_ids"])
+
+
+@pytest.mark.asyncio
+async def test_skip_pixel_values_returns_source_bytes_not_tensors(
+    vision_client,
+    local_asset_server,
+):
+    """``skip_pixel_values`` must swap the payload carrier without changing
+    anything the downstream engine keys off: tokens, hashes and placeholders
+    stay identical, only the tensors are replaced by the source bytes."""
+
+    image = local_asset_server.get_image_asset("RGBA_comp.png")
+    data_url = encode_image_url(image, format="PNG")
+    source_bytes = pybase64.b64decode(data_url.split(",", 1)[1], validate=True)
+
+    def payload(skip: bool) -> dict:
+        return {
+            "model": VISION_MODEL_NAME,
+            "messages": [
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "image_url", "image_url": {"url": data_url}},
+                        {"type": "text", "text": "What's in this image?"},
+                    ],
+                }
+            ],
+            "skip_pixel_values": skip,
+        }
+
+    full_resp = await vision_client.post(
+        "/v1/chat/completions/render", json=payload(False)
+    )
+    skipped_resp = await vision_client.post(
+        "/v1/chat/completions/render", json=payload(True)
+    )
+    full = full_resp.json()
+    skipped = skipped_resp.json()
+
+    assert skipped["token_ids"] == full["token_ids"]
+    assert skipped["features"]["mm_hashes"] == full["features"]["mm_hashes"]
+    assert skipped["features"]["mm_placeholders"] == full["features"]["mm_placeholders"]
+
+    assert full["features"]["kwargs_data"] is not None
+    assert skipped["features"]["kwargs_data"] is None
+
+    raw_images = skipped["features"]["raw_images"]
+    assert list(raw_images) == ["image"]
+    assert len(raw_images["image"]) == len(skipped["features"]["mm_hashes"]["image"])
+    # The exact bytes the renderer loaded, so the /generate side recomputes
+    # the same hash when it reloads them.
+    assert pybase64.b64decode(raw_images["image"][0], validate=True) == source_bytes
+
+    # The point of the flag: source bytes are far cheaper than pixel_values.
+    assert len(skipped_resp.content) < len(full_resp.content)

tests/entrypoints/scale_out/token_in_token_out/test_protocol.py

@@ -9,7 +9,12 @@
 
 import json
 
-from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest
+import pytest
+
+from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
+    GenerateRequest,
+    MultiModalFeatures,
+)
 from vllm.sampling_params import SamplingParams
 
 
@@ -68,3 +73,43 @@ def test_internal_instance_construction_treats_all_as_provided():
     assert req.is_sampling_param_provided("temperature")
     # And keys we never touched should also count as provided in this path.
     assert req.is_sampling_param_provided("top_p")
+
+
+def _features_payload(**overrides) -> dict:
+    payload = {
+        "mm_hashes": {"image": ["h0"]},
+        "mm_placeholders": {"image": [{"offset": 3, "length": 4}]},
+    }
+    payload.update(overrides)
+    return payload
+
+
+def test_raw_images_and_kwargs_data_are_mutually_exclusive():
+    """The two payload carriers describe the same items in different forms;
+    accepting both would leave the consumer no way to pick a winner."""
+    with pytest.raises(ValueError, match="mutually exclusive"):
+        MultiModalFeatures.model_validate(
+            _features_payload(kwargs_data={"image": ["x"]}, raw_images={"image": ["y"]})
+        )
+
+
+def test_raw_images_must_align_with_mm_hashes():
+    """`raw_images[m][i]` is positional against `mm_hashes[m][i]`, so a
+    length or key disagreement means the consumer would mispair items."""
+    with pytest.raises(ValueError, match="raw_images"):
+        MultiModalFeatures.model_validate(
+            _features_payload(raw_images={"image": ["a", "b"]})
+        )
+
+    with pytest.raises(ValueError, match="absent from mm_hashes"):
+        MultiModalFeatures.model_validate(
+            _features_payload(raw_images={"video": ["a"]})
+        )
+
+
+def test_raw_images_alone_is_accepted():
+    features = MultiModalFeatures.model_validate(
+        _features_payload(raw_images={"image": ["a"]})
+    )
+    assert features.kwargs_data is None
+    assert features.raw_images == {"image": ["a"]}