Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""A placeholder token written by the caller must not take an item's position.

Prompt updates are matched in prompt order, so a bare `<|image_pad|>` sitting
in front of the placeholder the chat template rendered consumes the image and
leaves the rendered placeholder unexpanded (vllm-project/vllm#57740).
"""

import pytest
from PIL import Image

from vllm.exceptions import VLLMValidationError
from vllm.multimodal import MULTIMODAL_REGISTRY

from ...utils import build_model_context

pytestmark = pytest.mark.cpu_test

# placeholder, bare placeholder token, surrounding tokens
_QWEN_VL = (
"<|vision_start|><|image_pad|><|vision_end|>",
"<|image_pad|>",
("<|vision_start|>", "<|vision_end|>"),
)

_LEAD = "Describe this for me: "


@pytest.mark.parametrize(
("model_id", "placeholder", "pad", "wrapper"),
[
("Qwen/Qwen2.5-VL-3B-Instruct", *_QWEN_VL),
("Qwen/Qwen3-VL-4B-Instruct", *_QWEN_VL),
("llava-hf/llava-1.5-7b-hf", "<image>", "<image>", None),
],
)
def test_extra_placeholder_token_in_prompt(
model_id: str,
placeholder: str,
pad: str,
wrapper: tuple[str, str] | None,
):
ctx = build_model_context(model_id, limit_mm_per_prompt={"image": 1})
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
tokenizer = processor.info.get_tokenizer()
pad_id = tokenizer.get_vocab()[pad]
mm_data = {"image": [Image.new("RGB", (224, 224), "white")]}

def process(prompt: str):
return processor(prompt, mm_items=processor.info.parse_mm_data(mm_data))

with pytest.raises(VLLMValidationError, match="Found more"):
process(f"{_LEAD}what does {pad} mean? {placeholder}")

processed_inputs = process(f"{_LEAD}{placeholder}")
prompt_ids = processed_inputs["prompt_token_ids"]
(image_range,) = processed_inputs["mm_placeholders"]["image"]

# The image lands on the placeholder, with the lead text before it.
assert image_range.offset > 0
assert pad_id not in prompt_ids[: image_range.offset]
assert prompt_ids[image_range.offset] == pad_id

if wrapper is not None:
vocab = tokenizer.get_vocab()
start_id, end_id = (vocab[token] for token in wrapper)
assert prompt_ids[image_range.offset - 1] == start_id
assert prompt_ids[image_range.offset + image_range.length] == end_id
168 changes: 168 additions & 0 deletions tests/multimodal/test_processing.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

import time
from contextlib import nullcontext
from types import SimpleNamespace

import numpy as np
import pytest
Expand Down Expand Up @@ -1119,6 +1120,7 @@ def decode(self, token_ids: list[int], **kwargs) -> str:
class _FakeProcessingInfo:
def __init__(self, tokenizer) -> None:
self._tokenizer = tokenizer
self.ctx = SimpleNamespace(tokenizer=tokenizer)

def get_tokenizer(self):
return self._tokenizer
Expand Down Expand Up @@ -1207,6 +1209,172 @@ def test_apply_prompt_updates_falls_back_with_index_targets():
assert [p.tokens for p in placeholders["image"]] == [[200, 201], [9]]


def _replacement(modality: str, target, replacement, item_idx: int = 0):
return PromptReplacement(modality, target, replacement).resolve(item_idx)


def _validate(prompt_ids, mm_updates, mm_placeholders, mm_item_counts):
_text_fallback_processor()._validate_mm_placeholders(
mm_placeholders,
mm_item_counts,
prompt_ids=prompt_ids,
mm_updates=mm_updates,
)


def test_validate_mm_placeholders_allows_exact_target_count():
_validate(
[ord("a"), 200, 200, ord("b")],
{"image": [[_replacement("image", [200], [200, 200])]]},
{"image": [_placeholder("image", 0, 1, [200, 200])]},
{"image": 1},
)


def test_validate_mm_placeholders_allows_one_target_per_item():
_validate(
[200, 200, ord("a"), 200, 200],
{
"image": [
[_replacement("image", [200], [200, 200], 0)],
[_replacement("image", [200], [200, 200], 1)],
]
},
{
"image": [
_placeholder("image", 0, 0, [200, 200]),
_placeholder("image", 1, 3, [200, 200]),
]
},
{"image": 2},
)


def test_validate_mm_placeholders_counts_targets_across_modalities():
"""A video replacement may be built out of image pads, as in
`llava_onevision2` and `glm4_1v`."""
_validate(
[200, 200, 301, 200, 200, 301],
{
"image": [[_replacement("image", [200], [200, 200])]],
"video": [[_replacement("video", [301], [301, 200, 200, 301])]],
},
{
"image": [_placeholder("image", 0, 0, [200, 200])],
"video": [_placeholder("video", 0, 2, [301, 200, 200, 301])],
},
{"image": 1, "video": 1},
)


def test_validate_mm_placeholders_counts_a_shared_span_once():
"""Under `use_audio_in_video` the Qwen Omni models record an audio and a
video placeholder over the same tokens, which must not count twice."""
tokens = [200, 301, 200, 301]

with pytest.raises(VLLMValidationError, match="Found more"):
_validate(
tokens + [200],
{
"audio": [[_replacement("audio", [200], tokens)]],
"video": [[_replacement("video", [301], tokens)]],
},
{
"audio": [_placeholder("audio", 0, 0, tokens)],
"video": [_placeholder("video", 0, 0, tokens)],
},
{"audio": 1, "video": 1},
)


def test_validate_mm_placeholders_rejects_target_outside_the_placeholders():
"""A target token that no placeholder covers means the item was bound to
the wrong one, as in vllm-project/vllm#57740."""
with pytest.raises(VLLMValidationError, match="Found more"):
_validate(
[200, 200, ord("a"), 200],
{"image": [[_replacement("image", [200], [200, 200])]]},
{"image": [_placeholder("image", 0, 0, [200, 200])]},
{"image": 1},
)


def test_validate_mm_placeholders_only_counts_the_target():
"""The replacement may carry other tokens that recur freely: `qwen3_vl`
wraps each pruned video frame in `<|vision_start|>`/`<|vision_end|>`."""
_validate(
[300, 200, 200, 301, 300, 301],
{"image": [[_replacement("image", [200], [300, 200, 200, 301])]]},
{"image": [_placeholder("image", 0, 0, [300, 200, 200, 301])]},
{"image": 1},
)


def test_validate_mm_placeholders_skips_insertions():
"""An insertion keeps its target instead of consuming it."""
_validate(
[300, 200, 200],
{"image": [[PromptInsertion("image", [300], [200, 200]).resolve(0)]]},
{"image": [_placeholder("image", 0, 1, [200, 200])]},
{"image": 1},
)


def test_validate_mm_placeholders_skips_index_targets():
"""A `PromptIndex` target is a position, not a token to count."""
_validate(
[200, 200, 200],
{"image": [[_replacement("image", PromptIndexTargets.start(), [200, 200])]]},
{"image": [_placeholder("image", 0, 0, [200, 200])]},
{"image": 1},
)


def test_validate_mm_placeholders_skips_targets_the_replacement_drops():
"""Some models swap the target out for a different token (`phi3v`,
`molmo2`), so the target itself is not conserved."""
_validate(
[200, 301, 301, 200],
{"image": [[_replacement("image", [200], [301, 301])]]},
{"image": [_placeholder("image", 0, 0, [301, 301])]},
{"image": 1},
)


def test_validate_mm_placeholders_skips_multi_token_targets():
"""Only a lone target token can be reproduced by accident. `glm4v` and
`moondream3` anchor on a sequence whose `<|endoftext|>` recurs per turn."""
_validate(
[300, 200, 301, 300, 200, 301],
{"image": [[_replacement("image", [300, 200, 301], [300, 200, 301])]]},
{"image": [_placeholder("image", 0, 0, [300, 200, 301])]},
{"image": 1},
)


def test_validate_mm_placeholders_after_apply_prompt_updates():
"""Run the real applier, so the tokens fed to the check are the ones it
records rather than hand-built ones."""
processor = _text_fallback_processor()
mm_updates = {"image": [[_replacement("image", [200], [200, 200])]]}

def check(prompt_ids: list[int]):
new_token_ids, placeholders = processor._apply_prompt_updates(
prompt_ids, mm_updates
)
processor._validate_mm_placeholders(
placeholders,
{"image": 1},
prompt_ids=new_token_ids,
mm_updates=mm_updates,
)

check([200, ord("a")])

with pytest.raises(VLLMValidationError, match="Found more"):
check([200, ord("a"), 200])


@pytest.mark.skip_global_cleanup
def test_overlay_modality_mm_kwargs_scoped_video_does_not_leak_to_image():
"""HF-style videos_kwargs must overlay only when modality is video."""
Expand Down
7 changes: 6 additions & 1 deletion vllm/model_executor/models/qwen2_5_omni_thinker.py
Original file line number Diff line number Diff line change
Expand Up @@ -673,7 +673,12 @@ def _maybe_apply_prompt_updates(
mm_prompt_updates,
)

self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
self._validate_mm_placeholders(
mm_placeholders,
mm_item_counts,
prompt_ids=prompt_ids,
mm_updates=mm_prompt_updates,
)

return prompt_ids, mm_placeholders

Expand Down
7 changes: 6 additions & 1 deletion vllm/model_executor/models/qwen3_omni_moe_thinker.py
Original file line number Diff line number Diff line change
Expand Up @@ -1345,7 +1345,12 @@ def _maybe_apply_prompt_updates(
mm_prompt_updates,
)

self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
self._validate_mm_placeholders(
mm_placeholders,
mm_item_counts,
prompt_ids=prompt_ids,
mm_updates=mm_prompt_updates,
)

return prompt_ids, mm_placeholders

Expand Down
75 changes: 74 additions & 1 deletion vllm/multimodal/processing/processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
from typing_extensions import TypeVar, assert_never

from vllm.config import SchedulerConfig
from vllm.exceptions import VLLMValidationError
from vllm.inputs import (
MultiModalEncDecInput,
MultiModalHashes,
Expand Down Expand Up @@ -1721,11 +1722,31 @@ def _validate_mm_updates(
"sure you have applied it before calling `LLM.generate`."
)

def _get_single_token_replace_targets(
self,
mm_updates: MultiModalPromptUpdates,
) -> set[int]:
"""The token IDs that a replacement consumes one at a time."""
return {
update.target[0]
for updates in mm_updates.values()
for item_updates in updates
for update in item_updates
if update.mode == UpdateMode.REPLACE
and isinstance(update.target, list)
and len(update.target) == 1
}

def _validate_mm_placeholders(
self,
mm_placeholders: Mapping[str, list[PlaceholderFeaturesInfo]],
mm_item_counts: Mapping[str, int],
*,
prompt_ids: list[int] | None = None,
mm_updates: MultiModalPromptUpdates | None = None,
) -> None:
# `RuntimeError` below is for an inconsistent processor implementation;
# `VLLMValidationError` is for a prompt the caller can fix.
for modality, item_count in mm_item_counts.items():
placeholders = mm_placeholders.get(modality, [])

Expand All @@ -1738,6 +1759,53 @@ def _validate_mm_placeholders(
"and `_get_mm_fields_config` are consistent with each other."
)

if prompt_ids is None or mm_updates is None:
return

# One span can carry more than one placeholder: under
# `use_audio_in_video` the Qwen Omni models record an audio and a video
# placeholder over the same tokens, which would otherwise count twice.
spans = {
(p.start_idx, len(p.tokens)): p.tokens
for placeholders in mm_placeholders.values()
for p in placeholders
}

# Targets are matched in prompt order, so a target token the caller
# wrote themselves is matched before the one the chat template
# rendered and takes the item's position, leaving the rendered one
# unexpanded. The counts above do not see this: the item is still
# bound exactly once. Count over all modalities at once, since one
# modality's target may sit inside another modality's placeholder.
for token_id in self._get_single_token_replace_targets(mm_updates):
num_placed = sum(tokens.count(token_id) for tokens in spans.values())

# The replacement dropped the target rather than re-emitting it
# (`phi3v`, `molmo2`), or the target is not what the update
# consumed, as on the Transformers backend's offsets path. Either
# way there is nothing to conserve.
if num_placed == 0:
continue

num_found = prompt_ids.count(token_id)

if num_found > num_placed:
tokenizer = self.info.ctx.tokenizer
token = (
f"token id {token_id}"
if tokenizer is None
else repr(tokenizer.decode([token_id]))
)
raise VLLMValidationError(
f"Found more {token} tokens in the prompt ({num_found}) "
f"than the multi-modal items account for ({num_placed})! "
"Placeholder tokens are matched in prompt order, so the "
"extra ones take the items' positions and the real "
"placeholders are left unexpanded. Remove them from the "
"prompt.",
parameter="prompt",
)

def _maybe_apply_prompt_updates(
self,
mm_items: MultiModalDataItems,
Expand All @@ -1752,7 +1820,12 @@ def _maybe_apply_prompt_updates(
mm_res.prompt_updates,
)

self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
self._validate_mm_placeholders(
mm_placeholders,
mm_item_counts,
prompt_ids=prompt_ids,
mm_updates=mm_res.prompt_updates,
)

return prompt_ids, mm_placeholders

Expand Down
Loading