diff --git a/tests/models/multimodal/processing/test_qwen2_5_vl.py b/tests/models/multimodal/processing/test_qwen2_5_vl.py
new file mode 100644
index 000000000000..9b6e00a8c0a0
--- /dev/null
+++ b/tests/models/multimodal/processing/test_qwen2_5_vl.py
@@ -0,0 +1,501 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Qwen2-VL family prompt expansion tests without downloading a checkpoint."""
+
+from types import SimpleNamespace
+
+import pytest
+import torch
+from tokenizers import Tokenizer, models, pre_tokenizers
+from transformers import PreTrainedTokenizerFast
+
+from vllm.model_executor.models.openpangu_vl import (
+ OpenPanguVLMultiModalProcessor,
+)
+from vllm.model_executor.models.qwen2_5_vl import (
+ Qwen2_5_VLForConditionalGeneration,
+ Qwen2_5_VLMultiModalProcessor,
+)
+from vllm.model_executor.models.qwen2_vl import (
+ Qwen2VLDummyInputsBuilder,
+ Qwen2VLForConditionalGeneration,
+ Qwen2VLMultiModalProcessor,
+ _get_vision_token_offset,
+)
+from vllm.multimodal.inputs import PlaceholderRange
+
+pytestmark = [pytest.mark.cpu_test, pytest.mark.skip_global_cleanup]
+
+
+def _make_processor(processor_cls, special_tokens, hf_processor, placeholder_strs):
+ vocab = {"[UNK]": 0, "before": 1, "after": 2}
+ backend = Tokenizer(models.WordLevel(vocab=vocab, unk_token="[UNK]"))
+ backend.pre_tokenizer = pre_tokenizers.Whitespace()
+ tokenizer = PreTrainedTokenizerFast(
+ tokenizer_object=backend,
+ additional_special_tokens=special_tokens,
+ )
+
+ def _placeholder_fn(modality, i):
+ result = placeholder_strs.get(modality)
+ if isinstance(result, BaseException):
+ raise result
+ return result
+
+ model_cls = SimpleNamespace(get_placeholder_str=_placeholder_fn)
+
+ processor = object.__new__(processor_cls)
+ # Processors expose the model class via _get_model_cls() (no
+ # model_loader indirection); point it at the mock.
+ processor._get_model_cls = lambda: model_cls
+ processor.info = SimpleNamespace(
+ get_hf_processor=lambda **kwargs: hf_processor,
+ get_image_processor=lambda **kwargs: SimpleNamespace(merge_size=2),
+ get_tokenizer=lambda: tokenizer,
+ ctx=SimpleNamespace(
+ model_config=SimpleNamespace(),
+ get_mm_config=lambda: SimpleNamespace(video_pruning_rate=None),
+ ),
+ )
+ return processor
+
+
+def _make_dummy_builder(processor):
+ """Create a dummy builder sharing the processor's mock model class."""
+ builder = Qwen2VLDummyInputsBuilder(processor.info)
+ builder._get_model_cls = processor._get_model_cls
+ return builder
+
+
+@pytest.fixture
+def processor():
+ return _make_processor(
+ Qwen2_5_VLMultiModalProcessor,
+ [
+ "<|vision_start|>",
+ "<|image_pad|>",
+ "<|vision_end|>",
+ "<|video_pad|>",
+ ],
+ SimpleNamespace(
+ image_token="<|image_pad|>",
+ video_token="<|video_pad|>",
+ ),
+ {
+ "image": "<|vision_start|><|image_pad|><|vision_end|>",
+ "video": "<|vision_start|><|video_pad|><|vision_end|>",
+ },
+ )
+
+
+@pytest.fixture
+def qwen2_vl_processor():
+ return _make_processor(
+ Qwen2VLMultiModalProcessor,
+ [
+ "<|vision_start|>",
+ "<|image_pad|>",
+ "<|vision_end|>",
+ "<|video_pad|>",
+ ],
+ SimpleNamespace(
+ image_token="<|image_pad|>",
+ video_token="<|video_pad|>",
+ ),
+ {
+ "image": "<|vision_start|><|image_pad|><|vision_end|>",
+ "video": "<|vision_start|><|video_pad|><|vision_end|>",
+ },
+ )
+
+
+@pytest.fixture
+def exaone_style_processor():
+ # EXAONE 4.5 reuses Qwen2VLMultiModalProcessor but its tokenizer has
+ # / instead of <|vision_start|>/<|vision_end|>.
+ return _make_processor(
+ Qwen2VLMultiModalProcessor,
+ [
+ "",
+ "",
+ "<|image_pad|>",
+ "<|video_pad|>",
+ ],
+ SimpleNamespace(
+ image_token="<|image_pad|>",
+ video_token="<|video_pad|>",
+ ),
+ {
+ "image": "<|image_pad|>",
+ "video": "<|video_pad|>",
+ },
+ )
+
+
+@pytest.fixture
+def dots_ocr_style_processor():
+ # dots.ocr registers Qwen2VLMultiModalProcessor directly with its own
+ # <|img|><|imgpad|><|endofimg|> wrapper.
+ return _make_processor(
+ Qwen2VLMultiModalProcessor,
+ [
+ "<|img|>",
+ "<|imgpad|>",
+ "<|endofimg|>",
+ "<|video_pad|>",
+ ],
+ SimpleNamespace(
+ image_token="<|imgpad|>",
+ video_token="<|video_pad|>",
+ ),
+ {
+ "image": "<|img|><|imgpad|><|endofimg|>",
+ "video": None,
+ },
+ )
+
+
+@pytest.fixture
+def jina_style_processor():
+ # Jina-VL inherits Qwen2VLMultiModalProcessor; its get_placeholder_str
+ # raises for video instead of returning None.
+ return _make_processor(
+ Qwen2VLMultiModalProcessor,
+ [
+ "<|vision_start|>",
+ "<|image_pad|>",
+ "<|vision_end|>",
+ "<|video_pad|>",
+ ],
+ SimpleNamespace(
+ image_token="<|image_pad|>",
+ video_token="<|video_pad|>",
+ ),
+ {
+ "image": "<|vision_start|><|image_pad|><|vision_end|>",
+ "video": ValueError("Only image modality is supported"),
+ },
+ )
+
+
+@pytest.fixture
+def openpangu_vl_processor():
+ return _make_processor(
+ OpenPanguVLMultiModalProcessor,
+ ["[unused18]", "[unused19]", "[unused20]", "[unused32]"],
+ SimpleNamespace(
+ image_token="[unused19]",
+ video_token="[unused32]",
+ ),
+ {
+ "image": "[unused18][unused19][unused20]",
+ "video": "[unused18][unused32][unused20]",
+ },
+ )
+
+
+def _resolve_image_update(processor):
+ updates = processor._get_prompt_updates(
+ mm_items={},
+ hf_processor_mm_kwargs={},
+ out_mm_kwargs={
+ "image": [
+ {"image_grid_thw": SimpleNamespace(data=torch.tensor([1, 2, 4]))}
+ ],
+ "video": [],
+ },
+ )
+ return [updates[0].resolve(0)]
+
+
+def test_qwen25_vl_does_not_match_bare_image_pad_in_user_text(processor):
+ tokenizer = processor.info.get_tokenizer()
+ vocab = tokenizer.get_vocab()
+ image_pad = vocab["<|image_pad|>"]
+ vision_start = vocab["<|vision_start|>"]
+ vision_end = vocab["<|vision_end|>"]
+
+ image_update = _resolve_image_update(processor)
+ prompt = tokenizer.encode(
+ "before <|image_pad|> <|vision_start|><|image_pad|><|vision_end|> after",
+ add_special_tokens=False,
+ )
+
+ new_prompt, placeholders = processor._apply_prompt_updates(
+ prompt,
+ {"image": [image_update]},
+ )
+
+ assert new_prompt.count(image_pad) == 3
+ assert new_prompt.count(vision_start) == 1
+ assert new_prompt.count(vision_end) == 1
+ assert placeholders["image"][0].tokens == [
+ vision_start,
+ image_pad,
+ image_pad,
+ vision_end,
+ ]
+ assert placeholders["image"][0].is_embed.tolist() == [False, True, True, False]
+
+
+def test_qwen2_vl_does_not_match_bare_image_pad_in_user_text(
+ qwen2_vl_processor,
+):
+ tokenizer = qwen2_vl_processor.info.get_tokenizer()
+ vocab = tokenizer.get_vocab()
+ image_pad = vocab["<|image_pad|>"]
+ vision_start = vocab["<|vision_start|>"]
+ vision_end = vocab["<|vision_end|>"]
+
+ image_update = _resolve_image_update(qwen2_vl_processor)
+ prompt = tokenizer.encode(
+ "before <|image_pad|> <|vision_start|><|image_pad|><|vision_end|> after",
+ add_special_tokens=False,
+ )
+
+ new_prompt, placeholders = qwen2_vl_processor._apply_prompt_updates(
+ prompt,
+ {"image": [image_update]},
+ )
+
+ assert new_prompt.count(image_pad) == 3
+ assert new_prompt.count(vision_start) == 1
+ assert new_prompt.count(vision_end) == 1
+ assert placeholders["image"][0].tokens == [
+ vision_start,
+ image_pad,
+ image_pad,
+ vision_end,
+ ]
+ assert placeholders["image"][0].is_embed.tolist() == [False, True, True, False]
+
+
+def test_exaone_style_wrapper_derived_from_placeholder_str(
+ exaone_style_processor,
+):
+ # EXAONE 4.5's tokenizer lacks <|vision_start|>; the target must be
+ # derived from its own placeholder string, not hardcoded.
+ tokenizer = exaone_style_processor.info.get_tokenizer()
+ vocab = tokenizer.get_vocab()
+ image_pad = vocab["<|image_pad|>"]
+ vision_start = vocab[""]
+ vision_end = vocab[""]
+
+ image_update = _resolve_image_update(exaone_style_processor)
+ prompt = tokenizer.encode(
+ "before <|image_pad|> <|image_pad|> after",
+ add_special_tokens=False,
+ )
+
+ new_prompt, placeholders = exaone_style_processor._apply_prompt_updates(
+ prompt,
+ {"image": [image_update]},
+ )
+
+ assert new_prompt.count(image_pad) == 3
+ assert new_prompt.count(vision_start) == 1
+ assert new_prompt.count(vision_end) == 1
+ assert placeholders["image"][0].tokens == [
+ vision_start,
+ image_pad,
+ image_pad,
+ vision_end,
+ ]
+ assert placeholders["image"][0].is_embed.tolist() == [False, True, True, False]
+
+
+def test_dots_ocr_style_wrapper_derived_from_placeholder_str(
+ dots_ocr_style_processor,
+):
+ # dots.ocr uses <|img|><|imgpad|><|endofimg|>; the shared processor must
+ # not assume the Qwen wrapper.
+ tokenizer = dots_ocr_style_processor.info.get_tokenizer()
+ vocab = tokenizer.get_vocab()
+ image_pad = vocab["<|imgpad|>"]
+ img_start = vocab["<|img|>"]
+ img_end = vocab["<|endofimg|>"]
+
+ image_update = _resolve_image_update(dots_ocr_style_processor)
+ prompt = tokenizer.encode(
+ "before <|imgpad|> <|img|><|imgpad|><|endofimg|> after",
+ add_special_tokens=False,
+ )
+
+ new_prompt, placeholders = dots_ocr_style_processor._apply_prompt_updates(
+ prompt,
+ {"image": [image_update]},
+ )
+
+ assert new_prompt.count(image_pad) == 3
+ assert new_prompt.count(img_start) == 1
+ assert new_prompt.count(img_end) == 1
+ assert placeholders["image"][0].tokens == [
+ img_start,
+ image_pad,
+ image_pad,
+ img_end,
+ ]
+ assert placeholders["image"][0].is_embed.tolist() == [False, True, True, False]
+
+
+def test_jina_style_video_falls_back_to_bare_pad(jina_style_processor):
+ # get_placeholder_str raises for video; the processor must fall back
+ # to the bare pad token instead of crashing.
+ updates = jina_style_processor._get_prompt_updates(
+ mm_items={},
+ hf_processor_mm_kwargs={},
+ out_mm_kwargs={"image": [], "video": []},
+ )
+ tokenizer = jina_style_processor.info.get_tokenizer()
+ video_pad = tokenizer.get_vocab()["<|video_pad|>"]
+
+ assert updates[1].target == [video_pad]
+
+
+def test_openpangu_vl_does_not_match_bare_image_pad_in_user_text(
+ openpangu_vl_processor,
+):
+ tokenizer = openpangu_vl_processor.info.get_tokenizer()
+ vocab = tokenizer.get_vocab()
+ image_pad = vocab["[unused19]"]
+ vision_start = vocab["[unused18]"]
+ vision_end = vocab["[unused20]"]
+
+ image_update = _resolve_image_update(openpangu_vl_processor)
+ prompt = tokenizer.encode(
+ "before [unused19] [unused18][unused19][unused20] after",
+ add_special_tokens=False,
+ )
+
+ new_prompt, placeholders = openpangu_vl_processor._apply_prompt_updates(
+ prompt,
+ {"image": [image_update]},
+ )
+
+ assert new_prompt.count(image_pad) == 3
+ assert new_prompt.count(vision_start) == 1
+ assert new_prompt.count(vision_end) == 1
+ assert placeholders["image"][0].tokens == [
+ vision_start,
+ image_pad,
+ image_pad,
+ vision_end,
+ ]
+ assert placeholders["image"][0].is_embed.tolist() == [False, True, True, False]
+
+
+def test_dummy_text_emits_full_wrapper(qwen2_vl_processor):
+ # The dummy prompt must contain the complete wrapper so that it matches
+ # the replacement targets (used for profiling).
+ builder = _make_dummy_builder(qwen2_vl_processor)
+
+ assert builder.get_dummy_text({"image": 1}) == (
+ "<|vision_start|><|image_pad|><|vision_end|>"
+ )
+ assert builder.get_dummy_text({"image": 1, "video": 1}) == (
+ "<|vision_start|><|image_pad|><|vision_end|>"
+ "<|vision_start|><|video_pad|><|vision_end|>"
+ )
+
+
+def test_dummy_text_emits_full_wrapper_dots_ocr(dots_ocr_style_processor):
+ builder = _make_dummy_builder(dots_ocr_style_processor)
+
+ assert builder.get_dummy_text({"image": 2}) == (
+ "<|img|><|imgpad|><|endofimg|><|img|><|imgpad|><|endofimg|>"
+ )
+
+
+def test_dummy_text_falls_back_to_bare_pad_for_jina_video(
+ jina_style_processor,
+):
+ # Jina-VL defines no video placeholder string; the dummy must fall back
+ # to the bare pad token, consistent with get_wrapper_token_ids, so that
+ # profiling with video still matches the video replacement target.
+ builder = _make_dummy_builder(jina_style_processor)
+
+ assert builder.get_dummy_text({"image": 1, "video": 1}) == (
+ "<|vision_start|><|image_pad|><|vision_end|><|video_pad|>"
+ )
+
+
+def test_vision_token_offset_skips_leading_non_embedded_tokens():
+ # mm_position.offset points at the opening wrapper token; the first
+ # vision token starts after the leading non-embedded wrapper tokens.
+ assert (
+ _get_vision_token_offset(
+ PlaceholderRange(
+ offset=5,
+ length=6,
+ is_embed=torch.tensor([False, True, True, True, True, False]),
+ )
+ )
+ == 6
+ )
+ # No embed mask (or a bare pad target): the offset already points at
+ # the first vision token.
+ assert _get_vision_token_offset(PlaceholderRange(offset=5, length=4)) == 5
+ assert (
+ _get_vision_token_offset(
+ PlaceholderRange(offset=5, length=1, is_embed=torch.tensor([True]))
+ )
+ == 5
+ )
+
+
+def test_mrope_positions_skip_wrapper_prefix():
+ # The prompt-replacement target is the complete wrapper, so
+ # mm_position.offset points at <|vision_start|>. M-RoPE must assign
+ # vision positions starting from the first <|image_pad|> instead of
+ # shifting every position by one.
+ model = object.__new__(Qwen2VLForConditionalGeneration)
+ model.config = SimpleNamespace(
+ vision_config=SimpleNamespace(spatial_merge_size=2, tokens_per_second=1.0)
+ )
+ feature = SimpleNamespace(
+ modality="image",
+ mm_position=PlaceholderRange(
+ offset=5,
+ length=6,
+ is_embed=torch.tensor([False, True, True, True, True, False]),
+ ),
+ data={"image_grid_thw": SimpleNamespace(data=torch.tensor([1, 4, 4]))},
+ )
+ # 5 text tokens, wrapper (6 tokens), 2 text tokens.
+ positions, _ = model.get_mrope_input_positions(list(range(13)), [feature])
+
+ assert positions.shape == (3, 13)
+ # t positions: text 0-5, vision grid (1,2,2) t=0 -> 6, text 8-10.
+ assert positions[0].tolist() == [0, 1, 2, 3, 4, 5, 6, 6, 6, 6, 8, 9, 10]
+ # h positions: vision grid h=[0,0,1,1] -> [6,6,7,7].
+ assert positions[1].tolist() == [0, 1, 2, 3, 4, 5, 6, 6, 7, 7, 8, 9, 10]
+ # w positions: vision grid w=[0,1,0,1] -> [6,7,6,7].
+ assert positions[2].tolist() == [0, 1, 2, 3, 4, 5, 6, 7, 6, 7, 8, 9, 10]
+
+
+def test_qwen25_vl_mrope_positions_skip_wrapper_prefix():
+ # Qwen2.5-VL's model class does not inherit from Qwen2-VL's model class,
+ # so it needs its own M-RoPE implementation: falling back to
+ # SupportsMRoPE's empty method returns None and breaks unpacking in
+ # the runner. This also covers EXAONE 4.5 and OpenCUA, which inherit
+ # from Qwen2.5-VL's model class.
+ model = object.__new__(Qwen2_5_VLForConditionalGeneration)
+ model.config = SimpleNamespace(
+ vision_config=SimpleNamespace(spatial_merge_size=2, tokens_per_second=1.0)
+ )
+ feature = SimpleNamespace(
+ modality="image",
+ mm_position=PlaceholderRange(
+ offset=5,
+ length=6,
+ is_embed=torch.tensor([False, True, True, True, True, False]),
+ ),
+ data={"image_grid_thw": SimpleNamespace(data=torch.tensor([1, 4, 4]))},
+ )
+ # 5 text tokens, wrapper (6 tokens), 2 text tokens.
+ positions, _ = model.get_mrope_input_positions(list(range(13)), [feature])
+
+ assert positions is not None
+ assert positions.shape == (3, 13)
+ # h positions: text 0-5, vision grid h=[0,0,1,1] -> [6,6,7,7], text 8-10.
+ assert positions[1].tolist() == [0, 1, 2, 3, 4, 5, 6, 6, 7, 7, 8, 9, 10]
diff --git a/vllm/model_executor/models/dots_ocr.py b/vllm/model_executor/models/dots_ocr.py
index adcc560ba5ea..e26dcbbd1665 100644
--- a/vllm/model_executor/models/dots_ocr.py
+++ b/vllm/model_executor/models/dots_ocr.py
@@ -96,9 +96,12 @@ class DotsOCRImageEmbeddingInputs(TensorSchema):
class DotsOCRDummyInputsBuilder(Qwen2VLDummyInputsBuilder):
- def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
- num_images = mm_counts.get("image", 0)
- return IMAGE_TOKEN * num_images
+ def _get_model_cls(self) -> type[nn.Module]:
+ return DotsOCRForCausalLM
+
+ # get_dummy_text is inherited: it emits the complete wrapper derived
+ # from get_placeholder_str ("<|img|><|imgpad|><|endofimg|>"), which
+ # matches the replacement targets in _get_prompt_updates.
def get_dummy_mm_data(
self,
@@ -578,8 +581,13 @@ def forward(
return hidden_states
+class DotsOCRMultiModalProcessor(Qwen2VLMultiModalProcessor):
+ def _get_model_cls(self) -> type[nn.Module]:
+ return DotsOCRForCausalLM
+
+
@MULTIMODAL_REGISTRY.register_processor(
- Qwen2VLMultiModalProcessor,
+ DotsOCRMultiModalProcessor,
info=DotsOCRProcessingInfo,
dummy_inputs=DotsOCRDummyInputsBuilder,
)
diff --git a/vllm/model_executor/models/exaone4_5.py b/vllm/model_executor/models/exaone4_5.py
index 5a53e60b7d5e..bbb880d228dc 100644
--- a/vllm/model_executor/models/exaone4_5.py
+++ b/vllm/model_executor/models/exaone4_5.py
@@ -50,8 +50,7 @@
)
from vllm.multimodal import MULTIMODAL_REGISTRY
-from .qwen2_vl import Qwen2VLDummyInputsBuilder as Exaone4_5_DummyInputsBuilder
-from .qwen2_vl import Qwen2VLMultiModalProcessor as Exaone4_5_MultiModalProcessor
+from .qwen2_vl import Qwen2VLDummyInputsBuilder, Qwen2VLMultiModalProcessor
from .utils import (
AutoWeightsLoader,
WeightsMapper,
@@ -315,6 +314,16 @@ def get_hf_processor(self, **kwargs: object) -> Exaone4_5_Processor:
)
+class Exaone4_5_MultiModalProcessor(Qwen2VLMultiModalProcessor):
+ def _get_model_cls(self) -> type[nn.Module]:
+ return Exaone4_5_ForConditionalGeneration
+
+
+class Exaone4_5_DummyInputsBuilder(Qwen2VLDummyInputsBuilder):
+ def _get_model_cls(self) -> type[nn.Module]:
+ return Exaone4_5_ForConditionalGeneration
+
+
@MULTIMODAL_REGISTRY.register_processor(
Exaone4_5_MultiModalProcessor,
info=Exaone4_5_ProcessingInfo,
diff --git a/vllm/model_executor/models/jina_vl.py b/vllm/model_executor/models/jina_vl.py
index 971284561926..fda71d2efd69 100644
--- a/vllm/model_executor/models/jina_vl.py
+++ b/vllm/model_executor/models/jina_vl.py
@@ -59,6 +59,9 @@ def forward(self, x, **kwargs):
class JinaVLMultiModalProcessor(Qwen2VLMultiModalProcessor):
+ def _get_model_cls(self) -> type[nn.Module]:
+ return JinaVLForSequenceClassification
+
def _cached_apply_hf_processor(
self,
inputs: ProcessorInputs,
@@ -96,10 +99,15 @@ def _cached_apply_hf_processor(
)
+class JinaVLDummyInputsBuilder(Qwen2VLDummyInputsBuilder):
+ def _get_model_cls(self) -> type[nn.Module]:
+ return JinaVLForSequenceClassification
+
+
@MULTIMODAL_REGISTRY.register_processor(
JinaVLMultiModalProcessor,
info=Qwen2VLProcessingInfo,
- dummy_inputs=Qwen2VLDummyInputsBuilder,
+ dummy_inputs=JinaVLDummyInputsBuilder,
)
class JinaVLForSequenceClassification(
Qwen2VLForConditionalGeneration,
diff --git a/vllm/model_executor/models/openpangu_vl.py b/vllm/model_executor/models/openpangu_vl.py
index 8a62aa6ba6b4..b72ce1bf47ce 100644
--- a/vllm/model_executor/models/openpangu_vl.py
+++ b/vllm/model_executor/models/openpangu_vl.py
@@ -60,6 +60,10 @@
Qwen2_5_VLMultiModalProcessor,
Qwen2_5_VLProcessingInfo,
)
+from vllm.model_executor.models.qwen2_vl import (
+ _get_vision_token_offset,
+ get_wrapper_token_ids,
+)
from vllm.model_executor.models.utils import (
AutoWeightsLoader,
WeightsMapper,
@@ -761,18 +765,25 @@ def _get_prompt_updates(
image_processor = self.info.get_image_processor(**hf_processor_mm_kwargs)
tokenizer = self.info.get_tokenizer()
vocab = tokenizer.get_vocab()
+
image_token = hf_processor.image_token
video_token = hf_processor.video_token
- vision_start_token = hf_processor.vision_start_token
- vision_end_token = hf_processor.vision_end_token
image_token_id = vocab[image_token]
video_token_id = vocab[video_token]
- vision_start_token_id = vocab[vision_start_token]
- vision_end_token_id = vocab[vision_end_token]
placeholder = {
"image": image_token_id,
"video": video_token_id,
}
+ # Match the complete wrapper emitted by the chat template (derived
+ # from the model's own placeholder string:
+ # [unused18][unused19][unused20] for images,
+ # [unused18][unused32][unused20] for videos).
+ targets = {
+ modality: get_wrapper_token_ids(
+ OpenPanguVLForConditionalGeneration, tokenizer, modality, pad_id
+ )
+ for modality, pad_id in placeholder.items()
+ }
merge_length = image_processor.merge_size**2
@@ -781,31 +792,31 @@ def get_replacement_openpangu_vision(item_idx: int, modality: str):
grid_thw = out_item[f"{modality}_grid_thw"].data
if not isinstance(grid_thw, torch.Tensor):
raise TypeError("Expected 'grid_thw' to be a Tensor")
+
+ pad_id = placeholder[modality]
+ wrapper_ids = targets[modality]
+ # get_wrapper_token_ids guarantees the pad token occurs exactly
+ # once in a derived wrapper, or the target is the bare pad token.
+ pad_idx = wrapper_ids.index(pad_id)
+ prefix = wrapper_ids[:pad_idx]
+ suffix = wrapper_ids[pad_idx + 1 :]
+
if modality == "image":
- image_token_id_total = [image_token_id] * (
- int(grid_thw.prod()) // merge_length
- )
- return image_token_id_total
+ num_tokens = int(grid_thw.prod()) // merge_length
+ replacement = prefix + [pad_id] * num_tokens + suffix
else:
- # When modality is video
+ # Video placeholders are expanded per frame, with the outer
+ # wrapper tokens stripped (preserving the original layout).
grid_t, grid_h, grid_w = grid_thw
- video_seq_length_per_time = (grid_h * grid_w).item() // merge_length
- video_token_id_per_time = (
- [vision_start_token_id]
- + [video_token_id] * video_seq_length_per_time
- + [vision_end_token_id]
- )
- video_token_id_total = video_token_id_per_time * grid_t.item()
- video_token_id_middle = video_token_id_total[1:-1]
- return PromptUpdateDetails.select_token_id(
- video_token_id_middle,
- embed_token_id=video_token_id,
- )
+ tokens_per_frame = (grid_h * grid_w).item() // merge_length
+ per_frame = prefix + [pad_id] * tokens_per_frame + suffix
+ replacement = (per_frame * grid_t.item())[1:-1]
+ return PromptUpdateDetails.select_token_id(replacement, pad_id)
return [
PromptReplacement(
modality=modality,
- target=[placeholder[modality]],
+ target=targets[modality],
replacement=partial(
get_replacement_openpangu_vision, modality=modality
),
@@ -815,7 +826,8 @@ def get_replacement_openpangu_vision(item_idx: int, modality: str):
class OpenPanguVLDummyInputsBuilder(Qwen2_5_VLDummyInputsBuilder):
- pass
+ def _get_model_cls(self) -> type[nn.Module]:
+ return OpenPanguVLForConditionalGeneration
@MULTIMODAL_REGISTRY.register_processor(
@@ -1174,7 +1186,9 @@ def iter_mm_grid_thw(
) -> Iterator[tuple[str, int, int, int, int]]:
spatial_merge_size = self.config.vision_config.spatial_merge_size
for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
- offset = mm_feature.mm_position.offset
+ # mm_position.offset points at the opening wrapper token;
+ # M-RoPE needs the offset of the first vision token.
+ offset = _get_vision_token_offset(mm_feature.mm_position)
modality = mm_feature.modality
feature_data = mm_feature.data
assert feature_data is not None
diff --git a/vllm/model_executor/models/qwen2_5_vl.py b/vllm/model_executor/models/qwen2_5_vl.py
index 581ff7730351..828c8b74af03 100644
--- a/vllm/model_executor/models/qwen2_5_vl.py
+++ b/vllm/model_executor/models/qwen2_5_vl.py
@@ -81,7 +81,6 @@
VideoItem,
)
from vllm.multimodal.parse import MultiModalDataItems
-from vllm.multimodal.processing import PromptReplacement, PromptUpdate
from vllm.multimodal.processing.processor import HFMultiModalInputs
from vllm.multimodal.video_prune.evs import (
compute_mrope_for_media,
@@ -112,9 +111,11 @@
)
from .qwen2_vl import (
Qwen2VLDummyInputsBuilder,
+ Qwen2VLImageProcessor,
Qwen2VLMultiModalDataParser,
Qwen2VLMultiModalProcessor,
Qwen2VLProcessingInfo,
+ _get_vision_token_offset,
)
from .utils import (
AutoWeightsLoader,
@@ -1148,6 +1149,9 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
class Qwen2_5_VLDummyInputsBuilder(Qwen2VLDummyInputsBuilder):
+ def _get_model_cls(self) -> type[nn.Module]:
+ return Qwen2_5_VLForConditionalGeneration
+
def _get_dummy_videos(
self,
*,
@@ -1201,6 +1205,9 @@ def get_data_parser(self):
class Qwen2_5_VLMultiModalProcessor(Qwen2VLMultiModalProcessor):
+ def _get_model_cls(self) -> type[nn.Module]:
+ return Qwen2_5_VLForConditionalGeneration
+
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
@@ -1231,59 +1238,39 @@ def _get_hf_mm_inputs(
]
return hf_inputs
- def _get_prompt_updates(
+ def _get_num_pad_tokens(
self,
- mm_items: MultiModalDataItems,
- hf_processor_mm_kwargs: Mapping[str, Any],
+ item_idx: int,
+ modality: str,
out_mm_kwargs: MultiModalKwargsItems,
- ) -> Sequence[PromptUpdate]:
- hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
- image_processor = self.info.get_image_processor(**hf_processor_mm_kwargs)
- tokenizer = self.info.get_tokenizer()
- vocab = tokenizer.get_vocab()
-
- placeholder = {
- "image": vocab[hf_processor.image_token],
- "video": vocab[hf_processor.video_token],
- }
-
- merge_length = image_processor.merge_size**2
-
- def get_replacement_qwen2vl(item_idx: int, modality: str):
- out_item = out_mm_kwargs[modality][item_idx]
- grid_thw = out_item[f"{modality}_grid_thw"].data
- assert isinstance(grid_thw, torch.Tensor)
-
- num_tokens = int(grid_thw.prod()) // merge_length
-
- # EVS-specific code
- video_pruning_rate = self.info.ctx.get_mm_config().video_pruning_rate
- if (
- modality == "video"
- and video_pruning_rate is not None
- and video_pruning_rate > 0.0
- ):
- T, H, W = map(int, grid_thw)
- tokens_per_frame = (H // image_processor.merge_size) * (
- W // image_processor.merge_size
- )
- num_tokens = compute_retained_tokens_count(
- tokens_per_frame,
- T,
- video_pruning_rate,
- )
- # End of EVS-specific code
-
- return [placeholder[modality]] * num_tokens
-
- return [
- PromptReplacement(
- modality=modality,
- target=[placeholder[modality]],
- replacement=partial(get_replacement_qwen2vl, modality=modality),
+ image_processor: Qwen2VLImageProcessor,
+ merge_length: int,
+ ) -> int:
+ out_item = out_mm_kwargs[modality][item_idx]
+ grid_thw = out_item[f"{modality}_grid_thw"].data
+ assert isinstance(grid_thw, torch.Tensor)
+
+ num_tokens = int(grid_thw.prod()) // merge_length
+
+ # EVS: Explored Visual-token Selection.
+ # Prune redundant video tokens to reduce prefill cost.
+ video_pruning_rate = self.info.ctx.get_mm_config().video_pruning_rate
+ if (
+ modality == "video"
+ and video_pruning_rate is not None
+ and video_pruning_rate > 0.0
+ ):
+ T, H, W = map(int, grid_thw)
+ tokens_per_frame = (H // image_processor.merge_size) * (
+ W // image_processor.merge_size
)
- for modality in ("image", "video")
- ]
+ num_tokens = compute_retained_tokens_count(
+ tokens_per_frame,
+ T,
+ video_pruning_rate,
+ )
+
+ return num_tokens
@MULTIMODAL_REGISTRY.register_processor(
@@ -1340,7 +1327,9 @@ def iter_mm_grid_thw(
spatial_merge_size = self.config.vision_config.spatial_merge_size
tokens_per_second = getattr(self.config.vision_config, "tokens_per_second", 1.0)
for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
- offset = mm_feature.mm_position.offset
+ # mm_position.offset points at the opening wrapper token;
+ # M-RoPE needs the offset of the first vision token.
+ offset = _get_vision_token_offset(mm_feature.mm_position)
data = mm_feature.data
assert data is not None
if mm_feature.modality == "image":
diff --git a/vllm/model_executor/models/qwen2_vl.py b/vllm/model_executor/models/qwen2_vl.py
index 7f0ed0e6a779..807a392004c6 100644
--- a/vllm/model_executor/models/qwen2_vl.py
+++ b/vllm/model_executor/models/qwen2_vl.py
@@ -72,6 +72,7 @@
MultiModalFieldConfig,
MultiModalKwargsItem,
MultiModalKwargsItems,
+ PlaceholderRange,
VideoItem,
)
from vllm.multimodal.parse import (
@@ -87,8 +88,10 @@
BaseProcessingInfo,
PromptReplacement,
PromptUpdate,
+ PromptUpdateDetails,
)
from vllm.sequence import IntermediateTensors
+from vllm.tokenizers.protocol import TokenizerLike
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.attention.backends.registry import AttentionBackendEnum
@@ -1193,16 +1196,86 @@ def get_max_video_tokens(
)
+def safe_placeholder_str(
+ model_cls: type[nn.Module],
+ modality: str,
+) -> str | None:
+ """Return ``model_cls.get_placeholder_str(modality, 1)``, or None if the
+ model does not support the modality (returns None or raises)."""
+ try:
+ return model_cls.get_placeholder_str(modality, 1)
+ except Exception:
+ return None
+
+
+def get_wrapper_token_ids(
+ model_cls: type[nn.Module],
+ tokenizer: TokenizerLike,
+ modality: str,
+ pad_id: int,
+) -> list[int]:
+ """Derive the prompt-replacement target for `modality` from the model
+ class's own placeholder string.
+
+ The target is the complete wrapper emitted by the chat template
+ (e.g. ``<|vision_start|><|image_pad|><|vision_end|>`` for Qwen2-VL,
+ ``<|image_pad|>`` for EXAONE 4.5, or
+ ``<|img|><|imgpad|><|endofimg|>`` for dots.ocr), so that a bare pad
+ token typed by the user in text is not mistaken for a placeholder.
+
+ Falls back to the bare pad token if the wrapper cannot be determined.
+ """
+ placeholder_str = safe_placeholder_str(model_cls, modality)
+ if placeholder_str:
+ wrapper_ids = tokenizer.encode(placeholder_str, add_special_tokens=False)
+ if wrapper_ids.count(pad_id) == 1:
+ return wrapper_ids
+
+ return [pad_id]
+
+
+def _get_vision_token_offset(mm_position: PlaceholderRange) -> int:
+ """Return the prompt offset of the first vision token.
+
+ The prompt-replacement target is the complete wrapper (e.g.
+ ``<|vision_start|><|image_pad|><|vision_end|>``), so
+ ``mm_position.offset`` points at the opening wrapper token. M-RoPE
+ position generation expects the offset of the first vision token,
+ so skip any leading non-embedded wrapper tokens.
+ """
+ offset = mm_position.offset
+ is_embed = mm_position.is_embed
+ if is_embed is not None:
+ embed_idx = torch.nonzero(is_embed.flatten())
+ if embed_idx.numel():
+ offset += int(embed_idx[0])
+ return offset
+
+
class Qwen2VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen2VLProcessingInfo]):
+ def _get_model_cls(self) -> type[nn.Module]:
+ return Qwen2VLForConditionalGeneration
+
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
+ model_cls = self._get_model_cls()
+
num_images = mm_counts.get("image", 0)
num_videos = mm_counts.get("video", 0)
+ # Emit the complete wrapper (not the bare pad token) so that the
+ # dummy prompt matches the replacement targets in _get_prompt_updates.
+ # Fall back to the bare pad token when the model defines no
+ # placeholder string for the modality (e.g. Jina-VL video),
+ # consistent with the fallback in get_wrapper_token_ids.
hf_processor = self.info.get_hf_processor()
- image_token: str = hf_processor.image_token
- video_token: str = hf_processor.video_token
+ image_placeholder = (
+ safe_placeholder_str(model_cls, "image") or hf_processor.image_token
+ )
+ video_placeholder = (
+ safe_placeholder_str(model_cls, "video") or hf_processor.video_token
+ )
- return image_token * num_images + video_token * num_videos
+ return image_placeholder * num_images + video_placeholder * num_videos
def get_dummy_mm_data(
self,
@@ -1233,6 +1306,9 @@ def get_dummy_mm_data(
class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor[Qwen2VLProcessingInfo]):
+ def _get_model_cls(self) -> type[nn.Module]:
+ return Qwen2VLForConditionalGeneration
+
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
@@ -1244,30 +1320,62 @@ def _get_prompt_updates(
tokenizer = self.info.get_tokenizer()
vocab = tokenizer.get_vocab()
+ model_cls = self._get_model_cls()
+
placeholder = {
"image": vocab[hf_processor.image_token],
"video": vocab[hf_processor.video_token],
}
+ # Match the complete wrapper emitted by the chat template (derived
+ # from each model's own placeholder string) instead of the bare pad
+ # token, so that a pad token typed by the user is left untouched.
+ targets = {
+ modality: get_wrapper_token_ids(model_cls, tokenizer, modality, pad_id)
+ for modality, pad_id in placeholder.items()
+ }
merge_length = image_processor.merge_size**2
def get_replacement_qwen2vl(item_idx: int, modality: str):
- out_item = out_mm_kwargs[modality][item_idx]
- grid_thw = out_item[f"{modality}_grid_thw"].data
- assert isinstance(grid_thw, torch.Tensor)
+ num_tokens = self._get_num_pad_tokens(
+ item_idx, modality, out_mm_kwargs, image_processor, merge_length
+ )
- num_tokens = int(grid_thw.prod()) // merge_length
- return [placeholder[modality]] * num_tokens
+ pad_id = placeholder[modality]
+ wrapper_ids = targets[modality]
+ # get_wrapper_token_ids guarantees the pad token occurs exactly
+ # once in a derived wrapper, or the target is the bare pad token.
+ pad_idx = wrapper_ids.index(pad_id)
+ replacement = (
+ wrapper_ids[:pad_idx]
+ + [pad_id] * num_tokens
+ + wrapper_ids[pad_idx + 1 :]
+ )
+ return PromptUpdateDetails.select_token_id(replacement, pad_id)
return [
PromptReplacement(
modality=modality,
- target=[placeholder[modality]],
+ target=targets[modality],
replacement=partial(get_replacement_qwen2vl, modality=modality),
)
for modality in ("image", "video")
]
+ def _get_num_pad_tokens(
+ self,
+ item_idx: int,
+ modality: str,
+ out_mm_kwargs: MultiModalKwargsItems,
+ image_processor: Qwen2VLImageProcessor,
+ merge_length: int,
+ ) -> int:
+ out_item = out_mm_kwargs[modality][item_idx]
+ grid_thw = out_item[f"{modality}_grid_thw"].data
+ assert isinstance(grid_thw, torch.Tensor)
+
+ return int(grid_thw.prod()) // merge_length
+
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
@@ -1322,7 +1430,9 @@ def iter_mm_grid_thw(
spatial_merge_size = self.config.vision_config.spatial_merge_size
tokens_per_second = getattr(self.config.vision_config, "tokens_per_second", 1.0)
for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
- offset = mm_feature.mm_position.offset
+ # mm_position.offset points at the opening wrapper token;
+ # M-RoPE needs the offset of the first vision token.
+ offset = _get_vision_token_offset(mm_feature.mm_position)
data = mm_feature.data
assert data is not None
if mm_feature.modality == "image":