diff --git a/tests/models/multimodal/processing/test_qwen2_5_vl.py b/tests/models/multimodal/processing/test_qwen2_5_vl.py new file mode 100644 index 000000000000..9b6e00a8c0a0 --- /dev/null +++ b/tests/models/multimodal/processing/test_qwen2_5_vl.py @@ -0,0 +1,501 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Qwen2-VL family prompt expansion tests without downloading a checkpoint.""" + +from types import SimpleNamespace + +import pytest +import torch +from tokenizers import Tokenizer, models, pre_tokenizers +from transformers import PreTrainedTokenizerFast + +from vllm.model_executor.models.openpangu_vl import ( + OpenPanguVLMultiModalProcessor, +) +from vllm.model_executor.models.qwen2_5_vl import ( + Qwen2_5_VLForConditionalGeneration, + Qwen2_5_VLMultiModalProcessor, +) +from vllm.model_executor.models.qwen2_vl import ( + Qwen2VLDummyInputsBuilder, + Qwen2VLForConditionalGeneration, + Qwen2VLMultiModalProcessor, + _get_vision_token_offset, +) +from vllm.multimodal.inputs import PlaceholderRange + +pytestmark = [pytest.mark.cpu_test, pytest.mark.skip_global_cleanup] + + +def _make_processor(processor_cls, special_tokens, hf_processor, placeholder_strs): + vocab = {"[UNK]": 0, "before": 1, "after": 2} + backend = Tokenizer(models.WordLevel(vocab=vocab, unk_token="[UNK]")) + backend.pre_tokenizer = pre_tokenizers.Whitespace() + tokenizer = PreTrainedTokenizerFast( + tokenizer_object=backend, + additional_special_tokens=special_tokens, + ) + + def _placeholder_fn(modality, i): + result = placeholder_strs.get(modality) + if isinstance(result, BaseException): + raise result + return result + + model_cls = SimpleNamespace(get_placeholder_str=_placeholder_fn) + + processor = object.__new__(processor_cls) + # Processors expose the model class via _get_model_cls() (no + # model_loader indirection); point it at the mock. + processor._get_model_cls = lambda: model_cls + processor.info = SimpleNamespace( + get_hf_processor=lambda **kwargs: hf_processor, + get_image_processor=lambda **kwargs: SimpleNamespace(merge_size=2), + get_tokenizer=lambda: tokenizer, + ctx=SimpleNamespace( + model_config=SimpleNamespace(), + get_mm_config=lambda: SimpleNamespace(video_pruning_rate=None), + ), + ) + return processor + + +def _make_dummy_builder(processor): + """Create a dummy builder sharing the processor's mock model class.""" + builder = Qwen2VLDummyInputsBuilder(processor.info) + builder._get_model_cls = processor._get_model_cls + return builder + + +@pytest.fixture +def processor(): + return _make_processor( + Qwen2_5_VLMultiModalProcessor, + [ + "<|vision_start|>", + "<|image_pad|>", + "<|vision_end|>", + "<|video_pad|>", + ], + SimpleNamespace( + image_token="<|image_pad|>", + video_token="<|video_pad|>", + ), + { + "image": "<|vision_start|><|image_pad|><|vision_end|>", + "video": "<|vision_start|><|video_pad|><|vision_end|>", + }, + ) + + +@pytest.fixture +def qwen2_vl_processor(): + return _make_processor( + Qwen2VLMultiModalProcessor, + [ + "<|vision_start|>", + "<|image_pad|>", + "<|vision_end|>", + "<|video_pad|>", + ], + SimpleNamespace( + image_token="<|image_pad|>", + video_token="<|video_pad|>", + ), + { + "image": "<|vision_start|><|image_pad|><|vision_end|>", + "video": "<|vision_start|><|video_pad|><|vision_end|>", + }, + ) + + +@pytest.fixture +def exaone_style_processor(): + # EXAONE 4.5 reuses Qwen2VLMultiModalProcessor but its tokenizer has + # / instead of <|vision_start|>/<|vision_end|>. + return _make_processor( + Qwen2VLMultiModalProcessor, + [ + "", + "", + "<|image_pad|>", + "<|video_pad|>", + ], + SimpleNamespace( + image_token="<|image_pad|>", + video_token="<|video_pad|>", + ), + { + "image": "<|image_pad|>", + "video": "<|video_pad|>", + }, + ) + + +@pytest.fixture +def dots_ocr_style_processor(): + # dots.ocr registers Qwen2VLMultiModalProcessor directly with its own + # <|img|><|imgpad|><|endofimg|> wrapper. + return _make_processor( + Qwen2VLMultiModalProcessor, + [ + "<|img|>", + "<|imgpad|>", + "<|endofimg|>", + "<|video_pad|>", + ], + SimpleNamespace( + image_token="<|imgpad|>", + video_token="<|video_pad|>", + ), + { + "image": "<|img|><|imgpad|><|endofimg|>", + "video": None, + }, + ) + + +@pytest.fixture +def jina_style_processor(): + # Jina-VL inherits Qwen2VLMultiModalProcessor; its get_placeholder_str + # raises for video instead of returning None. + return _make_processor( + Qwen2VLMultiModalProcessor, + [ + "<|vision_start|>", + "<|image_pad|>", + "<|vision_end|>", + "<|video_pad|>", + ], + SimpleNamespace( + image_token="<|image_pad|>", + video_token="<|video_pad|>", + ), + { + "image": "<|vision_start|><|image_pad|><|vision_end|>", + "video": ValueError("Only image modality is supported"), + }, + ) + + +@pytest.fixture +def openpangu_vl_processor(): + return _make_processor( + OpenPanguVLMultiModalProcessor, + ["[unused18]", "[unused19]", "[unused20]", "[unused32]"], + SimpleNamespace( + image_token="[unused19]", + video_token="[unused32]", + ), + { + "image": "[unused18][unused19][unused20]", + "video": "[unused18][unused32][unused20]", + }, + ) + + +def _resolve_image_update(processor): + updates = processor._get_prompt_updates( + mm_items={}, + hf_processor_mm_kwargs={}, + out_mm_kwargs={ + "image": [ + {"image_grid_thw": SimpleNamespace(data=torch.tensor([1, 2, 4]))} + ], + "video": [], + }, + ) + return [updates[0].resolve(0)] + + +def test_qwen25_vl_does_not_match_bare_image_pad_in_user_text(processor): + tokenizer = processor.info.get_tokenizer() + vocab = tokenizer.get_vocab() + image_pad = vocab["<|image_pad|>"] + vision_start = vocab["<|vision_start|>"] + vision_end = vocab["<|vision_end|>"] + + image_update = _resolve_image_update(processor) + prompt = tokenizer.encode( + "before <|image_pad|> <|vision_start|><|image_pad|><|vision_end|> after", + add_special_tokens=False, + ) + + new_prompt, placeholders = processor._apply_prompt_updates( + prompt, + {"image": [image_update]}, + ) + + assert new_prompt.count(image_pad) == 3 + assert new_prompt.count(vision_start) == 1 + assert new_prompt.count(vision_end) == 1 + assert placeholders["image"][0].tokens == [ + vision_start, + image_pad, + image_pad, + vision_end, + ] + assert placeholders["image"][0].is_embed.tolist() == [False, True, True, False] + + +def test_qwen2_vl_does_not_match_bare_image_pad_in_user_text( + qwen2_vl_processor, +): + tokenizer = qwen2_vl_processor.info.get_tokenizer() + vocab = tokenizer.get_vocab() + image_pad = vocab["<|image_pad|>"] + vision_start = vocab["<|vision_start|>"] + vision_end = vocab["<|vision_end|>"] + + image_update = _resolve_image_update(qwen2_vl_processor) + prompt = tokenizer.encode( + "before <|image_pad|> <|vision_start|><|image_pad|><|vision_end|> after", + add_special_tokens=False, + ) + + new_prompt, placeholders = qwen2_vl_processor._apply_prompt_updates( + prompt, + {"image": [image_update]}, + ) + + assert new_prompt.count(image_pad) == 3 + assert new_prompt.count(vision_start) == 1 + assert new_prompt.count(vision_end) == 1 + assert placeholders["image"][0].tokens == [ + vision_start, + image_pad, + image_pad, + vision_end, + ] + assert placeholders["image"][0].is_embed.tolist() == [False, True, True, False] + + +def test_exaone_style_wrapper_derived_from_placeholder_str( + exaone_style_processor, +): + # EXAONE 4.5's tokenizer lacks <|vision_start|>; the target must be + # derived from its own placeholder string, not hardcoded. + tokenizer = exaone_style_processor.info.get_tokenizer() + vocab = tokenizer.get_vocab() + image_pad = vocab["<|image_pad|>"] + vision_start = vocab[""] + vision_end = vocab[""] + + image_update = _resolve_image_update(exaone_style_processor) + prompt = tokenizer.encode( + "before <|image_pad|> <|image_pad|> after", + add_special_tokens=False, + ) + + new_prompt, placeholders = exaone_style_processor._apply_prompt_updates( + prompt, + {"image": [image_update]}, + ) + + assert new_prompt.count(image_pad) == 3 + assert new_prompt.count(vision_start) == 1 + assert new_prompt.count(vision_end) == 1 + assert placeholders["image"][0].tokens == [ + vision_start, + image_pad, + image_pad, + vision_end, + ] + assert placeholders["image"][0].is_embed.tolist() == [False, True, True, False] + + +def test_dots_ocr_style_wrapper_derived_from_placeholder_str( + dots_ocr_style_processor, +): + # dots.ocr uses <|img|><|imgpad|><|endofimg|>; the shared processor must + # not assume the Qwen wrapper. + tokenizer = dots_ocr_style_processor.info.get_tokenizer() + vocab = tokenizer.get_vocab() + image_pad = vocab["<|imgpad|>"] + img_start = vocab["<|img|>"] + img_end = vocab["<|endofimg|>"] + + image_update = _resolve_image_update(dots_ocr_style_processor) + prompt = tokenizer.encode( + "before <|imgpad|> <|img|><|imgpad|><|endofimg|> after", + add_special_tokens=False, + ) + + new_prompt, placeholders = dots_ocr_style_processor._apply_prompt_updates( + prompt, + {"image": [image_update]}, + ) + + assert new_prompt.count(image_pad) == 3 + assert new_prompt.count(img_start) == 1 + assert new_prompt.count(img_end) == 1 + assert placeholders["image"][0].tokens == [ + img_start, + image_pad, + image_pad, + img_end, + ] + assert placeholders["image"][0].is_embed.tolist() == [False, True, True, False] + + +def test_jina_style_video_falls_back_to_bare_pad(jina_style_processor): + # get_placeholder_str raises for video; the processor must fall back + # to the bare pad token instead of crashing. + updates = jina_style_processor._get_prompt_updates( + mm_items={}, + hf_processor_mm_kwargs={}, + out_mm_kwargs={"image": [], "video": []}, + ) + tokenizer = jina_style_processor.info.get_tokenizer() + video_pad = tokenizer.get_vocab()["<|video_pad|>"] + + assert updates[1].target == [video_pad] + + +def test_openpangu_vl_does_not_match_bare_image_pad_in_user_text( + openpangu_vl_processor, +): + tokenizer = openpangu_vl_processor.info.get_tokenizer() + vocab = tokenizer.get_vocab() + image_pad = vocab["[unused19]"] + vision_start = vocab["[unused18]"] + vision_end = vocab["[unused20]"] + + image_update = _resolve_image_update(openpangu_vl_processor) + prompt = tokenizer.encode( + "before [unused19] [unused18][unused19][unused20] after", + add_special_tokens=False, + ) + + new_prompt, placeholders = openpangu_vl_processor._apply_prompt_updates( + prompt, + {"image": [image_update]}, + ) + + assert new_prompt.count(image_pad) == 3 + assert new_prompt.count(vision_start) == 1 + assert new_prompt.count(vision_end) == 1 + assert placeholders["image"][0].tokens == [ + vision_start, + image_pad, + image_pad, + vision_end, + ] + assert placeholders["image"][0].is_embed.tolist() == [False, True, True, False] + + +def test_dummy_text_emits_full_wrapper(qwen2_vl_processor): + # The dummy prompt must contain the complete wrapper so that it matches + # the replacement targets (used for profiling). + builder = _make_dummy_builder(qwen2_vl_processor) + + assert builder.get_dummy_text({"image": 1}) == ( + "<|vision_start|><|image_pad|><|vision_end|>" + ) + assert builder.get_dummy_text({"image": 1, "video": 1}) == ( + "<|vision_start|><|image_pad|><|vision_end|>" + "<|vision_start|><|video_pad|><|vision_end|>" + ) + + +def test_dummy_text_emits_full_wrapper_dots_ocr(dots_ocr_style_processor): + builder = _make_dummy_builder(dots_ocr_style_processor) + + assert builder.get_dummy_text({"image": 2}) == ( + "<|img|><|imgpad|><|endofimg|><|img|><|imgpad|><|endofimg|>" + ) + + +def test_dummy_text_falls_back_to_bare_pad_for_jina_video( + jina_style_processor, +): + # Jina-VL defines no video placeholder string; the dummy must fall back + # to the bare pad token, consistent with get_wrapper_token_ids, so that + # profiling with video still matches the video replacement target. + builder = _make_dummy_builder(jina_style_processor) + + assert builder.get_dummy_text({"image": 1, "video": 1}) == ( + "<|vision_start|><|image_pad|><|vision_end|><|video_pad|>" + ) + + +def test_vision_token_offset_skips_leading_non_embedded_tokens(): + # mm_position.offset points at the opening wrapper token; the first + # vision token starts after the leading non-embedded wrapper tokens. + assert ( + _get_vision_token_offset( + PlaceholderRange( + offset=5, + length=6, + is_embed=torch.tensor([False, True, True, True, True, False]), + ) + ) + == 6 + ) + # No embed mask (or a bare pad target): the offset already points at + # the first vision token. + assert _get_vision_token_offset(PlaceholderRange(offset=5, length=4)) == 5 + assert ( + _get_vision_token_offset( + PlaceholderRange(offset=5, length=1, is_embed=torch.tensor([True])) + ) + == 5 + ) + + +def test_mrope_positions_skip_wrapper_prefix(): + # The prompt-replacement target is the complete wrapper, so + # mm_position.offset points at <|vision_start|>. M-RoPE must assign + # vision positions starting from the first <|image_pad|> instead of + # shifting every position by one. + model = object.__new__(Qwen2VLForConditionalGeneration) + model.config = SimpleNamespace( + vision_config=SimpleNamespace(spatial_merge_size=2, tokens_per_second=1.0) + ) + feature = SimpleNamespace( + modality="image", + mm_position=PlaceholderRange( + offset=5, + length=6, + is_embed=torch.tensor([False, True, True, True, True, False]), + ), + data={"image_grid_thw": SimpleNamespace(data=torch.tensor([1, 4, 4]))}, + ) + # 5 text tokens, wrapper (6 tokens), 2 text tokens. + positions, _ = model.get_mrope_input_positions(list(range(13)), [feature]) + + assert positions.shape == (3, 13) + # t positions: text 0-5, vision grid (1,2,2) t=0 -> 6, text 8-10. + assert positions[0].tolist() == [0, 1, 2, 3, 4, 5, 6, 6, 6, 6, 8, 9, 10] + # h positions: vision grid h=[0,0,1,1] -> [6,6,7,7]. + assert positions[1].tolist() == [0, 1, 2, 3, 4, 5, 6, 6, 7, 7, 8, 9, 10] + # w positions: vision grid w=[0,1,0,1] -> [6,7,6,7]. + assert positions[2].tolist() == [0, 1, 2, 3, 4, 5, 6, 7, 6, 7, 8, 9, 10] + + +def test_qwen25_vl_mrope_positions_skip_wrapper_prefix(): + # Qwen2.5-VL's model class does not inherit from Qwen2-VL's model class, + # so it needs its own M-RoPE implementation: falling back to + # SupportsMRoPE's empty method returns None and breaks unpacking in + # the runner. This also covers EXAONE 4.5 and OpenCUA, which inherit + # from Qwen2.5-VL's model class. + model = object.__new__(Qwen2_5_VLForConditionalGeneration) + model.config = SimpleNamespace( + vision_config=SimpleNamespace(spatial_merge_size=2, tokens_per_second=1.0) + ) + feature = SimpleNamespace( + modality="image", + mm_position=PlaceholderRange( + offset=5, + length=6, + is_embed=torch.tensor([False, True, True, True, True, False]), + ), + data={"image_grid_thw": SimpleNamespace(data=torch.tensor([1, 4, 4]))}, + ) + # 5 text tokens, wrapper (6 tokens), 2 text tokens. + positions, _ = model.get_mrope_input_positions(list(range(13)), [feature]) + + assert positions is not None + assert positions.shape == (3, 13) + # h positions: text 0-5, vision grid h=[0,0,1,1] -> [6,6,7,7], text 8-10. + assert positions[1].tolist() == [0, 1, 2, 3, 4, 5, 6, 6, 7, 7, 8, 9, 10] diff --git a/vllm/model_executor/models/dots_ocr.py b/vllm/model_executor/models/dots_ocr.py index adcc560ba5ea..e26dcbbd1665 100644 --- a/vllm/model_executor/models/dots_ocr.py +++ b/vllm/model_executor/models/dots_ocr.py @@ -96,9 +96,12 @@ class DotsOCRImageEmbeddingInputs(TensorSchema): class DotsOCRDummyInputsBuilder(Qwen2VLDummyInputsBuilder): - def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str: - num_images = mm_counts.get("image", 0) - return IMAGE_TOKEN * num_images + def _get_model_cls(self) -> type[nn.Module]: + return DotsOCRForCausalLM + + # get_dummy_text is inherited: it emits the complete wrapper derived + # from get_placeholder_str ("<|img|><|imgpad|><|endofimg|>"), which + # matches the replacement targets in _get_prompt_updates. def get_dummy_mm_data( self, @@ -578,8 +581,13 @@ def forward( return hidden_states +class DotsOCRMultiModalProcessor(Qwen2VLMultiModalProcessor): + def _get_model_cls(self) -> type[nn.Module]: + return DotsOCRForCausalLM + + @MULTIMODAL_REGISTRY.register_processor( - Qwen2VLMultiModalProcessor, + DotsOCRMultiModalProcessor, info=DotsOCRProcessingInfo, dummy_inputs=DotsOCRDummyInputsBuilder, ) diff --git a/vllm/model_executor/models/exaone4_5.py b/vllm/model_executor/models/exaone4_5.py index 5a53e60b7d5e..bbb880d228dc 100644 --- a/vllm/model_executor/models/exaone4_5.py +++ b/vllm/model_executor/models/exaone4_5.py @@ -50,8 +50,7 @@ ) from vllm.multimodal import MULTIMODAL_REGISTRY -from .qwen2_vl import Qwen2VLDummyInputsBuilder as Exaone4_5_DummyInputsBuilder -from .qwen2_vl import Qwen2VLMultiModalProcessor as Exaone4_5_MultiModalProcessor +from .qwen2_vl import Qwen2VLDummyInputsBuilder, Qwen2VLMultiModalProcessor from .utils import ( AutoWeightsLoader, WeightsMapper, @@ -315,6 +314,16 @@ def get_hf_processor(self, **kwargs: object) -> Exaone4_5_Processor: ) +class Exaone4_5_MultiModalProcessor(Qwen2VLMultiModalProcessor): + def _get_model_cls(self) -> type[nn.Module]: + return Exaone4_5_ForConditionalGeneration + + +class Exaone4_5_DummyInputsBuilder(Qwen2VLDummyInputsBuilder): + def _get_model_cls(self) -> type[nn.Module]: + return Exaone4_5_ForConditionalGeneration + + @MULTIMODAL_REGISTRY.register_processor( Exaone4_5_MultiModalProcessor, info=Exaone4_5_ProcessingInfo, diff --git a/vllm/model_executor/models/jina_vl.py b/vllm/model_executor/models/jina_vl.py index 971284561926..fda71d2efd69 100644 --- a/vllm/model_executor/models/jina_vl.py +++ b/vllm/model_executor/models/jina_vl.py @@ -59,6 +59,9 @@ def forward(self, x, **kwargs): class JinaVLMultiModalProcessor(Qwen2VLMultiModalProcessor): + def _get_model_cls(self) -> type[nn.Module]: + return JinaVLForSequenceClassification + def _cached_apply_hf_processor( self, inputs: ProcessorInputs, @@ -96,10 +99,15 @@ def _cached_apply_hf_processor( ) +class JinaVLDummyInputsBuilder(Qwen2VLDummyInputsBuilder): + def _get_model_cls(self) -> type[nn.Module]: + return JinaVLForSequenceClassification + + @MULTIMODAL_REGISTRY.register_processor( JinaVLMultiModalProcessor, info=Qwen2VLProcessingInfo, - dummy_inputs=Qwen2VLDummyInputsBuilder, + dummy_inputs=JinaVLDummyInputsBuilder, ) class JinaVLForSequenceClassification( Qwen2VLForConditionalGeneration, diff --git a/vllm/model_executor/models/openpangu_vl.py b/vllm/model_executor/models/openpangu_vl.py index 8a62aa6ba6b4..b72ce1bf47ce 100644 --- a/vllm/model_executor/models/openpangu_vl.py +++ b/vllm/model_executor/models/openpangu_vl.py @@ -60,6 +60,10 @@ Qwen2_5_VLMultiModalProcessor, Qwen2_5_VLProcessingInfo, ) +from vllm.model_executor.models.qwen2_vl import ( + _get_vision_token_offset, + get_wrapper_token_ids, +) from vllm.model_executor.models.utils import ( AutoWeightsLoader, WeightsMapper, @@ -761,18 +765,25 @@ def _get_prompt_updates( image_processor = self.info.get_image_processor(**hf_processor_mm_kwargs) tokenizer = self.info.get_tokenizer() vocab = tokenizer.get_vocab() + image_token = hf_processor.image_token video_token = hf_processor.video_token - vision_start_token = hf_processor.vision_start_token - vision_end_token = hf_processor.vision_end_token image_token_id = vocab[image_token] video_token_id = vocab[video_token] - vision_start_token_id = vocab[vision_start_token] - vision_end_token_id = vocab[vision_end_token] placeholder = { "image": image_token_id, "video": video_token_id, } + # Match the complete wrapper emitted by the chat template (derived + # from the model's own placeholder string: + # [unused18][unused19][unused20] for images, + # [unused18][unused32][unused20] for videos). + targets = { + modality: get_wrapper_token_ids( + OpenPanguVLForConditionalGeneration, tokenizer, modality, pad_id + ) + for modality, pad_id in placeholder.items() + } merge_length = image_processor.merge_size**2 @@ -781,31 +792,31 @@ def get_replacement_openpangu_vision(item_idx: int, modality: str): grid_thw = out_item[f"{modality}_grid_thw"].data if not isinstance(grid_thw, torch.Tensor): raise TypeError("Expected 'grid_thw' to be a Tensor") + + pad_id = placeholder[modality] + wrapper_ids = targets[modality] + # get_wrapper_token_ids guarantees the pad token occurs exactly + # once in a derived wrapper, or the target is the bare pad token. + pad_idx = wrapper_ids.index(pad_id) + prefix = wrapper_ids[:pad_idx] + suffix = wrapper_ids[pad_idx + 1 :] + if modality == "image": - image_token_id_total = [image_token_id] * ( - int(grid_thw.prod()) // merge_length - ) - return image_token_id_total + num_tokens = int(grid_thw.prod()) // merge_length + replacement = prefix + [pad_id] * num_tokens + suffix else: - # When modality is video + # Video placeholders are expanded per frame, with the outer + # wrapper tokens stripped (preserving the original layout). grid_t, grid_h, grid_w = grid_thw - video_seq_length_per_time = (grid_h * grid_w).item() // merge_length - video_token_id_per_time = ( - [vision_start_token_id] - + [video_token_id] * video_seq_length_per_time - + [vision_end_token_id] - ) - video_token_id_total = video_token_id_per_time * grid_t.item() - video_token_id_middle = video_token_id_total[1:-1] - return PromptUpdateDetails.select_token_id( - video_token_id_middle, - embed_token_id=video_token_id, - ) + tokens_per_frame = (grid_h * grid_w).item() // merge_length + per_frame = prefix + [pad_id] * tokens_per_frame + suffix + replacement = (per_frame * grid_t.item())[1:-1] + return PromptUpdateDetails.select_token_id(replacement, pad_id) return [ PromptReplacement( modality=modality, - target=[placeholder[modality]], + target=targets[modality], replacement=partial( get_replacement_openpangu_vision, modality=modality ), @@ -815,7 +826,8 @@ def get_replacement_openpangu_vision(item_idx: int, modality: str): class OpenPanguVLDummyInputsBuilder(Qwen2_5_VLDummyInputsBuilder): - pass + def _get_model_cls(self) -> type[nn.Module]: + return OpenPanguVLForConditionalGeneration @MULTIMODAL_REGISTRY.register_processor( @@ -1174,7 +1186,9 @@ def iter_mm_grid_thw( ) -> Iterator[tuple[str, int, int, int, int]]: spatial_merge_size = self.config.vision_config.spatial_merge_size for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset): - offset = mm_feature.mm_position.offset + # mm_position.offset points at the opening wrapper token; + # M-RoPE needs the offset of the first vision token. + offset = _get_vision_token_offset(mm_feature.mm_position) modality = mm_feature.modality feature_data = mm_feature.data assert feature_data is not None diff --git a/vllm/model_executor/models/qwen2_5_vl.py b/vllm/model_executor/models/qwen2_5_vl.py index 581ff7730351..828c8b74af03 100644 --- a/vllm/model_executor/models/qwen2_5_vl.py +++ b/vllm/model_executor/models/qwen2_5_vl.py @@ -81,7 +81,6 @@ VideoItem, ) from vllm.multimodal.parse import MultiModalDataItems -from vllm.multimodal.processing import PromptReplacement, PromptUpdate from vllm.multimodal.processing.processor import HFMultiModalInputs from vllm.multimodal.video_prune.evs import ( compute_mrope_for_media, @@ -112,9 +111,11 @@ ) from .qwen2_vl import ( Qwen2VLDummyInputsBuilder, + Qwen2VLImageProcessor, Qwen2VLMultiModalDataParser, Qwen2VLMultiModalProcessor, Qwen2VLProcessingInfo, + _get_vision_token_offset, ) from .utils import ( AutoWeightsLoader, @@ -1148,6 +1149,9 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: class Qwen2_5_VLDummyInputsBuilder(Qwen2VLDummyInputsBuilder): + def _get_model_cls(self) -> type[nn.Module]: + return Qwen2_5_VLForConditionalGeneration + def _get_dummy_videos( self, *, @@ -1201,6 +1205,9 @@ def get_data_parser(self): class Qwen2_5_VLMultiModalProcessor(Qwen2VLMultiModalProcessor): + def _get_model_cls(self) -> type[nn.Module]: + return Qwen2_5_VLForConditionalGeneration + def _get_mm_fields_config( self, hf_inputs: BatchFeature, @@ -1231,59 +1238,39 @@ def _get_hf_mm_inputs( ] return hf_inputs - def _get_prompt_updates( + def _get_num_pad_tokens( self, - mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, Any], + item_idx: int, + modality: str, out_mm_kwargs: MultiModalKwargsItems, - ) -> Sequence[PromptUpdate]: - hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) - image_processor = self.info.get_image_processor(**hf_processor_mm_kwargs) - tokenizer = self.info.get_tokenizer() - vocab = tokenizer.get_vocab() - - placeholder = { - "image": vocab[hf_processor.image_token], - "video": vocab[hf_processor.video_token], - } - - merge_length = image_processor.merge_size**2 - - def get_replacement_qwen2vl(item_idx: int, modality: str): - out_item = out_mm_kwargs[modality][item_idx] - grid_thw = out_item[f"{modality}_grid_thw"].data - assert isinstance(grid_thw, torch.Tensor) - - num_tokens = int(grid_thw.prod()) // merge_length - - # EVS-specific code - video_pruning_rate = self.info.ctx.get_mm_config().video_pruning_rate - if ( - modality == "video" - and video_pruning_rate is not None - and video_pruning_rate > 0.0 - ): - T, H, W = map(int, grid_thw) - tokens_per_frame = (H // image_processor.merge_size) * ( - W // image_processor.merge_size - ) - num_tokens = compute_retained_tokens_count( - tokens_per_frame, - T, - video_pruning_rate, - ) - # End of EVS-specific code - - return [placeholder[modality]] * num_tokens - - return [ - PromptReplacement( - modality=modality, - target=[placeholder[modality]], - replacement=partial(get_replacement_qwen2vl, modality=modality), + image_processor: Qwen2VLImageProcessor, + merge_length: int, + ) -> int: + out_item = out_mm_kwargs[modality][item_idx] + grid_thw = out_item[f"{modality}_grid_thw"].data + assert isinstance(grid_thw, torch.Tensor) + + num_tokens = int(grid_thw.prod()) // merge_length + + # EVS: Explored Visual-token Selection. + # Prune redundant video tokens to reduce prefill cost. + video_pruning_rate = self.info.ctx.get_mm_config().video_pruning_rate + if ( + modality == "video" + and video_pruning_rate is not None + and video_pruning_rate > 0.0 + ): + T, H, W = map(int, grid_thw) + tokens_per_frame = (H // image_processor.merge_size) * ( + W // image_processor.merge_size ) - for modality in ("image", "video") - ] + num_tokens = compute_retained_tokens_count( + tokens_per_frame, + T, + video_pruning_rate, + ) + + return num_tokens @MULTIMODAL_REGISTRY.register_processor( @@ -1340,7 +1327,9 @@ def iter_mm_grid_thw( spatial_merge_size = self.config.vision_config.spatial_merge_size tokens_per_second = getattr(self.config.vision_config, "tokens_per_second", 1.0) for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset): - offset = mm_feature.mm_position.offset + # mm_position.offset points at the opening wrapper token; + # M-RoPE needs the offset of the first vision token. + offset = _get_vision_token_offset(mm_feature.mm_position) data = mm_feature.data assert data is not None if mm_feature.modality == "image": diff --git a/vllm/model_executor/models/qwen2_vl.py b/vllm/model_executor/models/qwen2_vl.py index 7f0ed0e6a779..807a392004c6 100644 --- a/vllm/model_executor/models/qwen2_vl.py +++ b/vllm/model_executor/models/qwen2_vl.py @@ -72,6 +72,7 @@ MultiModalFieldConfig, MultiModalKwargsItem, MultiModalKwargsItems, + PlaceholderRange, VideoItem, ) from vllm.multimodal.parse import ( @@ -87,8 +88,10 @@ BaseProcessingInfo, PromptReplacement, PromptUpdate, + PromptUpdateDetails, ) from vllm.sequence import IntermediateTensors +from vllm.tokenizers.protocol import TokenizerLike from vllm.utils.tensor_schema import TensorSchema, TensorShape from vllm.utils.torch_utils import async_tensor_h2d from vllm.v1.attention.backends.registry import AttentionBackendEnum @@ -1193,16 +1196,86 @@ def get_max_video_tokens( ) +def safe_placeholder_str( + model_cls: type[nn.Module], + modality: str, +) -> str | None: + """Return ``model_cls.get_placeholder_str(modality, 1)``, or None if the + model does not support the modality (returns None or raises).""" + try: + return model_cls.get_placeholder_str(modality, 1) + except Exception: + return None + + +def get_wrapper_token_ids( + model_cls: type[nn.Module], + tokenizer: TokenizerLike, + modality: str, + pad_id: int, +) -> list[int]: + """Derive the prompt-replacement target for `modality` from the model + class's own placeholder string. + + The target is the complete wrapper emitted by the chat template + (e.g. ``<|vision_start|><|image_pad|><|vision_end|>`` for Qwen2-VL, + ``<|image_pad|>`` for EXAONE 4.5, or + ``<|img|><|imgpad|><|endofimg|>`` for dots.ocr), so that a bare pad + token typed by the user in text is not mistaken for a placeholder. + + Falls back to the bare pad token if the wrapper cannot be determined. + """ + placeholder_str = safe_placeholder_str(model_cls, modality) + if placeholder_str: + wrapper_ids = tokenizer.encode(placeholder_str, add_special_tokens=False) + if wrapper_ids.count(pad_id) == 1: + return wrapper_ids + + return [pad_id] + + +def _get_vision_token_offset(mm_position: PlaceholderRange) -> int: + """Return the prompt offset of the first vision token. + + The prompt-replacement target is the complete wrapper (e.g. + ``<|vision_start|><|image_pad|><|vision_end|>``), so + ``mm_position.offset`` points at the opening wrapper token. M-RoPE + position generation expects the offset of the first vision token, + so skip any leading non-embedded wrapper tokens. + """ + offset = mm_position.offset + is_embed = mm_position.is_embed + if is_embed is not None: + embed_idx = torch.nonzero(is_embed.flatten()) + if embed_idx.numel(): + offset += int(embed_idx[0]) + return offset + + class Qwen2VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen2VLProcessingInfo]): + def _get_model_cls(self) -> type[nn.Module]: + return Qwen2VLForConditionalGeneration + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str: + model_cls = self._get_model_cls() + num_images = mm_counts.get("image", 0) num_videos = mm_counts.get("video", 0) + # Emit the complete wrapper (not the bare pad token) so that the + # dummy prompt matches the replacement targets in _get_prompt_updates. + # Fall back to the bare pad token when the model defines no + # placeholder string for the modality (e.g. Jina-VL video), + # consistent with the fallback in get_wrapper_token_ids. hf_processor = self.info.get_hf_processor() - image_token: str = hf_processor.image_token - video_token: str = hf_processor.video_token + image_placeholder = ( + safe_placeholder_str(model_cls, "image") or hf_processor.image_token + ) + video_placeholder = ( + safe_placeholder_str(model_cls, "video") or hf_processor.video_token + ) - return image_token * num_images + video_token * num_videos + return image_placeholder * num_images + video_placeholder * num_videos def get_dummy_mm_data( self, @@ -1233,6 +1306,9 @@ def get_dummy_mm_data( class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor[Qwen2VLProcessingInfo]): + def _get_model_cls(self) -> type[nn.Module]: + return Qwen2VLForConditionalGeneration + def _get_prompt_updates( self, mm_items: MultiModalDataItems, @@ -1244,30 +1320,62 @@ def _get_prompt_updates( tokenizer = self.info.get_tokenizer() vocab = tokenizer.get_vocab() + model_cls = self._get_model_cls() + placeholder = { "image": vocab[hf_processor.image_token], "video": vocab[hf_processor.video_token], } + # Match the complete wrapper emitted by the chat template (derived + # from each model's own placeholder string) instead of the bare pad + # token, so that a pad token typed by the user is left untouched. + targets = { + modality: get_wrapper_token_ids(model_cls, tokenizer, modality, pad_id) + for modality, pad_id in placeholder.items() + } merge_length = image_processor.merge_size**2 def get_replacement_qwen2vl(item_idx: int, modality: str): - out_item = out_mm_kwargs[modality][item_idx] - grid_thw = out_item[f"{modality}_grid_thw"].data - assert isinstance(grid_thw, torch.Tensor) + num_tokens = self._get_num_pad_tokens( + item_idx, modality, out_mm_kwargs, image_processor, merge_length + ) - num_tokens = int(grid_thw.prod()) // merge_length - return [placeholder[modality]] * num_tokens + pad_id = placeholder[modality] + wrapper_ids = targets[modality] + # get_wrapper_token_ids guarantees the pad token occurs exactly + # once in a derived wrapper, or the target is the bare pad token. + pad_idx = wrapper_ids.index(pad_id) + replacement = ( + wrapper_ids[:pad_idx] + + [pad_id] * num_tokens + + wrapper_ids[pad_idx + 1 :] + ) + return PromptUpdateDetails.select_token_id(replacement, pad_id) return [ PromptReplacement( modality=modality, - target=[placeholder[modality]], + target=targets[modality], replacement=partial(get_replacement_qwen2vl, modality=modality), ) for modality in ("image", "video") ] + def _get_num_pad_tokens( + self, + item_idx: int, + modality: str, + out_mm_kwargs: MultiModalKwargsItems, + image_processor: Qwen2VLImageProcessor, + merge_length: int, + ) -> int: + out_item = out_mm_kwargs[modality][item_idx] + grid_thw = out_item[f"{modality}_grid_thw"].data + assert isinstance(grid_thw, torch.Tensor) + + return int(grid_thw.prod()) // merge_length + def _get_mm_fields_config( self, hf_inputs: BatchFeature, @@ -1322,7 +1430,9 @@ def iter_mm_grid_thw( spatial_merge_size = self.config.vision_config.spatial_merge_size tokens_per_second = getattr(self.config.vision_config, "tokens_per_second", 1.0) for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset): - offset = mm_feature.mm_position.offset + # mm_position.offset points at the opening wrapper token; + # M-RoPE needs the offset of the first vision token. + offset = _get_vision_token_offset(mm_feature.mm_position) data = mm_feature.data assert data is not None if mm_feature.modality == "image":