From 1bc8578e56edb411485027f38be3c51d6caabb8f Mon Sep 17 00:00:00 2001 From: Luyi Xiao <81631911+luyixiao95@users.noreply.github.com> Date: Tue, 1 Sep 2026 21:31:08 -0700 Subject: [PATCH 1/5] [Bugfix] Reject tokenizer-less Qwen VL processor init Signed-off-by: Luyi Xiao <81631911+luyixiao95@users.noreply.github.com> --- .../models/multimodal/processing/test_qwen2_vl.py | 14 ++++++++++++++ vllm/model_executor/models/qwen2_5_vl.py | 1 + vllm/model_executor/models/qwen2_vl.py | 8 ++++++++ 3 files changed, 23 insertions(+) diff --git a/tests/models/multimodal/processing/test_qwen2_vl.py b/tests/models/multimodal/processing/test_qwen2_vl.py index 5f3f5b05e106..7aeab830db5a 100644 --- a/tests/models/multimodal/processing/test_qwen2_vl.py +++ b/tests/models/multimodal/processing/test_qwen2_vl.py @@ -1,21 +1,35 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from unittest.mock import Mock + import pytest import torch from packaging.version import Version from PIL import Image from transformers import __version__ as TRANSFORMERS_VERSION +from vllm.model_executor.models.qwen2_5_vl import Qwen2_5_VLProcessingInfo +from vllm.model_executor.models.qwen2_vl import Qwen2VLProcessingInfo from vllm.model_executor.models.vision import FusedInputNorm from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.cache import MultiModalProcessorOnlyCache from vllm.multimodal.inputs import batched_tensors_equal +from vllm.multimodal.processing import InputProcessingContext from ....conftest import ImageTestAssets from ...utils import build_model_context +@pytest.mark.parametrize("info_cls", [Qwen2VLProcessingInfo, Qwen2_5_VLProcessingInfo]) +def test_hf_processor_requires_tokenizer(info_cls) -> None: + ctx = InputProcessingContext(model_config=Mock(), tokenizer=None) + info = info_cls(ctx) + + with pytest.raises(ValueError, match="`skip_tokenizer_init=True`"): + info.get_hf_processor() + + def test_jina_vl_processing_order() -> None: """Jina's document-first prompt keeps cached features and hashes aligned.""" ctx = build_model_context( diff --git a/vllm/model_executor/models/qwen2_5_vl.py b/vllm/model_executor/models/qwen2_5_vl.py index 8b214f6acb43..85d6179e753c 100644 --- a/vllm/model_executor/models/qwen2_5_vl.py +++ b/vllm/model_executor/models/qwen2_5_vl.py @@ -1141,6 +1141,7 @@ def get_hf_config(self): return self.ctx.get_hf_config(Qwen2_5_VLConfig) def get_hf_processor(self, **kwargs: object) -> Qwen2_5_VLProcessor: + self._require_tokenizer_for_hf_processor() return self.ctx.get_hf_processor( Qwen2_5_VLProcessor, use_fast=kwargs.pop("use_fast", True), diff --git a/vllm/model_executor/models/qwen2_vl.py b/vllm/model_executor/models/qwen2_vl.py index 2b0d67419870..468ba834192b 100644 --- a/vllm/model_executor/models/qwen2_vl.py +++ b/vllm/model_executor/models/qwen2_vl.py @@ -844,7 +844,15 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo): def get_hf_config(self): return self.ctx.get_hf_config(Qwen2VLConfig) + def _require_tokenizer_for_hf_processor(self) -> None: + if self.ctx.tokenizer is None: + raise ValueError( + "Cannot initialize the Qwen2-VL multimodal processor when " + "`skip_tokenizer_init=True`. Disable `--skip-tokenizer-init`." + ) + def get_hf_processor(self, **kwargs: object) -> Qwen2VLProcessor: + self._require_tokenizer_for_hf_processor() return self.ctx.get_hf_processor( Qwen2VLProcessor, use_fast=kwargs.pop("use_fast", True), From cbc97c085399be1ab9cdd02a6e51583bf2d55a82 Mon Sep 17 00:00:00 2001 From: Luyi Xiao <81631911+luyixiao95@users.noreply.github.com> Date: Tue, 1 Sep 2026 23:40:34 -0700 Subject: [PATCH 2/5] [Bugfix] Generalize multimodal tokenizer validation Signed-off-by: Luyi Xiao <81631911+luyixiao95@users.noreply.github.com> --- .../multimodal/processing/test_qwen2_vl.py | 14 ---------- tests/multimodal/test_processing.py | 28 +++++++++++++++++++ vllm/model_executor/models/qwen2_5_vl.py | 1 - vllm/model_executor/models/qwen2_vl.py | 8 ------ vllm/model_executor/models/terratorch.py | 3 ++ vllm/multimodal/processing/processor.py | 9 ++++++ 6 files changed, 40 insertions(+), 23 deletions(-) diff --git a/tests/models/multimodal/processing/test_qwen2_vl.py b/tests/models/multimodal/processing/test_qwen2_vl.py index 7aeab830db5a..5f3f5b05e106 100644 --- a/tests/models/multimodal/processing/test_qwen2_vl.py +++ b/tests/models/multimodal/processing/test_qwen2_vl.py @@ -1,35 +1,21 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from unittest.mock import Mock - import pytest import torch from packaging.version import Version from PIL import Image from transformers import __version__ as TRANSFORMERS_VERSION -from vllm.model_executor.models.qwen2_5_vl import Qwen2_5_VLProcessingInfo -from vllm.model_executor.models.qwen2_vl import Qwen2VLProcessingInfo from vllm.model_executor.models.vision import FusedInputNorm from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.cache import MultiModalProcessorOnlyCache from vllm.multimodal.inputs import batched_tensors_equal -from vllm.multimodal.processing import InputProcessingContext from ....conftest import ImageTestAssets from ...utils import build_model_context -@pytest.mark.parametrize("info_cls", [Qwen2VLProcessingInfo, Qwen2_5_VLProcessingInfo]) -def test_hf_processor_requires_tokenizer(info_cls) -> None: - ctx = InputProcessingContext(model_config=Mock(), tokenizer=None) - info = info_cls(ctx) - - with pytest.raises(ValueError, match="`skip_tokenizer_init=True`"): - info.get_hf_processor() - - def test_jina_vl_processing_order() -> None: """Jina's document-first prompt keeps cached features and hashes aligned.""" ctx = build_model_context( diff --git a/tests/multimodal/test_processing.py b/tests/multimodal/test_processing.py index 6e508af2338a..36061e385b1b 100644 --- a/tests/multimodal/test_processing.py +++ b/tests/multimodal/test_processing.py @@ -1122,10 +1122,14 @@ def decode(self, token_ids: list[int], **kwargs) -> str: class _FakeProcessingInfo: def __init__(self, tokenizer) -> None: self._tokenizer = tokenizer + self.ctx = type("Context", (), {"tokenizer": tokenizer})() def get_tokenizer(self): return self._tokenizer + def get_data_parser(self): + return MultiModalDataParser() + class _TextFallbackProcessor(BaseMultiModalProcessor): """Only `self.info.get_tokenizer()` is needed by the text fallback.""" @@ -1140,6 +1144,30 @@ def _get_prompt_updates(self, mm_items, hf_processor_mm_kwargs, out_mm_kwargs): raise NotImplementedError +class _TokenizerFreeProcessor(_TextFallbackProcessor): + requires_tokenizer = False + + +def test_multimodal_processor_requires_tokenizer_by_default(): + info = _FakeProcessingInfo(tokenizer=None) + + with pytest.raises(ValueError, match="`skip_tokenizer_init=True`"): + BaseMultiModalProcessor.__init__( + _TextFallbackProcessor.__new__(_TextFallbackProcessor), + info, + dummy_inputs=None, + ) + + +def test_multimodal_processor_can_allow_tokenizer_free_init(): + info = _FakeProcessingInfo(tokenizer=None) + processor = _TokenizerFreeProcessor.__new__(_TokenizerFreeProcessor) + + BaseMultiModalProcessor.__init__(processor, info, dummy_inputs=None) + + assert processor.info is info + + def _text_fallback_processor() -> BaseMultiModalProcessor: return _TextFallbackProcessor(_FakeTokenizer()) diff --git a/vllm/model_executor/models/qwen2_5_vl.py b/vllm/model_executor/models/qwen2_5_vl.py index 85d6179e753c..8b214f6acb43 100644 --- a/vllm/model_executor/models/qwen2_5_vl.py +++ b/vllm/model_executor/models/qwen2_5_vl.py @@ -1141,7 +1141,6 @@ def get_hf_config(self): return self.ctx.get_hf_config(Qwen2_5_VLConfig) def get_hf_processor(self, **kwargs: object) -> Qwen2_5_VLProcessor: - self._require_tokenizer_for_hf_processor() return self.ctx.get_hf_processor( Qwen2_5_VLProcessor, use_fast=kwargs.pop("use_fast", True), diff --git a/vllm/model_executor/models/qwen2_vl.py b/vllm/model_executor/models/qwen2_vl.py index 468ba834192b..2b0d67419870 100644 --- a/vllm/model_executor/models/qwen2_vl.py +++ b/vllm/model_executor/models/qwen2_vl.py @@ -844,15 +844,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo): def get_hf_config(self): return self.ctx.get_hf_config(Qwen2VLConfig) - def _require_tokenizer_for_hf_processor(self) -> None: - if self.ctx.tokenizer is None: - raise ValueError( - "Cannot initialize the Qwen2-VL multimodal processor when " - "`skip_tokenizer_init=True`. Disable `--skip-tokenizer-init`." - ) - def get_hf_processor(self, **kwargs: object) -> Qwen2VLProcessor: - self._require_tokenizer_for_hf_processor() return self.ctx.get_hf_processor( Qwen2VLProcessor, use_fast=kwargs.pop("use_fast", True), diff --git a/vllm/model_executor/models/terratorch.py b/vllm/model_executor/models/terratorch.py index 78bd541b7aaf..7181680fe833 100644 --- a/vllm/model_executor/models/terratorch.py +++ b/vllm/model_executor/models/terratorch.py @@ -169,6 +169,9 @@ def get_dummy_mm_data( class TerratorchMultiModalProcessor(BaseMultiModalProcessor[TerratorchProcessingInfo]): + # Terratorch models, including Prithvi, consume tokenizer-free inputs. + requires_tokenizer = False + def _get_mm_fields_config( self, hf_inputs: BatchFeature, diff --git a/vllm/multimodal/processing/processor.py b/vllm/multimodal/processing/processor.py index afae85476246..803b638f1603 100644 --- a/vllm/multimodal/processing/processor.py +++ b/vllm/multimodal/processing/processor.py @@ -1033,6 +1033,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): Not to be confused with `transformers.ProcessorMixin`. """ + requires_tokenizer = True + def __init__( self, info: _I, @@ -1042,6 +1044,13 @@ def __init__( ) -> None: super().__init__() + if self.requires_tokenizer and info.ctx.tokenizer is None: + raise ValueError( + f"{type(self).__name__} requires a tokenizer and cannot be " + "initialized when `skip_tokenizer_init=True`. Disable " + "`--skip-tokenizer-init`." + ) + self.info = info self.dummy_inputs = dummy_inputs self.cache = cache From 78d70557f9b6905fdc2ea35784763624be79bea2 Mon Sep 17 00:00:00 2001 From: Luyi Xiao <81631911+luyixiao95@users.noreply.github.com> Date: Tue, 1 Sep 2026 23:45:13 -0700 Subject: [PATCH 3/5] [Bugfix] Mark tokenizer requirement as a class variable Signed-off-by: Luyi Xiao <81631911+luyixiao95@users.noreply.github.com> --- vllm/multimodal/processing/processor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vllm/multimodal/processing/processor.py b/vllm/multimodal/processing/processor.py index 803b638f1603..3db820458880 100644 --- a/vllm/multimodal/processing/processor.py +++ b/vllm/multimodal/processing/processor.py @@ -8,6 +8,7 @@ from functools import lru_cache from typing import ( TYPE_CHECKING, + ClassVar, Generic, NamedTuple, Protocol, @@ -1033,7 +1034,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): Not to be confused with `transformers.ProcessorMixin`. """ - requires_tokenizer = True + requires_tokenizer: ClassVar[bool] = True def __init__( self, From 65548c375aa9c42fc3b3feb3edf10d44cf22ac7b Mon Sep 17 00:00:00 2001 From: Luyi Xiao <81631911+luyixiao95@users.noreply.github.com> Date: Tue, 1 Sep 2026 23:49:02 -0700 Subject: [PATCH 4/5] Remove tokenizer flag tests Signed-off-by: Luyi Xiao <81631911+luyixiao95@users.noreply.github.com> --- tests/multimodal/test_processing.py | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/tests/multimodal/test_processing.py b/tests/multimodal/test_processing.py index 36061e385b1b..6e508af2338a 100644 --- a/tests/multimodal/test_processing.py +++ b/tests/multimodal/test_processing.py @@ -1122,14 +1122,10 @@ def decode(self, token_ids: list[int], **kwargs) -> str: class _FakeProcessingInfo: def __init__(self, tokenizer) -> None: self._tokenizer = tokenizer - self.ctx = type("Context", (), {"tokenizer": tokenizer})() def get_tokenizer(self): return self._tokenizer - def get_data_parser(self): - return MultiModalDataParser() - class _TextFallbackProcessor(BaseMultiModalProcessor): """Only `self.info.get_tokenizer()` is needed by the text fallback.""" @@ -1144,30 +1140,6 @@ def _get_prompt_updates(self, mm_items, hf_processor_mm_kwargs, out_mm_kwargs): raise NotImplementedError -class _TokenizerFreeProcessor(_TextFallbackProcessor): - requires_tokenizer = False - - -def test_multimodal_processor_requires_tokenizer_by_default(): - info = _FakeProcessingInfo(tokenizer=None) - - with pytest.raises(ValueError, match="`skip_tokenizer_init=True`"): - BaseMultiModalProcessor.__init__( - _TextFallbackProcessor.__new__(_TextFallbackProcessor), - info, - dummy_inputs=None, - ) - - -def test_multimodal_processor_can_allow_tokenizer_free_init(): - info = _FakeProcessingInfo(tokenizer=None) - processor = _TokenizerFreeProcessor.__new__(_TokenizerFreeProcessor) - - BaseMultiModalProcessor.__init__(processor, info, dummy_inputs=None) - - assert processor.info is info - - def _text_fallback_processor() -> BaseMultiModalProcessor: return _TextFallbackProcessor(_FakeTokenizer()) From 0f061b47b5a78f00b1c6735c5105e4eca4778fcc Mon Sep 17 00:00:00 2001 From: Luyi Xiao <81631911+luyixiao95@users.noreply.github.com> Date: Wed, 2 Sep 2026 11:32:34 -0700 Subject: [PATCH 5/5] Fix Qwen3 Omni test context Signed-off-by: Luyi Xiao <81631911+luyixiao95@users.noreply.github.com> --- tests/model_executor/test_qwen3_omni.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/model_executor/test_qwen3_omni.py b/tests/model_executor/test_qwen3_omni.py index 54d2f2672f7b..708460c92c84 100644 --- a/tests/model_executor/test_qwen3_omni.py +++ b/tests/model_executor/test_qwen3_omni.py @@ -144,6 +144,7 @@ def test_qwen3_omni_get_updates_use_audio_in_video( # Create a mock context mock_ctx = Mock(spec=InputProcessingContext) + mock_ctx.tokenizer = mock_tokenizer # Create processing info info = Qwen3OmniMoeThinkerProcessingInfo(mock_ctx)