From 45fd97a3cd6b2f819ed97ff46775b776f3a01df3 Mon Sep 17 00:00:00 2001 From: Isotr0py Date: Wed, 4 Mar 2026 23:42:49 +0800 Subject: [PATCH 1/7] lazy import Signed-off-by: Isotr0py --- vllm/model_executor/models/deepseek_vl2.py | 3 +- vllm/transformers_utils/processor.py | 23 +++++++++++- .../transformers_utils/processors/__init__.py | 36 +++++++++++++------ 3 files changed, 49 insertions(+), 13 deletions(-) diff --git a/vllm/model_executor/models/deepseek_vl2.py b/vllm/model_executor/models/deepseek_vl2.py index e0395a5b13fa..469d7fb7119a 100644 --- a/vllm/model_executor/models/deepseek_vl2.py +++ b/vllm/model_executor/models/deepseek_vl2.py @@ -48,7 +48,6 @@ MlpProjectorConfig, VisionEncoderConfig, ) -from vllm.transformers_utils.processors.deepseek_vl2 import DeepseekVLV2Processor from vllm.utils.tensor_schema import TensorSchema, TensorShape from vllm.utils.torch_utils import set_default_torch_dtype @@ -160,7 +159,7 @@ def get_hf_config(self): return self.ctx.get_hf_config(DeepseekVLV2Config) def get_hf_processor(self, **kwargs: object): - return self.ctx.get_hf_processor(DeepseekVLV2Processor, **kwargs) + return self.ctx.get_hf_processor(**kwargs) def get_supported_mm_limits(self) -> Mapping[str, int | None]: return {"image": None} diff --git a/vllm/transformers_utils/processor.py b/vllm/transformers_utils/processor.py index 9190c82f50e6..20e82b75c74c 100644 --- a/vllm/transformers_utils/processor.py +++ b/vllm/transformers_utils/processor.py @@ -20,7 +20,9 @@ from typing_extensions import TypeVar from vllm.logger import init_logger +from vllm.transformers_utils import processors from vllm.transformers_utils.gguf_utils import is_gguf +from vllm.transformers_utils.repo_utils import get_hf_file_to_dict from vllm.transformers_utils.utils import convert_model_repo_to_path from vllm.utils.func_utils import get_allowed_kwarg_only_overrides @@ -139,6 +141,15 @@ def _merge_mm_kwargs( return allowed_kwargs +def get_processor_cls_name_from_config(processor_name: str) -> str | None: + config_file = ["preprocessor_config.json", "processor_config.json"] + for file in config_file: + config = get_hf_file_to_dict(file, processor_name) + if config and "processor_class" in config: + return config["processor_class"] + return None + + def get_processor( processor_name: str, *args: Any, @@ -152,8 +163,18 @@ def get_processor( revision = "main" try: processor_name = convert_model_repo_to_path(processor_name) + registered_cls_name = get_processor_cls_name_from_config(processor_name) + registered_processor_cls = ( + getattr(processors, registered_cls_name, None) + if registered_cls_name + else None + ) + registered_processor_cls = cast(type[_P] | None, registered_processor_cls) + # Use registered processor class when it's available + # and explicit processor_cls is not set. if isinstance(processor_cls, tuple) or processor_cls == ProcessorMixin: - processor = AutoProcessor.from_pretrained( + processor_cls = registered_processor_cls or AutoProcessor + processor = processor_cls.from_pretrained( processor_name, *args, revision=revision, diff --git a/vllm/transformers_utils/processors/__init__.py b/vllm/transformers_utils/processors/__init__.py index 0660a62ea262..ddc43f853ebf 100644 --- a/vllm/transformers_utils/processors/__init__.py +++ b/vllm/transformers_utils/processors/__init__.py @@ -8,16 +8,19 @@ - There is a need to override the existing processor to support vLLM. """ -from vllm.transformers_utils.processors.bagel import BagelProcessor -from vllm.transformers_utils.processors.deepseek_vl2 import DeepseekVLV2Processor -from vllm.transformers_utils.processors.fireredasr2_processor import ( - FireRedASR2Processor, -) -from vllm.transformers_utils.processors.funasr_processor import FunASRProcessor -from vllm.transformers_utils.processors.hunyuan_vl import HunYuanVLProcessor -from vllm.transformers_utils.processors.hunyuan_vl_image import HunYuanVLImageProcessor -from vllm.transformers_utils.processors.ovis import OvisProcessor -from vllm.transformers_utils.processors.ovis2_5 import Ovis2_5Processor +import importlib + +_CLASS_TO_MODULE: dict[str, str] = { + "BagelProcessor": "vllm.transformers_utils.processors.bagel", + "DeepseekVLV2Processor": "vllm.transformers_utils.processors.deepseek_vl2", + "FireRedASR2Processor": "vllm.transformers_utils.processors.fireredasr2_processor", + "FunASRProcessor": "vllm.transformers_utils.processors.funasr_processor", + "HunYuanVLProcessor": "vllm.transformers_utils.processors.hunyuan_vl", + "HunYuanVLImageProcessor": "vllm.transformers_utils.processors.hunyuan_vl_image", + "OvisProcessor": "vllm.transformers_utils.processors.ovis", + "Ovis2_5Processor": "vllm.transformers_utils.processors.ovis2_5", +} + __all__ = [ "BagelProcessor", @@ -29,3 +32,16 @@ "OvisProcessor", "Ovis2_5Processor", ] + + +def __getattr__(name: str): + if name in _CLASS_TO_MODULE: + module_name = _CLASS_TO_MODULE[name] + module = importlib.import_module(module_name) + return getattr(module, name) + + raise AttributeError(f"module 'processors' has no attribute '{name}'") + + +def __dir__(): + return sorted(list(__all__)) From 6abbbb3eea98ee61557d71d7e250668f1fa443af Mon Sep 17 00:00:00 2001 From: Isotr0py Date: Thu, 5 Mar 2026 11:02:45 +0800 Subject: [PATCH 2/7] remove autoprocessor registry Signed-off-by: Isotr0py --- vllm/transformers_utils/processors/bagel.py | 4 ---- vllm/transformers_utils/processors/deepseek_ocr.py | 5 +---- vllm/transformers_utils/processors/deepseek_vl2.py | 5 +---- vllm/transformers_utils/processors/fireredasr2_processor.py | 2 -- vllm/transformers_utils/processors/funasr_processor.py | 2 -- vllm/transformers_utils/processors/hunyuan_vl.py | 4 ---- vllm/transformers_utils/processors/ovis.py | 5 +---- vllm/transformers_utils/processors/qwen3_asr.py | 3 --- 8 files changed, 3 insertions(+), 27 deletions(-) diff --git a/vllm/transformers_utils/processors/bagel.py b/vllm/transformers_utils/processors/bagel.py index 09b2e31b3724..3226d7b0c83d 100644 --- a/vllm/transformers_utils/processors/bagel.py +++ b/vllm/transformers_utils/processors/bagel.py @@ -3,7 +3,6 @@ # Copyright 2025 Bytedance Ltd. and/or its affiliates. """BAGEL processor for image and text inputs.""" -from transformers import AutoProcessor from transformers.feature_extraction_utils import BatchFeature from transformers.image_utils import ImageInput from transformers.processing_utils import ProcessingKwargs, ProcessorMixin, Unpack @@ -79,6 +78,3 @@ def model_input_names(self): tokenizer_input_names = self.tokenizer.model_input_names image_processor_input_names = self.image_processor.model_input_names return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names)) - - -AutoProcessor.register("BagelProcessor", BagelProcessor) diff --git a/vllm/transformers_utils/processors/deepseek_ocr.py b/vllm/transformers_utils/processors/deepseek_ocr.py index 77e49483640a..68a2b1aaaa02 100644 --- a/vllm/transformers_utils/processors/deepseek_ocr.py +++ b/vllm/transformers_utils/processors/deepseek_ocr.py @@ -8,7 +8,7 @@ import torch import torchvision.transforms as T from PIL import Image, ImageOps -from transformers import AutoProcessor, BatchFeature, LlamaTokenizerFast +from transformers import BatchFeature, LlamaTokenizerFast from transformers.processing_utils import ProcessorMixin # TODO(Isotr0py): change modes for variants @@ -453,6 +453,3 @@ def tokenize_with_images( num_image_tokens, image_shapes, ) - - -AutoProcessor.register("DeepseekOCRProcessor", DeepseekOCRProcessor) diff --git a/vllm/transformers_utils/processors/deepseek_vl2.py b/vllm/transformers_utils/processors/deepseek_vl2.py index 5ef258b9be29..5a3c986c1307 100644 --- a/vllm/transformers_utils/processors/deepseek_vl2.py +++ b/vllm/transformers_utils/processors/deepseek_vl2.py @@ -29,7 +29,7 @@ import torch import torchvision.transforms as T from PIL import Image, ImageOps -from transformers import AutoProcessor, BatchFeature, LlamaTokenizerFast +from transformers import BatchFeature, LlamaTokenizerFast from transformers.processing_utils import ProcessorMixin @@ -401,6 +401,3 @@ def tokenize_with_images( images_spatial_crop, num_image_tokens, ) - - -AutoProcessor.register("DeepseekVLV2Processor", DeepseekVLV2Processor) diff --git a/vllm/transformers_utils/processors/fireredasr2_processor.py b/vllm/transformers_utils/processors/fireredasr2_processor.py index 98c99ec39181..4bde53015003 100644 --- a/vllm/transformers_utils/processors/fireredasr2_processor.py +++ b/vllm/transformers_utils/processors/fireredasr2_processor.py @@ -8,7 +8,6 @@ import torch.nn.functional as F from transformers import ( AutoFeatureExtractor, - AutoProcessor, BatchFeature, ) from transformers.feature_extraction_sequence_utils import SequenceFeatureExtractor @@ -345,4 +344,3 @@ def get_prompt_ids(self, text: str, return_tensors="np"): AutoFeatureExtractor.register( "FireRedASR2FeatureExtractor", FireRedASR2FeatureExtractor ) -AutoProcessor.register("FireRedASR2Processor", FireRedASR2Processor) diff --git a/vllm/transformers_utils/processors/funasr_processor.py b/vllm/transformers_utils/processors/funasr_processor.py index c4cb2a2c44e9..23cd00a2765e 100644 --- a/vllm/transformers_utils/processors/funasr_processor.py +++ b/vllm/transformers_utils/processors/funasr_processor.py @@ -9,7 +9,6 @@ from torch.nn.utils.rnn import pad_sequence from transformers import ( AutoFeatureExtractor, - AutoProcessor, BatchFeature, ) from transformers.feature_extraction_sequence_utils import SequenceFeatureExtractor @@ -501,4 +500,3 @@ def get_prompt_ids(self, text: str, return_tensors="np"): AutoFeatureExtractor.register("FunASRFeatureExtractor", FunASRFeatureExtractor) -AutoProcessor.register("FunASRProcessor", FunASRProcessor) diff --git a/vllm/transformers_utils/processors/hunyuan_vl.py b/vllm/transformers_utils/processors/hunyuan_vl.py index 924c679e71c9..2d0e4db97a6f 100644 --- a/vllm/transformers_utils/processors/hunyuan_vl.py +++ b/vllm/transformers_utils/processors/hunyuan_vl.py @@ -5,7 +5,6 @@ import numpy as np import torch -from transformers import AutoProcessor from transformers.feature_extraction_utils import BatchFeature from transformers.image_utils import ImageInput from transformers.processing_utils import ProcessorMixin @@ -225,6 +224,3 @@ def split_image_into_patch_blocks( patches = img.reshape(-1, 3, patch_size, patch_size) return patches - - -AutoProcessor.register("HunYuanVLProcessor", HunYuanVLProcessor) diff --git a/vllm/transformers_utils/processors/ovis.py b/vllm/transformers_utils/processors/ovis.py index bd5de95914c2..da80f24e75c0 100644 --- a/vllm/transformers_utils/processors/ovis.py +++ b/vllm/transformers_utils/processors/ovis.py @@ -26,7 +26,7 @@ import PIL import torch -from transformers import AutoProcessor, BatchFeature +from transformers import BatchFeature from transformers.image_utils import ImageInput from transformers.processing_utils import ProcessingKwargs, ProcessorMixin, Unpack from transformers.tokenization_utils_base import PreTokenizedInput, TextInput @@ -453,6 +453,3 @@ def model_input_names(self): dict.fromkeys(tokenizer_input_names + image_processor_input_names) ) return names_from_processor + ["second_per_grid_ts"] - - -AutoProcessor.register("OvisProcessor", OvisProcessor) diff --git a/vllm/transformers_utils/processors/qwen3_asr.py b/vllm/transformers_utils/processors/qwen3_asr.py index 677326e25c0d..55d38537928d 100644 --- a/vllm/transformers_utils/processors/qwen3_asr.py +++ b/vllm/transformers_utils/processors/qwen3_asr.py @@ -227,6 +227,3 @@ def model_input_names(self): + ["feature_attention_mask"] ) ) - - -AutoProcessor.register("Qwen3ASRProcessor", Qwen3ASRProcessor) From 229405acb21e9c8751f07548962ef0710147a001 Mon Sep 17 00:00:00 2001 From: Isotr0py Date: Thu, 5 Mar 2026 21:21:24 +0800 Subject: [PATCH 3/7] rename and clean Signed-off-by: Isotr0py --- vllm/model_executor/models/fireredasr2.py | 2 +- vllm/model_executor/models/funasr.py | 2 +- .../processors/{fireredasr2_processor.py => fireredasr2.py} | 0 .../processors/{funasr_processor.py => funasr.py} | 0 vllm/transformers_utils/processors/ovis2_5.py | 5 +---- 5 files changed, 3 insertions(+), 6 deletions(-) rename vllm/transformers_utils/processors/{fireredasr2_processor.py => fireredasr2.py} (100%) rename vllm/transformers_utils/processors/{funasr_processor.py => funasr.py} (100%) diff --git a/vllm/model_executor/models/fireredasr2.py b/vllm/model_executor/models/fireredasr2.py index f0d3e124c03b..91d7070a362a 100644 --- a/vllm/model_executor/models/fireredasr2.py +++ b/vllm/model_executor/models/fireredasr2.py @@ -41,7 +41,7 @@ PromptUpdateDetails, ) from vllm.transformers_utils.processor import cached_processor_from_config -from vllm.transformers_utils.processors.fireredasr2_processor import ( +from vllm.transformers_utils.processors.fireredasr2 import ( FireRedASR2FeatureExtractor, ) from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/model_executor/models/funasr.py b/vllm/model_executor/models/funasr.py index 25ede72f1fff..72bcd151cf33 100644 --- a/vllm/model_executor/models/funasr.py +++ b/vllm/model_executor/models/funasr.py @@ -50,7 +50,7 @@ PromptUpdate, ) from vllm.transformers_utils.processor import cached_processor_from_config -from vllm.transformers_utils.processors.funasr_processor import FunASRFeatureExtractor +from vllm.transformers_utils.processors.funasr import FunASRFeatureExtractor from vllm.utils.jsontree import json_map_leaves from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/transformers_utils/processors/fireredasr2_processor.py b/vllm/transformers_utils/processors/fireredasr2.py similarity index 100% rename from vllm/transformers_utils/processors/fireredasr2_processor.py rename to vllm/transformers_utils/processors/fireredasr2.py diff --git a/vllm/transformers_utils/processors/funasr_processor.py b/vllm/transformers_utils/processors/funasr.py similarity index 100% rename from vllm/transformers_utils/processors/funasr_processor.py rename to vllm/transformers_utils/processors/funasr.py diff --git a/vllm/transformers_utils/processors/ovis2_5.py b/vllm/transformers_utils/processors/ovis2_5.py index f1bcefc1a09c..22c29b2aa593 100644 --- a/vllm/transformers_utils/processors/ovis2_5.py +++ b/vllm/transformers_utils/processors/ovis2_5.py @@ -6,7 +6,7 @@ import numpy as np import PIL import torch -from transformers import AutoProcessor, BatchFeature +from transformers import BatchFeature from transformers.image_utils import ImageInput from transformers.processing_utils import ProcessingKwargs, ProcessorMixin, Unpack from transformers.tokenization_utils_base import PreTokenizedInput, TextInput @@ -476,6 +476,3 @@ def preprocess_multidata( visual_placeholders, torch.tensor([[grid_t, grid_h, grid_w]]), ) - - -AutoProcessor.register("Ovis2_5Processor", Ovis2_5Processor) From aba17622a438f943d20df1a8bad8be6fcb286aa1 Mon Sep 17 00:00:00 2001 From: Isotr0py Date: Thu, 5 Mar 2026 21:56:32 +0800 Subject: [PATCH 4/7] fix Signed-off-by: Isotr0py --- vllm/transformers_utils/processor.py | 6 +++++- vllm/transformers_utils/processors/__init__.py | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/vllm/transformers_utils/processor.py b/vllm/transformers_utils/processor.py index 20e82b75c74c..bb0838c9a9bb 100644 --- a/vllm/transformers_utils/processor.py +++ b/vllm/transformers_utils/processor.py @@ -142,7 +142,11 @@ def _merge_mm_kwargs( def get_processor_cls_name_from_config(processor_name: str) -> str | None: - config_file = ["preprocessor_config.json", "processor_config.json"] + config_file = [ + "processor_config.json", + "preprocessor_config.json", + "tokenizer_config.json", + ] for file in config_file: config = get_hf_file_to_dict(file, processor_name) if config and "processor_class" in config: diff --git a/vllm/transformers_utils/processors/__init__.py b/vllm/transformers_utils/processors/__init__.py index ddc43f853ebf..f7a0b7af0f5f 100644 --- a/vllm/transformers_utils/processors/__init__.py +++ b/vllm/transformers_utils/processors/__init__.py @@ -13,8 +13,8 @@ _CLASS_TO_MODULE: dict[str, str] = { "BagelProcessor": "vllm.transformers_utils.processors.bagel", "DeepseekVLV2Processor": "vllm.transformers_utils.processors.deepseek_vl2", - "FireRedASR2Processor": "vllm.transformers_utils.processors.fireredasr2_processor", - "FunASRProcessor": "vllm.transformers_utils.processors.funasr_processor", + "FireRedASR2Processor": "vllm.transformers_utils.processors.fireredasr2", + "FunASRProcessor": "vllm.transformers_utils.processors.funasr", "HunYuanVLProcessor": "vllm.transformers_utils.processors.hunyuan_vl", "HunYuanVLImageProcessor": "vllm.transformers_utils.processors.hunyuan_vl_image", "OvisProcessor": "vllm.transformers_utils.processors.ovis", From 9f09bdee093d29c42101c9e405c40d08dade15c6 Mon Sep 17 00:00:00 2001 From: Isotr0py Date: Thu, 5 Mar 2026 22:16:31 +0800 Subject: [PATCH 5/7] fix missing revision Signed-off-by: Isotr0py --- vllm/transformers_utils/processor.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/vllm/transformers_utils/processor.py b/vllm/transformers_utils/processor.py index bb0838c9a9bb..d16361b05cbe 100644 --- a/vllm/transformers_utils/processor.py +++ b/vllm/transformers_utils/processor.py @@ -141,14 +141,17 @@ def _merge_mm_kwargs( return allowed_kwargs -def get_processor_cls_name_from_config(processor_name: str) -> str | None: +def get_processor_cls_name_from_config( + processor_name: str, + revision: str | None = "main", +) -> str | None: config_file = [ "processor_config.json", "preprocessor_config.json", "tokenizer_config.json", ] for file in config_file: - config = get_hf_file_to_dict(file, processor_name) + config = get_hf_file_to_dict(file, processor_name, revision=revision) if config and "processor_class" in config: return config["processor_class"] return None @@ -167,7 +170,9 @@ def get_processor( revision = "main" try: processor_name = convert_model_repo_to_path(processor_name) - registered_cls_name = get_processor_cls_name_from_config(processor_name) + registered_cls_name = get_processor_cls_name_from_config( + processor_name, revision=revision + ) registered_processor_cls = ( getattr(processors, registered_cls_name, None) if registered_cls_name From db78ccd24f3698e2cc15e7e2f11f83f70fc8ce92 Mon Sep 17 00:00:00 2001 From: Isotr0py Date: Thu, 5 Mar 2026 23:26:41 +0800 Subject: [PATCH 6/7] fix Signed-off-by: Isotr0py --- vllm/transformers_utils/processor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vllm/transformers_utils/processor.py b/vllm/transformers_utils/processor.py index d16361b05cbe..1319e2943562 100644 --- a/vllm/transformers_utils/processor.py +++ b/vllm/transformers_utils/processor.py @@ -182,8 +182,8 @@ def get_processor( # Use registered processor class when it's available # and explicit processor_cls is not set. if isinstance(processor_cls, tuple) or processor_cls == ProcessorMixin: - processor_cls = registered_processor_cls or AutoProcessor - processor = processor_cls.from_pretrained( + _processor_cls = registered_processor_cls or AutoProcessor + processor = _processor_cls.from_pretrained( processor_name, *args, revision=revision, From 4ca304066197e97a8a8230309b25ed25c41a71fc Mon Sep 17 00:00:00 2001 From: Isotr0py Date: Fri, 6 Mar 2026 14:06:29 +0800 Subject: [PATCH 7/7] fix qwen3asr Signed-off-by: Isotr0py --- tests/models/registry.py | 7 ++----- vllm/transformers_utils/processors/__init__.py | 2 ++ 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/models/registry.py b/tests/models/registry.py index 3c9bb77e7c68..11f73d37e3f2 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -1019,18 +1019,15 @@ def check_available_online( min_transformers_version="4.57", ), "Qwen3ASRForConditionalGeneration": _HfExamplesInfo( - "Qwen/Qwen3-ASR-1.7B", + "Qwen/Qwen3-ASR-0.6B", max_model_len=4096, min_transformers_version="4.57", - is_available_online=False, ), "Qwen3ASRRealtimeGeneration": _HfExamplesInfo( - "Qwen/Qwen3-ASR-1.7B", + "Qwen/Qwen3-ASR-0.6B", max_model_len=4096, min_transformers_version="4.57", - enforce_eager=True, hf_overrides={"architectures": ["Qwen3ASRRealtimeGeneration"]}, - is_available_online=False, ), "RForConditionalGeneration": _HfExamplesInfo("YannQi/R-4B", trust_remote_code=True), "SkyworkR1VChatModel": _HfExamplesInfo( diff --git a/vllm/transformers_utils/processors/__init__.py b/vllm/transformers_utils/processors/__init__.py index f7a0b7af0f5f..ff2263f3e72c 100644 --- a/vllm/transformers_utils/processors/__init__.py +++ b/vllm/transformers_utils/processors/__init__.py @@ -19,6 +19,7 @@ "HunYuanVLImageProcessor": "vllm.transformers_utils.processors.hunyuan_vl_image", "OvisProcessor": "vllm.transformers_utils.processors.ovis", "Ovis2_5Processor": "vllm.transformers_utils.processors.ovis2_5", + "Qwen3ASRProcessor": "vllm.transformers_utils.processors.qwen3_asr", } @@ -31,6 +32,7 @@ "HunYuanVLImageProcessor", "OvisProcessor", "Ovis2_5Processor", + "Qwen3ASRProcessor", ]