From 11e08bfe43bbe9ab2156a68050faf748ccf68c0c Mon Sep 17 00:00:00 2001 From: Harshal Janjani Date: Mon, 27 Jul 2026 09:07:46 +0000 Subject: [PATCH 1/2] feat: Support Voxtral audio generation in the Transformers backend Signed-off-by: Harshal Janjani --- .../generation/test_transformers_audio.py | 3 +++ .../processing/test_transformers_audio.py | 13 +------------ vllm/config/model.py | 4 ++++ vllm/config/vllm.py | 7 +++++++ vllm/multimodal/processing/context.py | 12 ++++++++---- 5 files changed, 23 insertions(+), 16 deletions(-) diff --git a/tests/models/multimodal/generation/test_transformers_audio.py b/tests/models/multimodal/generation/test_transformers_audio.py index 919557987796..be7cff5490ff 100644 --- a/tests/models/multimodal/generation/test_transformers_audio.py +++ b/tests/models/multimodal/generation/test_transformers_audio.py @@ -40,6 +40,9 @@ "gpu_memory_utilization": 0.85, }, }, + "mistralai/Voxtral-Mini-3B-2507": { + "prompt": ("[INST][AUDIO]What can you tell me about this audio?[/INST]"), + }, "microsoft/VibeVoice-ASR-HF": { "prompt": ( "<|im_start|>system\n" diff --git a/tests/models/multimodal/processing/test_transformers_audio.py b/tests/models/multimodal/processing/test_transformers_audio.py index 3ff3e04379fc..923cf67642d1 100644 --- a/tests/models/multimodal/processing/test_transformers_audio.py +++ b/tests/models/multimodal/processing/test_transformers_audio.py @@ -58,18 +58,7 @@ [ "ibm-granite/granite-speech-3.3-2b", "nvidia/audio-flamingo-3-hf", - pytest.param( - "mistralai/Voxtral-Mini-3B-2507", - marks=pytest.mark.xfail( - reason="MistralCommonBackend.encode does not produce the audio " - "placeholder token (ID 24) from raw text. apply_chat_template " - "yields token IDs with placeholders, but MultiModalProcessor." - "apply() decodes the prompt back to text and re-tokenizes, at " - "which point the placeholders are lost. Fix belongs in " - "mistral_common or in the Voxtral-specific path.", - strict=False, - ), - ), + "mistralai/Voxtral-Mini-3B-2507", "microsoft/VibeVoice-ASR-HF", "zai-org/GLM-ASR-Nano-2512", ], diff --git a/vllm/config/model.py b/vllm/config/model.py index d64bd57e87dd..3d5e42f383ad 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -560,6 +560,10 @@ def __post_init__( ): raise ValueError("cumem allocator is not supported on current platform.") + # The Transformers backend needs the HF config, not Mistral's params.json + if self.model_impl == "transformers" and self.config_format == "auto": + self.config_format = "hf" + hf_config = get_config( self.hf_config_path or self.model, self.trust_remote_code, diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index a7f185cad407..c64514c462c5 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -2080,6 +2080,13 @@ def try_verify_and_update_config(self): f"Model: {self.model_config.model}" ) + # The Transformers backend needs HF weights, not Mistral's consolidated + if ( + self.model_config.model_impl == "transformers" + and self.load_config.load_format == "auto" + ): + self.load_config.load_format = "hf" + def compile_debug_dump_path(self) -> Path | None: """Returns a rank-aware path for dumping torch.compile debug information. diff --git a/vllm/multimodal/processing/context.py b/vllm/multimodal/processing/context.py index bc893d836db9..b39c8e52150a 100644 --- a/vllm/multimodal/processing/context.py +++ b/vllm/multimodal/processing/context.py @@ -195,17 +195,21 @@ def get_hf_processor( typ = ProcessorMixin - tokenizer = self.tokenizer - if is_mistral_tokenizer(tokenizer): - tokenizer = tokenizer.transformers_tokenizer # type: ignore[union-attr] + from transformers import MistralCommonBackend + tokenizer = self.tokenizer merged_kwargs = self.get_merged_mm_kwargs(kwargs) merged_kwargs.pop("tokenizer", None) + # `MistralCommonBackend` rejects a forwarded `tokenizer` kwarg + if not is_mistral_tokenizer(tokenizer) and not isinstance( + tokenizer, MistralCommonBackend + ): + merged_kwargs["tokenizer"] = tokenizer + return cached_processor_from_config( self.model_config, processor_cls=typ, - tokenizer=tokenizer, **merged_kwargs, ) From eb822f0c18acd7b4c20be2ee8be943b2c04d8c28 Mon Sep 17 00:00:00 2001 From: Harshal Janjani Date: Mon, 27 Jul 2026 11:07:37 +0000 Subject: [PATCH 2/2] nit: Update comments for clarity on HF config req Signed-off-by: Harshal Janjani --- vllm/config/model.py | 2 +- vllm/config/vllm.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vllm/config/model.py b/vllm/config/model.py index 3d5e42f383ad..620a83feb0fc 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -560,7 +560,7 @@ def __post_init__( ): raise ValueError("cumem allocator is not supported on current platform.") - # The Transformers backend needs the HF config, not Mistral's params.json + # AutoModel.from_config resolves by class, needs typed HF config not params.json if self.model_impl == "transformers" and self.config_format == "auto": self.config_format = "hf" diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index c64514c462c5..8fa2ea3c421e 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -2080,7 +2080,7 @@ def try_verify_and_update_config(self): f"Model: {self.model_config.model}" ) - # The Transformers backend needs HF weights, not Mistral's consolidated + # Mistral's consolidated weight names don't map to HF modules; use HF weights if ( self.model_config.model_impl == "transformers" and self.load_config.load_format == "auto"