Support Voxtral audio generation in the Transformers backend

tests/models/multimodal/generation/test_transformers_audio.py

@@ -40,6 +40,9 @@
             "gpu_memory_utilization": 0.85,
         },
     },
+    "mistralai/Voxtral-Mini-3B-2507": {
+        "prompt": ("[INST][AUDIO]What can you tell me about this audio?[/INST]"),
+    },
     "microsoft/VibeVoice-ASR-HF": {
         "prompt": (
             "<|im_start|>system\n"

tests/models/multimodal/processing/test_transformers_audio.py

@@ -58,18 +58,7 @@
     [
         "ibm-granite/granite-speech-3.3-2b",
         "nvidia/audio-flamingo-3-hf",
-        pytest.param(
-            "mistralai/Voxtral-Mini-3B-2507",
-            marks=pytest.mark.xfail(
-                reason="MistralCommonBackend.encode does not produce the audio "
-                "placeholder token (ID 24) from raw text. apply_chat_template "
-                "yields token IDs with placeholders, but MultiModalProcessor."
-                "apply() decodes the prompt back to text and re-tokenizes, at "
-                "which point the placeholders are lost. Fix belongs in "
-                "mistral_common or in the Voxtral-specific path.",
-                strict=False,
-            ),
-        ),
+        "mistralai/Voxtral-Mini-3B-2507",
         "microsoft/VibeVoice-ASR-HF",
         "zai-org/GLM-ASR-Nano-2512",
     ],

vllm/config/model.py

@@ -560,6 +560,10 @@ def __post_init__(
         ):
             raise ValueError("cumem allocator is not supported on current platform.")
 
+        # AutoModel.from_config resolves by class, needs typed HF config not params.json
+        if self.model_impl == "transformers" and self.config_format == "auto":
+            self.config_format = "hf"
+
         hf_config = get_config(
             self.hf_config_path or self.model,
             self.trust_remote_code,

vllm/config/vllm.py

@@ -2080,6 +2080,13 @@ def try_verify_and_update_config(self):
                     f"Model: {self.model_config.model}"
                 )
 
+        # Mistral's consolidated weight names don't map to HF modules; use HF weights
+        if (
+            self.model_config.model_impl == "transformers"
+            and self.load_config.load_format == "auto"
+        ):
+            self.load_config.load_format = "hf"
+
     def compile_debug_dump_path(self) -> Path | None:
         """Returns a rank-aware path for dumping
         torch.compile debug information.

vllm/multimodal/processing/context.py

@@ -195,17 +195,21 @@ def get_hf_processor(
 
             typ = ProcessorMixin
 
-        tokenizer = self.tokenizer
-        if is_mistral_tokenizer(tokenizer):
-            tokenizer = tokenizer.transformers_tokenizer  # type: ignore[union-attr]
+        from transformers import MistralCommonBackend
 
+        tokenizer = self.tokenizer
         merged_kwargs = self.get_merged_mm_kwargs(kwargs)
         merged_kwargs.pop("tokenizer", None)
 
+        # `MistralCommonBackend` rejects a forwarded `tokenizer` kwarg
+        if not is_mistral_tokenizer(tokenizer) and not isinstance(
+            tokenizer, MistralCommonBackend
+        ):
+            merged_kwargs["tokenizer"] = tokenizer
+
         return cached_processor_from_config(
             self.model_config,
             processor_cls=typ,
-            tokenizer=tokenizer,
             **merged_kwargs,
         )