feat[vLLM × v5]: Add audio support for the Transformers backend

docs/models/supported_models.md

@@ -19,7 +19,7 @@ vLLM also supports model implementations that are available in Transformers. We
 
 Currently, the Transformers modeling backend works for the following:
 
-- Modalities: embedding models, language models and vision-language models*
+- Modalities: embedding models, language models, vision-language models* and audio-language models
 - Architectures: encoder-only, decoder-only, mixture-of-experts
 - Attention types: full attention and/or sliding attention
 
@@ -607,7 +607,8 @@ Some models are supported only via the [Transformers modeling backend](#transfor
 
 | Architecture | Models | Inputs | Example HF Models | [LoRA](../features/lora.md) | [PP](../serving/parallelism_scaling.md) |
 | ------------ | ------ | ------ | ----------------- | --------------------------- | --------------------------------------- |
-| `Emu3ForConditionalGeneration` | Emu3 | T + I | `BAAI/Emu3-Chat-hf` | ✅︎ | ✅︎ |
+| `Emu3ForConditionalGeneration` | Emu3 | T + I<sup>+</sup> | `BAAI/Emu3-Chat-hf` | ✅︎ | ✅︎ |
+| `VibeVoiceAsrForConditionalGeneration` | VibeVoice-ASR | T + A<sup>+</sup> | `microsoft/VibeVoice-ASR-HF` | ✅︎ | ✅︎ |
 
 <sup>^</sup> You need to set the architecture name via `--hf-overrides` to match the one in vLLM.</br>
 <sup>E</sup> Pre-computed embeddings can be inputted for this modality.</br>

tests/models/multimodal/generation/test_transformers_audio.py

@@ -0,0 +1,133 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from typing import Any
+
+import pytest
+from transformers import AutoModelForSeq2SeqLM
+
+from vllm.assets.audio import AudioAsset
+from vllm.envs import disable_envs_cache
+from vllm.lora.request import LoRARequest
+from vllm.multimodal.audio import AudioResampler
+
+from ....conftest import HfRunner, VllmRunner
+from ...utils import check_logprobs_close
+
+AUDIO_ASSET = AudioAsset("mary_had_lamb")
+
+AUDIO_MODEL_SETTINGS: dict[str, dict[str, Any]] = {
+    "ibm-granite/granite-speech-3.3-2b": {
+        "prompt": (
+            "<|start_of_role|>system<|end_of_role|>"
+            "You are a helpful AI assistant<|end_of_text|>\n"
+            "<|start_of_role|>user<|end_of_role|>"
+            "<|audio|>can you transcribe the speech into a written format?"
+            "<|end_of_text|>\n"
+            "<|start_of_role|>assistant<|end_of_role|>"
+        ),
+        "audio_lora_path": "ibm-granite/granite-speech-3.3-2b",
+    },
+    "nvidia/audio-flamingo-3-hf": {
+        "prompt": (
+            "<|im_start|>system\n"
+            "You are a helpful assistant.<|im_end|>\n"
+            "<|im_start|>user\n"
+            "<sound>Transcribe the input speech.<|im_end|>\n"
+            "<|im_start|>assistant\n"
+        ),
+        "vllm_runner_kwargs": {
+            "gpu_memory_utilization": 0.85,
+        },
+    },
+    "microsoft/VibeVoice-ASR-HF": {
+        "prompt": (
+            "<|im_start|>system\n"
+            "You are a helpful assistant that transcribes audio input "
+            "into text output in JSON format.<|im_end|>\n"
+            "<|im_start|>user\n"
+            "<|object_ref_start|><|box_start|><|object_ref_end|>\n"
+            "This is a 1.0 seconds audio, please transcribe it with "
+            "these keys: Start time, End time, Speaker ID, Content"
+            "<|im_end|>\n"
+            "<|im_start|>assistant\n"
+        ),
+        "sampling_rate": 24000,
+        "vllm_runner_kwargs": {
+            "max_num_batched_tokens": 2048,
+            "gpu_memory_utilization": 0.85,
+        },
+    },
+    "zai-org/GLM-ASR-Nano-2512": {
+        "prompt": (
+            "<|user|>\n"
+            "<|begin_of_audio|><|pad|><|end_of_audio|><|user|>\n"
+            "Please transcribe this audio into text"
+            "<|assistant|>\n"
+        ),
+    },
+}
+
+
+@pytest.mark.parametrize("model_id", list(AUDIO_MODEL_SETTINGS))
+def test_transformers_audio_generation(
+    hf_runner: type[HfRunner],
+    vllm_runner: type[VllmRunner],
+    monkeypatch,
+    model_id: str,
+):
+    """Single-process workaround for V1 fork safety deadlock issue
+    (vllm-project/vllm/issues/17676). Running multiple audio models together
+    under pytest can cause (possibly flaky) hangs, so they are grouped under
+    the same config. Using VLLM_WORKER_MULTIPROC_METHOD=spawn avoids the
+    deadlock and allows worker processes to terminate cleanly, and release
+    GPU memory between test runs until the issue is fixed."""
+    # TODO: Remove monkeypatch once
+    # https://github.com/vllm-project/vllm/issues/17676 is fixed.
+    disable_envs_cache()
+    monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
+
+    settings = AUDIO_MODEL_SETTINGS[model_id]
+    audio_lora_path = settings.get("audio_lora_path")
+
+    audio, orig_sr = AUDIO_ASSET.audio_and_sample_rate
+    target_sr = settings.get("sampling_rate", orig_sr)
+    if orig_sr != target_sr:
+        audio = AudioResampler(target_sr=target_sr).resample(audio, orig_sr=orig_sr)
+    audio = (audio, target_sr)
+
+    with vllm_runner(
+        model_id,
+        model_impl="transformers",
+        dtype="bfloat16",
+        max_model_len=2048,
+        enforce_eager=True,
+        limit_mm_per_prompt={"audio": 1},
+        enable_lora=audio_lora_path is not None,
+        max_lora_rank=64,
+        **settings.get("vllm_runner_kwargs", {}),
+    ) as vllm_model:
+        lora_request = (
+            LoRARequest("audio", 1, audio_lora_path) if audio_lora_path else None
+        )
+        vllm_outputs = vllm_model.generate_greedy_logprobs(
+            [settings["prompt"]],
+            128,
+            num_logprobs=10,
+            audios=[audio],
+            lora_request=lora_request,
+        )
+
+    with hf_runner(
+        model_id, dtype="bfloat16", auto_cls=AutoModelForSeq2SeqLM
+    ) as hf_model:
+        hf_outputs = hf_model.generate_greedy_logprobs_limit(
+            [settings["prompt"]], 128, num_logprobs=10, audios=[audio]
+        )
+
+    check_logprobs_close(
+        outputs_0_lst=hf_outputs,
+        outputs_1_lst=vllm_outputs,
+        name_0="hf",
+        name_1="vllm",
+    )

tests/models/multimodal/processing/test_common.py

@@ -452,6 +452,14 @@ def test_processing_correctness(
             "audio placeholders from processed audio lengths. Its vLLM "
             "processor paths are covered by test_moss_audio.py."
         )
+    # TODO: Remove when transformers 5.15.0 is released, which contains
+    # https://github.com/huggingface/transformers/pull/47483.
+    if model_id == "microsoft/VibeVoice-ASR-HF":
+        pytest.skip(
+            "VibeVoice ASR requires audio as a positional argument and hence "
+            "cannot pass the processing correctness test as is. Its generation "
+            "is covered by test_transformers_audio.py."
+        )
     if model_id == "lmms-lab-encoder/LLaVA-OneVision-2-8B-Instruct":
         pytest.skip(
             "LLaVA-OneVision-2 video processing routes frames through custom "

tests/models/multimodal/processing/test_transformers_audio.py

@@ -0,0 +1,138 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import numpy as np
+import pytest
+
+from vllm.config import ModelConfig
+from vllm.multimodal import MULTIMODAL_REGISTRY
+
+AUDIO_MODEL_SETTINGS = {
+    "ibm-granite/granite-speech-3.3-2b": {
+        "prompt": (
+            "<|start_of_role|>system<|end_of_role|>"
+            "You are a helpful AI assistant<|end_of_text|>\n"
+            "<|start_of_role|>user<|end_of_role|>"
+            "<|audio|>can you transcribe the speech into a written format?"
+            "<|end_of_text|>\n"
+            "<|start_of_role|>assistant<|end_of_role|>"
+        ),
+    },
+    "nvidia/audio-flamingo-3-hf": {
+        "prompt": (
+            "<|im_start|>system\n"
+            "You are a helpful assistant.<|im_end|>\n"
+            "<|im_start|>user\n"
+            "<sound>Transcribe the input speech.<|im_end|>\n"
+            "<|im_start|>assistant\n"
+        ),
+    },
+    "mistralai/Voxtral-Mini-3B-2507": {
+        "prompt": ("[INST][AUDIO]What can you tell me about this audio?[/INST]"),
+    },
+    "microsoft/VibeVoice-ASR-HF": {
+        "prompt": (
+            "<|im_start|>system\n"
+            "You are a helpful assistant that transcribes audio input "
+            "into text output in JSON format.<|im_end|>\n"
+            "<|im_start|>user\n"
+            "<|object_ref_start|><|box_start|><|object_ref_end|>\n"
+            "This is a 1.0 seconds audio, please transcribe it with "
+            "these keys: Start time, End time, Speaker ID, Content"
+            "<|im_end|>\n"
+            "<|im_start|>assistant\n"
+        ),
+    },
+    "zai-org/GLM-ASR-Nano-2512": {
+        "prompt": (
+            "<|user|>\n"
+            "<|begin_of_audio|><|pad|><|end_of_audio|><|user|>\n"
+            "Please transcribe this audio into text"
+            "<|assistant|>\n"
+        ),
+    },
+}
+
+
+@pytest.mark.parametrize(
+    "model_id",
+    [
+        "ibm-granite/granite-speech-3.3-2b",
+        "nvidia/audio-flamingo-3-hf",
+        pytest.param(
+            "mistralai/Voxtral-Mini-3B-2507",
+            marks=pytest.mark.xfail(
+                reason="MistralCommonBackend.encode does not produce the audio "
+                "placeholder token (ID 24) from raw text. apply_chat_template "
+                "yields token IDs with placeholders, but MultiModalProcessor."
+                "apply() decodes the prompt back to text and re-tokenizes, at "
+                "which point the placeholders are lost. Fix belongs in "
+                "mistral_common or in the Voxtral-specific path.",
+                strict=False,
+            ),
+        ),
+        "microsoft/VibeVoice-ASR-HF",
+        "zai-org/GLM-ASR-Nano-2512",
+    ],
+)
+def test_audio_multimodal_processor(model_id):
+    settings = AUDIO_MODEL_SETTINGS[model_id]
+
+    model_config = ModelConfig(
+        model=model_id,
+        model_impl="transformers",
+    )
+
+    mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
+
+    audio = np.zeros(16000, dtype=np.float32)
+    mm_data = {"audio": (audio, 16000)}
+
+    result = mm_processor(
+        prompt=settings["prompt"],
+        mm_items=mm_processor.info.parse_mm_data(mm_data),
+        hf_processor_mm_kwargs={},
+    )
+
+    assert "prompt_token_ids" in result
+    assert len(result["prompt_token_ids"]) > 0
+
+    mm_placeholders = result.get("mm_placeholders", {})
+    assert "audio" in mm_placeholders, f"No audio placeholders found for {model_id}"
+    assert len(mm_placeholders["audio"]) == 1
+
+    placeholder = mm_placeholders["audio"][0]
+    assert placeholder.length > 0
+    assert placeholder.offset >= 0
+
+    audio_items = result.get("mm_kwargs", {}).get("audio", [])
+    assert len(audio_items) == 1, f"Expected 1 audio item, got {len(audio_items)}"
+    item_keys = list(audio_items[0].keys())
+    has_features = "input_features" in item_keys or "input_values" in item_keys
+    assert has_features, (
+        f"No audio features (input_features/input_values) in {item_keys} for {model_id}"
+    )
+
+
+def test_audio_multiple_inputs():
+    """Multiple audios per prompt are each detected as a separate placeholder
+    and multi-modal item by the Transformers backend."""
+    model_id = "ibm-granite/granite-speech-3.3-2b"
+    model_config = ModelConfig(model=model_id, model_impl="transformers")
+    mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
+
+    audio_token = mm_processor.info.get_hf_processor().audio_token
+    # One token per audio; the processor expands each to its placeholder run.
+    prompt = (
+        "<|start_of_role|>user<|end_of_role|>"
+        f"{audio_token} and {audio_token} transcribe<|end_of_text|>\n"
+    )
+    audios = [np.zeros(16000, dtype=np.float32), np.zeros(24000, dtype=np.float32)]
+
+    result = mm_processor(
+        prompt=prompt,
+        mm_items=mm_processor.info.parse_mm_data({"audio": audios}),
+        hf_processor_mm_kwargs={},
+    )
+
+    assert len(result["mm_placeholders"]["audio"]) == 2
+    assert len(result["mm_kwargs"]["audio"]) == 2

tests/models/multimodal/processing/test_transformers.py → tests/models/multimodal/processing/test_transformers_image.py

@@ -54,3 +54,26 @@ def test_multimodal_processor(model_id):
         str_processed_inputs["prompt_token_ids"]
         == ids_processed_inputs["prompt_token_ids"]
     )
+
+
+def test_image_multiple_inputs():
+    """Multiple images per prompt are each detected as a separate placeholder
+    and multi-modal item by the Transformers backend."""
+    model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
+    model_config = ModelConfig(model=model_id, model_impl="transformers")
+    mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
+
+    image = ImageAsset("cherry_blossom").pil_image
+    prompt = (
+        "<|im_start|>user <image>\n and <image>\n"
+        "What do these images show?<|im_end|><|im_start|>assistant\n"
+    )
+
+    result = mm_processor(
+        prompt=prompt,
+        mm_items=mm_processor.info.parse_mm_data({"image": [image, image]}),
+        hf_processor_mm_kwargs={},
+    )
+
+    assert len(result["mm_placeholders"]["image"]) == 2
+    assert len(result["mm_kwargs"]["image"]) == 2

tests/models/registry.py

@@ -1394,6 +1394,10 @@ def check_available_online(
         "fixie-ai/ultravox-v0_5-llama-3_2-1b",
         trust_remote_code=True,
     ),
+    "VibeVoiceAsrForConditionalGeneration": _HfExamplesInfo(
+        "microsoft/VibeVoice-ASR-HF",
+        min_transformers_version="5.13.0",
+    ),
     "VoxtralForConditionalGeneration": _HfExamplesInfo(
         "mistralai/Voxtral-Mini-3B-2507",
         tokenizer_mode="mistral",

vllm/model_executor/models/registry.py

@@ -665,6 +665,10 @@
         "transformers",
         "TransformersMultiModalForCausalLM",
     ),
+    "VibeVoiceAsrForConditionalGeneration": (
+        "transformers",
+        "TransformersMultiModalForCausalLM",
+    ),
 }
 
 _TRANSFORMERS_BACKEND_MODELS = {