Support per-video use_audio_in_video for Qwen Omni

tests/models/multimodal/processing/test_audio_in_video.py

@@ -113,3 +113,50 @@ def run(processor):
         f"  baseline : {baseline_ids}\n"
         f"  cache-hit: {second_ids}"
     )
+
+
+@pytest.mark.parametrize("model_id", MODELS)
+def test_audio_in_video_per_video_mask_mixed(model_id: str) -> None:
+    ctx = build_model_context(
+        model_id,
+        limit_mm_per_prompt={"audio": 1, "image": 0, "video": 2},
+    )
+    processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config, cache=None)
+
+    hf_config = processor.info.get_hf_config()
+    video_token_id = hf_config.video_token_id
+    audio_token_id = hf_config.audio_token_id
+
+    mm_data = create_mm_data(num_videos=2)
+    mm_data["audio"] = mm_data["audio"][:1]
+
+    processed = processor(
+        [video_token_id, video_token_id],
+        mm_items=processor.info.parse_mm_data(mm_data),
+        hf_processor_mm_kwargs={"use_audio_in_video": [True, False]},
+    )
+
+    prompt_token_ids = processed["prompt_token_ids"]
+    placeholders = processed["mm_placeholders"]
+
+    assert len(placeholders["video"]) == 2
+    assert len(placeholders["audio"]) == 1
+
+    audio_placeholder = placeholders["audio"][0]
+    first_video_placeholder = placeholders["video"][0]
+    second_video_placeholder = placeholders["video"][1]
+
+    assert audio_placeholder.offset == first_video_placeholder.offset
+    assert audio_placeholder.length == first_video_placeholder.length
+
+    first_video_tokens = prompt_token_ids[
+        first_video_placeholder.offset : first_video_placeholder.offset
+        + first_video_placeholder.length
+    ]
+    second_video_tokens = prompt_token_ids[
+        second_video_placeholder.offset : second_video_placeholder.offset
+        + second_video_placeholder.length
+    ]
+
+    assert audio_token_id in first_video_tokens
+    assert audio_token_id not in second_video_tokens

vllm/entrypoints/chat_utils.py

@@ -5,7 +5,7 @@
 import json
 from abc import ABC, abstractmethod
 from collections import Counter, defaultdict
-from collections.abc import Awaitable, Callable, Iterable
+from collections.abc import Awaitable, Callable, Iterable, Sequence
 from dataclasses import dataclass
 from functools import cached_property, lru_cache, partial
 from itertools import accumulate
@@ -914,6 +914,26 @@ def parse_video(self, video_url: str | None, uuid: str | None = None) -> None:
         raise NotImplementedError
 
 
+def _use_audio_in_video_for_idx(
+    mm_processor_kwargs: dict[str, Any] | None,
+    video_idx: int,
+) -> bool:
+    if not mm_processor_kwargs:
+        return False
+
+    value = mm_processor_kwargs.get("use_audio_in_video", False)
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, Sequence) and not isinstance(value, (str, bytes)):
+        if video_idx >= len(value):
+            raise ValueError(
+                "use_audio_in_video must contain one boolean per video, "
+                f"but found {len(value)} values for video index {video_idx}."
+            )
+        return bool(value[video_idx])
+    return bool(value)
+
+
 class MultiModalContentParser(BaseMultiModalContentParser):
     def __init__(
         self,
@@ -932,6 +952,7 @@ def __init__(
         )
 
         self._mm_processor_kwargs = mm_processor_kwargs
+        self._num_parsed_videos = 0
 
     @property
     def model_config(self) -> ModelConfig:
@@ -1039,6 +1060,9 @@ def parse_input_audio(
         return self.parse_audio(audio_url, uuid)
 
     def parse_video(self, video_url: str | None, uuid: str | None = None) -> None:
+        video_idx = self._num_parsed_videos
+        self._num_parsed_videos += 1
+
         video = (
             self._connector.fetch_video(
                 video_url=video_url,
@@ -1052,10 +1076,8 @@ def parse_video(self, video_url: str | None, uuid: str | None = None) -> None:
         self._add_placeholder("video", placeholder)
 
         # Extract audio from video if use_audio_in_video is True
-        if (
-            video_url
-            and self._mm_processor_kwargs
-            and self._mm_processor_kwargs.get("use_audio_in_video", False)
+        if video_url and _use_audio_in_video_for_idx(
+            self._mm_processor_kwargs, video_idx
         ):
             audio = self._connector.fetch_audio(video_url) if video_url else None
             audio_placeholder = self._tracker.add("audio", (audio, uuid))
@@ -1078,6 +1100,7 @@ def __init__(
             allowed_media_domains=tracker.allowed_media_domains,
         )
         self._mm_processor_kwargs: dict[str, Any] | None = mm_processor_kwargs
+        self._num_parsed_videos = 0
 
     @property
     def model_config(self) -> ModelConfig:
@@ -1226,16 +1249,17 @@ async def _video_with_uuid_async(self, video_url: str | None, uuid: str | None):
         return video, uuid
 
     def parse_video(self, video_url: str | None, uuid: str | None = None) -> None:
+        video_idx = self._num_parsed_videos
+        self._num_parsed_videos += 1
+
         placeholder = self._tracker.add(
             "video", partial(self._video_with_uuid_async, video_url, uuid)
         )
         self._add_placeholder("video", placeholder)
 
         # Extract audio from video if use_audio_in_video is True
-        if (
-            video_url
-            and self._mm_processor_kwargs
-            and self._mm_processor_kwargs.get("use_audio_in_video", False)
+        if video_url and _use_audio_in_video_for_idx(
+            self._mm_processor_kwargs, video_idx
         ):
             audio_placeholder = self._tracker.add(
                 "audio", partial(self._audio_with_uuid_async, video_url, uuid)