From 933c21c36526c93263d5c3153ab49d7cf2071812 Mon Sep 17 00:00:00 2001 From: Lanze Liu Date: Thu, 27 Aug 2026 14:38:42 -0700 Subject: [PATCH] [Bugfix][Multimodal] Bound PyAV FFmpeg decode threads via num_ffmpeg_threads The PyAV video backend set thread_type = "SLICE" without a thread_count, so FFmpeg auto-sized min(cpu_count + 1, 16) slice workers per open codec. Under serving concurrency this multiplies across media-pool slots and processes into far more decode threads than cores, and decode threads hang permanently in pthread_cond_wait inside avpriv_slicethread_execute (PyAV-Org/PyAV#615 has the identical stack). Expose num_ffmpeg_threads for the pyav backend through media_io_kwargs, mirroring the torchcodec option (0 = FFmpeg auto sizing), and default it to a bounded 4 so the stock serving configuration cannot oversubscribe. FIX #53973 Signed-off-by: Lanze Liu --- docs/features/multimodal_inputs.md | 20 ++++++- tests/multimodal/test_video.py | 62 ++++++++++++++++++++-- vllm/multimodal/video.py | 11 ++-- vllm/multimodal/video_decoders/__init__.py | 4 +- vllm/multimodal/video_decoders/pyav.py | 14 ++++- 5 files changed, 100 insertions(+), 11 deletions(-) diff --git a/docs/features/multimodal_inputs.md b/docs/features/multimodal_inputs.md index 387049f7cc71..6a0f2d021202 100644 --- a/docs/features/multimodal_inputs.md +++ b/docs/features/multimodal_inputs.md @@ -868,7 +868,7 @@ vllm serve Qwen/Qwen3-VL-30B-A3B-Instruct \ **TorchCodec-specific parameters:** -The following parameters only apply to the `torchcodec` backend: +The following parameters apply to the `torchcodec` backend: - `num_ffmpeg_threads`: Number of FFmpeg decoding threads. `0` (default) relies on the FFmpeg default, which is `min(cpu_count + 1, 16)`. This allows you to @@ -884,6 +884,24 @@ vllm serve Qwen/Qwen3-VL-30B-A3B-Instruct \ --media-io-kwargs '{"video": {"backend": "torchcodec", "seek_mode": "approximate", "num_ffmpeg_threads": 4}}' ``` +**PyAV-specific parameters:** + +The following parameters apply to the `pyav` backend: + +- `num_ffmpeg_threads`: Number of FFmpeg decoding threads — the same option as + `torchcodec`, but defaulting to `4` here to bound the per-request + slice-thread pool. With the FFmpeg default sizing (`0`, + `min(cpu_count + 1, 16)` workers per open codec), concurrent requests + multiply into far more decode threads than cores under serving load, which + can oversubscribe the host until decodes hang. Pass `0` to restore the + FFmpeg default sizing. + +```bash +# Example: PyAV with a stricter 2-thread decode bound +vllm serve Qwen/Qwen3-VL-30B-A3B-Instruct \ + --media-io-kwargs '{"video": {"backend": "pyav", "num_ffmpeg_threads": 2}}' +``` + **PyNvVideoCodec-specific parameters:** - `hw_decoders`: Maximum number of concurrent hardware decoder slots retained diff --git a/tests/multimodal/test_video.py b/tests/multimodal/test_video.py index 4020900696f3..54a1e542174e 100644 --- a/tests/multimodal/test_video.py +++ b/tests/multimodal/test_video.py @@ -6,6 +6,7 @@ import sys import threading from contextlib import ExitStack, contextmanager +from io import BytesIO from pathlib import Path import numpy as np @@ -31,6 +32,7 @@ get_video_loader_backend_for_processor, ) from vllm.multimodal.video_decoders import decode_video, resolve_video_backend_kwargs +from vllm.multimodal.video_decoders.pyav import PyAVVideoBackendMixin from vllm.multimodal.video_decoders.pynvvideocodec import ( PYNVVIDEOCODEC_DECODER_CACHE_SIZE, PyNvVideoCodecDecoderSlot, @@ -181,6 +183,19 @@ def fake_import_module(name: str, package: str): {"max_frames": 16}, {"pool_size": 3, "timeout_sec": 10.0}, ), + ( + "pyav", + {"min_frames": 4, "num_ffmpeg_threads": 2}, + {"min_frames": 4}, + {"num_ffmpeg_threads": 2}, + ), + # PyAV must default to a bounded thread count, not FFmpeg auto (#53973). + ( + "pyav", + {"min_frames": 4}, + {"min_frames": 4}, + {"num_ffmpeg_threads": 4}, + ), ], ) def test_video_backend_kwargs_are_separated_from_sampling_kwargs( @@ -199,9 +214,15 @@ def test_video_backend_kwargs_are_separated_from_sampling_kwargs( def test_video_backend_rejects_options_for_another_decoder(): with pytest.raises( - ValueError, match="num_ffmpeg_threads is not supported by the 'pyav' backend" + ValueError, match="seek_mode is not supported by the 'pyav' backend" ): - resolve_video_backend_kwargs("pyav", {"num_ffmpeg_threads": 2}) + resolve_video_backend_kwargs("pyav", {"seek_mode": "exact"}) + + with pytest.raises( + ValueError, + match="num_ffmpeg_threads is not supported by the 'opencv' backend", + ): + resolve_video_backend_kwargs("opencv", {"num_ffmpeg_threads": 2}) @pytest.mark.parametrize( @@ -1150,7 +1171,9 @@ def test_pyav_backend_loads_frames(dummy_video_path, monkeypatch: pytest.MonkeyP video_data = f.read() loader = VIDEO_LOADER_REGISTRY.load("opencv") - frames, metadata = loader.load_bytes(video_data, num_frames=8, backend="pyav") + frames, metadata = loader.load_bytes( + video_data, num_frames=8, backend="pyav", num_ffmpeg_threads=2 + ) assert frames.ndim == 4 assert frames.shape[3] == 3 # RGB @@ -1227,6 +1250,39 @@ def test_pyav_backend_returns_target_frames_not_keyframes(): ) +@pytest.mark.parametrize("num_ffmpeg_threads", [2, 0]) +def test_pyav_backend_bounds_decode_threads(num_ffmpeg_threads: int): + """Regression test for #53973: PyAV must bound the FFmpeg slice-thread pool. + + Left at FFmpeg's auto sizing, every open codec allocates roughly one + worker per host CPU, which multiplies across concurrent decodes and + hangs decode threads under serving load. ``0`` remains the documented + escape hatch back to FFmpeg auto sizing. + """ + import av + + video_bytes = create_long_gop_video(num_frames=30) + + with av.open(BytesIO(video_bytes)) as container: + source = PyAVVideoBackendMixin.get_metadata(container) + frames, valid = PyAVVideoBackendMixin.decode_frames( + container, + [0, 15], + source.original_fps, + source.duration, + num_ffmpeg_threads=num_ffmpeg_threads, + ) + thread_count = container.streams.video[0].codec_context.thread_count + if num_ffmpeg_threads > 0: + assert thread_count == num_ffmpeg_threads + else: + # FFmpeg resolves auto sizing to a host-dependent count >= 1. + assert thread_count >= 1 + + assert valid == [0, 15] + assert frames.shape[0] == 2 + + # ============================================================================ # TorchCodec Backend Tests # ============================================================================ diff --git a/vllm/multimodal/video.py b/vllm/multimodal/video.py index f1a605311a16..6d3fb861bb8e 100644 --- a/vllm/multimodal/video.py +++ b/vllm/multimodal/video.py @@ -237,11 +237,12 @@ def load_bytes( kwargs: Codec-specific options, validated against and forwarded to ``backend``: - - ``num_ffmpeg_threads`` (TorchCodec): number of FFmpeg - decoding threads; ``0`` (default) relies on the FFmpeg - default value which is ``min(cpu_count + 1, 16)``. - OpenCV will always use ``min(cpu_count, 16)`` while pyav - will always use ``min(cpu_count, (height + 15) / 16)``. + - ``num_ffmpeg_threads`` (TorchCodec, PyAV): number of FFmpeg + decoding threads; ``0`` relies on the FFmpeg default value + which is ``min(cpu_count + 1, 16)``. TorchCodec defaults to + ``0`` while PyAV defaults to ``4`` to bound the per-decode + slice-thread pool under concurrent serving load. OpenCV + will always use ``min(cpu_count, 16)``. - ``seek_mode`` (TorchCodec): ``"exact"`` (default) guarantees frame-accurate sampling by scanning the file on creation, while ``"approximate"`` skips that scan for faster decoder diff --git a/vllm/multimodal/video_decoders/__init__.py b/vllm/multimodal/video_decoders/__init__.py index c62310431652..04a72f1dbe0d 100644 --- a/vllm/multimodal/video_decoders/__init__.py +++ b/vllm/multimodal/video_decoders/__init__.py @@ -18,7 +18,9 @@ _BACKEND_OPTION_DEFAULTS: dict[str, dict[str, Any]] = { "opencv": {}, - "pyav": {}, + "pyav": { + "num_ffmpeg_threads": 4, + }, "torchcodec": { "num_ffmpeg_threads": 0, "seek_mode": "exact", diff --git a/vllm/multimodal/video_decoders/pyav.py b/vllm/multimodal/video_decoders/pyav.py index 80c282966668..56140961ac59 100644 --- a/vllm/multimodal/video_decoders/pyav.py +++ b/vllm/multimodal/video_decoders/pyav.py @@ -25,6 +25,8 @@ def decode_pyav( data: bytes, target: VideoTargetMetadata, sampling_kwargs: dict, + *, + num_ffmpeg_threads: int = 4, ) -> tuple[npt.NDArray, VideoSourceMetadata, list[int], list[int]]: with av.open(BytesIO(data)) as container: stream = container.streams.video[0] @@ -36,7 +38,11 @@ def decode_pyav( source=source, target=target, **sampling_kwargs ) frames, valid = PyAVVideoBackendMixin.decode_frames( - container, frame_idx, source.original_fps, source.duration + container, + frame_idx, + source.original_fps, + source.duration, + num_ffmpeg_threads=num_ffmpeg_threads, ) return frames, source, frame_idx, valid @@ -78,12 +84,18 @@ def decode_frames( frame_indices: list[int], fps: float, duration: float, + *, + num_ffmpeg_threads: int = 4, ) -> tuple[npt.NDArray, list[int]]: """Decode target frames via per-frame seek + forward decode to PTS.""" stream = container.streams.video[0] # SLICE parallelizes within a single frame without the # one-frame-per-thread latency penalty of FRAME threading. stream.thread_type = "SLICE" + # FFmpeg auto sizing (0) allocates min(cpu_count + 1, 16) workers + # per open codec, which multiplies across concurrent requests and + # deadlocks under serving load (#53973). + stream.thread_count = num_ffmpeg_threads time_base = stream.time_base frames_list: list[npt.NDArray] = []