From 32f4bd07f7f976df5afacdc12c4e01c1531807fd Mon Sep 17 00:00:00 2001 From: Arif Ahmed Date: Thu, 3 Sep 2026 18:25:00 -0700 Subject: [PATCH] [Multimodal] Add TorchCodec device selection with host-frame conversion TorchCodec supports CUDA (NVDEC) decoding when built with CUDA support, but the vLLM backend never forwarded a device, so users could not select it through --media-io-kwargs. Its CUDA output tensors also cannot be converted to NumPy directly. - register a torchcodec-only ``device`` option with the backend kwargs validator and forward it to ``VideoDecoder`` only when set, keeping the CPU default and TorchCodec's own validation untouched - copy only the sampled frames to the host before the NumPy conversion, a no-op for the CPU decoder - treat ``backend=torchcodec, device=cuda`` as a GPU video backend so the API server's frontend GPU memory reservation applies, and account the sampled frames against the ``--mm-ipc-gpu-memory-gb`` pool like the other GPU backends - document the option and the MPS / reservation guidance; add unit tests and a CUDA-gated frame-correctness test against the CPU decoder Co-Authored-By: Claude Fable 5.1 Signed-off-by: Arif Ahmed --- docs/features/multimodal_inputs.md | 22 +++- tests/config/test_multimodal_config.py | 15 +++ tests/multimodal/test_gpu_ipc_memory.py | 18 +++ tests/multimodal/test_video.py | 110 ++++++++++++++++++- vllm/config/multimodal.py | 10 +- vllm/multimodal/video.py | 5 + vllm/multimodal/video_decoders/__init__.py | 1 + vllm/multimodal/video_decoders/torchcodec.py | 62 +++++++++-- 8 files changed, 225 insertions(+), 18 deletions(-) diff --git a/docs/features/multimodal_inputs.md b/docs/features/multimodal_inputs.md index 64dca49d2848..92a1446e2db5 100644 --- a/docs/features/multimodal_inputs.md +++ b/docs/features/multimodal_inputs.md @@ -850,13 +850,13 @@ backends are supported: | Backend | Device | Description | | --- | --- | --- | | `opencv` (default) | CPU | OpenCV-based decoder | -| `torchcodec` | CPU | TorchCodec (PyTorch-native) decoder | +| `torchcodec` | CPU / GPU | TorchCodec (PyTorch-native) decoder; CUDA (NVDEC) decode is optional | | `pynvvideocodec` | GPU | NVIDIA PyNvVideoCodec decoder | | `deepstream` | GPU | NVIDIA DeepStream decoder | -The two CPU backends are ultimately backed by FFmpeg. `torchcodec` lets you -choose which FFmpeg version is used while `opencv` relies on whichever -FFmpeg build it was linked against. +The `opencv` backend and the default CPU mode of `torchcodec` are ultimately +backed by FFmpeg. `torchcodec` lets you choose which FFmpeg version is used +while `opencv` relies on whichever FFmpeg build it was linked against. Select the backend by passing the `backend` parameter via `--media-io-kwargs`: @@ -876,6 +876,14 @@ The following parameters only apply to the `torchcodec` backend: frame-accurate sampling by scanning the file when the decoder is created. `"approximate"` skips that scan for faster decoder creation, at the cost of relying on the file's metadata (which may yield less accurate seeking). +- `device`: Device used for decoding. Unset (default) or `"cpu"` decodes with + FFmpeg on the CPU; `"cuda"` decodes with NVDEC when the installed TorchCodec + build has CUDA support. Sampled frames are copied to host memory before + multimodal preprocessing either way, so the rest of the pipeline is + unchanged. GPU decoding runs in the API server process, so the + [PyNvVideoCodec guidance](#gpu-video-decoding-with-pynvvideocodec-nvdec) + on CUDA MPS and `--mm-ipc-gpu-memory-gb` applies to `"cuda"` as well, and + vLLM applies the same frontend GPU memory reservation. ```bash # Example: TorchCodec with approximate seek mode and 4 FFmpeg threads @@ -883,6 +891,12 @@ vllm serve Qwen/Qwen3-VL-30B-A3B-Instruct \ --media-io-kwargs '{"video": {"backend": "torchcodec", "seek_mode": "approximate", "num_ffmpeg_threads": 4}}' ``` +```bash +# Example: TorchCodec decoding on the GPU with NVDEC +vllm serve Qwen/Qwen3-VL-30B-A3B-Instruct \ + --media-io-kwargs '{"video": {"backend": "torchcodec", "device": "cuda"}}' +``` + **PyNvVideoCodec-specific parameters:** - `hw_decoders`: Maximum number of concurrent hardware decoder slots retained diff --git a/tests/config/test_multimodal_config.py b/tests/config/test_multimodal_config.py index 589eabf91d67..6fb719502c35 100644 --- a/tests/config/test_multimodal_config.py +++ b/tests/config/test_multimodal_config.py @@ -66,6 +66,21 @@ def test_use_gpu_video_backend_from_media_io_kwargs(backend_arg: str): assert config.use_gpu_video_backend() +@pytest.mark.parametrize( + ("device", "expected"), + [("cuda", True), ("cuda:1", True), ("cpu", False), (None, False)], +) +def test_use_gpu_video_backend_for_torchcodec_device( + device: str | None, expected: bool +): + video_kwargs: dict[str, object] = {"backend": "torchcodec"} + if device is not None: + video_kwargs["device"] = device + config = MultiModalConfig(media_io_kwargs={"video": video_kwargs}) + + assert config.use_gpu_video_backend() is expected + + def test_mm_encoder_fp8_scale_path_requires_fp8(): with pytest.raises(ValueError, match="mm_encoder_attn_dtype"): MultiModalConfig(mm_encoder_fp8_scale_path="/tmp/scales.json") diff --git a/tests/multimodal/test_gpu_ipc_memory.py b/tests/multimodal/test_gpu_ipc_memory.py index 9afbb3a1c76b..648174cf6e25 100644 --- a/tests/multimodal/test_gpu_ipc_memory.py +++ b/tests/multimodal/test_gpu_ipc_memory.py @@ -209,6 +209,24 @@ def test_reserve_mm_ipc_gpu_memory_includes_pynvvideocodec_decoder_budget( ) +def test_reserve_mm_ipc_gpu_memory_includes_torchcodec_cuda_decoder_budget( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv") + available_bytes = 4 * GiB_bytes + cpu_config = MultiModalConfig( + media_io_kwargs={"video": {"backend": "torchcodec"}}, + ) + cuda_config = MultiModalConfig( + media_io_kwargs={"video": {"backend": "torchcodec", "device": "cuda"}}, + ) + + assert reserve_mm_ipc_gpu_memory(available_bytes, cpu_config) == available_bytes + assert reserve_mm_ipc_gpu_memory(available_bytes, cuda_config) == ( + available_bytes - _pynvvideocodec_decoder_budget(hw_decoders=1) + ) + + def test_reserve_mm_ipc_gpu_memory_uses_env_video_backend( monkeypatch: pytest.MonkeyPatch, ): diff --git a/tests/multimodal/test_video.py b/tests/multimodal/test_video.py index 9c24b5f96932..bf61d27836c7 100644 --- a/tests/multimodal/test_video.py +++ b/tests/multimodal/test_video.py @@ -7,6 +7,7 @@ import threading from contextlib import ExitStack, contextmanager from pathlib import Path +from types import SimpleNamespace import numpy as np import numpy.typing as npt @@ -175,7 +176,7 @@ def fake_import_module(name: str, package: str): "torchcodec", {"min_frames": 4, "num_ffmpeg_threads": 2, "seek_mode": "approximate"}, {"min_frames": 4}, - {"num_ffmpeg_threads": 2, "seek_mode": "approximate"}, + {"num_ffmpeg_threads": 2, "seek_mode": "approximate", "device": None}, ), ( "deepstream", @@ -206,6 +207,16 @@ def test_video_backend_rejects_options_for_another_decoder(): resolve_video_backend_kwargs("opencv", {"num_ffmpeg_threads": 2}) +def test_video_backend_device_option_belongs_to_torchcodec(): + _, backend_kwargs = resolve_video_backend_kwargs("torchcodec", {"device": "cuda"}) + assert backend_kwargs["device"] == "cuda" + + with pytest.raises( + ValueError, match="device is not supported by the 'opencv' backend" + ): + resolve_video_backend_kwargs("opencv", {"device": "cuda"}) + + @pytest.mark.parametrize( ("backend", "error"), [ @@ -1205,6 +1216,103 @@ def test_torchcodec_backend_rejects_frame_recovery(dummy_video_path): ) +def test_torchcodec_backend_forwards_decode_device(monkeypatch: pytest.MonkeyPatch): + """``device`` reaches ``VideoDecoder`` only when set, and sampled frames + are moved to the host before the NumPy conversion.""" + from vllm.multimodal.video_decoders import torchcodec as torchcodec_module + + constructor_kwargs: list[dict[str, object]] = [] + requested_indices: list[list[int]] = [] + cpu_calls: list[None] = [] + acquired: list[int] = [] + + class RecordingPool: + @contextmanager + def acquire(self, size: int): + acquired.append(size) + yield + + class FakeTensor: + def cpu(self): + cpu_calls.append(None) + return self + + def numpy(self): + return np.zeros((len(requested_indices[-1]), 20, 10, 3), dtype=np.uint8) + + class FakeDecoder: + metadata = SimpleNamespace( + num_frames=4, average_fps=2.0, duration_seconds=2.0, width=10, height=20 + ) + + def __init__(self, data: bytes, **kwargs): + constructor_kwargs.append(kwargs) + + def get_frames_at(self, frame_indices: list[int]): + requested_indices.append(list(frame_indices)) + return SimpleNamespace(data=FakeTensor()) + + monkeypatch.setattr(torchcodec_module, "check_torchcodec_available", lambda: None) + monkeypatch.setattr(torchcodec_module, "VideoDecoder", FakeDecoder) + monkeypatch.setattr( + "vllm.multimodal.gpu_ipc_memory.get_mm_gpu_ipc_pool", RecordingPool + ) + + frames, metadata = VideoBackend.load_bytes( + b"fake video", num_frames=2, backend="torchcodec" + ) + assert "device" not in constructor_kwargs[-1] + assert constructor_kwargs[-1]["dimension_order"] == "NHWC" + assert frames.shape == (2, 20, 10, 3) + # CPU decode never touches the frontend GPU budget. + assert acquired == [] + + frames, metadata = VideoBackend.load_bytes( + b"fake video", num_frames=2, backend="torchcodec", device="cuda" + ) + assert constructor_kwargs[-1]["device"] == "cuda" + assert frames.shape == (2, 20, 10, 3) + assert metadata["frames_indices"] == requested_indices[-1] + assert len(cpu_calls) == 2 + # CUDA decode accounts the sampled frames against the frontend budget. + assert acquired == [2 * 20 * 10 * 3] + + +@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA") +def test_torchcodec_cuda_backend_matches_cpu_frames(): + """CUDA decode must select the same frames as CPU decode and return host + frames; NVDEC and FFmpeg color conversion may differ by a few levels.""" + torchcodec = pytest.importorskip("torchcodec") + num_frames, height, width = 50, 64, 64 + video_bytes = create_long_gop_video( + num_frames=num_frames, width=width, height=height + ) + try: + torchcodec.decoders.VideoDecoder(video_bytes, device="cuda") + except Exception as exc: # CPU-only TorchCodec build or no NVDEC + pytest.skip(f"TorchCodec CUDA decode unavailable: {exc}") + + loader = VIDEO_LOADER_REGISTRY.load("opencv") + cpu_frames, cpu_meta = loader.load_bytes( + video_bytes, num_frames=4, backend="torchcodec" + ) + gpu_frames, gpu_meta = loader.load_bytes( + video_bytes, num_frames=4, backend="torchcodec", device="cuda" + ) + + assert isinstance(gpu_frames, np.ndarray) + assert gpu_frames.shape == cpu_frames.shape == (4, height, width, 3) + assert gpu_meta["frames_indices"] == cpu_meta["frames_indices"] + + # The long-GOP fixture stamps a per-frame green marker at the center. + cpu_markers = [int(f[height // 2, width // 2, 1]) for f in cpu_frames] + gpu_markers = [int(f[height // 2, width // 2, 1]) for f in gpu_frames] + assert len(set(gpu_markers)) == 4 + assert all(abs(a - b) <= 8 for a, b in zip(cpu_markers, gpu_markers)) + diff = np.abs(gpu_frames.astype(np.int16) - cpu_frames.astype(np.int16)) + assert diff.mean() < 4.0 + + def test_torchcodec_backend_returns_target_frames_not_keyframes(): """Regression test: torchcodec must return the requested frames, not the GOP keyframe they seek back to. diff --git a/vllm/config/multimodal.py b/vllm/config/multimodal.py index ce2217d9fefd..bca61f24193a 100644 --- a/vllm/config/multimodal.py +++ b/vllm/config/multimodal.py @@ -526,10 +526,16 @@ def use_gpu_video_backend(self) -> bool: video_kwargs.get("video_backend") or envs.VLLM_VIDEO_LOADER_BACKEND ) codec_backend = video_kwargs.get("backend") - return VIDEO_LOADER_REGISTRY.backend_requires_gpu(video_loader_backend) or ( + if VIDEO_LOADER_REGISTRY.backend_requires_gpu(video_loader_backend) or ( codec_backend is not None and VIDEO_LOADER_REGISTRY.backend_requires_gpu(codec_backend) - ) + ): + return True + # TorchCodec decodes on the CPU by default but runs NVDEC inside the + # API server process when ``device`` selects CUDA. + return codec_backend == "torchcodec" and str( + video_kwargs.get("device") or "" + ).startswith("cuda") def is_multimodal_pruning_enabled(self): return self.get_video_pruning_spec() is not None diff --git a/vllm/multimodal/video.py b/vllm/multimodal/video.py index 854b5572b3ef..801646673331 100644 --- a/vllm/multimodal/video.py +++ b/vllm/multimodal/video.py @@ -265,6 +265,11 @@ def load_bytes( creation at the cost of relying on the file's metadata. See https://meta-pytorch.org/torchcodec/stable/generated_examples/decoding/approximate_mode.html for details. + - ``device`` (TorchCodec): decoder device. ``None`` (default) + or ``"cpu"`` decodes with FFmpeg on the CPU; ``"cuda"`` + decodes with NVDEC when the installed TorchCodec build has + CUDA support. Sampled frames are copied to host memory for + multimodal preprocessing either way. - ``hw_decoders`` (PyNvVideoCodec): maximum number of concurrent decoder slots. Defaults to 2 and must be a positive integer. diff --git a/vllm/multimodal/video_decoders/__init__.py b/vllm/multimodal/video_decoders/__init__.py index 1e6106e19937..a8a05b395b54 100644 --- a/vllm/multimodal/video_decoders/__init__.py +++ b/vllm/multimodal/video_decoders/__init__.py @@ -19,6 +19,7 @@ "torchcodec": { "num_ffmpeg_threads": 0, "seek_mode": "exact", + "device": None, }, PYNVVIDEOCODEC_VIDEO_BACKEND: { "hw_decoders": PYNVVIDEOCODEC_DEFAULT_HW_DECODERS, diff --git a/vllm/multimodal/video_decoders/torchcodec.py b/vllm/multimodal/video_decoders/torchcodec.py index f1f3290629a2..5907daf4e716 100644 --- a/vllm/multimodal/video_decoders/torchcodec.py +++ b/vllm/multimodal/video_decoders/torchcodec.py @@ -1,7 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from typing import Literal +from contextlib import AbstractContextManager, nullcontext +from typing import TYPE_CHECKING, Any, Literal import numpy as np import numpy.typing as npt @@ -14,6 +15,9 @@ check_frame_pixel_limit, ) +if TYPE_CHECKING: + import torch + try: from torchcodec.decoders import VideoDecoder except (ImportError, RuntimeError): @@ -30,12 +34,14 @@ def decode_torchcodec( *, num_ffmpeg_threads: int = 0, seek_mode: Literal["exact", "approximate"] = "exact", + device: "str | torch.device | None" = None, ) -> tuple[npt.NDArray, VideoSourceMetadata, list[int], list[int]]: check_torchcodec_available() decoder = TorchCodecVideoBackendMixin.make_torchcodec_decoder( data, num_ffmpeg_threads=num_ffmpeg_threads, seek_mode=seek_mode, + device=device, ) check_frame_pixel_limit( decoder.metadata.width or 0, @@ -47,18 +53,43 @@ def decode_torchcodec( frame_idx = loader_cls.compute_frames_index_to_sample( source=source, target=target, **sampling_kwargs ) - frames, valid = TorchCodecVideoBackendMixin.decode_torchcodec_frames( - decoder, frame_idx - ) + budget: AbstractContextManager[Any] = nullcontext() + if _is_cuda_device(device): + # GPU decode runs in the API server process. Account for the sampled + # frames against the frontend budget like the other GPU backends do. + from vllm.multimodal.gpu_ipc_memory import get_mm_gpu_ipc_pool + + pool = get_mm_gpu_ipc_pool() + raw_frame_bytes = ( + len(frame_idx) + * (decoder.metadata.height or 0) + * (decoder.metadata.width or 0) + * 3 + ) + if pool is not None and raw_frame_bytes > 0: + budget = pool.acquire(raw_frame_bytes) + with budget: + frames, valid = TorchCodecVideoBackendMixin.decode_torchcodec_frames( + decoder, frame_idx + ) return frames, source, frame_idx, valid +def _is_cuda_device(device: "str | torch.device | None") -> bool: + return device is not None and str(device).startswith("cuda") + + class TorchCodecVideoBackendMixin: """TorchCodec (FFmpeg-backed, PyTorch-native) codec utilities. Builds a :class:`~torchcodec.decoders.VideoDecoder` over the in-memory bytes and extracts the sampled indices with a single batched ``get_frames_at`` call, while releasing the GIL during decode. + + ``device`` selects TorchCodec's decoder device. ``None`` keeps the CPU + (FFmpeg) default; ``"cuda"`` decodes with NVDEC when the installed + TorchCodec build has CUDA support. Sampled frames are always returned as + host NumPy arrays, which is what the multimodal processors consume. """ @staticmethod @@ -67,15 +98,20 @@ def make_torchcodec_decoder( *, num_ffmpeg_threads: int = 0, seek_mode: Literal["exact", "approximate"] = "exact", + device: "str | torch.device | None" = None, ) -> "VideoDecoder": # NHWC matches the (num_frames, H, W, 3) uint8 RGB layout the rest # of the pipeline expects, avoiding a transpose. - return VideoDecoder( - data, - dimension_order="NHWC", - num_ffmpeg_threads=num_ffmpeg_threads, - seek_mode=seek_mode, - ) + decoder_kwargs: dict[str, Any] = { + "dimension_order": "NHWC", + "num_ffmpeg_threads": num_ffmpeg_threads, + "seek_mode": seek_mode, + } + # Forward ``device`` only when one was selected so the TorchCodec + # default (CPU) and its own validation errors stay untouched. + if device is not None: + decoder_kwargs["device"] = device + return VideoDecoder(data, **decoder_kwargs) @staticmethod def get_torchcodec_metadata(decoder: "VideoDecoder") -> VideoSourceMetadata: @@ -97,4 +133,8 @@ def decode_torchcodec_frames( return np.empty((0,), dtype=np.uint8), [] # Note: torchcodec releases the GIL for the entire call batch = decoder.get_frames_at(frame_indices) - return batch.data.numpy(), list(frame_indices) + # Multimodal processors consume host NumPy frames. A CUDA decoder + # returns device tensors, which cannot be converted directly, so copy + # only the sampled frames at this boundary. ``cpu()`` is a no-op for + # the default CPU decoder. + return batch.data.cpu().numpy(), list(frame_indices)