[Multimodal] Add TorchCodec device option (CUDA/NVDEC decode) with host-frame conversion

docs/features/multimodal_inputs.md

@@ -850,13 +850,13 @@ backends are supported:
 | Backend | Device | Description |
 | --- | --- | --- |
 | `opencv` (default) | CPU | OpenCV-based decoder |
-| `torchcodec` | CPU | TorchCodec (PyTorch-native) decoder |
+| `torchcodec` | CPU / GPU | TorchCodec (PyTorch-native) decoder; CUDA (NVDEC) decode is optional |
 | `pynvvideocodec` | GPU | NVIDIA PyNvVideoCodec decoder |
 | `deepstream` | GPU | NVIDIA DeepStream decoder |
 
-The two CPU backends are ultimately backed by FFmpeg. `torchcodec` lets you
-choose which FFmpeg version is used while `opencv` relies on whichever
-FFmpeg build it was linked against.
+The `opencv` backend and the default CPU mode of `torchcodec` are ultimately
+backed by FFmpeg. `torchcodec` lets you choose which FFmpeg version is used
+while `opencv` relies on whichever FFmpeg build it was linked against.
 
 Select the backend by passing the `backend` parameter via `--media-io-kwargs`:
 
@@ -876,13 +876,27 @@ The following parameters only apply to the `torchcodec` backend:
   frame-accurate sampling by scanning the file when the decoder is created.
   `"approximate"` skips that scan for faster decoder creation, at the cost of
   relying on the file's metadata (which may yield less accurate seeking).
+- `device`: Device used for decoding. Unset (default) or `"cpu"` decodes with
+  FFmpeg on the CPU; `"cuda"` decodes with NVDEC when the installed TorchCodec
+  build has CUDA support. Sampled frames are copied to host memory before
+  multimodal preprocessing either way, so the rest of the pipeline is
+  unchanged. GPU decoding runs in the API server process, so the
+  [PyNvVideoCodec guidance](#gpu-video-decoding-with-pynvvideocodec-nvdec)
+  on CUDA MPS and `--mm-ipc-gpu-memory-gb` applies to `"cuda"` as well, and
+  vLLM applies the same frontend GPU memory reservation.
 
 ```bash
 # Example: TorchCodec with approximate seek mode and 4 FFmpeg threads
 vllm serve Qwen/Qwen3-VL-30B-A3B-Instruct \
   --media-io-kwargs '{"video": {"backend": "torchcodec", "seek_mode": "approximate", "num_ffmpeg_threads": 4}}'
 ```
 
+```bash
+# Example: TorchCodec decoding on the GPU with NVDEC
+vllm serve Qwen/Qwen3-VL-30B-A3B-Instruct \
+  --media-io-kwargs '{"video": {"backend": "torchcodec", "device": "cuda"}}'
+```
+
 **PyNvVideoCodec-specific parameters:**
 
 - `hw_decoders`: Maximum number of concurrent hardware decoder slots retained

tests/config/test_multimodal_config.py

@@ -66,6 +66,21 @@ def test_use_gpu_video_backend_from_media_io_kwargs(backend_arg: str):
     assert config.use_gpu_video_backend()
 
 
+@pytest.mark.parametrize(
+    ("device", "expected"),
+    [("cuda", True), ("cuda:1", True), ("cpu", False), (None, False)],
+)
+def test_use_gpu_video_backend_for_torchcodec_device(
+    device: str | None, expected: bool
+):
+    video_kwargs: dict[str, object] = {"backend": "torchcodec"}
+    if device is not None:
+        video_kwargs["device"] = device
+    config = MultiModalConfig(media_io_kwargs={"video": video_kwargs})
+
+    assert config.use_gpu_video_backend() is expected
+
+
 def test_mm_encoder_fp8_scale_path_requires_fp8():
     with pytest.raises(ValueError, match="mm_encoder_attn_dtype"):
         MultiModalConfig(mm_encoder_fp8_scale_path="/tmp/scales.json")

tests/multimodal/test_gpu_ipc_memory.py

@@ -209,6 +209,24 @@ def test_reserve_mm_ipc_gpu_memory_includes_pynvvideocodec_decoder_budget(
     )
 
 
+def test_reserve_mm_ipc_gpu_memory_includes_torchcodec_cuda_decoder_budget(
+    monkeypatch: pytest.MonkeyPatch,
+):
+    monkeypatch.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv")
+    available_bytes = 4 * GiB_bytes
+    cpu_config = MultiModalConfig(
+        media_io_kwargs={"video": {"backend": "torchcodec"}},
+    )
+    cuda_config = MultiModalConfig(
+        media_io_kwargs={"video": {"backend": "torchcodec", "device": "cuda"}},
+    )
+
+    assert reserve_mm_ipc_gpu_memory(available_bytes, cpu_config) == available_bytes
+    assert reserve_mm_ipc_gpu_memory(available_bytes, cuda_config) == (
+        available_bytes - _pynvvideocodec_decoder_budget(hw_decoders=1)
+    )
+
+
 def test_reserve_mm_ipc_gpu_memory_uses_env_video_backend(
     monkeypatch: pytest.MonkeyPatch,
 ):

tests/multimodal/test_video.py

@@ -7,6 +7,7 @@
 import threading
 from contextlib import ExitStack, contextmanager
 from pathlib import Path
+from types import SimpleNamespace
 
 import numpy as np
 import numpy.typing as npt
@@ -175,7 +176,7 @@ def fake_import_module(name: str, package: str):
             "torchcodec",
             {"min_frames": 4, "num_ffmpeg_threads": 2, "seek_mode": "approximate"},
             {"min_frames": 4},
-            {"num_ffmpeg_threads": 2, "seek_mode": "approximate"},
+            {"num_ffmpeg_threads": 2, "seek_mode": "approximate", "device": None},
         ),
         (
             "deepstream",
@@ -206,6 +207,16 @@ def test_video_backend_rejects_options_for_another_decoder():
         resolve_video_backend_kwargs("opencv", {"num_ffmpeg_threads": 2})
 
 
+def test_video_backend_device_option_belongs_to_torchcodec():
+    _, backend_kwargs = resolve_video_backend_kwargs("torchcodec", {"device": "cuda"})
+    assert backend_kwargs["device"] == "cuda"
+
+    with pytest.raises(
+        ValueError, match="device is not supported by the 'opencv' backend"
+    ):
+        resolve_video_backend_kwargs("opencv", {"device": "cuda"})
+
+
 @pytest.mark.parametrize(
     ("backend", "error"),
     [
@@ -1205,6 +1216,103 @@ def test_torchcodec_backend_rejects_frame_recovery(dummy_video_path):
         )
 
 
+def test_torchcodec_backend_forwards_decode_device(monkeypatch: pytest.MonkeyPatch):
+    """``device`` reaches ``VideoDecoder`` only when set, and sampled frames
+    are moved to the host before the NumPy conversion."""
+    from vllm.multimodal.video_decoders import torchcodec as torchcodec_module
+
+    constructor_kwargs: list[dict[str, object]] = []
+    requested_indices: list[list[int]] = []
+    cpu_calls: list[None] = []
+    acquired: list[int] = []
+
+    class RecordingPool:
+        @contextmanager
+        def acquire(self, size: int):
+            acquired.append(size)
+            yield
+
+    class FakeTensor:
+        def cpu(self):
+            cpu_calls.append(None)
+            return self
+
+        def numpy(self):
+            return np.zeros((len(requested_indices[-1]), 20, 10, 3), dtype=np.uint8)
+
+    class FakeDecoder:
+        metadata = SimpleNamespace(
+            num_frames=4, average_fps=2.0, duration_seconds=2.0, width=10, height=20
+        )
+
+        def __init__(self, data: bytes, **kwargs):
+            constructor_kwargs.append(kwargs)
+
+        def get_frames_at(self, frame_indices: list[int]):
+            requested_indices.append(list(frame_indices))
+            return SimpleNamespace(data=FakeTensor())
+
+    monkeypatch.setattr(torchcodec_module, "check_torchcodec_available", lambda: None)
+    monkeypatch.setattr(torchcodec_module, "VideoDecoder", FakeDecoder)
+    monkeypatch.setattr(
+        "vllm.multimodal.gpu_ipc_memory.get_mm_gpu_ipc_pool", RecordingPool
+    )
+
+    frames, metadata = VideoBackend.load_bytes(
+        b"fake video", num_frames=2, backend="torchcodec"
+    )
+    assert "device" not in constructor_kwargs[-1]
+    assert constructor_kwargs[-1]["dimension_order"] == "NHWC"
+    assert frames.shape == (2, 20, 10, 3)
+    # CPU decode never touches the frontend GPU budget.
+    assert acquired == []
+
+    frames, metadata = VideoBackend.load_bytes(
+        b"fake video", num_frames=2, backend="torchcodec", device="cuda"
+    )
+    assert constructor_kwargs[-1]["device"] == "cuda"
+    assert frames.shape == (2, 20, 10, 3)
+    assert metadata["frames_indices"] == requested_indices[-1]
+    assert len(cpu_calls) == 2
+    # CUDA decode accounts the sampled frames against the frontend budget.
+    assert acquired == [2 * 20 * 10 * 3]
+
+
+@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
+def test_torchcodec_cuda_backend_matches_cpu_frames():
+    """CUDA decode must select the same frames as CPU decode and return host
+    frames; NVDEC and FFmpeg color conversion may differ by a few levels."""
+    torchcodec = pytest.importorskip("torchcodec")
+    num_frames, height, width = 50, 64, 64
+    video_bytes = create_long_gop_video(
+        num_frames=num_frames, width=width, height=height
+    )
+    try:
+        torchcodec.decoders.VideoDecoder(video_bytes, device="cuda")
+    except Exception as exc:  # CPU-only TorchCodec build or no NVDEC
+        pytest.skip(f"TorchCodec CUDA decode unavailable: {exc}")
+
+    loader = VIDEO_LOADER_REGISTRY.load("opencv")
+    cpu_frames, cpu_meta = loader.load_bytes(
+        video_bytes, num_frames=4, backend="torchcodec"
+    )
+    gpu_frames, gpu_meta = loader.load_bytes(
+        video_bytes, num_frames=4, backend="torchcodec", device="cuda"
+    )
+
+    assert isinstance(gpu_frames, np.ndarray)
+    assert gpu_frames.shape == cpu_frames.shape == (4, height, width, 3)
+    assert gpu_meta["frames_indices"] == cpu_meta["frames_indices"]
+
+    # The long-GOP fixture stamps a per-frame green marker at the center.
+    cpu_markers = [int(f[height // 2, width // 2, 1]) for f in cpu_frames]
+    gpu_markers = [int(f[height // 2, width // 2, 1]) for f in gpu_frames]
+    assert len(set(gpu_markers)) == 4
+    assert all(abs(a - b) <= 8 for a, b in zip(cpu_markers, gpu_markers))
+    diff = np.abs(gpu_frames.astype(np.int16) - cpu_frames.astype(np.int16))
+    assert diff.mean() < 4.0
+
+
 def test_torchcodec_backend_returns_target_frames_not_keyframes():
     """Regression test: torchcodec must return the requested frames, not the
     GOP keyframe they seek back to.

vllm/config/multimodal.py

@@ -526,10 +526,16 @@ def use_gpu_video_backend(self) -> bool:
             video_kwargs.get("video_backend") or envs.VLLM_VIDEO_LOADER_BACKEND
         )
         codec_backend = video_kwargs.get("backend")
-        return VIDEO_LOADER_REGISTRY.backend_requires_gpu(video_loader_backend) or (
+        if VIDEO_LOADER_REGISTRY.backend_requires_gpu(video_loader_backend) or (
             codec_backend is not None
             and VIDEO_LOADER_REGISTRY.backend_requires_gpu(codec_backend)
-        )
+        ):
+            return True
+        # TorchCodec decodes on the CPU by default but runs NVDEC inside the
+        # API server process when ``device`` selects CUDA.
+        return codec_backend == "torchcodec" and str(
+            video_kwargs.get("device") or ""
+        ).startswith("cuda")
 
     def is_multimodal_pruning_enabled(self):
         return self.get_video_pruning_spec() is not None

vllm/multimodal/video.py

@@ -265,6 +265,11 @@ def load_bytes(
                   creation at the cost of relying on the file's metadata. See
                   https://meta-pytorch.org/torchcodec/stable/generated_examples/decoding/approximate_mode.html
                   for details.
+                - ``device`` (TorchCodec): decoder device. ``None`` (default)
+                  or ``"cpu"`` decodes with FFmpeg on the CPU; ``"cuda"``
+                  decodes with NVDEC when the installed TorchCodec build has
+                  CUDA support. Sampled frames are copied to host memory for
+                  multimodal preprocessing either way.
                 - ``hw_decoders`` (PyNvVideoCodec): maximum number of
                   concurrent decoder slots. Defaults to 2 and must be a
                   positive integer.

vllm/multimodal/video_decoders/__init__.py

@@ -19,6 +19,7 @@
     "torchcodec": {
         "num_ffmpeg_threads": 0,
         "seek_mode": "exact",
+        "device": None,
     },
     PYNVVIDEOCODEC_VIDEO_BACKEND: {
         "hw_decoders": PYNVVIDEOCODEC_DEFAULT_HW_DECODERS,

vllm/multimodal/video_decoders/torchcodec.py

@@ -1,7 +1,8 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 
-from typing import Literal
+from contextlib import AbstractContextManager, nullcontext
+from typing import TYPE_CHECKING, Any, Literal
 
 import numpy as np
 import numpy.typing as npt
@@ -14,6 +15,9 @@
     check_frame_pixel_limit,
 )
 
+if TYPE_CHECKING:
+    import torch
+
 try:
     from torchcodec.decoders import VideoDecoder
 except (ImportError, RuntimeError):
@@ -30,12 +34,14 @@ def decode_torchcodec(
     *,
     num_ffmpeg_threads: int = 0,
     seek_mode: Literal["exact", "approximate"] = "exact",
+    device: "str | torch.device | None" = None,
 ) -> tuple[npt.NDArray, VideoSourceMetadata, list[int], list[int]]:
     check_torchcodec_available()
     decoder = TorchCodecVideoBackendMixin.make_torchcodec_decoder(
         data,
         num_ffmpeg_threads=num_ffmpeg_threads,
         seek_mode=seek_mode,
+        device=device,
     )
     check_frame_pixel_limit(
         decoder.metadata.width or 0,
@@ -47,18 +53,43 @@ def decode_torchcodec(
     frame_idx = loader_cls.compute_frames_index_to_sample(
         source=source, target=target, **sampling_kwargs
     )
-    frames, valid = TorchCodecVideoBackendMixin.decode_torchcodec_frames(
-        decoder, frame_idx
-    )
+    budget: AbstractContextManager[Any] = nullcontext()
+    if _is_cuda_device(device):
+        # GPU decode runs in the API server process. Account for the sampled
+        # frames against the frontend budget like the other GPU backends do.
+        from vllm.multimodal.gpu_ipc_memory import get_mm_gpu_ipc_pool
+
+        pool = get_mm_gpu_ipc_pool()
+        raw_frame_bytes = (
+            len(frame_idx)
+            * (decoder.metadata.height or 0)
+            * (decoder.metadata.width or 0)
+            * 3
+        )
+        if pool is not None and raw_frame_bytes > 0:
+            budget = pool.acquire(raw_frame_bytes)
+    with budget:
+        frames, valid = TorchCodecVideoBackendMixin.decode_torchcodec_frames(
+            decoder, frame_idx
+        )
     return frames, source, frame_idx, valid
 
 
+def _is_cuda_device(device: "str | torch.device | None") -> bool:
+    return device is not None and str(device).startswith("cuda")
+
+
 class TorchCodecVideoBackendMixin:
     """TorchCodec (FFmpeg-backed, PyTorch-native) codec utilities.
 
     Builds a :class:`~torchcodec.decoders.VideoDecoder` over the in-memory
     bytes and extracts the sampled indices with a single batched
     ``get_frames_at`` call, while releasing the GIL during decode.
+
+    ``device`` selects TorchCodec's decoder device. ``None`` keeps the CPU
+    (FFmpeg) default; ``"cuda"`` decodes with NVDEC when the installed
+    TorchCodec build has CUDA support. Sampled frames are always returned as
+    host NumPy arrays, which is what the multimodal processors consume.
     """
 
     @staticmethod
@@ -67,15 +98,20 @@ def make_torchcodec_decoder(
         *,
         num_ffmpeg_threads: int = 0,
         seek_mode: Literal["exact", "approximate"] = "exact",
+        device: "str | torch.device | None" = None,
     ) -> "VideoDecoder":
         # NHWC matches the (num_frames, H, W, 3) uint8 RGB layout the rest
         # of the pipeline expects, avoiding a transpose.
-        return VideoDecoder(
-            data,
-            dimension_order="NHWC",
-            num_ffmpeg_threads=num_ffmpeg_threads,
-            seek_mode=seek_mode,
-        )
+        decoder_kwargs: dict[str, Any] = {
+            "dimension_order": "NHWC",
+            "num_ffmpeg_threads": num_ffmpeg_threads,
+            "seek_mode": seek_mode,
+        }
+        # Forward ``device`` only when one was selected so the TorchCodec
+        # default (CPU) and its own validation errors stay untouched.
+        if device is not None:
+            decoder_kwargs["device"] = device
+        return VideoDecoder(data, **decoder_kwargs)
 
     @staticmethod
     def get_torchcodec_metadata(decoder: "VideoDecoder") -> VideoSourceMetadata:
@@ -97,4 +133,8 @@ def decode_torchcodec_frames(
             return np.empty((0,), dtype=np.uint8), []
         # Note: torchcodec releases the GIL for the entire call
         batch = decoder.get_frames_at(frame_indices)
-        return batch.data.numpy(), list(frame_indices)
+        # Multimodal processors consume host NumPy frames. A CUDA decoder
+        # returns device tensors, which cannot be converted directly, so copy
+        # only the sampled frames at this boundary. ``cpu()`` is a no-op for
+        # the default CPU decoder.
+        return batch.data.cpu().numpy(), list(frame_indices)