diff --git a/components/src/dynamo/common/multimodal/media_source.py b/components/src/dynamo/common/multimodal/media_source.py new file mode 100644 index 000000000000..cf3f2fdfaf5d --- /dev/null +++ b/components/src/dynamo/common/multimodal/media_source.py @@ -0,0 +1,83 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Read media bytes from the non-HTTP sources the hardware decoder can use. + +``fetch_bytes`` only speaks HTTP(S), so ``file://`` and ``data:`` media never +produced bytes at the routing layer and could not reach NVDEC -- they fell +through to the software decoder instead. The codec-compliant runtime images +ship no software video decoder, so that fallback no longer resolves and those +schemes lost video support entirely. This module supplies the missing bytes so +H.264/H.265 from a local file or a data URI decodes on the GPU like any other +source. + +``file://`` stays behind ``validate_local_path``: local access is refused +unless ``DYN_MM_LOCAL_PATH`` is set, paths are resolved before the prefix check +(so symlinks cannot escape), and this adds no read surface beyond what the +existing media connectors already allow under the same policy. +""" + +from __future__ import annotations + +import asyncio +import base64 +import binascii +import logging +from urllib.parse import unquote, urlparse +from urllib.request import url2pathname + +from dynamo.common.http.url_validator import ( + UrlValidationError, + UrlValidationPolicy, + validate_local_path, +) + +logger = logging.getLogger(__name__) + +# Schemes this module can turn into bytes. http(s) is deliberately absent: it +# belongs to fetch_bytes, which applies SSRF revalidation on every redirect hop. +LOCAL_MEDIA_SCHEMES = frozenset({"file", "data"}) + + +def is_local_media_url(url: str) -> bool: + """True when :func:`read_local_media_bytes` can produce bytes for ``url``.""" + return urlparse(url).scheme in LOCAL_MEDIA_SCHEMES + + +def _decode_data_uri(url: str) -> bytes: + """Decode a ``data:`` URI body to bytes. + + Only base64 payloads are accepted: a percent-encoded body would have to be + re-encoded to bytes by guessing a charset, and media data URIs are base64 + in practice. + """ + _, _, remainder = url.partition(":") + meta, sep, payload = remainder.partition(",") + if not sep: + raise UrlValidationError("Malformed data URI: missing ',' separator") + if "base64" not in meta.split(";"): + raise UrlValidationError("Unsupported data URI: expected base64 payload") + try: + return base64.b64decode(unquote(payload), validate=True) + except (binascii.Error, ValueError) as exc: + raise UrlValidationError(f"Malformed base64 in data URI: {exc}") from exc + + +async def read_local_media_bytes(url: str, policy: UrlValidationPolicy) -> bytes: + """Return the bytes behind a ``file://`` or ``data:`` media URL. + + Raises ``UrlValidationError`` when the scheme is unsupported, when local + access is disabled or the path escapes ``allowed_local_path``, or when a + data URI is malformed. + """ + scheme = urlparse(url).scheme + if scheme == "data": + return _decode_data_uri(url) + if scheme != "file": + raise UrlValidationError(f"Unsupported local media scheme: {scheme!r}") + + # url2pathname handles the platform's file:// -> path rules; the policy + # check below is what actually authorizes the read. + parsed = urlparse(url) + path = validate_local_path(url2pathname(parsed.path), policy) + return await asyncio.to_thread(path.read_bytes) diff --git a/components/src/dynamo/common/multimodal/nvdec_decoder.py b/components/src/dynamo/common/multimodal/nvdec_decoder.py new file mode 100644 index 000000000000..7e411d6685a0 --- /dev/null +++ b/components/src/dynamo/common/multimodal/nvdec_decoder.py @@ -0,0 +1,282 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""GPU (NVDEC) video decode for H.264/H.265 via PyNvVideoCodec. + +Dual-path companion to the VP8/VP9 in-tree FFmpeg decoders. Royalty-free codecs +(VP8/VP9/AV1) stay on the existing CPU path; H.264/H.265 decode on the GPU +through NVDEC, which links ``libnvcuvid`` at runtime and carries no bundled +software codec. NVDEC does not use SMs beyond a small YUV->RGB conversion, so the +inference-time impact is minimal. + +This module is backend-agnostic. Each backend's decode site (vLLM ``VideoLoader``, +TRT-LLM ``multimodal_processor``, SGLang encode worker) routes on the probed codec: + + codec = probe_video_codec(video_bytes) + if should_use_nvdec(codec): + frames, meta = decode_video_nvdec(video_bytes, num_frames) + else: + frames, meta = + +The returned frames match the existing ``VideoLoader`` contract: a host numpy +array of shape ``(T, H, W, 3)``, dtype ``uint8``, RGB, C-contiguous, plus a +metadata dict ``{"fps", "frames_indices", "total_num_frames"}``. + +Gating: NVDEC is used when PyNvVideoCodec is importable and ``DYN_DISABLE_NVDEC`` +is not set. When it is unavailable (CPU image, unsupported profile) the caller +surfaces the actionable "unsupported codec" error. +""" + +from __future__ import annotations + +import functools +import importlib.util +import logging +import os +import tempfile +import warnings + +import numpy as np + +from dynamo.common.utils.env import env_bool + +logger = logging.getLogger(__name__) + +DISABLE_ENV = "DYN_DISABLE_NVDEC" +GPU_ID_ENV = "DYN_NVDEC_GPU_ID" + +# Used only when the decoder cannot report a source frame rate (see +# _source_fps). Consumers divide by this to build frame timestamps, so it must +# stay non-zero; 30 fps is a common capture rate and keeps timestamps in a +# plausible range rather than failing the request outright. +FALLBACK_FPS = 30.0 + +# Codecs routed to NVDEC. VP8/VP9/AV1 stay on the existing royalty-free path. +HW_ROUTED_CODECS = frozenset({"h264", "hevc"}) + +# Container byte markers -> codec id. A video file carries exactly one video +# codec, so the first match wins. Covers ISO-BMFF (mp4/mov) sample-entry fourccs +# and Matroska/WebM CodecID strings. HEVC/AV1 checked before H.264 so a more +# specific marker is not shadowed. This is a routing hint, not a full demux -- +# PyNvVideoCodec re-parses the stream authoritatively at decode time. +_CODEC_MARKERS: tuple[tuple[bytes, str], ...] = ( + (b"hev1", "hevc"), + (b"hvc1", "hevc"), + (b"V_MPEGH/ISO/HEVC", "hevc"), + (b"av01", "av1"), + (b"V_AV1", "av1"), + (b"avc1", "h264"), + (b"avc3", "h264"), + (b"V_MPEG4/ISO/AVC", "h264"), + (b"vp09", "vp9"), + (b"V_VP9", "vp9"), + (b"vp08", "vp8"), + (b"V_VP8", "vp8"), +) + + +def probe_video_codec(data: bytes) -> str | None: + """Best-effort codec identification from container bytes. + + Returns a codec id (``"h264"``/``"hevc"``/``"vp9"``/``"vp8"``/``"av1"``) or + ``None`` when it can't tell. Scans the whole buffer because an mp4 ``moov`` + (which holds the codec fourcc) may sit at the end of a non-faststart file; + multimodal clips are small and already in memory. + """ + if not data: + return None + for marker, codec in _CODEC_MARKERS: + if marker in data: + return codec + return None + + +def should_use_nvdec(codec: str | None) -> bool: + """True if `codec` is one we route to NVDEC and NVDEC is available.""" + return codec in HW_ROUTED_CODECS and nvdec_available() + + +@functools.lru_cache(maxsize=1) +def nvdec_available() -> bool: + """True if NVDEC decode can run in this process. + + Cached: PyNvVideoCodec is only installed in GPU images, and the import is the + reliable signal. ``DYN_DISABLE_NVDEC`` forces the software path. A GPU that is + absent at decode time still raises and is caught by the caller's fallback. + """ + if env_bool(DISABLE_ENV): + return False + try: + with warnings.catch_warnings(): + # A third-party deprecation must not decide whether the capability + # exists. PyNvVideoCodec 2.2.0's __init__ does `from ast import Str`, + # which warns on Python 3.12; under pytest's filterwarnings=error + # that warning is *raised*, the import "fails", and this lru_cache + # then pins False for the whole process -- silently disabling NVDEC + # for every test in the session (a second import would have + # succeeded, since the warning is only raised once). + warnings.simplefilter("ignore") + import PyNvVideoCodec # noqa: F401 + except Exception as exc: # noqa: BLE001 + # Two very different situations reach here, and they need different + # log levels. The wheel being absent is normal on CPU-only images, so + # it stays at debug. The wheel being *installed* but unimportable is a + # misconfiguration -- almost always a container that did not request + # the "video" driver capability, so libnvcuvid is not mounted -- and it + # silently costs H.264/H.265 decode, which the codec-compliant images + # have no software fallback for. That deserves a warning; keeping it at + # debug is how this class of failure goes unnoticed. + if importlib.util.find_spec("PyNvVideoCodec") is None: + logger.debug( + "PyNvVideoCodec is not installed; NVDEC decode disabled (%s)", exc + ) + else: + logger.warning( + "PyNvVideoCodec is installed but failed to import (%s); NVDEC " + "hardware decode is DISABLED. This usually means the container " + "was not given the 'video' driver capability, so libnvcuvid is " + "unavailable -- set NVIDIA_DRIVER_CAPABILITIES to include " + "'video'. H.264/H.265 video input has no software decoder in " + "the codec-compliant images and will fail until this is fixed.", + exc, + ) + return False + return True + + +def _gpu_id() -> int: + raw = os.environ.get(GPU_ID_ENV, "").strip() + if not raw: + return 0 + try: + return int(raw) + except ValueError: + logger.warning("invalid %s=%r; using GPU 0", GPU_ID_ENV, raw) + return 0 + + +def _frame_to_rgb_hwc(frame) -> np.ndarray: + """Copy a decoded (device) RGB frame to a host ``(H, W, 3)`` uint8 array. + + A 2.x ``DecodedFrame`` (``output_color_type=RGB``) holds a CUDA buffer and + supports the DLPack protocol, so torch wraps it zero-copy on the GPU and + ``.cpu()`` copies to host. Validated on PyNvVideoCodec 2.1.1 for H.264/H.265. + """ + import torch + + try: + tensor = torch.from_dlpack(frame) + except Exception: # noqa: BLE001 - fall back to the CUDA-array-interface path + tensor = torch.as_tensor(frame, device="cuda") + arr = tensor.cpu().numpy() + if arr.dtype != np.uint8: + arr = arr.astype(np.uint8) + return arr + + +def _source_fps(decoder) -> float: + """Source frame rate for the metadata dict. Never returns 0. + + This value is not cosmetic. ``load_video`` hands ``(frames, metadata)`` + straight to vLLM as ``mm_data["video"]``, and Qwen3-VL turns sampled frame + indices into timestamps with ``idx / fps`` -- a zero raises + ZeroDivisionError and the request 500s. + + PyNvVideoCodec 2.2.0 exposes the rate as + ``get_stream_metadata().average_fps``; older probes are kept for other + versions, then ``num_frames / duration``, then a documented fallback so a + future API change degrades timestamp accuracy instead of failing requests. + """ + meta = getattr(decoder, "get_stream_metadata", None) + if callable(meta): + try: + info = meta() + val = getattr(info, "average_fps", None) + if val and float(val) > 0: + return float(val) + # Derive it when the rate is absent but the span is known. + frames = float(getattr(info, "num_frames", 0) or 0) + duration = float(getattr(info, "duration", 0) or 0) + if frames > 0 and duration > 0: + return frames / duration + except Exception: # noqa: BLE001 - metadata only, never fatal + pass + + for attr in ("get_fps", "fps", "GetFPS"): + val = getattr(decoder, attr, None) + try: + val = val() if callable(val) else val + if val and float(val) > 0: + return float(val) + except Exception: # noqa: BLE001 - metadata only, never fatal + continue + + logger.warning( + "NVDEC could not determine source fps; falling back to %.1f. Frame " + "timestamps handed to the model will be approximate.", + FALLBACK_FPS, + ) + return FALLBACK_FPS + + +def decode_video_nvdec( + data: bytes, num_frames: int, gpu_id: int | None = None +) -> tuple[np.ndarray, dict]: + """Decode H.264/H.265 (or any NVDEC-supported codec) bytes to sampled frames. + + Returns ``(frames, metadata)`` where ``frames`` is host numpy + ``(num_frames, H, W, 3)`` uint8 RGB C-contiguous, uniformly sampled across the + clip, and ``metadata`` has ``fps``/``frames_indices``/``total_num_frames`` -- + the same contract as ``VideoLoader.load_video``. Raises on decode failure so + the caller can fall back. + """ + import PyNvVideoCodec as nvc + + if gpu_id is None: + gpu_id = _gpu_id() + if num_frames < 1: + raise ValueError(f"num_frames must be >= 1, got {num_frames}") + + # SimpleDecoder takes a file PATH (not bytes) and reads frames lazily, so the + # temp file must stay alive for the whole decode -- keep it inside the context. + with tempfile.NamedTemporaryFile(suffix=".mp4") as tmp: + tmp.write(data) + tmp.flush() + decoder = nvc.SimpleDecoder( + tmp.name, + gpu_id=gpu_id, + output_color_type=nvc.OutputColorType.RGB, + use_device_memory=False, + ) + total = len(decoder) + if total <= 0: + raise RuntimeError("NVDEC decode produced no frames") + n = min(num_frames, total) + indices = np.unique(np.linspace(0, total - 1, n).astype(int)) + frames = [_frame_to_rgb_hwc(decoder[int(i)]) for i in indices] + fps = _source_fps(decoder) + + stacked = np.ascontiguousarray(np.stack(frames)).astype(np.uint8, copy=False) + if stacked.ndim != 4 or stacked.shape[-1] != 3: + raise RuntimeError( + f"NVDEC frames have unexpected shape {stacked.shape}; expected (T,H,W,3)" + ) + metadata = { + "fps": fps, + "frames_indices": indices.tolist(), + "total_num_frames": int(total), + # Whether the consumer may still sample these frames. vLLM's own video + # loader reports `len(frames_indices) == total_num_frames`: true when the + # loader handed over every source frame, false when it already picked a + # subset. Qwen3-VL treats a MISSING flag as false, so omitting it here + # made short clips -- the case where we return everything -- look + # pre-sampled, and the model's own fps policy never ran. Reported with a + # reproduction against vLLM 0.26.0's Qwen3-VL processor: the 10-frame + # fixture yielded video_grid_thw [[5,16,20]] instead of [[2,16,20]]. + # + # Reported by @Chokoyo on #11836. Note the E2E video tests cannot see + # this: they assert on what the model says about the clip, which does not + # change when the frame count does. The unit test is the check that can. + "do_sample_frames": len(indices) == total, + } + return stacked, metadata diff --git a/components/src/dynamo/common/multimodal/video_loader.py b/components/src/dynamo/common/multimodal/video_loader.py index 8f8600d4c27e..ca81c29fe981 100644 --- a/components/src/dynamo/common/multimodal/video_loader.py +++ b/components/src/dynamo/common/multimodal/video_loader.py @@ -27,6 +27,15 @@ UrlValidationPolicy, validate_media_url, ) +from dynamo.common.multimodal.media_source import ( + is_local_media_url, + read_local_media_bytes, +) +from dynamo.common.multimodal.nvdec_decoder import ( + decode_video_nvdec, + probe_video_codec, + should_use_nvdec, +) from dynamo.common.utils.runtime import run_async logger = logging.getLogger(__name__) @@ -125,6 +134,27 @@ async def _load_video_with_vllm( content = await fetch_bytes( normalized_url, self._http_timeout, policy=self._url_policy ) + # Dual decode path: hardware-decode H.264/H.265 on NVDEC; every other + # codec falls through to vLLM's software decoder below. Note the + # runtime images purge the software decode wheels (opencv/av/decord/ + # torchcodec) for codec compliance, so that fallback only resolves + # where a decoder has been installed separately. + decoded = await self._maybe_decode_with_nvdec(content) + if decoded is not None: + return decoded + return await asyncio.to_thread(media_io.load_bytes, content) + + # file:// and data: never touch the network, but they still deserve + # hardware decode: without this they reach only the software decoder, + # which the codec-compliant images do not ship, so H.264/H.265 from a + # local file or data URI would fail despite NVDEC being available and + # able to decode it. Reading is gated by the same url policy the vLLM + # connector below uses, so this adds no local-read surface. + if is_local_media_url(normalized_url): + content = await read_local_media_bytes(normalized_url, self._url_policy) + decoded = await self._maybe_decode_with_nvdec(content) + if decoded is not None: + return decoded return await asyncio.to_thread(media_io.load_bytes, content) connector = self._get_vllm_media_connector() @@ -132,6 +162,31 @@ async def _load_video_with_vllm( normalized_url, media_io, fetch_timeout=self._http_timeout ) + async def _maybe_decode_with_nvdec( + self, content: bytes + ) -> tuple[np.ndarray, Dict[str, Any]] | None: + """Hardware-decode H.264/H.265 via NVDEC, or None to use the software path. + + Returns None for VP8/VP9/AV1 (handled by the existing decoder), when NVDEC + is unavailable, or when NVDEC fails -- the caller then falls back to the + software decoder, which surfaces the actionable "unsupported codec" error + if it also cannot decode. + """ + codec = probe_video_codec(content) + if not should_use_nvdec(codec): + return None + try: + return await asyncio.to_thread( + decode_video_nvdec, content, self._num_frames + ) + except Exception as exc: # noqa: BLE001 - fall back to software decode + logger.warning( + "NVDEC decode failed for a %s clip (%s); using software decode", + codec, + exc, + ) + return None + async def load_video(self, video_url: str) -> tuple[np.ndarray, Dict[str, Any]]: try: frames, metadata = await self._load_video_with_vllm(video_url) diff --git a/components/src/dynamo/common/tests/multimodal/test_media_source.py b/components/src/dynamo/common/tests/multimodal/test_media_source.py new file mode 100644 index 000000000000..63180e7400f1 --- /dev/null +++ b/components/src/dynamo/common/tests/multimodal/test_media_source.py @@ -0,0 +1,112 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for reading file:// and data: media bytes. + +These schemes exist so H.264/H.265 from a local file or data URI can reach +NVDEC. The codec-compliant images ship no software video decoder, so without +this path those schemes have no decoder at all. The security boundary is +``validate_local_path``: local reads stay refused unless DYN_MM_LOCAL_PATH is +set, and resolved paths must stay inside it. +""" + +from __future__ import annotations + +import base64 + +import pytest + +from dynamo.common.http.url_validator import UrlValidationError, UrlValidationPolicy +from dynamo.common.multimodal.media_source import ( + is_local_media_url, + read_local_media_bytes, +) + +pytestmark = [ + pytest.mark.unit, + pytest.mark.gpu_0, + pytest.mark.pre_merge, +] + +PAYLOAD = b"\x00\x00\x00\x18ftypisom....avc1 fake h264 bytes" + + +@pytest.mark.parametrize( + "url,expected", + [ + ("file:///tmp/clip.mp4", True), + ("data:video/mp4;base64,AAAA", True), + ("http://example.com/clip.mp4", False), + ("https://example.com/clip.mp4", False), + ("s3://bucket/clip.mp4", False), + ], +) +def test_is_local_media_url(url, expected): + assert is_local_media_url(url) is expected + + +async def test_reads_file_url_inside_allowed_prefix(tmp_path): + clip = tmp_path / "clip.mp4" + clip.write_bytes(PAYLOAD) + policy = UrlValidationPolicy(allowed_local_path=str(tmp_path)) + + assert await read_local_media_bytes(clip.as_uri(), policy) == PAYLOAD + + +async def test_file_url_refused_when_local_access_disabled(tmp_path): + """Default policy has allowed_local_path unset -- reads must stay refused.""" + clip = tmp_path / "clip.mp4" + clip.write_bytes(PAYLOAD) + + with pytest.raises(UrlValidationError, match="not permitted"): + await read_local_media_bytes(clip.as_uri(), UrlValidationPolicy()) + + +async def test_file_url_cannot_escape_allowed_prefix(tmp_path): + """Path traversal out of the sandbox must be refused.""" + allowed = tmp_path / "allowed" + allowed.mkdir() + outside = tmp_path / "secret.mp4" + outside.write_bytes(PAYLOAD) + policy = UrlValidationPolicy(allowed_local_path=str(allowed)) + + with pytest.raises(UrlValidationError): + await read_local_media_bytes(outside.as_uri(), policy) + + +async def test_file_url_cannot_escape_via_symlink(tmp_path): + """resolve() happens before the prefix check, so symlinks cannot escape.""" + allowed = tmp_path / "allowed" + allowed.mkdir() + outside = tmp_path / "secret.mp4" + outside.write_bytes(PAYLOAD) + link = allowed / "innocent.mp4" + link.symlink_to(outside) + policy = UrlValidationPolicy(allowed_local_path=str(allowed)) + + with pytest.raises(UrlValidationError): + await read_local_media_bytes(link.as_uri(), policy) + + +async def test_reads_base64_data_uri(): + url = "data:video/mp4;base64," + base64.b64encode(PAYLOAD).decode() + # data: carries its own bytes, so it needs no local-path permission. + assert await read_local_media_bytes(url, UrlValidationPolicy()) == PAYLOAD + + +@pytest.mark.parametrize( + "url,match", + [ + ("data:video/mp4;base64", "missing"), # no comma + ("data:video/mp4,notbase64", "expected base64"), # not base64-tagged + ("data:video/mp4;base64,!!!not-base64!!!", "Malformed base64"), + ], +) +async def test_malformed_data_uri_rejected(url, match): + with pytest.raises(UrlValidationError, match=match): + await read_local_media_bytes(url, UrlValidationPolicy()) + + +async def test_unsupported_scheme_rejected(): + with pytest.raises(UrlValidationError, match="Unsupported local media scheme"): + await read_local_media_bytes("s3://bucket/clip.mp4", UrlValidationPolicy()) diff --git a/components/src/dynamo/common/tests/multimodal/test_nvdec_decoder.py b/components/src/dynamo/common/tests/multimodal/test_nvdec_decoder.py new file mode 100644 index 000000000000..a99577859661 --- /dev/null +++ b/components/src/dynamo/common/tests/multimodal/test_nvdec_decoder.py @@ -0,0 +1,306 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for the NVDEC video decoder (PyNvVideoCodec mocked, no GPU).""" + +from __future__ import annotations + +import logging +import sys +import types + +import numpy as np +import pytest + +from dynamo.common.multimodal import nvdec_decoder as nd + +pytestmark = [pytest.mark.pre_merge, pytest.mark.unit, pytest.mark.gpu_0] + + +@pytest.fixture(autouse=True) +def _clear_cache(): + nd.nvdec_available.cache_clear() + yield + nd.nvdec_available.cache_clear() + + +def _fake_pynv(num_frames: int = 10, h: int = 4, w: int = 6): + """A stand-in PyNvVideoCodec module returning deterministic HWC uint8 frames.""" + mod = types.ModuleType("PyNvVideoCodec") + + class OutputColorType: + RGB = "RGB" + + class SimpleDecoder: + def __init__( + self, src, gpu_id=0, output_color_type=None, use_device_memory=False + ): + self._n = num_frames + + def __len__(self): + return self._n + + def __getitem__(self, i): + return np.full((h, w, 3), i % 256, dtype=np.uint8) + + def get_fps(self): + return 30.0 + + mod.OutputColorType = OutputColorType + mod.SimpleDecoder = SimpleDecoder + return mod + + +@pytest.mark.parametrize( + "data,expected", + [ + (b"\x00\x00\x00\x18ftypisom....avc1", "h264"), + (b"....hev1....", "hevc"), + (b"....hvc1....", "hevc"), + (b"....vp09....", "vp9"), + (b"....av01....", "av1"), + (b"\x1aE\xdf\xa3....V_VP9....", "vp9"), + (b"....V_MPEG4/ISO/AVC....", "h264"), + (b"....V_MPEGH/ISO/HEVC....", "hevc"), + (b"random bytes, no codec marker", None), + (b"", None), + ], +) +def test_probe_video_codec(data, expected): + assert nd.probe_video_codec(data) == expected + + +def test_nvdec_available_false_when_disabled(monkeypatch): + monkeypatch.setenv(nd.DISABLE_ENV, "1") + monkeypatch.setitem(sys.modules, "PyNvVideoCodec", _fake_pynv()) + assert nd.nvdec_available() is False + + +def test_nvdec_available_false_when_not_importable(monkeypatch): + monkeypatch.delenv(nd.DISABLE_ENV, raising=False) + monkeypatch.setitem(sys.modules, "PyNvVideoCodec", None) # -> ImportError + assert nd.nvdec_available() is False + + +def test_nvdec_available_true_when_importable(monkeypatch): + monkeypatch.delenv(nd.DISABLE_ENV, raising=False) + monkeypatch.setitem(sys.modules, "PyNvVideoCodec", _fake_pynv()) + assert nd.nvdec_available() is True + + +def test_nvdec_available_false_when_import_raises_runtime(monkeypatch): + # PyNvVideoCodec raises RuntimeError (not ImportError) at import when the + # NVDEC/NVENC driver libs aren't exposed (no NVIDIA_DRIVER_CAPABILITIES=video). + monkeypatch.delenv(nd.DISABLE_ENV, raising=False) + monkeypatch.delitem(sys.modules, "PyNvVideoCodec", raising=False) + import builtins + + real_import = builtins.__import__ + + def fake_import(name, *args, **kwargs): + if name == "PyNvVideoCodec": + raise RuntimeError("Failed to load NVENC library: libnvidia-encode.so.1") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", fake_import) + assert nd.nvdec_available() is False + + +def _force_import_failure(monkeypatch, exc): + monkeypatch.delenv(nd.DISABLE_ENV, raising=False) + monkeypatch.delitem(sys.modules, "PyNvVideoCodec", raising=False) + import builtins + + real_import = builtins.__import__ + + def fake_import(name, *args, **kwargs): + if name == "PyNvVideoCodec": + raise exc + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", fake_import) + + +def test_installed_but_unimportable_warns_about_video_capability(monkeypatch, caplog): + """A misconfigured container must not fail silently. + + The wheel being present but unimportable means libnvcuvid is missing -- the + container was not granted the 'video' driver capability. That silently costs + H.264/H.265 decode, which the codec-compliant images cannot do in software, + so it has to be visible above debug level. + """ + _force_import_failure(monkeypatch, RuntimeError("libnvcuvid.so.1: not found")) + monkeypatch.setattr( + nd.importlib.util, "find_spec", lambda name: object() + ) # wheel IS installed + with caplog.at_level(logging.WARNING): + assert nd.nvdec_available() is False + assert any(r.levelno >= logging.WARNING for r in caplog.records) + assert "video" in caplog.text and "NVIDIA_DRIVER_CAPABILITIES" in caplog.text + + +def test_absent_wheel_stays_quiet(monkeypatch, caplog): + """CPU-only images legitimately have no wheel; that must not warn.""" + _force_import_failure(monkeypatch, ImportError("No module named PyNvVideoCodec")) + monkeypatch.setattr(nd.importlib.util, "find_spec", lambda name: None) + with caplog.at_level(logging.WARNING): + assert nd.nvdec_available() is False + assert not [r for r in caplog.records if r.levelno >= logging.WARNING] + + +def test_should_use_nvdec_routes_only_h264_hevc(monkeypatch): + monkeypatch.delenv(nd.DISABLE_ENV, raising=False) + monkeypatch.setitem(sys.modules, "PyNvVideoCodec", _fake_pynv()) + assert nd.should_use_nvdec("h264") is True + assert nd.should_use_nvdec("hevc") is True + assert nd.should_use_nvdec("vp9") is False + assert nd.should_use_nvdec("av1") is False + assert nd.should_use_nvdec(None) is False + + +def test_should_use_nvdec_false_when_unavailable(monkeypatch): + monkeypatch.setenv(nd.DISABLE_ENV, "1") + assert nd.should_use_nvdec("h264") is False + + +def test_decode_matches_frame_contract(monkeypatch): + monkeypatch.setitem( + sys.modules, "PyNvVideoCodec", _fake_pynv(num_frames=10, h=4, w=6) + ) + # The real frame->host conversion is torch/DLPack on the GPU (validated on + # hardware); stub it so this test runs on CPU CI. + monkeypatch.setattr( + nd, "_frame_to_rgb_hwc", lambda f: np.asarray(f, dtype=np.uint8) + ) + frames, meta = nd.decode_video_nvdec(b"fakebytes", num_frames=5) + assert frames.shape == (5, 4, 6, 3) # THWC + assert frames.dtype == np.uint8 + assert frames.flags["C_CONTIGUOUS"] + assert meta["total_num_frames"] == 10 + assert len(meta["frames_indices"]) == 5 + assert meta["fps"] == 30.0 + + +def test_decode_samples_uniformly(monkeypatch): + monkeypatch.setitem(sys.modules, "PyNvVideoCodec", _fake_pynv(num_frames=100)) + monkeypatch.setattr( + nd, "_frame_to_rgb_hwc", lambda f: np.asarray(f, dtype=np.uint8) + ) + _, meta = nd.decode_video_nvdec(b"x", num_frames=10) + assert meta["frames_indices"][0] == 0 + assert meta["frames_indices"][-1] == 99 + + +def test_decode_caps_at_total_frames(monkeypatch): + monkeypatch.setitem(sys.modules, "PyNvVideoCodec", _fake_pynv(num_frames=3)) + monkeypatch.setattr( + nd, "_frame_to_rgb_hwc", lambda f: np.asarray(f, dtype=np.uint8) + ) + frames, _ = nd.decode_video_nvdec(b"x", num_frames=32) + assert frames.shape[0] == 3 # cannot sample more frames than exist + + +@pytest.mark.parametrize( + "total,num_frames,expect_flag,expect_returned", + [ + (3, 32, True, 3), # fewer frames than asked for -> every source frame + (32, 32, True, 32), # exactly as many -> still every source frame + (100, 10, False, 10), # more than asked for -> we already sampled + (1, 5, True, 1), # single-frame clip + ], +) +def test_decode_reports_whether_it_already_sampled( + monkeypatch, total, num_frames, expect_flag, expect_returned +): + """`do_sample_frames` must say whether the consumer may still sample. + + vLLM's own loader reports `len(frames_indices) == total_num_frames`, and + Qwen3-VL reads a MISSING flag as False. Omitting it therefore made short + clips -- exactly the case where every source frame is returned -- look + pre-sampled, so the model's fps policy never ran and all frames were consumed + instead of the four it would have chosen. + + Asserted here rather than end to end on purpose: the serve tests check what + the model says about the clip, and that answer does not change when the frame + count does. This is the level that can see it. + """ + monkeypatch.setitem(sys.modules, "PyNvVideoCodec", _fake_pynv(num_frames=total)) + monkeypatch.setattr( + nd, "_frame_to_rgb_hwc", lambda f: np.asarray(f, dtype=np.uint8) + ) + frames, meta = nd.decode_video_nvdec(b"x", num_frames=num_frames) + + assert meta["do_sample_frames"] is expect_flag + assert frames.shape[0] == expect_returned + # The flag is exactly vLLM's rule, restated against what we returned. + assert meta["do_sample_frames"] == ( + len(meta["frames_indices"]) == meta["total_num_frames"] + ) + + +def test_decode_raises_on_empty_stream(monkeypatch): + monkeypatch.setitem(sys.modules, "PyNvVideoCodec", _fake_pynv(num_frames=0)) + with pytest.raises(RuntimeError): + nd.decode_video_nvdec(b"x", num_frames=5) + + +def test_decode_rejects_bad_num_frames(monkeypatch): + monkeypatch.setitem(sys.modules, "PyNvVideoCodec", _fake_pynv()) + with pytest.raises(ValueError): + nd.decode_video_nvdec(b"x", num_frames=0) + + +# --------------------------------------------------------------------------- +# _source_fps -- metadata["fps"] reaches vLLM as mm_data["video"], and Qwen3-VL +# computes timestamps as `idx / fps`, so a 0 here 500s the request. +# --------------------------------------------------------------------------- + + +class _Meta: + def __init__(self, average_fps=None, num_frames=None, duration=None): + if average_fps is not None: + self.average_fps = average_fps + if num_frames is not None: + self.num_frames = num_frames + if duration is not None: + self.duration = duration + + +class _Decoder: + def __init__(self, meta=None, **attrs): + if meta is not None: + self.get_stream_metadata = lambda: meta + for k, v in attrs.items(): + setattr(self, k, v) + + +def test_source_fps_prefers_stream_metadata(): + """PyNvVideoCodec 2.2.0 reports the rate via get_stream_metadata().""" + dec = _Decoder(meta=_Meta(average_fps=10.0), get_fps=lambda: 30.0) + assert nd._source_fps(dec) == 10.0 + + +def test_source_fps_derives_from_frames_and_duration(): + dec = _Decoder(meta=_Meta(num_frames=24, duration=2.0)) + assert nd._source_fps(dec) == 12.0 + + +def test_source_fps_falls_back_to_legacy_attrs(): + assert nd._source_fps(_Decoder(get_fps=lambda: 25.0)) == 25.0 + + +def test_source_fps_never_returns_zero(): + """Regression: a 0 fps crashed the vLLM NVDEC serve path with + ZeroDivisionError in Qwen3-VL's _calculate_timestamps.""" + # Nothing usable: no metadata, no legacy attrs, and a zero-valued one. + for dec in (_Decoder(), _Decoder(get_fps=lambda: 0), _Decoder(meta=_Meta())): + fps = nd._source_fps(dec) + assert fps > 0, f"fps must never be 0 (got {fps})" + assert fps == nd.FALLBACK_FPS + + +def test_decode_metadata_fps_is_nonzero(monkeypatch): + """End of the chain: the dict handed to vLLM must carry a usable fps.""" + monkeypatch.setitem(sys.modules, "PyNvVideoCodec", _fake_pynv(num_frames=10)) + _, metadata = nd.decode_video_nvdec(b"x", num_frames=4) + assert metadata["fps"] > 0 diff --git a/components/src/dynamo/common/tests/multimodal/test_nvdec_decoder_gpu.py b/components/src/dynamo/common/tests/multimodal/test_nvdec_decoder_gpu.py new file mode 100644 index 000000000000..6d43ebc7b14e --- /dev/null +++ b/components/src/dynamo/common/tests/multimodal/test_nvdec_decoder_gpu.py @@ -0,0 +1,67 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""GPU integration test for NVDEC decode (needs a GPU + PyNvVideoCodec). + +Decodes the committed H.264/H.265 fixtures through the real ``nvdec_decoder`` +and asserts the frame contract, mirroring the hardware validation done on +gpu-ts. Skips cleanly where NVDEC is unavailable, so it is a no-op on CPU lanes +and images without PyNvVideoCodec. + +The clips are committed fixtures rather than synthesized at run time on purpose: +generating them needs an H.264/H.265 *encoder*, which the codec-compliant image +deliberately does not ship (the in-tree ffmpeg is VP9-only, and imageio-ffmpeg +is installed with --no-binary). Synthesizing here made the test unable to run in +the very image it is meant to validate. +""" + +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import pytest + +from dynamo.common.multimodal import nvdec_decoder as nd + +pytestmark = [ + pytest.mark.integration, + pytest.mark.post_merge, + pytest.mark.gpu_1, + pytest.mark.vllm, +] + + +_MEDIA_DIR = ( + Path(__file__).resolve().parents[6] / "lib" / "llm" / "tests" / "data" / "media" +) + +# Committed fixtures, re-encoded from the VP9 240p_10.mp4 clip. +_FIXTURES = {"h264": "240p_10_h264.mp4", "hevc": "240p_10_h265.mp4"} + + +@pytest.mark.parametrize("codec", sorted(_FIXTURES)) +def test_nvdec_decodes_real_clip(codec): + if not nd.nvdec_available(): + pytest.skip("PyNvVideoCodec/NVDEC not available (needs the video capability)") + + path = _MEDIA_DIR / _FIXTURES[codec] + if not path.is_file(): + pytest.skip(f"fixture not available: {path}") + data = path.read_bytes() + # Guard against an unresolved Git LFS pointer masquerading as the clip. + if data[:7] == b"version": + pytest.skip(f"fixture is an unresolved LFS pointer: {path}") + + # Sanity: the probe classifies the clip as an NVDEC-routed codec. + assert nd.probe_video_codec(data) in nd.HW_ROUTED_CODECS + + frames, metadata = nd.decode_video_nvdec(data, num_frames=8) + + assert frames.ndim == 4 and frames.shape[-1] == 3 # (T, H, W, 3) + assert frames.dtype == np.uint8 + assert frames.flags["C_CONTIGUOUS"] + assert frames.shape[0] == 8 + assert frames[:, :, :, :].max() > 0 # real pixels, not a black clip + assert metadata["total_num_frames"] >= 8 + assert len(metadata["frames_indices"]) == 8 diff --git a/components/src/dynamo/common/tests/multimodal/test_video_loader.py b/components/src/dynamo/common/tests/multimodal/test_video_loader.py index 966742115f73..db26ac272d38 100644 --- a/components/src/dynamo/common/tests/multimodal/test_video_loader.py +++ b/components/src/dynamo/common/tests/multimodal/test_video_loader.py @@ -151,3 +151,56 @@ async def test_load_video_batch_reads_decoded_variant_with_metadata(monkeypatch) decoded_item, return_metadata=True, ) + + +@pytest.mark.asyncio +async def test_maybe_decode_with_nvdec_routes_h264(monkeypatch): + loader = VideoLoader() + frames = np.zeros((2, 4, 6, 3), dtype=np.uint8) + meta = {"fps": 30.0, "frames_indices": [0, 1], "total_num_frames": 2} + monkeypatch.setattr(video_loader_module, "probe_video_codec", lambda b: "h264") + monkeypatch.setattr(video_loader_module, "should_use_nvdec", lambda c: True) + monkeypatch.setattr( + video_loader_module, "decode_video_nvdec", lambda b, n: (frames, meta) + ) + + result = await loader._maybe_decode_with_nvdec(b"h264-bytes") + + assert result is not None + got_frames, got_meta = result + np.testing.assert_array_equal(got_frames, frames) + assert got_meta == meta + + +@pytest.mark.asyncio +async def test_maybe_decode_with_nvdec_skips_royalty_free(monkeypatch): + loader = VideoLoader() + monkeypatch.setattr(video_loader_module, "probe_video_codec", lambda b: "vp9") + monkeypatch.setattr(video_loader_module, "should_use_nvdec", lambda c: False) + called = {"decode": False} + + def _decode(*a, **k): + called["decode"] = True + + monkeypatch.setattr(video_loader_module, "decode_video_nvdec", _decode) + + result = await loader._maybe_decode_with_nvdec(b"vp9-bytes") + + assert result is None # VP9 stays on the software path + assert called["decode"] is False # NVDEC not invoked + + +@pytest.mark.asyncio +async def test_maybe_decode_with_nvdec_falls_back_on_failure(monkeypatch): + loader = VideoLoader() + monkeypatch.setattr(video_loader_module, "probe_video_codec", lambda b: "hevc") + monkeypatch.setattr(video_loader_module, "should_use_nvdec", lambda c: True) + + def _boom(*a, **k): + raise RuntimeError("nvdec session limit") + + monkeypatch.setattr(video_loader_module, "decode_video_nvdec", _boom) + + result = await loader._maybe_decode_with_nvdec(b"hevc-bytes") + + assert result is None # a NVDEC failure falls back, never raises diff --git a/components/src/dynamo/common/tests/test_video_utils.py b/components/src/dynamo/common/tests/test_video_utils.py index fddb373412af..7371965c6f8c 100644 --- a/components/src/dynamo/common/tests/test_video_utils.py +++ b/components/src/dynamo/common/tests/test_video_utils.py @@ -41,7 +41,7 @@ def _mock_iio_v2(self): iio.get_writer = MagicMock(return_value=writer) return iio, writer - def test_mp4_selects_h264_nvenc_codec(self): + def test_mp4_selects_vp9_codec(self): from dynamo.common.utils.video_utils import encode_to_video_bytes iio = self._mock_iio_v3() @@ -56,7 +56,8 @@ def test_mp4_selects_h264_nvenc_codec(self): iio.imwrite.assert_called_once() _, kwargs = iio.imwrite.call_args - assert kwargs.get("codec") == "h264_nvenc" + # Royalty-free: mp4 output uses VP9, not h264_nvenc. + assert kwargs.get("codec") == "libvpx-vp9" assert kwargs.get("fps") == 8 def test_webm_selects_libvpx_vp9_codec(self): @@ -154,3 +155,58 @@ def test_v2_api_fallback_writes_all_frames(self): assert writer.append_data.call_count == 4 writer.close.assert_called_once() + + +# --------------------------------------------------------------------------- +# frames_to_numpy +# --------------------------------------------------------------------------- + + +class TestFramesToNumpy: + """Tests for frames_to_numpy() — must accept both PIL and numpy frames.""" + + def test_numpy_float_frames_scale_to_uint8(self): + """Regression: diffusion pipelines emit float [0, 1] numpy frames. + + The PIL-only implementation crashed with "'numpy.ndarray' object has no + attribute 'convert'" on the Wan2.1 T2V serve path. + """ + from dynamo.common.utils.video_utils import frames_to_numpy + + frames = [np.full((4, 4, 3), 0.5, dtype=np.float32) for _ in range(3)] + out = frames_to_numpy(frames) + + assert out.shape == (3, 4, 4, 3) + assert out.dtype == np.uint8 + # 0.5 * 255 -> 128 (rounded) + assert np.all(out == 128) + + def test_numpy_uint8_frames_pass_through(self): + from dynamo.common.utils.video_utils import frames_to_numpy + + frames = [np.full((4, 4, 3), 200, dtype=np.uint8) for _ in range(2)] + out = frames_to_numpy(frames) + + assert out.shape == (2, 4, 4, 3) + assert out.dtype == np.uint8 + assert np.all(out == 200) + + def test_pil_frames_still_supported(self): + from PIL import Image + + from dynamo.common.utils.video_utils import frames_to_numpy + + frames = [ + Image.fromarray(np.full((4, 4, 3), 77, dtype=np.uint8)) for _ in range(2) + ] + out = frames_to_numpy(frames) + + assert out.shape == (2, 4, 4, 3) + assert out.dtype == np.uint8 + assert np.all(out == 77) + + def test_empty_raises(self): + from dynamo.common.utils.video_utils import frames_to_numpy + + with pytest.raises(ValueError, match="No images provided"): + frames_to_numpy([]) diff --git a/components/src/dynamo/common/utils/video_utils.py b/components/src/dynamo/common/utils/video_utils.py index 347df37e4bcd..c7cc59f1be0c 100644 --- a/components/src/dynamo/common/utils/video_utils.py +++ b/components/src/dynamo/common/utils/video_utils.py @@ -83,10 +83,15 @@ def normalize_video_frames(images: list) -> list: def frames_to_numpy(images: list) -> np.ndarray: - """Convert a list of PIL Images to a numpy array suitable for video encoding. + """Convert a list of video frames to a numpy array suitable for encoding. + + Accepts either PIL Images or numpy arrays. Diffusion video pipelines (e.g. + Wan2.1 T2V) emit numpy frames — float in ``[0, 1]`` by default — while other + stages hand back PIL Images; this normalizes both to ``uint8`` RGB, matching + the conversion ``diffusers.export_to_video`` performs internally. Args: - images: List of PIL Image objects (video frames). + images: List of PIL Image objects or ``np.ndarray`` frames (H, W, 3). Returns: Numpy array of shape ``(num_frames, height, width, 3)`` with dtype @@ -100,7 +105,14 @@ def frames_to_numpy(images: list) -> np.ndarray: frames = [] for img in images: - arr = np.array(img.convert("RGB")) + if isinstance(img, np.ndarray): + arr = img + if arr.dtype != np.uint8: + # Diffusers convention: numpy frames are float in [0, 1]. + arr = (arr * 255.0).round().clip(0, 255).astype(np.uint8) + else: + # PIL Image. + arr = np.array(img.convert("RGB")) frames.append(arr) # Validate consistent sizes @@ -154,15 +166,16 @@ def encode_to_mp4( logger.info(f"Encoding {len(frames)} frames to {output_path} at {fps} fps") try: - # Use imageio to write MP4. We use h264_nvenc (NVIDIA HW encoder) instead - # of libx264 because the in-tree ffmpeg build is LGPL-only and libx264 - # is GPL-licensed; see container/templates/wheel_builder.Dockerfile. - # Requires a CUDA-capable GPU at runtime. + # Encode with VP9 (libvpx-vp9). The in-tree ffmpeg build is LGPL-only and + # royalty-free: it carries no H.264 codec (not even the h264_nvenc HW + # encoder), so VP9 is the video encoder we ship. VP9-in-mp4 is valid and + # decodes with our VP8/VP9 decoder allowlist; see + # container/templates/wheel_builder.Dockerfile. if hasattr(iio, "imwrite"): - iio.imwrite(output_path, frames, fps=fps, codec="h264_nvenc") + iio.imwrite(output_path, frames, fps=fps, codec="libvpx-vp9") else: # Fall back to v2 API - writer = iio.get_writer(output_path, fps=fps, codec="h264_nvenc") # type: ignore[attr-defined] + writer = iio.get_writer(output_path, fps=fps, codec="libvpx-vp9") # type: ignore[attr-defined] try: for frame in frames: writer.append_data(frame) @@ -213,11 +226,11 @@ def encode_to_video_bytes( try: buffer = io.BytesIO() + # VP9 (libvpx-vp9) for both containers: the in-tree ffmpeg is royalty-free + # and carries no H.264 encoder. VP9-in-mp4 and VP9-in-webm are both valid. kwargs: dict = {"fps": fps} - if output_format == "webm": + if output_format in ("webm", "mp4"): kwargs["codec"] = "libvpx-vp9" - elif output_format == "mp4": - kwargs["codec"] = "h264_nvenc" else: raise ValueError(f"No codec specified for response format: {output_format}") diff --git a/components/src/dynamo/sglang/request_handlers/multimodal/encode_worker_handler.py b/components/src/dynamo/sglang/request_handlers/multimodal/encode_worker_handler.py index 6d56271dd5d9..b4ada2d99d81 100644 --- a/components/src/dynamo/sglang/request_handlers/multimodal/encode_worker_handler.py +++ b/components/src/dynamo/sglang/request_handlers/multimodal/encode_worker_handler.py @@ -2,10 +2,13 @@ # SPDX-License-Identifier: Apache-2.0 import asyncio +import importlib import json import logging from typing import Any, AsyncIterator, Dict, Optional +from urllib.parse import urlparse +import numpy as np import torch from blake3 import blake3 @@ -20,13 +23,31 @@ from transformers import AutoTokenizer from dynamo._core import Client, Context +from dynamo.common.http import fetch_bytes +from dynamo.common.http.url_validator import ( + UrlValidationError, + UrlValidationPolicy, + validate_media_url, +) from dynamo.common.memory.multimodal_embedding_cache_manager import ( CachedEmbedding, MultimodalEmbeddingCacheManager, ) from dynamo.common.multimodal import EMBEDDING_SENDER_FACTORIES from dynamo.common.multimodal.cache_uuid import reject_unsupported_multimodal_uuids +from dynamo.common.multimodal.media_source import ( + is_local_media_url, + read_local_media_bytes, +) +from dynamo.common.multimodal.nvdec_decoder import ( + DISABLE_ENV, + nvdec_available, + probe_video_codec, + should_use_nvdec, +) +from dynamo.common.multimodal.video_loader import VideoLoader from dynamo.common.utils import nvtx_utils as _nvtx +from dynamo.common.utils.env import env_bool from dynamo.llm import MultimodalEmbeddingCachePublisher from dynamo.sglang.args import Config from dynamo.sglang.protocol import ( @@ -36,6 +57,10 @@ SglangMultimodalRequest, ) from dynamo.sglang.request_handlers.handler_base import BaseWorkerHandler +from dynamo.sglang.request_handlers.multimodal.nvdec_video_decoder import ( + SGLANG_VIDEO_DECODER_AVAILABLE, + NvdecVideoDecoder, +) logger = logging.getLogger(__name__) @@ -55,6 +80,149 @@ IMAGE_URL_KEY = "image_url" VIDEO_URL_KEY = "video_url" +# SGLang model types whose video preprocessing needs per-frame timestamps from +# ``video_metadata``. For these, ``_process_video_items`` runs +# ``for m in video_metadata: m.get("fps")`` (sglang.srt.disaggregation.encode_server, +# v0.5.15). With the metadata shim below every pre-decoded frame carries a valid +# metadata dict (so that branch no longer crashes on ``None``), but routing these +# models through NVDEC is still gated pending end-to-end timestamp validation. The +# qwen2.5 family takes the ``video_grid_thw`` branch and does not read the metadata, +# so it is unaffected either way. SGLang has no in-image software decoder for the +# gated models either, so keeping them on the URL path is no worse than status quo. +_NVDEC_UNSAFE_MODEL_TYPES = frozenset( + {"qwen3_vl", "qwen3_vl_moe", "qwen3_5", "qwen3_5_moe", "intern_s2_preview"} +) + +# Fallback source fps stamped into the synthesized ``video_metadata`` for +# pre-decoded NVDEC frames (see ``_install_nvdec_video_metadata_shim``). The +# supported qwen2_5_vl model_type ignores ``video_metadata`` entirely; the value +# only feeds the qwen3_vl timestamp branch, which is gated off today. +_NVDEC_SHIM_FPS = 24.0 + + +def _install_load_video_passthrough() -> None: + """Let a pre-constructed decoder survive SGLang's ``load_video``. + + ``load_video`` (sglang.srt.utils.common, v0.5.16) returns + ``list``/``tuple``/``Tensor``/``ndarray`` untouched, but has no case for an + object that is already a ``VideoDecoderWrapper``: it falls through to + ``_normalize_video_input``, which returns ``None`` for that type, and raises + ``ValueError: Unsupported video input type``. Dynamo hands the encode path an + ``NvdecVideoDecoder`` so SGLang can drive frame selection itself, so teach + ``load_video`` to pass such objects through. + + Patch the name **as bound in each importing module**: they do + ``from sglang.srt.utils import load_video``, so rebinding + ``sglang.srt.utils.load_video`` alone leaves those call sites untouched. + ``encode_server`` is the one this handler actually goes through -- its + ``_flatten_and_load_videos`` calls its own binding -- and omitting it is why + an earlier revision still raised ``ValueError: Unsupported video input type`` + end to end while every unit test passed. + + Verified against ``v0.5.16``: ``encode_server``, ``base_processor`` and + ``utils`` are the ``srt`` modules that bind the name. Patch every one that + imports successfully and log which, so a future SGLang bump that moves the + call site shows up as a missing module rather than silently reverting to the + URL path. Idempotent; a no-op when SGLang is unavailable. + """ + if not SGLANG_VIDEO_DECODER_AVAILABLE: + return + try: + from sglang.srt.utils.video_decoder import VideoDecoderWrapper + except (ImportError, OSError): + return + + def _wrap(module, orig): + def _load_video_passthrough(video_file, *args, **kwargs): + if isinstance(video_file, VideoDecoderWrapper): + return video_file + return orig(video_file, *args, **kwargs) + + _load_video_passthrough._dynamo_nvdec_passthrough = True # type: ignore[attr-defined] + module.load_video = _load_video_passthrough + + patched: list[str] = [] + for module_path in ( + # The encode worker's own call site -- the one that matters here. + "sglang.srt.disaggregation.encode_server", + "sglang.srt.multimodal.processors.base_processor", + "sglang.srt.utils", + ): + try: + module = importlib.import_module(module_path) + except (ImportError, OSError): + continue + orig = getattr(module, "load_video", None) + if orig is None: + continue + if getattr(orig, "_dynamo_nvdec_passthrough", False): + patched.append(module_path) + continue + _wrap(module, orig) + patched.append(module_path) + + if "sglang.srt.disaggregation.encode_server" not in patched: + # Without this one the decoder reaches load_video unpatched and the + # request fails with "Unsupported video input type" rather than falling + # back, so make the gap visible instead of waiting for a 400. + logger.warning( + "load_video passthrough not installed on encode_server (patched: %s); " + "NVDEC video decoding will not work on this SGLang version", + patched or "none", + ) + else: + logger.debug("load_video passthrough installed on: %s", patched) + + +def _install_nvdec_video_metadata_shim() -> None: + """Let SGLang accept pre-decoded frames under transformers >= 5.12. + + SGLang's ``preprocess_video`` returns ``(frames, None)`` for a pre-decoded + ndarray, and ``_flatten_and_load_videos`` then sets + ``video_processor_kwargs["video_metadata"] = [None]`` -- its ``if + video_metadata:`` treats the ``[None]`` list as truthy. transformers >= 5.12 + strict-validates that kwarg and rejects ``[None]`` (it accepts + ``VideoMetadata`` / ``dict`` / ``list[dict]`` / ``None``), so + ``MMEncoder._encode`` raises ``BadRequestError``. Wrap ``preprocess_video`` so + a pre-decoded ndarray instead carries a valid metadata dict -- the same shape + SGLang builds on its own torchvision path -- which ``list[dict]`` validation + accepts. This unblocks any pre-decoded-pixel path -- frontend decoding + (``--frontend-decoding``) being the one that reaches it -- and supplies the + ``fps`` / ``frames_indices`` the qwen3_vl branch reads. Idempotent; a no-op + when SGLang is unavailable or already patched. Validated on GPU hardware + against the real ``MMEncoder._encode`` (before FAIL -> after PASS). + + This is **not** the NVDEC path. NVDEC hands SGLang an ``NvdecVideoDecoder``, + which takes ``preprocess_video``'s decoder branch and returns real metadata, + so the synthesized values below never apply to it. The shim remains only for + genuine pre-decoded arrays. + """ + try: + from sglang.srt.disaggregation import encode_server as es + except (ImportError, OSError): + return + + orig = getattr(es, "preprocess_video", None) + if orig is None or getattr(orig, "_dynamo_nvdec_shim", False): + return + + async def _preprocess_video_with_metadata(vr, *args, **kwargs): + video, metadata = await orig(vr, *args, **kwargs) + if metadata is None and isinstance(video, np.ndarray) and video.ndim >= 1: + num_frames = int(video.shape[0]) + fps = _NVDEC_SHIM_FPS + metadata = { + "fps": fps, + "duration": (num_frames / fps) if fps else 0.0, + "total_num_frames": num_frames, + "frames_indices": list(range(num_frames)), + "video_backend": "nvdec", + } + return video, metadata + + _preprocess_video_with_metadata._dynamo_nvdec_shim = True # type: ignore[attr-defined] + es.preprocess_video = _preprocess_video_with_metadata + class MultimodalEncodeWorkerHandler(BaseWorkerHandler[SglangMultimodalRequest, str]): """ @@ -80,6 +248,14 @@ def __init__( self.model = config.server_args.model_path self._missing_video_cache_key_config_warned = False + # NVDEC hardware decode for H.264/H.265 video input. #11836 strips the + # software video decoders (av/decord/torchcodec) from the SGLang image, + # so these codecs otherwise have no decoder. Reuse the shared default so + # this worker and the vLLM/TRT-LLM backends agree on + # DYN_MM_VIDEO_NUM_FRAMES; VP8/VP9/AV1 stay on the frontend path. + self.num_video_frames = max(1, VideoLoader.NUM_FRAMES_DEFAULT) + self._url_policy = UrlValidationPolicy.from_env() + if MMEncoder is None: raise RuntimeError( "MMEncoder is not available. " @@ -94,6 +270,16 @@ def __init__( rank=0, ) + # Let SGLang accept the NVDEC-backed decoder this handler builds, so it + # keeps ownership of frame selection (see _install_load_video_passthrough). + _install_load_video_passthrough() + + # Make SGLang's video processor accept pre-decoded frame arrays on + # transformers >= 5.12 (see _install_nvdec_video_metadata_shim). Safe to + # install unconditionally: it only rewrites the ``None`` metadata SGLang + # emits for pre-decoded pixels, leaving the software-decoder path intact. + _install_nvdec_video_metadata_shim() + # Load tokenizer to convert image token string to integer ID self.tokenizer = AutoTokenizer.from_pretrained( self.model, trust_remote_code=config.server_args.trust_remote_code @@ -319,6 +505,137 @@ def _aux_value_for_item( ) return cls._jsonable_media_value(value) + def _nvdec_video_enabled(self) -> bool: + """Whether to route video URLs through NVDEC for this encoder. + + Off when explicitly disabled, when PyNvVideoCodec/CUDA is unavailable, + or when the model's SGLang video preprocessing cannot accept + pre-decoded frames (see ``_NVDEC_UNSAFE_MODEL_TYPES``). + """ + # env_bool, not a truthiness test on the raw value: everywhere else this + # switch is read that way, and a bare os.environ.get makes + # DYN_DISABLE_NVDEC=0 *disable* NVDEC here while leaving it enabled in + # vLLM and TensorRT-LLM -- the same setting meaning opposite things + # depending on the backend. + if env_bool(DISABLE_ENV): + return False + if not nvdec_available(): + return False + model_type = getattr(self.encoder, "model_type", "") or "" + return model_type not in _NVDEC_UNSAFE_MODEL_TYPES + + async def _maybe_nvdec_decoder(self, url: str) -> Optional[Any]: + """Resolve a video URL to what SGLang should decode. + + Reads the URL (SSRF-validated for http(s); policy-gated for ``file://`` + and ``data:``), probes the container codec, and returns: + + * an ``NvdecVideoDecoder`` for H.264/H.265 -- hardware decode; + * the **fetched bytes** for any other codec, or if building the decoder + fails -- SGLang decodes what we already have; + * ``None`` only when nothing was fetched (unsupported scheme, or the + fetch itself failed), leaving the caller to pass the URL through. + + Returning the bytes rather than the URL matters for three reasons, all + reported by Codex on #11836. SGLang would otherwise download the same + payload a second time, doubling ingress and latency for every non-NVDEC + video. A one-use signed URL would fail on that second fetch. And SGLang + applies no URL policy of its own -- ``_normalize_video_input`` fetches + http(s) straight through ``get_mm_http_session`` -- so the bytes it + decoded were never the validated ones, and a redirect could resolve + differently between the two fetches. Handing over the bytes we validated + and already hold closes all three. + + ``load_video`` takes ``Union[str, bytes, VideoData]`` and + ``_normalize_video_input`` returns ``bytes`` untouched (sglang v0.5.16), + so this needs no shim on the SGLang side. + + A decoder rather than decoded frames: SGLang's ``preprocess_video`` reads + the source frame count and fps off the decoder and applies the model's + own ``vision_config.video`` policy to choose which frames to pull. Handing + it pre-sampled pixels takes the ``return vr, None`` branch instead, which + silently replaces that policy with a backend-global frame count and + fabricated metadata. Passing the decoder keeps frame selection in SGLang + and reports true source values. + + The fetch stays here rather than in SGLang because SGLang's + ``_normalize_video_input`` downloads http(s) with no policy check, which + would drop our SSRF validation. + + ``file://`` and ``data:`` are included because SGLang's own decoders are + stripped from the codec-compliant image: without hardware decode those + schemes have no decoder at all, so excluding them here would drop local + and inline video entirely rather than merely skipping acceleration. + """ + content: bytes | None = None + try: + normalized = await validate_media_url(url, self._url_policy) + scheme = urlparse(normalized).scheme + if scheme in ("http", "https"): + content = await fetch_bytes(normalized, 30.0, policy=self._url_policy) + elif is_local_media_url(normalized): + content = await read_local_media_bytes(normalized, self._url_policy) + else: + return None + if not should_use_nvdec(probe_video_codec(content)): + # Not a hardware codec, but the bytes are already here and were + # fetched under policy. Hand them over instead of the URL. + return content + # Constructing the decoder opens the container and reads its frame + # index, so keep it off the event loop. + return await asyncio.to_thread(NvdecVideoDecoder, content) + except UrlValidationError: + # A policy refusal is not a decode failure and must not degrade to + # the URL fallback. SGLang applies no policy of its own: it fetches + # http(s) through get_mm_http_session and resolves file:// to a bare + # path, so passing a rejected URL on turns "denied" into an + # unvalidated fetch or local read. + # + # Confirmed on GPU hardware before this guard existed: a loopback + # URL the policy refused was served to SGLang (38128 bytes fetched + # from a blocked address), and a refused file:// path resolved to a + # readable local file. + # + # UrlValidationError subclasses ValueError, which is how this + # handler already reports a bad request, so the caller surfaces it + # as one instead of silently widening what the deployment accepts. + raise + except Exception as exc: # noqa: BLE001 - additive; never blocks the path + # If the fetch itself failed there are no bytes and the URL is all + # the caller has. If it succeeded and only the decoder construction + # failed, pass the validated bytes on rather than making SGLang + # fetch them again. + logger.warning( + "NVDEC decode failed for video URL (%s); falling back to %s", + exc, + "the fetched bytes" if content is not None else "URL passthrough", + ) + return content + + async def _build_encode_inputs( + self, urls: list[str], modality_name: str + ) -> list[Any]: + """Map video URLs to NVDEC-backed decoders where applicable. + + Returns a list positionally aligned with ``urls``: each entry is an + ``NvdecVideoDecoder`` (H.264/H.265), the fetched bytes (any other codec, + so SGLang does not re-download what we already validated and hold), or + the original URL string when nothing was fetched. + Non-video modalities and disabled/ineligible cases return the URLs + unchanged. + + Called from both the cached and uncached encode paths. The embedding + cache is disabled by default, so routing this only through the cached + path would leave hardware decode unreachable in a stock deployment. + """ + if modality_name != "VIDEO" or not self._nvdec_video_enabled(): + return list(urls) + encode_inputs: list[Any] = [] + for url in urls: + decoder = await self._maybe_nvdec_decoder(url) + encode_inputs.append(decoder if decoder is not None else url) + return encode_inputs + async def _encode_with_cache( self, media_urls: list[str], modality: Any ) -> tuple[Any, torch.Tensor, list[CachedEmbedding]]: @@ -352,8 +669,14 @@ async def _encode_with_cache( # SGLang's _encode outputs are already on CPU; use CPU as target for consistency target_device = torch.device("cpu") if uncached_urls: + # H.264/H.265 URLs are hardware-decoded to frames here; SGLang's + # _encode accepts pre-decoded ndarrays in place of URLs. Cache keys + # above remain keyed on the original URL, so caching is unaffected. + encode_inputs = await self._build_encode_inputs( + uncached_urls, modality_name + ) grid_dim, new_embeddings, aux_data = await self.encoder._encode( - uncached_urls, modality + encode_inputs, modality ) # Verify SGLang output is on CPU as expected if new_embeddings.device != target_device: @@ -582,8 +905,18 @@ async def generate( cached_entries, ) = await self._encode_with_cache(urls, modality_enum) else: + # The embedding cache is off by default, so this is the + # path a stock deployment takes. It must apply the same + # NVDEC conversion as the cached branch -- otherwise raw + # URLs reach SGLang, which has no video decoder in the + # codec-compliant image, and every H.264/H.265 request + # fails. Not hoisted above the branch because + # _encode_with_cache converts only its uncached subset. + encode_inputs = await self._build_encode_inputs( + urls, modality_name + ) grid_dim, embeddings, aux_data = await self.encoder._encode( - urls, modality_enum + encode_inputs, modality_enum ) grid_list = self._ensure_batched_grid(grid_dim, len(urls)) diff --git a/components/src/dynamo/sglang/request_handlers/multimodal/nvdec_video_decoder.py b/components/src/dynamo/sglang/request_handlers/multimodal/nvdec_video_decoder.py new file mode 100644 index 000000000000..018892604894 --- /dev/null +++ b/components/src/dynamo/sglang/request_handlers/multimodal/nvdec_video_decoder.py @@ -0,0 +1,198 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""An NVDEC-backed decoder that satisfies SGLang's ``VideoDecoderWrapper`` contract. + +SGLang chooses which frames a video model sees. ``preprocess_video`` reads the +*source* frame count and frame rate, applies the model's ``vision_config.video`` +policy (``fps`` / ``nframes`` / ``min_frames`` / ``max_frames``) via +``smart_nframes``, and only then asks the decoder for those specific frames:: + + is_video_obj = isinstance(vr, VideoDecoderWrapper) + if not is_video_obj: + return vr, None + total_frames, video_fps = len(vr), vr.avg_fps + nframes = smart_nframes(video_config, total_frames=total_frames, video_fps=video_fps) + idx = np.unique(np.linspace(0, total_frames - 1, num=nframes, dtype=np.int64)) + video = vr.get_frames_as_tensor(idx.tolist()) + +Handing SGLang a pre-sampled ndarray takes the ``return vr, None`` branch, which +discards that policy: the model gets a backend-global frame count instead of the +count its own config asks for, with no true source metadata. Handing it this +class instead keeps frame selection where it belongs and reports real values. + +The gate is ``isinstance``, so subclassing is required -- duck typing does not +pass. Three properties of the upstream class make that safe (verified against +``v0.5.16:python/sglang/srt/utils/video_decoder.py``): + +* The module imports cleanly with neither torchcodec nor decord installed -- + ``_BACKEND`` falls back to ``"decord"`` and the import of decord is deferred to + construction. So the base class is importable in the codec-compliant image even + though both software decoders are stripped from it. +* ``close()`` touches only ``self._tmp_path`` and ``__del__`` just calls + ``close()``. ``SimpleDecoder`` needs a file path and reads lazily, so the temp + file has to outlive construction; recording it in ``_tmp_path`` makes the + inherited cleanup exactly right. +* Only ``__len__``, ``avg_fps`` and ``get_frames_as_tensor`` are on the + ``preprocess_video`` path. +""" + +from __future__ import annotations + +import logging +import os +import tempfile +from typing import Any + +import numpy as np + +from dynamo.common.multimodal.nvdec_decoder import ( + _frame_to_rgb_hwc, + _gpu_id, + _source_fps, +) + +logger = logging.getLogger(__name__) + +try: + from sglang.srt.utils.video_decoder import VideoDecoderWrapper + + SGLANG_VIDEO_DECODER_AVAILABLE = True +except Exception: # noqa: BLE001 - SGLang absent (unit tests) or interface moved + VideoDecoderWrapper = object # type: ignore[assignment,misc] + SGLANG_VIDEO_DECODER_AVAILABLE = False + +# Latched so the pinned-memory regression warning is emitted once, not per request. +_PIN_MEMORY_WARNED = False + + +class NvdecVideoDecoder(VideoDecoderWrapper): # type: ignore[misc,valid-type] + """Random-access NVDEC decoder over in-memory video bytes. + + Construct from the bytes Dynamo has already fetched and policy-validated, so + the SSRF checks in ``validate_media_url`` / ``fetch_bytes`` still apply. + SGLang's own ``_normalize_video_input`` would fetch http(s) with no policy + check at all, which is why the bytes are passed in rather than the URL. + """ + + def __init__(self, data: bytes, gpu_id: int | None = None) -> None: + import PyNvVideoCodec as nvc + + if gpu_id is None: + gpu_id = _gpu_id() + + # SimpleDecoder takes a PATH and reads frames lazily, so the file must + # outlive __init__. Mirrors what the parent's decord-bytes branch does. + fd, tmp_path = tempfile.mkstemp(suffix=".mp4") + try: + os.write(fd, data) + finally: + os.close(fd) + + # Deliberately not calling super().__init__(): it would construct a + # decord/torchcodec decoder, which this image does not ship. Set the + # attributes it would have set so inherited members still work -- + # close()/__del__ need _tmp_path, and source_bytes reads the rest. + self._source = tmp_path + self._source_bytes = data + self._source_path: str | None = None + # Annotated because close() clears it; without this mypy infers `str` + # from the assignment and rejects the reset. + self._tmp_path: str | None = tmp_path + self._num_decode_threads = 1 + self._tc_kwargs: dict[str, Any] = {} + + try: + self._decoder = nvc.SimpleDecoder( + tmp_path, + gpu_id=gpu_id, + output_color_type=nvc.OutputColorType.RGB, + use_device_memory=False, + ) + if len(self._decoder) <= 0: + raise RuntimeError("NVDEC opened the video but reports no frames") + except Exception: + # Do not leak the temp file when construction fails -- __del__ never + # runs for an object whose __init__ raised. + self.close() + raise + + # __len__, close and __del__ are defined here rather than inherited. The + # parent's versions are equivalent, but when SGLang is unavailable the base + # collapses to `object` (see the import guard above) and inheriting nothing + # would silently break len() and leak temp files under unit test. + + def __len__(self) -> int: + return len(self._decoder) + + def close(self) -> None: + """Remove the backing temp file. Idempotent; safe before _decoder exists.""" + tmp_path = getattr(self, "_tmp_path", None) + if tmp_path is not None: + if os.path.exists(tmp_path): + os.unlink(tmp_path) + self._tmp_path = None + + def __del__(self): + try: + self.close() + except Exception: # noqa: BLE001 - never raise from a finalizer + pass + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + @property + def avg_fps(self) -> float: + """True source frame rate. Never 0 -- see ``_source_fps``.""" + return _source_fps(self._decoder) + + def get_frames_as_tensor(self, indices: list): + """Frames at ``indices`` as an NHWC uint8 CPU tensor in pinned memory. + + NHWC because ``preprocess_video`` immediately does + ``video.permute(0, 3, 1, 2)``. Pinned host memory matches the parent's + decord branch and keeps the tensor off the GPU, which also avoids the + ``pin_memory()`` failure that a CUDA-resident tensor would raise. + """ + import torch + + if not indices: + raise ValueError("get_frames_as_tensor requires at least one index") + frames = [_frame_to_rgb_hwc(self._decoder[int(i)]) for i in indices] + arr = np.ascontiguousarray(np.stack(frames)).astype(np.uint8, copy=False) + if arr.ndim != 4 or arr.shape[-1] != 3: + raise RuntimeError( + f"NVDEC frames have unexpected shape {arr.shape}; expected (N,H,W,3)" + ) + tensor = torch.from_numpy(arr) + try: + return tensor.pin_memory() + except RuntimeError as exc: + # Pinned host memory is the point of this return type: it lets the + # downstream host-to-device copy run async instead of staging through + # a bounce buffer, and it is what the upstream decord branch does. + # Losing it is a real throughput regression, so say so loudly -- once, + # since this sits on the per-request path. The only expected cause is + # a host whose default accelerator is not CUDA (a non-CUDA dev box); + # on a deployed encode worker, which requires CUDA for NVDEC anyway, + # this should never fire. Serving unpinned still beats failing the + # request outright. + global _PIN_MEMORY_WARNED + if not _PIN_MEMORY_WARNED: + _PIN_MEMORY_WARNED = True + logger.warning( + "pin_memory() failed (%s); serving video frames from unpinned " + "host memory. Host-to-device transfer is slower than designed " + "-- expect reduced encode throughput. This is not expected on " + "a CUDA host.", + exc, + ) + return tensor + + def __getitem__(self, idx): + """Single frame as numpy NHWC uint8, matching the parent's contract.""" + return _frame_to_rgb_hwc(self._decoder[int(idx)]) diff --git a/components/src/dynamo/sglang/request_handlers/video_generation/video_generation_handler.py b/components/src/dynamo/sglang/request_handlers/video_generation/video_generation_handler.py index 091d1c5efb22..f94c83dd8335 100644 --- a/components/src/dynamo/sglang/request_handlers/video_generation/video_generation_handler.py +++ b/components/src/dynamo/sglang/request_handlers/video_generation/video_generation_handler.py @@ -259,7 +259,7 @@ async def _generate_video( return video_bytes async def _frames_to_video( - self, frames: list, fps: int, codec: str = "h264_nvenc" + self, frames: list, fps: int, codec: str = "libvpx-vp9" ) -> bytes: """Convert list of frames to video bytes. @@ -302,15 +302,9 @@ def encode_with_codec(codec_name: str) -> bytes: output_buffer.seek(0) return output_buffer.read() - try: - return encode_with_codec(codec) - except OSError: - if codec != "h264_nvenc": - raise - logger.warning( - "h264_nvenc failed; retrying video encoding with libx264" - ) - return encode_with_codec("libx264") + # VP9 (libvpx-vp9) is a royalty-free CPU encoder in the in-tree LGPL + # ffmpeg; no HW/GPU fallback and no libx264 (GPL, H.264) fallback. + return encode_with_codec(codec) except ImportError as e: raise RuntimeError( diff --git a/components/src/dynamo/sglang/tests/test_sglang_multimodal_embedding_cache.py b/components/src/dynamo/sglang/tests/test_sglang_multimodal_embedding_cache.py index f0b218348334..ca7a2757e4ea 100644 --- a/components/src/dynamo/sglang/tests/test_sglang_multimodal_embedding_cache.py +++ b/components/src/dynamo/sglang/tests/test_sglang_multimodal_embedding_cache.py @@ -308,3 +308,368 @@ async def test_video_cache_key_includes_sampling_config( assert cache_handler.encoder.encode_mock.await_count == 2 assert cache_handler._embedding_cache.get(first_key) is not None assert cache_handler._embedding_cache.get(second_key) is not None + + +# --- NVDEC video routing ------------------------------------------------------- + +_HANDLER_MOD = "dynamo.sglang.request_handlers.multimodal.encode_worker_handler" + + +@pytest.fixture +def nvdec_handler(cache_handler) -> MultimodalEncodeWorkerHandler: + """cache_handler wired with the attributes the NVDEC path reads.""" + cache_handler.num_video_frames = 32 + cache_handler._url_policy = SimpleNamespace() + return cache_handler + + +def test_nvdec_video_enabled_gating(nvdec_handler, monkeypatch) -> None: + monkeypatch.setattr(f"{_HANDLER_MOD}.nvdec_available", lambda: True) + + # Disabled explicitly. + monkeypatch.setenv("DYN_DISABLE_NVDEC", "1") + assert nvdec_handler._nvdec_video_enabled() is False + monkeypatch.delenv("DYN_DISABLE_NVDEC", raising=False) + + # No PyNvVideoCodec / CUDA. + monkeypatch.setattr(f"{_HANDLER_MOD}.nvdec_available", lambda: False) + assert nvdec_handler._nvdec_video_enabled() is False + + # Available + a model type whose pre-decoded video path crashes upstream. + monkeypatch.setattr(f"{_HANDLER_MOD}.nvdec_available", lambda: True) + nvdec_handler.encoder.model_type = "qwen3_vl" + assert nvdec_handler._nvdec_video_enabled() is False + + # Available + a safe model type. + nvdec_handler.encoder.model_type = "qwen2_5_vl" + assert nvdec_handler._nvdec_video_enabled() is True + + +@pytest.mark.asyncio +async def test_build_encode_inputs_non_video_passthrough( + nvdec_handler, monkeypatch +) -> None: + """Image modality never touches NVDEC; URLs pass through unchanged.""" + called = False + + async def _fail(_url): + nonlocal called + called = True + return object() + + monkeypatch.setattr(nvdec_handler, "_maybe_nvdec_decoder", _fail) + urls = ["https://x/a.jpg", "https://x/b.jpg"] + out = await nvdec_handler._build_encode_inputs(urls, "IMAGE") + assert out == urls + assert called is False + + +@pytest.mark.asyncio +async def test_build_encode_inputs_swaps_only_nvdec_hits( + nvdec_handler, monkeypatch +) -> None: + """Each URL maps positionally to a decoder, bytes, or the URL itself. + + Three outcomes, not two: a decoder for H.264/H.265, the already-fetched + bytes when the codec is not hardware-routed, and the URL only when nothing + was fetched at all. + """ + monkeypatch.setattr(f"{_HANDLER_MOD}.nvdec_available", lambda: True) + nvdec_handler.encoder.model_type = "qwen2_5_vl" + decoder = object() # stand-in for the NvdecVideoDecoder + + async def _decode(url): + if url.endswith("h264.mp4"): + return decoder + if url.endswith("vp9.webm"): + return b"vp9-bytes" # fetched under policy, handed straight over + return None # nothing fetched (unsupported scheme) + + monkeypatch.setattr(nvdec_handler, "_maybe_nvdec_decoder", _decode) + urls = ["https://x/h264.mp4", "https://x/vp9.webm", "ftp://x/other.mkv"] + out = await nvdec_handler._build_encode_inputs(urls, "VIDEO") + assert out[0] is decoder + assert out[1] == b"vp9-bytes" + assert out[2] == "ftp://x/other.mkv" + + +@pytest.mark.asyncio +async def test_maybe_nvdec_decoder_wraps_h264_only(nvdec_handler, monkeypatch) -> None: + """H.264 yields a decoder built from the fetched bytes, not decoded frames.""" + decoder = object() + monkeypatch.setattr( + f"{_HANDLER_MOD}.validate_media_url", + AsyncMock(return_value="https://x/clip.mp4"), + ) + monkeypatch.setattr(f"{_HANDLER_MOD}.fetch_bytes", AsyncMock(return_value=b"bytes")) + monkeypatch.setattr(f"{_HANDLER_MOD}.probe_video_codec", lambda _b: "h264") + monkeypatch.setattr(f"{_HANDLER_MOD}.should_use_nvdec", lambda c: c == "h264") + seen: dict = {} + + def _make(data): + seen["data"] = data + return decoder + + monkeypatch.setattr(f"{_HANDLER_MOD}.NvdecVideoDecoder", _make) + assert await nvdec_handler._maybe_nvdec_decoder("https://x/clip.mp4") is decoder + # The decoder is handed the validated bytes, so SGLang never refetches the URL. + assert seen["data"] == b"bytes" + + +@pytest.mark.asyncio +async def test_maybe_nvdec_decoder_returns_bytes_for_non_hw_codec( + nvdec_handler, monkeypatch +) -> None: + """VP9 (not HW-routed) yields the fetched bytes, never the URL. + + Returning the URL made SGLang download the same payload a second time: it + doubles ingress and latency, fails outright on a one-use signed URL, and -- + because SGLang applies no URL policy of its own, fetching http(s) straight + through ``get_mm_http_session`` -- means the bytes it decoded were never the + ones this handler validated, with a redirect free to resolve differently + between the two fetches. + + Reported by Codex on #11836. Measured on GPU against a counting HTTP server: + two origin fetches for one request before this change, one after. + """ + monkeypatch.setattr( + f"{_HANDLER_MOD}.validate_media_url", + AsyncMock(return_value="https://x/clip.webm"), + ) + fetch = AsyncMock(return_value=b"vp9-bytes") + monkeypatch.setattr(f"{_HANDLER_MOD}.fetch_bytes", fetch) + monkeypatch.setattr(f"{_HANDLER_MOD}.probe_video_codec", lambda _b: "vp9") + monkeypatch.setattr(f"{_HANDLER_MOD}.should_use_nvdec", lambda c: c == "h264") + + out = await nvdec_handler._maybe_nvdec_decoder("https://x/clip.webm") + + assert out == b"vp9-bytes" + assert fetch.await_count == 1 # fetched once, and that copy is what travels on + + +@pytest.mark.asyncio +async def test_maybe_nvdec_decoder_rejects_non_http_scheme( + nvdec_handler, monkeypatch +) -> None: + """A scheme that is neither http(s) nor a permitted local source is skipped. + + Note file:// and data: are *not* in that group -- they are read through + read_local_media_bytes when the policy allows -- so this uses a scheme with + no reader at all. Validation is stubbed as succeeding so the assertion is + about scheme dispatch rather than about the policy refusing it. + """ + monkeypatch.setattr( + f"{_HANDLER_MOD}.validate_media_url", + AsyncMock(return_value="ftp://example.invalid/clip.mp4"), + ) + fetch = AsyncMock() + monkeypatch.setattr(f"{_HANDLER_MOD}.fetch_bytes", fetch) + assert ( + await nvdec_handler._maybe_nvdec_decoder("ftp://example.invalid/clip.mp4") + is None + ) + fetch.assert_not_called() + fetch.assert_not_called() + + +@pytest.mark.asyncio +async def test_maybe_nvdec_decoder_falls_back_on_error( + nvdec_handler, monkeypatch +) -> None: + """A failed FETCH degrades to URL passthrough, because there are no bytes. + + Distinct from a failed decoder build, which returns the bytes it already has + (see test_decode_failure_falls_back_to_the_fetched_bytes). The URL is the + fallback only when nothing was retrieved. + """ + monkeypatch.setattr( + f"{_HANDLER_MOD}.validate_media_url", + AsyncMock(return_value="https://x/clip.mp4"), + ) + monkeypatch.setattr( + f"{_HANDLER_MOD}.fetch_bytes", AsyncMock(side_effect=RuntimeError("boom")) + ) + assert await nvdec_handler._maybe_nvdec_decoder("https://x/clip.mp4") is None + + +@pytest.mark.asyncio +async def test_policy_rejection_is_not_swallowed_into_url_passthrough( + nvdec_handler, monkeypatch +) -> None: + """A refused URL must fail the request, not fall through to SGLang. + + SGLang applies no URL policy: it fetches http(s) directly and resolves + file:// to a bare path. Returning None here restores the original URL, so a + refusal became an unvalidated fetch. Verified on GPU before the guard + existed -- a loopback URL the policy rejected was fetched anyway, and a + refused file:// path resolved to a readable local file. + """ + from dynamo.common.http.url_validator import UrlValidationError + + monkeypatch.setattr(f"{_HANDLER_MOD}.nvdec_available", lambda: True) + nvdec_handler.encoder.model_type = "qwen2_5_vl" + monkeypatch.setattr( + f"{_HANDLER_MOD}.validate_media_url", + AsyncMock(side_effect=UrlValidationError("blocked IP")), + ) + fetch = AsyncMock() + monkeypatch.setattr(f"{_HANDLER_MOD}.fetch_bytes", fetch) + + with pytest.raises(UrlValidationError): + await nvdec_handler._maybe_nvdec_decoder( + "http://169.254.169.254/latest/meta-data" + ) + fetch.assert_not_called() + + # ...and it propagates through the batch builder rather than being mapped + # back to the URL there. + with pytest.raises(UrlValidationError): + await nvdec_handler._build_encode_inputs( + ["http://169.254.169.254/latest/meta-data"], "VIDEO" + ) + + +@pytest.mark.asyncio +async def test_decode_failure_falls_back_to_the_fetched_bytes( + nvdec_handler, monkeypatch +) -> None: + """A genuine decode failure degrades to the bytes, not to an error or the URL. + + Two things at once: the refusal guard must not turn a decode failure into a + raised error, and the fallback must not hand back the URL -- the bytes are + already here and were fetched under policy, so making SGLang refetch them + would reintroduce the second request for the one case NVDEC was meant to + help. + """ + monkeypatch.setattr(f"{_HANDLER_MOD}.nvdec_available", lambda: True) + nvdec_handler.encoder.model_type = "qwen2_5_vl" + monkeypatch.setattr( + f"{_HANDLER_MOD}.validate_media_url", + AsyncMock(return_value="https://x/clip.mp4"), + ) + monkeypatch.setattr(f"{_HANDLER_MOD}.fetch_bytes", AsyncMock(return_value=b"bytes")) + monkeypatch.setattr(f"{_HANDLER_MOD}.probe_video_codec", lambda _b: "h264") + monkeypatch.setattr(f"{_HANDLER_MOD}.should_use_nvdec", lambda _c: True) + + def _boom(_data): + raise RuntimeError("NVDEC init failed") + + monkeypatch.setattr(f"{_HANDLER_MOD}.NvdecVideoDecoder", _boom) + assert await nvdec_handler._maybe_nvdec_decoder("https://x/clip.mp4") == b"bytes" + out = await nvdec_handler._build_encode_inputs(["https://x/clip.mp4"], "VIDEO") + assert out == [b"bytes"] + + +@pytest.mark.asyncio +async def test_video_routes_through_nvdec_with_cache_disabled( + nvdec_handler, monkeypatch +) -> None: + """NVDEC must run when the embedding cache is off -- the default config. + + Regression for the case where the NVDEC conversion lived only inside + ``_encode_with_cache``. ``--multimodal-embedding-cache-capacity-gb`` defaults + to 0, so a stock deployment took the no-cache branch, handed SGLang the raw + URL, and every H.264/H.265 request failed for want of a decoder in the + codec-compliant image. The earlier serve tests all set a non-zero capacity, + which hid it. + """ + nvdec_handler._embedding_cache = None # the default + monkeypatch.setattr(f"{_HANDLER_MOD}.nvdec_available", lambda: True) + nvdec_handler.encoder.model_type = "qwen2_5_vl" + + decoder = object() # stand-in for the NvdecVideoDecoder + + async def _decode(_url): + return decoder + + monkeypatch.setattr(nvdec_handler, "_maybe_nvdec_decoder", _decode) + + video_url = "https://example.com/clip.mp4" + nvdec_handler.encoder.encode_mock.return_value = ( + torch.tensor([2, 3, 4]), + torch.arange(24, dtype=torch.float32).reshape(6, 4), + {"second_per_grid_ts": [0.5], "video_timestamps": [[0.25, 0.75]]}, + ) + + transfer_future = asyncio.get_running_loop().create_future() + transfer_future.set_result(None) + + class _DummyEmbeddingSender: + async def send_embeddings(self, embeddings): + return ( + TransferRequest( + embeddings_shape=list(embeddings.shape), + embedding_dtype_str=str(embeddings.dtype), + serialized_request={"kind": "mock-transfer"}, + ), + transfer_future, + ) + + class _DummyPdWorkerClient: + async def round_robin(self, request_json, context=None): + async def _responses(): + yield json.dumps({"token_ids": [7], "finished": True, "text": ""}) + + return _responses() + + nvdec_handler.embedding_sender = _DummyEmbeddingSender() + nvdec_handler.pd_worker_client = _DummyPdWorkerClient() + + raw_request = { + "token_ids": [101, nvdec_handler.video_token_id, 102], + "stop_conditions": {"max_tokens": 8}, + "sampling_options": {"temperature": 0.0}, + "multi_modal_data": {"video_url": [{"Url": video_url}]}, + } + + async for _ in nvdec_handler.generate(raw_request, context=None): + pass + + # The decoder reaches SGLang, not the URL. Asserting on the URL would pass + # against the buggy version. + nvdec_handler.encoder.encode_mock.assert_awaited_once_with( + [decoder], Modality.VIDEO + ) + + +def test_load_video_passthrough_patches_the_encode_server_binding() -> None: + """The passthrough must reach the binding encode_server actually calls. + + Regression: an earlier revision patched base_processor and sglang.srt.utils + but not encode_server, which does its own ``from sglang.srt.utils import + load_video``. Every unit test passed -- they stubbed the decoder factory, so + load_video was never reached -- while the deployed encode worker rejected + each video request with "Unsupported video input type" and returned 400. + """ + encode_server = pytest.importorskip( + "sglang.srt.disaggregation.encode_server", + reason="SGLang required to verify the patch target", + ) + from sglang.srt.utils.video_decoder import VideoDecoderWrapper + + from dynamo.sglang.request_handlers.multimodal.encode_worker_handler import ( + _install_load_video_passthrough, + ) + + _install_load_video_passthrough() + patched = encode_server.load_video + assert getattr(patched, "_dynamo_nvdec_passthrough", False), ( + "encode_server.load_video is unpatched; NVDEC decoders will be rejected " + "by SGLang at request time" + ) + + # A pre-built decoder passes straight through instead of raising. + # __new__ skips __init__, so set _tmp_path explicitly: the finaliser reads it + # and would otherwise raise AttributeError during GC, which pytest surfaces + # as PytestUnraisableExceptionWarning -- an error under this repo's + # filterwarnings=error. + sentinel = VideoDecoderWrapper.__new__(VideoDecoderWrapper) + sentinel._tmp_path = None + assert patched(sentinel) is sentinel + + # Anything else still goes to the original implementation. + with pytest.raises(Exception): + patched(object()) + + _install_load_video_passthrough() # idempotent + assert encode_server.load_video is patched diff --git a/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py b/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py index a50485c031d9..09f26891a4ea 100644 --- a/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py +++ b/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import numpy as np import pytest import torch @@ -142,3 +143,46 @@ def create_multimodal_video_item( mm_item["second_per_grid_ts"], torch.tensor([0.5], dtype=torch.float32) ) assert mm_item["video_timestamps"] == [[0.25, 0.75]] + + +async def test_nvdec_video_metadata_shim_stamps_valid_metadata(): + """The metadata shim gives pre-decoded ndarray frames a valid metadata dict. + + SGLang's ``preprocess_video`` returns ``(frames, None)`` for a pre-decoded + ndarray, and transformers >= 5.12 strict-rejects the resulting + ``video_metadata=[None]``. ``_install_nvdec_video_metadata_shim`` wraps it so + the ndarray carries a valid dict instead (validated end-to-end on gpu-ts + against the real ``MMEncoder._encode``: before FAIL -> after PASS). + """ + es = pytest.importorskip("sglang.srt.disaggregation.encode_server") + from dynamo.sglang.request_handlers.multimodal.encode_worker_handler import ( + _NVDEC_SHIM_FPS, + _install_nvdec_video_metadata_shim, + ) + + saved = es.preprocess_video + try: + _install_nvdec_video_metadata_shim() + assert getattr(es.preprocess_video, "_dynamo_nvdec_shim", False) + + # Idempotent: a second install must not double-wrap. + wrapped = es.preprocess_video + _install_nvdec_video_metadata_shim() + assert es.preprocess_video is wrapped + + frames = np.zeros((5, 8, 8, 3), dtype=np.uint8) + video, meta = await es.preprocess_video(frames) + assert video is frames + assert meta == { + "fps": _NVDEC_SHIM_FPS, + "duration": 5 / _NVDEC_SHIM_FPS, + "total_num_frames": 5, + "frames_indices": [0, 1, 2, 3, 4], + "video_backend": "nvdec", + } + + # Non-ndarray inputs keep None metadata (the shim only touches pixels). + _, meta_none = await es.preprocess_video("not-an-array") + assert meta_none is None + finally: + es.preprocess_video = saved diff --git a/components/src/dynamo/sglang/tests/test_sglang_nvdec_video_decoder.py b/components/src/dynamo/sglang/tests/test_sglang_nvdec_video_decoder.py new file mode 100644 index 000000000000..41e1ce7a028d --- /dev/null +++ b/components/src/dynamo/sglang/tests/test_sglang_nvdec_video_decoder.py @@ -0,0 +1,169 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for the NVDEC-backed SGLang video decoder. + +PyNvVideoCodec is mocked throughout -- these never touch a GPU. The point is the +*contract* SGLang's ``preprocess_video`` relies on: an ``isinstance`` check +against ``VideoDecoderWrapper``, then ``len()``, ``avg_fps`` and +``get_frames_as_tensor(indices)``. +""" + +from __future__ import annotations + +import os +import sys +from types import SimpleNamespace +from unittest.mock import MagicMock + +import numpy as np +import pytest +import torch + +from dynamo.sglang.request_handlers.multimodal import nvdec_video_decoder as mod +from dynamo.sglang.request_handlers.multimodal.nvdec_video_decoder import ( + NvdecVideoDecoder, +) + +pytestmark = [ + pytest.mark.unit, + pytest.mark.sglang, + pytest.mark.gpu_1, # sglang tests run on GPU-enabled workers + pytest.mark.profiled_vram_gib(0), + pytest.mark.pre_merge, +] + +_WIDTH, _HEIGHT, _TOTAL, _FPS = 8, 6, 50, 10.0 + + +class _FakeSimpleDecoder: + """Stands in for nvc.SimpleDecoder: len() plus random access by index.""" + + def __init__(self, path, **kwargs): + self.path = path + self.kwargs = kwargs + self.requested: list[int] = [] + + def __len__(self) -> int: + return _TOTAL + + def __getitem__(self, idx): + self.requested.append(idx) + # Encode the index into the pixels so tests can prove which frames came back. + return np.full((_HEIGHT, _WIDTH, 3), idx % 256, dtype=np.uint8) + + def get_stream_metadata(self): + return SimpleNamespace(average_fps=_FPS, num_frames=_TOTAL, duration=5.0) + + +@pytest.fixture +def fake_nvc(monkeypatch): + """Install a fake PyNvVideoCodec module and a passthrough frame converter.""" + nvc = MagicMock() + nvc.SimpleDecoder = _FakeSimpleDecoder + nvc.OutputColorType.RGB = "RGB" + monkeypatch.setitem(sys.modules, "PyNvVideoCodec", nvc) + # The real converter expects a CUDA DecodedFrame; our fake yields numpy already. + monkeypatch.setattr(mod, "_frame_to_rgb_hwc", lambda frame: frame) + return nvc + + +@pytest.fixture +def decoder(fake_nvc): + d = NvdecVideoDecoder(b"fake-mp4-bytes") + yield d + d.close() + + +def test_reports_true_source_length_and_fps(decoder) -> None: + """SGLang derives its frame policy from these, so they must be the source values.""" + assert len(decoder) == _TOTAL + assert decoder.avg_fps == _FPS + + +def test_get_frames_as_tensor_returns_nhwc_uint8_cpu(decoder) -> None: + """preprocess_video does video.permute(0, 3, 1, 2), so NHWC is required.""" + out = decoder.get_frames_as_tensor([0, 10, 20]) + assert isinstance(out, torch.Tensor) + assert out.shape == (3, _HEIGHT, _WIDTH, 3) # NHWC + assert out.dtype == torch.uint8 + assert out.device.type == "cpu" + + +@pytest.mark.skipif( + not torch.cuda.is_available(), reason="pinned host memory requires CUDA" +) +def test_frames_are_pinned_on_cuda(decoder) -> None: + """Pinned memory is the performance contract of this return type. + + It lets the downstream host-to-device copy overlap instead of staging through + a bounce buffer. The implementation degrades to unpinned rather than failing + a request, so without this assertion a regression would show up only as lost + throughput -- silently, and never in CI. + """ + out = decoder.get_frames_as_tensor([0, 1, 2]) + assert out.is_pinned(), "video frames must be returned in pinned host memory" + + +def test_get_frames_as_tensor_returns_exactly_the_requested_frames(decoder) -> None: + """Frame selection belongs to SGLang: return its indices, not a resample.""" + indices = [1, 17, 33, 49] + out = decoder.get_frames_as_tensor(indices) + assert out.shape[0] == len(indices) + assert decoder._decoder.requested == indices + # Pixel value encodes the source index, proving identity rather than count. + assert [int(out[i, 0, 0, 0]) for i in range(len(indices))] == indices + + +def test_get_frames_as_tensor_rejects_empty_indices(decoder) -> None: + with pytest.raises(ValueError): + decoder.get_frames_as_tensor([]) + + +def test_close_removes_the_backing_temp_file(fake_nvc) -> None: + """SimpleDecoder needs a real path, so the temp file must be cleaned up.""" + d = NvdecVideoDecoder(b"fake-mp4-bytes") + tmp_path = d._tmp_path + assert tmp_path is not None and os.path.exists(tmp_path) + d.close() + assert not os.path.exists(tmp_path) + d.close() # idempotent + + +def test_temp_file_not_leaked_when_construction_fails(fake_nvc, monkeypatch) -> None: + """__del__ never runs for an object whose __init__ raised.""" + seen: dict = {} + + class _Boom: + def __init__(self, path, **kwargs): + seen["path"] = path + raise RuntimeError("decoder init failed") + + fake_nvc.SimpleDecoder = _Boom + with pytest.raises(RuntimeError): + NvdecVideoDecoder(b"fake-mp4-bytes") + assert not os.path.exists(seen["path"]) + + +def test_rejects_a_video_with_no_frames(fake_nvc) -> None: + class _Empty(_FakeSimpleDecoder): + def __len__(self) -> int: + return 0 + + fake_nvc.SimpleDecoder = _Empty + with pytest.raises(RuntimeError, match="no frames"): + NvdecVideoDecoder(b"fake-mp4-bytes") + + +@pytest.mark.skipif( + not mod.SGLANG_VIDEO_DECODER_AVAILABLE, reason="SGLang not installed" +) +def test_satisfies_sglangs_isinstance_gate(decoder) -> None: + """preprocess_video gates on isinstance -- duck typing would silently no-op. + + Without this, preprocess_video takes its ``return vr, None`` branch and + SGLang's frame-selection policy is skipped entirely. + """ + from sglang.srt.utils.video_decoder import VideoDecoderWrapper + + assert isinstance(decoder, VideoDecoderWrapper) diff --git a/components/src/dynamo/sglang/tests/test_video_encode_real.py b/components/src/dynamo/sglang/tests/test_video_encode_real.py new file mode 100644 index 000000000000..754a64db7e74 --- /dev/null +++ b/components/src/dynamo/sglang/tests/test_video_encode_real.py @@ -0,0 +1,80 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Real (un-mocked) video-encode regression test for the SGLang output path. + +Drives VideoGenerationWorkerHandler._frames_to_video through the ACTUAL ffmpeg +baked into the shipped runtime image. This is the pre_merge guard for the gap +that let SGLang ship with no ffmpeg at all (video generation would fail at the +encode step) go green through the PR pipeline. Runs on the CUDA image only +(gpu_0, no xpu) because the in-tree VP9 ffmpeg is copied into the CUDA runtime; +encoding VP9 (libvpx-vp9) is CPU-only, so no GPU is used. +""" + +import os +import subprocess +import tempfile + +import numpy as np +import pytest + +try: + from PIL import Image + + from dynamo.sglang.request_handlers.video_generation.video_generation_handler import ( + VideoGenerationWorkerHandler, + ) +except ImportError: + pytest.skip( + "SGLang video-generation dependencies not available", allow_module_level=True + ) + +pytestmark = [ + pytest.mark.unit, + pytest.mark.sglang, + pytest.mark.pre_merge, + pytest.mark.gpu_0, + pytest.mark.timeout(120), +] + + +def _synthetic_pil_frames(n: int = 8, size: int = 64) -> list: + return [ + Image.fromarray(np.full((size, size, 3), (i * 24) % 256, dtype=np.uint8)) + for i in range(n) + ] + + +def _probe_video_codec(video_bytes: bytes) -> str: + """Return the video stream codec of encoded bytes, via the shipped ffmpeg.""" + exe = os.environ.get("IMAGEIO_FFMPEG_EXE") + if not exe: + try: + import imageio_ffmpeg + + exe = imageio_ffmpeg.get_ffmpeg_exe() + except Exception: + exe = "ffmpeg" + with tempfile.NamedTemporaryFile(suffix=".mp4") as tmp: + tmp.write(video_bytes) + tmp.flush() + stderr = subprocess.run( + [exe, "-hide_banner", "-i", tmp.name], + capture_output=True, + text=True, + ).stderr + for line in stderr.splitlines(): + if "Video:" in line: + return line.split("Video:", 1)[1].split(",")[0].split()[0] + return "?" + + +@pytest.mark.asyncio +async def test_sglang_video_output_is_vp9_in_shipped_image(): + # _frames_to_video uses no instance state, so bypass the engine-bound + # constructor and call it directly. + handler = VideoGenerationWorkerHandler.__new__(VideoGenerationWorkerHandler) + video_bytes = await handler._frames_to_video(_synthetic_pil_frames(), 8) + assert video_bytes, "encoder produced no bytes" + codec = _probe_video_codec(video_bytes) + assert codec == "vp9", f"expected vp9-encoded output, got codec={codec!r}" diff --git a/components/src/dynamo/trtllm/multimodal_processor.py b/components/src/dynamo/trtllm/multimodal_processor.py index 84dcf4ac7278..ba4de88c243a 100644 --- a/components/src/dynamo/trtllm/multimodal_processor.py +++ b/components/src/dynamo/trtllm/multimodal_processor.py @@ -25,6 +25,7 @@ import torch from safetensors.torch import load as safetensors_load from safetensors.torch import load_file as safetensors_load_file +from tensorrt_llm.inputs.multimodal_data import VideoData from tensorrt_llm.inputs.utils import async_load_video from tensorrt_llm.llmapi.tokenizer import tokenizer_factory @@ -35,12 +36,37 @@ validate_media_url, ) from dynamo.common.multimodal.image_loader import ImageLoader +from dynamo.common.multimodal.nvdec_decoder import probe_video_codec, should_use_nvdec from dynamo.common.multimodal.video_loader import VideoLoader from dynamo.runtime.logging import configure_dynamo_logging configure_dynamo_logging() +def _nvdec_video_data(content: bytes, num_frames: int) -> VideoData: + """Decode H.264/H.265 via NVDEC into TRT-LLM's ``VideoData`` (format="pt"). + + Mirrors ``tensorrt_llm.inputs.media_io._load_video_by_cv2``'s "pt" output for + the pinned v1.3.0rc21: RGB, a list of ``(C, H, W)`` float32 [0,1] tensors, + with the same metadata keys. Synchronous -- call via ``asyncio.to_thread``. + """ + from dynamo.common.multimodal.nvdec_decoder import decode_video_nvdec + + frames_np, meta = decode_video_nvdec(content, num_frames) # (N,H,W,3) uint8 RGB + stacked = frames_np.astype("float32") * (1.0 / 255.0) + nchw = torch.from_numpy(stacked).permute(0, 3, 1, 2).contiguous() + frames_pt = list(torch.unbind(nchw, dim=0)) + fps = float(meta.get("fps") or 0.0) + total = int(meta["total_num_frames"]) + metadata = { + "total_num_frames": total, + "fps": fps, + "duration": (total / fps) if fps > 0 else 0.0, + "frames_indices": meta["frames_indices"], + } + return VideoData(frames=frames_pt, metadata=metadata, audio=None) + + class TokenizerProtocol(Protocol): """ A protocol for tokenizers that defines a decode method. @@ -450,14 +476,33 @@ async def process_openai_request( content = await fetch_bytes( normalized_url, 30.0, policy=self._url_policy ) - with tempfile.NamedTemporaryFile(suffix=".mp4") as video_file: - await asyncio.to_thread(video_file.write, content) - await asyncio.to_thread(video_file.flush) - videos.append( - await async_load_video( - video_file.name, self.num_video_frames + # Dual decode path: H.264/H.265 via NVDEC (hardware); other + # codecs via the vendor cv2 loader. NVDEC failure falls back. + nvdec_video = None + if should_use_nvdec(probe_video_codec(content)): + try: + nvdec_video = await asyncio.to_thread( + _nvdec_video_data, content, self.num_video_frames + ) + except Exception as exc: # noqa: BLE001 - fall back + logging.warning( + "NVDEC decode failed (%s); using the vendor " + "video decoder", + exc, + ) + if nvdec_video is not None: + videos.append(nvdec_video) + else: + with tempfile.NamedTemporaryFile( + suffix=".mp4" + ) as video_file: + await asyncio.to_thread(video_file.write, content) + await asyncio.to_thread(video_file.flush) + videos.append( + await async_load_video( + video_file.name, self.num_video_frames + ) ) - ) else: videos.append( await async_load_video( diff --git a/components/src/dynamo/trtllm/tests/test_trtllm_multimodal_processor.py b/components/src/dynamo/trtllm/tests/test_trtllm_multimodal_processor.py index 88580251eabd..0e43cbe0243d 100644 --- a/components/src/dynamo/trtllm/tests/test_trtllm_multimodal_processor.py +++ b/components/src/dynamo/trtllm/tests/test_trtllm_multimodal_processor.py @@ -88,3 +88,165 @@ async def test_internal_video_uses_dynamo_fetcher_when_allowed(monkeypatch) -> N fetch.assert_awaited_once_with(url, 30.0, policy=processor._url_policy) assert load_video.await_args.args[0] != url + + +def test_nvdec_video_data_builds_pt_videodata(monkeypatch) -> None: + """_nvdec_video_data turns NVDEC's (T,H,W,3) uint8 RGB frames into TRT-LLM's + VideoData "pt" form: a list of (C,H,W) float32 [0,1] tensors + metadata that + matches the vendor async_load_video output. Decode is mocked.""" + import numpy as np + + from dynamo.common.multimodal import nvdec_decoder + + n, h, w = 3, 4, 5 + # Mod in the default int dtype, then cast: `uint8_array % 256` raises + # OverflowError under NumPy 2.x / NEP 50 (256 is out of uint8 range). + frames_np = (np.arange(n * h * w * 3) % 256).astype(np.uint8).reshape(n, h, w, 3) + meta = {"total_num_frames": 24, "fps": 12.0, "frames_indices": [0, 8, 16]} + # _nvdec_video_data imports decode_video_nvdec lazily from this module, so + # patching the source attribute is picked up on the fresh import. + monkeypatch.setattr( + nvdec_decoder, + "decode_video_nvdec", + lambda content, num_frames: (frames_np, meta), + ) + + vd = mmp._nvdec_video_data(b"h264-bytes", 3) + + assert isinstance(vd.frames, list) and len(vd.frames) == n + f0 = vd.frames[0] + assert isinstance(f0, torch.Tensor) + assert tuple(f0.shape) == (3, h, w) # (C, H, W) + assert f0.dtype == torch.float32 + assert 0.0 <= float(f0.min()) and float(f0.max()) <= 1.0 + expected = torch.from_numpy(frames_np[0].astype("float32") * (1.0 / 255.0)).permute( + 2, 0, 1 + ) + assert torch.allclose(f0, expected, atol=1e-6) + assert set(vd.metadata) == { + "total_num_frames", + "fps", + "duration", + "frames_indices", + } + assert vd.metadata["total_num_frames"] == 24 + assert vd.metadata["fps"] == 12.0 + assert vd.metadata["duration"] == 24 / 12.0 + assert vd.metadata["frames_indices"] == [0, 8, 16] + assert vd.audio is None + + +@pytest.mark.asyncio +async def test_h264_video_routes_through_nvdec(monkeypatch) -> None: + """H.264 video input is hardware-decoded via NVDEC; the vendor + async_load_video is bypassed. Other codecs fall back to it (covered by + test_internal_video_uses_dynamo_fetcher_when_allowed).""" + monkeypatch.setenv("DYN_MM_ALLOW_INTERNAL", "1") + processor = MultimodalRequestProcessor( + model_type="multimodal", + model_dir="unused", + max_file_size_mb=10, + tokenizer=MagicMock(), + ) + sentinel = object() + monkeypatch.setattr(mmp, "fetch_bytes", AsyncMock(return_value=b"h264 bytes")) + monkeypatch.setattr(mmp, "probe_video_codec", lambda content: "h264") + monkeypatch.setattr(mmp, "should_use_nvdec", lambda codec: codec == "h264") + nvdec = MagicMock(return_value=sentinel) + monkeypatch.setattr(mmp, "_nvdec_video_data", nvdec) + load_video = AsyncMock(return_value=object()) + monkeypatch.setattr(mmp, "async_load_video", load_video) + + await processor.process_openai_request( + { + "multi_modal_data": { + "video_url": [{"Url": "http://169.254.169.254/v.mp4"}] + }, + "token_ids": [1], + }, + embeddings=None, + ep_disaggregated_params=None, + ) + + nvdec.assert_called_once() # the NVDEC transform ran ... + assert nvdec.call_args.args[0] == b"h264 bytes" + load_video.assert_not_awaited() # ... and the vendor decoder was bypassed + + +@pytest.mark.asyncio +async def test_vp9_video_reports_an_unsupported_codec_error(monkeypatch) -> None: + """VP9 has no decoder in this image, and the request must say so. + + The TensorRT-LLM images ship no cv2, and TensorRT-LLM decodes video through + it (`_load_video_by_cv2`). NVDEC covers H.264/H.265 only, so VP8/VP9/AV1 have + no decode path at all here -- a documented consequence of the codec-compliant + build, not an accident. + + Assert the failure rather than leave the gap untested, the approach NVIDIA + DALI took when it trimmed its own FFmpeg build (NVIDIA/DALI#6352). Without + this, a base-image bump that quietly reintroduced cv2 -- or a routing change + that sent VP9 somewhere unexpected -- would go unnoticed, since no test + exercises a codec this image cannot decode. + + The contract asserted is what a client sees: HTTP 400 naming the video, + not a 500 or a hang. + + Confirmed against a real TensorRT-LLM image (1.3.0rc22, no cv2 installed), + which returned exactly: + + HTTP 400 ... Failed to load video (): OpenCV (cv2) is required for + video decoding but is not installed. Install it with + `pip install opencv-python-headless`. + + Only the prefix and the 400 are ours (`multimodal_processor.py`); the rest is + TensorRT-LLM's own guard, so this asserts the wording we own and that the + upstream reason is *preserved* rather than swallowed -- not the vendor's + exact phrasing, which any version bump may reword. + """ + monkeypatch.setenv("DYN_MM_ALLOW_INTERNAL", "1") + processor = MultimodalRequestProcessor( + model_type="multimodal", + model_dir="unused", + max_file_size_mb=10, + tokenizer=MagicMock(), + ) + monkeypatch.setattr(mmp, "fetch_bytes", AsyncMock(return_value=b"vp9 bytes")) + monkeypatch.setattr(mmp, "probe_video_codec", lambda content: "vp9") + # Real routing: only H.264/H.265 are hardware-decoded. + monkeypatch.setattr( + mmp, "should_use_nvdec", lambda codec: codec in ("h264", "hevc") + ) + nvdec = MagicMock() + monkeypatch.setattr(mmp, "_nvdec_video_data", nvdec) + # The vendor loader failing on an image with no cv2, reproducing the message + # observed on hardware verbatim. Our handler catches bare `Exception`, so the + # type is immaterial -- the message is what reaches the client. + upstream_error = ImportError( + "OpenCV (cv2) is required for video decoding but is not installed. " + "Install it with `pip install opencv-python-headless`." + ) + monkeypatch.setattr(mmp, "async_load_video", AsyncMock(side_effect=upstream_error)) + + with pytest.raises(HttpStatusError) as excinfo: + await processor.process_openai_request( + { + "multi_modal_data": { + "video_url": [{"Url": "http://169.254.169.254/v.webm"}] + }, + "token_ids": [1], + }, + embeddings=None, + ep_disaggregated_params=None, + ) + + # A client-visible 400, not a 500: the input is unsupported, not broken server. + assert excinfo.value.status == 400 + message = str(excinfo.value) + # Ours: the prefix and the offending URL, so the client knows which input failed. + assert "Failed to load video" in message + assert "http://169.254.169.254/v.webm" in message + # The upstream reason is preserved verbatim rather than swallowed. Asserted as + # "whatever the cause said survives", not as specific vendor wording. + assert str(upstream_error) in message + + nvdec.assert_not_called() # VP9 must never take the hardware path diff --git a/components/src/dynamo/trtllm/tests/test_video_encode_real.py b/components/src/dynamo/trtllm/tests/test_video_encode_real.py new file mode 100644 index 000000000000..f5f0809f6eb6 --- /dev/null +++ b/components/src/dynamo/trtllm/tests/test_video_encode_real.py @@ -0,0 +1,67 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Real (un-mocked) video-encode regression test for the TRT-LLM output path. + +Drives encode_to_video_bytes (the exact call TRT-LLM's VideoGenerationHandler +makes) through the ACTUAL ffmpeg baked into the shipped runtime image, and +asserts the produced stream is VP9. pre_merge guard for the shipped-image codec +gap. Encoding VP9 (libvpx-vp9) is CPU-only, so no GPU is used (gpu_0). +""" + +import os +import subprocess +import tempfile + +import numpy as np +import pytest + +try: + from dynamo.common.utils.video_utils import encode_to_video_bytes +except ImportError: + pytest.skip("video_utils not available", allow_module_level=True) + +pytestmark = [ + pytest.mark.unit, + pytest.mark.trtllm, + pytest.mark.pre_merge, + pytest.mark.gpu_0, + pytest.mark.timeout(120), +] + + +def _synthetic_frames(n: int = 8, size: int = 64) -> np.ndarray: + return np.stack( + [np.full((size, size, 3), (i * 24) % 256, dtype=np.uint8) for i in range(n)] + ) + + +def _probe_video_codec(video_bytes: bytes) -> str: + """Return the video stream codec of encoded bytes, via the shipped ffmpeg.""" + exe = os.environ.get("IMAGEIO_FFMPEG_EXE") + if not exe: + try: + import imageio_ffmpeg + + exe = imageio_ffmpeg.get_ffmpeg_exe() + except Exception: + exe = "ffmpeg" + with tempfile.NamedTemporaryFile(suffix=".mp4") as tmp: + tmp.write(video_bytes) + tmp.flush() + stderr = subprocess.run( + [exe, "-hide_banner", "-i", tmp.name], + capture_output=True, + text=True, + ).stderr + for line in stderr.splitlines(): + if "Video:" in line: + return line.split("Video:", 1)[1].split(",")[0].split()[0] + return "?" + + +def test_trtllm_video_output_is_vp9_in_shipped_image(): + video_bytes = encode_to_video_bytes(_synthetic_frames(), fps=8, output_format="mp4") + assert video_bytes, "encoder produced no bytes" + codec = _probe_video_codec(video_bytes) + assert codec == "vp9", f"expected vp9-encoded output, got codec={codec!r}" diff --git a/components/src/dynamo/vllm/omni/output_formatter.py b/components/src/dynamo/vllm/omni/output_formatter.py index 384a90c2127d..c51ebc6174da 100644 --- a/components/src/dynamo/vllm/omni/output_formatter.py +++ b/components/src/dynamo/vllm/omni/output_formatter.py @@ -11,7 +11,6 @@ import asyncio import base64 import logging -import tempfile import time import uuid from io import BytesIO @@ -20,7 +19,6 @@ import numpy as np import soundfile as sf import torch -from diffusers.utils.export_utils import export_to_video from dynamo.common.protocols.audio_protocol import AudioData, NvAudioSpeechResponse from dynamo.common.protocols.image_protocol import ImageData, NvImagesResponse @@ -28,7 +26,11 @@ from dynamo.common.storage import upload_to_fs from dynamo.common.utils.engine_response import normalize_finish_reason from dynamo.common.utils.output_modalities import RequestType -from dynamo.common.utils.video_utils import normalize_video_frames +from dynamo.common.utils.video_utils import ( + encode_to_video_bytes, + frames_to_numpy, + normalize_video_frames, +) from dynamo.vllm.handlers import build_prompt_tokens_details from dynamo.vllm.omni.utils import is_empty_payload @@ -141,12 +143,16 @@ async def _encode_video( ) try: start_time = time.time() - frame_list = normalize_video_frames(images) - with tempfile.NamedTemporaryFile( - suffix=f".{output_format}", delete=True - ) as tmp: - await asyncio.to_thread(export_to_video, frame_list, tmp.name, fps) - video_bytes = tmp.read() + # Encode with the in-tree VP9 (libvpx-vp9) encoder rather + # than diffusers.export_to_video, whose imageio backend defaults to the + # H.264 codec that the codec-compliant image no longer ships (it would + # fail with "No valid H.264 encoder was found"). encode_to_video_bytes + # is the same shared helper the TRT-LLM video handler uses; VP9-in-mp4 + # is valid and decodes with our VP8/VP9 allowlist. + frames_np = frames_to_numpy(normalize_video_frames(images)) + video_bytes = await asyncio.to_thread( + encode_to_video_bytes, frames_np, fps=fps, output_format=output_format + ) if response_format == "b64_json": video_data = VideoData( diff --git a/components/src/dynamo/vllm/tests/omni/test_output_formatter.py b/components/src/dynamo/vllm/tests/omni/test_output_formatter.py index 1945c1eefe06..aa9d2cba8411 100644 --- a/components/src/dynamo/vllm/tests/omni/test_output_formatter.py +++ b/components/src/dynamo/vllm/tests/omni/test_output_formatter.py @@ -533,7 +533,14 @@ def _patches(self): "dynamo.vllm.omni.output_formatter.normalize_video_frames", return_value=[MagicMock()], ), - _patch("dynamo.vllm.omni.output_formatter.export_to_video"), + _patch( + "dynamo.vllm.omni.output_formatter.frames_to_numpy", + return_value=MagicMock(), + ), + _patch( + "dynamo.vllm.omni.output_formatter.encode_to_video_bytes", + return_value=b"video-bytes", + ), _patch( "dynamo.vllm.omni.output_formatter.upload_to_fs", return_value="http://x/v.mp4", @@ -553,8 +560,8 @@ async def test_video_url_response_format(self): stage = MagicMock() stage.images = [MagicMock()] - p1, p2, p3, p4 = self._patches() - with p1, p2, p3 as mock_upload, p4: + p1, p2, p3, p4, p5 = self._patches() + with p1, p2, p3, p4 as mock_upload, p5: result = await f.format( stage, "r5", @@ -580,8 +587,8 @@ async def test_video_b64_response_format(self): stage = MagicMock() stage.images = [MagicMock()] - p1, p2, p3, p4 = self._patches() - with p1, p2, p3 as mock_upload, p4: + p1, p2, p3, p4, p5 = self._patches() + with p1, p2, p3, p4 as mock_upload, p5: result = await f.format( stage, "r6", @@ -607,8 +614,8 @@ async def test_video_default_response_format_is_url(self): stage = MagicMock() stage.images = [MagicMock()] - p1, p2, p3, p4 = self._patches() - with p1, p2, p3 as mock_upload, p4: + p1, p2, p3, p4, p5 = self._patches() + with p1, p2, p3, p4 as mock_upload, p5: result = await f.format( stage, "r7", request_type=RequestType.VIDEO_GENERATION, fps=16 ) diff --git a/components/src/dynamo/vllm/tests/omni/test_video_encode_real.py b/components/src/dynamo/vllm/tests/omni/test_video_encode_real.py new file mode 100644 index 000000000000..be2cbb036fb1 --- /dev/null +++ b/components/src/dynamo/vllm/tests/omni/test_video_encode_real.py @@ -0,0 +1,129 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Real (un-mocked) video-encode regression test for the vLLM-Omni output path. + +Drives DiffusionFormatter._encode_video through the ACTUAL ffmpeg baked into the +shipped runtime image (no imageio/encode mock, no pip install). This is the +pre_merge guard for the gap that let two failures ship green: + - the image shipping no VP9 encoder (encode raises -> status "failed"), and + - the handler defaulting to a removed H.264 encoder (would also fail). +Runs on the CUDA image only (gpu_0, no xpu) because the in-tree ffmpeg is copied +into the CUDA runtime; encoding VP9 (libvpx-vp9) is CPU-only, so no GPU is used. +""" + +import base64 +import os +import subprocess +import tempfile + +import numpy as np +import pytest + +try: + from PIL import Image + + from dynamo.vllm.omni.output_formatter import DiffusionFormatter +except ImportError: + pytest.skip("vLLM omni dependencies not available", allow_module_level=True) + +pytestmark = [ + pytest.mark.unit, + pytest.mark.vllm, + pytest.mark.pre_merge, + pytest.mark.gpu_0, + pytest.mark.profiled_vram_gib(0), + pytest.mark.timeout(120), +] + + +def _synthetic_pil_frames(n: int = 8, size: int = 64) -> list: + return [ + Image.fromarray(np.full((size, size, 3), (i * 24) % 256, dtype=np.uint8)) + for i in range(n) + ] + + +def _synthetic_numpy_frames(n: int = 8, size: int = 64) -> list: + """A single float32 [0, 1] video tensor, as diffusion pipelines emit it. + + Wraps one ``(n, size, size, 3)`` array in a length-1 list so it flows through + ``normalize_video_frames`` exactly like ``stage_output.images`` does for the + Wan2.1 T2V serve path — the shape that regressed with 'numpy.ndarray' object + has no attribute 'convert'. + """ + video = np.stack( + [ + np.full((size, size, 3), (i * 24) % 256, dtype=np.float32) / 255.0 + for i in range(n) + ] + ) + return [video] + + +def _probe_video_codec(video_bytes: bytes) -> str: + """Return the video stream codec of encoded bytes, via the shipped ffmpeg.""" + exe = os.environ.get("IMAGEIO_FFMPEG_EXE") + if not exe: + try: + import imageio_ffmpeg + + exe = imageio_ffmpeg.get_ffmpeg_exe() + except Exception: + exe = "ffmpeg" + with tempfile.NamedTemporaryFile(suffix=".mp4") as tmp: + tmp.write(video_bytes) + tmp.flush() + stderr = subprocess.run( + [exe, "-hide_banner", "-i", tmp.name], + capture_output=True, + text=True, + ).stderr + for line in stderr.splitlines(): + if "Video:" in line: + return line.split("Video:", 1)[1].split(",")[0].split()[0] + return "?" + + +@pytest.mark.asyncio +async def test_omni_video_output_is_vp9_in_shipped_image(): + formatter = DiffusionFormatter( + model_name="test", media_fs=None, media_http_url=None + ) + result = await formatter._encode_video( + _synthetic_pil_frames(), + "req-vp9", + fps=8, + response_format="b64_json", + ) + # A missing ffmpeg or a regression to the H.264 default surfaces here as a + # "failed" status rather than a raise (the handler catches encode errors). + assert result["status"] == "completed", result.get("error") + video_bytes = base64.b64decode(result["data"][0]["b64_json"]) + assert video_bytes, "encoder produced no bytes" + codec = _probe_video_codec(video_bytes) + assert codec == "vp9", f"expected vp9-encoded output, got codec={codec!r}" + + +@pytest.mark.asyncio +async def test_omni_video_output_accepts_numpy_frames(): + """Regression: real diffusion pipelines emit float numpy frames, not PIL. + + The PIL-only ``frames_to_numpy`` crashed the Wan2.1 T2V serve path with + "'numpy.ndarray' object has no attribute 'convert'" — green in unit tests + that fed PIL frames. This drives the same numpy shape the pipeline produces. + """ + formatter = DiffusionFormatter( + model_name="test", media_fs=None, media_http_url=None + ) + result = await formatter._encode_video( + _synthetic_numpy_frames(), + "req-vp9-np", + fps=8, + response_format="b64_json", + ) + assert result["status"] == "completed", result.get("error") + video_bytes = base64.b64decode(result["data"][0]["b64_json"]) + assert video_bytes, "encoder produced no bytes" + codec = _probe_video_codec(video_bytes) + assert codec == "vp9", f"expected vp9-encoded output, got codec={codec!r}" diff --git a/container/compliance/enumerate_bundled_decoders.py b/container/compliance/enumerate_bundled_decoders.py new file mode 100644 index 000000000000..134d64e38e17 --- /dev/null +++ b/container/compliance/enumerate_bundled_decoders.py @@ -0,0 +1,84 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Enumerate what a vendored libavcodec actually registers. + +`scan_codecs.py` gates which media libraries may appear in an image. This answers +the different question of what a library that is *allowed* to be there actually +implements — needed to review a third-party waiver, since a bundled libavcodec +may or may not carry software decoders for the formats we exclude. + +Neither obvious shortcut works. These libraries are stripped, so `nm` reports +nothing even for codecs that are certainly present; and `strings` gives false +positives, because a decoder's long name survives in the binary when only its +parser or bitstream filter is retained. The authoritative answer is what the +library registers at runtime, so walk `av_codec_iterate` and ask +`av_codec_is_decoder` per entry. + +Usage: + python3 enumerate_bundled_decoders.py + +Exit code is 1 when any excluded decoder is registered, so it can gate a check. +""" + +from __future__ import annotations + +import ctypes +import sys + + +class AVCodec(ctypes.Structure): + _fields_ = [ + ("name", ctypes.c_char_p), + ("long_name", ctypes.c_char_p), + ("type", ctypes.c_int), + ("id", ctypes.c_int), + ] + + +def main(path: str) -> int: + lib = ctypes.CDLL(path) + lib.av_codec_iterate.restype = ctypes.c_void_p + lib.av_codec_iterate.argtypes = [ctypes.POINTER(ctypes.c_void_p)] + lib.av_codec_is_decoder.restype = ctypes.c_int + lib.av_codec_is_decoder.argtypes = [ctypes.c_void_p] + + opaque = ctypes.c_void_p(None) + decoders: set[str] = set() + encoders: set[str] = set() + while True: + c = lib.av_codec_iterate(ctypes.byref(opaque)) + if not c: + break + entry = ctypes.cast(c, ctypes.POINTER(AVCodec)).contents + name = entry.name.decode() + if lib.av_codec_is_decoder(c): + decoders.add(name) + else: + encoders.add(name) + + print(f"library : {path}") + print(f"registers: {len(decoders)} decoders, {len(encoders)} encoders\n") + + trimmed = ["h264", "hevc", "aac", "aac_fixed", "aac_latm"] + kept = ["vp8", "vp9", "mjpeg", "av1"] + + print(" decoders we exclude from the software stack:") + for name in trimmed: + mark = "PRESENT" if name in decoders else "absent" + print(f" {name:<12} decoder: {mark}") + print("\n expected present (sanity check -- a broken probe fails here):") + for name in kept: + mark = "present" if name in decoders else "ABSENT" + print(f" {name:<12} decoder: {mark}") + + still_there = [n for n in trimmed if n in decoders] + print() + if still_there: + print(f"VERDICT: registers excluded decoders: {', '.join(still_there)}") + return 1 + print("VERDICT: none of the excluded decoders are registered") + return 0 + + +sys.exit(main(sys.argv[1])) diff --git a/container/compliance/native_packages.yaml b/container/compliance/native_packages.yaml index 7d7340e55f57..f532e8d7de55 100644 --- a/container/compliance/native_packages.yaml +++ b/container/compliance/native_packages.yaml @@ -60,11 +60,11 @@ packages: # NOTE: NIXL ships as the `nixl` Python wheel (attributed by the python # generator), so it is intentionally NOT duplicated here. - # LGPL media stack — built --disable-gpl in wheel_builder (h264_nvenc + libvpx) - # and copied into vllm/sglang runtime (container/templates/vllm_runtime.Dockerfile - # + sglang_runtime.Dockerfile). Replaces the purged GPL apt ffmpeg. + # LGPL media stack — built --disable-gpl in wheel_builder with libvpx (VP9) and + # no H.264 capability, then copied into the vLLM, SGLang, and TensorRT-LLM + # runtimes for the VP9 video-encode path. - name: ffmpeg - version: "8.1" + version: "8.1.2" license: LGPL-2.1-or-later source: https://ffmpeg.org/ images: @@ -72,9 +72,12 @@ packages: - vllm-runtime-efa - sglang-runtime - sglang-runtime-efa + - trtllm-runtime + - trtllm-runtime-efa - name: libvpx - # VP8/VP9 codec library, ffmpeg's only non-NVIDIA encoder dependency. + # VP8/VP9 codec library — the in-tree ffmpeg's only video encoder now that + # H.264/NVENC is removed. Shipped in every image that carries the ffmpeg libs. version: v1.14.1 license: BSD-3-Clause source: https://chromium.googlesource.com/webm/libvpx @@ -83,6 +86,8 @@ packages: - vllm-runtime-efa - sglang-runtime - sglang-runtime-efa + - trtllm-runtime + - trtllm-runtime-efa # UCX — Unified Communication X, built from source in wheel_builder and used by # the NIXL transport. Installed to /usr/local (no dpkg metadata). diff --git a/container/compliance/policy/codec_policy.yaml b/container/compliance/policy/codec_policy.yaml new file mode 100644 index 000000000000..fe2193967f2f --- /dev/null +++ b/container/compliance/policy/codec_policy.yaml @@ -0,0 +1,144 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Media-codec allowlist gate — consumed by container/compliance/scan_codecs.py. +# +# We ship an in-tree ffmpeg built to a narrow media-codec set (see +# wheel_builder.Dockerfile). This gate keeps the finished image aligned with +# that: it fails the image BUILD if a media-codec library/binary appears outside +# our in-tree allowlist (allow_paths) or a reasoned exception — catching a distro +# libav*, a wheel-bundled libavcodec, libx264/265, or a stray/imageio-bundled +# ffmpeg that slipped in via a base-image or dependency bump. +# +# Filesystem-first because bundled codec .so files don't appear as SBOM +# components; deny_components adds a supplementary SBOM version floor. +# +# Keep this in sync with the wheel_builder.Dockerfile ffmpeg allowlist and the +# vllm/sglang media-library cleanup. New third-party bundled media libraries must +# be either removed or added to `exceptions` with a reason + owner — never +# silently allowlisted. + +# Filesystem globs (matched against absolute paths under the scan root) for +# media-codec libraries / binaries that must not ship. libav*/libsw* are matched +# wholesale and re-permitted only for our in-tree copy via allow_paths, so any +# THIRD-PARTY bundled libavcodec (distro, a vendored wheel) is caught. +deny_globs: + # Both the plain soname and the auditwheel hash-renamed variant (e.g. a wheel + # that vendors libx264-.so under its .libs/) must be caught. + - "**/libx264.so*" + - "**/libx264-*.so*" + - "**/libx265.so*" + - "**/libx265-*.so*" + - "**/libfdk-aac.so*" + - "**/libfdk-aac-*.so*" + - "**/libavcodec.so*" + - "**/libavcodec-*.so*" + - "**/libavdevice.so*" + - "**/libavdevice-*.so*" + - "**/libavformat.so*" + - "**/libavformat-*.so*" + - "**/libavfilter.so*" + - "**/libavfilter-*.so*" + - "**/libavutil.so*" + - "**/libavutil-*.so*" + - "**/libswscale.so*" + - "**/libswscale-*.so*" + - "**/libswresample.so*" + - "**/libswresample-*.so*" + - "**/libpostproc.so*" + - "**/libpostproc-*.so*" + # Standalone ffmpeg/ffprobe CLIs (e.g. a distro /usr/bin/ffmpeg). Our own copy + # at /usr/local/bin/ffmpeg is re-permitted by allow_paths below. + - "**/bin/ffmpeg" + - "**/bin/ffprobe" + # GPL prebuilt ffmpeg binary that imageio-ffmpeg bundles under site-packages. + - "**/imageio_ffmpeg/binaries/ffmpeg-*" + +# CycloneDX component gate (only applied when --sbom is passed). `min_fixed_version` +# flags any version strictly below a known-good floor; kept current with upstream +# ffmpeg maintenance releases. +deny_components: + - name: ffmpeg + min_fixed_version: "8.1.2" + - name: x264 + - name: x265 + - name: fdk-aac + +# Absolute-path prefixes whose media libraries are OUR in-tree ffmpeg, built with +# the narrow decoder/encoder allowlist in wheel_builder.Dockerfile. Matches here +# are reported as ALLOWED and never fail the build. +allow_paths: + - "/usr/local/lib/libav" + - "/usr/local/lib/libsw" + - "/usr/local/lib/libpostproc" + - "/usr/local/bin/ffmpeg" + - "/usr/local/src/ffmpeg" # retained source tree (kept for legal reasons) + +# Explicit third-party exceptions: LOGGED in the report but do NOT fail the build. +# Every entry must carry a reason and an owner so the waiver is auditable. +exceptions: + - glob: "**/nvidia/dali/.libs/*" + reason: >- + NVIDIA DALI (nvidia-dali-cuda130) vendors its own ffmpeg stack (libav* plus + the libsw*/libpostproc companions) under .libs/ for its video reader. It is + owned and handled by its own component team and tracked separately rather + than gated here. The glob covers the whole vendored dir so the libsw* + companions are waived too, not just libavcodec. + + Resolved by a version bump, not by narrowing. Measured with + container/compliance/enumerate_bundled_decoders.py against the shipped + image: DALI 2.1.0 registers 446 decoders including h264, hevc, aac, + aac_fixed and aac_latm. DALI restricted its own ffmpeg build to drop those + (NVIDIA/DALI_deps#162, NVIDIA/DALI#6352, May 2026), first released in + 2.1.1 -- as a cherry-pick, so checking whether the merge commit is an + ancestor of the tag gives the wrong answer; the release notes are the + reliable source. Upgrading the wheel in place to 2.1.1 or 2.2.0 leaves 440 + decoders with all five absent, vp8/vp9/mjpeg/av1 still present, and the + image importing and passing its unit suite. + + This waiver cannot be narrowed to the non-codec companions, which an + earlier revision of this note claimed. libavcodec, libavfilter, libavformat, + libavutil and libswscale are all DT_NEEDED by libdali.so, libdali_kernels.so, + libdali_operators.so and every python_function_plugin, so removing the codec + libraries on their own breaks `import nvidia.dali` outright. The vendored + set is all-or-nothing with the package. + + The entry stays even so. trtllm_runtime.Dockerfile now pins + nvidia-dali-cuda130>=2.1.1, so the vendored libraries no longer register + those decoders -- but they are still on disk and still match the deny globs + above, which gate on presence rather than content. What changed is the + reason for the waiver: it now covers a vendored ffmpeg that carries no + software h264/hevc/aac, rather than one that does. + + Enforcement lives at build time, not here. Two guards in + trtllm_runtime.Dockerfile enumerate every bundled libavcodec -- one after + the upgrade, and one after the overlay COPY, which is the only stage where + a missed whiteout can leave the base image's older copy sitting beside the + upgraded one -- and tests/dependencies/test_no_software_video_codecs.py + repeats the check against the shipped image. + + Deliberately NOT expressed as a deny_components version floor: + nvidia-dali-cuda130 appears in the base-image SBOM and is therefore + subtracted from the generated delta, so such an entry would never match + anything and would read as a check while doing nothing. + + Re-measure when the vendored version moves. + owner: DALI + # PyNvVideoCodec (added for NVDEC hardware video decode) vendors libavutil and + # libavformat under its package dir to demux input containers before handing the + # elementary stream to NVDEC. It ships NO libavcodec (the scan would flag it) and + # no libx264/265/fdk-aac: no software video/audio codec implementation is present; + # the decode runs on NVDEC via libnvcuvid. Scoped to the two non-codec libs only, + # so a libavcodec bundled by a future version would still fail the build. + - glob: "**/PyNvVideoCodec/libavutil.so*" + reason: >- + PyNvVideoCodec bundles libavutil (utility routines, no codec) to support its + NVDEC container demux path; the package ships no libavcodec, so no software + codec implementation is present. Decode runs on NVDEC (libnvcuvid). + owner: PyNvVideoCodec + - glob: "**/PyNvVideoCodec/libavformat.so*" + reason: >- + PyNvVideoCodec bundles libavformat (container demuxers/muxers only, no codec) + to parse the input container before NVDEC decode; the package ships no + libavcodec. Companion waiver to the PyNvVideoCodec libavutil entry above. + owner: PyNvVideoCodec diff --git a/container/compliance/scan_codecs.py b/container/compliance/scan_codecs.py new file mode 100644 index 000000000000..66e9325288d0 --- /dev/null +++ b/container/compliance/scan_codecs.py @@ -0,0 +1,290 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Media-codec allowlist gate for finished images. + +We ship an in-tree ffmpeg built to a narrow media-codec set, so the shipped +filesystem should not contain media-codec libraries/binaries from anywhere else. +Because SBOMs miss statically-bundled codec `.so` files, this is primarily a +FILESYSTEM scan of the built image; an optional `--sbom` adds a CycloneDX +component/version gate (an ffmpeg version floor). + +A denylist hit is classified as: + - ALLOWED — under an `allow_paths` prefix (our own in-tree ffmpeg) + - EXCEPTION — matches a reasoned `exceptions` entry (logged, does not fail) + - VIOLATION — anything else + +With --fail-on-findings, any VIOLATION exits non-zero, failing the image build +when this runs in the compliance licenses stage. + +Policy: container/compliance/policy/codec_policy.yaml +""" + +from __future__ import annotations + +import argparse +import fnmatch +import json +import logging +import os +import sys +from pathlib import Path + +import yaml + +try: + from packaging.version import Version +except ImportError: # packaging is optional; fall back to a numeric-tuple compare + Version = None + +logger = logging.getLogger("compliance.scan_codecs") + +# Virtual kernel filesystems only — pruned at the scan root because walking them +# is meaningless (and /proc is effectively unbounded). Real directories such as +# /tmp and /run are NOT pruned: a codec binary left there still ships in the +# image and must be scanned. +_PRUNE_DIRS = {"proc", "sys", "dev"} + + +def _glob_to_fullpath_pattern(glob: str) -> str: + """Translate a `**/`-style policy glob into an fnmatch pattern applied to the + full POSIX path. `**/` becomes `*` (fnmatch's `*` already spans `/`), so + `**/libx264.so*` -> `*libx264.so*` and `**/nvidia/dali/.libs/libav*` -> + `*nvidia/dali/.libs/libav*`.""" + return "*" + glob[3:] if glob.startswith("**/") else glob + + +def _matches_any(path: str, globs: list[str]) -> str | None: + for g in globs: + if fnmatch.fnmatch(path, _glob_to_fullpath_pattern(g)): + return g + return None + + +class CodecPolicy: + def __init__(self, doc: dict): + self.deny_globs: list[str] = doc.get("deny_globs", []) or [] + self.allow_paths: list[str] = doc.get("allow_paths", []) or [] + self.deny_components: list[dict] = doc.get("deny_components", []) or [] + self.exceptions: list[dict] = doc.get("exceptions", []) or [] + + @classmethod + def load(cls, path: Path) -> "CodecPolicy": + # Fail closed: a compliance gate that silently passes every image because + # of a typo'd/missing key is worse than no gate. Validate the shape up + # front so a malformed policy aborts the scan instead of allow-listing. + doc = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(doc, dict): + raise ValueError(f"codec policy {path} must be a YAML mapping") + if not isinstance(doc.get("deny_globs"), list) or not doc["deny_globs"]: + raise ValueError( + f"codec policy {path} requires a non-empty deny_globs list" + ) + # allow_paths is the key that fails OPEN, so it needs the strictest check. + # `classify` tests `abspath.startswith(p) for p in self.allow_paths`; a + # scalar string iterates per character, one of which is "/", and every + # absolute path starts with "/" -- so a single missing "- " turns the + # whole gate into a pass. Nothing downstream would report it, because + # every finding comes back verdict "allowed". + allow_paths = doc.get("allow_paths") + if allow_paths is not None and ( + not isinstance(allow_paths, list) + or not all(isinstance(p, str) and p for p in allow_paths) + ): + raise ValueError( + f"codec policy {path}: allow_paths must be a list of non-empty " + "strings (a bare string would allow-list every path)" + ) + for exc in doc.get("exceptions") or []: + if not all(exc.get(k) for k in ("glob", "reason", "owner")): + raise ValueError( + f"codec policy {path}: each exception needs glob, reason, and owner" + ) + return cls(doc) + + def classify(self, abspath: str) -> tuple[str, str | None]: + """Return (verdict, detail) for a path already known to hit a deny glob. + verdict is 'allowed' | 'exception' | 'violation'.""" + if any(abspath.startswith(p) for p in self.allow_paths): + return "allowed", None + for exc in self.exceptions: + if fnmatch.fnmatch(abspath, _glob_to_fullpath_pattern(exc.get("glob", ""))): + return "exception", (exc.get("reason") or "").strip() + return "violation", None + + +def scan_filesystem(root: Path, policy: CodecPolicy): + """Walk `root`, returning (violations, exceptions, allowed) lists of dicts.""" + violations, exceptions, allowed = [], [], [] + root_str = os.path.normpath(str(root)) + for dirpath, dirnames, filenames in os.walk(root, followlinks=False): + # Prune virtual dirs only at the scan root (don't prune a legitimately + # named 'run'/'tmp' deeper in a package tree). + if os.path.normpath(dirpath) == root_str: + dirnames[:] = [d for d in dirnames if d not in _PRUNE_DIRS] + for name in filenames: + full = os.path.join(dirpath, name) + # Report paths as absolute-from-root ("/usr/...") so allow_paths and + # exceptions read naturally regardless of where root is mounted. + rel = os.path.relpath(full, root_str) + abspath = "/" + rel.replace(os.sep, "/") + hit = _matches_any(abspath, policy.deny_globs) + if not hit: + continue + verdict, detail = policy.classify(abspath) + entry = {"path": abspath, "glob": hit, "detail": detail} + {"allowed": allowed, "exception": exceptions, "violation": violations}[ + verdict + ].append(entry) + return violations, exceptions, allowed + + +def _version_lt(a: str, b: str) -> bool: + """a < b. Uses packaging.version when available, else a numeric-tuple fallback. + A malformed version under packaging raises (InvalidVersion) and fails the scan + rather than silently degrading to the naive comparator.""" + if Version is not None: + return Version(a) < Version(b) + + def key(v: str) -> list[int]: + return [int(x) if x.isdigit() else 0 for x in v.replace("-", ".").split(".")] + + return key(a) < key(b) + + +def scan_sbom(sbom_path: Path, policy: CodecPolicy) -> list[dict]: + """Flag SBOM components matching deny_components (optionally below a fixed + version). Returns a list of violation dicts.""" + doc = json.loads(sbom_path.read_text(encoding="utf-8")) + out: list[dict] = [] + denied = {d["name"].lower(): d for d in policy.deny_components if d.get("name")} + for comp in doc.get("components", []) or []: + name = (comp.get("name") or "").lower() + rule = denied.get(name) + if not rule: + continue + version = str(comp.get("version") or "") + floor = rule.get("min_fixed_version") + if floor: + # No version ⇒ we cannot prove the component is at/above the fixed + # floor, so flag it rather than let an unversioned denied component + # (e.g. an ffmpeg with no version in the SBOM) silently pass. + if not version: + out.append( + { + "path": f"sbom:{name}@(no version)", + "glob": f"version unknown, cannot verify >= {floor}", + "detail": f"no version in SBOM; fixed floor is {floor}", + } + ) + elif _version_lt(version, floor): + out.append( + { + "path": f"sbom:{name}@{version}", + "glob": f"version < {floor}", + "detail": f"below fixed version {floor}", + } + ) + else: + out.append( + { + "path": f"sbom:{name}@{version}", + "glob": "denied component", + "detail": None, + } + ) + return out + + +def _emit_summary(image, violations, exceptions, allowed): + summary = os.environ.get("GITHUB_STEP_SUMMARY") + if not summary: + return + with open(summary, "a", encoding="utf-8") as s: + s.write(f"### Codec gate{' — ' + image if image else ''}\n\n") + s.write( + f"- violations: **{len(violations)}** | " + f"exceptions: {len(exceptions)} | allowed (in-tree): {len(allowed)}\n\n" + ) + if violations: + s.write("| path | matched | note |\n|---|---|---|\n") + for v in violations: + s.write(f"| `{v['path']}` | `{v['glob']}` | {v['detail'] or ''} |\n") + else: + s.write("✅ no media-codec artifacts outside the allowlist.\n") + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(prog="compliance.scan_codecs") + p.add_argument( + "--root", type=Path, default=Path("/"), help="filesystem root to scan" + ) + p.add_argument( + "--policy", + type=Path, + default=Path(__file__).parent / "policy" / "codec_policy.yaml", + ) + p.add_argument( + "--sbom", + type=Path, + default=None, + help="optional CycloneDX SBOM for the component gate", + ) + p.add_argument("--image", default="", help="image label for the report header") + p.add_argument("--report", type=Path, default=None) + p.add_argument( + "--fail-on-findings", action="store_true", help="exit non-zero on any violation" + ) + p.add_argument("-v", "--verbose", action="store_true") + args = p.parse_args(argv) + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="%(levelname)s [%(name)s]: %(message)s", + ) + + policy = CodecPolicy.load(args.policy) + violations, exceptions, allowed = scan_filesystem(args.root, policy) + if args.sbom: + # An explicitly-supplied SBOM that is missing must fail loudly — silently + # skipping it would disable the version-floor gate without anyone noticing. + if not args.sbom.is_file(): + logger.error("--sbom given but not found: %s", args.sbom) + return 1 + violations.extend(scan_sbom(args.sbom, policy)) + + hdr = f"Codec gate{' for ' + args.image if args.image else ''}" + print(hdr) + print( + f" scanned root: {args.root} | violations: {len(violations)} | " + f"exceptions: {len(exceptions)} | allowed (in-tree): {len(allowed)}" + ) + for label, rows in ( + ("ALLOWED", allowed), + ("EXCEPTION", exceptions), + ("VIOLATION", violations), + ): + for r in rows: + line = f" {label:9} {r['path']}" + if r.get("detail"): + line += f" — {r['detail']}" + (logger.debug if label == "ALLOWED" else print)(line) + + if args.report: + args.report.write_text( + "\n".join(f"{r['path']}\t{r['glob']}" for r in violations) + "\n", + encoding="utf-8", + ) + _emit_summary(args.image, violations, exceptions, allowed) + + if violations: + logger.error( + "%d media-codec artifact(s) present outside the allowlist; " + "remove them or add a reasoned exception to codec_policy.yaml", + len(violations), + ) + if args.fail_on_findings: + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/container/compliance/tests/test_scan_codecs.py b/container/compliance/tests/test_scan_codecs.py new file mode 100644 index 000000000000..eda99393d1d2 --- /dev/null +++ b/container/compliance/tests/test_scan_codecs.py @@ -0,0 +1,253 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Tests for the media-codec allowlist gate (compliance.scan_codecs). + +Run from the repo root with the compliance package on the path: + + PYTHONPATH=container python -m pytest container/compliance/tests/test_scan_codecs.py +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from compliance.scan_codecs import CodecPolicy, main, scan_filesystem, scan_sbom + +pytestmark = [ + pytest.mark.pre_merge, + pytest.mark.post_merge, + pytest.mark.gpu_0, + pytest.mark.unit, +] + +# The real shipped policy — the tests assert against it, not a fixture, so a +# policy edit that would let a non-allowlisted media codec through fails a test here. +_POLICY = CodecPolicy.load( + Path(__file__).resolve().parents[1] / "policy" / "codec_policy.yaml" +) + + +def _touch(root: Path, rel: str) -> None: + p = root / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_bytes(b"\x7fELF") + + +def test_in_tree_ffmpeg_is_allowed(tmp_path: Path): + # Our own /usr/local ffmpeg libs — must classify as allowed, never violate. + for rel in ( + "usr/local/lib/libavcodec.so.62", + "usr/local/lib/libswscale.so.9", + "usr/local/bin/ffmpeg", + ): + _touch(tmp_path, rel) + violations, _exceptions, allowed = scan_filesystem(tmp_path, _POLICY) + assert violations == [] + assert {a["path"] for a in allowed} == { + "/usr/local/lib/libavcodec.so.62", + "/usr/local/lib/libswscale.so.9", + "/usr/local/bin/ffmpeg", + } + + +def test_dali_bundled_ffmpeg_is_a_logged_exception(tmp_path: Path): + # DALI's whole vendored .libs/ is waived — both libavcodec and the libsw* + # companions (the latter regressed the trtllm build when the glob was libav* only). + dali = "usr/local/lib/python3.12/dist-packages/nvidia/dali/.libs" + _touch(tmp_path, f"{dali}/libavcodec-73c99a8b.so.62") + _touch(tmp_path, f"{dali}/libswscale-8cfd67c4.so.9") + violations, exceptions, _allowed = scan_filesystem(tmp_path, _POLICY) + assert violations == [] + assert len(exceptions) == 2 + assert all("DALI" in (e["detail"] or "") for e in exceptions) + + +def test_pynvvideocodec_demux_libs_waived_but_codec_denied(tmp_path: Path): + # PyNvVideoCodec vendors libavutil + libavformat (container demux, no codec) + # for its NVDEC path — waived. The exception is scoped to those two libs, so a + # libavcodec bundled by a future version must STILL fail the gate. + pkg = "usr/local/lib/python3.12/dist-packages/PyNvVideoCodec" + for rel in ( + f"{pkg}/libavutil.so.60.26.102", + f"{pkg}/libavutil.so", + f"{pkg}/libavformat.so.62", + f"{pkg}/libavformat.so", + ): + _touch(tmp_path, rel) + _touch(tmp_path, f"{pkg}/libavcodec.so.62") # a real codec must NOT be waived + violations, exceptions, _allowed = scan_filesystem(tmp_path, _POLICY) + assert [v["path"] for v in violations] == [f"/{pkg}/libavcodec.so.62"] + assert len(exceptions) == 4 + assert all("PyNvVideoCodec" in (e["detail"] or "") for e in exceptions) + + +@pytest.mark.parametrize( + "rel", + [ + "usr/lib/x86_64-linux-gnu/libx264.so.164", + "usr/lib/x86_64-linux-gnu/libx265.so.199", + # A third-party bundled libavcodec NOT under /usr/local and NOT DALI. + "usr/local/lib/python3.12/dist-packages/opencv_python_headless.libs/libavcodec-156beeea.so.62.11.100", + "opt/venv/lib/python3.12/site-packages/imageio_ffmpeg/binaries/ffmpeg-linux-x86_64-v7.1", + ], +) +def test_nonallowlisted_media_artifacts_are_violations(tmp_path: Path, rel: str): + _touch(tmp_path, rel) + violations, _, _ = scan_filesystem(tmp_path, _POLICY) + assert [v["path"] for v in violations] == ["/" + rel] + + +def test_unrelated_files_are_ignored(tmp_path: Path): + for rel in ("usr/lib/libc.so.6", "usr/local/lib/libvpx.so.9", "opt/dynamo/foo.py"): + _touch(tmp_path, rel) + violations, exceptions, allowed = scan_filesystem(tmp_path, _POLICY) + assert (violations, exceptions, allowed) == ([], [], []) + + +def test_sbom_flags_ffmpeg_below_cve_floor(tmp_path: Path): + sbom = tmp_path / "s.cdx.json" + sbom.write_text( + json.dumps( + { + "components": [ + {"name": "ffmpeg", "version": "8.0.1"}, # < 8.1.2 -> flagged + {"name": "ffmpeg", "version": "8.1.2"}, # == floor -> ok + {"name": "libvpx", "version": "1.14.1"}, # not denied + ] + } + ) + ) + hits = scan_sbom(sbom, _POLICY) + assert [h["path"] for h in hits] == ["sbom:ffmpeg@8.0.1"] + + +def test_codec_under_tmp_is_scanned(tmp_path: Path): + # /tmp is a real image directory (not a virtual kernel fs) — a codec left + # there still ships and must be flagged, not pruned from the walk. + _touch(tmp_path, "tmp/libx264.so.164") + violations, _, _ = scan_filesystem(tmp_path, _POLICY) + assert [v["path"] for v in violations] == ["/tmp/libx264.so.164"] + + +def test_sbom_unversioned_denied_component_is_flagged(tmp_path: Path): + # A denied component with no version cannot be proven >= the fixed floor, + # so it must be flagged rather than silently pass the version gate. + sbom = tmp_path / "s.cdx.json" + sbom.write_text(json.dumps({"components": [{"name": "ffmpeg"}]})) + hits = scan_sbom(sbom, _POLICY) + assert len(hits) == 1 and "no version" in hits[0]["path"] + + +def test_missing_sbom_path_fails(tmp_path: Path): + # An explicitly-supplied but missing --sbom must fail, not silently skip + # (which would disable the version-floor gate unnoticed). + assert main(["--root", str(tmp_path), "--sbom", str(tmp_path / "nope.json")]) == 1 + + +def test_fail_on_findings_exit_code(tmp_path: Path): + _touch(tmp_path, "usr/lib/x86_64-linux-gnu/libx264.so.164") + assert main(["--root", str(tmp_path), "--fail-on-findings"]) == 1 + # Same tree, report-only: findings reported but exit 0. + assert main(["--root", str(tmp_path)]) == 0 + + +def test_clean_tree_passes(tmp_path: Path): + _touch(tmp_path, "usr/local/lib/libavcodec.so.62") # ours -> allowed + assert main(["--root", str(tmp_path), "--fail-on-findings"]) == 0 + + +# --- Fail-closed policy loading: a malformed policy must abort, not allow-list --- + + +def _write_policy(tmp_path: Path, text: str) -> Path: + p = tmp_path / "policy.yaml" + p.write_text(text, encoding="utf-8") + return p + + +def test_load_rejects_missing_deny_globs(tmp_path: Path): + # No deny_globs -> nothing is classified -> every image would pass silently. + pol = _write_policy(tmp_path, "allow_paths: ['/usr/local/lib/libav']\n") + with pytest.raises(ValueError, match="deny_globs"): + CodecPolicy.load(pol) + + +def test_load_rejects_empty_deny_globs(tmp_path: Path): + pol = _write_policy(tmp_path, "deny_globs: []\n") + with pytest.raises(ValueError, match="deny_globs"): + CodecPolicy.load(pol) + + +def test_load_rejects_non_mapping(tmp_path: Path): + pol = _write_policy(tmp_path, "- just\n- a\n- list\n") + with pytest.raises(ValueError, match="mapping"): + CodecPolicy.load(pol) + + +def test_load_rejects_exception_missing_owner(tmp_path: Path): + pol = _write_policy( + tmp_path, + "deny_globs: ['**/libavcodec.so*']\n" + "exceptions:\n" + " - glob: '**/foo/*'\n" + " reason: because\n", # owner intentionally omitted + ) + with pytest.raises(ValueError, match="owner"): + CodecPolicy.load(pol) + + +def test_load_rejects_scalar_allow_paths(tmp_path: Path): + """A bare string for allow_paths must abort, not allow-list everything. + + `classify` tests `abspath.startswith(p) for p in self.allow_paths`. A scalar + iterates per character, one of which is "/", so every absolute path matches + and every violation comes back "allowed" -- the gate passes any image while + still printing a report. One missing "- " is all it takes, which is why this + key is validated more strictly than the others. + """ + pol = _write_policy( + tmp_path, + "deny_globs: ['**/libx264.so*']\nallow_paths: /usr/local/lib\n", + ) + with pytest.raises(ValueError, match="allow_paths"): + CodecPolicy.load(pol) + + +def test_scalar_allow_paths_would_have_allow_listed_everything(): + """The bug the check above prevents, pinned so the reason stays visible. + + Constructed directly rather than through `load`, which now rejects it. + """ + policy = CodecPolicy( + {"deny_globs": ["**/libx264.so*"], "allow_paths": "/usr/local/lib"} + ) + assert policy.classify("/opt/evil/libx264.so.164") == ("allowed", None) + # Spelled correctly, the same path is a violation. + ok = CodecPolicy( + {"deny_globs": ["**/libx264.so*"], "allow_paths": ["/usr/local/lib"]} + ) + assert ok.classify("/opt/evil/libx264.so.164") == ("violation", None) + + +def test_load_accepts_a_well_formed_allow_paths_list(tmp_path: Path): + """Paired positive case: the valid shape must still load and still allow.""" + pol = _write_policy( + tmp_path, + "deny_globs: ['**/libx264.so*']\nallow_paths:\n - /usr/local/lib\n", + ) + policy = CodecPolicy.load(pol) + assert policy.allow_paths == ["/usr/local/lib"] + assert policy.classify("/usr/local/lib/libx264.so.164") == ("allowed", None) + assert policy.classify("/opt/evil/libx264.so.164") == ("violation", None) + + +def test_wheel_renamed_codec_lib_is_denied(tmp_path: Path): + # auditwheel hash-renamed libx264-.so vendored under a wheel .libs/ must + # be caught by the hashed deny-glob, not just the plain soname. + _touch( + tmp_path, + "usr/local/lib/python3.12/dist-packages/foo.libs/libx264-abc123.so", + ) + assert main(["--root", str(tmp_path), "--fail-on-findings"]) == 1 diff --git a/container/context.yaml b/container/context.yaml index f7f57f337091..32de1c671e30 100644 --- a/container/context.yaml +++ b/container/context.yaml @@ -51,7 +51,10 @@ dynamo: enable_kvbm: "true" enable_media_ffmpeg: "false" enable_gpu_memory_service: "true" - ffmpeg_version: "8.1" + # 8.1.2 is an upstream maintenance release that picks up security fixes over + # 8.1; combined with the narrowed decoder set in wheel_builder it trims the + # media decode surface. Keep in sync with native_packages.yaml's ffmpeg entry. + ffmpeg_version: "8.1.2" # ffmpeg build inputs (only consumed when ENABLE_MEDIA_FFMPEG=true). nv_codec_headers_ref: "n13.0.19.0" libvpx_ref: "v1.14.1" @@ -108,9 +111,10 @@ sglang: # lmsysorg/sglang is built FROM this nvidia/cuda cudnn-devel base, so this # attributes everything sglang adds over the clean NVIDIA CUDA+cuDNN floor # (framework, python stack, system libs) rather than hiding it. Per-arch stem; - # the licenses stage appends -${TARGETARCH}.cdx.json. (sglang ships no system - # GPL ffmpeg/codec dpkgs; the GPL imageio-ffmpeg pip binary is replaced by an - # in-tree LGPL ffmpeg in sglang_runtime.Dockerfile — no dpkg purge needed.) + # the licenses stage appends -${TARGETARCH}.cdx.json. SGLang's codec-bearing + # Python wheels are purged in sglang_runtime.Dockerfile; only the in-tree VP9 + # ffmpeg is copied in (CUDA, for the video-generation encode path) and the + # Dynamo Rust wheel is built without media-ffmpeg. baseline_sbom: cuda@8b2705ea xpu: base_image: intel/deep-learning-essentials @@ -129,7 +133,7 @@ sglang: # NIXL Python stack — its wheel COPY is narrowed to ai_dynamo*.whl so the SDK # build doesn't leak into the runtime image. nixl_ref: v1.3.0 - enable_media_ffmpeg: "true" + enable_media_ffmpeg: "false" enable_gpu_memory_service: "true" enable_kvbm: "false" enable_modelexpress: "true" diff --git a/container/deps/requirements.sglang.txt b/container/deps/requirements.sglang.txt index e813813faf18..403c5459c3dd 100644 --- a/container/deps/requirements.sglang.txt +++ b/container/deps/requirements.sglang.txt @@ -1,14 +1,26 @@ # SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # -# Third-party Python dependencies for the sglang runtime image. Installed -# with --force-reinstall --no-deps to replace the upstream lmsysorg/sglang -# base image's imageio-ffmpeg wheel (which ships a GPL-encumbered prebuilt -# ffmpeg binary) with a source build that leaves no binary on disk. -# IMAGEIO_FFMPEG_EXE points imageio at the in-tree LGPL ffmpeg CLI. +# Third-party Python dependencies for the SGLang runtime image. Installed with +# --force-reinstall --no-deps so the upstream SGLang dependency stack remains +# otherwise unchanged. Codec-bearing video-DECODE wheels (av, decord, opencv, +# torchcodec) are removed separately in sglang_runtime.Dockerfile; the in-tree +# VP9 ffmpeg + imageio-ffmpeg below are kept for the video-generation encode +# path. +# Install imageio-ffmpeg from source so no prebuilt ffmpeg binary lands in +# /imageio_ffmpeg/binaries/. IMAGEIO_FFMPEG_EXE (set in +# sglang_runtime.Dockerfile) points imageio at the in-tree VP9 CLI copied from +# wheel_builder. The video-generation handler encodes with libvpx-vp9. --no-binary imageio-ffmpeg blake3>=1.0.0,<2.0.0 # Dynamo SGLang multimodal request handlers import blake3 at startup -imageio-ffmpeg>=0.6.0 # binary skipped per --no-binary directive at top of file +imageio-ffmpeg>=0.6.0 # binary skipped per --no-binary directive above +# NVDEC hardware decode for H.264/H.265 video input (dynamo common.multimodal. +# nvdec_decoder), restoring those codecs after the video-DECODE purge in +# sglang_runtime.Dockerfile. Pin >=2.2.0: it bundles only libavutil + libavformat +# (container demux, no software codec); older 2.0.x bundles a full FFmpeg incl. +# libavcodec that the codec gate rejects. Links libnvcuvid at runtime; requires +# NVIDIA_DRIVER_CAPABILITIES to include "video" (set in sglang_runtime.Dockerfile). +PyNvVideoCodec>=2.2.0 zstandard==0.23.0 diff --git a/container/deps/requirements.trtllm.txt b/container/deps/requirements.trtllm.txt index 9938b210a182..0fb8303ab3bd 100644 --- a/container/deps/requirements.trtllm.txt +++ b/container/deps/requirements.trtllm.txt @@ -19,5 +19,12 @@ imageio-ffmpeg>=0.6.0 # binary skipped per --no-binary directive at top of file # does not need this). msgspec>=0.19.0 # NIXL is unpinned here; trtllm_runtime.Dockerfile installs it from NIXL_REF. +# NVDEC hardware decode for H.264/H.265 video input (dynamo common.multimodal. +# nvdec_decoder). Pin >=2.2.0: it bundles only libavutil + libavformat (container +# demux, no software codec); older 2.0.x bundles a full FFmpeg incl. libavcodec +# that the codec gate rejects. Links libnvcuvid at runtime. Upstream +# tensorrt-llm/release does not bundle it. Requires NVIDIA_DRIVER_CAPABILITIES to +# include "video" (set in trtllm_runtime.Dockerfile) or it cannot import. +PyNvVideoCodec>=2.2.0 # Required by ai_dynamo_runtime (companion to msgspec above). uvloop>=0.21.0 diff --git a/container/deps/requirements.vllm.txt b/container/deps/requirements.vllm.txt index 5e596820f44d..ec620782b0a7 100644 --- a/container/deps/requirements.vllm.txt +++ b/container/deps/requirements.vllm.txt @@ -12,3 +12,10 @@ --no-binary imageio-ffmpeg imageio-ffmpeg>=0.6.0 # binary skipped per --no-binary directive at top of file +# NVDEC hardware decode for H.264/H.265 video input (common/multimodal/ +# nvdec_decoder). Pin >=2.2.0 and force-reinstall (see the vllm_runtime.Dockerfile +# install): the vllm-openai base ships PyNvVideoCodec 2.0.4, which bundles a full +# FFmpeg (libavcodec + libavdevice/libavfilter/libsw*) that the codec gate +# rejects; 2.2.0 bundles only libavutil + libavformat (container demux, no codec). +# Requires NVIDIA_DRIVER_CAPABILITIES to include "video" or it cannot import. +PyNvVideoCodec>=2.2.0 diff --git a/container/templates/args.Dockerfile b/container/templates/args.Dockerfile index 9b91e38410ad..505662d937cd 100644 --- a/container/templates/args.Dockerfile +++ b/container/templates/args.Dockerfile @@ -52,7 +52,6 @@ ARG ETCD_VERSION={{ context.dynamo.etcd_version }} ARG ENABLE_MEDIA_FFMPEG={{ context[framework].enable_media_ffmpeg }} ARG FFMPEG_VERSION={{ context.dynamo.ffmpeg_version }} -ARG NV_CODEC_HEADERS_REF={{ context.dynamo.nv_codec_headers_ref }} ARG LIBVPX_REF={{ context.dynamo.libvpx_ref }} {% if device == "cuda" -%} ARG ENABLE_GPU_MEMORY_SERVICE={{ context[framework].enable_gpu_memory_service }} diff --git a/container/templates/compliance.Dockerfile b/container/templates/compliance.Dockerfile index 382b27f4e42d..e61648775e53 100644 --- a/container/templates/compliance.Dockerfile +++ b/container/templates/compliance.Dockerfile @@ -123,6 +123,28 @@ RUN python3 -m compliance.policy.validate \ --policy /opt/compliance/policy/licenses.toml \ --input /legal/osrb-deps.csv +# Media-codec allowlist gate: scans THIS stage's filesystem (== +# the shipped image tree, since licenses is FROM pre_runtime) and fails the build +# if a media-codec library/binary (a third-party libav*, libx264/265, or a stray +# or imageio-bundled ffmpeg) ships outside our in-tree allowlist or a reasoned +# exception. Feeds the generated delta SBOM in too, for an ffmpeg-version floor. +# Files, not just the SBOM, because statically-bundled codec .so's don't appear +# as components. +# +# CUDA only for now. The XPU image is not currently published on NGC, so it does +# not yet require the codecs to be removed. It would also fail this gate today: +# the purge the gate depends on sits in the `device == "cuda"` block of +# vllm_runtime.Dockerfile, so the XPU image still carries its base image's media +# stack. Enable both together if that changes. +{% if device == "cuda" %} +RUN python3 -m compliance.scan_codecs \ + --root / \ + --policy /opt/compliance/policy/codec_policy.yaml \ + --sbom /legal/osrb.cdx.json \ + --image {{ framework }}-{{ target }} \ + --fail-on-findings -v +{% endif %} + ####################################### ####### Compliance: artifact ########## diff --git a/container/templates/sglang_runtime.Dockerfile b/container/templates/sglang_runtime.Dockerfile index feb54f991f08..05c8f6014662 100644 --- a/container/templates/sglang_runtime.Dockerfile +++ b/container/templates/sglang_runtime.Dockerfile @@ -67,25 +67,6 @@ $NIXL_PLUGIN_DIR:\ ${LD_LIBRARY_PATH:-} {% endif %} -# Copy ffmpeg from wheel_builder: versioned shared libs (libav*.so*, -# libsw*.so*) for the Rust media-ffmpeg decoder, plus the LGPL CLI binary -# (built with h264_nvenc + libvpx_vp9 encoders) that imageio targets via -# IMAGEIO_FFMPEG_EXE for video encoding. Ungated by enable_media_ffmpeg -# because the upstream lmsysorg/sglang base image always ships -# imageio-ffmpeg with a GPL-encumbered prebuilt binary that we replace -# unconditionally below; the LGPL CLI must be present so imageio has -# something to target. -RUN --mount=type=bind,from=wheel_builder,source=/usr/local/,target=/tmp/usr/local/ \ - mkdir -p /usr/local/lib/pkgconfig && \ - cp -rnL /tmp/usr/local/include/libav* /tmp/usr/local/include/libsw* /usr/local/include/ && \ - cp -nL /tmp/usr/local/lib/libav*.so* /tmp/usr/local/lib/libsw*.so* /usr/local/lib/ && \ - cp -nL /tmp/usr/local/lib/lib*vpx*.so* /usr/local/lib/ 2>/dev/null || true && \ - cp -nL /tmp/usr/local/lib/pkgconfig/libav*.pc /tmp/usr/local/lib/pkgconfig/libsw*.pc /usr/local/lib/pkgconfig/ && \ - cp -nL /tmp/usr/local/bin/ffmpeg /usr/local/bin/ffmpeg && \ - cp -r /tmp/usr/local/src/ffmpeg /usr/local/src/ && \ - ldconfig -ENV IMAGEIO_FFMPEG_EXE=/usr/local/bin/ffmpeg - {% if target not in ("dev", "local-dev") %} # Runtime target installs only the prebuilt Dynamo wheels. SGLang and its NIXL # packages come from the upstream lmsysorg/sglang runtime image; --no-deps keeps @@ -155,17 +136,111 @@ RUN --mount=type=bind,source=./container/deps/requirements.common.txt,target=/tm export PIP_CACHE_DIR=/root/.cache/pip && \ pip install --break-system-packages --no-deps $(grep -E '^nvtx==' /tmp/requirements.common.txt) -# Replace the upstream lmsysorg/sglang image's imageio-ffmpeg (which ships a -# GPL-encumbered prebuilt ffmpeg binary in /imageio_ffmpeg/binaries/) -# with a source install that leaves no binary on disk. IMAGEIO_FFMPEG_EXE points -# imageio at the LGPL CLI we copied from wheel_builder above. The --no-binary -# directive lives in the requirements file itself. +# Install SGLang-specific runtime dependencies without changing the upstream +# dependency solution. imageio-ffmpeg is installed from source (no bundled +# binary) for the VP9 video-encode path; see requirements.sglang.txt. RUN --mount=type=bind,source=./container/deps/requirements.sglang.txt,target=/tmp/requirements.sglang.txt \ --mount=type=cache,target=/root/.cache/pip,sharing=locked \ export PIP_CACHE_DIR=/root/.cache/pip && \ pip install --break-system-packages --force-reinstall --no-deps \ --requirement /tmp/requirements.sglang.txt +# Remove the codec-bearing video-DECODE components from the upstream SGLang image +# (PyAV, decord, OpenCV, torchcodec + any base ffmpeg/libav*), then copy the +# VP9-only in-tree ffmpeg from wheel_builder below for the video-generation +# encode path. H.264/H.265/AAC encoders must never appear (guarded at the end). +# +# Inkling image preprocessing uses Pillow. Its audio feature extractor imports +# soundfile and uses torchaudio only for resampling, so those paths remain +# available for formats supported by libsndfile (for example WAV and FLAC). +# AAC-backed M4A stays removed; video encode is VP9 only. +# Transitional: this exists only until the base image stops shipping these. +# The permanent statement of what may ship is +# tests/dependencies/test_no_software_video_codecs.py -- when this purge +# becomes redundant, delete it and keep those assertions. +# +# CUDA only, and it has to be: the replacement ffmpeg is copied in under +# `device == "cuda"` just below. Running the purge on XPU strips the base image's +# media stack and puts nothing back, leaving that image with no ffmpeg at all and +# an empty IMAGEIO_FFMPEG_EXE -- worse than either leaving it alone or replacing +# it properly. The XPU image is not published, so there is nothing to remove from +# it in the first place; the codec gate skips it for the same reason. Matches the +# equivalent purge in vllm_runtime.Dockerfile. +{% if device == "cuda" %} +RUN set -eux; \ + python3 -m pip uninstall --yes \ + av \ + decord \ + decord2 \ + opencv-python \ + opencv-python-headless \ + torchcodec; \ + SITE_PACKAGES="$(python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"; \ + rm -rf \ + "${SITE_PACKAGES}"/av \ + "${SITE_PACKAGES}"/av-*.dist-info \ + "${SITE_PACKAGES}"/av.libs \ + "${SITE_PACKAGES}"/cv2 \ + "${SITE_PACKAGES}"/decord \ + "${SITE_PACKAGES}"/decord-*.dist-info \ + "${SITE_PACKAGES}"/decord.libs \ + "${SITE_PACKAGES}"/decord2 \ + "${SITE_PACKAGES}"/decord2-*.dist-info \ + "${SITE_PACKAGES}"/decord2.libs \ + "${SITE_PACKAGES}"/opencv_python*.dist-info \ + "${SITE_PACKAGES}"/opencv_python*.libs \ + "${SITE_PACKAGES}"/torchcodec \ + "${SITE_PACKAGES}"/torchcodec-*.dist-info \ + /usr/local/bin/ffmpeg \ + /usr/local/bin/ffprobe \ + /usr/local/include/libav* \ + /usr/local/include/libsw* \ + /usr/local/lib/libav* \ + /usr/local/lib/libpostproc* \ + /usr/local/lib/libsw* \ + /usr/local/lib/pkgconfig/libav*.pc \ + /usr/local/lib/pkgconfig/libpostproc*.pc \ + /usr/local/lib/pkgconfig/libsw*.pc \ + /usr/local/src/ffmpeg \ + /root/.cache/pip; \ + ldconfig +{% endif %} + +{% if device == "cuda" %} +# Copy the in-tree VP9 ffmpeg from wheel_builder: versioned shared libs +# (libav*.so*, libsw*.so*) + libvpx + the in-tree CLI binary that imageio targets +# via IMAGEIO_FFMPEG_EXE. The upstream ffmpeg was purged above, so this +# VP9-only build is the only ffmpeg present; the video-generation +# handler (CUDA DiffGenerator) encodes libvpx-vp9 with it. Same pattern as +# vllm_runtime.Dockerfile. +RUN --mount=type=bind,from=wheel_builder,source=/usr/local/,target=/tmp/usr/local/ \ + mkdir -p /usr/local/lib/pkgconfig && \ + cp -rnL /tmp/usr/local/include/libav* /tmp/usr/local/include/libsw* /usr/local/include/ && \ + cp -nL /tmp/usr/local/lib/libav*.so* /tmp/usr/local/lib/libsw*.so* /usr/local/lib/ && \ + cp -nL /tmp/usr/local/lib/lib*vpx*.so* /usr/local/lib/ 2>/dev/null || true && \ + cp -nL /tmp/usr/local/lib/pkgconfig/libav*.pc /tmp/usr/local/lib/pkgconfig/libsw*.pc /usr/local/lib/pkgconfig/ && \ + cp -nL /tmp/usr/local/bin/ffmpeg /usr/local/bin/ffmpeg && \ + cp -r /tmp/usr/local/src/ffmpeg /usr/local/src/ && \ + ldconfig +ENV IMAGEIO_FFMPEG_EXE=/usr/local/bin/ffmpeg + +# Positive codec guard: the shipped ffmpeg MUST expose the VP9 encoder and MUST +# NOT expose any H.264/H.265/AAC/NVENC encoder. A missing/broken copy (no VP9) +# or a codec regression fails the build here rather than at runtime — closing the +# gap where SGLang shipping no encoder passed every PR gate. +RUN set -eu; \ + ff="${IMAGEIO_FFMPEG_EXE:-ffmpeg}"; \ + "$ff" -hide_banner -encoders 2>/dev/null | grep -qiE 'libvpx[-_]vp9' \ + || { echo "ERROR: shipped ffmpeg ($ff) has no VP9 encoder" >&2; exit 1; }; \ + if "$ff" -hide_banner -encoders 2>/dev/null \ + | grep -iE 'h\.?264|h\.?265|hevc|(^| )aac|nvenc|cuvid|nvdec'; then \ + echo "ERROR: shipped ffmpeg ($ff) exposes an H.264/H.265/AAC/NVENC encoder" >&2; \ + exit 1; \ + fi +{% else %} +ENV IMAGEIO_FFMPEG_EXE= +{% endif %} + {% if device != "xpu" and target not in ("dev", "local-dev") %} # Add generic UCX aliases beside NIXL's private libraries so native consumers # use the same implementation without breaking UCX's $ORIGIN-based core and @@ -189,6 +264,10 @@ COPY --chmod=775 --chown=dynamo:0 components/src/dynamo/frontend /workspace/comp COPY --chmod=775 --chown=dynamo:0 components/src/dynamo/sglang /workspace/components/src/dynamo/sglang COPY --chmod=775 --chown=dynamo:0 components/src/dynamo/mocker /workspace/components/src/dynamo/mocker COPY --chmod=775 --chown=dynamo:0 recipes/ /workspace/recipes/ +# The vllm and trtllm images already copy lib/; sglang did not, so its tests had +# no lib/llm/tests/data/media fixtures and any test needing a local media clip +# could only skip. ~32 MB, negligible here and it keeps the three consistent. +COPY --chown=dynamo:0 lib /workspace/lib COPY --chmod=664 --chown=dynamo:0 LICENSE /workspace/ # Enable forceful shutdown of inflight requests @@ -224,6 +303,36 @@ RUN SITE_PACKAGES="$(python3 -c 'import site; print(site.getsitepackages()[0])') (python3 -m compileall -q -j0 /sgl-workspace/sglang/python || true) {%- endif %} +# Belt-and-suspenders guard at the end of the populated runtime stage: fail the +# build if an H.264/H.265/AAC codec *implementation* library appears. The VP9 +# ffmpeg (ffmpeg + libav*/libsw* + libvpx) copied above is intentionally +# shipped for the video-encode path, so it is NOT flagged here — the in-tree +# build is VP9-only by construction (wheel_builder's post-build codec-surface +# guard) and the media-codec scan below re-permits it only under /usr/local while +# still catching any stray third-party libav*/ffmpeg. This adds the extra +# non-FFmpeg AAC/H.264 implementation names the scan's deny_globs don't list. +# +# CUDA only, for the same reason as the purge above: this asserts that the purge +# worked, and on XPU there is no purge to assert. Left ungated it would fail that +# build on the base image's own libraries -- which is precisely what happened to +# the first version of this change, where gating the purge alone turned a working +# XPU build into a failing one. +{% if device == "cuda" %} +RUN set -eux; \ + remaining="$(find /usr /opt /workspace /sgl-workspace -xdev \ + \( -type f -o -type l \) \ + \( -name 'libx264*.so*' -o -name 'libx265*.so*' \ + -o -name 'libopenh264*.so*' -o -name 'libfdk-aac*.so*' \ + -o -name 'libfaac*.so*' -o -name 'libvo-aacenc*.so*' \ + -o -name 'libaacplus*.so*' \) -print)"; \ + if [ -n "${remaining}" ]; then \ + echo "ERROR: H.264/H.265/AAC codec libraries remain in the SGLang image:" >&2; \ + echo "${remaining}" >&2; \ + exit 1; \ + fi; \ + python3 -c 'import soundfile, torchaudio; from PIL import Image' +{% endif %} + USER dynamo ARG DYNAMO_COMMIT_SHA ENV DYNAMO_COMMIT_SHA=${DYNAMO_COMMIT_SHA} @@ -250,6 +359,10 @@ CMD [] ####################################### FROM pre_runtime AS runtime +# NVDEC (PyNvVideoCodec, added in requirements.sglang.txt) needs the driver +# "video" capability at runtime so libnvcuvid is exposed; without it +# PyNvVideoCodec cannot import. Ensure the K8s pod/runtimeClass does not drop it. +ENV NVIDIA_DRIVER_CAPABILITIES=video,compute,utility {% if target not in ("dev", "local-dev") %} COPY --from=licenses /legal /legal {% endif %} diff --git a/container/templates/trtllm_runtime.Dockerfile b/container/templates/trtllm_runtime.Dockerfile index 4cf65c3696be..d11a6148379d 100644 --- a/container/templates/trtllm_runtime.Dockerfile +++ b/container/templates/trtllm_runtime.Dockerfile @@ -167,6 +167,97 @@ RUN --mount=type=bind,from=wheel_builder,source=/usr/local/,target=/tmp/usr/loca ldconfig ENV IMAGEIO_FFMPEG_EXE=/usr/local/bin/ffmpeg +# Positive codec guard: the shipped ffmpeg MUST expose the VP9 encoder and MUST +# NOT expose any H.264/H.265/AAC/NVENC encoder. A missing/broken copy (no VP9) +# or a codec regression fails the build here rather than at runtime — closing the +# gap where an image with no working encoder passed every PR gate. +RUN set -eu; \ + ff="${IMAGEIO_FFMPEG_EXE:-ffmpeg}"; \ + "$ff" -hide_banner -encoders 2>/dev/null | grep -qiE 'libvpx[-_]vp9' \ + || { echo "ERROR: shipped ffmpeg ($ff) has no VP9 encoder" >&2; exit 1; }; \ + if "$ff" -hide_banner -encoders 2>/dev/null \ + | grep -iE 'h\.?264|h\.?265|hevc|(^| )aac|nvenc|cuvid|nvdec'; then \ + echo "ERROR: shipped ffmpeg ($ff) exposes an H.264/H.265/AAC/NVENC encoder" >&2; \ + exit 1; \ + fi + +# Drop upstream's preinstalled opencv-python-headless (and the libav*/ffmpeg +# copies its wheel vendors under opencv_python_headless.libs/). TRT-LLM made +# cv2 an optional video extra (NVIDIA/TensorRT-LLM#16206) and its imports are +# already function-local in 1.3.x, so nothing in the image needs it at import +# time; video-decode users can install it explicitly per upstream docs. +# Must run via the system interpreter: with VIRTUAL_ENV set, plain `pip` +# targets the venv and exits 0 WITHOUT touching the system-site install. +# The guards fail the build if the package survives. Mirrored by the +# pre_runtime whiteout below — the squash COPY cannot represent deletions. +RUN /usr/bin/python3 -m pip uninstall -y --break-system-packages opencv-python-headless && \ + ! /usr/bin/python3 -c "import cv2" 2>/dev/null && \ + [ ! -e /usr/local/lib/python3.12/dist-packages/opencv_python_headless.libs ] + +# Upgrade DALI past its own media-codec cleanup. Upstream restricted DALI's +# vendored ffmpeg build to drop the software h264/hevc/aac decoders +# (NVIDIA/DALI_deps#162, NVIDIA/DALI#6352), first released in 2.1.1; the base +# image here carries 2.1.0, which predates it. +# +# Upgrade rather than remove. The vendored libav*/libsw* set cannot be trimmed +# on its own -- every one of them is DT_NEEDED by libdali.so and its siblings, so +# deleting the codec libraries breaks `import nvidia.dali` outright -- and while +# nothing in this image imports DALI today, it belongs to TensorRT-LLM rather +# than to us. A version bump removes the codec surface without taking a package +# out of someone else's image. +# +# Measured on the shipped image: 2.1.0 registers 446 decoders including h264, +# hevc, aac, aac_fixed and aac_latm; 2.1.1 and 2.2.0 each register 440 with all +# five absent and vp8/vp9/mjpeg/av1 retained. +# +# Pinned exactly, not a floor. `>=2.1.1` resolves to whatever is newest -- 2.2.0 +# at the time of writing -- which is a minor-version jump into a component this +# repo does not own, taken silently at build time. 2.1.1 is the smallest change +# that removes the decoders, so it is the one to take. +# +# A pin can go stale in the one direction that matters: if TensorRT-LLM later +# ships a base image with a newer DALI, installing 2.1.1 over it is a downgrade. +# The check below turns that into a build failure with instructions rather than a +# silent regression, which also makes this block self-retiring -- when upstream +# catches up, the build tells whoever is looking to delete it. +# +# THIS BLOCK IS TRANSITIONAL. It exists because the base image predates upstream's +# own cleanup. The guards after it and the bundled-libavcodec assertion in +# tests/dependencies/test_no_software_video_codecs.py are NOT transitional: they +# are the permanent statement of what this image may contain, and they must +# outlive the workaround. +# +# System interpreter for the same reason as the opencv removal above: with +# VIRTUAL_ENV set, plain pip targets the venv and leaves the system-site copy in +# place. The guard enumerates what the library actually registers rather than +# trusting the version string, so a wheel that reintroduces a decoder fails the +# build. Mirrored by the pre_runtime whiteout below, which matters more here than +# for a deletion: DALI's libraries are hash-named, so an upgrade RENAMES them and +# the squash COPY would otherwise leave the old codec-carrying copy in place +# beside the new one. +RUN --mount=type=bind,source=./container/compliance/enumerate_bundled_decoders.py,target=/tmp/enumerate_bundled_decoders.py \ + set -eu; \ + before=$(/usr/bin/python3 -c 'import importlib.metadata as m; print(m.version("nvidia-dali-cuda130"))'); \ + echo "DALI in base image: $before"; \ + newest=$(printf '%s\n2.1.1\n' "$before" | sort -V | tail -1); \ + if [ "$newest" != "2.1.1" ]; then \ + echo "ERROR: base image already carries DALI $before, so pinning 2.1.1 would" >&2; \ + echo " downgrade it. Upstream has caught up -- delete this RUN and let" >&2; \ + echo " the base version stand. Keep the guards below and the bundled-" >&2; \ + echo " libavcodec assertion in tests/dependencies/, which are what stop" >&2; \ + echo " this from regressing." >&2; \ + exit 1; \ + fi; \ + /usr/bin/python3 -m pip install --break-system-packages --no-cache-dir \ + --extra-index-url https://pypi.nvidia.com 'nvidia-dali-cuda130==2.1.1'; \ + v=$(/usr/bin/python3 -c 'import importlib.metadata as m; print(m.version("nvidia-dali-cuda130"))'); \ + echo "DALI version: $v"; \ + [ "$v" = "2.1.1" ] || { echo "ERROR: wanted DALI 2.1.1, got $v" >&2; exit 1; }; \ + for lib in $(find /usr/local/lib/python3.12/dist-packages/nvidia/dali/.libs -name 'libavcodec*.so*'); do \ + /usr/bin/python3 /tmp/enumerate_bundled_decoders.py "$lib"; \ + done; \ + /usr/bin/python3 -c 'import nvidia.dali' + # Pull /workspace_src (incl. LICENSE) from the transport stage and # wire up the launch screen in a single RUN — saves the standalone workspace COPY layer. RUN --mount=type=bind,from=workspace_files,source=/workspace_src,target=/tmp/workspace_src \ @@ -194,11 +285,42 @@ CMD ["/bin/bash"] # single layer. Only Dynamo-specific env needs redeclaring below. FROM ${RUNTIME_IMAGE}:${RUNTIME_IMAGE_TAG} AS pre_runtime # Whiteout paths runtime_full removed — COPY can't represent deletions, so -# without this, upstream's /workspace, /home/ubuntu, and single-file -# /usr/local/bin/etcd would leak alongside our content. -RUN rm -rf /workspace /home/ubuntu /usr/local/bin/etcd +# without this, upstream's /workspace, /home/ubuntu, single-file +# /usr/local/bin/etcd, and preinstalled opencv (cv2/ + vendored +# opencv_python_headless.libs/ + dist-info) would leak alongside our content. +# Keep this list in sync with any deletion RUNs in the stages above. +# +# DALI is here for a different reason: runtime_full UPGRADES it rather than +# removing it, and its vendored libraries are hash-named +# (libavcodec-847b0373.so.62 -> libavcodec-7e11e4e3.so.62). An overlay COPY only +# replaces paths that match, so without dropping the old tree first, the base +# image's 2.1.0 libraries — the ones carrying h264/hevc/aac — would ship +# alongside the upgraded ones and the upgrade would buy nothing. +RUN rm -rf /workspace /home/ubuntu /usr/local/bin/etcd \ + /usr/local/lib/python3.12/dist-packages/cv2 \ + /usr/local/lib/python3.12/dist-packages/opencv_python_headless* \ + /usr/local/lib/python3.12/dist-packages/nvidia/dali \ + /usr/local/lib/python3.12/dist-packages/nvidia_dali_* && \ + ! /usr/bin/python3 -c "import cv2" 2>/dev/null COPY --from=runtime_full / / +# Post-overlay guard for the DALI whiteout above. This is the only stage where +# the failure can appear: runtime_full holds exactly one DALI, and the whiteout +# is what keeps the overlay from re-adding the base image's copy beside it. +# Because those libraries are hash-named, a missed whiteout leaves TWO -- the +# upgraded one and the 2.1.0 one that registers h264/hevc/aac -- and the image +# then reports version 2.1.0 while carrying both. Verified by building this +# stage with and without the rm -rf: without it, two copies and the codecs are +# back; with it, one copy and clean. +# +# Every match is checked, not just the first: the whole point is catching the +# case where more than one exists. +RUN --mount=type=bind,source=./container/compliance/enumerate_bundled_decoders.py,target=/tmp/enumerate_bundled_decoders.py \ + set -eu; \ + for lib in $(find /usr/local/lib/python3.12/dist-packages/nvidia/dali/.libs -name 'libavcodec*.so*' 2>/dev/null); do \ + /usr/bin/python3 /tmp/enumerate_bundled_decoders.py "$lib"; \ + done + # Mirrors runtime_full's ENV — must stay in sync. Re-declaration is required # because `FROM ${RUNTIME_IMAGE}` here does not inherit runtime_full's config. # dev/local-dev create their own venv in a later stage, so the venv ENV is left @@ -243,6 +365,10 @@ CMD ["/bin/bash"] FROM pre_runtime AS runtime +# NVDEC (PyNvVideoCodec, added in requirements.trtllm.txt) needs the driver +# "video" capability at runtime so libnvcuvid is exposed; without it +# PyNvVideoCodec cannot import. Ensure the K8s pod/runtimeClass does not drop it. +ENV NVIDIA_DRIVER_CAPABILITIES=video,compute,utility {% if target not in ("dev", "local-dev") %} COPY --from=licenses /legal /legal {% endif %} diff --git a/container/templates/vllm_runtime.Dockerfile b/container/templates/vllm_runtime.Dockerfile index 52dd34fe9361..9bedc272a811 100644 --- a/container/templates/vllm_runtime.Dockerfile +++ b/container/templates/vllm_runtime.Dockerfile @@ -287,6 +287,20 @@ RUN --mount=type=bind,from=wheel_builder,source=/usr/local/,target=/tmp/usr/loca cp -r /tmp/usr/local/src/ffmpeg /usr/local/src/ && \ ldconfig ENV IMAGEIO_FFMPEG_EXE=/usr/local/bin/ffmpeg + +# Positive codec guard: the shipped ffmpeg MUST expose the VP9 encoder and MUST +# NOT expose any H.264/H.265/AAC/NVENC encoder. A missing/broken copy (no VP9) +# or a codec regression fails the build here rather than at runtime — closing the +# gap where an image with no working encoder passed every PR gate. +RUN set -eu; \ + ff="${IMAGEIO_FFMPEG_EXE:-ffmpeg}"; \ + "$ff" -hide_banner -encoders 2>/dev/null | grep -qiE 'libvpx[-_]vp9' \ + || { echo "ERROR: shipped ffmpeg ($ff) has no VP9 encoder" >&2; exit 1; }; \ + if "$ff" -hide_banner -encoders 2>/dev/null \ + | grep -iE 'h\.?264|h\.?265|hevc|(^| )aac|nvenc|cuvid|nvdec'; then \ + echo "ERROR: shipped ffmpeg ($ff) exposes an H.264/H.265/AAC/NVENC encoder" >&2; \ + exit 1; \ + fi {% endif %} # Replace the upstream vllm/vllm-openai image's imageio-ffmpeg (which ships a @@ -294,17 +308,84 @@ ENV IMAGEIO_FFMPEG_EXE=/usr/local/bin/ffmpeg # with a source install that leaves no binary on disk. On cuda, IMAGEIO_FFMPEG_EXE # (set above) points imageio at the LGPL CLI copied from wheel_builder. The # --no-binary directive lives in the requirements file itself. +{% if device == "cuda" %} RUN --mount=type=bind,source=./container/deps/requirements.vllm.txt,target=/tmp/requirements.vllm.txt \ --mount=type=cache,id=uv-root-{{ context.dynamo.uv_version }},target=/root/.cache/uv,sharing=locked \ export UV_CACHE_DIR=/root/.cache/uv && \ + uv pip install {{ pip_target }} \ + --reinstall-package imageio-ffmpeg --reinstall-package PyNvVideoCodec \ + --no-deps --requirement /tmp/requirements.vllm.txt +{% else %} +# PyNvVideoCodec decodes on NVDEC through libnvcuvid, so it is inert on a +# non-NVIDIA device. Drop it from the shared requirements rather than ship an +# unusable NVIDIA codec wheel in, for example, the Intel XPU image. The pattern +# is anchored to the line start so it cannot match inside another requirement, +# and the import check fails the build if the package arrives by another route +# -- a filter that silently stopped matching would otherwise look like success. +# +# Whole-RUN branches, rather than a conditional inside one RUN: a `{% raw %}{% if %}{% endraw %}` in the +# middle of a `\`-continued command emits a blank line that ends the command +# early, and a `#` comment there is joined onto the previous line, commenting +# out the rest. Both break the build in ways the template does not show. +RUN --mount=type=bind,source=./container/deps/requirements.vllm.txt,target=/tmp/requirements.vllm.txt \ + --mount=type=cache,id=uv-root-{{ context.dynamo.uv_version }},target=/root/.cache/uv,sharing=locked \ + export UV_CACHE_DIR=/root/.cache/uv && \ + grep -v '^PyNvVideoCodec' /tmp/requirements.vllm.txt > /tmp/requirements.vllm.nonvidia.txt && \ uv pip install {{ pip_target }} --reinstall-package imageio-ffmpeg --no-deps \ - --requirement /tmp/requirements.vllm.txt + --requirement /tmp/requirements.vllm.nonvidia.txt && \ + ! /opt/venv/bin/python -c "import PyNvVideoCodec" 2>/dev/null +{% endif %} # Remove the vLLM source tree shipped in the base image to avoid pytest # collection conflicts (duplicate conftest plugin registration) and stale # tool scripts referencing files not present in Dynamo's build context. RUN rm -rf /workspace/vllm +{% if device == "cuda" %} +# Transitional: this exists only until the base image stops shipping these. +# The permanent statement of what may ship is +# tests/dependencies/test_no_software_video_codecs.py -- when this purge +# becomes redundant, delete it and keep those assertions. +# Remove the codec-bearing video-DECODE wheels inherited from the vllm-openai +# base. Each bundles its own full ffmpeg carrying software H.264/H.265/AAC; +# PyAV and decord additionally ship GPL libx264/libx265. Dynamo's vLLM component +# imports none of the removed wheels, so they are unused decode-side dead weight. +# (PyNvVideoCodec is KEPT for NVDEC hardware decode -- see the note below.) The in-tree +# LGPL ffmpeg + imageio-ffmpeg installed above are intentionally KEPT for the +# omni video-encode path, which uses the royalty-free VP9 (libvpx_vp9) encoder — +# no H.264 is built. Direct rm makes the removal robust regardless of how the +# base image's pip is configured; the guards fail the build if any of them survive. +RUN set -eux; \ + python3 -m pip uninstall --yes \ + av decord decord2 opencv-python opencv-python-headless torchcodec \ + || true; \ + SITE_PACKAGES="$(python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"; \ + rm -rf \ + "${SITE_PACKAGES}"/av "${SITE_PACKAGES}"/av-*.dist-info "${SITE_PACKAGES}"/av.libs \ + "${SITE_PACKAGES}"/cv2 "${SITE_PACKAGES}"/opencv_python*.dist-info "${SITE_PACKAGES}"/opencv_python*.libs \ + "${SITE_PACKAGES}"/decord "${SITE_PACKAGES}"/decord-*.dist-info "${SITE_PACKAGES}"/decord.libs \ + "${SITE_PACKAGES}"/decord2 "${SITE_PACKAGES}"/decord2-*.dist-info "${SITE_PACKAGES}"/decord2.libs \ + "${SITE_PACKAGES}"/torchcodec "${SITE_PACKAGES}"/torchcodec-*.dist-info \ + /root/.cache/pip; \ + # Guard EVERY purged wheel: the uninstall above is `|| true`, so a package + # that survived (rename, new bundling) would otherwise pass silently. decord2 + # imports as `decord`, so both are covered by the one check. + ! python3 -c "import cv2" 2>/dev/null; \ + ! python3 -c "import av" 2>/dev/null; \ + ! python3 -c "import decord" 2>/dev/null; \ + ! python3 -c "import torchcodec" 2>/dev/null + +# PyNvVideoCodec is KEPT (removed from the purge above) but UPGRADED to >=2.2.0 by +# the requirements install: the base image's 2.0.4 bundles a full FFmpeg (incl. +# libavcodec) that the codec gate rejects, while 2.2.0 bundles only libavutil + +# libavformat (container demux, no software codec). It provides the built-in +# H.264/H.265 video-input path (NVDEC hardware decode via libnvcuvid; +# common/multimodal/nvdec_decoder.py) and requires the driver "video" capability +# at runtime or it cannot import; set it in the image and ensure the K8s +# pod/runtimeClass does not drop it. +ENV NVIDIA_DRIVER_CAPABILITIES=video,compute,utility +{% endif %} + USER dynamo # Copy the workspace surface needed by the current vLLM pre-merge test image. diff --git a/container/templates/wheel_builder.Dockerfile b/container/templates/wheel_builder.Dockerfile index 7ea02cbf9aa6..4f8a752ba87a 100644 --- a/container/templates/wheel_builder.Dockerfile +++ b/container/templates/wheel_builder.Dockerfile @@ -292,15 +292,38 @@ RUN mkdir -p /tmp/native-sources ENV SCCACHE_BUCKET=${USE_SCCACHE:+${SCCACHE_BUCKET}} \ SCCACHE_REGION=${USE_SCCACHE:+${SCCACHE_REGION}} -# Always build FFmpeg so libs are available for Rust checks in CI. -# We also build the ffmpeg CLI with h264_nvenc + libvpx_vp9 encoders so Python -# code can encode video without the GPL-licensed binary shipped by imageio-ffmpeg. -# Stays LGPL-only: --disable-gpl --disable-nonfree are preserved; H.264 comes from -# NVIDIA's NVENC (proprietary HW encoder, already a runtime dependency of these -# GPU images) and VP9 from libvpx (BSD). +# Build FFmpeg for every framework's video-encode path, SGLang included. The +# build is VP9-only (libvpx) — it contains no H.264, H.265, or AAC encoder in +# any form — so SGLang's video-generation handler gets a VP9 encoder to write +# with, matching vLLM/TRT-LLM. +# Build FFmpeg so libs are available for Rust checks in CI. +# We build the ffmpeg CLI with the libvpx_vp9 encoder so Python code can encode +# video without the GPL-licensed binary shipped by imageio-ffmpeg. +# Stays LGPL-only AND royalty-free: --disable-gpl --disable-nonfree are preserved, +# and no H.264/H.265/AAC codec is built in any form. Video encode is VP9 only +# (libvpx, BSD). NVENC is intentionally NOT enabled: the hardware H.264 encoder is +# still a distributable H.264 codec surface, so it is omitted entirely — see the +# post-build guard below that fails the build if any H.264 surface reappears. +# +# MEDIA CODEC ALLOWLIST: the in-tree libavcodec should carry only +# the media formats we actually build and use, not ffmpeg's full default decoder +# set. A blanket --disable-decoders/--disable-demuxers/--disable-parsers plus a +# narrow allowlist keeps the shipped libav*.so limited to that set. The allowlist +# covers exactly two paths: (1) the encode CLI ingesting rawvideo frames from +# imageio over a pipe and encoding with libvpx_vp9, and (2) the Rust media-ffmpeg +# VideoDecoder decoding VP8/VP9 in mp4/webm/mkv (test fixtures are VP9-in-mp4). +# No H.264 parser/decoder/encoder is enabled — H.264 is not built at all. Image +# decode does not use ffmpeg (it goes through the Rust `image` crate), so no +# still-image decoders are enabled here. +# The `fd` protocol is enabled alongside `pipe`: `ffmpeg -i -` reads stdin via +# the `fd:` protocol on ffmpeg 8.x (not `pipe:`), so omitting it breaks the +# imageio encode path with "Protocol not found. Did you mean file:fd:?". Both +# are pure fd/stream I/O and carry no codec implementation. +# +# Combined with the 8.1 -> 8.1.2 bump below (an upstream maintenance release), +# this also trims the decoder surface to what we ship. # Do not delete the source tarball for legal reasons. ARG FFMPEG_VERSION -ARG NV_CODEC_HEADERS_REF ARG LIBVPX_REF RUN --mount=type=secret,id=aws-web-identity-token,target=/run/secrets/aws-token \ --mount=type=secret,id=aws-role-arn,env=AWS_ROLE_ARN \ @@ -315,12 +338,10 @@ RUN --mount=type=secret,id=aws-web-identity-token,target=/run/secrets/aws-token elif [ "$DEVICE" = "cuda" ]; then \ dnf install -y --setopt=tsflags=nocontexts pkg-config xz git yasm; \ fi && \ - # nv-codec-headers: provides the NVENC/NVDEC API headers ffmpeg compiles against. - # Header-only, no runtime dep here; libcuda/libnvidia-encode are loaded at runtime - # in the consuming container. + # No nv-codec-headers: NVENC/NVDEC are not built, so the NVIDIA codec API + # headers are not needed. This keeps H.264 (incl. the h264_nvenc HW encoder) + # out of the in-tree ffmpeg entirely. cd /tmp && \ - git clone --depth 1 --branch ${NV_CODEC_HEADERS_REF} https://github.com/FFmpeg/nv-codec-headers.git && \ - make -C nv-codec-headers PREFIX=/usr/local install && \ # libvpx: BSD-licensed VP9 encoder needed for the WebM output path. Built from # source so we don't need to track distro package names (libvpx-dev on Debian # vs libvpx-devel via EPEL on RHEL/manylinux). @@ -331,7 +352,18 @@ RUN --mount=type=secret,id=aws-web-identity-token,target=/run/secrets/aws-token make install && \ ldconfig && \ cd /tmp && \ - curl --retry 5 --retry-delay 3 -LO https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.xz && \ + # Retry the ffmpeg fetch in a shell loop: curl's own --retry does not cover + # SSL/connection-level failures (e.g. `curl: (35) SSL_ERROR_SYSCALL`) unless + # --retry-all-errors is used, which needs curl >= 7.71 (the manylinux build + # base ships 7.61). The loop retries on ANY failure and is version-agnostic. + for attempt in 1 2 3 4 5; do \ + curl --retry 3 --retry-delay 5 --retry-connrefused --connect-timeout 30 -fL \ + -o ffmpeg-${FFMPEG_VERSION}.tar.xz \ + https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.xz && break; \ + echo "ffmpeg download attempt ${attempt}/5 failed; retrying in 10s" >&2; \ + sleep 10; \ + done && \ + test -s ffmpeg-${FFMPEG_VERSION}.tar.xz && \ tar xf ffmpeg-${FFMPEG_VERSION}.tar.xz && \ cd ffmpeg-${FFMPEG_VERSION} && \ ./configure \ @@ -346,15 +378,37 @@ RUN --mount=type=secret,id=aws-web-identity-token,target=/run/secrets/aws-token --disable-devices \ --disable-libdrm \ --enable-shared \ - --enable-nvenc \ --enable-libvpx \ --disable-encoders \ - --enable-encoder=h264_nvenc,libvpx_vp9 \ + --enable-encoder=libvpx_vp9 \ + --disable-decoders \ + --enable-decoder=vp8,vp9,rawvideo \ --disable-muxers \ --enable-muxer=mov,mp4,matroska,webm \ - --enable-protocol=file,pipe && \ + --disable-demuxers \ + --enable-demuxer=mov,matroska,rawvideo \ + --disable-parsers \ + --enable-parser=vp8,vp9 \ + --disable-protocols \ + --enable-protocol=file,pipe,fd && \ make -j$(nproc) && \ make install && \ + # Compliance guard: fail the build if any royalty-bearing / HW codec surface + # leaked into the in-tree ffmpeg. By construction this build is VP9-only, so a + # match here means a config regression. Check the implementation-carrying + # surfaces (encoders/decoders/parsers), not -codecs (lists names even when no + # implementation is built) and not -bsfs: bitstream filters (e.g. + # aac_adtstoasc, h264_mp4toannexb) only reframe an already-encoded stream, are + # pulled in as mov/mp4 muxer dependencies, and carry no codec implementation. + for surface in encoders decoders parsers; do \ + if /usr/local/bin/ffmpeg -hide_banner "-${surface}" 2>/dev/null \ + | grep -qiE 'h\.?264|h\.?265|hevc|(^| )aac|nvenc|cuvid|nvdec'; then \ + echo "ERROR: in-tree ffmpeg exposes a disallowed codec via -${surface}" >&2; \ + /usr/local/bin/ffmpeg -hide_banner "-${surface}" 2>/dev/null \ + | grep -iE 'h\.?264|h\.?265|hevc|(^| )aac|nvenc|cuvid|nvdec' >&2; \ + exit 1; \ + fi; \ + done && \ /tmp/use-sccache.sh show-stats "FFMPEG" && \ ldconfig && \ mkdir -p /usr/local/src/ffmpeg && \ @@ -506,7 +560,9 @@ COPY components/ /opt/dynamo/components/ # Build ai-dynamo (pure Python) and ai-dynamo-runtime (maturin) wheels ARG USE_SCCACHE +{% if framework != "sglang" %} ARG ENABLE_MEDIA_FFMPEG +{% endif %} RUN --mount=type=secret,id=aws-web-identity-token,target=/run/secrets/aws-token \ --mount=type=secret,id=aws-role-arn,env=AWS_ROLE_ARN \ --mount=type=cache,target=/root/.cargo/registry,sharing=shared \ @@ -523,12 +579,13 @@ RUN --mount=type=secret,id=aws-web-identity-token,target=/run/secrets/aws-token cd /opt/dynamo && \ uv build --wheel --out-dir /opt/dynamo/dist && \ cd /opt/dynamo/lib/bindings/python && \ - if [ "$ENABLE_MEDIA_FFMPEG" = "true" ]; then \ +{% if framework == "sglang" %} maturin build --release --features "kv-indexer,slot-tracker,select-service,mm-routing,aic-forward-pass,request-trace-s3{% if target == "planner" %},mocker-kvbm-offload{% endif %}" --out /opt/dynamo/dist && \ +{% else %} if [ "$ENABLE_MEDIA_FFMPEG" = "true" ]; then \ maturin build --release --features "media-ffmpeg,kv-indexer,slot-tracker,select-service,mm-routing,aic-forward-pass,request-trace-s3{% if target == "planner" %},mocker-kvbm-offload{% endif %}" --out /opt/dynamo/dist; \ else \ maturin build --release --features "kv-indexer,slot-tracker,select-service,mm-routing,aic-forward-pass,request-trace-s3{% if target == "planner" %},mocker-kvbm-offload{% endif %}" --out /opt/dynamo/dist; \ fi && \ - /tmp/use-sccache.sh show-stats "Dynamo Runtime" +{% endif %} /tmp/use-sccache.sh show-stats "Dynamo Runtime" {% if target == "planner" %} # AI Simulate is a separate Python distribution. Build it only for the planner diff --git a/docs/fern/index.yml b/docs/fern/index.yml index 49afa031a76a..4f6c48799c14 100644 --- a/docs/fern/index.yml +++ b/docs/fern/index.yml @@ -377,6 +377,9 @@ navigation: path: pages/use-cases/multimodal-serving/multimodal-kv-routing.md - page: Encoder Disaggregation path: pages/use-cases/multimodal-serving/encoder-disaggregation.md + - page: Video Decode GPU Requirements + slug: video-decode-gpu-requirements + path: pages/use-cases/multimodal-serving/video-decode-gpu-requirements.md - section: Diffusion collapsed: true icon: wand-magic-sparkles diff --git a/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/multimodal.md b/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/multimodal.md index 309ff4751fe1..2801449f495b 100644 --- a/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/multimodal.md +++ b/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/multimodal.md @@ -12,9 +12,32 @@ This document provides a comprehensive guide for multimodal inference using SGLa |----------|--------------|------------|---------------|-------| | **Image** | HTTP/HTTPS URL | Yes | Yes | Vision encoder generates embeddings | | **Image** | Data URL (Base64) | No | No | | -| **Video** | HTTP/HTTPS/`file://` URL | Yes | Yes | Vision encoder generates embeddings | +| **Video** | HTTP/HTTPS, `file://`, `data:` | No | Yes, H.264/H.265, Qwen2-family only | Needs the encode worker; decoded on NVDEC, then the vision encoder produces embeddings | | **Audio** | HTTP/HTTPS URL | No | No | Not supported in SGLang backend | +> [!IMPORTANT] +> **Video input is limited to H.264 and H.265, and requires a separate encode +> worker.** The runtime image ships no software video decoder, so H.264 and H.265 +> video is decoded on the GPU by NVDEC. Video in any other format (VP8, VP9, AV1) +> cannot be decoded at all. +> +> "Encode worker" here means Dynamo's `--disaggregation-mode encode` component, +> which runs the model's vision encoder to turn frames into embeddings. It does +> not encode video — nothing in this path produces a video stream. Hardware +> decode is wired into that worker (as used by `multimodal_epd.sh`). In an +> aggregated deployment SGLang resolves and decodes the media URL itself, so +> Dynamo never sees the bytes and cannot route them to NVDEC — video input is +> therefore unavailable in aggregated deployments of this image. +> +> Video is also skipped for model types whose preprocessing cannot accept +> pre-decoded frames (the Qwen3-VL family); those requests fall back to the URL +> path, which has no decoder in this image. Use a Qwen2-family vision model. +> +> NVDEC requires a GPU with a video decode engine and a container granted the +> `video` driver capability — see +> [Video Decode GPU Requirements](../../../../../use-cases/multimodal-serving/video-decode-gpu-requirements.md). +> `file://` sources additionally require `DYN_MM_LOCAL_PATH` to permit local reads. + ### Supported URL Formats | Format | Example | Description | diff --git a/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/tensorrt-llm/multimodal.md b/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/tensorrt-llm/multimodal.md index ba795398f101..9056e1c9161f 100644 --- a/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/tensorrt-llm/multimodal.md +++ b/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/tensorrt-llm/multimodal.md @@ -18,7 +18,7 @@ You can provide multimodal inputs in the following ways: |----------|--------------|------------|---------------|-------| | **Image** | HTTP/HTTPS URL | Yes | Yes | Full support for all image models | | **Image** | Pre-computed Embeddings (.safetensors) | Yes | Yes | Direct embedding files | -| **Video** | HTTP/HTTPS URL | No | No | Not implemented | +| **Video** | HTTP/HTTPS URL | Yes | Yes | Decoded per request; H.264/H.265 on NVDEC, other codecs via the TensorRT-LLM loader | | **Audio** | HTTP/HTTPS URL | No | No | Not implemented | ### Supported URL Formats @@ -434,8 +434,11 @@ await register_model( ## Known Limitations -- **No video support** - No video encoder implementation - **No audio support** - No audio encoder implementation +- **Video decode requires NVDEC** - H.264/H.265 video is decoded on the GPU; the + runtime image ships no software video decoder. This needs a GPU with a video + decode engine and a container granted the `video` driver capability — see + [Video Decode GPU Requirements](../../../../../use-cases/multimodal-serving/video-decode-gpu-requirements.md). - **Multimodal preprocessing/tokenization happens in Python** - Rust may forward token_ids, but multimodal requests are parsed and re-tokenized in the Python worker - **Multi-node H100 limitation** - Loading `meta-llama/Llama-4-Maverick-17B-128E-Instruct` with 8 nodes of H100 with TP=16 is not possible due to head count divisibility (`num_attention_heads: 40` not divisible by `tp_size: 16`) - **llava-v1.6-mistral-7b-hf model crash** - Known issue with TRTLLM backend compatibility with `TensorRT LLM version: 1.2.0rc6.post1`. To use Llava model download revision `revision='52320fb52229` locally using HF. diff --git a/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/vllm/multimodal.md b/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/vllm/multimodal.md index d4b3f50025ba..8e1e0a656a76 100644 --- a/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/vllm/multimodal.md +++ b/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/vllm/multimodal.md @@ -20,9 +20,18 @@ This document provides a comprehensive guide for multimodal inference using the | Modality | Aggregated | P/D | Separate encode worker | | --- | --- | --- | --- | | **Image** | Yes | Yes | Legacy entry point only | -| **Video** | Yes | Yes | Processed by the language-model worker | +| **Video** | Yes, H.264/H.265 | Yes, H.264/H.265 | Processed by the language-model worker | | **Audio** | Yes | Yes, with decode reload | Not routed to the separate encoder | +> [!IMPORTANT] +> **Video input is limited to H.264 and H.265.** The runtime images ship no software +> video decoder, so these codecs are decoded on the GPU by NVDEC and no other codec +> (VP8, VP9, AV1) has a decoder available. NVDEC decode requires a GPU with a video +> decode engine and a container granted the `video` driver capability — see +> [Video Decode GPU Requirements](../../../../../use-cases/multimodal-serving/video-decode-gpu-requirements.md). +> `http`, `https`, `file://` and `data:` sources are all hardware-decoded; `file://` +> additionally requires `DYN_MM_LOCAL_PATH` to permit local reads. + ### Supported URL Formats | Format | Example | Description | diff --git a/docs/fern/pages/use-cases/diffusion/overview.md b/docs/fern/pages/use-cases/diffusion/overview.md index 989b43373275..f3e0f3234263 100644 --- a/docs/fern/pages/use-cases/diffusion/overview.md +++ b/docs/fern/pages/use-cases/diffusion/overview.md @@ -77,7 +77,7 @@ The built-in backends expose OpenAI-compatible endpoints for images (`/v1/images **Supports:** Text-to-image and text-to-video. - **Main limitation:** Not recommended for production. Video output requires an NVENC-capable NVIDIA GPU. + **Main limitation:** Not recommended for production. Kubernetes @@ -165,7 +165,14 @@ The built-in backends expose OpenAI-compatible endpoints for images (`/v1/images - Text-to-video output requires `imageio`, ffmpeg, and an NVENC-capable NVIDIA GPU. Dynamo's TensorRT-LLM runtime image includes an ffmpeg build with `h264_nvenc`. + Text-to-video output requires `imageio` and ffmpeg. Dynamo's runtime images include an ffmpeg build that encodes VP9 (`libvpx-vp9`), so `.mp4` output is a VP9 stream in an MP4 container and no GPU encoder is needed. + + > [!IMPORTANT] + > **Changed:** `output_format: "mp4"` previously returned H.264 and now returns VP9. The file extension is unchanged, so a client that passes the bytes straight to a player may fail without an error from Dynamo. VP9-in-MP4 plays in Chrome, Firefox and ffmpeg, but not in Safari or QuickTime. + > + > VP9 is used on every GPU because NVIDIA omits the hardware H.264 encoder (NVENC) from the datacenter line — A100, H100, H200, B200 and GB200 have none — so a hardware H.264 path would work only on L4/L40S/RTX-class parts. The alternative, a software H.264 encoder, is what the codec-compliant images deliberately exclude. + > + > To get H.264, transcode on the client: `ffmpeg -i output.mp4 -c:v libx264 output-h264.mp4`. Outside the container, install the Python wrapper without its bundled binary and point it to your ffmpeg: diff --git a/docs/fern/pages/use-cases/multimodal-serving/video-decode-gpu-requirements.md b/docs/fern/pages/use-cases/multimodal-serving/video-decode-gpu-requirements.md new file mode 100644 index 000000000000..2f118cce3522 --- /dev/null +++ b/docs/fern/pages/use-cases/multimodal-serving/video-decode-gpu-requirements.md @@ -0,0 +1,168 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +title: Video Decode GPU Requirements +--- + +Dynamo decodes H.264 and H.265 (HEVC) video input on the GPU using NVDEC, NVIDIA's +dedicated hardware video decoder, through +[PyNvVideoCodec](https://pypi.org/project/PyNvVideoCodec/). + +Other formats — VP8, VP9 and AV1 — have **no video-input decoder** in the shipped images. +The in-tree VP8/VP9 FFmpeg serves the video *output* (generation) path; it is not wired to +video input, and the Rust `media-ffmpeg` decoder is not built into these images. Video +input decodes through Python carriers (OpenCV, PyAV, decord) that the images deliberately +omit, so a VP8/VP9/AV1 clip fails with an unsupported-codec error unless one of those +packages is installed alongside. + +This page covers which GPUs provide NVDEC, what the container must expose, and how +Dynamo behaves when hardware decode is unavailable. + +## GPU support + +NVDEC is a fixed-function decode engine, separate from the SMs used for inference, so +decoding adds negligible load to the GPU beyond a small YUV-to-RGB conversion. + +A common misconception is that datacenter GPUs have no video engines. That applies to +**NVENC**, the hardware *encoder*, which NVIDIA omits from datacenter parts. The +*decoder* is present: + +| GPU | Architecture | NVDEC (decode) | NVENC (encode) | H.264 | HEVC | +|-----|--------------|----------------|----------------|-------|------| +| A100 | Ampere | 5 engines | none | Yes | Yes | +| H100, H200 | Hopper | 7 engines | none | Yes | Yes (Main, Main 10) | +| B200, GB200 | Blackwell | 7 engines | none | Yes | Yes (Main, Main 10, 422 10/12) | +| L4 | Ada Lovelace | 4 engines | 2 | Yes | Yes | +| L40, L40S | Ada Lovelace | 3 engines | 3 | Yes | Yes | +| RTX 6000 Ada | Ada Lovelace | 3 engines | 3 | Yes (8/10-bit) | Yes (8/10/12-bit, up to 4:4:4) | + +Every GPU above decodes both codecs Dynamo routes to hardware, so H.264 and H.265 video +input works across the datacenter lineup. Hopper's NVDEC matches Turing's feature set and +does **not** decode AV1; Blackwell adds AV1 decode. The Ada parts (L4, L40S, RTX 6000 Ada) +decode AV1 as well, and unlike the datacenter accelerators they also carry NVENC. + +Because no datacenter GPU ships NVENC, Dynamo's video *generation* path encodes with a +CPU VP9 encoder rather than a hardware H.264 encoder. + +> [!NOTE] +> Under Multi-Instance GPU (MIG), NVDEC engines are divided across instances. A given MIG +> profile may expose fewer decoders than the full GPU, and some profiles expose none. +> Verify decode works in the exact profile you deploy. + +## Container requirements + +NVDEC links `libnvcuvid` at runtime, which the NVIDIA container runtime only mounts when +the **`video` driver capability** is requested. Without it, `import PyNvVideoCodec` fails +and Dynamo falls back to software decode. + +Dynamo's runtime images already declare it: + +```dockerfile +ENV NVIDIA_DRIVER_CAPABILITIES=video,compute,utility +``` + +With Docker that is usually enough, because the toolkit reads the capability from the +image. To be explicit, or when overriding the variable for other reasons: + +```bash +docker run --gpus all -e NVIDIA_DRIVER_CAPABILITIES=video,compute,utility ... +``` + +### Kubernetes + +> [!IMPORTANT] +> On Kubernetes the image's `ENV` is **not** reliably sufficient. Set the variable on the +> container spec as well. Dynamo's own GPU test runners shipped images carrying the `ENV` +> and still had no hardware decode until the pod spec set it explicitly. + +Add it to the container that runs the worker: + +```yaml +spec: + containers: + - name: worker + env: + - name: NVIDIA_DRIVER_CAPABILITIES + value: video,compute,utility + resources: + limits: + nvidia.com/gpu: "1" +``` + +If it still does not take effect, the capability is being dropped below the pod. Check +`supported-driver-capabilities` in `/etc/nvidia-container-runtime/config.toml` on the +node, and — if the cluster runs in CDI mode — whether the generated device spec includes +the video libraries, since in that mode capabilities come from the spec rather than the +environment variable. + +> [!WARNING] +> Missing the `video` capability is the most common cause of hardware decode being +> silently unavailable. The GPU itself is fine; the container simply cannot see the +> decoder. + +## Verifying hardware decode + +```bash +python3 -c " +from dynamo.common.multimodal.nvdec_decoder import nvdec_available +print('NVDEC available:', nvdec_available()) +" +``` + +`False` means Dynamo will not use hardware decode in that container. Check, in order: the +`video` driver capability, that `PyNvVideoCodec` is installed, and that +`DYN_DISABLE_NVDEC` is unset. + +To distinguish "capability missing" from every other cause, look for the decode library +itself. It is mounted by the container runtime, not installed by the image, so its absence +points squarely at the capability: + +```bash +ldconfig -p | grep -i nvcuvid +``` + +Expect `libnvcuvid.so.1`. Nothing means the `video` capability did not reach this +container. If it is present on the node but not inside, the capability is being dropped +between the two. + +## Behavior when NVDEC is unavailable + +Hardware decode is additive and never blocks a request on its own: routing falls through +to the software decode path where one exists. + +> [!IMPORTANT] +> In the shipped images there is no software decode path for video input, for any format. +> The Python carriers that decode video input (OpenCV, PyAV, decord) are deliberately not +> installed, and the in-tree VP8/VP9 FFmpeg serves the video *output* path rather than +> input. So if NVDEC is unavailable, H.264 and H.265 fail with an unsupported-codec error +> — and VP8, VP9 and AV1 fail the same way whether NVDEC is available or not, since NVDEC +> does not decode them either. +> +> Grant the container the `video` driver capability so NVDEC can serve H.264 and H.265. +> For the other formats, install a decode carrier alongside, or transcode the input to +> H.264/H.265 before sending it. + +## Hardware encode (NVENC) + +There is nothing to enable. Dynamo does not use NVENC on any path. + +Video **output** — the generation path — encodes VP9 on the CPU with the in-tree FFmpeg. +That is deliberate and works everywhere: no datacenter GPU ships an encoder, so a +hardware encode path would be unavailable on exactly the parts most deployments run. On +workstation parts that do have NVENC (L4, L40S, RTX 6000 Ada) it simply stays unused. + +The same `video` driver capability governs both engines, so a container configured for +NVDEC as above needs no additional change. Encode performance does not depend on it. + +## Configuration + +| Variable | Default | Purpose | +|----------|---------|---------| +| `DYN_DISABLE_NVDEC` | unset | Set to `1` to skip hardware decode. In a shipped image that leaves video input with no decoder at all, so it is a debugging switch rather than a fallback. Read as a boolean: `1`/`true`/`yes` disable, anything else does not. | +| `DYN_NVDEC_GPU_ID` | `0` | GPU ordinal used for decode. | +| `DYN_MM_VIDEO_NUM_FRAMES` | `32` | Frames sampled uniformly from each clip. | + +## Sources + +- [NVIDIA Video Encode and Decode GPU Support Matrix](https://developer.nvidia.com/video-encode-and-decode-gpu-support-matrix-new) +- [NVDEC Application Note](https://docs.nvidia.com/video-technologies/video-codec-sdk/13.1/nvdec-application-note/index.html) diff --git a/lib/llm/src/preprocessor/media/README.md b/lib/llm/src/preprocessor/media/README.md index ac9f8aa805f6..b21f7d052f1a 100644 --- a/lib/llm/src/preprocessor/media/README.md +++ b/lib/llm/src/preprocessor/media/README.md @@ -52,6 +52,9 @@ register_model( > [!WARNING] > **Video decoding**: Video decoding needs to be enabled via the `dynamo-llm/media-ffmpeg` rust feature. The following ffmpeg dynamic libraries must be available on the system: `libavcodec`, `libavdevice`, `libavfilter`, `libavformat`, `libswresample`, `libswscale`. These are available in dynamo dockerfiles rendered with `enable_media_ffmpeg` set to true in `container/context.yaml`. +> [!WARNING] +> **Supported input codecs**: The in-tree ffmpeg is built with a narrow decoder allowlist (VP8/VP9 video in mp4/webm/mkv) — it carries only the media formats we build and use, not ffmpeg's full default set (see `container/templates/wheel_builder.Dockerfile`). Other codecs, including H.264 and H.265, are intentionally **not** decodable in software; decoding them would require enabling the NVDEC hardware decoders (`h264_cuvid`/`hevc_cuvid`), which is not wired up today. + ## Image decoding options ### Limits (not overridable at runtime via `media_io_kwargs`) diff --git a/lib/llm/src/preprocessor/media/decoders/video.rs b/lib/llm/src/preprocessor/media/decoders/video.rs index 649823bf1ac7..dd13ede877fa 100644 --- a/lib/llm/src/preprocessor/media/decoders/video.rs +++ b/lib/llm/src/preprocessor/media/decoders/video.rs @@ -315,6 +315,9 @@ mod tests { /// Load test video and parse expected dimensions from filename. /// Filename format: "{resolution}_{frames}.mp4" (e.g., "240p_10.mp4" -> 320x240, 10 frames) + /// Fixtures are VP9-in-mp4: the in-tree ffmpeg only decodes a narrow + /// allowlist (VP8/VP9), so H.264 fixtures would not decode. Regenerate with + /// `ffmpeg -f lavfi -i testsrc2=size=WxH:rate=1 -frames:v N -c:v libvpx-vp9 -g 1 -strict -2 {resolution}_{frames}.mp4`. fn load_test_video(filename: &str) -> (EncodedMediaData, u32, u32, u32) { let path = format!( "{}/tests/data/media/{}", diff --git a/lib/llm/tests/data/media/.gitattributes b/lib/llm/tests/data/media/.gitattributes index aa69db3a7e72..d96ce1643e3b 100644 --- a/lib/llm/tests/data/media/.gitattributes +++ b/lib/llm/tests/data/media/.gitattributes @@ -4,3 +4,5 @@ llm-optimize-deploy-graphic.png filter=lfs diff=lfs merge=lfs -text 2160p_10.mp4 filter=lfs diff=lfs merge=lfs -text 240p_100.mp4 filter=lfs diff=lfs merge=lfs -text 240p_10.mp4 filter=lfs diff=lfs merge=lfs -text +240p_10_h264.mp4 filter=lfs diff=lfs merge=lfs -text +240p_10_h265.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/lib/llm/tests/data/media/2160p_10.mp4 b/lib/llm/tests/data/media/2160p_10.mp4 index 017ae1c3fee9..5ffc09eb959f 100644 --- a/lib/llm/tests/data/media/2160p_10.mp4 +++ b/lib/llm/tests/data/media/2160p_10.mp4 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f82a62d48a38e29ea004d0991adea5cdf2838e17647f24e8350aecc8297b8916 -size 5416 +oid sha256:462fa796a3099539df042a684b944e38acb2b19a2ddcceb760f88c092a1b805d +size 1009737 diff --git a/lib/llm/tests/data/media/240p_1.mp4 b/lib/llm/tests/data/media/240p_1.mp4 index 5d2a0fddf0f0..13cac1d3cb8e 100644 --- a/lib/llm/tests/data/media/240p_1.mp4 +++ b/lib/llm/tests/data/media/240p_1.mp4 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4442d098b32b29a3e7f9babac257a8c6ab3c4ed41cd2e1ad62df1e4734b92b6b -size 1597 +oid sha256:ac4dd818eb2b1371b0756704d7322bc64ef8ae9c68f8a6da1a4c12a26326485e +size 6541 diff --git a/lib/llm/tests/data/media/240p_10.mp4 b/lib/llm/tests/data/media/240p_10.mp4 index 86b57b122372..fd1f51bf0494 100644 --- a/lib/llm/tests/data/media/240p_10.mp4 +++ b/lib/llm/tests/data/media/240p_10.mp4 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:125d25fa5f09f44ae09e79e09d2e6c85a417ae8279a069eaab310b0e28cab18d -size 1913 +oid sha256:e72a3b7b83ec10000db8071bcafec959f110907d02f88da92ff792463ccb3eed +size 65804 diff --git a/lib/llm/tests/data/media/240p_100.mp4 b/lib/llm/tests/data/media/240p_100.mp4 index 7e021c9a7e82..77b2854b68e5 100644 --- a/lib/llm/tests/data/media/240p_100.mp4 +++ b/lib/llm/tests/data/media/240p_100.mp4 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c534d517ee77150ec971b3568dd999c09bbd4ec7850abed417f063bddb8e514f -size 4579 +oid sha256:13211b149c490748c2518313eae17181fca1082d7449e9b16a11e6728a0db3b4 +size 659773 diff --git a/lib/llm/tests/data/media/240p_10_h264.mp4 b/lib/llm/tests/data/media/240p_10_h264.mp4 new file mode 100644 index 000000000000..46e8b8acf5e8 --- /dev/null +++ b/lib/llm/tests/data/media/240p_10_h264.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d919aca7f53329421011b0c41ca2b3afc42fe982a40a561d65ffbf4481aa3b1b +size 38128 diff --git a/lib/llm/tests/data/media/240p_10_h265.mp4 b/lib/llm/tests/data/media/240p_10_h265.mp4 new file mode 100644 index 000000000000..b791d8109f7c --- /dev/null +++ b/lib/llm/tests/data/media/240p_10_h265.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65ec2be4415439a597b15111fc8b96a530c99cc2f6a5fb853a21db93197da7d4 +size 31151 diff --git a/lib/llm/tests/data/media/2p_10.mp4 b/lib/llm/tests/data/media/2p_10.mp4 index ce80424c5f7a..658c1965415e 100644 --- a/lib/llm/tests/data/media/2p_10.mp4 +++ b/lib/llm/tests/data/media/2p_10.mp4 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6cb1a8ca5c3ee2391a1606fac4004685e57fcc0ac5277773ccfdada534bb63f3 -size 1832 +oid sha256:ccf744a1e29f99d50d4988637f90a7d1110f3387ea059589c994a8f8ebdc25d4 +size 1180 diff --git a/lib/llm/tests/data/media/make_triangle_fixture.py b/lib/llm/tests/data/media/make_triangle_fixture.py new file mode 100644 index 000000000000..8fe1a3eeb03c --- /dev/null +++ b/lib/llm/tests/data/media/make_triangle_fixture.py @@ -0,0 +1,83 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Generate the moving-triangle source frames for the video serve fixtures. + +Why a triangle. The current fixtures are ffmpeg `testsrc2` colour bars, and the +serve tests assert on whatever a vision model happens to call them. That is +unstable: the same clip drew "a test pattern or a color calibration screen" in CI +and "a digital or pixel art animation" on a GPU box. A single unmistakable object +gives a stable token to assert. + +Why one source, three encodings. Emitting raw RGB here and letting the encoder +vary means VP9, H.264 and H.265 carry *identical* frames. Every codec path can +then assert the same word, and a difference in model output points at decode +rather than at phrasing. It also allows a direct cross-codec frame comparison, +which is a far stronger check than any word list. + +Why motion. A moving triangle also exercises temporal sampling: a regression that +collapses frame selection to a single frame changes what the model can say about +movement, where a static image would hide it. + +Regenerate all three fixtures (needs an ffmpeg with libx264/libx265/libvpx-vp9; +the Dynamo runtime images deliberately ship no such encoder): + + python3 make_triangle_fixture.py > frames.rgb + for spec in "libvpx-vp9 -strict -2:triangle_240p_10.mp4" \\ + "libx264:triangle_240p_10_h264.mp4" \\ + "libx265 -tag:v hvc1:triangle_240p_10_h265.mp4"; do + codec="${spec%%:*}"; out="${spec##*:}" + ffmpeg -y -f rawvideo -pix_fmt rgb24 -s 320x240 -r 10 -i frames.rgb \\ + -c:v $codec -g 1 -pix_fmt yuv420p "$out" + done + +Encoding all three from the same raw frames is the point: it keeps the decoded +content identical across codecs. Measured round-trip against the source is +1.59 / 1.68 / 1.63 mean absolute per-pixel difference. +""" + +from __future__ import annotations + +import sys + +import numpy as np + +WIDTH, HEIGHT, FRAMES = 320, 240, 10 +BACKGROUND = (16, 16, 48) # near-black blue; high contrast, not "colourful" +TRIANGLE = (250, 230, 40) # saturated yellow + + +def _frame(index: int) -> np.ndarray: + """One frame: a filled triangle translated horizontally across the clip.""" + img = np.zeros((HEIGHT, WIDTH, 3), dtype=np.uint8) + img[:, :] = BACKGROUND + + # Sweep left to right so motion is unambiguous across the sampled frames. + travel = WIDTH - 160 + x0 = 20 + int(travel * index / max(FRAMES - 1, 1)) + apex = (x0 + 60, 40) + left = (x0, HEIGHT - 40) + right = (x0 + 120, HEIGHT - 40) + + # Fill by half-plane tests: inside iff on the same side of all three edges. + ys, xs = np.mgrid[0:HEIGHT, 0:WIDTH] + + def side(a, b): + return (b[0] - a[0]) * (ys - a[1]) - (b[1] - a[1]) * (xs - a[0]) + + d1, d2, d3 = side(apex, left), side(left, right), side(right, apex) + inside = ((d1 >= 0) & (d2 >= 0) & (d3 >= 0)) | ((d1 <= 0) & (d2 <= 0) & (d3 <= 0)) + img[inside] = TRIANGLE + return img + + +def main() -> int: + out = sys.stdout.buffer + for i in range(FRAMES): + out.write(_frame(i).tobytes()) + out.flush() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/lib/llm/tests/data/media/triangle_240p_10.mp4 b/lib/llm/tests/data/media/triangle_240p_10.mp4 new file mode 100644 index 000000000000..31dac5ea0d27 Binary files /dev/null and b/lib/llm/tests/data/media/triangle_240p_10.mp4 differ diff --git a/lib/llm/tests/data/media/triangle_240p_10_h264.mp4 b/lib/llm/tests/data/media/triangle_240p_10_h264.mp4 new file mode 100644 index 000000000000..c8f366611faa Binary files /dev/null and b/lib/llm/tests/data/media/triangle_240p_10_h264.mp4 differ diff --git a/lib/llm/tests/data/media/triangle_240p_10_h265.mp4 b/lib/llm/tests/data/media/triangle_240p_10_h265.mp4 new file mode 100644 index 000000000000..ab5a21beb5ca Binary files /dev/null and b/lib/llm/tests/data/media/triangle_240p_10_h265.mp4 differ diff --git a/pyproject.toml b/pyproject.toml index ed307e1baed1..d50564a547b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -279,6 +279,7 @@ markers = [ "integration: marks tests as integration tests", "unit: marks tests as unit tests", "wheel_smoke: marks tests that validate wheels copied from runtime images", + "installs_extra_dependencies: marks tests that pip-install a package the runtime image intentionally omits (via DYN_TEST_ONLY_PIP_INSTALL). The feature under test may not work in an unmodified shipped image", "stress: marks tests as stress tests", "performance: marks tests as performance tests", "benchmark: marks tests as benchmark tests", @@ -386,6 +387,14 @@ module = ["tensorrt_llm", "tensorrt_llm.*"] follow_imports = "skip" ignore_missing_imports = true +[[tool.mypy.overrides]] +# PyNvVideoCodec: GPU-only NVDEC codec (common/multimodal/nvdec_decoder.py). +# Absent from the CPU lint env and ships no type stubs, so mypy would raise +# import-not-found on the guarded `import PyNvVideoCodec`. +module = ["PyNvVideoCodec", "PyNvVideoCodec.*"] +follow_imports = "skip" +ignore_missing_imports = true + [[tool.mypy.overrides]] # WAR mypy 1.18.x crash with numpy 1.26.x stubs: # "Should never get here in normal mode, got TypeAlias:numpy.float64 instead of TypeInfo" diff --git a/tests/dependencies/test_no_opencv.py b/tests/dependencies/test_no_opencv.py new file mode 100644 index 000000000000..605868f05fb6 --- /dev/null +++ b/tests/dependencies/test_no_opencv.py @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Regression guard: opencv-python-headless is not installed in TRT-LLM images. + +Upstream TRT-LLM made cv2 an optional video extra +(https://github.com/NVIDIA/TensorRT-LLM/pull/16206) and its 1.3.x imports are +already function-local, so nothing needs it at import time. Dynamo's +trtllm_runtime.Dockerfile drops the preinstall (including the vendored media +libraries under opencv_python_headless.libs/) in both the runtime_full stage and +the pre_runtime whiteout, slimming the image. This test keeps a base-image bump +or Dockerfile refactor from silently reintroducing it. +""" + +import importlib.util +import pathlib + +import pytest + +pytestmark = [ + pytest.mark.trtllm, + pytest.mark.pre_merge, + pytest.mark.post_merge, + pytest.mark.gpu_0, + pytest.mark.unit, + pytest.mark.skipif( + importlib.util.find_spec("tensorrt_llm") is None, + reason="TRT-LLM images only (other frameworks may ship opencv)", + ), +] + + +def test_opencv_not_installed(): + """cv2 must not be importable anywhere on the image's python path.""" + spec = importlib.util.find_spec("cv2") + assert spec is None, ( + f"opencv-python-headless is installed (cv2 at {spec.origin}); " + "it must be removed from TRT-LLM images — see " + "trtllm_runtime.Dockerfile (uninstall RUN + pre_runtime whiteout)" + ) + + +def test_no_vendored_opencv_libs(): + """The wheel's vendored shared libraries must be gone from dist-packages.""" + leftovers = list( + pathlib.Path("/usr/local/lib/python3.12/dist-packages").glob( + "opencv_python_headless*" + ) + ) + assert not leftovers, ( + f"vendored opencv artifacts still on disk: {leftovers}; " + "the pre_runtime whiteout in trtllm_runtime.Dockerfile must remove them" + ) diff --git a/tests/dependencies/test_no_software_video_codecs.py b/tests/dependencies/test_no_software_video_codecs.py new file mode 100644 index 000000000000..841ef1580b3d --- /dev/null +++ b/tests/dependencies/test_no_software_video_codecs.py @@ -0,0 +1,257 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Assert the shipped runtime images carry no software H.264/H.265/AAC decoder. + +The in-tree FFmpeg is already guarded where it is built: `wheel_builder.Dockerfile` +enumerates `-encoders -decoders -parsers` and fails the build on a match. That +guard runs in the builder stage, so it proves what was *produced*, not what the +runtime image finally *ships* — a base-image bump, a stray copy, or an ffmpeg +earlier on `PATH` would all slip past it. + +This closes that gap from the application's point of view: resolve FFmpeg the way +Dynamo's encode path does, and assert the formats we deliberately do not carry in +software are absent. NVIDIA DALI took the same approach when it trimmed its own +FFmpeg build (NVIDIA/DALI#6352): rather than skipping tests for codecs it no +longer supports, it asserts the failure, so a silent reintroduction is caught. + +Every negative assertion here is paired with a positive one. A missing or broken +FFmpeg would otherwise satisfy "no H.264 decoder" trivially and the test would +pass while proving nothing. + +This module is the PERMANENT half of the codec work. The removal logic in the +image templates is not: purging wheels the vLLM and SGLang base images preinstall, +and pinning DALI past its own cleanup, are all workarounds for upstreams that had +not yet trimmed their own builds. Each is expected to disappear as those upstreams +catch up -- the DALI pin already fails the build with instructions when its base +image passes it. What must survive every one of those removals is this file: it +states what the shipped images may contain, independently of which upstream +currently needs help meeting it. Delete the workarounds as they become redundant; +do not delete the assertions with them. +""" + +from __future__ import annotations + +import ctypes +import importlib.util +import os +import re +import subprocess +import sys + +import pytest + +# Deliberately NO framework marker here. tests/conftest.py skips an item when any +# framework marker it carries names a module that is absent, so listing all three +# would mean "requires vllm AND sglang AND tensorrt_llm" -- true in no image, and +# the whole module would skip everywhere while looking present. Each image is +# covered by its own single-marker entry point at the bottom of this file. +pytestmark = [ + pytest.mark.unit, + pytest.mark.pre_merge, + pytest.mark.post_merge, + pytest.mark.gpu_0, +] + +# Formats intentionally absent from the software stack. Hardware decode covers +# H.264/H.265 via NVDEC; audio has no hardware path and is opt-in only. +# Mirrors the build-time guard's pattern in wheel_builder.Dockerfile: `aac` is +# anchored to a word start so it cannot match inside an unrelated identifier. +_DISALLOWED_RE = re.compile(r"h\.?264|h\.?265|hevc|(?:^|\s)aac|nvenc|cuvid|nvdec", re.I) +# Present by construction, so a broken FFmpeg cannot masquerade as a pass. +_REQUIRED = ("vp9",) + +_SURFACES = ("encoders", "decoders", "parsers") + + +def _ffmpeg() -> str: + """Resolve FFmpeg the way the encode path does.""" + exe = os.environ.get("IMAGEIO_FFMPEG_EXE") + if exe: + return exe + try: + import imageio_ffmpeg + + return imageio_ffmpeg.get_ffmpeg_exe() + except Exception: + return "ffmpeg" + + +def _surface(exe: str, surface: str) -> str: + proc = subprocess.run( + [exe, "-hide_banner", f"-{surface}"], capture_output=True, text=True + ) + return proc.stdout + + +def _surfaces() -> dict[str, str]: + exe = _ffmpeg() + try: + out = {s: _surface(exe, s) for s in _SURFACES} + except (OSError, FileNotFoundError) as exc: + pytest.fail(f"could not run the shipped ffmpeg ({exe}): {exc}") + if not any(out.values()): + pytest.fail(f"ffmpeg at {exe} produced no codec listing; cannot verify") + return out + + +def _assert_required_codecs_present(surfaces: dict[str, str]) -> None: + """Sanity: the listings are real, so the absence checks mean something.""" + listing = "\n".join(surfaces.values()).lower() + for name in _REQUIRED: + assert name in listing, ( + f"{name} missing from the shipped ffmpeg -- the build is broken, and " + "the absence assertions here would pass vacuously" + ) + + +def _assert_no_software_codecs(surfaces: dict[str, str]) -> None: + """No implementation-carrying surface may expose the excluded formats. + + Checks encoders/decoders/parsers rather than ``-codecs``, which lists names + even when nothing is built. Bitstream filters are deliberately not checked: + they reframe an already-encoded stream, carry no codec implementation, and + ``h264_mp4toannexb`` is needed to feed hardware decode. This mirrors the + build-time guard in wheel_builder.Dockerfile, including its pattern -- `aac` + is anchored so it cannot match inside an unrelated word. + """ + for surface in _SURFACES: + listing = surfaces[surface] + hits = [ln for ln in listing.splitlines() if _DISALLOWED_RE.search(ln)] + assert not hits, ( + f"shipped ffmpeg exposes an excluded format via -{surface}; the " + "runtime image carries a software implementation it should not:\n" + + "\n".join(hits) + ) + + +def _assert_python_carriers_absent() -> None: + """The Python decode carriers are absent from an unmodified image. + + They are installable at runtime behind an opt-in switch, and tests that do so + are marked ``installs_extra_dependencies``. An unmodified image must not have + them, or a green multimodal run says nothing about what customers receive. + """ + for module, package in (("cv2", "opencv-python"), ("av", "PyAV")): + spec = importlib.util.find_spec(module) + assert spec is None, ( + f"{package} is installed ({module} at {spec.origin}); an unmodified " + "runtime image is expected to ship without it" + ) + + +def _bundled_libavcodecs() -> list[str]: + """Every third-party libavcodec on disk, excluding our own in-tree copy. + + A wheel may vendor a complete ffmpeg under its package directory, where the + ``ffmpeg`` CLI checks above cannot see it. DALI does exactly that. + """ + found: list[str] = [] + for root in {p for p in sys.path if p and os.path.isdir(p)}: + for dirpath, _dirnames, filenames in os.walk(root): + if dirpath.startswith("/usr/local/lib") and "/dist-packages" not in dirpath: + continue # our in-tree ffmpeg, covered by the surface checks + for name in filenames: + if name.startswith("libavcodec") and ".so" in name: + found.append(os.path.join(dirpath, name)) + return found + + +def _registered_decoders(lib_path: str) -> set[str]: + """What a libavcodec actually registers, via av_codec_iterate. + + These libraries are stripped, so ``nm`` sees nothing; ``strings`` gives false + positives because a decoder's long name survives when only its parser or + bitstream filter is kept. Asking the library at runtime is the only + authoritative answer. Mirrors container/compliance/enumerate_bundled_decoders.py. + """ + + class AVCodec(ctypes.Structure): + _fields_ = [ + ("name", ctypes.c_char_p), + ("long_name", ctypes.c_char_p), + ("type", ctypes.c_int), + ("id", ctypes.c_int), + ] + + lib = ctypes.CDLL(lib_path) + lib.av_codec_iterate.restype = ctypes.c_void_p + lib.av_codec_iterate.argtypes = [ctypes.POINTER(ctypes.c_void_p)] + lib.av_codec_is_decoder.restype = ctypes.c_int + lib.av_codec_is_decoder.argtypes = [ctypes.c_void_p] + + opaque = ctypes.c_void_p(None) + decoders: set[str] = set() + while True: + c = lib.av_codec_iterate(ctypes.byref(opaque)) + if not c: + break + entry = ctypes.cast(c, ctypes.POINTER(AVCodec)).contents + if lib.av_codec_is_decoder(c): + decoders.add(entry.name.decode()) + return decoders + + +# Formats no bundled libavcodec may register. Narrower than _DISALLOWED_RE: that +# pattern matches listing lines, this matches exact codec names. +_EXCLUDED_DECODERS = ("h264", "hevc", "aac", "aac_fixed", "aac_latm") + + +def _assert_bundled_libavcodecs_carry_no_software_codecs() -> None: + """A vendored libavcodec must not register the formats we exclude. + + The TensorRT-LLM base image ships DALI, which vendors its own ffmpeg for its + video reader. DALI 2.1.0 registered h264, hevc and the aac family; upstream + restricted that build (NVIDIA/DALI_deps#162, NVIDIA/DALI#6352) from 2.1.1, + and the image pins past it. This keeps that pin honest: a base-image bump + that reverted to an older DALI, or a new dependency vendoring its own ffmpeg, + fails here rather than shipping. + + The libraries cannot simply be deleted — they are DT_NEEDED by libdali.so, so + removing them breaks the package outright, which is why this asserts on + content rather than absence. + """ + for lib_path in _bundled_libavcodecs(): + try: + decoders = _registered_decoders(lib_path) + except OSError as exc: + pytest.fail(f"could not load bundled libavcodec {lib_path}: {exc}") + assert decoders, ( + f"{lib_path} registered no decoders at all; the probe is broken and " + "the exclusion check below would pass vacuously" + ) + present = sorted(d for d in _EXCLUDED_DECODERS if d in decoders) + assert not present, ( + f"bundled libavcodec {lib_path} registers excluded decoder(s): " + f"{', '.join(present)}. It ships {len(decoders)} decoders in total; " + "if this is DALI, the image needs 2.1.1 or newer." + ) + + +def _check_image() -> None: + surfaces = _surfaces() + _assert_required_codecs_present(surfaces) + _assert_no_software_codecs(surfaces) + _assert_python_carriers_absent() + _assert_bundled_libavcodecs_carry_no_software_codecs() + + +# One entry point per image. Each carries a single framework marker so it runs in +# that backend's lane and is skipped in the others -- tests/conftest.py skips an +# item whose framework marker names an absent module, so a single module marked +# with all three would skip everywhere and silently prove nothing. + + +@pytest.mark.vllm +def test_vllm_image_ships_no_software_video_codecs() -> None: + _check_image() + + +@pytest.mark.sglang +def test_sglang_image_ships_no_software_video_codecs() -> None: + _check_image() + + +@pytest.mark.trtllm +def test_trtllm_image_ships_no_software_video_codecs() -> None: + _check_image() diff --git a/tests/router/test_router_e2e_with_sglang.py b/tests/router/test_router_e2e_with_sglang.py index 9e2dde73cf6f..80837f0e07b3 100644 --- a/tests/router/test_router_e2e_with_sglang.py +++ b/tests/router/test_router_e2e_with_sglang.py @@ -381,12 +381,30 @@ def test_router_decisions_sglang_disagg( ) -# DYN-2784: Fixture setup hangs silently in nightly only (worker #2 dies -# in SGLangProcess launch, KvRouter blocks forever on min_initial_workers=2; -# pytest.mark.timeout signal gets swallowed at the C-level syscall). -# Passes reliably in pre_merge/post_merge runs, so scope the skip to the -# nightly pipeline via skip_in_nightly, which nightly-ci.yml excludes from -# its sglang single-GPU marker filter. Remove once DYN-2784 lands a real fix. +# DYN-2784: fixture setup hangs instead of failing. Worker #2 dies during +# SGLangProcess launch and KvRouter then blocks forever on +# min_initial_workers=2; the timeout below cannot fire because the signal is +# swallowed at a C-level syscall. +# +# Skipped outright, because the containment this previously relied on does not +# hold. The note here used to read "passes reliably in pre_merge/post_merge, so +# scope the skip to nightly" -- measured over five days that is 82% of nightly +# runs but also 63% of post_merge, and the runs that do hang burn a whole job: +# observed hangs of 3659s, 4235s and 6077s, each holding a 12 GiB reservation +# with three more workers queued behind it, because the VRAM-aware parallel +# orchestrator has no hard-kill for a child that outlives its declared timeout. +# That came to roughly 220 runner-hours in five days. +# +# skip_in_nightly is kept but is not what makes this safe: it only drops the +# test from the nightly marker filter, and the hang is not nightly-specific. +# +# Re-enable once the launch race is fixed, or once the orchestrator can kill a +# hung child -- either one alone turns this from a lost job into a normal +# failure. The scenario itself is worth keeping; it is the hang that is not +# affordable. +@pytest.mark.skip( + reason="hangs to the job step cap instead of failing; see the note above" +) @pytest.mark.e2e @pytest.mark.model(MODEL_NAME) @pytest.mark.skip_in_nightly diff --git a/tests/serve/common.py b/tests/serve/common.py index 4a6e76142e40..c284bdbe22d9 100644 --- a/tests/serve/common.py +++ b/tests/serve/common.py @@ -6,6 +6,8 @@ import dataclasses import logging import os +import subprocess +import sys import time from collections.abc import Callable, Mapping from copy import deepcopy @@ -270,6 +272,41 @@ def _prepare_deployment( ) +# EngineConfig.env key naming a whitespace-separated list of pip packages to +# install into the runtime container before the server launches. Some runtime +# images intentionally omit certain media-decoder libraries; the few serve tests +# that exercise a decode path install the decoder here at test time so coverage +# is retained without the shipped image carrying it. No-op when the key is unset. +TEST_ONLY_PIP_ENV_KEY = "DYN_TEST_ONLY_PIP_INSTALL" + +# Session-level guard so the same package set is installed at most once even +# though every parametrized deployment (and each retry) calls the installer. +_test_only_pip_done: set[str] = set() + + +def _install_test_only_packages(config: EngineConfig) -> None: + """Install any test-only pip packages a config requested via its env. + + Runs inside the same runtime container/interpreter the server subprocess + inherits, so the worker can import the freshly installed module. + """ + spec = config.env.get(TEST_ONLY_PIP_ENV_KEY, "").strip() + if not spec or spec in _test_only_pip_done: + return + packages = spec.split() + logging.getLogger(__name__).info( + "Installing test-only package(s) into runtime container: %s", + " ".join(packages), + ) + # --break-system-packages: runtime images use an externally-managed system + # python (PEP 668); this is the ephemeral test container, not a shipped image. + subprocess.run( + [sys.executable, "-m", "pip", "install", "--break-system-packages", *packages], + check=True, + ) + _test_only_pip_done.add(spec) + + def run_serve_deployment( config: EngineConfig, request: Any, @@ -296,6 +333,10 @@ def run_serve_deployment( logger.info("Using model: %s", config.model) logger.info("Script: %s", config.script_name) + # Install any decoder a codec-stripped image needs for this test, before the + # server launches, so the worker can import it. No-op unless the config opts in. + _install_test_only_packages(config) + prep = _prepare_deployment(config, request, ports=ports, extra_env=extra_env) config = prep.config merged_env = prep.merged_env diff --git a/tests/serve/conftest.py b/tests/serve/conftest.py index 3c8b963976c6..eb510a996361 100644 --- a/tests/serve/conftest.py +++ b/tests/serve/conftest.py @@ -14,9 +14,83 @@ # Shared constants for multimodal testing IMAGE_SERVER_PORT = allocate_port(8765) MULTIMODAL_IMG_URL = f"http://localhost:{IMAGE_SERVER_PORT}/llm-graphic.png" +# A yellow triangle sweeping left to right on a near-black background, 320x240, +# 10 frames. Deliberately one unmistakable object rather than a test pattern: the +# assertion below needs a word a vision model reliably reaches for. +# +# The three encodings come from ONE set of raw frames (see +# lib/llm/tests/data/media/make_triangle_fixture.py), so VP9, H.264 and H.265 carry identical +# content. That means every codec path can assert the same word, and a difference +# in output points at decode rather than at phrasing. Measured round-trip against +# the source: 1.59 / 1.68 / 1.63 mean absolute per-pixel difference -- within 0.1 +# of each other, so the decoded frames are directly comparable across codecs. The +# previous fixtures differed by 17.77 between codecs, which made that impossible. +# +# The triangle moves, but do NOT read the motion as coverage of temporal +# sampling. Asked to describe the clip, Qwen2.5-VL-3B returns "A yellow triangle +# appears in the center of the screen. The triangle is static and does not move." +# -- it reports the object reliably and the movement not at all. A regression +# that collapsed frame selection to a single frame would therefore not show up in +# the response text. Cross-codec frame comparison is the check to build on for +# that, not the model's wording. MULTIMODAL_VIDEO_PATH = os.path.join( - WORKSPACE_DIR, "lib/llm/tests/data/media/240p_10.mp4" + WORKSPACE_DIR, "lib/llm/tests/data/media/triangle_240p_10.mp4" ) +# The H.264/H.265 clips exercise the NVDEC hardware-decode path, served over http +# by the image_server fixture. +# +# An earlier revision of this note claimed file:// video is not hardware-decoded. +# That is wrong for vLLM and SGLang: both read local media through +# read_local_media_bytes and then route it to NVDEC exactly as they do http(s) +# (common/multimodal/video_loader.py, sglang encode_worker_handler.py), because +# otherwise a local H.264 file would reach only the software decoder these images +# do not ship. TensorRT-LLM is the exception -- it has its own +# allowed_local_media_path handling and no NVDEC routing for local paths. +# MULTIMODAL_VIDEO_H264_FILE_URI below covers the local path. +_MEDIA_DIR = os.path.join(WORKSPACE_DIR, "lib/llm/tests/data/media") +_HTTP_SERVED_VIDEOS = ( + "triangle_240p_10.mp4", + "triangle_240p_10_h264.mp4", + "triangle_240p_10_h265.mp4", +) +MULTIMODAL_VIDEO_H264_URL = ( + f"http://localhost:{IMAGE_SERVER_PORT}/triangle_240p_10_h264.mp4" +) +MULTIMODAL_VIDEO_H265_URL = ( + f"http://localhost:{IMAGE_SERVER_PORT}/triangle_240p_10_h265.mp4" +) +# VP9 source clip over http, for backends whose video path needs a URL rather +# than a local file. Prefer this over any third-party URL: a remote host is an +# availability dependency the test does not control. +MULTIMODAL_VIDEO_URL = f"http://localhost:{IMAGE_SERVER_PORT}/triangle_240p_10.mp4" +# The same H.264 clip addressed as a local file. Reading it is gated by +# DYN_MM_LOCAL_PATH, which must name a directory containing the clip -- +# MULTIMODAL_MEDIA_DIR below. Worth covering separately from http: the local +# branch has its own read path (read_local_media_bytes rather than a fetch) +# before it reaches the same NVDEC routing, so an http-only suite leaves the +# whole local read, its policy gate, and its hand-off to the decoder untested. +MULTIMODAL_VIDEO_H264_FILE_URI = ( + f"file://{os.path.join(_MEDIA_DIR, 'triangle_240p_10_h264.mp4')}" +) +# Value for DYN_MM_LOCAL_PATH in tests that use the file:// URI above. +MULTIMODAL_MEDIA_DIR = _MEDIA_DIR +# What the clip depicts, for expected_response assertions. The subject is chosen +# so the answer is unambiguous: a model that decoded the frames says "triangle", +# and one that did not cannot say it by luck. +# +# Confirmed against the real deployment on both H.264 and H.265, which returned +# the same sentence for each: "The video begins with a black screen. A yellow +# triangle appears in the center of the screen." One noun, no hedging -- a much +# wider margin than a word list over an abstract pattern. +# +# Verified to be capable of failing, too: forcing an impossible expected value +# makes the test fail, so a pass here is not vacuous. +# +# This replaces an earlier ["red", "static", "still"] against testsrc2 colour +# bars, which described content those clips never contained and passed only when +# the model happened to use one of the words. Matching is case-insensitive +# substring with OR logic (tests/utils/payloads.py). +MULTIMODAL_VIDEO_EXPECTED = ["triangle"] def get_multimodal_test_image_bytes() -> bytes: @@ -93,14 +167,19 @@ def _handler(request: Request) -> Response: httpserver.expect_request("/llm-graphic.png").respond_with_handler(_handler) - # Serve video file for multimodal video tests (guard against LFS pointers) - if os.path.isfile(MULTIMODAL_VIDEO_PATH): - with open(MULTIMODAL_VIDEO_PATH, "rb") as vf: + # Serve the video fixtures over http (VP9 + the H.264/H.265 NVDEC clips) for + # multimodal video tests. Guard against unpulled Git-LFS pointer files. + for _fname in _HTTP_SERVED_VIDEOS: + _fpath = os.path.join(_MEDIA_DIR, _fname) + if not os.path.isfile(_fpath): + continue + with open(_fpath, "rb") as vf: video_data = vf.read() - if not video_data.startswith(b"version "): - httpserver.expect_request("/240p_10.mp4").respond_with_data( - video_data, content_type="video/mp4" - ) + if video_data.startswith(b"version "): # unresolved LFS pointer + continue + httpserver.expect_request(f"/{_fname}").respond_with_data( + video_data, content_type="video/mp4" + ) return httpserver diff --git a/tests/serve/multimodal_profiles/README.md b/tests/serve/multimodal_profiles/README.md index ef68b9cbd972..6f5c8ce0c887 100644 --- a/tests/serve/multimodal_profiles/README.md +++ b/tests/serve/multimodal_profiles/README.md @@ -112,7 +112,7 @@ VLLM_MULTIMODAL_PROFILES = [ delayed_start=60, profiled_vram_gib=8.2, requested_vllm_kv_cache_bytes=1_719_075_000, - tests=[MmCase(payload=make_video_payload(["red", "static", "still"]))], + tests=[MmCase(payload=make_video_payload(MULTIMODAL_VIDEO_EXPECTED))], ), ... }, @@ -158,7 +158,9 @@ profiles: the LFS-tracked PNG as a `data:image/png;base64,...` URL. Materialization is deferred to first `.body` read so pytest collection doesn't fail when LFS hasn't pulled the image yet -- `make_video_payload(expected_phrases)` — local test video +- `make_video_payload(expected_phrases)` — local test video (the triangle + fixture; pass `MULTIMODAL_VIDEO_EXPECTED` rather than a literal, so the + expectation cannot drift from the footage) - `make_audio_payload(expected_phrases)` — remote test WAV Backend-specific files (`vllm.py`, future `trtllm.py`, etc.) compose diff --git a/tests/serve/multimodal_profiles/vllm.py b/tests/serve/multimodal_profiles/vllm.py index 35b9bdd3b687..cf564163fdc4 100644 --- a/tests/serve/multimodal_profiles/vllm.py +++ b/tests/serve/multimodal_profiles/vllm.py @@ -5,13 +5,17 @@ import pytest +from dynamo.common.multimodal.nvdec_decoder import nvdec_available from dynamo.common.utils.paths import WORKSPACE_DIR +from tests.serve.conftest import MULTIMODAL_VIDEO_EXPECTED from tests.utils.multimodal import ( MmCase, MultimodalModelProfile, TopologyConfig, make_audio_payload, make_custom_encoder_payload, + make_h264_video_payload, + make_hevc_video_payload, make_image_payload, make_image_payload_b64, make_image_payload_cached_tokens, @@ -24,6 +28,15 @@ image_token_metrics_payload, ) +# NVDEC H.264/H.265 serve cases hardware-decode via libnvcuvid, which needs the +# container's driver "video" capability (NVIDIA_DRIVER_CAPABILITIES=...,video). +# Runners without it can't decode H.264/H.265 and the codec-compliant image +# ships no software fallback, so skip rather than fail. Evaluated once at +# collection; mirrors the guard in +# components/src/dynamo/common/tests/multimodal/test_nvdec_decoder_gpu.py. +# NVDEC is still validated on gpu-ts. +_NVDEC_UNAVAILABLE = not nvdec_available() + # LLaVA 1.5 color-identification reference set: the model legitimately # emits these colors (though the order/subset varies across CUDA backends # under vLLM 0.20). Reused by every LLaVA topology that runs the standard @@ -49,6 +62,8 @@ VLLM_TOPOLOGY_SCRIPTS: dict[str, str] = { "agg": "agg_multimodal.sh", "agg_video": "agg_multimodal.sh", + # H.264/H.265 (NVDEC hardware decode) reuse the same aggregated script. + "agg_video_nvdec": "agg_multimodal.sh", # Aggregated MM-aware router. Default uses the Rust frontend with the # `mm-routing` feature; the `_chat_processor` variant uses the vLLM # Python preprocessor (`--dyn-chat-processor=vllm`) to enable the @@ -106,12 +121,52 @@ ], ), "agg_video": TopologyConfig( - marks=[pytest.mark.pre_merge], + marks=[pytest.mark.pre_merge, pytest.mark.installs_extra_dependencies], timeout_s=600, delayed_start=60, profiled_vram_gib=8.2, requested_vllm_kv_cache_bytes=1_719_075_000, - tests=[MmCase(payload=make_video_payload(["red", "static", "still"]))], + # Backend video decode goes through vLLM's VideoMediaIO (opencv); + # the shipped image omits it as a media-codec carrier, so install + # it for this test only. See common._install_test_only_packages. + # installs_extra_dependencies: this VP9 case therefore proves the routing + # works *given* a decoder, not that VP9 decodes in a shipped + # image -- it does not, until the opt-in decoders land. + env={"DYN_TEST_ONLY_PIP_INSTALL": "opencv-python-headless"}, + tests=[MmCase(payload=make_video_payload(MULTIMODAL_VIDEO_EXPECTED))], + ), + # NVDEC hardware-decode path: H.264/H.265 video input decoded on the + # GPU (PyNvVideoCodec, baked into the image) — no per-test decoder + # install, unlike the VP9 agg_video case above. Clips are served over + # http so the NVDEC route triggers (file:// video is not + # hardware-decoded). Skips when the runner lacks the driver "video" + # capability that libnvcuvid requires (see _NVDEC_UNAVAILABLE); NVDEC + # is validated on gpu-ts. + "agg_video_nvdec": TopologyConfig( + marks=[ + pytest.mark.post_merge, + pytest.mark.skipif( + _NVDEC_UNAVAILABLE, + reason=( + "NVDEC/PyNvVideoCodec unavailable; needs the driver " + "'video' capability (NVIDIA_DRIVER_CAPABILITIES)" + ), + ), + ], + timeout_s=600, + delayed_start=60, + profiled_vram_gib=8.2, + requested_vllm_kv_cache_bytes=1_719_075_000, + tests=[ + MmCase( + suffix="h264", + payload=make_h264_video_payload(MULTIMODAL_VIDEO_EXPECTED), + ), + MmCase( + suffix="h265", + payload=make_hevc_video_payload(MULTIMODAL_VIDEO_EXPECTED), + ), + ], ), # Post_merge MM-routing coverage for the Qwen3-VL family — the # smaller Qwen3.5-0.8B (`agg_router` below) is the pre_merge gater. @@ -184,13 +239,16 @@ tests=[MmCase(payload=make_image_payload(["green"]))], ), "epd_video": TopologyConfig( - marks=[pytest.mark.post_merge], + marks=[pytest.mark.post_merge, pytest.mark.installs_extra_dependencies], timeout_s=600, delayed_start=60, single_gpu=True, profiled_vram_gib=19.7, requested_vllm_kv_cache_bytes=1_714_881_000, - tests=[MmCase(payload=make_video_payload(["red", "static", "still"]))], + # See agg_video: install the opencv backend decoder for this test + # only, so this is likewise a installs_extra_dependencies case. + env={"DYN_TEST_ONLY_PIP_INSTALL": "opencv-python-headless"}, + tests=[MmCase(payload=make_video_payload(MULTIMODAL_VIDEO_EXPECTED))], ), "p_d": TopologyConfig( marks=[pytest.mark.post_merge], diff --git a/tests/serve/multimodal_profiles/vllm_xpu.py b/tests/serve/multimodal_profiles/vllm_xpu.py index e8ac27c847e3..43ae69d59b56 100644 --- a/tests/serve/multimodal_profiles/vllm_xpu.py +++ b/tests/serve/multimodal_profiles/vllm_xpu.py @@ -4,6 +4,7 @@ import pytest from tests.serve.common import WORKSPACE_DIR +from tests.serve.conftest import MULTIMODAL_VIDEO_EXPECTED from tests.utils.multimodal import ( MmCase, MultimodalModelProfile, @@ -105,7 +106,7 @@ requested_vllm_kv_cache_bytes=1_719_075_000, tests=[ MmCase( - payload=make_video_payload(["red", "static", "still"]), + payload=make_video_payload(MULTIMODAL_VIDEO_EXPECTED), env={"DYN_MM_LOCAL_PATH": WORKSPACE_DIR}, ) ], diff --git a/tests/serve/test_sglang.py b/tests/serve/test_sglang.py index 33633e38eb32..8bf7296e0329 100644 --- a/tests/serve/test_sglang.py +++ b/tests/serve/test_sglang.py @@ -9,12 +9,21 @@ import pytest +from dynamo.common.multimodal.nvdec_decoder import nvdec_available from tests.serve.common import ( SERVE_TEST_DIR, WORKSPACE_DIR, params_with_model_mark, run_serve_deployment, ) +from tests.serve.conftest import ( + MULTIMODAL_MEDIA_DIR, + MULTIMODAL_VIDEO_EXPECTED, + MULTIMODAL_VIDEO_H264_FILE_URI, + MULTIMODAL_VIDEO_H264_URL, + MULTIMODAL_VIDEO_H265_URL, + MULTIMODAL_VIDEO_URL, +) from tests.serve.lora_utils import DEFAULT_LORA_REPO, MinioLoraConfig from tests.serve.multimodal_profiles.sglang import ( SGLANG_MULTIMODAL_PROFILES, @@ -76,9 +85,15 @@ class SGLangConfig(EngineConfig): sglang_dir = os.environ.get("SGLANG_DIR") or os.path.join( WORKSPACE_DIR, "examples/backends/sglang" ) -REMOTE_VIDEO_TEST_URI = ( - "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Omni/demo/draw.mp4" -) +# Video clips are served by the image_server fixture from the in-repo media +# fixtures. This used to point at a third-party host, which made the SGLang +# video tests depend on that host being reachable: a read timeout fetching the +# clip failed the run with no relation to the code under test. +VIDEO_TEST_URI = MULTIMODAL_VIDEO_URL + +# Evaluated once at collection, like the vLLM profile's guard: NVDEC needs +# the container's driver "video" capability, which CI runners do not grant. +_NVDEC_UNAVAILABLE = not nvdec_available() # Generated multimodal configs from profile definitions (mirrors test_vllm.py). # Each profile expands into one config per MmCase per topology with the @@ -530,6 +545,10 @@ class SGLangConfig(EngineConfig): marks=[ pytest.mark.multimodal, pytest.mark.gpu_1, + # Installs decord below, which the shipped image omits: this proves + # the routing works given a decoder, not that the codec decodes in a + # shipped image. + pytest.mark.installs_extra_dependencies, # Bisected with tests/utils/profile_pytest.py: minimum = 4368 # tokens, 2x safety = 8736. Peak 20.5 GiB at 8736 tokens. Without # the cap, sglang's default 65% fraction allocates ~278k tokens @@ -545,6 +564,10 @@ class SGLangConfig(EngineConfig): "Qwen/Qwen2-VL-7B-Instruct", ], timeout=360, + # SGLang's video path decodes with decord; the shipped image omits it as + # a media-codec carrier, so install it for this test only. + # See common._install_test_only_packages. + env={"DYN_TEST_ONLY_PIP_INSTALL": "decord2"}, frontend_port=DefaultPort.FRONTEND.value, request_payloads=[ chat_payload( @@ -552,14 +575,100 @@ class SGLangConfig(EngineConfig): {"type": "text", "text": "Describe the video in detail"}, { "type": "video_url", - "video_url": {"url": REMOTE_VIDEO_TEST_URI}, + "video_url": {"url": VIDEO_TEST_URI}, }, ], repeat_count=1, - expected_response=["guitar", "tablet", "draw"], + expected_response=MULTIMODAL_VIDEO_EXPECTED, + temperature=0.0, + max_tokens=100, + ) + ], + ), + "video_e_pd_qwen_nvdec": SGLangConfig( + # H.264/H.265 video input decoded on the GPU by NVDEC, which is the only + # video decoder in the shipped image -- so unlike the cases above this + # installs nothing and exercises what a deployment actually gets. It is + # the only end-to-end cover for the video_metadata shim in + # encode_worker_handler, which SGLang's pre-decoded-frame path needs. + # + # Must be the E/PD topology: NVDEC is wired into the encode worker, and + # the aggregated path hands the URL to SGLang, which resolves and decodes + # it internally with no interception point. + # + # Must also be a qwen2-family model. Qwen3-VL is in + # _NVDEC_UNSAFE_MODEL_TYPES (its preprocessing cannot take pre-decoded + # frames), so NVDEC is deliberately skipped for it and the request would + # fall back to the URL path, which has no decoder here. + name="video_e_pd_qwen_nvdec", + directory=sglang_dir, + script_name="multimodal_epd.sh", + marks=[ + pytest.mark.multimodal, + pytest.mark.gpu_1, + # CI runners lack the driver "video" capability libnvcuvid needs, so + # this skips there and is exercised on GPU hardware instead. + pytest.mark.skipif( + _NVDEC_UNAVAILABLE, + reason=( + "NVDEC/PyNvVideoCodec unavailable; needs the driver " + "'video' capability (NVIDIA_DRIVER_CAPABILITIES)" + ), + ), + pytest.mark.timeout(420), + pytest.mark.post_merge, + ], + model="Qwen/Qwen2.5-VL-3B-Instruct", + # Deliberately does NOT set --multimodal-embedding-cache-capacity-gb, so + # the embedding cache stays at its default of 0 (disabled) and this + # exercises the branch a stock deployment actually takes. An earlier + # revision set 0.1 here, which routed the request through the cached + # encode path -- the only path that performed the NVDEC conversion -- + # and so passed while default deployments failed outright. The cached + # path keeps its coverage in the encode-worker unit tests. + script_args=[ + "--model", + "Qwen/Qwen2.5-VL-3B-Instruct", + "--chat-template", + "qwen2-vl", + "--single-gpu", + ], + timeout=420, + env={ + "DYN_ENCODE_GPU_MEM": "0.1", + "DYN_WORKER_GPU_MEM": "0.4", + "DYN_SGL_EMBEDDING_TRANSFER_MODE": "local", + # The clips come from the image_server over plain http on localhost, + # which the URL policy rejects by default. Without this the encode + # worker refuses the URL before NVDEC sees it and falls back to the + # URL path, which has no decoder in this image. + "DYN_MM_ALLOW_INTERNAL": "1", + # Permits the file:// payload below. The local branch reads through + # read_local_media_bytes instead of fetching, then joins the same + # NVDEC routing, so without a file:// case that read, its policy + # gate, and its hand-off to the decoder are never exercised. + "DYN_MM_LOCAL_PATH": MULTIMODAL_MEDIA_DIR, + }, + frontend_port=DefaultPort.FRONTEND.value, + request_payloads=[ + chat_payload( + [ + {"type": "text", "text": "Describe the video in detail"}, + {"type": "video_url", "video_url": {"url": url}}, + ], + repeat_count=1, + expected_response=MULTIMODAL_VIDEO_EXPECTED, temperature=0.0, max_tokens=100, ) + # The same clip over http and as a local file. All three are the + # identical triangle footage, so a difference in the answer points at + # the transport rather than at the model or the encoding. + for url in ( + MULTIMODAL_VIDEO_H264_URL, + MULTIMODAL_VIDEO_H265_URL, + MULTIMODAL_VIDEO_H264_FILE_URI, + ) ], ), "video_e_pd_qwen": SGLangConfig( @@ -571,6 +680,9 @@ class SGLangConfig(EngineConfig): marks=[ pytest.mark.multimodal, pytest.mark.gpu_1, + # See video_agg_qwen: installs decord, so likewise a + # installs_extra_dependencies case. + pytest.mark.installs_extra_dependencies, # No profiled_vram_gib: multimodal_epd.sh uses explicit # --mem-fraction-static via DYN_ENCODE_GPU_MEM / DYN_WORKER_GPU_MEM. pytest.mark.timeout(360), @@ -591,6 +703,9 @@ class SGLangConfig(EngineConfig): "DYN_ENCODE_GPU_MEM": "0.1", "DYN_WORKER_GPU_MEM": "0.4", "DYN_SGL_EMBEDDING_TRANSFER_MODE": "local", + # SGLang's video path decodes with decord; the shipped image omits it + # as a media-codec carrier, so install it for this test only. + "DYN_TEST_ONLY_PIP_INSTALL": "decord2", }, frontend_port=DefaultPort.FRONTEND.value, request_payloads=[ @@ -599,11 +714,11 @@ class SGLangConfig(EngineConfig): {"type": "text", "text": "Describe the video in detail"}, { "type": "video_url", - "video_url": {"url": REMOTE_VIDEO_TEST_URI}, + "video_url": {"url": VIDEO_TEST_URI}, }, ], repeat_count=1, - expected_response=["guitar", "tablet", "draw"], + expected_response=MULTIMODAL_VIDEO_EXPECTED, temperature=0.0, max_tokens=100, ), @@ -612,11 +727,11 @@ class SGLangConfig(EngineConfig): {"type": "text", "text": "Describe the video in detail"}, { "type": "video_url", - "video_url": {"url": REMOTE_VIDEO_TEST_URI}, + "video_url": {"url": VIDEO_TEST_URI}, }, ], repeat_count=1, - expected_response=["guitar", "tablet", "draw"], + expected_response=MULTIMODAL_VIDEO_EXPECTED, expected_log=["Embedding cache hit for VIDEO URL index 0"], temperature=0.0, max_tokens=100, diff --git a/tests/serve/test_trtllm.py b/tests/serve/test_trtllm.py index ec7f7ea7fa26..027667de8c0a 100644 --- a/tests/serve/test_trtllm.py +++ b/tests/serve/test_trtllm.py @@ -9,12 +9,18 @@ import pytest import yaml +from dynamo.common.multimodal.nvdec_decoder import nvdec_available from tests.serve.common import ( SERVE_TEST_DIR, WORKSPACE_DIR, params_with_model_mark, run_serve_deployment, ) +from tests.serve.conftest import ( + MULTIMODAL_VIDEO_EXPECTED, + MULTIMODAL_VIDEO_H264_URL, + MULTIMODAL_VIDEO_H265_URL, +) from tests.utils.constants import DefaultPort from tests.utils.engine_process import EngineConfig from tests.utils.multimodal import make_image_payload_cached_tokens @@ -44,6 +50,11 @@ class TRTLLMConfig(EngineConfig): trtllm_dir = os.environ.get("TRTLLM_DIR") or os.path.join( WORKSPACE_DIR, "examples/backends/trtllm" ) + +# Evaluated once at collection: NVDEC needs the container's driver "video" +# capability, which CI runners do not grant. +_NVDEC_UNAVAILABLE = not nvdec_available() + qwen3_vl_engine_config_dir = os.path.join( WORKSPACE_DIR, "examples/backends/trtllm/engine_configs/qwen3-vl-2b-instruct", @@ -308,6 +319,57 @@ class TRTLLMConfig(EngineConfig): ) ], ), + "aggregated_multimodal_video_nvdec": TRTLLMConfig( + # The only serve-level cover for video input on this backend. TensorRT-LLM + # supports video for the Qwen-VL families, and multimodal_processor routes + # video_url through NVDEC for H.264/H.265, but nothing exercised it + # end-to-end: the NVDEC VideoData transform was verified only by mocked + # unit tests and by hand on GPU hardware. + # + # Installs no decoder: NVDEC is the only video decoder in the shipped + # image, so this is what a deployment actually gets. Both codecs run + # against one deployment to avoid a second model load. + name="aggregated_multimodal_video_nvdec", + directory=trtllm_dir, + script_name="agg_multimodal.sh", + marks=[ + pytest.mark.gpu_1, + pytest.mark.trtllm, + pytest.mark.multimodal, + # CI runners lack the driver "video" capability libnvcuvid needs, so + # this skips there and is exercised on GPU hardware instead. + pytest.mark.skipif( + _NVDEC_UNAVAILABLE, + reason=( + "NVDEC/PyNvVideoCodec unavailable; needs the driver " + "'video' capability (NVIDIA_DRIVER_CAPABILITIES)" + ), + ), + pytest.mark.post_merge, + pytest.mark.profiled_vram_gib(12.0), + pytest.mark.requested_trtllm_kv_tokens(32768), + pytest.mark.timeout(960), + ], + model="Qwen/Qwen3-VL-2B-Instruct", + frontend_port=DefaultPort.FRONTEND.value, + timeout=900, + delayed_start=60, + # The clips are served from localhost by the image_server fixture. + env={"DYN_MM_ALLOW_INTERNAL": "1"}, + request_payloads=[ + chat_payload( + [ + {"type": "text", "text": "Describe the video in detail"}, + {"type": "video_url", "video_url": {"url": url}}, + ], + repeat_count=1, + expected_response=MULTIMODAL_VIDEO_EXPECTED, + temperature=0.0, + max_tokens=100, + ) + for url in (MULTIMODAL_VIDEO_H264_URL, MULTIMODAL_VIDEO_H265_URL) + ], + ), # TensorRT-LLM EPD (Encode-Prefill-Decode) multimodal test for pre-merge CI # Uses Qwen3-VL-2B-Instruct model with 1 GPU (all workers share same GPU) # diff --git a/tests/serve/test_vllm_xpu.py b/tests/serve/test_vllm_xpu.py index c115735d9896..b3661ed3a8d6 100644 --- a/tests/serve/test_vllm_xpu.py +++ b/tests/serve/test_vllm_xpu.py @@ -16,7 +16,11 @@ params_with_model_mark, run_serve_deployment, ) -from tests.serve.conftest import MULTIMODAL_IMG_URL, get_multimodal_test_image_bytes +from tests.serve.conftest import ( + MULTIMODAL_IMG_URL, + MULTIMODAL_VIDEO_EXPECTED, + get_multimodal_test_image_bytes, +) from tests.serve.lora_utils import MinioLoraConfig from tests.serve.multimodal_profiles.vllm_xpu import ( VLLM_MULTIMODAL_PROFILES, @@ -490,7 +494,7 @@ class VLLMConfig(EngineConfig): }, ], repeat_count=1, - expected_response=["red", "static", "still"], + expected_response=MULTIMODAL_VIDEO_EXPECTED, temperature=0.0, max_tokens=100, ) diff --git a/tests/utils/multimodal.py b/tests/utils/multimodal.py index 789387be716c..beff0fe7064e 100644 --- a/tests/utils/multimodal.py +++ b/tests/utils/multimodal.py @@ -12,13 +12,22 @@ import pytest from dynamo.common.utils.paths import WORKSPACE_DIR -from tests.serve.conftest import MULTIMODAL_IMG_URL, get_multimodal_test_image_bytes +from tests.serve.conftest import ( + MULTIMODAL_IMG_URL, + MULTIMODAL_VIDEO_H264_URL, + MULTIMODAL_VIDEO_H265_URL, + get_multimodal_test_image_bytes, +) from tests.utils.engine_process import EngineConfig from tests.utils.payload_builder import chat_payload from tests.utils.payloads import BasePayload, CachedTokensChatPayload, ChatPayload +# Same triangle clip the http fixtures use (see tests/serve/conftest.py), so the +# local-file and NVDEC paths describe identical content and can assert the same +# word. Keeping them different subjects is what let expectations drift from the +# footage they were written against. LOCAL_VIDEO_TEST_PATH = Path( - WORKSPACE_DIR, "lib/llm/tests/data/media/240p_10.mp4" + WORKSPACE_DIR, "lib/llm/tests/data/media/triangle_240p_10.mp4" ).resolve() LOCAL_VIDEO_TEST_URI = LOCAL_VIDEO_TEST_PATH.as_uri() @@ -365,6 +374,30 @@ def make_video_payload(expected_response: list[str]) -> ChatPayload: ) +def _make_http_video_payload(url: str, expected_response: list[str]) -> ChatPayload: + return chat_payload( + [ + {"type": "text", "text": "Describe the video in detail"}, + {"type": "video_url", "video_url": {"url": url}}, + ], + repeat_count=1, + expected_response=expected_response, + temperature=0.0, + max_tokens=100, + ) + + +def make_h264_video_payload(expected_response: list[str]) -> ChatPayload: + """Video payload whose clip is fetched over http so the request exercises the + NVDEC hardware-decode path (H.264). file:// video is not hardware-decoded.""" + return _make_http_video_payload(MULTIMODAL_VIDEO_H264_URL, expected_response) + + +def make_hevc_video_payload(expected_response: list[str]) -> ChatPayload: + """H.265/HEVC counterpart of make_h264_video_payload (NVDEC hardware decode).""" + return _make_http_video_payload(MULTIMODAL_VIDEO_H265_URL, expected_response) + + def make_audio_payload(expected_response: list[str]) -> ChatPayload: """Standard audio transcription payload using the remote test WAV.""" return chat_payload(