diff --git a/components/src/dynamo/common/multimodal/codec_errors.py b/components/src/dynamo/common/multimodal/codec_errors.py index a5473288a93f..8d5ff5ed308d 100644 --- a/components/src/dynamo/common/multimodal/codec_errors.py +++ b/components/src/dynamo/common/multimodal/codec_errors.py @@ -20,8 +20,11 @@ from __future__ import annotations +import importlib +import importlib.metadata + from dynamo.common.multimodal.nvdec_decoder import HW_ROUTED_CODECS, nvdec_available -from dynamo.common.utils.install_media_decoders import VALIDATED_SPECS +from dynamo.common.utils.install_media_decoders import VALIDATED_SPECS, installer_covers INSTALLER_CMD = "python -m dynamo.common.utils.install_media_decoders" @@ -35,14 +38,71 @@ class MissingMediaDecoderError(RuntimeError): """ -def _install_hint(backend: str, package: str) -> str: - spec = VALIDATED_SPECS[package] +def _carrier_present(module: str) -> bool: + """Return whether the carrier imports; keep import errors out of diagnostics.""" + try: + importlib.import_module(module) + except Exception: # noqa: BLE001 - any import failure means unusable here + return False + return True + + +def _needs_binary_wheel(backend: str, package: str) -> bool: + """Whether this package must come from a wheel: vLLM's OpenCV sdist has no codecs.""" + return backend == "vllm" and package == "opencv-python-headless" + + +def _is_vllm_source_built_cv2(backend: str, package: str, module: str) -> bool: + """Whether this is the codec-free OpenCV build shipped by vLLM.""" return ( - f"install the validated decoder with `pip install --no-deps '{spec}'` " - f"(or `{INSTALLER_CMD} {backend}`)" + _needs_binary_wheel(backend, package) + and module == "cv2" + and _carrier_present(module) ) +def _installed_version(package: str) -> str | None: + """The installed distribution version, or None when it has no metadata.""" + try: + return importlib.metadata.version(package) + except importlib.metadata.PackageNotFoundError: + return None + + +def _install_hint(backend: str, package: str, module: str) -> str: + """Use the installed OpenCV version for vLLM replacements, validated bounds otherwise.""" + if _is_vllm_source_built_cv2(backend, package, module): + installed = _installed_version(package) + if installed: + hint = ( + "replace the shipped build with the binary wheel of the same " + f"version: `pip install --no-deps --force-reinstall " + f"--only-binary {package} '{package}=={installed}'`" + ) + else: + # cv2 omits the distribution packaging revision. + hint = ( + "first determine the shipped cv2 version with " + '`python -c "import cv2; print(cv2.__version__)"`, then replace ' + "it with the binary wheel built from that same version -- the " + "distribution adds a packaging revision, so match on the prefix: " + f"`pip install --no-deps --force-reinstall --only-binary {package} " + f"'{package}==.*'`" + ) + else: + only_binary = ( + f"--only-binary {package} " if _needs_binary_wheel(backend, package) else "" + ) + hint = ( + "install the validated decoder with " + f"`pip install --no-deps --force-reinstall " + f"{only_binary}'{VALIDATED_SPECS[package]}'`" + ) + if installer_covers(backend, package): + hint += f" (or `{INSTALLER_CMD} {backend}`)" + return hint + + def _with_cause(message: str, cause: str | None) -> str: """Append the underlying decoder text so diagnostics survive the wrap. @@ -79,6 +139,13 @@ def video_decoder_missing( "but NVDEC is unavailable in this container. Grant the 'video' " "driver capability (NVIDIA_DRIVER_CAPABILITIES) to enable it, or " ) + elif _is_vllm_source_built_cv2(backend, package, module): + lead = ( + f"this video ({codec_desc}) has no decoder in this image: shipped " + "images decode only H.264/H.265 (in hardware, via NVDEC), and the " + f"'{module}' they ship is built without a video backend. " + "Re-encode the input to H.264/H.265, or " + ) else: lead = ( f"this video ({codec_desc}) has no decoder in this image: shipped " @@ -88,7 +155,10 @@ def video_decoder_missing( ) return MissingMediaDecoderError( _with_cause( - "Cannot decode video: " + lead + _install_hint(backend, package) + ".", + "Cannot decode video: " + + lead + + _install_hint(backend, package, module) + + ".", cause, ) ) @@ -106,7 +176,7 @@ def audio_decoder_missing( _with_cause( "Cannot decode audio: this input needs the PyAV decoder ('av'), which " "this image deliberately does not ship, and NVDEC does not decode " - "audio. To enable audio input, " + _install_hint(backend, "av") + ".", + "audio. To enable audio input, " + _install_hint(backend, "av", "av") + ".", cause, ) ) diff --git a/components/src/dynamo/common/multimodal/video_loader.py b/components/src/dynamo/common/multimodal/video_loader.py index 6c39c0d51fc9..f9134305003c 100644 --- a/components/src/dynamo/common/multimodal/video_loader.py +++ b/components/src/dynamo/common/multimodal/video_loader.py @@ -14,8 +14,10 @@ # limitations under the License. import asyncio +import functools import logging import os +import re from typing import Any, Awaitable, Dict, Final, List from urllib.parse import urlparse @@ -45,6 +47,34 @@ logger = logging.getLogger(__name__) +def _attributable_to_cv2(exc: BaseException, media_io: Any | None = None) -> bool: + """Identify OpenCV errors without relabeling failures from other video backends.""" + if getattr(exc, "name", None) == "cv2" or "cv2" in str(exc): + return True + return ( + isinstance(exc, ValueError) + and str(exc) == "Could not open video stream" + and (getattr(media_io, "kwargs", None) or {}).get("backend", "opencv") + == "opencv" + ) + + +@functools.lru_cache(maxsize=1) +def _cv2_lacks_video_backend() -> bool: + """Return whether OpenCV is broken or lacks FFmpeg and GStreamer.""" + try: + import cv2 + except ImportError: + return False + except Exception: # noqa: BLE001 - a broken import has no usable video backend + return True + build_info = cv2.getBuildInformation() + return not any( + re.search(rf"^\s*{backend}:\s*YES", build_info, re.MULTILINE | re.IGNORECASE) + for backend in ("FFMPEG", "GSTREAMER") + ) + + URL_VARIANT_KEY: Final = "Url" DECODED_VARIANT_KEY: Final = "Decoded" @@ -165,13 +195,8 @@ async def _decode_video_bytes( ) -> tuple[np.ndarray, Dict[str, Any]]: """Decode video bytes: H.264/H.265 on NVDEC, all else software. - The runtime images purge the software decode wheels (opencv/av/decord/ - torchcodec) for codec compliance, so the software fallback only - resolves where a decoder was installed separately. When it is absent, - vLLM's lazy import surfaces a bare ``No module named 'cv2'`` with no - codec and no remedy -- convert that into the actionable - unsupported-codec error, which can name the codec because the probe - already ran here. + vLLM ships OpenCV for still images without video backends. Translate + its decode failures into install guidance; preserve other backends. The NVDEC path honors the two ``media_io_kwargs`` that decide *which* frames come back -- ``num_frames`` and ``fps`` -- because it samples @@ -198,9 +223,22 @@ async def _decode_video_bytes( try: return await asyncio.to_thread(media_io.load_bytes, content) except ImportError as exc: + if not _attributable_to_cv2(exc): + raise raise video_decoder_missing( "vllm", "opencv-python-headless", "cv2", codec, cause=str(exc) ) from exc + except (OSError, SystemError, ValueError) as exc: + # Confirm both the error source and missing video support. + if not (_attributable_to_cv2(exc, media_io) and _cv2_lacks_video_backend()): + raise + raise video_decoder_missing( + "vllm", + "opencv-python-headless", + "cv2", + codec, + cause=str(exc), + ) from exc def _extract_nvdec_args(self, media_io: Any) -> dict[str, Any]: num_frames = getattr(media_io, "num_frames", self._num_frames) diff --git a/components/src/dynamo/common/tests/multimodal/conftest.py b/components/src/dynamo/common/tests/multimodal/conftest.py index 5fa2acd9b033..c9b7e8524975 100644 --- a/components/src/dynamo/common/tests/multimodal/conftest.py +++ b/components/src/dynamo/common/tests/multimodal/conftest.py @@ -15,10 +15,15 @@ import importlib +import pytest import pytest_asyncio from dynamo.common.http import close_http_client +# The decode carriers ``codec_errors`` probes. Only these names are answered by +# the ``carrier_imports`` fixture; every other import runs for real. +_MEDIA_CARRIERS = ("cv2", "av", "decord") + # Cached results of probing optional deps used by multimodal unit tests. # `None` = not attempted, `True` = importable, `False` = raised. _torch_importable: bool | None = None @@ -70,6 +75,32 @@ def pytest_ignore_collect(collection_path, config) -> bool | None: return None +@pytest.fixture +def carrier_imports(monkeypatch): + """Mock media-carrier imports while leaving unrelated imports intact.""" + # Import after collection's optional-dependency guard: the multimodal + # package imports torch, which CPU-only runtime images may not ship. + from dynamo.common.multimodal import codec_errors + + def _set( + present: tuple[str, ...] = (), error: str | BaseException | None = None + ) -> None: + real = importlib.import_module + + def _fake(name: str, *args, **kwargs): + if name in _MEDIA_CARRIERS: + if name in present: + return object() + if isinstance(error, BaseException): + raise error + raise ImportError(error or f"No module named '{name}'") + return real(name, *args, **kwargs) + + monkeypatch.setattr(codec_errors.importlib, "import_module", _fake) + + return _set + + @pytest_asyncio.fixture(autouse=True) async def _close_shared_http_client(): yield diff --git a/components/src/dynamo/common/tests/multimodal/test_audio_loader.py b/components/src/dynamo/common/tests/multimodal/test_audio_loader.py index 81497dd38f54..11e6634e1fcf 100644 --- a/components/src/dynamo/common/tests/multimodal/test_audio_loader.py +++ b/components/src/dynamo/common/tests/multimodal/test_audio_loader.py @@ -6,9 +6,9 @@ import numpy as np import pytest -import dynamo.common.multimodal.audio_loader as audio_loader_module from dynamo.common.http import HttpStatusError from dynamo.common.http.url_validator import UrlValidationError, UrlValidationPolicy +from dynamo.common.multimodal import audio_loader as audio_loader_module from dynamo.common.multimodal.audio_loader import AudioLoader from dynamo.common.multimodal.codec_errors import MissingMediaDecoderError from dynamo.common.utils.install_media_decoders import VALIDATED_SPECS @@ -187,11 +187,14 @@ async def test_load_audio_batch_reads_decoded_variant(monkeypatch): @pytest.mark.asyncio -async def test_load_audio_missing_decoder_is_actionable(): +async def test_load_audio_missing_decoder_is_actionable(carrier_imports): """vLLM's own hint here is `pip install vllm[audio]`, which drags in an unpinned stack; the wrap must point at the validated bounded install and say there is no hardware alternative for audio.""" loader = AudioLoader() + # 'av' genuinely absent, independent of what the test machine has + # installed: _install_hint asks the running interpreter. + carrier_imports() loader._load_audio_with_vllm = AsyncMock( # type: ignore[method-assign] side_effect=ImportError("Please install vllm[audio] for audio support") ) diff --git a/components/src/dynamo/common/tests/multimodal/test_codec_errors.py b/components/src/dynamo/common/tests/multimodal/test_codec_errors.py index 5e2d45755a35..1a4c60341a3e 100644 --- a/components/src/dynamo/common/tests/multimodal/test_codec_errors.py +++ b/components/src/dynamo/common/tests/multimodal/test_codec_errors.py @@ -5,7 +5,7 @@ import pytest -import dynamo.common.multimodal.codec_errors as codec_errors +from dynamo.common.multimodal import codec_errors from dynamo.common.multimodal.codec_errors import ( MissingMediaDecoderError, audio_decoder_missing, @@ -16,7 +16,13 @@ pytestmark = [pytest.mark.unit, pytest.mark.pre_merge, pytest.mark.gpu_0] -def test_video_message_names_codec_spec_and_installer(monkeypatch): +@pytest.fixture(autouse=True) +def _carrier_absent(carrier_imports): + """Make missing-carrier messages independent of installed packages.""" + carrier_imports() + + +def test_video_message_names_codec_and_spec(monkeypatch): monkeypatch.setattr(codec_errors, "nvdec_available", lambda: True) err = video_decoder_missing("vllm", "opencv-python-headless", "cv2", "vp9") @@ -32,6 +38,61 @@ def test_video_message_names_codec_spec_and_installer(monkeypatch): assert "H.264/H.265" in msg +def test_present_but_unusable_carrier_pins_the_installed_version( + monkeypatch, carrier_imports +): + """Replacing codec-free OpenCV must preserve the installed version.""" + carrier_imports(present=("cv2",)) + monkeypatch.setattr( + codec_errors.importlib.metadata, "version", lambda p: "5.0.0.93" + ) + msg = str(video_decoder_missing("vllm", "opencv-python-headless", "cv2", "vp9")) + assert "--force-reinstall" in msg + assert "--only-binary opencv-python-headless" in msg + assert "opencv-python-headless==5.0.0.93" in msg + assert VALIDATED_SPECS["opencv-python-headless"] not in msg + + +def test_present_carrier_without_metadata_requests_its_version( + monkeypatch, carrier_imports +): + """Without metadata, match the library version plus its packaging revision.""" + carrier_imports(present=("cv2",)) + + def _missing(_package): + raise codec_errors.importlib.metadata.PackageNotFoundError + + monkeypatch.setattr(codec_errors.importlib.metadata, "version", _missing) + msg = str(video_decoder_missing("vllm", "opencv-python-headless", "cv2", "vp9")) + assert "print(cv2.__version__)" in msg + assert "opencv-python-headless==.*" in msg + assert VALIDATED_SPECS["opencv-python-headless"] not in msg + assert "--force-reinstall" in msg + + +def test_broken_carrier_is_not_treated_as_the_source_build(carrier_imports): + """Broken imports need a reinstall, not a missing-video-backend diagnosis.""" + carrier_imports(error=OSError("libGL.so.1: cannot open shared object file")) + + msg = str( + video_decoder_missing( + "vllm", + "opencv-python-headless", + "cv2", + "vp9", + cause="libGL.so.1: cannot open shared object file", + ) + ) + assert "built without a video backend" not in msg + assert VALIDATED_SPECS["opencv-python-headless"] in msg + assert "--force-reinstall" in msg + # Without this, pip may rebuild the codec-free sdist the operator is + # trying to replace. + assert "--only-binary opencv-python-headless" in msg + # The carrier's own words are what identify the real fault. + assert "libGL.so.1" in msg + + def test_hw_codec_without_nvdec_points_at_driver_capability(monkeypatch): """H.264 with NVDEC unavailable is a capability problem first. diff --git a/components/src/dynamo/common/tests/multimodal/test_video_loader.py b/components/src/dynamo/common/tests/multimodal/test_video_loader.py index f45c47a5ab8f..9f213ce0e873 100644 --- a/components/src/dynamo/common/tests/multimodal/test_video_loader.py +++ b/components/src/dynamo/common/tests/multimodal/test_video_loader.py @@ -1,14 +1,16 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import sys from unittest.mock import AsyncMock import numpy as np import pytest -import dynamo.common.multimodal.video_loader as video_loader_module from dynamo.common.http import HttpStatusError from dynamo.common.http.url_validator import UrlValidationError, UrlValidationPolicy +from dynamo.common.multimodal import codec_errors +from dynamo.common.multimodal import video_loader as video_loader_module from dynamo.common.multimodal.codec_errors import MissingMediaDecoderError from dynamo.common.multimodal.video_loader import VideoLoader from dynamo.common.utils.install_media_decoders import VALIDATED_SPECS @@ -287,7 +289,9 @@ def load_bytes(self, content: bytes): @pytest.mark.asyncio -async def test_decode_video_bytes_missing_decoder_is_actionable(monkeypatch): +async def test_decode_video_bytes_missing_decoder_is_actionable( + monkeypatch, carrier_imports +): """A bare `No module named 'cv2'` must become the actionable codec error. Reproduced on a real runtime image before this existed: the user-visible @@ -297,6 +301,8 @@ async def test_decode_video_bytes_missing_decoder_is_actionable(monkeypatch): loader = VideoLoader() monkeypatch.setattr(video_loader_module, "probe_video_codec", lambda b: "vp9") monkeypatch.setattr(video_loader_module, "should_use_nvdec", lambda c: False) + # cv2 genuinely absent, independent of what the test machine has installed. + carrier_imports() with pytest.raises(MissingMediaDecoderError) as exc_info: await loader._decode_video_bytes(b"vp9-bytes", _ImportErrorMediaIO()) @@ -304,10 +310,197 @@ async def test_decode_video_bytes_missing_decoder_is_actionable(monkeypatch): msg = str(exc_info.value) assert "'vp9'" in msg # names the codec assert VALIDATED_SPECS["opencv-python-headless"] in msg # bounded spec - assert "install_media_decoders vllm" in msg # installer command + assert "pip install" in msg assert "cv2" in msg +class _SystemErrorMediaIO: + """What a cv2 with no video backend does: fail inside VideoCapture.""" + + def load_bytes(self, content: bytes): + raise SystemError( + " returned a result with an exception set" + ) + + +@pytest.mark.asyncio +async def test_decode_video_bytes_backendless_cv2_is_actionable( + monkeypatch, carrier_imports +): + """Translate codec-free OpenCV failures into actionable decoder guidance.""" + loader = VideoLoader() + monkeypatch.setattr(video_loader_module, "probe_video_codec", lambda b: "vp9") + monkeypatch.setattr(video_loader_module, "should_use_nvdec", lambda c: False) + monkeypatch.setattr(video_loader_module, "_cv2_lacks_video_backend", lambda: True) + + carrier_imports(present=("cv2",)) + monkeypatch.setattr( + codec_errors.importlib.metadata, "version", lambda p: "5.0.0.93" + ) + + with pytest.raises(MissingMediaDecoderError) as exc_info: + await loader._decode_video_bytes(b"vp9-bytes", _SystemErrorMediaIO()) + + msg = str(exc_info.value) + assert "'vp9'" in msg + assert "opencv-python-headless==5.0.0.93" in msg + assert "--force-reinstall" in msg + assert "video backend" in msg + + +class _FailedOpenMediaIO: + """What OpenCV's own video backend raises when VideoCapture opens nothing.""" + + def load_bytes(self, content: bytes): + raise ValueError("Could not open video stream") + + +@pytest.mark.asyncio +async def test_decode_video_bytes_failed_open_value_error_is_actionable( + monkeypatch, carrier_imports +): + """Translate OpenCV failed-open errors when its video backend is absent.""" + loader = VideoLoader() + monkeypatch.setattr(video_loader_module, "probe_video_codec", lambda b: "vp9") + monkeypatch.setattr(video_loader_module, "should_use_nvdec", lambda c: False) + monkeypatch.setattr(video_loader_module, "_cv2_lacks_video_backend", lambda: True) + + carrier_imports(present=("cv2",)) + monkeypatch.setattr( + codec_errors.importlib.metadata, "version", lambda p: "5.0.0.93" + ) + + with pytest.raises(MissingMediaDecoderError) as exc_info: + await loader._decode_video_bytes(b"vp9-bytes", _FailedOpenMediaIO()) + + msg = str(exc_info.value) + assert "'vp9'" in msg + assert "opencv-python-headless==5.0.0.93" in msg + assert "--force-reinstall" in msg + assert "video backend" in msg + + +class _InitErrorMediaIO: + """What a lazily initialised cv2 raises when its own libraries fail to load.""" + + def load_bytes(self, content: bytes): + raise OSError("cv2: libavcodec.so.60: cannot open shared object file") + + +@pytest.mark.asyncio +async def test_decode_video_bytes_cv2_os_error_is_actionable( + monkeypatch, carrier_imports +): + """Keep the remedy visible when cv2 initialisation fails inside the decode.""" + loader = VideoLoader() + monkeypatch.setattr(video_loader_module, "probe_video_codec", lambda b: "vp9") + monkeypatch.setattr(video_loader_module, "should_use_nvdec", lambda c: False) + monkeypatch.setattr(video_loader_module, "_cv2_lacks_video_backend", lambda: True) + + carrier_imports(present=("cv2",)) + monkeypatch.setattr( + codec_errors.importlib.metadata, "version", lambda p: "5.0.0.93" + ) + + with pytest.raises(MissingMediaDecoderError) as exc_info: + await loader._decode_video_bytes(b"vp9-bytes", _InitErrorMediaIO()) + + msg = str(exc_info.value) + assert "'vp9'" in msg + assert "opencv-python-headless==5.0.0.93" in msg + assert "--force-reinstall" in msg + + +@pytest.mark.asyncio +async def test_unrelated_os_error_is_not_blamed_on_opencv(monkeypatch): + """An I/O failure that never mentions the carrier keeps its own error.""" + loader = VideoLoader() + monkeypatch.setattr(video_loader_module, "probe_video_codec", lambda b: "vp9") + monkeypatch.setattr(video_loader_module, "should_use_nvdec", lambda c: False) + monkeypatch.setattr(video_loader_module, "_cv2_lacks_video_backend", lambda: True) + + class _DiskFullMediaIO: + def load_bytes(self, content: bytes): + raise OSError("[Errno 28] No space left on device: '/tmp/clip.mp4'") + + with pytest.raises(OSError, match="No space left"): + await loader._decode_video_bytes(b"vp9-bytes", _DiskFullMediaIO()) + + +@pytest.mark.asyncio +async def test_backendless_cv2_does_not_pre_empt_a_working_decode(monkeypatch): + """Allow configured video backends to decode before checking OpenCV.""" + loader = VideoLoader() + monkeypatch.setattr(video_loader_module, "probe_video_codec", lambda b: "vp9") + monkeypatch.setattr(video_loader_module, "should_use_nvdec", lambda c: False) + monkeypatch.setattr(video_loader_module, "_cv2_lacks_video_backend", lambda: True) + + frames = np.zeros((2, 4, 4, 3), dtype=np.uint8) + + class _WorkingMediaIO: + def load_bytes(self, content: bytes): + return frames, {"frames_indices": [0, 1]} + + decoded, metadata = await loader._decode_video_bytes( + b"vp9-bytes", _WorkingMediaIO() + ) + assert decoded is frames + assert metadata == {"frames_indices": [0, 1]} + + +@pytest.mark.asyncio +async def test_other_backend_import_error_is_not_blamed_on_opencv(monkeypatch): + """Preserve import errors from other video backends.""" + loader = VideoLoader() + monkeypatch.setattr(video_loader_module, "probe_video_codec", lambda b: "vp9") + monkeypatch.setattr(video_loader_module, "should_use_nvdec", lambda c: False) + + class _PyAvMediaIO: + def load_bytes(self, content: bytes): + raise ModuleNotFoundError("No module named 'av'", name="av") + + with pytest.raises(ModuleNotFoundError, match="'av'"): + await loader._decode_video_bytes(b"vp9-bytes", _PyAvMediaIO()) + + +@pytest.mark.asyncio +async def test_other_backend_system_error_is_not_blamed_on_opencv(monkeypatch): + """Preserve other backend failures even when OpenCV lacks video support.""" + loader = VideoLoader() + monkeypatch.setattr(video_loader_module, "probe_video_codec", lambda b: "vp9") + monkeypatch.setattr(video_loader_module, "should_use_nvdec", lambda c: False) + monkeypatch.setattr(video_loader_module, "_cv2_lacks_video_backend", lambda: True) + + class _OtherBackendMediaIO: + def load_bytes(self, content: bytes): + raise SystemError("torchcodec decoder failed to initialise") + + with pytest.raises(SystemError, match="torchcodec"): + await loader._decode_video_bytes(b"vp9-bytes", _OtherBackendMediaIO()) + + +@pytest.mark.asyncio +async def test_unrelated_system_error_keeps_its_own_message(monkeypatch): + """A SystemError from anywhere else must not be blamed on the codec.""" + loader = VideoLoader() + monkeypatch.setattr(video_loader_module, "probe_video_codec", lambda b: "vp9") + monkeypatch.setattr(video_loader_module, "should_use_nvdec", lambda c: False) + monkeypatch.setattr(video_loader_module, "_cv2_lacks_video_backend", lambda: False) + + with pytest.raises(SystemError, match="VideoCapture"): + await loader._decode_video_bytes(b"vp9-bytes", _SystemErrorMediaIO()) + + +def test_cv2_lacks_video_backend_is_false_when_cv2_absent(monkeypatch): + """An absent cv2 is not this case: the ImportError path names it better.""" + video_loader_module._cv2_lacks_video_backend.cache_clear() + monkeypatch.setitem(sys.modules, "cv2", None) + try: + assert video_loader_module._cv2_lacks_video_backend() is False + finally: + video_loader_module._cv2_lacks_video_backend.cache_clear() + + @pytest.mark.asyncio async def test_load_video_batch_preserves_missing_decoder_error(): """The batch aggregate wraps failures in a generic Exception; the diff --git a/components/src/dynamo/common/tests/test_install_media_decoders.py b/components/src/dynamo/common/tests/test_install_media_decoders.py index 03576888e075..b9e608a693a3 100644 --- a/components/src/dynamo/common/tests/test_install_media_decoders.py +++ b/components/src/dynamo/common/tests/test_install_media_decoders.py @@ -35,6 +35,11 @@ def sandboxed(monkeypatch): monkeypatch.setattr( install_media_decoders, "_modules_missing_fresh", lambda mods: list(mods) ) + monkeypatch.setattr( + install_media_decoders, + "_vllm_opencv_spec", + lambda: install_media_decoders.VALIDATED_SPECS["opencv-python-headless"], + ) return monkeypatch @@ -133,21 +138,26 @@ def test_already_present_installs_nothing(sandboxed): assert calls == [] -def test_vllm_installs_bounded_video_and_audio_specs(sandboxed): +def test_vllm_installs_video_and_audio(sandboxed): present: set[str] = set() _set_available(sandboxed, present) + sandboxed.setattr( + install_media_decoders, + "_vllm_opencv_spec", + lambda: "opencv-python-headless==5.0.0.93", + ) calls = _record_pip_and_mark(sandboxed, present, "cv2", "av") installed = install_media_decoders.install_media_decoders("vllm") assert installed == [ - install_media_decoders.VALIDATED_SPECS["opencv-python-headless"], + "opencv-python-headless==5.0.0.93", install_media_decoders.VALIDATED_SPECS["av"], ] (cmd,) = calls assert "--break-system-packages" in cmd + assert "--force-reinstall" in cmd + assert cmd[cmd.index("--only-binary") + 1] == "opencv-python-headless" for spec in installed: assert spec in cmd - # Never installed: pynvvideocodec because the image already ships it as the - # NVDEC path, and the rest because no vLLM decode path imports them. for banned in ("torchcodec", "pynvvideocodec", "decord2", "libx264"): assert not any(banned in part for part in cmd) @@ -174,16 +184,6 @@ def test_trtllm_installs_opencv_only(sandboxed): assert install_media_decoders.VALIDATED_SPECS["av"] not in cmd -def test_installs_only_missing_modules(sandboxed): - present = {"cv2"} # video carrier present, audio missing - _set_available(sandboxed, present) - calls = _record_pip_and_mark(sandboxed, present, "av") - installed = install_media_decoders.install_media_decoders("vllm") - assert installed == [install_media_decoders.VALIDATED_SPECS["av"]] - (cmd,) = calls - assert install_media_decoders.VALIDATED_SPECS["opencv-python-headless"] not in cmd - - def test_every_install_uses_no_deps(sandboxed): """--no-deps is unconditional now that custom specs are out of scope -- the installer must never be able to shift the image's pinned stack.""" @@ -256,29 +256,19 @@ def test_dry_run_reports_without_installing(sandboxed): assert calls == [] -def test_pending_subset_installs_only_still_missing(sandboxed): - """A racing process may install part of the set while we wait on the lock. - - Probe round 1 (pre-check) sees both vLLM carriers missing; round 2 - (post-lock re-check) sees cv2 already installed by the racing process, so - only the audio carrier installs; round 3 (post-verify) sees everything. - """ +def test_racing_install_between_probe_and_lock_runs_no_pip(sandboxed): + """Skip pip when a concurrent installer makes both decoders usable.""" rounds = {"n": 0} def probe(mods): rounds["n"] += 1 - if rounds["n"] == 1: - return list(mods) - if rounds["n"] == 2: - return [m for m in mods if m != "cv2"] - return [] + return list(mods) if rounds["n"] == 1 else [] sandboxed.setattr(install_media_decoders, "_modules_missing_fresh", probe) calls = _record_pip(sandboxed) - installed = install_media_decoders.install_media_decoders("vllm") - assert installed == [install_media_decoders.VALIDATED_SPECS["av"]] - (cmd,) = calls - assert install_media_decoders.VALIDATED_SPECS["opencv-python-headless"] not in cmd + + assert install_media_decoders.install_media_decoders("vllm") == [] + assert calls == [] def test_modules_missing_fresh_real_probe(): @@ -310,6 +300,32 @@ def test_probe_treats_present_but_broken_package_as_missing(tmp_path, monkeypatc assert install_media_decoders._modules_missing_fresh(["brokenmod"]) == ["brokenmod"] +def test_opencv_spec_falls_back_when_the_version_probe_fails(monkeypatch): + """An unreadable OpenCV version must not abort the install. + + A cv2 that raises on import (rather than being absent) kills the probe + child, and check=True turns that into CalledProcessError in the caller. + The bounded spec is the right answer there, exactly as for an empty probe. + """ + + def _boom(cmd, **kwargs): + raise subprocess.CalledProcessError(1, cmd) + + monkeypatch.setattr(install_media_decoders.subprocess, "run", _boom) + assert ( + install_media_decoders._vllm_opencv_spec() + == install_media_decoders.VALIDATED_SPECS["opencv-python-headless"] + ) + + +def test_opencv_version_probe_survives_a_broken_cv2(tmp_path, monkeypatch): + """The probe child must exit cleanly when importing cv2 raises.""" + (tmp_path / "cv2.py").write_text("raise RuntimeError('native libs gone')\n") + monkeypatch.setenv("PYTHONPATH", str(tmp_path)) + spec = install_media_decoders._vllm_opencv_spec() + assert spec.startswith("opencv-python-headless") + + def test_lock_refuses_symlink_and_preserves_target(tmp_path, monkeypatch): """A pre-planted symlink at the fixed lock path must not be followed. diff --git a/components/src/dynamo/common/utils/install_media_decoders.py b/components/src/dynamo/common/utils/install_media_decoders.py index 4a4140c7d2b6..2261ffabd205 100644 --- a/components/src/dynamo/common/utils/install_media_decoders.py +++ b/components/src/dynamo/common/utils/install_media_decoders.py @@ -38,8 +38,9 @@ broadens the codec surface of the running container; review that against your organization's distribution and security policies before rolling it out. -Each package installs at a version validated against Dynamo's multimodal test -suite (see the specs below): the lower bound is the exact validated version and +For vLLM, codec-free OpenCV is replaced with a binary wheel matching the installed +version. Other packages install at a version validated against Dynamo's multimodal +test suite (see the specs below): the lower bound is the exact validated version and the upper bound excludes the next major, so a fresh install cannot silently pick up an unvalidated release. This installer deliberately covers ONLY those tested combinations; to pin different versions or install a custom subset, run @@ -49,7 +50,7 @@ The default install runs with ``--no-deps`` so it cannot change the image's pinned dependency stack (e.g. numpy under torch/vLLM); the carriers need only numpy, which the backend already provides. The install is idempotent (skipped -when the module already imports), serialized across processes with a file +when imports and OpenCV video support are available), serialized with a file lock, and bounded by a timeout so a stalled index cannot hang forever. The encode path (video output) is unaffected and stays on the in-tree FFmpeg. The optional Rust frontend decoder links FFmpeg's compiled-in decoders and is @@ -140,33 +141,25 @@ class _Decoder: } +def installer_covers(backend: str, package: str) -> bool: + """Return whether the backend installer covers this package.""" + return any(d.package == package for d in _BACKEND_DECODERS.get(backend, ())) + + def _modules_missing_fresh(modules: Sequence[str]) -> list[str]: - """Return the subset of `modules` a FRESH interpreter cannot IMPORT. - - Fresh interpreter, not this process: running as a non-root user, pip - defaults to a user-site install, and a user-site directory created after - the parent interpreter started is never added to its ``sys.path`` - (``site.py`` only does that at startup, and - ``importlib.invalidate_caches()`` cannot add path entries). Verified on - all three runtime images: the same-process check reported the install - missing while a fresh process imported it fine. What matters - operationally is the worker process launched after this command -- - which is exactly a fresh interpreter. - - Real import, not ``find_spec``: a package whose files are present but - whose native libraries cannot load (a broken or partially removed wheel) - has a spec and would be treated as installed, silently skipping the - install and deferring the failure to request time. Importing in the - probe subprocess keeps this process's module state untouched. - """ + """Check imports and OpenCV video support in a fresh worker-like interpreter.""" if not modules: return [] probe = ( - "import importlib, sys\n" + "import importlib, re, sys\n" "missing = []\n" "for m in sys.argv[1:]:\n" " try:\n" - " importlib.import_module(m)\n" + " module = importlib.import_module(m)\n" + " if m == 'cv2' and not re.search(\n" + " r'^\\s*(FFMPEG|GSTREAMER):\\s*YES',\n" + " module.getBuildInformation(), re.M | re.I):\n" + " missing.append(m)\n" " except Exception:\n" " missing.append(m)\n" "print(' '.join(missing))\n" @@ -185,6 +178,42 @@ def _modules_missing_fresh(modules: Sequence[str]) -> list[str]: return out.stdout.split() +def _vllm_opencv_spec() -> str: + """Preserve the image's OpenCV version when replacing its source build.""" + probe = ( + "import importlib.metadata\n" + "version = ''\n" + "for package in ('opencv-python-headless', 'opencv-python'):\n" + " try:\n" + " version = importlib.metadata.version(package)\n" + " break\n" + " except importlib.metadata.PackageNotFoundError:\n" + " pass\n" + "if not version:\n" + " try:\n" + " import cv2\n" + # A cv2 that raises anything is as unusable as a missing one. + " except Exception:\n" + " pass\n" + " else:\n" + " version = cv2.__version__ + '.*'\n" + "print(version)\n" + ) + try: + out = subprocess.run( + [sys.executable, "-c", probe], + capture_output=True, + text=True, + timeout=120, + check=True, + ) + except Exception as exc: # noqa: BLE001 - an unreadable version is not fatal + logger.warning("OpenCV version probe failed to run: %s", exc) + return _OPENCV.spec + version = out.stdout.strip() + return f"{_OPENCV.package}=={version}" if version else _OPENCV.spec + + def _redact(text: str) -> str: """Mask URL userinfo (user:token@host) so pip args never leak secrets.""" return _CRED_RE.sub(r"\1***@", text) @@ -253,9 +282,9 @@ def install_media_decoders( ) -> list[str]: """Install `backend`'s media-decoder package(s); return the specs installed. - Installs the backend's validated, version-bounded specs with ``--no-deps``, - skipping any package whose module already imports. Deliberately covers only - the tested combinations: for custom versions or subsets, run pip directly. + Installs with ``--no-deps`` and skips usable decoders. vLLM OpenCV is + replaced with a same-version binary wheel; other packages use validated + bounds. For custom versions or subsets, run pip directly. Returns the list of pip specs actually installed (empty when everything was already present). ``dry_run`` returns what would install without running @@ -267,9 +296,7 @@ def install_media_decoders( raise ValueError( f"unknown backend {backend!r}; expected one of {sorted(_BACKEND_DECODERS)}" ) - # Install only what does not already import cleanly. The probe does a - # real import in a fresh interpreter, so a present-but-broken package - # counts as missing rather than being silently skipped. + # Include import failures and OpenCV builds without a video backend. missing_names = set(_modules_missing_fresh([d.module for d in decoders])) missing = [d for d in decoders if d.module in missing_names] if not missing: @@ -278,10 +305,18 @@ def install_media_decoders( backend, ) return [] - specs = [d.spec for d in missing] + specs = [ + _vllm_opencv_spec() if backend == "vllm" and d.module == "cv2" else d.spec + for d in missing + ] verify_modules = [d.module for d in missing] if dry_run: + if backend == "vllm" and "cv2" in verify_modules: + logger.info( + "dry run: OpenCV replacement requires " + "--force-reinstall --only-binary opencv-python-headless" + ) logger.info( "dry run: would install for %s: %s", backend, _redact(" ".join(specs)) ) @@ -306,13 +341,17 @@ def install_media_decoders( return [] specs = [spec for spec, _ in pending] verify_modules = [mod for _, mod in pending] - _pip_install(specs, pip_args, timeout_s=timeout_s) + extra_args = list(pip_args) + if backend == "vllm" and "cv2" in verify_modules: + specs[verify_modules.index("cv2")] = _vllm_opencv_spec() + extra_args += ["--force-reinstall", "--only-binary", _OPENCV.package] + _pip_install(specs, extra_args, timeout_s=timeout_s) importlib.invalidate_caches() still_missing = _modules_missing_fresh(verify_modules) if still_missing: raise RuntimeError( - "media decoder module(s) still not importable after install: " + "media decoder module(s) still not importable or video-capable after install: " + " ".join(still_missing) + ". If a package is present but broken, pip may have treated the " "requirement as already satisfied -- pip uninstall it and rerun, " @@ -329,8 +368,8 @@ def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser( prog="python -m dynamo.common.utils.install_media_decoders", description=( - "Install a Dynamo backend's media-decoder package(s) at validated, " - "version-bounded releases. Explicit by design: nothing installs " + "Install a Dynamo backend's media decoders, preserving vLLM's OpenCV " + "version and using validated bounds otherwise. Nothing installs " "unless an operator runs this." ), ) @@ -348,7 +387,6 @@ def main(argv: list[str] | None = None) -> int: 'string. Use the = form -- --pip-args="--no-index --find-links ' '/wheels" -- so a value starting with a dash is not mistaken ' "for an option (for air-gapped hosts)" - "air-gapped hosts)" ), ) parser.add_argument( diff --git a/container/templates/vllm_runtime.Dockerfile b/container/templates/vllm_runtime.Dockerfile index 2454a4befb82..51d923226ef9 100644 --- a/container/templates/vllm_runtime.Dockerfile +++ b/container/templates/vllm_runtime.Dockerfile @@ -468,13 +468,19 @@ RUN rm -rf /workspace/vllm # Remove the codec-bearing video-DECODE wheels inherited from the vllm-openai # base. Each bundles its own full ffmpeg carrying software H.264/H.265/AAC; # PyAV and decord additionally ship GPL libx264/libx265. Dynamo's vLLM component -# imports none of the removed wheels, so they are unused decode-side dead weight. +# keeps OpenCV for mistral_common's cv2.resize, rebuilt at the base image's +# version with video backends disabled. Other codec-bearing wheels are removed. # (PyNvVideoCodec is KEPT for NVDEC hardware decode -- see the note below.) The in-tree # LGPL ffmpeg + imageio-ffmpeg installed above are intentionally KEPT for the # omni video-encode path, which uses the royalty-free VP9 (libvpx_vp9) encoder — # no H.264 is built. Direct rm makes the removal robust regardless of how the # base image's pip is configured; the guards fail the build if any of them survive. RUN set -eux; \ + OPENCV_VERSION="$(python3 -m pip show opencv-python-headless 2>/dev/null | awk '/^Version:/{print $2}')"; \ + if [ -z "${OPENCV_VERSION}" ]; then \ + OPENCV_VERSION="$(python3 -m pip show opencv-python 2>/dev/null | awk '/^Version:/{print $2}')"; \ + fi; \ + test -n "${OPENCV_VERSION}" || { echo "ERROR: base image must provide version metadata for opencv-python-headless or opencv-python" >&2; exit 1; }; \ python3 -m pip uninstall --yes \ av decord decord2 opencv-python opencv-python-headless torchcodec \ || true; \ @@ -492,7 +498,14 @@ RUN set -eux; \ ! python3 -c "import cv2" 2>/dev/null; \ ! python3 -c "import av" 2>/dev/null; \ ! python3 -c "import decord" 2>/dev/null; \ - ! python3 -c "import torchcodec" 2>/dev/null + ! python3 -c "import torchcodec" 2>/dev/null; \ + ENABLE_HEADLESS=1 ENABLE_CONTRIB=0 MAKEFLAGS="-j$(nproc)" \ + CMAKE_ARGS="-DWITH_FFMPEG=OFF -DWITH_GSTREAMER=OFF -DVIDEOIO_ENABLE_PLUGINS=OFF -DWITH_1394=OFF -DWITH_V4L=OFF -DBUILD_TESTS=OFF -DBUILD_PERF_TESTS=OFF -DBUILD_EXAMPLES=OFF -DBUILD_opencv_apps=OFF -DENABLE_CCACHE=OFF" \ + python3 -m pip install --no-binary opencv-python-headless "opencv-python-headless==${OPENCV_VERSION}"; \ + rm -rf /root/.cache/pip; \ + python3 -c "import cv2; cv2.resize"; \ + ! ls -d "${SITE_PACKAGES}"/opencv_python*.libs 2>/dev/null; \ + python3 -c "import cv2,re,sys; enabled=[name for name,value in re.findall(r'^\s*(FFMPEG|GSTREAMER):\s*(\S+)', cv2.getBuildInformation(), re.M|re.I) if value.upper()=='YES']; sys.exit('ERROR: cv2 was built with video backends: '+', '.join(enabled) if enabled else 0)" # PyNvVideoCodec is KEPT (removed from the purge above) but UPGRADED to >=2.2.0 by # the requirements install: the base image's 2.0.4 bundles a full FFmpeg (incl. diff --git a/docs/fern/pages/use-cases/multimodal-serving/additional-media-decoders.md b/docs/fern/pages/use-cases/multimodal-serving/additional-media-decoders.md index 36bb82eda59b..d0f3d370c981 100644 --- a/docs/fern/pages/use-cases/multimodal-serving/additional-media-decoders.md +++ b/docs/fern/pages/use-cases/multimodal-serving/additional-media-decoders.md @@ -26,20 +26,20 @@ Each backend decodes such input through a specific Python package whose wheel bu | Backend | Input | Package (validated version bounds) | Import | |---------|-------|------------------------------------|--------| -| vLLM | video | `opencv-python-headless>=4.13.0.92,<5` | `cv2` | +| vLLM | video | `opencv-python-headless`, matching the installed version | `cv2` | | vLLM | audio | `av>=18.0.0,<19` | `av` | | SGLang | video | `decord2>=3.4.0,<4` | `decord` | | TensorRT-LLM | video | `opencv-python-headless>=4.13.0.92,<5` | `cv2` | -The lower bound of each spec is the version validated against Dynamo's multimodal test suite; the upper bound excludes the next major release so an install cannot silently pick up an unvalidated version. PyNvVideoCodec is not in this list because every CUDA runtime image already ships it — it is the NVDEC path, not a fallback, so there is nothing to install. torchcodec is left out because no Dynamo decode path imports it. +For vLLM video, the installer replaces codec-free OpenCV with a binary wheel of the installed version. If OpenCV is absent, it uses `opencv-python-headless>=4.13.0.92,<5`. For the bounded specs, the lower bound is the version validated against Dynamo's multimodal test suite; the upper bound excludes the next major release so an install cannot silently pick up an unvalidated version. PyNvVideoCodec is not in this list because every CUDA runtime image already ships it — it is the NVDEC path, not a fallback, so there is nothing to install. torchcodec is left out because no Dynamo decode path imports it. ## Install with pip The table above is the contract; these commands are its direct translation, and work with the installer of your choice (`pip`, `uv pip`, ...): ```bash -# vLLM: video + audio input -pip install --no-deps 'opencv-python-headless>=4.13.0.92,<5' 'av>=18.0.0,<19' +# vLLM: audio input +pip install --no-deps 'av>=18.0.0,<19' # SGLang: video input pip install --no-deps 'decord2>=3.4.0,<4' @@ -52,13 +52,13 @@ pip install --no-deps 'opencv-python-headless>=4.13.0.92,<5' ## Install with the bundled installer -For exactly the tested combinations above, every runtime image also ships an installer as part of the `ai-dynamo` package. Run it in the worker container (not the frontend) for the backend you deploy: +Every runtime image ships an installer for the packages above as part of the `ai-dynamo` package. Run it in the worker container (not the frontend) for the backend you deploy: ```bash python -m dynamo.common.utils.install_media_decoders vllm ``` -Compared to the raw pip commands it adds: skip-if-already-importable, a cross-process lock for concurrent runs, a post-install import verification in a fresh interpreter, and a non-zero exit if anything did not land. It deliberately installs only the validated specs — it has no package-selection flags. To see what it would do first: +It skips usable decoders, serializes concurrent installs, and verifies imports in a fresh interpreter. For OpenCV, it also checks for a video backend. On vLLM, it replaces codec-free OpenCV with a same-version binary wheel using `--force-reinstall --only-binary` and installs missing PyAV. Other packages use the validated bounds. It exits non-zero if verification fails. To see what it would do first: ```bash python -m dynamo.common.utils.install_media_decoders vllm --dry-run @@ -98,4 +98,21 @@ The default pip timeout is 600 seconds (`--timeout-s` overrides it; `0` disables - For H.264 and H.265, prefer NVDEC. Granting the container the `video` driver capability decodes those formats on the GPU with no extra package. Install a software decoder when that is not an option, or when the input is audio. - Installing a decoder package brings in that wheel's bundled media libraries. The runtime images are scanned for media components at build time; a package installed afterwards is not covered by that scan. Review what your deployment ships — a baked image layer keeps the change visible and reviewable. - On TensorRT-LLM, the install puts back `opencv-python-headless`, which those images deliberately do not ship. H.264 and H.265 already decode there through NVDEC, so install it only for a host where NVDEC is unavailable. +- The vLLM images ship OpenCV already, rebuilt from source with every video backend disabled. It covers still images — which multimodal Mistral models need, because `mistral_common` resizes every image through `cv2` — and decodes no video at all. Video input on vLLM goes through NVDEC. To decode video in software instead, swap that build for the PyPI wheel of the same version: + +```bash +VERSION=$(pip show opencv-python-headless | awk '/^Version:/{print $2}') +if [ -n "$VERSION" ]; then + SPEC="opencv-python-headless==${VERSION}" +# Without metadata, match the library version plus its packaging revision. +elif VERSION=$(python -c 'import cv2; print(cv2.__version__)') && [ -n "$VERSION" ]; then + SPEC="opencv-python-headless==${VERSION}.*" +else + SPEC='opencv-python-headless>=4.13.0.92,<5' +fi +pip install --no-deps --force-reinstall --only-binary opencv-python-headless "$SPEC" +``` + +`--force-reinstall --only-binary` replaces the source build even when its version already satisfies pip. Pinning preserves the image's OpenCV version; with neither metadata nor an importable `cv2`, the command falls back to the bounded range, which is what the bundled installer does too. The wheel restores bundled FFmpeg and codecs, so review it against your distribution policy. The bundled vLLM installer performs this replacement and installs missing PyAV. + - The optional Rust frontend decoder (`--frontend-decoding`) links FFmpeg's compiled-in decoders and always decodes VP8/VP9 regardless of installed Python packages; backend decoding is what an install extends. Re-encoding an input to VP9 (`ffmpeg -i input.mp4 -c:v libvpx-vp9 -an output.webm`) is an alternative that needs no additional packages. diff --git a/docs/fern/pages/use-cases/multimodal-serving/video-decode-gpu-requirements.md b/docs/fern/pages/use-cases/multimodal-serving/video-decode-gpu-requirements.md index 0a77b9ec92b2..fa41b29c14f4 100644 --- a/docs/fern/pages/use-cases/multimodal-serving/video-decode-gpu-requirements.md +++ b/docs/fern/pages/use-cases/multimodal-serving/video-decode-gpu-requirements.md @@ -12,8 +12,9 @@ Other formats — VP8, VP9 and AV1 — have **no video-input decoder** in the sh The in-tree VP8/VP9 FFmpeg serves the video *output* (generation) path; it is not wired to video input, and the Rust `media-ffmpeg` decoder is not built into these images. Video input decodes through Python carriers (OpenCV, PyAV, decord) that the images deliberately -omit, so a VP8/VP9/AV1 clip fails with an unsupported-codec error unless one of those -packages is installed alongside. +omit — the vLLM images do ship OpenCV, but built without any video backend, for still-image +work only — so a VP8/VP9/AV1 clip fails with an unsupported-codec error unless a carrier +that decodes video is installed alongside. This page covers which GPUs provide NVDEC, what the container must expose, and how Dynamo behaves when hardware decode is unavailable. @@ -133,8 +134,9 @@ to the software decode path where one exists. > [!IMPORTANT] > In the shipped images there is no software decode path for video input, for any format. > The Python carriers that decode video input (OpenCV, PyAV, decord) are deliberately not -> installed, and the in-tree VP8/VP9 FFmpeg serves the video *output* path rather than -> input. So if NVDEC is unavailable, H.264 and H.265 fail with an unsupported-codec error +> installed — the vLLM images ship OpenCV built without a video backend, which resizes +> still images and opens no video — and the in-tree VP8/VP9 FFmpeg serves the video +> *output* path rather than input. So if NVDEC is unavailable, H.264 and H.265 fail with an unsupported-codec error > — and VP8, VP9 and AV1 fail the same way whether NVDEC is available or not, since NVDEC > does not decode them either. > @@ -148,8 +150,8 @@ To decode a format NVDEC does not cover — or H.264/H.265 on a host with no NVD explicitly install the backend's decode package at the validated version bounds: ```bash -# vLLM: video + audio input -pip install --no-deps 'opencv-python-headless>=4.13.0.92,<5' 'av>=18.0.0,<19' +# vLLM: audio input +pip install --no-deps 'av>=18.0.0,<19' # SGLang: video input pip install --no-deps 'decord2>=3.4.0,<4' @@ -158,9 +160,30 @@ pip install --no-deps 'decord2>=3.4.0,<4' pip install --no-deps 'opencv-python-headless>=4.13.0.92,<5' ``` +vLLM **video** input is the exception, because those images already ship OpenCV built +without a video backend. Adding video decode there means replacing that build with the +binary wheel of the same version, not installing a range: + +```bash +VERSION=$(pip show opencv-python-headless | awk '/^Version:/{print $2}') +if [ -n "$VERSION" ]; then + SPEC="opencv-python-headless==${VERSION}" +# Without metadata, match the library version plus its packaging revision. +elif VERSION=$(python -c 'import cv2; print(cv2.__version__)') && [ -n "$VERSION" ]; then + SPEC="opencv-python-headless==${VERSION}.*" +else + SPEC='opencv-python-headless>=4.13.0.92,<5' +fi +pip install --no-deps --force-reinstall --only-binary opencv-python-headless "$SPEC" +``` + +With neither pip metadata nor an importable `cv2`, this falls back to the bounded range instead of +an unresolvable `==.*` pin. + Nothing installs automatically — this is a deliberate operator step. The images also ship an installer with the same bounds plus idempotency and air-gap support -(`python -m dynamo.common.utils.install_media_decoders `); see +(`python -m dynamo.common.utils.install_media_decoders `). For vLLM, it replaces +codec-free OpenCV with a same-version binary wheel and installs missing PyAV. See [Additional Media Decoders](additional-media-decoders.md) for the full workflow, including baking the install into an image layer for Kubernetes. diff --git a/pyproject.toml b/pyproject.toml index 50c0f3d73ccc..1a19cc35cef3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -459,6 +459,12 @@ module = ["PyNvVideoCodec", "PyNvVideoCodec.*"] follow_imports = "skip" ignore_missing_imports = true +[[tool.mypy.overrides]] +# OpenCV is optional and absent from the lint environment. +module = ["cv2", "cv2.*"] +follow_imports = "skip" +ignore_missing_imports = true + [[tool.mypy.overrides]] # WAR mypy 1.18.x crash with numpy 1.26.x stubs: # "Should never get here in normal mode, got TypeAlias:numpy.float64 instead of TypeInfo" diff --git a/tests/dependencies/test_no_software_video_codecs.py b/tests/dependencies/test_no_software_video_codecs.py index 841ef1580b3d..4916015e2b99 100644 --- a/tests/dependencies/test_no_software_video_codecs.py +++ b/tests/dependencies/test_no_software_video_codecs.py @@ -57,7 +57,9 @@ # H.264/H.265 via NVDEC; audio has no hardware path and is opt-in only. # Mirrors the build-time guard's pattern in wheel_builder.Dockerfile: `aac` is # anchored to a word start so it cannot match inside an unrelated identifier. -_DISALLOWED_RE = re.compile(r"h\.?264|h\.?265|hevc|(?:^|\s)aac|nvenc|cuvid|nvdec", re.I) +_DISALLOWED_RE = re.compile( + r"h\.?264|h\.?265|hevc|(?:^|\s)aac|nvenc|cuvid|nvdec", re.IGNORECASE +) # Present by construction, so a broken FFmpeg cannot masquerade as a pass. _REQUIRED = ("vp9",) @@ -125,14 +127,19 @@ def _assert_no_software_codecs(surfaces: dict[str, str]) -> None: ) -def _assert_python_carriers_absent() -> None: +def _assert_python_carriers_absent(ships_cv2: bool) -> None: """The Python decode carriers are absent from an unmodified image. They are installable at runtime behind an opt-in switch, and tests that do so are marked ``installs_extra_dependencies``. An unmodified image must not have them, or a green multimodal run says nothing about what customers receive. + + vLLM's codec-free OpenCV is checked by _assert_cv2_carries_no_codec. """ - for module, package in (("cv2", "opencv-python"), ("av", "PyAV")): + carriers = [("av", "PyAV")] + if not ships_cv2: + carriers.append(("cv2", "opencv-python")) + for module, package in carriers: spec = importlib.util.find_spec(module) assert spec is None, ( f"{package} is installed ({module} at {spec.origin}); an unmodified " @@ -140,6 +147,32 @@ def _assert_python_carriers_absent() -> None: ) +def _assert_cv2_carries_no_codec() -> None: + """Reject video backends and vendored media libraries in shipped OpenCV.""" + import cv2 + + build_info = cv2.getBuildInformation() + for backend in ("FFMPEG", "GSTREAMER"): + match = re.search( + rf"^\s*{backend}:\s*(\S+)", build_info, re.MULTILINE | re.IGNORECASE + ) + assert not (match and match.group(1).upper() == "YES"), ( + f"the shipped cv2 was built with {backend}; it must be built with " + f"-DWITH_{backend}=OFF so the image carries no software video codec" + ) + + vendored = [ + os.path.join(root, name) + for root in {p for p in sys.path if p and os.path.isdir(p)} + for name in os.listdir(root) + if name.startswith("opencv_python") and name.endswith(".libs") + ] + assert not vendored, ( + f"cv2 ships vendored media libraries ({', '.join(vendored)}); the image " + "is expected to carry the source build, which vendors none" + ) + + def _bundled_libavcodecs() -> list[str]: """Every third-party libavcodec on disk, excluding our own in-tree copy. @@ -228,11 +261,13 @@ def _assert_bundled_libavcodecs_carry_no_software_codecs() -> None: ) -def _check_image() -> None: +def _check_image(ships_cv2: bool = False) -> None: surfaces = _surfaces() _assert_required_codecs_present(surfaces) _assert_no_software_codecs(surfaces) - _assert_python_carriers_absent() + _assert_python_carriers_absent(ships_cv2) + if ships_cv2: + _assert_cv2_carries_no_codec() _assert_bundled_libavcodecs_carry_no_software_codecs() @@ -244,7 +279,10 @@ def _check_image() -> None: @pytest.mark.vllm def test_vllm_image_ships_no_software_video_codecs() -> None: - _check_image() + import torch + + # Only the CUDA image rebuilds OpenCV; no visible GPU is required here. + _check_image(ships_cv2=torch.version.cuda is not None) @pytest.mark.sglang