From cd97b1a2042debe555ead977c95d0c56761b67ab Mon Sep 17 00:00:00 2001 From: Harrison King Saturley-Hall Date: Thu, 16 Jul 2026 21:35:09 -0400 Subject: [PATCH 1/2] fix(vllm): use Dynamo FFmpeg for media bindings Signed-off-by: Harrison King Saturley-Hall --- .../deps/vllm/requirements.media-source.txt | 10 + .../deps/vllm/validate_ffmpeg_provider.py | 209 ++++++++++++++++++ container/templates/vllm_runtime.Dockerfile | 124 ++++++++++- 3 files changed, 331 insertions(+), 12 deletions(-) create mode 100644 container/deps/vllm/requirements.media-source.txt create mode 100644 container/deps/vllm/validate_ffmpeg_provider.py diff --git a/container/deps/vllm/requirements.media-source.txt b/container/deps/vllm/requirements.media-source.txt new file mode 100644 index 000000000000..1def1c1c7469 --- /dev/null +++ b/container/deps/vllm/requirements.media-source.txt @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Build these distributions from source against Dynamo's allowlisted FFmpeg. +# The hashes pin the PyPI source archives; binary wheels must never be used. + +av==18.0.0 \ + --hash=sha256:4ef7e72c3d3a872584a1215173b16e0226811037f40dcdbf75992631098df1ba +opencv-python==5.0.0.93 \ + --hash=sha256:66aac3e5b5faa48d4025816592f3af19e4bfc2c68dec067bae2dbb4ca10aa9e2 diff --git a/container/deps/vllm/validate_ffmpeg_provider.py b/container/deps/vllm/validate_ffmpeg_provider.py new file mode 100644 index 000000000000..79b44db38add --- /dev/null +++ b/container/deps/vllm/validate_ffmpeg_provider.py @@ -0,0 +1,209 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Validate that vLLM uses only Dynamo's allowlisted FFmpeg installation.""" + +import importlib +import importlib.metadata +import os +import re +import subprocess +from pathlib import Path + +FFMPEG = Path("/usr/local/bin/ffmpeg") +ALLOWED_LIBRARY_ROOT = Path("/usr/local/lib") +SEARCH_ROOTS = (Path("/usr"), Path("/opt"), Path("/workspace")) +ALLOWED_DECODERS = {"rawvideo", "vp8", "vp9"} +ALLOWED_ENCODERS = {"h264_nvenc", "libvpx_vp9"} +REQUIRED_BUILD_CONFIGURATION = { + "--disable-bsfs", + "--disable-decoders", + "--disable-demuxers", + "--disable-encoders", + "--disable-parsers", + "--disable-protocols", + "--enable-decoder=vp8,vp9,rawvideo", + "--enable-demuxer=mov,matroska,rawvideo", + "--enable-encoder=h264_nvenc,libvpx_vp9", + "--enable-parser=vp8,vp9", +} +FORBIDDEN_DISTRIBUTIONS = { + "decord", + "decord2", + "opencv-python-headless", + "pynvvideocodec", +} +MEDIA_LIBRARY_RE = re.compile( + r"^(?:libav(?:codec|device|filter|format|util)|libpostproc|" + r"libsw(?:resample|scale)|libx26[45]|libopenh264|libfdk-aac|libfaac|" + r"libvo-aacenc|libaacplus).*\.so(?:\..*)?$" +) +MEDIA_BINARY_RE = re.compile(r"^ff(?:mpeg|probe)(?:[-_].*)?$") +COMPONENT_RE = re.compile(r"^[A-Z.]{6}\s+(\S+)") + + +def run(*command: str) -> str: + """Run a validation command and return stdout.""" + + return subprocess.run( + command, + check=True, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ).stdout + + +def normalize_distribution_name(name: str) -> str: + return re.sub(r"[-_.]+", "-", name).lower() + + +def installed_distributions() -> dict[str, importlib.metadata.Distribution]: + return { + normalize_distribution_name(distribution.metadata["Name"]): distribution + for distribution in importlib.metadata.distributions() + if distribution.metadata["Name"] + } + + +def ffmpeg_components(kind: str) -> set[str]: + output = run(str(FFMPEG), "-hide_banner", f"-{kind}") + components = set() + for line in output.splitlines(): + match = COMPONENT_RE.match(line.strip()) + if match is not None: + components.add(match.group(1)) + return components + + +def unexpected_media_artifacts() -> list[Path]: + artifacts = [] + for root in SEARCH_ROOTS: + for directory, _, filenames in os.walk(root): + directory_path = Path(directory) + if directory_path.is_relative_to(Path("/usr/local/src/ffmpeg")): + continue + for filename in filenames: + path = directory_path / filename + if MEDIA_LIBRARY_RE.match(filename): + if not path.resolve().is_relative_to(ALLOWED_LIBRARY_ROOT): + artifacts.append(path) + elif MEDIA_BINARY_RE.match(filename) and os.access(path, os.X_OK): + if path.resolve() != FFMPEG: + artifacts.append(path) + return artifacts + + +def extension_libraries(module_name: str, pattern: str = "*.so") -> list[Path]: + module = importlib.import_module(module_name) + module_file = Path(module.__file__).resolve() + package_root = module_file if module_file.suffix == ".so" else module_file.parent + if package_root.is_file(): + return [package_root] + return sorted(package_root.rglob(pattern)) + + +def assert_links_to_dynamo_ffmpeg(module_name: str, pattern: str = "*.so") -> None: + extensions = extension_libraries(module_name, pattern) + if not extensions: + raise RuntimeError(f"{module_name} contains no extension libraries to inspect") + + resolved_media_libraries = set() + for extension in extensions: + for line in run("ldd", str(extension)).splitlines(): + if not re.search(r"lib(?:av|sw)", line): + continue + if "not found" in line: + raise RuntimeError( + f"unresolved FFmpeg dependency for {extension}: {line}" + ) + match = re.search(r"=>\s+(/\S+)", line) + if match is None: + continue + resolved = Path(match.group(1)).resolve() + resolved_media_libraries.add(resolved) + if not resolved.is_relative_to(ALLOWED_LIBRARY_ROOT): + raise RuntimeError( + f"{extension} loads FFmpeg outside {ALLOWED_LIBRARY_ROOT}: {resolved}" + ) + + if not resolved_media_libraries: + raise RuntimeError(f"{module_name} does not link to Dynamo's FFmpeg libraries") + + +def main() -> None: + if not FFMPEG.is_file(): + raise RuntimeError(f"missing Dynamo FFmpeg executable: {FFMPEG}") + + expected_version = os.environ["EXPECTED_FFMPEG_VERSION"] + first_version_line = run(str(FFMPEG), "-version").splitlines()[0] + if not first_version_line.startswith(f"ffmpeg version {expected_version}"): + raise RuntimeError( + f"expected FFmpeg {expected_version}, found: {first_version_line}" + ) + + build_configuration = run(str(FFMPEG), "-buildconf") + missing_configuration = REQUIRED_BUILD_CONFIGURATION - set( + build_configuration.split() + ) + if missing_configuration: + raise RuntimeError( + "FFmpeg is missing required allowlist configuration: " + + ", ".join(sorted(missing_configuration)) + ) + + decoders = ffmpeg_components("decoders") + if decoders != ALLOWED_DECODERS: + raise RuntimeError( + f"unexpected FFmpeg decoder set: expected {sorted(ALLOWED_DECODERS)}, " + f"found {sorted(decoders)}" + ) + + encoders = ffmpeg_components("encoders") + if encoders != ALLOWED_ENCODERS: + raise RuntimeError( + f"unexpected FFmpeg encoder set: expected {sorted(ALLOWED_ENCODERS)}, " + f"found {sorted(encoders)}" + ) + + distributions = installed_distributions() + unexpected_distributions = FORBIDDEN_DISTRIBUTIONS & distributions.keys() + if unexpected_distributions: + raise RuntimeError( + "codec-bearing upstream distributions remain installed: " + + ", ".join(sorted(unexpected_distributions)) + ) + for required_distribution in ("av", "opencv-python", "torchcodec"): + if required_distribution not in distributions: + raise RuntimeError( + f"required distribution is missing: {required_distribution}" + ) + + unexpected_artifacts = unexpected_media_artifacts() + if unexpected_artifacts: + raise RuntimeError( + "FFmpeg or prohibited codec artifacts remain outside Dynamo's " + "installation:\n" + + "\n".join(str(path) for path in sorted(unexpected_artifacts)) + ) + + for module_name in ("av", "cv2"): + assert_links_to_dynamo_ffmpeg(module_name) + ffmpeg_major = expected_version.partition(".")[0] + assert_links_to_dynamo_ffmpeg("torchcodec", f"libtorchcodec_core{ffmpeg_major}.so") + + imageio_ffmpeg = importlib.import_module("imageio_ffmpeg") + selected_ffmpeg = Path(imageio_ffmpeg.get_ffmpeg_exe()).resolve() + if selected_ffmpeg != FFMPEG: + raise RuntimeError( + f"imageio-ffmpeg selected {selected_ffmpeg}, expected {FFMPEG}" + ) + + print( + f"validated FFmpeg {expected_version}: one provider, " + f"decoders={sorted(decoders)}, encoders={sorted(encoders)}" + ) + + +if __name__ == "__main__": + main() diff --git a/container/templates/vllm_runtime.Dockerfile b/container/templates/vllm_runtime.Dockerfile index 2b922fa4e8f5..61731d69289a 100644 --- a/container/templates/vllm_runtime.Dockerfile +++ b/container/templates/vllm_runtime.Dockerfile @@ -10,11 +10,55 @@ {% if platform == "multi" %} FROM --platform=linux/amd64 ${RUNTIME_IMAGE}:${RUNTIME_IMAGE_TAG} AS vllm_runtime_amd64 FROM --platform=linux/arm64 ${RUNTIME_IMAGE}:${RUNTIME_IMAGE_TAG} AS vllm_runtime_arm64 -FROM vllm_runtime_${TARGETARCH} AS pre_runtime +FROM vllm_runtime_${TARGETARCH} AS vllm_runtime_base {% else %} -FROM ${RUNTIME_IMAGE}:${RUNTIME_IMAGE_TAG} AS pre_runtime +FROM ${RUNTIME_IMAGE}:${RUNTIME_IMAGE_TAG} AS vllm_runtime_base {% endif %} +{% if device == "cuda" %} +# The upstream PyAV and OpenCV wheels bundle their own FFmpeg libraries. Build +# replacements against Dynamo's /usr/local FFmpeg so the runtime has exactly one +# libav provider and one audited codec surface. +FROM vllm_runtime_base AS vllm_media_wheel_builder +USER root +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + build-essential \ + cmake \ + ninja-build \ + pkg-config \ + python3-dev && \ + rm -rf /var/lib/apt/lists/* + +RUN --mount=type=bind,from=wheel_builder,source=/usr/local/,target=/tmp/usr/local/ \ + rm -rf /usr/local/include/libav* /usr/local/include/libsw* && \ + rm -f /usr/local/lib/libav*.so* /usr/local/lib/libsw*.so* /usr/local/lib/lib*vpx*.so* && \ + rm -f /usr/local/lib/pkgconfig/libav*.pc /usr/local/lib/pkgconfig/libsw*.pc && \ + mkdir -p /usr/local/include /usr/local/lib/pkgconfig && \ + cp -rL /tmp/usr/local/include/libav* /tmp/usr/local/include/libsw* /usr/local/include/ && \ + cp -L /tmp/usr/local/lib/libav*.so* /tmp/usr/local/lib/libsw*.so* /usr/local/lib/ && \ + find /tmp/usr/local/lib -maxdepth 1 -name 'lib*vpx*.so*' -exec cp -L {} /usr/local/lib/ \; && \ + cp -L /tmp/usr/local/lib/pkgconfig/libav*.pc /tmp/usr/local/lib/pkgconfig/libsw*.pc /usr/local/lib/pkgconfig/ && \ + ldconfig + +ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig \ + LD_LIBRARY_PATH=/usr/local/lib +RUN --mount=type=bind,source=./container/deps/vllm/requirements.media-source.txt,target=/tmp/requirements.media-source.txt,readonly \ + --mount=type=cache,target=/root/.cache/pip,sharing=locked \ + set -eux; \ + export CMAKE_BUILD_PARALLEL_LEVEL="$(nproc)"; \ + export CMAKE_PREFIX_PATH=/usr/local; \ + export CMAKE_ARGS="-DWITH_FFMPEG=ON -DOPENCV_FFMPEG_USE_FIND_PACKAGE=OFF -DOPENCV_FFMPEG_ENABLE_LIBAVDEVICE=OFF -DOPENCV_FFMPEG_SKIP_DOWNLOAD=ON -DWITH_GSTREAMER=OFF -DWITH_QT=OFF -DWITH_GTK=OFF -DWITH_WIN32UI=OFF -DCMAKE_BUILD_RPATH=/usr/local/lib -DCMAKE_INSTALL_RPATH=/usr/local/lib"; \ + python3 -m pip wheel \ + --wheel-dir /opt/vllm-media-wheels \ + --no-deps \ + --no-binary=av,opencv-python \ + --require-hashes \ + --requirement /tmp/requirements.media-source.txt +{% endif %} + +FROM vllm_runtime_base AS pre_runtime + ARG PYTHON_VERSION ARG ENABLE_KVBM ARG ENABLE_GPU_MEMORY_SERVICE @@ -22,6 +66,7 @@ ARG VLLM_OMNI_REF ARG NIXL_REF {% if device == "cuda" %} ARG CUDA_MAJOR +ARG FFMPEG_VERSION {% endif %} ARG MODELEXPRESS_VERSION @@ -211,10 +256,11 @@ RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked \ # against libx264/libx265/libmp3lame. Purge ONLY the explicitly-named ffmpeg + # codec packages and replace them with the LGPL-only in-tree ffmpeg built in # wheel_builder (--disable-gpl --disable-nonfree; H.264 via NVENC, VP9 via -# libvpx). PyAV, torchaudio, torchvision, soundfile and Pillow all bundle their -# own libraries and do not link the system ffmpeg/codecs, so removing them is -# safe. dpkg-query keeps the match robust across base-image/arch version -# suffixes (e.g. libavcodec58 vs 60). +# libvpx). PyAV and OpenCV are handled separately below because their upstream +# wheels carry private FFmpeg libraries that apt cannot see. torchaudio, +# torchvision, soundfile, and Pillow do not link libav. dpkg-query keeps the +# package match robust across base-image/arch version suffixes (for example, +# libavcodec58 versus libavcodec60). # # This grep is the COMPLETE, auditable set of what leaves the image: there is # deliberately NO apt-get autoremove, so the removal can never cascade into @@ -248,15 +294,59 @@ RUN --mount=type=bind,source=./container/deps/vllm/validate_torch_compile_smoke. # was just purged, so the LGPL CLI must always be present for the omni # video-export path to have something to encode with. RUN --mount=type=bind,from=wheel_builder,source=/usr/local/,target=/tmp/usr/local/ \ + rm -rf /usr/local/include/libav* /usr/local/include/libsw* /usr/local/src/ffmpeg && \ + rm -f /usr/local/bin/ffmpeg /usr/local/bin/ffprobe && \ + rm -f /usr/local/lib/libav*.so* /usr/local/lib/libsw*.so* /usr/local/lib/lib*vpx*.so* && \ + rm -f /usr/local/lib/pkgconfig/libav*.pc /usr/local/lib/pkgconfig/libsw*.pc && \ mkdir -p /usr/local/lib/pkgconfig && \ - cp -rnL /tmp/usr/local/include/libav* /tmp/usr/local/include/libsw* /usr/local/include/ && \ - cp -nL /tmp/usr/local/lib/libav*.so* /tmp/usr/local/lib/libsw*.so* /usr/local/lib/ && \ - cp -nL /tmp/usr/local/lib/lib*vpx*.so* /usr/local/lib/ 2>/dev/null || true && \ - cp -nL /tmp/usr/local/lib/pkgconfig/libav*.pc /tmp/usr/local/lib/pkgconfig/libsw*.pc /usr/local/lib/pkgconfig/ && \ - cp -nL /tmp/usr/local/bin/ffmpeg /usr/local/bin/ffmpeg && \ + cp -rL /tmp/usr/local/include/libav* /tmp/usr/local/include/libsw* /usr/local/include/ && \ + cp -L /tmp/usr/local/lib/libav*.so* /tmp/usr/local/lib/libsw*.so* /usr/local/lib/ && \ + find /tmp/usr/local/lib -maxdepth 1 -name 'lib*vpx*.so*' -exec cp -L {} /usr/local/lib/ \; && \ + cp -L /tmp/usr/local/lib/pkgconfig/libav*.pc /tmp/usr/local/lib/pkgconfig/libsw*.pc /usr/local/lib/pkgconfig/ && \ + cp -L /tmp/usr/local/bin/ffmpeg /usr/local/bin/ffmpeg && \ cp -r /tmp/usr/local/src/ffmpeg /usr/local/src/ && \ ldconfig ENV IMAGEIO_FFMPEG_EXE=/usr/local/bin/ffmpeg + +# Replace upstream media wheels after vLLM-Omni has completed dependency +# resolution. PyAV and OpenCV are rebuilt in vllm_media_wheel_builder against +# the shared FFmpeg above; PyNvVideoCodec is removed because it provides a +# separate NVDEC/libav codec path rather than using Dynamo's allowlisted build. +COPY --from=vllm_media_wheel_builder /opt/vllm-media-wheels/ /opt/dynamo/vllm-media-wheels/ +RUN set -eux; \ + python3 -m pip uninstall --yes \ + av \ + decord \ + decord2 \ + opencv-python \ + opencv-python-headless \ + PyNvVideoCodec; \ + SITE_PACKAGES="$(python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"; \ + rm -rf \ + "${SITE_PACKAGES}"/av \ + "${SITE_PACKAGES}"/av-*.dist-info \ + "${SITE_PACKAGES}"/av.libs \ + "${SITE_PACKAGES}"/cv2 \ + "${SITE_PACKAGES}"/decord \ + "${SITE_PACKAGES}"/decord-*.dist-info \ + "${SITE_PACKAGES}"/decord.libs \ + "${SITE_PACKAGES}"/decord2 \ + "${SITE_PACKAGES}"/decord2-*.dist-info \ + "${SITE_PACKAGES}"/decord2.libs \ + "${SITE_PACKAGES}"/opencv_python-*.dist-info \ + "${SITE_PACKAGES}"/opencv_python.libs \ + "${SITE_PACKAGES}"/opencv_python_headless-*.dist-info \ + "${SITE_PACKAGES}"/opencv_python_headless.libs \ + "${SITE_PACKAGES}"/PyNvVideoCodec* \ + "${SITE_PACKAGES}"/pynvvideocodec*; \ + find "${SITE_PACKAGES}" \( -type f -o -type l \) \ + | grep -E '/(libav(codec|device|filter|format|util)|libpostproc|libsw(resample|scale)|libx264|libx265|libopenh264|libfdk-aac|libfaac|libvo-aacenc|libaacplus)[^/]*\.so' \ + | xargs --no-run-if-empty rm -f; \ + uv pip install --system --no-deps \ + /opt/dynamo/vllm-media-wheels/av-*.whl \ + /opt/dynamo/vllm-media-wheels/opencv_python-*.whl; \ + rm -rf /opt/dynamo/vllm-media-wheels /root/.cache/pip /root/.cache/uv; \ + ldconfig {% endif %} # Replace the upstream vllm/vllm-openai image's imageio-ffmpeg (which ships a @@ -268,7 +358,9 @@ RUN --mount=type=bind,source=./container/deps/requirements.vllm.txt,target=/tmp/ --mount=type=cache,target=/root/.cache/uv,sharing=locked \ export UV_CACHE_DIR=/root/.cache/uv && \ uv pip install {{ pip_target }} --reinstall-package imageio-ffmpeg --no-deps \ - --requirement /tmp/requirements.vllm.txt + --requirement /tmp/requirements.vllm.txt && \ + SITE_PACKAGES="$(python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')" && \ + rm -rf "${SITE_PACKAGES}"/imageio_ffmpeg/binaries # Remove the vLLM source tree shipped in the base image to avoid pytest # collection conflicts (duplicate conftest plugin registration) and stale @@ -295,6 +387,14 @@ RUN --mount=type=bind,source=./container/launch_message/runtime.txt,target=/opt/ chmod 755 /opt/dynamo/.launch_screen && \ echo 'cat /opt/dynamo/.launch_screen' >> /etc/bash.bashrc +{% if device == "cuda" %} +# Keep this after every package install and workspace COPY so a later layer +# cannot silently restore an upstream FFmpeg or broaden the codec surface. +RUN --mount=type=bind,source=./container/deps/vllm/validate_ffmpeg_provider.py,target=/tmp/validate_ffmpeg_provider.py,readonly \ + EXPECTED_FFMPEG_VERSION="${FFMPEG_VERSION}" \ + python3 /tmp/validate_ffmpeg_provider.py +{% endif %} + USER dynamo ARG DYNAMO_COMMIT_SHA From 9fb746e3a2f2093c3ea1c78ffd0dcc18e923d07e Mon Sep 17 00:00:00 2001 From: Harrison King Saturley-Hall Date: Thu, 16 Jul 2026 21:48:17 -0400 Subject: [PATCH 2/2] fix(vllm): remove H.264 NVENC support Signed-off-by: Harrison King Saturley-Hall --- .../src/dynamo/vllm/omni/output_formatter.py | 24 ++++++------ .../vllm/tests/omni/test_output_formatter.py | 37 +++++++++++++++---- .../deps/vllm/validate_ffmpeg_provider.py | 8 +++- container/templates/vllm_runtime.Dockerfile | 10 ++--- container/templates/wheel_builder.Dockerfile | 29 ++++++++++++--- 5 files changed, 76 insertions(+), 32 deletions(-) diff --git a/components/src/dynamo/vllm/omni/output_formatter.py b/components/src/dynamo/vllm/omni/output_formatter.py index 384a90c2127d..f481d9fca82c 100644 --- a/components/src/dynamo/vllm/omni/output_formatter.py +++ b/components/src/dynamo/vllm/omni/output_formatter.py @@ -11,7 +11,6 @@ import asyncio import base64 import logging -import tempfile import time import uuid from io import BytesIO @@ -20,7 +19,6 @@ import numpy as np import soundfile as sf import torch -from diffusers.utils.export_utils import export_to_video from dynamo.common.protocols.audio_protocol import AudioData, NvAudioSpeechResponse from dynamo.common.protocols.image_protocol import ImageData, NvImagesResponse @@ -28,7 +26,10 @@ from dynamo.common.storage import upload_to_fs from dynamo.common.utils.engine_response import normalize_finish_reason from dynamo.common.utils.output_modalities import RequestType -from dynamo.common.utils.video_utils import normalize_video_frames +from dynamo.common.utils.video_utils import ( + encode_to_video_bytes, + normalize_video_frames, +) from dynamo.vllm.handlers import build_prompt_tokens_details from dynamo.vllm.omni.utils import is_empty_payload @@ -129,24 +130,25 @@ async def _encode_video( response_format: Optional[str] = None, output_format: Optional[str] = None, ) -> Dict[str, Any] | None: - output_format = output_format or "mp4" + output_format = output_format or "webm" response_format = response_format or "url" if response_format not in ("url", "b64_json"): raise ValueError( f"Unsupported response_format: {response_format!r}; expected 'url' or 'b64_json'" ) - if output_format != "mp4": + if output_format != "webm": raise ValueError( - f"Unsupported output_format: {output_format!r}; only 'mp4' is supported" + f"Unsupported output_format: {output_format!r}; only 'webm' is supported" ) try: start_time = time.time() frame_list = normalize_video_frames(images) - with tempfile.NamedTemporaryFile( - suffix=f".{output_format}", delete=True - ) as tmp: - await asyncio.to_thread(export_to_video, frame_list, tmp.name, fps) - video_bytes = tmp.read() + video_bytes = await asyncio.to_thread( + encode_to_video_bytes, + np.asarray(frame_list), + fps=fps, + output_format=output_format, + ) if response_format == "b64_json": video_data = VideoData( diff --git a/components/src/dynamo/vllm/tests/omni/test_output_formatter.py b/components/src/dynamo/vllm/tests/omni/test_output_formatter.py index 1945c1eefe06..3c572e41afc0 100644 --- a/components/src/dynamo/vllm/tests/omni/test_output_formatter.py +++ b/components/src/dynamo/vllm/tests/omni/test_output_formatter.py @@ -5,6 +5,7 @@ from unittest.mock import MagicMock, patch +import numpy as np import pytest try: @@ -523,7 +524,7 @@ async def test_audio_data_carries_output_format_url_path(self): class TestDiffusionFormatterVideoOutputFormat: - """_encode_video always sets VideoData.output_format='mp4'.""" + """_encode_video uses WebM/VP9 so vLLM does not require H.264.""" def _patches(self): from unittest.mock import patch as _patch @@ -531,12 +532,15 @@ def _patches(self): return ( _patch( "dynamo.vllm.omni.output_formatter.normalize_video_frames", - return_value=[MagicMock()], + return_value=[np.zeros((2, 2, 3), dtype=np.uint8)], + ), + _patch( + "dynamo.vllm.omni.output_formatter.encode_to_video_bytes", + return_value=b"fake-webm", ), - _patch("dynamo.vllm.omni.output_formatter.export_to_video"), _patch( "dynamo.vllm.omni.output_formatter.upload_to_fs", - return_value="http://x/v.mp4", + return_value="http://x/v.webm", ), _patch( "dynamo.vllm.omni.output_formatter.asyncio.to_thread", @@ -564,8 +568,8 @@ async def test_video_url_response_format(self): ) assert result is not None - assert result["data"][0]["output_format"] == "mp4" - assert result["data"][0]["url"] == "http://x/v.mp4" + assert result["data"][0]["output_format"] == "webm" + assert result["data"][0]["url"] == "http://x/v.webm" assert result["data"][0].get("b64_json") is None mock_upload.assert_called_once() @@ -591,7 +595,7 @@ async def test_video_b64_response_format(self): ) assert result is not None - assert result["data"][0]["output_format"] == "mp4" + assert result["data"][0]["output_format"] == "webm" assert result["data"][0].get("url") is None assert result["data"][0]["b64_json"] is not None base64.b64decode(result["data"][0]["b64_json"]) # must be valid base64 @@ -614,5 +618,22 @@ async def test_video_default_response_format_is_url(self): ) assert result is not None - assert result["data"][0]["url"] == "http://x/v.mp4" + assert result["data"][0]["url"] == "http://x/v.webm" mock_upload.assert_called_once() + + @pytest.mark.asyncio + async def test_video_mp4_output_is_rejected(self): + from dynamo.common.utils.output_modalities import RequestType + from dynamo.vllm.omni.output_formatter import DiffusionFormatter + + f = DiffusionFormatter(model_name="test", media_fs=None, media_http_url=None) + stage = MagicMock() + stage.images = [MagicMock()] + + with pytest.raises(ValueError, match="only 'webm' is supported"): + await f.format( + stage, + "r8", + request_type=RequestType.VIDEO_GENERATION, + output_format="mp4", + ) diff --git a/container/deps/vllm/validate_ffmpeg_provider.py b/container/deps/vllm/validate_ffmpeg_provider.py index 79b44db38add..71ed388d462e 100644 --- a/container/deps/vllm/validate_ffmpeg_provider.py +++ b/container/deps/vllm/validate_ffmpeg_provider.py @@ -14,18 +14,22 @@ ALLOWED_LIBRARY_ROOT = Path("/usr/local/lib") SEARCH_ROOTS = (Path("/usr"), Path("/opt"), Path("/workspace")) ALLOWED_DECODERS = {"rawvideo", "vp8", "vp9"} -ALLOWED_ENCODERS = {"h264_nvenc", "libvpx_vp9"} +ALLOWED_ENCODERS = {"libvpx-vp9"} REQUIRED_BUILD_CONFIGURATION = { "--disable-bsfs", "--disable-decoders", "--disable-demuxers", "--disable-encoders", + "--disable-muxers", "--disable-parsers", "--disable-protocols", "--enable-decoder=vp8,vp9,rawvideo", "--enable-demuxer=mov,matroska,rawvideo", - "--enable-encoder=h264_nvenc,libvpx_vp9", + "--enable-encoder=libvpx_vp9", + "--enable-libvpx", + "--enable-muxer=matroska,webm", "--enable-parser=vp8,vp9", + "--enable-protocol=file,pipe", } FORBIDDEN_DISTRIBUTIONS = { "decord", diff --git a/container/templates/vllm_runtime.Dockerfile b/container/templates/vllm_runtime.Dockerfile index 61731d69289a..c9f33de258a5 100644 --- a/container/templates/vllm_runtime.Dockerfile +++ b/container/templates/vllm_runtime.Dockerfile @@ -255,9 +255,9 @@ RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked \ # The upstream vllm/vllm-openai base image ships a GPL/GPL-3.0 ffmpeg built # against libx264/libx265/libmp3lame. Purge ONLY the explicitly-named ffmpeg + # codec packages and replace them with the LGPL-only in-tree ffmpeg built in -# wheel_builder (--disable-gpl --disable-nonfree; H.264 via NVENC, VP9 via -# libvpx). PyAV and OpenCV are handled separately below because their upstream -# wheels carry private FFmpeg libraries that apt cannot see. torchaudio, +# wheel_builder (--disable-gpl --disable-nonfree; VP9 encoding via libvpx; no +# H.264/H.265/AAC codec). PyAV and OpenCV are handled separately below because +# their upstream wheels carry private FFmpeg libraries that apt cannot see. torchaudio, # torchvision, soundfile, and Pillow do not link libav. dpkg-query keeps the # package match robust across base-image/arch version suffixes (for example, # libavcodec58 versus libavcodec60). @@ -289,10 +289,10 @@ RUN --mount=type=bind,source=./container/deps/vllm/validate_torch_compile_smoke. python3 /tmp/validate_torch_compile_smoke.py # Copy the LGPL ffmpeg from wheel_builder: versioned shared libs (libav*.so*, -# libsw*.so*) + libvpx + the LGPL CLI binary that imageio/diffusers target via +# libsw*.so*) + libvpx + the LGPL CLI binary that imageio targets via # IMAGEIO_FFMPEG_EXE. Ungated by enable_media_ffmpeg because the base GPL ffmpeg # was just purged, so the LGPL CLI must always be present for the omni -# video-export path to have something to encode with. +# WebM/VP9 video-export path to have something to encode with. RUN --mount=type=bind,from=wheel_builder,source=/usr/local/,target=/tmp/usr/local/ \ rm -rf /usr/local/include/libav* /usr/local/include/libsw* /usr/local/src/ffmpeg && \ rm -f /usr/local/bin/ffmpeg /usr/local/bin/ffprobe && \ diff --git a/container/templates/wheel_builder.Dockerfile b/container/templates/wheel_builder.Dockerfile index 3597cf1bd96a..8a0a255ba8a3 100644 --- a/container/templates/wheel_builder.Dockerfile +++ b/container/templates/wheel_builder.Dockerfile @@ -289,11 +289,10 @@ ENV SCCACHE_BUCKET=${USE_SCCACHE:+${SCCACHE_BUCKET}} \ SCCACHE_REGION=${USE_SCCACHE:+${SCCACHE_REGION}} # Always build FFmpeg so libs are available for Rust checks in CI. -# We also build the ffmpeg CLI with h264_nvenc + libvpx_vp9 encoders so Python -# code can encode video without the GPL-licensed binary shipped by imageio-ffmpeg. -# Stays LGPL-only: --disable-gpl --disable-nonfree are preserved; H.264 comes from -# NVIDIA's NVENC (proprietary HW encoder, already a runtime dependency of these -# GPU images) and VP9 from libvpx (BSD). +# vLLM uses a narrow VP8/VP9/raw-video decode surface and VP9 encoding. Other +# frameworks retain their existing media configuration. +# Do not treat a hardware implementation as changing a codec's patent status: +# the vLLM build deliberately does not compile the NVENC H.264 encoder. # Do not delete the source tarball for legal reasons. ARG FFMPEG_VERSION ARG NV_CODEC_HEADERS_REF @@ -311,12 +310,14 @@ RUN --mount=type=secret,id=aws-web-identity-token,target=/run/secrets/aws-token elif [ "$DEVICE" = "cuda" ]; then \ dnf install -y --setopt=tsflags=nocontexts pkg-config xz git yasm; \ fi && \ + cd /tmp && \ +{% if framework != "vllm" %} # nv-codec-headers: provides the NVENC/NVDEC API headers ffmpeg compiles against. # Header-only, no runtime dep here; libcuda/libnvidia-encode are loaded at runtime # in the consuming container. - cd /tmp && \ git clone --depth 1 --branch ${NV_CODEC_HEADERS_REF} https://github.com/FFmpeg/nv-codec-headers.git && \ make -C nv-codec-headers PREFIX=/usr/local install && \ +{% endif %} # libvpx: BSD-licensed VP9 encoder needed for the WebM output path. Built from # source so we don't need to track distro package names (libvpx-dev on Debian # vs libvpx-devel via EPEL on RHEL/manylinux). @@ -342,6 +343,21 @@ RUN --mount=type=secret,id=aws-web-identity-token,target=/run/secrets/aws-token --disable-devices \ --disable-libdrm \ --enable-shared \ +{% if framework == "vllm" %} + --disable-decoders \ + --enable-decoder=vp8,vp9,rawvideo \ + --disable-demuxers \ + --enable-demuxer=mov,matroska,rawvideo \ + --disable-parsers \ + --enable-parser=vp8,vp9 \ + --enable-libvpx \ + --disable-encoders \ + --enable-encoder=libvpx_vp9 \ + --disable-muxers \ + --enable-muxer=matroska,webm \ + --disable-protocols \ + --enable-protocol=file,pipe && \ +{% else %} --enable-nvenc \ --enable-libvpx \ --disable-encoders \ @@ -349,6 +365,7 @@ RUN --mount=type=secret,id=aws-web-identity-token,target=/run/secrets/aws-token --disable-muxers \ --enable-muxer=mov,mp4,matroska,webm \ --enable-protocol=file,pipe && \ +{% endif %} make -j$(nproc) && \ make install && \ /tmp/use-sccache.sh show-stats "FFMPEG" && \