fix(vllm): use Dynamo FFmpeg for media bindings

components/src/dynamo/vllm/omni/output_formatter.py

@@ -11,7 +11,6 @@
 import asyncio
 import base64
 import logging
-import tempfile
 import time
 import uuid
 from io import BytesIO
@@ -20,15 +19,17 @@
 import numpy as np
 import soundfile as sf
 import torch
-from diffusers.utils.export_utils import export_to_video
 
 from dynamo.common.protocols.audio_protocol import AudioData, NvAudioSpeechResponse
 from dynamo.common.protocols.image_protocol import ImageData, NvImagesResponse
 from dynamo.common.protocols.video_protocol import NvVideosResponse, VideoData
 from dynamo.common.storage import upload_to_fs
 from dynamo.common.utils.engine_response import normalize_finish_reason
 from dynamo.common.utils.output_modalities import RequestType
-from dynamo.common.utils.video_utils import normalize_video_frames
+from dynamo.common.utils.video_utils import (
+    encode_to_video_bytes,
+    normalize_video_frames,
+)
 from dynamo.vllm.handlers import build_prompt_tokens_details
 from dynamo.vllm.omni.utils import is_empty_payload
 
@@ -129,24 +130,25 @@ async def _encode_video(
         response_format: Optional[str] = None,
         output_format: Optional[str] = None,
     ) -> Dict[str, Any] | None:
-        output_format = output_format or "mp4"
+        output_format = output_format or "webm"
         response_format = response_format or "url"
         if response_format not in ("url", "b64_json"):
             raise ValueError(
                 f"Unsupported response_format: {response_format!r}; expected 'url' or 'b64_json'"
             )
-        if output_format != "mp4":
+        if output_format != "webm":
             raise ValueError(
-                f"Unsupported output_format: {output_format!r}; only 'mp4' is supported"
+                f"Unsupported output_format: {output_format!r}; only 'webm' is supported"
             )
         try:
             start_time = time.time()
             frame_list = normalize_video_frames(images)
-            with tempfile.NamedTemporaryFile(
-                suffix=f".{output_format}", delete=True
-            ) as tmp:
-                await asyncio.to_thread(export_to_video, frame_list, tmp.name, fps)
-                video_bytes = tmp.read()
+            video_bytes = await asyncio.to_thread(
+                encode_to_video_bytes,
+                np.asarray(frame_list),
+                fps=fps,
+                output_format=output_format,
+            )
 
             if response_format == "b64_json":
                 video_data = VideoData(

components/src/dynamo/vllm/tests/omni/test_output_formatter.py

@@ -5,6 +5,7 @@
 
 from unittest.mock import MagicMock, patch
 
+import numpy as np
 import pytest
 
 try:
@@ -523,20 +524,23 @@ async def test_audio_data_carries_output_format_url_path(self):
 
 
 class TestDiffusionFormatterVideoOutputFormat:
-    """_encode_video always sets VideoData.output_format='mp4'."""
+    """_encode_video uses WebM/VP9 so vLLM does not require H.264."""
 
     def _patches(self):
         from unittest.mock import patch as _patch
 
         return (
             _patch(
                 "dynamo.vllm.omni.output_formatter.normalize_video_frames",
-                return_value=[MagicMock()],
+                return_value=[np.zeros((2, 2, 3), dtype=np.uint8)],
+            ),
+            _patch(
+                "dynamo.vllm.omni.output_formatter.encode_to_video_bytes",
+                return_value=b"fake-webm",
             ),
-            _patch("dynamo.vllm.omni.output_formatter.export_to_video"),
             _patch(
                 "dynamo.vllm.omni.output_formatter.upload_to_fs",
-                return_value="http://x/v.mp4",
+                return_value="http://x/v.webm",
             ),
             _patch(
                 "dynamo.vllm.omni.output_formatter.asyncio.to_thread",
@@ -564,8 +568,8 @@ async def test_video_url_response_format(self):
             )
 
         assert result is not None
-        assert result["data"][0]["output_format"] == "mp4"
-        assert result["data"][0]["url"] == "http://x/v.mp4"
+        assert result["data"][0]["output_format"] == "webm"
+        assert result["data"][0]["url"] == "http://x/v.webm"
         assert result["data"][0].get("b64_json") is None
         mock_upload.assert_called_once()
 
@@ -591,7 +595,7 @@ async def test_video_b64_response_format(self):
             )
 
         assert result is not None
-        assert result["data"][0]["output_format"] == "mp4"
+        assert result["data"][0]["output_format"] == "webm"
         assert result["data"][0].get("url") is None
         assert result["data"][0]["b64_json"] is not None
         base64.b64decode(result["data"][0]["b64_json"])  # must be valid base64
@@ -614,5 +618,22 @@ async def test_video_default_response_format_is_url(self):
             )
 
         assert result is not None
-        assert result["data"][0]["url"] == "http://x/v.mp4"
+        assert result["data"][0]["url"] == "http://x/v.webm"
         mock_upload.assert_called_once()
+
+    @pytest.mark.asyncio
+    async def test_video_mp4_output_is_rejected(self):
+        from dynamo.common.utils.output_modalities import RequestType
+        from dynamo.vllm.omni.output_formatter import DiffusionFormatter
+
+        f = DiffusionFormatter(model_name="test", media_fs=None, media_http_url=None)
+        stage = MagicMock()
+        stage.images = [MagicMock()]
+
+        with pytest.raises(ValueError, match="only 'webm' is supported"):
+            await f.format(
+                stage,
+                "r8",
+                request_type=RequestType.VIDEO_GENERATION,
+                output_format="mp4",
+            )

container/deps/vllm/requirements.media-source.txt

@@ -0,0 +1,10 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Build these distributions from source against Dynamo's allowlisted FFmpeg.
+# The hashes pin the PyPI source archives; binary wheels must never be used.
+
+av==18.0.0 \
+    --hash=sha256:4ef7e72c3d3a872584a1215173b16e0226811037f40dcdbf75992631098df1ba
+opencv-python==5.0.0.93 \
+    --hash=sha256:66aac3e5b5faa48d4025816592f3af19e4bfc2c68dec067bae2dbb4ca10aa9e2

container/deps/vllm/validate_ffmpeg_provider.py

@@ -0,0 +1,213 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Validate that vLLM uses only Dynamo's allowlisted FFmpeg installation."""
+
+import importlib
+import importlib.metadata
+import os
+import re
+import subprocess
+from pathlib import Path
+
+FFMPEG = Path("/usr/local/bin/ffmpeg")
+ALLOWED_LIBRARY_ROOT = Path("/usr/local/lib")
+SEARCH_ROOTS = (Path("/usr"), Path("/opt"), Path("/workspace"))
+ALLOWED_DECODERS = {"rawvideo", "vp8", "vp9"}
+ALLOWED_ENCODERS = {"libvpx-vp9"}
+REQUIRED_BUILD_CONFIGURATION = {
+    "--disable-bsfs",
+    "--disable-decoders",
+    "--disable-demuxers",
+    "--disable-encoders",
+    "--disable-muxers",
+    "--disable-parsers",
+    "--disable-protocols",
+    "--enable-decoder=vp8,vp9,rawvideo",
+    "--enable-demuxer=mov,matroska,rawvideo",
+    "--enable-encoder=libvpx_vp9",
+    "--enable-libvpx",
+    "--enable-muxer=matroska,webm",
+    "--enable-parser=vp8,vp9",
+    "--enable-protocol=file,pipe",
+}
+FORBIDDEN_DISTRIBUTIONS = {
+    "decord",
+    "decord2",
+    "opencv-python-headless",
+    "pynvvideocodec",
+}
+MEDIA_LIBRARY_RE = re.compile(
+    r"^(?:libav(?:codec|device|filter|format|util)|libpostproc|"
+    r"libsw(?:resample|scale)|libx26[45]|libopenh264|libfdk-aac|libfaac|"
+    r"libvo-aacenc|libaacplus).*\.so(?:\..*)?$"
+)
+MEDIA_BINARY_RE = re.compile(r"^ff(?:mpeg|probe)(?:[-_].*)?$")
+COMPONENT_RE = re.compile(r"^[A-Z.]{6}\s+(\S+)")
+
+
+def run(*command: str) -> str:
+    """Run a validation command and return stdout."""
+
+    return subprocess.run(
+        command,
+        check=True,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    ).stdout
+
+
+def normalize_distribution_name(name: str) -> str:
+    return re.sub(r"[-_.]+", "-", name).lower()
+
+
+def installed_distributions() -> dict[str, importlib.metadata.Distribution]:
+    return {
+        normalize_distribution_name(distribution.metadata["Name"]): distribution
+        for distribution in importlib.metadata.distributions()
+        if distribution.metadata["Name"]
+    }
+
+
+def ffmpeg_components(kind: str) -> set[str]:
+    output = run(str(FFMPEG), "-hide_banner", f"-{kind}")
+    components = set()
+    for line in output.splitlines():
+        match = COMPONENT_RE.match(line.strip())
+        if match is not None:
+            components.add(match.group(1))
+    return components
+
+
+def unexpected_media_artifacts() -> list[Path]:
+    artifacts = []
+    for root in SEARCH_ROOTS:
+        for directory, _, filenames in os.walk(root):
+            directory_path = Path(directory)
+            if directory_path.is_relative_to(Path("/usr/local/src/ffmpeg")):
+                continue
+            for filename in filenames:
+                path = directory_path / filename
+                if MEDIA_LIBRARY_RE.match(filename):
+                    if not path.resolve().is_relative_to(ALLOWED_LIBRARY_ROOT):
+                        artifacts.append(path)
+                elif MEDIA_BINARY_RE.match(filename) and os.access(path, os.X_OK):
+                    if path.resolve() != FFMPEG:
+                        artifacts.append(path)
+    return artifacts
+
+
+def extension_libraries(module_name: str, pattern: str = "*.so") -> list[Path]:
+    module = importlib.import_module(module_name)
+    module_file = Path(module.__file__).resolve()
+    package_root = module_file if module_file.suffix == ".so" else module_file.parent
+    if package_root.is_file():
+        return [package_root]
+    return sorted(package_root.rglob(pattern))
+
+
+def assert_links_to_dynamo_ffmpeg(module_name: str, pattern: str = "*.so") -> None:
+    extensions = extension_libraries(module_name, pattern)
+    if not extensions:
+        raise RuntimeError(f"{module_name} contains no extension libraries to inspect")
+
+    resolved_media_libraries = set()
+    for extension in extensions:
+        for line in run("ldd", str(extension)).splitlines():
+            if not re.search(r"lib(?:av|sw)", line):
+                continue
+            if "not found" in line:
+                raise RuntimeError(
+                    f"unresolved FFmpeg dependency for {extension}: {line}"
+                )
+            match = re.search(r"=>\s+(/\S+)", line)
+            if match is None:
+                continue
+            resolved = Path(match.group(1)).resolve()
+            resolved_media_libraries.add(resolved)
+            if not resolved.is_relative_to(ALLOWED_LIBRARY_ROOT):
+                raise RuntimeError(
+                    f"{extension} loads FFmpeg outside {ALLOWED_LIBRARY_ROOT}: {resolved}"
+                )
+
+    if not resolved_media_libraries:
+        raise RuntimeError(f"{module_name} does not link to Dynamo's FFmpeg libraries")
+
+
+def main() -> None:
+    if not FFMPEG.is_file():
+        raise RuntimeError(f"missing Dynamo FFmpeg executable: {FFMPEG}")
+
+    expected_version = os.environ["EXPECTED_FFMPEG_VERSION"]
+    first_version_line = run(str(FFMPEG), "-version").splitlines()[0]
+    if not first_version_line.startswith(f"ffmpeg version {expected_version}"):
+        raise RuntimeError(
+            f"expected FFmpeg {expected_version}, found: {first_version_line}"
+        )
+
+    build_configuration = run(str(FFMPEG), "-buildconf")
+    missing_configuration = REQUIRED_BUILD_CONFIGURATION - set(
+        build_configuration.split()
+    )
+    if missing_configuration:
+        raise RuntimeError(
+            "FFmpeg is missing required allowlist configuration: "
+            + ", ".join(sorted(missing_configuration))
+        )
+
+    decoders = ffmpeg_components("decoders")
+    if decoders != ALLOWED_DECODERS:
+        raise RuntimeError(
+            f"unexpected FFmpeg decoder set: expected {sorted(ALLOWED_DECODERS)}, "
+            f"found {sorted(decoders)}"
+        )
+
+    encoders = ffmpeg_components("encoders")
+    if encoders != ALLOWED_ENCODERS:
+        raise RuntimeError(
+            f"unexpected FFmpeg encoder set: expected {sorted(ALLOWED_ENCODERS)}, "
+            f"found {sorted(encoders)}"
+        )
+
+    distributions = installed_distributions()
+    unexpected_distributions = FORBIDDEN_DISTRIBUTIONS & distributions.keys()
+    if unexpected_distributions:
+        raise RuntimeError(
+            "codec-bearing upstream distributions remain installed: "
+            + ", ".join(sorted(unexpected_distributions))
+        )
+    for required_distribution in ("av", "opencv-python", "torchcodec"):
+        if required_distribution not in distributions:
+            raise RuntimeError(
+                f"required distribution is missing: {required_distribution}"
+            )
+
+    unexpected_artifacts = unexpected_media_artifacts()
+    if unexpected_artifacts:
+        raise RuntimeError(
+            "FFmpeg or prohibited codec artifacts remain outside Dynamo's "
+            "installation:\n"
+            + "\n".join(str(path) for path in sorted(unexpected_artifacts))
+        )
+
+    for module_name in ("av", "cv2"):
+        assert_links_to_dynamo_ffmpeg(module_name)
+    ffmpeg_major = expected_version.partition(".")[0]
+    assert_links_to_dynamo_ffmpeg("torchcodec", f"libtorchcodec_core{ffmpeg_major}.so")
+
+    imageio_ffmpeg = importlib.import_module("imageio_ffmpeg")
+    selected_ffmpeg = Path(imageio_ffmpeg.get_ffmpeg_exe()).resolve()
+    if selected_ffmpeg != FFMPEG:
+        raise RuntimeError(
+            f"imageio-ffmpeg selected {selected_ffmpeg}, expected {FFMPEG}"
+        )
+
+    print(
+        f"validated FFmpeg {expected_version}: one provider, "
+        f"decoders={sorted(decoders)}, encoders={sorted(encoders)}"
+    )
+
+
+if __name__ == "__main__":
+    main()