From 869ddafe0b61aeef7299c7ffb0d08d2fdb4ded23 Mon Sep 17 00:00:00 2001 From: Dmitry Tokarev Date: Tue, 14 Jul 2026 00:53:44 +0200 Subject: [PATCH 1/8] build(container): restrict the in-tree ffmpeg to a narrow media-codec allowlist (OPS-7665) The in-tree ffmpeg restricted encoders/muxers but still shipped ffmpeg's full default decoder set (291 video + 212 audio decoders) in the libav*.so copied into every runtime image. Narrow it to the media formats we actually build and use. Switch the wheel_builder configure to an allowlist: blanket --disable-decoders/--disable-demuxers/--disable-parsers/--disable-protocols plus exactly the components the two real paths need: - encode CLI (imageio): rawvideo demux/decode over a pipe -> h264_nvenc/vp9 - Rust media-ffmpeg VideoDecoder: VP8/VP9 in mp4/webm/mkv Image decode does not use ffmpeg (Rust `image` crate), so no still-image decoders are enabled. HW NVDEC (h264_cuvid/hevc_cuvid) can be re-added if a decode feature needs it. Also bump ffmpeg 8.1 -> 8.1.2 (an upstream maintenance release) to pick up security fixes and trim the decode surface further. Test fixtures are re-encoded to VP9-in-mp4; filenames and the Rust/Python tests that consume them are unchanged. Validated by building ffmpeg 8.1.2 with this exact allowlist and decoding every fixture to the expected frame counts. Co-Authored-By: Claude Fable 5 Signed-off-by: Dmitry Tokarev --- container/compliance/native_packages.yaml | 2 +- container/context.yaml | 5 ++++- container/templates/wheel_builder.Dockerfile | 21 +++++++++++++++++++ lib/llm/src/preprocessor/media/README.md | 3 +++ .../src/preprocessor/media/decoders/video.rs | 3 +++ lib/llm/tests/data/media/2160p_10.mp4 | 4 ++-- lib/llm/tests/data/media/240p_1.mp4 | 4 ++-- lib/llm/tests/data/media/240p_10.mp4 | 4 ++-- lib/llm/tests/data/media/240p_100.mp4 | 4 ++-- lib/llm/tests/data/media/2p_10.mp4 | 4 ++-- 10 files changed, 42 insertions(+), 12 deletions(-) diff --git a/container/compliance/native_packages.yaml b/container/compliance/native_packages.yaml index bd4be516c300..36140e3922f0 100644 --- a/container/compliance/native_packages.yaml +++ b/container/compliance/native_packages.yaml @@ -81,7 +81,7 @@ packages: # and copied into vllm/sglang runtime (container/templates/vllm_runtime.Dockerfile # + sglang_runtime.Dockerfile). Replaces the purged GPL apt ffmpeg. - name: ffmpeg - version: "8.1" + version: "8.1.2" license: LGPL-2.1-or-later source: https://ffmpeg.org/ images: diff --git a/container/context.yaml b/container/context.yaml index c4e03430b26b..38703a501628 100644 --- a/container/context.yaml +++ b/container/context.yaml @@ -43,7 +43,10 @@ dynamo: enable_kvbm: "true" enable_media_ffmpeg: "false" enable_gpu_memory_service: "true" - ffmpeg_version: "8.1" + # 8.1.2 is an upstream maintenance release that picks up security fixes over + # 8.1; combined with the narrowed decoder set in wheel_builder it trims the + # media decode surface. Keep in sync with native_packages.yaml's ffmpeg entry. + ffmpeg_version: "8.1.2" # ffmpeg build inputs (only consumed when ENABLE_MEDIA_FFMPEG=true). nv_codec_headers_ref: "n13.0.19.0" libvpx_ref: "v1.14.1" diff --git a/container/templates/wheel_builder.Dockerfile b/container/templates/wheel_builder.Dockerfile index 3597cf1bd96a..65d199f4223a 100644 --- a/container/templates/wheel_builder.Dockerfile +++ b/container/templates/wheel_builder.Dockerfile @@ -294,6 +294,20 @@ ENV SCCACHE_BUCKET=${USE_SCCACHE:+${SCCACHE_BUCKET}} \ # Stays LGPL-only: --disable-gpl --disable-nonfree are preserved; H.264 comes from # NVIDIA's NVENC (proprietary HW encoder, already a runtime dependency of these # GPU images) and VP9 from libvpx (BSD). +# +# MEDIA CODEC ALLOWLIST (see OPS-7665): the in-tree libavcodec should carry only +# the media formats we actually build and use, not ffmpeg's full default decoder +# set. A blanket --disable-decoders/--disable-demuxers/--disable-parsers plus a +# narrow allowlist keeps the shipped libav*.so limited to that set (HW NVDEC can +# be re-added explicitly if a decode feature ever needs H.264/H.265). The +# allowlist covers exactly two paths: (1) the encode CLI ingesting rawvideo +# frames from imageio over a pipe, and (2) the Rust media-ffmpeg VideoDecoder +# decoding VP8/VP9 in mp4/webm/mkv (test fixtures are VP9-in-mp4). Image decode +# does not use ffmpeg (it goes through the Rust `image` crate), so no still-image +# decoders are enabled here. +# +# Combined with the 8.1 -> 8.1.2 bump below (an upstream maintenance release), +# this also trims the decoder surface to what we ship. # Do not delete the source tarball for legal reasons. ARG FFMPEG_VERSION ARG NV_CODEC_HEADERS_REF @@ -346,8 +360,15 @@ RUN --mount=type=secret,id=aws-web-identity-token,target=/run/secrets/aws-token --enable-libvpx \ --disable-encoders \ --enable-encoder=h264_nvenc,libvpx_vp9 \ + --disable-decoders \ + --enable-decoder=vp8,vp9,rawvideo \ --disable-muxers \ --enable-muxer=mov,mp4,matroska,webm \ + --disable-demuxers \ + --enable-demuxer=mov,matroska,rawvideo \ + --disable-parsers \ + --enable-parser=vp8,vp9 \ + --disable-protocols \ --enable-protocol=file,pipe && \ make -j$(nproc) && \ make install && \ diff --git a/lib/llm/src/preprocessor/media/README.md b/lib/llm/src/preprocessor/media/README.md index ac9f8aa805f6..21c73e7d36f6 100644 --- a/lib/llm/src/preprocessor/media/README.md +++ b/lib/llm/src/preprocessor/media/README.md @@ -52,6 +52,9 @@ register_model( > [!WARNING] > **Video decoding**: Video decoding needs to be enabled via the `dynamo-llm/media-ffmpeg` rust feature. The following ffmpeg dynamic libraries must be available on the system: `libavcodec`, `libavdevice`, `libavfilter`, `libavformat`, `libswresample`, `libswscale`. These are available in dynamo dockerfiles rendered with `enable_media_ffmpeg` set to true in `container/context.yaml`. +> [!WARNING] +> **Supported input codecs**: The in-tree ffmpeg is built with a narrow decoder allowlist (VP8/VP9 video in mp4/webm/mkv) — it carries only the media formats we build and use, not ffmpeg's full default set (see `container/templates/wheel_builder.Dockerfile` and OPS-7665). Other codecs, including H.264 and H.265, are intentionally **not** decodable in software; decoding them would require enabling the NVDEC hardware decoders (`h264_cuvid`/`hevc_cuvid`), which is not wired up today. + ## Image decoding options ### Limits (not overridable at runtime via `media_io_kwargs`) diff --git a/lib/llm/src/preprocessor/media/decoders/video.rs b/lib/llm/src/preprocessor/media/decoders/video.rs index 649823bf1ac7..645061061fd3 100644 --- a/lib/llm/src/preprocessor/media/decoders/video.rs +++ b/lib/llm/src/preprocessor/media/decoders/video.rs @@ -315,6 +315,9 @@ mod tests { /// Load test video and parse expected dimensions from filename. /// Filename format: "{resolution}_{frames}.mp4" (e.g., "240p_10.mp4" -> 320x240, 10 frames) + /// Fixtures are VP9-in-mp4: the in-tree ffmpeg only decodes a narrow + /// allowlist (VP8/VP9), so H.264 fixtures would not decode. Regenerate with + /// `ffmpeg -f lavfi -i testsrc2=size=WxH:rate=1 -frames:v N -c:v libvpx-vp9 -g 1 -strict -2`. fn load_test_video(filename: &str) -> (EncodedMediaData, u32, u32, u32) { let path = format!( "{}/tests/data/media/{}", diff --git a/lib/llm/tests/data/media/2160p_10.mp4 b/lib/llm/tests/data/media/2160p_10.mp4 index 017ae1c3fee9..5ffc09eb959f 100644 --- a/lib/llm/tests/data/media/2160p_10.mp4 +++ b/lib/llm/tests/data/media/2160p_10.mp4 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f82a62d48a38e29ea004d0991adea5cdf2838e17647f24e8350aecc8297b8916 -size 5416 +oid sha256:462fa796a3099539df042a684b944e38acb2b19a2ddcceb760f88c092a1b805d +size 1009737 diff --git a/lib/llm/tests/data/media/240p_1.mp4 b/lib/llm/tests/data/media/240p_1.mp4 index 5d2a0fddf0f0..13cac1d3cb8e 100644 --- a/lib/llm/tests/data/media/240p_1.mp4 +++ b/lib/llm/tests/data/media/240p_1.mp4 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4442d098b32b29a3e7f9babac257a8c6ab3c4ed41cd2e1ad62df1e4734b92b6b -size 1597 +oid sha256:ac4dd818eb2b1371b0756704d7322bc64ef8ae9c68f8a6da1a4c12a26326485e +size 6541 diff --git a/lib/llm/tests/data/media/240p_10.mp4 b/lib/llm/tests/data/media/240p_10.mp4 index 86b57b122372..fd1f51bf0494 100644 --- a/lib/llm/tests/data/media/240p_10.mp4 +++ b/lib/llm/tests/data/media/240p_10.mp4 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:125d25fa5f09f44ae09e79e09d2e6c85a417ae8279a069eaab310b0e28cab18d -size 1913 +oid sha256:e72a3b7b83ec10000db8071bcafec959f110907d02f88da92ff792463ccb3eed +size 65804 diff --git a/lib/llm/tests/data/media/240p_100.mp4 b/lib/llm/tests/data/media/240p_100.mp4 index 7e021c9a7e82..77b2854b68e5 100644 --- a/lib/llm/tests/data/media/240p_100.mp4 +++ b/lib/llm/tests/data/media/240p_100.mp4 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c534d517ee77150ec971b3568dd999c09bbd4ec7850abed417f063bddb8e514f -size 4579 +oid sha256:13211b149c490748c2518313eae17181fca1082d7449e9b16a11e6728a0db3b4 +size 659773 diff --git a/lib/llm/tests/data/media/2p_10.mp4 b/lib/llm/tests/data/media/2p_10.mp4 index ce80424c5f7a..658c1965415e 100644 --- a/lib/llm/tests/data/media/2p_10.mp4 +++ b/lib/llm/tests/data/media/2p_10.mp4 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6cb1a8ca5c3ee2391a1606fac4004685e57fcc0ac5277773ccfdada534bb63f3 -size 1832 +oid sha256:ccf744a1e29f99d50d4988637f90a7d1110f3387ea059589c994a8f8ebdc25d4 +size 1180 From ccdf44193f5a67666876af176d4e6a019998399a Mon Sep 17 00:00:00 2001 From: Harrison King Saturley-Hall Date: Wed, 15 Jul 2026 21:35:24 -0400 Subject: [PATCH 2/8] build(sglang): remove FFmpeg codec support Remove codec-bearing Python packages inherited from the SGLang base, build the Dynamo runtime without media-ffmpeg, and fail the image build if FFmpeg, H.264, H.265, or AAC libraries remain. Signed-off-by: Harrison King Saturley-Hall --- container/compliance/native_packages.yaml | 7 +- container/context.yaml | 8 +- container/deps/requirements.sglang.txt | 12 +-- container/templates/sglang_runtime.Dockerfile | 95 ++++++++++++++----- container/templates/wheel_builder.Dockerfile | 15 ++- 5 files changed, 92 insertions(+), 45 deletions(-) diff --git a/container/compliance/native_packages.yaml b/container/compliance/native_packages.yaml index 36140e3922f0..7ca910dd97ae 100644 --- a/container/compliance/native_packages.yaml +++ b/container/compliance/native_packages.yaml @@ -78,8 +78,7 @@ packages: # generator), so it is intentionally NOT duplicated here. # LGPL media stack — built --disable-gpl in wheel_builder (h264_nvenc + libvpx) - # and copied into vllm/sglang runtime (container/templates/vllm_runtime.Dockerfile - # + sglang_runtime.Dockerfile). Replaces the purged GPL apt ffmpeg. + # and copied into the vLLM runtime. SGLang intentionally contains no FFmpeg. - name: ffmpeg version: "8.1.2" license: LGPL-2.1-or-later @@ -87,8 +86,6 @@ packages: images: - vllm-runtime - vllm-runtime-efa - - sglang-runtime - - sglang-runtime-efa - name: libvpx # VP8/VP9 codec library, ffmpeg's only non-NVIDIA encoder dependency. @@ -98,8 +95,6 @@ packages: images: - vllm-runtime - vllm-runtime-efa - - sglang-runtime - - sglang-runtime-efa # UCX — Unified Communication X, built from source in wheel_builder and used by # the NIXL transport. Installed to /usr/local (no dpkg metadata). diff --git a/container/context.yaml b/container/context.yaml index 38703a501628..7122f4506941 100644 --- a/container/context.yaml +++ b/container/context.yaml @@ -103,9 +103,9 @@ sglang: # lmsysorg/sglang is built FROM this nvidia/cuda cudnn-devel base, so this # attributes everything sglang adds over the clean NVIDIA CUDA+cuDNN floor # (framework, python stack, system libs) rather than hiding it. Per-arch stem; - # the licenses stage appends -${TARGETARCH}.cdx.json. (sglang ships no system - # GPL ffmpeg/codec dpkgs; the GPL imageio-ffmpeg pip binary is replaced by an - # in-tree LGPL ffmpeg in sglang_runtime.Dockerfile — no dpkg purge needed.) + # the licenses stage appends -${TARGETARCH}.cdx.json. SGLang's codec-bearing + # Python wheels are purged in sglang_runtime.Dockerfile; FFmpeg is not copied + # into the image and the Dynamo Rust wheel is built without media-ffmpeg. baseline_sbom: cuda@8b2705ea xpu: base_image: intel/deep-learning-essentials @@ -124,7 +124,7 @@ sglang: # NIXL Python stack — its wheel COPY is narrowed to ai_dynamo*.whl so the SDK # build doesn't leak into the runtime image. nixl_ref: v1.0.1 - enable_media_ffmpeg: "true" + enable_media_ffmpeg: "false" enable_gpu_memory_service: "true" enable_kvbm: "false" enable_modelexpress: "true" diff --git a/container/deps/requirements.sglang.txt b/container/deps/requirements.sglang.txt index e813813faf18..64c60a9e66cf 100644 --- a/container/deps/requirements.sglang.txt +++ b/container/deps/requirements.sglang.txt @@ -1,14 +1,10 @@ # SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # -# Third-party Python dependencies for the sglang runtime image. Installed -# with --force-reinstall --no-deps to replace the upstream lmsysorg/sglang -# base image's imageio-ffmpeg wheel (which ships a GPL-encumbered prebuilt -# ffmpeg binary) with a source build that leaves no binary on disk. -# IMAGEIO_FFMPEG_EXE points imageio at the in-tree LGPL ffmpeg CLI. - ---no-binary imageio-ffmpeg +# Third-party Python dependencies for the SGLang runtime image. Installed with +# --force-reinstall --no-deps so the upstream SGLang dependency stack remains +# otherwise unchanged. FFmpeg and codec-bearing wheels are removed separately +# in sglang_runtime.Dockerfile. blake3>=1.0.0,<2.0.0 # Dynamo SGLang multimodal request handlers import blake3 at startup -imageio-ffmpeg>=0.6.0 # binary skipped per --no-binary directive at top of file zstandard==0.23.0 diff --git a/container/templates/sglang_runtime.Dockerfile b/container/templates/sglang_runtime.Dockerfile index 9ba7a6672aa5..c95a1046eba5 100644 --- a/container/templates/sglang_runtime.Dockerfile +++ b/container/templates/sglang_runtime.Dockerfile @@ -67,25 +67,6 @@ $NIXL_PLUGIN_DIR:\ ${LD_LIBRARY_PATH:-} {% endif %} -# Copy ffmpeg from wheel_builder: versioned shared libs (libav*.so*, -# libsw*.so*) for the Rust media-ffmpeg decoder, plus the LGPL CLI binary -# (built with h264_nvenc + libvpx_vp9 encoders) that imageio targets via -# IMAGEIO_FFMPEG_EXE for video encoding. Ungated by enable_media_ffmpeg -# because the upstream lmsysorg/sglang base image always ships -# imageio-ffmpeg with a GPL-encumbered prebuilt binary that we replace -# unconditionally below; the LGPL CLI must be present so imageio has -# something to target. -RUN --mount=type=bind,from=wheel_builder,source=/usr/local/,target=/tmp/usr/local/ \ - mkdir -p /usr/local/lib/pkgconfig && \ - cp -rnL /tmp/usr/local/include/libav* /tmp/usr/local/include/libsw* /usr/local/include/ && \ - cp -nL /tmp/usr/local/lib/libav*.so* /tmp/usr/local/lib/libsw*.so* /usr/local/lib/ && \ - cp -nL /tmp/usr/local/lib/lib*vpx*.so* /usr/local/lib/ 2>/dev/null || true && \ - cp -nL /tmp/usr/local/lib/pkgconfig/libav*.pc /tmp/usr/local/lib/pkgconfig/libsw*.pc /usr/local/lib/pkgconfig/ && \ - cp -nL /tmp/usr/local/bin/ffmpeg /usr/local/bin/ffmpeg && \ - cp -r /tmp/usr/local/src/ffmpeg /usr/local/src/ && \ - ldconfig -ENV IMAGEIO_FFMPEG_EXE=/usr/local/bin/ffmpeg - {% if target not in ("dev", "local-dev") %} # Runtime target installs only the prebuilt Dynamo wheels. SGLang and its NIXL # packages come from the upstream lmsysorg/sglang runtime image; --no-deps keeps @@ -151,17 +132,60 @@ RUN --mount=type=bind,source=./container/deps/requirements.common.txt,target=/tm export PIP_CACHE_DIR=/root/.cache/pip && \ pip install --break-system-packages --no-deps $(grep -E '^nvtx==' /tmp/requirements.common.txt) -# Replace the upstream lmsysorg/sglang image's imageio-ffmpeg (which ships a -# GPL-encumbered prebuilt ffmpeg binary in /imageio_ffmpeg/binaries/) -# with a source install that leaves no binary on disk. IMAGEIO_FFMPEG_EXE points -# imageio at the LGPL CLI we copied from wheel_builder above. The --no-binary -# directive lives in the requirements file itself. +# Install SGLang-specific runtime dependencies without changing the upstream +# dependency solution. imageio-ffmpeg is intentionally absent. RUN --mount=type=bind,source=./container/deps/requirements.sglang.txt,target=/tmp/requirements.sglang.txt \ --mount=type=cache,target=/root/.cache/pip,sharing=locked \ export PIP_CACHE_DIR=/root/.cache/pip && \ pip install --break-system-packages --force-reinstall --no-deps \ --requirement /tmp/requirements.sglang.txt +# Remove every codec-bearing component found in the upstream SGLang image and +# fail the build if an executable or shared library for FFmpeg, H.264, H.265, or +# AAC remains in the merged runtime filesystem. +# +# Inkling image preprocessing uses Pillow. Its audio feature extractor imports +# soundfile and uses torchaudio only for resampling, so those paths remain +# available for formats supported by libsndfile (for example WAV and FLAC). +# AAC-backed M4A and all video encode/decode support are intentionally removed. +RUN set -eux; \ + python3 -m pip uninstall --yes \ + av \ + decord \ + imageio-ffmpeg \ + opencv-python \ + opencv-python-headless \ + torchcodec; \ + SITE_PACKAGES="$(python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"; \ + rm -rf \ + "${SITE_PACKAGES}"/av \ + "${SITE_PACKAGES}"/av-*.dist-info \ + "${SITE_PACKAGES}"/av.libs \ + "${SITE_PACKAGES}"/cv2 \ + "${SITE_PACKAGES}"/decord \ + "${SITE_PACKAGES}"/decord-*.dist-info \ + "${SITE_PACKAGES}"/decord.libs \ + "${SITE_PACKAGES}"/imageio_ffmpeg \ + "${SITE_PACKAGES}"/imageio_ffmpeg-*.dist-info \ + "${SITE_PACKAGES}"/opencv_python*.dist-info \ + "${SITE_PACKAGES}"/opencv_python*.libs \ + "${SITE_PACKAGES}"/torchcodec \ + "${SITE_PACKAGES}"/torchcodec-*.dist-info \ + /usr/local/bin/ffmpeg \ + /usr/local/bin/ffprobe \ + /usr/local/include/libav* \ + /usr/local/include/libsw* \ + /usr/local/lib/libav* \ + /usr/local/lib/libpostproc* \ + /usr/local/lib/libsw* \ + /usr/local/lib/pkgconfig/libav*.pc \ + /usr/local/lib/pkgconfig/libpostproc*.pc \ + /usr/local/lib/pkgconfig/libsw*.pc \ + /usr/local/src/ffmpeg \ + /root/.cache/pip; \ + ldconfig +ENV IMAGEIO_FFMPEG_EXE= + # Copy tests, deploy and components for CI with correct ownership COPY --chmod=775 --chown=dynamo:0 tests /workspace/tests COPY --chmod=775 --chown=dynamo:0 examples /workspace/examples @@ -207,6 +231,29 @@ RUN SITE_PACKAGES="$(python3 -c 'import site; print(site.getsitepackages()[0])') (python3 -m compileall -q -j0 /sgl-workspace/sglang/python || true) {%- endif %} +# Keep this guard at the end of the populated runtime stage so later COPY/RUN +# steps cannot silently reintroduce a codec. The extra AAC library names cover +# common non-FFmpeg implementations even though the current Syft baseline did +# not find them. +RUN set -eux; \ + remaining="$(find /usr /opt /workspace /sgl-workspace -xdev \ + \( -type f -o -type l \) \ + \( -name ffmpeg -o -name ffprobe \ + -o -name 'libavcodec*.so*' -o -name 'libavdevice*.so*' \ + -o -name 'libavfilter*.so*' -o -name 'libavformat*.so*' \ + -o -name 'libavutil*.so*' -o -name 'libpostproc*.so*' \ + -o -name 'libswresample*.so*' -o -name 'libswscale*.so*' \ + -o -name 'libx264*.so*' -o -name 'libx265*.so*' \ + -o -name 'libopenh264*.so*' -o -name 'libfdk-aac*.so*' \ + -o -name 'libfaac*.so*' -o -name 'libvo-aacenc*.so*' \ + -o -name 'libaacplus*.so*' \) -print)"; \ + if [ -n "${remaining}" ]; then \ + echo "ERROR: codec-bearing files remain in the SGLang image:" >&2; \ + echo "${remaining}" >&2; \ + exit 1; \ + fi; \ + python3 -c 'import soundfile, torchaudio; from PIL import Image' + USER dynamo ARG DYNAMO_COMMIT_SHA ENV DYNAMO_COMMIT_SHA=${DYNAMO_COMMIT_SHA} diff --git a/container/templates/wheel_builder.Dockerfile b/container/templates/wheel_builder.Dockerfile index 65d199f4223a..e9b91d904188 100644 --- a/container/templates/wheel_builder.Dockerfile +++ b/container/templates/wheel_builder.Dockerfile @@ -288,7 +288,12 @@ RUN mkdir -p /tmp/native-sources ENV SCCACHE_BUCKET=${USE_SCCACHE:+${SCCACHE_BUCKET}} \ SCCACHE_REGION=${USE_SCCACHE:+${SCCACHE_REGION}} -# Always build FFmpeg so libs are available for Rust checks in CI. +# Build FFmpeg for frameworks that retain media encode/decode support. SGLang +# deliberately omits it: its Inkling image/audio path uses Pillow and +# soundfile/torchaudio, and the SGLang runtime must not contain H.264, H.265, or +# AAC implementations. +{% if framework != "sglang" %} +# Build FFmpeg so libs are available for Rust checks in CI. # We also build the ffmpeg CLI with h264_nvenc + libvpx_vp9 encoders so Python # code can encode video without the GPL-licensed binary shipped by imageio-ffmpeg. # Stays LGPL-only: --disable-gpl --disable-nonfree are preserved; H.264 comes from @@ -377,6 +382,7 @@ RUN --mount=type=secret,id=aws-web-identity-token,target=/run/secrets/aws-token mkdir -p /usr/local/src/ffmpeg && \ find /tmp/ffmpeg-${FFMPEG_VERSION} \( -name config.log -o -name config.status \) -delete && \ mv /tmp/ffmpeg-${FFMPEG_VERSION}* /usr/local/src/ffmpeg/ +{% endif %} # Build and install UCX RUN --mount=type=secret,id=aws-web-identity-token,target=/run/secrets/aws-token \ @@ -524,7 +530,9 @@ COPY components/ /opt/dynamo/components/ # Build ai-dynamo (pure Python) and ai-dynamo-runtime (maturin) wheels ARG USE_SCCACHE +{% if framework != "sglang" %} ARG ENABLE_MEDIA_FFMPEG +{% endif %} RUN --mount=type=secret,id=aws-web-identity-token,target=/run/secrets/aws-token \ --mount=type=secret,id=aws-role-arn,env=AWS_ROLE_ARN \ --mount=type=cache,target=/root/.cargo/registry,sharing=shared \ @@ -541,12 +549,13 @@ RUN --mount=type=secret,id=aws-web-identity-token,target=/run/secrets/aws-token cd /opt/dynamo && \ uv build --wheel --out-dir /opt/dynamo/dist && \ cd /opt/dynamo/lib/bindings/python && \ - if [ "$ENABLE_MEDIA_FFMPEG" = "true" ]; then \ +{% if framework == "sglang" %} maturin build --release --features "kv-indexer,slot-tracker,select-service,mm-routing,aic-forward-pass" --out /opt/dynamo/dist && \ +{% else %} if [ "$ENABLE_MEDIA_FFMPEG" = "true" ]; then \ maturin build --release --features "media-ffmpeg,kv-indexer,slot-tracker,select-service,mm-routing,aic-forward-pass" --out /opt/dynamo/dist; \ else \ maturin build --release --features "kv-indexer,slot-tracker,select-service,mm-routing,aic-forward-pass" --out /opt/dynamo/dist; \ fi && \ - /tmp/use-sccache.sh show-stats "Dynamo Runtime" +{% endif %} /tmp/use-sccache.sh show-stats "Dynamo Runtime" # Compliance: harvest each crate's real LICENSE files from the cargo registry # source cache so the rust NOTICES generator can inline upstream license text From 3569f302860aa17870b78c28d55715cdb1ff72b7 Mon Sep 17 00:00:00 2001 From: Harrison King Saturley-Hall Date: Wed, 15 Jul 2026 21:49:39 -0400 Subject: [PATCH 3/8] fix(sglang): remove decord2 codec libraries The ARM64 Inkling base installs the distribution as decord2, whose wheel bundles FFmpeg, x264, and x265 shared libraries. Remove that package and its residual files before the final codec guard. Signed-off-by: Harrison King Saturley-Hall --- container/templates/sglang_runtime.Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/container/templates/sglang_runtime.Dockerfile b/container/templates/sglang_runtime.Dockerfile index c95a1046eba5..05c10805c53a 100644 --- a/container/templates/sglang_runtime.Dockerfile +++ b/container/templates/sglang_runtime.Dockerfile @@ -152,6 +152,7 @@ RUN set -eux; \ python3 -m pip uninstall --yes \ av \ decord \ + decord2 \ imageio-ffmpeg \ opencv-python \ opencv-python-headless \ @@ -165,6 +166,9 @@ RUN set -eux; \ "${SITE_PACKAGES}"/decord \ "${SITE_PACKAGES}"/decord-*.dist-info \ "${SITE_PACKAGES}"/decord.libs \ + "${SITE_PACKAGES}"/decord2 \ + "${SITE_PACKAGES}"/decord2-*.dist-info \ + "${SITE_PACKAGES}"/decord2.libs \ "${SITE_PACKAGES}"/imageio_ffmpeg \ "${SITE_PACKAGES}"/imageio_ffmpeg-*.dist-info \ "${SITE_PACKAGES}"/opencv_python*.dist-info \ From c45909b18239516035f447e2007de25749338dc9 Mon Sep 17 00:00:00 2001 From: Dmitry Tokarev Date: Fri, 17 Jul 2026 19:38:11 +0200 Subject: [PATCH 4/8] docs(container): address review nits on the media-codec allowlist - Drop internal ticket references from the wheel_builder + media README comments. - Make the fixture-regeneration ffmpeg command in the video.rs docstring executable by adding the output filename. Co-Authored-By: Claude Opus 4.8 Signed-off-by: Dmitry Tokarev --- container/templates/wheel_builder.Dockerfile | 2 +- lib/llm/src/preprocessor/media/README.md | 2 +- lib/llm/src/preprocessor/media/decoders/video.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/container/templates/wheel_builder.Dockerfile b/container/templates/wheel_builder.Dockerfile index e9b91d904188..bb86ecda0486 100644 --- a/container/templates/wheel_builder.Dockerfile +++ b/container/templates/wheel_builder.Dockerfile @@ -300,7 +300,7 @@ ENV SCCACHE_BUCKET=${USE_SCCACHE:+${SCCACHE_BUCKET}} \ # NVIDIA's NVENC (proprietary HW encoder, already a runtime dependency of these # GPU images) and VP9 from libvpx (BSD). # -# MEDIA CODEC ALLOWLIST (see OPS-7665): the in-tree libavcodec should carry only +# MEDIA CODEC ALLOWLIST: the in-tree libavcodec should carry only # the media formats we actually build and use, not ffmpeg's full default decoder # set. A blanket --disable-decoders/--disable-demuxers/--disable-parsers plus a # narrow allowlist keeps the shipped libav*.so limited to that set (HW NVDEC can diff --git a/lib/llm/src/preprocessor/media/README.md b/lib/llm/src/preprocessor/media/README.md index 21c73e7d36f6..b21f7d052f1a 100644 --- a/lib/llm/src/preprocessor/media/README.md +++ b/lib/llm/src/preprocessor/media/README.md @@ -53,7 +53,7 @@ register_model( > **Video decoding**: Video decoding needs to be enabled via the `dynamo-llm/media-ffmpeg` rust feature. The following ffmpeg dynamic libraries must be available on the system: `libavcodec`, `libavdevice`, `libavfilter`, `libavformat`, `libswresample`, `libswscale`. These are available in dynamo dockerfiles rendered with `enable_media_ffmpeg` set to true in `container/context.yaml`. > [!WARNING] -> **Supported input codecs**: The in-tree ffmpeg is built with a narrow decoder allowlist (VP8/VP9 video in mp4/webm/mkv) — it carries only the media formats we build and use, not ffmpeg's full default set (see `container/templates/wheel_builder.Dockerfile` and OPS-7665). Other codecs, including H.264 and H.265, are intentionally **not** decodable in software; decoding them would require enabling the NVDEC hardware decoders (`h264_cuvid`/`hevc_cuvid`), which is not wired up today. +> **Supported input codecs**: The in-tree ffmpeg is built with a narrow decoder allowlist (VP8/VP9 video in mp4/webm/mkv) — it carries only the media formats we build and use, not ffmpeg's full default set (see `container/templates/wheel_builder.Dockerfile`). Other codecs, including H.264 and H.265, are intentionally **not** decodable in software; decoding them would require enabling the NVDEC hardware decoders (`h264_cuvid`/`hevc_cuvid`), which is not wired up today. ## Image decoding options diff --git a/lib/llm/src/preprocessor/media/decoders/video.rs b/lib/llm/src/preprocessor/media/decoders/video.rs index 645061061fd3..dd13ede877fa 100644 --- a/lib/llm/src/preprocessor/media/decoders/video.rs +++ b/lib/llm/src/preprocessor/media/decoders/video.rs @@ -317,7 +317,7 @@ mod tests { /// Filename format: "{resolution}_{frames}.mp4" (e.g., "240p_10.mp4" -> 320x240, 10 frames) /// Fixtures are VP9-in-mp4: the in-tree ffmpeg only decodes a narrow /// allowlist (VP8/VP9), so H.264 fixtures would not decode. Regenerate with - /// `ffmpeg -f lavfi -i testsrc2=size=WxH:rate=1 -frames:v N -c:v libvpx-vp9 -g 1 -strict -2`. + /// `ffmpeg -f lavfi -i testsrc2=size=WxH:rate=1 -frames:v N -c:v libvpx-vp9 -g 1 -strict -2 {resolution}_{frames}.mp4`. fn load_test_video(filename: &str) -> (EncodedMediaData, u32, u32, u32) { let path = format!( "{}/tests/data/media/{}", From 642de0e3e1994a3609eb56d8ae65fcce41990ef6 Mon Sep 17 00:00:00 2001 From: Dmitry Tokarev Date: Fri, 17 Jul 2026 19:52:49 +0200 Subject: [PATCH 5/8] build(container): keep the h264 parser for h264_nvenc mp4 muxing The TRT-LLM diffusion encode path (imageio -> ffmpeg -c:v h264_nvenc -> mp4) regressed after the decoder allowlist. The h264 *parser* (not the H.264 decoder) is needed for the mp4 muxer to package the h264_nvenc bitstream, so add it back. A parser carries no codec implementation, so this does not reintroduce any software H.264 encode/decode surface; h264_nvenc remains the NVIDIA hardware encoder (the sanctioned path). Co-Authored-By: Claude Opus 4.8 Signed-off-by: Dmitry Tokarev --- container/templates/wheel_builder.Dockerfile | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/container/templates/wheel_builder.Dockerfile b/container/templates/wheel_builder.Dockerfile index bb86ecda0486..658ab34802fe 100644 --- a/container/templates/wheel_builder.Dockerfile +++ b/container/templates/wheel_builder.Dockerfile @@ -306,10 +306,13 @@ ENV SCCACHE_BUCKET=${USE_SCCACHE:+${SCCACHE_BUCKET}} \ # narrow allowlist keeps the shipped libav*.so limited to that set (HW NVDEC can # be re-added explicitly if a decode feature ever needs H.264/H.265). The # allowlist covers exactly two paths: (1) the encode CLI ingesting rawvideo -# frames from imageio over a pipe, and (2) the Rust media-ffmpeg VideoDecoder -# decoding VP8/VP9 in mp4/webm/mkv (test fixtures are VP9-in-mp4). Image decode -# does not use ffmpeg (it goes through the Rust `image` crate), so no still-image -# decoders are enabled here. +# frames from imageio over a pipe and encoding with h264_nvenc (NVIDIA's HW +# encoder — the sanctioned path) or libvpx_vp9, and (2) the Rust media-ffmpeg +# VideoDecoder decoding VP8/VP9 in mp4/webm/mkv (test fixtures are VP9-in-mp4). +# The h264 *parser* is enabled — not the H.264 decoder — because the mp4 muxer +# needs it to package the h264_nvenc bitstream (extract SPS/PPS); a parser +# carries no codec implementation. Image decode does not use ffmpeg (it goes +# through the Rust `image` crate), so no still-image decoders are enabled here. # # Combined with the 8.1 -> 8.1.2 bump below (an upstream maintenance release), # this also trims the decoder surface to what we ship. @@ -372,7 +375,7 @@ RUN --mount=type=secret,id=aws-web-identity-token,target=/run/secrets/aws-token --disable-demuxers \ --enable-demuxer=mov,matroska,rawvideo \ --disable-parsers \ - --enable-parser=vp8,vp9 \ + --enable-parser=vp8,vp9,h264 \ --disable-protocols \ --enable-protocol=file,pipe && \ make -j$(nproc) && \ From eaabac23c6e57999575479a2f145781cebac7dcb Mon Sep 17 00:00:00 2001 From: Dmitry Tokarev Date: Fri, 17 Jul 2026 20:34:22 +0200 Subject: [PATCH 6/8] build(vllm): remove codec-bearing video-decode wheels from vLLM images The vllm-openai base ships PyAV, opencv, decord, torchcodec, and PyNvVideoCodec, each bundling its own full ffmpeg with software H.264/H.265/AAC (PyAV and decord also ship GPL libx264/libx265). Dynamo's vLLM component imports none of them, so they are unused decode-side dead weight. Uninstall + remove them, with a guard that fails the build if cv2/av survive. The in-tree LGPL ffmpeg and imageio-ffmpeg are kept for the omni HW video-encode path (h264_nvenc, the sanctioned NVIDIA hardware encoder). This drops software video *decode* from the vLLM images (mirrors the SGLang codec removal). Co-Authored-By: Claude Opus 4.8 Signed-off-by: Dmitry Tokarev --- container/templates/vllm_runtime.Dockerfile | 24 +++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/container/templates/vllm_runtime.Dockerfile b/container/templates/vllm_runtime.Dockerfile index 2b922fa4e8f5..81687f5f4c2e 100644 --- a/container/templates/vllm_runtime.Dockerfile +++ b/container/templates/vllm_runtime.Dockerfile @@ -275,6 +275,30 @@ RUN --mount=type=bind,source=./container/deps/requirements.vllm.txt,target=/tmp/ # tool scripts referencing files not present in Dynamo's build context. RUN rm -rf /workspace/vllm +# Remove the codec-bearing video-DECODE wheels inherited from the vllm-openai +# base. Each bundles its own full ffmpeg carrying software H.264/H.265/AAC; +# PyAV and decord additionally ship GPL libx264/libx265. Dynamo's vLLM component +# imports none of them, so they are unused decode-side dead weight. The in-tree +# LGPL ffmpeg + imageio-ffmpeg installed above are intentionally KEPT for the +# omni HW video-encode path (h264_nvenc — the sanctioned path). Direct rm makes +# the removal robust regardless of how the base image's pip is configured; the +# guards fail the build if any of them survive. +RUN set -eux; \ + python3 -m pip uninstall --yes \ + av decord decord2 opencv-python opencv-python-headless torchcodec PyNvVideoCodec \ + || true; \ + SITE_PACKAGES="$(python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"; \ + rm -rf \ + "${SITE_PACKAGES}"/av "${SITE_PACKAGES}"/av-*.dist-info "${SITE_PACKAGES}"/av.libs \ + "${SITE_PACKAGES}"/cv2 "${SITE_PACKAGES}"/opencv_python*.dist-info "${SITE_PACKAGES}"/opencv_python*.libs \ + "${SITE_PACKAGES}"/decord "${SITE_PACKAGES}"/decord-*.dist-info "${SITE_PACKAGES}"/decord.libs \ + "${SITE_PACKAGES}"/decord2 "${SITE_PACKAGES}"/decord2-*.dist-info "${SITE_PACKAGES}"/decord2.libs \ + "${SITE_PACKAGES}"/torchcodec "${SITE_PACKAGES}"/torchcodec-*.dist-info \ + "${SITE_PACKAGES}"/PyNvVideoCodec "${SITE_PACKAGES}"/PyNvVideoCodec-*.dist-info "${SITE_PACKAGES}"/PyNvVideoCodec.libs \ + /root/.cache/pip; \ + ! python3 -c "import cv2" 2>/dev/null; \ + ! python3 -c "import av" 2>/dev/null + USER dynamo # Copy the workspace surface needed by the current vLLM pre-merge test image. From ddd5c0598e57081f83600435111cea0a4349bf80 Mon Sep 17 00:00:00 2001 From: Dmitry Tokarev Date: Fri, 17 Jul 2026 22:09:17 +0200 Subject: [PATCH 7/8] fix(container): enable ffmpeg fd protocol for stdin encode path The narrow in-tree ffmpeg protocol allowlist enabled only file,pipe. On ffmpeg 8.x, `ffmpeg -i -` (reading rawvideo frames from stdin, as the imageio encode path does) resolves stdin through the `fd:` protocol, not `pipe:`. With fd disabled, opening the stdin input fails immediately with "Error opening input: Protocol not found. Did you mean file:fd:?", breaking the video encode path. Add fd to the protocol allowlist. It is pure file-descriptor I/O and carries no codec implementation, so it does not widen the media-format surface. Reproduced and confirmed against a locally built allowlisted ffmpeg: file,pipe fails to open stdin; file,pipe,fd opens it cleanly. Co-Authored-By: Claude Opus 4.8 Signed-off-by: Dmitry Tokarev --- container/templates/wheel_builder.Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/container/templates/wheel_builder.Dockerfile b/container/templates/wheel_builder.Dockerfile index 658ab34802fe..188b6d3adf42 100644 --- a/container/templates/wheel_builder.Dockerfile +++ b/container/templates/wheel_builder.Dockerfile @@ -313,6 +313,10 @@ ENV SCCACHE_BUCKET=${USE_SCCACHE:+${SCCACHE_BUCKET}} \ # needs it to package the h264_nvenc bitstream (extract SPS/PPS); a parser # carries no codec implementation. Image decode does not use ffmpeg (it goes # through the Rust `image` crate), so no still-image decoders are enabled here. +# The `fd` protocol is enabled alongside `pipe`: `ffmpeg -i -` reads stdin via +# the `fd:` protocol on ffmpeg 8.x (not `pipe:`), so omitting it breaks the +# imageio encode path with "Protocol not found. Did you mean file:fd:?". Both +# are pure fd/stream I/O and carry no codec implementation. # # Combined with the 8.1 -> 8.1.2 bump below (an upstream maintenance release), # this also trims the decoder surface to what we ship. @@ -377,7 +381,7 @@ RUN --mount=type=secret,id=aws-web-identity-token,target=/run/secrets/aws-token --disable-parsers \ --enable-parser=vp8,vp9,h264 \ --disable-protocols \ - --enable-protocol=file,pipe && \ + --enable-protocol=file,pipe,fd && \ make -j$(nproc) && \ make install && \ /tmp/use-sccache.sh show-stats "FFMPEG" && \ From 09391d2dbd9516e4b0fc205f11a908eb156b5b55 Mon Sep 17 00:00:00 2001 From: Dmitry Tokarev Date: Fri, 17 Jul 2026 23:38:27 +0200 Subject: [PATCH 8/8] test(serve): install video decoder at test time for codec-stripped images The vllm and sglang runtime images intentionally omit certain media-decoder libraries (opencv for vLLM's VideoMediaIO backend decode, decord for SGLang's video path). The video multimodal serve tests exercise those decode paths, so without the decoder present they fail at request time ("No module named 'cv2'" / "No module named 'decord'"). Rather than skip that coverage, let a serve config opt into an ephemeral, test-only decoder install via a reserved env key (DYN_TEST_ONLY_PIP_INSTALL). common.run_serve_deployment installs the named package(s) into the runtime container before the server launches, so the worker can import them; the shipped image is unaffected. Session-deduped and a no-op unless a config sets the key. Wired for the three vLLM video topologies (opencv-python-headless) and the two SGLang video configs (decord2). Verified on faithful codec-stripped rc.11 images: each decode path reproduces the exact failure when stripped and decodes successfully once the package is installed. Co-Authored-By: Claude Opus 4.8 Signed-off-by: Dmitry Tokarev --- tests/serve/common.py | 41 +++++++++++++++++++++++++ tests/serve/multimodal_profiles/vllm.py | 8 +++++ tests/serve/test_sglang.py | 7 +++++ 3 files changed, 56 insertions(+) diff --git a/tests/serve/common.py b/tests/serve/common.py index a58f4b6c9185..45a4d39f6f0f 100644 --- a/tests/serve/common.py +++ b/tests/serve/common.py @@ -9,6 +9,8 @@ import logging import os import signal +import subprocess +import sys import time from collections.abc import Callable, Mapping from copy import deepcopy @@ -460,6 +462,41 @@ def run_prefill_drain_deployment( deallocate_port(p) +# EngineConfig.env key naming a whitespace-separated list of pip packages to +# install into the runtime container before the server launches. Some runtime +# images intentionally omit certain media-decoder libraries; the few serve tests +# that exercise a decode path install the decoder here at test time so coverage +# is retained without the shipped image carrying it. No-op when the key is unset. +TEST_ONLY_PIP_ENV_KEY = "DYN_TEST_ONLY_PIP_INSTALL" + +# Session-level guard so the same package set is installed at most once even +# though every parametrized deployment (and each retry) calls the installer. +_test_only_pip_done: set[str] = set() + + +def _install_test_only_packages(config: EngineConfig) -> None: + """Install any test-only pip packages a config requested via its env. + + Runs inside the same runtime container/interpreter the server subprocess + inherits, so the worker can import the freshly installed module. + """ + spec = (getattr(config, "env", None) or {}).get(TEST_ONLY_PIP_ENV_KEY, "").strip() + if not spec or spec in _test_only_pip_done: + return + packages = spec.split() + logging.getLogger(__name__).info( + "Installing test-only package(s) into runtime container: %s", + " ".join(packages), + ) + # --break-system-packages: runtime images use an externally-managed system + # python (PEP 668); this is the ephemeral test container, not a shipped image. + subprocess.run( + [sys.executable, "-m", "pip", "install", "--break-system-packages", *packages], + check=True, + ) + _test_only_pip_done.add(spec) + + def run_serve_deployment( config: EngineConfig, request: Any, @@ -486,6 +523,10 @@ def run_serve_deployment( logger.info("Using model: %s", config.model) logger.info("Script: %s", config.script_name) + # Install any decoder a codec-stripped image needs for this test, before the + # server launches, so the worker can import it. No-op unless the config opts in. + _install_test_only_packages(config) + prep = _prepare_deployment(config, request, ports=ports, extra_env=extra_env) config = prep.config merged_env = prep.merged_env diff --git a/tests/serve/multimodal_profiles/vllm.py b/tests/serve/multimodal_profiles/vllm.py index 8d061656c9f1..76e85bbedba4 100644 --- a/tests/serve/multimodal_profiles/vllm.py +++ b/tests/serve/multimodal_profiles/vllm.py @@ -109,6 +109,10 @@ delayed_start=60, profiled_vram_gib=8.2, requested_vllm_kv_cache_bytes=1_719_075_000, + # Backend video decode goes through vLLM's VideoMediaIO (opencv); + # the shipped image omits it as a media-codec carrier, so install + # it for this test only. See common._install_test_only_packages. + env={"DYN_TEST_ONLY_PIP_INSTALL": "opencv-python-headless"}, tests=[MmCase(payload=make_video_payload(["red", "static", "still"]))], ), "agg_unified": TopologyConfig( @@ -165,6 +169,8 @@ delayed_start=60, profiled_vram_gib=8.2, requested_vllm_kv_cache_bytes=1_719_075_000, + # See agg_video: install the opencv backend decoder for this test only. + env={"DYN_TEST_ONLY_PIP_INSTALL": "opencv-python-headless"}, tests=[ MmCase( payload=make_video_payload(["red", "static", "still"]), @@ -249,6 +255,8 @@ single_gpu=True, profiled_vram_gib=19.7, requested_vllm_kv_cache_bytes=1_714_881_000, + # See agg_video: install the opencv backend decoder for this test only. + env={"DYN_TEST_ONLY_PIP_INSTALL": "opencv-python-headless"}, tests=[MmCase(payload=make_video_payload(["red", "static", "still"]))], ), "p_d": TopologyConfig( diff --git a/tests/serve/test_sglang.py b/tests/serve/test_sglang.py index 2d4b10d53215..1bb07cac6bcb 100644 --- a/tests/serve/test_sglang.py +++ b/tests/serve/test_sglang.py @@ -558,6 +558,10 @@ class SGLangConfig(EngineConfig): "Qwen/Qwen2-VL-7B-Instruct", ], timeout=360, + # SGLang's video path decodes with decord; the shipped image omits it as + # a media-codec carrier, so install it for this test only. + # See common._install_test_only_packages. + env={"DYN_TEST_ONLY_PIP_INSTALL": "decord2"}, frontend_port=DefaultPort.FRONTEND.value, request_payloads=[ chat_payload( @@ -604,6 +608,9 @@ class SGLangConfig(EngineConfig): "DYN_ENCODE_GPU_MEM": "0.1", "DYN_WORKER_GPU_MEM": "0.4", "DYN_SGL_EMBEDDING_TRANSFER_MODE": "local", + # SGLang's video path decodes with decord; the shipped image omits it + # as a media-codec carrier, so install it for this test only. + "DYN_TEST_ONLY_PIP_INSTALL": "decord2", }, frontend_port=DefaultPort.FRONTEND.value, request_payloads=[