From e9c49058a67fb65d4f1d9dd21e1c042f06e60cf5 Mon Sep 17 00:00:00 2001 From: Dmitry Tokarev Date: Wed, 5 Aug 2026 20:26:31 -0400 Subject: [PATCH 1/2] fix(multimodal): copy NVDEC frames out of decoder-owned buffers _frame_to_rgb_hwc converted frames with tensor.cpu().numpy(). The decoder is constructed with use_device_memory=False, so the DLPack-wrapped frame is already host memory: .cpu() is a no-op and .numpy() aliases memory the decoder owns and may recycle between indexed reads. Correct behaviour relied on PyNvVideoCodec handing each DecodedFrame its own buffer, which is undocumented. Copy each frame into owned memory instead; the docstring also still described the device-memory path, so rewrite it. Covers both call sites: decode_video_nvdec and the SGLang NvdecVideoDecoder.get_frames_as_tensor convert through this helper. The new regression test drives the helper with a numpy source (numpy exports DLPack exactly like a DecodedFrame): it fails on the previous code and passes with the copy. Raised by automated review on the release cherry-pick (#12703). Co-Authored-By: Claude Fable 5 Signed-off-by: Dmitry Tokarev --- .../dynamo/common/multimodal/nvdec_decoder.py | 20 ++++++++++--------- .../tests/multimodal/test_nvdec_decoder.py | 17 ++++++++++++++++ 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/components/src/dynamo/common/multimodal/nvdec_decoder.py b/components/src/dynamo/common/multimodal/nvdec_decoder.py index 7e411d6685a0..3149383fb3bb 100644 --- a/components/src/dynamo/common/multimodal/nvdec_decoder.py +++ b/components/src/dynamo/common/multimodal/nvdec_decoder.py @@ -156,11 +156,16 @@ def _gpu_id() -> int: def _frame_to_rgb_hwc(frame) -> np.ndarray: - """Copy a decoded (device) RGB frame to a host ``(H, W, 3)`` uint8 array. - - A 2.x ``DecodedFrame`` (``output_color_type=RGB``) holds a CUDA buffer and - supports the DLPack protocol, so torch wraps it zero-copy on the GPU and - ``.cpu()`` copies to host. Validated on PyNvVideoCodec 2.1.1 for H.264/H.265. + """Copy a decoded RGB frame into an owned host ``(H, W, 3)`` uint8 array. + + A 2.x ``DecodedFrame`` (``output_color_type=RGB``) supports the DLPack + protocol, so torch wraps its buffer zero-copy. The decoder is constructed + with ``use_device_memory=False``, so that buffer is already host memory: + ``.cpu()`` is a no-op and ``.numpy()`` would keep aliasing decoder-owned + memory, which the decoder is free to recycle for the next indexed read. + The explicit ``np.array(..., copy=True)`` is what guarantees every + collected frame owns its pixels (and normalizes dtype in the same step). + Validated on PyNvVideoCodec 2.1.1 for H.264/H.265. """ import torch @@ -168,10 +173,7 @@ def _frame_to_rgb_hwc(frame) -> np.ndarray: tensor = torch.from_dlpack(frame) except Exception: # noqa: BLE001 - fall back to the CUDA-array-interface path tensor = torch.as_tensor(frame, device="cuda") - arr = tensor.cpu().numpy() - if arr.dtype != np.uint8: - arr = arr.astype(np.uint8) - return arr + return np.array(tensor.cpu().numpy(), dtype=np.uint8, copy=True) def _source_fps(decoder) -> float: diff --git a/components/src/dynamo/common/tests/multimodal/test_nvdec_decoder.py b/components/src/dynamo/common/tests/multimodal/test_nvdec_decoder.py index a99577859661..365e19c7b212 100644 --- a/components/src/dynamo/common/tests/multimodal/test_nvdec_decoder.py +++ b/components/src/dynamo/common/tests/multimodal/test_nvdec_decoder.py @@ -163,6 +163,23 @@ def test_should_use_nvdec_false_when_unavailable(monkeypatch): assert nd.should_use_nvdec("h264") is False +def test_frame_to_rgb_hwc_copies_out_of_decoder_memory(): + """The decoder hands over host frames (``use_device_memory=False``) and is + free to recycle the underlying buffer between indexed reads, so the + conversion must copy. numpy arrays export DLPack exactly like a + ``DecodedFrame`` does: without the copy, torch wraps the buffer zero-copy + and mutating the source would corrupt the already-collected frame. + """ + source = np.full((4, 6, 3), 7, dtype=np.uint8) + + converted = nd._frame_to_rgb_hwc(source) + source[:] = 99 # the decoder reuses its buffer for the next frame + + assert converted.dtype == np.uint8 + assert converted.shape == (4, 6, 3) + np.testing.assert_array_equal(converted, np.full((4, 6, 3), 7, dtype=np.uint8)) + + def test_decode_matches_frame_contract(monkeypatch): monkeypatch.setitem( sys.modules, "PyNvVideoCodec", _fake_pynv(num_frames=10, h=4, w=6) From 3e0dfdbbf2b1568ffb42566dea72036067b26858 Mon Sep 17 00:00:00 2001 From: Dmitry Tokarev Date: Wed, 5 Aug 2026 20:26:31 -0400 Subject: [PATCH 2/2] build(container): drop PyNvVideoCodec from the non-CUDA SGLang image The shared requirements.sglang.txt install was device-unguarded, so the NVIDIA-only NVDEC wheel landed in the Intel XPU image, where it is inert (no libnvcuvid). Mirror the existing branch in vllm_runtime.Dockerfile: filter it out with a line-anchored grep for non-CUDA devices and fail the build if the package still imports, so a silently broken filter cannot look like success. Rendered CUDA output is unchanged. Raised by automated review on the release cherry-pick (#12703). Co-Authored-By: Claude Fable 5 Signed-off-by: Dmitry Tokarev --- container/templates/sglang_runtime.Dockerfile | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/container/templates/sglang_runtime.Dockerfile b/container/templates/sglang_runtime.Dockerfile index 05c8f6014662..5c9a88572aa2 100644 --- a/container/templates/sglang_runtime.Dockerfile +++ b/container/templates/sglang_runtime.Dockerfile @@ -139,11 +139,29 @@ RUN --mount=type=bind,source=./container/deps/requirements.common.txt,target=/tm # Install SGLang-specific runtime dependencies without changing the upstream # dependency solution. imageio-ffmpeg is installed from source (no bundled # binary) for the VP9 video-encode path; see requirements.sglang.txt. +{% if device == "cuda" %} RUN --mount=type=bind,source=./container/deps/requirements.sglang.txt,target=/tmp/requirements.sglang.txt \ --mount=type=cache,target=/root/.cache/pip,sharing=locked \ export PIP_CACHE_DIR=/root/.cache/pip && \ pip install --break-system-packages --force-reinstall --no-deps \ --requirement /tmp/requirements.sglang.txt +{% else %} +# PyNvVideoCodec decodes on NVDEC through libnvcuvid, so it is inert on a +# non-NVIDIA device. Drop it from the shared requirements rather than ship an +# unusable NVIDIA codec wheel in the Intel XPU image. The pattern is anchored +# to the line start so it cannot match inside another requirement, and the +# import check fails the build if the package arrives by another route -- a +# filter that silently stopped matching would otherwise look like success. +# Whole-RUN branches rather than a conditional inside one RUN, for the reasons +# documented at the equivalent block in vllm_runtime.Dockerfile. +RUN --mount=type=bind,source=./container/deps/requirements.sglang.txt,target=/tmp/requirements.sglang.txt \ + --mount=type=cache,target=/root/.cache/pip,sharing=locked \ + export PIP_CACHE_DIR=/root/.cache/pip && \ + grep -v '^PyNvVideoCodec' /tmp/requirements.sglang.txt > /tmp/requirements.sglang.nonvidia.txt && \ + pip install --break-system-packages --force-reinstall --no-deps \ + --requirement /tmp/requirements.sglang.nonvidia.txt && \ + ! python3 -c "import PyNvVideoCodec" 2>/dev/null +{% endif %} # Remove the codec-bearing video-DECODE components from the upstream SGLang image # (PyAV, decord, OpenCV, torchcodec + any base ffmpeg/libav*), then copy the