Skip to content
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 78 additions & 0 deletions components/src/dynamo/common/protocols/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,91 @@
- video_protocol: NvCreateVideoRequest, NvVideosResponse for video generation
"""

from typing import Any, Dict, Optional

from dynamo.common.protocols.video_protocol import (
NvCreateVideoRequest,
NvVideosResponse,
VideoData,
)

MEDIA_PASSTHROUGH_KEY = "media_passthrough"
"""Key under a media request's ``extra_args`` where the frontend nests
unknown top-level request fields (an OpenAI client's ``extra_body``)."""


class MediaPassthroughRejected(ValueError):
"""A request tried to pass a knob the deployment owns, not the caller."""


# Keys a caller must never set through the passthrough bucket. These name a
# thing to load or a safety/policy control, and the bucket is fully caller
# controlled, so honoring them would let a request point the engine at code
# to load or turn a guardrail off.
_RESERVED_PASSTHROUGH_KEYS = frozenset(
{
"extra_args",
"guardrails",
"frame_interpolation_model_path",
}
)

# A knob whose name contains one of these is treated the same way: the first
# group names a fetch or load location, the second a safety/policy switch.
_UNSAFE_KEY_MARKERS = (
"path",
"checkpoint",
"ckpt",
"repo",
"hub",
"url",
"uri",
"local_dir",
"guardrail",
"safety",
"nsfw",
"moderation",
)


def _passthrough_key_rejected(key: str) -> bool:
if key in _RESERVED_PASSTHROUGH_KEYS or key.startswith("_"):
return True
lowered = key.lower()
return any(marker in lowered for marker in _UNSAFE_KEY_MARKERS)


def sanitize_media_passthrough(
extra_args: Optional[Dict[str, Any]],
) -> Dict[str, Any]:
"""Return the passthrough knobs that are safe to hand a backend.

The frontend nests a request's unknown top-level fields (an OpenAI
client's ``extra_body``) under ``extra_args[MEDIA_PASSTHROUGH_KEY]``.
Those are untyped and fully caller controlled, so a knob that names a
model or checkpoint path, a fetch location, or a safety or policy
control is refused here rather than passed to the engine, where it could
reach a load path or a guardrail switch. Everything else is returned
unchanged for the caller to apply.

Raises MediaPassthroughRejected (a ValueError) when a rejected knob is
present, so request building fails before the engine runs.
"""
knobs = (extra_args or {}).get(MEDIA_PASSTHROUGH_KEY) or {}
rejected = sorted(k for k in knobs if _passthrough_key_rejected(k))
if rejected:
raise MediaPassthroughRejected(
"these request fields cannot be set per request (a model or "
"checkpoint path, a fetch location, or a safety or policy "
"control is deployment configuration): " + ", ".join(rejected)
)
return dict(knobs)


__all__ = [
"MEDIA_PASSTHROUGH_KEY",
"MediaPassthroughRejected",
"sanitize_media_passthrough",
"NvCreateVideoRequest",
"NvVideosResponse",
"VideoData",
Expand Down
7 changes: 6 additions & 1 deletion components/src/dynamo/common/protocols/audio_protocol.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
manually and must be kept in sync.
"""

from typing import Literal, Optional
from typing import Any, Dict, Literal, Optional

from pydantic import BaseModel, Field

Expand All @@ -34,6 +34,11 @@ class NvCreateAudioSpeechRequest(BaseModel):
Follows vLLM-Omni's OpenAICreateSpeechRequest format.
"""

extra_args: Optional[Dict[str, Any]] = None
"""Worker-boundary passthrough. The frontend nests unknown top-level
request fields (an OpenAI client's extra_body) under the
"media_passthrough" key."""

# Standard OpenAI params
input: str
"""The text to synthesize into speech."""
Expand Down
7 changes: 6 additions & 1 deletion components/src/dynamo/common/protocols/image_protocol.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
# SPDX-License-Identifier: Apache-2.0


from typing import Optional
from typing import Any, Dict, Optional

from pydantic import BaseModel

Expand Down Expand Up @@ -38,6 +38,11 @@ class NvCreateImageRequest(BaseModel):
Matches the flattened Rust NvCreateImageRequest in lib/llm/src/protocols/openai/images.rs
"""

extra_args: Optional[Dict[str, Any]] = None
"""Worker-boundary passthrough. The frontend nests unknown top-level
request fields (an OpenAI client's extra_body) under the
"media_passthrough" key."""

prompt: str
"""The text prompt for image generation."""

Expand Down
7 changes: 6 additions & 1 deletion components/src/dynamo/common/protocols/video_protocol.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
"""
# TODO: Replace these Pydantic models with Python bindings to the Rust protocol types once PyO3 bindings are available.

from typing import Literal, Optional
from typing import Any, Dict, Literal, Optional

from pydantic import BaseModel

Expand Down Expand Up @@ -53,6 +53,11 @@ class NvCreateVideoRequest(BaseModel):
Matches Rust NvCreateVideoRequest in lib/llm/src/protocols/openai/videos.rs.
"""

extra_args: Optional[Dict[str, Any]] = None
"""Worker-boundary passthrough. The frontend nests unknown top-level
request fields (an OpenAI client's extra_body) under the
"media_passthrough" key."""

# Required fields
prompt: str
"""The text prompt for video generation."""
Expand Down
80 changes: 80 additions & 0 deletions components/src/dynamo/common/tests/test_media_passthrough.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Unit tests for the media passthrough sanitizer.

These run without any engine or model dependency: the point of the
sanitizer is to refuse a dangerous request field before anything is loaded,
so the test proves the rejection with no network and no model activity.
"""

import pytest
from dynamo.common.protocols import (
MEDIA_PASSTHROUGH_KEY,
MediaPassthroughRejected,
sanitize_media_passthrough,
)

pytestmark = [
pytest.mark.unit,
pytest.mark.gpu_0,
pytest.mark.pre_merge,
]


def _wrap(knobs):
return {MEDIA_PASSTHROUGH_KEY: knobs}


def test_none_and_empty_are_empty():
assert sanitize_media_passthrough(None) == {}
assert sanitize_media_passthrough({}) == {}
assert sanitize_media_passthrough(_wrap({})) == {}


def test_benign_knobs_pass_through():
knobs = {"backend_custom_knob": 0.5, "denoise_strength": 0.8, "generate_sound": True}
assert sanitize_media_passthrough(_wrap(knobs)) == knobs


@pytest.mark.parametrize(
"key",
[
"frame_interpolation_model_path", # the reported RCE field
"guardrails", # the reported guardrail bypass
"extra_args", # the container itself
"vae_checkpoint",
"unet_ckpt",
"lora_repo",
"hf_hub_id",
"weights_url",
"source_uri",
"custom_local_dir",
"safety_checker",
"nsfw_filter",
"content_moderation",
"_private_attr",
],
)
def test_reserved_and_shaped_keys_are_rejected(key):
with pytest.raises(MediaPassthroughRejected):
sanitize_media_passthrough(_wrap({key: "x", "harmless": 1}))


def test_rejection_names_the_offending_key():
with pytest.raises(MediaPassthroughRejected, match="frame_interpolation_model_path"):
sanitize_media_passthrough(
_wrap({"frame_interpolation_model_path": "attacker/repo"})
)


def test_rejected_is_a_value_error():
# Request building catches ValueError and returns a client error, so the
# sanitizer's exception must be a ValueError subclass.
assert issubclass(MediaPassthroughRejected, ValueError)


def test_original_bucket_is_not_mutated():
bucket = _wrap({"backend_custom_knob": 1})
sanitize_media_passthrough(bucket)["backend_custom_knob"] = 999
assert bucket[MEDIA_PASSTHROUGH_KEY]["backend_custom_knob"] == 1
7 changes: 7 additions & 0 deletions components/src/dynamo/vllm/omni/audio_handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
except ImportError:
Qwen3TTSPromptEmbedsBuilder = None # type: ignore[assignment, misc]

from dynamo.common.protocols import sanitize_media_passthrough
from dynamo.common.protocols.audio_protocol import NvCreateAudioSpeechRequest
from dynamo.common.utils.output_modalities import RequestType

Expand Down Expand Up @@ -260,6 +261,12 @@ async def _engine_inputs_tts(
if task_type == "VoiceDesign":
tts_params["non_streaming_mode"] = [True]

# Frontend-forwarded passthrough knobs join the engine params;
# fields the handler already set win. A knob naming a path or a
# policy control is refused here (see sanitize_media_passthrough).
for key, value in sanitize_media_passthrough(req.extra_args).items():
tts_params.setdefault(key, [value])

estimated_len = self._estimate_tts_prompt_len(tts_params)

prompt = {
Expand Down
32 changes: 32 additions & 0 deletions components/src/dynamo/vllm/omni/omni_handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@

from dynamo._core import Context
from dynamo.common.multimodal import ImageLoader
from dynamo.common.protocols import sanitize_media_passthrough
from dynamo.common.protocols.audio_protocol import NvCreateAudioSpeechRequest
from dynamo.common.protocols.image_protocol import ImageNvExt, NvCreateImageRequest
from dynamo.common.protocols.video_protocol import NvCreateVideoRequest, VideoNvExt
Expand Down Expand Up @@ -65,6 +66,35 @@
DEFAULT_VIDEO_FPS = 16


def _apply_media_passthrough(
sp: OmniDiffusionSamplingParams, extra_args: Optional[Dict[str, Any]]
) -> None:
"""Hand frontend-forwarded passthrough knobs to the engine.

The frontend nests a request's unknown top-level fields (an OpenAI
client's ``extra_body``) under ``extra_args["media_passthrough"]``.
``sanitize_media_passthrough`` drops the request if any knob names a
path/checkpoint or a policy control, then the rest ride ``sp.extra_args``
to the engine. Nothing is set on the sampling params by attribute name:
a caller-controlled key must not choose which attribute it writes.
"""
knobs = sanitize_media_passthrough(extra_args)
if not knobs:
return
existing = getattr(sp, "extra_args", None)
if isinstance(existing, dict):
existing.update(knobs)
else:
try:
sp.extra_args = knobs
except (AttributeError, TypeError):
logger.warning(
"Dropping media passthrough knobs %s: sampling params expose "
"no extra_args",
sorted(knobs),
)


@dataclass
class EngineInputs:
"""Parsed engine inputs ready for AsyncOmni.generate().
Expand Down Expand Up @@ -682,6 +712,7 @@ def _engine_inputs_from_image(self, req: NvCreateImageRequest) -> EngineInputs:
sp.seed = (
nvext.seed if nvext.seed is not None else random.randint(0, 2**32 - 1)
)
_apply_media_passthrough(sp, req.extra_args)

sampling_params_list = self._build_sampling_params_list(sp)
lora_request = self._resolve_and_apply_lora(req.model, sampling_params_list)
Expand Down Expand Up @@ -743,6 +774,7 @@ def _engine_inputs_from_video(
self._update_if_not_none(sp, "boundary_ratio", nvext.boundary_ratio)
self._update_if_not_none(sp, "guidance_scale_2", nvext.guidance_scale_2)
self._update_if_not_none(sp, "fps", fps)
_apply_media_passthrough(sp, req.extra_args)

sampling_params_list = self._build_sampling_params_list(sp)
lora_request = self._resolve_and_apply_lora(req.model, sampling_params_list)
Expand Down
54 changes: 54 additions & 0 deletions components/src/dynamo/vllm/tests/omni/test_omni_handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -263,6 +263,60 @@ async def test_i2v_nvext_params_on_sampling_params(self):
assert sp.guidance_scale_2 == 1.0
assert sp.num_inference_steps == 40

async def test_media_passthrough_reaches_sampling_params(self):
"""A top-level SDK extra_body field, nested by the frontend under
extra_args["media_passthrough"], rides sampling params extra_args to
the engine. Nothing is set on the sampling params by attribute name."""
handler = _make_handler()
req = NvCreateVideoRequest(
prompt="a cat by the sea",
model="test-model",
size="832x480",
extra_args={
"media_passthrough": {
"backend_custom_knob": 0.5,
"denoise_strength": 0.8,
}
},
)
result = await handler.build_engine_inputs(req, RequestType.VIDEO_GENERATION)
sp = result.sampling_params_list[0]
assert sp.extra_args["backend_custom_knob"] == 0.5
assert sp.extra_args["denoise_strength"] == 0.8

@pytest.mark.parametrize(
"bad_knob",
[
{"frame_interpolation_model_path": "attacker/repository"},
{"guardrails": False},
{"safety_checker": None},
{"vae_checkpoint_url": "http://evil/x.pkl"},
],
)
async def test_media_passthrough_rejects_load_and_policy_knobs(self, bad_knob):
"""A path/checkpoint field or a policy control is refused while the
request is being built, before any engine call, so it cannot reach a
model load or a guardrail switch."""
handler = _make_handler()
handler.engine_client.generate = MagicMock(
side_effect=AssertionError("engine must not run for a rejected request")
)
req = NvCreateVideoRequest(
prompt="a cat",
model="test-model",
size="832x480",
extra_args={"media_passthrough": bad_knob},
)
with pytest.raises(ValueError):
await handler.build_engine_inputs(req, RequestType.VIDEO_GENERATION)

def test_media_passthrough_absent_is_a_no_op(self):
req = NvCreateVideoRequest(prompt="a cat", model="test-model")
assert req.extra_args is None
handler = _make_handler()
inputs = handler._engine_inputs_from_video(req)
assert inputs.sampling_params_list is not None

def test_i2v_protocol_roundtrip(self):
"""VideoNvExt and NvCreateVideoRequest serialize/deserialize I2V fields correctly."""
req = NvCreateVideoRequest(
Expand Down
Loading
Loading