From a5dc5d977e6cc8a905ec9e24b5496940fd6207e8 Mon Sep 17 00:00:00 2001 From: Atindra Jha Date: Wed, 26 Aug 2026 07:56:31 +0000 Subject: [PATCH 1/5] feat(llm): add extra_body passthrough to media request structs Signed-off-by: Atindra Jha --- lib/llm/src/protocols/openai/audios.rs | 28 +++++++++++++++++++ lib/llm/src/protocols/openai/images.rs | 36 +++++++++++++++++++++++++ lib/llm/src/protocols/openai/videos.rs | 37 ++++++++++++++++++++++++++ 3 files changed, 101 insertions(+) diff --git a/lib/llm/src/protocols/openai/audios.rs b/lib/llm/src/protocols/openai/audios.rs index d74607b15727..afba5cd1c190 100644 --- a/lib/llm/src/protocols/openai/audios.rs +++ b/lib/llm/src/protocols/openai/audios.rs @@ -71,6 +71,12 @@ pub struct NvCreateAudioSpeechRequest { #[serde(skip_serializing_if = "Option::is_none")] pub user: Option, + /// Extra parameters passed through to the backend without strict + /// validation. Meant for backend-specific knobs that have no typed + /// field yet. Stable ones can be promoted to typed fields over time. + #[serde(skip_serializing_if = "Option::is_none")] + pub extra_body: Option>, + /// NVIDIA extensions (reserved for future use) #[serde(skip_serializing_if = "Option::is_none")] pub nvext: Option, @@ -245,12 +251,34 @@ mod tests { ref_text: None, max_new_tokens: None, user: None, + extra_body: None, nvext: None, }; let json = serde_json::to_string(&req).unwrap(); assert!(!json.contains("data_source")); } + #[test] + fn audio_request_extra_body_round_trips() { + let json = r#"{"input":"hello","extra_body":{"emotion":"calm","pitch":1.2}}"#; + let req: NvCreateAudioSpeechRequest = serde_json::from_str(json).unwrap(); + let extra = req.extra_body.as_ref().unwrap(); + assert_eq!(extra["emotion"], serde_json::json!("calm")); + assert_eq!(extra["pitch"], serde_json::json!(1.2)); + + let out = serde_json::to_string(&req).unwrap(); + let back: NvCreateAudioSpeechRequest = serde_json::from_str(&out).unwrap(); + assert_eq!(back.extra_body, req.extra_body); + } + + #[test] + fn audio_request_extra_body_absent_is_none_and_omitted() { + let json = r#"{"input":"hello"}"#; + let req: NvCreateAudioSpeechRequest = serde_json::from_str(json).unwrap(); + assert_eq!(req.extra_body, None); + assert!(!serde_json::to_string(&req).unwrap().contains("extra_body")); + } + // --- AudioData --- #[test] diff --git a/lib/llm/src/protocols/openai/images.rs b/lib/llm/src/protocols/openai/images.rs index 52f43a0f1d88..5a239f4748c9 100644 --- a/lib/llm/src/protocols/openai/images.rs +++ b/lib/llm/src/protocols/openai/images.rs @@ -20,6 +20,12 @@ pub struct NvCreateImageRequest { #[serde(skip_serializing_if = "Option::is_none")] pub input_reference: Option, + /// Extra parameters passed through to the backend without strict + /// validation. Meant for backend-specific knobs that have no typed + /// field yet. Stable ones can be promoted to typed fields over time. + #[serde(skip_serializing_if = "Option::is_none")] + pub extra_body: Option>, + #[serde(skip_serializing_if = "Option::is_none")] pub nvext: Option, } @@ -85,3 +91,33 @@ impl AnnotationsProvider for NvCreateImageRequest { .unwrap_or(false) } } + +#[cfg(test)] +mod tests { + use super::*; + + // --- NvCreateImageRequest --- + + #[test] + fn image_request_extra_body_round_trips_beside_flattened_inner() { + let json = r#"{"prompt":"a cat","extra_body":{"think_mode":true,"size_override":{"h":512,"w":768}}}"#; + let req: NvCreateImageRequest = serde_json::from_str(json).unwrap(); + assert_eq!(req.inner.prompt, "a cat"); + let extra = req.extra_body.as_ref().unwrap(); + assert_eq!(extra["think_mode"], serde_json::json!(true)); + assert_eq!(extra["size_override"]["h"], serde_json::json!(512)); + + let out = serde_json::to_string(&req).unwrap(); + let back: NvCreateImageRequest = serde_json::from_str(&out).unwrap(); + assert_eq!(back.extra_body, req.extra_body); + assert_eq!(back.inner.prompt, "a cat"); + } + + #[test] + fn image_request_extra_body_absent_is_none_and_omitted() { + let json = r#"{"prompt":"a cat"}"#; + let req: NvCreateImageRequest = serde_json::from_str(json).unwrap(); + assert_eq!(req.extra_body, None); + assert!(!serde_json::to_string(&req).unwrap().contains("extra_body")); + } +} diff --git a/lib/llm/src/protocols/openai/videos.rs b/lib/llm/src/protocols/openai/videos.rs index 0b96f5d57d1f..9fbe05a2b694 100644 --- a/lib/llm/src/protocols/openai/videos.rs +++ b/lib/llm/src/protocols/openai/videos.rs @@ -50,6 +50,12 @@ pub struct NvCreateVideoRequest { #[serde(skip_serializing_if = "Option::is_none")] pub stream: Option, + /// Extra parameters passed through to the backend without strict + /// validation. Meant for backend-specific knobs that have no typed + /// field yet. Stable ones can be promoted to typed fields over time. + #[serde(skip_serializing_if = "Option::is_none")] + pub extra_body: Option>, + /// NVIDIA extensions #[serde(skip_serializing_if = "Option::is_none")] pub nvext: Option, @@ -212,12 +218,43 @@ mod tests { response_format: None, output_format: None, stream: None, + extra_body: None, nvext: None, }; let json = serde_json::to_string(&req).unwrap(); assert!(!json.contains("stream")); } + #[test] + fn video_request_extra_body_round_trips() { + let json = r#"{"prompt":"cat","model":"cosmos","extra_body":{"generate_sound":true,"strength":0.7}}"#; + let req: NvCreateVideoRequest = serde_json::from_str(json).unwrap(); + let extra = req.extra_body.as_ref().unwrap(); + assert_eq!(extra["generate_sound"], serde_json::json!(true)); + assert_eq!(extra["strength"], serde_json::json!(0.7)); + + let out = serde_json::to_string(&req).unwrap(); + let back: NvCreateVideoRequest = serde_json::from_str(&out).unwrap(); + assert_eq!(back.extra_body, req.extra_body); + } + + #[test] + fn video_request_extra_body_absent_is_none_and_omitted() { + let json = r#"{"prompt":"cat","model":"wan"}"#; + let req: NvCreateVideoRequest = serde_json::from_str(json).unwrap(); + assert_eq!(req.extra_body, None); + assert!(!serde_json::to_string(&req).unwrap().contains("extra_body")); + } + + #[test] + fn video_request_unknown_top_level_fields_still_ignored() { + // extra_body is the explicit passthrough. Stray unknown fields keep + // getting dropped rather than captured. + let json = r#"{"prompt":"cat","model":"wan","not_a_field":1}"#; + let req: NvCreateVideoRequest = serde_json::from_str(json).unwrap(); + assert_eq!(req.extra_body, None); + } + #[test] fn video_request_output_format_optional_absent_is_none() { let json = r#"{"prompt":"cat","model":"wan"}"#; From 0b00ddce01531404391866a0657d8fe14f92a3ba Mon Sep 17 00:00:00 2001 From: Atindra Jha Date: Fri, 28 Aug 2026 07:50:08 +0000 Subject: [PATCH 2/5] fix(llm): align media passthrough with extra_body wire semantics Signed-off-by: Atindra Jha --- .../dynamo/common/protocols/audio_protocol.py | 7 +- .../dynamo/common/protocols/image_protocol.py | 7 +- .../dynamo/common/protocols/video_protocol.py | 7 +- lib/llm/src/protocols/openai/audios.rs | 38 +++-- lib/llm/src/protocols/openai/images.rs | 139 +++++++++++++++--- lib/llm/src/protocols/openai/videos.rs | 50 ++++--- 6 files changed, 188 insertions(+), 60 deletions(-) diff --git a/components/src/dynamo/common/protocols/audio_protocol.py b/components/src/dynamo/common/protocols/audio_protocol.py index f8b12fad51fa..c0c44e25a282 100644 --- a/components/src/dynamo/common/protocols/audio_protocol.py +++ b/components/src/dynamo/common/protocols/audio_protocol.py @@ -14,7 +14,7 @@ from typing import Literal, Optional -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field class AudioNvExt(BaseModel): @@ -34,6 +34,11 @@ class NvCreateAudioSpeechRequest(BaseModel): Follows vLLM-Omni's OpenAICreateSpeechRequest format. """ + model_config = ConfigDict(extra="allow") + """Unknown top-level fields are backend passthrough knobs. Keep them + so handlers can read and forward them instead of dropping them at + the worker boundary.""" + # Standard OpenAI params input: str """The text to synthesize into speech.""" diff --git a/components/src/dynamo/common/protocols/image_protocol.py b/components/src/dynamo/common/protocols/image_protocol.py index 2c621d44735c..bf186fd475ff 100644 --- a/components/src/dynamo/common/protocols/image_protocol.py +++ b/components/src/dynamo/common/protocols/image_protocol.py @@ -4,7 +4,7 @@ from typing import Optional -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict # For omni models, we need to support raw_request parsing and json output format. We need to have these protocols defined here for serialization and deserialization. # TODO: Replace these Pydantic models with Python bindings to the Rust protocol types once PyO3 bindings are available. @@ -38,6 +38,11 @@ class NvCreateImageRequest(BaseModel): Matches the flattened Rust NvCreateImageRequest in lib/llm/src/protocols/openai/images.rs """ + model_config = ConfigDict(extra="allow") + """Unknown top-level fields are backend passthrough knobs. Keep them + so handlers can read and forward them instead of dropping them at + the worker boundary.""" + prompt: str """The text prompt for image generation.""" diff --git a/components/src/dynamo/common/protocols/video_protocol.py b/components/src/dynamo/common/protocols/video_protocol.py index 13f11a8905c6..49d4247a31b6 100644 --- a/components/src/dynamo/common/protocols/video_protocol.py +++ b/components/src/dynamo/common/protocols/video_protocol.py @@ -10,7 +10,7 @@ from typing import Literal, Optional -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict class VideoNvExt(BaseModel): @@ -53,6 +53,11 @@ class NvCreateVideoRequest(BaseModel): Matches Rust NvCreateVideoRequest in lib/llm/src/protocols/openai/videos.rs. """ + model_config = ConfigDict(extra="allow") + """Unknown top-level fields are backend passthrough knobs. Keep them + so handlers can read and forward them instead of dropping them at + the worker boundary.""" + # Required fields prompt: str """The text prompt for video generation.""" diff --git a/lib/llm/src/protocols/openai/audios.rs b/lib/llm/src/protocols/openai/audios.rs index afba5cd1c190..aa0edc5f06fc 100644 --- a/lib/llm/src/protocols/openai/audios.rs +++ b/lib/llm/src/protocols/openai/audios.rs @@ -71,15 +71,16 @@ pub struct NvCreateAudioSpeechRequest { #[serde(skip_serializing_if = "Option::is_none")] pub user: Option, - /// Extra parameters passed through to the backend without strict - /// validation. Meant for backend-specific knobs that have no typed - /// field yet. Stable ones can be promoted to typed fields over time. - #[serde(skip_serializing_if = "Option::is_none")] - pub extra_body: Option>, - /// NVIDIA extensions (reserved for future use) #[serde(skip_serializing_if = "Option::is_none")] pub nvext: Option, + + /// Unknown top-level fields are retained here and forwarded to the + /// backend without strict validation. This matches the OpenAI client's + /// extra_body option, which merges into the top level of the body. + /// Stable knobs can be promoted to typed fields over time. + #[serde(default, flatten)] + pub passthrough: serde_json::Map, } /// Audio data in response @@ -251,32 +252,37 @@ mod tests { ref_text: None, max_new_tokens: None, user: None, - extra_body: None, nvext: None, + passthrough: serde_json::Map::new(), }; let json = serde_json::to_string(&req).unwrap(); assert!(!json.contains("data_source")); } #[test] - fn audio_request_extra_body_round_trips() { - let json = r#"{"input":"hello","extra_body":{"emotion":"calm","pitch":1.2}}"#; + fn audio_request_captures_unknown_top_level_fields() { + // The OpenAI client's extra_body option merges into the top level of + // the body, so that is where backend knobs arrive. + let json = r#"{"input":"hello","emotion":"calm","pitch":1.2}"#; let req: NvCreateAudioSpeechRequest = serde_json::from_str(json).unwrap(); - let extra = req.extra_body.as_ref().unwrap(); - assert_eq!(extra["emotion"], serde_json::json!("calm")); - assert_eq!(extra["pitch"], serde_json::json!(1.2)); + assert_eq!(req.passthrough["emotion"], serde_json::json!("calm")); + assert_eq!(req.passthrough["pitch"], serde_json::json!(1.2)); + assert!(!req.passthrough.contains_key("input")); let out = serde_json::to_string(&req).unwrap(); let back: NvCreateAudioSpeechRequest = serde_json::from_str(&out).unwrap(); - assert_eq!(back.extra_body, req.extra_body); + assert_eq!(back.passthrough, req.passthrough); + assert!(out.contains("\"emotion\":\"calm\"")); } #[test] - fn audio_request_extra_body_absent_is_none_and_omitted() { + fn audio_request_empty_passthrough_adds_nothing() { let json = r#"{"input":"hello"}"#; let req: NvCreateAudioSpeechRequest = serde_json::from_str(json).unwrap(); - assert_eq!(req.extra_body, None); - assert!(!serde_json::to_string(&req).unwrap().contains("extra_body")); + assert!(req.passthrough.is_empty()); + let out: serde_json::Value = + serde_json::from_str(&serde_json::to_string(&req).unwrap()).unwrap(); + assert_eq!(out, serde_json::json!({"input":"hello"})); } // --- AudioData --- diff --git a/lib/llm/src/protocols/openai/images.rs b/lib/llm/src/protocols/openai/images.rs index 5a239f4748c9..21c8e4f6653e 100644 --- a/lib/llm/src/protocols/openai/images.rs +++ b/lib/llm/src/protocols/openai/images.rs @@ -11,23 +11,93 @@ mod nvext; pub use nvext::{NvExt, NvExtProvider}; /// Image generation request with NVIDIA extensions. -#[derive(Serialize, Deserialize, Validate, Debug, Clone)] +/// +/// Serde is hand-rolled: `inner` occupies the top level of the wire body, +/// and a derived flattened map next to a flattened struct would capture the +/// struct's keys too. The manual impls keep one external contract, typed +/// fields at the top level plus unknown top-level fields retained in +/// [`Self::passthrough`]. +#[derive(Validate, Debug, Clone)] pub struct NvCreateImageRequest { - #[serde(flatten)] pub inner: dynamo_protocols::types::CreateImageRequest, /// Optional image reference that guides generation (for I2I/TI2I). - #[serde(skip_serializing_if = "Option::is_none")] pub input_reference: Option, - /// Extra parameters passed through to the backend without strict - /// validation. Meant for backend-specific knobs that have no typed - /// field yet. Stable ones can be promoted to typed fields over time. - #[serde(skip_serializing_if = "Option::is_none")] - pub extra_body: Option>, - - #[serde(skip_serializing_if = "Option::is_none")] pub nvext: Option, + + /// Unknown top-level fields are retained here and forwarded to the + /// backend without strict validation. This matches the OpenAI client's + /// extra_body option, which merges into the top level of the body. + /// Stable knobs can be promoted to typed fields over time. + pub passthrough: serde_json::Map, +} + +impl<'de> Deserialize<'de> for NvCreateImageRequest { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let mut body = serde_json::Map::deserialize(deserializer)?; + let input_reference = match body.remove("input_reference") { + Some(value) => serde_json::from_value(value).map_err(serde::de::Error::custom)?, + None => None, + }; + let nvext = match body.remove("nvext") { + Some(value) => serde_json::from_value(value).map_err(serde::de::Error::custom)?, + None => None, + }; + let inner: dynamo_protocols::types::CreateImageRequest = + serde_json::from_value(serde_json::Value::Object(body.clone())) + .map_err(serde::de::Error::custom)?; + // Keys the typed request consumed stay out of the passthrough. + if let serde_json::Value::Object(consumed) = + serde_json::to_value(&inner).map_err(serde::de::Error::custom)? + { + for key in consumed.keys() { + body.remove(key); + } + } + Ok(Self { + inner, + input_reference, + nvext, + passthrough: body, + }) + } +} + +impl Serialize for NvCreateImageRequest { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let mut body = match serde_json::to_value(&self.inner).map_err(serde::ser::Error::custom)? { + serde_json::Value::Object(map) => map, + _ => { + return Err(serde::ser::Error::custom( + "image request must serialize to an object", + )) + } + }; + // Typed fields win over passthrough entries of the same name. + for (key, value) in &self.passthrough { + body.entry(key.clone()).or_insert_with(|| value.clone()); + } + if let Some(input_reference) = &self.input_reference { + body.insert( + "input_reference".to_string(), + serde_json::Value::String(input_reference.clone()), + ); + } + if let Some(nvext) = &self.nvext { + body.insert( + "nvext".to_string(), + serde_json::to_value(nvext).map_err(serde::ser::Error::custom)?, + ); + } + body.serialize(serializer) + } } /// A response structure for image generation responses, embedding OpenAI's @@ -99,25 +169,56 @@ mod tests { // --- NvCreateImageRequest --- #[test] - fn image_request_extra_body_round_trips_beside_flattened_inner() { - let json = r#"{"prompt":"a cat","extra_body":{"think_mode":true,"size_override":{"h":512,"w":768}}}"#; + fn image_request_captures_unknown_top_level_fields() { + // The OpenAI client's extra_body option merges into the top level of + // the body, so that is where backend knobs arrive. + let json = r#"{"prompt":"a cat","think_mode":true,"size_override":{"h":512,"w":768}}"#; let req: NvCreateImageRequest = serde_json::from_str(json).unwrap(); assert_eq!(req.inner.prompt, "a cat"); - let extra = req.extra_body.as_ref().unwrap(); - assert_eq!(extra["think_mode"], serde_json::json!(true)); - assert_eq!(extra["size_override"]["h"], serde_json::json!(512)); + assert_eq!(req.passthrough["think_mode"], serde_json::json!(true)); + assert_eq!( + req.passthrough["size_override"]["h"], + serde_json::json!(512) + ); let out = serde_json::to_string(&req).unwrap(); let back: NvCreateImageRequest = serde_json::from_str(&out).unwrap(); - assert_eq!(back.extra_body, req.extra_body); assert_eq!(back.inner.prompt, "a cat"); + assert_eq!(back.passthrough, req.passthrough); + } + + #[test] + fn image_request_typed_fields_stay_out_of_passthrough() { + let json = r#"{"prompt":"a cat","n":2,"input_reference":"ref.png","nvext":{"seed":7},"custom_knob":1}"#; + let req: NvCreateImageRequest = serde_json::from_str(json).unwrap(); + assert_eq!(req.inner.prompt, "a cat"); + assert_eq!(req.input_reference.as_deref(), Some("ref.png")); + assert_eq!(req.nvext.as_ref().and_then(|n| n.seed), Some(7)); + assert_eq!(req.passthrough["custom_knob"], serde_json::json!(1)); + for consumed in ["prompt", "n", "input_reference", "nvext"] { + assert!(!req.passthrough.contains_key(consumed), "{consumed}"); + } + } + + #[test] + fn image_request_round_trips_all_field_kinds_at_top_level() { + let json = + r#"{"prompt":"a cat","input_reference":"ref.png","nvext":{"seed":7},"knob":"x"}"#; + let req: NvCreateImageRequest = serde_json::from_str(json).unwrap(); + let out: serde_json::Value = + serde_json::from_str(&serde_json::to_string(&req).unwrap()).unwrap(); + assert_eq!(out["prompt"], serde_json::json!("a cat")); + assert_eq!(out["input_reference"], serde_json::json!("ref.png")); + assert_eq!(out["nvext"]["seed"], serde_json::json!(7)); + assert_eq!(out["knob"], serde_json::json!("x")); + assert!(out.get("passthrough").is_none()); + assert!(out.get("inner").is_none()); } #[test] - fn image_request_extra_body_absent_is_none_and_omitted() { + fn image_request_empty_passthrough_stays_empty() { let json = r#"{"prompt":"a cat"}"#; let req: NvCreateImageRequest = serde_json::from_str(json).unwrap(); - assert_eq!(req.extra_body, None); - assert!(!serde_json::to_string(&req).unwrap().contains("extra_body")); + assert!(req.passthrough.is_empty()); } } diff --git a/lib/llm/src/protocols/openai/videos.rs b/lib/llm/src/protocols/openai/videos.rs index 9fbe05a2b694..f9ebdd6ce3dd 100644 --- a/lib/llm/src/protocols/openai/videos.rs +++ b/lib/llm/src/protocols/openai/videos.rs @@ -50,15 +50,16 @@ pub struct NvCreateVideoRequest { #[serde(skip_serializing_if = "Option::is_none")] pub stream: Option, - /// Extra parameters passed through to the backend without strict - /// validation. Meant for backend-specific knobs that have no typed - /// field yet. Stable ones can be promoted to typed fields over time. - #[serde(skip_serializing_if = "Option::is_none")] - pub extra_body: Option>, - /// NVIDIA extensions #[serde(skip_serializing_if = "Option::is_none")] pub nvext: Option, + + /// Unknown top-level fields are retained here and forwarded to the + /// backend without strict validation. This matches the OpenAI client's + /// extra_body option, which merges into the top level of the body. + /// Stable knobs can be promoted to typed fields over time. + #[serde(default, flatten)] + pub passthrough: serde_json::Map, } /// Video data in response @@ -218,41 +219,46 @@ mod tests { response_format: None, output_format: None, stream: None, - extra_body: None, nvext: None, + passthrough: serde_json::Map::new(), }; let json = serde_json::to_string(&req).unwrap(); assert!(!json.contains("stream")); } #[test] - fn video_request_extra_body_round_trips() { - let json = r#"{"prompt":"cat","model":"cosmos","extra_body":{"generate_sound":true,"strength":0.7}}"#; + fn video_request_captures_unknown_top_level_fields() { + // The OpenAI client's extra_body option merges into the top level of + // the body, so that is where backend knobs arrive. + let json = r#"{"prompt":"cat","model":"cosmos","generate_sound":true,"strength":0.7}"#; let req: NvCreateVideoRequest = serde_json::from_str(json).unwrap(); - let extra = req.extra_body.as_ref().unwrap(); - assert_eq!(extra["generate_sound"], serde_json::json!(true)); - assert_eq!(extra["strength"], serde_json::json!(0.7)); + assert_eq!(req.passthrough["generate_sound"], serde_json::json!(true)); + assert_eq!(req.passthrough["strength"], serde_json::json!(0.7)); let out = serde_json::to_string(&req).unwrap(); let back: NvCreateVideoRequest = serde_json::from_str(&out).unwrap(); - assert_eq!(back.extra_body, req.extra_body); + assert_eq!(back.passthrough, req.passthrough); + assert!(out.contains("\"generate_sound\":true")); } #[test] - fn video_request_extra_body_absent_is_none_and_omitted() { - let json = r#"{"prompt":"cat","model":"wan"}"#; + fn video_request_typed_fields_stay_out_of_passthrough() { + let json = r#"{"prompt":"cat","model":"wan","stream":true,"custom_knob":1}"#; let req: NvCreateVideoRequest = serde_json::from_str(json).unwrap(); - assert_eq!(req.extra_body, None); - assert!(!serde_json::to_string(&req).unwrap().contains("extra_body")); + assert_eq!(req.stream, Some(true)); + assert!(!req.passthrough.contains_key("prompt")); + assert!(!req.passthrough.contains_key("stream")); + assert_eq!(req.passthrough["custom_knob"], serde_json::json!(1)); } #[test] - fn video_request_unknown_top_level_fields_still_ignored() { - // extra_body is the explicit passthrough. Stray unknown fields keep - // getting dropped rather than captured. - let json = r#"{"prompt":"cat","model":"wan","not_a_field":1}"#; + fn video_request_empty_passthrough_adds_nothing() { + let json = r#"{"prompt":"cat","model":"wan"}"#; let req: NvCreateVideoRequest = serde_json::from_str(json).unwrap(); - assert_eq!(req.extra_body, None); + assert!(req.passthrough.is_empty()); + let out: serde_json::Value = + serde_json::from_str(&serde_json::to_string(&req).unwrap()).unwrap(); + assert_eq!(out, serde_json::json!({"prompt":"cat","model":"wan"})); } #[test] From 62b88f9e7cd7780b15139f8e0ebd2c11e8c7b4f1 Mon Sep 17 00:00:00 2001 From: Atindra Jha Date: Sat, 29 Aug 2026 20:43:56 +0000 Subject: [PATCH 3/5] feat(llm): carry media passthrough to workers as namespaced extra_args Signed-off-by: Atindra Jha --- .../src/dynamo/common/protocols/__init__.py | 5 ++ .../dynamo/common/protocols/audio_protocol.py | 12 ++--- .../dynamo/common/protocols/image_protocol.py | 12 ++--- .../dynamo/common/protocols/video_protocol.py | 12 ++--- .../src/dynamo/vllm/omni/audio_handler.py | 8 ++++ .../src/dynamo/vllm/omni/omni_handler.py | 36 ++++++++++++++ .../vllm/tests/omni/test_omni_handler.py | 29 +++++++++++ lib/llm/src/http/service/openai.rs | 10 ++-- lib/llm/src/protocols/openai.rs | 22 +++++++++ lib/llm/src/protocols/openai/audios.rs | 31 ++++++++++++ lib/llm/src/protocols/openai/images.rs | 47 +++++++++++++++++- lib/llm/src/protocols/openai/videos.rs | 48 +++++++++++++++++++ 12 files changed, 250 insertions(+), 22 deletions(-) diff --git a/components/src/dynamo/common/protocols/__init__.py b/components/src/dynamo/common/protocols/__init__.py index 55f2125b21e8..d4134e66c552 100644 --- a/components/src/dynamo/common/protocols/__init__.py +++ b/components/src/dynamo/common/protocols/__init__.py @@ -13,7 +13,12 @@ VideoData, ) +MEDIA_PASSTHROUGH_KEY = "media_passthrough" +"""Key under a media request's ``extra_args`` where the frontend nests +unknown top-level request fields (an OpenAI client's ``extra_body``).""" + __all__ = [ + "MEDIA_PASSTHROUGH_KEY", "NvCreateVideoRequest", "NvVideosResponse", "VideoData", diff --git a/components/src/dynamo/common/protocols/audio_protocol.py b/components/src/dynamo/common/protocols/audio_protocol.py index c0c44e25a282..17ea95da55d0 100644 --- a/components/src/dynamo/common/protocols/audio_protocol.py +++ b/components/src/dynamo/common/protocols/audio_protocol.py @@ -12,9 +12,9 @@ manually and must be kept in sync. """ -from typing import Literal, Optional +from typing import Any, Dict, Literal, Optional -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, Field class AudioNvExt(BaseModel): @@ -34,10 +34,10 @@ class NvCreateAudioSpeechRequest(BaseModel): Follows vLLM-Omni's OpenAICreateSpeechRequest format. """ - model_config = ConfigDict(extra="allow") - """Unknown top-level fields are backend passthrough knobs. Keep them - so handlers can read and forward them instead of dropping them at - the worker boundary.""" + extra_args: Optional[Dict[str, Any]] = None + """Worker-boundary passthrough. The frontend nests unknown top-level + request fields (an OpenAI client's extra_body) under the + "media_passthrough" key.""" # Standard OpenAI params input: str diff --git a/components/src/dynamo/common/protocols/image_protocol.py b/components/src/dynamo/common/protocols/image_protocol.py index bf186fd475ff..f4c60f8d7510 100644 --- a/components/src/dynamo/common/protocols/image_protocol.py +++ b/components/src/dynamo/common/protocols/image_protocol.py @@ -2,9 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 -from typing import Optional +from typing import Any, Dict, Optional -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel # For omni models, we need to support raw_request parsing and json output format. We need to have these protocols defined here for serialization and deserialization. # TODO: Replace these Pydantic models with Python bindings to the Rust protocol types once PyO3 bindings are available. @@ -38,10 +38,10 @@ class NvCreateImageRequest(BaseModel): Matches the flattened Rust NvCreateImageRequest in lib/llm/src/protocols/openai/images.rs """ - model_config = ConfigDict(extra="allow") - """Unknown top-level fields are backend passthrough knobs. Keep them - so handlers can read and forward them instead of dropping them at - the worker boundary.""" + extra_args: Optional[Dict[str, Any]] = None + """Worker-boundary passthrough. The frontend nests unknown top-level + request fields (an OpenAI client's extra_body) under the + "media_passthrough" key.""" prompt: str """The text prompt for image generation.""" diff --git a/components/src/dynamo/common/protocols/video_protocol.py b/components/src/dynamo/common/protocols/video_protocol.py index 49d4247a31b6..644cea28e161 100644 --- a/components/src/dynamo/common/protocols/video_protocol.py +++ b/components/src/dynamo/common/protocols/video_protocol.py @@ -8,9 +8,9 @@ """ # TODO: Replace these Pydantic models with Python bindings to the Rust protocol types once PyO3 bindings are available. -from typing import Literal, Optional +from typing import Any, Dict, Literal, Optional -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel class VideoNvExt(BaseModel): @@ -53,10 +53,10 @@ class NvCreateVideoRequest(BaseModel): Matches Rust NvCreateVideoRequest in lib/llm/src/protocols/openai/videos.rs. """ - model_config = ConfigDict(extra="allow") - """Unknown top-level fields are backend passthrough knobs. Keep them - so handlers can read and forward them instead of dropping them at - the worker boundary.""" + extra_args: Optional[Dict[str, Any]] = None + """Worker-boundary passthrough. The frontend nests unknown top-level + request fields (an OpenAI client's extra_body) under the + "media_passthrough" key.""" # Required fields prompt: str diff --git a/components/src/dynamo/vllm/omni/audio_handler.py b/components/src/dynamo/vllm/omni/audio_handler.py index ffb527f04f7c..d7ba15c21de4 100644 --- a/components/src/dynamo/vllm/omni/audio_handler.py +++ b/components/src/dynamo/vllm/omni/audio_handler.py @@ -21,6 +21,7 @@ except ImportError: Qwen3TTSPromptEmbedsBuilder = None # type: ignore[assignment, misc] +from dynamo.common.protocols import MEDIA_PASSTHROUGH_KEY from dynamo.common.protocols.audio_protocol import NvCreateAudioSpeechRequest from dynamo.common.utils.output_modalities import RequestType @@ -260,6 +261,13 @@ async def _engine_inputs_tts( if task_type == "VoiceDesign": tts_params["non_streaming_mode"] = [True] + # Frontend-forwarded passthrough knobs join the engine params; + # fields the handler already set win. + for key, value in ( + (req.extra_args or {}).get(MEDIA_PASSTHROUGH_KEY) or {} + ).items(): + tts_params.setdefault(key, [value]) + estimated_len = self._estimate_tts_prompt_len(tts_params) prompt = { diff --git a/components/src/dynamo/vllm/omni/omni_handler.py b/components/src/dynamo/vllm/omni/omni_handler.py index 9cd40d44eaf2..9f9913d4e9a5 100644 --- a/components/src/dynamo/vllm/omni/omni_handler.py +++ b/components/src/dynamo/vllm/omni/omni_handler.py @@ -26,6 +26,7 @@ from dynamo._core import Context from dynamo.common.multimodal import ImageLoader +from dynamo.common.protocols import MEDIA_PASSTHROUGH_KEY from dynamo.common.protocols.audio_protocol import NvCreateAudioSpeechRequest from dynamo.common.protocols.image_protocol import ImageNvExt, NvCreateImageRequest from dynamo.common.protocols.video_protocol import NvCreateVideoRequest, VideoNvExt @@ -65,6 +66,39 @@ DEFAULT_VIDEO_FPS = 16 +def _apply_media_passthrough( + sp: OmniDiffusionSamplingParams, extra_args: Optional[Dict[str, Any]] +) -> None: + """Apply frontend-forwarded passthrough knobs to diffusion sampling params. + + The frontend nests a request's unknown top-level fields (an OpenAI + client's ``extra_body``) under ``extra_args["media_passthrough"]``. + Knobs matching a sampling-params attribute are set directly; the rest + land in ``sp.extra_args`` for the engine to interpret. + """ + knobs = (extra_args or {}).get(MEDIA_PASSTHROUGH_KEY) or {} + unmatched: Dict[str, Any] = {} + for key, value in knobs.items(): + if key != "extra_args" and hasattr(sp, key): + setattr(sp, key, value) + else: + unmatched[key] = value + if not unmatched: + return + existing = getattr(sp, "extra_args", None) + if isinstance(existing, dict): + existing.update(unmatched) + else: + try: + sp.extra_args = unmatched + except (AttributeError, TypeError): + logger.warning( + "Dropping media passthrough knobs %s: sampling params expose " + "no matching attribute and no extra_args", + sorted(unmatched), + ) + + @dataclass class EngineInputs: """Parsed engine inputs ready for AsyncOmni.generate(). @@ -682,6 +716,7 @@ def _engine_inputs_from_image(self, req: NvCreateImageRequest) -> EngineInputs: sp.seed = ( nvext.seed if nvext.seed is not None else random.randint(0, 2**32 - 1) ) + _apply_media_passthrough(sp, req.extra_args) sampling_params_list = self._build_sampling_params_list(sp) lora_request = self._resolve_and_apply_lora(req.model, sampling_params_list) @@ -743,6 +778,7 @@ def _engine_inputs_from_video( self._update_if_not_none(sp, "boundary_ratio", nvext.boundary_ratio) self._update_if_not_none(sp, "guidance_scale_2", nvext.guidance_scale_2) self._update_if_not_none(sp, "fps", fps) + _apply_media_passthrough(sp, req.extra_args) sampling_params_list = self._build_sampling_params_list(sp) lora_request = self._resolve_and_apply_lora(req.model, sampling_params_list) diff --git a/components/src/dynamo/vllm/tests/omni/test_omni_handler.py b/components/src/dynamo/vllm/tests/omni/test_omni_handler.py index 923be332ec1e..9d2e183a5da0 100644 --- a/components/src/dynamo/vllm/tests/omni/test_omni_handler.py +++ b/components/src/dynamo/vllm/tests/omni/test_omni_handler.py @@ -263,6 +263,35 @@ async def test_i2v_nvext_params_on_sampling_params(self): assert sp.guidance_scale_2 == 1.0 assert sp.num_inference_steps == 40 + async def test_media_passthrough_reaches_sampling_params(self): + """A top-level SDK extra_body field, nested by the frontend under + extra_args["media_passthrough"], reaches the engine input.""" + handler = _make_handler() + req = NvCreateVideoRequest( + prompt="a cat by the sea", + model="test-model", + size="832x480", + extra_args={ + "media_passthrough": { + "num_inference_steps": 12, + "backend_custom_knob": 0.5, + } + }, + ) + result = await handler.build_engine_inputs(req, RequestType.VIDEO_GENERATION) + sp = result.sampling_params_list[0] + # Knobs matching a sampling-params attribute apply directly. + assert sp.num_inference_steps == 12 + # The rest ride sampling params extra_args to the engine. + assert sp.extra_args["backend_custom_knob"] == 0.5 + + def test_media_passthrough_absent_is_a_no_op(self): + req = NvCreateVideoRequest(prompt="a cat", model="test-model") + assert req.extra_args is None + handler = _make_handler() + inputs = handler._engine_inputs_from_video(req) + assert inputs.sampling_params_list is not None + def test_i2v_protocol_roundtrip(self): """VideoNvExt and NvCreateVideoRequest serialize/deserialize I2V fields correctly.""" req = NvCreateVideoRequest( diff --git a/lib/llm/src/http/service/openai.rs b/lib/llm/src/http/service/openai.rs index 6ad9988504fe..8cba60aca668 100644 --- a/lib/llm/src/http/service/openai.rs +++ b/lib/llm/src/http/service/openai.rs @@ -4264,13 +4264,14 @@ pub fn responses_router( async fn images( State(state): State>, headers: HeaderMap, - Json(request): Json, + Json(mut request): Json, ) -> Result { // return a 503 if the service is not ready // (per-model readiness check is deferred until after we resolve the // ImageModel enum into a string; see below) check_ready(&state)?; + request.nest_passthrough(); let request_id = get_or_create_request_id(&headers); let request = context_from_headers(request, request_id, &headers)?; let request_id = request.id().to_string(); @@ -4405,12 +4406,13 @@ pub fn images_router( async fn videos( State(state): State>, headers: HeaderMap, - Json(request): Json, + Json(mut request): Json, ) -> Result { // return a 503 if the service or model is not ready check_ready(&state)?; check_model_serving_ready(&state, &request.model)?; + request.nest_passthrough(); let request_id = get_or_create_request_id(&headers); let request = context_from_headers(request, request_id, &headers)?; let request_id = request.id().to_string(); @@ -4527,11 +4529,12 @@ async fn videos( async fn video_stream( State(state): State>, headers: HeaderMap, - Json(request): Json, + Json(mut request): Json, ) -> Result { check_ready(&state)?; check_model_serving_ready(&state, &request.model)?; + request.nest_passthrough(); let request_id = get_or_create_request_id(&headers); let request = context_from_headers(request, request_id, &headers)?; let model = request.model.clone(); @@ -4746,6 +4749,7 @@ async fn handler_audio_speech( .get_or_insert_default() .frontend_accepts_audio_chunks = Some(true); } + request.nest_passthrough(); let request = context_from_headers(request, request_id, &headers)?; // model is optional in the request; fall back to a model that can actually diff --git a/lib/llm/src/protocols/openai.rs b/lib/llm/src/protocols/openai.rs index a594a3c595b1..37cdf573f0f4 100644 --- a/lib/llm/src/protocols/openai.rs +++ b/lib/llm/src/protocols/openai.rs @@ -37,6 +37,28 @@ use validate::{ TEMPERATURE_RANGE, validate_range, validate_top_p, }; +/// Key under `extra_args` where media handlers nest a request's captured +/// top-level passthrough before dispatching it to a worker. +pub const MEDIA_PASSTHROUGH_KEY: &str = "media_passthrough"; + +/// Move a media request's captured top-level unknowns under an explicit +/// `extra_args["media_passthrough"]` entry. Handlers call this before +/// dispatch so the worker boundary carries one nested, namespaced field +/// instead of loose top-level unknowns. +pub(crate) fn nest_media_passthrough( + passthrough: &mut serde_json::Map, + extra_args: &mut Option>, +) { + if passthrough.is_empty() { + return; + } + let nested = std::mem::take(passthrough); + extra_args.get_or_insert_with(serde_json::Map::new).insert( + MEDIA_PASSTHROUGH_KEY.to_string(), + serde_json::Value::Object(nested), + ); +} + /// Side from which prompt tokens are truncated. #[derive(ToSchema, Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)] #[serde(rename_all = "snake_case")] diff --git a/lib/llm/src/protocols/openai/audios.rs b/lib/llm/src/protocols/openai/audios.rs index aa0edc5f06fc..82f9568acf8f 100644 --- a/lib/llm/src/protocols/openai/audios.rs +++ b/lib/llm/src/protocols/openai/audios.rs @@ -75,6 +75,14 @@ pub struct NvCreateAudioSpeechRequest { #[serde(skip_serializing_if = "Option::is_none")] pub nvext: Option, + /// Worker-boundary contract, not a public field: the frontend moves + /// `passthrough` under `extra_args["media_passthrough"]` before + /// dispatch (see [`Self::nest_passthrough`]) so workers read one + /// explicit nested entry. A client-sent `extra_args` lands in + /// `passthrough` like any other unknown field. + #[serde(default, skip_deserializing, skip_serializing_if = "Option::is_none")] + pub extra_args: Option>, + /// Unknown top-level fields are retained here and forwarded to the /// backend without strict validation. This matches the OpenAI client's /// extra_body option, which merges into the top level of the body. @@ -83,6 +91,14 @@ pub struct NvCreateAudioSpeechRequest { pub passthrough: serde_json::Map, } +impl NvCreateAudioSpeechRequest { + /// Nest captured top-level unknowns under `extra_args["media_passthrough"]` + /// for dispatch to a worker. + pub fn nest_passthrough(&mut self) { + super::nest_media_passthrough(&mut self.passthrough, &mut self.extra_args); + } +} + /// Audio data in response #[derive(Serialize, Deserialize, Debug, Clone)] pub struct AudioData { @@ -253,6 +269,7 @@ mod tests { max_new_tokens: None, user: None, nvext: None, + extra_args: None, passthrough: serde_json::Map::new(), }; let json = serde_json::to_string(&req).unwrap(); @@ -285,6 +302,20 @@ mod tests { assert_eq!(out, serde_json::json!({"input":"hello"})); } + #[test] + fn audio_request_nests_passthrough_for_workers() { + let json = r#"{"input":"hello","emotion":"calm"}"#; + let mut req: NvCreateAudioSpeechRequest = serde_json::from_str(json).unwrap(); + req.nest_passthrough(); + assert!(req.passthrough.is_empty()); + let out = serde_json::to_value(&req).unwrap(); + assert_eq!( + out["extra_args"]["media_passthrough"]["emotion"], + serde_json::json!("calm") + ); + assert!(out.get("emotion").is_none()); + } + // --- AudioData --- #[test] diff --git a/lib/llm/src/protocols/openai/images.rs b/lib/llm/src/protocols/openai/images.rs index 21c8e4f6653e..fa00bde87b7f 100644 --- a/lib/llm/src/protocols/openai/images.rs +++ b/lib/llm/src/protocols/openai/images.rs @@ -26,6 +26,13 @@ pub struct NvCreateImageRequest { pub nvext: Option, + /// Worker-boundary contract, not a public field: the frontend moves + /// `passthrough` under `extra_args["media_passthrough"]` before + /// dispatch (see [`Self::nest_passthrough`]) so workers read one + /// explicit nested entry. A client-sent `extra_args` lands in + /// `passthrough` like any other unknown field. + pub extra_args: Option>, + /// Unknown top-level fields are retained here and forwarded to the /// backend without strict validation. This matches the OpenAI client's /// extra_body option, which merges into the top level of the body. @@ -33,6 +40,14 @@ pub struct NvCreateImageRequest { pub passthrough: serde_json::Map, } +impl NvCreateImageRequest { + /// Nest captured top-level unknowns under `extra_args["media_passthrough"]` + /// for dispatch to a worker. + pub fn nest_passthrough(&mut self) { + super::nest_media_passthrough(&mut self.passthrough, &mut self.extra_args); + } +} + impl<'de> Deserialize<'de> for NvCreateImageRequest { fn deserialize(deserializer: D) -> Result where @@ -62,6 +77,7 @@ impl<'de> Deserialize<'de> for NvCreateImageRequest { inner, input_reference, nvext, + extra_args: None, passthrough: body, }) } @@ -77,7 +93,7 @@ impl Serialize for NvCreateImageRequest { _ => { return Err(serde::ser::Error::custom( "image request must serialize to an object", - )) + )); } }; // Typed fields win over passthrough entries of the same name. @@ -96,6 +112,12 @@ impl Serialize for NvCreateImageRequest { serde_json::to_value(nvext).map_err(serde::ser::Error::custom)?, ); } + if let Some(extra_args) = &self.extra_args { + body.insert( + "extra_args".to_string(), + serde_json::Value::Object(extra_args.clone()), + ); + } body.serialize(serializer) } } @@ -221,4 +243,27 @@ mod tests { let req: NvCreateImageRequest = serde_json::from_str(json).unwrap(); assert!(req.passthrough.is_empty()); } + + #[test] + fn image_request_nests_passthrough_for_workers() { + let json = r#"{"prompt":"a cat","think_mode":true}"#; + let mut req: NvCreateImageRequest = serde_json::from_str(json).unwrap(); + req.nest_passthrough(); + assert!(req.passthrough.is_empty()); + let out = serde_json::to_value(&req).unwrap(); + assert_eq!( + out["extra_args"]["media_passthrough"]["think_mode"], + serde_json::json!(true) + ); + assert!(out.get("think_mode").is_none()); + assert_eq!(out["prompt"], serde_json::json!("a cat")); + } + + #[test] + fn image_request_client_extra_args_is_not_the_worker_field() { + let json = r#"{"prompt":"a cat","extra_args":{"x":1}}"#; + let req: NvCreateImageRequest = serde_json::from_str(json).unwrap(); + assert!(req.extra_args.is_none()); + assert_eq!(req.passthrough["extra_args"]["x"], serde_json::json!(1)); + } } diff --git a/lib/llm/src/protocols/openai/videos.rs b/lib/llm/src/protocols/openai/videos.rs index f9ebdd6ce3dd..9b325286048c 100644 --- a/lib/llm/src/protocols/openai/videos.rs +++ b/lib/llm/src/protocols/openai/videos.rs @@ -54,6 +54,14 @@ pub struct NvCreateVideoRequest { #[serde(skip_serializing_if = "Option::is_none")] pub nvext: Option, + /// Worker-boundary contract, not a public field: the frontend moves + /// `passthrough` under `extra_args["media_passthrough"]` before + /// dispatch (see [`Self::nest_passthrough`]) so workers read one + /// explicit nested entry. A client-sent `extra_args` lands in + /// `passthrough` like any other unknown field. + #[serde(default, skip_deserializing, skip_serializing_if = "Option::is_none")] + pub extra_args: Option>, + /// Unknown top-level fields are retained here and forwarded to the /// backend without strict validation. This matches the OpenAI client's /// extra_body option, which merges into the top level of the body. @@ -62,6 +70,14 @@ pub struct NvCreateVideoRequest { pub passthrough: serde_json::Map, } +impl NvCreateVideoRequest { + /// Nest captured top-level unknowns under `extra_args["media_passthrough"]` + /// for dispatch to a worker. + pub fn nest_passthrough(&mut self) { + super::nest_media_passthrough(&mut self.passthrough, &mut self.extra_args); + } +} + /// Video data in response #[derive(Serialize, Deserialize, Debug, Clone)] pub struct VideoData { @@ -220,6 +236,7 @@ mod tests { output_format: None, stream: None, nvext: None, + extra_args: None, passthrough: serde_json::Map::new(), }; let json = serde_json::to_string(&req).unwrap(); @@ -261,6 +278,37 @@ mod tests { assert_eq!(out, serde_json::json!({"prompt":"cat","model":"wan"})); } + #[test] + fn video_request_nests_passthrough_for_workers() { + let json = r#"{"prompt":"cat","model":"cosmos","generate_sound":true}"#; + let mut req: NvCreateVideoRequest = serde_json::from_str(json).unwrap(); + req.nest_passthrough(); + assert!(req.passthrough.is_empty()); + let out = serde_json::to_value(&req).unwrap(); + assert_eq!( + out["extra_args"]["media_passthrough"]["generate_sound"], + serde_json::json!(true) + ); + assert!(out.get("generate_sound").is_none()); + } + + #[test] + fn video_request_nest_without_unknowns_adds_nothing() { + let json = r#"{"prompt":"cat","model":"wan"}"#; + let mut req: NvCreateVideoRequest = serde_json::from_str(json).unwrap(); + req.nest_passthrough(); + let out = serde_json::to_value(&req).unwrap(); + assert!(out.get("extra_args").is_none()); + } + + #[test] + fn video_request_client_extra_args_is_not_the_worker_field() { + let json = r#"{"prompt":"cat","model":"wan","extra_args":{"x":1}}"#; + let req: NvCreateVideoRequest = serde_json::from_str(json).unwrap(); + assert!(req.extra_args.is_none()); + assert_eq!(req.passthrough["extra_args"]["x"], serde_json::json!(1)); + } + #[test] fn video_request_output_format_optional_absent_is_none() { let json = r#"{"prompt":"cat","model":"wan"}"#; From b9dabff39643143a1238126720d9ee87ea8c5ab6 Mon Sep 17 00:00:00 2001 From: Atindra Jha Date: Tue, 1 Sep 2026 22:36:02 +0000 Subject: [PATCH 4/5] fix(omni): reject load-path and policy knobs in media passthrough Signed-off-by: Atindra Jha --- .../src/dynamo/common/protocols/__init__.py | 73 +++++++++++++++++ .../common/tests/test_media_passthrough.py | 80 +++++++++++++++++++ .../src/dynamo/vllm/omni/audio_handler.py | 9 +-- .../src/dynamo/vllm/omni/omni_handler.py | 28 +++---- .../vllm/tests/omni/test_omni_handler.py | 35 ++++++-- 5 files changed, 199 insertions(+), 26 deletions(-) create mode 100644 components/src/dynamo/common/tests/test_media_passthrough.py diff --git a/components/src/dynamo/common/protocols/__init__.py b/components/src/dynamo/common/protocols/__init__.py index d4134e66c552..c4bbf793b64d 100644 --- a/components/src/dynamo/common/protocols/__init__.py +++ b/components/src/dynamo/common/protocols/__init__.py @@ -7,6 +7,8 @@ - video_protocol: NvCreateVideoRequest, NvVideosResponse for video generation """ +from typing import Any, Dict, Optional + from dynamo.common.protocols.video_protocol import ( NvCreateVideoRequest, NvVideosResponse, @@ -17,8 +19,79 @@ """Key under a media request's ``extra_args`` where the frontend nests unknown top-level request fields (an OpenAI client's ``extra_body``).""" + +class MediaPassthroughRejected(ValueError): + """A request tried to pass a knob the deployment owns, not the caller.""" + + +# Keys a caller must never set through the passthrough bucket. These name a +# thing to load or a safety/policy control, and the bucket is fully caller +# controlled, so honoring them would let a request point the engine at code +# to load or turn a guardrail off. +_RESERVED_PASSTHROUGH_KEYS = frozenset( + { + "extra_args", + "guardrails", + "frame_interpolation_model_path", + } +) + +# A knob whose name contains one of these is treated the same way: the first +# group names a fetch or load location, the second a safety/policy switch. +_UNSAFE_KEY_MARKERS = ( + "path", + "checkpoint", + "ckpt", + "repo", + "hub", + "url", + "uri", + "local_dir", + "guardrail", + "safety", + "nsfw", + "moderation", +) + + +def _passthrough_key_rejected(key: str) -> bool: + if key in _RESERVED_PASSTHROUGH_KEYS or key.startswith("_"): + return True + lowered = key.lower() + return any(marker in lowered for marker in _UNSAFE_KEY_MARKERS) + + +def sanitize_media_passthrough( + extra_args: Optional[Dict[str, Any]], +) -> Dict[str, Any]: + """Return the passthrough knobs that are safe to hand a backend. + + The frontend nests a request's unknown top-level fields (an OpenAI + client's ``extra_body``) under ``extra_args[MEDIA_PASSTHROUGH_KEY]``. + Those are untyped and fully caller controlled, so a knob that names a + model or checkpoint path, a fetch location, or a safety or policy + control is refused here rather than passed to the engine, where it could + reach a load path or a guardrail switch. Everything else is returned + unchanged for the caller to apply. + + Raises MediaPassthroughRejected (a ValueError) when a rejected knob is + present, so request building fails before the engine runs. + """ + knobs = (extra_args or {}).get(MEDIA_PASSTHROUGH_KEY) or {} + rejected = sorted(k for k in knobs if _passthrough_key_rejected(k)) + if rejected: + raise MediaPassthroughRejected( + "these request fields cannot be set per request (a model or " + "checkpoint path, a fetch location, or a safety or policy " + "control is deployment configuration): " + ", ".join(rejected) + ) + return dict(knobs) + + __all__ = [ "MEDIA_PASSTHROUGH_KEY", + "MediaPassthroughRejected", + "sanitize_media_passthrough", "NvCreateVideoRequest", "NvVideosResponse", "VideoData", diff --git a/components/src/dynamo/common/tests/test_media_passthrough.py b/components/src/dynamo/common/tests/test_media_passthrough.py new file mode 100644 index 000000000000..5344e9b99d87 --- /dev/null +++ b/components/src/dynamo/common/tests/test_media_passthrough.py @@ -0,0 +1,80 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for the media passthrough sanitizer. + +These run without any engine or model dependency: the point of the +sanitizer is to refuse a dangerous request field before anything is loaded, +so the test proves the rejection with no network and no model activity. +""" + +import pytest +from dynamo.common.protocols import ( + MEDIA_PASSTHROUGH_KEY, + MediaPassthroughRejected, + sanitize_media_passthrough, +) + +pytestmark = [ + pytest.mark.unit, + pytest.mark.gpu_0, + pytest.mark.pre_merge, +] + + +def _wrap(knobs): + return {MEDIA_PASSTHROUGH_KEY: knobs} + + +def test_none_and_empty_are_empty(): + assert sanitize_media_passthrough(None) == {} + assert sanitize_media_passthrough({}) == {} + assert sanitize_media_passthrough(_wrap({})) == {} + + +def test_benign_knobs_pass_through(): + knobs = {"backend_custom_knob": 0.5, "denoise_strength": 0.8, "generate_sound": True} + assert sanitize_media_passthrough(_wrap(knobs)) == knobs + + +@pytest.mark.parametrize( + "key", + [ + "frame_interpolation_model_path", # the reported RCE field + "guardrails", # the reported guardrail bypass + "extra_args", # the container itself + "vae_checkpoint", + "unet_ckpt", + "lora_repo", + "hf_hub_id", + "weights_url", + "source_uri", + "custom_local_dir", + "safety_checker", + "nsfw_filter", + "content_moderation", + "_private_attr", + ], +) +def test_reserved_and_shaped_keys_are_rejected(key): + with pytest.raises(MediaPassthroughRejected): + sanitize_media_passthrough(_wrap({key: "x", "harmless": 1})) + + +def test_rejection_names_the_offending_key(): + with pytest.raises(MediaPassthroughRejected, match="frame_interpolation_model_path"): + sanitize_media_passthrough( + _wrap({"frame_interpolation_model_path": "attacker/repo"}) + ) + + +def test_rejected_is_a_value_error(): + # Request building catches ValueError and returns a client error, so the + # sanitizer's exception must be a ValueError subclass. + assert issubclass(MediaPassthroughRejected, ValueError) + + +def test_original_bucket_is_not_mutated(): + bucket = _wrap({"backend_custom_knob": 1}) + sanitize_media_passthrough(bucket)["backend_custom_knob"] = 999 + assert bucket[MEDIA_PASSTHROUGH_KEY]["backend_custom_knob"] == 1 diff --git a/components/src/dynamo/vllm/omni/audio_handler.py b/components/src/dynamo/vllm/omni/audio_handler.py index d7ba15c21de4..4f77d86fdc08 100644 --- a/components/src/dynamo/vllm/omni/audio_handler.py +++ b/components/src/dynamo/vllm/omni/audio_handler.py @@ -21,7 +21,7 @@ except ImportError: Qwen3TTSPromptEmbedsBuilder = None # type: ignore[assignment, misc] -from dynamo.common.protocols import MEDIA_PASSTHROUGH_KEY +from dynamo.common.protocols import sanitize_media_passthrough from dynamo.common.protocols.audio_protocol import NvCreateAudioSpeechRequest from dynamo.common.utils.output_modalities import RequestType @@ -262,10 +262,9 @@ async def _engine_inputs_tts( tts_params["non_streaming_mode"] = [True] # Frontend-forwarded passthrough knobs join the engine params; - # fields the handler already set win. - for key, value in ( - (req.extra_args or {}).get(MEDIA_PASSTHROUGH_KEY) or {} - ).items(): + # fields the handler already set win. A knob naming a path or a + # policy control is refused here (see sanitize_media_passthrough). + for key, value in sanitize_media_passthrough(req.extra_args).items(): tts_params.setdefault(key, [value]) estimated_len = self._estimate_tts_prompt_len(tts_params) diff --git a/components/src/dynamo/vllm/omni/omni_handler.py b/components/src/dynamo/vllm/omni/omni_handler.py index 9f9913d4e9a5..7c16c86fc59a 100644 --- a/components/src/dynamo/vllm/omni/omni_handler.py +++ b/components/src/dynamo/vllm/omni/omni_handler.py @@ -26,7 +26,7 @@ from dynamo._core import Context from dynamo.common.multimodal import ImageLoader -from dynamo.common.protocols import MEDIA_PASSTHROUGH_KEY +from dynamo.common.protocols import sanitize_media_passthrough from dynamo.common.protocols.audio_protocol import NvCreateAudioSpeechRequest from dynamo.common.protocols.image_protocol import ImageNvExt, NvCreateImageRequest from dynamo.common.protocols.video_protocol import NvCreateVideoRequest, VideoNvExt @@ -69,33 +69,29 @@ def _apply_media_passthrough( sp: OmniDiffusionSamplingParams, extra_args: Optional[Dict[str, Any]] ) -> None: - """Apply frontend-forwarded passthrough knobs to diffusion sampling params. + """Hand frontend-forwarded passthrough knobs to the engine. The frontend nests a request's unknown top-level fields (an OpenAI client's ``extra_body``) under ``extra_args["media_passthrough"]``. - Knobs matching a sampling-params attribute are set directly; the rest - land in ``sp.extra_args`` for the engine to interpret. + ``sanitize_media_passthrough`` drops the request if any knob names a + path/checkpoint or a policy control, then the rest ride ``sp.extra_args`` + to the engine. Nothing is set on the sampling params by attribute name: + a caller-controlled key must not choose which attribute it writes. """ - knobs = (extra_args or {}).get(MEDIA_PASSTHROUGH_KEY) or {} - unmatched: Dict[str, Any] = {} - for key, value in knobs.items(): - if key != "extra_args" and hasattr(sp, key): - setattr(sp, key, value) - else: - unmatched[key] = value - if not unmatched: + knobs = sanitize_media_passthrough(extra_args) + if not knobs: return existing = getattr(sp, "extra_args", None) if isinstance(existing, dict): - existing.update(unmatched) + existing.update(knobs) else: try: - sp.extra_args = unmatched + sp.extra_args = knobs except (AttributeError, TypeError): logger.warning( "Dropping media passthrough knobs %s: sampling params expose " - "no matching attribute and no extra_args", - sorted(unmatched), + "no extra_args", + sorted(knobs), ) diff --git a/components/src/dynamo/vllm/tests/omni/test_omni_handler.py b/components/src/dynamo/vllm/tests/omni/test_omni_handler.py index 9d2e183a5da0..e4a864035a14 100644 --- a/components/src/dynamo/vllm/tests/omni/test_omni_handler.py +++ b/components/src/dynamo/vllm/tests/omni/test_omni_handler.py @@ -265,7 +265,8 @@ async def test_i2v_nvext_params_on_sampling_params(self): async def test_media_passthrough_reaches_sampling_params(self): """A top-level SDK extra_body field, nested by the frontend under - extra_args["media_passthrough"], reaches the engine input.""" + extra_args["media_passthrough"], rides sampling params extra_args to + the engine. Nothing is set on the sampling params by attribute name.""" handler = _make_handler() req = NvCreateVideoRequest( prompt="a cat by the sea", @@ -273,17 +274,41 @@ async def test_media_passthrough_reaches_sampling_params(self): size="832x480", extra_args={ "media_passthrough": { - "num_inference_steps": 12, "backend_custom_knob": 0.5, + "denoise_strength": 0.8, } }, ) result = await handler.build_engine_inputs(req, RequestType.VIDEO_GENERATION) sp = result.sampling_params_list[0] - # Knobs matching a sampling-params attribute apply directly. - assert sp.num_inference_steps == 12 - # The rest ride sampling params extra_args to the engine. assert sp.extra_args["backend_custom_knob"] == 0.5 + assert sp.extra_args["denoise_strength"] == 0.8 + + @pytest.mark.parametrize( + "bad_knob", + [ + {"frame_interpolation_model_path": "attacker/repository"}, + {"guardrails": False}, + {"safety_checker": None}, + {"vae_checkpoint_url": "http://evil/x.pkl"}, + ], + ) + async def test_media_passthrough_rejects_load_and_policy_knobs(self, bad_knob): + """A path/checkpoint field or a policy control is refused while the + request is being built, before any engine call, so it cannot reach a + model load or a guardrail switch.""" + handler = _make_handler() + handler.engine_client.generate = MagicMock( + side_effect=AssertionError("engine must not run for a rejected request") + ) + req = NvCreateVideoRequest( + prompt="a cat", + model="test-model", + size="832x480", + extra_args={"media_passthrough": bad_knob}, + ) + with pytest.raises(ValueError): + await handler.build_engine_inputs(req, RequestType.VIDEO_GENERATION) def test_media_passthrough_absent_is_a_no_op(self): req = NvCreateVideoRequest(prompt="a cat", model="test-model") From 399a66e34f2e5d2ec85db5ce237af045f13326ed Mon Sep 17 00:00:00 2001 From: Ryan McCormick Date: Thu, 3 Sep 2026 08:31:56 -0700 Subject: [PATCH 5/5] style(common): format media passthrough tests Signed-off-by: Ryan McCormick --- .../src/dynamo/common/tests/test_media_passthrough.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/components/src/dynamo/common/tests/test_media_passthrough.py b/components/src/dynamo/common/tests/test_media_passthrough.py index 5344e9b99d87..7de75ad259ec 100644 --- a/components/src/dynamo/common/tests/test_media_passthrough.py +++ b/components/src/dynamo/common/tests/test_media_passthrough.py @@ -9,6 +9,7 @@ """ import pytest + from dynamo.common.protocols import ( MEDIA_PASSTHROUGH_KEY, MediaPassthroughRejected, @@ -33,7 +34,11 @@ def test_none_and_empty_are_empty(): def test_benign_knobs_pass_through(): - knobs = {"backend_custom_knob": 0.5, "denoise_strength": 0.8, "generate_sound": True} + knobs = { + "backend_custom_knob": 0.5, + "denoise_strength": 0.8, + "generate_sound": True, + } assert sanitize_media_passthrough(_wrap(knobs)) == knobs @@ -62,7 +67,9 @@ def test_reserved_and_shaped_keys_are_rejected(key): def test_rejection_names_the_offending_key(): - with pytest.raises(MediaPassthroughRejected, match="frame_interpolation_model_path"): + with pytest.raises( + MediaPassthroughRejected, match="frame_interpolation_model_path" + ): sanitize_media_passthrough( _wrap({"frame_interpolation_model_path": "attacker/repo"}) )