Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion components/src/dynamo/common/protocols/audio_protocol.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

from typing import Literal, Optional

from pydantic import BaseModel, Field
from pydantic import BaseModel, ConfigDict, Field


class AudioNvExt(BaseModel):
Expand All @@ -34,6 +34,11 @@ class NvCreateAudioSpeechRequest(BaseModel):
Follows vLLM-Omni's OpenAICreateSpeechRequest format.
"""

model_config = ConfigDict(extra="allow")
"""Unknown top-level fields are backend passthrough knobs. Keep them
so handlers can read and forward them instead of dropping them at
the worker boundary."""

# Standard OpenAI params
input: str
"""The text to synthesize into speech."""
Expand Down
7 changes: 6 additions & 1 deletion components/src/dynamo/common/protocols/image_protocol.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

from typing import Optional

from pydantic import BaseModel
from pydantic import BaseModel, ConfigDict

# For omni models, we need to support raw_request parsing and json output format. We need to have these protocols defined here for serialization and deserialization.
# TODO: Replace these Pydantic models with Python bindings to the Rust protocol types once PyO3 bindings are available.
Expand Down Expand Up @@ -38,6 +38,11 @@ class NvCreateImageRequest(BaseModel):
Matches the flattened Rust NvCreateImageRequest in lib/llm/src/protocols/openai/images.rs
"""

model_config = ConfigDict(extra="allow")
"""Unknown top-level fields are backend passthrough knobs. Keep them
so handlers can read and forward them instead of dropping them at
the worker boundary."""

prompt: str
"""The text prompt for image generation."""

Expand Down
7 changes: 6 additions & 1 deletion components/src/dynamo/common/protocols/video_protocol.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

from typing import Literal, Optional

from pydantic import BaseModel
from pydantic import BaseModel, ConfigDict


class VideoNvExt(BaseModel):
Expand Down Expand Up @@ -53,6 +53,11 @@ class NvCreateVideoRequest(BaseModel):
Matches Rust NvCreateVideoRequest in lib/llm/src/protocols/openai/videos.rs.
"""

model_config = ConfigDict(extra="allow")
"""Unknown top-level fields are backend passthrough knobs. Keep them
so handlers can read and forward them instead of dropping them at
the worker boundary."""

# Required fields
prompt: str
"""The text prompt for video generation."""
Expand Down
34 changes: 34 additions & 0 deletions lib/llm/src/protocols/openai/audios.rs
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,13 @@ pub struct NvCreateAudioSpeechRequest {
/// NVIDIA extensions (reserved for future use)
#[serde(skip_serializing_if = "Option::is_none")]
pub nvext: Option<NvExt>,

/// Unknown top-level fields are retained here and forwarded to the
/// backend without strict validation. This matches the OpenAI client's
/// extra_body option, which merges into the top level of the body.
/// Stable knobs can be promoted to typed fields over time.
#[serde(default, flatten)]
pub passthrough: serde_json::Map<String, serde_json::Value>,
}

/// Audio data in response
Expand Down Expand Up @@ -246,11 +253,38 @@ mod tests {
max_new_tokens: None,
user: None,
nvext: None,
passthrough: serde_json::Map::new(),
};
let json = serde_json::to_string(&req).unwrap();
assert!(!json.contains("data_source"));
}

#[test]
fn audio_request_captures_unknown_top_level_fields() {
// The OpenAI client's extra_body option merges into the top level of
// the body, so that is where backend knobs arrive.
let json = r#"{"input":"hello","emotion":"calm","pitch":1.2}"#;
let req: NvCreateAudioSpeechRequest = serde_json::from_str(json).unwrap();
assert_eq!(req.passthrough["emotion"], serde_json::json!("calm"));
assert_eq!(req.passthrough["pitch"], serde_json::json!(1.2));
assert!(!req.passthrough.contains_key("input"));

let out = serde_json::to_string(&req).unwrap();
let back: NvCreateAudioSpeechRequest = serde_json::from_str(&out).unwrap();
assert_eq!(back.passthrough, req.passthrough);
assert!(out.contains("\"emotion\":\"calm\""));
}

#[test]
fn audio_request_empty_passthrough_adds_nothing() {
let json = r#"{"input":"hello"}"#;
let req: NvCreateAudioSpeechRequest = serde_json::from_str(json).unwrap();
assert!(req.passthrough.is_empty());
let out: serde_json::Value =
serde_json::from_str(&serde_json::to_string(&req).unwrap()).unwrap();
assert_eq!(out, serde_json::json!({"input":"hello"}));
}

// --- AudioData ---

#[test]
Expand Down
145 changes: 141 additions & 4 deletions lib/llm/src/protocols/openai/images.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,17 +11,93 @@ mod nvext;
pub use nvext::{NvExt, NvExtProvider};

/// Image generation request with NVIDIA extensions.
#[derive(Serialize, Deserialize, Validate, Debug, Clone)]
///
/// Serde is hand-rolled: `inner` occupies the top level of the wire body,
/// and a derived flattened map next to a flattened struct would capture the
/// struct's keys too. The manual impls keep one external contract, typed
/// fields at the top level plus unknown top-level fields retained in
/// [`Self::passthrough`].
#[derive(Validate, Debug, Clone)]
pub struct NvCreateImageRequest {
#[serde(flatten)]
pub inner: dynamo_protocols::types::CreateImageRequest,

/// Optional image reference that guides generation (for I2I/TI2I).
#[serde(skip_serializing_if = "Option::is_none")]
pub input_reference: Option<String>,

#[serde(skip_serializing_if = "Option::is_none")]
pub nvext: Option<NvExt>,

/// Unknown top-level fields are retained here and forwarded to the
/// backend without strict validation. This matches the OpenAI client's
/// extra_body option, which merges into the top level of the body.
/// Stable knobs can be promoted to typed fields over time.
pub passthrough: serde_json::Map<String, serde_json::Value>,
}

impl<'de> Deserialize<'de> for NvCreateImageRequest {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
let mut body = serde_json::Map::deserialize(deserializer)?;
let input_reference = match body.remove("input_reference") {
Some(value) => serde_json::from_value(value).map_err(serde::de::Error::custom)?,
None => None,
};
let nvext = match body.remove("nvext") {
Some(value) => serde_json::from_value(value).map_err(serde::de::Error::custom)?,
None => None,
};
let inner: dynamo_protocols::types::CreateImageRequest =
serde_json::from_value(serde_json::Value::Object(body.clone()))
.map_err(serde::de::Error::custom)?;
// Keys the typed request consumed stay out of the passthrough.
if let serde_json::Value::Object(consumed) =
serde_json::to_value(&inner).map_err(serde::de::Error::custom)?
{
for key in consumed.keys() {
body.remove(key);
}
}
Ok(Self {
inner,
input_reference,
nvext,
passthrough: body,
})
}
}

impl Serialize for NvCreateImageRequest {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
let mut body = match serde_json::to_value(&self.inner).map_err(serde::ser::Error::custom)? {
serde_json::Value::Object(map) => map,
_ => {
return Err(serde::ser::Error::custom(
"image request must serialize to an object",
))
}
};
// Typed fields win over passthrough entries of the same name.
for (key, value) in &self.passthrough {
body.entry(key.clone()).or_insert_with(|| value.clone());
}
if let Some(input_reference) = &self.input_reference {
body.insert(
"input_reference".to_string(),
serde_json::Value::String(input_reference.clone()),
);
}
if let Some(nvext) = &self.nvext {
body.insert(
"nvext".to_string(),
serde_json::to_value(nvext).map_err(serde::ser::Error::custom)?,
);
}
body.serialize(serializer)
}
}

/// A response structure for image generation responses, embedding OpenAI's
Expand Down Expand Up @@ -85,3 +161,64 @@ impl AnnotationsProvider for NvCreateImageRequest {
.unwrap_or(false)
}
}

#[cfg(test)]
mod tests {
use super::*;

// --- NvCreateImageRequest ---

#[test]
fn image_request_captures_unknown_top_level_fields() {
// The OpenAI client's extra_body option merges into the top level of
// the body, so that is where backend knobs arrive.
let json = r#"{"prompt":"a cat","think_mode":true,"size_override":{"h":512,"w":768}}"#;
let req: NvCreateImageRequest = serde_json::from_str(json).unwrap();
assert_eq!(req.inner.prompt, "a cat");
assert_eq!(req.passthrough["think_mode"], serde_json::json!(true));
assert_eq!(
req.passthrough["size_override"]["h"],
serde_json::json!(512)
);

let out = serde_json::to_string(&req).unwrap();
let back: NvCreateImageRequest = serde_json::from_str(&out).unwrap();
assert_eq!(back.inner.prompt, "a cat");
assert_eq!(back.passthrough, req.passthrough);
}

#[test]
fn image_request_typed_fields_stay_out_of_passthrough() {
let json = r#"{"prompt":"a cat","n":2,"input_reference":"ref.png","nvext":{"seed":7},"custom_knob":1}"#;
let req: NvCreateImageRequest = serde_json::from_str(json).unwrap();
assert_eq!(req.inner.prompt, "a cat");
assert_eq!(req.input_reference.as_deref(), Some("ref.png"));
assert_eq!(req.nvext.as_ref().and_then(|n| n.seed), Some(7));
assert_eq!(req.passthrough["custom_knob"], serde_json::json!(1));
for consumed in ["prompt", "n", "input_reference", "nvext"] {
assert!(!req.passthrough.contains_key(consumed), "{consumed}");
}
}

#[test]
fn image_request_round_trips_all_field_kinds_at_top_level() {
let json =
r#"{"prompt":"a cat","input_reference":"ref.png","nvext":{"seed":7},"knob":"x"}"#;
let req: NvCreateImageRequest = serde_json::from_str(json).unwrap();
let out: serde_json::Value =
serde_json::from_str(&serde_json::to_string(&req).unwrap()).unwrap();
assert_eq!(out["prompt"], serde_json::json!("a cat"));
assert_eq!(out["input_reference"], serde_json::json!("ref.png"));
assert_eq!(out["nvext"]["seed"], serde_json::json!(7));
assert_eq!(out["knob"], serde_json::json!("x"));
assert!(out.get("passthrough").is_none());
assert!(out.get("inner").is_none());
}

#[test]
fn image_request_empty_passthrough_stays_empty() {
let json = r#"{"prompt":"a cat"}"#;
let req: NvCreateImageRequest = serde_json::from_str(json).unwrap();
assert!(req.passthrough.is_empty());
}
}
43 changes: 43 additions & 0 deletions lib/llm/src/protocols/openai/videos.rs
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,13 @@ pub struct NvCreateVideoRequest {
/// NVIDIA extensions
#[serde(skip_serializing_if = "Option::is_none")]
pub nvext: Option<NvExt>,

/// Unknown top-level fields are retained here and forwarded to the
/// backend without strict validation. This matches the OpenAI client's
/// extra_body option, which merges into the top level of the body.
/// Stable knobs can be promoted to typed fields over time.
#[serde(default, flatten)]
pub passthrough: serde_json::Map<String, serde_json::Value>,
}

/// Video data in response
Expand Down Expand Up @@ -213,11 +220,47 @@ mod tests {
output_format: None,
stream: None,
nvext: None,
passthrough: serde_json::Map::new(),
};
let json = serde_json::to_string(&req).unwrap();
assert!(!json.contains("stream"));
}

#[test]
fn video_request_captures_unknown_top_level_fields() {
// The OpenAI client's extra_body option merges into the top level of
// the body, so that is where backend knobs arrive.
let json = r#"{"prompt":"cat","model":"cosmos","generate_sound":true,"strength":0.7}"#;
let req: NvCreateVideoRequest = serde_json::from_str(json).unwrap();
assert_eq!(req.passthrough["generate_sound"], serde_json::json!(true));
assert_eq!(req.passthrough["strength"], serde_json::json!(0.7));

let out = serde_json::to_string(&req).unwrap();
let back: NvCreateVideoRequest = serde_json::from_str(&out).unwrap();
assert_eq!(back.passthrough, req.passthrough);
assert!(out.contains("\"generate_sound\":true"));
}

#[test]
fn video_request_typed_fields_stay_out_of_passthrough() {
let json = r#"{"prompt":"cat","model":"wan","stream":true,"custom_knob":1}"#;
let req: NvCreateVideoRequest = serde_json::from_str(json).unwrap();
assert_eq!(req.stream, Some(true));
assert!(!req.passthrough.contains_key("prompt"));
assert!(!req.passthrough.contains_key("stream"));
assert_eq!(req.passthrough["custom_knob"], serde_json::json!(1));
}

#[test]
fn video_request_empty_passthrough_adds_nothing() {
let json = r#"{"prompt":"cat","model":"wan"}"#;
let req: NvCreateVideoRequest = serde_json::from_str(json).unwrap();
assert!(req.passthrough.is_empty());
let out: serde_json::Value =
serde_json::from_str(&serde_json::to_string(&req).unwrap()).unwrap();
assert_eq!(out, serde_json::json!({"prompt":"cat","model":"wan"}));
}

#[test]
fn video_request_output_format_optional_absent_is_none() {
let json = r#"{"prompt":"cat","model":"wan"}"#;
Expand Down
Loading