diff --git a/crates/mcp/src/transform/transformer.rs b/crates/mcp/src/transform/transformer.rs index c9507db9c0..dc44c1d0f3 100644 --- a/crates/mcp/src/transform/transformer.rs +++ b/crates/mcp/src/transform/transformer.rs @@ -54,6 +54,24 @@ pub fn extract_embedded_openai_responses(result: &serde_json::Value) -> Vec` so they +/// serialize as absent (not `null`) when the MCP server did not surface them. +#[derive(Default)] +struct ImageGenerationFields { + image_b64: Option, + revised_prompt: Option, + action: Option, + background: Option, + output_format: Option, + quality: Option, + size: Option, +} + /// Transforms MCP CallToolResult to OpenAI Responses API output items. pub struct ResponseTransformer; @@ -245,6 +263,9 @@ impl ResponseTransformer { /// Extracts fields from the MCP `CallToolResult`: /// - `result` / `image_base64` / `b64_json`: base64-encoded image bytes /// - `revised_prompt`: optional rewritten prompt preserved for replay + /// - `action` / `background` / `output_format` / `quality` / `size`: + /// OpenAI-side output metadata. Forwarded verbatim when the MCP server + /// surfaces them so cloud passthrough does not drop them. /// /// Also probes embedded text blocks carrying `{"openai_response": {...}}` /// payloads for the same fields, mirroring the pattern used by @@ -253,27 +274,31 @@ impl ResponseTransformer { result: &serde_json::Value, tool_call_id: &str, ) -> ResponseOutputItem { - let (image_b64, revised_prompt) = Self::extract_image_generation_fields(result); + let fields = Self::extract_image_generation_fields(result); ResponseOutputItem::ImageGenerationCall { id: format!("ig_{tool_call_id}"), + action: fields.action, + background: fields.background, + output_format: fields.output_format, + quality: fields.quality, // `result` is non-optional in the spec; fall back to an empty // string when the MCP server returns no image data so the shape // still matches `ResponseOutputItem::ImageGenerationCall`. Router // wiring is responsible for surfacing an error status when // `image_b64` is missing. - result: image_b64.unwrap_or_default(), - revised_prompt, + result: fields.image_b64.unwrap_or_default(), + revised_prompt: fields.revised_prompt, + size: fields.size, status: ImageGenerationCallStatus::Completed, } } - /// Extract `(image_base64, revised_prompt)` from an MCP result payload. - fn extract_image_generation_fields( - result: &serde_json::Value, - ) -> (Option, Option) { - let mut image_b64: Option = None; - let mut revised_prompt: Option = None; + /// Extract the image-generation fields SMG surfaces from an MCP result + /// payload. Returns a struct (not a tuple) so adding/removing fields + /// does not ripple through call sites. + fn extract_image_generation_fields(result: &serde_json::Value) -> ImageGenerationFields { + let mut fields = ImageGenerationFields::default(); // Merge helper: fills any still-unset output slot from an object. // Using mutable accumulation (instead of early-returning on first @@ -281,20 +306,48 @@ impl ResponseTransformer { // across multiple text blocks — e.g. `revised_prompt` in one and // `image_base64` in another. First occurrence wins for each slot. let mut update_fields = |obj: &serde_json::Map| { - if image_b64.is_none() { - image_b64 = obj + if fields.image_b64.is_none() { + fields.image_b64 = obj .get("result") .and_then(|v| v.as_str()) .or_else(|| obj.get("image_base64").and_then(|v| v.as_str())) .or_else(|| obj.get("b64_json").and_then(|v| v.as_str())) .map(String::from); } - if revised_prompt.is_none() { - revised_prompt = obj + if fields.revised_prompt.is_none() { + fields.revised_prompt = obj .get("revised_prompt") .and_then(|v| v.as_str()) .map(String::from); } + // Metadata passthrough: the five OpenAI-documented + // `image_generation_call` output-item metadata fields. Each is + // optional and independently extracted so an MCP server that + // surfaces only some of them (e.g. just `size`) is preserved. + if fields.action.is_none() { + fields.action = obj.get("action").and_then(|v| v.as_str()).map(String::from); + } + if fields.background.is_none() { + fields.background = obj + .get("background") + .and_then(|v| v.as_str()) + .map(String::from); + } + if fields.output_format.is_none() { + fields.output_format = obj + .get("output_format") + .and_then(|v| v.as_str()) + .map(String::from); + } + if fields.quality.is_none() { + fields.quality = obj + .get("quality") + .and_then(|v| v.as_str()) + .map(String::from); + } + if fields.size.is_none() { + fields.size = obj.get("size").and_then(|v| v.as_str()).map(String::from); + } }; // 1) Direct object fields: { result | image_base64 | b64_json, revised_prompt } @@ -342,7 +395,7 @@ impl ResponseTransformer { } } - (image_b64, revised_prompt) + fields } /// Extract web sources from MCP result. @@ -1029,6 +1082,7 @@ mod tests { result, revised_prompt, status, + .. } => { assert_eq!(id, "ig_req-img-1"); assert_eq!(result, "BASE64_IMAGE_BYTES"); @@ -1245,6 +1299,7 @@ mod tests { result, revised_prompt, status, + .. } => { assert_eq!(id, "ig_req-img-4"); assert!(result.is_empty()); @@ -1255,12 +1310,160 @@ mod tests { } } + /// When the MCP tool result surfaces the OpenAI-side metadata + /// (`action`, `background`, `output_format`, `quality`, `size`) on the + /// direct object, the transformer forwards them verbatim onto the + /// `image_generation_call` output item so cloud-passthrough fidelity is + /// preserved and integration assertions can read them. + #[test] + fn test_image_generation_transform_forwards_metadata_direct_object() { + let result = json!({ + "result": "BASE64_IMAGE_BYTES", + "revised_prompt": "a serene mountain at sunrise", + "action": "generate", + "background": "opaque", + "output_format": "png", + "quality": "high", + "size": "1024x1024", + }); + + let transformed = ResponseTransformer::transform( + &result, + &ResponseFormat::ImageGenerationCall, + "req-img-meta", + "server", + "image_generation", + "{}", + ); + + match transformed { + ResponseOutputItem::ImageGenerationCall { + id, + action, + background, + output_format, + quality, + result, + revised_prompt, + size, + status, + } => { + assert_eq!(id, "ig_req-img-meta"); + assert_eq!(action.as_deref(), Some("generate")); + assert_eq!(background.as_deref(), Some("opaque")); + assert_eq!(output_format.as_deref(), Some("png")); + assert_eq!(quality.as_deref(), Some("high")); + assert_eq!(result, "BASE64_IMAGE_BYTES"); + assert_eq!( + revised_prompt.as_deref(), + Some("a serene mountain at sunrise") + ); + assert_eq!(size.as_deref(), Some("1024x1024")); + assert_eq!(status, ImageGenerationCallStatus::Completed); + } + _ => panic!("Expected ImageGenerationCall"), + } + } + + /// Metadata forwarding also works from a top-level text-block payload + /// (the shape produced by FastMCP-style servers that return a plain + /// `dict`). The extractor must pick metadata out of that payload the + /// same way it already reads `result`/`revised_prompt`. + #[test] + fn test_image_generation_transform_forwards_metadata_from_text_block() { + let result = json!([ + { + "type": "text", + "text": r#"{ + "result": "TOP_LEVEL_BYTES", + "revised_prompt": "a happy cat", + "action": "generate", + "background": "transparent", + "output_format": "webp", + "quality": "medium", + "size": "1536x1024", + "status": "completed" + }"# + } + ]); + + let transformed = ResponseTransformer::transform( + &result, + &ResponseFormat::ImageGenerationCall, + "req-img-meta-textblock", + "server", + "image_generation", + "{}", + ); + + match transformed { + ResponseOutputItem::ImageGenerationCall { + action, + background, + output_format, + quality, + size, + .. + } => { + assert_eq!(action.as_deref(), Some("generate")); + assert_eq!(background.as_deref(), Some("transparent")); + assert_eq!(output_format.as_deref(), Some("webp")); + assert_eq!(quality.as_deref(), Some("medium")); + assert_eq!(size.as_deref(), Some("1536x1024")); + } + _ => panic!("Expected ImageGenerationCall"), + } + } + + /// An MCP server that surfaces no metadata emits `None` for every + /// optional field — the transformer must not invent defaults. Guards + /// against an accidental `unwrap_or("")` that would pin an empty string + /// on the wire and round-trip as `""` instead of being skipped entirely. + #[test] + fn test_image_generation_transform_metadata_absent_when_not_surfaced() { + let result = json!({ + "result": "BASE64_IMAGE_BYTES", + }); + + let transformed = ResponseTransformer::transform( + &result, + &ResponseFormat::ImageGenerationCall, + "req-img-nometa", + "server", + "image_generation", + "{}", + ); + + match transformed { + ResponseOutputItem::ImageGenerationCall { + action, + background, + output_format, + quality, + size, + .. + } => { + assert!(action.is_none()); + assert!(background.is_none()); + assert!(output_format.is_none()); + assert!(quality.is_none()); + assert!(size.is_none()); + } + _ => panic!("Expected ImageGenerationCall"), + } + } + #[test] fn test_compact_image_generation_output_strips_base64() { let mut item = ResponseOutputItem::ImageGenerationCall { id: "ig_abc".to_string(), + action: None, + background: None, + output_format: None, + quality: None, result: "A_VERY_LONG_BASE64_STRING".to_string(), revised_prompt: Some("mountain".to_string()), + size: None, status: ImageGenerationCallStatus::Completed, }; @@ -1272,6 +1475,7 @@ mod tests { result, revised_prompt, status, + .. } => { assert_eq!(id, "ig_abc"); assert!(result.is_empty()); diff --git a/crates/protocols/src/responses.rs b/crates/protocols/src/responses.rs index 4afd8d1937..7993035141 100644 --- a/crates/protocols/src/responses.rs +++ b/crates/protocols/src/responses.rs @@ -1554,7 +1554,8 @@ pub enum ResponseInputOutputItem { /// flow): clients may resubmit only `{ type, id }` to reference a prior /// generation by identifier, so `result` and `status` are accepted as /// absent on the input side. The full shape is - /// `{ id, result?: base64 string, revised_prompt?, status?, type }`. + /// `{ id, action?, background?, output_format?, quality?, result?: base64, + /// revised_prompt?, size?, status?, type }`. /// /// This mirrors the OpenAI Python SDK 2.8.x /// `response_input_item_param.ImageGenerationCall` TypedDict: while the @@ -1563,11 +1564,35 @@ pub enum ResponseInputOutputItem { /// image-generation tool guide), and `skip_serializing_if` keeps the /// serialized form spec-compatible when a full item is round-tripped. /// The server-side `ResponseOutputItem::ImageGenerationCall` variant - /// remains strict because the gateway always populates those fields - /// on emit. + /// carries the same metadata so real OpenAI responses + /// (`action`/`background`/`output_format`/`quality`/`size`) survive + /// cloud-passthrough and persistence round-trips. + /// + /// The metadata fields (`action`, `background`, `output_format`, + /// `quality`, `size`) are typed as `Option` rather than + /// narrow enums so unknown or future-added values pass through + /// unchanged — this mirrors `ImageGenerationTool` on the input-tool + /// side. #[serde(rename = "image_generation_call")] ImageGenerationCall { id: String, + /// `"generate" | "edit" | "auto"` — which image-generation action the + /// prior turn dispatched. Preserved free-form so future actions pass + /// through without a wire break. + #[serde(default, skip_serializing_if = "Option::is_none")] + action: Option, + /// `"transparent" | "opaque" | "auto"`. Matches the + /// `image_generation` tool input knob of the same name. + #[serde(default, skip_serializing_if = "Option::is_none")] + background: Option, + /// `"png" | "webp" | "jpeg"`. Matches the `image_generation` tool + /// input knob of the same name. + #[serde(default, skip_serializing_if = "Option::is_none")] + output_format: Option, + /// `"auto" | "low" | "medium" | "high" | "standard" | "hd"`. Matches + /// the `image_generation` tool input knob of the same name. + #[serde(default, skip_serializing_if = "Option::is_none")] + quality: Option, /// Base64-encoded image bytes. Omitted on id-only references. #[serde(default, skip_serializing_if = "Option::is_none")] result: Option, @@ -1576,6 +1601,10 @@ pub enum ResponseInputOutputItem { /// not drop it on replay. #[serde(default, skip_serializing_if = "Option::is_none")] revised_prompt: Option, + /// `"auto" | "1024x1024" | "1024x1536" | "1536x1024"`. Matches the + /// `image_generation` tool input knob of the same name. + #[serde(default, skip_serializing_if = "Option::is_none")] + size: Option, /// Generation status. Omitted on id-only references. #[serde(default, skip_serializing_if = "Option::is_none")] status: Option, @@ -2064,10 +2093,39 @@ pub enum ResponseOutputItem { }, /// `type: "image_generation_call"` — output item carrying a base64 image /// produced by the `image_generation` built-in tool. Spec: - /// `{ id, result: base64 string, revised_prompt?, status, type }`. + /// `{ id, action?, background?, output_format?, quality?, result: base64, + /// revised_prompt?, size?, status, type }`. + /// + /// Real OpenAI production responses include the five metadata fields + /// (`action`, `background`, `output_format`, `quality`, `size`) even + /// though the OpenAI Rust SDK v2.8.1 omits them. We carry them as + /// `Option` so cloud passthrough and persistence round-trips + /// preserve them verbatim — and so downstream consumers can read them + /// without a second round-trip to the provider. + /// + /// The metadata fields are typed as `Option` rather than narrow + /// enums so unknown or future-added values pass through unchanged; + /// this mirrors `ImageGenerationTool` on the input-tool side. #[serde(rename = "image_generation_call")] ImageGenerationCall { id: String, + /// `"generate" | "edit" | "auto"` — which image-generation action + /// this call dispatched. Preserved free-form so future actions pass + /// through without a wire break. + #[serde(default, skip_serializing_if = "Option::is_none")] + action: Option, + /// `"transparent" | "opaque" | "auto"`. Mirrors the + /// `image_generation` tool input knob of the same name. + #[serde(default, skip_serializing_if = "Option::is_none")] + background: Option, + /// `"png" | "webp" | "jpeg"`. Mirrors the `image_generation` tool + /// input knob of the same name. + #[serde(default, skip_serializing_if = "Option::is_none")] + output_format: Option, + /// `"auto" | "low" | "medium" | "high" | "standard" | "hd"`. Mirrors + /// the `image_generation` tool input knob of the same name. + #[serde(default, skip_serializing_if = "Option::is_none")] + quality: Option, /// Base64-encoded image bytes. result: String, /// Prompt text the mainline model rewrote before dispatching the @@ -2075,6 +2133,10 @@ pub enum ResponseOutputItem { /// not drop it on replay. #[serde(default, skip_serializing_if = "Option::is_none")] revised_prompt: Option, + /// `"auto" | "1024x1024" | "1024x1536" | "1536x1024"`. Mirrors the + /// `image_generation` tool input knob of the same name. + #[serde(default, skip_serializing_if = "Option::is_none")] + size: Option, status: ImageGenerationCallStatus, }, /// `type: "compaction"` — server-emitted item carrying an opaque diff --git a/crates/protocols/tests/responses.rs b/crates/protocols/tests/responses.rs index 5653ec77bf..da621f2d2b 100644 --- a/crates/protocols/tests/responses.rs +++ b/crates/protocols/tests/responses.rs @@ -1117,6 +1117,7 @@ fn image_generation_call_output_item_round_trips_spec_shape() { result, revised_prompt, status, + .. } => { assert_eq!(id, "ig_1"); assert_eq!(result, "aGVsbG8="); @@ -1151,6 +1152,7 @@ fn image_generation_call_output_item_round_trips_with_revised_prompt() { result, revised_prompt, status, + .. } => { assert_eq!(id, "ig_1"); assert_eq!(result, "aGVsbG8="); @@ -1184,6 +1186,7 @@ fn image_generation_call_input_item_round_trips_spec_shape() { result, revised_prompt, status, + .. } => { assert_eq!(id, "ig_2"); assert_eq!(result.as_deref(), Some("d29ybGQ=")); @@ -1215,6 +1218,7 @@ fn image_generation_call_input_item_accepts_id_only_reference() { result, revised_prompt, status, + .. } => { assert_eq!(id, "ig_3"); assert!(result.is_none()); @@ -1249,6 +1253,7 @@ fn image_generation_call_input_item_round_trips_with_revised_prompt() { result, revised_prompt, status, + .. } => { assert_eq!(id, "ig_2"); assert_eq!(result.as_deref(), Some("d29ybGQ=")); @@ -1263,6 +1268,191 @@ fn image_generation_call_input_item_round_trips_with_revised_prompt() { assert_eq!(serde_json::to_value(&item).expect("serialize"), payload); } +/// Real OpenAI production responses carry additional metadata on +/// `image_generation_call` output items (`action`, `background`, +/// `output_format`, `quality`, `size`) that the OpenAI Rust SDK v2.8.1 omits. +/// This roundtrip pins the full OpenAI-shaped output item and verifies every +/// metadata field is preserved through (de)serialization — silently dropping +/// any of these would break cloud-passthrough fidelity and integration +/// test assertions. +#[test] +fn image_generation_call_output_item_round_trips_with_full_metadata() { + let payload = json!({ + "type": "image_generation_call", + "id": "ig_prod_1", + "action": "generate", + "background": "opaque", + "output_format": "png", + "quality": "high", + "result": "aGVsbG8=", + "revised_prompt": "A red fox sitting on a mossy rock at golden hour.", + "size": "1024x1024", + "status": "completed", + }); + let item: ResponseOutputItem = serde_json::from_value(payload.clone()) + .expect("full-metadata image_generation_call output item deserialize"); + match &item { + ResponseOutputItem::ImageGenerationCall { + id, + action, + background, + output_format, + quality, + result, + revised_prompt, + size, + status, + } => { + assert_eq!(id, "ig_prod_1"); + assert_eq!(action.as_deref(), Some("generate")); + assert_eq!(background.as_deref(), Some("opaque")); + assert_eq!(output_format.as_deref(), Some("png")); + assert_eq!(quality.as_deref(), Some("high")); + assert_eq!(result, "aGVsbG8="); + assert_eq!( + revised_prompt.as_deref(), + Some("A red fox sitting on a mossy rock at golden hour.") + ); + assert_eq!(size.as_deref(), Some("1024x1024")); + assert_eq!(*status, ImageGenerationCallStatus::Completed); + } + other => panic!("expected ImageGenerationCall, got {other:?}"), + } + assert_eq!(serde_json::to_value(&item).expect("serialize"), payload); +} + +/// A minimal `image_generation_call` output item (only `id`, `result`, +/// `status`) must round-trip without introducing `null` entries for the +/// absent metadata fields — `skip_serializing_if = "Option::is_none"` keeps +/// the wire shape spec-minimal. Guards against an accidental +/// `Option::default()` that would materialize `"action": null` etc. on emit. +#[test] +fn image_generation_call_output_item_round_trips_minimal_without_metadata() { + let payload = json!({ + "type": "image_generation_call", + "id": "ig_min_1", + "result": "aGVsbG8=", + "status": "completed", + }); + let item: ResponseOutputItem = serde_json::from_value(payload.clone()) + .expect("minimal image_generation_call output item deserialize"); + match &item { + ResponseOutputItem::ImageGenerationCall { + id, + action, + background, + output_format, + quality, + result, + revised_prompt, + size, + status, + } => { + assert_eq!(id, "ig_min_1"); + assert!(action.is_none()); + assert!(background.is_none()); + assert!(output_format.is_none()); + assert!(quality.is_none()); + assert_eq!(result, "aGVsbG8="); + assert!(revised_prompt.is_none()); + assert!(size.is_none()); + assert_eq!(*status, ImageGenerationCallStatus::Completed); + } + other => panic!("expected ImageGenerationCall, got {other:?}"), + } + // Must be byte-for-byte equal — no `"action": null`, `"size": null` etc. + assert_eq!(serde_json::to_value(&item).expect("serialize"), payload); +} + +/// Symmetry with the output-side variant — the input-side +/// `ImageGenerationCall` also carries the five metadata fields so a client +/// resubmitting a prior-turn image generation item gets byte-identical +/// round-tripping. Without this, stateless multi-turn replay strips metadata +/// that the output item emitted. +#[test] +fn image_generation_call_input_item_round_trips_with_full_metadata() { + let payload = json!({ + "type": "image_generation_call", + "id": "ig_prod_2", + "action": "edit", + "background": "transparent", + "output_format": "webp", + "quality": "medium", + "result": "d29ybGQ=", + "revised_prompt": "A winter cabin with snow falling softly outside.", + "size": "1024x1536", + "status": "completed", + }); + let item: ResponseInputOutputItem = serde_json::from_value(payload.clone()) + .expect("full-metadata image_generation_call input item deserialize"); + match &item { + ResponseInputOutputItem::ImageGenerationCall { + id, + action, + background, + output_format, + quality, + result, + revised_prompt, + size, + status, + } => { + assert_eq!(id, "ig_prod_2"); + assert_eq!(action.as_deref(), Some("edit")); + assert_eq!(background.as_deref(), Some("transparent")); + assert_eq!(output_format.as_deref(), Some("webp")); + assert_eq!(quality.as_deref(), Some("medium")); + assert_eq!(result.as_deref(), Some("d29ybGQ=")); + assert_eq!( + revised_prompt.as_deref(), + Some("A winter cabin with snow falling softly outside.") + ); + assert_eq!(size.as_deref(), Some("1024x1536")); + assert_eq!(*status, Some(ImageGenerationCallStatus::Completed)); + } + other => panic!("expected ImageGenerationCall, got {other:?}"), + } + assert_eq!(serde_json::to_value(&item).expect("serialize"), payload); +} + +/// Minimal id-only input-side image_generation_call must keep serializing +/// without any `null` metadata fields — the documented multi-turn reference +/// form stays byte-identical after the metadata-field additions. +#[test] +fn image_generation_call_input_item_round_trips_minimal_without_metadata() { + let payload = json!({ + "type": "image_generation_call", + "id": "ig_min_2", + }); + let item: ResponseInputOutputItem = serde_json::from_value(payload.clone()) + .expect("minimal image_generation_call input item deserialize"); + match &item { + ResponseInputOutputItem::ImageGenerationCall { + id, + action, + background, + output_format, + quality, + result, + revised_prompt, + size, + status, + } => { + assert_eq!(id, "ig_min_2"); + assert!(action.is_none()); + assert!(background.is_none()); + assert!(output_format.is_none()); + assert!(quality.is_none()); + assert!(result.is_none()); + assert!(revised_prompt.is_none()); + assert!(size.is_none()); + assert!(status.is_none()); + } + other => panic!("expected ImageGenerationCall, got {other:?}"), + } + assert_eq!(serde_json::to_value(&item).expect("serialize"), payload); +} + #[test] fn compaction_output_item_round_trips_spec_shape() { let payload = json!({ diff --git a/model_gateway/src/routers/grpc/regular/responses/conversions.rs b/model_gateway/src/routers/grpc/regular/responses/conversions.rs index 41e9e1950e..72b32ee384 100644 --- a/model_gateway/src/routers/grpc/regular/responses/conversions.rs +++ b/model_gateway/src/routers/grpc/regular/responses/conversions.rs @@ -560,8 +560,13 @@ mod tests { let req = ResponsesRequest { input: ResponseInput::Items(vec![ResponseInputOutputItem::ImageGenerationCall { id: "ig_test".to_string(), + action: None, + background: None, + output_format: None, + quality: None, result: Some("base64data".to_string()), revised_prompt: Some("a cat".to_string()), + size: None, status: None, }]), ..Default::default()