From 6dd4dc823a74973b4a4196787239b055dacfe33e Mon Sep 17 00:00:00 2001 From: Chang Su Date: Fri, 27 Mar 2026 07:26:31 +0000 Subject: [PATCH] fix(multimodal): fix Phi-3-vision image placeholder injection and field layouts Phi-3-vision multimodal requests were failing for four reasons: 1. Wrong placeholder token: The Phi3VisionSpec used `` as the placeholder token, but the Phi-3-vision tokenizer uses `<|image|>` (token id 32044). This caused a 400 error on every multimodal request. 2. Missing placeholder injection: Phi-3-vision's chat template uses string content format, which strips image_url parts during message processing. No placeholder tokens were inserted into the text, so the tokenized prompt had zero image markers and expand_tokens found nothing to replace. This mirrors vLLM's approach of injecting model-specific placeholders during content formatting. The fix threads an optional image_placeholder through process_chat_messages/process_messages -> transform_content_field/ format_content_parts, resolved early via resolve_placeholder_token(). Multimodal context (model_id, tokenizer_source, components) is now computed once and reused for both placeholder resolution and process_multimodal, eliminating duplicate lookups. 3. Wrong field layout for image_sizes: The image_sizes tensor was treated as SharedField (default) but vLLM expects it as BatchedField, matching its _get_mm_fields_config. This caused a shape validation error when processing multiple images. 4. Fixed num_img_tokens: prompt_replacements used the fixed config value (144) instead of the actual per-image token count computed by the HD transform preprocessor, causing placeholder/embedding size mismatches. Signed-off-by: Chang Su --- bindings/golang/src/client.rs | 2 +- bindings/golang/src/policy.rs | 2 +- bindings/golang/src/preprocessor.rs | 2 +- crates/multimodal/src/registry/phi3_v.rs | 26 ++-- model_gateway/src/routers/grpc/multimodal.rs | 28 ++++ .../grpc/regular/stages/chat/preparation.rs | 130 ++++++++++------- .../regular/stages/messages/preparation.rs | 132 ++++++++++-------- .../src/routers/grpc/utils/chat_utils.rs | 61 +++++--- .../src/routers/grpc/utils/message_utils.rs | 53 ++++--- 9 files changed, 282 insertions(+), 154 deletions(-) diff --git a/bindings/golang/src/client.rs b/bindings/golang/src/client.rs index 96e16f552e..8e5918c69e 100644 --- a/bindings/golang/src/client.rs +++ b/bindings/golang/src/client.rs @@ -155,7 +155,7 @@ pub unsafe extern "C" fn sgl_client_chat_completion_stream( }; // Process messages and apply chat template - let processed_messages = match process_chat_messages(&chat_request, tokenizer.as_ref()) { + let processed_messages = match process_chat_messages(&chat_request, tokenizer.as_ref(), None) { Ok(msgs) => msgs, Err(e) => { set_error_message(error_out, &format!("Failed to process messages: {e}")); diff --git a/bindings/golang/src/policy.rs b/bindings/golang/src/policy.rs index 72a3ffdd7e..b0844ab432 100644 --- a/bindings/golang/src/policy.rs +++ b/bindings/golang/src/policy.rs @@ -499,7 +499,7 @@ pub unsafe extern "C" fn sgl_multi_client_chat_completion_stream( }; // Process messages and apply chat template - let processed_messages = match process_chat_messages(&chat_request, tokenizer.as_ref()) { + let processed_messages = match process_chat_messages(&chat_request, tokenizer.as_ref(), None) { Ok(msgs) => msgs, Err(e) => { set_error_message(error_out, &format!("Failed to process messages: {e}")); diff --git a/bindings/golang/src/preprocessor.rs b/bindings/golang/src/preprocessor.rs index a23c02773d..c562a6107e 100644 --- a/bindings/golang/src/preprocessor.rs +++ b/bindings/golang/src/preprocessor.rs @@ -37,7 +37,7 @@ fn preprocess_impl( tokenizer: &dyn Tokenizer, ) -> Result { // Process chat messages (apply chat_template) - let processed_messages = process_chat_messages(chat_request, tokenizer).map_err(|e| { + let processed_messages = process_chat_messages(chat_request, tokenizer, None).map_err(|e| { ( SglErrorCode::ParsingError, format!("Failed to process chat messages: {e}"), diff --git a/crates/multimodal/src/registry/phi3_v.rs b/crates/multimodal/src/registry/phi3_v.rs index 134784c77a..f5d95a9f3d 100644 --- a/crates/multimodal/src/registry/phi3_v.rs +++ b/crates/multimodal/src/registry/phi3_v.rs @@ -4,7 +4,7 @@ use serde_json::{json, Value}; use crate::{ registry::{ModelMetadata, ModelProcessorSpec, RegistryResult}, - types::{Modality, PromptReplacement, TokenId}, + types::{FieldLayout, Modality, PromptReplacement, TokenId}, vision::image_processor::PreprocessedImages, }; @@ -32,11 +32,11 @@ impl ModelProcessorSpec for Phi3VisionSpec { } fn placeholder_token(&self, _metadata: &ModelMetadata) -> RegistryResult { - Ok("".to_owned()) + Ok("<|image|>".to_owned()) } fn placeholder_token_id(&self, metadata: &ModelMetadata) -> RegistryResult { - metadata.token_id("") + metadata.token_id("<|image|>") } fn modality_limits( @@ -50,6 +50,13 @@ impl ModelProcessorSpec for Phi3VisionSpec { Ok(json!({})) } + fn field_layouts(&self) -> HashMap { + HashMap::from([ + ("pixel_values".to_string(), FieldLayout::Batched), + ("image_sizes".to_string(), FieldLayout::Batched), + ]) + } + fn prompt_replacements( &self, metadata: &ModelMetadata, @@ -57,11 +64,14 @@ impl ModelProcessorSpec for Phi3VisionSpec { ) -> RegistryResult> { let token_id = self.placeholder_token_id(metadata)?; let token = self.placeholder_token(metadata)?; - let count = Self::tokens_per_image(metadata); + let fallback = Self::tokens_per_image(metadata); Ok(preprocessed - .image_sizes + .num_img_tokens .iter() - .map(|_| PromptReplacement::repeated(Modality::Image, &token, token_id, count)) + .map(|&count| { + let n = if count > 0 { count } else { fallback }; + PromptReplacement::repeated(Modality::Image, &token, token_id, n) + }) .collect()) } } @@ -77,7 +87,7 @@ mod tests { #[test] fn phi3_uses_num_img_tokens() { - let tokenizer = TestTokenizer::new(&[("", 555)]); + let tokenizer = TestTokenizer::new(&[("<|image|>", 555)]); let config = json!({ "model_type": "phi3_v", "img_processor": {"num_img_tokens": 144} @@ -98,7 +108,7 @@ mod tests { #[test] fn phi3_matches_alias_via_model_type() { - let tokenizer = TestTokenizer::new(&[("", 555)]); + let tokenizer = TestTokenizer::new(&[("<|image|>", 555)]); let config = json!({ "model_type": "phi3_v", "img_processor": {"num_img_tokens": 144} diff --git a/model_gateway/src/routers/grpc/multimodal.rs b/model_gateway/src/routers/grpc/multimodal.rs index 4a6cb0220d..aad81fe4f1 100644 --- a/model_gateway/src/routers/grpc/multimodal.rs +++ b/model_gateway/src/routers/grpc/multimodal.rs @@ -156,6 +156,34 @@ pub(crate) struct MultimodalIntermediate { pub keep_on_cpu_keys: Vec, } +/// Resolve the placeholder token string for a multimodal model. +/// +/// Loads the model config and looks up the model spec to get the placeholder +/// token (e.g. `"<|image|>"` for Phi-3-vision). Returns `None` if the model +/// is not recognized as multimodal. +pub(crate) async fn resolve_placeholder_token( + model_id: &str, + tokenizer: &dyn TokenizerTrait, + components: &MultimodalComponents, + tokenizer_source: &str, +) -> Result> { + let model_config = components + .get_or_load_config(model_id, tokenizer_source) + .await?; + let metadata = ModelMetadata { + model_id, + tokenizer, + config: &model_config.config, + }; + let spec = match components.model_registry.lookup(&metadata) { + Some(s) => s, + None => return Ok(None), + }; + Ok(Some(spec.placeholder_token(&metadata).map_err(|e| { + anyhow::anyhow!("Failed to get placeholder token: {e}") + })?)) +} + /// Check if any messages in the request contain multimodal content (images). pub(crate) fn has_multimodal_content(messages: &[ChatMessage]) -> bool { messages.iter().any(|msg| { diff --git a/model_gateway/src/routers/grpc/regular/stages/chat/preparation.rs b/model_gateway/src/routers/grpc/regular/stages/chat/preparation.rs index 33238332e5..e082532044 100644 --- a/model_gateway/src/routers/grpc/regular/stages/chat/preparation.rs +++ b/model_gateway/src/routers/grpc/regular/stages/chat/preparation.rs @@ -48,32 +48,12 @@ impl ChatPreparationStage { // Step 1: Filter tools if needed let body_ref = utils::filter_chat_request_by_tool_choice(request); - // Step 2: Process messages and apply chat template - let processed_messages = match utils::process_chat_messages(&body_ref, &*tokenizer) { - Ok(msgs) => msgs, - Err(e) => { - error!(function = "ChatPreparationStage::execute", error = %e, "Failed to process chat messages"); - return Err(error::bad_request("process_messages_failed", e)); - } - }; - - // Step 3: Tokenize the processed text (no special tokens - chat template already handles them) - let encoding = match tokenizer.encode(&processed_messages.text, false) { - Ok(encoding) => encoding, - Err(e) => { - error!(function = "ChatPreparationStage::execute", error = %e, "Tokenization failed"); - return Err(error::internal_error( - "tokenization_failed", - format!("Tokenization failed: {e}"), - )); - } - }; - - let mut token_ids = encoding.token_ids().to_vec(); - - // Step 3.5: Full multimodal processing (fetch + preprocess + expand tokens + hash) - let mut multimodal_intermediate = None; - if multimodal::has_multimodal_content(&request.messages) { + // Resolve multimodal context once: placeholder token, model_id, tokenizer_source. + // The placeholder is passed to process_chat_messages so that string-format chat + // templates insert it per image instead of stripping image parts. The remaining + // fields are reused by process_multimodal to avoid duplicate lookups. + let is_multimodal = multimodal::has_multimodal_content(&request.messages); + let (image_placeholder, mm_context) = if is_multimodal { if let Some(mm_components) = ctx.components.multimodal.as_ref() { let model_id = ctx.input.model_id.as_str(); let tokenizer_source = ctx @@ -96,37 +76,20 @@ impl ChatPreparationStage { )); } - match multimodal::process_multimodal( - &request.messages, + let placeholder = multimodal::resolve_placeholder_token( model_id, &*tokenizer, - token_ids, mm_components, &tokenizer_source, ) .await - { - Ok(output) => { - debug!( - function = "ChatPreparationStage::execute", - expanded_tokens = output.expanded_token_ids.len(), - "Multimodal processing complete" - ); - token_ids = output.expanded_token_ids; - multimodal_intermediate = Some(output.intermediate); - } - Err(e) => { - error!( - function = "ChatPreparationStage::execute", - error = %e, - "Multimodal processing failed" - ); - return Err(error::bad_request( - "multimodal_processing_failed", - format!("Multimodal processing failed: {e}"), - )); - } - } + .ok() + .flatten(); + + ( + placeholder, + Some((mm_components, model_id, tokenizer_source)), + ) } else { error!( function = "ChatPreparationStage::execute", @@ -137,6 +100,71 @@ impl ChatPreparationStage { "Multimodal content detected but multimodal processing is not available", )); } + } else { + (None, None) + }; + + // Step 2: Process messages and apply chat template + let processed_messages = match utils::process_chat_messages( + &body_ref, + &*tokenizer, + image_placeholder.as_deref(), + ) { + Ok(msgs) => msgs, + Err(e) => { + error!(function = "ChatPreparationStage::execute", error = %e, "Failed to process chat messages"); + return Err(error::bad_request("process_messages_failed", e)); + } + }; + + // Step 3: Tokenize the processed text (no special tokens - chat template already handles them) + let encoding = match tokenizer.encode(&processed_messages.text, false) { + Ok(encoding) => encoding, + Err(e) => { + error!(function = "ChatPreparationStage::execute", error = %e, "Tokenization failed"); + return Err(error::internal_error( + "tokenization_failed", + format!("Tokenization failed: {e}"), + )); + } + }; + + let mut token_ids = encoding.token_ids().to_vec(); + + // Step 4: Full multimodal processing (fetch + preprocess + expand tokens + hash) + let mut multimodal_intermediate = None; + if let Some((mm_components, model_id, tokenizer_source)) = mm_context { + match multimodal::process_multimodal( + &request.messages, + model_id, + &*tokenizer, + token_ids, + mm_components, + &tokenizer_source, + ) + .await + { + Ok(output) => { + debug!( + function = "ChatPreparationStage::execute", + expanded_tokens = output.expanded_token_ids.len(), + "Multimodal processing complete" + ); + token_ids = output.expanded_token_ids; + multimodal_intermediate = Some(output.intermediate); + } + Err(e) => { + error!( + function = "ChatPreparationStage::execute", + error = %e, + "Multimodal processing failed" + ); + return Err(error::bad_request( + "multimodal_processing_failed", + format!("Multimodal processing failed: {e}"), + )); + } + } } // Step 4: Build tool constraints if needed diff --git a/model_gateway/src/routers/grpc/regular/stages/messages/preparation.rs b/model_gateway/src/routers/grpc/regular/stages/messages/preparation.rs index e54dcb1ee6..90f3c6531a 100644 --- a/model_gateway/src/routers/grpc/regular/stages/messages/preparation.rs +++ b/model_gateway/src/routers/grpc/regular/stages/messages/preparation.rs @@ -71,36 +71,9 @@ impl MessagePreparationStage { Some(filtered_tools.as_slice()) }; - // Step 2: Process messages and apply chat template - let processed_messages = match message_utils::process_messages( - request, - &*tokenizer, - tools_for_template, - ) { - Ok(msgs) => msgs, - Err(e) => { - error!(function = "MessagePreparationStage::execute", error = %e, "Failed to process messages"); - return Err(error::bad_request("process_messages_failed", e)); - } - }; - - // Step 3: Tokenize the processed text - let encoding = match tokenizer.encode(&processed_messages.text, false) { - Ok(encoding) => encoding, - Err(e) => { - error!(function = "MessagePreparationStage::execute", error = %e, "Tokenization failed"); - return Err(error::internal_error( - "tokenization_failed", - format!("Tokenization failed: {e}"), - )); - } - }; - - let mut token_ids = encoding.token_ids().to_vec(); - - // Step 3.5: Multimodal processing (fetch + preprocess + expand tokens + hash) - let mut multimodal_intermediate = None; - if multimodal::has_multimodal_content_messages(&request.messages) { + // Resolve multimodal context once (see chat/preparation.rs for details). + let is_multimodal = multimodal::has_multimodal_content_messages(&request.messages); + let (image_placeholder, mm_context) = if is_multimodal { if let Some(mm_components) = ctx.components.multimodal.as_ref() { let model_id = ctx.input.model_id.as_str(); let tokenizer_source = ctx @@ -123,37 +96,20 @@ impl MessagePreparationStage { )); } - match multimodal::process_multimodal_messages( - &request.messages, + let placeholder = multimodal::resolve_placeholder_token( model_id, &*tokenizer, - token_ids, mm_components, &tokenizer_source, ) .await - { - Ok(output) => { - debug!( - function = "MessagePreparationStage::execute", - expanded_tokens = output.expanded_token_ids.len(), - "Multimodal processing complete" - ); - token_ids = output.expanded_token_ids; - multimodal_intermediate = Some(output.intermediate); - } - Err(e) => { - error!( - function = "MessagePreparationStage::execute", - error = %e, - "Multimodal processing failed" - ); - return Err(error::bad_request( - "multimodal_processing_failed", - format!("Multimodal processing failed: {e}"), - )); - } - } + .ok() + .flatten(); + + ( + placeholder, + Some((mm_components, model_id, tokenizer_source)), + ) } else { error!( function = "MessagePreparationStage::execute", @@ -164,6 +120,72 @@ impl MessagePreparationStage { "Multimodal content detected but multimodal processing is not available", )); } + } else { + (None, None) + }; + + // Step 2: Process messages and apply chat template + let processed_messages = match message_utils::process_messages( + request, + &*tokenizer, + tools_for_template, + image_placeholder.as_deref(), + ) { + Ok(msgs) => msgs, + Err(e) => { + error!(function = "MessagePreparationStage::execute", error = %e, "Failed to process messages"); + return Err(error::bad_request("process_messages_failed", e)); + } + }; + + // Step 3: Tokenize the processed text + let encoding = match tokenizer.encode(&processed_messages.text, false) { + Ok(encoding) => encoding, + Err(e) => { + error!(function = "MessagePreparationStage::execute", error = %e, "Tokenization failed"); + return Err(error::internal_error( + "tokenization_failed", + format!("Tokenization failed: {e}"), + )); + } + }; + + let mut token_ids = encoding.token_ids().to_vec(); + + // Step 4: Multimodal processing (fetch + preprocess + expand tokens + hash) + let mut multimodal_intermediate = None; + if let Some((mm_components, model_id, tokenizer_source)) = mm_context { + match multimodal::process_multimodal_messages( + &request.messages, + model_id, + &*tokenizer, + token_ids, + mm_components, + &tokenizer_source, + ) + .await + { + Ok(output) => { + debug!( + function = "MessagePreparationStage::execute", + expanded_tokens = output.expanded_token_ids.len(), + "Multimodal processing complete" + ); + token_ids = output.expanded_token_ids; + multimodal_intermediate = Some(output.intermediate); + } + Err(e) => { + error!( + function = "MessagePreparationStage::execute", + error = %e, + "Multimodal processing failed" + ); + return Err(error::bad_request( + "multimodal_processing_failed", + format!("Multimodal processing failed: {e}"), + )); + } + } } // Step 4: Build tool constraints if tools present diff --git a/model_gateway/src/routers/grpc/utils/chat_utils.rs b/model_gateway/src/routers/grpc/utils/chat_utils.rs index 52f00b3bd1..5fcb95daf1 100644 --- a/model_gateway/src/routers/grpc/utils/chat_utils.rs +++ b/model_gateway/src/routers/grpc/utils/chat_utils.rs @@ -122,6 +122,7 @@ pub(crate) fn process_tool_call_arguments(messages: &mut [Value]) -> Result<(), pub(crate) fn process_content_format( messages: &[ChatMessage], content_format: ChatTemplateContentFormat, + image_placeholder: Option<&str>, ) -> Result, String> { messages .iter() @@ -131,7 +132,7 @@ pub(crate) fn process_content_format( if let Some(obj) = message_json.as_object_mut() { if let Some(content_value) = obj.get_mut("content") { - transform_content_field(content_value, content_format); + transform_content_field(content_value, content_format, image_placeholder); } } @@ -140,29 +141,40 @@ pub(crate) fn process_content_format( .collect() } -/// Transform a single content field based on content format -fn transform_content_field(content_value: &mut Value, content_format: ChatTemplateContentFormat) { +/// Transform a single content field based on content format. +/// +/// When `image_placeholder` is provided and the content format is `String`, +/// each `image_url` part is replaced with the placeholder string instead of +/// being stripped. This mirrors vLLM's behavior of injecting model-specific +/// placeholder tokens (e.g. `"<|image|>"`) so that the tokenizer produces +/// token IDs the multimodal expansion step can find and replace. +fn transform_content_field( + content_value: &mut Value, + content_format: ChatTemplateContentFormat, + image_placeholder: Option<&str>, +) { let Some(content_array) = content_value.as_array() else { return; // Not multimodal, keep as-is }; match content_format { ChatTemplateContentFormat::String => { - // Extract and join text parts only + // Extract text parts; optionally replace image parts with placeholders let text_parts: Vec = content_array .iter() .filter_map(|part| { - part.as_object()? - .get("type")? - .as_str() - .filter(|&t| t == "text") - .and_then(|_| part.as_object()?.get("text")?.as_str()) - .map(String::from) + let obj = part.as_object()?; + let type_str = obj.get("type")?.as_str()?; + match type_str { + "text" => obj.get("text")?.as_str().map(String::from), + "image_url" => image_placeholder.map(String::from), + _ => None, + } }) .collect(); if !text_parts.is_empty() { - *content_value = Value::String(text_parts.join(" ")); + *content_value = Value::String(text_parts.join("\n")); } } ChatTemplateContentFormat::OpenAI => { @@ -359,11 +371,13 @@ pub(crate) fn filter_chat_request_by_tool_choice( pub fn process_chat_messages( request: &ChatCompletionRequest, tokenizer: &dyn Tokenizer, + image_placeholder: Option<&str>, ) -> Result { let formatted_text = { // Get content format and transform messages accordingly let content_format = tokenizer.chat_template_content_format(); - let mut transformed_messages = process_content_format(&request.messages, content_format)?; + let mut transformed_messages = + process_content_format(&request.messages, content_format, image_placeholder)?; // Process tool call arguments in assistant messages process_tool_call_arguments(&mut transformed_messages)?; @@ -691,15 +705,16 @@ mod tests { name: None, }]; - let result = process_content_format(&messages, ChatTemplateContentFormat::String).unwrap(); + let result = + process_content_format(&messages, ChatTemplateContentFormat::String, None).unwrap(); assert_eq!(result.len(), 1); let transformed_message = &result[0]; - // Should flatten multimodal content to text only + // Should flatten multimodal content to text only (image stripped, no placeholder) assert_eq!( transformed_message["content"].as_str().unwrap(), - "Hello World" + "Hello\nWorld" ); assert_eq!(transformed_message["role"].as_str().unwrap(), "user"); } @@ -721,7 +736,8 @@ mod tests { name: None, }]; - let result = process_content_format(&messages, ChatTemplateContentFormat::OpenAI).unwrap(); + let result = + process_content_format(&messages, ChatTemplateContentFormat::OpenAI, None).unwrap(); assert_eq!(result.len(), 1); let transformed_message = &result[0]; @@ -745,7 +761,8 @@ mod tests { name: None, }]; - let result = process_content_format(&messages, ChatTemplateContentFormat::String).unwrap(); + let result = + process_content_format(&messages, ChatTemplateContentFormat::String, None).unwrap(); assert_eq!(result.len(), 1); let transformed_message = &result[0]; @@ -780,7 +797,8 @@ mod tests { }, ]; - let result = process_content_format(&messages, ChatTemplateContentFormat::String).unwrap(); + let result = + process_content_format(&messages, ChatTemplateContentFormat::String, None).unwrap(); assert_eq!(result.len(), 2); @@ -805,7 +823,8 @@ mod tests { name: None, }]; - let result = process_content_format(&messages, ChatTemplateContentFormat::String).unwrap(); + let result = + process_content_format(&messages, ChatTemplateContentFormat::String, None).unwrap(); assert_eq!(result.len(), 1); let transformed_message = &result[0]; @@ -838,14 +857,14 @@ mod tests { ]; let result_string = - process_content_format(&messages, ChatTemplateContentFormat::String).unwrap(); + process_content_format(&messages, ChatTemplateContentFormat::String, None).unwrap(); assert_eq!(result_string.len(), 2); assert_eq!(result_string[0]["content"].as_str().unwrap(), "Plain text"); assert_eq!(result_string[1]["content"].as_str().unwrap(), "With image"); let result_openai = - process_content_format(&messages, ChatTemplateContentFormat::OpenAI).unwrap(); + process_content_format(&messages, ChatTemplateContentFormat::OpenAI, None).unwrap(); assert_eq!(result_openai.len(), 2); assert_eq!(result_openai[0]["content"].as_str().unwrap(), "Plain text"); diff --git a/model_gateway/src/routers/grpc/utils/message_utils.rs b/model_gateway/src/routers/grpc/utils/message_utils.rs index dd4e65ba34..0a83f2de81 100644 --- a/model_gateway/src/routers/grpc/utils/message_utils.rs +++ b/model_gateway/src/routers/grpc/utils/message_utils.rs @@ -36,12 +36,13 @@ pub fn process_messages( request: &CreateMessageRequest, tokenizer: &dyn Tokenizer, chat_tools: Option<&[ChatTool]>, + image_placeholder: Option<&str>, ) -> Result { let content_format = tokenizer.chat_template_content_format(); // Step 1: Convert InputMessages to chat template JSON values let mut transformed_messages = - process_message_content_format(&request.messages, content_format)?; + process_message_content_format(&request.messages, content_format, image_placeholder)?; // Step 2: Prepend system message if present if let Some(system) = &request.system { @@ -135,11 +136,17 @@ pub fn process_messages( pub(crate) fn process_message_content_format( messages: &[InputMessage], content_format: ChatTemplateContentFormat, + image_placeholder: Option<&str>, ) -> Result, String> { messages.iter().try_fold(Vec::new(), |mut result, message| { match message.role { messages::Role::User => { - convert_user_message(&message.content, content_format, &mut result); + convert_user_message( + &message.content, + content_format, + image_placeholder, + &mut result, + ); } messages::Role::Assistant => { result.push(convert_assistant_message(&message.content, content_format)); @@ -157,6 +164,7 @@ pub(crate) fn process_message_content_format( fn convert_user_message( content: &InputContent, content_format: ChatTemplateContentFormat, + image_placeholder: Option<&str>, result: &mut Vec, ) { match content { @@ -191,7 +199,7 @@ fn convert_user_message( ); if !user_parts.is_empty() { - let content = format_content_parts(user_parts, content_format); + let content = format_content_parts(user_parts, content_format, image_placeholder); result.push(json!({"role": "user", "content": content})); } result.extend(tool_msgs); @@ -276,20 +284,27 @@ fn convert_assistant_message( /// /// - `String` format: join text parts into a single string /// - `OpenAI` format: keep as array of typed parts -fn format_content_parts(parts: Vec, content_format: ChatTemplateContentFormat) -> Value { +fn format_content_parts( + parts: Vec, + content_format: ChatTemplateContentFormat, + image_placeholder: Option<&str>, +) -> Value { match content_format { ChatTemplateContentFormat::String => { - // Extract text parts and join + // Extract text parts; optionally replace image parts with placeholders let text: String = parts .iter() .filter_map(|p| { - p.as_object() - .and_then(|obj| obj.get("type")?.as_str().filter(|&t| t == "text")) - .and_then(|_| p.as_object()?.get("text")?.as_str()) - .map(String::from) + let obj = p.as_object()?; + let type_str = obj.get("type")?.as_str()?; + match type_str { + "text" => obj.get("text")?.as_str().map(String::from), + "image" => image_placeholder.map(String::from), + _ => None, + } }) .collect::>() - .join(" "); + .join("\n"); Value::String(text) } ChatTemplateContentFormat::OpenAI => Value::Array(parts), @@ -418,7 +433,8 @@ mod tests { }]; let result = - process_message_content_format(&messages, ChatTemplateContentFormat::String).unwrap(); + process_message_content_format(&messages, ChatTemplateContentFormat::String, None) + .unwrap(); assert_eq!(result.len(), 1); assert_eq!(result[0]["role"], "user"); assert_eq!(result[0]["content"], "Hello"); @@ -436,7 +452,8 @@ mod tests { }]; let result = - process_message_content_format(&messages, ChatTemplateContentFormat::String).unwrap(); + process_message_content_format(&messages, ChatTemplateContentFormat::String, None) + .unwrap(); assert_eq!(result.len(), 1); assert_eq!(result[0]["role"], "assistant"); assert_eq!(result[0]["content"], "Hi there"); @@ -462,7 +479,8 @@ mod tests { }]; let result = - process_message_content_format(&messages, ChatTemplateContentFormat::String).unwrap(); + process_message_content_format(&messages, ChatTemplateContentFormat::String, None) + .unwrap(); assert_eq!(result.len(), 1); assert_eq!(result[0]["role"], "assistant"); assert_eq!(result[0]["content"], "Let me check."); @@ -486,7 +504,8 @@ mod tests { }]; let result = - process_message_content_format(&messages, ChatTemplateContentFormat::String).unwrap(); + process_message_content_format(&messages, ChatTemplateContentFormat::String, None) + .unwrap(); // Tool result becomes a "tool" role message, not a "user" message assert_eq!(result.len(), 1); assert_eq!(result[0]["role"], "tool"); @@ -513,7 +532,8 @@ mod tests { }]; let result = - process_message_content_format(&messages, ChatTemplateContentFormat::String).unwrap(); + process_message_content_format(&messages, ChatTemplateContentFormat::String, None) + .unwrap(); assert_eq!(result.len(), 1); assert_eq!(result[0]["role"], "assistant"); assert_eq!(result[0]["content"], "The answer is 42."); @@ -540,7 +560,8 @@ mod tests { }]; let result = - process_message_content_format(&messages, ChatTemplateContentFormat::String).unwrap(); + process_message_content_format(&messages, ChatTemplateContentFormat::String, None) + .unwrap(); assert_eq!( result[0]["reasoning_content"], "First thought.\nSecond thought."