diff --git a/multimodal/Cargo.toml b/multimodal/Cargo.toml index c606f4048b..cb676dc4be 100644 --- a/multimodal/Cargo.toml +++ b/multimodal/Cargo.toml @@ -18,7 +18,7 @@ name = "llm_multimodal" [dependencies] async-trait.workspace = true base64 = "0.22" -bytes = "1.8.0" +bytes = { version = "1.8.0", features = ["serde"] } image = { version = "0.25.4", default-features = false, features = ["png", "jpeg", "gif", "bmp", "ico", "tiff", "webp"] } ndarray = "0.16" once_cell = "1.21.3" diff --git a/multimodal/src/media.rs b/multimodal/src/media.rs index d44633128f..f1e0501f45 100644 --- a/multimodal/src/media.rs +++ b/multimodal/src/media.rs @@ -2,7 +2,6 @@ use std::{collections::HashSet, path::PathBuf, sync::Arc, time::Duration}; use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine}; use bytes::Bytes; -use image::DynamicImage; use reqwest::Client; use tokio::{fs, task}; use url::Url; @@ -196,11 +195,14 @@ impl MediaConnector { detail: ImageDetail, source: ImageSource, ) -> Result, MediaConnectorError> { - let raw: Arc> = Arc::new(bytes.to_vec()); - let raw_clone = raw.clone(); - let image: DynamicImage = - task::spawn_blocking(move || image::load_from_memory(&raw_clone)).await??; + // Use bytes directly for decoding + let cursor = std::io::Cursor::new(bytes.clone()); + let reader = image::ImageReader::new(cursor).with_guessed_format()?; - Ok(Arc::new(ImageFrame::new(image, raw, detail, source))) + let image = task::spawn_blocking(move || reader.decode()) + .await + .map_err(MediaConnectorError::Blocking)??; + + Ok(Arc::new(ImageFrame::new(image, bytes, detail, source))) } } diff --git a/multimodal/src/types.rs b/multimodal/src/types.rs index fe3bb1dc95..4eabd3908f 100644 --- a/multimodal/src/types.rs +++ b/multimodal/src/types.rs @@ -121,10 +121,10 @@ pub enum ImageSource { } /// Concrete image payload captured by the media connector. -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct ImageFrame { - image: DynamicImage, - raw_bytes: Arc>, + pub image: DynamicImage, + pub raw_bytes: bytes::Bytes, pub detail: ImageDetail, pub source: ImageSource, } @@ -132,7 +132,7 @@ pub struct ImageFrame { impl ImageFrame { pub fn new( image: DynamicImage, - raw_bytes: Arc>, + raw_bytes: bytes::Bytes, detail: ImageDetail, source: ImageSource, ) -> Self { @@ -149,7 +149,7 @@ impl ImageFrame { } pub fn raw_bytes(&self) -> &[u8] { - self.raw_bytes.as_slice() + &self.raw_bytes } pub fn source(&self) -> &ImageSource { @@ -198,8 +198,7 @@ pub struct PlaceholderRange { pub struct MultiModalTensor { pub shape: Vec, pub dtype: String, - #[serde(with = "serde_bytes")] - pub data: Vec, + pub data: bytes::Bytes, } #[derive(Debug, Clone, Serialize, Deserialize)]