perf(multimodal): optimize payload handling with zero-copy

multimodal/Cargo.toml

@@ -18,7 +18,7 @@ name = "llm_multimodal"
 [dependencies]
 async-trait.workspace = true
 base64 = "0.22"
-bytes = "1.8.0"
+bytes = { version = "1.8.0", features = ["serde"] }
 image = { version = "0.25.4", default-features = false, features = ["png", "jpeg", "gif", "bmp", "ico", "tiff", "webp"] }
 ndarray = "0.16"
 once_cell = "1.21.3"

multimodal/src/media.rs

@@ -2,7 +2,6 @@ use std::{collections::HashSet, path::PathBuf, sync::Arc, time::Duration};
 
 use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine};
 use bytes::Bytes;
-use image::DynamicImage;
 use reqwest::Client;
 use tokio::{fs, task};
 use url::Url;
@@ -196,11 +195,14 @@ impl MediaConnector {
         detail: ImageDetail,
         source: ImageSource,
     ) -> Result<Arc<ImageFrame>, MediaConnectorError> {
-        let raw: Arc<Vec<u8>> = Arc::new(bytes.to_vec());
-        let raw_clone = raw.clone();
-        let image: DynamicImage =
-            task::spawn_blocking(move || image::load_from_memory(&raw_clone)).await??;
+        // Use bytes directly for decoding
+        let cursor = std::io::Cursor::new(bytes.clone());
+        let reader = image::ImageReader::new(cursor).with_guessed_format()?;
 
-        Ok(Arc::new(ImageFrame::new(image, raw, detail, source)))
+        let image = task::spawn_blocking(move || reader.decode())
+            .await
+            .map_err(MediaConnectorError::Blocking)??;
+
+        Ok(Arc::new(ImageFrame::new(image, bytes, detail, source)))
     }
 }

multimodal/src/types.rs

@@ -121,18 +121,18 @@ pub enum ImageSource {
 }
 
 /// Concrete image payload captured by the media connector.
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct ImageFrame {
-    image: DynamicImage,
-    raw_bytes: Arc<Vec<u8>>,
+    pub image: DynamicImage,
+    pub raw_bytes: bytes::Bytes,
     pub detail: ImageDetail,
     pub source: ImageSource,
 }
 
 impl ImageFrame {
     pub fn new(
         image: DynamicImage,
-        raw_bytes: Arc<Vec<u8>>,
+        raw_bytes: bytes::Bytes,
         detail: ImageDetail,
         source: ImageSource,
     ) -> Self {
@@ -149,7 +149,7 @@ impl ImageFrame {
     }
 
     pub fn raw_bytes(&self) -> &[u8] {
-        self.raw_bytes.as_slice()
+        &self.raw_bytes
     }
 
     pub fn source(&self) -> &ImageSource {
@@ -198,8 +198,7 @@ pub struct PlaceholderRange {
 pub struct MultiModalTensor {
     pub shape: Vec<usize>,
     pub dtype: String,
-    #[serde(with = "serde_bytes")]
-    pub data: Vec<u8>,
+    pub data: bytes::Bytes,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]