feat(grpc): send preprocessed multimodal data to vLLM with hashing and structured tokens

grpc_client/proto/vllm_engine.proto

@@ -92,10 +92,47 @@ message TokenizedInput {
   repeated uint32 input_ids = 2;  // Actual token IDs to process
 }
 
+// A typed tensor: raw little-endian bytes + shape + dtype.
+message TensorData {
+  bytes data = 1;              // Raw little-endian bytes (f32/i64/u32)
+  repeated uint32 shape = 2;   // Dimension sizes
+  string dtype = 3;            // "float32", "int64", "uint32"
+}
+
+message PlaceholderRange {
+  uint32 offset = 1;
+  uint32 length = 2;
+}
+
 // Multimodal inputs for vision/audio models
 message MultimodalInputs {
-  // Raw image bytes (JPEG/PNG) — vLLM handles preprocessing internally
-  repeated bytes image_data = 1;
+  // Reserved: field 1 was image_data (raw bytes), removed in favor of
+  // preprocessed pixel_values.
+  reserved 1;
+
+  // Preprocessed pixel values tensor
+  TensorData pixel_values = 2;
+
+  // Model-specific tensors (image_grid_thw, aspect_ratios, etc.)
+  map<string, TensorData> model_specific_tensors = 3;
+
+  // Image token ID used for placeholder expansion
+  optional uint32 im_token_id = 4;
+
+  // Placeholder offsets: where each image's tokens are in input_ids
+  repeated PlaceholderRange mm_placeholders = 5;
+
+  // Per-image blake3 hex hashes for encoder output caching
+  repeated string mm_hashes = 6;
+
+  // Tensor keys whose first dimension is per-image (batched).
+  // Keys not listed here are shared across all images.
+  repeated string batched_keys = 7;
+
+  // Tensor keys that need flat slicing: maps tensor name → sizes tensor name.
+  // e.g. {"pixel_values": "patches_per_image"} means pixel_values should be
+  // split per-item using sizes from the patches_per_image tensor.
+  map<string, string> flat_keys = 8;
 }
 
 // =====================

model_gateway/src/routers/grpc/mod.rs

@@ -1,8 +1,5 @@
 //! gRPC router implementations
 
-use std::sync::Arc;
-
-use llm_multimodal::ImageFrame;
 use openai_protocol::common::StringOrArray;
 
 pub mod client; // Used by core/
@@ -24,8 +21,8 @@ pub use proto_wrapper::{MultimodalData, TensorBytes};
 #[derive(Debug)]
 pub struct ProcessedMessages {
     pub text: String,
-    /// Raw fetched images (Phase 1). Backend-specific preprocessing
-    /// happens in Phase 2 at request building time.
-    pub multimodal_images: Option<Vec<Arc<ImageFrame>>>,
+    /// Preprocessed multimodal data (pixel values, placeholders, hashes).
+    /// Populated during preparation when multimodal content is detected.
+    pub multimodal_data: Option<MultimodalData>,
     pub stop_sequences: Option<StringOrArray>,
 }