From c864e7c566a0b934d833f0e0157a03b5d5064a5e Mon Sep 17 00:00:00 2001 From: Kangyan Zhou Date: Tue, 31 Mar 2026 16:08:51 -0700 Subject: [PATCH 01/23] feat(multimodal): add Kimi-K2.5 vision support for gRPC router Add ModelProcessorSpec and ImagePreProcessor for moonshotai/Kimi-K2.5 so the gRPC PD router can handle multimodal (image) requests. - KimiK25VisionSpec: matches "kimi" + "k2" model IDs, uses <|media_pad|> placeholder (media_placeholder_token_id from config), NaViT-style field layouts identical to Qwen-VL family - KimiK25Processor: wraps QwenVLProcessorBase with Kimi-specific defaults (patch_size=14, merge_size=2, normalization=[0.5,0.5,0.5], max_pixels=3,211,264 from in_patch_limit=16384) - Fix get_zmq_socket import path for sglang main compat Co-Authored-By: Claude Opus 4.6 (1M context) --- crates/multimodal/src/registry/kimi_k25.rs | 211 ++++++++++++++++ crates/multimodal/src/registry/mod.rs | 3 + .../multimodal/src/vision/image_processor.rs | 10 + .../src/vision/processors/kimi_k25.rs | 225 ++++++++++++++++++ .../multimodal/src/vision/processors/mod.rs | 2 + 5 files changed, 451 insertions(+) create mode 100644 crates/multimodal/src/registry/kimi_k25.rs create mode 100644 crates/multimodal/src/vision/processors/kimi_k25.rs diff --git a/crates/multimodal/src/registry/kimi_k25.rs b/crates/multimodal/src/registry/kimi_k25.rs new file mode 100644 index 0000000000..16488d1f8c --- /dev/null +++ b/crates/multimodal/src/registry/kimi_k25.rs @@ -0,0 +1,211 @@ +use std::collections::HashMap; + +use serde_json::{json, Value}; + +use crate::{ + registry::{ModelMetadata, ModelProcessorSpec, ModelRegistryError, RegistryResult}, + types::{FieldLayout, Modality, PromptReplacement, TokenId}, + vision::image_processor::PreprocessedImages, +}; + +pub(super) struct KimiK25VisionSpec; + +impl KimiK25VisionSpec { + /// The repeated pad token (`<|media_pad|>`) — `media_placeholder_token_id` in config. + fn pad_token_id(metadata: &ModelMetadata) -> RegistryResult { + metadata + .config_u32(&["media_placeholder_token_id"]) + .map(|v| v as TokenId) + .ok_or_else(|| ModelRegistryError::MissingConfigField { + field: "media_placeholder_token_id".to_string(), + }) + } +} + +impl ModelProcessorSpec for KimiK25VisionSpec { + fn name(&self) -> &'static str { + "kimi_k25" + } + + fn matches(&self, metadata: &ModelMetadata) -> bool { + let id = metadata.model_id.to_ascii_lowercase(); + id.contains("kimi") && id.contains("k2") + } + + fn placeholder_token(&self, _metadata: &ModelMetadata) -> RegistryResult { + Ok("<|media_pad|>".to_string()) + } + + fn placeholder_token_id(&self, metadata: &ModelMetadata) -> RegistryResult { + Self::pad_token_id(metadata) + } + + fn modality_limits( + &self, + _metadata: &ModelMetadata, + ) -> RegistryResult> { + Ok(HashMap::from([(Modality::Image, 10)])) + } + + fn processor_kwargs(&self, _metadata: &ModelMetadata) -> RegistryResult { + Ok(json!({})) + } + + fn prompt_replacements( + &self, + metadata: &ModelMetadata, + preprocessed: &PreprocessedImages, + ) -> RegistryResult> { + let pad_token_id = Self::pad_token_id(metadata)?; + let placeholder_token = self.placeholder_token(metadata)?; + Ok(preprocessed + .num_img_tokens + .iter() + .map(|&num_tokens| { + PromptReplacement::repeated( + Modality::Image, + &placeholder_token, + pad_token_id, + num_tokens, + ) + }) + .collect()) + } + + fn field_layouts(&self) -> HashMap { + // Kimi-K2.5 uses NaViT-style patchification like Qwen-VL: + // pixel_values is [total_patches, patch_features], split by patches_per_image. + // image_grid_thw is [num_images, 3]. + HashMap::from([ + ( + "pixel_values".to_string(), + FieldLayout::flat("patches_per_image"), + ), + ("image_grid_thw".to_string(), FieldLayout::Batched), + ("patches_per_image".to_string(), FieldLayout::Batched), + ]) + } + + fn keep_on_cpu_keys(&self) -> Vec { + vec!["image_grid_thw".to_string()] + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use crate::{ + registry::{test_helpers::*, ModelMetadata, ModelRegistry}, + types::ImageSize, + }; + + #[test] + fn kimi_k25_matches_model_id() { + let tokenizer = TestTokenizer::new(&[("<|media_pad|>", 163605)]); + let config = json!({ + "model_type": "kimi_k25", + "media_placeholder_token_id": 163605 + }); + let metadata = ModelMetadata { + model_id: "moonshotai/Kimi-K2.5", + tokenizer: &tokenizer, + config: &config, + }; + let registry = ModelRegistry::new(); + let spec = registry.lookup(&metadata).expect("kimi_k25 spec"); + assert_eq!(spec.name(), "kimi_k25"); + } + + #[test] + fn kimi_k25_prompt_replacements() { + let tokenizer = TestTokenizer::new(&[("<|media_pad|>", 163605)]); + let config = json!({ + "model_type": "kimi_k25", + "media_placeholder_token_id": 163605 + }); + let metadata = ModelMetadata { + model_id: "moonshotai/Kimi-K2.5", + tokenizer: &tokenizer, + config: &config, + }; + let registry = ModelRegistry::new(); + let spec = registry.lookup(&metadata).expect("kimi_k25 spec"); + + let replacements = spec + .prompt_replacements( + &metadata, + &test_preprocessed_with_tokens(&[ImageSize::new(448, 448)], &[256]), + ) + .unwrap(); + + // 256 pad tokens (no start/end wrapper — SGLang handles that in the chat template) + assert_eq!(replacements[0].tokens.len(), 256); + assert!(replacements[0].tokens.iter().all(|&t| t == 163605)); + } + + #[test] + fn kimi_k25_prompt_replacements_multiple_images() { + let tokenizer = TestTokenizer::new(&[("<|media_pad|>", 163605)]); + let config = json!({ + "model_type": "kimi_k25", + "media_placeholder_token_id": 163605 + }); + let metadata = ModelMetadata { + model_id: "moonshotai/Kimi-K2.5", + tokenizer: &tokenizer, + config: &config, + }; + let registry = ModelRegistry::new(); + let spec = registry.lookup(&metadata).expect("kimi_k25 spec"); + + let replacements = spec + .prompt_replacements( + &metadata, + &test_preprocessed_with_tokens( + &[ImageSize::new(448, 448), ImageSize::new(224, 224)], + &[256, 64], + ), + ) + .unwrap(); + + assert_eq!(replacements.len(), 2); + assert_eq!(replacements[0].tokens.len(), 256); + assert_eq!(replacements[1].tokens.len(), 64); + assert!(replacements[1].tokens.iter().all(|&t| t == 163605)); + } + + #[test] + fn kimi_k25_matches_kimi_k2_variant() { + let tokenizer = TestTokenizer::new(&[("<|media_pad|>", 163605)]); + let config = json!({ + "model_type": "kimi_k25", + "media_placeholder_token_id": 163605 + }); + let metadata = ModelMetadata { + model_id: "moonshotai/Kimi-K2-VL", + tokenizer: &tokenizer, + config: &config, + }; + let registry = ModelRegistry::new(); + let spec = registry.lookup(&metadata); + assert!(spec.is_some(), "Should match Kimi-K2 variants"); + } + + #[test] + fn kimi_k25_does_not_match_kimi_k1() { + let tokenizer = TestTokenizer::new(&[("<|media_pad|>", 163605)]); + let config = json!({ + "model_type": "kimi_k1", + "media_placeholder_token_id": 163605 + }); + let metadata = ModelMetadata { + model_id: "moonshotai/Kimi-K1-VL", + tokenizer: &tokenizer, + config: &config, + }; + let registry = ModelRegistry::new(); + let spec = registry.lookup(&metadata); + assert!(spec.is_none(), "Should not match Kimi-K1"); + } +} diff --git a/crates/multimodal/src/registry/mod.rs b/crates/multimodal/src/registry/mod.rs index 22ab15cc63..6ddb366da2 100644 --- a/crates/multimodal/src/registry/mod.rs +++ b/crates/multimodal/src/registry/mod.rs @@ -1,3 +1,4 @@ +mod kimi_k25; mod llama4; mod llava; mod phi3_v; @@ -5,6 +6,7 @@ mod qwen3_vl; mod qwen_vl; mod traits; +use kimi_k25::KimiK25VisionSpec; use llama4::Llama4Spec; use llava::{LlavaNextSpec, LlavaSpec}; use once_cell::sync::Lazy; @@ -22,6 +24,7 @@ impl ModelRegistry { pub fn new() -> Self { Self { specs: vec![ + LazySpec::new("kimi_k25", || Box::new(KimiK25VisionSpec)), LazySpec::new("llama4", || Box::new(Llama4Spec)), // LlavaNext must be registered before Llava so "llava_next" model_type matches first. LazySpec::new("llava_next", || Box::new(LlavaNextSpec)), diff --git a/crates/multimodal/src/vision/image_processor.rs b/crates/multimodal/src/vision/image_processor.rs index 1297335351..97cc870ecb 100644 --- a/crates/multimodal/src/vision/image_processor.rs +++ b/crates/multimodal/src/vision/image_processor.rs @@ -464,6 +464,16 @@ impl ImageProcessorRegistry { Box::new(super::processors::Llama4VisionProcessor::new()), ); + // Register Kimi-K2.5 Vision + registry.register( + "kimi-k2", + Box::new(super::processors::KimiK25Processor::new()), + ); + registry.register( + "kimi_k2", + Box::new(super::processors::KimiK25Processor::new()), + ); + registry } } diff --git a/crates/multimodal/src/vision/processors/kimi_k25.rs b/crates/multimodal/src/vision/processors/kimi_k25.rs new file mode 100644 index 0000000000..553ece9b87 --- /dev/null +++ b/crates/multimodal/src/vision/processors/kimi_k25.rs @@ -0,0 +1,225 @@ +//! Kimi-K2.5 image processor. +//! +//! This module provides the Kimi-K2.5 processor which wraps the shared +//! `QwenVLProcessorBase` with Kimi-K2.5 specific default parameters. +//! +//! Kimi-K2.5 uses MoonViT with NaViT-style patchification that is structurally +//! identical to Qwen2-VL's processing pipeline, differing in normalization +//! (simple [0.5, 0.5, 0.5] instead of CLIP mean/std) and default max_pixels +//! (3,211,264 vs 1,003,520). +//! +//! # Kimi-K2.5 Parameters +//! +//! - patch_size: 14 +//! - merge_size: 2 +//! - factor: 28 (patch_size * merge_size) +//! - normalization: [0.5, 0.5, 0.5] mean/std + +use std::ops::Deref; + +use image::DynamicImage; + +use super::qwen_vl_base::{QwenVLConfig, QwenVLProcessorBase}; +use crate::vision::{ + image_processor::{ImagePreProcessor, PreprocessedImages}, + preprocessor_config::PreProcessorConfig, + transforms::TransformError, +}; + +/// Kimi-K2.5 normalization mean values. +pub const KIMI_K25_MEAN: [f64; 3] = [0.5, 0.5, 0.5]; + +/// Kimi-K2.5 normalization std values. +pub const KIMI_K25_STD: [f64; 3] = [0.5, 0.5, 0.5]; + +/// Default minimum pixels for Kimi-K2.5 (256 * 28 * 28 = 200,704) +pub const DEFAULT_MIN_PIXELS: usize = 256 * 28 * 28; + +/// Default maximum pixels for Kimi-K2.5 (16384 * 14 * 14 = 3,211,264) +/// Based on in_patch_limit=16384 from preprocessor config. +pub const DEFAULT_MAX_PIXELS: usize = 16384 * 14 * 14; + +/// Default patch size +pub const DEFAULT_PATCH_SIZE: usize = 14; + +/// Default merge size for token reduction +pub const DEFAULT_MERGE_SIZE: usize = 2; + +/// Default temporal patch size (for video frames) +pub const DEFAULT_TEMPORAL_PATCH_SIZE: usize = 2; + +/// Kimi-K2.5 image processor. +/// +/// This is a thin wrapper around `QwenVLProcessorBase` with Kimi-K2.5 +/// specific default parameters: +/// - patch_size: 14 +/// - merge_size: 2 +/// - [0.5, 0.5, 0.5] normalization mean/std +#[derive(Debug, Clone)] +pub struct KimiK25Processor { + inner: QwenVLProcessorBase, +} + +impl Default for KimiK25Processor { + fn default() -> Self { + Self::new() + } +} + +impl KimiK25Processor { + pub fn new() -> Self { + Self { + inner: QwenVLProcessorBase::new(QwenVLConfig { + patch_size: DEFAULT_PATCH_SIZE, + merge_size: DEFAULT_MERGE_SIZE, + min_pixels: DEFAULT_MIN_PIXELS, + max_pixels: DEFAULT_MAX_PIXELS, + temporal_patch_size: DEFAULT_TEMPORAL_PATCH_SIZE, + mean: KIMI_K25_MEAN, + std: KIMI_K25_STD, + model_name: "kimi-k2.5", + }), + } + } + + pub fn from_preprocessor_config(config: &PreProcessorConfig) -> Self { + Self { + inner: QwenVLProcessorBase::new(QwenVLConfig { + patch_size: config.get_patch_size(DEFAULT_PATCH_SIZE), + merge_size: config.merge_size.unwrap_or(DEFAULT_MERGE_SIZE), + min_pixels: config.min_pixels.unwrap_or(DEFAULT_MIN_PIXELS), + max_pixels: config.max_pixels.unwrap_or(DEFAULT_MAX_PIXELS), + temporal_patch_size: config + .temporal_patch_size + .unwrap_or(DEFAULT_TEMPORAL_PATCH_SIZE), + mean: KIMI_K25_MEAN, + std: KIMI_K25_STD, + model_name: "kimi-k2.5", + }), + } + } +} + +impl Deref for KimiK25Processor { + type Target = QwenVLProcessorBase; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl ImagePreProcessor for KimiK25Processor { + fn default_mean(&self) -> [f64; 3] { + self.inner.default_mean() + } + + fn default_std(&self) -> [f64; 3] { + self.inner.default_std() + } + + fn preprocess( + &self, + images: &[DynamicImage], + config: &PreProcessorConfig, + ) -> Result { + self.inner.preprocess(images, config) + } + + fn calculate_num_tokens(&self, width: u32, height: u32, config: &PreProcessorConfig) -> usize { + self.inner.calculate_num_tokens(width, height, config) + } + + fn model_name(&self) -> &'static str { + self.inner.model_name() + } + + fn get_processed_size(&self, config: &PreProcessorConfig) -> Option<(u32, u32)> { + self.inner.get_processed_size(config) + } +} + +#[cfg(test)] +mod tests { + use image::{Rgb, RgbImage}; + + use super::*; + use crate::vision::image_processor::ModelSpecificValue; + + fn create_test_image(width: u32, height: u32, color: Rgb) -> DynamicImage { + DynamicImage::from(RgbImage::from_pixel(width, height, color)) + } + + #[test] + fn test_kimi_k25_processor_default() { + let processor = KimiK25Processor::new(); + assert_eq!(processor.patch_size(), 14); + assert_eq!(processor.merge_size(), 2); + assert_eq!(processor.min_pixels(), DEFAULT_MIN_PIXELS); + assert_eq!(processor.max_pixels(), DEFAULT_MAX_PIXELS); + assert_eq!(processor.get_factor(), 28); // 14 * 2 + } + + #[test] + fn test_default_mean_std() { + let processor = KimiK25Processor::new(); + assert_eq!(processor.default_mean(), KIMI_K25_MEAN); + assert_eq!(processor.default_std(), KIMI_K25_STD); + } + + #[test] + fn test_model_name() { + let processor = KimiK25Processor::new(); + assert_eq!(processor.model_name(), "kimi-k2.5"); + } + + #[test] + fn test_kimi_k25_preprocess() { + let processor = KimiK25Processor::new(); + let config = PreProcessorConfig { + do_resize: Some(true), + do_normalize: Some(true), + image_mean: Some(KIMI_K25_MEAN.to_vec()), + image_std: Some(KIMI_K25_STD.to_vec()), + ..Default::default() + }; + + let image = create_test_image(600, 400, Rgb([128, 128, 128])); + let result = processor.preprocess(&[image], &config).unwrap(); + + // pixel_values is patchified: [total_patches, patch_features] + assert_eq!(result.pixel_values.ndim(), 2); + assert!(result.pixel_values.shape()[0] > 0); + + // Check image_grid_thw and patches_per_image are present + assert!(result.model_specific.contains_key("image_grid_thw")); + assert!(result.model_specific.contains_key("patches_per_image")); + + assert!(result.num_img_tokens[0] > 0); + } + + #[test] + fn test_kimi_k25_preprocess_multiple() { + let processor = KimiK25Processor::new(); + let config = PreProcessorConfig::default(); + + let images = vec![ + create_test_image(600, 400, Rgb([100, 100, 100])), + create_test_image(400, 600, Rgb([150, 150, 150])), + ]; + + let result = processor.preprocess(&images, &config).unwrap(); + + assert_eq!(result.image_sizes.len(), 2); + assert_eq!(result.num_img_tokens.len(), 2); + assert_eq!(result.pixel_values.ndim(), 2); + + if let Some(ModelSpecificValue::IntTensor { data, shape }) = + result.model_specific.get("image_grid_thw") + { + assert_eq!(shape, &[2, 3]); + assert_eq!(data.len(), 6); + } else { + panic!("Expected image_grid_thw to be IntTensor"); + } + } +} diff --git a/crates/multimodal/src/vision/processors/mod.rs b/crates/multimodal/src/vision/processors/mod.rs index 4b2c48d2c0..eeded2639d 100644 --- a/crates/multimodal/src/vision/processors/mod.rs +++ b/crates/multimodal/src/vision/processors/mod.rs @@ -15,6 +15,7 @@ //! - **LLaMA 4 Vision** (`llama4_vision`): Tile-based processing with 336x336 tiles and global tile //! - **Pixtral/Mistral3** (`pixtral`): CLIP-based preprocessing with dynamic resolution +pub mod kimi_k25; pub mod llama4_vision; pub mod llava; pub mod phi3_vision; @@ -24,6 +25,7 @@ pub mod qwen2_vl; pub mod qwen3_vl; pub mod qwen_vl_base; +pub use kimi_k25::KimiK25Processor; pub use llama4_vision::Llama4VisionProcessor; pub use llava::{ImageAspectRatio, LlavaNextProcessor, LlavaProcessor}; pub use phi3_vision::Phi3VisionProcessor; From c658a4f2fb5b81fafa7799e34e7a475d01e732ba Mon Sep 17 00:00:00 2001 From: Kangyan Zhou Date: Tue, 31 Mar 2026 17:14:51 -0700 Subject: [PATCH 02/23] fix(multimodal): download config.json from HF Hub when not available locally When the tokenizer source is a HuggingFace model ID (e.g., "moonshotai/Kimi-K2.5") rather than a local directory, the gRPC router cannot read config.json and preprocessor_config.json from disk. This causes multimodal requests to fail with "Failed to read config.json". Make get_or_load_config async and fall back to downloading the two config files from HF Hub via the new download_files_from_hf helper when the local path doesn't exist. Co-Authored-By: Claude Opus 4.6 (1M context) --- crates/tokenizer/src/hub.rs | 31 ++++- model_gateway/src/routers/grpc/multimodal.rs | 138 +++++++++++-------- 2 files changed, 107 insertions(+), 62 deletions(-) diff --git a/crates/tokenizer/src/hub.rs b/crates/tokenizer/src/hub.rs index 0879c0545e..ffdc1f75fc 100644 --- a/crates/tokenizer/src/hub.rs +++ b/crates/tokenizer/src/hub.rs @@ -1,4 +1,7 @@ -use std::path::{Path, PathBuf}; +use std::{ + collections::HashMap, + path::{Path, PathBuf}, +}; use hf_hub::api::tokio::{Api, ApiBuilder}; @@ -63,6 +66,32 @@ fn is_chat_template_file(filename: &str) -> bool { || filename == "chat_template.json" // JSON file containing Jinja template } +/// Download specific files from a HuggingFace model repo. +/// +/// Returns a map of filename → local cache path for each successfully downloaded file. +/// Files that don't exist in the repo are silently skipped. +pub async fn download_files_from_hf( + model_id: &str, + filenames: &[&str], +) -> anyhow::Result> { + let api = build_api()?; + let repo = api.model(model_id.to_string()); + let mut results = HashMap::new(); + + for &filename in filenames { + match repo.get(filename).await { + Ok(path) => { + results.insert(filename.to_string(), path); + } + Err(_) => { + // File doesn't exist in repo, skip + } + } + } + + Ok(results) +} + /// Attempt to download tokenizer files from Hugging Face /// Returns the directory containing the downloaded tokenizer files pub async fn download_tokenizer_from_hf(model_id: impl AsRef) -> anyhow::Result { diff --git a/model_gateway/src/routers/grpc/multimodal.rs b/model_gateway/src/routers/grpc/multimodal.rs index a68ec14580..6019049cc1 100644 --- a/model_gateway/src/routers/grpc/multimodal.rs +++ b/model_gateway/src/routers/grpc/multimodal.rs @@ -9,7 +9,7 @@ //! functions differ because they work with different input types (`ChatMessage` vs //! `InputMessage`). -use std::{collections::HashMap, sync::Arc}; +use std::{collections::HashMap, path::Path, sync::Arc}; use anyhow::{Context, Result}; use dashmap::DashMap; @@ -70,6 +70,9 @@ impl MultimodalComponents { } /// Load or retrieve cached model config for a given model and tokenizer source path. + /// + /// If `tokenizer_source` is a local directory containing `config.json`, loads from disk. + /// Otherwise, treats it as a HuggingFace model ID and downloads the config files. pub async fn get_or_load_config( &self, model_id: &str, @@ -79,13 +82,32 @@ impl MultimodalComponents { return Ok(cached.clone()); } - let base_dir = llm_multimodal::hub::resolve_model_config_dir(tokenizer_source) - .await - .with_context(|| { - format!("Failed to resolve model config directory for '{tokenizer_source}'") - })?; + let base_dir = Path::new(tokenizer_source); + let config_path = base_dir.join("config.json"); + + // If local config.json exists, load from disk; otherwise download from HF Hub. + let (config, preprocessor_config) = if config_path.is_file() { + Self::load_configs_from_disk(base_dir)? + } else { + debug!( + model_id = model_id, + source = tokenizer_source, + "config.json not found locally, downloading from HuggingFace Hub" + ); + Self::load_configs_from_hf(tokenizer_source).await? + }; + + let model_config = Arc::new(MultimodalModelConfig { + config, + preprocessor_config, + }); + + self.model_configs + .insert(model_id.to_string(), model_config.clone()); + Ok(model_config) + } - // Load config.json + fn load_configs_from_disk(base_dir: &Path) -> Result<(serde_json::Value, PreProcessorConfig)> { let config_path = base_dir.join("config.json"); let config: serde_json::Value = std::fs::read_to_string(&config_path) .with_context(|| format!("Failed to read config.json at {}", config_path.display())) @@ -95,7 +117,6 @@ impl MultimodalComponents { }) })?; - // Load preprocessor_config.json let pp_config_path = base_dir.join("preprocessor_config.json"); let preprocessor_config = std::fs::read_to_string(&pp_config_path) .with_context(|| { @@ -113,14 +134,53 @@ impl MultimodalComponents { }) })?; - let model_config = Arc::new(MultimodalModelConfig { - config, - preprocessor_config, - }); + Ok((config, preprocessor_config)) + } - self.model_configs - .insert(model_id.to_string(), model_config.clone()); - Ok(model_config) + async fn load_configs_from_hf( + model_id: &str, + ) -> Result<(serde_json::Value, PreProcessorConfig)> { + let files = llm_tokenizer::hub::download_files_from_hf( + model_id, + &["config.json", "preprocessor_config.json"], + ) + .await + .with_context(|| { + format!("Failed to download config files from HuggingFace for model: {model_id}") + })?; + + let config_path = files.get("config.json").ok_or_else(|| { + anyhow::anyhow!("config.json not found in HuggingFace repo: {model_id}") + })?; + let config: serde_json::Value = std::fs::read_to_string(config_path) + .with_context(|| { + format!( + "Failed to read downloaded config.json at {}", + config_path.display() + ) + }) + .and_then(|s| { + serde_json::from_str(&s) + .with_context(|| format!("Failed to parse config.json for model: {model_id}")) + })?; + + let pp_config_path = files.get("preprocessor_config.json").ok_or_else(|| { + anyhow::anyhow!("preprocessor_config.json not found in HuggingFace repo: {model_id}") + })?; + let preprocessor_config = std::fs::read_to_string(pp_config_path) + .with_context(|| { + format!( + "Failed to read downloaded preprocessor_config.json at {}", + pp_config_path.display() + ) + }) + .and_then(|s| { + PreProcessorConfig::from_json(&s).with_context(|| { + format!("Failed to parse preprocessor_config.json for model: {model_id}") + }) + })?; + + Ok((config, preprocessor_config)) } } @@ -156,34 +216,6 @@ pub(crate) struct MultimodalIntermediate { pub keep_on_cpu_keys: Vec, } -/// Resolve the placeholder token string for a multimodal model. -/// -/// Loads the model config and looks up the model spec to get the placeholder -/// token (e.g. `"<|image|>"` for Phi-3-vision). Returns `None` if the model -/// is not recognized as multimodal. -pub(crate) async fn resolve_placeholder_token( - model_id: &str, - tokenizer: &dyn TokenizerTrait, - components: &MultimodalComponents, - tokenizer_source: &str, -) -> Result> { - let model_config = components - .get_or_load_config(model_id, tokenizer_source) - .await?; - let metadata = ModelMetadata { - model_id, - tokenizer, - config: &model_config.config, - }; - let spec = match components.model_registry.lookup(&metadata) { - Some(s) => s, - None => return Ok(None), - }; - Ok(Some(spec.placeholder_token(&metadata).map_err(|e| { - anyhow::anyhow!("Failed to get placeholder token: {e}") - })?)) -} - /// Check if any messages in the request contain multimodal content (images). pub(crate) fn has_multimodal_content(messages: &[ChatMessage]) -> bool { messages.iter().any(|msg| { @@ -409,7 +441,6 @@ async fn process_multimodal_parts( debug!( image_count = images.len(), - image_sizes = ?images.iter().map(|f| (f.image.width(), f.image.height())).collect::>(), "Fetched images for multimodal processing" ); @@ -417,10 +448,6 @@ async fn process_multimodal_parts( let model_config = components .get_or_load_config(model_id, tokenizer_source) .await?; - let model_type = model_config - .config - .get("model_type") - .and_then(|v| v.as_str()); let metadata = ModelMetadata { model_id, tokenizer, @@ -433,7 +460,7 @@ async fn process_multimodal_parts( let image_processor = components .image_processor_registry - .find(model_id, model_type) + .find(model_id) .ok_or_else(|| anyhow::anyhow!("No image processor found for model: {model_id}"))?; // ImagePreProcessor::preprocess takes &[DynamicImage]; images are behind Arc. @@ -685,18 +712,7 @@ fn serialize_pixel_values(preprocessed: &PreprocessedImages) -> (Vec, Vec Date: Tue, 31 Mar 2026 17:19:44 -0700 Subject: [PATCH 03/23] fix(multimodal): improve error handling in HF Hub config download Address review findings: - Log errors from HF Hub downloads instead of silently swallowing them - Add explicit error when local model directory exists but config.json is missing (prevents misleading fallback to HF Hub) - Upgrade fallback log from debug to warn for production visibility Co-Authored-By: Claude Opus 4.6 (1M context) --- crates/tokenizer/src/hub.rs | 9 +++++++-- model_gateway/src/routers/grpc/multimodal.rs | 9 ++++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/crates/tokenizer/src/hub.rs b/crates/tokenizer/src/hub.rs index ffdc1f75fc..c8251bb1c7 100644 --- a/crates/tokenizer/src/hub.rs +++ b/crates/tokenizer/src/hub.rs @@ -83,8 +83,13 @@ pub async fn download_files_from_hf( Ok(path) => { results.insert(filename.to_string(), path); } - Err(_) => { - // File doesn't exist in repo, skip + Err(e) => { + tracing::warn!( + filename = filename, + model_id = model_id, + error = %e, + "Failed to download file from HuggingFace, skipping" + ); } } } diff --git a/model_gateway/src/routers/grpc/multimodal.rs b/model_gateway/src/routers/grpc/multimodal.rs index 6019049cc1..a73b24d433 100644 --- a/model_gateway/src/routers/grpc/multimodal.rs +++ b/model_gateway/src/routers/grpc/multimodal.rs @@ -88,8 +88,15 @@ impl MultimodalComponents { // If local config.json exists, load from disk; otherwise download from HF Hub. let (config, preprocessor_config) = if config_path.is_file() { Self::load_configs_from_disk(base_dir)? + } else if base_dir.is_dir() { + // Local directory exists but config.json is missing + anyhow::bail!( + "config.json not found in local model directory: {}. \ + Ensure the directory contains config.json and preprocessor_config.json.", + base_dir.display() + ); } else { - debug!( + warn!( model_id = model_id, source = tokenizer_source, "config.json not found locally, downloading from HuggingFace Hub" From fc07c18050428c89e654f4c60cdeb7572c6eac68 Mon Sep 17 00:00:00 2001 From: Kangyan Zhou Date: Tue, 31 Mar 2026 18:46:25 -0700 Subject: [PATCH 04/23] fix(tokenizer): use encode_with_special_tokens for tiktoken encoding TiktokenTokenizer::encode() was using encode_ordinary() which ignores special tokens in the input text. This caused chat-template-rendered special tokens like <|media_pad|> to be split into BPE sub-tokens instead of being recognized as single token IDs. Switch to encode_with_special_tokens() unconditionally, matching HuggingFace tokenizer behavior where added special tokens are always recognized in input text. This fixes Kimi-K2.5 multimodal where the chat template inserts <|media_pad|> (ID 163605) but the tokenizer was producing sub-tokens that expand_tokens couldn't find. Co-Authored-By: Claude Opus 4.6 (1M context) --- crates/tokenizer/src/tiktoken.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/tokenizer/src/tiktoken.rs b/crates/tokenizer/src/tiktoken.rs index 83ee373063..d4ba0d7d13 100644 --- a/crates/tokenizer/src/tiktoken.rs +++ b/crates/tokenizer/src/tiktoken.rs @@ -437,7 +437,12 @@ pub fn is_tiktoken_file(path: &Path) -> bool { impl Encoder for TiktokenTokenizer { fn encode(&self, input: &str, _add_special_tokens: bool) -> Result { - let tokens = self.tokenizer.encode_ordinary(input); + // Always use encode_with_special_tokens so that special token strings + // in the input (e.g., <|media_pad|> from chat templates) are recognized + // as single tokens rather than split into BPE sub-tokens. + // This matches HuggingFace tokenizer behavior where added special tokens + // are always recognized in input text regardless of add_special_tokens. + let tokens = self.tokenizer.encode_with_special_tokens(input); Ok(Encoding::Tiktoken(tokens)) } From d7e26116594bdee3d140648d22c48620b03181a6 Mon Sep 17 00:00:00 2001 From: Kangyan Zhou Date: Tue, 31 Mar 2026 19:27:21 -0700 Subject: [PATCH 05/23] fix(multimodal): rename image_grid_thw to grid_thws for Kimi-K2.5 Kimi-K2.5 engine accesses `item.grid_thws` (plural) on MultimodalDataItem, but the gateway was sending `image_grid_thw` (Qwen-VL convention). Rename the key in the processor output and update field_layouts/keep_on_cpu_keys to match. Co-Authored-By: Claude Opus 4.6 (1M context) --- crates/multimodal/src/registry/kimi_k25.rs | 6 +++--- .../src/vision/processors/kimi_k25.rs | 17 ++++++++++++----- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/crates/multimodal/src/registry/kimi_k25.rs b/crates/multimodal/src/registry/kimi_k25.rs index 16488d1f8c..d472b42570 100644 --- a/crates/multimodal/src/registry/kimi_k25.rs +++ b/crates/multimodal/src/registry/kimi_k25.rs @@ -75,19 +75,19 @@ impl ModelProcessorSpec for KimiK25VisionSpec { fn field_layouts(&self) -> HashMap { // Kimi-K2.5 uses NaViT-style patchification like Qwen-VL: // pixel_values is [total_patches, patch_features], split by patches_per_image. - // image_grid_thw is [num_images, 3]. + // grid_thws is [num_images, 3] (Kimi uses "grid_thws" instead of Qwen's "image_grid_thw"). HashMap::from([ ( "pixel_values".to_string(), FieldLayout::flat("patches_per_image"), ), - ("image_grid_thw".to_string(), FieldLayout::Batched), + ("grid_thws".to_string(), FieldLayout::Batched), ("patches_per_image".to_string(), FieldLayout::Batched), ]) } fn keep_on_cpu_keys(&self) -> Vec { - vec!["image_grid_thw".to_string()] + vec!["grid_thws".to_string()] } } diff --git a/crates/multimodal/src/vision/processors/kimi_k25.rs b/crates/multimodal/src/vision/processors/kimi_k25.rs index 553ece9b87..494a24c126 100644 --- a/crates/multimodal/src/vision/processors/kimi_k25.rs +++ b/crates/multimodal/src/vision/processors/kimi_k25.rs @@ -122,7 +122,14 @@ impl ImagePreProcessor for KimiK25Processor { images: &[DynamicImage], config: &PreProcessorConfig, ) -> Result { - self.inner.preprocess(images, config) + let mut result = self.inner.preprocess(images, config)?; + + // Kimi-K2.5 engine expects "grid_thws" instead of Qwen-VL's "image_grid_thw" + if let Some(val) = result.model_specific.remove("image_grid_thw") { + result.model_specific.insert("grid_thws".to_string(), val); + } + + Ok(result) } fn calculate_num_tokens(&self, width: u32, height: u32, config: &PreProcessorConfig) -> usize { @@ -190,8 +197,8 @@ mod tests { assert_eq!(result.pixel_values.ndim(), 2); assert!(result.pixel_values.shape()[0] > 0); - // Check image_grid_thw and patches_per_image are present - assert!(result.model_specific.contains_key("image_grid_thw")); + // Check grid_thws and patches_per_image are present + assert!(result.model_specific.contains_key("grid_thws")); assert!(result.model_specific.contains_key("patches_per_image")); assert!(result.num_img_tokens[0] > 0); @@ -214,12 +221,12 @@ mod tests { assert_eq!(result.pixel_values.ndim(), 2); if let Some(ModelSpecificValue::IntTensor { data, shape }) = - result.model_specific.get("image_grid_thw") + result.model_specific.get("grid_thws") { assert_eq!(shape, &[2, 3]); assert_eq!(data.len(), 6); } else { - panic!("Expected image_grid_thw to be IntTensor"); + panic!("Expected grid_thws to be IntTensor"); } } } From 39a1d9ced4bfc1f39445a3266dc6b8603143c9ba Mon Sep 17 00:00:00 2001 From: Kangyan Zhou Date: Tue, 31 Mar 2026 20:16:30 -0700 Subject: [PATCH 06/23] refactor(multimodal): make KimiK25Processor fully standalone Remove QwenVLProcessorBase dependency. Kimi-K2.5's MoonViT expects pixel_values as [N, C, patch_size, patch_size] (4D), not flattened [N, C*T*patch_size*patch_size] (2D) like Qwen-VL. The model's PatchEmbed3d applies Conv2d on each patch directly. Implement smart_resize and extract_patches independently, producing [total_patches, 3*14*14] = [N, 588] patches that the engine reconstructs as [N, 3, 14, 14] for Conv2d input. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../src/vision/processors/kimi_k25.rs | 375 ++++++++++++++---- 1 file changed, 289 insertions(+), 86 deletions(-) diff --git a/crates/multimodal/src/vision/processors/kimi_k25.rs b/crates/multimodal/src/vision/processors/kimi_k25.rs index 494a24c126..dcf5c2fd99 100644 --- a/crates/multimodal/src/vision/processors/kimi_k25.rs +++ b/crates/multimodal/src/vision/processors/kimi_k25.rs @@ -1,63 +1,67 @@ -//! Kimi-K2.5 image processor. +//! Kimi-K2.5 (MoonViT) image processor. //! -//! This module provides the Kimi-K2.5 processor which wraps the shared -//! `QwenVLProcessorBase` with Kimi-K2.5 specific default parameters. +//! Kimi-K2.5 uses MoonViT which expects pixel_values as +//! `[total_patches, C, patch_size, patch_size]` (4D on the engine side). +//! The model's PatchEmbed3d applies Conv2d on each patch, so patches must NOT +//! be flattened across the temporal dimension in the preprocessor. //! -//! Kimi-K2.5 uses MoonViT with NaViT-style patchification that is structurally -//! identical to Qwen2-VL's processing pipeline, differing in normalization -//! (simple [0.5, 0.5, 0.5] instead of CLIP mean/std) and default max_pixels -//! (3,211,264 vs 1,003,520). +//! # Processing Pipeline //! -//! # Kimi-K2.5 Parameters +//! 1. NaViT-style resize to fit within min/max pixel bounds +//! 2. Align dimensions to (patch_size * merge_size) boundary +//! 3. Convert to tensor and normalize with [0.5, 0.5, 0.5] mean/std +//! 4. Extract patches as [C, patch_size, patch_size] per spatial position +//! +//! # Token Calculation +//! +//! ```text +//! grid_t = 1 (for images) +//! grid_h = resized_height / patch_size +//! grid_w = resized_width / patch_size +//! num_tokens = (grid_t * grid_h * grid_w) / merge_size² +//! ``` +//! +//! # Parameters //! //! - patch_size: 14 //! - merge_size: 2 //! - factor: 28 (patch_size * merge_size) //! - normalization: [0.5, 0.5, 0.5] mean/std +//! - max_pixels: 3,211,264 (from in_patch_limit=16384) -use std::ops::Deref; - -use image::DynamicImage; +use image::{DynamicImage, GenericImageView}; +use ndarray::{Array2, Array3}; -use super::qwen_vl_base::{QwenVLConfig, QwenVLProcessorBase}; use crate::vision::{ - image_processor::{ImagePreProcessor, PreprocessedImages}, + image_processor::{ImagePreProcessor, ModelSpecificValue, PreprocessedImages}, preprocessor_config::PreProcessorConfig, - transforms::TransformError, + transforms::{normalize, pil_to_filter, resize, to_tensor, TransformError}, }; -/// Kimi-K2.5 normalization mean values. pub const KIMI_K25_MEAN: [f64; 3] = [0.5, 0.5, 0.5]; - -/// Kimi-K2.5 normalization std values. pub const KIMI_K25_STD: [f64; 3] = [0.5, 0.5, 0.5]; -/// Default minimum pixels for Kimi-K2.5 (256 * 28 * 28 = 200,704) -pub const DEFAULT_MIN_PIXELS: usize = 256 * 28 * 28; - -/// Default maximum pixels for Kimi-K2.5 (16384 * 14 * 14 = 3,211,264) -/// Based on in_patch_limit=16384 from preprocessor config. -pub const DEFAULT_MAX_PIXELS: usize = 16384 * 14 * 14; - -/// Default patch size pub const DEFAULT_PATCH_SIZE: usize = 14; - -/// Default merge size for token reduction pub const DEFAULT_MERGE_SIZE: usize = 2; +pub const DEFAULT_MIN_PIXELS: usize = 256 * 28 * 28; // 200,704 +pub const DEFAULT_MAX_PIXELS: usize = 16384 * 14 * 14; // 3,211,264 (in_patch_limit=16384) + +/// Python-compatible rounding (banker's rounding / round half to even). +#[inline] +fn round_half_to_even(x: f64) -> f64 { + let rounded = x.round(); + if (x - x.floor() - 0.5).abs() < 1e-9 && rounded as i64 % 2 != 0 { + return rounded - 1.0; + } + rounded +} -/// Default temporal patch size (for video frames) -pub const DEFAULT_TEMPORAL_PATCH_SIZE: usize = 2; - -/// Kimi-K2.5 image processor. -/// -/// This is a thin wrapper around `QwenVLProcessorBase` with Kimi-K2.5 -/// specific default parameters: -/// - patch_size: 14 -/// - merge_size: 2 -/// - [0.5, 0.5, 0.5] normalization mean/std #[derive(Debug, Clone)] pub struct KimiK25Processor { - inner: QwenVLProcessorBase, + patch_size: usize, + merge_size: usize, + min_pixels: usize, + max_pixels: usize, } impl Default for KimiK25Processor { @@ -69,52 +73,131 @@ impl Default for KimiK25Processor { impl KimiK25Processor { pub fn new() -> Self { Self { - inner: QwenVLProcessorBase::new(QwenVLConfig { - patch_size: DEFAULT_PATCH_SIZE, - merge_size: DEFAULT_MERGE_SIZE, - min_pixels: DEFAULT_MIN_PIXELS, - max_pixels: DEFAULT_MAX_PIXELS, - temporal_patch_size: DEFAULT_TEMPORAL_PATCH_SIZE, - mean: KIMI_K25_MEAN, - std: KIMI_K25_STD, - model_name: "kimi-k2.5", - }), + patch_size: DEFAULT_PATCH_SIZE, + merge_size: DEFAULT_MERGE_SIZE, + min_pixels: DEFAULT_MIN_PIXELS, + max_pixels: DEFAULT_MAX_PIXELS, } } pub fn from_preprocessor_config(config: &PreProcessorConfig) -> Self { Self { - inner: QwenVLProcessorBase::new(QwenVLConfig { - patch_size: config.get_patch_size(DEFAULT_PATCH_SIZE), - merge_size: config.merge_size.unwrap_or(DEFAULT_MERGE_SIZE), - min_pixels: config.min_pixels.unwrap_or(DEFAULT_MIN_PIXELS), - max_pixels: config.max_pixels.unwrap_or(DEFAULT_MAX_PIXELS), - temporal_patch_size: config - .temporal_patch_size - .unwrap_or(DEFAULT_TEMPORAL_PATCH_SIZE), - mean: KIMI_K25_MEAN, - std: KIMI_K25_STD, - model_name: "kimi-k2.5", - }), + patch_size: config.get_patch_size(DEFAULT_PATCH_SIZE), + merge_size: config.merge_size.unwrap_or(DEFAULT_MERGE_SIZE), + min_pixels: config.min_pixels.unwrap_or(DEFAULT_MIN_PIXELS), + max_pixels: config.max_pixels.unwrap_or(DEFAULT_MAX_PIXELS), } } -} -impl Deref for KimiK25Processor { - type Target = QwenVLProcessorBase; + pub fn patch_size(&self) -> usize { + self.patch_size + } + + pub fn merge_size(&self) -> usize { + self.merge_size + } + + /// factor = patch_size * merge_size — dimensions must be divisible by this. + #[inline] + pub fn factor(&self) -> usize { + self.patch_size * self.merge_size + } + + /// NaViT-style smart resize: fit within min/max pixel bounds while + /// preserving aspect ratio and aligning to factor boundary. + pub fn smart_resize( + &self, + height: usize, + width: usize, + ) -> Result<(usize, usize), TransformError> { + let factor = self.factor(); + + if height < factor || width < factor { + return Err(TransformError::InvalidShape { + expected: format!("dimensions >= {factor} (patch_size * merge_size)"), + actual: vec![height, width], + }); + } + + let max_dim = height.max(width) as f64; + let min_dim = height.min(width) as f64; + if max_dim / min_dim > 200.0 { + return Err(TransformError::InvalidShape { + expected: "aspect ratio < 200:1".to_string(), + actual: vec![height, width], + }); + } + + let mut h_bar = round_half_to_even(height as f64 / factor as f64) as usize * factor; + let mut w_bar = round_half_to_even(width as f64 / factor as f64) as usize * factor; + h_bar = h_bar.max(factor); + w_bar = w_bar.max(factor); + + if h_bar * w_bar > self.max_pixels { + let beta = ((height * width) as f64 / self.max_pixels as f64).sqrt(); + h_bar = ((height as f64 / beta / factor as f64).floor() as usize) * factor; + w_bar = ((width as f64 / beta / factor as f64).floor() as usize) * factor; + h_bar = h_bar.max(factor); + w_bar = w_bar.max(factor); + } else if h_bar * w_bar < self.min_pixels { + let beta = (self.min_pixels as f64 / (height * width) as f64).sqrt(); + h_bar = ((height as f64 * beta / factor as f64).ceil() as usize) * factor; + w_bar = ((width as f64 * beta / factor as f64).ceil() as usize) * factor; + } + + Ok((h_bar, w_bar)) + } + + /// Extract [C, patch_size, patch_size] patches from a [C, H, W] tensor. + /// + /// Returns flattened `Vec` with layout `[num_patches, C * patch_size * patch_size]`. + /// The engine reconstructs this as `[num_patches, C, patch_size, patch_size]`. + fn extract_patches( + tensor: &Array3, + patch_size: usize, + ) -> Result, TransformError> { + let channels = tensor.shape()[0]; + let height = tensor.shape()[1]; + let width = tensor.shape()[2]; + + if !height.is_multiple_of(patch_size) || !width.is_multiple_of(patch_size) { + return Err(TransformError::ShapeError(format!( + "Image dimensions [{height}, {width}] not divisible by patch_size {patch_size}" + ))); + } + + let grid_h = height / patch_size; + let grid_w = width / patch_size; + let num_patches = grid_h * grid_w; + let patch_features = channels * patch_size * patch_size; + + let mut patches = Vec::with_capacity(num_patches * patch_features); + + for gh in 0..grid_h { + for gw in 0..grid_w { + let h_start = gh * patch_size; + let w_start = gw * patch_size; + for c in 0..channels { + for ph in 0..patch_size { + for pw in 0..patch_size { + patches.push(tensor[[c, h_start + ph, w_start + pw]]); + } + } + } + } + } - fn deref(&self) -> &Self::Target { - &self.inner + Ok(patches) } } impl ImagePreProcessor for KimiK25Processor { fn default_mean(&self) -> [f64; 3] { - self.inner.default_mean() + KIMI_K25_MEAN } fn default_std(&self) -> [f64; 3] { - self.inner.default_std() + KIMI_K25_STD } fn preprocess( @@ -122,26 +205,97 @@ impl ImagePreProcessor for KimiK25Processor { images: &[DynamicImage], config: &PreProcessorConfig, ) -> Result { - let mut result = self.inner.preprocess(images, config)?; + if images.is_empty() { + return Err(TransformError::EmptyBatch); + } - // Kimi-K2.5 engine expects "grid_thws" instead of Qwen-VL's "image_grid_thw" - if let Some(val) = result.model_specific.remove("image_grid_thw") { - result.model_specific.insert("grid_thws".to_string(), val); + let image_sizes: Vec<(u32, u32)> = images.iter().map(|img| img.dimensions()).collect(); + let mean = config.get_image_mean(); + let std = config.get_image_std(); + let filter = pil_to_filter(config.resampling); + + // Kimi patches are [C, patch_size, patch_size] — no temporal flattening + let patch_features = 3 * self.patch_size * self.patch_size; + + let mut all_patches: Vec = Vec::new(); + let mut patches_per_image: Vec = Vec::with_capacity(images.len()); + let mut grid_thw_data = Vec::with_capacity(images.len() * 3); + let mut num_img_tokens = Vec::with_capacity(images.len()); + + for image in images { + let (w, h) = image.dimensions(); + let (target_h, target_w) = self.smart_resize(h as usize, w as usize)?; + + let resized = if config.do_resize.unwrap_or(true) { + resize(image, target_w as u32, target_h as u32, filter) + } else { + image.clone() + }; + + let mut tensor = to_tensor(&resized); + if config.do_normalize.unwrap_or(true) { + normalize(&mut tensor, &mean, &std); + } + + let grid_h = target_h / self.patch_size; + let grid_w = target_w / self.patch_size; + let grid_t = 1usize; // images always have temporal=1 + + grid_thw_data.push(grid_t as i64); + grid_thw_data.push(grid_h as i64); + grid_thw_data.push(grid_w as i64); + + let num_patches = grid_h * grid_w; + // Token count after merge: (grid_t * grid_h * grid_w) / merge_size² + let tokens = (grid_t * grid_h * grid_w) / (self.merge_size * self.merge_size); + num_img_tokens.push(tokens); + + let patches = Self::extract_patches(&tensor, self.patch_size)?; + all_patches.extend(patches); + patches_per_image.push(num_patches as i64); } + let total_patches: usize = patches_per_image.iter().map(|&n| n as usize).sum(); + let pixel_values = Array2::from_shape_vec((total_patches, patch_features), all_patches) + .map_err(|e| { + TransformError::ShapeError(format!( + "Failed to create pixel_values [{total_patches}, {patch_features}]: {e}" + )) + })?; + + let result = + PreprocessedImages::new_dynamic(pixel_values.into_dyn(), num_img_tokens, image_sizes) + .with_extra( + "grid_thws", + ModelSpecificValue::int_2d(grid_thw_data, images.len(), 3), + ) + .with_extra( + "patches_per_image", + ModelSpecificValue::int_1d(patches_per_image), + ); + Ok(result) } - fn calculate_num_tokens(&self, width: u32, height: u32, config: &PreProcessorConfig) -> usize { - self.inner.calculate_num_tokens(width, height, config) + fn calculate_num_tokens(&self, width: u32, height: u32, _config: &PreProcessorConfig) -> usize { + let (new_height, new_width) = match self.smart_resize(height as usize, width as usize) { + Ok((h, w)) => (h, w), + Err(_) => { + let factor = self.factor(); + (factor, factor) + } + }; + let grid_h = new_height / self.patch_size; + let grid_w = new_width / self.patch_size; + (grid_h * grid_w) / (self.merge_size * self.merge_size) } fn model_name(&self) -> &'static str { - self.inner.model_name() + "kimi-k2.5" } - fn get_processed_size(&self, config: &PreProcessorConfig) -> Option<(u32, u32)> { - self.inner.get_processed_size(config) + fn get_processed_size(&self, _config: &PreProcessorConfig) -> Option<(u32, u32)> { + None // dynamic sizing } } @@ -150,7 +304,7 @@ mod tests { use image::{Rgb, RgbImage}; use super::*; - use crate::vision::image_processor::ModelSpecificValue; + use crate::vision::preprocessor_config::PatchSize; fn create_test_image(width: u32, height: u32, color: Rgb) -> DynamicImage { DynamicImage::from(RgbImage::from_pixel(width, height, color)) @@ -161,9 +315,7 @@ mod tests { let processor = KimiK25Processor::new(); assert_eq!(processor.patch_size(), 14); assert_eq!(processor.merge_size(), 2); - assert_eq!(processor.min_pixels(), DEFAULT_MIN_PIXELS); - assert_eq!(processor.max_pixels(), DEFAULT_MAX_PIXELS); - assert_eq!(processor.get_factor(), 28); // 14 * 2 + assert_eq!(processor.factor(), 28); } #[test] @@ -180,7 +332,23 @@ mod tests { } #[test] - fn test_kimi_k25_preprocess() { + fn test_smart_resize_within_bounds() { + let processor = KimiK25Processor::new(); + let (h, w) = processor.smart_resize(500, 500).unwrap(); + assert_eq!(h % 28, 0); + assert_eq!(w % 28, 0); + assert!(h * w >= processor.min_pixels); + assert!(h * w <= processor.max_pixels); + } + + #[test] + fn test_smart_resize_extreme_aspect_ratio() { + let processor = KimiK25Processor::new(); + assert!(processor.smart_resize(100, 30000).is_err()); + } + + #[test] + fn test_preprocess_patch_shape() { let processor = KimiK25Processor::new(); let config = PreProcessorConfig { do_resize: Some(true), @@ -193,19 +361,18 @@ mod tests { let image = create_test_image(600, 400, Rgb([128, 128, 128])); let result = processor.preprocess(&[image], &config).unwrap(); - // pixel_values is patchified: [total_patches, patch_features] + // Kimi: [total_patches, 3 * 14 * 14] = [total_patches, 588] assert_eq!(result.pixel_values.ndim(), 2); + assert_eq!(result.pixel_values.shape()[1], 3 * 14 * 14); assert!(result.pixel_values.shape()[0] > 0); - // Check grid_thws and patches_per_image are present assert!(result.model_specific.contains_key("grid_thws")); assert!(result.model_specific.contains_key("patches_per_image")); - assert!(result.num_img_tokens[0] > 0); } #[test] - fn test_kimi_k25_preprocess_multiple() { + fn test_preprocess_multiple_images() { let processor = KimiK25Processor::new(); let config = PreProcessorConfig::default(); @@ -218,7 +385,7 @@ mod tests { assert_eq!(result.image_sizes.len(), 2); assert_eq!(result.num_img_tokens.len(), 2); - assert_eq!(result.pixel_values.ndim(), 2); + assert_eq!(result.pixel_values.shape()[1], 588); if let Some(ModelSpecificValue::IntTensor { data, shape }) = result.model_specific.get("grid_thws") @@ -228,5 +395,41 @@ mod tests { } else { panic!("Expected grid_thws to be IntTensor"); } + + // patches_per_image sum matches total_patches + if let Some(ModelSpecificValue::IntTensor { data, .. }) = + result.model_specific.get("patches_per_image") + { + let total: i64 = data.iter().sum(); + assert_eq!(total as usize, result.pixel_values.shape()[0]); + } + } + + #[test] + fn test_calculate_num_tokens() { + let processor = KimiK25Processor::new(); + let config = PreProcessorConfig::default(); + // 448x448 → grid 32x32 → 1024 patches → 1024/4 = 256 tokens + let tokens = processor.calculate_num_tokens(448, 448, &config); + assert_eq!(tokens, 256); + } + + #[test] + fn test_from_preprocessor_config() { + let config = PreProcessorConfig { + patch_size: Some(PatchSize { + height: Some(14), + width: Some(14), + }), + merge_size: Some(2), + min_pixels: Some(100000), + max_pixels: Some(500000), + ..Default::default() + }; + let processor = KimiK25Processor::from_preprocessor_config(&config); + assert_eq!(processor.patch_size(), 14); + assert_eq!(processor.merge_size(), 2); + assert_eq!(processor.min_pixels, 100000); + assert_eq!(processor.max_pixels, 500000); } } From ac10e7c53ea822671ba4f5fe51c9eea4fe5cdef8 Mon Sep 17 00:00:00 2001 From: Kangyan Zhou Date: Tue, 31 Mar 2026 20:51:07 -0700 Subject: [PATCH 07/23] fix(multimodal): serialize Kimi pixel_values as 4D [N, 3, 14, 14] The engine's PatchEmbed3d Conv2d expects 4D input [N, C, H, W] but the gateway was serializing pixel_values as 2D [N, C*patch_size*patch_size]. Store as ndarray::Array4 so the proto shape field is [N, 3, 14, 14], which the engine reconstructs correctly for Conv2d. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../src/vision/processors/kimi_k25.rs | 34 +++++++++++-------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/crates/multimodal/src/vision/processors/kimi_k25.rs b/crates/multimodal/src/vision/processors/kimi_k25.rs index dcf5c2fd99..6c54e0bd70 100644 --- a/crates/multimodal/src/vision/processors/kimi_k25.rs +++ b/crates/multimodal/src/vision/processors/kimi_k25.rs @@ -30,7 +30,7 @@ //! - max_pixels: 3,211,264 (from in_patch_limit=16384) use image::{DynamicImage, GenericImageView}; -use ndarray::{Array2, Array3}; +use ndarray::Array3; use crate::vision::{ image_processor::{ImagePreProcessor, ModelSpecificValue, PreprocessedImages}, @@ -214,9 +214,6 @@ impl ImagePreProcessor for KimiK25Processor { let std = config.get_image_std(); let filter = pil_to_filter(config.resampling); - // Kimi patches are [C, patch_size, patch_size] — no temporal flattening - let patch_features = 3 * self.patch_size * self.patch_size; - let mut all_patches: Vec = Vec::new(); let mut patches_per_image: Vec = Vec::with_capacity(images.len()); let mut grid_thw_data = Vec::with_capacity(images.len() * 3); @@ -256,12 +253,18 @@ impl ImagePreProcessor for KimiK25Processor { } let total_patches: usize = patches_per_image.iter().map(|&n| n as usize).sum(); - let pixel_values = Array2::from_shape_vec((total_patches, patch_features), all_patches) - .map_err(|e| { - TransformError::ShapeError(format!( - "Failed to create pixel_values [{total_patches}, {patch_features}]: {e}" - )) - })?; + // Store as 4D [total_patches, C, patch_size, patch_size] so the engine + // receives the correct shape for Conv2d input via the proto shape field. + let pixel_values = ndarray::Array4::from_shape_vec( + (total_patches, 3, self.patch_size, self.patch_size), + all_patches, + ) + .map_err(|e| { + TransformError::ShapeError(format!( + "Failed to create pixel_values [{total_patches}, 3, {}, {}]: {e}", + self.patch_size, self.patch_size + )) + })?; let result = PreprocessedImages::new_dynamic(pixel_values.into_dyn(), num_img_tokens, image_sizes) @@ -361,9 +364,11 @@ mod tests { let image = create_test_image(600, 400, Rgb([128, 128, 128])); let result = processor.preprocess(&[image], &config).unwrap(); - // Kimi: [total_patches, 3 * 14 * 14] = [total_patches, 588] - assert_eq!(result.pixel_values.ndim(), 2); - assert_eq!(result.pixel_values.shape()[1], 3 * 14 * 14); + // Kimi: [total_patches, 3, 14, 14] — 4D for Conv2d + assert_eq!(result.pixel_values.ndim(), 4); + assert_eq!(result.pixel_values.shape()[1], 3); + assert_eq!(result.pixel_values.shape()[2], 14); + assert_eq!(result.pixel_values.shape()[3], 14); assert!(result.pixel_values.shape()[0] > 0); assert!(result.model_specific.contains_key("grid_thws")); @@ -385,7 +390,8 @@ mod tests { assert_eq!(result.image_sizes.len(), 2); assert_eq!(result.num_img_tokens.len(), 2); - assert_eq!(result.pixel_values.shape()[1], 588); + assert_eq!(result.pixel_values.ndim(), 4); + assert_eq!(result.pixel_values.shape()[1], 3); if let Some(ModelSpecificValue::IntTensor { data, shape }) = result.model_specific.get("grid_thws") From c55c24f8adb7ca0361169d7b3da63f36a5c7b10f Mon Sep 17 00:00:00 2001 From: Kangyan Zhou Date: Tue, 31 Mar 2026 21:41:34 -0700 Subject: [PATCH 08/23] fix(multimodal): match HF Kimi preprocessor resize+pad pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous smart_resize (from Qwen-VL) resized images directly to factor-aligned dimensions, stretching the content. The HF Kimi preprocessor instead: 1. Computes scale capped at 1.0 (never upscales) 2. Resizes with BICUBIC interpolation 3. Zero-pads to factor-aligned dimensions This mismatch caused degraded image quality — the model was trained with zero-padded images, not stretched ones. Rewrite to match the HF navit_resize_image + resize_image pipeline exactly. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../src/vision/processors/kimi_k25.rs | 353 +++++++++--------- 1 file changed, 185 insertions(+), 168 deletions(-) diff --git a/crates/multimodal/src/vision/processors/kimi_k25.rs b/crates/multimodal/src/vision/processors/kimi_k25.rs index 6c54e0bd70..72026d71ba 100644 --- a/crates/multimodal/src/vision/processors/kimi_k25.rs +++ b/crates/multimodal/src/vision/processors/kimi_k25.rs @@ -1,33 +1,17 @@ //! Kimi-K2.5 (MoonViT) image processor. //! -//! Kimi-K2.5 uses MoonViT which expects pixel_values as -//! `[total_patches, C, patch_size, patch_size]` (4D on the engine side). -//! The model's PatchEmbed3d applies Conv2d on each patch, so patches must NOT -//! be flattened across the temporal dimension in the preprocessor. +//! Matches the HuggingFace `KimiK25VisionProcessor` preprocessing pipeline: //! -//! # Processing Pipeline +//! 1. Compute scale to fit within patch limits (never upscale) +//! 2. Resize with BICUBIC interpolation +//! 3. Zero-pad to make dimensions divisible by factor (patch_size * merge_size) +//! 4. Normalize with [0.5, 0.5, 0.5] mean/std +//! 5. Extract patches as [N, C, patch_size, patch_size] //! -//! 1. NaViT-style resize to fit within min/max pixel bounds -//! 2. Align dimensions to (patch_size * merge_size) boundary -//! 3. Convert to tensor and normalize with [0.5, 0.5, 0.5] mean/std -//! 4. Extract patches as [C, patch_size, patch_size] per spatial position -//! -//! # Token Calculation -//! -//! ```text -//! grid_t = 1 (for images) -//! grid_h = resized_height / patch_size -//! grid_w = resized_width / patch_size -//! num_tokens = (grid_t * grid_h * grid_w) / merge_size² -//! ``` -//! -//! # Parameters -//! -//! - patch_size: 14 -//! - merge_size: 2 -//! - factor: 28 (patch_size * merge_size) -//! - normalization: [0.5, 0.5, 0.5] mean/std -//! - max_pixels: 3,211,264 (from in_patch_limit=16384) +//! Key difference from Qwen-VL: Kimi resizes then zero-pads for alignment, +//! while Qwen-VL resizes directly to factor-aligned dimensions. The model was +//! trained with zero-padded images, so using the wrong resize strategy degrades +//! image quality. use image::{DynamicImage, GenericImageView}; use ndarray::Array3; @@ -35,7 +19,7 @@ use ndarray::Array3; use crate::vision::{ image_processor::{ImagePreProcessor, ModelSpecificValue, PreprocessedImages}, preprocessor_config::PreProcessorConfig, - transforms::{normalize, pil_to_filter, resize, to_tensor, TransformError}, + transforms::{normalize, to_tensor, TransformError}, }; pub const KIMI_K25_MEAN: [f64; 3] = [0.5, 0.5, 0.5]; @@ -43,25 +27,26 @@ pub const KIMI_K25_STD: [f64; 3] = [0.5, 0.5, 0.5]; pub const DEFAULT_PATCH_SIZE: usize = 14; pub const DEFAULT_MERGE_SIZE: usize = 2; -pub const DEFAULT_MIN_PIXELS: usize = 256 * 28 * 28; // 200,704 -pub const DEFAULT_MAX_PIXELS: usize = 16384 * 14 * 14; // 3,211,264 (in_patch_limit=16384) - -/// Python-compatible rounding (banker's rounding / round half to even). -#[inline] -fn round_half_to_even(x: f64) -> f64 { - let rounded = x.round(); - if (x - x.floor() - 0.5).abs() < 1e-9 && rounded as i64 % 2 != 0 { - return rounded - 1.0; - } - rounded +/// Maximum total patches before merge (from preprocessor_config.json in_patch_limit) +pub const DEFAULT_IN_PATCH_LIMIT: usize = 16384; +/// Maximum patches along one spatial dimension +pub const DEFAULT_PATCH_LIMIT_ON_ONE_SIDE: usize = 512; + +/// Kimi-K2.5 resize configuration for a single image. +struct ResizeConfig { + new_width: usize, + new_height: usize, + pad_width: usize, + pad_height: usize, + num_tokens: usize, } #[derive(Debug, Clone)] pub struct KimiK25Processor { patch_size: usize, merge_size: usize, - min_pixels: usize, - max_pixels: usize, + in_patch_limit: usize, + patch_limit_on_one_side: usize, } impl Default for KimiK25Processor { @@ -75,8 +60,8 @@ impl KimiK25Processor { Self { patch_size: DEFAULT_PATCH_SIZE, merge_size: DEFAULT_MERGE_SIZE, - min_pixels: DEFAULT_MIN_PIXELS, - max_pixels: DEFAULT_MAX_PIXELS, + in_patch_limit: DEFAULT_IN_PATCH_LIMIT, + patch_limit_on_one_side: DEFAULT_PATCH_LIMIT_ON_ONE_SIDE, } } @@ -84,8 +69,8 @@ impl KimiK25Processor { Self { patch_size: config.get_patch_size(DEFAULT_PATCH_SIZE), merge_size: config.merge_size.unwrap_or(DEFAULT_MERGE_SIZE), - min_pixels: config.min_pixels.unwrap_or(DEFAULT_MIN_PIXELS), - max_pixels: config.max_pixels.unwrap_or(DEFAULT_MAX_PIXELS), + in_patch_limit: DEFAULT_IN_PATCH_LIMIT, + patch_limit_on_one_side: DEFAULT_PATCH_LIMIT_ON_ONE_SIDE, } } @@ -97,75 +82,82 @@ impl KimiK25Processor { self.merge_size } - /// factor = patch_size * merge_size — dimensions must be divisible by this. #[inline] - pub fn factor(&self) -> usize { + fn factor(&self) -> usize { self.patch_size * self.merge_size } - /// NaViT-style smart resize: fit within min/max pixel bounds while - /// preserving aspect ratio and aligning to factor boundary. - pub fn smart_resize( - &self, - height: usize, - width: usize, - ) -> Result<(usize, usize), TransformError> { - let factor = self.factor(); - - if height < factor || width < factor { - return Err(TransformError::InvalidShape { - expected: format!("dimensions >= {factor} (patch_size * merge_size)"), - actual: vec![height, width], - }); - } + /// Compute resize dimensions and padding, matching HF `navit_resize_image`. + /// + /// Never upscales (scale capped at 1.0). Pads with zeros to align to factor. + fn compute_resize_config(&self, width: usize, height: usize) -> ResizeConfig { + let ps = self.patch_size; + let patches_w = (width / ps).max(1) as f64; + let patches_h = (height / ps).max(1) as f64; + + let s1 = (self.in_patch_limit as f64 / (patches_w * patches_h)).sqrt(); + let s2 = (self.patch_limit_on_one_side * ps) as f64 / width as f64; + let s3 = (self.patch_limit_on_one_side * ps) as f64 / height as f64; + let scale = f64::min(1.0, f64::min(s1, f64::min(s2, s3))); + + let new_w = ((width as f64 * scale) as usize).max(1); + let new_h = ((height as f64 * scale) as usize).max(1); + let new_w = new_w.min(self.patch_limit_on_one_side * ps); + let new_h = new_h.min(self.patch_limit_on_one_side * ps); - let max_dim = height.max(width) as f64; - let min_dim = height.min(width) as f64; - if max_dim / min_dim > 200.0 { - return Err(TransformError::InvalidShape { - expected: "aspect ratio < 200:1".to_string(), - actual: vec![height, width], - }); + let factor = self.factor(); + let pad_width = (factor - new_w % factor) % factor; + let pad_height = (factor - new_h % factor) % factor; + + let token_height = (new_h + pad_height) / factor; + let token_width = (new_w + pad_width) / factor; + let num_tokens = token_height * token_width; + + ResizeConfig { + new_width: new_w, + new_height: new_h, + pad_width, + pad_height, + num_tokens, } + } - let mut h_bar = round_half_to_even(height as f64 / factor as f64) as usize * factor; - let mut w_bar = round_half_to_even(width as f64 / factor as f64) as usize * factor; - h_bar = h_bar.max(factor); - w_bar = w_bar.max(factor); - - if h_bar * w_bar > self.max_pixels { - let beta = ((height * width) as f64 / self.max_pixels as f64).sqrt(); - h_bar = ((height as f64 / beta / factor as f64).floor() as usize) * factor; - w_bar = ((width as f64 / beta / factor as f64).floor() as usize) * factor; - h_bar = h_bar.max(factor); - w_bar = w_bar.max(factor); - } else if h_bar * w_bar < self.min_pixels { - let beta = (self.min_pixels as f64 / (height * width) as f64).sqrt(); - h_bar = ((height as f64 * beta / factor as f64).ceil() as usize) * factor; - w_bar = ((width as f64 * beta / factor as f64).ceil() as usize) * factor; + /// Resize image and zero-pad to factor-aligned dimensions. + /// + /// Returns a [C, H_padded, W_padded] tensor where H_padded and W_padded + /// are divisible by factor. + fn resize_and_pad(image: &DynamicImage, cfg: &ResizeConfig) -> Array3 { + let padded_h = cfg.new_height + cfg.pad_height; + let padded_w = cfg.new_width + cfg.pad_width; + + // Resize using BICUBIC + let resized = image.resize_exact( + cfg.new_width as u32, + cfg.new_height as u32, + image::imageops::FilterType::CatmullRom, + ); + + // Convert to tensor [C, H, W] with values in [0, 1] + let tensor = to_tensor(&resized); + + if cfg.pad_width == 0 && cfg.pad_height == 0 { + return tensor; } - Ok((h_bar, w_bar)) + // Zero-pad: create padded array, copy resized data into top-left + let mut padded = Array3::::zeros((3, padded_h, padded_w)); + padded + .slice_mut(ndarray::s![.., ..cfg.new_height, ..cfg.new_width]) + .assign(&tensor); + padded } /// Extract [C, patch_size, patch_size] patches from a [C, H, W] tensor. - /// - /// Returns flattened `Vec` with layout `[num_patches, C * patch_size * patch_size]`. - /// The engine reconstructs this as `[num_patches, C, patch_size, patch_size]`. - fn extract_patches( - tensor: &Array3, - patch_size: usize, - ) -> Result, TransformError> { + fn extract_patches(tensor: &Array3, patch_size: usize) -> Vec { let channels = tensor.shape()[0]; let height = tensor.shape()[1]; let width = tensor.shape()[2]; - if !height.is_multiple_of(patch_size) || !width.is_multiple_of(patch_size) { - return Err(TransformError::ShapeError(format!( - "Image dimensions [{height}, {width}] not divisible by patch_size {patch_size}" - ))); - } - let grid_h = height / patch_size; let grid_w = width / patch_size; let num_patches = grid_h * grid_w; @@ -187,7 +179,7 @@ impl KimiK25Processor { } } - Ok(patches) + patches } } @@ -212,7 +204,6 @@ impl ImagePreProcessor for KimiK25Processor { let image_sizes: Vec<(u32, u32)> = images.iter().map(|img| img.dimensions()).collect(); let mean = config.get_image_mean(); let std = config.get_image_std(); - let filter = pil_to_filter(config.resampling); let mut all_patches: Vec = Vec::new(); let mut patches_per_image: Vec = Vec::with_capacity(images.len()); @@ -221,40 +212,35 @@ impl ImagePreProcessor for KimiK25Processor { for image in images { let (w, h) = image.dimensions(); - let (target_h, target_w) = self.smart_resize(h as usize, w as usize)?; + let cfg = self.compute_resize_config(w as usize, h as usize); - let resized = if config.do_resize.unwrap_or(true) { - resize(image, target_w as u32, target_h as u32, filter) - } else { - image.clone() - }; + // Resize + zero-pad + let mut tensor = Self::resize_and_pad(image, &cfg); - let mut tensor = to_tensor(&resized); + // Normalize if config.do_normalize.unwrap_or(true) { normalize(&mut tensor, &mean, &std); } - let grid_h = target_h / self.patch_size; - let grid_w = target_w / self.patch_size; - let grid_t = 1usize; // images always have temporal=1 + let padded_h = cfg.new_height + cfg.pad_height; + let padded_w = cfg.new_width + cfg.pad_width; + let grid_h = padded_h / self.patch_size; + let grid_w = padded_w / self.patch_size; + let grid_t = 1usize; grid_thw_data.push(grid_t as i64); grid_thw_data.push(grid_h as i64); grid_thw_data.push(grid_w as i64); let num_patches = grid_h * grid_w; - // Token count after merge: (grid_t * grid_h * grid_w) / merge_size² - let tokens = (grid_t * grid_h * grid_w) / (self.merge_size * self.merge_size); - num_img_tokens.push(tokens); + num_img_tokens.push(cfg.num_tokens); - let patches = Self::extract_patches(&tensor, self.patch_size)?; + let patches = Self::extract_patches(&tensor, self.patch_size); all_patches.extend(patches); patches_per_image.push(num_patches as i64); } let total_patches: usize = patches_per_image.iter().map(|&n| n as usize).sum(); - // Store as 4D [total_patches, C, patch_size, patch_size] so the engine - // receives the correct shape for Conv2d input via the proto shape field. let pixel_values = ndarray::Array4::from_shape_vec( (total_patches, 3, self.patch_size, self.patch_size), all_patches, @@ -281,16 +267,8 @@ impl ImagePreProcessor for KimiK25Processor { } fn calculate_num_tokens(&self, width: u32, height: u32, _config: &PreProcessorConfig) -> usize { - let (new_height, new_width) = match self.smart_resize(height as usize, width as usize) { - Ok((h, w)) => (h, w), - Err(_) => { - let factor = self.factor(); - (factor, factor) - } - }; - let grid_h = new_height / self.patch_size; - let grid_w = new_width / self.patch_size; - (grid_h * grid_w) / (self.merge_size * self.merge_size) + self.compute_resize_config(width as usize, height as usize) + .num_tokens } fn model_name(&self) -> &'static str { @@ -298,7 +276,7 @@ impl ImagePreProcessor for KimiK25Processor { } fn get_processed_size(&self, _config: &PreProcessorConfig) -> Option<(u32, u32)> { - None // dynamic sizing + None } } @@ -314,47 +292,72 @@ mod tests { } #[test] - fn test_kimi_k25_processor_default() { - let processor = KimiK25Processor::new(); - assert_eq!(processor.patch_size(), 14); - assert_eq!(processor.merge_size(), 2); - assert_eq!(processor.factor(), 28); + fn test_defaults() { + let p = KimiK25Processor::new(); + assert_eq!(p.patch_size(), 14); + assert_eq!(p.merge_size(), 2); + assert_eq!(p.factor(), 28); } #[test] - fn test_default_mean_std() { - let processor = KimiK25Processor::new(); - assert_eq!(processor.default_mean(), KIMI_K25_MEAN); - assert_eq!(processor.default_std(), KIMI_K25_STD); + fn test_mean_std() { + let p = KimiK25Processor::new(); + assert_eq!(p.default_mean(), KIMI_K25_MEAN); + assert_eq!(p.default_std(), KIMI_K25_STD); } #[test] fn test_model_name() { - let processor = KimiK25Processor::new(); - assert_eq!(processor.model_name(), "kimi-k2.5"); + assert_eq!(KimiK25Processor::new().model_name(), "kimi-k2.5"); + } + + #[test] + fn test_resize_config_no_upscale() { + let p = KimiK25Processor::new(); + // Small image should NOT be upscaled (scale capped at 1.0) + let cfg = p.compute_resize_config(100, 100); + assert!(cfg.new_width <= 100); + assert!(cfg.new_height <= 100); + // Padded dimensions must be factor-aligned + assert_eq!((cfg.new_height + cfg.pad_height) % 28, 0); + assert_eq!((cfg.new_width + cfg.pad_width) % 28, 0); } #[test] - fn test_smart_resize_within_bounds() { - let processor = KimiK25Processor::new(); - let (h, w) = processor.smart_resize(500, 500).unwrap(); - assert_eq!(h % 28, 0); - assert_eq!(w % 28, 0); - assert!(h * w >= processor.min_pixels); - assert!(h * w <= processor.max_pixels); + fn test_resize_config_large_image_downscaled() { + let p = KimiK25Processor::new(); + // Large image should be downscaled + let cfg = p.compute_resize_config(4000, 3000); + // Resized dimensions should be smaller than original + assert!(cfg.new_width < 4000); + assert!(cfg.new_height < 3000); + // Per-side patch limit must be respected (HF assertion) + let padded_h = cfg.new_height + cfg.pad_height; + let padded_w = cfg.new_width + cfg.pad_width; + assert!(padded_h / 14 <= DEFAULT_PATCH_LIMIT_ON_ONE_SIDE * 2); + assert!(padded_w / 14 <= DEFAULT_PATCH_LIMIT_ON_ONE_SIDE * 2); } #[test] - fn test_smart_resize_extreme_aspect_ratio() { - let processor = KimiK25Processor::new(); - assert!(processor.smart_resize(100, 30000).is_err()); + fn test_resize_config_matches_hf_reference() { + let p = KimiK25Processor::new(); + // 600x400 image: scale=1.0 (small enough), resize to 600x400, + // pad to (600+4=) → let's compute: + // factor=28, 400 % 28 = 400 - 14*28 = 400-392 = 8, pad_h = 28-8 = 20 + // 600 % 28 = 600 - 21*28 = 600-588 = 12, pad_w = 28-12 = 16 + let cfg = p.compute_resize_config(600, 400); + assert_eq!(cfg.new_width, 600); + assert_eq!(cfg.new_height, 400); + assert_eq!(cfg.pad_height, 20); + assert_eq!(cfg.pad_width, 16); + // Padded: 420 x 616, grid: 30 x 44, tokens: (30*44)/(2*2) = 330 + assert_eq!(cfg.num_tokens, 330); } #[test] - fn test_preprocess_patch_shape() { - let processor = KimiK25Processor::new(); + fn test_preprocess_4d_output() { + let p = KimiK25Processor::new(); let config = PreProcessorConfig { - do_resize: Some(true), do_normalize: Some(true), image_mean: Some(KIMI_K25_MEAN.to_vec()), image_std: Some(KIMI_K25_STD.to_vec()), @@ -362,14 +365,13 @@ mod tests { }; let image = create_test_image(600, 400, Rgb([128, 128, 128])); - let result = processor.preprocess(&[image], &config).unwrap(); + let result = p.preprocess(&[image], &config).unwrap(); - // Kimi: [total_patches, 3, 14, 14] — 4D for Conv2d + // 4D output: [total_patches, 3, 14, 14] assert_eq!(result.pixel_values.ndim(), 4); assert_eq!(result.pixel_values.shape()[1], 3); assert_eq!(result.pixel_values.shape()[2], 14); assert_eq!(result.pixel_values.shape()[3], 14); - assert!(result.pixel_values.shape()[0] > 0); assert!(result.model_specific.contains_key("grid_thws")); assert!(result.model_specific.contains_key("patches_per_image")); @@ -378,15 +380,14 @@ mod tests { #[test] fn test_preprocess_multiple_images() { - let processor = KimiK25Processor::new(); + let p = KimiK25Processor::new(); let config = PreProcessorConfig::default(); - let images = vec![ create_test_image(600, 400, Rgb([100, 100, 100])), create_test_image(400, 600, Rgb([150, 150, 150])), ]; - let result = processor.preprocess(&images, &config).unwrap(); + let result = p.preprocess(&images, &config).unwrap(); assert_eq!(result.image_sizes.len(), 2); assert_eq!(result.num_img_tokens.len(), 2); @@ -402,7 +403,6 @@ mod tests { panic!("Expected grid_thws to be IntTensor"); } - // patches_per_image sum matches total_patches if let Some(ModelSpecificValue::IntTensor { data, .. }) = result.model_specific.get("patches_per_image") { @@ -413,11 +413,10 @@ mod tests { #[test] fn test_calculate_num_tokens() { - let processor = KimiK25Processor::new(); + let p = KimiK25Processor::new(); let config = PreProcessorConfig::default(); - // 448x448 → grid 32x32 → 1024 patches → 1024/4 = 256 tokens - let tokens = processor.calculate_num_tokens(448, 448, &config); - assert_eq!(tokens, 256); + let tokens = p.calculate_num_tokens(600, 400, &config); + assert_eq!(tokens, 330); } #[test] @@ -428,14 +427,32 @@ mod tests { width: Some(14), }), merge_size: Some(2), - min_pixels: Some(100000), - max_pixels: Some(500000), ..Default::default() }; - let processor = KimiK25Processor::from_preprocessor_config(&config); - assert_eq!(processor.patch_size(), 14); - assert_eq!(processor.merge_size(), 2); - assert_eq!(processor.min_pixels, 100000); - assert_eq!(processor.max_pixels, 500000); + let p = KimiK25Processor::from_preprocessor_config(&config); + assert_eq!(p.patch_size(), 14); + assert_eq!(p.merge_size(), 2); + } + + #[test] + fn test_zero_padding_applied() { + let p = KimiK25Processor::new(); + let config = PreProcessorConfig { + do_normalize: Some(false), // skip normalization to check raw values + ..Default::default() + }; + + // 100x100 image with white pixels + let image = create_test_image(100, 100, Rgb([255, 255, 255])); + let result = p.preprocess(&[image], &config).unwrap(); + + // The padded region should contain zeros (from zero-padding) + let flat = result.pixel_values_flat(); + let has_zeros = flat.contains(&0.0); + assert!(has_zeros, "Expected zero-padded regions in output"); + + // The image region should contain non-zero values (white = 1.0) + let has_ones = flat.iter().any(|&v| (v - 1.0).abs() < 1e-6); + assert!(has_ones, "Expected non-zero image values in output"); } } From 50fde3b60a08997527d100db548083fa2542c319 Mon Sep 17 00:00:00 2001 From: Kangyan Zhou Date: Tue, 31 Mar 2026 22:11:39 -0700 Subject: [PATCH 09/23] fix(multimodal): use download_tokenizer_from_hf for config resolution download_files_from_hf was silently failing in production (likely hf-hub crate issue). Switch to download_tokenizer_from_hf which already works for tokenizer loading and returns the HF cache directory containing config.json and preprocessor_config.json. Co-Authored-By: Claude Opus 4.6 (1M context) --- model_gateway/src/routers/grpc/multimodal.rs | 46 ++++---------------- 1 file changed, 8 insertions(+), 38 deletions(-) diff --git a/model_gateway/src/routers/grpc/multimodal.rs b/model_gateway/src/routers/grpc/multimodal.rs index a73b24d433..584302f033 100644 --- a/model_gateway/src/routers/grpc/multimodal.rs +++ b/model_gateway/src/routers/grpc/multimodal.rs @@ -147,47 +147,17 @@ impl MultimodalComponents { async fn load_configs_from_hf( model_id: &str, ) -> Result<(serde_json::Value, PreProcessorConfig)> { - let files = llm_tokenizer::hub::download_files_from_hf( - model_id, - &["config.json", "preprocessor_config.json"], - ) - .await - .with_context(|| { - format!("Failed to download config files from HuggingFace for model: {model_id}") - })?; - - let config_path = files.get("config.json").ok_or_else(|| { - anyhow::anyhow!("config.json not found in HuggingFace repo: {model_id}") - })?; - let config: serde_json::Value = std::fs::read_to_string(config_path) - .with_context(|| { - format!( - "Failed to read downloaded config.json at {}", - config_path.display() - ) - }) - .and_then(|s| { - serde_json::from_str(&s) - .with_context(|| format!("Failed to parse config.json for model: {model_id}")) - })?; - - let pp_config_path = files.get("preprocessor_config.json").ok_or_else(|| { - anyhow::anyhow!("preprocessor_config.json not found in HuggingFace repo: {model_id}") - })?; - let preprocessor_config = std::fs::read_to_string(pp_config_path) + // Reuse the tokenizer download function which downloads tokenizer files + // AND config files to the HF cache, returning the cache directory path. + // If files were already downloaded (e.g., during tokenizer init), this + // returns the cached path without re-downloading. + let cache_dir = llm_tokenizer::hub::download_tokenizer_from_hf(model_id) + .await .with_context(|| { - format!( - "Failed to read downloaded preprocessor_config.json at {}", - pp_config_path.display() - ) - }) - .and_then(|s| { - PreProcessorConfig::from_json(&s).with_context(|| { - format!("Failed to parse preprocessor_config.json for model: {model_id}") - }) + format!("Failed to resolve HuggingFace cache for model: {model_id}") })?; - Ok((config, preprocessor_config)) + Self::load_configs_from_disk(&cache_dir) } } From 2a6d58a91b68e8862cc91b6da52c60d032381e03 Mon Sep 17 00:00:00 2001 From: Kangyan Zhou Date: Tue, 31 Mar 2026 22:38:44 -0700 Subject: [PATCH 10/23] fix(multimodal): add download_model_configs_from_hf for config.json download_tokenizer_from_hf only downloads tokenizer files (filtered by is_tokenizer_file), not config.json or preprocessor_config.json. Add a dedicated download_model_configs_from_hf that fetches these two files on the first multimodal request. Co-Authored-By: Claude Opus 4.6 (1M context) --- crates/tokenizer/src/hub.rs | 37 +++++++++----------- model_gateway/src/routers/grpc/multimodal.rs | 8 ++--- 2 files changed, 18 insertions(+), 27 deletions(-) diff --git a/crates/tokenizer/src/hub.rs b/crates/tokenizer/src/hub.rs index c8251bb1c7..237eec8c95 100644 --- a/crates/tokenizer/src/hub.rs +++ b/crates/tokenizer/src/hub.rs @@ -1,7 +1,4 @@ -use std::{ - collections::HashMap, - path::{Path, PathBuf}, -}; +use std::path::{Path, PathBuf}; use hf_hub::api::tokio::{Api, ApiBuilder}; @@ -66,35 +63,33 @@ fn is_chat_template_file(filename: &str) -> bool { || filename == "chat_template.json" // JSON file containing Jinja template } -/// Download specific files from a HuggingFace model repo. +/// Download model config files (config.json, preprocessor_config.json) from HuggingFace. /// -/// Returns a map of filename → local cache path for each successfully downloaded file. -/// Files that don't exist in the repo are silently skipped. -pub async fn download_files_from_hf( - model_id: &str, - filenames: &[&str], -) -> anyhow::Result> { +/// Returns the cache directory containing the downloaded files. +/// Uses the same HF Hub cache as tokenizer downloads. +pub async fn download_model_configs_from_hf(model_id: &str) -> anyhow::Result { let api = build_api()?; let repo = api.model(model_id.to_string()); - let mut results = HashMap::new(); - for &filename in filenames { + let config_files = ["config.json", "preprocessor_config.json"]; + let mut cache_dir = None; + + for filename in &config_files { match repo.get(filename).await { Ok(path) => { - results.insert(filename.to_string(), path); + if cache_dir.is_none() { + cache_dir = path.parent().map(|p| p.to_path_buf()); + } } Err(e) => { - tracing::warn!( - filename = filename, - model_id = model_id, - error = %e, - "Failed to download file from HuggingFace, skipping" - ); + return Err(anyhow::anyhow!( + "Failed to download '{filename}' from model '{model_id}': {e}" + )); } } } - Ok(results) + cache_dir.ok_or_else(|| anyhow::anyhow!("No config files downloaded for model '{model_id}'")) } /// Attempt to download tokenizer files from Hugging Face diff --git a/model_gateway/src/routers/grpc/multimodal.rs b/model_gateway/src/routers/grpc/multimodal.rs index 584302f033..7f190b7c77 100644 --- a/model_gateway/src/routers/grpc/multimodal.rs +++ b/model_gateway/src/routers/grpc/multimodal.rs @@ -147,14 +147,10 @@ impl MultimodalComponents { async fn load_configs_from_hf( model_id: &str, ) -> Result<(serde_json::Value, PreProcessorConfig)> { - // Reuse the tokenizer download function which downloads tokenizer files - // AND config files to the HF cache, returning the cache directory path. - // If files were already downloaded (e.g., during tokenizer init), this - // returns the cached path without re-downloading. - let cache_dir = llm_tokenizer::hub::download_tokenizer_from_hf(model_id) + let cache_dir = llm_tokenizer::hub::download_model_configs_from_hf(model_id) .await .with_context(|| { - format!("Failed to resolve HuggingFace cache for model: {model_id}") + format!("Failed to download config files from HuggingFace for model: {model_id}") })?; Self::load_configs_from_disk(&cache_dir) From 1252b938d26e5b331412883d6479dbe6dd0b88f3 Mon Sep 17 00:00:00 2001 From: Kangyan Zhou Date: Tue, 31 Mar 2026 23:28:11 -0700 Subject: [PATCH 11/23] debug(multimodal): add info logging for image preprocessing pipeline Add detailed logging at key points: - Image dimensions, color type, raw bytes size after fetch - Pixel values shape, token counts, first/last pixels, min/max - Serialized pixel_values bytes and shape - Token expansion details (search_token_id, placeholders, offsets) Also use download_model_configs_from_hf and remove dead code. Co-Authored-By: Claude Opus 4.6 (1M context) --- model_gateway/src/routers/grpc/multimodal.rs | 43 ++++++++++++++++---- 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/model_gateway/src/routers/grpc/multimodal.rs b/model_gateway/src/routers/grpc/multimodal.rs index 7f190b7c77..bbc2a1ac59 100644 --- a/model_gateway/src/routers/grpc/multimodal.rs +++ b/model_gateway/src/routers/grpc/multimodal.rs @@ -25,7 +25,7 @@ use openai_protocol::{ common::ContentPart, messages::{ImageSource, InputContent, InputContentBlock, InputMessage, Role}, }; -use tracing::{debug, warn}; +use tracing::{info, warn}; use crate::routers::grpc::{ client::GrpcClient, @@ -412,10 +412,19 @@ async fn process_multimodal_parts( )); } - debug!( - image_count = images.len(), - "Fetched images for multimodal processing" - ); + // Log image details at info level for debugging multimodal pipeline + for (i, img) in images.iter().enumerate() { + let (w, h) = (img.image.width(), img.image.height()); + let color_type = format!("{:?}", img.image.color()); + info!( + index = i, + width = w, + height = h, + color_type = %color_type, + raw_bytes_len = img.raw_bytes.len(), + "Fetched image for multimodal processing" + ); + } // Step 2: Resolve model spec and preprocess images let model_config = components @@ -444,11 +453,25 @@ async fn process_multimodal_parts( .preprocess(&dynamic_images, &model_config.preprocessor_config) .map_err(|e| anyhow::anyhow!("Image preprocessing failed: {e}"))?; - debug!( + info!( num_images = preprocessed.num_img_tokens.len(), total_tokens = preprocessed.num_img_tokens.iter().sum::(), + pixel_values_shape = ?preprocessed.pixel_values.shape(), + num_img_tokens = ?preprocessed.num_img_tokens, + model_specific_keys = ?preprocessed.model_specific.keys().collect::>(), "Image preprocessing complete" ); + // Log first few pixel values for debugging + let flat = preprocessed.pixel_values_flat(); + if flat.len() >= 10 { + info!( + first_10_pixels = ?&flat[..10], + last_10_pixels = ?&flat[flat.len()-10..], + pixel_min = flat.iter().copied().fold(f32::INFINITY, f32::min), + pixel_max = flat.iter().copied().fold(f32::NEG_INFINITY, f32::max), + "Pixel values sample" + ); + } // Step 3: Compute prompt replacements and expand tokens. let prompt_replacements = spec @@ -475,12 +498,13 @@ async fn process_multimodal_parts( &prompt_replacements, ); - debug!( + info!( original_len = token_ids.len(), expanded_len = expanded.token_ids.len(), placeholder_count = expanded.placeholders.len(), ?search_token_id, ?im_token_id, + placeholders = ?expanded.placeholders.iter().map(|p| (p.offset, p.length)).collect::>(), "Token expansion complete" ); @@ -612,6 +636,11 @@ pub(crate) fn assemble_multimodal_data( fn assemble_sglang(intermediate: MultimodalIntermediate) -> SglangMultimodalData { let (pixel_values, pixel_values_shape) = serialize_pixel_values(&intermediate.preprocessed); + info!( + pixel_values_bytes = pixel_values.len(), + pixel_values_shape = ?pixel_values_shape, + "Serialized pixel_values for sglang" + ); let model_specific_tensors = serialize_model_specific(intermediate.preprocessed.model_specific); let image_data = intermediate .images From ac570676ad96a4ad64e798c07ff71f1c37e79fa6 Mon Sep 17 00:00:00 2001 From: Kangyan Zhou Date: Tue, 31 Mar 2026 23:37:32 -0700 Subject: [PATCH 12/23] fix(multimodal): use Kimi normalization, not CLIP fallback KimiK25Processor::preprocess() was reading mean/std from PreProcessorConfig which falls back to CLIP values when the config can't be parsed (Kimi's preprocessor_config.json nests values under media_proc_cfg). This caused images to be normalized with CLIP mean=[0.48,0.46,0.41] std=[0.27,0.26,0.28] instead of Kimi's mean=[0.5,0.5,0.5] std=[0.5,0.5,0.5], producing wrong pixel values that made the model misinterpret images entirely. Use self.default_mean()/default_std() which are hardcoded to the correct Kimi values. Co-Authored-By: Claude Opus 4.6 (1M context) --- crates/multimodal/src/vision/processors/kimi_k25.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/multimodal/src/vision/processors/kimi_k25.rs b/crates/multimodal/src/vision/processors/kimi_k25.rs index 72026d71ba..d205620887 100644 --- a/crates/multimodal/src/vision/processors/kimi_k25.rs +++ b/crates/multimodal/src/vision/processors/kimi_k25.rs @@ -202,8 +202,10 @@ impl ImagePreProcessor for KimiK25Processor { } let image_sizes: Vec<(u32, u32)> = images.iter().map(|img| img.dimensions()).collect(); - let mean = config.get_image_mean(); - let std = config.get_image_std(); + // Always use Kimi-K2.5's own normalization values, not the preprocessor config's + // fallback (which defaults to CLIP mean/std when the config can't be parsed). + let mean = self.default_mean(); + let std = self.default_std(); let mut all_patches: Vec = Vec::new(); let mut patches_per_image: Vec = Vec::with_capacity(images.len()); From e7e15b4df1f1363af1d4088c3aa92bed8f6eb13e Mon Sep 17 00:00:00 2001 From: Kangyan Zhou Date: Tue, 31 Mar 2026 23:48:49 -0700 Subject: [PATCH 13/23] fix(multimodal): parse Kimi nested media_proc_cfg in preprocessor config Instead of hardcoding normalization values, parse the nested media_proc_cfg structure in Kimi's preprocessor_config.json to extract image_mean, image_std, patch_size, and merge_kernel_size. This ensures the correct values are used regardless of how the config is structured. The previous fix hardcoded [0.5,0.5,0.5] in the processor, which worked but would break if values changed. Now from_json() checks for media_proc_cfg when top-level fields are missing. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../src/vision/preprocessor_config.rs | 68 ++++++++++++++++++- .../src/vision/processors/kimi_k25.rs | 6 +- 2 files changed, 69 insertions(+), 5 deletions(-) diff --git a/crates/multimodal/src/vision/preprocessor_config.rs b/crates/multimodal/src/vision/preprocessor_config.rs index d237ebf216..6f828cd6a7 100644 --- a/crates/multimodal/src/vision/preprocessor_config.rs +++ b/crates/multimodal/src/vision/preprocessor_config.rs @@ -230,8 +230,43 @@ pub struct PreProcessorConfig { impl PreProcessorConfig { /// Parse from JSON string. + /// + /// Handles both standard HuggingFace format (top-level fields) and Kimi-K2.5's + /// nested format where values are under `media_proc_cfg`. pub fn from_json(json: &str) -> Result { - serde_json::from_str(json) + let mut config: Self = serde_json::from_str(json)?; + // If top-level image_mean/std are missing, try to extract from media_proc_cfg + if config.image_mean.is_none() || config.image_std.is_none() { + if let Ok(raw) = serde_json::from_str::(json) { + if let Some(media_cfg) = raw.get("media_proc_cfg") { + if config.image_mean.is_none() { + config.image_mean = media_cfg + .get("image_mean") + .and_then(|v| serde_json::from_value(v.clone()).ok()); + } + if config.image_std.is_none() { + config.image_std = media_cfg + .get("image_std") + .and_then(|v| serde_json::from_value(v.clone()).ok()); + } + if config.patch_size.is_none() { + config.patch_size = media_cfg.get("patch_size").and_then(|v| { + v.as_u64().map(|ps| PatchSize { + height: Some(ps as u32), + width: Some(ps as u32), + }) + }); + } + if config.merge_size.is_none() { + config.merge_size = media_cfg + .get("merge_kernel_size") + .and_then(|v| v.as_u64()) + .map(|v| v as usize); + } + } + } + } + Ok(config) } /// Parse from JSON value. @@ -467,4 +502,35 @@ mod tests { Some(&"bar".to_string()) ); } + + #[test] + fn test_parse_kimi_nested_media_proc_cfg() { + let json = r#"{ + "auto_map": { + "AutoProcessor": "kimi_k25_processor.KimiK25Processor" + }, + "media_proc_cfg": { + "in_patch_limit": 16384, + "patch_size": 14, + "image_mean": [0.5, 0.5, 0.5], + "image_std": [0.5, 0.5, 0.5], + "merge_kernel_size": 2, + "patch_limit_on_one_side": 512 + } + }"#; + + let config = PreProcessorConfig::from_json(json).unwrap(); + + // image_mean/std should be extracted from media_proc_cfg + let mean = config.get_image_mean(); + assert!((mean[0] - 0.5).abs() < 1e-6); + assert!((mean[1] - 0.5).abs() < 1e-6); + assert!((mean[2] - 0.5).abs() < 1e-6); + + let std = config.get_image_std(); + assert!((std[0] - 0.5).abs() < 1e-6); + + assert_eq!(config.get_patch_size(0), 14); + assert_eq!(config.merge_size, Some(2)); + } } diff --git a/crates/multimodal/src/vision/processors/kimi_k25.rs b/crates/multimodal/src/vision/processors/kimi_k25.rs index d205620887..72026d71ba 100644 --- a/crates/multimodal/src/vision/processors/kimi_k25.rs +++ b/crates/multimodal/src/vision/processors/kimi_k25.rs @@ -202,10 +202,8 @@ impl ImagePreProcessor for KimiK25Processor { } let image_sizes: Vec<(u32, u32)> = images.iter().map(|img| img.dimensions()).collect(); - // Always use Kimi-K2.5's own normalization values, not the preprocessor config's - // fallback (which defaults to CLIP mean/std when the config can't be parsed). - let mean = self.default_mean(); - let std = self.default_std(); + let mean = config.get_image_mean(); + let std = config.get_image_std(); let mut all_patches: Vec = Vec::new(); let mut patches_per_image: Vec = Vec::with_capacity(images.len()); From 4157f79d2460a82294bece99dc9b14ec081d4583 Mon Sep 17 00:00:00 2001 From: Kangyan Zhou Date: Tue, 31 Mar 2026 23:58:42 -0700 Subject: [PATCH 14/23] chore(multimodal): remove debug logging from image preprocessing Revert info-level diagnostic logging back to debug level now that the normalization root cause has been identified and fixed. Co-Authored-By: Claude Opus 4.6 (1M context) --- model_gateway/src/routers/grpc/multimodal.rs | 43 ++++---------------- 1 file changed, 7 insertions(+), 36 deletions(-) diff --git a/model_gateway/src/routers/grpc/multimodal.rs b/model_gateway/src/routers/grpc/multimodal.rs index bbc2a1ac59..7f190b7c77 100644 --- a/model_gateway/src/routers/grpc/multimodal.rs +++ b/model_gateway/src/routers/grpc/multimodal.rs @@ -25,7 +25,7 @@ use openai_protocol::{ common::ContentPart, messages::{ImageSource, InputContent, InputContentBlock, InputMessage, Role}, }; -use tracing::{info, warn}; +use tracing::{debug, warn}; use crate::routers::grpc::{ client::GrpcClient, @@ -412,19 +412,10 @@ async fn process_multimodal_parts( )); } - // Log image details at info level for debugging multimodal pipeline - for (i, img) in images.iter().enumerate() { - let (w, h) = (img.image.width(), img.image.height()); - let color_type = format!("{:?}", img.image.color()); - info!( - index = i, - width = w, - height = h, - color_type = %color_type, - raw_bytes_len = img.raw_bytes.len(), - "Fetched image for multimodal processing" - ); - } + debug!( + image_count = images.len(), + "Fetched images for multimodal processing" + ); // Step 2: Resolve model spec and preprocess images let model_config = components @@ -453,25 +444,11 @@ async fn process_multimodal_parts( .preprocess(&dynamic_images, &model_config.preprocessor_config) .map_err(|e| anyhow::anyhow!("Image preprocessing failed: {e}"))?; - info!( + debug!( num_images = preprocessed.num_img_tokens.len(), total_tokens = preprocessed.num_img_tokens.iter().sum::(), - pixel_values_shape = ?preprocessed.pixel_values.shape(), - num_img_tokens = ?preprocessed.num_img_tokens, - model_specific_keys = ?preprocessed.model_specific.keys().collect::>(), "Image preprocessing complete" ); - // Log first few pixel values for debugging - let flat = preprocessed.pixel_values_flat(); - if flat.len() >= 10 { - info!( - first_10_pixels = ?&flat[..10], - last_10_pixels = ?&flat[flat.len()-10..], - pixel_min = flat.iter().copied().fold(f32::INFINITY, f32::min), - pixel_max = flat.iter().copied().fold(f32::NEG_INFINITY, f32::max), - "Pixel values sample" - ); - } // Step 3: Compute prompt replacements and expand tokens. let prompt_replacements = spec @@ -498,13 +475,12 @@ async fn process_multimodal_parts( &prompt_replacements, ); - info!( + debug!( original_len = token_ids.len(), expanded_len = expanded.token_ids.len(), placeholder_count = expanded.placeholders.len(), ?search_token_id, ?im_token_id, - placeholders = ?expanded.placeholders.iter().map(|p| (p.offset, p.length)).collect::>(), "Token expansion complete" ); @@ -636,11 +612,6 @@ pub(crate) fn assemble_multimodal_data( fn assemble_sglang(intermediate: MultimodalIntermediate) -> SglangMultimodalData { let (pixel_values, pixel_values_shape) = serialize_pixel_values(&intermediate.preprocessed); - info!( - pixel_values_bytes = pixel_values.len(), - pixel_values_shape = ?pixel_values_shape, - "Serialized pixel_values for sglang" - ); let model_specific_tensors = serialize_model_specific(intermediate.preprocessed.model_specific); let image_data = intermediate .images From 9739ee2d74e0b9096f6608c94580eadcd78f9907 Mon Sep 17 00:00:00 2001 From: Kangyan Zhou Date: Wed, 1 Apr 2026 00:07:39 -0700 Subject: [PATCH 15/23] fix(multimodal): address PR review findings - Make from_value consistent with from_json by delegating to it, ensuring nested media_proc_cfg extraction applies to both paths - Add test for encode_with_special_tokens verifying special token strings in input produce single token IDs Co-Authored-By: Claude Opus 4.6 (1M context) --- .../multimodal/src/vision/preprocessor_config.rs | 5 ++++- crates/tokenizer/src/tiktoken.rs | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/crates/multimodal/src/vision/preprocessor_config.rs b/crates/multimodal/src/vision/preprocessor_config.rs index 6f828cd6a7..3d59ebea60 100644 --- a/crates/multimodal/src/vision/preprocessor_config.rs +++ b/crates/multimodal/src/vision/preprocessor_config.rs @@ -270,8 +270,11 @@ impl PreProcessorConfig { } /// Parse from JSON value. + /// + /// Handles nested `media_proc_cfg` the same way as `from_json`. pub fn from_value(value: serde_json::Value) -> Result { - serde_json::from_value(value) + let json_str = serde_json::to_string(&value)?; + Self::from_json(&json_str) } /// Get patch size as a simple usize. diff --git a/crates/tokenizer/src/tiktoken.rs b/crates/tokenizer/src/tiktoken.rs index d4ba0d7d13..6b64cf6464 100644 --- a/crates/tokenizer/src/tiktoken.rs +++ b/crates/tokenizer/src/tiktoken.rs @@ -804,4 +804,19 @@ mod tests { let decoded = tokenizer.decode(encoding.token_ids(), false).unwrap(); assert_eq!(decoded, text); } + + #[test] + fn test_encode_recognizes_special_tokens_in_input() { + // encode_with_special_tokens must recognize special token strings + // so that chat-template-rendered text (containing e.g. <|endoftext|>) + // produces single token IDs, not BPE sub-tokens. + let tokenizer = TiktokenTokenizer::new(TiktokenModel::Cl100kBase).unwrap(); + // <|endoftext|> is token 100257 in cl100k_base + let encoding = tokenizer.encode("hello<|endoftext|>world", false).unwrap(); + let ids = encoding.token_ids(); + assert!( + ids.contains(&100257), + "Special token <|endoftext|> should be recognized as single token, got: {ids:?}" + ); + } } From 5bc71cb65f88ab3d8e1dd9be95153235db1f5310 Mon Sep 17 00:00:00 2001 From: Kangyan Zhou Date: Wed, 1 Apr 2026 12:09:11 -0700 Subject: [PATCH 16/23] perf(multimodal): fuse resize+pad+normalize and optimize patch extraction Two optimizations for the Kimi-K2.5 image preprocessing pipeline: 1. Fuse resize + pad + normalize into a single pass using deinterleave_rgb_to_planes with precomputed scale/bias. Eliminates 2 intermediate Array3 allocations and 2 extra passes over pixel data. 2. Replace per-element scalar indexing in extract_patches with row-based extend_from_slice (14-element memcpy per row), enabling compiler auto-vectorization. Also take upstream multimodal.rs which has resolve_model_config_dir and updated image_processor_registry.find() API. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../src/vision/processors/kimi_k25.rs | 121 ++++++++++++------ model_gateway/src/routers/grpc/multimodal.rs | 111 ++++++++-------- 2 files changed, 146 insertions(+), 86 deletions(-) diff --git a/crates/multimodal/src/vision/processors/kimi_k25.rs b/crates/multimodal/src/vision/processors/kimi_k25.rs index 72026d71ba..0285b0863a 100644 --- a/crates/multimodal/src/vision/processors/kimi_k25.rs +++ b/crates/multimodal/src/vision/processors/kimi_k25.rs @@ -19,7 +19,7 @@ use ndarray::Array3; use crate::vision::{ image_processor::{ImagePreProcessor, ModelSpecificValue, PreprocessedImages}, preprocessor_config::PreProcessorConfig, - transforms::{normalize, to_tensor, TransformError}, + transforms::{self, TransformError}, }; pub const KIMI_K25_MEAN: [f64; 3] = [0.5, 0.5, 0.5]; @@ -122,13 +122,20 @@ impl KimiK25Processor { } } - /// Resize image and zero-pad to factor-aligned dimensions. + /// Fused resize + zero-pad + normalize into a single [C, H_padded, W_padded] tensor. /// - /// Returns a [C, H_padded, W_padded] tensor where H_padded and W_padded - /// are divisible by factor. - fn resize_and_pad(image: &DynamicImage, cfg: &ResizeConfig) -> Array3 { - let padded_h = cfg.new_height + cfg.pad_height; - let padded_w = cfg.new_width + cfg.pad_width; + /// Avoids intermediate allocations by: + /// 1. Allocating the final padded canvas directly + /// 2. Pre-filling with normalized black (bias value) + /// 3. Deinterleaving + normalizing the image region in one pass + fn resize_pad_and_normalize( + image: &DynamicImage, + cfg: &ResizeConfig, + mean: &[f64; 3], + std: &[f64; 3], + ) -> Array3 { + let canvas_h = cfg.new_height + cfg.pad_height; + let canvas_w = cfg.new_width + cfg.pad_width; // Resize using BICUBIC let resized = image.resize_exact( @@ -137,22 +144,51 @@ impl KimiK25Processor { image::imageops::FilterType::CatmullRom, ); - // Convert to tensor [C, H, W] with values in [0, 1] - let tensor = to_tensor(&resized); - - if cfg.pad_width == 0 && cfg.pad_height == 0 { - return tensor; + let (img_w, img_h, raw) = transforms::rgb_bytes(&resized); + let canvas_pixels = canvas_h * canvas_w; + + // Precompute fused scale/bias: pixel/255 → normalized + // output[c][i] = raw[i*3+c] / 255.0 * (1/std[c]) + (-mean[c]/std[c]) + let scale: [f32; 3] = std::array::from_fn(|c| 1.0 / (255.0 * std[c] as f32)); + let bias: [f32; 3] = std::array::from_fn(|c| -(mean[c] as f32) / (std[c] as f32)); + + let mut data = vec![0.0f32; 3 * canvas_pixels]; + let (r_plane, rest) = data.split_at_mut(canvas_pixels); + let (g_plane, b_plane) = rest.split_at_mut(canvas_pixels); + + // Pre-fill with normalized black: (0/255 - mean) / std = bias + r_plane.fill(bias[0]); + g_plane.fill(bias[1]); + b_plane.fill(bias[2]); + + // Overwrite image region row-by-row using vectorized deinterleave + let rw = img_w.min(canvas_w); + let rh = img_h.min(canvas_h); + for y in 0..rh { + let src_row = &raw[y * img_w * 3..y * img_w * 3 + rw * 3]; + let dst_offset = y * canvas_w; + transforms::deinterleave_rgb_to_planes( + src_row, + &mut r_plane[dst_offset..dst_offset + rw], + &mut g_plane[dst_offset..dst_offset + rw], + &mut b_plane[dst_offset..dst_offset + rw], + scale, + bias, + ); } - // Zero-pad: create padded array, copy resized data into top-left - let mut padded = Array3::::zeros((3, padded_h, padded_w)); - padded - .slice_mut(ndarray::s![.., ..cfg.new_height, ..cfg.new_width]) - .assign(&tensor); - padded + #[expect( + clippy::expect_used, + reason = "data has exactly 3*canvas_h*canvas_w elements by construction" + )] + Array3::from_shape_vec((3, canvas_h, canvas_w), data) + .expect("shape matches pre-allocated buffer") } - /// Extract [C, patch_size, patch_size] patches from a [C, H, W] tensor. + /// Extract [C, patch_size, patch_size] patches from a contiguous [C, H, W] tensor. + /// + /// Uses row-based `copy_from_slice` instead of per-element indexing so the + /// compiler can auto-vectorize the inner copy. fn extract_patches(tensor: &Array3, patch_size: usize) -> Vec { let channels = tensor.shape()[0]; let height = tensor.shape()[1]; @@ -165,15 +201,25 @@ impl KimiK25Processor { let mut patches = Vec::with_capacity(num_patches * patch_features); + // Get contiguous slice for direct row addressing + let flat = tensor.as_standard_layout(); + #[expect( + clippy::expect_used, + reason = "as_standard_layout guarantees contiguous C-order memory" + )] + let data = flat + .as_slice() + .expect("as_standard_layout guarantees contiguous memory"); + for gh in 0..grid_h { for gw in 0..grid_w { let h_start = gh * patch_size; let w_start = gw * patch_size; for c in 0..channels { + let plane_offset = c * height * width; for ph in 0..patch_size { - for pw in 0..patch_size { - patches.push(tensor[[c, h_start + ph, w_start + pw]]); - } + let row_start = plane_offset + (h_start + ph) * width + w_start; + patches.extend_from_slice(&data[row_start..row_start + patch_size]); } } } @@ -214,13 +260,8 @@ impl ImagePreProcessor for KimiK25Processor { let (w, h) = image.dimensions(); let cfg = self.compute_resize_config(w as usize, h as usize); - // Resize + zero-pad - let mut tensor = Self::resize_and_pad(image, &cfg); - - // Normalize - if config.do_normalize.unwrap_or(true) { - normalize(&mut tensor, &mean, &std); - } + // Fused resize + pad + normalize in one pass (avoids 2 extra allocations) + let tensor = Self::resize_pad_and_normalize(image, &cfg, &mean, &std); let padded_h = cfg.new_height + cfg.pad_height; let padded_w = cfg.new_width + cfg.pad_width; @@ -438,21 +479,29 @@ mod tests { fn test_zero_padding_applied() { let p = KimiK25Processor::new(); let config = PreProcessorConfig { - do_normalize: Some(false), // skip normalization to check raw values + image_mean: Some(KIMI_K25_MEAN.to_vec()), + image_std: Some(KIMI_K25_STD.to_vec()), ..Default::default() }; - // 100x100 image with white pixels + // 100x100 white image — after normalization: (255/255 - 0.5) / 0.5 = 1.0 + // Padded region: (0/255 - 0.5) / 0.5 = -1.0 let image = create_test_image(100, 100, Rgb([255, 255, 255])); let result = p.preprocess(&[image], &config).unwrap(); - // The padded region should contain zeros (from zero-padding) let flat = result.pixel_values_flat(); - let has_zeros = flat.contains(&0.0); - assert!(has_zeros, "Expected zero-padded regions in output"); + // Padded region should be normalized black (-1.0) + let has_neg_ones = flat.iter().any(|&v| (v - (-1.0)).abs() < 1e-6); + assert!( + has_neg_ones, + "Expected normalized-black padding (-1.0) in output" + ); - // The image region should contain non-zero values (white = 1.0) + // Image region should be normalized white (1.0) let has_ones = flat.iter().any(|&v| (v - 1.0).abs() < 1e-6); - assert!(has_ones, "Expected non-zero image values in output"); + assert!( + has_ones, + "Expected normalized-white image values (1.0) in output" + ); } } diff --git a/model_gateway/src/routers/grpc/multimodal.rs b/model_gateway/src/routers/grpc/multimodal.rs index 7f190b7c77..a68ec14580 100644 --- a/model_gateway/src/routers/grpc/multimodal.rs +++ b/model_gateway/src/routers/grpc/multimodal.rs @@ -9,7 +9,7 @@ //! functions differ because they work with different input types (`ChatMessage` vs //! `InputMessage`). -use std::{collections::HashMap, path::Path, sync::Arc}; +use std::{collections::HashMap, sync::Arc}; use anyhow::{Context, Result}; use dashmap::DashMap; @@ -70,9 +70,6 @@ impl MultimodalComponents { } /// Load or retrieve cached model config for a given model and tokenizer source path. - /// - /// If `tokenizer_source` is a local directory containing `config.json`, loads from disk. - /// Otherwise, treats it as a HuggingFace model ID and downloads the config files. pub async fn get_or_load_config( &self, model_id: &str, @@ -82,39 +79,13 @@ impl MultimodalComponents { return Ok(cached.clone()); } - let base_dir = Path::new(tokenizer_source); - let config_path = base_dir.join("config.json"); - - // If local config.json exists, load from disk; otherwise download from HF Hub. - let (config, preprocessor_config) = if config_path.is_file() { - Self::load_configs_from_disk(base_dir)? - } else if base_dir.is_dir() { - // Local directory exists but config.json is missing - anyhow::bail!( - "config.json not found in local model directory: {}. \ - Ensure the directory contains config.json and preprocessor_config.json.", - base_dir.display() - ); - } else { - warn!( - model_id = model_id, - source = tokenizer_source, - "config.json not found locally, downloading from HuggingFace Hub" - ); - Self::load_configs_from_hf(tokenizer_source).await? - }; - - let model_config = Arc::new(MultimodalModelConfig { - config, - preprocessor_config, - }); - - self.model_configs - .insert(model_id.to_string(), model_config.clone()); - Ok(model_config) - } + let base_dir = llm_multimodal::hub::resolve_model_config_dir(tokenizer_source) + .await + .with_context(|| { + format!("Failed to resolve model config directory for '{tokenizer_source}'") + })?; - fn load_configs_from_disk(base_dir: &Path) -> Result<(serde_json::Value, PreProcessorConfig)> { + // Load config.json let config_path = base_dir.join("config.json"); let config: serde_json::Value = std::fs::read_to_string(&config_path) .with_context(|| format!("Failed to read config.json at {}", config_path.display())) @@ -124,6 +95,7 @@ impl MultimodalComponents { }) })?; + // Load preprocessor_config.json let pp_config_path = base_dir.join("preprocessor_config.json"); let preprocessor_config = std::fs::read_to_string(&pp_config_path) .with_context(|| { @@ -141,19 +113,14 @@ impl MultimodalComponents { }) })?; - Ok((config, preprocessor_config)) - } - - async fn load_configs_from_hf( - model_id: &str, - ) -> Result<(serde_json::Value, PreProcessorConfig)> { - let cache_dir = llm_tokenizer::hub::download_model_configs_from_hf(model_id) - .await - .with_context(|| { - format!("Failed to download config files from HuggingFace for model: {model_id}") - })?; + let model_config = Arc::new(MultimodalModelConfig { + config, + preprocessor_config, + }); - Self::load_configs_from_disk(&cache_dir) + self.model_configs + .insert(model_id.to_string(), model_config.clone()); + Ok(model_config) } } @@ -189,6 +156,34 @@ pub(crate) struct MultimodalIntermediate { pub keep_on_cpu_keys: Vec, } +/// Resolve the placeholder token string for a multimodal model. +/// +/// Loads the model config and looks up the model spec to get the placeholder +/// token (e.g. `"<|image|>"` for Phi-3-vision). Returns `None` if the model +/// is not recognized as multimodal. +pub(crate) async fn resolve_placeholder_token( + model_id: &str, + tokenizer: &dyn TokenizerTrait, + components: &MultimodalComponents, + tokenizer_source: &str, +) -> Result> { + let model_config = components + .get_or_load_config(model_id, tokenizer_source) + .await?; + let metadata = ModelMetadata { + model_id, + tokenizer, + config: &model_config.config, + }; + let spec = match components.model_registry.lookup(&metadata) { + Some(s) => s, + None => return Ok(None), + }; + Ok(Some(spec.placeholder_token(&metadata).map_err(|e| { + anyhow::anyhow!("Failed to get placeholder token: {e}") + })?)) +} + /// Check if any messages in the request contain multimodal content (images). pub(crate) fn has_multimodal_content(messages: &[ChatMessage]) -> bool { messages.iter().any(|msg| { @@ -414,6 +409,7 @@ async fn process_multimodal_parts( debug!( image_count = images.len(), + image_sizes = ?images.iter().map(|f| (f.image.width(), f.image.height())).collect::>(), "Fetched images for multimodal processing" ); @@ -421,6 +417,10 @@ async fn process_multimodal_parts( let model_config = components .get_or_load_config(model_id, tokenizer_source) .await?; + let model_type = model_config + .config + .get("model_type") + .and_then(|v| v.as_str()); let metadata = ModelMetadata { model_id, tokenizer, @@ -433,7 +433,7 @@ async fn process_multimodal_parts( let image_processor = components .image_processor_registry - .find(model_id) + .find(model_id, model_type) .ok_or_else(|| anyhow::anyhow!("No image processor found for model: {model_id}"))?; // ImagePreProcessor::preprocess takes &[DynamicImage]; images are behind Arc. @@ -685,7 +685,18 @@ fn serialize_pixel_values(preprocessed: &PreprocessedImages) -> (Vec, Vec Date: Wed, 1 Apr 2026 16:14:26 -0700 Subject: [PATCH 17/23] debug(multimodal): add per-step timing for multimodal preprocessing Log timing breakdown: image fetch, config load, preprocessing, token expansion, and assembly/serialization. This helps identify which step dominates TTFT for multimodal gRPC requests. Co-Authored-By: Claude Opus 4.6 (1M context) --- model_gateway/src/routers/grpc/multimodal.rs | 37 ++++++++++++++++++-- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/model_gateway/src/routers/grpc/multimodal.rs b/model_gateway/src/routers/grpc/multimodal.rs index a68ec14580..6d2f807e8c 100644 --- a/model_gateway/src/routers/grpc/multimodal.rs +++ b/model_gateway/src/routers/grpc/multimodal.rs @@ -25,7 +25,7 @@ use openai_protocol::{ common::ContentPart, messages::{ImageSource, InputContent, InputContentBlock, InputMessage, Role}, }; -use tracing::{debug, warn}; +use tracing::{debug, info, warn}; use crate::routers::grpc::{ client::GrpcClient, @@ -374,6 +374,8 @@ async fn process_multimodal_parts( components: &MultimodalComponents, tokenizer_source: &str, ) -> Result { + let t0 = std::time::Instant::now(); + let mut tracker = AsyncMultiModalTracker::new(components.media_connector.clone()); for part in content_parts { @@ -407,6 +409,8 @@ async fn process_multimodal_parts( )); } + let t_fetch = t0.elapsed(); + debug!( image_count = images.len(), image_sizes = ?images.iter().map(|f| (f.image.width(), f.image.height())).collect::>(), @@ -436,6 +440,8 @@ async fn process_multimodal_parts( .find(model_id, model_type) .ok_or_else(|| anyhow::anyhow!("No image processor found for model: {model_id}"))?; + let t_config = t0.elapsed(); + // ImagePreProcessor::preprocess takes &[DynamicImage]; images are behind Arc. // Clone cost is negligible vs. the preprocessing work itself. let dynamic_images: Vec = images.iter().map(|f| f.image.clone()).collect(); @@ -444,6 +450,8 @@ async fn process_multimodal_parts( .preprocess(&dynamic_images, &model_config.preprocessor_config) .map_err(|e| anyhow::anyhow!("Image preprocessing failed: {e}"))?; + let t_preprocess = t0.elapsed(); + debug!( num_images = preprocessed.num_img_tokens.len(), total_tokens = preprocessed.num_img_tokens.iter().sum::(), @@ -475,6 +483,23 @@ async fn process_multimodal_parts( &prompt_replacements, ); + let t_expand = t0.elapsed(); + + info!( + fetch_ms = t_fetch.as_millis() as u64, + config_ms = (t_config - t_fetch).as_millis() as u64, + preprocess_ms = (t_preprocess - t_config).as_millis() as u64, + expand_ms = (t_expand - t_preprocess).as_millis() as u64, + total_ms = t_expand.as_millis() as u64, + image_count = images.len(), + image_pixels = images + .iter() + .map(|f| f.image.width() as u64 * f.image.height() as u64) + .sum::(), + total_tokens = preprocessed.num_img_tokens.iter().sum::(), + "Multimodal preprocessing timing" + ); + debug!( original_len = token_ids.len(), expanded_len = expanded.token_ids.len(), @@ -603,11 +628,17 @@ pub(crate) fn assemble_multimodal_data( intermediate: MultimodalIntermediate, client: &GrpcClient, ) -> MultimodalData { - match client { + let t0 = std::time::Instant::now(); + let result = match client { GrpcClient::Sglang(_) => MultimodalData::Sglang(assemble_sglang(intermediate)), GrpcClient::Vllm(_) => MultimodalData::Vllm(assemble_vllm(intermediate)), GrpcClient::Trtllm(_) => MultimodalData::Trtllm(assemble_trtllm(intermediate)), - } + }; + info!( + serialize_ms = t0.elapsed().as_millis() as u64, + "Multimodal assembly/serialization timing" + ); + result } fn assemble_sglang(intermediate: MultimodalIntermediate) -> SglangMultimodalData { From f653eca9ea8b43100d2588cf91eaceeea7fabeb6 Mon Sep 17 00:00:00 2001 From: Kangyan Zhou Date: Wed, 1 Apr 2026 16:53:02 -0700 Subject: [PATCH 18/23] perf(multimodal): spawn_blocking for preprocessing + strip decode mm_inputs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two optimizations to reduce gRPC multimodal TTFT: 1. Move image preprocessing (resize + pad + normalize + patchify) to tokio::task::spawn_blocking so CPU-intensive work doesn't block the async runtime. Under 200 concurrent requests, this prevents serialized preprocessing from inflating tail latencies. 2. Strip mm_inputs from decode worker requests in PD dual dispatch. The decode worker only needs the KV cache from prefill — sending ~40MB of pixel tensors to it was pure waste. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../grpc/common/stages/request_execution.rs | 5 ++- model_gateway/src/routers/grpc/multimodal.rs | 32 ++++++++++++------- .../src/routers/grpc/proto_wrapper.rs | 13 ++++++++ 3 files changed, 37 insertions(+), 13 deletions(-) diff --git a/model_gateway/src/routers/grpc/common/stages/request_execution.rs b/model_gateway/src/routers/grpc/common/stages/request_execution.rs index a8e20d322f..40e4d8e589 100644 --- a/model_gateway/src/routers/grpc/common/stages/request_execution.rs +++ b/model_gateway/src/routers/grpc/common/stages/request_execution.rs @@ -235,7 +235,10 @@ impl RequestExecutionStage { })?; let prefill_request = proto_request.clone_inner(); - let decode_request = proto_request; + // Strip multimodal data from decode request — the decode worker only + // needs the KV cache from prefill, not the pixel tensors (~40MB saved). + let mut decode_request = proto_request; + decode_request.clear_mm_inputs(); let (prefill_result, decode_result): (StreamResult, StreamResult) = tokio::join!( prefill_client.generate(prefill_request), diff --git a/model_gateway/src/routers/grpc/multimodal.rs b/model_gateway/src/routers/grpc/multimodal.rs index 6d2f807e8c..b6eddd5643 100644 --- a/model_gateway/src/routers/grpc/multimodal.rs +++ b/model_gateway/src/routers/grpc/multimodal.rs @@ -435,20 +435,28 @@ async fn process_multimodal_parts( .lookup(&metadata) .ok_or_else(|| anyhow::anyhow!("Multimodal not supported for model: {model_id}"))?; - let image_processor = components - .image_processor_registry - .find(model_id, model_type) - .ok_or_else(|| anyhow::anyhow!("No image processor found for model: {model_id}"))?; - let t_config = t0.elapsed(); - // ImagePreProcessor::preprocess takes &[DynamicImage]; images are behind Arc. - // Clone cost is negligible vs. the preprocessing work itself. - let dynamic_images: Vec = images.iter().map(|f| f.image.clone()).collect(); - - let preprocessed: PreprocessedImages = image_processor - .preprocess(&dynamic_images, &model_config.preprocessor_config) - .map_err(|e| anyhow::anyhow!("Image preprocessing failed: {e}"))?; + // Run CPU-intensive image preprocessing on a blocking thread pool so it + // doesn't block the tokio async runtime under concurrent load. + let pp_config = model_config.preprocessor_config.clone(); + let registry = components.image_processor_registry.clone(); + let model_id_owned = model_id.to_string(); + let model_type_owned = model_type.map(String::from); + let image_clones: Vec = images.iter().map(|f| f.image.clone()).collect(); + + let preprocessed: PreprocessedImages = tokio::task::spawn_blocking(move || { + let processor = registry + .find(&model_id_owned, model_type_owned.as_deref()) + .ok_or_else(|| { + anyhow::anyhow!("No image processor found for model: {model_id_owned}") + })?; + processor + .preprocess(&image_clones, &pp_config) + .map_err(|e| anyhow::anyhow!("Image preprocessing failed: {e}")) + }) + .await + .map_err(|e| anyhow::anyhow!("Preprocessing task panicked: {e}"))??; let t_preprocess = t0.elapsed(); diff --git a/model_gateway/src/routers/grpc/proto_wrapper.rs b/model_gateway/src/routers/grpc/proto_wrapper.rs index 56940d2936..83db9e3be2 100644 --- a/model_gateway/src/routers/grpc/proto_wrapper.rs +++ b/model_gateway/src/routers/grpc/proto_wrapper.rs @@ -402,6 +402,19 @@ impl ProtoGenerateRequest { self.clone() } + /// Strip multimodal inputs from the request. + /// + /// Used for the decode worker in PD disaggregation — the decode worker only + /// needs the KV cache from prefill, not the image pixel data. This avoids + /// transmitting ~40MB of pixel tensors to a worker that ignores them. + pub fn clear_mm_inputs(&mut self) { + match self { + Self::Sglang(req) => req.mm_inputs = None, + Self::Vllm(req) => req.mm_inputs = None, + Self::Trtllm(_) => {} // TRT-LLM proto has no mm_inputs field + } + } + /// Get request ID pub fn request_id(&self) -> &str { match self { From 66fa5ba7a66c226f1e6feccf4c2a97e423143dea Mon Sep 17 00:00:00 2001 From: Kangyan Zhou Date: Wed, 1 Apr 2026 17:39:58 -0700 Subject: [PATCH 19/23] perf(multimodal): use SIMD-accelerated resize for Kimi preprocessing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace image::resize_exact(CatmullRom) with transforms::resize() which uses fast_image_resize (AVX2/SSE4 SIMD). This is a drop-in replacement that gives 3-5x faster BICUBIC resize — the dominant CPU cost in the preprocessing pipeline. Co-Authored-By: Claude Opus 4.6 (1M context) --- crates/multimodal/src/vision/processors/kimi_k25.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/multimodal/src/vision/processors/kimi_k25.rs b/crates/multimodal/src/vision/processors/kimi_k25.rs index 0285b0863a..e9b15ed02a 100644 --- a/crates/multimodal/src/vision/processors/kimi_k25.rs +++ b/crates/multimodal/src/vision/processors/kimi_k25.rs @@ -137,8 +137,9 @@ impl KimiK25Processor { let canvas_h = cfg.new_height + cfg.pad_height; let canvas_w = cfg.new_width + cfg.pad_width; - // Resize using BICUBIC - let resized = image.resize_exact( + // Resize using SIMD-accelerated BICUBIC (fast_image_resize) + let resized = transforms::resize( + image, cfg.new_width as u32, cfg.new_height as u32, image::imageops::FilterType::CatmullRom, From 0bd6f07ed321b2b80d6ccdc12fe536d43df6860a Mon Sep 17 00:00:00 2001 From: Kangyan Zhou Date: Wed, 1 Apr 2026 17:55:40 -0700 Subject: [PATCH 20/23] chore(multimodal): remove per-step timing instrumentation Remove info-level timing logs (fetch_ms, config_ms, preprocess_ms, expand_ms, serialize_ms) now that performance analysis is complete. Co-Authored-By: Claude Opus 4.6 (1M context) --- model_gateway/src/routers/grpc/multimodal.rs | 37 ++------------------ 1 file changed, 3 insertions(+), 34 deletions(-) diff --git a/model_gateway/src/routers/grpc/multimodal.rs b/model_gateway/src/routers/grpc/multimodal.rs index b6eddd5643..32a699f09d 100644 --- a/model_gateway/src/routers/grpc/multimodal.rs +++ b/model_gateway/src/routers/grpc/multimodal.rs @@ -25,7 +25,7 @@ use openai_protocol::{ common::ContentPart, messages::{ImageSource, InputContent, InputContentBlock, InputMessage, Role}, }; -use tracing::{debug, info, warn}; +use tracing::{debug, warn}; use crate::routers::grpc::{ client::GrpcClient, @@ -374,8 +374,6 @@ async fn process_multimodal_parts( components: &MultimodalComponents, tokenizer_source: &str, ) -> Result { - let t0 = std::time::Instant::now(); - let mut tracker = AsyncMultiModalTracker::new(components.media_connector.clone()); for part in content_parts { @@ -409,8 +407,6 @@ async fn process_multimodal_parts( )); } - let t_fetch = t0.elapsed(); - debug!( image_count = images.len(), image_sizes = ?images.iter().map(|f| (f.image.width(), f.image.height())).collect::>(), @@ -435,8 +431,6 @@ async fn process_multimodal_parts( .lookup(&metadata) .ok_or_else(|| anyhow::anyhow!("Multimodal not supported for model: {model_id}"))?; - let t_config = t0.elapsed(); - // Run CPU-intensive image preprocessing on a blocking thread pool so it // doesn't block the tokio async runtime under concurrent load. let pp_config = model_config.preprocessor_config.clone(); @@ -458,8 +452,6 @@ async fn process_multimodal_parts( .await .map_err(|e| anyhow::anyhow!("Preprocessing task panicked: {e}"))??; - let t_preprocess = t0.elapsed(); - debug!( num_images = preprocessed.num_img_tokens.len(), total_tokens = preprocessed.num_img_tokens.iter().sum::(), @@ -491,23 +483,6 @@ async fn process_multimodal_parts( &prompt_replacements, ); - let t_expand = t0.elapsed(); - - info!( - fetch_ms = t_fetch.as_millis() as u64, - config_ms = (t_config - t_fetch).as_millis() as u64, - preprocess_ms = (t_preprocess - t_config).as_millis() as u64, - expand_ms = (t_expand - t_preprocess).as_millis() as u64, - total_ms = t_expand.as_millis() as u64, - image_count = images.len(), - image_pixels = images - .iter() - .map(|f| f.image.width() as u64 * f.image.height() as u64) - .sum::(), - total_tokens = preprocessed.num_img_tokens.iter().sum::(), - "Multimodal preprocessing timing" - ); - debug!( original_len = token_ids.len(), expanded_len = expanded.token_ids.len(), @@ -636,17 +611,11 @@ pub(crate) fn assemble_multimodal_data( intermediate: MultimodalIntermediate, client: &GrpcClient, ) -> MultimodalData { - let t0 = std::time::Instant::now(); - let result = match client { + match client { GrpcClient::Sglang(_) => MultimodalData::Sglang(assemble_sglang(intermediate)), GrpcClient::Vllm(_) => MultimodalData::Vllm(assemble_vllm(intermediate)), GrpcClient::Trtllm(_) => MultimodalData::Trtllm(assemble_trtllm(intermediate)), - }; - info!( - serialize_ms = t0.elapsed().as_millis() as u64, - "Multimodal assembly/serialization timing" - ); - result + } } fn assemble_sglang(intermediate: MultimodalIntermediate) -> SglangMultimodalData { From 038c73d702524a89b426e45fe5cb187214c2323c Mon Sep 17 00:00:00 2001 From: Kangyan Zhou Date: Sat, 4 Apr 2026 00:19:12 -0700 Subject: [PATCH 21/23] fix(multimodal): address PR review findings - Remove dead download_model_configs_from_hf (replaced by upstream resolve_model_config_dir) - Extract in_patch_limit/patch_limit_on_one_side from media_proc_cfg into config.extra, read in from_preprocessor_config - Always check media_proc_cfg for all fields, not just when image_mean/std are missing (fixes partial config overlap) - Log warning when placeholder_token_id fails instead of silent .ok() - Add config_model_type fallback to KimiK25VisionSpec::matches - Add tests: 1x1 image, empty batch, from_preprocessor_config limits - Improve tiktoken encode comment explaining why encode_with_special_tokens is used Co-Authored-By: Claude Opus 4.6 (1M context) --- crates/multimodal/src/registry/kimi_k25.rs | 3 + .../src/vision/preprocessor_config.rs | 14 ++++- .../src/vision/processors/kimi_k25.rs | 56 ++++++++++++++++++- crates/tokenizer/src/hub.rs | 29 ---------- crates/tokenizer/src/tiktoken.rs | 8 ++- model_gateway/src/routers/grpc/multimodal.rs | 16 ++++-- 6 files changed, 86 insertions(+), 40 deletions(-) diff --git a/crates/multimodal/src/registry/kimi_k25.rs b/crates/multimodal/src/registry/kimi_k25.rs index d472b42570..0c5cb23252 100644 --- a/crates/multimodal/src/registry/kimi_k25.rs +++ b/crates/multimodal/src/registry/kimi_k25.rs @@ -30,6 +30,9 @@ impl ModelProcessorSpec for KimiK25VisionSpec { fn matches(&self, metadata: &ModelMetadata) -> bool { let id = metadata.model_id.to_ascii_lowercase(); id.contains("kimi") && id.contains("k2") + || metadata + .config_model_type() + .is_some_and(|mt| mt == "kimi_k25") } fn placeholder_token(&self, _metadata: &ModelMetadata) -> RegistryResult { diff --git a/crates/multimodal/src/vision/preprocessor_config.rs b/crates/multimodal/src/vision/preprocessor_config.rs index 3d59ebea60..d1340fe794 100644 --- a/crates/multimodal/src/vision/preprocessor_config.rs +++ b/crates/multimodal/src/vision/preprocessor_config.rs @@ -235,8 +235,9 @@ impl PreProcessorConfig { /// nested format where values are under `media_proc_cfg`. pub fn from_json(json: &str) -> Result { let mut config: Self = serde_json::from_str(json)?; - // If top-level image_mean/std are missing, try to extract from media_proc_cfg - if config.image_mean.is_none() || config.image_std.is_none() { + // Extract values from nested media_proc_cfg (used by Kimi-K2.5 and similar models) + // when top-level fields are missing. + { if let Ok(raw) = serde_json::from_str::(json) { if let Some(media_cfg) = raw.get("media_proc_cfg") { if config.image_mean.is_none() { @@ -263,6 +264,15 @@ impl PreProcessorConfig { .and_then(|v| v.as_u64()) .map(|v| v as usize); } + // Also extract Kimi-specific limits into the extra map + // so processors can read them via get_extra() + for key in ["in_patch_limit", "patch_limit_on_one_side"] { + if !config.extra.contains_key(key) { + if let Some(v) = media_cfg.get(key) { + config.extra.insert(key.to_string(), v.clone()); + } + } + } } } } diff --git a/crates/multimodal/src/vision/processors/kimi_k25.rs b/crates/multimodal/src/vision/processors/kimi_k25.rs index e9b15ed02a..ef3b85b05e 100644 --- a/crates/multimodal/src/vision/processors/kimi_k25.rs +++ b/crates/multimodal/src/vision/processors/kimi_k25.rs @@ -69,8 +69,12 @@ impl KimiK25Processor { Self { patch_size: config.get_patch_size(DEFAULT_PATCH_SIZE), merge_size: config.merge_size.unwrap_or(DEFAULT_MERGE_SIZE), - in_patch_limit: DEFAULT_IN_PATCH_LIMIT, - patch_limit_on_one_side: DEFAULT_PATCH_LIMIT_ON_ONE_SIDE, + in_patch_limit: config + .get_extra::("in_patch_limit") + .unwrap_or(DEFAULT_IN_PATCH_LIMIT), + patch_limit_on_one_side: config + .get_extra::("patch_limit_on_one_side") + .unwrap_or(DEFAULT_PATCH_LIMIT_ON_ONE_SIDE), } } @@ -505,4 +509,52 @@ mod tests { "Expected normalized-white image values (1.0) in output" ); } + + #[test] + fn test_preprocess_tiny_image() { + // 1x1 image should not panic — padded to 28x28 + let p = KimiK25Processor::new(); + let config = PreProcessorConfig { + image_mean: Some(KIMI_K25_MEAN.to_vec()), + image_std: Some(KIMI_K25_STD.to_vec()), + ..Default::default() + }; + let image = create_test_image(1, 1, Rgb([128, 128, 128])); + let result = p.preprocess(&[image], &config).unwrap(); + assert_eq!(result.pixel_values.ndim(), 4); + assert!(result.pixel_values.shape()[0] > 0); + assert!(result.num_img_tokens[0] > 0); + } + + #[test] + fn test_preprocess_empty_batch_returns_error() { + let p = KimiK25Processor::new(); + let config = PreProcessorConfig::default(); + let result = p.preprocess(&[], &config); + assert!(result.is_err()); + } + + #[test] + fn test_from_preprocessor_config_reads_limits() { + let config = PreProcessorConfig { + patch_size: Some(PatchSize { + height: Some(14), + width: Some(14), + }), + merge_size: Some(2), + extra: [ + ("in_patch_limit".to_string(), serde_json::json!(8192)), + ( + "patch_limit_on_one_side".to_string(), + serde_json::json!(256), + ), + ] + .into_iter() + .collect(), + ..Default::default() + }; + let p = KimiK25Processor::from_preprocessor_config(&config); + assert_eq!(p.in_patch_limit, 8192); + assert_eq!(p.patch_limit_on_one_side, 256); + } } diff --git a/crates/tokenizer/src/hub.rs b/crates/tokenizer/src/hub.rs index 237eec8c95..0879c0545e 100644 --- a/crates/tokenizer/src/hub.rs +++ b/crates/tokenizer/src/hub.rs @@ -63,35 +63,6 @@ fn is_chat_template_file(filename: &str) -> bool { || filename == "chat_template.json" // JSON file containing Jinja template } -/// Download model config files (config.json, preprocessor_config.json) from HuggingFace. -/// -/// Returns the cache directory containing the downloaded files. -/// Uses the same HF Hub cache as tokenizer downloads. -pub async fn download_model_configs_from_hf(model_id: &str) -> anyhow::Result { - let api = build_api()?; - let repo = api.model(model_id.to_string()); - - let config_files = ["config.json", "preprocessor_config.json"]; - let mut cache_dir = None; - - for filename in &config_files { - match repo.get(filename).await { - Ok(path) => { - if cache_dir.is_none() { - cache_dir = path.parent().map(|p| p.to_path_buf()); - } - } - Err(e) => { - return Err(anyhow::anyhow!( - "Failed to download '{filename}' from model '{model_id}': {e}" - )); - } - } - } - - cache_dir.ok_or_else(|| anyhow::anyhow!("No config files downloaded for model '{model_id}'")) -} - /// Attempt to download tokenizer files from Hugging Face /// Returns the directory containing the downloaded tokenizer files pub async fn download_tokenizer_from_hf(model_id: impl AsRef) -> anyhow::Result { diff --git a/crates/tokenizer/src/tiktoken.rs b/crates/tokenizer/src/tiktoken.rs index 6b64cf6464..d032e95a89 100644 --- a/crates/tokenizer/src/tiktoken.rs +++ b/crates/tokenizer/src/tiktoken.rs @@ -440,8 +440,12 @@ impl Encoder for TiktokenTokenizer { // Always use encode_with_special_tokens so that special token strings // in the input (e.g., <|media_pad|> from chat templates) are recognized // as single tokens rather than split into BPE sub-tokens. - // This matches HuggingFace tokenizer behavior where added special tokens - // are always recognized in input text regardless of add_special_tokens. + // + // In the gateway, tiktoken tokenizers are only created via + // from_dir_with_chat_template (hub-loaded models like Kimi, DeepSeek). + // The input to encode() is always chat-template-rendered text containing + // special tokens that must be recognized. Raw user text is never encoded + // directly — it goes through the chat template first. let tokens = self.tokenizer.encode_with_special_tokens(input); Ok(Encoding::Tiktoken(tokens)) } diff --git a/model_gateway/src/routers/grpc/multimodal.rs b/model_gateway/src/routers/grpc/multimodal.rs index 32a699f09d..3db6c0cac0 100644 --- a/model_gateway/src/routers/grpc/multimodal.rs +++ b/model_gateway/src/routers/grpc/multimodal.rs @@ -470,11 +470,17 @@ async fn process_multimodal_parts( .placeholder_token(&metadata) .map_err(|e| anyhow::anyhow!("Failed to get placeholder token: {e}"))?; let search_token_id = tokenizer.token_to_id(&placeholder_token); - let im_token_id: Option = spec - .placeholder_token_id(&metadata) - .ok() - .map(|id| id as u32) - .or(search_token_id); + let im_token_id: Option = match spec.placeholder_token_id(&metadata) { + Ok(id) => Some(id as u32), + Err(e) => { + warn!( + error = %e, + ?search_token_id, + "Failed to resolve placeholder_token_id from config, falling back to tokenizer lookup" + ); + search_token_id + } + }; let expanded = expand_tokens( &token_ids, From adb44b8e6e3bf76be1dc602bada04b118aafa313 Mon Sep 17 00:00:00 2001 From: Kangyan Zhou Date: Mon, 6 Apr 2026 13:04:16 -0700 Subject: [PATCH 22/23] docs(multimodal): remove Qwen-VL references from Kimi comments Make Kimi-K2.5 code comments self-contained instead of comparing against Qwen-VL internals. Co-Authored-By: Claude Opus 4.6 (1M context) --- crates/multimodal/src/registry/kimi_k25.rs | 4 ++-- crates/multimodal/src/vision/processors/kimi_k25.rs | 7 +++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/crates/multimodal/src/registry/kimi_k25.rs b/crates/multimodal/src/registry/kimi_k25.rs index 0c5cb23252..1347bf2855 100644 --- a/crates/multimodal/src/registry/kimi_k25.rs +++ b/crates/multimodal/src/registry/kimi_k25.rs @@ -76,9 +76,9 @@ impl ModelProcessorSpec for KimiK25VisionSpec { } fn field_layouts(&self) -> HashMap { - // Kimi-K2.5 uses NaViT-style patchification like Qwen-VL: + // Kimi-K2.5 uses NaViT-style patchification: // pixel_values is [total_patches, patch_features], split by patches_per_image. - // grid_thws is [num_images, 3] (Kimi uses "grid_thws" instead of Qwen's "image_grid_thw"). + // grid_thws is [num_images, 3] with (temporal, height, width) grid dimensions. HashMap::from([ ( "pixel_values".to_string(), diff --git a/crates/multimodal/src/vision/processors/kimi_k25.rs b/crates/multimodal/src/vision/processors/kimi_k25.rs index ef3b85b05e..ffbe46ff76 100644 --- a/crates/multimodal/src/vision/processors/kimi_k25.rs +++ b/crates/multimodal/src/vision/processors/kimi_k25.rs @@ -8,10 +8,9 @@ //! 4. Normalize with [0.5, 0.5, 0.5] mean/std //! 5. Extract patches as [N, C, patch_size, patch_size] //! -//! Key difference from Qwen-VL: Kimi resizes then zero-pads for alignment, -//! while Qwen-VL resizes directly to factor-aligned dimensions. The model was -//! trained with zero-padded images, so using the wrong resize strategy degrades -//! image quality. +//! Kimi resizes then zero-pads to make dimensions divisible by the alignment +//! factor (patch_size * merge_size). The model was trained with zero-padded +//! images, so using direct resize-to-aligned would degrade image quality. use image::{DynamicImage, GenericImageView}; use ndarray::Array3; From a21f971bc7eeed2e014e272373672351c11790e0 Mon Sep 17 00:00:00 2001 From: Kangyan Zhou Date: Mon, 6 Apr 2026 15:44:56 -0700 Subject: [PATCH 23/23] fix(servicer): return MultimodalInputs object instead of dict from _parse_mm_inputs sglang v0.5.10 mm_utils.has_shm_features() accesses req.mm_inputs.mm_items via attribute, which fails when mm_inputs is a plain dict. Return a proper MultimodalInputs dataclass to fix the AttributeError crash on VLM requests in gRPC mode. --- .../smg_grpc_servicer/sglang/servicer.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/grpc_servicer/smg_grpc_servicer/sglang/servicer.py b/grpc_servicer/smg_grpc_servicer/sglang/servicer.py index d330216ca4..f0c5fa9de1 100644 --- a/grpc_servicer/smg_grpc_servicer/sglang/servicer.py +++ b/grpc_servicer/smg_grpc_servicer/sglang/servicer.py @@ -37,7 +37,7 @@ TokenizedEmbeddingReqInput, TokenizedGenerateReqInput, ) -from sglang.srt.managers.schedule_batch import Modality, MultimodalDataItem +from sglang.srt.managers.schedule_batch import Modality, MultimodalDataItem, MultimodalInputs from sglang.srt.sampling.sampling_params import SamplingParams as SGLSamplingParams from sglang.srt.server_args import ServerArgs from sglang.utils import get_exception_traceback @@ -768,8 +768,8 @@ def _decode_tensor_data(tensor_data): arr = np.frombuffer(tensor_data.data, dtype=np_dtype).reshape(shape) return torch.from_numpy(arr) - def _parse_mm_inputs(self, mm_proto) -> dict: - """Parse proto MultimodalInputs into the mm_inputs dict expected by scheduler.""" + def _parse_mm_inputs(self, mm_proto) -> MultimodalInputs: + """Parse proto MultimodalInputs into a MultimodalInputs object for the scheduler.""" # Decode pixel_values from typed TensorData field pixel_values = self._decode_tensor_data(mm_proto.pixel_values) @@ -795,12 +795,12 @@ def _parse_mm_inputs(self, mm_proto) -> dict: offsets=offsets, ) - result = {"mm_items": [mm_item]} + im_token_id = mm_proto.im_token_id if mm_proto.HasField("im_token_id") else None - if mm_proto.HasField("im_token_id"): - result["im_token_id"] = mm_proto.im_token_id - - return result + return MultimodalInputs( + mm_items=[mm_item], + im_token_id=im_token_id, + ) def _convert_embed_request( self, grpc_req: sglang_scheduler_pb2.EmbedRequest