feat(multimodal): add Kimi-K2.5 vision support for gRPC router

crates/multimodal/src/registry/kimi_k25.rs

@@ -0,0 +1,214 @@
+use std::collections::HashMap;
+
+use serde_json::{json, Value};
+
+use crate::{
+    registry::{ModelMetadata, ModelProcessorSpec, ModelRegistryError, RegistryResult},
+    types::{FieldLayout, Modality, PromptReplacement, TokenId},
+    vision::image_processor::PreprocessedImages,
+};
+
+pub(super) struct KimiK25VisionSpec;
+
+impl KimiK25VisionSpec {
+    /// The repeated pad token (`<|media_pad|>`) — `media_placeholder_token_id` in config.
+    fn pad_token_id(metadata: &ModelMetadata) -> RegistryResult<TokenId> {
+        metadata
+            .config_u32(&["media_placeholder_token_id"])
+            .map(|v| v as TokenId)
+            .ok_or_else(|| ModelRegistryError::MissingConfigField {
+                field: "media_placeholder_token_id".to_string(),
+            })
+    }
+}
+
+impl ModelProcessorSpec for KimiK25VisionSpec {
+    fn name(&self) -> &'static str {
+        "kimi_k25"
+    }
+
+    fn matches(&self, metadata: &ModelMetadata) -> bool {
+        let id = metadata.model_id.to_ascii_lowercase();
+        id.contains("kimi") && id.contains("k2")
+            || metadata
+                .config_model_type()
+                .is_some_and(|mt| mt == "kimi_k25")
+    }
+
+    fn placeholder_token(&self, _metadata: &ModelMetadata) -> RegistryResult<String> {
+        Ok("<|media_pad|>".to_string())
+    }
+
+    fn placeholder_token_id(&self, metadata: &ModelMetadata) -> RegistryResult<TokenId> {
+        Self::pad_token_id(metadata)
+    }
+
+    fn modality_limits(
+        &self,
+        _metadata: &ModelMetadata,
+    ) -> RegistryResult<HashMap<Modality, usize>> {
+        Ok(HashMap::from([(Modality::Image, 10)]))
+    }
+
+    fn processor_kwargs(&self, _metadata: &ModelMetadata) -> RegistryResult<Value> {
+        Ok(json!({}))
+    }
+
+    fn prompt_replacements(
+        &self,
+        metadata: &ModelMetadata,
+        preprocessed: &PreprocessedImages,
+    ) -> RegistryResult<Vec<PromptReplacement>> {
+        let pad_token_id = Self::pad_token_id(metadata)?;
+        let placeholder_token = self.placeholder_token(metadata)?;
+        Ok(preprocessed
+            .num_img_tokens
+            .iter()
+            .map(|&num_tokens| {
+                PromptReplacement::repeated(
+                    Modality::Image,
+                    &placeholder_token,
+                    pad_token_id,
+                    num_tokens,
+                )
+            })
+            .collect())
+    }
+
+    fn field_layouts(&self) -> HashMap<String, FieldLayout> {
+        // Kimi-K2.5 uses NaViT-style patchification:
+        // pixel_values is [total_patches, patch_features], split by patches_per_image.
+        // grid_thws is [num_images, 3] with (temporal, height, width) grid dimensions.
+        HashMap::from([
+            (
+                "pixel_values".to_string(),
+                FieldLayout::flat("patches_per_image"),
+            ),
+            ("grid_thws".to_string(), FieldLayout::Batched),
+            ("patches_per_image".to_string(), FieldLayout::Batched),
+        ])
+    }
+
+    fn keep_on_cpu_keys(&self) -> Vec<String> {
+        vec!["grid_thws".to_string()]
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use serde_json::json;
+
+    use crate::{
+        registry::{test_helpers::*, ModelMetadata, ModelRegistry},
+        types::ImageSize,
+    };
+
+    #[test]
+    fn kimi_k25_matches_model_id() {
+        let tokenizer = TestTokenizer::new(&[("<|media_pad|>", 163605)]);
+        let config = json!({
+            "model_type": "kimi_k25",
+            "media_placeholder_token_id": 163605
+        });
+        let metadata = ModelMetadata {
+            model_id: "moonshotai/Kimi-K2.5",
+            tokenizer: &tokenizer,
+            config: &config,
+        };
+        let registry = ModelRegistry::new();
+        let spec = registry.lookup(&metadata).expect("kimi_k25 spec");
+        assert_eq!(spec.name(), "kimi_k25");
+    }
+
+    #[test]
+    fn kimi_k25_prompt_replacements() {
+        let tokenizer = TestTokenizer::new(&[("<|media_pad|>", 163605)]);
+        let config = json!({
+            "model_type": "kimi_k25",
+            "media_placeholder_token_id": 163605
+        });
+        let metadata = ModelMetadata {
+            model_id: "moonshotai/Kimi-K2.5",
+            tokenizer: &tokenizer,
+            config: &config,
+        };
+        let registry = ModelRegistry::new();
+        let spec = registry.lookup(&metadata).expect("kimi_k25 spec");
+
+        let replacements = spec
+            .prompt_replacements(
+                &metadata,
+                &test_preprocessed_with_tokens(&[ImageSize::new(448, 448)], &[256]),
+            )
+            .unwrap();
+
+        // 256 pad tokens (no start/end wrapper — SGLang handles that in the chat template)
+        assert_eq!(replacements[0].tokens.len(), 256);
+        assert!(replacements[0].tokens.iter().all(|&t| t == 163605));
+    }
+
+    #[test]
+    fn kimi_k25_prompt_replacements_multiple_images() {
+        let tokenizer = TestTokenizer::new(&[("<|media_pad|>", 163605)]);
+        let config = json!({
+            "model_type": "kimi_k25",
+            "media_placeholder_token_id": 163605
+        });
+        let metadata = ModelMetadata {
+            model_id: "moonshotai/Kimi-K2.5",
+            tokenizer: &tokenizer,
+            config: &config,
+        };
+        let registry = ModelRegistry::new();
+        let spec = registry.lookup(&metadata).expect("kimi_k25 spec");
+
+        let replacements = spec
+            .prompt_replacements(
+                &metadata,
+                &test_preprocessed_with_tokens(
+                    &[ImageSize::new(448, 448), ImageSize::new(224, 224)],
+                    &[256, 64],
+                ),
+            )
+            .unwrap();
+
+        assert_eq!(replacements.len(), 2);
+        assert_eq!(replacements[0].tokens.len(), 256);
+        assert_eq!(replacements[1].tokens.len(), 64);
+        assert!(replacements[1].tokens.iter().all(|&t| t == 163605));
+    }
+
+    #[test]
+    fn kimi_k25_matches_kimi_k2_variant() {
+        let tokenizer = TestTokenizer::new(&[("<|media_pad|>", 163605)]);
+        let config = json!({
+            "model_type": "kimi_k25",
+            "media_placeholder_token_id": 163605
+        });
+        let metadata = ModelMetadata {
+            model_id: "moonshotai/Kimi-K2-VL",
+            tokenizer: &tokenizer,
+            config: &config,
+        };
+        let registry = ModelRegistry::new();
+        let spec = registry.lookup(&metadata);
+        assert!(spec.is_some(), "Should match Kimi-K2 variants");
+    }
+
+    #[test]
+    fn kimi_k25_does_not_match_kimi_k1() {
+        let tokenizer = TestTokenizer::new(&[("<|media_pad|>", 163605)]);
+        let config = json!({
+            "model_type": "kimi_k1",
+            "media_placeholder_token_id": 163605
+        });
+        let metadata = ModelMetadata {
+            model_id: "moonshotai/Kimi-K1-VL",
+            tokenizer: &tokenizer,
+            config: &config,
+        };
+        let registry = ModelRegistry::new();
+        let spec = registry.lookup(&metadata);
+        assert!(spec.is_none(), "Should not match Kimi-K1");
+    }
+}

crates/multimodal/src/registry/mod.rs

@@ -1,10 +1,12 @@
+mod kimi_k25;
 mod llama4;
 mod llava;
 mod phi3_v;
 mod qwen3_vl;
 mod qwen_vl;
 mod traits;
 
+use kimi_k25::KimiK25VisionSpec;
 use llama4::Llama4Spec;
 use llava::{LlavaNextSpec, LlavaSpec};
 use once_cell::sync::Lazy;
@@ -22,6 +24,7 @@ impl ModelRegistry {
     pub fn new() -> Self {
         Self {
             specs: vec![
+                LazySpec::new("kimi_k25", || Box::new(KimiK25VisionSpec)),
                 LazySpec::new("llama4", || Box::new(Llama4Spec)),
                 // LlavaNext must be registered before Llava so "llava_next" model_type matches first.
                 LazySpec::new("llava_next", || Box::new(LlavaNextSpec)),

crates/multimodal/src/vision/image_processor.rs

@@ -464,6 +464,16 @@ impl ImageProcessorRegistry {
             Box::new(super::processors::Llama4VisionProcessor::new()),
         );
 
+        // Register Kimi-K2.5 Vision
+        registry.register(
+            "kimi-k2",
+            Box::new(super::processors::KimiK25Processor::new()),
+        );
+        registry.register(
+            "kimi_k2",
+            Box::new(super::processors::KimiK25Processor::new()),
+        );
+
         registry
     }
 }

crates/multimodal/src/vision/preprocessor_config.rs

@@ -230,13 +230,63 @@ pub struct PreProcessorConfig {
 
 impl PreProcessorConfig {
     /// Parse from JSON string.
+    ///
+    /// Handles both standard HuggingFace format (top-level fields) and Kimi-K2.5's
+    /// nested format where values are under `media_proc_cfg`.
     pub fn from_json(json: &str) -> Result<Self, serde_json::Error> {
-        serde_json::from_str(json)
+        let raw: serde_json::Value = serde_json::from_str(json)?;
+        Self::from_value(raw)
     }
 
     /// Parse from JSON value.
+    ///
+    /// Handles both standard HuggingFace format (top-level fields) and Kimi-K2.5's
+    /// nested format where values are under `media_proc_cfg`.
     pub fn from_value(value: serde_json::Value) -> Result<Self, serde_json::Error> {
-        serde_json::from_value(value)
+        let mut config: Self = serde_json::from_value(value.clone())?;
+        Self::apply_nested_media_cfg(&mut config, &value);
+        Ok(config)
+    }
+
+    /// Extract values from nested `media_proc_cfg` (used by Kimi-K2.5 and
+    /// similar models) when top-level fields are missing.
+    fn apply_nested_media_cfg(config: &mut Self, raw: &serde_json::Value) {
+        let Some(media_cfg) = raw.get("media_proc_cfg") else {
+            return;
+        };
+        if config.image_mean.is_none() {
+            config.image_mean = media_cfg
+                .get("image_mean")
+                .and_then(|v| serde_json::from_value(v.clone()).ok());
+        }
+        if config.image_std.is_none() {
+            config.image_std = media_cfg
+                .get("image_std")
+                .and_then(|v| serde_json::from_value(v.clone()).ok());
+        }
+        if config.patch_size.is_none() {
+            config.patch_size = media_cfg.get("patch_size").and_then(|v| {
+                v.as_u64().map(|ps| PatchSize {
+                    height: Some(ps as u32),
+                    width: Some(ps as u32),
+                })
+            });
+        }
+        if config.merge_size.is_none() {
+            config.merge_size = media_cfg
+                .get("merge_kernel_size")
+                .and_then(|v| v.as_u64())
+                .map(|v| v as usize);
+        }
+        // Also extract Kimi-specific limits into the extra map
+        // so processors can read them via get_extra()
+        for key in ["in_patch_limit", "patch_limit_on_one_side"] {
+            if !config.extra.contains_key(key) {
+                if let Some(v) = media_cfg.get(key) {
+                    config.extra.insert(key.to_string(), v.clone());
+                }
+            }
+        }
     }
 
     /// Get patch size as a simple usize.
@@ -467,4 +517,35 @@ mod tests {
             Some(&"bar".to_string())
         );
     }
+
+    #[test]
+    fn test_parse_kimi_nested_media_proc_cfg() {
+        let json = r#"{
+            "auto_map": {
+                "AutoProcessor": "kimi_k25_processor.KimiK25Processor"
+            },
+            "media_proc_cfg": {
+                "in_patch_limit": 16384,
+                "patch_size": 14,
+                "image_mean": [0.5, 0.5, 0.5],
+                "image_std": [0.5, 0.5, 0.5],
+                "merge_kernel_size": 2,
+                "patch_limit_on_one_side": 512
+            }
+        }"#;
+
+        let config = PreProcessorConfig::from_json(json).unwrap();
+
+        // image_mean/std should be extracted from media_proc_cfg
+        let mean = config.get_image_mean();
+        assert!((mean[0] - 0.5).abs() < 1e-6);
+        assert!((mean[1] - 0.5).abs() < 1e-6);
+        assert!((mean[2] - 0.5).abs() < 1e-6);
+
+        let std = config.get_image_std();
+        assert!((std[0] - 0.5).abs() < 1e-6);
+
+        assert_eq!(config.get_patch_size(0), 14);
+        assert_eq!(config.merge_size, Some(2));
+    }
 }