Skip to content
Closed
Show file tree
Hide file tree
Changes from 20 commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
c864e7c
feat(multimodal): add Kimi-K2.5 vision support for gRPC router
Kangyan-Zhou Mar 31, 2026
c658a4f
fix(multimodal): download config.json from HF Hub when not available …
Kangyan-Zhou Apr 1, 2026
1aecc0b
fix(multimodal): improve error handling in HF Hub config download
Kangyan-Zhou Apr 1, 2026
fc07c18
fix(tokenizer): use encode_with_special_tokens for tiktoken encoding
Kangyan-Zhou Apr 1, 2026
d7e2611
fix(multimodal): rename image_grid_thw to grid_thws for Kimi-K2.5
Kangyan-Zhou Apr 1, 2026
39a1d9c
refactor(multimodal): make KimiK25Processor fully standalone
Kangyan-Zhou Apr 1, 2026
ac10e7c
fix(multimodal): serialize Kimi pixel_values as 4D [N, 3, 14, 14]
Kangyan-Zhou Apr 1, 2026
c55c24f
fix(multimodal): match HF Kimi preprocessor resize+pad pipeline
Kangyan-Zhou Apr 1, 2026
50fde3b
fix(multimodal): use download_tokenizer_from_hf for config resolution
Kangyan-Zhou Apr 1, 2026
2a6d58a
fix(multimodal): add download_model_configs_from_hf for config.json
Kangyan-Zhou Apr 1, 2026
1252b93
debug(multimodal): add info logging for image preprocessing pipeline
Kangyan-Zhou Apr 1, 2026
ac57067
fix(multimodal): use Kimi normalization, not CLIP fallback
Kangyan-Zhou Apr 1, 2026
e7e15b4
fix(multimodal): parse Kimi nested media_proc_cfg in preprocessor config
Kangyan-Zhou Apr 1, 2026
4157f79
chore(multimodal): remove debug logging from image preprocessing
Kangyan-Zhou Apr 1, 2026
9739ee2
fix(multimodal): address PR review findings
Kangyan-Zhou Apr 1, 2026
5bc71cb
perf(multimodal): fuse resize+pad+normalize and optimize patch extrac…
Kangyan-Zhou Apr 1, 2026
f6bad75
debug(multimodal): add per-step timing for multimodal preprocessing
Kangyan-Zhou Apr 1, 2026
f653eca
perf(multimodal): spawn_blocking for preprocessing + strip decode mm_…
Kangyan-Zhou Apr 1, 2026
66fa5ba
perf(multimodal): use SIMD-accelerated resize for Kimi preprocessing
Kangyan-Zhou Apr 2, 2026
0bd6f07
chore(multimodal): remove per-step timing instrumentation
Kangyan-Zhou Apr 2, 2026
038c73d
fix(multimodal): address PR review findings
Kangyan-Zhou Apr 4, 2026
adb44b8
docs(multimodal): remove Qwen-VL references from Kimi comments
Kangyan-Zhou Apr 6, 2026
a21f971
fix(servicer): return MultimodalInputs object instead of dict from _p…
Kangyan-Zhou Apr 6, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
211 changes: 211 additions & 0 deletions crates/multimodal/src/registry/kimi_k25.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,211 @@
use std::collections::HashMap;

use serde_json::{json, Value};

use crate::{
registry::{ModelMetadata, ModelProcessorSpec, ModelRegistryError, RegistryResult},
types::{FieldLayout, Modality, PromptReplacement, TokenId},
vision::image_processor::PreprocessedImages,
};

pub(super) struct KimiK25VisionSpec;

impl KimiK25VisionSpec {
/// The repeated pad token (`<|media_pad|>`) — `media_placeholder_token_id` in config.
fn pad_token_id(metadata: &ModelMetadata) -> RegistryResult<TokenId> {
metadata
.config_u32(&["media_placeholder_token_id"])
.map(|v| v as TokenId)
.ok_or_else(|| ModelRegistryError::MissingConfigField {
field: "media_placeholder_token_id".to_string(),
})
}
}

impl ModelProcessorSpec for KimiK25VisionSpec {
fn name(&self) -> &'static str {
"kimi_k25"
}

fn matches(&self, metadata: &ModelMetadata) -> bool {
let id = metadata.model_id.to_ascii_lowercase();
id.contains("kimi") && id.contains("k2")
}

fn placeholder_token(&self, _metadata: &ModelMetadata) -> RegistryResult<String> {
Ok("<|media_pad|>".to_string())
}

fn placeholder_token_id(&self, metadata: &ModelMetadata) -> RegistryResult<TokenId> {
Self::pad_token_id(metadata)
}

fn modality_limits(
&self,
_metadata: &ModelMetadata,
) -> RegistryResult<HashMap<Modality, usize>> {
Ok(HashMap::from([(Modality::Image, 10)]))
}

fn processor_kwargs(&self, _metadata: &ModelMetadata) -> RegistryResult<Value> {
Ok(json!({}))
}

fn prompt_replacements(
&self,
metadata: &ModelMetadata,
preprocessed: &PreprocessedImages,
) -> RegistryResult<Vec<PromptReplacement>> {
let pad_token_id = Self::pad_token_id(metadata)?;
let placeholder_token = self.placeholder_token(metadata)?;
Ok(preprocessed
.num_img_tokens
.iter()
.map(|&num_tokens| {
PromptReplacement::repeated(
Modality::Image,
&placeholder_token,
pad_token_id,
num_tokens,
)
})
.collect())
}

fn field_layouts(&self) -> HashMap<String, FieldLayout> {
// Kimi-K2.5 uses NaViT-style patchification like Qwen-VL:
// pixel_values is [total_patches, patch_features], split by patches_per_image.
// grid_thws is [num_images, 3] (Kimi uses "grid_thws" instead of Qwen's "image_grid_thw").
HashMap::from([
(
"pixel_values".to_string(),
FieldLayout::flat("patches_per_image"),
),
("grid_thws".to_string(), FieldLayout::Batched),
("patches_per_image".to_string(), FieldLayout::Batched),
])
}

fn keep_on_cpu_keys(&self) -> Vec<String> {
vec!["grid_thws".to_string()]
}
}

#[cfg(test)]
mod tests {
use serde_json::json;

use crate::{
registry::{test_helpers::*, ModelMetadata, ModelRegistry},
types::ImageSize,
};

#[test]
fn kimi_k25_matches_model_id() {
let tokenizer = TestTokenizer::new(&[("<|media_pad|>", 163605)]);
let config = json!({
"model_type": "kimi_k25",
"media_placeholder_token_id": 163605
});
let metadata = ModelMetadata {
model_id: "moonshotai/Kimi-K2.5",
tokenizer: &tokenizer,
config: &config,
};
let registry = ModelRegistry::new();
let spec = registry.lookup(&metadata).expect("kimi_k25 spec");
assert_eq!(spec.name(), "kimi_k25");
}

#[test]
fn kimi_k25_prompt_replacements() {
let tokenizer = TestTokenizer::new(&[("<|media_pad|>", 163605)]);
let config = json!({
"model_type": "kimi_k25",
"media_placeholder_token_id": 163605
});
let metadata = ModelMetadata {
model_id: "moonshotai/Kimi-K2.5",
tokenizer: &tokenizer,
config: &config,
};
let registry = ModelRegistry::new();
let spec = registry.lookup(&metadata).expect("kimi_k25 spec");

let replacements = spec
.prompt_replacements(
&metadata,
&test_preprocessed_with_tokens(&[ImageSize::new(448, 448)], &[256]),
)
.unwrap();

// 256 pad tokens (no start/end wrapper — SGLang handles that in the chat template)
assert_eq!(replacements[0].tokens.len(), 256);
assert!(replacements[0].tokens.iter().all(|&t| t == 163605));
}

#[test]
fn kimi_k25_prompt_replacements_multiple_images() {
let tokenizer = TestTokenizer::new(&[("<|media_pad|>", 163605)]);
let config = json!({
"model_type": "kimi_k25",
"media_placeholder_token_id": 163605
});
let metadata = ModelMetadata {
model_id: "moonshotai/Kimi-K2.5",
tokenizer: &tokenizer,
config: &config,
};
let registry = ModelRegistry::new();
let spec = registry.lookup(&metadata).expect("kimi_k25 spec");

let replacements = spec
.prompt_replacements(
&metadata,
&test_preprocessed_with_tokens(
&[ImageSize::new(448, 448), ImageSize::new(224, 224)],
&[256, 64],
),
)
.unwrap();

assert_eq!(replacements.len(), 2);
assert_eq!(replacements[0].tokens.len(), 256);
assert_eq!(replacements[1].tokens.len(), 64);
assert!(replacements[1].tokens.iter().all(|&t| t == 163605));
}

#[test]
fn kimi_k25_matches_kimi_k2_variant() {
let tokenizer = TestTokenizer::new(&[("<|media_pad|>", 163605)]);
let config = json!({
"model_type": "kimi_k25",
"media_placeholder_token_id": 163605
});
let metadata = ModelMetadata {
model_id: "moonshotai/Kimi-K2-VL",
tokenizer: &tokenizer,
config: &config,
};
let registry = ModelRegistry::new();
let spec = registry.lookup(&metadata);
assert!(spec.is_some(), "Should match Kimi-K2 variants");
}

#[test]
fn kimi_k25_does_not_match_kimi_k1() {
let tokenizer = TestTokenizer::new(&[("<|media_pad|>", 163605)]);
let config = json!({
"model_type": "kimi_k1",
"media_placeholder_token_id": 163605
});
let metadata = ModelMetadata {
model_id: "moonshotai/Kimi-K1-VL",
tokenizer: &tokenizer,
config: &config,
};
let registry = ModelRegistry::new();
let spec = registry.lookup(&metadata);
assert!(spec.is_none(), "Should not match Kimi-K1");
}
}
3 changes: 3 additions & 0 deletions crates/multimodal/src/registry/mod.rs
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
mod kimi_k25;
mod llama4;
mod llava;
mod phi3_v;
mod qwen3_vl;
mod qwen_vl;
mod traits;

use kimi_k25::KimiK25VisionSpec;
use llama4::Llama4Spec;
use llava::{LlavaNextSpec, LlavaSpec};
use once_cell::sync::Lazy;
Expand All @@ -22,6 +24,7 @@ impl ModelRegistry {
pub fn new() -> Self {
Self {
specs: vec![
LazySpec::new("kimi_k25", || Box::new(KimiK25VisionSpec)),
LazySpec::new("llama4", || Box::new(Llama4Spec)),
// LlavaNext must be registered before Llava so "llava_next" model_type matches first.
LazySpec::new("llava_next", || Box::new(LlavaNextSpec)),
Expand Down
10 changes: 10 additions & 0 deletions crates/multimodal/src/vision/image_processor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -464,6 +464,16 @@ impl ImageProcessorRegistry {
Box::new(super::processors::Llama4VisionProcessor::new()),
);

// Register Kimi-K2.5 Vision
registry.register(
"kimi-k2",
Box::new(super::processors::KimiK25Processor::new()),
);
registry.register(
"kimi_k2",
Box::new(super::processors::KimiK25Processor::new()),
);

registry
}
}
Expand Down
73 changes: 71 additions & 2 deletions crates/multimodal/src/vision/preprocessor_config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -230,13 +230,51 @@ pub struct PreProcessorConfig {

impl PreProcessorConfig {
/// Parse from JSON string.
///
/// Handles both standard HuggingFace format (top-level fields) and Kimi-K2.5's
/// nested format where values are under `media_proc_cfg`.
pub fn from_json(json: &str) -> Result<Self, serde_json::Error> {
serde_json::from_str(json)
let mut config: Self = serde_json::from_str(json)?;
// If top-level image_mean/std are missing, try to extract from media_proc_cfg
if config.image_mean.is_none() || config.image_std.is_none() {
if let Ok(raw) = serde_json::from_str::<serde_json::Value>(json) {
if let Some(media_cfg) = raw.get("media_proc_cfg") {
if config.image_mean.is_none() {
config.image_mean = media_cfg
.get("image_mean")
.and_then(|v| serde_json::from_value(v.clone()).ok());
}
if config.image_std.is_none() {
config.image_std = media_cfg
.get("image_std")
.and_then(|v| serde_json::from_value(v.clone()).ok());
}
if config.patch_size.is_none() {
config.patch_size = media_cfg.get("patch_size").and_then(|v| {
v.as_u64().map(|ps| PatchSize {
height: Some(ps as u32),
width: Some(ps as u32),
})
});
}
if config.merge_size.is_none() {
config.merge_size = media_cfg
.get("merge_kernel_size")
.and_then(|v| v.as_u64())
.map(|v| v as usize);
}
}
}
}
Ok(config)
}

/// Parse from JSON value.
///
/// Handles nested `media_proc_cfg` the same way as `from_json`.
pub fn from_value(value: serde_json::Value) -> Result<Self, serde_json::Error> {
serde_json::from_value(value)
let json_str = serde_json::to_string(&value)?;
Self::from_json(&json_str)
}

/// Get patch size as a simple usize.
Expand Down Expand Up @@ -467,4 +505,35 @@ mod tests {
Some(&"bar".to_string())
);
}

#[test]
fn test_parse_kimi_nested_media_proc_cfg() {
let json = r#"{
"auto_map": {
"AutoProcessor": "kimi_k25_processor.KimiK25Processor"
},
"media_proc_cfg": {
"in_patch_limit": 16384,
"patch_size": 14,
"image_mean": [0.5, 0.5, 0.5],
"image_std": [0.5, 0.5, 0.5],
"merge_kernel_size": 2,
"patch_limit_on_one_side": 512
}
}"#;

let config = PreProcessorConfig::from_json(json).unwrap();

// image_mean/std should be extracted from media_proc_cfg
let mean = config.get_image_mean();
assert!((mean[0] - 0.5).abs() < 1e-6);
assert!((mean[1] - 0.5).abs() < 1e-6);
assert!((mean[2] - 0.5).abs() < 1e-6);

let std = config.get_image_std();
assert!((std[0] - 0.5).abs() < 1e-6);

assert_eq!(config.get_patch_size(0), 14);
assert_eq!(config.merge_size, Some(2));
}
}
Loading
Loading