From df20252be590c0f983d11e084d1598378743b8cc Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 29 Jul 2026 08:34:03 +0000 Subject: [PATCH 1/3] kimi-k3 : fixes for loading and running the full-size model Graph node budget, LLM_TYPE for the 93-layer config, four hparams promoted to required, and an E8M0 NaN check during the MXFP4 repack. Details in the PR description. Assisted-by: Claude Code --- src/models/kimi-k3.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/models/kimi-k3.cpp b/src/models/kimi-k3.cpp index d952d72cdf1..e634e9507be 100644 --- a/src/models/kimi-k3.cpp +++ b/src/models/kimi-k3.cpp @@ -36,12 +36,16 @@ void llama_model_kimi_k3::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false); ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false); ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func); - ml.get_key(LLM_KV_EXPERT_LATENT_LENGTH, hparams.n_expert_latent, false); + // required: a silent default here loads cleanly and produces garbage + ml.get_key(LLM_KV_EXPERT_LATENT_LENGTH, hparams.n_expert_latent); ml.get_key(LLM_KV_ATTN_RES_BLOCK_SIZE, hparams.attn_res_block_size); ml.get_key(LLM_KV_ACTIVATION_SITU_BETA, hparams.situ_beta); ml.get_key(LLM_KV_ACTIVATION_SITU_LINEAR_BETA, hparams.situ_linear_beta); + GGML_ASSERT(hparams.attn_res_block_size > 0 && "Kimi-K3 requires attn_res.block_size"); + GGML_ASSERT(hparams.n_expert_latent > 0 && "Kimi-K3 requires expert_latent_length"); + switch (hparams.n_layer()) { case 93: type = LLM_TYPE_2_8T_A50B; break; // Kimi-K3 default: type = LLM_TYPE_UNKNOWN; From 27fd568200c3eceebc1e47ac4d620694276457b4 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 29 Jul 2026 08:34:03 +0000 Subject: [PATCH 2/3] kimi-k3 : add the MoonViT-3d vision tower (image path) New kimik3 projector type, its graph builder and the mmproj converter. Also adds an optional clip.%s.attention.head_dim so build_vit stops deriving d_head from n_embd, which is wrong whenever a tower's qkv width differs from n_embd. Assisted-by: Claude Code --- conversion/kimivl.py | 76 ++++++++++++++++++++++++++++++++++ gguf-py/gguf/constants.py | 1 + tools/mtmd/CMakeLists.txt | 1 + tools/mtmd/clip-impl.h | 2 + tools/mtmd/clip.cpp | 31 ++++++++++++++ tools/mtmd/models/kimik3.cpp | 80 ++++++++++++++++++++++++++++++++++++ tools/mtmd/models/models.h | 7 ++++ tools/mtmd/mtmd.cpp | 1 + 8 files changed, 199 insertions(+) create mode 100644 tools/mtmd/models/kimik3.cpp diff --git a/conversion/kimivl.py b/conversion/kimivl.py index ae60abf3098..d37bf74ea5c 100644 --- a/conversion/kimivl.py +++ b/conversion/kimivl.py @@ -171,3 +171,79 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter name = name.replace("mm_projector.linear_", "mm_projector.proj.linear_", 1) yield from super().modify_tensors(data_torch, name, bid) + + +@ModelBase.register("KimiK3ForConditionalGeneration") +class KimiK3VisionModel(MmprojModel): + """Kimi-K3 MoonViT-3d vision tower (image path). + + Structurally the Kimi-K2.5 tower with RMSNorm, no biases, a non-square fused QKV + (qkv_hidden_size 1536 vs vt_hidden_size 1024) and a post-norm patchmergerv2 projector. + Video is out of scope: for t == 1 the temporal pool and temporal position term vanish. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + assert self.hparams_vision is not None, "Kimi-K3 requires vision_config in config.json" + self.merge_kernel_size = tuple(self.hparams_vision.get("merge_kernel_size", [2, 2])) + self.patch_size = self.hparams_vision.get("patch_size", 14) + pos_emb_h = self.hparams_vision.get("init_pos_emb_height", 64) + self.hparams_vision["image_size"] = pos_emb_h * self.patch_size + + def set_gguf_parameters(self): + super().set_gguf_parameters() + assert self.hparams_vision is not None + self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.KIMIK3) + + # qkv width != n_embd, so the runtime cannot derive d_head + n_head = self.hparams_vision["vt_num_attention_heads"] + qkv_hidden = self.hparams_vision.get("qkv_hidden_size") or self.hparams_vision["vt_hidden_size"] + assert qkv_hidden % n_head == 0, f"qkv_hidden_size {qkv_hidden} not divisible by {n_head} heads" + self.gguf_writer.add_vision_head_dim(qkv_hidden // n_head) + + self.gguf_writer.add_vision_use_gelu(True) # activation_func is gelu_pytorch_tanh + self.gguf_writer.add_vision_attention_layernorm_eps( + self.hparams_vision.get("projector_ln_eps", 1e-5)) + self.gguf_writer.add_vision_projector_scale_factor(self.merge_kernel_size[0]) + + in_patch_limit = self.preprocessor_config.get("media_proc_cfg", {}).get( + "in_patch_limit", self.preprocessor_config.get("in_patch_limit", 16384)) + pixels_per_patch = self.patch_size ** 2 + self.gguf_writer.add_vision_min_pixels(8 * pixels_per_patch) + self.gguf_writer.add_vision_max_pixels(in_patch_limit * pixels_per_patch) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, _ = item + if not name.startswith(("vision_tower.", "mm_projector.")): + return None + return super().filter_tensors(item) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + assert self.hparams_vision is not None + n_head = self.hparams_vision["vt_num_attention_heads"] + + if "wqkv" in name and "weight" in name: + # de-interleave Q/K so the runtime can use build_rope_2d(interleave_freq=false) + out_dim = data_torch.shape[0] + qkv_dim = out_dim // 3 + head_dim = qkv_dim // n_head + wq, wk, wv = (data_torch[:qkv_dim], data_torch[qkv_dim:2 * qkv_dim], data_torch[2 * qkv_dim:]) + + def deinterleave(w: Tensor) -> Tensor: + return (w.reshape(n_head, head_dim // 4, 2, 2, w.shape[-1]) + .permute(0, 2, 1, 3, 4) + .reshape(w.shape[0], w.shape[-1])) + + data_torch = torch.cat([deinterleave(wq), deinterleave(wk), wv], dim=0) + + if "pos_emb.weight" in name: + # kept 3D: the runtime reads grid extents from ne[1]/ne[2] + pass + + if "mm_projector.proj.0." in name: + name = name.replace(".proj.0.", ".proj.linear_1.") + elif "mm_projector.proj.2." in name: + name = name.replace(".proj.2.", ".proj.linear_2.") + + yield from super().modify_tensors(data_torch, name, bid) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index d043c9b6ecc..f7ff5c30212 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -5422,6 +5422,7 @@ class VisionProjectorType: KIMIVL = "kimivl" PADDLEOCR = "paddleocr" KIMIK25 = "kimik25" + KIMIK3 = "kimik3" LIGHTONOCR = "lightonocr" COGVLM = "cogvlm" JANUS_PRO = "janus_pro" diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt index 769a44e0b73..5fab771830c 100644 --- a/tools/mtmd/CMakeLists.txt +++ b/tools/mtmd/CMakeLists.txt @@ -40,6 +40,7 @@ add_library(mtmd models/granite4-vision.cpp models/hunyuanvl.cpp models/internvl.cpp + models/kimik3.cpp models/kimivl.cpp models/kimik25.cpp models/nemotron-v2-vl.cpp diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index 2c9ea499ce1..29914791f88 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -478,6 +478,7 @@ enum projector_type { PROJECTOR_TYPE_YOUTUVL, PROJECTOR_TYPE_YASA2, PROJECTOR_TYPE_KIMIK25, + PROJECTOR_TYPE_KIMIK3, PROJECTOR_TYPE_NEMOTRON_V2_VL, PROJECTOR_TYPE_HUNYUANVL, PROJECTOR_TYPE_PARAKEET, @@ -540,6 +541,7 @@ static std::map PROJECTOR_TYPE_NAMES = { { PROJECTOR_TYPE_YOUTUVL, "youtuvl"}, { PROJECTOR_TYPE_YASA2, "yasa2"}, { PROJECTOR_TYPE_KIMIK25, "kimik25"}, + { PROJECTOR_TYPE_KIMIK3, "kimik3"}, { PROJECTOR_TYPE_NEMOTRON_V2_VL, "nemotron_v2_vl"}, { PROJECTOR_TYPE_EXAONE4_5, "exaone4_5"}, { PROJECTOR_TYPE_HUNYUANVL, "hunyuanvl"}, diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index b9dd5e8452e..bde73761f54 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -1007,6 +1007,10 @@ static std::unique_ptr clip_get_graph_builder(clip_ctx * ctx, const { builder = std::make_unique(ctx, img); } break; + case PROJECTOR_TYPE_KIMIK3: + { + builder = std::make_unique(ctx, img); + } break; case PROJECTOR_TYPE_COGVLM: { builder = std::make_unique(ctx, img); @@ -1526,6 +1530,23 @@ struct clip_model_loader { hparams.rope_theta = 10000.0f; get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); + int min_pixels = 0, max_pixels = 0; + get_u32(KEY_IMAGE_MIN_PIXELS, min_pixels, false); + get_u32(KEY_IMAGE_MAX_PIXELS, max_pixels, false); + if (min_pixels > 0 && max_pixels > 0) { + hparams.image_min_pixels = min_pixels; + hparams.image_max_pixels = max_pixels; + hparams.warmup_image_size = static_cast(std::sqrt(max_pixels)); + } else { + hparams.set_limit_image_tokens(2, 4096); + } + } break; + case PROJECTOR_TYPE_KIMIK3: + { + hparams.image_resize_algo = RESIZE_ALGO_BILINEAR; + hparams.rope_theta = 10000.0f; + get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); + int min_pixels = 0, max_pixels = 0; get_u32(KEY_IMAGE_MIN_PIXELS, min_pixels, false); get_u32(KEY_IMAGE_MAX_PIXELS, max_pixels, false); @@ -2645,6 +2666,13 @@ struct clip_model_loader { model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight")); model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias")); } break; + case PROJECTOR_TYPE_KIMIK3: + { + // patchmergerv2, bias-free, norm after the projection + model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight")); + model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight")); + model.mm_post_norm_w = get_tensor(string_format(TN_MM_POST_NORM, "weight")); + } break; case PROJECTOR_TYPE_KIMIVL: case PROJECTOR_TYPE_PADDLEOCR: case PROJECTOR_TYPE_KIMIK25: @@ -4067,6 +4095,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { case PROJECTOR_TYPE_LFM2: case PROJECTOR_TYPE_KIMIVL: case PROJECTOR_TYPE_KIMIK25: + case PROJECTOR_TYPE_KIMIK3: { // dynamic size int out_patch_size = params.patch_size * ctx->model.hparams.n_merge; @@ -4922,6 +4951,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { case PROJECTOR_TYPE_PIXTRAL: case PROJECTOR_TYPE_KIMIVL: case PROJECTOR_TYPE_KIMIK25: + case PROJECTOR_TYPE_KIMIK3: case PROJECTOR_TYPE_LIGHTONOCR: { // set the 2D positions @@ -5770,6 +5800,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { case PROJECTOR_TYPE_KIMIVL: case PROJECTOR_TYPE_PADDLEOCR: case PROJECTOR_TYPE_KIMIK25: + case PROJECTOR_TYPE_KIMIK3: case PROJECTOR_TYPE_YASA2: return ctx->model.mm_2_w->ne[1]; case PROJECTOR_TYPE_HUNYUANVL: diff --git a/tools/mtmd/models/kimik3.cpp b/tools/mtmd/models/kimik3.cpp new file mode 100644 index 00000000000..fe06c7fdbb3 --- /dev/null +++ b/tools/mtmd/models/kimik3.cpp @@ -0,0 +1,80 @@ +#include "models.h" + +#include +#include + +// Kimi-K3 MoonViT-3d, image path. +// Follows clip_graph_kimik25, but with RMSNorm, no biases, qkv width != n_embd, and a post-norm patchmergerv2 projector. +// Images only: at t == 1 the temporal pool and the temporal position term vanish. + +ggml_tensor * clip_graph_kimik3::resize_position_embeddings_3d(uint32_t interpolation_mode) { + ggml_tensor * pos_embd = model.position_embeddings; + const int height = img.ny() / patch_size; + const int width = img.nx() / patch_size; + + GGML_ASSERT(pos_embd); + + const int64_t stored_c = pos_embd->ne[0]; + const int64_t orig_w = pos_embd->ne[1]; + const int64_t orig_h = pos_embd->ne[2]; + + GGML_ASSERT(stored_c == n_embd); + + if (height == (int) orig_h && width == (int) orig_w) { + return ggml_cont_2d(ctx0, pos_embd, n_embd, width * height); + } + + pos_embd = ggml_permute(ctx0, pos_embd, 2, 1, 0, 3); + pos_embd = ggml_interpolate(ctx0, pos_embd, height, width, n_embd, 1, interpolation_mode); + pos_embd = ggml_permute(ctx0, pos_embd, 2, 1, 0, 3); + pos_embd = ggml_cont_2d(ctx0, pos_embd, n_embd, width * height); + return pos_embd; +} + +ggml_cgraph * clip_graph_kimik3::build() { + ggml_tensor * pos_h = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches); + ggml_set_name(pos_h, "pos_h"); + ggml_set_input(pos_h); + + ggml_tensor * pos_w = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches); + ggml_set_name(pos_w, "pos_w"); + ggml_set_input(pos_w); + + ggml_tensor * learned_pos_embd = resize_position_embeddings_3d(GGML_SCALE_MODE_BILINEAR); + + // Q/K are de-interleaved during conversion. + auto add_pos = [&](ggml_tensor * cur, const clip_layer &) { + return build_rope_2d(ctx0, cur, pos_w, pos_h, hparams.rope_theta, false); + }; + + ggml_tensor * inp = build_inp(); + inp = ggml_add(ctx0, inp, learned_pos_embd); + + ggml_tensor * cur = build_vit( + inp, n_patches, + NORM_TYPE_RMS, + hparams.ffn_op, + nullptr, + add_pos); + cb(cur, "vit_out", -1); + + { + const int scale_factor = model.hparams.n_merge; + cur = build_patch_merge_permute(cur, scale_factor); + + cur = build_ffn(cur, + model.mm_1_w, nullptr, + nullptr, nullptr, + model.mm_2_w, nullptr, + FFN_GELU, + -1); + cb(cur, "proj_mlp_out", -1); + + cur = build_norm(cur, model.mm_post_norm_w, nullptr, NORM_TYPE_RMS, hparams.eps, -1); + cb(cur, "proj_out", -1); + } + + ggml_build_forward_expand(gf, cur); + + return gf; +} diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h index 3631d849b65..59921c594b3 100644 --- a/tools/mtmd/models/models.h +++ b/tools/mtmd/models/models.h @@ -394,6 +394,13 @@ struct clip_graph_parakeet : clip_graph { ggml_cgraph * build() override; }; +struct clip_graph_kimik3 : clip_graph { + clip_graph_kimik3(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} + ggml_cgraph * build() override; + + ggml_tensor * resize_position_embeddings_3d(uint32_t interpolation_mode); +}; + struct clip_graph_exaone4_5 : clip_graph { clip_graph_exaone4_5(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} ggml_cgraph * build() override; diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 6596b2785f5..93356d5e969 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -797,6 +797,7 @@ struct mtmd_context { image_preproc = std::make_unique(ctx_v); } break; case PROJECTOR_TYPE_KIMIK25: + case PROJECTOR_TYPE_KIMIK3: { // GLM-5.2-V reuses the Kimi-K2.5 vision encoder and projector, but marks // images with its own tokens, so decide based on the text model vocab From edfd4c1a3b7a653303a85257ddac2a1f3ce39a2f Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Thu, 6 Aug 2026 08:04:05 +0000 Subject: [PATCH 3/3] kimi-k3 : declare ssm_a as SSM_A_NOSCAN so it stays on the GPU build_kda consumes ssm_a through a broadcast ggml_mul, never through ggml_ssm_scan, but it was registered as LLM_TENSOR_SSM_A, which llama-arch.cpp maps to GGML_OP_SSM_SCAN. create_tensor probes the buffer type with that op through weight_buft_supported, no backend offers SSM_SCAN for a [n_head] tensor, the probe fails, and ssm_a is placed on the CPU along with everything downstream of it. LLM_TENSOR_SSM_A_NOSCAN exists for exactly this case and maps to GGML_OP_MUL. Both spellings resolve to the same "blk.%d.ssm_a" name, so there is no GGUF change and no converter change. qwen3next.cpp and qwen35moe.cpp already declare their A this way. Measured on Kimi-K3-Mini across two GPUs, llama-bench pp2048: before 10705.91 +- 445.93 tok/s after 17009.96 +- 360.89 tok/s 1.59x The loader reports it directly. Before the change it prints "tensor 'token_embd.weight' (bf16) (and 4 others) cannot be used with preferred buffer type CUDA_Host, using CPU instead"; after it prints "(and 0 others)". --- src/models/kimi-k3.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/models/kimi-k3.cpp b/src/models/kimi-k3.cpp index e634e9507be..5eedee76852 100644 --- a/src/models/kimi-k3.cpp +++ b/src/models/kimi-k3.cpp @@ -97,8 +97,11 @@ void llama_model_kimi_k3::load_arch_tensors(llama_model_loader &) { layer.ssm_f_b = create_tensor(tn(LLM_TENSOR_SSM_F_B, "weight", i), {head_dim, d_inner}, 0); layer.ssm_beta = create_tensor(tn(LLM_TENSOR_SSM_BETA, "weight", i), {n_embd, n_head}, 0); - // K3's A_log is a plain 1-D [n_head] tensor (kimi-linear's is padded) - layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A, i), {n_head}, 0); + // K3's A_log is a plain 1-D [n_head] tensor (kimi-linear's is padded). + // NOSCAN: build_kda consumes it via a broadcast ggml_mul, not ggml_ssm_scan. + // Same "blk.%d.ssm_a" name, but SSM_A declares GGML_OP_SSM_SCAN, and no backend + // offers that at [n_head], so the create_tensor probe fails and it lands on CPU. + layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A_NOSCAN, i), {n_head}, 0); layer.ssm_dt_b = create_tensor(tn(LLM_TENSOR_SSM_DT, "bias", i), {d_inner}, 0); // K3 uses a single full-rank gate instead of kimi-linear's g_a/g_b pair