From 4bbf9aba39147465abca1d04e83853250dfc2a08 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Thu, 11 Jun 2026 18:28:20 -0600 Subject: [PATCH 01/29] feat(convert): Add conversion for GraniteSWAForCausalLM Branch: GraniteSWAForCausalLM AI-usage: full (Bob, OpenCode + Qwen3.6-35b) Signed-off-by: Gabe Goodhart --- conversion/__init__.py | 1 + conversion/granite.py | 40 +++++++++++++++++++++++++++++++++++++++ gguf-py/gguf/constants.py | 20 ++++++++++++++++++++ 3 files changed, 61 insertions(+) diff --git a/conversion/__init__.py b/conversion/__init__.py index 02102fac8d9..9bf378cd48b 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -100,6 +100,7 @@ "GraniteMoeSharedForCausalLM": "granite", "GraniteSpeechForConditionalGeneration": "granite", "GraniteSpeechPlusForConditionalGeneration": "granite", + "GraniteSWAForCausalLM": "granite", "Grok1ForCausalLM": "grok", "GrokForCausalLM": "grok", "GroveMoeForCausalLM": "grovemoe", diff --git a/conversion/granite.py b/conversion/granite.py index 8367ed225da..276297718e9 100644 --- a/conversion/granite.py +++ b/conversion/granite.py @@ -73,6 +73,46 @@ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Ca return super().filter_tensors(item) +@ModelBase.register("GraniteSWAForCausalLM") +class GraniteSWAModel(GraniteModel): + """Conversion for IBM's GraniteSWAForCausalLM (interleaved sliding window attention)""" + model_arch = gguf.MODEL_ARCH.GRANITE_SWA + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, gen = item + + # Add .weight suffix to sinks tensors (the C++ code expects blk.{bid}.attn_sinks.weight) + if "sinks" in name: + name += ".weight" + + return super().filter_tensors((name, gen)) + + def set_gguf_parameters(self): + """GraniteSWA uses Granite parameters plus sliding window configuration.""" + super().set_gguf_parameters() + + # Add sliding_window from config + sliding_window = self.hparams.get("sliding_window", 128) + self.gguf_writer.add_sliding_window(sliding_window) + logger.info("gguf: (granite_swa) sliding_window = %s", sliding_window) + + # Derive sliding_window_pattern from layer_types + if layer_types := self.hparams.get("layer_types"): + is_swa = [t == "sliding_attention" for t in layer_types] + self.gguf_writer.add_sliding_window_pattern(is_swa) + logger.info("gguf: (granite_swa) sliding_window_pattern = %d SWA layers / %d total", + sum(is_swa), len(is_swa)) + else: + # Fall back to period-based pattern: (i + 1) % 4 != 0 + # This matches the pattern for models without explicit layer_types + n_layers = self.block_count + is_swa = [(i + 1) % 4 != 0 for i in range(n_layers)] + self.gguf_writer.add_sliding_window_pattern(is_swa) + logger.info("gguf: (granite_swa) sliding_window_pattern (inferred) = %d SWA layers / %d total", + sum(is_swa), n_layers) + + @ModelBase.register("GraniteMoeForCausalLM", "GraniteMoeSharedForCausalLM") class GraniteMoeModel(GraniteModel): """Conversion for IBM's GraniteMoeForCausalLM""" diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 63ac2ed1f5c..33278ceae7e 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -499,6 +499,7 @@ class MODEL_ARCH(IntEnum): GRANITE = auto() GRANITE_MOE = auto() GRANITE_HYBRID = auto() + GRANITE_SWA = auto() CHAMELEON = auto() WAVTOKENIZER_DEC = auto() PLM = auto() @@ -1080,6 +1081,7 @@ class MODEL_TENSOR(IntEnum): MODEL_ARCH.GRANITE: "granite", MODEL_ARCH.GRANITE_MOE: "granitemoe", MODEL_ARCH.GRANITE_HYBRID: "granitehybrid", + MODEL_ARCH.GRANITE_SWA: "granite_swa", MODEL_ARCH.CHAMELEON: "chameleon", MODEL_ARCH.WAVTOKENIZER_DEC: "wavtokenizer-dec", MODEL_ARCH.PLM: "plm", @@ -3618,6 +3620,8 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.ATTN_SINKS, + MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.FFN_NORM, MODEL_TENSOR.FFN_GATE, MODEL_TENSOR.FFN_DOWN, @@ -3671,6 +3675,22 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, ], + MODEL_ARCH.GRANITE_SWA: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.ATTN_SINKS, + MODEL_TENSOR.ROPE_FREQS, + MODEL_TENSOR.FFN_NORM, + MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP, + ], MODEL_ARCH.CHAMELEON: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, From c74049a85f67a86c13db158883140e7d7f13f019 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Thu, 11 Jun 2026 18:30:23 -0600 Subject: [PATCH 02/29] feat(llama): Add granite_swa support Branch: GraniteSWAForCausalLM AI-usage: full (Bob, OpenCode + Qwen3.6-35b) Signed-off-by: Gabe Goodhart --- src/llama-arch.cpp | 1 + src/llama-arch.h | 1 + src/llama-model.cpp | 3 + src/models/granite-swa.cpp | 321 +++++++++++++++++++++++++++++++++++++ src/models/models.h | 28 ++++ 5 files changed, 354 insertions(+) create mode 100644 src/models/granite-swa.cpp diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 72968607db8..560aeb3c773 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -100,6 +100,7 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_GRANITE, "granite" }, { LLM_ARCH_GRANITE_MOE, "granitemoe" }, { LLM_ARCH_GRANITE_HYBRID, "granitehybrid" }, + { LLM_ARCH_GRANITE_SWA, "granite_swa" }, { LLM_ARCH_CHAMELEON, "chameleon" }, { LLM_ARCH_WAVTOKENIZER_DEC, "wavtokenizer-dec" }, { LLM_ARCH_PLM, "plm" }, diff --git a/src/llama-arch.h b/src/llama-arch.h index b74d53af4a3..61d885c0ea3 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -105,6 +105,7 @@ enum llm_arch { LLM_ARCH_GRANITE, LLM_ARCH_GRANITE_MOE, LLM_ARCH_GRANITE_HYBRID, + LLM_ARCH_GRANITE_SWA, LLM_ARCH_CHAMELEON, LLM_ARCH_WAVTOKENIZER_DEC, LLM_ARCH_PLM, diff --git a/src/llama-model.cpp b/src/llama-model.cpp index eaf3f35d2d8..2eb192d3e3d 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -232,6 +232,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params return new llama_model_minicpm(params); case LLM_ARCH_GRANITE_HYBRID: return new llama_model_granite_hybrid(params); + case LLM_ARCH_GRANITE_SWA: + return new llama_model_granite_swa(params); case LLM_ARCH_CHAMELEON: return new llama_model_chameleon(params); case LLM_ARCH_WAVTOKENIZER_DEC: @@ -2460,6 +2462,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { case LLM_ARCH_GRANITE: case LLM_ARCH_GRANITE_MOE: case LLM_ARCH_GRANITE_HYBRID: + case LLM_ARCH_GRANITE_SWA: case LLM_ARCH_CHAMELEON: case LLM_ARCH_BAILINGMOE: case LLM_ARCH_NEO_BERT: diff --git a/src/models/granite-swa.cpp b/src/models/granite-swa.cpp new file mode 100644 index 00000000000..d3069b8c894 --- /dev/null +++ b/src/models/granite-swa.cpp @@ -0,0 +1,321 @@ +#include "models.h" + +#include + +void llama_model_granite_swa::load_arch_hparams(llama_model_loader & ml) { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + ml.get_key(LLM_KV_LOGIT_SCALE, hparams.f_logit_scale); + ml.get_key(LLM_KV_RESIDUAL_SCALE, hparams.f_residual_scale, false); + ml.get_key(LLM_KV_EMBEDDING_SCALE, hparams.f_embedding_scale, false); + ml.get_key(LLM_KV_ATTENTION_SCALE, hparams.f_attention_scale, false); + + // iSWA configuration + ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer()); + ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); + hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; + + // Granite4 Vision uses array deepstack_mapping + ml.get_arr(LLM_KV_DEEPSTACK_MAPPING, hparams.deepstack_mapping_arr, false); + + // Count the unique deepstack input indices + std::unordered_set unique_deepstack_idxs; + for (const auto val : hparams.deepstack_mapping_arr) { + if (val >= 0) { + unique_deepstack_idxs.insert(val); + } + } + hparams.n_deepstack_layers = unique_deepstack_idxs.size(); + + // Ensure all values are valid (avoid overflow attacks) + for (const auto val : unique_deepstack_idxs) { + if (val > hparams.n_deepstack_layers) { + std::stringstream ss; + ss << "Invalid deepstack index: " << val << " > " << hparams.n_deepstack_layers; + throw std::runtime_error(ss.str()); + } + } + + // Granite uses rope_finetuned as a switch for rope, so default to true + bool rope_finetuned = true; + ml.get_key(LLM_KV_ROPE_SCALING_FINETUNED, rope_finetuned, false); + hparams.rope_finetuned = rope_finetuned; + + switch (hparams.n_layer()) { + case 32: type = LLM_TYPE_3B; break; + case 40: type = LLM_TYPE_3B; break; + // Add additional layer/vocab/etc checks here for other model sizes + default: type = LLM_TYPE_UNKNOWN; + } + + // For Granite MoE Shared + ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, /* required */ false); +} + +void llama_model_granite_swa::load_arch_tensors(llama_model_loader &) { + LLAMA_LOAD_LOCALS; + + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); + + // output + output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED); + + // if output is NULL, init from the input tok embed + if (output == NULL) { + output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED); + } + + for (int i = 0; i < n_layer; ++i) { + auto & layer = layers[i]; + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); + + create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0); + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0); + + // optional bias tensors + layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED); + + // Per-layer attention sinks for iSWA + layer.attn_sinks = create_tensor(tn(LLM_TENSOR_ATTN_SINKS, "weight", i), {n_head}, 0); + + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0); + + if (hparams.rope_scaling_type_train == LLAMA_ROPE_SCALING_TYPE_LONGROPE) { + layer.rope_long = create_tensor(tn(LLM_TENSOR_ROPE_FACTORS_LONG, "weight", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0)); + layer.rope_short = create_tensor(tn(LLM_TENSOR_ROPE_FACTORS_SHORT, "weight", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0)); + } + else { + layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, "weight", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0)); + } + + if (n_expert == 0) { + layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, 0); + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); + + // optional MLP bias + layer.ffn_gate_b = create_tensor(tn(LLM_TENSOR_FFN_GATE, "bias", i), {n_ff}, TENSOR_NOT_REQUIRED); + layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED); + layer.ffn_up_b = create_tensor(tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, TENSOR_NOT_REQUIRED); + } else { + layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0); + layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff, n_expert}, TENSOR_NOT_REQUIRED); + layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), { n_ff, n_embd, n_expert}, 0); + layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff, n_expert}, 0); + + // For Granite MoE Shared + if (hparams.n_ff_shexp > 0) { + layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, hparams.n_ff_shexp}, 0); + layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, hparams.n_ff_shexp}, 0); + layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {hparams.n_ff_shexp, n_embd}, 0); + } + } + } +} + +std::unique_ptr llama_model_granite_swa::build_arch_graph(const llm_graph_params & params) const { + return std::make_unique(*this, params); +} + +llama_model_granite_swa::graph::graph( + const llama_model & model, + const llm_graph_params & params) + : llm_graph_context(params) { + + const int64_t n_embd_head = hparams.n_embd_head_v(); + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); + GGML_ASSERT(n_embd_head == n_rot); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // inp_pos - built only if rope enabled + ggml_tensor * inp_pos = nullptr; + if (hparams.rope_finetuned) { + inp_pos = build_inp_pos(); + } + auto * inp_attn = build_attn_inp_kv_iswa(); + + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + for (int il = 0; il < n_layer; ++il) { + + // Granite Vision 4.1 deepstack: inject the projector stream that + // targets decoder layer `il` before the decoder runs. + // NOTE: skip the first deepstack layer since that's inpL + const auto & deepstack_emb_idx = hparams.deepstack_mapping_arr[il]; + if (il > 0 && deepstack_emb_idx >= 0) { + ggml_tensor * ds = ggml_view_2d(ctx0, + res->t_inp_embd, n_embd, n_tokens, + res->t_inp_embd->nb[1], + deepstack_emb_idx * n_embd * sizeof(float)); + inpL = ggml_add(ctx0, inpL, ds); + cb(inpL, "deepstack_in", il); + } + + ggml_tensor * inpSA = inpL; + + // norm + cur = build_norm(inpL, + model.layers[il].attn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + // self-attention + cur = build_attention_layer( + cur, inp_pos, inp_attn, + model, n_embd_head, il); + + if (il == n_layer - 1 && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + // ffn + cur = build_layer_ffn(cur, inpSA, model, il); + + // input for next layer + inpL = cur; + } + cur = inpL; + + cur = build_norm(cur, + model.output_norm, NULL, + LLM_NORM_RMS, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head + cur = build_lora_mm(model.output, cur, model.output_s); + + // For Granite architectures - scale logits + cur = ggml_scale(ctx0, cur, 1.0f / hparams.f_logit_scale); + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); +} + +ggml_tensor * llama_model_granite_swa::graph::build_attention_layer( + ggml_tensor * cur, + ggml_tensor * inp_pos, + llm_graph_input_attn_kv_iswa * inp_attn, + const llama_model & model, + const int64_t n_embd_head, + const int il) { + + auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, + n_embd_head, hparams.n_head(il), hparams.n_head_kv(il), il); + + const bool use_rope = hparams.rope_finetuned; + if (use_rope) { + ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); + Qcur = ggml_rope_ext( + ctx0, Qcur, inp_pos, rope_factors, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, rope_factors, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + } + + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + + const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f/sqrtf(float(n_embd_head)) : hparams.f_attention_scale; + + // Pass layer.attn_sinks to build_attn for sink-based attention modulation + cur = build_attn(inp_attn, + model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s, + Qcur, Kcur, Vcur, nullptr, model.layers[il].attn_sinks, nullptr, kq_scale, il); + cb(cur, "attn_out", il); + return cur; +} + +ggml_tensor * llama_model_granite_swa::graph::build_layer_ffn( + ggml_tensor * cur, + ggml_tensor * inpSA, + const llama_model & model, + const int il) { + + // For Granite architectures - scale residual + if (hparams.f_residual_scale) { + cur = ggml_scale(ctx0, cur, hparams.f_residual_scale); + } + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + // feed-forward network (non-MoE) + if (model.layers[il].ffn_gate_inp == nullptr) { + + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + cur = build_ffn(cur, + model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL, + model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, NULL, + model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + + } else { + // MoE branch + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + ggml_tensor * moe_out = build_moe_ffn(cur, + model.layers[il].ffn_gate_inp, + model.layers[il].ffn_up_exps, + model.layers[il].ffn_gate_exps, + model.layers[il].ffn_down_exps, + nullptr, + n_expert, n_expert_used, + LLM_FFN_SILU, true, + hparams.expert_weights_scale, + LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, + il); + cb(moe_out, "ffn_moe_out", il); + + // For Granite MoE Shared + if (hparams.n_ff_shexp > 0) { + ggml_tensor * ffn_shexp = build_ffn(cur, + model.layers[il].ffn_up_shexp, NULL, NULL, + model.layers[il].ffn_gate_shexp, NULL, NULL, + model.layers[il].ffn_down_shexp, NULL, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(ffn_shexp, "ffn_shexp", il); + + cur = ggml_add(ctx0, moe_out, ffn_shexp); + cb(cur, "ffn_out", il); + } else { + cur = moe_out; + } + } + + // For Granite architectures - scale residual + if (hparams.f_residual_scale) { + cur = ggml_scale(ctx0, cur, hparams.f_residual_scale); + } + cur = ggml_add(ctx0, cur, ffn_inp); + cb(cur, "ffn_out", il); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + return cur; +} diff --git a/src/models/models.h b/src/models/models.h index beab9f6bc7d..155678581f6 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -1563,6 +1563,34 @@ struct llama_model_granite_hybrid : public llama_model_base { }; +struct llama_model_granite_swa : public llama_model_base { + llama_model_granite_swa(const struct llama_model_params & params) : llama_model_base(params) {} + void load_arch_hparams(llama_model_loader & ml) override; + void load_arch_tensors(llama_model_loader & ml) override; + + struct graph : public llm_graph_context { + graph(const llama_model & model, const llm_graph_params & params); + + private: + ggml_tensor * build_attention_layer( + ggml_tensor * cur, + ggml_tensor * inp_pos, + llm_graph_input_attn_kv_iswa * inp_attn, + const llama_model & model, + const int64_t n_embd_head, + const int il); + + ggml_tensor * build_layer_ffn( + ggml_tensor * cur, + ggml_tensor * inpSA, + const llama_model & model, + const int il); + }; + + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; +}; + + struct llama_model_chameleon : public llama_model_base { llama_model_chameleon(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; From 1f3741a5c408490b437a55b47e23931f73933dfb Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Thu, 9 Jul 2026 11:45:28 -0600 Subject: [PATCH 03/29] feat(conversion): Add conversion infra for rope_pattern array NOTE: There is other work also targeting this, so this may be removed depending on merge order. Branch: GraniteSWAForCausalLM AI-usage: full (Bob) Signed-off-by: Gabe Goodhart --- gguf-py/gguf/constants.py | 2 ++ gguf-py/gguf/gguf_writer.py | 3 +++ 2 files changed, 5 insertions(+) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 33278ceae7e..80844648fe6 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -195,6 +195,8 @@ class Attention: SHARED_KV_LAYERS = "{arch}.attention.shared_kv_layers" SLIDING_WINDOW_PATTERN = "{arch}.attention.sliding_window_pattern" TEMPERATURE_SCALE = "{arch}.attention.temperature_scale" + ROPE_PATTERN = "{arch}.attention.rope_pattern" + class Indexer: HEAD_COUNT = "{arch}.attention.indexer.head_count" diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 1e277f0687c..9703929170d 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -810,6 +810,9 @@ def add_sliding_window_pattern(self, value: int | Sequence[bool]) -> None: else: self.add_array(key, value) + def add_rope_pattern(self, value: Sequence[bool]) -> None: + self.add_array(Keys.Attention.ROPE_PATTERN.format(arch=self.arch), value) + def add_dense_features_dims(self, dense:str, in_f:int, out_f:int) -> None: self.add_uint32(Keys.LLM.DENSE_FEAT_IN_SIZE.format(arch=self.arch, dense=dense), in_f) self.add_uint32(Keys.LLM.DENSE_FEAT_OUT_SIZE.format(arch=self.arch, dense=dense), out_f) From f7d057b7ed18493284fe48a4507ec345cf6ddbda Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Thu, 9 Jul 2026 11:46:05 -0600 Subject: [PATCH 04/29] fix(conversion): Fix SWA pattern logic and support for non-rope layers Branch: GraniteSWAForCausalLM AI-usage: full (Bob) Signed-off-by: Gabe Goodhart --- conversion/granite.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/conversion/granite.py b/conversion/granite.py index 276297718e9..091f2b98970 100644 --- a/conversion/granite.py +++ b/conversion/granite.py @@ -104,14 +104,21 @@ def set_gguf_parameters(self): logger.info("gguf: (granite_swa) sliding_window_pattern = %d SWA layers / %d total", sum(is_swa), len(is_swa)) else: - # Fall back to period-based pattern: (i + 1) % 4 != 0 - # This matches the pattern for models without explicit layer_types + # Fall back to period-based pattern: i % 4 != 0 + # This matches the transformers default pattern n_layers = self.block_count - is_swa = [(i + 1) % 4 != 0 for i in range(n_layers)] + is_swa = [i % 4 != 0 for i in range(n_layers)] self.gguf_writer.add_sliding_window_pattern(is_swa) logger.info("gguf: (granite_swa) sliding_window_pattern (inferred) = %d SWA layers / %d total", sum(is_swa), n_layers) + # Add rope_pattern from no_rope_layers + if no_rope_layers := self.hparams.get("no_rope_layers"): + # Convert 1/0 to bool (1 = use RoPE, 0 = NoPE) + rope_pattern = [bool(x) for x in no_rope_layers] + self.gguf_writer.add_rope_pattern(rope_pattern) + logger.info("gguf: (granite_swa) rope_pattern = %d RoPE layers / %d total", + sum(rope_pattern), len(rope_pattern)) @ModelBase.register("GraniteMoeForCausalLM", "GraniteMoeSharedForCausalLM") class GraniteMoeModel(GraniteModel): From 44215bb1415da6aec12a3fa8b45c7f5c536f04ad Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Thu, 9 Jul 2026 11:46:36 -0600 Subject: [PATCH 05/29] feat(conversion): Add support for GraniteMoeSWA Branch: GraniteSWAForCausalLM AI-usage: full (Bob) Signed-off-by: Gabe Goodhart --- conversion/granite.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/conversion/granite.py b/conversion/granite.py index 091f2b98970..fcaff3d8691 100644 --- a/conversion/granite.py +++ b/conversion/granite.py @@ -120,6 +120,43 @@ def set_gguf_parameters(self): logger.info("gguf: (granite_swa) rope_pattern = %d RoPE layers / %d total", sum(rope_pattern), len(rope_pattern)) + +@ModelBase.register("GraniteMoeSWAForCausalLM") +class GraniteMoeSWAModel(GraniteSWAModel): + """Conversion for IBM's GraniteMoeSWAForCausalLM (unified dense + MoE with iSWA)""" + model_arch = gguf.MODEL_ARCH.GRANITE_SWA + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + """Split merged MoE tensors (gate+up) following standard MoE pattern.""" + + # Handle expert FFN tensors (merged gate+up) + if name.endswith("block_sparse_moe.input_linear.weight"): + ffn_dim = self.hparams["intermediate_size"] + assert data_torch.shape[-2] == 2 * ffn_dim, "Merged FFN tensor size must be 2 * intermediate_size" + gate, up = data_torch.split(ffn_dim, dim=-2) + yield from ModelBase.modify_tensors(self, gate, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE_EXP, bid), bid) + yield from ModelBase.modify_tensors(self, up, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP_EXP, bid), bid) + return + + # Handle shared expert FFN tensors (if present) + if name.endswith("shared_mlp.input_linear.weight"): + ffn_dim = self.hparams.get("shared_intermediate_size", self.hparams["intermediate_size"]) + assert data_torch.shape[-2] == 2 * ffn_dim, "Merged FFN tensor size must be 2 * shared_intermediate_size" + gate, up = data_torch.split(ffn_dim, dim=-2) + yield from ModelBase.modify_tensors(self, gate, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE_SHEXP, bid), bid) + yield from ModelBase.modify_tensors(self, up, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP_SHEXP, bid), bid) + return + + # Handle shared expert output (if present) + if name.endswith("shared_mlp.output_linear.weight"): + yield from ModelBase.modify_tensors(self, data_torch, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_DOWN_SHEXP, bid), bid) + return + + # Pass through to parent for all other tensors (including sinks) + yield from super().modify_tensors(data_torch, name, bid) + + + @ModelBase.register("GraniteMoeForCausalLM", "GraniteMoeSharedForCausalLM") class GraniteMoeModel(GraniteModel): """Conversion for IBM's GraniteMoeForCausalLM""" From cf315175c3608af8c548bc6c89467eda06e65de3 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Thu, 9 Jul 2026 11:48:15 -0600 Subject: [PATCH 06/29] feat: Add llama_hparams::has_rope and arch constants NOTE: This shadows the work done for Granite Speech https://github.com/ggml-org/llama.cpp/pull/25107 Branch: GraniteSWAForCausalLM AI-usage: full (Bob) Signed-off-by: Gabe Goodhart --- src/llama-arch.cpp | 2 ++ src/llama-arch.h | 2 ++ src/llama-hparams.cpp | 14 ++++++++++++++ src/llama-hparams.h | 9 +++++++++ 4 files changed, 27 insertions(+) diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 560aeb3c773..82375d58d5c 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -241,6 +241,8 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, "%s.attention.relative_buckets_count" }, { LLM_KV_ATTENTION_SLIDING_WINDOW, "%s.attention.sliding_window" }, { LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, "%s.attention.sliding_window_pattern" }, + { LLM_KV_ATTENTION_ROPE_PATTERN, "%s.attention.rope_pattern" }, + { LLM_KV_ATTENTION_SCALE, "%s.attention.scale" }, { LLM_KV_ATTENTION_OUTPUT_SCALE, "%s.attention.output_scale" }, { LLM_KV_ATTENTION_VALUE_SCALE, "%s.attention.value_scale" }, diff --git a/src/llama-arch.h b/src/llama-arch.h index 61d885c0ea3..6b2289262fc 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -247,6 +247,8 @@ enum llm_kv { LLM_KV_ATTENTION_SLIDING_WINDOW, LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, LLM_KV_ATTENTION_SCALE, + LLM_KV_ATTENTION_ROPE_PATTERN, + LLM_KV_ATTENTION_OUTPUT_SCALE, LLM_KV_ATTENTION_VALUE_SCALE, LLM_KV_ATTENTION_TEMPERATURE_LENGTH, diff --git a/src/llama-hparams.cpp b/src/llama-hparams.cpp index 9d0683d2fec..2d7ff0013e6 100644 --- a/src/llama-hparams.cpp +++ b/src/llama-hparams.cpp @@ -241,6 +241,20 @@ bool llama_hparams::is_swa(uint32_t il) const { GGML_ABORT("%s: il (%u) out of bounds (n_layer_all: %u)\n", __func__, il, n_layer_all); } +bool llama_hparams::has_rope(uint32_t il) const { + // If rope_pattern is not set, fall back to rope_finetuned + if (rope_pattern[0] == 0 && rope_pattern[1] == 0) { + return rope_finetuned; + } + + if (il < n_layer_all) { + return rope_pattern[il] != 0; + } + + GGML_ABORT("%s: il (%u) out of bounds (n_layer_all: %u)\n", __func__, il, n_layer_all); +} + + bool llama_hparams::is_mla() const { assert((n_embd_head_k_mla_impl == 0 && n_embd_head_v_mla_impl == 0) || (n_embd_head_k_mla_impl != 0 && n_embd_head_v_mla_impl != 0)); diff --git a/src/llama-hparams.h b/src/llama-hparams.h index 8be5f28f39e..4b630e81aad 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -145,6 +145,11 @@ struct llama_hparams { // if is_swa_impl[il] == 1, then layer il is SWA // if is_swa_impl[il] == 0, then layer il is dense (i.e. non-SWA) + + // Per-layer RoPE enable flags (1 = use RoPE, 0 = NoPE) + // by default, all layers use RoPE (controlled by rope_finetuned) + std::array rope_pattern; + // by default, all layers are dense // note: using uint32_t type for compatibility reason std::array is_swa_impl; @@ -304,6 +309,10 @@ struct llama_hparams { void set_recr_pattern(uint32_t n_pattern, bool dense_first = false); + // Per-layer RoPE enable check (duplicated from PR #25107 for temporary compatibility) + bool has_rope(uint32_t il) const; + + // whether or not the given layer is recurrent (for hybrid models) bool is_recr(uint32_t il) const; From f5d4e9f4ed7e8bb49c2ea62c7c5e2f4a1336801e Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Thu, 9 Jul 2026 11:48:43 -0600 Subject: [PATCH 07/29] feat: Add support for per-layer rope determination Branch: GraniteSWAForCausalLM AI-usage: full (Bob) Signed-off-by: Gabe Goodhart --- src/models/granite-swa.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/models/granite-swa.cpp b/src/models/granite-swa.cpp index d3069b8c894..966ab9d4405 100644 --- a/src/models/granite-swa.cpp +++ b/src/models/granite-swa.cpp @@ -37,6 +37,10 @@ void llama_model_granite_swa::load_arch_hparams(llama_model_loader & ml) { // Granite uses rope_finetuned as a switch for rope, so default to true bool rope_finetuned = true; + + // Per-layer RoPE pattern (optional) + ml.get_arr(LLM_KV_ATTENTION_ROPE_PATTERN, hparams.rope_pattern, false); + ml.get_key(LLM_KV_ROPE_SCALING_FINETUNED, rope_finetuned, false); hparams.rope_finetuned = rope_finetuned; @@ -134,10 +138,7 @@ llama_model_granite_swa::graph::graph( inpL = build_inp_embd(model.tok_embd); // inp_pos - built only if rope enabled - ggml_tensor * inp_pos = nullptr; - if (hparams.rope_finetuned) { - inp_pos = build_inp_pos(); - } + ggml_tensor * inp_pos = build_inp_pos(); auto * inp_attn = build_attn_inp_kv_iswa(); ggml_tensor * inp_out_ids = build_inp_out_ids(); @@ -211,7 +212,7 @@ ggml_tensor * llama_model_granite_swa::graph::build_attention_layer( auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, hparams.n_head(il), hparams.n_head_kv(il), il); - const bool use_rope = hparams.rope_finetuned; + const bool use_rope = hparams.has_rope(il); if (use_rope) { ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); Qcur = ggml_rope_ext( From 9d75ea8a7568cc053008d38ec2e627c81097fae7 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Thu, 9 Jul 2026 16:26:27 -0600 Subject: [PATCH 08/29] style: Fix failing flake8 for extra newlines Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart --- conversion/granite.py | 1 - gguf-py/gguf/constants.py | 1 - 2 files changed, 2 deletions(-) diff --git a/conversion/granite.py b/conversion/granite.py index fcaff3d8691..a30569e2a48 100644 --- a/conversion/granite.py +++ b/conversion/granite.py @@ -156,7 +156,6 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter yield from super().modify_tensors(data_torch, name, bid) - @ModelBase.register("GraniteMoeForCausalLM", "GraniteMoeSharedForCausalLM") class GraniteMoeModel(GraniteModel): """Conversion for IBM's GraniteMoeForCausalLM""" diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 80844648fe6..2eff63d5de9 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -197,7 +197,6 @@ class Attention: TEMPERATURE_SCALE = "{arch}.attention.temperature_scale" ROPE_PATTERN = "{arch}.attention.rope_pattern" - class Indexer: HEAD_COUNT = "{arch}.attention.indexer.head_count" KEY_LENGTH = "{arch}.attention.indexer.key_length" From ef14e1c3b92e737e16475dda4ca94ccd7b6be61a Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Thu, 9 Jul 2026 16:54:04 -0600 Subject: [PATCH 09/29] test: Write out SLIDING_WINDOW_PATTERN in llama-model-saver Branch: GraniteSWAForCausalLM AI-usage: full (OpenCode + Qwen3.6-35b) Signed-off-by: Gabe Goodhart --- src/llama-model-saver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp index a3928523ba8..27d050be617 100644 --- a/src/llama-model-saver.cpp +++ b/src/llama-model-saver.cpp @@ -267,7 +267,7 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_ATTENTION_GATE_LORA_RANK, hparams.n_lora_gate); add_kv(LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, hparams.n_rel_attn_bkts); add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); - // add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, ???); + add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, true); add_kv(LLM_KV_ATTENTION_SCALE, hparams.f_attention_scale); add_kv(LLM_KV_ATTENTION_OUTPUT_SCALE, hparams.f_attn_out_scale); add_kv(LLM_KV_ATTENTION_VALUE_SCALE, hparams.f_attn_value_scale); From c32fe0bfee1b207db44f5dfd9ef4bcb30e0bf27d Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Mon, 13 Jul 2026 15:30:40 -0600 Subject: [PATCH 10/29] fix(convert): Fix missing registration for GraniteMoeSWAForCausalLM Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart --- conversion/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/conversion/__init__.py b/conversion/__init__.py index 9bf378cd48b..074e9bc2dac 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -101,6 +101,7 @@ "GraniteSpeechForConditionalGeneration": "granite", "GraniteSpeechPlusForConditionalGeneration": "granite", "GraniteSWAForCausalLM": "granite", + "GraniteMoeSWAForCausalLM": "granite", "Grok1ForCausalLM": "grok", "GrokForCausalLM": "grok", "GroveMoeForCausalLM": "grovemoe", From 36aa2e0221bdd919c6d36be48e9f20758e0dc9f2 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Tue, 14 Jul 2026 12:57:52 -0600 Subject: [PATCH 11/29] fix: Load MoE params as optional Branch: GraniteSWAForCausalLM AI-usage: draft (OpenCode + Qwen3.6-35b) Signed-off-by: Gabe Goodhart --- src/models/granite-swa.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/models/granite-swa.cpp b/src/models/granite-swa.cpp index 966ab9d4405..84ac30c0f16 100644 --- a/src/models/granite-swa.cpp +++ b/src/models/granite-swa.cpp @@ -9,7 +9,11 @@ void llama_model_granite_swa::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_EMBEDDING_SCALE, hparams.f_embedding_scale, false); ml.get_key(LLM_KV_ATTENTION_SCALE, hparams.f_attention_scale, false); - // iSWA configuration + // MoE expert configuration + ml.get_key(LLM_KV_EXPERT_COUNT, hparams.n_expert, false); + ml.get_key(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used, false); + + // iSWA configuration ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer()); ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; From 94d2643c0f9cf67aa3c63dd2ffb44dd40887a50e Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Tue, 14 Jul 2026 13:00:06 -0600 Subject: [PATCH 12/29] feat: Handle MoE params in conversion branch: GraniteSWAForCausalLM AI-usage: full (OpenCode + Qwen3.6-35b) Signed-off-by: Gabe Goodhart --- conversion/granite.py | 22 +++++++++++++++++++++- gguf-py/gguf/constants.py | 12 ++++++++++-- gguf-py/gguf/tensor_mapping.py | 1 + 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/conversion/granite.py b/conversion/granite.py index a30569e2a48..acf1ea719c8 100644 --- a/conversion/granite.py +++ b/conversion/granite.py @@ -126,10 +126,30 @@ class GraniteMoeSWAModel(GraniteSWAModel): """Conversion for IBM's GraniteMoeSWAForCausalLM (unified dense + MoE with iSWA)""" model_arch = gguf.MODEL_ARCH.GRANITE_SWA + def set_gguf_parameters(self): + super().set_gguf_parameters() + if shared_intermediate_size := self.hparams.get("shared_intermediate_size"): + self.gguf_writer.add_expert_shared_feed_forward_length(shared_intermediate_size) + logger.info("gguf: (granitemoewa) shared_intermediate_size = %s", shared_intermediate_size) + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: """Split merged MoE tensors (gate+up) following standard MoE pattern.""" - # Handle expert FFN tensors (merged gate+up) + # Handle expert FFN tensors (merged gate+up) - swash format: experts.gate_up_proj + if name.endswith("block_sparse_moe.experts.gate_up_proj"): + ffn_dim = self.hparams["intermediate_size"] + assert data_torch.shape[-2] == 2 * ffn_dim, f"Merged FFN tensor size must be 2 * intermediate_size, got {data_torch.shape[-2]}" + gate, up = data_torch.split(ffn_dim, dim=-2) + yield from ModelBase.modify_tensors(self, gate, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE_EXP, bid), bid) + yield from ModelBase.modify_tensors(self, up, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP_EXP, bid), bid) + return + + # Handle expert FFN down projection - swash format: experts.down_proj + if name.endswith("block_sparse_moe.experts.down_proj"): + yield from ModelBase.modify_tensors(self, data_torch, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_DOWN_EXP, bid), bid) + return + + # Handle expert FFN tensors (merged gate+up) - standard granite format: input_linear.weight if name.endswith("block_sparse_moe.input_linear.weight"): ffn_dim = self.hparams["intermediate_size"] assert data_torch.shape[-2] == 2 * ffn_dim, "Merged FFN tensor size must be 2 * intermediate_size" diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 2eff63d5de9..5249aaaee55 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -1170,8 +1170,8 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.ATTN_K_NORM: "blk.{bid}.attn_k_norm", MODEL_TENSOR.ATTN_OUT_NORM: "blk.{bid}.attn_output_norm", MODEL_TENSOR.ATTN_POST_NORM: "blk.{bid}.post_attention_norm", - MODEL_TENSOR.FFN_GATE_INP: "blk.{bid}.ffn_gate_inp", - MODEL_TENSOR.FFN_GATE_INP_SHEXP: "blk.{bid}.ffn_gate_inp_shexp", + MODEL_TENSOR.FFN_GATE_INP: "blk.{bid}.ffn_gate_inp.weight", + MODEL_TENSOR.FFN_GATE_INP_SHEXP: "blk.{bid}.ffn_gate_inp_shexp.weight", MODEL_TENSOR.FFN_NORM: "blk.{bid}.ffn_norm", MODEL_TENSOR.FFN_PRE_NORM: "blk.{bid}.ffn_norm", MODEL_TENSOR.FFN_POST_NORM: "blk.{bid}.post_ffw_norm", @@ -3691,6 +3691,14 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.FFN_GATE, MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, + # MoE (GraniteMoeSWA) + MODEL_TENSOR.FFN_GATE_INP, + MODEL_TENSOR.FFN_GATE_EXP, + MODEL_TENSOR.FFN_DOWN_EXP, + MODEL_TENSOR.FFN_UP_EXP, + MODEL_TENSOR.FFN_GATE_SHEXP, + MODEL_TENSOR.FFN_UP_SHEXP, + MODEL_TENSOR.FFN_DOWN_SHEXP, ], MODEL_ARCH.CHAMELEON: [ MODEL_TENSOR.TOKEN_EMBD, diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index 9efb36f8a44..09624933a03 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -451,6 +451,7 @@ class TensorNameMap: "transformer.decoder_layer.{bid}.router", # Grok "transformer.blocks.{bid}.ffn.router.layer", # dbrx "model.layers.{bid}.block_sparse_moe.router.layer", # granitemoe + "model.layers.{bid}.block_sparse_moe.router.weight", # granitemoeswa "model.layers.{bid}.feed_forward.router", # llama4 jamba "encoder.layers.{bid}.mlp.router.layer", # nomic-bert-moe "model.layers.{bid}.mlp.router", # openai-moe From 90c3d64600534d924ef8bea9c9c2b4eeb96c2896 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Wed, 12 Aug 2026 07:21:21 -0600 Subject: [PATCH 13/29] style: Remove unnecessary newline AI-usage: none Signed-off-by: Gabe Goodhart --- gguf-py/gguf/constants.py | 1 - 1 file changed, 1 deletion(-) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 53c299409ca..ed0eb1d9fbd 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -4111,7 +4111,6 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, ], - MODEL_ARCH.GRANITE_SWITCH: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, From 0b1df4c50511a7c925e1fb5259e2c7ae90256c55 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Wed, 12 Aug 2026 07:27:54 -0600 Subject: [PATCH 14/29] fix: Remove unnecessary tensor additions to GRANITE architecture Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart --- gguf-py/gguf/constants.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index ed0eb1d9fbd..c7e2b62f9c2 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -4056,8 +4056,6 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, MODEL_TENSOR.ATTN_OUT, - MODEL_TENSOR.ATTN_SINKS, - MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.FFN_NORM, MODEL_TENSOR.FFN_GATE, MODEL_TENSOR.FFN_DOWN, From 8865f1bde1dfdaa7a7d60b0487daf88a4e693025 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Wed, 12 Aug 2026 07:44:16 -0600 Subject: [PATCH 15/29] fix: Correctly handle naming for ffn gate inp Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart --- gguf-py/gguf/constants.py | 4 ++-- gguf-py/gguf/tensor_mapping.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index c7e2b62f9c2..b2b6996356f 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -1339,8 +1339,8 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.ATTN_K_NORM: "blk.{bid}.attn_k_norm", MODEL_TENSOR.ATTN_OUT_NORM: "blk.{bid}.attn_output_norm", MODEL_TENSOR.ATTN_POST_NORM: "blk.{bid}.post_attention_norm", - MODEL_TENSOR.FFN_GATE_INP: "blk.{bid}.ffn_gate_inp.weight", - MODEL_TENSOR.FFN_GATE_INP_SHEXP: "blk.{bid}.ffn_gate_inp_shexp.weight", + MODEL_TENSOR.FFN_GATE_INP: "blk.{bid}.ffn_gate_inp", + MODEL_TENSOR.FFN_GATE_INP_SHEXP: "blk.{bid}.ffn_gate_inp_shexp", MODEL_TENSOR.FFN_NORM: "blk.{bid}.ffn_norm", MODEL_TENSOR.FFN_PRE_NORM: "blk.{bid}.ffn_norm", MODEL_TENSOR.FFN_POST_NORM: "blk.{bid}.post_ffw_norm", diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index 57ce8ece385..321e6bcb8ec 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -451,7 +451,7 @@ class TensorNameMap: "transformer.decoder_layer.{bid}.router", # Grok "transformer.blocks.{bid}.ffn.router.layer", # dbrx "model.layers.{bid}.block_sparse_moe.router.layer", # granitemoe - "model.layers.{bid}.block_sparse_moe.router.weight", # granitemoeswa + "model.layers.{bid}.block_sparse_moe.router", # granite_swa "model.layers.{bid}.feed_forward.router", # llama4 jamba "encoder.layers.{bid}.mlp.router.layer", # nomic-bert-moe "model.layers.{bid}.mlp.router", # openai-moe From 8e5bcadbe30fc9a10b07178b827262c52ad2fb0f Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Wed, 12 Aug 2026 11:18:40 -0600 Subject: [PATCH 16/29] fix: Always default hparams.rope_pattern to 1s This isn't strictly necessary, but it will allow other models to rely on hparams.has_rope(il) without needting to prepopulate. Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart --- src/llama-model.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 7935f4fe796..af83eb069be 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1149,6 +1149,7 @@ void llama_model_base::load_hparams(llama_model_loader & ml) { std::fill(hparams.n_ff_arr.begin(), hparams.n_ff_arr.end(), 0); std::fill(hparams.rope_sections.begin(), hparams.rope_sections.end(), 0); + std::fill(hparams.rope_pattern.begin(), hparams.rope_pattern.end(), 1); std::fill(hparams.is_swa_impl.begin(), hparams.is_swa_impl.end(), 0); std::fill(hparams.is_recr_impl.begin(), hparams.is_recr_impl.end(), llm_arch_is_recurrent(ml.get_arch()) ? 1 : 0); std::fill(hparams.is_indexer_full_impl.begin(), hparams.is_indexer_full_impl.end(), 0); From 977437b02d01b346d7579b9e48b4bd74a986794e Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Wed, 12 Aug 2026 11:26:10 -0600 Subject: [PATCH 17/29] feat: Move to has_rope for all granite model architectures Now that we have a proper hparam for this, it's better to use it and not require a hacky fallback in the hparam method itself. Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart --- src/models/granite-hybrid.cpp | 7 +++---- src/models/granite-moe.cpp | 2 +- src/models/granite-swa.cpp | 6 ------ src/models/granite-switch.cpp | 6 +++--- src/models/granite.cpp | 7 +++---- 5 files changed, 10 insertions(+), 18 deletions(-) diff --git a/src/models/granite-hybrid.cpp b/src/models/granite-hybrid.cpp index eb23095aece..39536517606 100644 --- a/src/models/granite-hybrid.cpp +++ b/src/models/granite-hybrid.cpp @@ -16,7 +16,7 @@ void llama_model_granite_hybrid::load_arch_hparams(llama_model_loader & ml) { // Granite uses rope_finetuned as a switch for rope, so default to true bool rope_finetuned = true; ml.get_key(LLM_KV_ROPE_SCALING_FINETUNED, rope_finetuned, false); - hparams.rope_finetuned = rope_finetuned; + std::fill(hparams.rope_pattern.begin(), hparams.rope_pattern.end(), rope_finetuned); // A layer is recurrent IFF the n_head_kv value is set to 0 for (uint32_t i = 0; i < hparams.n_layer(); ++i) { @@ -147,7 +147,7 @@ llama_model_granite_hybrid::graph::graph(const llama_model & model, const llm_gr // Positional embeddings populated if rope enabled ggml_tensor * inp_pos = nullptr; - if (hparams.rope_finetuned) { + if (hparams.has_rope(0)) { inp_pos = build_inp_pos(); } @@ -206,8 +206,7 @@ ggml_tensor * llama_model_granite_hybrid::graph::build_attention_layer(ggml_tens const int il) { auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, hparams.n_head(il), hparams.n_head_kv(il), il); - const bool use_rope = hparams.rope_finetuned; - if (use_rope) { + if (hparams.has_rope(il)) { ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); diff --git a/src/models/granite-moe.cpp b/src/models/granite-moe.cpp index 115263c418f..ecefded9297 100644 --- a/src/models/granite-moe.cpp +++ b/src/models/granite-moe.cpp @@ -10,7 +10,7 @@ void llama_model_granite_moe::load_arch_hparams(llama_model_loader & ml) { // Granite uses rope_finetuned as a switch for rope, so default to true bool rope_finetuned = true; ml.get_key(LLM_KV_ROPE_SCALING_FINETUNED, rope_finetuned, false); - hparams.rope_finetuned = rope_finetuned; + std::fill(hparams.rope_pattern.begin(), hparams.rope_pattern.end(), rope_finetuned); switch (hparams.n_layer()) { case 32: type = LLM_TYPE_3B; break; diff --git a/src/models/granite-swa.cpp b/src/models/granite-swa.cpp index 84ac30c0f16..97e8adde3b3 100644 --- a/src/models/granite-swa.cpp +++ b/src/models/granite-swa.cpp @@ -39,15 +39,9 @@ void llama_model_granite_swa::load_arch_hparams(llama_model_loader & ml) { } } - // Granite uses rope_finetuned as a switch for rope, so default to true - bool rope_finetuned = true; - // Per-layer RoPE pattern (optional) ml.get_arr(LLM_KV_ATTENTION_ROPE_PATTERN, hparams.rope_pattern, false); - ml.get_key(LLM_KV_ROPE_SCALING_FINETUNED, rope_finetuned, false); - hparams.rope_finetuned = rope_finetuned; - switch (hparams.n_layer()) { case 32: type = LLM_TYPE_3B; break; case 40: type = LLM_TYPE_3B; break; diff --git a/src/models/granite-switch.cpp b/src/models/granite-switch.cpp index 80f6b86edc1..1b4c3c808ae 100644 --- a/src/models/granite-switch.cpp +++ b/src/models/granite-switch.cpp @@ -11,7 +11,7 @@ void llama_model_granite_switch::load_arch_hparams(llama_model_loader & ml) { bool rope_finetuned = true; ml.get_key(LLM_KV_ROPE_SCALING_FINETUNED, rope_finetuned, false); - hparams.rope_finetuned = rope_finetuned; + std::fill(hparams.rope_pattern.begin(), hparams.rope_pattern.end(), rope_finetuned); switch (hparams.n_layer()) { case 40: type = hparams.n_embd == 4096 ? LLM_TYPE_8B : LLM_TYPE_3B; break; @@ -254,7 +254,7 @@ llama_model_granite_switch::graph::graph( cb(inpL, "inp_embd", -1); ggml_tensor * inp_pos = nullptr; - if (hparams.rope_finetuned) { + if (hparams.has_rope(0)) { inp_pos = build_inp_pos(); } auto * inp_attn = build_attn_inp_kv(); @@ -361,7 +361,7 @@ ggml_tensor * llama_model_granite_switch::graph::build_attention_layer( Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); - if (hparams.rope_finetuned) { + if (hparams.has_rope(il)) { ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, diff --git a/src/models/granite.cpp b/src/models/granite.cpp index 4a75c5ff3cc..45e8fd9d500 100644 --- a/src/models/granite.cpp +++ b/src/models/granite.cpp @@ -33,7 +33,7 @@ void llama_model_granite::load_arch_hparams(llama_model_loader & ml) { // Granite uses rope_finetuned as a switch for rope, so default to true bool rope_finetuned = true; ml.get_key(LLM_KV_ROPE_SCALING_FINETUNED, rope_finetuned, false); - hparams.rope_finetuned = rope_finetuned; + std::fill(hparams.rope_pattern.begin(), hparams.rope_pattern.end(), rope_finetuned); switch (hparams.n_layer()) { case 32: type = LLM_TYPE_3B; break; @@ -127,7 +127,7 @@ llama_model_granite::graph::graph( // inp_pos - built only if rope enabled ggml_tensor * inp_pos = nullptr; - if (hparams.rope_finetuned) { + if (hparams.has_rope(0)) { inp_pos = build_inp_pos(); } auto * inp_attn = build_attn_inp_kv(); @@ -203,8 +203,7 @@ ggml_tensor * llama_model_granite::graph::build_attention_layer( auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, hparams.n_head(il), hparams.n_head_kv(il), il); - const bool use_rope = hparams.rope_finetuned; - if (use_rope) { + if (hparams.has_rope(il)) { ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); Qcur = ggml_rope_ext( ctx0, Qcur, inp_pos, rope_factors, From e400df6f08de136a74151aab30d2a089b08c898b Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Wed, 12 Aug 2026 11:26:38 -0600 Subject: [PATCH 18/29] feat: No hacky rope_finetuned fallback in has_rope Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart --- src/llama-hparams.cpp | 5 ----- src/llama-hparams.h | 9 ++++----- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/src/llama-hparams.cpp b/src/llama-hparams.cpp index 4008ff542cb..ec544adce07 100644 --- a/src/llama-hparams.cpp +++ b/src/llama-hparams.cpp @@ -284,11 +284,6 @@ bool llama_hparams::has_rope(uint32_t il) const { return false; } - // If rope_pattern is not set, fall back to rope_finetuned - if (rope_pattern[0] == 0 && rope_pattern[1] == 0) { - return rope_finetuned; - } - if (il < n_layer_all) { return rope_pattern[il] != 0; } diff --git a/src/llama-hparams.h b/src/llama-hparams.h index 9fc26d2c604..30e00046691 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -143,6 +143,10 @@ struct llama_hparams { std::array rope_sections; + // Per-layer RoPE enable flags (1 = use RoPE, 0 = NoPE) + // by default, all layers use RoPE (controlled by rope_finetuned) + std::array rope_pattern; + // Sliding Window Attention (SWA) llama_swa_type swa_type = LLAMA_SWA_TYPE_NONE; // the size of the sliding window (0 - no SWA) @@ -150,11 +154,6 @@ struct llama_hparams { // if is_swa_impl[il] == 1, then layer il is SWA // if is_swa_impl[il] == 0, then layer il is dense (i.e. non-SWA) - - // Per-layer RoPE enable flags (1 = use RoPE, 0 = NoPE) - // by default, all layers use RoPE (controlled by rope_finetuned) - std::array rope_pattern; - // by default, all layers are dense // note: using uint32_t type for compatibility reason std::array is_swa_impl; From e7695cf3672fdc9f2c28bed365922acfd1c94643 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Wed, 12 Aug 2026 11:31:20 -0600 Subject: [PATCH 19/29] fix: Fully remove rope hparam filling in granitemoe There are no granitemoe models that use NoPE (it's not actually used in the layer building below), so this was just dead code. Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart --- src/models/granite-moe.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/models/granite-moe.cpp b/src/models/granite-moe.cpp index ecefded9297..09be49393e3 100644 --- a/src/models/granite-moe.cpp +++ b/src/models/granite-moe.cpp @@ -7,11 +7,6 @@ void llama_model_granite_moe::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_EMBEDDING_SCALE, hparams.f_embedding_scale, false); ml.get_key(LLM_KV_ATTENTION_SCALE, hparams.f_attention_scale, false); - // Granite uses rope_finetuned as a switch for rope, so default to true - bool rope_finetuned = true; - ml.get_key(LLM_KV_ROPE_SCALING_FINETUNED, rope_finetuned, false); - std::fill(hparams.rope_pattern.begin(), hparams.rope_pattern.end(), rope_finetuned); - switch (hparams.n_layer()) { case 32: type = LLM_TYPE_3B; break; case 40: type = LLM_TYPE_3B; break; From 030670478d642ed81f62058935e7d3d1565ac3a0 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Wed, 12 Aug 2026 11:45:44 -0600 Subject: [PATCH 20/29] fix: Save out rope_pattern in model-saver Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart --- src/llama-model-saver.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp index ba901511c0f..ac665f7d341 100644 --- a/src/llama-model-saver.cpp +++ b/src/llama-model-saver.cpp @@ -268,6 +268,7 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_ATTENTION_VALUE_RESIDUAL_MIX_LORA_RANK, hparams.n_lora_value_res_mix); add_kv(LLM_KV_ATTENTION_GATE_LORA_RANK, hparams.n_lora_gate); add_kv(LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, hparams.n_rel_attn_bkts); + add_kv(LLM_KV_ATTENTION_ROPE_PATTERN, hparams.rope_pattern, true); add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, true); add_kv(LLM_KV_ATTENTION_SCALE, hparams.f_attention_scale); From 93f6cb8c4e86c1a6ac62f0a8c2c99db6fafe8373 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Wed, 12 Aug 2026 12:35:05 -0600 Subject: [PATCH 21/29] fix: Set hparams.rope_finetuned for round trip Since the value is _read_ from rope_finetuned, we need to persist it when the model is saved with the saver. Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart --- src/models/granite-hybrid.cpp | 1 + src/models/granite-switch.cpp | 1 + src/models/granite.cpp | 1 + 3 files changed, 3 insertions(+) diff --git a/src/models/granite-hybrid.cpp b/src/models/granite-hybrid.cpp index 39536517606..8a8f7e19ff0 100644 --- a/src/models/granite-hybrid.cpp +++ b/src/models/granite-hybrid.cpp @@ -16,6 +16,7 @@ void llama_model_granite_hybrid::load_arch_hparams(llama_model_loader & ml) { // Granite uses rope_finetuned as a switch for rope, so default to true bool rope_finetuned = true; ml.get_key(LLM_KV_ROPE_SCALING_FINETUNED, rope_finetuned, false); + hparams.rope_finetuned = rope_finetuned; // needed for round trip save std::fill(hparams.rope_pattern.begin(), hparams.rope_pattern.end(), rope_finetuned); // A layer is recurrent IFF the n_head_kv value is set to 0 diff --git a/src/models/granite-switch.cpp b/src/models/granite-switch.cpp index 1b4c3c808ae..7c9a901c8a4 100644 --- a/src/models/granite-switch.cpp +++ b/src/models/granite-switch.cpp @@ -11,6 +11,7 @@ void llama_model_granite_switch::load_arch_hparams(llama_model_loader & ml) { bool rope_finetuned = true; ml.get_key(LLM_KV_ROPE_SCALING_FINETUNED, rope_finetuned, false); + hparams.rope_finetuned = rope_finetuned; // needed for round trip save std::fill(hparams.rope_pattern.begin(), hparams.rope_pattern.end(), rope_finetuned); switch (hparams.n_layer()) { diff --git a/src/models/granite.cpp b/src/models/granite.cpp index 45e8fd9d500..9e9f97e94dc 100644 --- a/src/models/granite.cpp +++ b/src/models/granite.cpp @@ -33,6 +33,7 @@ void llama_model_granite::load_arch_hparams(llama_model_loader & ml) { // Granite uses rope_finetuned as a switch for rope, so default to true bool rope_finetuned = true; ml.get_key(LLM_KV_ROPE_SCALING_FINETUNED, rope_finetuned, false); + hparams.rope_finetuned = rope_finetuned; // needed for round trip save std::fill(hparams.rope_pattern.begin(), hparams.rope_pattern.end(), rope_finetuned); switch (hparams.n_layer()) { From 600615dc8309c8dd097112d71d6f136477977c0c Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Mon, 17 Aug 2026 12:05:14 -0600 Subject: [PATCH 22/29] fix: Code review cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Gabe Goodhart Co-authored-by: Sigbjørn Skjæret Co-authored-by: Sigbjørn Skjæret --- conversion/granite.py | 3 +-- src/models/granite-swa.cpp | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/conversion/granite.py b/conversion/granite.py index bb71041753e..0ae5a84eafb 100644 --- a/conversion/granite.py +++ b/conversion/granite.py @@ -82,8 +82,7 @@ class GraniteSWAModel(GraniteModel): def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: name, gen = item - # Add .weight suffix to sinks tensors (the C++ code expects blk.{bid}.attn_sinks.weight) - if "sinks" in name: + if name.endswith("sinks"): name += ".weight" return super().filter_tensors((name, gen)) diff --git a/src/models/granite-swa.cpp b/src/models/granite-swa.cpp index 97e8adde3b3..4bee09715d9 100644 --- a/src/models/granite-swa.cpp +++ b/src/models/granite-swa.cpp @@ -14,7 +14,7 @@ void llama_model_granite_swa::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used, false); // iSWA configuration - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer()); + ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl); ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; From 94fc69fd1980e1e4289fbf29cf3e510dac1bc26b Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Mon, 17 Aug 2026 15:56:24 -0600 Subject: [PATCH 23/29] refactor: Keep gate/up fused for MoE path Branch: GraniteSWAForCausalLM AI-usage: full (Claude + Sonnet 5) Signed-off-by: Gabe Goodhart --- conversion/granite.py | 12 ++++++------ gguf-py/gguf/constants.py | 1 + src/models/granite-swa.cpp | 6 +++--- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/conversion/granite.py b/conversion/granite.py index 0ae5a84eafb..525083f2c24 100644 --- a/conversion/granite.py +++ b/conversion/granite.py @@ -135,12 +135,12 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter """Split merged MoE tensors (gate+up) following standard MoE pattern.""" # Handle expert FFN tensors (merged gate+up) - swash format: experts.gate_up_proj + # Kept fused since inference (build_moe_ffn) supports a single gate_up_exps + # tensor for the routed experts. if name.endswith("block_sparse_moe.experts.gate_up_proj"): ffn_dim = self.hparams["intermediate_size"] assert data_torch.shape[-2] == 2 * ffn_dim, f"Merged FFN tensor size must be 2 * intermediate_size, got {data_torch.shape[-2]}" - gate, up = data_torch.split(ffn_dim, dim=-2) - yield from ModelBase.modify_tensors(self, gate, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE_EXP, bid), bid) - yield from ModelBase.modify_tensors(self, up, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP_EXP, bid), bid) + yield from ModelBase.modify_tensors(self, data_torch, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE_UP_EXP, bid), bid) return # Handle expert FFN down projection - swash format: experts.down_proj @@ -149,12 +149,12 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter return # Handle expert FFN tensors (merged gate+up) - standard granite format: input_linear.weight + # Kept fused since inference (build_moe_ffn) supports a single gate_up_exps + # tensor for the routed experts. if name.endswith("block_sparse_moe.input_linear.weight"): ffn_dim = self.hparams["intermediate_size"] assert data_torch.shape[-2] == 2 * ffn_dim, "Merged FFN tensor size must be 2 * intermediate_size" - gate, up = data_torch.split(ffn_dim, dim=-2) - yield from ModelBase.modify_tensors(self, gate, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE_EXP, bid), bid) - yield from ModelBase.modify_tensors(self, up, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP_EXP, bid), bid) + yield from ModelBase.modify_tensors(self, data_torch, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE_UP_EXP, bid), bid) return # Handle shared expert FFN tensors (if present) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index b2b6996356f..cdcc79c73bb 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -4142,6 +4142,7 @@ class MODEL_TENSOR(IntEnum): # MoE (GraniteMoeSWA) MODEL_TENSOR.FFN_GATE_INP, MODEL_TENSOR.FFN_GATE_EXP, + MODEL_TENSOR.FFN_GATE_UP_EXP, MODEL_TENSOR.FFN_DOWN_EXP, MODEL_TENSOR.FFN_UP_EXP, MODEL_TENSOR.FFN_GATE_SHEXP, diff --git a/src/models/granite-swa.cpp b/src/models/granite-swa.cpp index 4bee09715d9..2f049fb2850 100644 --- a/src/models/granite-swa.cpp +++ b/src/models/granite-swa.cpp @@ -102,9 +102,8 @@ void llama_model_granite_swa::load_arch_tensors(llama_model_loader &) { layer.ffn_up_b = create_tensor(tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, TENSOR_NOT_REQUIRED); } else { layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0); - layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff, n_expert}, TENSOR_NOT_REQUIRED); layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), { n_ff, n_embd, n_expert}, 0); - layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff, n_expert}, 0); + create_tensor_gate_up_exps(layer, i, n_embd, n_ff, n_expert, 0); // For Granite MoE Shared if (hparams.n_ff_shexp > 0) { @@ -286,7 +285,8 @@ ggml_tensor * llama_model_granite_swa::graph::build_layer_ffn( LLM_FFN_SILU, true, hparams.expert_weights_scale, LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, - il); + il, + nullptr, model.layers[il].ffn_gate_up_exps); cb(moe_out, "ffn_moe_out", il); // For Granite MoE Shared From c94d5ac9d09fb609cf9b109392e2857a57869368 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Mon, 17 Aug 2026 16:02:32 -0600 Subject: [PATCH 24/29] fix: Skip GRANITE_SWA in model saver https://github.com/ggml-org/llama.cpp/pull/25505#discussion_r3773175651 Keeping is_swa_impl in the saver can break other models. Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart --- src/llama-model-saver.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp index 61d639254ad..b9e0a600943 100644 --- a/src/llama-model-saver.cpp +++ b/src/llama-model-saver.cpp @@ -30,6 +30,7 @@ bool llama_model_saver_supports_arch(llm_arch arch) { case LLM_ARCH_MUSE_GLIMMER: case LLM_ARCH_MELLUM: case LLM_ARCH_LAGUNA: + case LLM_ARCH_GRANITE_SWA: return false; default: return true; @@ -274,7 +275,7 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, hparams.n_rel_attn_bkts); add_kv(LLM_KV_ATTENTION_ROPE_PATTERN, hparams.rope_pattern, true); add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); - add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, true); + // add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, ???); add_kv(LLM_KV_ATTENTION_SCALE, hparams.f_attention_scale); add_kv(LLM_KV_ATTENTION_OUTPUT_SCALE, hparams.f_attn_out_scale); add_kv(LLM_KV_ATTENTION_VALUE_SCALE, hparams.f_attn_value_scale); From b5b7d1d62b2bce5877c07f958c754031c80fde70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Tue, 18 Aug 2026 09:12:45 +0200 Subject: [PATCH 25/29] add sliding window pattern for model in test --- tests/test-llama-archs.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index 448f6754460..91a308ba95e 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -197,7 +197,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_ROPE_FREQ_BASE_SWA, 10000.0f); // SWA pattern: every 5th layer is full attention (matches E2B layer_types) ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, uint32_t(5)); - } else if (arch == LLM_ARCH_COHERE2MOE || arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_MUSE_GLIMMER) { + } else if (arch == LLM_ARCH_COHERE2MOE || arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_GRANITE_SWA) { std::vector pattern; pattern.reserve(n_layer); for (uint32_t il = 0; il < n_layer; il++) { From f7b13112db621b9c432184859176a2fa6a0b1780 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Tue, 18 Aug 2026 09:16:27 -0600 Subject: [PATCH 26/29] style: Fix indentation Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart --- src/models/granite-swa.cpp | 20 ++++++++++---------- src/models/models.h | 18 +++++++++--------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/models/granite-swa.cpp b/src/models/granite-swa.cpp index 2f049fb2850..7f100fe4b99 100644 --- a/src/models/granite-swa.cpp +++ b/src/models/granite-swa.cpp @@ -14,7 +14,7 @@ void llama_model_granite_swa::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used, false); // iSWA configuration - ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl); + ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl); ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; @@ -199,12 +199,12 @@ llama_model_granite_swa::graph::graph( } ggml_tensor * llama_model_granite_swa::graph::build_attention_layer( - ggml_tensor * cur, - ggml_tensor * inp_pos, + ggml_tensor * cur, + ggml_tensor * inp_pos, llm_graph_input_attn_kv_iswa * inp_attn, - const llama_model & model, - const int64_t n_embd_head, - const int il) { + const llama_model & model, + const int64_t n_embd_head, + const int il) { auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, hparams.n_head(il), hparams.n_head_kv(il), il); @@ -240,10 +240,10 @@ ggml_tensor * llama_model_granite_swa::graph::build_attention_layer( } ggml_tensor * llama_model_granite_swa::graph::build_layer_ffn( - ggml_tensor * cur, - ggml_tensor * inpSA, - const llama_model & model, - const int il) { + ggml_tensor * cur, + ggml_tensor * inpSA, + const llama_model & model, + const int il) { // For Granite architectures - scale residual if (hparams.f_residual_scale) { diff --git a/src/models/models.h b/src/models/models.h index 42d383f858b..1dd30dfd16d 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -1729,18 +1729,18 @@ struct llama_model_granite_swa : public llama_model_base { private: ggml_tensor * build_attention_layer( - ggml_tensor * cur, - ggml_tensor * inp_pos, + ggml_tensor * cur, + ggml_tensor * inp_pos, llm_graph_input_attn_kv_iswa * inp_attn, - const llama_model & model, - const int64_t n_embd_head, - const int il); + const llama_model & model, + const int64_t n_embd_head, + const int il); ggml_tensor * build_layer_ffn( - ggml_tensor * cur, - ggml_tensor * inpSA, - const llama_model & model, - const int il); + ggml_tensor * cur, + ggml_tensor * inpSA, + const llama_model & model, + const int il); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; From eeb83f3e2f1f7630931f9ac7d5e1260d2ee189bb Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Tue, 18 Aug 2026 18:51:48 -0600 Subject: [PATCH 27/29] fix: Fix \r\n Thanks Claude! Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart --- conversion/granite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conversion/granite.py b/conversion/granite.py index d46b6451586..3df5a8b5487 100644 --- a/conversion/granite.py +++ b/conversion/granite.py @@ -83,7 +83,7 @@ class GraniteSWAModel(GraniteModel): def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: name, gen = item - if name.endswith("sinks"): + if name.endswith("sinks"): name += ".weight" return super().filter_tensors((name, gen)) From d7a74f1ec17eb946989b89d7708be776ef0cce20 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Tue, 18 Aug 2026 19:14:10 -0600 Subject: [PATCH 28/29] feat: Keep shared expert fused Branch: GraniteSWAForCausalLM AI-usage: full (Claude + Sonnet 5) Signed-off-by: Gabe Goodhart --- conversion/granite.py | 8 ++++---- gguf-py/gguf/constants.py | 2 +- src/models/granite-swa.cpp | 11 +++++------ 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/conversion/granite.py b/conversion/granite.py index 3df5a8b5487..796d37cca26 100644 --- a/conversion/granite.py +++ b/conversion/granite.py @@ -158,13 +158,13 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter yield from ModelBase.modify_tensors(self, data_torch, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE_UP_EXP, bid), bid) return - # Handle shared expert FFN tensors (if present) + # Handle shared expert FFN tensors (if present) - kept fused since + # inference (build_ffn) supports a single ffn_up_shexp tensor with + # LLM_FFN_SWIGLU for the shared expert. if name.endswith("shared_mlp.input_linear.weight"): ffn_dim = self.hparams.get("shared_intermediate_size", self.hparams["intermediate_size"]) assert data_torch.shape[-2] == 2 * ffn_dim, "Merged FFN tensor size must be 2 * shared_intermediate_size" - gate, up = data_torch.split(ffn_dim, dim=-2) - yield from ModelBase.modify_tensors(self, gate, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE_SHEXP, bid), bid) - yield from ModelBase.modify_tensors(self, up, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP_SHEXP, bid), bid) + yield from ModelBase.modify_tensors(self, data_torch, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP_SHEXP, bid), bid) return # Handle shared expert output (if present) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index b643108d3c4..fad8d1fd8c2 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -4176,7 +4176,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.FFN_GATE_UP_EXP, MODEL_TENSOR.FFN_DOWN_EXP, MODEL_TENSOR.FFN_UP_EXP, - MODEL_TENSOR.FFN_GATE_SHEXP, + # Shared expert - gate+up kept fused in FFN_UP_SHEXP (LLM_FFN_SWIGLU) MODEL_TENSOR.FFN_UP_SHEXP, MODEL_TENSOR.FFN_DOWN_SHEXP, ], diff --git a/src/models/granite-swa.cpp b/src/models/granite-swa.cpp index 7f100fe4b99..3aa2b63b235 100644 --- a/src/models/granite-swa.cpp +++ b/src/models/granite-swa.cpp @@ -105,10 +105,9 @@ void llama_model_granite_swa::load_arch_tensors(llama_model_loader &) { layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), { n_ff, n_embd, n_expert}, 0); create_tensor_gate_up_exps(layer, i, n_embd, n_ff, n_expert, 0); - // For Granite MoE Shared + // For Granite MoE Shared - gate+up kept fused in ffn_up_shexp (see LLM_FFN_SWIGLU below) if (hparams.n_ff_shexp > 0) { - layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, hparams.n_ff_shexp}, 0); - layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, hparams.n_ff_shexp}, 0); + layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, 2*hparams.n_ff_shexp}, 0); layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {hparams.n_ff_shexp, n_embd}, 0); } } @@ -289,14 +288,14 @@ ggml_tensor * llama_model_granite_swa::graph::build_layer_ffn( nullptr, model.layers[il].ffn_gate_up_exps); cb(moe_out, "ffn_moe_out", il); - // For Granite MoE Shared + // For Granite MoE Shared - gate+up kept fused in ffn_up_shexp if (hparams.n_ff_shexp > 0) { ggml_tensor * ffn_shexp = build_ffn(cur, model.layers[il].ffn_up_shexp, NULL, NULL, - model.layers[il].ffn_gate_shexp, NULL, NULL, + NULL, NULL, NULL, model.layers[il].ffn_down_shexp, NULL, NULL, NULL, - LLM_FFN_SILU, LLM_FFN_PAR, il); + LLM_FFN_SWIGLU, LLM_FFN_SEQ, il); cb(ffn_shexp, "ffn_shexp", il); cur = ggml_add(ctx0, moe_out, ffn_shexp); From de5f6fd56dc6b821ec99852348a7673115cf14be Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Wed, 19 Aug 2026 06:32:16 -0600 Subject: [PATCH 29/29] style: More indentation fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Gabe Goodhart Co-authored-by: Sigbjørn Skjæret --- src/llama-arch.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 87ed957c5e4..955c2d79652 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -262,7 +262,7 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, "%s.attention.relative_buckets_count" }, { LLM_KV_ATTENTION_SLIDING_WINDOW, "%s.attention.sliding_window" }, { LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, "%s.attention.sliding_window_pattern" }, - { LLM_KV_ATTENTION_ROPE_PATTERN, "%s.attention.rope_pattern" }, + { LLM_KV_ATTENTION_ROPE_PATTERN, "%s.attention.rope_pattern" }, { LLM_KV_ATTENTION_SCALE, "%s.attention.scale" }, { LLM_KV_ATTENTION_OUTPUT_SCALE, "%s.attention.output_scale" },