Skip to content

Add GLM MoE Metal active expert patches #1063

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
i386 merged 1 commit into from
Jul 23, 2026
Merged

Add GLM MoE Metal active expert patches #1063

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 63 additions & 0 deletions third_party/llama.cpp/patches/0030-ggml-default-GLM-MoE-two-phase-Metal-path.patch
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
From 11fb71bbe0c6a1b525097a455d75810f0a4e43e4 Mon Sep 17 00:00:00 2001
From: Mesh-LLM CI <ci@mesh-llm.local>
Date: Sun, 19 Jul 2026 11:27:53 +1000
Subject: [PATCH 25/36] ggml: default GLM MoE two-phase Metal path

---
ggml/src/ggml-metal/ggml-metal-ops.cpp | 9 ++++++++-
tests/test-backend-ops.cpp | 12 ++++++++++++
2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp
index 36378f8e..684efdc6 100644
--- a/ggml/src/ggml-metal/ggml-metal-ops.cpp
+++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp
@@ -127,7 +127,14 @@ static bool ggml_metal_glm_dsa_moe_private_scratch_enabled() {

static bool ggml_metal_glm_dsa_moe_two_phase_enabled() {
const char * value = getenv("GGML_METAL_EXPERIMENTAL_GLM_MOE_TWO_PHASE");
- return value && atoi(value) != 0;
+ if (value) {
+ return atoi(value) != 0;
+ }
+ value = getenv("GGML_METAL_DISABLE_GLM_MOE_TWO_PHASE");
+ if (value) {
+ return atoi(value) == 0;
+ }
+ return true;
}

static bool ggml_metal_glm_dsa_moe_dual_lane_enabled() {
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index 8618ea52..56fb7b23 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -9832,6 +9832,17 @@ struct test_glm_ffn_two_phase_consumer : public test_glm_ffn_consumer {
return std::string("GLM_FFN_TWO_PHASE_") + variant;
}

+ test_status_t eval(
+ ggml_backend_t backend1,
+ ggml_backend_t backend2,
+ const char * op_names_filter,
+ printer * output_printer) override {
+ scoped_test_env motif_reference_env("GGML_METAL_ENABLE_GLM_MOE_DECODE_MOTIF_REFERENCE", "0");
+ scoped_test_env two_phase_env(
+ "GGML_METAL_EXPERIMENTAL_GLM_MOE_TWO_PHASE", two_phase ? "1" : "0");
+ return test_case::eval(backend1, backend2, op_names_filter, output_printer);
+ }
+
bool eval_perf(ggml_backend_t backend, const char * op_names_filter, printer * output_printer) override {
scoped_test_env motif_reference_env("GGML_METAL_ENABLE_GLM_MOE_DECODE_MOTIF_REFERENCE", "0");
scoped_test_env two_phase_env(
@@ -15138,6 +15149,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {

test_cases.emplace_back(new test_glm_ffn_chain_native_down(true, "CANDIDATE"));
test_cases.emplace_back(new test_glm_ffn_chain_two_phase(true, "CANDIDATE"));
+ test_cases.emplace_back(new test_glm_ffn_two_phase_consumer(true, "CANDIDATE"));
test_cases.emplace_back(new test_glm_ffn_dual_lane_consumer(true, "CANDIDATE"));

return test_cases;
--
2.54.0 (Apple Git-157)

221 changes: 221 additions & 0 deletions third_party/llama.cpp/patches/0031-ggml-add-GLM-MoE-Metal-selector-diagnostics.patch
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,221 @@
From 8797805107cd1e49bc018a498ce9d008493000f8 Mon Sep 17 00:00:00 2001
From: Mesh-LLM CI <ci@mesh-llm.local>
Date: Sun, 19 Jul 2026 16:31:29 +1000
Subject: [PATCH 26/36] ggml: add GLM MoE Metal selector diagnostics

---
ggml/src/ggml-metal/ggml-metal-ops.cpp | 75 +++++++++++++++++++++-----
tests/test-backend-ops.cpp | 39 ++++++++++++--
2 files changed, 98 insertions(+), 16 deletions(-)

diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp
index 684efdc6..ab41e1ec 100644
--- a/ggml/src/ggml-metal/ggml-metal-ops.cpp
+++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp
@@ -147,6 +147,11 @@ static bool ggml_metal_glm_dsa_moe_dual_lane_gate_slot4_enabled() {
return value && atoi(value) != 0;
}

+static bool ggml_metal_glm_dsa_moe_debug_terminal_outputs_enabled() {
+ const char * value = getenv("GGML_METAL_DEBUG_GLM_MOE_TERMINAL_OUTPUTS");
+ return value && atoi(value) != 0;
+}
+
static bool ggml_metal_glm_dsa_absorbed_qkv_phases_enabled() {
const char * value = getenv("GGML_METAL_EXPERIMENTAL_GLM_ABSORBED_QKV_PHASES");
return value && atoi(value) != 0;
@@ -3731,6 +3736,17 @@ static int ggml_metal_op_glm_moe_two_phase(
ctx->enc, (phase_args.n_out + 255)/256, 1, 1, 256, 1, 1);
ctx->set_fused_range_outputs(motif.final_out_offset);
} else {
+ const bool debug_terminal_outputs =
+ ggml_metal_glm_dsa_moe_debug_terminal_outputs_enabled();
+ if (debug_terminal_outputs) {
+ ggml_metal_encode_glm_moe_shared_mul_mv(
+ ctx,
+ motif.shared_down,
+ ggml_metal_get_buffer_id(motif.shared_glu),
+ ggml_metal_get_buffer_id(motif.shared_down));
+ ggml_metal_op_concurrency_reset(ctx);
+ }
+
const char * down_kernel = "kernel_glm_moe_two_phase_down";
auto down_pipeline = ggml_metal_library_get_pipeline(ctx->lib, down_kernel);
if (!down_pipeline.pipeline) {
@@ -3753,7 +3769,12 @@ static int ggml_metal_op_glm_moe_two_phase(
ggml_metal_encoder_set_threadgroup_memory_size(ctx->enc, 8*8*sizeof(float), 0);
ggml_metal_encoder_dispatch_threadgroups(
ctx->enc, (phase_args.n_out + 7)/8, 1, 1, 32, 8, 1);
- ctx->set_fused_range_outputs(motif.out_offset, motif.final_out_offset);
+ if (debug_terminal_outputs) {
+ ctx->set_fused_range_outputs(
+ motif.out_offset, motif.shared_down_offset, motif.final_out_offset);
+ } else {
+ ctx->set_fused_range_outputs(motif.out_offset, motif.final_out_offset);
+ }
}
ggml_metal_op_concurrency_reset(ctx);

@@ -4288,25 +4309,52 @@ static int ggml_metal_op_mul_mv_id_gate_up_swiglu(
if (dst->type == GGML_TYPE_F16 && !pair_sg) {
return 0;
}
- const bool pair_sg_slot8 = pair_sg && dst->type == GGML_TYPE_F32 &&
+ const bool pair_sg_slot8_requested =
ggml_metal_glm_dsa_q2_gate_up_swiglu_pair_sg_slot8_enabled();
- const bool pair_sg_slot2 = pair_sg && dst->type == GGML_TYPE_F32 &&
+ const bool pair_sg_slot2_requested =
ggml_metal_glm_dsa_q2_gate_up_swiglu_pair_sg_slot2_enabled();
- const bool pair_sg_slot4_dual_default = pair_sg_default_shape && src1_override != nullptr;
+ const bool pair_sg_slot4_dual_requested =
+ ggml_metal_glm_dsa_q2_gate_up_swiglu_pair_sg_slot4_dual_enabled();
+ const bool pair_sg_slot4_dual_r12_requested =
+ ggml_metal_glm_dsa_q2_gate_up_swiglu_pair_sg_slot4_dual_r12_enabled();
+ const bool pair_sg_slot4_dual_r16_requested =
+ ggml_metal_glm_dsa_q2_gate_up_swiglu_pair_sg_slot4_dual_r16_enabled();
+ const bool pair_sg_slot2_dual_requested =
+ ggml_metal_glm_dsa_q2_gate_up_swiglu_pair_sg_slot2_dual_enabled();
+ const bool pair_sg_slot8_split_requested =
+ ggml_metal_glm_dsa_q2_gate_up_swiglu_pair_sg_slot8_split_enabled();
+ const bool pair_sg_rowtile_requested =
+ ggml_metal_glm_dsa_q2_gate_up_swiglu_pair_sg_rowtile_enabled();
+ const bool pair_sg_slot1_dual_default =
+ pair_sg_default_shape &&
+ !pair_sg_slot8_requested &&
+ !pair_sg_slot2_requested &&
+ !pair_sg_slot4_dual_requested &&
+ !pair_sg_slot4_dual_r12_requested &&
+ !pair_sg_slot4_dual_r16_requested &&
+ !pair_sg_slot2_dual_requested &&
+ !pair_sg_slot8_split_requested &&
+ !pair_sg_rowtile_requested;
+ const bool pair_sg_slot8 = pair_sg && dst->type == GGML_TYPE_F32 &&
+ pair_sg_slot8_requested;
+ const bool pair_sg_slot2 = pair_sg && dst->type == GGML_TYPE_F32 &&
+ pair_sg_slot2_requested;
+ const bool pair_sg_slot4_dual_default = pair_sg_default_shape &&
+ src1_override != nullptr &&
+ pair_sg_slot1_dual_default;
const bool pair_sg_slot4_dual = pair_sg && dst->type == GGML_TYPE_F32 &&
- (ggml_metal_glm_dsa_q2_gate_up_swiglu_pair_sg_slot4_dual_enabled() ||
- pair_sg_slot4_dual_default);
+ (pair_sg_slot4_dual_requested || pair_sg_slot4_dual_default);
const bool pair_sg_slot4_dual_r12 = pair_sg && dst->type == GGML_TYPE_F32 &&
- ggml_metal_glm_dsa_q2_gate_up_swiglu_pair_sg_slot4_dual_r12_enabled();
+ pair_sg_slot4_dual_r12_requested;
const bool pair_sg_slot4_dual_r16 = pair_sg && dst->type == GGML_TYPE_F32 &&
- ggml_metal_glm_dsa_q2_gate_up_swiglu_pair_sg_slot4_dual_r16_enabled();
+ pair_sg_slot4_dual_r16_requested;
const bool pair_sg_slot1_dual = pair_sg && dst->type == GGML_TYPE_F32 &&
(ggml_metal_glm_dsa_q2_gate_up_swiglu_pair_sg_slot1_dual_enabled() ||
- (pair_sg_default_shape && !pair_sg_slot4_dual_default));
+ (pair_sg_slot1_dual_default && !pair_sg_slot4_dual_default));
const bool pair_sg_slot2_dual = pair_sg && dst->type == GGML_TYPE_F32 &&
- ggml_metal_glm_dsa_q2_gate_up_swiglu_pair_sg_slot2_dual_enabled();
+ pair_sg_slot2_dual_requested;
const bool pair_sg_slot8_split = pair_sg && dst->type == GGML_TYPE_F32 &&
- ggml_metal_glm_dsa_q2_gate_up_swiglu_pair_sg_slot8_split_enabled();
+ pair_sg_slot8_split_requested;
const bool pair_sg_share_y = pair_sg && dst->type == GGML_TYPE_F32 &&
ggml_metal_glm_dsa_q2_gate_up_swiglu_pair_sg_share_y_enabled() &&
ne00 == 6144 &&
@@ -4351,7 +4399,7 @@ static int ggml_metal_op_mul_mv_id_gate_up_swiglu(
ne21 == 1 &&
src1->type == GGML_TYPE_F32;
const bool pair_sg_rowtile = pair_sg && dst->type == GGML_TYPE_F32 &&
- ggml_metal_glm_dsa_q2_gate_up_swiglu_pair_sg_rowtile_enabled() &&
+ pair_sg_rowtile_requested &&
ne00 == 6144 &&
ne01 == 2048 &&
ne20 == 8 &&
@@ -4893,7 +4941,8 @@ static int ggml_metal_op_mul_mv_id_weighted_reduce(
enc, ggml_metal_get_buffer_id(down->src[2]));

const bool use_output_scratch =
- down->src[0]->type == GGML_TYPE_Q3_K && !preserve_routed_input;
+ down->src[0]->type == GGML_TYPE_Q3_K &&
+ !preserve_routed_input;
const ggml_metal_buffer_id weighted_reduce_dst = use_output_scratch ?
fusion_scratch : ggml_metal_get_buffer_id(weighted_sum);

diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index 56fb7b23..53489947 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -9575,6 +9575,7 @@ struct test_glm_ffn_consumer : public test_case {
ggml_tensor * selected_experts = nullptr;
ggml_tensor * weights = nullptr;
ggml_tensor * routed_out = nullptr;
+ ggml_tensor * shexp_out = nullptr;
ggml_tensor * out = nullptr;

test_glm_ffn_consumer(
@@ -9762,9 +9763,9 @@ struct test_glm_ffn_consumer : public test_case {
}
}

- ggml_tensor * shexp = ggml_mul_mat(ctx, downs, act);
- ggml_set_name(shexp, "ffn_shexp");
- return shexp;
+ shexp_out = ggml_mul_mat(ctx, downs, act);
+ ggml_set_name(shexp_out, "ffn_shexp");
+ return shexp_out;
}

ggml_tensor * build_graph(ggml_context * ctx) override {
@@ -9851,6 +9852,36 @@ struct test_glm_ffn_two_phase_consumer : public test_glm_ffn_consumer {
}
};

+struct test_glm_ffn_terminal_diagnostic : public test_glm_ffn_two_phase_consumer {
+ explicit test_glm_ffn_terminal_diagnostic(bool two_phase, const char * variant)
+ : test_glm_ffn_two_phase_consumer(two_phase, variant) {}
+
+ std::string op_desc(ggml_tensor * t) override {
+ GGML_UNUSED(t);
+ return std::string("GLM_FFN_TERMINAL_DIAGNOSTIC_") + variant;
+ }
+
+ std::vector<ggml_tensor *> fusion_test_nodes() override {
+ return { selected_experts, weights, routed_out, shexp_out, out };
+ }
+
+ test_status_t eval(
+ ggml_backend_t backend1,
+ ggml_backend_t backend2,
+ const char * op_names_filter,
+ printer * output_printer) override {
+ scoped_test_env terminal_outputs_env("GGML_METAL_DEBUG_GLM_MOE_TERMINAL_OUTPUTS", "1");
+ return test_glm_ffn_two_phase_consumer::eval(
+ backend1, backend2, op_names_filter, output_printer);
+ }
+
+ bool eval_perf(ggml_backend_t backend, const char * op_names_filter, printer * output_printer) override {
+ scoped_test_env terminal_outputs_env("GGML_METAL_DEBUG_GLM_MOE_TERMINAL_OUTPUTS", "1");
+ return test_glm_ffn_two_phase_consumer::eval_perf(
+ backend, op_names_filter, output_printer);
+ }
+};
+
struct test_glm_ffn_dual_lane_consumer : public test_glm_ffn_consumer {
const bool dual_lane;
const char * variant;
@@ -15150,6 +15181,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_glm_ffn_chain_native_down(true, "CANDIDATE"));
test_cases.emplace_back(new test_glm_ffn_chain_two_phase(true, "CANDIDATE"));
test_cases.emplace_back(new test_glm_ffn_two_phase_consumer(true, "CANDIDATE"));
+ test_cases.emplace_back(new test_glm_ffn_terminal_diagnostic(true, "CANDIDATE"));
test_cases.emplace_back(new test_glm_ffn_dual_lane_consumer(true, "CANDIDATE"));

return test_cases;
@@ -15520,6 +15552,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {
GGML_TYPE_Q4_K, GGML_TYPE_Q4_K, GGML_TYPE_Q4_K, 6144, 2048, true, true));
test_cases.emplace_back(new test_glm_ffn_consumer(GGML_TYPE_Q2_K, GGML_TYPE_Q2_K, GGML_TYPE_Q3_K,
GGML_TYPE_Q4_K, GGML_TYPE_Q4_K, GGML_TYPE_Q4_K, 6144, 2048, true, false, true));
+ test_cases.emplace_back(new test_glm_ffn_terminal_diagnostic(true, "CANDIDATE"));
test_cases.emplace_back(new test_glm_ffn_consumer(GGML_TYPE_TQ2_0, GGML_TYPE_TQ2_0, GGML_TYPE_TQ2_0,
GGML_TYPE_Q4_K, GGML_TYPE_Q4_K, GGML_TYPE_Q4_K, 6144, 2048, true, false));
test_cases.emplace_back(new test_glm_ffn_consumer(GGML_TYPE_Q1_0, GGML_TYPE_Q1_0, GGML_TYPE_Q1_0,
--
2.54.0 (Apple Git-157)

45 changes: 45 additions & 0 deletions third_party/llama.cpp/patches/0032-ggml-skip-zero-weight-GLM-MoE-gate-up-slots.patch
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
From 4523e19eecd43c47ca650939bb0c2dfc87c43920 Mon Sep 17 00:00:00 2001
From: James Dumay <james@mesh-llm.local>
Date: Sun, 19 Jul 2026 00:00:00 +1000
Subject: [PATCH 27/36] ggml: skip zero-weight GLM MoE gate-up slots

---
ggml/src/ggml-metal/ggml-metal.metal | 14 ++++++++------
1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metal
index b7b7e065..1b62d0dc 100644
--- a/ggml/src/ggml-metal/ggml-metal.metal
+++ b/ggml/src/ggml-metal/ggml-metal.metal
@@ -10773,9 +10773,15 @@ void kernel_mul_mv_id_q2_K_gate_up_swiglu_pair_sg_dual_impl(
return;
}

- if (slot < args.nei0 && token < args.nei1) {
+ const bool valid_output = slot < args.nei0 && token < args.nei1;
+ const float route_weight = valid_output && args.weighted != 0 ?
+ ((device const float *) (weights + uint64_t(slot)*args.weights_nb1 + uint64_t(token)*args.weights_nb2))[0] :
+ 1.0f;
+
+ if (valid_output) {
const int32_t expert_id = ((device const int32_t *) (ids + token*args.nbi1))[slot];
- const bool active_slot = args._pad0 <= 0 || slot < args._pad0;
+ const bool active_slot = (args._pad0 <= 0 || slot < args._pad0) &&
+ (args.weighted == 0 || route_weight != 0.0f);
if (active_slot && expert_id >= 0 && expert_id < args.ne02) {
const uint64_t offset0 = uint64_t(expert_id)*args.nb02 + uint64_t(first_row)*args.nb01;
const uint64_t offset1 = uint64_t(slot % args.ne11)*args.nb11 + uint64_t(token)*args.nb12;
@@ -10936,10 +10942,6 @@ void kernel_mul_mv_id_q2_K_gate_up_swiglu_pair_sg_dual_impl(
}
}

- const bool valid_output = slot < args.nei0 && token < args.nei1;
- const float route_weight = valid_output && args.weighted != 0 ?
- ((device const float *) (weights + uint64_t(slot)*args.weights_nb1 + uint64_t(token)*args.weights_nb2))[0] :
- 1.0f;
for (int row = 0; row < nr0; ++row) {
const float up_value = simd_sum(sum_up[row]);
const float gate_value = simd_sum(sum_gate[row]);
--
2.54.0 (Apple Git-157)

35 changes: 35 additions & 0 deletions third_party/llama.cpp/patches/0033-tests-cover-full-GLM-MoE-selected-chain.patch
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
From 37ee00411144af6570074a352ac8584ab8527ad6 Mon Sep 17 00:00:00 2001
From: James Dumay <james@mesh-llm.local>
Date: Sun, 19 Jul 2026 00:00:00 +1000
Subject: [PATCH 28/36] tests: cover full GLM MoE selected chain

---
tests/test-backend-ops.cpp | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index 53489947..2aaded1c 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -7824,7 +7824,9 @@ struct test_glm_moe_selected_chain : public test_case {
}
}

- double max_nmse_err() override { return 5e-4; }
+ double max_nmse_err() override {
+ return n_repeats > 1 ? 5e-3 : 5e-4;
+ }

bool eval_perf(ggml_backend_t backend, const char * op_names_filter, printer * output_printer) override {
scoped_test_env motif_reference_env("GGML_METAL_ENABLE_GLM_MOE_DECODE_MOTIF_REFERENCE", "1");
@@ -14945,6 +14947,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_glm_moe_q2_weight_roofline(true, "FIXED_IDS"));
test_cases.emplace_back(new test_glm_moe_q2_weight_roofline(false, "ROUTE_B"));
test_cases.emplace_back(new test_glm_moe_selected_chain(1, false, "CORRECTNESS"));
+ test_cases.emplace_back(new test_glm_moe_selected_chain(3, false, "USEFUL_A"));
for (ggml_type type : {GGML_TYPE_Q8_0, GGML_TYPE_Q6_K, GGML_TYPE_Q5_K, GGML_TYPE_Q4_K}) {
test_cases.emplace_back(new test_glm_attention_projection_chain(type, 2));
}
--
2.54.0 (Apple Git-157)

Loading
Loading