From 6ad03e3a22453eacc8fd3b093222e80ce8fea2c8 Mon Sep 17 00:00:00 2001 From: Christopher Albert Date: Tue, 23 Jun 2026 09:30:12 +0200 Subject: [PATCH 1/2] llama : meta split state for combined gate+up ffn_up (phi3) Architectures that pack gate and up into a single ffn_up tensor of width 2*n_ff (phi3) were split naively across devices, so the ffn_down source split state did not match and the meta backend aborted on GGML_ASSERT(split_states_equal(...)). Split such tensors into two n_ff groups, mirroring the fused ffn_gate_up handling. Fixes the test-llama-archs abort on phi3 with the meta (TP) backend. --- src/llama-model.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index d041a9ce3e2..572b681cec4 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -550,6 +550,11 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str GGML_ASSERT(tensor->ne[axis] == n_embd + 2*n_embd_gqa); return {{n_embd, 1}, {n_embd_gqa, 2}}; } + if ((std::regex_match(tensor_name, pattern_ffn_up_gate_weight) || + std::regex_match(tensor_name, pattern_ffn_up_gate_bias)) && + tensor->ne[axis] == 2*hparams.n_ff(il)) { + return {{hparams.n_ff(il), 2}}; + } if (std::regex_match(tensor_name, pattern_ffn_gate_up_weight)) { const int64_t n_ff_exp = hparams.n_ff_exp; GGML_ASSERT(tensor->ne[axis] == 2*n_ff_exp); From a0caeb18de503a2c77ef925a4cd14b2b8669b071 Mon Sep 17 00:00:00 2001 From: Christopher Albert Date: Fri, 3 Jul 2026 19:37:01 +0200 Subject: [PATCH 2/2] llama : guard combined ffn_up split, use per-layer n_ff Keep the 2*n_ff dimension check as a guard so ordinary separate ffn_up/ffn_gate tensors fall through to the default even split. The combined gate|up ffn_up is dense-only (phi3, modern-bert), so n_ff(il) is the correct width; fetch it once. --- src/llama-model.cpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 572b681cec4..444366e5d6a 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -550,10 +550,15 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str GGML_ASSERT(tensor->ne[axis] == n_embd + 2*n_embd_gqa); return {{n_embd, 1}, {n_embd_gqa, 2}}; } - if ((std::regex_match(tensor_name, pattern_ffn_up_gate_weight) || - std::regex_match(tensor_name, pattern_ffn_up_gate_bias)) && - tensor->ne[axis] == 2*hparams.n_ff(il)) { - return {{hparams.n_ff(il), 2}}; + // phi3 and modern-bert pack gate|up into a dense ffn_up of width 2*n_ff; + // ordinary separate ffn_up/ffn_gate stay at n_ff, so this stays a guard + // that falls through to the default even split rather than an assert. + if (std::regex_match(tensor_name, pattern_ffn_up_gate_weight) || + std::regex_match(tensor_name, pattern_ffn_up_gate_bias)) { + const int64_t n_ff = hparams.n_ff(il); + if (n_ff != 0 && tensor->ne[axis] == 2*n_ff) { + return {{n_ff, 2}}; + } } if (std::regex_match(tensor_name, pattern_ffn_gate_up_weight)) { const int64_t n_ff_exp = hparams.n_ff_exp;