diff --git a/third_party/llama.cpp/patches/0001-Add-Skippy-ABI-and-package-writer-foundation.patch b/third_party/llama.cpp/patches/0001-Add-Skippy-ABI-and-package-writer-foundation.patch index 920444261e..2a8d0d3366 100644 --- a/third_party/llama.cpp/patches/0001-Add-Skippy-ABI-and-package-writer-foundation.patch +++ b/third_party/llama.cpp/patches/0001-Add-Skippy-ABI-and-package-writer-foundation.patch @@ -1,7 +1,7 @@ -From 34be3b6d89d94ac218678e7f6ada201a1bcba4c1 Mon Sep 17 00:00:00 2001 +From 7e292e39190998abf873f50a8445d50b701cdb69 Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Fri, 26 Jun 2026 13:47:52 +1000 -Subject: [PATCH 01/45] Add Skippy ABI and package writer foundation +Subject: [PATCH 01/47] Add Skippy ABI and package writer foundation Squashes the initial Skippy ABI headers, model info entrypoints, single-stage runtime baseline, runtime-slice tensor filtering, activation-frame execution, GGUF slice writing, and package part composition. --- @@ -19,10 +19,10 @@ Squashes the initial Skippy ABI headers, model info entrypoints, single-stage ru create mode 100644 src/skippy.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt -index 81f23d7e..50ad6e57 100644 +index 3df1d82d..ad7652a0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt -@@ -245,6 +245,7 @@ set(LLAMA_BIN_INSTALL_DIR ${CMAKE_INSTALL_BINDIR} CACHE PATH "Location o +@@ -254,6 +254,7 @@ set(LLAMA_BIN_INSTALL_DIR ${CMAKE_INSTALL_BINDIR} CACHE PATH "Location o set(LLAMA_PUBLIC_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/include/llama.h @@ -346,7 +346,7 @@ index 320784c3..5298c337 100644 unicode-data.cpp unicode.cpp diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp -index c8ecb0a2..660dee46 100644 +index 6d1c8f4e..137dc9da 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -21,6 +21,20 @@ @@ -370,7 +370,7 @@ index c8ecb0a2..660dee46 100644 // dedup helpers static ggml_tensor * build_attn_inp_kq_mask( -@@ -2173,28 +2187,32 @@ ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd) const { +@@ -2184,28 +2198,32 @@ ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd) const { { auto & cur = inps[0]; @@ -444,7 +444,7 @@ index 7ed490ce..a7986a87 100644 enum llm_graph_type { LLM_GRAPH_TYPE_DEFAULT, diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp -index 43447f57..7f23dbbc 100644 +index 510586e9..6254771a 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -18,6 +18,16 @@ static const size_t kiB = 1024; diff --git a/third_party/llama.cpp/patches/0002-Add-early-staged-model-family-and-chat-support.patch b/third_party/llama.cpp/patches/0002-Add-early-staged-model-family-and-chat-support.patch index f26e2540fe..2d73c8459c 100644 --- a/third_party/llama.cpp/patches/0002-Add-early-staged-model-family-and-chat-support.patch +++ b/third_party/llama.cpp/patches/0002-Add-early-staged-model-family-and-chat-support.patch @@ -1,7 +1,7 @@ -From 0f00b168cc09677e844bf8822040aa3d71f304b5 Mon Sep 17 00:00:00 2001 +From 4a3154e28533e87a29222f219f882ef0ec28dce7 Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Fri, 26 Jun 2026 13:47:52 +1000 -Subject: [PATCH 02/45] Add early staged model-family and chat support +Subject: [PATCH 02/47] Add early staged model-family and chat support Squashes the first Qwen MoE staged-runtime support, native KV page basics, session reset, token EOG exposure, batched token verification, chat-template ABI, and the first wider model-family support pass. --- @@ -32,7 +32,7 @@ Squashes the first Qwen MoE staged-runtime support, native KV page basics, sessi create mode 100644 common/stage-chat.cpp diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt -index 99688f53..3928ba17 100644 +index 799d2275..a3cfaec1 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -98,6 +98,7 @@ add_library(${TARGET} @@ -42,7 +42,7 @@ index 99688f53..3928ba17 100644 + stage-chat.cpp speculative.cpp speculative.h - trie.cpp + subproc.cpp diff --git a/common/stage-chat.cpp b/common/stage-chat.cpp new file mode 100644 index 00000000..39d6331a @@ -319,7 +319,7 @@ index 7ab93d35..23a46e0f 100644 const char * path, struct skippy_model_info ** out_info, diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp -index 660dee46..cca99a8f 100644 +index 137dc9da..bed11342 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -22,6 +22,7 @@ @@ -455,7 +455,7 @@ index a7986a87..33d57061 100644 public: llm_graph_input_pos(uint32_t n_pos_per_embd) : n_pos_per_embd(n_pos_per_embd) {} diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp -index e25464c5..d7c7c220 100644 +index 44cb1668..638c9664 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp @@ -4,6 +4,7 @@ @@ -466,7 +466,7 @@ index e25464c5..d7c7c220 100644 #include #include -@@ -1224,6 +1225,352 @@ ggml_tensor * llama_kv_cache::get_k_storage(int32_t il) const { +@@ -1337,6 +1338,352 @@ ggml_tensor * llama_kv_cache::get_k_storage(int32_t il) const { return layers[ikv].k; } @@ -820,7 +820,7 @@ index e25464c5..d7c7c220 100644 uint32_t result = 0; diff --git a/src/llama-kv-cache.h b/src/llama-kv-cache.h -index 531d99db..d9301c19 100644 +index d5a92f44..e421bd1c 100644 --- a/src/llama-kv-cache.h +++ b/src/llama-kv-cache.h @@ -6,12 +6,14 @@ @@ -863,7 +863,7 @@ index 531d99db..d9301c19 100644 // graph_build API // diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp -index 7f23dbbc..c94aca78 100644 +index 6254771a..be9233c2 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -1082,13 +1082,6 @@ struct ggml_tensor * llama_model_loader::create_tensor( @@ -929,10 +929,10 @@ index 7f23dbbc..c94aca78 100644 if (info.op == GGML_OP_NONE || (flags & TENSOR_SKIP)) { const size_t nbytes = ggml_nbytes(t_meta); diff --git a/src/llama-model.cpp b/src/llama-model.cpp -index b100f601..07d3f4f3 100644 +index be0a0df5..4152ee61 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp -@@ -1398,13 +1398,13 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) { +@@ -1403,13 +1403,13 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) { } // MoE expert weight scales (per-expert, shape {n_expert}) diff --git a/third_party/llama.cpp/patches/0003-Add-staged-sampling-checkpoints-and-part-loading.patch b/third_party/llama.cpp/patches/0003-Add-staged-sampling-checkpoints-and-part-loading.patch index 8a296b15a8..27fc5f9a47 100644 --- a/third_party/llama.cpp/patches/0003-Add-staged-sampling-checkpoints-and-part-loading.patch +++ b/third_party/llama.cpp/patches/0003-Add-staged-sampling-checkpoints-and-part-loading.patch @@ -1,7 +1,7 @@ -From ab8b97e2a7169c1f6081a27c49f6bc64959f79a1 Mon Sep 17 00:00:00 2001 +From b2d816ab8938faef60abdd2d1f95fe3cc2a1e9d1 Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Fri, 26 Jun 2026 13:47:52 +1000 -Subject: [PATCH 03/45] Add staged sampling, checkpoints, and part loading +Subject: [PATCH 03/47] Add staged sampling, checkpoints, and part loading Squashes sampling configuration, batched verification frames, recurrent/native session checkpoints, logit bias, session trim, verification handoff fixes, split-metadata stripping, thinking toggle support, and ordered GGUF part loading. --- @@ -49,10 +49,10 @@ index 39d6331a..60d53fc4 100644 for (size_t i = 0; i < message_count; ++i) { diff --git a/include/llama.h b/include/llama.h -index 9fab6931..e54b64dc 100644 +index 3c6d22be..99efc134 100644 --- a/include/llama.h +++ b/include/llama.h -@@ -514,6 +514,13 @@ extern "C" { +@@ -515,6 +515,13 @@ extern "C" { size_t n_paths, struct llama_model_params params); @@ -245,7 +245,7 @@ index 23a46e0f..4dd985da 100644 size_t output_text_capacity, size_t * out_text_bytes, diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp -index c94aca78..ecafbdf2 100644 +index be9233c2..3070a8c6 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -532,6 +532,7 @@ llama_model_loader::llama_model_loader( @@ -321,10 +321,10 @@ index 92ad6430..f73fa907 100644 llama_load_mode load_mode, bool check_tensors, diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp -index caf7733a..7a3d105a 100644 +index 92ebc11b..03a6c34f 100644 --- a/src/llama-quant.cpp +++ b/src/llama-quant.cpp -@@ -885,7 +885,7 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: +@@ -893,7 +893,7 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: const llama_model_kv_override * kv_overrides = params->kv_overrides; std::vector splits = {}; llama_model_loader ml(/*metadata*/ nullptr, /*set_tensor_data*/ nullptr, /*set_tensor_data_ud*/ nullptr, @@ -334,10 +334,10 @@ index caf7733a..7a3d105a 100644 auto mparams = llama_model_default_params(); diff --git a/src/llama.cpp b/src/llama.cpp -index 11ac9656..48c578ee 100644 +index d22e4c81..37ddf52f 100644 --- a/src/llama.cpp +++ b/src/llama.cpp -@@ -299,9 +299,9 @@ static bool llama_prepare_model_devices(const llama_model_params & params, llama +@@ -302,9 +302,9 @@ static bool llama_prepare_model_devices(const llama_model_params & params, llama // Returns 0 on success, -1 on error, and -2 on cancellation via llama_progress_callback static std::pair llama_model_load(struct gguf_context * metadata, llama_model_set_tensor_data_t set_tensor_data, void * set_tensor_data_ud, @@ -349,7 +349,7 @@ index 11ac9656..48c578ee 100644 params.check_tensors, params.no_alloc, params.kv_overrides, params.tensor_buft_overrides); ml.print_info(); -@@ -366,6 +366,7 @@ static struct llama_model * llama_model_load_from_file_impl( +@@ -369,6 +369,7 @@ static struct llama_model * llama_model_load_from_file_impl( void * set_tensor_data_ud, const std::string & path_model, std::vector & splits, @@ -357,7 +357,7 @@ index 11ac9656..48c578ee 100644 FILE * file, struct llama_model_params params) { { -@@ -408,7 +409,7 @@ static struct llama_model * llama_model_load_from_file_impl( +@@ -411,7 +412,7 @@ static struct llama_model * llama_model_load_from_file_impl( }; } @@ -366,7 +366,7 @@ index 11ac9656..48c578ee 100644 GGML_ASSERT(status <= 0); if (status < 0) { if (status == -1) { -@@ -436,7 +437,7 @@ struct llama_model * llama_model_init_from_user( +@@ -439,7 +440,7 @@ struct llama_model * llama_model_init_from_user( std::vector splits = {}; params.load_mode = LLAMA_LOAD_MODE_NONE; params.use_extra_bufts = false; @@ -375,7 +375,7 @@ index 11ac9656..48c578ee 100644 } // deprecated struct llama_model * llama_load_model_from_file( -@@ -449,7 +450,7 @@ struct llama_model * llama_model_load_from_file( +@@ -452,7 +453,7 @@ struct llama_model * llama_model_load_from_file( const char * path_model, struct llama_model_params params) { std::vector splits = {}; @@ -384,7 +384,7 @@ index 11ac9656..48c578ee 100644 } struct llama_model * llama_model_load_from_splits( -@@ -465,7 +466,27 @@ struct llama_model * llama_model_load_from_splits( +@@ -468,7 +469,27 @@ struct llama_model * llama_model_load_from_splits( for (size_t i = 0; i < n_paths; ++i) { splits.push_back(paths[i]); } @@ -413,7 +413,7 @@ index 11ac9656..48c578ee 100644 } struct llama_model * llama_model_load_from_file_ptr(FILE * file, struct llama_model_params params) { -@@ -475,7 +496,7 @@ struct llama_model * llama_model_load_from_file_ptr(FILE * file, struct llama_mo +@@ -478,7 +499,7 @@ struct llama_model * llama_model_load_from_file_ptr(FILE * file, struct llama_mo } std::string path_model; std::vector splits = {}; @@ -422,7 +422,7 @@ index 11ac9656..48c578ee 100644 } void llama_model_save_to_file(const struct llama_model * model, const char * path_model) { -@@ -600,4 +621,3 @@ const char * llama_print_system_info(void) { +@@ -603,4 +624,3 @@ const char * llama_print_system_info(void) { return s.c_str(); } diff --git a/third_party/llama.cpp/patches/0004-Add-lanes-external-media-and-chat-grammar-support.patch b/third_party/llama.cpp/patches/0004-Add-lanes-external-media-and-chat-grammar-support.patch index 475ea40e7f..83b3fac54c 100644 --- a/third_party/llama.cpp/patches/0004-Add-lanes-external-media-and-chat-grammar-support.patch +++ b/third_party/llama.cpp/patches/0004-Add-lanes-external-media-and-chat-grammar-support.patch @@ -1,7 +1,7 @@ -From 59cdfb7bf5cf5e112f229ebec9a4e0fa3d954547 Mon Sep 17 00:00:00 2001 +From aec3d7b74786f7434166c15be069d882cdb39ea8 Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Fri, 26 Jun 2026 13:47:52 +1000 -Subject: [PATCH 04/45] Add lanes, external media, and chat grammar support +Subject: [PATCH 04/47] Add lanes, external media, and chat grammar support Squashes Qwen3 MoE staged execution, KV cache type and generation signal ABI, backend device selection, external media prefill, shared execution lanes, batch/flash-attention config, tool-aware chat templates, and chat grammar sampling. --- @@ -573,7 +573,7 @@ index 4dd985da..3ffdb879 100644 const char * path, struct skippy_model_info ** out_info, diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp -index d7c7c220..f74bb495 100644 +index 638c9664..41962096 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp @@ -196,12 +196,10 @@ llama_kv_cache::llama_kv_cache( @@ -593,7 +593,7 @@ index d7c7c220..f74bb495 100644 } // [TAG_V_CACHE_VARIABLE] -@@ -1225,352 +1223,6 @@ ggml_tensor * llama_kv_cache::get_k_storage(int32_t il) const { +@@ -1338,352 +1336,6 @@ ggml_tensor * llama_kv_cache::get_k_storage(int32_t il) const { return layers[ikv].k; } @@ -947,7 +947,7 @@ index d7c7c220..f74bb495 100644 uint32_t result = 0; diff --git a/src/llama-kv-cache.h b/src/llama-kv-cache.h -index d9301c19..92394d4c 100644 +index e421bd1c..6c92186d 100644 --- a/src/llama-kv-cache.h +++ b/src/llama-kv-cache.h @@ -13,8 +13,6 @@ struct llama_cparams; diff --git a/third_party/llama.cpp/patches/0005-Add-resident-prefix-cache-and-session-refinements.patch b/third_party/llama.cpp/patches/0005-Add-resident-prefix-cache-and-session-refinements.patch index b6165645af..e645173e72 100644 --- a/third_party/llama.cpp/patches/0005-Add-resident-prefix-cache-and-session-refinements.patch +++ b/third_party/llama.cpp/patches/0005-Add-resident-prefix-cache-and-session-refinements.patch @@ -1,12 +1,12 @@ -From 8d4cf568f56593d1fa8223ad12ca2965d680fee6 Mon Sep 17 00:00:00 2001 +From df9fea9e4a0d7bd91ce128ca3820ed9618aef466 Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Fri, 26 Jun 2026 13:47:52 +1000 -Subject: [PATCH 05/45] Add resident prefix cache and session refinements +Subject: [PATCH 05/47] Add resident prefix cache and session refinements Squashes resident prefix cache support, exact-prefix state restoration, borrowed prefix sessions, lazy generation signals, single-token allocation avoidance, server-like thread defaults, prefix cache borrowing, lane preservation, and recurrent-state remapping. --- include/skippy.h | 119 +++++- - src/llama-kv-cache.cpp | 369 +++++++++++++++++++ + src/llama-kv-cache.cpp | 369 ++++++++++++++++++ src/llama-kv-cache.h | 21 ++ src/skippy.cpp | 821 ++++++++++++++++++++++++++++++++++++++++- 4 files changed, 1320 insertions(+), 10 deletions(-) @@ -196,10 +196,10 @@ index 3ffdb879..deb5151e 100644 struct skippy_model * model, const char * text, diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp -index f74bb495..41a4e0de 100644 +index 41962096..942a1a4a 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp -@@ -1223,6 +1223,375 @@ ggml_tensor * llama_kv_cache::get_k_storage(int32_t il) const { +@@ -1336,6 +1336,375 @@ ggml_tensor * llama_kv_cache::get_k_storage(int32_t il) const { return layers[ikv].k; } @@ -576,7 +576,7 @@ index f74bb495..41a4e0de 100644 uint32_t result = 0; diff --git a/src/llama-kv-cache.h b/src/llama-kv-cache.h -index 92394d4c..be51c3c7 100644 +index 6c92186d..beaf005e 100644 --- a/src/llama-kv-cache.h +++ b/src/llama-kv-cache.h @@ -13,6 +13,8 @@ struct llama_cparams; diff --git a/third_party/llama.cpp/patches/0006-Expand-staged-execution-across-dense-and-recurrent-f.patch b/third_party/llama.cpp/patches/0006-Expand-staged-execution-across-dense-and-recurrent-f.patch index 709f3e5630..d140b4dece 100644 --- a/third_party/llama.cpp/patches/0006-Expand-staged-execution-across-dense-and-recurrent-f.patch +++ b/third_party/llama.cpp/patches/0006-Expand-staged-execution-across-dense-and-recurrent-f.patch @@ -1,7 +1,7 @@ -From 7432fb6267d078b2e6c79df8cc301517913cf6c0 Mon Sep 17 00:00:00 2001 +From 95a491c6dd9fd92ba8b7b3406b632c8821cbc5a6 Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Fri, 26 Jun 2026 13:47:53 +1000 -Subject: [PATCH 06/45] Expand staged execution across dense and recurrent +Subject: [PATCH 06/47] Expand staged execution across dense and recurrent families Squashes the dense and decoder family support waves through LFM2, Mamba, Jamba, RWKV6, Qwen2MoE, and native sequence id exposure. @@ -51,10 +51,10 @@ index deb5151e..84169398 100644 const struct skippy_session * session); diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp -index cca99a8f..f2618710 100644 +index bed11342..ad8f2fc6 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp -@@ -2269,10 +2269,15 @@ ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd) const { +@@ -2280,10 +2280,15 @@ ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd) const { res->t_inp_embd = cur; diff --git a/third_party/llama.cpp/patches/0007-Expand-staged-execution-across-VL-and-broad-model-fa.patch b/third_party/llama.cpp/patches/0007-Expand-staged-execution-across-VL-and-broad-model-fa.patch index 262b56fc3e..14faeea86a 100644 --- a/third_party/llama.cpp/patches/0007-Expand-staged-execution-across-VL-and-broad-model-fa.patch +++ b/third_party/llama.cpp/patches/0007-Expand-staged-execution-across-VL-and-broad-model-fa.patch @@ -1,7 +1,7 @@ -From a4be6bd95fb953270ad66f37355eb8e154387d6e Mon Sep 17 00:00:00 2001 +From de078a2e2458d364e6dbe3e895fd7a30e7296d0b Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Fri, 26 Jun 2026 13:47:53 +1000 -Subject: [PATCH 07/45] Expand staged execution across VL and broad model +Subject: [PATCH 07/47] Expand staged execution across VL and broad model families Squashes RWKV7 activation sideband support, Phi2, Granite, Hunyuan Dense/MoE, PhiMoE, tied-output handling, Qwen35/Hunyuan-VL, broad llama-family coverage, BitNet and StarCoder fixes, and DeepSeek OCR/Qwen3-VL-MoE support. @@ -84,10 +84,10 @@ index 84169398..1c6d2c9f 100644 struct skippy_session; struct skippy_model_info; diff --git a/src/llama-context.cpp b/src/llama-context.cpp -index eed041ee..c0bfd7db 100644 +index 9b399d60..dc61ddc8 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp -@@ -2352,6 +2352,10 @@ llm_graph_result * llama_context::get_gf_res_reserve() const { +@@ -2354,6 +2354,10 @@ llm_graph_result * llama_context::get_gf_res_reserve() const { return static_cast(gf_res_reserve.get()); } @@ -111,7 +111,7 @@ index bf91daa8..42442879 100644 // returns the result of ggml_backend_sched_graph_compute_async execution ggml_status graph_compute(ggml_cgraph * gf, bool batched); diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp -index f2618710..03873edb 100644 +index ad8f2fc6..70b61495 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -23,6 +23,7 @@ @@ -244,7 +244,7 @@ index 33d57061..d9e241a1 100644 std::vector t_layer_inp; diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp -index ecafbdf2..91589e1b 100644 +index 3070a8c6..3b1d48f4 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -19,6 +19,7 @@ static const size_t MiB = 1024*kiB; @@ -305,10 +305,10 @@ index f73fa907..5d693719 100644 enum llama_fver { GGUF_FILE_VERSION_V1 = 1, diff --git a/src/llama-model.cpp b/src/llama-model.cpp -index 07d3f4f3..b603f869 100644 +index 4152ee61..9cc29a51 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp -@@ -2763,7 +2763,8 @@ void llama_model_base::create_tensor_qkv(llama_layer & layer, int bid, +@@ -2770,7 +2770,8 @@ void llama_model_base::create_tensor_qkv(llama_layer & layer, int bid, int flags) { const int64_t n_embd_qkv = n_embd_q_ + n_embd_k_ + n_embd_v_; layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", bid), {n_embd_, n_embd_qkv}, TENSOR_NOT_REQUIRED | TENSOR_SKIP_IF_VIRTUAL); @@ -1803,7 +1803,7 @@ index 6e2bd9a3..5b91df3d 100644 model.output_norm, model.output_norm_b, LLM_NORM, -1); diff --git a/src/models/openai-moe.cpp b/src/models/openai-moe.cpp -index 6d74f9c7..d860df79 100644 +index c91bae1c..8c4d7bed 100644 --- a/src/models/openai-moe.cpp +++ b/src/models/openai-moe.cpp @@ -65,16 +65,21 @@ llama_model_openai_moe::graph::graph(const llama_model & model, const llm_graph_ @@ -1835,8 +1835,8 @@ index 6d74f9c7..d860df79 100644 cb(cur, "attn_out", il); } -- if (il == n_layer - 1) { -+ if (il == il_end - 1 && inp_out_ids) { +- if (il == n_layer - 1 && inp_out_ids && cparams.embeddings_nextn_masked) { ++ if (il == il_end - 1 && inp_out_ids && cparams.embeddings_nextn_masked) { // skip computing output for unused tokens cur = ggml_get_rows(ctx0, cur, inp_out_ids); inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); @@ -1851,9 +1851,9 @@ index 6d74f9c7..d860df79 100644 + return; + } + - cur = build_norm(cur, - model.output_norm, NULL, - LLM_NORM_RMS, -1); + res->t_h_nextn = cur; + + if (!cparams.embeddings_nextn_masked && inp_out_ids) { diff --git a/src/models/openelm.cpp b/src/models/openelm.cpp index 13120bd3..33ca072f 100644 --- a/src/models/openelm.cpp diff --git a/third_party/llama.cpp/patches/0008-Add-external-decode-media-prefill-and-newer-family-s.patch b/third_party/llama.cpp/patches/0008-Add-external-decode-media-prefill-and-newer-family-s.patch index 946db2893d..2e55eb3c20 100644 --- a/third_party/llama.cpp/patches/0008-Add-external-decode-media-prefill-and-newer-family-s.patch +++ b/third_party/llama.cpp/patches/0008-Add-external-decode-media-prefill-and-newer-family-s.patch @@ -1,7 +1,7 @@ -From 3653cfda98f3349e7f48490c90128825c8d47762 Mon Sep 17 00:00:00 2001 +From a40efb4320028532de6767cc26d63f500fb9dadd Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Fri, 26 Jun 2026 13:47:53 +1000 -Subject: [PATCH 08/45] Add external decode, media prefill, and newer family +Subject: [PATCH 08/47] Add external decode, media prefill, and newer family support Squashes tensor filtering accounting fixes, recurrent KV page export handling, external decode filtering, staged external media prefill, tensor element counts, activation-frame batch padding and position sideband, plus Gemma3n, Llama4, and Mistral4 slices. @@ -98,7 +98,7 @@ index 1c6d2c9f..f1ff7852 100644 struct skippy_session * session, llama_token token_id, diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp -index 03873edb..fb804ffd 100644 +index 70b61495..bf7b62f6 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -24,6 +24,7 @@ @@ -322,10 +322,10 @@ index d9e241a1..2e0d9c3b 100644 std::vector t_layer_inp; diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp -index 41a4e0de..0e6902aa 100644 +index 942a1a4a..ac10ba6c 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp -@@ -1302,14 +1302,16 @@ bool llama_kv_cache::stage_export_kv_page( +@@ -1419,14 +1419,16 @@ bool llama_kv_cache::stage_export_kv_page( if (il < static_cast(layer_start) || il >= static_cast(layer_end)) { continue; } @@ -347,7 +347,7 @@ index 41a4e0de..0e6902aa 100644 if (k_row > std::numeric_limits::max() || v_row > std::numeric_limits::max() || -@@ -1322,19 +1324,19 @@ bool llama_kv_cache::stage_export_kv_page( +@@ -1439,19 +1441,19 @@ bool llama_kv_cache::stage_export_kv_page( v_row_bytes = static_cast(v_row); v_element_bytes = static_cast(v_el); k_type = static_cast(layer.k_stream[strm]->type); @@ -372,7 +372,7 @@ index 41a4e0de..0e6902aa 100644 const uint64_t k_bytes = token_count * k_row; if (payload_bytes > std::numeric_limits::max() - k_bytes || payload_bytes + k_bytes > std::numeric_limits::max() - v_bytes) { -@@ -1388,6 +1390,9 @@ bool llama_kv_cache::stage_export_kv_page( +@@ -1505,6 +1507,9 @@ bool llama_kv_cache::stage_export_kv_page( if (!v_trans) { for (const auto * layer : selected) { @@ -382,7 +382,7 @@ index 41a4e0de..0e6902aa 100644 auto * v = layer->v_stream[strm]; for (uint32_t cell_idx : cell_idxs) { ggml_backend_tensor_get(v, dst, static_cast(cell_idx) * v_row_bytes, v_row_bytes); -@@ -1466,15 +1471,17 @@ bool llama_kv_cache::stage_import_kv_page( +@@ -1587,15 +1592,17 @@ bool llama_kv_cache::stage_import_kv_page( if (il < static_cast(desc.layer_start) || il >= static_cast(desc.layer_end)) { continue; } @@ -406,7 +406,7 @@ index 41a4e0de..0e6902aa 100644 desc.k_row_bytes != k_row || desc.v_row_bytes != v_row || desc.v_element_bytes != v_el) { -@@ -1557,6 +1564,9 @@ bool llama_kv_cache::stage_import_kv_page( +@@ -1682,6 +1689,9 @@ bool llama_kv_cache::stage_import_kv_page( auto & cells = v_cells[strm]; if (!v_trans) { for (const auto * layer : selected) { @@ -417,7 +417,7 @@ index 41a4e0de..0e6902aa 100644 for (uint32_t i = 0; i < n_tokens; ++i) { ggml_backend_tensor_set(v, src, static_cast(sinfo.idxs[0][i]) * desc.v_row_bytes, desc.v_row_bytes); diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp -index 91589e1b..a62c3644 100644 +index 3b1d48f4..ec530d62 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -1126,6 +1126,19 @@ struct ggml_tensor * llama_model_loader::create_tensor( diff --git a/third_party/llama.cpp/patches/0009-Add-chat-grammar-device-enumeration-and-runtime-even.patch b/third_party/llama.cpp/patches/0009-Add-chat-grammar-device-enumeration-and-runtime-even.patch index a830e8d881..0be171fe95 100644 --- a/third_party/llama.cpp/patches/0009-Add-chat-grammar-device-enumeration-and-runtime-even.patch +++ b/third_party/llama.cpp/patches/0009-Add-chat-grammar-device-enumeration-and-runtime-even.patch @@ -1,7 +1,7 @@ -From 8619285489ebdbaf28638a8cf598c00e2cbf66ec Mon Sep 17 00:00:00 2001 +From 2a4018835a0c94a87c132efabb74ccee8d0e42d0 Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Fri, 26 Jun 2026 13:47:53 +1000 -Subject: [PATCH 09/45] Add chat grammar, device enumeration, and runtime +Subject: [PATCH 09/47] Add chat grammar, device enumeration, and runtime events Squashes chat grammar exception containment and lazy triggering, backend device enumeration, min-p sampling, KV cell compaction, and model-open runtime events. @@ -446,10 +446,10 @@ index 00000000..382f1e69 + +#endif diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp -index 0e6902aa..131064e0 100644 +index ac10ba6c..c4517d65 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp -@@ -736,11 +736,28 @@ llama_memory_context_ptr llama_kv_cache::init_full() { +@@ -796,11 +796,28 @@ llama_memory_context_ptr llama_kv_cache::init_full() { } llama_memory_context_ptr llama_kv_cache::init_update(llama_context * lctx, bool optimize) { @@ -481,7 +481,7 @@ index 0e6902aa..131064e0 100644 } llama_kv_cache::slot_info_vec_t llama_kv_cache::prepare(const std::vector & ubatches) { -@@ -1167,6 +1184,93 @@ void llama_kv_cache::apply_ubatch(const slot_info & sinfo, const llama_ubatch & +@@ -1274,6 +1291,93 @@ void llama_kv_cache::apply_ubatch(const slot_info & sinfo, const llama_ubatch & } } @@ -575,7 +575,7 @@ index 0e6902aa..131064e0 100644 bool llama_kv_cache::get_can_shift() const { // Step35 uses per-layer RoPE dims; K-shift assumes a single global n_rot. if (model.arch == LLM_ARCH_STEP35) { -@@ -2884,8 +2988,9 @@ llama_kv_cache_context::llama_kv_cache_context( +@@ -3152,8 +3256,9 @@ llama_kv_cache_context::llama_kv_cache_context( llama_kv_cache * kv, llama_context * lctx, bool do_shift, @@ -588,7 +588,7 @@ index 0e6902aa..131064e0 100644 } } diff --git a/src/llama-kv-cache.h b/src/llama-kv-cache.h -index be51c3c7..f8569057 100644 +index beaf005e..b253d915 100644 --- a/src/llama-kv-cache.h +++ b/src/llama-kv-cache.h @@ -95,6 +95,12 @@ public: @@ -604,7 +604,7 @@ index be51c3c7..f8569057 100644 // TODO: refactor the memory instances to not depend on `llama_model` // instead pass all necessary info (e.g. hparams, dev layers, arch, etc.) directly // likely through `struct llama_memory_params` -@@ -217,6 +223,10 @@ public: +@@ -219,6 +225,10 @@ public: // emplace the ubatch context into slot: [sinfo.idxs[0...ubatch.n_tokens - 1]] void apply_ubatch(const slot_info & sinfo, const llama_ubatch & ubatch); @@ -615,7 +615,7 @@ index be51c3c7..f8569057 100644 // // input API // -@@ -359,7 +369,8 @@ public: +@@ -367,7 +377,8 @@ public: llama_kv_cache * kv, llama_context * lctx, bool do_shift, diff --git a/third_party/llama.cpp/patches/0010-Add-MTP-execution-support-and-sampling-cleanup.patch b/third_party/llama.cpp/patches/0010-Add-MTP-execution-support-and-sampling-cleanup.patch index ba24eab26b..8350669433 100644 --- a/third_party/llama.cpp/patches/0010-Add-MTP-execution-support-and-sampling-cleanup.patch +++ b/third_party/llama.cpp/patches/0010-Add-MTP-execution-support-and-sampling-cleanup.patch @@ -1,7 +1,7 @@ -From 42f14cd0687b5a880fa2a4f1db51ca035b5ba326 Mon Sep 17 00:00:00 2001 +From c16f8c23e5108b0295ea4f67ad295dd69ca6901f Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Fri, 26 Jun 2026 13:47:53 +1000 -Subject: [PATCH 10/45] Add MTP execution support and sampling cleanup +Subject: [PATCH 10/47] Add MTP execution support and sampling cleanup Squashes GLM MTP graph support, batched stage execution, native MTP sidecar support, batched MTP verification drafts, sampling and detokenization parity, unused ABI removal, MTP layer-count and sidecar fixes, VerifySpan copy avoidance, greedy sampling fast path, GLM chat fallback, and GLM DSA IndexShare support. --- @@ -296,7 +296,7 @@ index a6bd7fd9..cfb4a8fa 100644 #ifdef __cplusplus diff --git a/src/llama-context.cpp b/src/llama-context.cpp -index c0bfd7db..9ab7be4b 100644 +index dc61ddc8..abfe34f9 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -1989,9 +1989,22 @@ int llama_context::decode(const llama_batch & batch_inp) { @@ -324,10 +324,10 @@ index c0bfd7db..9ab7be4b 100644 } diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp -index fb804ffd..a75f0b75 100644 +index bf7b62f6..a84bf8b6 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp -@@ -2957,6 +2957,8 @@ ggml_tensor * llm_graph_context::build_attn( +@@ -2968,6 +2968,8 @@ ggml_tensor * llm_graph_context::build_attn( const auto & kq_mask = inp->get_kq_mask_mla(); @@ -337,10 +337,10 @@ index fb804ffd..a75f0b75 100644 ggml_tensor * kq_mask_all = ggml_fill(ctx0, kq_mask, -INFINITY); diff --git a/src/llama-model.cpp b/src/llama-model.cpp -index b603f869..10633a51 100644 +index 9cc29a51..442387e6 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp -@@ -2196,6 +2196,14 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, +@@ -2201,6 +2201,14 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, } } @@ -783,10 +783,10 @@ index df190e1f..1d90426d 100644 } diff --git a/src/models/models.h b/src/models/models.h -index 76daa8cc..f3b576c0 100644 +index 92ebfafa..f4fb02cc 100644 --- a/src/models/models.h +++ b/src/models/models.h -@@ -1068,6 +1068,10 @@ struct llama_model_deepseek2 : public llama_model_base { +@@ -1084,6 +1084,10 @@ struct llama_model_deepseek2 : public llama_model_base { graph(const llama_model & model, const llm_graph_params & params); }; @@ -797,7 +797,7 @@ index 76daa8cc..f3b576c0 100644 std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; -@@ -1220,6 +1224,7 @@ struct llama_model_glm_dsa : public llama_model_base { +@@ -1236,6 +1240,7 @@ struct llama_model_glm_dsa : public llama_model_base { struct graph : public llm_graph_context { graph(const llama_model & model, const llm_graph_params & params); }; diff --git a/third_party/llama.cpp/patches/0011-Pass-reasoning-format-through-stage-chat-templates.patch b/third_party/llama.cpp/patches/0011-Pass-reasoning-format-through-stage-chat-templates.patch index c4c30a9fdc..3156ebd82b 100644 --- a/third_party/llama.cpp/patches/0011-Pass-reasoning-format-through-stage-chat-templates.patch +++ b/third_party/llama.cpp/patches/0011-Pass-reasoning-format-through-stage-chat-templates.patch @@ -1,7 +1,7 @@ -From 3b3d6f4ab0c269931f5f75120f3d8b496917e3b7 Mon Sep 17 00:00:00 2001 +From 4522a9bafafd75bae57cb76b4db85c3661f084ea Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Fri, 3 Jul 2026 13:15:19 +1000 -Subject: [PATCH 11/45] Pass reasoning format through stage chat templates +Subject: [PATCH 11/47] Pass reasoning format through stage chat templates --- common/stage-chat.cpp | 4 ++++ diff --git a/third_party/llama.cpp/patches/0012-Wire-mmap-and-mlock-runtime-load-options.patch b/third_party/llama.cpp/patches/0012-Wire-mmap-and-mlock-runtime-load-options.patch index 64904604df..0df47da442 100644 --- a/third_party/llama.cpp/patches/0012-Wire-mmap-and-mlock-runtime-load-options.patch +++ b/third_party/llama.cpp/patches/0012-Wire-mmap-and-mlock-runtime-load-options.patch @@ -1,7 +1,7 @@ -From 08a6cf87151e9f2460482d2889cb9f3750d47316 Mon Sep 17 00:00:00 2001 +From 60092b7fe8f4cc756dd8fac7cc736c836058f304 Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Wed, 1 Jul 2026 23:52:26 -0400 -Subject: [PATCH 12/45] Wire mmap and mlock runtime load options +Subject: [PATCH 12/47] Wire mmap and mlock runtime load options --- include/skippy.h | 3 +++ diff --git a/third_party/llama.cpp/patches/0013-Add-external-MTP-draft-sidecar-attachment.patch b/third_party/llama.cpp/patches/0013-Add-external-MTP-draft-sidecar-attachment.patch index a824a13519..1032699e12 100644 --- a/third_party/llama.cpp/patches/0013-Add-external-MTP-draft-sidecar-attachment.patch +++ b/third_party/llama.cpp/patches/0013-Add-external-MTP-draft-sidecar-attachment.patch @@ -1,7 +1,7 @@ -From 4724a2467f59db12a2638a417b70577a5f1df6b4 Mon Sep 17 00:00:00 2001 +From b2a802de169d16a9dbbf19d33d134ea1e9c8e293 Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Fri, 3 Jul 2026 12:18:37 -0400 -Subject: [PATCH 13/45] Add external MTP draft sidecar attachment +Subject: [PATCH 13/47] Add external MTP draft sidecar attachment --- include/skippy.h | 6 ++++ diff --git a/third_party/llama.cpp/patches/0014-Add-non-frame-native-MTP-decode-ABI.patch b/third_party/llama.cpp/patches/0014-Add-non-frame-native-MTP-decode-ABI.patch index 45df7a4c2a..0dc0c4b21d 100644 --- a/third_party/llama.cpp/patches/0014-Add-non-frame-native-MTP-decode-ABI.patch +++ b/third_party/llama.cpp/patches/0014-Add-non-frame-native-MTP-decode-ABI.patch @@ -1,7 +1,7 @@ -From bc10f74e32c88213d092b45b46b24502a0d397cb Mon Sep 17 00:00:00 2001 +From 5c697fa4118abba8285af065bc77b791d4fe941b Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Fri, 3 Jul 2026 15:03:17 -0400 -Subject: [PATCH 14/45] Add non-frame native MTP decode ABI +Subject: [PATCH 14/47] Add non-frame native MTP decode ABI --- include/skippy.h | 9 +++++++++ diff --git a/third_party/llama.cpp/patches/0015-Fix-stage-activation-graph-input-allocation.patch b/third_party/llama.cpp/patches/0015-Fix-stage-activation-graph-input-allocation.patch index 308a81b550..029149eb40 100644 --- a/third_party/llama.cpp/patches/0015-Fix-stage-activation-graph-input-allocation.patch +++ b/third_party/llama.cpp/patches/0015-Fix-stage-activation-graph-input-allocation.patch @@ -1,14 +1,14 @@ -From 76ff9a9cdd614c8bf4a082d06b9d144bc6096dd4 Mon Sep 17 00:00:00 2001 +From 518d810ca858450ac5b81cb4e3a6b21f7b3e64de Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Wed, 15 Jul 2026 07:54:20 +1000 -Subject: [PATCH 15/45] Fix stage activation graph input allocation +Subject: [PATCH 15/47] Fix stage activation graph input allocation --- src/llama-graph.cpp | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp -index a75f0b75..6613747a 100644 +index a84bf8b6..1363deb5 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -135,8 +135,8 @@ void llm_graph_input_embd::set_input(const llama_ubatch * ubatch) { @@ -86,7 +86,7 @@ index a75f0b75..6613747a 100644 return res; } -@@ -2340,7 +2349,13 @@ ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd) const { +@@ -2351,7 +2360,13 @@ ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd) const { assert(ggml_are_same_shape (inps[0], inps[1])); assert(ggml_are_same_stride(inps[0], inps[1])); diff --git a/third_party/llama.cpp/patches/0016-Recognize-thinking-field-in-chat-auto-parser.patch b/third_party/llama.cpp/patches/0016-Recognize-thinking-field-in-chat-auto-parser.patch index 7a87d97086..a0f38fd4fe 100644 --- a/third_party/llama.cpp/patches/0016-Recognize-thinking-field-in-chat-auto-parser.patch +++ b/third_party/llama.cpp/patches/0016-Recognize-thinking-field-in-chat-auto-parser.patch @@ -1,7 +1,7 @@ -From 587f96c77401abfb93f56147b306c2f5641f2a54 Mon Sep 17 00:00:00 2001 +From 4f5f711be6c0acff41e0fcfdc175e46224da4e7b Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Tue, 14 Jul 2026 19:32:50 +1000 -Subject: [PATCH 16/45] Recognize thinking field in chat auto-parser +Subject: [PATCH 16/47] Recognize thinking field in chat auto-parser --- common/chat-diff-analyzer.cpp | 10 +++++++--- diff --git a/third_party/llama.cpp/patches/0017-Expose-stateful-N-gram-cache-ABI.patch b/third_party/llama.cpp/patches/0017-Expose-stateful-N-gram-cache-ABI.patch index 780dbc8fb4..ab80c7d460 100644 --- a/third_party/llama.cpp/patches/0017-Expose-stateful-N-gram-cache-ABI.patch +++ b/third_party/llama.cpp/patches/0017-Expose-stateful-N-gram-cache-ABI.patch @@ -1,7 +1,7 @@ -From c83720f81a6cebdfcba89d761afabb72bb6796f6 Mon Sep 17 00:00:00 2001 +From d5876c02fc1adef99acf71718638e56ce8d0e050 Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Mon, 20 Jul 2026 14:07:01 +1000 -Subject: [PATCH 17/45] Expose stateful N-gram cache ABI +Subject: [PATCH 17/47] Expose stateful N-gram cache ABI --- common/ngram-cache.cpp | 136 ++++++++++++++++++++++++++++++++++++++++ diff --git a/third_party/llama.cpp/patches/0018-Remove-legacy-session-checkpoint-ABI.patch b/third_party/llama.cpp/patches/0018-Remove-legacy-session-checkpoint-ABI.patch index bd91f1403b..f5ff307607 100644 --- a/third_party/llama.cpp/patches/0018-Remove-legacy-session-checkpoint-ABI.patch +++ b/third_party/llama.cpp/patches/0018-Remove-legacy-session-checkpoint-ABI.patch @@ -1,7 +1,7 @@ -From ac55c0b40d0c582878f2f0068315e5889c97f845 Mon Sep 17 00:00:00 2001 +From 312c8e1434d9864c39d619cd18c37eff79a59062 Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Mon, 20 Jul 2026 14:07:56 +1000 -Subject: [PATCH 18/45] Remove legacy session checkpoint ABI +Subject: [PATCH 18/47] Remove legacy session checkpoint ABI --- include/skippy.h | 10 ---- diff --git a/third_party/llama.cpp/patches/0019-Re-prime-native-MTP-after-state-restoration.patch b/third_party/llama.cpp/patches/0019-Re-prime-native-MTP-after-state-restoration.patch index 3d858bab5c..9487b8d480 100644 --- a/third_party/llama.cpp/patches/0019-Re-prime-native-MTP-after-state-restoration.patch +++ b/third_party/llama.cpp/patches/0019-Re-prime-native-MTP-after-state-restoration.patch @@ -1,7 +1,7 @@ -From 7462ace1495a306a658712c27034b7a70f936c6d Mon Sep 17 00:00:00 2001 +From aa826fd29d52eb836f8f3868ef04c7b5fe0bd380 Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Tue, 21 Jul 2026 01:38:34 +1000 -Subject: [PATCH 19/45] Re-prime native MTP after state restoration +Subject: [PATCH 19/47] Re-prime native MTP after state restoration --- src/skippy.cpp | 31 ++++++++++++++++++++++++++++++- diff --git a/third_party/llama.cpp/patches/0020-Fix-N-gram-confidence-threshold-indexing.patch b/third_party/llama.cpp/patches/0020-Fix-N-gram-confidence-threshold-indexing.patch index 5583220831..071ed58bcb 100644 --- a/third_party/llama.cpp/patches/0020-Fix-N-gram-confidence-threshold-indexing.patch +++ b/third_party/llama.cpp/patches/0020-Fix-N-gram-confidence-threshold-indexing.patch @@ -1,7 +1,7 @@ -From 99a2f74d756dc4b93b650b92609a2ccae1719fd2 Mon Sep 17 00:00:00 2001 +From e741d4f362840cf22f354845f65f4085a7aa32c6 Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Tue, 21 Jul 2026 02:44:00 +1000 -Subject: [PATCH 20/45] Fix N-gram confidence threshold indexing +Subject: [PATCH 20/47] Fix N-gram confidence threshold indexing When the configured N-gram range starts above one, ngrams_primary is a zero-based slice of that range. Indexing the confidence tables directly by diff --git a/third_party/llama.cpp/patches/0021-ggml-add-GLM-DSA-sparse-execution-primitives.patch b/third_party/llama.cpp/patches/0021-ggml-add-GLM-DSA-sparse-execution-primitives.patch index a67b1afff4..b15dd19917 100644 --- a/third_party/llama.cpp/patches/0021-ggml-add-GLM-DSA-sparse-execution-primitives.patch +++ b/third_party/llama.cpp/patches/0021-ggml-add-GLM-DSA-sparse-execution-primitives.patch @@ -1,7 +1,7 @@ -From 8bdf03a6a77a8acb38fd4bb35349a9d05f0f0038 Mon Sep 17 00:00:00 2001 +From bf407143bac0bfbf0234a02a17ede248019f8651 Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Tue, 14 Jul 2026 09:15:57 +1000 -Subject: [PATCH 21/45] ggml: add GLM-DSA sparse execution primitives +Subject: [PATCH 21/47] ggml: add GLM-DSA sparse execution primitives --- ggml/include/ggml-rpc.h | 2 +- @@ -16,13 +16,13 @@ Subject: [PATCH 21/45] ggml: add GLM-DSA sparse execution primitives ggml/src/ggml-metal/ggml-metal-device.h | 83 +- ggml/src/ggml-metal/ggml-metal-device.m | 152 +- ggml/src/ggml-metal/ggml-metal-impl.h | 331 +- - ggml/src/ggml-metal/ggml-metal-ops.cpp | 13555 +++++++++++++++----- + ggml/src/ggml-metal/ggml-metal-ops.cpp | 13517 +++++++++++++++----- ggml/src/ggml-metal/ggml-metal-ops.h | 13 + ggml/src/ggml-metal/ggml-metal.cpp | 5 + - ggml/src/ggml-metal/ggml-metal.metal | 9996 ++++++++++++--- + ggml/src/ggml-metal/ggml-metal.metal | 9710 ++++++++++++-- ggml/src/ggml.c | 235 +- src/llama-hparams.h | 4 + - 18 files changed, 23411 insertions(+), 5276 deletions(-) + 18 files changed, 23249 insertions(+), 5114 deletions(-) diff --git a/ggml/include/ggml-rpc.h b/ggml/include/ggml-rpc.h index 16ca3394..82d27410 100644 @@ -127,10 +127,10 @@ index 35f0c44e..7ef0ec88 100644 // custom operators diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp -index 87615921..c0f90e89 100644 +index 7f4e252d..875ce72e 100644 --- a/ggml/src/ggml-backend.cpp +++ b/ggml/src/ggml-backend.cpp -@@ -982,6 +982,89 @@ static void ggml_backend_sched_print_assignments(ggml_backend_sched_t sched, str +@@ -991,6 +991,89 @@ static void ggml_backend_sched_print_assignments(ggml_backend_sched_t sched, str } } @@ -220,7 +220,7 @@ index 87615921..c0f90e89 100644 static bool ggml_backend_sched_buffer_supported(ggml_backend_sched_t sched, struct ggml_tensor * t, int backend_id) { ggml_backend_buffer_t buf = t->view_src ? t->view_src->buffer : t->buffer; ggml_backend_buffer_type_t buft = NULL; -@@ -1378,6 +1461,7 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra +@@ -1387,6 +1470,7 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra if (sched->debug) { ggml_backend_sched_print_assignments(sched, graph); } @@ -1323,7 +1323,7 @@ index 32d97cd5..083949ce 100644 idx_start, idx_end, diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cpp -index 270c1411..088089aa 100644 +index 16e98eb5..b6f3c494 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.cpp +++ b/ggml/src/ggml-metal/ggml-metal-device.cpp @@ -5,6 +5,7 @@ @@ -4942,7 +4942,7 @@ index 270c1411..088089aa 100644 return res; } -@@ -1306,6 +3247,89 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_top_k_merge(ggml +@@ -1321,6 +3262,89 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_top_k_merge(ggml return res; } @@ -5032,7 +5032,7 @@ index 270c1411..088089aa 100644 ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_pad( ggml_metal_library_t lib, const struct ggml_tensor * op, -@@ -1523,7 +3547,7 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_v +@@ -1538,7 +3562,7 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_v const ggml_tensor * op, int32_t dv, int32_t nwg) { @@ -5042,7 +5042,7 @@ index 270c1411..088089aa 100644 char base[256]; char name[256]; diff --git a/ggml/src/ggml-metal/ggml-metal-device.h b/ggml/src/ggml-metal/ggml-metal-device.h -index b36fa811..03567502 100644 +index d0956df5..61aa5d76 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.h +++ b/ggml/src/ggml-metal/ggml-metal-device.h @@ -88,7 +88,7 @@ void ggml_metal_encoder_set_threadgroup_memory_size(ggml_metal_encoder_t encoder @@ -5090,7 +5090,7 @@ index b36fa811..03567502 100644 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_solve_tri (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_ext (ggml_metal_library_t lib, const struct ggml_tensor * op, int nsg, int nxpsg, int r1ptg); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm (ggml_metal_library_t lib, const struct ggml_tensor * op); -@@ -136,11 +154,66 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv +@@ -136,12 +154,67 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_map0 (ggml_metal_library_t lib, int ne02, int ne20); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_id (ggml_metal_library_t lib, const struct ggml_tensor * op); @@ -5145,6 +5145,7 @@ index b36fa811..03567502 100644 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argmax (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argsort (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argsort_merge (ggml_metal_library_t lib, const struct ggml_tensor * op); + struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_fwht (ggml_metal_library_t lib, int n); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_top_k (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_top_k_merge (ggml_metal_library_t lib, const struct ggml_tensor * op); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_topk_moe_route (ggml_metal_library_t lib); @@ -5157,7 +5158,7 @@ index b36fa811..03567502 100644 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_bin (ggml_metal_library_t lib, const struct ggml_tensor * op, int32_t n_fuse ); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_bin_one (ggml_metal_library_t lib, enum ggml_op op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_l2_norm (ggml_metal_library_t lib, const struct ggml_tensor * op); -@@ -309,6 +382,10 @@ void ggml_metal_buffer_free (ggml_metal_buffer_t buf); +@@ -310,6 +383,10 @@ void ggml_metal_buffer_free (ggml_metal_buffer_t buf); void * ggml_metal_buffer_get_base (ggml_metal_buffer_t buf); bool ggml_metal_buffer_is_shared(ggml_metal_buffer_t buf); @@ -5371,7 +5372,7 @@ index 4edd77c6..4531d2d5 100644 + return (struct ggml_metal_buffer_id) { buf->buffers[0].metal, 0 }; +} diff --git a/ggml/src/ggml-metal/ggml-metal-impl.h b/ggml/src/ggml-metal/ggml-metal-impl.h -index 330278d0..f7112ee5 100644 +index 9f350aad..6d20ae26 100644 --- a/ggml/src/ggml-metal/ggml-metal-impl.h +++ b/ggml/src/ggml-metal/ggml-metal-impl.h @@ -23,6 +23,8 @@ @@ -5689,9 +5690,9 @@ index 330278d0..f7112ee5 100644 typedef struct { int32_t ne00; int32_t ne01; -@@ -1157,6 +1435,57 @@ typedef struct { - int32_t len; - } ggml_metal_kargs_argsort_merge; +@@ -1161,6 +1439,57 @@ typedef struct { + int32_t nrows; + } ggml_metal_kargs_fwht; +typedef struct { + int32_t n_expert; @@ -5748,7 +5749,7 @@ index 330278d0..f7112ee5 100644 int64_t ne0; float start; diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp -index c716f118..36378f8e 100644 +index 76626a45..e22add77 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -10,8 +10,10 @@ @@ -11487,20 +11488,13 @@ index c716f118..36378f8e 100644 } break; case GGML_OP_GET_ROWS: { -@@ -482,62 +5706,1421 @@ static int ggml_metal_op_encode_impl(ggml_metal_op_t ctx, int idx) { - n_fuse = ggml_metal_op_count_equal(ctx, idx); - } break; - default: -- { -- GGML_LOG_ERROR("%s: error: node %3d, op = %8s not implemented\n", __func__, idx, ggml_op_name(node->op)); -- GGML_ABORT("fatal error"); -- } -+ { -+ GGML_LOG_ERROR("%s: error: node %3d, op = %8s not implemented\n", __func__, idx, ggml_op_name(node->op)); -+ GGML_ABORT("fatal error"); -+ } -+ } -+ +@@ -488,41 +5712,989 @@ static int ggml_metal_op_encode_impl(ggml_metal_op_t ctx, int idx) { + } + } + +- if (ctx->debug_graph > 0) { +- if (n_fuse > 1) { +- GGML_LOG_DEBUG("%s: fuse %d ops\n", __func__, n_fuse); +done: + if (ctx->debug_graph > 0) { + if (n_fuse > 1) { @@ -12323,9 +12317,14 @@ index c716f118..36378f8e 100644 + ggml_metal_encoder_set_buffer (enc, bid_dst, 2); + + ggml_metal_encoder_dispatch_threadgroups(enc, net0*ne01, ne02, ne03, nth, 1, 1); -+ } -+ } -+ + } + } + +- // update the mem ranges in the encoding context +- for (int i = 0; i < n_fuse; ++i) { +- if (!ggml_metal_op_concurrency_add(ctx, ctx->node(idx + i))) { +- ggml_metal_op_concurrency_reset(ctx); +- } + return 1; +} + @@ -12334,8 +12333,9 @@ index c716f118..36378f8e 100644 + const char * value = getenv("LLAMA_GLM_DSA_EXPERIMENTAL_PACKED_GATHER_ROWS_PER_TG"); + if (value == nullptr || value[0] == '\0') { + return props_dev->device_id == GGML_METAL_DEVICE_M3_ULTRA ? 16 : 1; -+ } -+ + } + +- return n_fuse; + switch (atoi(value)) { + case 2: + case 4: @@ -12347,15 +12347,22 @@ index c716f118..36378f8e 100644 + default: + return 1; + } -+} -+ + } + +-int ggml_metal_op_encode(ggml_metal_op_t ctx, int idx) { +- if (ctx->use_capture) { +- ggml_metal_encoder_debug_group_push(ctx->enc, ggml_op_desc(ctx->node(idx))); +static int32_t ggml_metal_glm_dsa_packed_gather_threads_per_row( + const ggml_metal_device_props * props_dev) { + const char * value = getenv("LLAMA_GLM_DSA_EXPERIMENTAL_PACKED_GATHER_THREADS_PER_ROW"); + if (value == nullptr || value[0] == '\0') { + return props_dev->device_id == GGML_METAL_DEVICE_M3_ULTRA ? 32 : 64; -+ } -+ + } + +- int res = ggml_metal_op_encode_impl(ctx, idx); +- if (idx + res > ctx->n_nodes()) { +- GGML_ABORT("fusion error: nodes spanning multiple encoders have been fused. this indicates a bug in the fusion logic %s", +- "https://github.com/ggml-org/llama.cpp/pull/14849"); + switch (atoi(value)) { + case 16: + case 32: @@ -12363,9 +12370,11 @@ index c716f118..36378f8e 100644 + return atoi(value); + default: + return 64; -+ } + } +} -+ + +- if (ctx->use_capture) { +- ggml_metal_encoder_debug_group_pop(ctx->enc); +int ggml_metal_op_get_rows(ggml_metal_op_t ctx, int idx) { + ggml_tensor * op = ctx->node(idx); + @@ -12385,8 +12394,9 @@ index c716f118..36378f8e 100644 + ggml_metal_tensor_name(op)); + } + return 1; -+ } -+ + } + +- return res; + ggml_metal_library_t lib = ctx->lib; + ggml_metal_encoder_t enc = ctx->enc; + @@ -12481,23 +12491,22 @@ index c716f118..36378f8e 100644 + ggml_metal_encoder_dispatch_threadgroups(enc, grid_x, grid_y, grid_z, nth, rows_per_tg, 1); + + return 1; -+} -+ + } + +-int ggml_metal_op_concat(ggml_metal_op_t ctx, int idx) { +int ggml_metal_op_set_rows(ggml_metal_op_t ctx, int idx) { -+ ggml_tensor * op = ctx->node(idx); -+ -+ ggml_metal_library_t lib = ctx->lib; -+ ggml_metal_encoder_t enc = ctx->enc; -+ -+ GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); -+ GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); -+ GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne); -+ GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb); -+ GGML_TENSOR_LOCALS( int32_t, ne, op, ne); -+ GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); -+ + ggml_tensor * op = ctx->node(idx); + + ggml_metal_library_t lib = ctx->lib; +@@ -535,63 +6707,98 @@ int ggml_metal_op_concat(ggml_metal_op_t ctx, int idx) { + GGML_TENSOR_LOCALS( int32_t, ne, op, ne); + GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); + +- const int32_t dim = ((const int32_t *) op->op_params)[0]; + auto pipeline = ggml_metal_library_get_pipeline_set_rows(lib, op->src[1]->type, op->type); -+ + +- ggml_metal_kargs_concat args = { +- /*.ne00 =*/ ne00, + const int32_t nk0 = ne0/ggml_blck_size(op->type); + + int nth = 32; // SIMD width @@ -12520,34 +12529,59 @@ index c716f118..36378f8e 100644 + + ggml_metal_kargs_set_rows args = { + /*.nk0 =*/ nk0, -+ /*.ne01 =*/ ne01, -+ /*.nb01 =*/ nb01, -+ /*.nb02 =*/ nb02, -+ /*.nb03 =*/ nb03, -+ /*.ne11 =*/ ne11, -+ /*.ne12 =*/ ne12, -+ /*.nb10 =*/ nb10, -+ /*.nb11 =*/ nb11, -+ /*.nb12 =*/ nb12, -+ /*.nb1 =*/ nb1, -+ /*.nb2 =*/ nb2, -+ /*.nb3 =*/ nb3, -+ }; -+ -+ ggml_metal_encoder_set_pipeline(enc, pipeline); -+ ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); -+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); -+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2); -+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3); -+ + /*.ne01 =*/ ne01, +- /*.ne02 =*/ ne02, +- /*.ne03 =*/ ne03, +- /*.nb00 =*/ nb00, + /*.nb01 =*/ nb01, + /*.nb02 =*/ nb02, + /*.nb03 =*/ nb03, +- /*.ne10 =*/ ne10, + /*.ne11 =*/ ne11, + /*.ne12 =*/ ne12, +- /*.ne13 =*/ ne13, + /*.nb10 =*/ nb10, + /*.nb11 =*/ nb11, + /*.nb12 =*/ nb12, +- /*.nb13 =*/ nb13, +- /*.ne0 =*/ ne0, +- /*.ne1 =*/ ne1, +- /*.ne2 =*/ ne2, +- /*.ne3 =*/ ne3, +- /*.nb0 =*/ nb0, + /*.nb1 =*/ nb1, + /*.nb2 =*/ nb2, + /*.nb3 =*/ nb3, +- /*.dim =*/ dim, + }; + +- auto pipeline = ggml_metal_library_get_pipeline_concat(lib, op->type); +- + ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3); + +- int nth = std::min(256, ne0); + ggml_metal_encoder_dispatch_threadgroups(enc, (ne01 + nrptg - 1)/nrptg, ne02, ne03, nth, nrptg, 1); -+ + +- // when rows are small, we can batch them together in a single threadgroup +- int nrptg = 1; +- if (nth < 256) { +- nrptg = std::min((256 + nth - 1) / nth, ne1); +- if (nrptg * nth > 256) { +- nrptg = 256 / nth; +- } +- } + return 1; +} -+ + +- const int nw0 = (ne1 + nrptg - 1) / nrptg; +int ggml_metal_op_diag(ggml_metal_op_t ctx, int idx) { + ggml_tensor * op = ctx->node(idx); -+ + +- ggml_metal_encoder_dispatch_threadgroups(enc, nw0, ne2, ne3, nth, nrptg, 1); + ggml_metal_library_t lib = ctx->lib; + ggml_metal_encoder_t enc = ctx->enc; + @@ -12583,28 +12617,47 @@ index c716f118..36378f8e 100644 + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 2); + + ggml_metal_encoder_dispatch_threadgroups(enc, ne1, ne2, ne3, 32, 1, 1); -+ -+ return 1; -+} -+ + + return 1; + } + +-int ggml_metal_op_repeat(ggml_metal_op_t ctx, int idx) { +int ggml_metal_op_soft_max(ggml_metal_op_t ctx, int idx) { -+ ggml_tensor * op = ctx->node(idx); -+ -+ ggml_metal_library_t lib = ctx->lib; -+ ggml_metal_encoder_t enc = ctx->enc; -+ -+ GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); -+ GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); + ggml_tensor * op = ctx->node(idx); + + ggml_metal_library_t lib = ctx->lib; +@@ -599,43 +6806,89 @@ int ggml_metal_op_repeat(ggml_metal_op_t ctx, int idx) { + + GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); + GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); + GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne); + GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb); + GGML_TENSOR_LOCALS( int32_t, ne2, op->src[2], ne); + GGML_TENSOR_LOCALS(uint64_t, nb2, op->src[2], nb); -+ GGML_TENSOR_LOCALS( int32_t, ne, op, ne); -+ GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); -+ + GGML_TENSOR_LOCALS( int32_t, ne, op, ne); + GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); + +- auto pipeline = ggml_metal_library_get_pipeline_repeat(lib, op->type); + float scale; + float max_bias; -+ + +- ggml_metal_kargs_repeat args = { +- /*.ne00 =*/ ne00, +- /*.ne01 =*/ ne01, +- /*.ne02 =*/ ne02, +- /*.ne03 =*/ ne03, +- /*.nb00 =*/ nb00, +- /*.nb01 =*/ nb01, +- /*.nb02 =*/ nb02, +- /*.nb03 =*/ nb03, +- /*.ne0 =*/ ne0, +- /*.ne1 =*/ ne1, +- /*.ne2 =*/ ne2, +- /*.ne3 =*/ ne3, +- /*.nb0 =*/ nb0, +- /*.nb1 =*/ nb1, +- /*.nb2 =*/ nb2, +- /*.nb3 =*/ nb3, + memcpy(&scale, ((const int32_t *) op->op_params) + 0, sizeof(scale)); + memcpy(&max_bias, ((const int32_t *) op->op_params) + 1, sizeof(max_bias)); + @@ -12637,8 +12690,8 @@ index c716f118..36378f8e 100644 + /*.m0 =*/ m0, + /*.m1 =*/ m1, + /*.n_head_log2 =*/ n_head_log2, -+ }; -+ + }; + + auto pipeline = ggml_metal_library_get_pipeline_soft_max(lib, op); + + int nth = 32; // SIMD width @@ -12655,73 +12708,145 @@ index c716f118..36378f8e 100644 + + const size_t smem = pipeline.smem; + -+ ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_pipeline(enc, pipeline); +- ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); +- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); +- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2); + ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[0]), 1); + if (op->src[1]) { + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[1]), 2); + } else { + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[0]), 2); - } ++ } + if (op->src[2]) { + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[2]), 3); + } else { + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[0]), 3); + } + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 4); -+ + +- const int nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline), ne0); + ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); -+ + +- ggml_metal_encoder_dispatch_threadgroups(enc, ne1, ne2, ne3, nth, 1, 1); + ggml_metal_encoder_dispatch_threadgroups(enc, ne01, ne02, ne03, nth, 1, 1); -+ -+ return 1; -+} -+ + + return 1; + } + +-int ggml_metal_op_acc(ggml_metal_op_t ctx, int idx) { +int ggml_metal_op_ssm_conv(ggml_metal_op_t ctx, int idx) { -+ ggml_tensor * op = ctx->node(idx); -+ -+ ggml_metal_library_t lib = ctx->lib; -+ ggml_metal_encoder_t enc = ctx->enc; -+ -+ GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); -+ GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); -+ GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne); -+ GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb); -+ GGML_TENSOR_LOCALS( int32_t, ne, op, ne); -+ GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); + ggml_tensor * op = ctx->node(idx); -- if (ctx->debug_graph > 0) { -- if (n_fuse > 1) { -- GGML_LOG_DEBUG("%s: fuse %d ops\n", __func__, n_fuse); -- } -- } -+ ggml_metal_kargs_ssm_conv args = { -+ /*.ne00 =*/ ne00, -+ /*.ne01 =*/ ne01, -+ /*.ne02 =*/ ne02, -+ /*.nb00 =*/ nb00, -+ /*.nb01 =*/ nb01, -+ /*.nb02 =*/ nb02, -+ /*.ne10 =*/ ne10, -+ /*.ne11 =*/ ne11, -+ /*.nb10 =*/ nb10, -+ /*.nb11 =*/ nb11, -+ /*.ne0 =*/ ne0, -+ /*.ne1 =*/ ne1, -+ /*.ne2 =*/ ne2, -+ /*.nb0 =*/ nb0, + ggml_metal_library_t lib = ctx->lib; +@@ -648,111 +6901,67 @@ int ggml_metal_op_acc(ggml_metal_op_t ctx, int idx) { + GGML_TENSOR_LOCALS( int32_t, ne, op, ne); + GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); + +- GGML_ASSERT(op->src[0]->type == GGML_TYPE_F32); +- GGML_ASSERT(op->src[1]->type == GGML_TYPE_F32); +- GGML_ASSERT(op->type == GGML_TYPE_F32); +- +- GGML_ASSERT(ggml_is_contiguous_rows(op->src[0])); +- GGML_ASSERT(ggml_is_contiguous_rows(op->src[1])); +- +- const size_t pnb1 = ((const int32_t *) op->op_params)[0]; +- const size_t pnb2 = ((const int32_t *) op->op_params)[1]; +- const size_t pnb3 = ((const int32_t *) op->op_params)[2]; +- const size_t offs = ((const int32_t *) op->op_params)[3]; +- +- const bool inplace = (bool) ((const int32_t *) op->op_params)[4]; +- +- if (!inplace) { +- // run a separate kernel to cpy src->dst +- // not sure how to avoid this +- // TODO: make a simpler cpy_bytes kernel +- +- //const id pipeline = ctx->pipelines[GGML_METAL_PIPELINE_TYPE_CPY_F32_F32].obj; +- auto pipeline = ggml_metal_library_get_pipeline_cpy(lib, op->src[0]->type, op->type); +- +- ggml_metal_kargs_cpy args = { +- /*.nk0 =*/ ne00, +- /*.ne00 =*/ ne00, +- /*.ne01 =*/ ne01, +- /*.ne02 =*/ ne02, +- /*.ne03 =*/ ne03, +- /*.nb00 =*/ nb00, +- /*.nb01 =*/ nb01, +- /*.nb02 =*/ nb02, +- /*.nb03 =*/ nb03, +- /*.ne0 =*/ ne0, +- /*.ne1 =*/ ne1, +- /*.ne2 =*/ ne2, +- /*.ne3 =*/ ne3, +- /*.nb0 =*/ nb0, +- /*.nb1 =*/ nb1, +- /*.nb2 =*/ nb2, +- /*.nb3 =*/ nb3, +- }; +- +- ggml_metal_encoder_set_pipeline(enc, pipeline); +- ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); +- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); +- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2); +- +- const int nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline), ne00); +- +- ggml_metal_encoder_dispatch_threadgroups(enc, ne01, ne02, ne03, nth, 1, 1); +- +- ggml_metal_op_concurrency_reset(ctx); +- } +- +- ggml_metal_kargs_bin args = { +- /*.ne00 =*/ ne10, +- /*.ne01 =*/ ne11, +- /*.ne02 =*/ ne12, +- /*.ne03 =*/ ne13, ++ ggml_metal_kargs_ssm_conv args = { ++ /*.ne00 =*/ ne00, ++ /*.ne01 =*/ ne01, ++ /*.ne02 =*/ ne02, + /*.nb00 =*/ nb00, +- /*.nb01 =*/ pnb1, +- /*.nb02 =*/ pnb2, +- /*.nb03 =*/ pnb3, ++ /*.nb01 =*/ nb01, ++ /*.nb02 =*/ nb02, + /*.ne10 =*/ ne10, + /*.ne11 =*/ ne11, +- /*.ne12 =*/ ne12, +- /*.ne13 =*/ ne13, + /*.nb10 =*/ nb10, + /*.nb11 =*/ nb11, +- /*.nb12 =*/ nb12, +- /*.nb13 =*/ nb13, +- /*.ne0 =*/ ne10, +- /*.ne1 =*/ ne11, +- /*.ne2 =*/ ne12, +- /*.ne3 =*/ ne13, ++ /*.ne0 =*/ ne0, ++ /*.ne1 =*/ ne1, ++ /*.ne2 =*/ ne2, + /*.nb0 =*/ nb0, +- /*.nb1 =*/ pnb1, +- /*.nb2 =*/ pnb2, +- /*.nb3 =*/ pnb3, +- /*.offs =*/ offs, +- /*.o1 =*/ { 0 }, + /*.nb1 =*/ nb1, + /*.nb2 =*/ nb2, -+ }; + }; -- // update the mem ranges in the encoding context -- for (int i = 0; i < n_fuse; ++i) { -- if (!ggml_metal_op_concurrency_add(ctx, ctx->node(idx + i))) { -- ggml_metal_op_concurrency_reset(ctx); -- } +- auto pipeline = ggml_metal_library_get_pipeline_bin_one(lib, GGML_OP_ADD); + // Use batched kernel for prefill (ne1 > 1) to reduce threadgroup dispatch overhead + const bool use_batched = (ne1 > 1); -+ + +- ggml_metal_encoder_set_pipeline(enc, pipeline); +- ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); +- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); +- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2); +- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3); + if (use_batched) { + // Determine the smallest power of 2 that's >= ne1, but <= 256 + int BATCH_SIZE; @@ -12732,22 +12857,28 @@ index c716f118..36378f8e 100644 + else if (ne1 > 8 ) BATCH_SIZE = 16; + else if (ne1 > 4 ) BATCH_SIZE = 8; + else BATCH_SIZE = 2; -+ + +- const int nth_max = MIN(256, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)); + auto pipeline = ggml_metal_library_get_pipeline_ssm_conv_batched(lib, op, BATCH_SIZE); -+ + +- int nth = 1; + ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[0]), 1); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[1]), 2); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 3); -+ + +- while (2*nth < args.ne0 && nth < nth_max) { +- nth *= 2; +- } + // Dispatch: ne01 rows, ceil(ne1/BATCH_SIZE) token batches, ne02 sequences + // Each threadgroup has BATCH_SIZE threads, each handling one token + const int n_token_batches = (ne1 + BATCH_SIZE - 1) / BATCH_SIZE; + ggml_metal_encoder_dispatch_threadgroups(enc, ne01, n_token_batches, ne02, BATCH_SIZE, 1, 1); + } else { + auto pipeline = ggml_metal_library_get_pipeline_ssm_conv(lib, op); -+ + +- ggml_metal_encoder_dispatch_threadgroups(enc, ne11, ne12, ne13, nth, 1, 1); + ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[0]), 1); @@ -12755,31 +12886,20 @@ index c716f118..36378f8e 100644 + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 3); + + ggml_metal_encoder_dispatch_threadgroups(enc, ne01, ne1, ne02, 1, 1, 1); - } ++ } -- return n_fuse; -+ return 1; + return 1; } --int ggml_metal_op_encode(ggml_metal_op_t ctx, int idx) { -- if (ctx->use_capture) { -- ggml_metal_encoder_debug_group_push(ctx->enc, ggml_op_desc(ctx->node(idx))); -- } +-int ggml_metal_op_unary(ggml_metal_op_t ctx, int idx) { +int ggml_metal_op_ssm_scan(ggml_metal_op_t ctx, int idx) { -+ ggml_tensor * op = ctx->node(idx); + ggml_tensor * op = ctx->node(idx); -- int res = ggml_metal_op_encode_impl(ctx, idx); -- if (idx + res > ctx->n_nodes()) { -- GGML_ABORT("fusion error: nodes spanning multiple encoders have been fused. this indicates a bug in the fusion logic %s", -- "https://github.com/ggml-org/llama.cpp/pull/14849"); -- } -+ ggml_metal_library_t lib = ctx->lib; -+ ggml_metal_encoder_t enc = ctx->enc; + ggml_metal_library_t lib = ctx->lib; +@@ -760,92 +6969,97 @@ int ggml_metal_op_unary(ggml_metal_op_t ctx, int idx) { -- if (ctx->use_capture) { -- ggml_metal_encoder_debug_group_pop(ctx->enc); -+ GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); -+ GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); + GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); + GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); + GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne); + GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb); + GGML_TENSOR_LOCALS( int32_t, ne2, op->src[2], ne); @@ -12792,26 +12912,77 @@ index c716f118..36378f8e 100644 + GGML_TENSOR_LOCALS(uint64_t, nb5, op->src[5], nb); + GGML_TENSOR_LOCALS( int32_t, ne6, op->src[6], ne); + GGML_TENSOR_LOCALS(uint64_t, nb6, op->src[6], nb); -+ GGML_TENSOR_LOCALS( int32_t, ne, op, ne); -+ GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); -+ + GGML_TENSOR_LOCALS( int32_t, ne, op, ne); + GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); + +- GGML_ASSERT(ggml_is_contiguous_rows(op->src[0])); +- +- ggml_metal_buffer_id bid_src0 = ggml_metal_get_buffer_id(op->src[0]); +- ggml_metal_buffer_id bid_dst = ggml_metal_get_buffer_id(op); +- +- ggml_metal_kargs_unary args = { +- /*.ne00 =*/ ne00, +- /*.ne01 =*/ ne01, +- /*.ne02 =*/ ne02, +- /*.ne03 =*/ ne03, +- /*.nb00 =*/ nb00, +- /*.nb01 =*/ nb01, +- /*.nb02 =*/ nb02, +- /*.nb03 =*/ nb03, +- /*.ne0 =*/ ne0, +- /*.ne1 =*/ ne1, +- /*.ne2 =*/ ne2, +- /*.ne3 =*/ ne3, +- /*.nb0 =*/ nb0, +- /*.nb1 =*/ nb1, +- /*.nb2 =*/ nb2, +- /*.nb3 =*/ nb3, +- /*.slope =*/ 0.0, +- /*.scale =*/ 0.0, +- /*.bias =*/ 0.0, +- /*.val =*/ 0.0, +- /*.min =*/ 0.0, +- /*.max =*/ 0.0, +- }; +- +- if (op->op == GGML_OP_LEAKY_RELU) { +- args.slope = ggml_get_op_params_f32(op, 0); +- } +- +- if (op->op == GGML_OP_SCALE) { +- args.scale = ggml_get_op_params_f32(op, 0); +- args.bias = ggml_get_op_params_f32(op, 1); +- } + const ggml_tensor * src3 = op->src[3]; + const ggml_tensor * src4 = op->src[4]; + const ggml_tensor * src5 = op->src[5]; + const ggml_tensor * src6 = op->src[6]; -+ + +- if (op->op == GGML_OP_FILL) { +- args.val = ggml_get_op_params_f32(op, 0); +- } + GGML_ASSERT(src3); + GGML_ASSERT(src4); + GGML_ASSERT(src5); + GGML_ASSERT(src6); -+ + +- if (op->op == GGML_OP_CLAMP) { +- args.min = ggml_get_op_params_f32(op, 0); +- args.max = ggml_get_op_params_f32(op, 1); +- } + const int64_t d_state = ne00; + const int64_t d_inner = ne01; + const int64_t n_head = ne02; + const int64_t n_group = ne41; + const int64_t n_seq_tokens = ne12; + const int64_t n_seqs = ne13; -+ + +- if (op->op == GGML_OP_UNARY && ggml_get_unary_op(op) == GGML_UNARY_OP_XIELU) { +- args.slope = ggml_get_op_params_f32(op, 1); // alpha_n +- args.scale = ggml_get_op_params_f32(op, 2); // alpha_p +- args.bias = ggml_get_op_params_f32(op, 3); // beta +- args.val = ggml_get_op_params_f32(op, 4); // eps +- } + ggml_metal_kargs_ssm_scan args = { + /*.d_state =*/ d_state, + /*.d_inner =*/ d_inner, @@ -12845,15 +13016,25 @@ index c716f118..36378f8e 100644 + /*.nb53 =*/ nb53, + /*.nb0 =*/ nb0, + }; -+ + +- auto pipeline = ggml_metal_library_get_pipeline_unary(lib, op); + auto pipeline = ggml_metal_library_get_pipeline_ssm_scan(lib, op); -+ + +- if (pipeline.c4) { +- args.ne00 = ne00/4; +- args.ne0 = ne0/4; +- } + GGML_ASSERT(d_state <= ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)); + + const size_t smem = pipeline.smem; -+ -+ ggml_metal_encoder_set_pipeline(enc, pipeline); -+ ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + + ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); +- ggml_metal_encoder_set_buffer (enc, bid_src0, 1); +- ggml_metal_encoder_set_buffer (enc, bid_dst, 2); +- +- if (pipeline.cnt) { +- const int n = pipeline.c4 ? ggml_nelements(op)/4 : ggml_nelements(op); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[2]), 3); @@ -12862,35 +13043,78 @@ index c716f118..36378f8e 100644 + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[5]), 6); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[6]), 7); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 8); -+ + +- ggml_metal_encoder_dispatch_threadgroups(enc, n, 1, 1, 1, 1, 1); +- } else { +- const int nth_max = MIN(256, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)); +- const int nth = MIN(args.ne00, nth_max); +- const int nk0 = (args.ne00 + nth - 1)/nth; + ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); -+ + +- ggml_metal_encoder_dispatch_threadgroups(enc, nk0*ne01, ne02, ne03, nth, 1, 1); +- } + ggml_metal_encoder_dispatch_threadgroups(enc, d_inner, n_head, n_seqs, d_state, 1, 1); -+ -+ return 1; -+} -+ + + return 1; + } + +-int ggml_metal_op_glu(ggml_metal_op_t ctx, int idx) { +int ggml_metal_op_rwkv(ggml_metal_op_t ctx, int idx) { -+ ggml_tensor * op = ctx->node(idx); -+ -+ ggml_metal_library_t lib = ctx->lib; -+ ggml_metal_encoder_t enc = ctx->enc; -+ -+ GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); -+ GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); -+ GGML_TENSOR_LOCALS( int32_t, ne, op, ne); -+ GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); -+ + ggml_tensor * op = ctx->node(idx); + + ggml_metal_library_t lib = ctx->lib; +@@ -853,94 +7067,115 @@ int ggml_metal_op_glu(ggml_metal_op_t ctx, int idx) { + + GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); + GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); +- GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne); +- GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb); + GGML_TENSOR_LOCALS( int32_t, ne, op, ne); + GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); + +- if (op->src[1]) { +- GGML_ASSERT(ggml_are_same_shape(op->src[0], op->src[1])); +- } +- +- auto pipeline = ggml_metal_library_get_pipeline_glu(lib, op); +- +- const int32_t swp = ggml_get_op_params_i32(op, 1); +- const float alpha = ggml_get_op_params_f32(op, 2); +- const float limit = ggml_get_op_params_f32(op, 3); +- +- const int32_t i00 = swp ? ne0 : 0; +- const int32_t i10 = swp ? 0 : ne0; +- +- ggml_metal_kargs_glu args = { +- /*.ne00 =*/ ne00, +- /*.nb01 =*/ nb01, +- /*.ne10 =*/ op->src[1] ? ne10 : ne00, +- /*.nb11 =*/ op->src[1] ? nb11 : nb01, +- /*.ne0 =*/ ne0, +- /*.nb1 =*/ nb1, +- /*.i00 =*/ op->src[1] ? 0 : i00, +- /*.i10 =*/ op->src[1] ? 0 : i10, +- /*.alpha=*/ alpha, +- /*.limit=*/ limit +- }; + const int64_t B = op->op == GGML_OP_RWKV_WKV6 ? op->src[5]->ne[1] : op->src[6]->ne[1]; + const int64_t T = op->src[0]->ne[2]; + const int64_t C = op->ne[0]; + const int64_t H = op->src[0]->ne[1]; -+ + +- const int64_t nrows = ggml_nrows(op->src[0]); + auto pipeline = ggml_metal_library_get_pipeline_rwkv(lib, op); -+ + +- const int32_t nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline), ne00/2); + int ida = 0; -+ -+ ggml_metal_encoder_set_pipeline(enc, pipeline); + + ggml_metal_encoder_set_pipeline(enc, pipeline); +- ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); +- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); +- if (op->src[1]) { +- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2); +- } else { +- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 2); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), ida++); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), ida++); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[2]), ida++); @@ -12900,49 +13124,72 @@ index c716f118..36378f8e 100644 + if (op->op == GGML_OP_RWKV_WKV7) { + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[6]), ida++); } +- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), ida++); + ggml_metal_encoder_set_bytes (enc, (void *) &B, sizeof(B), ida++); + ggml_metal_encoder_set_bytes (enc, (void *) &T, sizeof(T), ida++); + ggml_metal_encoder_set_bytes (enc, (void *) &C, sizeof(C), ida++); + ggml_metal_encoder_set_bytes (enc, (void *) &H, sizeof(H), ida++); -- return res; +- ggml_metal_encoder_dispatch_threadgroups(enc, nrows, 1, 1, nth, 1, 1); + ggml_metal_encoder_dispatch_threadgroups(enc, B * H, 1, 1, C/H, 1, 1); -+ -+ return 1; + + return 1; } --int ggml_metal_op_concat(ggml_metal_op_t ctx, int idx) { +-int ggml_metal_op_sum(ggml_metal_op_t ctx, int idx) { +- ggml_tensor * op = ctx->node(idx); +int ggml_metal_op_gated_delta_net(ggml_metal_op_t ctx, int idx) { - ggml_tensor * op = ctx->node(idx); ++ ggml_tensor * op = ctx->node(idx); ggml_metal_library_t lib = ctx->lib; ggml_metal_encoder_t enc = ctx->enc; -+ - GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); - GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); - GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne); - GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb); +- const uint64_t n = (uint64_t) ggml_nelements(op->src[0]); +- +- ggml_metal_kargs_sum args = { +- /*.np =*/ n, +- }; +- +- auto pipeline = ggml_metal_library_get_pipeline_sum(lib, op); + +- int nth = 32; // SIMD width ++ GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); ++ GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); ++ GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne); ++ GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb); + GGML_TENSOR_LOCALS( int32_t, ne2, op->src[2], ne); + GGML_TENSOR_LOCALS(uint64_t, nb2, op->src[2], nb); - GGML_TENSOR_LOCALS( int32_t, ne, op, ne); - GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); ++ GGML_TENSOR_LOCALS( int32_t, ne, op, ne); ++ GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); -- const int32_t dim = ((const int32_t *) op->op_params)[0]; +- while (nth < (int) n && nth < ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)) { +- nth *= 2; +- } + auto pipeline = ggml_metal_library_get_pipeline_gated_delta_net(lib, op); -- ggml_metal_kargs_concat args = { +- nth = std::min(nth, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)); +- nth = std::min(nth, (int) n); + int ida = 0; -+ + +- const int nsg = (nth + 31) / 32; + ggml_metal_kargs_gated_delta_net args = { - /*.ne00 =*/ ne00, - /*.ne01 =*/ ne01, - /*.ne02 =*/ ne02, -@@ -554,6 +7137,17 @@ int ggml_metal_op_concat(ggml_metal_op_t ctx, int idx) { - /*.nb11 =*/ nb11, - /*.nb12 =*/ nb12, - /*.nb13 =*/ nb13, ++ /*.ne00 =*/ ne00, ++ /*.ne01 =*/ ne01, ++ /*.ne02 =*/ ne02, ++ /*.ne03 =*/ ne03, ++ /*.nb00 =*/ nb00, ++ /*.nb01 =*/ nb01, ++ /*.nb02 =*/ nb02, ++ /*.nb03 =*/ nb03, ++ /*.ne10 =*/ ne10, ++ /*.ne11 =*/ ne11, ++ /*.ne12 =*/ ne12, ++ /*.ne13 =*/ ne13, ++ /*.nb10 =*/ nb10, ++ /*.nb11 =*/ nb11, ++ /*.nb12 =*/ nb12, ++ /*.nb13 =*/ nb13, + /*.ne20 =*/ ne20, + /*.ne21 =*/ ne21, + /*.ne22 =*/ ne22, @@ -12954,40 +13201,20 @@ index c716f118..36378f8e 100644 + /*.ns02 =*/ (int32_t) (nb02/sizeof(float)), + /*.ns12 =*/ (int32_t) (nb12/sizeof(float)), + /*.ns22 =*/ (int32_t) (nb22/sizeof(float)), - /*.ne0 =*/ ne0, - /*.ne1 =*/ ne1, - /*.ne2 =*/ ne2, -@@ -561,37 +7155,27 @@ int ggml_metal_op_concat(ggml_metal_op_t ctx, int idx) { - /*.nb0 =*/ nb0, - /*.nb1 =*/ nb1, - /*.nb2 =*/ nb2, -- /*.nb3 =*/ nb3, -- /*.dim =*/ dim, -- }; -- -- auto pipeline = ggml_metal_library_get_pipeline_concat(lib, op->type); -- -- ggml_metal_encoder_set_pipeline(enc, pipeline); -- ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); -- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); -- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2); -- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3); -- -- int nth = std::min(256, ne0); -- -- // when rows are small, we can batch them together in a single threadgroup -- int nrptg = 1; -- if (nth < 256) { -- nrptg = std::min((256 + nth - 1) / nth, ne1); -- if (nrptg * nth > 256) { -- nrptg = 256 / nth; -- } -- } ++ /*.ne0 =*/ ne0, ++ /*.ne1 =*/ ne1, ++ /*.ne2 =*/ ne2, ++ /*.ne3 =*/ ne3, ++ /*.nb0 =*/ nb0, ++ /*.nb1 =*/ nb1, ++ /*.nb2 =*/ nb2, + /*.nb3 =*/ nb3, + }; -- const int nw0 = (ne1 + nrptg - 1) / nrptg; -+ ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_pipeline(enc, pipeline); +- ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); +- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); +- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), ida++); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), ida++); // q + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), ida++); // k @@ -12997,20 +13224,21 @@ index c716f118..36378f8e 100644 + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[5]), ida++); // state + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), ida++); // dst -- ggml_metal_encoder_dispatch_threadgroups(enc, nw0, ne2, ne3, nth, nrptg, 1); +- ggml_metal_encoder_set_threadgroup_memory_size(enc, nsg * sizeof(float), 0); + const int nsg = pipeline.nsg; -+ + +- ggml_metal_encoder_dispatch_threadgroups(enc, 1, 1, 1, nth, 1, 1); + ggml_metal_encoder_dispatch_threadgroups(enc, op->src[2]->ne[0]/nsg, op->src[2]->ne[1], op->src[2]->ne[3], 32, nsg, 1); return 1; } --int ggml_metal_op_repeat(ggml_metal_op_t ctx, int idx) { +-int ggml_metal_op_sum_rows(ggml_metal_op_t ctx, int idx) { +int ggml_metal_op_solve_tri(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); ggml_metal_library_t lib = ctx->lib; -@@ -599,12 +7183,12 @@ int ggml_metal_op_repeat(ggml_metal_op_t ctx, int idx) { +@@ -948,15 +7183,12 @@ int ggml_metal_op_sum_rows(ggml_metal_op_t ctx, int idx) { GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); @@ -13019,14 +13247,17 @@ index c716f118..36378f8e 100644 GGML_TENSOR_LOCALS( int32_t, ne, op, ne); GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); -- auto pipeline = ggml_metal_library_get_pipeline_repeat(lib, op->type); +- GGML_ASSERT(ggml_is_contiguous_rows(op->src[0])); - -- ggml_metal_kargs_repeat args = { +- ggml_metal_buffer_id bid_src0 = ggml_metal_get_buffer_id(op->src[0]); +- ggml_metal_buffer_id bid_dst = ggml_metal_get_buffer_id(op); +- +- ggml_metal_kargs_sum_rows args = { + ggml_metal_kargs_solve_tri args = { /*.ne00 =*/ ne00, /*.ne01 =*/ ne01, /*.ne02 =*/ ne02, -@@ -613,6 +7197,14 @@ int ggml_metal_op_repeat(ggml_metal_op_t ctx, int idx) { +@@ -965,6 +7197,14 @@ int ggml_metal_op_sum_rows(ggml_metal_op_t ctx, int idx) { /*.nb01 =*/ nb01, /*.nb02 =*/ nb02, /*.nb03 =*/ nb03, @@ -13041,23 +13272,41 @@ index c716f118..36378f8e 100644 /*.ne0 =*/ ne0, /*.ne1 =*/ ne1, /*.ne2 =*/ ne2, -@@ -623,19 +7215,24 @@ int ggml_metal_op_repeat(ggml_metal_op_t ctx, int idx) { +@@ -975,78 +7215,57 @@ int ggml_metal_op_sum_rows(ggml_metal_op_t ctx, int idx) { /*.nb3 =*/ nb3, }; +- auto pipeline = ggml_metal_library_get_pipeline_sum_rows(lib, op); +- +- if (pipeline.c4) { +- args.ne00 = ne00/4; +- args.ne0 = ne0/4; +- } +- +- int nth = 32; // SIMD width +- +- while (nth < args.ne00 && nth < ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)) { +- nth *= 2; +- } +- +- nth = std::min(nth, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)); +- nth = std::min(nth, (int) args.ne00); +- +- const size_t smem = pipeline.smem; + auto pipeline = ggml_metal_library_get_pipeline_solve_tri(lib, op); -+ + ggml_metal_encoder_set_pipeline(enc, pipeline); ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); - ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); -- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2); +- ggml_metal_encoder_set_buffer (enc, bid_src0, 1); +- ggml_metal_encoder_set_buffer (enc, bid_dst, 2); ++ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3); -- const int nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline), ne0); +- ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); + const int nsg = pipeline.nsg; -- ggml_metal_encoder_dispatch_threadgroups(enc, ne1, ne2, ne3, nth, 1, 1); +- ggml_metal_encoder_dispatch_threadgroups(enc, ne01, ne02, ne03, nth, 1, 1); + ggml_metal_encoder_set_threadgroup_memory_size(enc, pipeline.smem, 0); + + ggml_metal_encoder_dispatch_threadgroups(enc, (ne10 + nsg - 1)/nsg, ne02, ne03, 32, nsg, 1); @@ -13065,93 +13314,201 @@ index c716f118..36378f8e 100644 return 1; } --int ggml_metal_op_acc(ggml_metal_op_t ctx, int idx) { +-int ggml_metal_op_cumsum(ggml_metal_op_t ctx, int idx) { +int ggml_metal_op_set(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); ggml_metal_library_t lib = ctx->lib; -@@ -648,12 +7245,9 @@ int ggml_metal_op_acc(ggml_metal_op_t ctx, int idx) { + ggml_metal_encoder_t enc = ctx->enc; + +- GGML_ASSERT(ggml_is_contiguous_rows(op->src[0])); +- + GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); + GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); ++ GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne); ++ GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb); GGML_TENSOR_LOCALS( int32_t, ne, op, ne); GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); -- GGML_ASSERT(op->src[0]->type == GGML_TYPE_F32); -- GGML_ASSERT(op->src[1]->type == GGML_TYPE_F32); -- GGML_ASSERT(op->type == GGML_TYPE_F32); +- auto pipeline_blk = ggml_metal_library_get_pipeline_cumsum_blk(lib, op); - -- GGML_ASSERT(ggml_is_contiguous_rows(op->src[0])); -- GGML_ASSERT(ggml_is_contiguous_rows(op->src[1])); -+ ggml_metal_buffer_id bid_src0 = ggml_metal_get_buffer_id(op->src[0]); +- int nth = 1; +- while (nth < ne00 && 2*nth <= ggml_metal_pipeline_max_theads_per_threadgroup(pipeline_blk)) { +- nth *= 2; +- } +- +- GGML_ASSERT(ne00 <= nth*nth); +- +- const int64_t net0 = (ne00 + nth - 1) / nth; +- const int64_t net1 = ne01; +- const int64_t net2 = ne02; +- const int64_t net3 = ne03; +- +- const uint64_t nbt0 = sizeof(float); +- const uint64_t nbt1 = net0*nbt0; +- const uint64_t nbt2 = net1*nbt1; +- const uint64_t nbt3 = net2*nbt2; +- +- const size_t smem = GGML_PAD(32*sizeof(float), 16); +- + ggml_metal_buffer_id bid_src0 = ggml_metal_get_buffer_id(op->src[0]); + ggml_metal_buffer_id bid_src1 = ggml_metal_get_buffer_id(op->src[1]); -+ ggml_metal_buffer_id bid_dst = ggml_metal_get_buffer_id(op); + ggml_metal_buffer_id bid_dst = ggml_metal_get_buffer_id(op); - const size_t pnb1 = ((const int32_t *) op->op_params)[0]; - const size_t pnb2 = ((const int32_t *) op->op_params)[1]; -@@ -692,8 +7286,8 @@ int ggml_metal_op_acc(ggml_metal_op_t ctx, int idx) { +- ggml_metal_buffer_id bid_tmp = bid_dst; +- bid_tmp.offs += ggml_nbytes(op); ++ const size_t pnb1 = ((const int32_t *) op->op_params)[0]; ++ const size_t pnb2 = ((const int32_t *) op->op_params)[1]; ++ const size_t pnb3 = ((const int32_t *) op->op_params)[2]; ++ const size_t offs = ((const int32_t *) op->op_params)[3]; ++ ++ const bool inplace = (bool) ((const int32_t *) op->op_params)[4]; ++ ++ if (!inplace) { ++ // run a separate kernel to cpy src->dst ++ // not sure how to avoid this ++ // TODO: make a simpler cpy_bytes kernel - ggml_metal_encoder_set_pipeline(enc, pipeline); - ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); -- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); -- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2); -+ ggml_metal_encoder_set_buffer (enc, bid_src0, 1); +- { +- ggml_metal_kargs_cumsum_blk args = { ++ //const id pipeline = ctx->pipelines[GGML_METAL_PIPELINE_TYPE_CPY_F32_F32].obj; ++ auto pipeline = ggml_metal_library_get_pipeline_cpy(lib, op->src[0]->type, op->type); ++ ++ ggml_metal_kargs_cpy args = { ++ /*.nk0 =*/ ne00, + /*.ne00 =*/ ne00, + /*.ne01 =*/ ne01, + /*.ne02 =*/ ne02, +@@ -1055,145 +7274,93 @@ int ggml_metal_op_cumsum(ggml_metal_op_t ctx, int idx) { + /*.nb01 =*/ nb01, + /*.nb02 =*/ nb02, + /*.nb03 =*/ nb03, +- /*.net0 =*/ net0, +- /*.net1 =*/ net1, +- /*.net2 =*/ net2, +- /*.net3 =*/ net3, +- /*.nbt0 =*/ nbt0, +- /*.nbt1 =*/ nbt1, +- /*.nbt2 =*/ nbt2, +- /*.nbt3 =*/ nbt3, +- /*.outb =*/ ne00 > nth, ++ /*.ne0 =*/ ne0, ++ /*.ne1 =*/ ne1, ++ /*.ne2 =*/ ne2, ++ /*.ne3 =*/ ne3, ++ /*.nb0 =*/ nb0, ++ /*.nb1 =*/ nb1, ++ /*.nb2 =*/ nb2, ++ /*.nb3 =*/ nb3, + }; + +- ggml_metal_encoder_set_pipeline(enc, pipeline_blk); ++ ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer (enc, bid_src0, 1); +- ggml_metal_encoder_set_buffer (enc, bid_tmp, 2); +- ggml_metal_encoder_set_buffer (enc, bid_dst, 3); + ggml_metal_encoder_set_buffer (enc, bid_dst, 2); - const int nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline), ne00); +- ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); ++ const int nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline), ne00); + +- ggml_metal_encoder_dispatch_threadgroups(enc, net0*ne01, ne02, ne03, nth, 1, 1); +- } ++ ggml_metal_encoder_dispatch_threadgroups(enc, ne01, ne02, ne03, nth, 1, 1); -@@ -702,245 +7296,71 @@ int ggml_metal_op_acc(ggml_metal_op_t ctx, int idx) { +- if (ne00 > nth) { ggml_metal_op_concurrency_reset(ctx); - } ++ } -- ggml_metal_kargs_bin args = { -- /*.ne00 =*/ ne10, -- /*.ne01 =*/ ne11, -- /*.ne02 =*/ ne12, -- /*.ne03 =*/ ne13, -- /*.nb00 =*/ nb00, -- /*.nb01 =*/ pnb1, -- /*.nb02 =*/ pnb2, -- /*.nb03 =*/ pnb3, -- /*.ne10 =*/ ne10, -- /*.ne11 =*/ ne11, -- /*.ne12 =*/ ne12, -- /*.ne13 =*/ ne13, -- /*.nb10 =*/ nb10, -- /*.nb11 =*/ nb11, -- /*.nb12 =*/ nb12, -- /*.nb13 =*/ nb13, -- /*.ne0 =*/ ne10, -- /*.ne1 =*/ ne11, -- /*.ne2 =*/ ne12, -- /*.ne3 =*/ ne13, -- /*.nb0 =*/ nb0, -- /*.nb1 =*/ pnb1, -- /*.nb2 =*/ pnb2, -- /*.nb3 =*/ pnb3, -- /*.offs =*/ offs, -- /*.o1 =*/ { 0 }, -- }; -- -- auto pipeline = ggml_metal_library_get_pipeline_bin_one(lib, GGML_OP_ADD); -- -- ggml_metal_encoder_set_pipeline(enc, pipeline); -- ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); -- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); -- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2); -- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3); -- -- const int nth_max = MIN(256, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)); -- -- int nth = 1; -- -- while (2*nth < args.ne0 && nth < nth_max) { -- nth *= 2; -- } +- { +- ggml_metal_kargs_cumsum_blk args = { +- /*.ne00 =*/ net0, +- /*.ne01 =*/ net1, +- /*.ne02 =*/ net2, +- /*.ne03 =*/ net3, +- /*.nb00 =*/ nbt0, +- /*.nb01 =*/ nbt1, +- /*.nb02 =*/ nbt2, +- /*.nb03 =*/ nbt3, +- /*.net0 =*/ net0, +- /*.net1 =*/ net1, +- /*.net2 =*/ net2, +- /*.net3 =*/ net3, +- /*.nbt0 =*/ nbt0, +- /*.nbt1 =*/ nbt1, +- /*.nbt2 =*/ nbt2, +- /*.nbt3 =*/ nbt3, +- /*.outb =*/ false, +- }; - -- ggml_metal_encoder_dispatch_threadgroups(enc, ne11, ne12, ne13, nth, 1, 1); +- ggml_metal_encoder_set_pipeline(enc, pipeline_blk); +- ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); +- ggml_metal_encoder_set_buffer (enc, bid_tmp, 1); +- ggml_metal_encoder_set_buffer (enc, bid_tmp, 2); +- ggml_metal_encoder_set_buffer (enc, bid_tmp, 3); - +- ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); ++ auto pipeline = ggml_metal_library_get_pipeline_cpy(lib, op->src[1]->type, op->type); + +- ggml_metal_encoder_dispatch_threadgroups(enc, net1, net2, net3, nth, 1, 1); +- } ++ GGML_ASSERT(ne10 % ggml_blck_size(op->src[1]->type) == 0); + +- ggml_metal_op_concurrency_reset(ctx); ++ int64_t nk0 = ne10; ++ if (ggml_is_quantized(op->src[1]->type)) { ++ nk0 = ne10/16; ++ } else if (ggml_is_quantized(op->type)) { ++ nk0 = ne10/ggml_blck_size(op->type); ++ } + +- { +- auto pipeline_add = ggml_metal_library_get_pipeline_cumsum_add(lib, op); ++ int nth = std::min(nk0*ne11, 256); + +- ggml_metal_kargs_cumsum_add args = { +- /*.ne00 =*/ ne00, +- /*.ne01 =*/ ne01, +- /*.ne02 =*/ ne02, +- /*.ne03 =*/ ne03, +- /*.nb00 =*/ nb00, +- /*.nb01 =*/ nb01, +- /*.nb02 =*/ nb02, +- /*.nb03 =*/ nb03, +- /*.net0 =*/ net0, +- /*.net1 =*/ net1, +- /*.net2 =*/ net2, +- /*.net3 =*/ net3, +- /*.nbt0 =*/ nbt0, +- /*.nbt1 =*/ nbt1, +- /*.nbt2 =*/ nbt2, +- /*.nbt3 =*/ nbt3, +- }; ++ // when rows are small, we can batch them together in a single threadgroup ++ int nrptg = 1; + +- ggml_metal_encoder_set_pipeline(enc, pipeline_add); +- ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); +- ggml_metal_encoder_set_buffer (enc, bid_tmp, 1); +- ggml_metal_encoder_set_buffer (enc, bid_dst, 2); ++ // TODO: relax this constraint in the future ++ if (ggml_blck_size(op->src[1]->type) == 1 && ggml_blck_size(op->type) == 1) { ++ if (nth > nk0) { ++ nrptg = (nth + nk0 - 1)/nk0; ++ nth = nk0; + +- ggml_metal_encoder_dispatch_threadgroups(enc, net0*ne01, ne02, ne03, nth, 1, 1); ++ if (nrptg*nth > 256) { ++ nrptg--; ++ } + } + } + - return 1; -} - --int ggml_metal_op_unary(ggml_metal_op_t ctx, int idx) { +-int ggml_metal_op_get_rows(ggml_metal_op_t ctx, int idx) { - ggml_tensor * op = ctx->node(idx); - - ggml_metal_library_t lib = ctx->lib; @@ -13159,180 +13516,27 @@ index c716f118..36378f8e 100644 - - GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); - GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); +- GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne); +- GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb); - GGML_TENSOR_LOCALS( int32_t, ne, op, ne); - GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); - -- GGML_ASSERT(ggml_is_contiguous_rows(op->src[0])); -- -- ggml_metal_buffer_id bid_src0 = ggml_metal_get_buffer_id(op->src[0]); -- ggml_metal_buffer_id bid_dst = ggml_metal_get_buffer_id(op); -- -- ggml_metal_kargs_unary args = { +- auto pipeline = ggml_metal_library_get_pipeline_get_rows(lib, op->src[0]->type); ++ nth = std::min(nth, nk0); + +- ggml_metal_kargs_get_rows args = { +- /*.ne00t =*/ ggml_is_quantized(op->src[0]->type) ? ne00/16 : ne00, - /*.ne00 =*/ ne00, -- /*.ne01 =*/ ne01, -- /*.ne02 =*/ ne02, -- /*.ne03 =*/ ne03, -- /*.nb00 =*/ nb00, - /*.nb01 =*/ nb01, - /*.nb02 =*/ nb02, - /*.nb03 =*/ nb03, -- /*.ne0 =*/ ne0, -- /*.ne1 =*/ ne1, -- /*.ne2 =*/ ne2, -- /*.ne3 =*/ ne3, -- /*.nb0 =*/ nb0, +- /*.ne10 =*/ ne10, +- /*.nb10 =*/ nb10, +- /*.nb11 =*/ nb11, +- /*.nb12 =*/ nb12, - /*.nb1 =*/ nb1, - /*.nb2 =*/ nb2, - /*.nb3 =*/ nb3, -- /*.slope =*/ 0.0, -- /*.scale =*/ 0.0, -- /*.bias =*/ 0.0, -- /*.val =*/ 0.0, -- /*.min =*/ 0.0, -- /*.max =*/ 0.0, -- }; -- -- if (op->op == GGML_OP_LEAKY_RELU) { -- args.slope = ggml_get_op_params_f32(op, 0); -- } -- -- if (op->op == GGML_OP_SCALE) { -- args.scale = ggml_get_op_params_f32(op, 0); -- args.bias = ggml_get_op_params_f32(op, 1); -- } -- -- if (op->op == GGML_OP_FILL) { -- args.val = ggml_get_op_params_f32(op, 0); -- } -- -- if (op->op == GGML_OP_CLAMP) { -- args.min = ggml_get_op_params_f32(op, 0); -- args.max = ggml_get_op_params_f32(op, 1); -- } -- -- if (op->op == GGML_OP_UNARY && ggml_get_unary_op(op) == GGML_UNARY_OP_XIELU) { -- args.slope = ggml_get_op_params_f32(op, 1); // alpha_n -- args.scale = ggml_get_op_params_f32(op, 2); // alpha_p -- args.bias = ggml_get_op_params_f32(op, 3); // beta -- args.val = ggml_get_op_params_f32(op, 4); // eps -- } -- -- auto pipeline = ggml_metal_library_get_pipeline_unary(lib, op); -- -- if (pipeline.c4) { -- args.ne00 = ne00/4; -- args.ne0 = ne0/4; -- } -- -- ggml_metal_encoder_set_pipeline(enc, pipeline); -- ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); -- ggml_metal_encoder_set_buffer (enc, bid_src0, 1); -- ggml_metal_encoder_set_buffer (enc, bid_dst, 2); -- -- if (pipeline.cnt) { -- const int n = pipeline.c4 ? ggml_nelements(op)/4 : ggml_nelements(op); -- -- ggml_metal_encoder_dispatch_threadgroups(enc, n, 1, 1, 1, 1, 1); -- } else { -- const int nth_max = MIN(256, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)); -- const int nth = MIN(args.ne00, nth_max); -- const int nk0 = (args.ne00 + nth - 1)/nth; -- -- ggml_metal_encoder_dispatch_threadgroups(enc, nk0*ne01, ne02, ne03, nth, 1, 1); -- } -- -- return 1; --} -- --int ggml_metal_op_glu(ggml_metal_op_t ctx, int idx) { -- ggml_tensor * op = ctx->node(idx); -- -- ggml_metal_library_t lib = ctx->lib; -- ggml_metal_encoder_t enc = ctx->enc; -- -- GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); -- GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); -- GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne); -- GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb); -- GGML_TENSOR_LOCALS( int32_t, ne, op, ne); -- GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); -- -- if (op->src[1]) { -- GGML_ASSERT(ggml_are_same_shape(op->src[0], op->src[1])); -- } -- -- auto pipeline = ggml_metal_library_get_pipeline_glu(lib, op); -- -- const int32_t swp = ggml_get_op_params_i32(op, 1); -- const float alpha = ggml_get_op_params_f32(op, 2); -- const float limit = ggml_get_op_params_f32(op, 3); -- -- const int32_t i00 = swp ? ne0 : 0; -- const int32_t i10 = swp ? 0 : ne0; -- -- ggml_metal_kargs_glu args = { -- /*.ne00 =*/ ne00, -- /*.nb01 =*/ nb01, -- /*.ne10 =*/ op->src[1] ? ne10 : ne00, -- /*.nb11 =*/ op->src[1] ? nb11 : nb01, -- /*.ne0 =*/ ne0, -- /*.nb1 =*/ nb1, -- /*.i00 =*/ op->src[1] ? 0 : i00, -- /*.i10 =*/ op->src[1] ? 0 : i10, -- /*.alpha=*/ alpha, -- /*.limit=*/ limit -- }; -- -- const int64_t nrows = ggml_nrows(op->src[0]); -+ auto pipeline = ggml_metal_library_get_pipeline_cpy(lib, op->src[1]->type, op->type); - -- const int32_t nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline), ne00/2); -+ GGML_ASSERT(ne10 % ggml_blck_size(op->src[1]->type) == 0); - -- ggml_metal_encoder_set_pipeline(enc, pipeline); -- ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); -- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); -- if (op->src[1]) { -- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2); -- } else { -- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 2); -+ int64_t nk0 = ne10; -+ if (ggml_is_quantized(op->src[1]->type)) { -+ nk0 = ne10/16; -+ } else if (ggml_is_quantized(op->type)) { -+ nk0 = ne10/ggml_blck_size(op->type); - } -- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3); - -- ggml_metal_encoder_dispatch_threadgroups(enc, nrows, 1, 1, nth, 1, 1); -+ int nth = std::min(nk0*ne11, 256); - -- return 1; --} -+ // when rows are small, we can batch them together in a single threadgroup -+ int nrptg = 1; - --int ggml_metal_op_sum(ggml_metal_op_t ctx, int idx) { -- ggml_tensor * op = ctx->node(idx); -+ // TODO: relax this constraint in the future -+ if (ggml_blck_size(op->src[1]->type) == 1 && ggml_blck_size(op->type) == 1) { -+ if (nth > nk0) { -+ nrptg = (nth + nk0 - 1)/nk0; -+ nth = nk0; - -- ggml_metal_library_t lib = ctx->lib; -- ggml_metal_encoder_t enc = ctx->enc; -+ if (nrptg*nth > 256) { -+ nrptg--; -+ } -+ } -+ } - -- const uint64_t n = (uint64_t) ggml_nelements(op->src[0]); -+ nth = std::min(nth, nk0); - -- ggml_metal_kargs_sum args = { -- /*.np =*/ n, + ggml_metal_kargs_cpy args = { + /*.nk0 =*/ nk0, + /*.ne00 =*/ ne10, @@ -13353,65 +13557,67 @@ index c716f118..36378f8e 100644 + /*.nb3 =*/ pnb3, }; -- auto pipeline = ggml_metal_library_get_pipeline_sum(lib, op); -- -- int nth = 32; // SIMD width -- -- while (nth < (int) n && nth < ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)) { -- nth *= 2; -- } -- -- nth = std::min(nth, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)); -- nth = std::min(nth, (int) n); +- const int nth = std::min(args.ne00t, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)); + const int nw0 = nrptg == 1 ? (nk0 + nth - 1)/nth : 1; -- const int nsg = (nth + 31) / 32; +- const int nw0 = (args.ne00t + nth - 1)/nth; + bid_dst.offs += offs; ggml_metal_encoder_set_pipeline(enc, pipeline); ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); - ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); -- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2); -- -- ggml_metal_encoder_set_threadgroup_memory_size(enc, nsg * sizeof(float), 0); +- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2); +- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3); + ggml_metal_encoder_set_buffer (enc, bid_src1, 1); + ggml_metal_encoder_set_buffer (enc, bid_dst, 2); -- ggml_metal_encoder_dispatch_threadgroups(enc, 1, 1, 1, nth, 1, 1); +- ggml_metal_encoder_dispatch_threadgroups(enc, nw0*ne10, ne11, ne12, nth, 1, 1); + ggml_metal_encoder_dispatch_threadgroups(enc, nw0*(ne11 + nrptg - 1)/nrptg, ne12, ne13, nth, nrptg, 1); return 1; } --int ggml_metal_op_sum_rows(ggml_metal_op_t ctx, int idx) { +-int ggml_metal_op_set_rows(ggml_metal_op_t ctx, int idx) { +int ggml_metal_op_cpy(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); ggml_metal_library_t lib = ctx->lib; -@@ -951,12 +7371,38 @@ int ggml_metal_op_sum_rows(ggml_metal_op_t ctx, int idx) { +@@ -1201,103 +7368,160 @@ int ggml_metal_op_set_rows(ggml_metal_op_t ctx, int idx) { + + GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); + GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); +- GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne); +- GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb); GGML_TENSOR_LOCALS( int32_t, ne, op, ne); GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); -- GGML_ASSERT(ggml_is_contiguous_rows(op->src[0])); +- auto pipeline = ggml_metal_library_get_pipeline_set_rows(lib, op); +- +- const int32_t nk0 = ne0/ggml_blck_size(op->type); + auto pipeline = ggml_metal_library_get_pipeline_cpy(lib, op->src[0]->type, op->type); -- ggml_metal_buffer_id bid_src0 = ggml_metal_get_buffer_id(op->src[0]); -- ggml_metal_buffer_id bid_dst = ggml_metal_get_buffer_id(op); +- int nth = 32; // SIMD width + GGML_ASSERT(ne00 % ggml_blck_size(op->src[0]->type) == 0); -- ggml_metal_kargs_sum_rows args = { +- while (nth < nk0 && nth < ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)) { +- nth *= 2; + int64_t nk0 = ne00; + if (ggml_is_quantized(op->src[0]->type)) { + nk0 = ne00/16; + } else if (ggml_is_quantized(op->type)) { + nk0 = ne00/ggml_blck_size(op->type); -+ } -+ + } + + int nth = std::min(nk0*ne01, 256); + + // when rows are small, we can batch them together in a single threadgroup -+ int nrptg = 1; -+ + int nrptg = 1; +- if (nth > nk0) { +- nrptg = (nth + nk0 - 1)/nk0; +- nth = nk0; + +- if (nrptg*nth > ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)) { +- nrptg--; + // TODO: relax this constraint in the future + if (ggml_blck_size(op->src[0]->type) == 1 && ggml_blck_size(op->type) == 1) { + if (nth > nk0) { @@ -13421,193 +13627,73 @@ index c716f118..36378f8e 100644 + if (nrptg*nth > 256) { + nrptg--; + } -+ } -+ } -+ + } + } + +- nth = std::min(nth, nk0); + nth = std::min(nth, nk0); -+ + +- ggml_metal_kargs_set_rows args = { + ggml_metal_kargs_cpy args = { -+ /*.nk0 =*/ nk0, - /*.ne00 =*/ ne00, + /*.nk0 =*/ nk0, ++ /*.ne00 =*/ ne00, /*.ne01 =*/ ne01, - /*.ne02 =*/ ne02, -@@ -975,180 +7421,67 @@ int ggml_metal_op_sum_rows(ggml_metal_op_t ctx, int idx) { ++ /*.ne02 =*/ ne02, ++ /*.ne03 =*/ ne03, ++ /*.nb00 =*/ nb00, + /*.nb01 =*/ nb01, + /*.nb02 =*/ nb02, + /*.nb03 =*/ nb03, +- /*.ne11 =*/ ne11, +- /*.ne12 =*/ ne12, +- /*.nb10 =*/ nb10, +- /*.nb11 =*/ nb11, +- /*.nb12 =*/ nb12, ++ /*.ne0 =*/ ne0, ++ /*.ne1 =*/ ne1, ++ /*.ne2 =*/ ne2, ++ /*.ne3 =*/ ne3, ++ /*.nb0 =*/ nb0, + /*.nb1 =*/ nb1, + /*.nb2 =*/ nb2, /*.nb3 =*/ nb3, }; -- auto pipeline = ggml_metal_library_get_pipeline_sum_rows(lib, op); -- -- if (pipeline.c4) { -- args.ne00 = ne00/4; -- args.ne0 = ne0/4; -- } -- -- int nth = 32; // SIMD width -- -- while (nth < args.ne00 && nth < ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)) { -- nth *= 2; -- } -- -- nth = std::min(nth, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)); -- nth = std::min(nth, (int) args.ne00); -- -- const size_t smem = pipeline.smem; + const int nw0 = nrptg == 1 ? (nk0 + nth - 1)/nth : 1; - - ggml_metal_encoder_set_pipeline(enc, pipeline); - ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); -- ggml_metal_encoder_set_buffer (enc, bid_src0, 1); -- ggml_metal_encoder_set_buffer (enc, bid_dst, 2); -- -- ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); ++ ++ ggml_metal_encoder_set_pipeline(enc, pipeline); ++ ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2); - -- ggml_metal_encoder_dispatch_threadgroups(enc, ne01, ne02, ne03, nth, 1, 1); ++ + ggml_metal_encoder_dispatch_threadgroups(enc, nw0*(ne01 + nrptg - 1)/nrptg, ne02, ne03, nth, nrptg, 1); - - return 1; - } - --int ggml_metal_op_cumsum(ggml_metal_op_t ctx, int idx) { ++ ++ return 1; ++} ++ +int ggml_metal_op_pool_1d(ggml_metal_op_t ctx, int idx) { - ggml_tensor * op = ctx->node(idx); - - ggml_metal_library_t lib = ctx->lib; - ggml_metal_encoder_t enc = ctx->enc; - -- GGML_ASSERT(ggml_is_contiguous_rows(op->src[0])); -- - GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); - GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); - GGML_TENSOR_LOCALS( int32_t, ne, op, ne); - GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); - -- auto pipeline_blk = ggml_metal_library_get_pipeline_cumsum_blk(lib, op); -- -- int nth = 1; -- while (nth < ne00 && 2*nth <= ggml_metal_pipeline_max_theads_per_threadgroup(pipeline_blk)) { -- nth *= 2; -- } -- -- GGML_ASSERT(ne00 <= nth*nth); -- -- const int64_t net0 = (ne00 + nth - 1) / nth; -- const int64_t net1 = ne01; -- const int64_t net2 = ne02; -- const int64_t net3 = ne03; -- -- const uint64_t nbt0 = sizeof(float); -- const uint64_t nbt1 = net0*nbt0; -- const uint64_t nbt2 = net1*nbt1; -- const uint64_t nbt3 = net2*nbt2; -- -- const size_t smem = GGML_PAD(32*sizeof(float), 16); -- -- ggml_metal_buffer_id bid_src0 = ggml_metal_get_buffer_id(op->src[0]); -- ggml_metal_buffer_id bid_dst = ggml_metal_get_buffer_id(op); -- -- ggml_metal_buffer_id bid_tmp = bid_dst; -- bid_tmp.offs += ggml_nbytes(op); -- -- { -- ggml_metal_kargs_cumsum_blk args = { -- /*.ne00 =*/ ne00, -- /*.ne01 =*/ ne01, -- /*.ne02 =*/ ne02, -- /*.ne03 =*/ ne03, -- /*.nb00 =*/ nb00, -- /*.nb01 =*/ nb01, -- /*.nb02 =*/ nb02, -- /*.nb03 =*/ nb03, -- /*.net0 =*/ net0, -- /*.net1 =*/ net1, -- /*.net2 =*/ net2, -- /*.net3 =*/ net3, -- /*.nbt0 =*/ nbt0, -- /*.nbt1 =*/ nbt1, -- /*.nbt2 =*/ nbt2, -- /*.nbt3 =*/ nbt3, -- /*.outb =*/ ne00 > nth, -- }; -- -- ggml_metal_encoder_set_pipeline(enc, pipeline_blk); -- ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); -- ggml_metal_encoder_set_buffer (enc, bid_src0, 1); -- ggml_metal_encoder_set_buffer (enc, bid_tmp, 2); -- ggml_metal_encoder_set_buffer (enc, bid_dst, 3); -- -- ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); -- -- ggml_metal_encoder_dispatch_threadgroups(enc, net0*ne01, ne02, ne03, nth, 1, 1); -- } -- -- if (ne00 > nth) { -- ggml_metal_op_concurrency_reset(ctx); -- -- { -- ggml_metal_kargs_cumsum_blk args = { -- /*.ne00 =*/ net0, -- /*.ne01 =*/ net1, -- /*.ne02 =*/ net2, -- /*.ne03 =*/ net3, -- /*.nb00 =*/ nbt0, -- /*.nb01 =*/ nbt1, -- /*.nb02 =*/ nbt2, -- /*.nb03 =*/ nbt3, -- /*.net0 =*/ net0, -- /*.net1 =*/ net1, -- /*.net2 =*/ net2, -- /*.net3 =*/ net3, -- /*.nbt0 =*/ nbt0, -- /*.nbt1 =*/ nbt1, -- /*.nbt2 =*/ nbt2, -- /*.nbt3 =*/ nbt3, -- /*.outb =*/ false, -- }; -- -- ggml_metal_encoder_set_pipeline(enc, pipeline_blk); -- ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); -- ggml_metal_encoder_set_buffer (enc, bid_tmp, 1); -- ggml_metal_encoder_set_buffer (enc, bid_tmp, 2); -- ggml_metal_encoder_set_buffer (enc, bid_tmp, 3); -- -- ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); ++ ggml_tensor * op = ctx->node(idx); ++ ++ ggml_metal_library_t lib = ctx->lib; ++ ggml_metal_encoder_t enc = ctx->enc; ++ ++ GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); ++ GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); ++ GGML_TENSOR_LOCALS( int32_t, ne, op, ne); ++ GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); ++ + const int32_t * opts = op->op_params; + ggml_op_pool op_pool = (ggml_op_pool) opts[0]; - -- ggml_metal_encoder_dispatch_threadgroups(enc, net1, net2, net3, nth, 1, 1); -- } ++ + const int32_t k0 = opts[1]; + const int32_t s0 = opts[2]; + const int32_t p0 = opts[3]; - -- ggml_metal_op_concurrency_reset(ctx); ++ + const int64_t IW = op->src[0]->ne[0]; + const int64_t OW = op->ne[0]; - -- { -- auto pipeline_add = ggml_metal_library_get_pipeline_cumsum_add(lib, op); ++ + const int64_t np = ggml_nelements(op); - -- ggml_metal_kargs_cumsum_add args = { -- /*.ne00 =*/ ne00, -- /*.ne01 =*/ ne01, -- /*.ne02 =*/ ne02, -- /*.ne03 =*/ ne03, -- /*.nb00 =*/ nb00, -- /*.nb01 =*/ nb01, -- /*.nb02 =*/ nb02, -- /*.nb03 =*/ nb03, -- /*.net0 =*/ net0, -- /*.net1 =*/ net1, -- /*.net2 =*/ net2, -- /*.net3 =*/ net3, -- /*.nbt0 =*/ nbt0, -- /*.nbt1 =*/ nbt1, -- /*.nbt2 =*/ nbt2, -- /*.nbt3 =*/ nbt3, -- }; ++ + ggml_metal_kargs_pool_1d args_pool_1d = { + /* .k0 = */ k0, + /* .s0 = */ s0, @@ -13616,78 +13702,159 @@ index c716f118..36378f8e 100644 + /* .OW = */ OW, + /* .np = */ np + }; ++ ++ auto pipeline = ggml_metal_library_get_pipeline_pool_1d(lib, op, op_pool); ++ ++ const int nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline), (int) np); ++ const int ntg = (np + nth - 1) / nth; ++ + ggml_metal_encoder_set_pipeline(enc, pipeline); +- ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); ++ ggml_metal_encoder_set_bytes (enc, &args_pool_1d, sizeof(args_pool_1d), 0); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); +- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2); +- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3); ++ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2); + +- ggml_metal_encoder_dispatch_threadgroups(enc, (ne01 + nrptg - 1)/nrptg, ne02, ne03, nth, nrptg, 1); ++ ggml_metal_encoder_dispatch_threadgroups(enc, ntg, 1, 1, nth, 1, 1); + + return 1; + } + +-int ggml_metal_op_diag(ggml_metal_op_t ctx, int idx) { ++// supported FWHT sizes, must stay in sync with the ++// kernel_fwht_f32_ templates in ggml-metal.metal ++static bool ggml_metal_fwht_supported_size(int64_t n) { ++ return n == 64 || n == 128 || n == 256 || n == 512; ++} ++ ++int ggml_metal_op_fwht(ggml_metal_op_t ctx, int idx) { + ggml_tensor * op = ctx->node(idx); + + ggml_metal_library_t lib = ctx->lib; + ggml_metal_encoder_t enc = ctx->enc; + +- GGML_TENSOR_LOCALS(int32_t, ne0, op->src[0], ne); +- GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); +- GGML_TENSOR_LOCALS(int32_t, ne, op, ne); +- GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); ++ ggml_tensor * src1 = op->src[1]; + +- ggml_metal_kargs_diag args = { +- /*.ne00 =*/ne00, +- /*.ne01 =*/ne01, +- /*.ne02 =*/ne02, +- /*.ne03 =*/ne03, +- /*.nb00 =*/nb00, +- /*.nb01 =*/nb01, +- /*.nb02 =*/nb02, +- /*.nb03 =*/nb03, +- /*.ne0 =*/ne0, +- /*.ne1 =*/ne1, +- /*.ne2 =*/ne2, +- /*.ne3 =*/ne3, +- /*.nb0 =*/nb0, +- /*.nb1 =*/nb1, +- /*.nb2 =*/nb2, +- /*.nb3 =*/nb3, ++ const int64_t n = src1->ne[0]; ++ const int64_t nrows = ggml_nrows(src1); ++ ++ ggml_metal_kargs_fwht args = { ++ /*.nrows = */ (int32_t) nrows, + }; -- ggml_metal_encoder_set_pipeline(enc, pipeline_add); -- ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); -- ggml_metal_encoder_set_buffer (enc, bid_tmp, 1); -- ggml_metal_encoder_set_buffer (enc, bid_dst, 2); -+ auto pipeline = ggml_metal_library_get_pipeline_pool_1d(lib, op, op_pool); +- auto pipeline = ggml_metal_library_get_pipeline_diag(lib, op); ++ auto pipeline = ggml_metal_library_get_pipeline_fwht(lib, n); -- ggml_metal_encoder_dispatch_threadgroups(enc, net0*ne01, ne02, ne03, nth, 1, 1); -- } -- } -+ const int nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline), (int) np); -+ const int ntg = (np + nth - 1) / nth; + ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0); +- ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[0]), 1); +- ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 2); ++ ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(src1), 1); ++ ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 2); + +- ggml_metal_encoder_dispatch_threadgroups(enc, ne1, ne2, ne3, 32, 1, 1); ++ const int th_max = ggml_metal_pipeline_max_theads_per_threadgroup(pipeline); ++ const int simd_size = 32; + -+ ggml_metal_encoder_set_pipeline(enc, pipeline); -+ ggml_metal_encoder_set_bytes (enc, &args_pool_1d, sizeof(args_pool_1d), 0); -+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); -+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2); ++ int sg_per_tg = 2; ++ sg_per_tg = std::min(sg_per_tg, th_max/simd_size); ++ sg_per_tg = std::max(sg_per_tg, 1); + -+ ggml_metal_encoder_dispatch_threadgroups(enc, ntg, 1, 1, nth, 1, 1); ++ const int64_t n_tg = (nrows + sg_per_tg - 1) / sg_per_tg; ++ ggml_metal_encoder_dispatch_threadgroups(enc, n_tg, 1, 1, 32*sg_per_tg, 1, 1); return 1; } --int ggml_metal_op_get_rows(ggml_metal_op_t ctx, int idx) { -+ +-int ggml_metal_op_soft_max(ggml_metal_op_t ctx, int idx) { +int ggml_metal_op_pool_2d(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); ggml_metal_library_t lib = ctx->lib; -@@ -1156,49 +7489,153 @@ int ggml_metal_op_get_rows(ggml_metal_op_t ctx, int idx) { +@@ -1305,94 +7529,165 @@ int ggml_metal_op_soft_max(ggml_metal_op_t ctx, int idx) { GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); - GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne); - GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb); +- GGML_TENSOR_LOCALS( int32_t, ne2, op->src[2], ne); +- GGML_TENSOR_LOCALS(uint64_t, nb2, op->src[2], nb); GGML_TENSOR_LOCALS( int32_t, ne, op, ne); GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); -- auto pipeline = ggml_metal_library_get_pipeline_get_rows(lib, op->src[0]->type); +- float scale; +- float max_bias; + const int32_t * opts = op->op_params; + ggml_op_pool op_pool = (ggml_op_pool) opts[0]; -- ggml_metal_kargs_get_rows args = { -- /*.ne00t =*/ ggml_is_quantized(op->src[0]->type) ? ne00/16 : ne00, -- /*.ne00 =*/ ne00, -- /*.nb01 =*/ nb01, -- /*.nb02 =*/ nb02, -- /*.nb03 =*/ nb03, -- /*.ne10 =*/ ne10, -- /*.nb10 =*/ nb10, -- /*.nb11 =*/ nb11, -- /*.nb12 =*/ nb12, -- /*.nb1 =*/ nb1, -- /*.nb2 =*/ nb2, -- /*.nb3 =*/ nb3, +- memcpy(&scale, ((const int32_t *) op->op_params) + 0, sizeof(scale)); +- memcpy(&max_bias, ((const int32_t *) op->op_params) + 1, sizeof(max_bias)); + const int32_t k0 = opts[1]; + const int32_t k1 = opts[2]; + const int32_t s0 = opts[3]; + const int32_t s1 = opts[4]; + const int32_t p0 = opts[5]; + const int32_t p1 = opts[6]; -+ + +- const uint32_t n_head = op->src[0]->ne[2]; +- const int32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head)); + const int64_t IH = op->src[0]->ne[1]; + const int64_t IW = op->src[0]->ne[0]; -+ + +- const float m0 = powf(2.0f, -(max_bias ) / n_head_log2); +- const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); + const int64_t N = op->ne[3]; + const int64_t OC = op->ne[2]; + const int64_t OH = op->ne[1]; + const int64_t OW = op->ne[0]; -+ + +- // softmax + const int64_t np = N * OC * OH * OW; -+ + +- ggml_metal_kargs_soft_max args = { +- /*.ne00 =*/ ne00, +- /*.ne01 =*/ ne01, +- /*.ne02 =*/ ne02, +- /*.nb01 =*/ nb01, +- /*.nb02 =*/ nb02, +- /*.nb03 =*/ nb03, +- /*.ne11 =*/ ne11, +- /*.ne12 =*/ ne12, +- /*.ne13 =*/ ne13, +- /*.nb11 =*/ nb11, +- /*.nb12 =*/ nb12, +- /*.nb13 =*/ nb13, +- /*.nb1 =*/ nb1, +- /*.nb2 =*/ nb2, +- /*.nb3 =*/ nb3, +- /*.scale =*/ scale, +- /*.max_bias =*/ max_bias, +- /*.m0 =*/ m0, +- /*.m1 =*/ m1, +- /*.n_head_log2 =*/ n_head_log2, + ggml_metal_kargs_pool_2d args_pool_2d = { + /* .k0 = */ k0, + /* .k1 = */ k1, @@ -13702,34 +13869,70 @@ index c716f118..36378f8e 100644 + /* .np = */ np }; -- const int nth = std::min(args.ne00t, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)); +- auto pipeline = ggml_metal_library_get_pipeline_soft_max(lib, op); +- +- int nth = 32; // SIMD width +- +- if (ne00%4 == 0) { +- while (nth < ne00/4 && nth*ne01*ne02*ne03 < 256) { +- nth *= 2; +- } +- } else { +- while (nth < ne00 && nth*ne01*ne02*ne03 < 256) { +- nth *= 2; +- } +- } + auto pipeline = ggml_metal_library_get_pipeline_pool_2d(lib, op, op_pool); -- const int nw0 = (args.ne00t + nth - 1)/nth; +- const size_t smem = pipeline.smem; + const int nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline), (int) np); + const int ntg = (np + nth - 1) / nth; ggml_metal_encoder_set_pipeline(enc, pipeline); -- ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); +- ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0); +- ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[0]), 1); +- if (op->src[1]) { +- ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[1]), 2); +- } else { +- ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[0]), 2); +- } +- if (op->src[2]) { +- ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[2]), 3); +- } else { +- ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[0]), 3); +- } +- ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 4); +- +- ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); + ggml_metal_encoder_set_bytes (enc, &args_pool_2d, sizeof(args_pool_2d), 0); - ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); -- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2); -- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3); ++ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2); -- ggml_metal_encoder_dispatch_threadgroups(enc, nw0*ne10, ne11, ne12, nth, 1, 1); +- ggml_metal_encoder_dispatch_threadgroups(enc, ne01, ne02, ne03, nth, 1, 1); + ggml_metal_encoder_dispatch_threadgroups(enc, ntg, 1, 1, nth, 1, 1); return 1; } --int ggml_metal_op_set_rows(ggml_metal_op_t ctx, int idx) { +-int ggml_metal_op_ssm_conv(ggml_metal_op_t ctx, int idx) { +int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); ggml_metal_library_t lib = ctx->lib; ggml_metal_encoder_t enc = ctx->enc; ++ const int32_t hint = ggml_get_op_params_i32(op, 1); ++ ++ if (hint == GGML_HINT_SRC0_IS_HADAMARD) { ++ if (op->src[1]->type == GGML_TYPE_F32 && ++ op->type == GGML_TYPE_F32 && ++ ggml_is_contiguous(op->src[1]) && ++ ggml_is_contiguous(op) && ++ ggml_are_same_shape(op->src[1], op) && ++ ggml_metal_fwht_supported_size(op->src[1]->ne[0])) { ++ return ggml_metal_op_fwht(ctx, idx); ++ } ++ } + const char * block_ceiling = getenv("GGML_METAL_EXPERIMENTAL_GLM_DECODE_BLOCK_BYTE_CEILING"); + if (block_ceiling != nullptr && atoi(block_ceiling) != 0 && + strcmp(ggml_metal_tensor_name(op->src[0]), "glm_decode_block_byte_ceiling_weights") == 0) { @@ -13822,32 +14025,40 @@ index c716f118..36378f8e 100644 GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne); -@@ -1206,475 +7643,518 @@ int ggml_metal_op_set_rows(ggml_metal_op_t ctx, int idx) { +@@ -1400,211 +7695,260 @@ int ggml_metal_op_ssm_conv(ggml_metal_op_t ctx, int idx) { GGML_TENSOR_LOCALS( int32_t, ne, op, ne); GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); -- auto pipeline = ggml_metal_library_get_pipeline_set_rows(lib, op); +- ggml_metal_kargs_ssm_conv args = { +- /*.ne00 =*/ ne00, +- /*.ne01 =*/ ne01, +- /*.ne02 =*/ ne02, +- /*.nb00 =*/ nb00, +- /*.nb01 =*/ nb01, +- /*.nb02 =*/ nb02, +- /*.ne10 =*/ ne10, +- /*.ne11 =*/ ne11, +- /*.nb10 =*/ nb10, +- /*.nb11 =*/ nb11, +- /*.ne0 =*/ ne0, +- /*.ne1 =*/ ne1, +- /*.ne2 =*/ ne2, +- /*.nb0 =*/ nb0, +- /*.nb1 =*/ nb1, +- /*.nb2 =*/ nb2, +- }; + GGML_ASSERT(ne00 == ne10); - -- const int32_t nk0 = ne0/ggml_blck_size(op->type); ++ + GGML_ASSERT(ne12 % ne02 == 0); + GGML_ASSERT(ne13 % ne03 == 0); - -- int nth = 32; // SIMD width ++ + const int16_t r2 = ne12/ne02; + const int16_t r3 = ne13/ne03; - -- while (nth < nk0 && nth < ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)) { -- nth *= 2; -- } ++ + // find the break-even point where the matrix-matrix kernel becomes more efficient compared + // to the matrix-vector kernel + const int ne11_mm_min = 8; - -- int nrptg = 1; -- if (nth > nk0) { -- nrptg = (nth + nk0 - 1)/nk0; -- nth = nk0; ++ + // first try to use small-batch mat-mv kernels + // these should be efficient for BS [2, ~8] + if (op->src[1]->type == GGML_TYPE_F32 && (ne00%128 == 0) && @@ -13889,8 +14100,8 @@ index c716f118..36378f8e 100644 + // + const int nsg = 2; // num simdgroups per threadgroup -- if (nrptg*nth > ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)) { -- nrptg--; +- // Use batched kernel for prefill (ne1 > 1) to reduce threadgroup dispatch overhead +- const bool use_batched = (ne1 > 1); + // num threads along row per simdgroup + int16_t nxpsg = 0; + if (ne00 % 256 == 0 && ne11 < 3) { @@ -13899,46 +14110,23 @@ index c716f118..36378f8e 100644 + nxpsg = 8; + } else { + nxpsg = 4; - } -- } -- -- nth = std::min(nth, nk0); -- -- ggml_metal_kargs_set_rows args = { -- /*.nk0 =*/ nk0, -- /*.ne01 =*/ ne01, -- /*.nb01 =*/ nb01, -- /*.nb02 =*/ nb02, -- /*.nb03 =*/ nb03, -- /*.ne11 =*/ ne11, -- /*.ne12 =*/ ne12, -- /*.nb10 =*/ nb10, -- /*.nb11 =*/ nb11, -- /*.nb12 =*/ nb12, -- /*.nb1 =*/ nb1, -- /*.nb2 =*/ nb2, -- /*.nb3 =*/ nb3, -- }; -- -- ggml_metal_encoder_set_pipeline(enc, pipeline); -- ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); -- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); -- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2); -- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3); -- -- ggml_metal_encoder_dispatch_threadgroups(enc, (ne01 + nrptg - 1)/nrptg, ne02, ne03, nth, nrptg, 1); ++ } -- return 1; --} -- --int ggml_metal_op_diag(ggml_metal_op_t ctx, int idx) { -- ggml_tensor * op = ctx->node(idx); +- if (use_batched) { +- // Determine the smallest power of 2 that's >= ne1, but <= 256 +- int BATCH_SIZE; +- if (ne1 > 128) BATCH_SIZE = 256; +- else if (ne1 > 64 ) BATCH_SIZE = 128; +- else if (ne1 > 32 ) BATCH_SIZE = 64; +- else if (ne1 > 16 ) BATCH_SIZE = 32; +- else if (ne1 > 8 ) BATCH_SIZE = 16; +- else if (ne1 > 4 ) BATCH_SIZE = 8; +- else BATCH_SIZE = 2; + const int16_t nypsg = 32/nxpsg; // num threads along col per simdgroup (i.e. a simdgroup processes that many src0 rows at a time) + const int16_t r0ptg = nypsg*nsg; // num src0 rows per threadgroup + int16_t r1ptg = 4; // num src1 rows per threadgroup -- ggml_metal_library_t lib = ctx->lib; -- ggml_metal_encoder_t enc = ctx->enc; +- auto pipeline = ggml_metal_library_get_pipeline_ssm_conv_batched(lib, op, BATCH_SIZE); + // note: not sure how optimal are those across all different hardware. there might be something cleverer + switch (ne11) { + case 2: @@ -13956,30 +14144,19 @@ index c716f118..36378f8e 100644 + GGML_ABORT("unsupported ne11"); + }; -- GGML_TENSOR_LOCALS(int32_t, ne0, op->src[0], ne); -- GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); -- GGML_TENSOR_LOCALS(int32_t, ne, op, ne); -- GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); +- ggml_metal_encoder_set_pipeline(enc, pipeline); +- ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0); +- ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[0]), 1); +- ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[1]), 2); +- ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 3); + auto pipeline = ggml_metal_library_get_pipeline_mul_mv_ext(lib, op, nsg, nxpsg, r1ptg); -- ggml_metal_kargs_diag args = { -- /*.ne00 =*/ne00, -- /*.ne01 =*/ne01, -- /*.ne02 =*/ne02, -- /*.ne03 =*/ne03, -- /*.nb00 =*/nb00, -- /*.nb01 =*/nb01, -- /*.nb02 =*/nb02, -- /*.nb03 =*/nb03, -- /*.ne0 =*/ne0, -- /*.ne1 =*/ne1, -- /*.ne2 =*/ne2, -- /*.ne3 =*/ne3, -- /*.nb0 =*/nb0, -- /*.nb1 =*/nb1, -- /*.nb2 =*/nb2, -- /*.nb3 =*/nb3, -- }; +- // Dispatch: ne01 rows, ceil(ne1/BATCH_SIZE) token batches, ne02 sequences +- // Each threadgroup has BATCH_SIZE threads, each handling one token +- const int n_token_batches = (ne1 + BATCH_SIZE - 1) / BATCH_SIZE; +- ggml_metal_encoder_dispatch_threadgroups(enc, ne01, n_token_batches, ne02, BATCH_SIZE, 1, 1); +- } else { +- auto pipeline = ggml_metal_library_get_pipeline_ssm_conv(lib, op); + ggml_metal_kargs_mul_mv_ext args = { + /*.ne00 =*/ ne00, + /*.ne01 =*/ ne01, @@ -14001,17 +14178,21 @@ index c716f118..36378f8e 100644 + /*.r3 =*/ r3, + }; -- auto pipeline = ggml_metal_library_get_pipeline_diag(lib, op); -+ ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_pipeline(enc, pipeline); +- ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0); +- ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[0]), 1); +- ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[1]), 2); +- ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 3); +- +- ggml_metal_encoder_dispatch_threadgroups(enc, ne01, ne1, ne02, 1, 1, 1); +- } + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3); - -- ggml_metal_encoder_set_pipeline(enc, pipeline); -- ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0); -- ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[0]), 1); -- ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 2); + +- return 1; +-} + ggml_metal_encoder_dispatch_threadgroups(enc, ((ne01 + r0ptg - 1)/r0ptg), ((ne11 + r1ptg - 1)/r1ptg), ne12*ne13, 32, nsg, 1); + } else if ( + !ggml_is_transposed(op->src[0]) && @@ -14021,7 +14202,8 @@ index c716f118..36378f8e 100644 + props_dev->has_simdgroup_mm && ne00 >= 64 && ne11 > ne11_mm_min) { + //GGML_LOG_INFO("matrix: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12); -- ggml_metal_encoder_dispatch_threadgroups(enc, ne1, ne2, ne3, 32, 1, 1); +-int ggml_metal_op_ssm_scan(ggml_metal_op_t ctx, int idx) { +- ggml_tensor * op = ctx->node(idx); + // some Metal matrix data types require aligned pointers + // ref: https://developer.apple.com/metal/Metal-Shading-Language-Specification.pdf (Table 2.5) + //switch (op->src[0]->type) { @@ -14031,12 +14213,26 @@ index c716f118..36378f8e 100644 + // default: break; + //} -- return 1; --} +- ggml_metal_library_t lib = ctx->lib; +- ggml_metal_encoder_t enc = ctx->enc; + auto pipeline = ggml_metal_library_get_pipeline_mul_mm(lib, op); --int ggml_metal_op_soft_max(ggml_metal_op_t ctx, int idx) { -- ggml_tensor * op = ctx->node(idx); +- GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); +- GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); +- GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne); +- GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb); +- GGML_TENSOR_LOCALS( int32_t, ne2, op->src[2], ne); +- GGML_TENSOR_LOCALS(uint64_t, nb2, op->src[2], nb); +- GGML_TENSOR_LOCALS( int32_t, ne3, op->src[3], ne); +- GGML_TENSOR_LOCALS(uint64_t, nb3, op->src[3], nb); +- GGML_TENSOR_LOCALS( int32_t, ne4, op->src[4], ne); +- GGML_TENSOR_LOCALS(uint64_t, nb4, op->src[4], nb); +- GGML_TENSOR_LOCALS( int32_t, ne5, op->src[5], ne); +- GGML_TENSOR_LOCALS(uint64_t, nb5, op->src[5], nb); +- GGML_TENSOR_LOCALS( int32_t, ne6, op->src[6], ne); +- GGML_TENSOR_LOCALS(uint64_t, nb6, op->src[6], nb); +- GGML_TENSOR_LOCALS( int32_t, ne, op, ne); +- GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); + ggml_metal_kargs_mul_mm args = { + /*.ne00 =*/ ne00, + /*.ne02 =*/ ne02, @@ -14054,71 +14250,90 @@ index c716f118..36378f8e 100644 + /*.r3 =*/ r3, + }; -- ggml_metal_library_t lib = ctx->lib; -- ggml_metal_encoder_t enc = ctx->enc; +- const ggml_tensor * src3 = op->src[3]; +- const ggml_tensor * src4 = op->src[4]; +- const ggml_tensor * src5 = op->src[5]; +- const ggml_tensor * src6 = op->src[6]; + ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3); -- GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); -- GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); -- GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne); -- GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb); -- GGML_TENSOR_LOCALS( int32_t, ne2, op->src[2], ne); -- GGML_TENSOR_LOCALS(uint64_t, nb2, op->src[2], nb); -- GGML_TENSOR_LOCALS( int32_t, ne, op, ne); -- GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); +- GGML_ASSERT(src3); +- GGML_ASSERT(src4); +- GGML_ASSERT(src5); +- GGML_ASSERT(src6); + const size_t smem = pipeline.smem; -- float scale; -- float max_bias; +- const int64_t d_state = ne00; +- const int64_t d_inner = ne01; +- const int64_t n_head = ne02; +- const int64_t n_group = ne41; +- const int64_t n_seq_tokens = ne12; +- const int64_t n_seqs = ne13; + ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); -- memcpy(&scale, ((const int32_t *) op->op_params) + 0, sizeof(scale)); -- memcpy(&max_bias, ((const int32_t *) op->op_params) + 1, sizeof(max_bias)); +- ggml_metal_kargs_ssm_scan args = { +- /*.d_state =*/ d_state, +- /*.d_inner =*/ d_inner, +- /*.n_head =*/ n_head, +- /*.n_group =*/ n_group, +- /*.n_seq_tokens =*/ n_seq_tokens, +- /*.n_seqs =*/ n_seqs, +- /*.s_off =*/ ggml_nelements(op->src[1]) * sizeof(float), +- /*.nb00 =*/ nb00, +- /*.nb01 =*/ nb01, +- /*.nb02 =*/ nb02, +- /*.nb03 =*/ nb03, +- /*.nb10 =*/ nb10, +- /*.nb11 =*/ nb11, +- /*.nb12 =*/ nb12, +- /*.ns12 =*/ nb12/nb10, +- /*.nb13 =*/ nb13, +- /*.nb20 =*/ nb20, +- /*.nb21 =*/ nb21, +- /*.ns21 =*/ nb21/nb20, +- /*.nb22 =*/ nb22, +- /*.ne30 =*/ ne30, +- /*.nb31 =*/ nb31, +- /*.nb41 =*/ nb41, +- /*.nb42 =*/ nb42, +- /*.ns42 =*/ nb42/nb40, +- /*.nb43 =*/ nb43, +- /*.nb51 =*/ nb51, +- /*.nb52 =*/ nb52, +- /*.ns52 =*/ nb52/nb50, +- /*.nb53 =*/ nb53, +- /*.nb0 =*/ nb0, +- }; + const int nr0 = pipeline.nr0; + const int nr1 = pipeline.nr1; + const int nsg = pipeline.nsg; -- const uint32_t n_head = op->src[0]->ne[2]; -- const int32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head)); +- auto pipeline = ggml_metal_library_get_pipeline_ssm_scan(lib, op); + ggml_metal_encoder_dispatch_threadgroups(enc, ((ne11 + nr1 - 1) / nr1), ((ne01 + nr0 - 1) / nr0), ne12 * ne13, 32, nsg, 1); + } else { + auto pipeline = ggml_metal_library_get_pipeline_mul_mv(lib, op); -- const float m0 = powf(2.0f, -(max_bias ) / n_head_log2); -- const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); +- GGML_ASSERT(d_state <= ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)); + const int nr0 = pipeline.nr0; + const int nr1 = pipeline.nr1; + const int nsg = pipeline.nsg; -- // softmax +- const size_t smem = pipeline.smem; + const size_t smem = pipeline.smem; -- ggml_metal_kargs_soft_max args = { -- /*.ne00 =*/ ne00, -- /*.ne01 =*/ ne01, -- /*.ne02 =*/ ne02, -- /*.nb01 =*/ nb01, -- /*.nb02 =*/ nb02, -- /*.nb03 =*/ nb03, -- /*.ne11 =*/ ne11, -- /*.ne12 =*/ ne12, -- /*.ne13 =*/ ne13, -- /*.nb11 =*/ nb11, -- /*.nb12 =*/ nb12, -- /*.nb13 =*/ nb13, -- /*.nb1 =*/ nb1, -- /*.nb2 =*/ nb2, -- /*.nb3 =*/ nb3, -- /*.scale =*/ scale, -- /*.max_bias =*/ max_bias, -- /*.m0 =*/ m0, -- /*.m1 =*/ m1, -- /*.n_head_log2 =*/ n_head_log2, -- }; +- ggml_metal_encoder_set_pipeline(enc, pipeline); +- ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); +- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); +- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2); +- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[2]), 3); +- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[3]), 4); +- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[4]), 5); +- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[5]), 6); +- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[6]), 7); +- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 8); + ggml_metal_kargs_mul_mv args = { + /*.ne00 =*/ ne00, + /*.ne01 =*/ ne01, @@ -14141,23 +14356,16 @@ index c716f118..36378f8e 100644 + /*.r3 =*/ r3, + }; -- auto pipeline = ggml_metal_library_get_pipeline_soft_max(lib, op); +- ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); + ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3); -- int nth = 32; // SIMD width +- ggml_metal_encoder_dispatch_threadgroups(enc, d_inner, n_head, n_seqs, d_state, 1, 1); + ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); - -- if (ne00%4 == 0) { -- while (nth < ne00/4 && nth*ne01*ne02*ne03 < 256) { -- nth *= 2; -- } -- } else { -- while (nth < ne00 && nth*ne01*ne02*ne03 < 256) { -- nth *= 2; ++ + if (op->src[0]->type == GGML_TYPE_F32 || + op->src[0]->type == GGML_TYPE_F16 || + op->src[0]->type == GGML_TYPE_BF16 || @@ -14165,126 +14373,74 @@ index c716f118..36378f8e 100644 + ggml_metal_encoder_dispatch_threadgroups(enc, ((ne01 + nr0 - 1)/(nr0)), ((ne11 + nr1 - 1)/nr1), ne12*ne13, 32, nsg, 1); + } else { + ggml_metal_encoder_dispatch_threadgroups(enc, ((ne01 + nr0*nsg - 1)/(nr0*nsg)), ((ne11 + nr1 - 1)/nr1), ne12*ne13, 32, nsg, 1); - } - } ++ } ++ } -- const size_t smem = pipeline.smem; -- -- ggml_metal_encoder_set_pipeline(enc, pipeline); -- ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0); -- ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[0]), 1); -- if (op->src[1]) { -- ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[1]), 2); -- } else { -- ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[0]), 2); -- } -- if (op->src[2]) { -- ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[2]), 3); -- } else { -- ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[0]), 3); -- } -- ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 4); -- -- ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); -- -- ggml_metal_encoder_dispatch_threadgroups(enc, ne01, ne02, ne03, nth, 1, 1); -- return 1; } --int ggml_metal_op_ssm_conv(ggml_metal_op_t ctx, int idx) { +-int ggml_metal_op_rwkv(ggml_metal_op_t ctx, int idx) { - ggml_tensor * op = ctx->node(idx); -- ++size_t ggml_metal_op_mul_mat_id_extra_tpe(const ggml_tensor * op) { ++ assert(op->op == GGML_OP_MUL_MAT_ID); + - ggml_metal_library_t lib = ctx->lib; - ggml_metal_encoder_t enc = ctx->enc; -- ++ const int64_t ne02 = op->src[0]->ne[2]; // n_expert + - GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); - GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); -- GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne); -- GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb); - GGML_TENSOR_LOCALS( int32_t, ne, op, ne); - GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); -- -- ggml_metal_kargs_ssm_conv args = { -- /*.ne00 =*/ ne00, -- /*.ne01 =*/ ne01, -- /*.ne02 =*/ ne02, -- /*.nb00 =*/ nb00, -- /*.nb01 =*/ nb01, -- /*.nb02 =*/ nb02, -- /*.ne10 =*/ ne10, -- /*.ne11 =*/ ne11, -- /*.nb10 =*/ nb10, -- /*.nb11 =*/ nb11, -- /*.ne0 =*/ ne0, -- /*.ne1 =*/ ne1, -- /*.ne2 =*/ ne2, -- /*.nb0 =*/ nb0, -- /*.nb1 =*/ nb1, -- /*.nb2 =*/ nb2, -- }; -+size_t ggml_metal_op_mul_mat_id_extra_tpe(const ggml_tensor * op) { -+ assert(op->op == GGML_OP_MUL_MAT_ID); - -- // Use batched kernel for prefill (ne1 > 1) to reduce threadgroup dispatch overhead -- const bool use_batched = (ne1 > 1); -+ const int64_t ne02 = op->src[0]->ne[2]; // n_expert - -- if (use_batched) { -- // Determine the smallest power of 2 that's >= ne1, but <= 256 -- int BATCH_SIZE; -- if (ne1 > 128) BATCH_SIZE = 256; -- else if (ne1 > 64 ) BATCH_SIZE = 128; -- else if (ne1 > 32 ) BATCH_SIZE = 64; -- else if (ne1 > 16 ) BATCH_SIZE = 32; -- else if (ne1 > 8 ) BATCH_SIZE = 16; -- else if (ne1 > 4 ) BATCH_SIZE = 8; -- else BATCH_SIZE = 2; + return ggml_type_size(GGML_TYPE_I32)*ne02; +} -- auto pipeline = ggml_metal_library_get_pipeline_ssm_conv_batched(lib, op, BATCH_SIZE); +- const int64_t B = op->op == GGML_OP_RWKV_WKV6 ? op->src[5]->ne[1] : op->src[6]->ne[1]; +- const int64_t T = op->src[0]->ne[2]; +- const int64_t C = op->ne[0]; +- const int64_t H = op->src[0]->ne[1]; +size_t ggml_metal_op_mul_mat_id_extra_ids(const ggml_tensor * op) { + assert(op->op == GGML_OP_MUL_MAT_ID); -- ggml_metal_encoder_set_pipeline(enc, pipeline); -- ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0); -- ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[0]), 1); -- ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[1]), 2); -- ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 3); +- auto pipeline = ggml_metal_library_get_pipeline_rwkv(lib, op); + const int64_t ne02 = op->src[0]->ne[2]; // n_expert + const int64_t ne21 = op->src[2]->ne[1]; // n_token -- // Dispatch: ne01 rows, ceil(ne1/BATCH_SIZE) token batches, ne02 sequences -- // Each threadgroup has BATCH_SIZE threads, each handling one token -- const int n_token_batches = (ne1 + BATCH_SIZE - 1) / BATCH_SIZE; -- ggml_metal_encoder_dispatch_threadgroups(enc, ne01, n_token_batches, ne02, BATCH_SIZE, 1, 1); -- } else { -- auto pipeline = ggml_metal_library_get_pipeline_ssm_conv(lib, op); +- int ida = 0; + return ggml_type_size(GGML_TYPE_I32)*ne02*ne21; +} -- ggml_metal_encoder_set_pipeline(enc, pipeline); -- ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0); -- ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[0]), 1); -- ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[1]), 2); -- ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 3); +- ggml_metal_encoder_set_pipeline(enc, pipeline); +- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), ida++); +- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), ida++); +- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[2]), ida++); +- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[3]), ida++); +- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[4]), ida++); +- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[5]), ida++); +- if (op->op == GGML_OP_RWKV_WKV7) { +- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[6]), ida++); +- } +- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), ida++); +- ggml_metal_encoder_set_bytes (enc, (void *) &B, sizeof(B), ida++); +- ggml_metal_encoder_set_bytes (enc, (void *) &T, sizeof(T), ida++); +- ggml_metal_encoder_set_bytes (enc, (void *) &C, sizeof(C), ida++); +- ggml_metal_encoder_set_bytes (enc, (void *) &H, sizeof(H), ida++); +size_t ggml_metal_op_mul_mat_id_extra_src1_scratch(const ggml_tensor * op) { + assert(op->op == GGML_OP_MUL_MAT_ID); -- ggml_metal_encoder_dispatch_threadgroups(enc, ne01, ne1, ne02, 1, 1, 1); +- ggml_metal_encoder_dispatch_threadgroups(enc, B * H, 1, 1, C/H, 1, 1); + if (!ggml_metal_tensor_name_contains(op, "ffn_moe_down") || + (op->src[0]->type != GGML_TYPE_Q2_K && op->src[0]->type != GGML_TYPE_Q3_K) || + (op->src[1]->type != GGML_TYPE_F32 && op->src[1]->type != GGML_TYPE_F16) || + op->src[2]->ne[0] != 8) { + return 0; - } ++ } - return 1; + return ggml_nbytes(op->src[1]); } --int ggml_metal_op_ssm_scan(ggml_metal_op_t ctx, int idx) { +-int ggml_metal_op_gated_delta_net(ggml_metal_op_t ctx, int idx) { +int ggml_metal_op_mul_mat_id(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); @@ -14292,97 +14448,75 @@ index c716f118..36378f8e 100644 ggml_metal_encoder_t enc = ctx->enc; + const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx->dev); -+ + GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); - GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne); - GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb); - GGML_TENSOR_LOCALS( int32_t, ne2, op->src[2], ne); - GGML_TENSOR_LOCALS(uint64_t, nb2, op->src[2], nb); -- GGML_TENSOR_LOCALS( int32_t, ne3, op->src[3], ne); -- GGML_TENSOR_LOCALS(uint64_t, nb3, op->src[3], nb); -- GGML_TENSOR_LOCALS( int32_t, ne4, op->src[4], ne); -- GGML_TENSOR_LOCALS(uint64_t, nb4, op->src[4], nb); -- GGML_TENSOR_LOCALS( int32_t, ne5, op->src[5], ne); -- GGML_TENSOR_LOCALS(uint64_t, nb5, op->src[5], nb); -- GGML_TENSOR_LOCALS( int32_t, ne6, op->src[6], ne); -- GGML_TENSOR_LOCALS(uint64_t, nb6, op->src[6], nb); +@@ -1615,66 +7959,254 @@ int ggml_metal_op_gated_delta_net(ggml_metal_op_t ctx, int idx) { GGML_TENSOR_LOCALS( int32_t, ne, op, ne); GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); -- const ggml_tensor * src3 = op->src[3]; -- const ggml_tensor * src4 = op->src[4]; -- const ggml_tensor * src5 = op->src[5]; -- const ggml_tensor * src6 = op->src[6]; +- auto pipeline = ggml_metal_library_get_pipeline_gated_delta_net(lib, op); + // src2 = ids + GGML_ASSERT(op->src[2]->type == GGML_TYPE_I32); -- GGML_ASSERT(src3); -- GGML_ASSERT(src4); -- GGML_ASSERT(src5); -- GGML_ASSERT(src6); +- int ida = 0; + GGML_ASSERT(!ggml_is_transposed(op->src[0])); + GGML_ASSERT(!ggml_is_transposed(op->src[1])); -- const int64_t d_state = ne00; -- const int64_t d_inner = ne01; -- const int64_t n_head = ne02; -- const int64_t n_group = ne41; -- const int64_t n_seq_tokens = ne12; -- const int64_t n_seqs = ne13; +- ggml_metal_kargs_gated_delta_net args = { +- /*.ne00 =*/ ne00, +- /*.ne01 =*/ ne01, +- /*.ne02 =*/ ne02, +- /*.ne03 =*/ ne03, +- /*.nb00 =*/ nb00, +- /*.nb01 =*/ nb01, +- /*.nb02 =*/ nb02, +- /*.nb03 =*/ nb03, +- /*.ne10 =*/ ne10, +- /*.ne11 =*/ ne11, +- /*.ne12 =*/ ne12, +- /*.ne13 =*/ ne13, +- /*.nb10 =*/ nb10, +- /*.nb11 =*/ nb11, +- /*.nb12 =*/ nb12, +- /*.nb13 =*/ nb13, +- /*.ne20 =*/ ne20, +- /*.ne21 =*/ ne21, +- /*.ne22 =*/ ne22, +- /*.ne23 =*/ ne23, +- /*.nb20 =*/ nb20, +- /*.nb21 =*/ nb21, +- /*.nb22 =*/ nb22, +- /*.nb23 =*/ nb23, +- /*.ns02 =*/ (int32_t) (nb02/sizeof(float)), +- /*.ns12 =*/ (int32_t) (nb12/sizeof(float)), +- /*.ns22 =*/ (int32_t) (nb22/sizeof(float)), +- /*.ne0 =*/ ne0, +- /*.ne1 =*/ ne1, +- /*.ne2 =*/ ne2, +- /*.ne3 =*/ ne3, +- /*.nb0 =*/ nb0, +- /*.nb1 =*/ nb1, +- /*.nb2 =*/ nb2, +- /*.nb3 =*/ nb3, +- }; + GGML_ASSERT(ne03 == 1); + GGML_ASSERT(ne13 == 1); - -- ggml_metal_kargs_ssm_scan args = { -- /*.d_state =*/ d_state, -- /*.d_inner =*/ d_inner, -- /*.n_head =*/ n_head, -- /*.n_group =*/ n_group, -- /*.n_seq_tokens =*/ n_seq_tokens, -- /*.n_seqs =*/ n_seqs, -- /*.s_off =*/ ggml_nelements(op->src[1]) * sizeof(float), -- /*.nb00 =*/ nb00, -- /*.nb01 =*/ nb01, -- /*.nb02 =*/ nb02, -- /*.nb03 =*/ nb03, -- /*.nb10 =*/ nb10, -- /*.nb11 =*/ nb11, -- /*.nb12 =*/ nb12, -- /*.ns12 =*/ nb12/nb10, -- /*.nb13 =*/ nb13, -- /*.nb20 =*/ nb20, -- /*.nb21 =*/ nb21, -- /*.ns21 =*/ nb21/nb20, -- /*.nb22 =*/ nb22, -- /*.ne30 =*/ ne30, -- /*.nb31 =*/ nb31, -- /*.nb41 =*/ nb41, -- /*.nb42 =*/ nb42, -- /*.ns42 =*/ nb42/nb40, -- /*.nb43 =*/ nb43, -- /*.nb51 =*/ nb51, -- /*.nb52 =*/ nb52, -- /*.ns52 =*/ nb52/nb50, -- /*.nb53 =*/ nb53, -- /*.nb0 =*/ nb0, -- }; ++ + ggml_metal_buffer_id bid_src0 = ggml_metal_get_buffer_id(op->src[0]); + ggml_metal_buffer_id bid_src1 = ggml_metal_get_buffer_id(op->src[1]); + ggml_metal_buffer_id bid_src2 = ggml_metal_get_buffer_id(op->src[2]); + ggml_metal_buffer_id bid_dst = ggml_metal_get_buffer_id(op); - -- auto pipeline = ggml_metal_library_get_pipeline_ssm_scan(lib, op); ++ + const uint32_t r2 = 1; + const uint32_t r3 = 1; - -- GGML_ASSERT(d_state <= ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)); ++ + // find the break-even point where the matrix-matrix kernel becomes more efficient compared + // to the matrix-vector kernel + // ne20 = n_used_experts + // ne21 = n_rows (batch size) + const int ne21_mm_id_min = ggml_metal_glm_dsa_mul_mm_id_min_tokens_requested(); - -- const size_t smem = pipeline.smem; ++ + if (props_dev->has_simdgroup_mm && ne00 >= 64 && (ne21 >= ne21_mm_id_min)) { + // some Metal matrix data types require aligned pointers + // ref: https://developer.apple.com/metal/Metal-Shading-Language-Specification.pdf (Table 2.5) @@ -14392,26 +14526,14 @@ index c716f118..36378f8e 100644 + // case GGML_TYPE_BF16: GGML_ASSERT(nb01 % 8 == 0); break; + // default: break; + //} - -- ggml_metal_encoder_set_pipeline(enc, pipeline); -- ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); -- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); -- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2); -- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[2]), 3); -- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[3]), 4); -- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[4]), 5); -- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[5]), 6); -- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[6]), 7); -- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 8); ++ + // extra buffers for intermediate id mapping + ggml_metal_buffer_id bid_tpe = bid_dst; + bid_tpe.offs += ggml_nbytes(op); - -- ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); ++ + ggml_metal_buffer_id bid_ids = bid_tpe; + bid_ids.offs += ggml_metal_op_mul_mat_id_extra_tpe(op); - -- ggml_metal_encoder_dispatch_threadgroups(enc, d_inner, n_head, n_seqs, d_state, 1, 1); ++ + { + ggml_metal_kargs_mul_mm_id_map0 args = { + ne02, @@ -14423,66 +14545,32 @@ index c716f118..36378f8e 100644 + ne20, // n_expert_used + nb21, + }; - -- return 1; --} ++ + auto pipeline = ggml_metal_library_get_pipeline_mul_mm_id_map0(lib, ne02, ne20); - --int ggml_metal_op_rwkv(ggml_metal_op_t ctx, int idx) { -- ggml_tensor * op = ctx->node(idx); ++ + const size_t smem = pipeline.smem; - -- ggml_metal_library_t lib = ctx->lib; -- ggml_metal_encoder_t enc = ctx->enc; ++ + GGML_ASSERT(ne02 <= ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)); - -- GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); -- GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); -- GGML_TENSOR_LOCALS( int32_t, ne, op, ne); -- GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); ++ + GGML_ASSERT(smem <= props_dev->max_theadgroup_memory_size); - -- const int64_t B = op->op == GGML_OP_RWKV_WKV6 ? op->src[5]->ne[1] : op->src[6]->ne[1]; -- const int64_t T = op->src[0]->ne[2]; -- const int64_t C = op->ne[0]; -- const int64_t H = op->src[0]->ne[1]; ++ + ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer (enc, bid_src2, 1); + ggml_metal_encoder_set_buffer (enc, bid_tpe, 2); + ggml_metal_encoder_set_buffer (enc, bid_ids, 3); - -- auto pipeline = ggml_metal_library_get_pipeline_rwkv(lib, op); ++ + ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); - -- int ida = 0; ++ + ggml_metal_encoder_dispatch_threadgroups(enc, 1, 1, 1, ne02, 1, 1); + } - -- ggml_metal_encoder_set_pipeline(enc, pipeline); -- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), ida++); -- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), ida++); -- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[2]), ida++); -- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[3]), ida++); -- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[4]), ida++); -- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[5]), ida++); -- if (op->op == GGML_OP_RWKV_WKV7) { -- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[6]), ida++); -- } -- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), ida++); -- ggml_metal_encoder_set_bytes (enc, (void *) &B, sizeof(B), ida++); -- ggml_metal_encoder_set_bytes (enc, (void *) &T, sizeof(T), ida++); -- ggml_metal_encoder_set_bytes (enc, (void *) &C, sizeof(C), ida++); -- ggml_metal_encoder_set_bytes (enc, (void *) &H, sizeof(H), ida++); ++ + // this barrier is always needed because the next kernel has to wait for the id maps to be computed + ggml_metal_op_concurrency_reset(ctx); - -- ggml_metal_encoder_dispatch_threadgroups(enc, B * H, 1, 1, C/H, 1, 1); ++ + { + auto pipeline = ggml_metal_library_get_pipeline_mul_mm_id(lib, op); - -- return 1; --} ++ + ggml_metal_kargs_mul_mm_id args = { + /*.ne00 =*/ ne00, + /*.ne02 =*/ ne02, @@ -14501,9 +14589,7 @@ index c716f118..36378f8e 100644 + /*.r2 =*/ r2, + /*.r3 =*/ r3, + }; - --int ggml_metal_op_gated_delta_net(ggml_metal_op_t ctx, int idx) { -- ggml_tensor * op = ctx->node(idx); ++ + ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer (enc, bid_src0, 1); @@ -14511,21 +14597,11 @@ index c716f118..36378f8e 100644 + ggml_metal_encoder_set_buffer (enc, bid_tpe, 3); + ggml_metal_encoder_set_buffer (enc, bid_ids, 4); + ggml_metal_encoder_set_buffer (enc, bid_dst, 5); - -- ggml_metal_library_t lib = ctx->lib; -- ggml_metal_encoder_t enc = ctx->enc; ++ + const size_t smem = pipeline.smem; - ++ + ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); - -- GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); -- GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); -- GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne); -- GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb); -- GGML_TENSOR_LOCALS( int32_t, ne2, op->src[2], ne); -- GGML_TENSOR_LOCALS(uint64_t, nb2, op->src[2], nb); -- GGML_TENSOR_LOCALS( int32_t, ne, op, ne); -- GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); ++ + const int grid_x = (ne21 + 31)/32; + const int grid_y = (ne01 + 63)/64; + const int grid_z = ne02; @@ -14554,52 +14630,13 @@ index c716f118..36378f8e 100644 + } + } else { + auto pipeline = ggml_metal_library_get_pipeline_mul_mv_id(lib, op); - -- auto pipeline = ggml_metal_library_get_pipeline_gated_delta_net(lib, op); ++ + const int nr0 = pipeline.nr0; + const int nr1 = pipeline.nr1; + const int nsg = pipeline.nsg; - -- int ida = 0; ++ + const size_t smem = pipeline.smem; - -- ggml_metal_kargs_gated_delta_net args = { -- /*.ne00 =*/ ne00, -- /*.ne01 =*/ ne01, -- /*.ne02 =*/ ne02, -- /*.ne03 =*/ ne03, -- /*.nb00 =*/ nb00, -- /*.nb01 =*/ nb01, -- /*.nb02 =*/ nb02, -- /*.nb03 =*/ nb03, -- /*.ne10 =*/ ne10, -- /*.ne11 =*/ ne11, -- /*.ne12 =*/ ne12, -- /*.ne13 =*/ ne13, -- /*.nb10 =*/ nb10, -- /*.nb11 =*/ nb11, -- /*.nb12 =*/ nb12, -- /*.nb13 =*/ nb13, -- /*.ne20 =*/ ne20, -- /*.ne21 =*/ ne21, -- /*.ne22 =*/ ne22, -- /*.ne23 =*/ ne23, -- /*.nb20 =*/ nb20, -- /*.nb21 =*/ nb21, -- /*.nb22 =*/ nb22, -- /*.nb23 =*/ nb23, -- /*.ns02 =*/ (int32_t) (nb02/sizeof(float)), -- /*.ns12 =*/ (int32_t) (nb12/sizeof(float)), -- /*.ns22 =*/ (int32_t) (nb22/sizeof(float)), -- /*.ne0 =*/ ne0, -- /*.ne1 =*/ ne1, -- /*.ne2 =*/ ne2, -- /*.ne3 =*/ ne3, -- /*.nb0 =*/ nb0, -- /*.nb1 =*/ nb1, -- /*.nb2 =*/ nb2, -- /*.nb3 =*/ nb3, -- }; ++ + ggml_metal_kargs_mul_mv_id args = { + /*.nei0 =*/ ne20, + /*.nei1 =*/ ne21, @@ -14622,6 +14659,17 @@ index c716f118..36378f8e 100644 + /*.nb1 =*/ nb1, + /*.nr0 =*/ nr0, + }; ++ ++ if (ggml_is_quantized(op->src[0]->type)) { ++ GGML_ASSERT(ne00 >= nsg*nr0); ++ } ++ ++ ggml_metal_encoder_set_pipeline(enc, pipeline); ++ ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0); ++ ggml_metal_encoder_set_buffer(enc, bid_src0, 1); ++ ggml_metal_encoder_set_buffer(enc, bid_src1, 2); ++ ggml_metal_encoder_set_buffer(enc, bid_dst, 3); ++ ggml_metal_encoder_set_buffer(enc, bid_src2, 4); - ggml_metal_encoder_set_pipeline(enc, pipeline); - ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), ida++); @@ -14632,24 +14680,13 @@ index c716f118..36378f8e 100644 - ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[4]), ida++); // beta - ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[5]), ida++); // state - ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), ida++); // dst -+ if (ggml_is_quantized(op->src[0]->type)) { -+ GGML_ASSERT(ne00 >= nsg*nr0); -+ } ++ const int64_t _ne1 = 1; ++ const int64_t ne123 = ne20*ne21; - const int nsg = pipeline.nsg; -+ ggml_metal_encoder_set_pipeline(enc, pipeline); -+ ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0); -+ ggml_metal_encoder_set_buffer(enc, bid_src0, 1); -+ ggml_metal_encoder_set_buffer(enc, bid_src1, 2); -+ ggml_metal_encoder_set_buffer(enc, bid_dst, 3); -+ ggml_metal_encoder_set_buffer(enc, bid_src2, 4); ++ ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); - ggml_metal_encoder_dispatch_threadgroups(enc, op->src[2]->ne[0]/nsg, op->src[2]->ne[1], op->src[2]->ne[3], 32, nsg, 1); -+ const int64_t _ne1 = 1; -+ const int64_t ne123 = ne20*ne21; -+ -+ ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); -+ + int grid_x = 0; + int grid_y = 0; + const int grid_z = ne123; @@ -14720,7 +14757,7 @@ index c716f118..36378f8e 100644 ggml_tensor * op = ctx->node(idx); ggml_metal_library_t lib = ctx->lib; -@@ -1684,1610 +8164,2148 @@ int ggml_metal_op_solve_tri(ggml_metal_op_t ctx, int idx) { +@@ -1684,1662 +8216,2148 @@ int ggml_metal_op_solve_tri(ggml_metal_op_t ctx, int idx) { GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne); GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb); @@ -14955,19 +14992,22 @@ index c716f118..36378f8e 100644 + } + if (tiles >= 4) { + return 4; -+ } + } + if (tiles >= 2) { + return 2; + } + return 1; +} -+ + +- int nth = std::min(nk0*ne11, 256); +static int32_t ggml_metal_glm_dsa_selected_row_flash_tiled_nwg_for_top_k(int64_t top_k) { + const char * value = getenv("LLAMA_GLM_DSA_SELECTED_ROW_FLASH_NWG"); + if (value != nullptr && value[0] != '\0') { + return ggml_metal_glm_dsa_selected_row_flash_nwg_requested(); + } -+ + +- // when rows are small, we can batch them together in a single threadgroup +- int nrptg = 1; + const int64_t tiles = (top_k + 63)/64; + if (tiles >= 32) { + return 32; @@ -14986,7 +15026,12 @@ index c716f118..36378f8e 100644 + } + return 1; +} -+ + +- // TODO: relax this constraint in the future +- if (ggml_blck_size(op->src[1]->type) == 1 && ggml_blck_size(op->type) == 1) { +- if (nth > nk0) { +- nrptg = (nth + nk0 - 1)/nk0; +- nth = nk0; +static bool ggml_metal_glm_dsa_compact_flash_shape( + const ggml_tensor * op, + bool has_mask, @@ -15006,7 +15051,10 @@ index c716f118..36378f8e 100644 + op->src[0]->ne[1] == 1 && + op->src[1]->ne[1] == 2048; +} -+ + +- if (nrptg*nth > 256) { +- nrptg--; +- } +static int32_t ggml_metal_flash_attn_ext_vec_nwg( + const ggml_tensor * op, + bool has_mask, @@ -15018,10 +15066,31 @@ index c716f118..36378f8e 100644 + if (ggml_metal_glm_dsa_selected_row_flash_tiled_selected(op) && + ggml_metal_glm_dsa_selected_row_flash_tiled_shape(op)) { + return ggml_metal_glm_dsa_selected_row_flash_tiled_nwg(op); -+ } + } + return ggml_metal_glm_dsa_selected_row_flash_nwg_requested(); -+ } -+ + } + +- nth = std::min(nth, nk0); +- +- ggml_metal_kargs_cpy args = { +- /*.nk0 =*/ nk0, +- /*.ne00 =*/ ne10, +- /*.ne01 =*/ ne11, +- /*.ne02 =*/ ne12, +- /*.ne03 =*/ ne13, +- /*.nb00 =*/ nb10, +- /*.nb01 =*/ nb11, +- /*.nb02 =*/ nb12, +- /*.nb03 =*/ nb13, +- /*.ne0 =*/ ne10, +- /*.ne1 =*/ ne11, +- /*.ne2 =*/ ne12, +- /*.ne3 =*/ ne13, +- /*.nb0 =*/ ggml_element_size(op), +- /*.nb1 =*/ pnb1, +- /*.nb2 =*/ pnb2, +- /*.nb3 =*/ pnb3, +- }; + if (ggml_metal_glm_dsa_compact_flash_shape(op, has_mask, has_sinks, has_bias, has_scap, has_kvpad)) { + const char * nwg8_tensor = getenv("GGML_METAL_EXPERIMENTAL_GLM_COMPACT_NWG8_TENSOR"); + if (nwg8_tensor != nullptr && nwg8_tensor[0] != '\0' && @@ -15030,13 +15099,19 @@ index c716f118..36378f8e 100644 + } + return ggml_metal_glm_dsa_compact_flash_nwg_requested(); + } -+ + +- const int nw0 = nrptg == 1 ? (nk0 + nth - 1)/nth : 1; + return 32; +} -+ + +- bid_dst.offs += offs; +size_t ggml_metal_op_flash_attn_ext_extra_pad(const ggml_tensor * op) { + assert(op->op == GGML_OP_FLASH_ATTN_EXT); -+ + +- ggml_metal_encoder_set_pipeline(enc, pipeline); +- ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); +- ggml_metal_encoder_set_buffer (enc, bid_src1, 1); +- ggml_metal_encoder_set_buffer (enc, bid_dst, 2); + GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); + GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); + GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne); @@ -15045,20 +15120,31 @@ index c716f118..36378f8e 100644 + GGML_TENSOR_LOCALS(uint64_t, nb2, op->src[2], nb); + GGML_TENSOR_LOCALS( int32_t, ne3, op->src[3], ne); + GGML_TENSOR_LOCALS(uint64_t, nb3, op->src[3], nb); -+ + +- ggml_metal_encoder_dispatch_threadgroups(enc, nw0*(ne11 + nrptg - 1)/nrptg, ne12, ne13, nth, nrptg, 1); + size_t res = 0; -+ + +- return 1; +-} + const bool has_mask = op->src[3] != nullptr; -+ + +-int ggml_metal_op_cpy(ggml_metal_op_t ctx, int idx) { +- ggml_tensor * op = ctx->node(idx); + // note: the non-vec kernel requires more extra memory, so always reserve for it + GGML_ASSERT(OP_FLASH_ATTN_EXT_NCPSG >= OP_FLASH_ATTN_EXT_VEC_NCPSG); -+ + +- ggml_metal_library_t lib = ctx->lib; +- ggml_metal_encoder_t enc = ctx->enc; + //if (ggml_metal_op_flash_attn_ext_use_vec(op)) { + if (false) { + // note: always reserve the padding space to avoid graph reallocations + //const bool has_kvpad = ne11 % OP_FLASH_ATTN_EXT_VEC_NCPSG != 0; + const bool has_kvpad = true; -+ + +- GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); +- GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); +- GGML_TENSOR_LOCALS( int32_t, ne, op, ne); +- GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); + if (has_kvpad) { + res += OP_FLASH_ATTN_EXT_VEC_NCPSG*( + nb11*ne12*ne13 + @@ -15068,22 +15154,30 @@ index c716f118..36378f8e 100644 + } else { + //const bool has_kvpad = ne11 % OP_FLASH_ATTN_EXT_NCPSG != 0; + const bool has_kvpad = true; -+ + +- auto pipeline = ggml_metal_library_get_pipeline_cpy(lib, op->src[0]->type, op->type); + if (has_kvpad) { + res += OP_FLASH_ATTN_EXT_NCPSG*( + nb11*ne12*ne13 + + nb21*ne22*ne23 + + (has_mask ? ggml_type_size(GGML_TYPE_F16)*ne31*ne32*ne33 : 0)); + } - } ++ } -- int nth = std::min(nk0*ne11, 256); +- GGML_ASSERT(ne00 % ggml_blck_size(op->src[0]->type) == 0); + return res; +} -+ + +- int64_t nk0 = ne00; +- if (ggml_is_quantized(op->src[0]->type)) { +- nk0 = ne00/16; +- } else if (ggml_is_quantized(op->type)) { +- nk0 = ne00/ggml_blck_size(op->type); +- } +size_t ggml_metal_op_flash_attn_ext_extra_blk(const ggml_tensor * op) { + assert(op->op == GGML_OP_FLASH_ATTN_EXT); -+ + +- int nth = std::min(nk0*ne01, 256); + GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); + //GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); + //GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne); @@ -15098,7 +15192,7 @@ index c716f118..36378f8e 100644 + size_t res = 0; - // TODO: relax this constraint in the future -- if (ggml_blck_size(op->src[1]->type) == 1 && ggml_blck_size(op->type) == 1) { +- if (ggml_blck_size(op->src[0]->type) == 1 && ggml_blck_size(op->type) == 1) { - if (nth > nk0) { - nrptg = (nth + nk0 - 1)/nk0; - nth = nk0; @@ -15113,54 +15207,52 @@ index c716f118..36378f8e 100644 } - nth = std::min(nth, nk0); -- ++ const bool is_vec = ggml_metal_op_flash_attn_ext_use_vec(op); + - ggml_metal_kargs_cpy args = { - /*.nk0 =*/ nk0, -- /*.ne00 =*/ ne10, -- /*.ne01 =*/ ne11, -- /*.ne02 =*/ ne12, -- /*.ne03 =*/ ne13, -- /*.nb00 =*/ nb10, -- /*.nb01 =*/ nb11, -- /*.nb02 =*/ nb12, -- /*.nb03 =*/ nb13, -- /*.ne0 =*/ ne10, -- /*.ne1 =*/ ne11, -- /*.ne2 =*/ ne12, -- /*.ne3 =*/ ne13, -- /*.nb0 =*/ ggml_element_size(op), -- /*.nb1 =*/ pnb1, -- /*.nb2 =*/ pnb2, -- /*.nb3 =*/ pnb3, +- /*.ne00 =*/ ne00, +- /*.ne01 =*/ ne01, +- /*.ne02 =*/ ne02, +- /*.ne03 =*/ ne03, +- /*.nb00 =*/ nb00, +- /*.nb01 =*/ nb01, +- /*.nb02 =*/ nb02, +- /*.nb03 =*/ nb03, +- /*.ne0 =*/ ne0, +- /*.ne1 =*/ ne1, +- /*.ne2 =*/ ne2, +- /*.ne3 =*/ ne3, +- /*.nb0 =*/ nb0, +- /*.nb1 =*/ nb1, +- /*.nb2 =*/ nb2, +- /*.nb3 =*/ nb3, - }; -+ const bool is_vec = ggml_metal_op_flash_attn_ext_use_vec(op); - -- const int nw0 = nrptg == 1 ? (nk0 + nth - 1)/nth : 1; + // this optimization is not useful for the vector kernels + // note: always reserve the blk buffer to avoid graph reallocations + //if (is_vec) { + // return res; + //} -- bid_dst.offs += offs; +- const int nw0 = nrptg == 1 ? (nk0 + nth - 1)/nth : 1; + const int nqptg = is_vec ? OP_FLASH_ATTN_EXT_VEC_NQPSG : OP_FLASH_ATTN_EXT_NQPSG; + const int ncpsg = is_vec ? OP_FLASH_ATTN_EXT_VEC_NCPSG : OP_FLASH_ATTN_EXT_NCPSG; - ggml_metal_encoder_set_pipeline(enc, pipeline); - ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); -- ggml_metal_encoder_set_buffer (enc, bid_src1, 1); -- ggml_metal_encoder_set_buffer (enc, bid_dst, 2); +- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); +- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2); + const int64_t ne1 = (ne01 + nqptg - 1)/nqptg; + const int64_t ne0 = (ne30 + ncpsg - 1)/ncpsg; -- ggml_metal_encoder_dispatch_threadgroups(enc, nw0*(ne11 + nrptg - 1)/nrptg, ne12, ne13, nth, nrptg, 1); +- ggml_metal_encoder_dispatch_threadgroups(enc, nw0*(ne01 + nrptg - 1)/nrptg, ne02, ne03, nth, nrptg, 1); + res += GGML_PAD(ggml_type_size(GGML_TYPE_I8)*ne0*ne1*ne32*ne33, 32); - return 1; + return res; } --int ggml_metal_op_cpy(ggml_metal_op_t ctx, int idx) { +-int ggml_metal_op_pool_1d(ggml_metal_op_t ctx, int idx) { - ggml_tensor * op = ctx->node(idx); - - ggml_metal_library_t lib = ctx->lib; @@ -15173,9 +15265,17 @@ index c716f118..36378f8e 100644 - GGML_TENSOR_LOCALS( int32_t, ne, op, ne); - GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); - -- auto pipeline = ggml_metal_library_get_pipeline_cpy(lib, op->src[0]->type, op->type); +- const int32_t * opts = op->op_params; +- ggml_op_pool op_pool = (ggml_op_pool) opts[0]; - -- GGML_ASSERT(ne00 % ggml_blck_size(op->src[0]->type) == 0); +- const int32_t k0 = opts[1]; +- const int32_t s0 = opts[2]; +- const int32_t p0 = opts[3]; +- +- const int64_t IW = op->src[0]->ne[0]; +- const int64_t OW = op->ne[0]; +- +- const int64_t np = ggml_nelements(op); + //GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne); + //GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb); + GGML_TENSOR_LOCALS( int32_t, ne2, op->src[2], ne); @@ -15183,15 +15283,17 @@ index c716f118..36378f8e 100644 + //GGML_TENSOR_LOCALS( int32_t, ne3, op->src[3], ne); + //GGML_TENSOR_LOCALS(uint64_t, nb3, op->src[3], nb); -- int64_t nk0 = ne00; -- if (ggml_is_quantized(op->src[0]->type)) { -- nk0 = ne00/16; -- } else if (ggml_is_quantized(op->type)) { -- nk0 = ne00/ggml_blck_size(op->type); -- } +- ggml_metal_kargs_pool_1d args_pool_1d = { +- /* .k0 = */ k0, +- /* .s0 = */ s0, +- /* .p0 = */ p0, +- /* .IW = */ IW, +- /* .OW = */ OW, +- /* .np = */ np +- }; + size_t res = 0; -- int nth = std::min(nk0*ne01, 256); +- auto pipeline = ggml_metal_library_get_pipeline_pool_1d(lib, op, op_pool); + // note: always reserve the temp buffer to avoid graph reallocations + //if (ggml_metal_op_flash_attn_ext_use_vec(op)) { + if (true) { @@ -15212,23 +15314,20 @@ index c716f118..36378f8e 100644 + return res; + } -- // when rows are small, we can batch them together in a single threadgroup -- int nrptg = 1; +- const int nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline), (int) np); +- const int ntg = (np + nth - 1) / nth; + const int64_t ne01_max = std::min(ne01, 32); -- // TODO: relax this constraint in the future -- if (ggml_blck_size(op->src[0]->type) == 1 && ggml_blck_size(op->type) == 1) { -- if (nth > nk0) { -- nrptg = (nth + nk0 - 1)/nk0; -- nth = nk0; +- ggml_metal_encoder_set_pipeline(enc, pipeline); +- ggml_metal_encoder_set_bytes (enc, &args_pool_1d, sizeof(args_pool_1d), 0); +- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); +- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2); + // temp buffer for writing the results from each workgroup + // - ne20: the size of the Value head + // - + 2: the S and M values for each intermediate result + res += ggml_type_size(GGML_TYPE_F32)*(ne01_max*ne02*ne03*nwg*(ne20 + 2)); -- if (nrptg*nth > 256) { -- nrptg--; -- } +- ggml_metal_encoder_dispatch_threadgroups(enc, ntg, 1, 1, nth, 1, 1); + if (ggml_metal_glm_compact_split_exact_enabled() && + !has_mask && !has_sinks && !has_bias && !has_scap && !has_kvpad && + op->src[1]->type == GGML_TYPE_F16 && @@ -15239,52 +15338,28 @@ index c716f118..36378f8e 100644 + op->src[0]->ne[3] == 1 && + op->src[1]->ne[1] == 2048 && + op->src[1]->ne[2] == 1 && -+ op->src[2]->ne[0] == 512 && -+ op->src[2]->ne[2] == 1) { -+ constexpr size_t chunk_rows = OP_FLASH_ATTN_EXT_VEC_NCPSG; -+ const size_t heads = size_t(op->src[0]->ne[2])*size_t(op->src[0]->ne[3]); -+ const size_t rows = size_t(op->src[1]->ne[1]); -+ const size_t chunks = (rows + chunk_rows - 1)/chunk_rows; -+ const size_t score_values = heads*rows; -+ const size_t chunk_ms_values = heads*chunks; -+ const size_t chunk_v_values = heads*chunks*size_t(op->src[2]->ne[0]); -+ res += ggml_type_size(GGML_TYPE_F32)* -+ (score_values + chunk_ms_values + chunk_v_values); - } - } - -- nth = std::min(nth, nk0); -- -- ggml_metal_kargs_cpy args = { -- /*.nk0 =*/ nk0, -- /*.ne00 =*/ ne00, -- /*.ne01 =*/ ne01, -- /*.ne02 =*/ ne02, -- /*.ne03 =*/ ne03, -- /*.nb00 =*/ nb00, -- /*.nb01 =*/ nb01, -- /*.nb02 =*/ nb02, -- /*.nb03 =*/ nb03, -- /*.ne0 =*/ ne0, -- /*.ne1 =*/ ne1, -- /*.ne2 =*/ ne2, -- /*.ne3 =*/ ne3, -- /*.nb0 =*/ nb0, -- /*.nb1 =*/ nb1, -- /*.nb2 =*/ nb2, -- /*.nb3 =*/ nb3, -- }; -- -- const int nw0 = nrptg == 1 ? (nk0 + nth - 1)/nth : 1; -- -- ggml_metal_encoder_set_pipeline(enc, pipeline); -- ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); -- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); -- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2); ++ op->src[2]->ne[0] == 512 && ++ op->src[2]->ne[2] == 1) { ++ constexpr size_t chunk_rows = OP_FLASH_ATTN_EXT_VEC_NCPSG; ++ const size_t heads = size_t(op->src[0]->ne[2])*size_t(op->src[0]->ne[3]); ++ const size_t rows = size_t(op->src[1]->ne[1]); ++ const size_t chunks = (rows + chunk_rows - 1)/chunk_rows; ++ const size_t score_values = heads*rows; ++ const size_t chunk_ms_values = heads*chunks; ++ const size_t chunk_v_values = heads*chunks*size_t(op->src[2]->ne[0]); ++ res += ggml_type_size(GGML_TYPE_F32)* ++ (score_values + chunk_ms_values + chunk_v_values); ++ } ++ } + +- return 1; + return res; -+} + } -- ggml_metal_encoder_dispatch_threadgroups(enc, nw0*(ne01 + nrptg - 1)/nrptg, ne02, ne03, nth, nrptg, 1); +-// supported FWHT sizes, must stay in sync with the +-// kernel_fwht_f32_ templates in ggml-metal.metal +-static bool ggml_metal_fwht_supported_size(int64_t n) { +- return n == 64 || n == 128 || n == 256 || n == 512; +size_t ggml_metal_op_dsa_sparse_attn_extra_tmp(const ggml_tensor * op) { + assert(op->op == GGML_OP_DSA_SPARSE_ATTN); + @@ -15298,14 +15373,13 @@ index c716f118..36378f8e 100644 + op->src[4]->ne[0] <= 0 || op->src[4]->ne[0] > 4096) { + return 0; + } - -- return 1; ++ + constexpr size_t max_nwg = 32; + const size_t nrows = size_t(op->ne[1])*size_t(op->ne[2])*size_t(op->ne[3]); + return sizeof(float)*nrows*max_nwg*size_t(op->ne[0] + 2); } --int ggml_metal_op_pool_1d(ggml_metal_op_t ctx, int idx) { +-int ggml_metal_op_fwht(ggml_metal_op_t ctx, int idx) { +static int ggml_metal_op_selected_row_flash_vec(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); + if (!ggml_metal_selected_row_flash_vec_shape_ok(op)) { @@ -15321,10 +15395,10 @@ index c716f118..36378f8e 100644 ggml_metal_library_t lib = ctx->lib; ggml_metal_encoder_t enc = ctx->enc; -- GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); -- GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); -- GGML_TENSOR_LOCALS( int32_t, ne, op, ne); -- GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); +- ggml_tensor * src1 = op->src[1]; +- +- const int64_t n = src1->ne[0]; +- const int64_t nrows = ggml_nrows(src1); + if (ggml_metal_glm_selected_row_flash_noop_enabled()) { + auto pipeline = ggml_metal_library_get_pipeline_zero_f32(lib); + const int nth = 256; @@ -15336,8 +15410,8 @@ index c716f118..36378f8e 100644 + return 1; + } -- const int32_t * opts = op->op_params; -- ggml_op_pool op_pool = (ggml_op_pool) opts[0]; +- ggml_metal_kargs_fwht args = { +- /*.nrows = */ (int32_t) nrows, + ggml_metal_kargs_selected_row_flash args = { + /*.ne00 =*/ (int32_t) q->ne[0], + /*.ne01 =*/ (int32_t) q->ne[1], @@ -15388,11 +15462,9 @@ index c716f118..36378f8e 100644 + /*.nb2 =*/ op->nb[2], + /*.nb3 =*/ op->nb[3], + /*.scale =*/ ggml_get_op_params_f32(op, 0), -+ }; + }; -- const int32_t k0 = opts[1]; -- const int32_t s0 = opts[2]; -- const int32_t p0 = opts[3]; +- auto pipeline = ggml_metal_library_get_pipeline_fwht(lib, n); + const bool use_tiled = + ggml_metal_glm_dsa_selected_row_flash_tiled_selected(op) && + ggml_metal_glm_dsa_selected_row_flash_tiled_shape(op); @@ -15413,33 +15485,19 @@ index c716f118..36378f8e 100644 + const int32_t dispatch_nsg = use_tiled ? 4 : 1; + GGML_ASSERT(dispatch_nsg*32 <= ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)); -- const int64_t IW = op->src[0]->ne[0]; -- const int64_t OW = op->ne[0]; + ggml_metal_buffer_id bid_dst = ggml_metal_get_buffer_id(op); + ggml_metal_buffer_id bid_pad = bid_dst; + bid_pad.offs += ggml_nbytes(op); - -- const int64_t np = ggml_nelements(op); ++ + ggml_metal_buffer_id bid_blk = bid_pad; + bid_blk.offs += ggml_metal_op_flash_attn_ext_extra_pad(op); - -- ggml_metal_kargs_pool_1d args_pool_1d = { -- /* .k0 = */ k0, -- /* .s0 = */ s0, -- /* .p0 = */ p0, -- /* .IW = */ IW, -- /* .OW = */ OW, -- /* .np = */ np -- }; ++ + ggml_metal_buffer_id bid_tmp = bid_blk; + bid_tmp.offs += ggml_metal_op_flash_attn_ext_extra_blk(op); - -- auto pipeline = ggml_metal_library_get_pipeline_pool_1d(lib, op, op_pool); ++ + ggml_metal_buffer_id bid_v = ggml_metal_get_buffer_id(packed_kv); + bid_v.offs += view->view_offs; - -- const int nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline), (int) np); -- const int ntg = (np + nth - 1) / nth; ++ +#define SELECTED_ROW_FLASH_VEC_SMEM(nsg_) (GGML_PAD(((GGML_PAD(args.ne00, 128) + 4*OP_FLASH_ATTN_EXT_VEC_NCPSG + 2*GGML_PAD(args.ne20, 128))*(nsg_))*(sizeof(float)/2), 16)) +#define SELECTED_ROW_FLASH_TILED_SMEM (GGML_PAD((8*576 + 2*8*512 + 2*8*64 + 4*4*16*8)*sizeof(uint16_t), 16)) + const size_t smem = use_tiled ? SELECTED_ROW_FLASH_TILED_SMEM : SELECTED_ROW_FLASH_VEC_SMEM(nsg); @@ -15448,12 +15506,12 @@ index c716f118..36378f8e 100644 + + const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx->dev); + GGML_ASSERT(smem <= props_dev->max_theadgroup_memory_size); - ++ + ggml_metal_op_concurrency_reset(ctx); ggml_metal_encoder_set_pipeline(enc, pipeline); -- ggml_metal_encoder_set_bytes (enc, &args_pool_1d, sizeof(args_pool_1d), 0); -- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); -- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2); +- ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0); +- ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(src1), 1); +- ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 2); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(q), 1); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(packed_kv), 2); @@ -15478,13 +15536,19 @@ index c716f118..36378f8e 100644 + ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); + ggml_metal_encoder_dispatch_threadgroups(enc, grid_x, grid_y, grid_z, 32, dispatch_nsg, 1); -- ggml_metal_encoder_dispatch_threadgroups(enc, ntg, 1, 1, nth, 1, 1); +- const int th_max = ggml_metal_pipeline_max_theads_per_threadgroup(pipeline); +- const int simd_size = 32; + ggml_metal_op_concurrency_reset(ctx); -+ + +- int sg_per_tg = 2; +- sg_per_tg = std::min(sg_per_tg, th_max/simd_size); +- sg_per_tg = std::max(sg_per_tg, 1); + ggml_metal_kargs_flash_attn_ext_vec_reduce args0 = { + (int32_t) (args.ne1*args.ne2*args.ne3), + }; -+ + +- const int64_t n_tg = (nrows + sg_per_tg - 1) / sg_per_tg; +- ggml_metal_encoder_dispatch_threadgroups(enc, n_tg, 1, 1, 32*sg_per_tg, 1, 1); + auto pipeline0 = ggml_metal_library_get_pipeline_flash_attn_ext_vec_reduce(lib, op, args.ne20, nwg); + ggml_metal_encoder_set_pipeline(enc, pipeline0); + ggml_metal_encoder_set_bytes (enc, &args0, sizeof(args0), 0); @@ -15521,7 +15585,6 @@ index c716f118..36378f8e 100644 return 1; } -- -int ggml_metal_op_pool_2d(ggml_metal_op_t ctx, int idx) { +int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); @@ -15578,19 +15641,6 @@ index c716f118..36378f8e 100644 + float scale; + float max_bias; + float logit_softcap; -+ -+ memcpy(&scale, ((const int32_t *) op->op_params) + 0, sizeof(scale)); -+ memcpy(&max_bias, ((const int32_t *) op->op_params) + 1, sizeof(max_bias)); -+ memcpy(&logit_softcap, ((const int32_t *) op->op_params) + 2, sizeof(logit_softcap)); -+ -+ if (logit_softcap != 0.0f) { -+ scale /= logit_softcap; -+ } -+ -+ const bool has_mask = op->src[3] != NULL; -+ const bool has_sinks = op->src[4] != NULL; -+ const bool has_bias = max_bias != 0.0f; -+ const bool has_scap = logit_softcap != 0.0f; - ggml_metal_kargs_pool_2d args_pool_2d = { - /* .k0 = */ k0, @@ -15605,45 +15655,69 @@ index c716f118..36378f8e 100644 - /* .OW = */ OW, - /* .np = */ np - }; -+ const uint32_t n_head = op->src[0]->ne[2]; -+ const int32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head)); ++ memcpy(&scale, ((const int32_t *) op->op_params) + 0, sizeof(scale)); ++ memcpy(&max_bias, ((const int32_t *) op->op_params) + 1, sizeof(max_bias)); ++ memcpy(&logit_softcap, ((const int32_t *) op->op_params) + 2, sizeof(logit_softcap)); ++ ++ if (logit_softcap != 0.0f) { ++ scale /= logit_softcap; ++ } ++ ++ const bool has_mask = op->src[3] != NULL; ++ const bool has_sinks = op->src[4] != NULL; ++ const bool has_bias = max_bias != 0.0f; ++ const bool has_scap = logit_softcap != 0.0f; - auto pipeline = ggml_metal_library_get_pipeline_pool_2d(lib, op, op_pool); -+ const float m0 = powf(2.0f, -(max_bias ) / n_head_log2); -+ const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); ++ const uint32_t n_head = op->src[0]->ne[2]; ++ const int32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head)); - const int nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline), (int) np); - const int ntg = (np + nth - 1) / nth; -+ GGML_ASSERT(ne01 < 65536); ++ const float m0 = powf(2.0f, -(max_bias ) / n_head_log2); ++ const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); - ggml_metal_encoder_set_pipeline(enc, pipeline); - ggml_metal_encoder_set_bytes (enc, &args_pool_2d, sizeof(args_pool_2d), 0); - ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); - ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2); ++ GGML_ASSERT(ne01 < 65536); + +- ggml_metal_encoder_dispatch_threadgroups(enc, ntg, 1, 1, nth, 1, 1); + ggml_metal_buffer_id bid_src0 = ggml_metal_get_buffer_id(op->src[0]); + ggml_metal_buffer_id bid_src1 = ggml_metal_get_buffer_id(op->src[1]); + ggml_metal_buffer_id bid_src2 = ggml_metal_get_buffer_id(op->src[2]); + ggml_metal_buffer_id bid_src3 = has_mask ? ggml_metal_get_buffer_id(op->src[3]) : bid_src0; + ggml_metal_buffer_id bid_src4 = has_sinks ? ggml_metal_get_buffer_id(op->src[4]) : bid_src0; -- ggml_metal_encoder_dispatch_threadgroups(enc, ntg, 1, 1, nth, 1, 1); -+ ggml_metal_buffer_id bid_dst = ggml_metal_get_buffer_id(op); - - return 1; -} -+ ggml_metal_buffer_id bid_pad = bid_dst; -+ bid_pad.offs += ggml_nbytes(op); ++ ggml_metal_buffer_id bid_dst = ggml_metal_get_buffer_id(op); -int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) { - ggml_tensor * op = ctx->node(idx); -+ ggml_metal_buffer_id bid_blk = bid_pad; -+ bid_blk.offs += ggml_metal_op_flash_attn_ext_extra_pad(op); ++ ggml_metal_buffer_id bid_pad = bid_dst; ++ bid_pad.offs += ggml_nbytes(op); - ggml_metal_library_t lib = ctx->lib; - ggml_metal_encoder_t enc = ctx->enc; ++ ggml_metal_buffer_id bid_blk = bid_pad; ++ bid_blk.offs += ggml_metal_op_flash_attn_ext_extra_pad(op); + +- const int32_t hint = ggml_get_op_params_i32(op, 1); + ggml_metal_buffer_id bid_tmp = bid_blk; + bid_tmp.offs += ggml_metal_op_flash_attn_ext_extra_blk(op); +- if (hint == GGML_HINT_SRC0_IS_HADAMARD) { +- if (op->src[1]->type == GGML_TYPE_F32 && +- op->type == GGML_TYPE_F32 && +- ggml_is_contiguous(op->src[1]) && +- ggml_is_contiguous(op) && +- ggml_are_same_shape(op->src[1], op) && +- ggml_metal_fwht_supported_size(op->src[1]->ne[0])) { +- return ggml_metal_op_fwht(ctx, idx); +- } +- } - const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx->dev); + if (!ggml_metal_op_flash_attn_ext_use_vec(op)) { + // half8x8 kernel @@ -18117,7 +18191,7 @@ index c716f118..36378f8e 100644 ggml_tensor * op = ctx->node(idx); ggml_metal_library_t lib = ctx->lib; -@@ -3295,352 +10313,285 @@ int ggml_metal_op_l2_norm(ggml_metal_op_t ctx, int idx) { +@@ -3347,352 +10365,285 @@ int ggml_metal_op_l2_norm(ggml_metal_op_t ctx, int idx) { GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); @@ -18664,7 +18738,7 @@ index c716f118..36378f8e 100644 ggml_tensor * op = ctx->node(idx); ggml_metal_library_t lib = ctx->lib; -@@ -3648,79 +10599,55 @@ int ggml_metal_op_im2col(ggml_metal_op_t ctx, int idx) { +@@ -3700,79 +10651,55 @@ int ggml_metal_op_im2col(ggml_metal_op_t ctx, int idx) { GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); @@ -18776,7 +18850,7 @@ index c716f118..36378f8e 100644 ggml_tensor * op = ctx->node(idx); ggml_metal_library_t lib = ctx->lib; -@@ -3728,77 +10655,63 @@ int ggml_metal_op_conv_2d(ggml_metal_op_t ctx, int idx) { +@@ -3780,77 +10707,63 @@ int ggml_metal_op_conv_2d(ggml_metal_op_t ctx, int idx) { GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); @@ -18893,7 +18967,7 @@ index c716f118..36378f8e 100644 ggml_tensor * op = ctx->node(idx); ggml_metal_library_t lib = ctx->lib; -@@ -3806,150 +10719,103 @@ int ggml_metal_op_conv_2d_dw(ggml_metal_op_t ctx, int idx) { +@@ -3858,150 +10771,103 @@ int ggml_metal_op_conv_2d_dw(ggml_metal_op_t ctx, int idx) { GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); @@ -19096,7 +19170,7 @@ index c716f118..36378f8e 100644 ggml_tensor * op = ctx->node(idx); ggml_metal_library_t lib = ctx->lib; -@@ -3957,134 +10823,113 @@ int ggml_metal_op_conv_transpose_1d(ggml_metal_op_t ctx, int idx) { +@@ -4009,134 +10875,113 @@ int ggml_metal_op_conv_transpose_1d(ggml_metal_op_t ctx, int idx) { GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); @@ -19293,7 +19367,7 @@ index c716f118..36378f8e 100644 ggml_tensor * op = ctx->node(idx); ggml_metal_library_t lib = ctx->lib; -@@ -4092,81 +10937,75 @@ int ggml_metal_op_conv_transpose_2d(ggml_metal_op_t ctx, int idx) { +@@ -4144,81 +10989,75 @@ int ggml_metal_op_conv_transpose_2d(ggml_metal_op_t ctx, int idx) { GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); @@ -19411,7 +19485,7 @@ index c716f118..36378f8e 100644 /*.ne00 =*/ ne00, /*.ne01 =*/ ne01, /*.ne02 =*/ ne02, -@@ -4179,85 +11018,64 @@ int ggml_metal_op_upscale(ggml_metal_op_t ctx, int idx) { +@@ -4231,85 +11070,64 @@ int ggml_metal_op_upscale(ggml_metal_op_t ctx, int idx) { /*.ne1 =*/ ne1, /*.ne2 =*/ ne2, /*.ne3 =*/ ne3, @@ -19536,7 +19610,7 @@ index c716f118..36378f8e 100644 ggml_tensor * op = ctx->node(idx); ggml_metal_library_t lib = ctx->lib; -@@ -4265,50 +11083,146 @@ int ggml_metal_op_pad(ggml_metal_op_t ctx, int idx) { +@@ -4317,50 +11135,146 @@ int ggml_metal_op_pad(ggml_metal_op_t ctx, int idx) { GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); @@ -19714,7 +19788,7 @@ index c716f118..36378f8e 100644 ggml_tensor * op = ctx->node(idx); ggml_metal_library_t lib = ctx->lib; -@@ -4316,79 +11230,318 @@ int ggml_metal_op_pad_reflect_1d(ggml_metal_op_t ctx, int idx) { +@@ -4368,79 +11282,318 @@ int ggml_metal_op_pad_reflect_1d(ggml_metal_op_t ctx, int idx) { GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); @@ -20074,7 +20148,7 @@ index c716f118..36378f8e 100644 ggml_tensor * op = ctx->node(idx); ggml_metal_library_t lib = ctx->lib; -@@ -4396,33 +11549,65 @@ int ggml_metal_op_timestep_embedding(ggml_metal_op_t ctx, int idx) { +@@ -4448,33 +11601,65 @@ int ggml_metal_op_timestep_embedding(ggml_metal_op_t ctx, int idx) { GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); @@ -20154,7 +20228,7 @@ index c716f118..36378f8e 100644 ggml_tensor * op = ctx->node(idx); ggml_metal_library_t lib = ctx->lib; -@@ -4430,144 +11615,234 @@ int ggml_metal_op_argmax(ggml_metal_op_t ctx, int idx) { +@@ -4482,144 +11667,234 @@ int ggml_metal_op_argmax(ggml_metal_op_t ctx, int idx) { GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); @@ -20491,7 +20565,7 @@ index c716f118..36378f8e 100644 ggml_tensor * op = ctx->node(idx); diff --git a/ggml/src/ggml-metal/ggml-metal-ops.h b/ggml/src/ggml-metal/ggml-metal-ops.h -index 89a6ad82..5d22c181 100644 +index 2783ecb8..b31b8e8f 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.h +++ b/ggml/src/ggml-metal/ggml-metal-ops.h @@ -11,6 +11,8 @@ typedef struct ggml_metal_op * ggml_metal_op_t; @@ -20558,7 +20632,7 @@ index a1003b3a..3807bf76 100644 case GGML_OP_ARGSORT: { diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metal -index 969fddfa..b7b7e065 100644 +index f14ee079..2f1fe190 100644 --- a/ggml/src/ggml-metal/ggml-metal.metal +++ b/ggml/src/ggml-metal/ggml-metal.metal @@ -1562,6 +1562,60 @@ typedef decltype(kernel_swiglu) kernel_swiglu_t; @@ -21449,7 +21523,7 @@ index 969fddfa..b7b7e065 100644 typedef void (im2col_t)( constant ggml_metal_kargs_im2col & args, device const float * x, -@@ -6016,136 +6786,624 @@ kernel void kernel_argsort_f32_i32( +@@ -6016,112 +6786,600 @@ kernel void kernel_argsort_f32_i32( template [[host_name("kernel_argsort_f32_i32_asc")]] kernel argsort_t kernel_argsort_f32_i32; template [[host_name("kernel_argsort_f32_i32_desc")]] kernel argsort_t kernel_argsort_f32_i32; @@ -21461,21 +21535,6 @@ index 969fddfa..b7b7e065 100644 - uint3 tgpig[[threadgroup_position_in_grid]], - ushort3 tpitg[[thread_position_in_threadgroup]], - ushort3 ntg[[threads_per_threadgroup]]); -- --template --kernel void kernel_argsort_merge_f32_i32( -- constant ggml_metal_kargs_argsort_merge & args, -- device const char * src0, -- device const int32_t * tmp, -- device int32_t * dst, -- uint3 tgpig[[threadgroup_position_in_grid]], -- ushort3 tpitg[[thread_position_in_threadgroup]], -- ushort3 ntg[[threads_per_threadgroup]]) { -- -- const int im = tgpig[0] / args.ne01; -- const int i01 = tgpig[0] % args.ne01; -- const int i02 = tgpig[1]; -- const int i03 = tgpig[2]; +kernel void kernel_topk_moe_route_f32_i32( + constant ggml_metal_kargs_topk_moe_route & args, + device const char * logits, @@ -21501,7 +21560,15 @@ index 969fddfa..b7b7e065 100644 + return; + } -- const int start = im * (2 * args.len); +-template +-kernel void kernel_argsort_merge_f32_i32( +- constant ggml_metal_kargs_argsort_merge & args, +- device const char * src0, +- device const int32_t * tmp, +- device int32_t * dst, +- uint3 tgpig[[threadgroup_position_in_grid]], +- ushort3 tpitg[[thread_position_in_threadgroup]], +- ushort3 ntg[[threads_per_threadgroup]]) { + float prob = 0.0f; + float score = -INFINITY; + if (tid < args.n_expert) { @@ -21513,21 +21580,21 @@ index 969fddfa..b7b7e065 100644 + probs[tid] = prob; + expert_ids[tid] = tid < args.n_expert ? tid : -1; -- const int len0 = MIN(args.len, MAX(0, args.ne0 - (int)(start))); -- const int len1 = MIN(args.len, MAX(0, args.ne0 - (int)(start + args.len))); +- const int im = tgpig[0] / args.ne01; +- const int i01 = tgpig[0] % args.ne01; +- const int i02 = tgpig[1]; +- const int i03 = tgpig[2]; + threadgroup_barrier(mem_flags::mem_threadgroup); -- const int total = len0 + len1; +- const int start = im * (2 * args.len); + for (int32_t top = 0; top < args.top_k; ++top) { + work_scores[tid] = scores[tid]; + work_probs[tid] = probs[tid]; + work_ids[tid] = expert_ids[tid]; + threadgroup_barrier(mem_flags::mem_threadgroup); -- device const int32_t * tmp0 = tmp + start -- + i01*args.ne0 -- + i02*args.ne0*args.ne01 -- + i03*args.ne0*args.ne01*args.ne02; +- const int len0 = MIN(args.len, MAX(0, args.ne0 - (int)(start))); +- const int len1 = MIN(args.len, MAX(0, args.ne0 - (int)(start + args.len))); + for (int32_t stride = MAX_EXPERTS/2; stride > 0; stride >>= 1) { + if (tid < stride) { + const float lhs_score = work_scores[tid]; @@ -21547,7 +21614,7 @@ index 969fddfa..b7b7e065 100644 + threadgroup_barrier(mem_flags::mem_threadgroup); + } -- device const int32_t * tmp1 = tmp0 + args.len; +- const int total = len0 + len1; + if (tid == 0) { + top_probs[top] = work_probs[0]; + top_ids[top] = work_ids[0]; @@ -21558,10 +21625,10 @@ index 969fddfa..b7b7e065 100644 + threadgroup_barrier(mem_flags::mem_threadgroup); + } -- dst += start -- + i01*args.top_k -- + i02*args.top_k*args.ne01 -- + i03*args.top_k*args.ne01*args.ne02; +- device const int32_t * tmp0 = tmp + start +- + i01*args.ne0 +- + i02*args.ne0*args.ne01 +- + i03*args.ne0*args.ne01*args.ne02; + if (tid == 0) { + const int32_t active_top_k = args._pad1 > 0 ? min(args.top_k, args._pad1) : args.top_k; + float sum = 0.0f; @@ -21572,10 +21639,7 @@ index 969fddfa..b7b7e065 100644 + sum = max(sum, args.clamp_min); + } -- device const float * src0_row = (device const float *)(src0 -- + args.nb01*i01 -- + args.nb02*i02 -- + args.nb03*i03); +- device const int32_t * tmp1 = tmp0 + args.len; + for (int32_t i = 0; i < args.top_k; ++i) { + float weight = i < active_top_k ? top_probs[i] : 0.0f; + if (args.norm) { @@ -21584,8 +21648,10 @@ index 969fddfa..b7b7e065 100644 + top_probs[i] = weight * args.scale; + } -- if (total == 0) { -- return; +- dst += start +- + i01*args.top_k +- + i02*args.top_k*args.ne01 +- + i03*args.top_k*args.ne01*args.ne02; + if (args._pad0 != 0) { + for (int32_t i = 0; i < args.top_k; ++i) { + for (int32_t j = i + 1; j < args.top_k; ++j) { @@ -21603,23 +21669,20 @@ index 969fddfa..b7b7e065 100644 + } + } + } - } ++ } + threadgroup_barrier(mem_flags::mem_threadgroup); -- const int chunk = (total + ntg.x - 1) / ntg.x; -- -- const int k0 = tpitg.x * chunk; -- const int k1 = MIN(MIN(k0 + chunk, total), args.top_k); -- -- if (k0 >= args.top_k) { -- return; +- device const float * src0_row = (device const float *)(src0 +- + args.nb01*i01 +- + args.nb02*i02 +- + args.nb03*i03); + if (tid < args.top_k) { + ((device int32_t *) (ids + tid*args.ids_nb0 + token*args.ids_nb1))[0] = top_ids[tid]; + ((device float *) (weights + tid*args.weights_nb1 + token*args.weights_nb2))[0] = top_probs[tid]; - } ++ } +} -- if (k0 >= total) { +- if (total == 0) { +kernel void kernel_topk_moe_route_f32_i32_sg_reduce( + constant ggml_metal_kargs_topk_moe_route & args, + device const char * logits, @@ -21648,15 +21711,7 @@ index 969fddfa..b7b7e065 100644 return; } -- int low = k0 > len1 ? k0 - len1 : 0; -- int high = MIN(k0, len0); -- -- // binary-search partition (i, j) such that i + j = k -- while (low < high) { -- const int mid = (low + high) >> 1; -- -- const int32_t idx0 = tmp0[mid]; -- const int32_t idx1 = tmp1[k0 - mid - 1]; +- const int chunk = (total + ntg.x - 1) / ntg.x; + float lane_prob = 0.0f; + float lane_score = -INFINITY; + const int32_t expert_id = tid; @@ -21666,8 +21721,8 @@ index 969fddfa..b7b7e065 100644 + lane_score = lane_prob + (args.has_bias ? ((device const float *) (bias + tid*args.bias_nb0))[0] : 0.0f); + } -- const float val0 = src0_row[idx0]; -- const float val1 = src0_row[idx1]; +- const int k0 = tpitg.x * chunk; +- const int k1 = MIN(MIN(k0 + chunk, total), args.top_k); + for (int32_t top = 0; top < args.top_k; ++top) { + const float max_score = simd_max(lane_score); + const int32_t lane_id = lane_score == max_score ? expert_id : MAX_EXPERTS; @@ -21675,38 +21730,30 @@ index 969fddfa..b7b7e065 100644 + const float selected_prob = expert_id == max_id ? lane_prob : 0.0f; + const float max_prob = simd_sum(selected_prob); -- bool take_left; -- if (order == GGML_SORT_ORDER_ASC) { -- take_left = (val0 <= val1); -- } else { -- take_left = (val0 >= val1); +- if (k0 >= args.top_k) { +- return; +- } + if (lane == 0) { + const int32_t candidate = simdgroup*args.top_k + top; + candidate_scores[candidate] = max_score; + candidate_probs[candidate] = max_prob; + candidate_ids[candidate] = max_id == MAX_EXPERTS ? -1 : max_id; - } ++ } -- if (take_left) { -- low = mid + 1; -- } else { -- high = mid; +- if (k0 >= total) { +- return; + if (expert_id == max_id) { + lane_score = -INFINITY; - } ++ } } -- int i = low; -- int j = k0 - i; +- int low = k0 > len1 ? k0 - len1 : 0; +- int high = MIN(k0, len0); + threadgroup_barrier(mem_flags::mem_threadgroup); -- // keep the merge fronts into registers -- int32_t idx0 = 0; -- float val0 = 0.0f; -- if (i < len0) { -- idx0 = tmp0[i]; -- val0 = src0_row[idx0]; -- } +- // binary-search partition (i, j) such that i + j = k +- while (low < high) { +- const int mid = (low + high) >> 1; + if (simdgroup == 0) { + const int32_t n_candidates = N_SIMDGROUPS*args.top_k; + float local_scores[4]; @@ -21726,12 +21773,8 @@ index 969fddfa..b7b7e065 100644 + } + } -- int32_t idx1 = 0; -- float val1 = 0.0f; -- if (j < len1) { -- idx1 = tmp1[j]; -- val1 = src0_row[idx1]; -- } +- const int32_t idx0 = tmp0[mid]; +- const int32_t idx1 = tmp1[k0 - mid - 1]; + for (int32_t top = 0; top < args.top_k; ++top) { + float best_score = local_scores[0]; + float best_prob = local_probs[0]; @@ -21748,41 +21791,35 @@ index 969fddfa..b7b7e065 100644 + } + } -- for (int k = k0; k < k1; ++k) { -- int32_t out_idx; +- const float val0 = src0_row[idx0]; +- const float val1 = src0_row[idx1]; + const float max_score = simd_max(best_score); + const int32_t winner_id = best_score == max_score ? best_id : MAX_EXPERTS; + const int32_t max_id = simd_min(winner_id); + const float selected_prob = best_id == max_id ? best_prob : 0.0f; + const float max_prob = simd_sum(selected_prob); -- if (i >= len0) { -- while (k < k1) { -- dst[k++] = tmp1[j++]; -- } -- break; -- } else if (j >= len1) { -- while (k < k1) { -- dst[k++] = tmp0[i++]; +- bool take_left; +- if (order == GGML_SORT_ORDER_ASC) { +- take_left = (val0 <= val1); +- } else { +- take_left = (val0 >= val1); + if (lane == 0) { + top_probs[top] = max_prob; + top_ids[top] = max_id == MAX_EXPERTS ? -1 : max_id; - } -- break; -- } else { -- bool take_left; - -- if (order == GGML_SORT_ORDER_ASC) { -- take_left = (val0 <= val1); -- } else { -- take_left = (val0 >= val1); ++ } ++ + for (int32_t i = 0; i < 4; ++i) { + if (local_ids[i] == max_id) { + local_scores[i] = -INFINITY; + } + } -+ } -+ + } + +- if (take_left) { +- low = mid + 1; +- } else { +- high = mid; + float sum = 0.0f; + if (lane == 0) { + const int32_t active_top_k = args._pad1 > 0 ? min(args.top_k, args._pad1) : args.top_k; @@ -21823,10 +21860,12 @@ index 969fddfa..b7b7e065 100644 + ((device int32_t *) (ids + i*args.ids_nb0 + token*args.ids_nb1))[0] = top_ids[i]; + ((device float *) (weights + i*args.weights_nb1 + token*args.weights_nb2))[0] = top_probs[i]; + } -+ } -+ } + } + } +} -+ + +- int i = low; +- int j = k0 - i; +kernel void kernel_topk_moe_route_glm_256_8_sg32( + constant ggml_metal_kargs_topk_moe_route & args, + device const char * logits, @@ -21840,14 +21879,23 @@ index 969fddfa..b7b7e065 100644 + constexpr int32_t TOP_K = 8; + constexpr int32_t LANES = 32; + constexpr int32_t EXPERTS_PER_LANE = N_EXPERTS/LANES; -+ + +- // keep the merge fronts into registers +- int32_t idx0 = 0; +- float val0 = 0.0f; +- if (i < len0) { +- idx0 = tmp0[i]; +- val0 = src0_row[idx0]; + const int32_t token = tgpig.x; + const int32_t lane = tiisg; + + if (token >= args.n_tokens || args.n_expert != N_EXPERTS || args.top_k != TOP_K || tpitg.x >= LANES) { + return; -+ } -+ + } + +- int32_t idx1 = 0; +- float val1 = 0.0f; +- if (j < len1) { + float local_scores[EXPERTS_PER_LANE]; + float local_probs[EXPERTS_PER_LANE]; + int32_t local_ids[EXPERTS_PER_LANE]; @@ -22149,34 +22197,10 @@ index 969fddfa..b7b7e065 100644 + int32_t idx1 = 0; + float val1 = 0.0f; + if (j < len1) { -+ idx1 = tmp1[j]; -+ val1 = src0_row[idx1]; -+ } -+ -+ for (int k = k0; k < k1; ++k) { -+ int32_t out_idx; -+ -+ if (i >= len0) { -+ while (k < k1) { -+ dst[k++] = tmp1[j++]; -+ } -+ break; -+ } else if (j >= len1) { -+ while (k < k1) { -+ dst[k++] = tmp0[i++]; -+ } -+ break; -+ } else { -+ bool take_left; -+ -+ if (order == GGML_SORT_ORDER_ASC) { -+ take_left = (val0 <= val1); -+ } else { -+ take_left = (val0 >= val1); - } - - if (take_left) { -@@ -7797,14 +9055,20 @@ kernel void kernel_flash_attn_ext_vec_reduce( + idx1 = tmp1[j]; + val1 = src0_row[idx1]; + } +@@ -7859,14 +9117,20 @@ kernel void kernel_flash_attn_ext_vec_reduce( const uint64_t rid = tgpig; const short iwg = tiisg; @@ -22200,7 +22224,7 @@ index 969fddfa..b7b7e065 100644 S = simd_sum(S*ms); S = S == 0.0f ? 0.0f : 1.0f/S; -@@ -7815,7 +9079,11 @@ kernel void kernel_flash_attn_ext_vec_reduce( +@@ -7877,7 +9141,11 @@ kernel void kernel_flash_attn_ext_vec_reduce( device float4 * dst4 = (device float4 *) dst + rid*DV4; for (short i = sgitg; i < DV4; i += NWG) { @@ -22213,7 +22237,7 @@ index 969fddfa..b7b7e065 100644 if (iwg == 0) { dst4[i] = v*S; -@@ -8026,7 +9294,7 @@ template [[host_name("kernel_concat_i16")]] kernel kernel_concat_t kernel_conca +@@ -8088,7 +9356,7 @@ template [[host_name("kernel_concat_i16")]] kernel kernel_concat_t kernel_conca template [[host_name("kernel_concat_i32")]] kernel kernel_concat_t kernel_concat; template [[host_name("kernel_concat_i64")]] kernel kernel_concat_t kernel_concat; @@ -22222,7 +22246,7 @@ index 969fddfa..b7b7e065 100644 void kernel_mul_mv_q2_K_f32_impl( args_t args, device const char * src0, -@@ -8038,7 +9306,7 @@ void kernel_mul_mv_q2_K_f32_impl( +@@ -8100,7 +9368,7 @@ void kernel_mul_mv_q2_K_f32_impl( ushort sgitg) { const short NSG = FC_mul_mv_nsg; @@ -22231,7 +22255,7 @@ index 969fddfa..b7b7e065 100644 const int r0 = tgpig.x; const int r1 = tgpig.y; -@@ -8131,299 +9399,615 @@ kernel void kernel_mul_mv_q2_K_f32( +@@ -8193,160 +9461,4638 @@ kernel void kernel_mul_mv_q2_K_f32( kernel_mul_mv_q2_K_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); } @@ -22280,14 +22304,24 @@ index 969fddfa..b7b7e065 100644 - //const uint16_t kmask1 = 0x3030; - //const uint16_t kmask2 = 0x0f0f; -- ++ const short ix = tiisg/8; ++ const short it = tiisg%8; ++ const short iq = it/4; ++ const short ir = it%4; ++ const short is = (8*ir)/16; + - const short tid = tiisg/4; - const short ix = tiisg%4; - const short ip = tid/4; // 0 or 1 - const short il = 2*((tid%4)/2); // 0 or 2 - const short ir = tid%2; - const short l0 = 8*ir; -- ++ if (extra._pad0 != 0) { ++ const int32_t expert_id = ((device const int32_t *) (ids + token*args.nbi1))[0]; ++ if (expert_id < 0 || expert_id >= args.ne02) { ++ return; ++ } + - // One would think that the Metal compiler would figure out that ip and il can only have - // 4 possible states, and optimize accordingly. Well, no. It needs help, and we do it - // with these two tales. @@ -22297,41 +22331,6 @@ index 969fddfa..b7b7e065 100644 - {0x0004, 0x0400, 0x0008, 0x0800}, // ip = 0, il = 2 - {0x0010, 0x1000, 0x0020, 0x2000}, // ip = 1, il = 0 - {0x0040, 0x4000, 0x0080, 0x8000}}; // ip = 1, il = 2 -- -- // Possible masks for the low 2 bits -- const int4 qm[2] = {{0x0003, 0x0300, 0x000c, 0x0c00}, {0x0030, 0x3000, 0x00c0, 0xc000}}; -- -- const ushort4 hm = mm[2*ip + il/2]; -- -- const short shift = 2*il; -- -- const float v1 = il == 0 ? 4.f : 64.f; -- const float v2 = 4.f * v1; -- -- const uint16_t s_shift1 = 4*ip; -- const uint16_t s_shift2 = s_shift1 + il; -- -- const short q_offset = 32*ip + l0; -- const short y_offset = 128*ip + 32*il + l0; -+ const short ix = tiisg/8; -+ const short it = tiisg%8; -+ const short iq = it/4; -+ const short ir = it%4; -+ const short is = (8*ir)/16; - -- device const float * y1 = yy + ix*QK_K + y_offset; -- -- uint32_t scales32, aux32; -- thread uint16_t * scales16 = (thread uint16_t *)&scales32; -- thread const int8_t * scales = (thread const int8_t *)&scales32; -+ if (extra._pad0 != 0) { -+ const int32_t expert_id = ((device const int32_t *) (ids + token*args.nbi1))[0]; -+ if (expert_id < 0 || expert_id >= args.ne02) { -+ return; -+ } - -- float sumf1[nr0] = {0.f}; -- float sumf2[nr0] = {0.f}; + device const char * src0_cur = src0s + uint64_t(expert_id)*args.nb02; + device const char * src1_cur = src1 + uint64_t(token)*args.nb12; + ggml_metal_kargs_mul_mv args0 = { @@ -22367,52 +22366,33 @@ index 969fddfa..b7b7e065 100644 + return; + } -- for (int i = ix; i < nb; i += 4) { -- for (short l = 0; l < 8; ++l) { -- yl[l+ 0] = y1[l+ 0]; -- yl[l+ 8] = y1[l+16]; -- yl[l+16] = y1[l+32]; -- yl[l+24] = y1[l+48]; +- // Possible masks for the low 2 bits +- const int4 qm[2] = {{0x0003, 0x0300, 0x000c, 0x0c00}, {0x0030, 0x3000, 0x00c0, 0xc000}}; + for (int slot = 0; slot < args.nei0; ++slot) { + const int32_t expert_id = ((device const int32_t *) (ids + token*args.nbi1))[slot]; + if (expert_id < 0 || expert_id >= args.ne02) { + continue; - } ++ } -- device const uint16_t * q = (device const uint16_t *)(x[i].qs + q_offset); -- device const uint16_t * h = (device const uint16_t *)(x[i].hmask + l0); -- device const uint16_t * a = (device const uint16_t *)(x[i].scales); -- device const half * dh = &x[i].d; +- const ushort4 hm = mm[2*ip + il/2]; + const float route_weight = extra.already_weighted ? 1.0f : + ((device const float *) (weights + slot*extra.weights_nb1 + token*extra.weights_nb2))[0]; + if (route_weight == 0.0f) { + continue; + } -- for (short row = 0; row < nr0; ++row) { -- const float d_all = (float)dh[0]; +- const short shift = 2*il; + const uint64_t offset0 = uint64_t(expert_id)*args.nb02 + uint64_t(first_row)*args.nb01; + const uint64_t offset1 = uint64_t(slot)*args.nb11 + uint64_t(token)*args.nb12; -- scales16[0] = a[4]; -- scales16[1] = a[5]; -- aux32 = ((scales32 >> s_shift2) << 4) & 0x30303030; -- scales16[0] = a[il+0]; -- scales16[1] = a[il+1]; -- scales32 = ((scales32 >> s_shift1) & 0x0f0f0f0f) | aux32; +- const float v1 = il == 0 ? 4.f : 64.f; +- const float v2 = 4.f * v1; + device const block_q2_K * x = (device const block_q2_K *) (src0s + offset0); + device const float * y = (device const float *) (src1 + offset1); + device const float * y4 = y + ix * QK_K + 128 * iq + 8 * ir; -- float s1 = 0, s2 = 0, s3 = 0, s4 = 0, s5 = 0, s6 = 0; -- for (short l = 0; l < 8; l += 2) { -- const int32_t qs = q[l/2]; -- s1 += yl[l+0] * (qs & qm[il/2][0]); -- s2 += yl[l+1] * (qs & qm[il/2][1]); -- s3 += ((h[l/2] & hm[0]) ? 0.f : yl[l+0]) + ((h[l/2] & hm[1]) ? 0.f : yl[l+1]); -- s4 += yl[l+16] * (qs & qm[il/2][2]); -- s5 += yl[l+17] * (qs & qm[il/2][3]); -- s6 += ((h[l/2] & hm[2]) ? 0.f : yl[l+16]) + ((h[l/2] & hm[3]) ? 0.f : yl[l+17]); +- const uint16_t s_shift1 = 4*ip; +- const uint16_t s_shift2 = s_shift1 + il; + for (int ib = ix; ib < nb; ib += 4) { + float4 sumy = {0.f, 0.f, 0.f, 0.f}; + for (short i = 0; i < 8; ++i) { @@ -22420,21 +22400,10 @@ index 969fddfa..b7b7e065 100644 + yl[i+ 8] = y4[i+32]; sumy[1] += yl[i+ 8]; + yl[i+16] = y4[i+64]; sumy[2] += yl[i+16]; + yl[i+24] = y4[i+96]; sumy[3] += yl[i+24]; - } -- float d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1); -- float d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2); -- sumf1[row] += d1 * (scales[0] - 32); -- sumf2[row] += d2 * (scales[2] - 32); ++ } -- s1 = s2 = s3 = s4 = s5 = s6 = 0; -- for (short l = 0; l < 8; l += 2) { -- const int32_t qs = q[l/2+8]; -- s1 += yl[l+8] * (qs & qm[il/2][0]); -- s2 += yl[l+9] * (qs & qm[il/2][1]); -- s3 += ((h[l/2+8] & hm[0]) ? 0.f : yl[l+8]) + ((h[l/2+8] & hm[1]) ? 0.f : yl[l+9]); -- s4 += yl[l+24] * (qs & qm[il/2][2]); -- s5 += yl[l+25] * (qs & qm[il/2][3]); -- s6 += ((h[l/2+8] & hm[2]) ? 0.f : yl[l+24]) + ((h[l/2+8] & hm[3]) ? 0.f : yl[l+25]); +- const short q_offset = 32*ip + l0; +- const short y_offset = 128*ip + 32*il + l0; + device const uint8_t * sc = (device const uint8_t *)x[ib].scales + 8*iq + is; + device const uint16_t * qs = (device const uint16_t *)x[ib].qs + 16 * iq + 4 * ir; + device const half * dh = &x[ib].d; @@ -22466,112 +22435,38 @@ index 969fddfa..b7b7e065 100644 + qs += args.nb01/2; + sc += args.nb01; + dh += args.nb01/2; - } -- d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1); -- d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2); -- sumf1[row] += d1 * (scales[1] - 32); -- sumf2[row] += d2 * (scales[3] - 32); ++ } -- q += args.nb01/2; -- h += args.nb01/2; -- a += args.nb01/2; -- dh += args.nb01/2; +- device const float * y1 = yy + ix*QK_K + y_offset; + y4 += 4 * QK_K; - } -- -- y1 += 4 * QK_K; -- } -- -- for (int row = 0; row < nr0; ++row) { -- const float sumf = (sumf1[row] + 0.25f * sumf2[row]) / (1 << shift); -- sumf1[row] = simd_sum(sumf); - } ++ } ++ } -- device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; -- -- if (tiisg == 0) { -- for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) { -- dst_f32[first_row + row] = sumf1[row]; + for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) { + float sum_all = simd_sum(sumf[row]); + if (tiisg == 0) { + ((device float *) (dst + uint64_t(first_row + row)*extra.dst_nb0 + uint64_t(token)*extra.dst_nb1))[0] = sum_all; - } - } - } - --[[host_name("kernel_mul_mv_q3_K_f32")]] --kernel void kernel_mul_mv_q3_K_f32( -- constant ggml_metal_kargs_mul_mv & args, -- device const char * src0, ++ } ++ } ++} ++ +template +void kernel_mul_mv_id_q2_K_weighted_reduce_slots_sg_impl( + constant ggml_metal_kargs_mul_mv_id & args, + device const char * src0s, - device const char * src1, - device char * dst, -+ device const char * ids, -+ device const char * weights, -+ constant ggml_metal_kargs_mul_mv_id_weighted_reduce_extra & extra, -+ threadgroup float * partials [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]]) { -- -- kernel_mul_mv_q3_K_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); --} -- --template --void kernel_mul_mv_q4_K_f32_impl( -- args_t args, -- device const char * src0, -- device const char * src1, -- device char * dst, -- threadgroup char * shmem, -- uint3 tgpig, -- ushort tiisg, -- ushort sgitg) { -- const short NSG = FC_mul_mv_nsg; -- -- constexpr uint16_t kmask1 = 0x3f3f; -- constexpr uint16_t kmask2 = 0x0f0f; -- constexpr uint16_t kmask3 = 0xc0c0; -- -- const short ix = tiisg/8; // 0...3 -- const short it = tiisg%8; // 0...7 -- const short iq = it/4; // 0 or 1 -- const short ir = it%4; // 0...3 -- -+ constexpr int n_slots = 8; - const int nb = args.ne00/QK_K; - - const int r0 = tgpig.x; -- const int r1 = tgpig.y; -- const int im = tgpig.z; -- -- const int first_row = (r0 * NSG + sgitg) * nr0; -- -- const uint i12 = im%FC_mul_mv_ne12; -- const uint i13 = im/FC_mul_mv_ne12; -- -- const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; -- const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; -- -- device const block_q4_K * x = (device const block_q4_K *) (src0 + offset0); -- device const float * y = (device const float *) (src1 + offset1); -- -- float yl[16]; -- float yh[16]; -- -- float sumf[nr0]={0.f}; -- -- device const float * y4 = y + ix * QK_K + 64 * iq + 8 * ir; -- -- uint16_t sc16[4]; -- thread const uint8_t * sc8 = (thread const uint8_t *)sc16; -- -- for (int ib = ix; ib < nb; ib += 4) { -- float4 sumy = {0.f, 0.f, 0.f, 0.f}; ++ device const char * src1, ++ device char * dst, ++ device const char * ids, ++ device const char * weights, ++ constant ggml_metal_kargs_mul_mv_id_weighted_reduce_extra & extra, ++ threadgroup float * partials [[threadgroup(0)]], ++ uint3 tgpig[[threadgroup_position_in_grid]], ++ ushort tiisg[[thread_index_in_simdgroup]], ++ ushort sgitg[[simdgroup_index_in_threadgroup]]) { ++ constexpr int n_slots = 8; ++ const int nb = args.ne00/QK_K; ++ ++ const int r0 = tgpig.x; + const int token = tgpig.y; + const int slot = sgitg; + const int first_row = r0 * nr0; @@ -22604,12 +22499,7 @@ index 969fddfa..b7b7e065 100644 + } + } + } - -- for (short i = 0; i < 8; ++i) { -- yl[i+0] = y4[i+ 0]; sumy[0] += yl[i+0]; -- yl[i+8] = y4[i+ 32]; sumy[1] += yl[i+8]; -- yh[i+0] = y4[i+128]; sumy[2] += yh[i+0]; -- yh[i+8] = y4[i+160]; sumy[3] += yh[i+8]; ++ + if (tiisg == 0) { + for (int row = 0; row < nr0; ++row) { + partials[slot*nr0 + row] = first_row + row < args.ne0 ? simd_sum(scan_sum[row]) : 0.0f; @@ -22622,24 +22512,16 @@ index 969fddfa..b7b7e065 100644 + sum += partials[s*nr0 + tiisg]; + } + ((device float *) (dst + uint64_t(first_row + tiisg)*extra.dst_nb0 + uint64_t(token)*extra.dst_nb1))[0] = sum; - } ++ } + return; + } - -- device const uint16_t * sc = (device const uint16_t *)x[ib].scales + iq; -- device const uint16_t * q1 = (device const uint16_t *)x[ib].qs + 16 * iq + 4 * ir; -- device const half * dh = &x[ib].d; ++ + float sumf[nr0]; + for (int row = 0; row < nr0; ++row) { + sumf[row] = 0.0f; + } + float yl[32]; - -- for (short row = 0; row < nr0; row++) { -- sc16[0] = sc[0] & kmask1; -- sc16[1] = sc[2] & kmask1; -- sc16[2] = ((sc[4] >> 0) & kmask2) | ((sc[0] & kmask3) >> 2); -- sc16[3] = ((sc[4] >> 4) & kmask2) | ((sc[2] & kmask3) >> 2); ++ + const short ix = tiisg/8; + const short it = tiisg%8; + const short iq = it/4; @@ -22668,8 +22550,7 @@ index 969fddfa..b7b7e065 100644 + yl[i+16] = float(y4[i+64]); sumy[2] += yl[i+16]; + yl[i+24] = float(y4[i+96]); sumy[3] += yl[i+24]; + } - -- device const uint16_t * q2 = q1 + 32; ++ + device const uint8_t * sc = (device const uint8_t *)x[ib].scales + 8*iq + is; + device const uint16_t * qs = (device const uint16_t *)x[ib].qs + 16 * iq + 4 * ir; + device const half * dh = &x[ib].d; @@ -22722,87 +22603,56 @@ index 969fddfa..b7b7e065 100644 + sumy[2] * (sc[4] & 0xF0) + sumy[3] * (sc[6] & 0xF0)); + } + sumf[row] += route_weight * dot_val; - -- float4 acc1 = {0.f, 0.f, 0.f, 0.f}; -- float4 acc2 = {0.f, 0.f, 0.f, 0.f}; ++ + qs += args.nb01/2; + sc += args.nb01; + dh += args.nb01/2; + } - -- FOR_UNROLL (short i = 0; i < 4; ++i) { -- acc1[0] += yl[2*i + 0] * (q1[i] & 0x000F); -- acc1[1] += yl[2*i + 1] * (q1[i] & 0x0F00); -- acc1[2] += yl[2*i + 8] * (q1[i] & 0x00F0); -- acc1[3] += yl[2*i + 9] * (q1[i] & 0xF000); -- acc2[0] += yh[2*i + 0] * (q2[i] & 0x000F); -- acc2[1] += yh[2*i + 1] * (q2[i] & 0x0F00); -- acc2[2] += yh[2*i + 8] * (q2[i] & 0x00F0); -- acc2[3] += yh[2*i + 9] * (q2[i] & 0xF000); ++ + y4 += 4 * QK_K; + } - } ++ } + } + } - -- sumf[row] += dh[0] * ((acc1[0] + 1.f/256.f * acc1[1]) * sc8[0] + -- (acc1[2] + 1.f/256.f * acc1[3]) * sc8[1] * 1.f/16.f + -- (acc2[0] + 1.f/256.f * acc2[1]) * sc8[4] + -- (acc2[2] + 1.f/256.f * acc2[3]) * sc8[5] * 1.f/16.f) - -- dh[1] * (sumy[0] * sc8[2] + sumy[1] * sc8[3] + sumy[2] * sc8[6] + sumy[3] * sc8[7]); ++ + float reduced[nr0]; + for (int row = 0; row < nr0; ++row) { + reduced[row] = simd_sum(sumf[row]); + } - -- q1 += args.nb01/2; -- sc += args.nb01/2; -- dh += args.nb01/2; ++ + if (tiisg == 0) { + for (int row = 0; row < nr0; ++row) { + partials[slot*nr0 + row] = first_row + row < args.ne0 ? reduced[row] : 0.0f; - } -- -- y4 += 4 * QK_K; - } - -- device float * dst_f32 = (device float *) dst + (int64_t)im*args.ne0*args.ne1 + (int64_t)r1*args.ne0; ++ } ++ } ++ + threadgroup_barrier(mem_flags::mem_threadgroup); - -- for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) { -- float sum_all = simd_sum(sumf[row]); -- if (tiisg == 0) { -- dst_f32[first_row + row] = sum_all; ++ + if (sgitg == 0 && tiisg < nr0 && first_row + tiisg < args.ne0) { + float sum = 0.0f; + for (int s = 0; s < n_slots; ++s) { + sum += partials[s*nr0 + tiisg]; - } ++ } + ((device float *) (dst + uint64_t(first_row + tiisg)*extra.dst_nb0 + uint64_t(token)*extra.dst_nb1))[0] = sum; - } - } - --[[host_name("kernel_mul_mv_q4_K_f32")]] --kernel void kernel_mul_mv_q4_K_f32( -- constant ggml_metal_kargs_mul_mv & args, -- device const char * src0, ++ } ++} ++ +kernel void kernel_mul_mv_id_q2_K_weighted_reduce_slots_sg_r8_nb8( + constant ggml_metal_kargs_mul_mv_id & args, + device const char * src0s, - device const char * src1, - device char * dst, ++ device const char * src1, ++ device char * dst, + device const char * ids, + device const char * weights, + constant ggml_metal_kargs_mul_mv_id_weighted_reduce_extra & extra, + threadgroup float * partials [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]]) { ++ uint3 tgpig[[threadgroup_position_in_grid]], ++ ushort tiisg[[thread_index_in_simdgroup]], ++ ushort sgitg[[simdgroup_index_in_threadgroup]]) { + kernel_mul_mv_id_q2_K_weighted_reduce_slots_sg_impl<8>( + args, src0s, src1, dst, ids, weights, extra, partials, tgpig, tiisg, sgitg); +} - -- kernel_mul_mv_q4_K_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); ++ +kernel void kernel_mul_mv_id_q2_K_weighted_reduce_slots_sg_r8_nb8_f16( + constant ggml_metal_kargs_mul_mv_id & args, + device const char * src0s, @@ -22817,18 +22667,13 @@ index 969fddfa..b7b7e065 100644 + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + kernel_mul_mv_id_q2_K_weighted_reduce_slots_sg_impl<8, half>( + args, src0s, src1, dst, ids, weights, extra, partials, tgpig, tiisg, sgitg); - } - --template --void kernel_mul_mv_q5_K_f32_impl( -- args_t args, -- device const char * src0, ++} ++ +kernel void kernel_mul_mv_id_q2_K_weighted_reduce_slots_sg_r8_nb8_shifted( + constant ggml_metal_kargs_mul_mv_id & args, + device const char * src0s, - device const char * src1, - device char * dst, -- threadgroup char * shmem, ++ device const char * src1, ++ device char * dst, + device const char * ids, + device const char * weights, + constant ggml_metal_kargs_mul_mv_id_weighted_reduce_extra & extra, @@ -23072,102 +22917,57 @@ index 969fddfa..b7b7e065 100644 + device const char * src1, + device char * dst, + float route_weight, - uint3 tgpig, - ushort tiisg, - ushort sgitg) { -@@ -8443,1377 +10027,6793 @@ void kernel_mul_mv_q5_K_f32_impl( - const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; - -- device const block_q5_K * x = (device const block_q5_K *) (src0 + offset0); -- device const float * yy = (device const float *) (src1 + offset1); -- -- float sumf[nr0]={0.f}; -- -- float yl[16], yh[16]; -- -- constexpr uint16_t kmask1 = 0x3f3f; -- constexpr uint16_t kmask2 = 0x0f0f; -- constexpr uint16_t kmask3 = 0xc0c0; -- -- const short tid = tiisg/4; -- const short ix = tiisg%4; -- const short iq = tid/4; -- const short ir = tid%4; -- -- const short l0 = 8*ir; -- const short q_offset = 32*iq + l0; -- const short y_offset = 64*iq + l0; -- -- const uint8_t hm1 = 1u << (2*iq); -- const uint8_t hm2 = hm1 << 1; -- const uint8_t hm3 = hm1 << 4; -- const uint8_t hm4 = hm2 << 4; ++ uint3 tgpig, ++ ushort tiisg, ++ ushort sgitg) { ++ const short NSG = FC_mul_mv_nsg; ++ ++ const int nb = args.ne00/QK_K; ++ ++ const int r0 = tgpig.x; ++ const int r1 = tgpig.y; ++ const int im = tgpig.z; ++ ++ const int first_row = (r0 * NSG + sgitg) * nr0; ++ ++ const uint i12 = im%FC_mul_mv_ne12; ++ const uint i13 = im/FC_mul_mv_ne12; ++ ++ const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; ++ const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; ++ + device const block_q2_K * x_up = (device const block_q2_K *) (src0_up + offset0); + device const block_q2_K * x_gate = (device const block_q2_K *) (src0_gate + offset0); + device const float * y = (device const float *) (src1 + offset1); - -- uint16_t sc16[4]; -- thread const uint8_t * sc8 = (thread const uint8_t *)sc16; ++ + float yl[32]; + float sum_up[nr0] = {0.f}; + float sum_gate[nr0] = {0.f}; - -- device const float * y1 = yy + ix*QK_K + y_offset; ++ + const short ix = tiisg/8; + const short it = tiisg%8; + const short iq = it/4; + const short ir = it%4; + const short is = (8*ir)/16; - -- for (int i = ix; i < nb; i += 4) { -- device const uint8_t * q1 = x[i].qs + q_offset; -- device const uint8_t * qh = x[i].qh + l0; -- device const half * dh = &x[i].d; -- device const uint16_t * a = (device const uint16_t *)x[i].scales + iq; ++ + device const float * y4 = y + ix * QK_K + 128 * iq + 8 * ir; - -- device const float * y2 = y1 + 128; ++ + for (int ib = ix; ib < nb; ib += 4) { - float4 sumy = {0.f, 0.f, 0.f, 0.f}; -- for (short l = 0; l < 8; ++l) { -- yl[l+0] = y1[l+ 0]; sumy[0] += yl[l+0]; -- yl[l+8] = y1[l+32]; sumy[1] += yl[l+8]; -- yh[l+0] = y2[l+ 0]; sumy[2] += yh[l+0]; -- yh[l+8] = y2[l+32]; sumy[3] += yh[l+8]; ++ float4 sumy = {0.f, 0.f, 0.f, 0.f}; + for (short i = 0; i < 8; ++i) { + yl[i+ 0] = y4[i+ 0]; sumy[0] += yl[i+ 0]; + yl[i+ 8] = y4[i+32]; sumy[1] += yl[i+ 8]; + yl[i+16] = y4[i+64]; sumy[2] += yl[i+16]; + yl[i+24] = y4[i+96]; sumy[3] += yl[i+24]; - } - -- for (short row = 0; row < nr0; ++row) { -- device const uint8_t * q2 = q1 + 64; -- -- sc16[0] = a[0] & kmask1; -- sc16[1] = a[2] & kmask1; -- sc16[2] = ((a[4] >> 0) & kmask2) | ((a[0] & kmask3) >> 2); -- sc16[3] = ((a[4] >> 4) & kmask2) | ((a[2] & kmask3) >> 2); ++ } ++ + device const uint8_t * sc_up = (device const uint8_t *)x_up[ib].scales + 8*iq + is; + device const uint16_t * qs_up = (device const uint16_t *)x_up[ib].qs + 16*iq + 4*ir; + device const half * dh_up = &x_up[ib].d; + device const uint8_t * sc_gate = (device const uint8_t *)x_gate[ib].scales + 8*iq + is; + device const uint16_t * qs_gate = (device const uint16_t *)x_gate[ib].qs + 16*iq + 4*ir; + device const half * dh_gate = &x_gate[ib].d; - -- float4 acc1 = {0.f}; -- float4 acc2 = {0.f}; -- FOR_UNROLL (short l = 0; l < 8; ++l) { -- uint8_t h = qh[l]; -- acc1[0] += yl[l+0] * (q1[l] & 0x0F); -- acc1[1] += yl[l+8] * (q1[l] & 0xF0); -- acc1[2] += yh[l+0] * (q2[l] & 0x0F); -- acc1[3] += yh[l+8] * (q2[l] & 0xF0); -- acc2[0] += h & hm1 ? yl[l+0] : 0.f; -- acc2[1] += h & hm2 ? yl[l+8] : 0.f; -- acc2[2] += h & hm3 ? yh[l+0] : 0.f; -- acc2[3] += h & hm4 ? yh[l+8] : 0.f; ++ + for (short row = 0; row < nr0; row++) { + float4 acc1_up = {0.f, 0.f, 0.f, 0.f}; + float4 acc2_up = {0.f, 0.f, 0.f, 0.f}; @@ -23234,115 +23034,65 @@ index 969fddfa..b7b7e065 100644 + (acc1_gate[2] + 1.f/256.f * acc2_gate[2]) * (sc_gate[4] & 0xF) * 1.f/16.f + + (acc1_gate[3] + 1.f/256.f * acc2_gate[3]) * (sc_gate[6] & 0xF) * 1.f/64.f) - + dmin_gate * (sumy[0] * (sc_gate[0] & 0xF0) + sumy[1] * (sc_gate[2] & 0xF0) + sumy[2] * (sc_gate[4] & 0xF0) + sumy[3] * (sc_gate[6] & 0xF0)); - } - -- sumf[row] += dh[0] * (sc8[0] * (acc1[0] + 16.f*acc2[0]) + -- sc8[1] * (acc1[1]/16.f + 16.f*acc2[1]) + -- sc8[4] * (acc1[2] + 16.f*acc2[2]) + -- sc8[5] * (acc1[3]/16.f + 16.f*acc2[3])) - -- dh[1] * (sumy[0] * sc8[2] + sumy[1] * sc8[3] + sumy[2] * sc8[6] + sumy[3] * sc8[7]); -- -- q1 += args.nb01; -- qh += args.nb01; -- dh += args.nb01/2; -- a += args.nb01/2; ++ } ++ + qs_up += args.nb01/2; + sc_up += args.nb01; + dh_up += args.nb01/2; + qs_gate += args.nb01/2; + sc_gate += args.nb01; + dh_gate += args.nb01/2; - } - -- y1 += 4 * QK_K; ++ } ++ + y4 += 4 * QK_K; - } - - device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; - - for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) { -- const float tot = simd_sum(sumf[row]); ++ } ++ ++ device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; ++ ++ for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) { + const float up_value = simd_sum(sum_up[row]); -+ const float gate_value = simd_sum(sum_gate[row]); - if (tiisg == 0) { -- dst_f32[first_row + row] = tot; -+ const float silu = gate_value / (1.0f + exp(-gate_value)); -+ dst_f32[first_row + row] = up_value * silu * route_weight; - } - } - } - --[[host_name("kernel_mul_mv_q5_K_f32")]] --kernel void kernel_mul_mv_q5_K_f32( -- constant ggml_metal_kargs_mul_mv & args, -- device const char * src0, ++ const float gate_value = simd_sum(sum_gate[row]); ++ if (tiisg == 0) { ++ const float silu = gate_value / (1.0f + exp(-gate_value)); ++ dst_f32[first_row + row] = up_value * silu * route_weight; ++ } ++ } ++} ++ +kernel void kernel_mul_mv_id_q2_K_gate_up_swiglu( + constant ggml_metal_kargs_mul_mv_id_gate_up_swiglu & args, + device const char * src0_up_all, + device const char * src0_gate_all, - device const char * src1, - device char * dst, ++ device const char * src1, ++ device char * dst, + device const char * ids, + device const char * weights, + threadgroup char * shmem [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], ++ uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiitg[[thread_index_in_threadgroup]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]]) { ++ ushort tiisg[[thread_index_in_simdgroup]], ++ ushort sgitg[[simdgroup_index_in_threadgroup]]) { + const int iid1 = tgpig.z/args.nei0; + const int idx = tgpig.z%args.nei0; - -- kernel_mul_mv_q5_K_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); --} ++ + tgpig.z = 0; - --template --void kernel_mul_mv_q6_K_f32_impl( -- args_t args, -- device const char * src0, -- device const char * src1, -- device char * dst, -- threadgroup char * shmem, -- uint3 tgpig, -- ushort tiisg, -- ushort sgitg) { -- const short NSG = FC_mul_mv_nsg; ++ + const int32_t i02 = ((device const int32_t *) (ids + iid1*args.nbi1))[idx]; + const float route_weight = args.weighted != 0 ? + ((device const float *) (weights + uint64_t(idx)*args.weights_nb1 + uint64_t(iid1)*args.weights_nb2))[0] : + 1.0f; - -- constexpr uint8_t kmask1 = 0x03; -- constexpr uint8_t kmask2 = 0x0C; -- constexpr uint8_t kmask3 = 0x30; -- constexpr uint8_t kmask4 = 0xC0; ++ + const int64_t i11 = idx % args.ne11; + const int64_t i12 = iid1; - -- const int nb = args.ne00/QK_K; ++ + const int64_t i1 = idx; + const int64_t i2 = i12; - -- const int r0 = tgpig.x; -- const int r1 = tgpig.y; -- const int im = tgpig.z; -- -- const int first_row = (r0 * NSG + sgitg) * nr0; -- -- const uint i12 = im%FC_mul_mv_ne12; -- const uint i13 = im/FC_mul_mv_ne12; -- -- const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; -- const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; -- -- device const block_q6_K * x = (device const block_q6_K *) (src0 + offset0); -- device const float * yy = (device const float *) (src1 + offset1); ++ + device const char * src0_up_cur = src0_up_all + i02*args.nb02; + device const char * src0_gate_cur = src0_gate_all + i02*args.nb02; + device const char * src1_cur = src1 + i11*args.nb11 + i12*args.nb12; + device char * dst_cur = dst + (i1*args.ne0 + i2*args.ne1*args.ne0)*sizeof(float); - -- float sumf[nr0] = { 0.f }; ++ + ggml_metal_kargs_mul_mv args0 = { + /*.ne00 =*/ args.ne00, + /*.ne01 =*/ args.ne01, @@ -23364,8 +23114,7 @@ index 969fddfa..b7b7e065 100644 + /*.r2 =*/ 1, + /*.r3 =*/ 1, + }; - -- float yl[16]; ++ + switch (args.nr0) { + case 1: + kernel_mul_mv_q2_K_gate_up_swiglu_impl<1>( @@ -23388,20 +23137,11 @@ index 969fddfa..b7b7e065 100644 + args0, src0_up_cur, src0_gate_cur, src1_cur, dst_cur, route_weight, tgpig, tiisg, sgitg); + break; + } - -- const short tid = tiisg/2; -- const short ix = tiisg%2; -- const short ip = tid/8; // 0 or 1 -- const short il = tid%8; -- const short l0 = 4*il; -- const short is = 8*ip + l0/16; ++ + (void) shmem; + (void) tiitg; +} - -- const short y_offset = 128*ip + l0; -- const short q_offset_l = 64*ip + l0; -- const short q_offset_h = 32*ip + l0; ++ +kernel void kernel_mul_mv_id_q2_K_gate_up_swiglu_vecscale( + constant ggml_metal_kargs_mul_mv_id_gate_up_swiglu & args, + device const char * src0_up_all, @@ -23417,47 +23157,25 @@ index 969fddfa..b7b7e065 100644 + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + const int iid1 = tgpig.z/args.nei0; + const int idx = tgpig.z%args.nei0; - -- for (int i = ix; i < nb; i += 2) { -- device const uint8_t * q1 = x[i].ql + q_offset_l; -- device const uint8_t * q2 = q1 + 32; -- device const uint8_t * qh = x[i].qh + q_offset_h; -- device const int8_t * sc = x[i].scales + is; -- device const half * dh = &x[i].d; ++ + tgpig.z = 0; - -- device const float * y = yy + i * QK_K + y_offset; ++ + const int32_t i02 = ((device const int32_t *) (ids + iid1*args.nbi1))[idx]; + const float route_weight = args.weighted != 0 ? + ((device const float *) (weights + uint64_t(idx)*args.weights_nb1 + uint64_t(iid1)*args.weights_nb2))[0] : + 1.0f; - -- for (short l = 0; l < 4; ++l) { -- yl[4*l + 0] = y[l + 0]; -- yl[4*l + 1] = y[l + 32]; -- yl[4*l + 2] = y[l + 64]; -- yl[4*l + 3] = y[l + 96]; -- } ++ + const int64_t i11 = idx % args.ne11; + const int64_t i12 = iid1; - -- for (short row = 0; row < nr0; ++row) { -- float4 sums = {0.f, 0.f, 0.f, 0.f}; ++ + const int64_t i1 = idx; + const int64_t i2 = i12; - -- FOR_UNROLL (short l = 0; l < 4; ++l) { -- sums[0] += yl[4*l + 0] * ((int8_t)((q1[l] & 0xF) | ((qh[l] & kmask1) << 4)) - 32); -- sums[1] += yl[4*l + 1] * ((int8_t)((q2[l] & 0xF) | ((qh[l] & kmask2) << 2)) - 32); -- sums[2] += yl[4*l + 2] * ((int8_t)((q1[l] >> 4) | ((qh[l] & kmask3) << 0)) - 32); -- sums[3] += yl[4*l + 3] * ((int8_t)((q2[l] >> 4) | ((qh[l] & kmask4) >> 2)) - 32); -- } ++ + device const char * src0_up_cur = src0_up_all + i02*args.nb02; + device const char * src0_gate_cur = src0_gate_all + i02*args.nb02; + device const char * src1_cur = src1 + i11*args.nb11 + i12*args.nb12; + device char * dst_cur = dst + (i1*args.ne0 + i2*args.ne1*args.ne0)*sizeof(float); - -- sumf[row] += dh[0] * (sums[0] * sc[0] + sums[1] * sc[2] + sums[2] * sc[4] + sums[3] * sc[6]); ++ + ggml_metal_kargs_mul_mv args0 = { + /*.ne00 =*/ args.ne00, + /*.ne01 =*/ args.ne01, @@ -23479,13 +23197,7 @@ index 969fddfa..b7b7e065 100644 + /*.r2 =*/ 1, + /*.r3 =*/ 1, + }; - -- q1 += args.nb01; -- q2 += args.nb01; -- qh += args.nb01; -- sc += args.nb01; -- dh += args.nb01/2; -- } ++ + switch (args.nr0) { + case 1: + kernel_mul_mv_q2_K_gate_up_swiglu_impl<1, true>( @@ -23507,106 +23219,36 @@ index 969fddfa..b7b7e065 100644 + kernel_mul_mv_q2_K_gate_up_swiglu_impl( + args0, src0_up_cur, src0_gate_cur, src1_cur, dst_cur, route_weight, tgpig, tiisg, sgitg); + break; - } - -- device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; -- -- for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) { -- float sum_all = simd_sum(sumf[row]); -- if (tiisg == 0) { -- dst_f32[first_row + row] = sum_all; -- } -- } ++ } ++ + (void) shmem; + (void) tiitg; - } - --[[host_name("kernel_mul_mv_q6_K_f32")]] --kernel void kernel_mul_mv_q6_K_f32( -- constant ggml_metal_kargs_mul_mv & args, -- device const char * src0, ++} ++ +template +void kernel_mul_mv_id_q2_K_gate_up_swiglu_pair_sg_impl( + constant ggml_metal_kargs_mul_mv_id_gate_up_swiglu & args, + device const char * src0_up_all, + device const char * src0_gate_all, - device const char * src1, - device char * dst, ++ device const char * src1, ++ device char * dst, + device const char * ids, + device const char * weights, + threadgroup float * partials [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]]) { -- -- kernel_mul_mv_q6_K_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); --} -- --// ======================= "True" 2-bit -- --template --void kernel_mul_mv_iq2_xxs_f32_impl( -- args_t args, -- device const char * src0, -- device const char * src1, -- device char * dst, -- threadgroup char * shmem, -- uint3 tgpig, -- ushort tiisg, -- ushort sgitg) { -- const short NSG = FC_mul_mv_nsg; -- - const int nb = args.ne00/QK_K; - - const int r0 = tgpig.x; -- const int r1 = tgpig.y; -- const int im = tgpig.z; -- -- const int first_row = (r0 * NSG + sgitg) * nr0; -- -- const uint i12 = im%FC_mul_mv_ne12; -- const uint i13 = im/FC_mul_mv_ne12; -- -- const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; -- const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; -- -- device const block_iq2_xxs * x = (device const block_iq2_xxs *) (src0 + offset0); -- device const float * y = (device const float *) (src1 + offset1); -- -- float yl[32]; -- float sumf[nr0]={0.f}; ++ uint3 tgpig[[threadgroup_position_in_grid]], ++ ushort tiisg[[thread_index_in_simdgroup]], ++ ushort sgitg[[simdgroup_index_in_threadgroup]]) { ++ const int nb = args.ne00/QK_K; ++ ++ const int r0 = tgpig.x; + const int token = tgpig.z / ((args.nei0 + slots_per_tg - 1) / slots_per_tg); + const int slot_group = tgpig.z % ((args.nei0 + slots_per_tg - 1) / slots_per_tg); + const int slot = slot_group * slots_per_tg + int(sgitg / 2); + const bool is_gate = (sgitg & 1) != 0; + const int first_row = r0 * nr0; - -- const int nb32 = nb * (QK_K / 32); -- -- threadgroup uint64_t * svalues = (threadgroup uint64_t *)(shmem); -- threadgroup uint8_t * ssigns = (threadgroup uint8_t *)(svalues + 256); -- { -- int nval = 4; -- int pos = (32*sgitg + tiisg)*nval; -- for (int i = 0; i < nval; ++i) svalues[pos + i] = iq2xxs_grid[pos + i]; -- nval = 2; -- pos = (32*sgitg + tiisg)*nval; -- for (int i = 0; i < nval; ++i) ssigns[pos+i] = ksigns_iq2xs[pos+i]; -- threadgroup_barrier(mem_flags::mem_threadgroup); -- } -- -- const int ix = tiisg; -- -- device const float * y4 = y + 32 * ix; -- -- for (int ib32 = ix; ib32 < nb32; ib32 += 32) { -- for (short i = 0; i < 32; ++i) { -- yl[i] = y4[i]; -- } ++ + float sumf[nr0] = {0.f}; - -- const int ibl = ib32 / (QK_K / 32); -- const int ib = ib32 % (QK_K / 32); ++ + if (slot < args.nei0 && token < args.nei1) { + const int32_t expert_id = ((device const int32_t *) (ids + token*args.nbi1))[slot]; + const bool active_slot = args._pad0 <= 0 || slot < args._pad0; @@ -23655,10 +23297,7 @@ index 969fddfa..b7b7e065 100644 + yl[i+24] = y_t(y4[i+96]); sumy[3] += float(yl[i+24]); + } + } - -- device const block_iq2_xxs * xr = x + ibl; -- device const uint16_t * q2 = xr->qs + 4 * ib; -- device const half * dh = &xr->d; ++ + int qy[32]; + int4 sumq = {0, 0, 0, 0}; + float4 d8 = {0.f, 0.f, 0.f, 0.f}; @@ -23679,12 +23318,7 @@ index 969fddfa..b7b7e065 100644 + } + } + } - -- for (short row = 0; row < nr0; row++) { -- const float db = dh[0]; -- device const uint8_t * aux8 = (device const uint8_t *)q2; -- const uint32_t aux32 = q2[2] | (q2[3] << 16); -- const float d = db * (0.5f + (aux32 >> 28)); ++ + for (short row = 0; row < nr0; row++) { + device const block_q2_K * xb = use_row_tile ? + (x_expert + uint64_t(first_row/nr0)*uint64_t(nb*nr0) + uint64_t(ib)*uint64_t(nr0) + uint64_t(row)) : @@ -23754,38 +23388,21 @@ index 969fddfa..b7b7e065 100644 + } + sumf[row] += dot_val; + } - -- float sum = 0; -- for (short l = 0; l < 4; ++l) { -- const threadgroup uint8_t * grid = (const threadgroup uint8_t *)(svalues + aux8[l]); -- const uint8_t signs = ssigns[(aux32 >> 7*l) & 127]; -- for (short j = 0; j < 8; ++j) { -- sum += yl[8*l + j] * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f); ++ + y4 += 4 * QK_K; -+ if (share_y) { -+ threadgroup_barrier(mem_flags::mem_threadgroup); - } - } -- sumf[row] += d * sum; -- -- dh += args.nb01/2; -- q2 += args.nb01/2; - } -- -- y4 += 32 * 32; - } - -- device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; -- -- for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) { -- float sum_all = simd_sum(sumf[row]); ++ if (share_y) { ++ threadgroup_barrier(mem_flags::mem_threadgroup); ++ } ++ } ++ } ++ } ++ + for (int row = 0; row < nr0; ++row) { + const float row_sum = simd_sum(sumf[row]); - if (tiisg == 0) { -- dst_f32[first_row + row] = sum_all * 0.25f; ++ if (tiisg == 0) { + partials[sgitg*nr0 + row] = first_row + row < args.ne0 ? row_sum : 0.0f; - } - } ++ } ++ } + + threadgroup_barrier(mem_flags::mem_threadgroup); + @@ -23799,45 +23416,30 @@ index 969fddfa..b7b7e065 100644 + ((device dst_t *) dst)[uint64_t(slot)*args.ne0 + uint64_t(token)*args.ne1*args.ne0 + first_row + tiisg] = + dst_t(up_value * silu * route_weight); + } - } - --[[host_name("kernel_mul_mv_iq2_xxs_f32")]] --kernel void kernel_mul_mv_iq2_xxs_f32( -- constant ggml_metal_kargs_mul_mv & args, -- device const char * src0, ++} ++ +kernel void kernel_mul_mv_id_q2_K_gate_up_swiglu_pair_sg( + constant ggml_metal_kargs_mul_mv_id_gate_up_swiglu & args, + device const char * src0_up_all, + device const char * src0_gate_all, - device const char * src1, - device char * dst, -- threadgroup char * shmem [[threadgroup(0)]], ++ device const char * src1, ++ device char * dst, + device const char * ids, + device const char * weights, + threadgroup float * partials [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]]) { -- kernel_mul_mv_iq2_xxs_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); ++ uint3 tgpig[[threadgroup_position_in_grid]], ++ ushort tiisg[[thread_index_in_simdgroup]], ++ ushort sgitg[[simdgroup_index_in_threadgroup]]) { + kernel_mul_mv_id_q2_K_gate_up_swiglu_pair_sg_impl<8, float>( + args, src0_up_all, src0_gate_all, src1, dst, ids, weights, partials, tgpig, tiisg, sgitg); - } - --template --void kernel_mul_mv_iq2_xs_f32_impl( -- args_t args, -- device const char * src0, ++} ++ +kernel void kernel_mul_mv_id_q2_K_gate_up_swiglu_pair_sg_share_y( + constant ggml_metal_kargs_mul_mv_id_gate_up_swiglu & args, + device const char * src0_up_all, + device const char * src0_gate_all, - device const char * src1, - device char * dst, -- threadgroup char * shmem, -- uint3 tgpig, -- ushort tiisg, -- ushort sgitg) { -- const short NSG = FC_mul_mv_nsg; ++ device const char * src1, ++ device char * dst, + device const char * ids, + device const char * weights, + threadgroup float * partials [[threadgroup(0)]], @@ -23847,8 +23449,7 @@ index 969fddfa..b7b7e065 100644 + kernel_mul_mv_id_q2_K_gate_up_swiglu_pair_sg_impl<8, float, 4, true>( + args, src0_up_all, src0_gate_all, src1, dst, ids, weights, partials, tgpig, tiisg, sgitg); +} - -- const int nb = args.ne00/QK_K; ++ +kernel void kernel_mul_mv_id_q2_K_gate_up_swiglu_pair_sg_vecscale( + constant ggml_metal_kargs_mul_mv_id_gate_up_swiglu & args, + device const char * src0_up_all, @@ -23864,10 +23465,7 @@ index 969fddfa..b7b7e065 100644 + kernel_mul_mv_id_q2_K_gate_up_swiglu_pair_sg_impl<8, float, 4, false, true>( + args, src0_up_all, src0_gate_all, src1, dst, ids, weights, partials, tgpig, tiisg, sgitg); +} - -- const int r0 = tgpig.x; -- const int r1 = tgpig.y; -- const int im = tgpig.z; ++ +kernel void kernel_mul_mv_id_q2_K_gate_up_swiglu_pair_sg_q8_act( + constant ggml_metal_kargs_mul_mv_id_gate_up_swiglu & args, + device const char * src0_up_all, @@ -23883,8 +23481,7 @@ index 969fddfa..b7b7e065 100644 + kernel_mul_mv_id_q2_K_gate_up_swiglu_pair_sg_impl<8, float, 4, false, false, true>( + args, src0_up_all, src0_gate_all, src1, dst, ids, weights, partials, tgpig, tiisg, sgitg); +} - -- const int first_row = (r0 * NSG + sgitg) * nr0; ++ +kernel void kernel_mul_mv_id_q2_K_gate_up_swiglu_pair_sg_half_y( + constant ggml_metal_kargs_mul_mv_id_gate_up_swiglu & args, + device const char * src0_up_all, @@ -23900,9 +23497,7 @@ index 969fddfa..b7b7e065 100644 + kernel_mul_mv_id_q2_K_gate_up_swiglu_pair_sg_impl<8, float, 4, false, false, false, half>( + args, src0_up_all, src0_gate_all, src1, dst, ids, weights, partials, tgpig, tiisg, sgitg); +} - -- const uint i12 = im%FC_mul_mv_ne12; -- const uint i13 = im/FC_mul_mv_ne12; ++ +kernel void kernel_mul_mv_id_q2_K_gate_up_swiglu_pair_sg_rowtile( + constant ggml_metal_kargs_mul_mv_id_gate_up_swiglu & args, + device const char * src0_up_all, @@ -23918,9 +23513,7 @@ index 969fddfa..b7b7e065 100644 + kernel_mul_mv_id_q2_K_gate_up_swiglu_pair_sg_impl<8, float, 4, false, false, false, float, true>( + args, src0_up_all, src0_gate_all, src1, dst, ids, weights, partials, tgpig, tiisg, sgitg); +} - -- const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; -- const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; ++ +kernel void kernel_mul_mv_id_q2_K_gate_up_swiglu_pair_sg_r12( + constant ggml_metal_kargs_mul_mv_id_gate_up_swiglu & args, + device const char * src0_up_all, @@ -23936,9 +23529,7 @@ index 969fddfa..b7b7e065 100644 + kernel_mul_mv_id_q2_K_gate_up_swiglu_pair_sg_impl<12, float>( + args, src0_up_all, src0_gate_all, src1, dst, ids, weights, partials, tgpig, tiisg, sgitg); +} - -- device const block_iq2_xs * x = (device const block_iq2_xs *) (src0 + offset0); -- device const float * y = (device const float *) (src1 + offset1); ++ +kernel void kernel_mul_mv_id_q2_K_gate_up_swiglu_pair_sg_r16( + constant ggml_metal_kargs_mul_mv_id_gate_up_swiglu & args, + device const char * src0_up_all, @@ -23954,9 +23545,7 @@ index 969fddfa..b7b7e065 100644 + kernel_mul_mv_id_q2_K_gate_up_swiglu_pair_sg_impl<16, float>( + args, src0_up_all, src0_gate_all, src1, dst, ids, weights, partials, tgpig, tiisg, sgitg); +} - -- float yl[32]; -- float sumf[nr0]={0.f}; ++ +kernel void kernel_mul_mv_id_q2_K_gate_up_swiglu_pair_sg_slot2( + constant ggml_metal_kargs_mul_mv_id_gate_up_swiglu & args, + device const char * src0_up_all, @@ -23972,8 +23561,7 @@ index 969fddfa..b7b7e065 100644 + kernel_mul_mv_id_q2_K_gate_up_swiglu_pair_sg_impl<8, float, 2>( + args, src0_up_all, src0_gate_all, src1, dst, ids, weights, partials, tgpig, tiisg, sgitg); +} - -- const int nb32 = nb * (QK_K / 32); ++ +kernel void kernel_mul_mv_id_q2_K_gate_up_swiglu_pair_sg_f16( + constant ggml_metal_kargs_mul_mv_id_gate_up_swiglu & args, + device const char * src0_up_all, @@ -23989,18 +23577,7 @@ index 969fddfa..b7b7e065 100644 + kernel_mul_mv_id_q2_K_gate_up_swiglu_pair_sg_impl<8, half>( + args, src0_up_all, src0_gate_all, src1, dst, ids, weights, partials, tgpig, tiisg, sgitg); +} - -- threadgroup uint64_t * svalues = (threadgroup uint64_t *)(shmem); -- threadgroup uint8_t * ssigns = (threadgroup uint8_t *)(svalues + 512); -- { -- int nval = 8; -- int pos = (32*sgitg + tiisg)*nval; -- for (int i = 0; i < nval; ++i) svalues[pos + i] = iq2xs_grid[pos + i]; -- nval = 2; -- pos = (32*sgitg + tiisg)*nval; -- for (int i = 0; i < nval; ++i) ssigns[pos+i] = ksigns_iq2xs[pos+i]; -- threadgroup_barrier(mem_flags::mem_threadgroup); -- } ++ +kernel void kernel_mul_mv_id_q2_K_gate_up_swiglu_pair_sg_slot8_split( + constant ggml_metal_kargs_mul_mv_id_gate_up_swiglu & args, + device const char * src0_up_all, @@ -24016,8 +23593,7 @@ index 969fddfa..b7b7e065 100644 + kernel_mul_mv_id_q2_K_gate_up_swiglu_pair_sg_impl<8, float, 8>( + args, src0_up_all, src0_gate_all, src1, dst, ids, weights, partials, tgpig, tiisg, sgitg); +} - -- const int ix = tiisg; ++ +static inline float4 q2_repacked_unpack4(uint8_t packed) { + return float4( + float( packed & 0x3), @@ -24025,18 +23601,14 @@ index 969fddfa..b7b7e065 100644 + float((packed >> 4) & 0x3), + float((packed >> 6) & 0x3)); +} - -- device const float * y4 = y + 32 * ix; ++ +static inline float q2_repacked_dot8(device const uint8_t * packed, thread const float * values) { + const float4 y0 = float4(values[0], values[1], values[2], values[3]); + const float4 y1 = float4(values[4], values[5], values[6], values[7]); + return dot(y0, q2_repacked_unpack4(packed[0])) + + dot(y1, q2_repacked_unpack4(packed[1])); +} - -- for (int ib32 = ix; ib32 < nb32; ib32 += 32) { -- for (short i = 0; i < 32; ++i) { -- yl[i] = y4[i]; ++ +template< + int slots_per_tg, + int nr0 = 8, @@ -24092,10 +23664,8 @@ index 969fddfa..b7b7e065 100644 + } + } + } - } - -- const int ibl = ib32 / (QK_K / 32); -- const int ib = ib32 % (QK_K / 32); ++ } ++ + const bool valid_output = slot < args.nei0 && token < args.nei1; + for (int row = 0; row < nr0; ++row) { + const float up_value = simd_sum(sum_up[row]); @@ -24111,11 +23681,7 @@ index 969fddfa..b7b7e065 100644 + (void) partials; + return; + } - -- device const block_iq2_xs * xr = x + ibl; -- device const uint16_t * q2 = xr->qs + 4 * ib; -- device const uint8_t * sc = xr->scales + ib; -- device const half * dh = &xr->d; ++ + if (slot < args.nei0 && token < args.nei1) { + const int32_t expert_id = ((device const int32_t *) (ids + token*args.nbi1))[slot]; + const bool active_slot = args._pad0 <= 0 || slot < args._pad0; @@ -24162,13 +23728,7 @@ index 969fddfa..b7b7e065 100644 + yl[i+24] = y4[i+96]; sumy[3] += yl[i+24]; + } + } - -- for (short row = 0; row < nr0; row++) { -- const float db = dh[0]; -- const uint8_t ls1 = sc[0] & 0xf; -- const uint8_t ls2 = sc[0] >> 4; -- const float d1 = db * (0.5f + ls1); -- const float d2 = db * (0.5f + ls2); ++ + device const uint8_t * sc_up = (device const uint8_t *)x_up[ib].scales + 8*iq + is; + device const uint16_t * qs_up = (device const uint16_t *)x_up[ib].qs + 16 * iq + 4 * ir; + device const half * dh_up = &x_up[ib].d; @@ -24250,13 +23810,7 @@ index 969fddfa..b7b7e065 100644 + acc1_gate[3] += yl[i+24] * (qs_gate[i/2] & 0x00c0); + acc2_gate[3] += yl[i+25] * (qs_gate[i/2] & 0xc000); + } - -- float sum1 = 0, sum2 = 0; -- for (short l = 0; l < 2; ++l) { -- const threadgroup uint8_t * grid = (const threadgroup uint8_t *)(svalues + (q2[l] & 511)); -- const uint8_t signs = ssigns[(q2[l] >> 9)]; -- for (short j = 0; j < 8; ++j) { -- sum1 += yl[8*l + j] * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f); ++ + float dall = dh_up[0]; + float dmin = dh_up[1] * 1.f/16.f; + sum_up[row] += dall * ((acc1_up[0] + 1.f/256.f * acc2_up[0]) * (sc_up[0] & 0xF) * 1.f/ 1.f + @@ -24282,19 +23836,12 @@ index 969fddfa..b7b7e065 100644 + qs_gate += args.nb01/2; + sc_gate += args.nb01; + dh_gate += args.nb01/2; - } -- } -- for (short l = 2; l < 4; ++l) { -- const threadgroup uint8_t * grid = (const threadgroup uint8_t *)(svalues + (q2[l] & 511)); -- const uint8_t signs = ssigns[(q2[l] >> 9)]; -- for (short j = 0; j < 8; ++j) { -- sum2 += yl[8*l + j] * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f); ++ } + + if (!use_prequant_q8) { + y4 += 4 * QK_K; - } - } -- sumf[row] += d1 * sum1 + d2 * sum2; ++ } ++ } + } + } + @@ -25934,10 +25481,12 @@ index 969fddfa..b7b7e065 100644 + device const float * y1 = yy + ix*QK_K + y_offset; + + float yl[32]; -+ uint32_t scales32, aux32; -+ thread uint16_t * scales16 = (thread uint16_t *)&scales32; -+ thread const int8_t * scales = (thread const int8_t *)&scales32; -+ + uint32_t scales32, aux32; + thread uint16_t * scales16 = (thread uint16_t *)&scales32; + thread const int8_t * scales = (thread const int8_t *)&scales32; + +- float sumf1[nr0] = {0.f}; +- float sumf2[nr0] = {0.f}; + float sumf1[nr0] = {0.f}; + float sumf2[nr0] = {0.f}; + @@ -27318,22 +26867,64 @@ index 969fddfa..b7b7e065 100644 + const short NSG = FC_mul_mv_nsg; + + const int nb = args.ne00/QK_K; -+ + +- for (int i = ix; i < nb; i += 4) { +- for (short l = 0; l < 8; ++l) { +- yl[l+ 0] = y1[l+ 0]; +- yl[l+ 8] = y1[l+16]; +- yl[l+16] = y1[l+32]; +- yl[l+24] = y1[l+48]; +- } + const int r0 = tgpig.x; + const int r1 = tgpig.y; + const int im = tgpig.z; -+ + +- device const uint16_t * q = (device const uint16_t *)(x[i].qs + q_offset); +- device const uint16_t * h = (device const uint16_t *)(x[i].hmask + l0); +- device const uint16_t * a = (device const uint16_t *)(x[i].scales); +- device const half * dh = &x[i].d; + const int first_row = (r0 * NSG + sgitg) * nr0; -+ + +- for (short row = 0; row < nr0; ++row) { +- const float d_all = (float)dh[0]; + const uint i12 = im%FC_mul_mv_ne12; + const uint i13 = im/FC_mul_mv_ne12; -+ + +- scales16[0] = a[4]; +- scales16[1] = a[5]; +- aux32 = ((scales32 >> s_shift2) << 4) & 0x30303030; +- scales16[0] = a[il+0]; +- scales16[1] = a[il+1]; +- scales32 = ((scales32 >> s_shift1) & 0x0f0f0f0f) | aux32; + const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; -+ + +- float s1 = 0, s2 = 0, s3 = 0, s4 = 0, s5 = 0, s6 = 0; +- for (short l = 0; l < 8; l += 2) { +- const int32_t qs = q[l/2]; +- s1 += yl[l+0] * (qs & qm[il/2][0]); +- s2 += yl[l+1] * (qs & qm[il/2][1]); +- s3 += ((h[l/2] & hm[0]) ? 0.f : yl[l+0]) + ((h[l/2] & hm[1]) ? 0.f : yl[l+1]); +- s4 += yl[l+16] * (qs & qm[il/2][2]); +- s5 += yl[l+17] * (qs & qm[il/2][3]); +- s6 += ((h[l/2] & hm[2]) ? 0.f : yl[l+16]) + ((h[l/2] & hm[3]) ? 0.f : yl[l+17]); +- } +- float d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1); +- float d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2); +- sumf1[row] += d1 * (scales[0] - 32); +- sumf2[row] += d2 * (scales[2] - 32); + device const block_iq1_s * x = (device const block_iq1_s *) (src0 + offset0); + device const float * y = (device const float *) (src1 + offset1); -+ + +- s1 = s2 = s3 = s4 = s5 = s6 = 0; +- for (short l = 0; l < 8; l += 2) { +- const int32_t qs = q[l/2+8]; +- s1 += yl[l+8] * (qs & qm[il/2][0]); +- s2 += yl[l+9] * (qs & qm[il/2][1]); +- s3 += ((h[l/2+8] & hm[0]) ? 0.f : yl[l+8]) + ((h[l/2+8] & hm[1]) ? 0.f : yl[l+9]); +- s4 += yl[l+24] * (qs & qm[il/2][2]); +- s5 += yl[l+25] * (qs & qm[il/2][3]); +- s6 += ((h[l/2+8] & hm[2]) ? 0.f : yl[l+24]) + ((h[l/2+8] & hm[3]) ? 0.f : yl[l+25]); + float yl[32]; + float sumf[nr0]={0.f}; + @@ -27370,197 +26961,295 @@ index 969fddfa..b7b7e065 100644 + + yl[j+ 8] * (grid2[j] & 0xf) + yl[j+12] * (grid2[j] >> 4) + + yl[j+16] * (grid3[j] & 0xf) + yl[j+20] * (grid3[j] >> 4) + + yl[j+24] * (grid4[j] & 0xf) + yl[j+28] * (grid4[j] >> 4); -+ } + } +- d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1); +- d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2); +- sumf1[row] += d1 * (scales[1] - 32); +- sumf2[row] += d2 * (scales[3] - 32); + sumf[row] += (float)dh[0] * (sum + sumy * (qh[0] & 0x8000 ? -1 - IQ1S_DELTA : -1 + IQ1S_DELTA)) * (2*((qh[0] >> 12) & 7) + 1); -+ -+ dh += args.nb01/2; + +- q += args.nb01/2; +- h += args.nb01/2; +- a += args.nb01/2; + dh += args.nb01/2; + qs += args.nb01; + qh += args.nb01/2; -+ } -+ + } + +- y1 += 4 * QK_K; +- } +- +- for (int row = 0; row < nr0; ++row) { +- const float sumf = (sumf1[row] + 0.25f * sumf2[row]) / (1 << shift); +- sumf1[row] = simd_sum(sumf); + y4 += 32 * 32; -+ } -+ -+ device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; -+ + } + + device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; + +- if (tiisg == 0) { +- for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) { +- dst_f32[first_row + row] = sumf1[row]; + for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) { + float sum_all = simd_sum(sumf[row]); + if (tiisg == 0) { + dst_f32[first_row + row] = sum_all; -+ } -+ } -+} -+ -+[[host_name("kernel_mul_mv_iq1_s_f32")]] -+kernel void kernel_mul_mv_iq1_s_f32( -+ constant ggml_metal_kargs_mul_mv & args, -+ device const char * src0, -+ device const char * src1, -+ device char * dst, -+ uint3 tgpig[[threadgroup_position_in_grid]], -+ ushort tiisg[[thread_index_in_simdgroup]], -+ ushort sgitg[[simdgroup_index_in_threadgroup]]) { -+ + } + } + } + +-[[host_name("kernel_mul_mv_q3_K_f32")]] +-kernel void kernel_mul_mv_q3_K_f32( ++[[host_name("kernel_mul_mv_iq1_s_f32")]] ++kernel void kernel_mul_mv_iq1_s_f32( + constant ggml_metal_kargs_mul_mv & args, + device const char * src0, + device const char * src1, +@@ -8355,11 +14101,11 @@ kernel void kernel_mul_mv_q3_K_f32( + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + +- kernel_mul_mv_q3_K_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); + kernel_mul_mv_iq1_s_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); -+} -+ -+template + } + + template +-void kernel_mul_mv_q4_K_f32_impl( +void kernel_mul_mv_iq1_m_f32_impl( -+ args_t args, -+ device const char * src0, -+ device const char * src1, -+ device char * dst, -+ threadgroup char * shmem, -+ uint3 tgpig, -+ ushort tiisg, -+ ushort sgitg) { -+ const short NSG = FC_mul_mv_nsg; -+ -+ const int nb = args.ne00/QK_K; -+ -+ const int r0 = tgpig.x; -+ const int r1 = tgpig.y; -+ const int im = tgpig.z; -+ -+ const int first_row = (r0 * NSG + sgitg) * nr0; -+ -+ const uint i12 = im%FC_mul_mv_ne12; -+ const uint i13 = im/FC_mul_mv_ne12; -+ -+ const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; -+ const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; -+ + args_t args, + device const char * src0, + device const char * src1, +@@ -8370,15 +14116,6 @@ void kernel_mul_mv_q4_K_f32_impl( + ushort sgitg) { + const short NSG = FC_mul_mv_nsg; + +- constexpr uint16_t kmask1 = 0x3f3f; +- constexpr uint16_t kmask2 = 0x0f0f; +- constexpr uint16_t kmask3 = 0xc0c0; +- +- const short ix = tiisg/8; // 0...3 +- const short it = tiisg%8; // 0...7 +- const short iq = it/4; // 0 or 1 +- const short ir = it%4; // 0...3 +- + const int nb = args.ne00/QK_K; + + const int r0 = tgpig.x; +@@ -8393,70 +14130,67 @@ void kernel_mul_mv_q4_K_f32_impl( + const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + +- device const block_q4_K * x = (device const block_q4_K *) (src0 + offset0); +- device const float * y = (device const float *) (src1 + offset1); +- +- float yl[16]; +- float yh[16]; + device const block_iq1_m * x = (device const block_iq1_m *) (src0 + offset0); + device const float * y = (device const float *) (src1 + offset1); -+ + + float yl[32]; -+ float sumf[nr0]={0.f}; -+ + float sumf[nr0]={0.f}; + +- device const float * y4 = y + ix * QK_K + 64 * iq + 8 * ir; + const int nb32 = nb * (QK_K / 32); -+ + +- uint16_t sc16[4]; +- thread const uint8_t * sc8 = (thread const uint8_t *)sc16; + const short ix = tiisg; -+ + +- for (int ib = ix; ib < nb; ib += 4) { +- float4 sumy = {0.f, 0.f, 0.f, 0.f}; + device const float * y4 = y + 32 * ix; -+ + + iq1m_scale_t scale; + + for (int ib32 = ix; ib32 < nb32; ib32 += 32) { + float4 sumy = {0.f}; -+ for (short i = 0; i < 8; ++i) { + for (short i = 0; i < 8; ++i) { +- yl[i+0] = y4[i+ 0]; sumy[0] += yl[i+0]; +- yl[i+8] = y4[i+ 32]; sumy[1] += yl[i+8]; +- yh[i+0] = y4[i+128]; sumy[2] += yh[i+0]; +- yh[i+8] = y4[i+160]; sumy[3] += yh[i+8]; + yl[i+ 0] = y4[i+ 0]; sumy[0] += yl[i+ 0]; + yl[i+ 8] = y4[i+ 8]; sumy[1] += yl[i+ 8]; + yl[i+16] = y4[i+16]; sumy[2] += yl[i+16]; + yl[i+24] = y4[i+24]; sumy[3] += yl[i+24]; -+ } -+ + } + +- device const uint16_t * sc = (device const uint16_t *)x[ib].scales + iq; +- device const uint16_t * q1 = (device const uint16_t *)x[ib].qs + 16 * iq + 4 * ir; +- device const half * dh = &x[ib].d; + const int ibl = ib32 / (QK_K / 32); + const int ib = ib32 % (QK_K / 32); -+ + +- for (short row = 0; row < nr0; row++) { +- sc16[0] = sc[0] & kmask1; +- sc16[1] = sc[2] & kmask1; +- sc16[2] = ((sc[4] >> 0) & kmask2) | ((sc[0] & kmask3) >> 2); +- sc16[3] = ((sc[4] >> 4) & kmask2) | ((sc[2] & kmask3) >> 2); + device const block_iq1_m * xr = x + ibl; + device const uint8_t * qs = xr->qs + 4 * ib; + device const uint8_t * qh = xr->qh + 2 * ib; + device const uint16_t * sc = (device const uint16_t *)xr->scales; -+ + +- device const uint16_t * q2 = q1 + 32; + for (short row = 0; row < nr0; row++) { + scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000); -+ + +- float4 acc1 = {0.f, 0.f, 0.f, 0.f}; +- float4 acc2 = {0.f, 0.f, 0.f, 0.f}; + constant uint8_t * grid1 = (constant uint8_t *)(iq1s_grid_gpu + (qs[0] | ((qh[0] << 8) & 0x700))); + constant uint8_t * grid2 = (constant uint8_t *)(iq1s_grid_gpu + (qs[1] | ((qh[0] << 4) & 0x700))); + constant uint8_t * grid3 = (constant uint8_t *)(iq1s_grid_gpu + (qs[2] | ((qh[1] << 8) & 0x700))); + constant uint8_t * grid4 = (constant uint8_t *)(iq1s_grid_gpu + (qs[3] | ((qh[1] << 4) & 0x700))); -+ + +- FOR_UNROLL (short i = 0; i < 4; ++i) { +- acc1[0] += yl[2*i + 0] * (q1[i] & 0x000F); +- acc1[1] += yl[2*i + 1] * (q1[i] & 0x0F00); +- acc1[2] += yl[2*i + 8] * (q1[i] & 0x00F0); +- acc1[3] += yl[2*i + 9] * (q1[i] & 0xF000); +- acc2[0] += yh[2*i + 0] * (q2[i] & 0x000F); +- acc2[1] += yh[2*i + 1] * (q2[i] & 0x0F00); +- acc2[2] += yh[2*i + 8] * (q2[i] & 0x00F0); +- acc2[3] += yh[2*i + 9] * (q2[i] & 0xF000); + float2 sum = {0.f}; + for (short j = 0; j < 4; ++j) { + sum[0] += yl[j+ 0] * (grid1[j] & 0xf) + yl[j+ 4] * (grid1[j] >> 4) + + yl[j+ 8] * (grid2[j] & 0xf) + yl[j+12] * (grid2[j] >> 4); + sum[1] += yl[j+16] * (grid3[j] & 0xf) + yl[j+20] * (grid3[j] >> 4) + + yl[j+24] * (grid4[j] & 0xf) + yl[j+28] * (grid4[j] >> 4); -+ } + } + const float delta1 = sumy[0] * (qh[0] & 0x08 ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA) + sumy[1] * (qh[0] & 0x80 ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA); + const float delta2 = sumy[2] * (qh[1] & 0x08 ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA) + sumy[3] * (qh[1] & 0x80 ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA); -+ + +- sumf[row] += dh[0] * ((acc1[0] + 1.f/256.f * acc1[1]) * sc8[0] + +- (acc1[2] + 1.f/256.f * acc1[3]) * sc8[1] * 1.f/16.f + +- (acc2[0] + 1.f/256.f * acc2[1]) * sc8[4] + +- (acc2[2] + 1.f/256.f * acc2[3]) * sc8[5] * 1.f/16.f) - +- dh[1] * (sumy[0] * sc8[2] + sumy[1] * sc8[3] + sumy[2] * sc8[6] + sumy[3] * sc8[7]); + sumf[row] += (float)scale.f16 * ((sum[0] + delta1) * (2*((sc[ib/2] >> (6*(ib%2)+0)) & 7) + 1) + + (sum[1] + delta2) * (2*((sc[ib/2] >> (6*(ib%2)+3)) & 7) + 1)); -+ -+ sc += args.nb01/2; + +- q1 += args.nb01/2; + sc += args.nb01/2; +- dh += args.nb01/2; + qs += args.nb01; + qh += args.nb01; -+ } -+ + } + +- y4 += 4 * QK_K; + y4 += 32 * 32; -+ } -+ + } + +- device float * dst_f32 = (device float *) dst + (int64_t)im*args.ne0*args.ne1 + (int64_t)r1*args.ne0; + device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; -+ -+ for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) { -+ float sum_all = simd_sum(sumf[row]); -+ if (tiisg == 0) { -+ dst_f32[first_row + row] = sum_all; -+ } -+ } -+} -+ + + for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) { + float sum_all = simd_sum(sumf[row]); +@@ -8466,8 +14200,8 @@ void kernel_mul_mv_q4_K_f32_impl( + } + } + +-[[host_name("kernel_mul_mv_q4_K_f32")]] +-kernel void kernel_mul_mv_q4_K_f32( +[[host_name("kernel_mul_mv_iq1_m_f32")]] +kernel void kernel_mul_mv_iq1_m_f32( -+ constant ggml_metal_kargs_mul_mv & args, -+ device const char * src0, -+ device const char * src1, -+ device char * dst, -+ uint3 tgpig[[threadgroup_position_in_grid]], -+ ushort tiisg[[thread_index_in_simdgroup]], -+ ushort sgitg[[simdgroup_index_in_threadgroup]]) { -+ + constant ggml_metal_kargs_mul_mv & args, + device const char * src0, + device const char * src1, +@@ -8476,11 +14210,11 @@ kernel void kernel_mul_mv_q4_K_f32( + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + +- kernel_mul_mv_q4_K_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); + kernel_mul_mv_iq1_m_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); -+} -+ + } + +-template +-void kernel_mul_mv_q5_K_f32_impl( +template +void kernel_mul_mv_iq4_nl_f32_impl( -+ args_t args, -+ device const char * src0, -+ device const char * src1, -+ device char * dst, -+ threadgroup char * shmem, -+ uint3 tgpig, -+ ushort tiisg, -+ ushort sgitg) { -+ const short NSG = FC_mul_mv_nsg; -+ + args_t args, + device const char * src0, + device const char * src1, +@@ -8491,13 +14225,13 @@ void kernel_mul_mv_q5_K_f32_impl( + ushort sgitg) { + const short NSG = FC_mul_mv_nsg; + +- const int nb = args.ne00/QK_K; + threadgroup float * shmem_f32 = (threadgroup float *) shmem; -+ -+ const int r0 = tgpig.x; -+ const int r1 = tgpig.y; -+ const int im = tgpig.z; -+ + + const int r0 = tgpig.x; + const int r1 = tgpig.y; + const int im = tgpig.z; + +- const int first_row = (r0 * NSG + sgitg) * nr0; + const int first_row = (r0 * NSG + sgitg) * NR0; -+ -+ const uint i12 = im%FC_mul_mv_ne12; -+ const uint i13 = im/FC_mul_mv_ne12; -+ -+ const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; -+ const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; -+ + + const uint i12 = im%FC_mul_mv_ne12; + const uint i13 = im/FC_mul_mv_ne12; +@@ -8505,113 +14239,92 @@ void kernel_mul_mv_q5_K_f32_impl( + const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + +- device const block_q5_K * x = (device const block_q5_K *) (src0 + offset0); +- device const float * yy = (device const float *) (src1 + offset1); +- +- float sumf[nr0]={0.f}; +- +- float yl[16], yh[16]; + device const block_iq4_nl * x = (device const block_iq4_nl *) (src0 + offset0); + device const float * y = (device const float *) (src1 + offset1); -+ + +- constexpr uint16_t kmask1 = 0x3f3f; +- constexpr uint16_t kmask2 = 0x0f0f; +- constexpr uint16_t kmask3 = 0xc0c0; + const int nb = args.ne00/QK4_NL; + const int ns01 = args.nb01/args.nb00; -+ + +- const short tid = tiisg/4; +- const short ix = tiisg%4; +- const short iq = tid/4; +- const short ir = tid%4; + const short ix = tiisg/2; // 0...15 + const short it = tiisg%2; // 0 or 1 -+ + +- const short l0 = 8*ir; +- const short q_offset = 32*iq + l0; +- const short y_offset = 64*iq + l0; + shmem_f32[tiisg] = kvalues_iq4nl_f[tiisg%16]; + threadgroup_barrier(mem_flags::mem_threadgroup); -+ + +- const uint8_t hm1 = 1u << (2*iq); +- const uint8_t hm2 = hm1 << 1; +- const uint8_t hm3 = hm1 << 4; +- const uint8_t hm4 = hm2 << 4; + float4 yl[4]; + float sumf[NR0]={0.f}; -+ + +- uint16_t sc16[4]; +- thread const uint8_t * sc8 = (thread const uint8_t *)sc16; + device const float * yb = y + ix*QK4_NL + it*8; -+ + +- device const float * y1 = yy + ix*QK_K + y_offset; + uint32_t aux32[2]; + thread const uint8_t * q8 = (thread const uint8_t *)aux32; -+ + +- for (int i = ix; i < nb; i += 4) { +- device const uint8_t * q1 = x[i].qs + q_offset; +- device const uint8_t * qh = x[i].qh + l0; +- device const half * dh = &x[i].d; +- device const uint16_t * a = (device const uint16_t *)x[i].scales + iq; + float4 qf1, qf2; -+ + +- device const float * y2 = y1 + 128; +- float4 sumy = {0.f, 0.f, 0.f, 0.f}; +- for (short l = 0; l < 8; ++l) { +- yl[l+0] = y1[l+ 0]; sumy[0] += yl[l+0]; +- yl[l+8] = y1[l+32]; sumy[1] += yl[l+8]; +- yh[l+0] = y2[l+ 0]; sumy[2] += yh[l+0]; +- yh[l+8] = y2[l+32]; sumy[3] += yh[l+8]; +- } + // [TAG_MUL_MV_WEIRD] + for (int ib = ix; ib < nb && ib < ns01; ib += 16) { + device const float4 * y4 = (device const float4 *)yb; @@ -27568,13 +27257,32 @@ index 969fddfa..b7b7e065 100644 + yl[1] = y4[4]; + yl[2] = y4[1]; + yl[3] = y4[5]; -+ + +- for (short row = 0; row < nr0; ++row) { +- device const uint8_t * q2 = q1 + 64; + for (short row = 0; row < NR0; row++) { + device const block_iq4_nl & xb = x[row*ns01 + ib]; + device const uint16_t * q4 = (device const uint16_t *)(xb.qs + 8*it); -+ + +- sc16[0] = a[0] & kmask1; +- sc16[1] = a[2] & kmask1; +- sc16[2] = ((a[4] >> 0) & kmask2) | ((a[0] & kmask3) >> 2); +- sc16[3] = ((a[4] >> 4) & kmask2) | ((a[2] & kmask3) >> 2); + float4 acc1 = {0.f}, acc2 = {0.f}; -+ + +- float4 acc1 = {0.f}; +- float4 acc2 = {0.f}; +- FOR_UNROLL (short l = 0; l < 8; ++l) { +- uint8_t h = qh[l]; +- acc1[0] += yl[l+0] * (q1[l] & 0x0F); +- acc1[1] += yl[l+8] * (q1[l] & 0xF0); +- acc1[2] += yh[l+0] * (q2[l] & 0x0F); +- acc1[3] += yh[l+8] * (q2[l] & 0xF0); +- acc2[0] += h & hm1 ? yl[l+0] : 0.f; +- acc2[1] += h & hm2 ? yl[l+8] : 0.f; +- acc2[2] += h & hm3 ? yh[l+0] : 0.f; +- acc2[3] += h & hm4 ? yh[l+8] : 0.f; +- } + aux32[0] = q4[0] | (q4[1] << 16); + aux32[1] = (aux32[0] >> 4) & 0x0f0f0f0f; + aux32[0] &= 0x0f0f0f0f; @@ -27582,7 +27290,12 @@ index 969fddfa..b7b7e065 100644 + qf2 = {shmem_f32[q8[4]], shmem_f32[q8[5]], shmem_f32[q8[6]], shmem_f32[q8[7]]}; + acc1 += yl[0] * qf1; + acc2 += yl[1] * qf2; -+ + +- sumf[row] += dh[0] * (sc8[0] * (acc1[0] + 16.f*acc2[0]) + +- sc8[1] * (acc1[1]/16.f + 16.f*acc2[1]) + +- sc8[4] * (acc1[2] + 16.f*acc2[2]) + +- sc8[5] * (acc1[3]/16.f + 16.f*acc2[3])) - +- dh[1] * (sumy[0] * sc8[2] + sumy[1] * sc8[3] + sumy[2] * sc8[6] + sumy[3] * sc8[7]); + aux32[0] = q4[2] | (q4[3] << 16); + aux32[1] = (aux32[0] >> 4) & 0x0f0f0f0f; + aux32[0] &= 0x0f0f0f0f; @@ -27590,88 +27303,134 @@ index 969fddfa..b7b7e065 100644 + qf2 = {shmem_f32[q8[4]], shmem_f32[q8[5]], shmem_f32[q8[6]], shmem_f32[q8[7]]}; + acc1 += yl[2] * qf1; + acc2 += yl[3] * qf2; -+ + +- q1 += args.nb01; +- qh += args.nb01; +- dh += args.nb01/2; +- a += args.nb01/2; + acc1 += acc2; + + sumf[row] += (float)xb.d * (acc1[0] + acc1[1] + acc1[2] + acc1[3]); -+ } -+ + } + +- y1 += 4 * QK_K; + yb += 16 * QK4_NL; -+ } -+ -+ device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; -+ + } + + device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; + +- for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) { +- const float tot = simd_sum(sumf[row]); + for (int row = 0; row < NR0 && first_row + row < args.ne0; ++row) { + float sum_all = simd_sum(sumf[row]); -+ if (tiisg == 0) { + if (tiisg == 0) { +- dst_f32[first_row + row] = tot; + dst_f32[first_row + row] = sum_all; -+ } -+ } -+} -+ + } + } + } + +-[[host_name("kernel_mul_mv_q5_K_f32")]] +-kernel void kernel_mul_mv_q5_K_f32( +[[host_name("kernel_mul_mv_iq4_nl_f32")]] +kernel void kernel_mul_mv_iq4_nl_f32( -+ constant ggml_metal_kargs_mul_mv & args, -+ device const char * src0, -+ device const char * src1, -+ device char * dst, + constant ggml_metal_kargs_mul_mv & args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem [[threadgroup(0)]], -+ uint3 tgpig[[threadgroup_position_in_grid]], -+ ushort tiisg[[thread_index_in_simdgroup]], -+ ushort sgitg[[simdgroup_index_in_threadgroup]]) { -+ + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + +- kernel_mul_mv_q5_K_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); + kernel_mul_mv_iq4_nl_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); -+} -+ + } + +-template +-void kernel_mul_mv_q6_K_f32_impl( +template +void kernel_mul_mv_iq4_xs_f32_impl( -+ args_t args, -+ device const char * src0, -+ device const char * src1, -+ device char * dst, -+ threadgroup char * shmem, -+ uint3 tgpig, -+ ushort tiisg, -+ ushort sgitg) { -+ const short NSG = FC_mul_mv_nsg; -+ + args_t args, + device const char * src0, + device const char * src1, +@@ -8622,18 +14335,12 @@ void kernel_mul_mv_q6_K_f32_impl( + ushort sgitg) { + const short NSG = FC_mul_mv_nsg; + +- constexpr uint8_t kmask1 = 0x03; +- constexpr uint8_t kmask2 = 0x0C; +- constexpr uint8_t kmask3 = 0x30; +- constexpr uint8_t kmask4 = 0xC0; +- +- const int nb = args.ne00/QK_K; + threadgroup float * shmem_f32 = (threadgroup float *) shmem; -+ -+ const int r0 = tgpig.x; -+ const int r1 = tgpig.y; -+ const int im = tgpig.z; + + const int r0 = tgpig.x; + const int r1 = tgpig.y; + const int im = tgpig.z; +- +- const int first_row = (r0 * NSG + sgitg) * nr0; + const int first_row = (r0 * NSG + sgitg) * NR0; -+ -+ const uint i12 = im%FC_mul_mv_ne12; -+ const uint i13 = im/FC_mul_mv_ne12; -+ -+ const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; -+ const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; -+ + + const uint i12 = im%FC_mul_mv_ne12; + const uint i13 = im/FC_mul_mv_ne12; +@@ -8641,63 +14348,70 @@ void kernel_mul_mv_q6_K_f32_impl( + const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + +- device const block_q6_K * x = (device const block_q6_K *) (src0 + offset0); +- device const float * yy = (device const float *) (src1 + offset1); + device const block_iq4_xs * x = (device const block_iq4_xs *) (src0 + offset0); + device const float * y = (device const float *) (src1 + offset1); -+ + +- float sumf[nr0] = { 0.f }; + const int nb = args.ne00/QK_K; + const int ns01 = args.nb01/args.nb00; -+ + +- float yl[16]; + const short ix = tiisg/16; // 0 or 1 + const short it = tiisg%16; // 0...15 + const short ib = it/2; + const short il = it%2; -+ + +- const short tid = tiisg/2; +- const short ix = tiisg%2; +- const short ip = tid/8; // 0 or 1 +- const short il = tid%8; +- const short l0 = 4*il; +- const short is = 8*ip + l0/16; + shmem_f32[tiisg] = kvalues_iq4nl_f[tiisg%16]; + threadgroup_barrier(mem_flags::mem_threadgroup); -+ + +- const short y_offset = 128*ip + l0; +- const short q_offset_l = 64*ip + l0; +- const short q_offset_h = 32*ip + l0; + float4 yl[4]; + float sumf[NR0]={0.f}; -+ + +- for (int i = ix; i < nb; i += 2) { +- device const uint8_t * q1 = x[i].ql + q_offset_l; +- device const uint8_t * q2 = q1 + 32; +- device const uint8_t * qh = x[i].qh + q_offset_h; +- device const int8_t * sc = x[i].scales + is; +- device const half * dh = &x[i].d; + device const float * yb = y + ix * QK_K + ib * 32 + il * 8; -+ + +- device const float * y = yy + i * QK_K + y_offset; + uint32_t aux32[2]; + thread const uint8_t * q8 = (thread const uint8_t *)aux32; -+ + +- for (short l = 0; l < 4; ++l) { +- yl[4*l + 0] = y[l + 0]; +- yl[4*l + 1] = y[l + 32]; +- yl[4*l + 2] = y[l + 64]; +- yl[4*l + 3] = y[l + 96]; +- } + float4 qf1, qf2; -+ + +- for (short row = 0; row < nr0; ++row) { +- float4 sums = {0.f, 0.f, 0.f, 0.f}; + // [TAG_MUL_MV_WEIRD] + for (int ibl = ix; ibl < nb && ibl < ns01; ibl += 2) { + device const float4 * y4 = (device const float4 *)yb; @@ -27679,13 +27438,25 @@ index 969fddfa..b7b7e065 100644 + yl[1] = y4[4]; + yl[2] = y4[1]; + yl[3] = y4[5]; -+ + +- FOR_UNROLL (short l = 0; l < 4; ++l) { +- sums[0] += yl[4*l + 0] * ((int8_t)((q1[l] & 0xF) | ((qh[l] & kmask1) << 4)) - 32); +- sums[1] += yl[4*l + 1] * ((int8_t)((q2[l] & 0xF) | ((qh[l] & kmask2) << 2)) - 32); +- sums[2] += yl[4*l + 2] * ((int8_t)((q1[l] >> 4) | ((qh[l] & kmask3) << 0)) - 32); +- sums[3] += yl[4*l + 3] * ((int8_t)((q2[l] >> 4) | ((qh[l] & kmask4) >> 2)) - 32); +- } + for (short row = 0; row < NR0; ++row) { + device const block_iq4_xs & xb = x[row*ns01 + ibl]; + device const uint32_t * q4 = (device const uint32_t *)(xb.qs + 16*ib + 8*il); -+ + +- sumf[row] += dh[0] * (sums[0] * sc[0] + sums[1] * sc[2] + sums[2] * sc[4] + sums[3] * sc[6]); + float4 acc1 = {0.f}, acc2 = {0.f}; -+ + +- q1 += args.nb01; +- q2 += args.nb01; +- qh += args.nb01; +- sc += args.nb01; +- dh += args.nb01/2; + aux32[0] = (q4[0] ) & 0x0f0f0f0f; + aux32[1] = (q4[0] >> 4) & 0x0f0f0f0f; + qf1 = {shmem_f32[q8[0]], shmem_f32[q8[1]], shmem_f32[q8[2]], shmem_f32[q8[3]]}; @@ -27704,129 +27475,196 @@ index 969fddfa..b7b7e065 100644 + + const int ls = (((xb.scales_l[ib/2] >> 4*(ib%2)) & 0xf) | (((xb.scales_h >> 2*ib) & 3) << 4)) - 32; + sumf[row] += (float)xb.d * ls * (acc1[0] + acc1[1] + acc1[2] + acc1[3]); -+ } + } + + yb += 2 * QK_K; -+ } -+ -+ device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; -+ + } + + device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; + +- for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) { + for (int row = 0; row < NR0 && first_row + row < args.ne0; ++row) { -+ float sum_all = simd_sum(sumf[row]); -+ if (tiisg == 0) { -+ dst_f32[first_row + row] = sum_all; -+ } -+ } -+} -+ + float sum_all = simd_sum(sumf[row]); + if (tiisg == 0) { + dst_f32[first_row + row] = sum_all; +@@ -8705,23 +14419,22 @@ void kernel_mul_mv_q6_K_f32_impl( + } + } + +-[[host_name("kernel_mul_mv_q6_K_f32")]] +-kernel void kernel_mul_mv_q6_K_f32( +[[host_name("kernel_mul_mv_iq4_xs_f32")]] +kernel void kernel_mul_mv_iq4_xs_f32( -+ constant ggml_metal_kargs_mul_mv & args, -+ device const char * src0, -+ device const char * src1, -+ device char * dst, + constant ggml_metal_kargs_mul_mv & args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem [[threadgroup(0)]], -+ uint3 tgpig[[threadgroup_position_in_grid]], -+ ushort tiisg[[thread_index_in_simdgroup]], -+ ushort sgitg[[simdgroup_index_in_threadgroup]]) { -+ + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + +- kernel_mul_mv_q6_K_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); + kernel_mul_mv_iq4_xs_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); -+} -+ + } + +-// ======================= "True" 2-bit +- +-template +-void kernel_mul_mv_iq2_xxs_f32_impl( +template +void kernel_mul_mv_mxfp4_f32_impl( -+ args_t args, -+ device const char * src0, -+ device const char * src1, -+ device char * dst, -+ threadgroup char * shmem, -+ uint3 tgpig, -+ ushort tiisg, -+ ushort sgitg) { -+ const short NSG = FC_mul_mv_nsg; -+ + args_t args, + device const char * src0, + device const char * src1, +@@ -8732,13 +14445,13 @@ void kernel_mul_mv_iq2_xxs_f32_impl( + ushort sgitg) { + const short NSG = FC_mul_mv_nsg; + +- const int nb = args.ne00/QK_K; + threadgroup float * shmem_f32 = (threadgroup float *) shmem; -+ -+ const int r0 = tgpig.x; -+ const int r1 = tgpig.y; -+ const int im = tgpig.z; -+ + + const int r0 = tgpig.x; + const int r1 = tgpig.y; + const int im = tgpig.z; + +- const int first_row = (r0 * NSG + sgitg) * nr0; + const int first_row = (r0 * NSG + sgitg) * NR0; -+ -+ const uint i12 = im%FC_mul_mv_ne12; -+ const uint i13 = im/FC_mul_mv_ne12; -+ -+ const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; -+ const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; -+ + + const uint i12 = im%FC_mul_mv_ne12; + const uint i13 = im/FC_mul_mv_ne12; +@@ -8746,77 +14459,62 @@ void kernel_mul_mv_iq2_xxs_f32_impl( + const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + +- device const block_iq2_xxs * x = (device const block_iq2_xxs *) (src0 + offset0); +- device const float * y = (device const float *) (src1 + offset1); +- +- float yl[32]; +- float sumf[nr0]={0.f}; + device const block_mxfp4 * x = (device const block_mxfp4 *) (src0 + offset0); + device const float * y = (device const float *) (src1 + offset1); -+ + +- const int nb32 = nb * (QK_K / 32); + const int nb = args.ne00/QK_MXFP4; + const int ns01 = args.nb01/args.nb00; // this can be larger than nb for permuted src0 tensors -+ + +- threadgroup uint64_t * svalues = (threadgroup uint64_t *)(shmem); +- threadgroup uint8_t * ssigns = (threadgroup uint8_t *)(svalues + 256); +- { +- int nval = 4; +- int pos = (32*sgitg + tiisg)*nval; +- for (int i = 0; i < nval; ++i) svalues[pos + i] = iq2xxs_grid[pos + i]; +- nval = 2; +- pos = (32*sgitg + tiisg)*nval; +- for (int i = 0; i < nval; ++i) ssigns[pos+i] = ksigns_iq2xs[pos+i]; +- threadgroup_barrier(mem_flags::mem_threadgroup); +- } + const short ix = tiisg/2; // 0...15 + const short it = tiisg%2; // 0 or 1 -+ + +- const int ix = tiisg; + shmem_f32[tiisg] = kvalues_mxfp4_f[tiisg%16]; + threadgroup_barrier(mem_flags::mem_threadgroup); -+ + +- device const float * y4 = y + 32 * ix; + float4 yl[4]; + float sumf[NR0]={0.f}; -+ + +- for (int ib32 = ix; ib32 < nb32; ib32 += 32) { +- for (short i = 0; i < 32; ++i) { +- yl[i] = y4[i]; +- } + device const float * yb = y + ix*QK_MXFP4 + it*8; -+ + +- const int ibl = ib32 / (QK_K / 32); +- const int ib = ib32 % (QK_K / 32); + // note: just the check `ib < nb` is enough, but adding the redundant `&& ib < ns01` check makes the kernel a bit faster + // no idea why that is - needs some deeper investigation [TAG_MUL_MV_WEIRD] + for (int ib = ix; ib < nb && ib < ns01; ib += 16) { + device const float4 * y4 = (device const float4 *) yb; -+ + +- device const block_iq2_xxs * xr = x + ibl; +- device const uint16_t * q2 = xr->qs + 4 * ib; +- device const half * dh = &xr->d; + yl[0] = y4[0]; + yl[1] = y4[4]; + yl[2] = y4[1]; + yl[3] = y4[5]; -+ + +- for (short row = 0; row < nr0; row++) { +- const float db = dh[0]; +- device const uint8_t * aux8 = (device const uint8_t *)q2; +- const uint32_t aux32 = q2[2] | (q2[3] << 16); +- const float d = db * (0.5f + (aux32 >> 28)); + FOR_UNROLL (short row = 0; row < NR0; row++) { + device const block_mxfp4 & xb = x[row*ns01 + ib]; + device const uint8_t * q2 = (device const uint8_t *)(xb.qs + 8*it); -+ + +- float sum = 0; +- for (short l = 0; l < 4; ++l) { +- const threadgroup uint8_t * grid = (const threadgroup uint8_t *)(svalues + aux8[l]); +- const uint8_t signs = ssigns[(aux32 >> 7*l) & 127]; +- for (short j = 0; j < 8; ++j) { +- sum += yl[8*l + j] * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f); +- } +- } +- sumf[row] += d * sum; + float4 acc1 = yl[0]*float4(shmem_f32[q2[0] & 0x0F], shmem_f32[q2[1] & 0x0F], shmem_f32[q2[2] & 0x0F], shmem_f32[q2[3] & 0x0F]); + float4 acc2 = yl[1]*float4(shmem_f32[q2[0] >> 4 ], shmem_f32[q2[1] >> 4 ], shmem_f32[q2[2] >> 4 ], shmem_f32[q2[3] >> 4 ]); + float4 acc3 = yl[2]*float4(shmem_f32[q2[4] & 0x0F], shmem_f32[q2[5] & 0x0F], shmem_f32[q2[6] & 0x0F], shmem_f32[q2[7] & 0x0F]); + float4 acc4 = yl[3]*float4(shmem_f32[q2[4] >> 4 ], shmem_f32[q2[5] >> 4 ], shmem_f32[q2[6] >> 4 ], shmem_f32[q2[7] >> 4 ]); -+ + +- dh += args.nb01/2; +- q2 += args.nb01/2; + acc1 = (acc1 + acc3) + (acc2 + acc4); + + sumf[row] += e8m0_to_fp32(xb.e) * ((acc1[0] + acc1[1]) + (acc1[2] + acc1[3])); -+ } -+ + } + +- y4 += 32 * 32; + yb += 16 * QK_MXFP4; -+ } -+ -+ device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; -+ + } + + device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; + +- for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) { + for (int row = 0; row < NR0 && first_row + row < args.ne0; ++row) { -+ float sum_all = simd_sum(sumf[row]); -+ if (tiisg == 0) { + float sum_all = simd_sum(sumf[row]); + if (tiisg == 0) { +- dst_f32[first_row + row] = sum_all * 0.25f; + dst_f32[first_row + row] = sum_all; -+ } -+ } -+} -+ + } + } + } + +-[[host_name("kernel_mul_mv_iq2_xxs_f32")]] +-kernel void kernel_mul_mv_iq2_xxs_f32( +[[host_name("kernel_mul_mv_mxfp4_f32")]] +kernel void kernel_mul_mv_mxfp4_f32( -+ constant ggml_metal_kargs_mul_mv & args, -+ device const char * src0, -+ device const char * src1, -+ device char * dst, -+ threadgroup char * shmem [[threadgroup(0)]], -+ uint3 tgpig[[threadgroup_position_in_grid]], -+ ushort tiisg[[thread_index_in_simdgroup]], -+ ushort sgitg[[simdgroup_index_in_threadgroup]]) { + constant ggml_metal_kargs_mul_mv & args, + device const char * src0, + device const char * src1, +@@ -8825,1057 +14523,2359 @@ kernel void kernel_mul_mv_iq2_xxs_f32( + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { +- kernel_mul_mv_iq2_xxs_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + + kernel_mul_mv_mxfp4_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); -+} -+ + } + +-template +-void kernel_mul_mv_iq2_xs_f32_impl( +- args_t args, +- device const char * src0, +- device const char * src1, +- device char * dst, +- threadgroup char * shmem, +- uint3 tgpig, +- ushort tiisg, +- ushort sgitg) { +- const short NSG = FC_mul_mv_nsg; +template +kernel void kernel_get_rows_q( + constant ggml_metal_kargs_get_rows & args, @@ -27840,24 +27678,35 @@ index 969fddfa..b7b7e065 100644 + const int32_t i10 = tgpig.x%args.ne10; + const int32_t i11 = tgpig.y; + const int32_t i12 = tgpig.z; -+ + +- const int nb = args.ne00/QK_K; + const int32_t r = ((const device int32_t *) ((const device char *) src1 + i12*args.nb12 + i11*args.nb11 + i10*args.nb10))[0]; -+ + +- const int r0 = tgpig.x; +- const int r1 = tgpig.y; +- const int im = tgpig.z; + const int32_t i02 = i11; + const int32_t i03 = i12; -+ + +- const int first_row = (r0 * NSG + sgitg) * nr0; + auto psrc = (device const block_q *) ((const device char *) src0 + i03*args.nb03 + i02*args.nb02 + r*args.nb01); + auto pdst = (device float4x4 *) (( device char *) dst + i12*args.nb3 + i11*args.nb2 + i10*args.nb1); -+ + +- const uint i12 = im%FC_mul_mv_ne12; +- const uint i13 = im/FC_mul_mv_ne12; + for (int ind = iw0*ntg.x + tiitg; ind < args.ne00t;) { + float4x4 temp; + dequantize_func(psrc + ind/nl, ind%nl, temp); + pdst[ind] = temp; -+ + +- const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; +- const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + break; + } +} -+ + +- device const block_iq2_xs * x = (device const block_iq2_xs *) (src0 + offset0); +- device const float * y = (device const float *) (src1 + offset1); +template +kernel void kernel_get_rows_f( + constant ggml_metal_kargs_get_rows & args, @@ -27871,12 +27720,25 @@ index 969fddfa..b7b7e065 100644 + const int32_t i10 = tgpig.x%args.ne10; + const int32_t i11 = tgpig.y; + const int32_t i12 = tgpig.z; -+ + +- float yl[32]; +- float sumf[nr0]={0.f}; + const int32_t r = ((const device int32_t *) ((const device char *) src1 + i12*args.nb12 + i11*args.nb11 + i10*args.nb10))[0]; -+ + +- const int nb32 = nb * (QK_K / 32); + const int32_t i02 = i11; + const int32_t i03 = i12; -+ + +- threadgroup uint64_t * svalues = (threadgroup uint64_t *)(shmem); +- threadgroup uint8_t * ssigns = (threadgroup uint8_t *)(svalues + 512); +- { +- int nval = 8; +- int pos = (32*sgitg + tiisg)*nval; +- for (int i = 0; i < nval; ++i) svalues[pos + i] = iq2xs_grid[pos + i]; +- nval = 2; +- pos = (32*sgitg + tiisg)*nval; +- for (int i = 0; i < nval; ++i) ssigns[pos+i] = ksigns_iq2xs[pos+i]; +- threadgroup_barrier(mem_flags::mem_threadgroup); + auto psrc = (const device T0 *) ((const device char *) src0 + i03*args.nb03 + i02*args.nb02 + r*args.nb01); + auto pdst = ( device T *) (( device char *) dst + i12*args.nb3 + i11*args.nb2 + i10*args.nb1); + @@ -27884,9 +27746,10 @@ index 969fddfa..b7b7e065 100644 + pdst[ind] = psrc[ind]; + + break; -+ } + } +} -+ + +- const int ix = tiisg; +kernel void kernel_get_rows_f16_f16_vec4( + constant ggml_metal_kargs_get_rows & args, + device const void * src0, @@ -27899,9 +27762,13 @@ index 969fddfa..b7b7e065 100644 + const int32_t i10 = tgpig.x%args.ne10; + const int32_t i11 = tgpig.y; + const int32_t i12 = tgpig.z; -+ + +- device const float * y4 = y + 32 * ix; + const int32_t r = ((const device int32_t *) ((const device char *) src1 + i12*args.nb12 + i11*args.nb11 + i10*args.nb10))[0]; -+ + +- for (int ib32 = ix; ib32 < nb32; ib32 += 32) { +- for (short i = 0; i < 32; ++i) { +- yl[i] = y4[i]; + const int32_t i02 = i11; + const int32_t i03 = i12; + @@ -27916,12 +27783,18 @@ index 969fddfa..b7b7e065 100644 + for (int i = base; i < args.ne00; ++i) { + pdst[i] = psrc[i]; + } -+ } -+ + } + +- const int ibl = ib32 / (QK_K / 32); +- const int ib = ib32 % (QK_K / 32); + break; + } +} -+ + +- device const block_iq2_xs * xr = x + ibl; +- device const uint16_t * q2 = xr->qs + 4 * ib; +- device const uint8_t * sc = xr->scales + ib; +- device const half * dh = &xr->d; +kernel void kernel_get_rows_f16_f16_packed_rows( + constant ggml_metal_kargs_get_rows & args, + device const void * src0, @@ -27934,12 +27807,34 @@ index 969fddfa..b7b7e065 100644 + if (i10 >= args.ne10) { + return; + } -+ + +- for (short row = 0; row < nr0; row++) { +- const float db = dh[0]; +- const uint8_t ls1 = sc[0] & 0xf; +- const uint8_t ls2 = sc[0] >> 4; +- const float d1 = db * (0.5f + ls1); +- const float d2 = db * (0.5f + ls2); + const int32_t i11 = tgpig.y; + const int32_t i12 = tgpig.z; + const int32_t r = ((const device int32_t *) ((const device char *) src1 + + i12*args.nb12 + i11*args.nb11 + i10*args.nb10))[0]; -+ + +- float sum1 = 0, sum2 = 0; +- for (short l = 0; l < 2; ++l) { +- const threadgroup uint8_t * grid = (const threadgroup uint8_t *)(svalues + (q2[l] & 511)); +- const uint8_t signs = ssigns[(q2[l] >> 9)]; +- for (short j = 0; j < 8; ++j) { +- sum1 += yl[8*l + j] * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f); +- } +- } +- for (short l = 2; l < 4; ++l) { +- const threadgroup uint8_t * grid = (const threadgroup uint8_t *)(svalues + (q2[l] & 511)); +- const uint8_t signs = ssigns[(q2[l] >> 9)]; +- for (short j = 0; j < 8; ++j) { +- sum2 += yl[8*l + j] * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f); +- } +- } +- sumf[row] += d1 * sum1 + d2 * sum2; + const int32_t i02 = i11; + const int32_t i03 = i12; + @@ -30879,7 +30774,7 @@ index 969fddfa..b7b7e065 100644 template [[host_name("kernel_get_rows_q1_0")]] kernel get_rows_q_t kernel_get_rows_q; template [[host_name("kernel_get_rows_q2_0")]] kernel get_rows_q_t kernel_get_rows_q; -@@ -9837,101 +16837,126 @@ template [[host_name("kernel_get_rows_iq1_s")]] kernel get_rows_q_t kernel_get +@@ -9899,101 +16899,126 @@ template [[host_name("kernel_get_rows_iq1_s")]] kernel get_rows_q_t kernel_get template [[host_name("kernel_get_rows_iq1_m")]] kernel get_rows_q_t kernel_get_rows_q; template [[host_name("kernel_get_rows_iq4_nl")]] kernel get_rows_q_t kernel_get_rows_q; template [[host_name("kernel_get_rows_iq4_xs")]] kernel get_rows_q_t kernel_get_rows_q; @@ -31079,7 +30974,7 @@ index 969fddfa..b7b7e065 100644 kernel void kernel_diag_f32( constant ggml_metal_kargs_diag & args, -@@ -10675,6 +17700,76 @@ kernel void kernel_mul_mm_id( +@@ -10737,6 +17762,76 @@ kernel void kernel_mul_mm_id( } } @@ -31156,7 +31051,7 @@ index 969fddfa..b7b7e065 100644 // // matrix-matrix multiplication // -@@ -10928,6 +18023,7 @@ template [[host_name("kernel_mul_mv_id_q8_0_f32")]] kernel kernel_mul_mv_id_t +@@ -10990,6 +18085,7 @@ template [[host_name("kernel_mul_mv_id_q8_0_f32")]] kernel kernel_mul_mv_id_t template [[host_name("kernel_mul_mv_id_q1_0_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; template [[host_name("kernel_mul_mv_id_q2_0_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; @@ -31164,7 +31059,7 @@ index 969fddfa..b7b7e065 100644 template [[host_name("kernel_mul_mv_id_q4_0_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; template [[host_name("kernel_mul_mv_id_q4_1_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; template [[host_name("kernel_mul_mv_id_q5_0_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; -@@ -10936,7 +18032,15 @@ template [[host_name("kernel_mul_mv_id_q5_1_f32")]] kernel kernel_mul_mv_id_t +@@ -10998,7 +18094,15 @@ template [[host_name("kernel_mul_mv_id_q5_1_f32")]] kernel kernel_mul_mv_id_t template [[host_name("kernel_mul_mv_id_mxfp4_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; template [[host_name("kernel_mul_mv_id_q2_K_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; @@ -31466,13 +31361,13 @@ index a7d1fe7d..cdc85b08 100644 struct ggml_hash_set ggml_hash_set_new(size_t size) { diff --git a/src/llama-hparams.h b/src/llama-hparams.h -index 747754fc..6cca1d33 100644 +index fc770bf0..a9f04795 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h -@@ -226,6 +226,10 @@ struct llama_hparams { - uint32_t indexer_n_head = 0; - uint32_t indexer_head_size = 0; - uint32_t indexer_top_k = 0; +@@ -232,6 +232,10 @@ struct llama_hparams { + uint32_t indexer_local_blocks = 0; + // MSA stores its indexer keys in the main KV cache (k_idx tensors); + bool indexer_kv = false; + uint32_t indexer_top_k_freq = 0; + uint32_t indexer_skip_top_k_offset = 0; + bool indexer_types_present = false; diff --git a/third_party/llama.cpp/patches/0022-skippy-expose-GLM-DSA-staged-runtime-controls.patch b/third_party/llama.cpp/patches/0022-skippy-expose-GLM-DSA-staged-runtime-controls.patch index 88487057d3..2d4cfec132 100644 --- a/third_party/llama.cpp/patches/0022-skippy-expose-GLM-DSA-staged-runtime-controls.patch +++ b/third_party/llama.cpp/patches/0022-skippy-expose-GLM-DSA-staged-runtime-controls.patch @@ -1,7 +1,7 @@ -From 02b458479060c5ae055c4a9973a66884ed55c6e9 Mon Sep 17 00:00:00 2001 +From 338da87701a5aafef4e1cab202ae4dd0adaad427 Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Tue, 14 Jul 2026 09:16:19 +1000 -Subject: [PATCH 22/45] skippy: expose GLM-DSA staged runtime controls +Subject: [PATCH 22/47] skippy: expose GLM-DSA staged runtime controls --- include/skippy.h | 20 + @@ -56,7 +56,7 @@ index 50c08515..8f166e9c 100644 struct skippy_tensor_info { diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp -index 6613747a..244a400a 100644 +index 1363deb5..126a434b 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -25,6 +25,7 @@ static thread_local skippy_graph_filter g_skippy_graph_filter; diff --git a/third_party/llama.cpp/patches/0023-tests-cover-native-GLM-DSA-execution-paths.patch b/third_party/llama.cpp/patches/0023-tests-cover-native-GLM-DSA-execution-paths.patch index 537f0285a0..24122f2c5f 100644 --- a/third_party/llama.cpp/patches/0023-tests-cover-native-GLM-DSA-execution-paths.patch +++ b/third_party/llama.cpp/patches/0023-tests-cover-native-GLM-DSA-execution-paths.patch @@ -1,7 +1,7 @@ -From dbce275745b31af70ea3af5d023d2012da818c6d Mon Sep 17 00:00:00 2001 +From 05c9d6a2ccf5435db13abdca6026f10341147678 Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Tue, 14 Jul 2026 09:16:32 +1000 -Subject: [PATCH 23/45] tests: cover native GLM-DSA execution paths +Subject: [PATCH 23/47] tests: cover native GLM-DSA execution paths --- scripts/glm52-phase-d-policy-report.sh | 287 ++ @@ -1262,7 +1262,7 @@ index 00000000..8a1e0f38 + +echo "wrote ${summary}" diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp -index e7cd6d0c..0a7c2997 100644 +index b4061e35..ce4801ba 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -51,6 +51,87 @@ @@ -6897,7 +6897,7 @@ index e7cd6d0c..0a7c2997 100644 for (ggml_type type : all_types) { for (int b : {1, 7}) { for (bool v : {false, true}) { -@@ -8807,6 +14052,18 @@ static std::vector> make_test_cases_eval() { +@@ -8810,6 +14055,18 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 8192, 512, 5120, {128, 1}, {1, 1})); #endif @@ -6916,7 +6916,7 @@ index e7cd6d0c..0a7c2997 100644 for (ggml_type type_a : all_types) { for (int i = 1; i < 10; ++i) { test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 16, i, 256, { 1, 1}, {1, 1})); -@@ -9604,6 +14861,218 @@ static std::vector> make_test_cases_eval() { +@@ -9607,6 +14864,218 @@ static std::vector> make_test_cases_eval() { } } } @@ -7135,7 +7135,7 @@ index e7cd6d0c..0a7c2997 100644 test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 1, 1)); test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 16, 1, 1)); -@@ -9671,6 +15140,10 @@ static std::vector> make_test_cases_eval() { +@@ -9674,6 +15143,10 @@ static std::vector> make_test_cases_eval() { } } @@ -7146,7 +7146,7 @@ index e7cd6d0c..0a7c2997 100644 return test_cases; } #ifdef _MSC_VER -@@ -9827,6 +15300,21 @@ static std::vector> make_test_cases_perf() { +@@ -9834,6 +15307,21 @@ static std::vector> make_test_cases_perf() { } } @@ -7168,7 +7168,7 @@ index e7cd6d0c..0a7c2997 100644 // qwen3-30b-a3b for (int bs : {1, 4, 8, 32, 64, 128, 256, 512}) { for (ggml_type type_a : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, GGML_TYPE_Q4_K, GGML_TYPE_Q6_K, GGML_TYPE_IQ2_XS}) { -@@ -9846,6 +15334,266 @@ static std::vector> make_test_cases_perf() { +@@ -9853,6 +15341,266 @@ static std::vector> make_test_cases_perf() { } } @@ -7436,7 +7436,7 @@ index e7cd6d0c..0a7c2997 100644 // gpt-oss-20b for (int bs : {1, 4, 8, 512}) { diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp -index 86c3051c..a2e79125 100644 +index d02e65c9..659ea9e9 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -14,6 +14,7 @@ @@ -7486,14 +7486,16 @@ index 86c3051c..a2e79125 100644 } else if (arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE) { n_layer = 3; } else if (arch == LLM_ARCH_CHAMELEON) { -@@ -199,8 +219,33 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { - } - - ms.add_kv(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, uint32_t(1)); -- ms.add_kv(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, uint32_t(64)); -- ms.add_kv(LLM_KV_ATTENTION_INDEXER_TOP_K, uint32_t(8)); -+ ms.add_kv(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, arch == LLM_ARCH_GLM_DSA ? uint32_t(128) : uint32_t(64)); -+ ms.add_kv(LLM_KV_ATTENTION_INDEXER_TOP_K, arch == LLM_ARCH_GLM_DSA ? glm_dsa_indexer_top_k : uint32_t(8)); +@@ -204,10 +224,35 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { + // MSA requires one indexer head per GQA (KV) head, unlike the DSA archs where the + // indexer head count is independent of the main attention head count. + ms.add_kv(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, arch == LLM_ARCH_MINIMAX_M3 ? n_head : uint32_t(1)); +- ms.add_kv(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, uint32_t(64)); +- ms.add_kv(LLM_KV_ATTENTION_INDEXER_TOP_K, uint32_t(8)); ++ ms.add_kv(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, arch == LLM_ARCH_GLM_DSA ? uint32_t(128) : uint32_t(64)); ++ ms.add_kv(LLM_KV_ATTENTION_INDEXER_TOP_K, arch == LLM_ARCH_GLM_DSA ? glm_dsa_indexer_top_k : uint32_t(8)); + ms.add_kv(LLM_KV_ATTENTION_INDEXER_BLOCK_SIZE, uint32_t(4)); + ms.add_kv(LLM_KV_ATTENTION_INDEXER_LOCAL_BLOCKS, uint32_t(1)); + if (arch == LLM_ARCH_GLM_DSA) { + if (glm_dsa_indexshare == glm_dsa_indexshare_fixture::CONFLICTING_METADATA) { + ms.add_kv(LLM_KV_ATTENTION_INDEXER_TOP_K_FREQUENCY, uint32_t(1)); @@ -7522,7 +7524,7 @@ index 86c3051c..a2e79125 100644 ms.add_kv(LLM_KV_ROPE_DIMENSION_SECTIONS, std::vector({n_embd_head/4, n_embd_head/4, n_embd_head/4, n_embd_head/4})); ms.add_kv(LLM_KV_TOKENIZER_MODEL, "no_vocab"); // ms.add_kv(LLM_KV_DENSE_2_FEAT_OUT, n_embd); -@@ -254,9 +299,242 @@ static bool silent_model_load_progress(float /*progress*/, void * /*user_data*/) +@@ -261,9 +306,242 @@ static bool silent_model_load_progress(float /*progress*/, void * /*user_data*/) return true; } @@ -7766,7 +7768,7 @@ index 86c3051c..a2e79125 100644 GGML_ASSERT((gguf_ctx == nullptr) != (file == nullptr)); llama_model_params model_params = llama_model_default_params(); model_params.progress_callback = silent_model_load_progress; -@@ -272,6 +550,9 @@ static std::pair get_model_and_ctx( +@@ -279,6 +557,9 @@ static std::pair get_model_and_ctx( if (!encode) { ctx_params.n_ubatch = 64; } @@ -7776,7 +7778,7 @@ index 86c3051c..a2e79125 100644 size_t tmp = seed; llama_model_ptr model(gguf_ctx != nullptr ? -@@ -287,6 +568,656 @@ static std::pair get_model_and_ctx( +@@ -294,6 +575,656 @@ static std::pair get_model_and_ctx( return std::make_pair(std::move(model), std::move(lctx)); } @@ -8433,7 +8435,7 @@ index 86c3051c..a2e79125 100644 static std::vector get_logits( llama_model * model, llama_context * lctx, const std::vector & tokens, bool encode = false) { const uint32_t n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model)); -@@ -430,6 +1361,20 @@ static bool arch_supported(const llm_arch arch) { +@@ -438,6 +1369,20 @@ static bool arch_supported(const llm_arch arch) { return true; } @@ -8454,7 +8456,7 @@ index 86c3051c..a2e79125 100644 static int save_models(const llm_arch target_arch, const size_t seed, const ggml_log_level log_level, const std::string & dir) { struct user_data_t { struct { -@@ -543,6 +1488,33 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const gg +@@ -551,6 +1496,33 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const gg const std::string template_row_res = "%15s %10s|%20s|\n"; bool all_ok = true; @@ -8488,7 +8490,7 @@ index 86c3051c..a2e79125 100644 common_log_flush(common_log_main()); printf(template_header.c_str(), "Model arch.", "Device", "Config", "NMSE vs. CPU", "Roundtrip"); printf("|"); -@@ -592,6 +1564,9 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const gg +@@ -600,6 +1572,9 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const gg std::string status_roundtrip = "\033[1;33mSKIP\033[0m"; char nmse_str[12] = {0}; bool skip = !arch_supported(arch) || (dc.split_mode == LLAMA_SPLIT_MODE_TENSOR && dc.devs.empty()); diff --git a/third_party/llama.cpp/patches/0024-Support-GLM-DSA-fused-KV_B-tensors.patch b/third_party/llama.cpp/patches/0024-Support-GLM-DSA-fused-KV_B-tensors.patch index 6cbe4bfd16..226f18f708 100644 --- a/third_party/llama.cpp/patches/0024-Support-GLM-DSA-fused-KV_B-tensors.patch +++ b/third_party/llama.cpp/patches/0024-Support-GLM-DSA-fused-KV_B-tensors.patch @@ -1,7 +1,7 @@ -From 358fdff04cab3ca3d85cd51dbc5592a1378df9fc Mon Sep 17 00:00:00 2001 +From e040dfebf0ff158ce57fe8d063b90ed84646ff8e Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Wed, 24 Jun 2026 15:35:23 +1000 -Subject: [PATCH 24/45] Support GLM-DSA fused KV_B tensors +Subject: [PATCH 24/47] Support GLM-DSA fused KV_B tensors --- src/models/deepseek2.cpp | 23 +++++++++++++++++------ diff --git a/third_party/llama.cpp/patches/0025-Run-GLM-DSA-through-IndexShare-graph.patch b/third_party/llama.cpp/patches/0025-Run-GLM-DSA-through-IndexShare-graph.patch index cb59f5881c..f33ed194d1 100644 --- a/third_party/llama.cpp/patches/0025-Run-GLM-DSA-through-IndexShare-graph.patch +++ b/third_party/llama.cpp/patches/0025-Run-GLM-DSA-through-IndexShare-graph.patch @@ -1,7 +1,7 @@ -From 89bf8ceca0a8dfdccf9abed3f859a1c80af21948 Mon Sep 17 00:00:00 2001 +From 03e3ece73ac2c07e17f909bda82bfd9ba2a83ec0 Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Thu, 25 Jun 2026 15:51:20 +1000 -Subject: [PATCH 25/45] Run GLM-DSA through IndexShare graph +Subject: [PATCH 25/47] Run GLM-DSA through IndexShare graph --- src/models/glm-dsa.cpp | 43 ++++++++++++++++++++++++++++++++++++++---- diff --git a/third_party/llama.cpp/patches/0026-Bump-Skippy-ABI-for-GLM-DSA-runtime-config.patch b/third_party/llama.cpp/patches/0026-Bump-Skippy-ABI-for-GLM-DSA-runtime-config.patch index 7ee7eb102c..24a153662c 100644 --- a/third_party/llama.cpp/patches/0026-Bump-Skippy-ABI-for-GLM-DSA-runtime-config.patch +++ b/third_party/llama.cpp/patches/0026-Bump-Skippy-ABI-for-GLM-DSA-runtime-config.patch @@ -1,7 +1,7 @@ -From f9e182d9e48b35b1e9e445b6267bac73fc472291 Mon Sep 17 00:00:00 2001 +From 7aed9c3cb636118b9a06351931fc8785ac425f6d Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Tue, 21 Jul 2026 15:31:56 +1000 -Subject: [PATCH 26/45] Bump Skippy ABI for GLM DSA runtime config +Subject: [PATCH 26/47] Bump Skippy ABI for GLM DSA runtime config --- include/skippy/common.h | 2 +- diff --git a/third_party/llama.cpp/patches/0027-Fix-GLM-DSA-Metal-get_rows-placement.patch b/third_party/llama.cpp/patches/0027-Fix-GLM-DSA-Metal-get_rows-placement.patch index 50416e2eb2..f3dfac6a47 100644 --- a/third_party/llama.cpp/patches/0027-Fix-GLM-DSA-Metal-get_rows-placement.patch +++ b/third_party/llama.cpp/patches/0027-Fix-GLM-DSA-Metal-get_rows-placement.patch @@ -1,17 +1,17 @@ -From 9b1b59b34069b19c59f280e1fb4c4480e3b173d0 Mon Sep 17 00:00:00 2001 +From 84cfaaf9d48e6a146ef446e779b29db3f63e628e Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Wed, 22 Jul 2026 15:13:16 +1000 -Subject: [PATCH 27/45] Fix GLM DSA Metal get_rows placement +Subject: [PATCH 27/47] Fix GLM DSA Metal get_rows placement --- ggml/src/ggml-metal/ggml-metal.metal | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metal -index b7b7e065..7a6df158 100644 +index 2f1fe190..7674aef1 100644 --- a/ggml/src/ggml-metal/ggml-metal.metal +++ b/ggml/src/ggml-metal/ggml-metal.metal -@@ -16815,28 +16815,6 @@ kernel void kernel_dsa_sparse_attn_decode_grouped_impl( +@@ -16877,28 +16877,6 @@ kernel void kernel_dsa_sparse_attn_decode_grouped_impl( reduce[reduce_offset + tid] = local_active_top_end; threadgroup_barrier(mem_flags::mem_threadgroup); diff --git a/third_party/llama.cpp/patches/0028-ggml-default-GLM-MoE-two-phase-Metal-path.patch b/third_party/llama.cpp/patches/0028-ggml-default-GLM-MoE-two-phase-Metal-path.patch index 2d5d3b6e2a..adb6e15abb 100644 --- a/third_party/llama.cpp/patches/0028-ggml-default-GLM-MoE-two-phase-Metal-path.patch +++ b/third_party/llama.cpp/patches/0028-ggml-default-GLM-MoE-two-phase-Metal-path.patch @@ -1,7 +1,7 @@ -From 1f9c781f915ef3ea0f71f48660272b1c31e19d9e Mon Sep 17 00:00:00 2001 +From 78bd4db176b5437a43ed58fd51a8f6c304465798 Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Sun, 19 Jul 2026 11:27:53 +1000 -Subject: [PATCH 28/45] ggml: default GLM MoE two-phase Metal path +Subject: [PATCH 28/47] ggml: default GLM MoE two-phase Metal path --- ggml/src/ggml-metal/ggml-metal-ops.cpp | 9 ++++++++- @@ -9,7 +9,7 @@ Subject: [PATCH 28/45] ggml: default GLM MoE two-phase Metal path 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp -index 36378f8e..684efdc6 100644 +index e22add77..c7e21f34 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -127,7 +127,14 @@ static bool ggml_metal_glm_dsa_moe_private_scratch_enabled() { @@ -29,7 +29,7 @@ index 36378f8e..684efdc6 100644 static bool ggml_metal_glm_dsa_moe_dual_lane_enabled() { diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp -index 0a7c2997..cc7703c8 100644 +index ce4801ba..477c51cd 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -9835,6 +9835,17 @@ struct test_glm_ffn_two_phase_consumer : public test_glm_ffn_consumer { @@ -50,7 +50,7 @@ index 0a7c2997..cc7703c8 100644 bool eval_perf(ggml_backend_t backend, const char * op_names_filter, printer * output_printer) override { scoped_test_env motif_reference_env("GGML_METAL_ENABLE_GLM_MOE_DECODE_MOTIF_REFERENCE", "0"); scoped_test_env two_phase_env( -@@ -15142,6 +15153,7 @@ static std::vector> make_test_cases_eval() { +@@ -15145,6 +15156,7 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_glm_ffn_chain_native_down(true, "CANDIDATE")); test_cases.emplace_back(new test_glm_ffn_chain_two_phase(true, "CANDIDATE")); diff --git a/third_party/llama.cpp/patches/0029-ggml-add-GLM-MoE-Metal-selector-diagnostics.patch b/third_party/llama.cpp/patches/0029-ggml-add-GLM-MoE-Metal-selector-diagnostics.patch index 34f60b209c..02fcaac5af 100644 --- a/third_party/llama.cpp/patches/0029-ggml-add-GLM-MoE-Metal-selector-diagnostics.patch +++ b/third_party/llama.cpp/patches/0029-ggml-add-GLM-MoE-Metal-selector-diagnostics.patch @@ -1,7 +1,7 @@ -From d2550d768c3d538b50266b13861dc1ad39e266fa Mon Sep 17 00:00:00 2001 +From f67dc3032d5e7c66a7f6b312d88100588bd240b0 Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Sun, 19 Jul 2026 16:31:29 +1000 -Subject: [PATCH 29/45] ggml: add GLM MoE Metal selector diagnostics +Subject: [PATCH 29/47] ggml: add GLM MoE Metal selector diagnostics --- ggml/src/ggml-metal/ggml-metal-ops.cpp | 75 +++++++++++++++++++++----- @@ -9,7 +9,7 @@ Subject: [PATCH 29/45] ggml: add GLM MoE Metal selector diagnostics 2 files changed, 98 insertions(+), 16 deletions(-) diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp -index 684efdc6..ab41e1ec 100644 +index c7e21f34..d2fdef79 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -147,6 +147,11 @@ static bool ggml_metal_glm_dsa_moe_dual_lane_gate_slot4_enabled() { @@ -139,7 +139,7 @@ index 684efdc6..ab41e1ec 100644 fusion_scratch : ggml_metal_get_buffer_id(weighted_sum); diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp -index cc7703c8..50ec5225 100644 +index 477c51cd..e45661b0 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -9578,6 +9578,7 @@ struct test_glm_ffn_consumer : public test_case { @@ -200,7 +200,7 @@ index cc7703c8..50ec5225 100644 struct test_glm_ffn_dual_lane_consumer : public test_glm_ffn_consumer { const bool dual_lane; const char * variant; -@@ -15154,6 +15185,7 @@ static std::vector> make_test_cases_eval() { +@@ -15157,6 +15188,7 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_glm_ffn_chain_native_down(true, "CANDIDATE")); test_cases.emplace_back(new test_glm_ffn_chain_two_phase(true, "CANDIDATE")); test_cases.emplace_back(new test_glm_ffn_two_phase_consumer(true, "CANDIDATE")); @@ -208,7 +208,7 @@ index cc7703c8..50ec5225 100644 test_cases.emplace_back(new test_glm_ffn_dual_lane_consumer(true, "CANDIDATE")); return test_cases; -@@ -15524,6 +15556,7 @@ static std::vector> make_test_cases_perf() { +@@ -15531,6 +15563,7 @@ static std::vector> make_test_cases_perf() { GGML_TYPE_Q4_K, GGML_TYPE_Q4_K, GGML_TYPE_Q4_K, 6144, 2048, true, true)); test_cases.emplace_back(new test_glm_ffn_consumer(GGML_TYPE_Q2_K, GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, GGML_TYPE_Q4_K, GGML_TYPE_Q4_K, GGML_TYPE_Q4_K, 6144, 2048, true, false, true)); diff --git a/third_party/llama.cpp/patches/0030-ggml-skip-zero-weight-GLM-MoE-gate-up-slots.patch b/third_party/llama.cpp/patches/0030-ggml-skip-zero-weight-GLM-MoE-gate-up-slots.patch index 2bb8c01fc4..cbcc764f53 100644 --- a/third_party/llama.cpp/patches/0030-ggml-skip-zero-weight-GLM-MoE-gate-up-slots.patch +++ b/third_party/llama.cpp/patches/0030-ggml-skip-zero-weight-GLM-MoE-gate-up-slots.patch @@ -1,17 +1,17 @@ -From 8c3753398c77d59baa5cad70aa89875354d9573e Mon Sep 17 00:00:00 2001 +From 669be8e843c86ae71ceb754a334f6ec2b1071e54 Mon Sep 17 00:00:00 2001 From: James Dumay Date: Sun, 19 Jul 2026 00:00:00 +1000 -Subject: [PATCH 30/45] ggml: skip zero-weight GLM MoE gate-up slots +Subject: [PATCH 30/47] ggml: skip zero-weight GLM MoE gate-up slots --- ggml/src/ggml-metal/ggml-metal.metal | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metal -index 7a6df158..b5e37c11 100644 +index 7674aef1..c1e7e938 100644 --- a/ggml/src/ggml-metal/ggml-metal.metal +++ b/ggml/src/ggml-metal/ggml-metal.metal -@@ -10773,9 +10773,15 @@ void kernel_mul_mv_id_q2_K_gate_up_swiglu_pair_sg_dual_impl( +@@ -10835,9 +10835,15 @@ void kernel_mul_mv_id_q2_K_gate_up_swiglu_pair_sg_dual_impl( return; } @@ -29,7 +29,7 @@ index 7a6df158..b5e37c11 100644 if (active_slot && expert_id >= 0 && expert_id < args.ne02) { const uint64_t offset0 = uint64_t(expert_id)*args.nb02 + uint64_t(first_row)*args.nb01; const uint64_t offset1 = uint64_t(slot % args.ne11)*args.nb11 + uint64_t(token)*args.nb12; -@@ -10936,10 +10942,6 @@ void kernel_mul_mv_id_q2_K_gate_up_swiglu_pair_sg_dual_impl( +@@ -10998,10 +11004,6 @@ void kernel_mul_mv_id_q2_K_gate_up_swiglu_pair_sg_dual_impl( } } diff --git a/third_party/llama.cpp/patches/0031-tests-cover-full-GLM-MoE-selected-chain.patch b/third_party/llama.cpp/patches/0031-tests-cover-full-GLM-MoE-selected-chain.patch index 007865421a..07fbb8c596 100644 --- a/third_party/llama.cpp/patches/0031-tests-cover-full-GLM-MoE-selected-chain.patch +++ b/third_party/llama.cpp/patches/0031-tests-cover-full-GLM-MoE-selected-chain.patch @@ -1,14 +1,14 @@ -From a00ce995595582f131c501bb930578b3e049f879 Mon Sep 17 00:00:00 2001 +From fa5dc54744490caaa81acc230b9d232577c440e3 Mon Sep 17 00:00:00 2001 From: James Dumay Date: Sun, 19 Jul 2026 00:00:00 +1000 -Subject: [PATCH 31/45] tests: cover full GLM MoE selected chain +Subject: [PATCH 31/47] tests: cover full GLM MoE selected chain --- tests/test-backend-ops.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp -index 50ec5225..32f1215d 100644 +index e45661b0..3edcf06f 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -7826,7 +7826,9 @@ struct test_glm_moe_selected_chain : public test_case { @@ -22,7 +22,7 @@ index 50ec5225..32f1215d 100644 bool eval_perf(ggml_backend_t backend, const char * op_names_filter, printer * output_printer) override { scoped_test_env motif_reference_env("GGML_METAL_ENABLE_GLM_MOE_DECODE_MOTIF_REFERENCE", "1"); -@@ -14949,6 +14951,7 @@ static std::vector> make_test_cases_eval() { +@@ -14952,6 +14954,7 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_glm_moe_q2_weight_roofline(true, "FIXED_IDS")); test_cases.emplace_back(new test_glm_moe_q2_weight_roofline(false, "ROUTE_B")); test_cases.emplace_back(new test_glm_moe_selected_chain(1, false, "CORRECTNESS")); diff --git a/third_party/llama.cpp/patches/0032-tests-add-GLM-Q2Q3-selected-weight-roofline.patch b/third_party/llama.cpp/patches/0032-tests-add-GLM-Q2Q3-selected-weight-roofline.patch index 4ca91999ac..d6c0bbc4ac 100644 --- a/third_party/llama.cpp/patches/0032-tests-add-GLM-Q2Q3-selected-weight-roofline.patch +++ b/third_party/llama.cpp/patches/0032-tests-add-GLM-Q2Q3-selected-weight-roofline.patch @@ -1,14 +1,14 @@ -From 223525c271739859819dd21e8164e3247196b7ad Mon Sep 17 00:00:00 2001 +From 1dc45ffa1cf72ed14b28aa5af8d63259ad3dd33c Mon Sep 17 00:00:00 2001 From: James Dumay Date: Sun, 19 Jul 2026 00:00:00 +1000 -Subject: [PATCH 32/45] tests: add GLM Q2Q3 selected weight roofline +Subject: [PATCH 32/47] tests: add GLM Q2Q3 selected weight roofline --- tests/test-backend-ops.cpp | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp -index 32f1215d..9f566612 100644 +index 3edcf06f..67dbbdb6 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -7685,6 +7685,30 @@ struct test_glm_moe_q2_weight_roofline : public test_glm_moe_routed_q2_final_out @@ -42,7 +42,7 @@ index 32f1215d..9f566612 100644 struct test_glm_moe_selected_chain : public test_case { static constexpr int64_t n_expert = 32; static constexpr int64_t n_expert_used = 8; -@@ -14950,6 +14974,9 @@ static std::vector> make_test_cases_eval() { +@@ -14953,6 +14977,9 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_glm_moe_q2_weight_roofline(false, "ROUTE_A")); test_cases.emplace_back(new test_glm_moe_q2_weight_roofline(true, "FIXED_IDS")); test_cases.emplace_back(new test_glm_moe_q2_weight_roofline(false, "ROUTE_B")); @@ -52,7 +52,7 @@ index 32f1215d..9f566612 100644 test_cases.emplace_back(new test_glm_moe_selected_chain(1, false, "CORRECTNESS")); test_cases.emplace_back(new test_glm_moe_selected_chain(3, false, "USEFUL_A")); for (ggml_type type : {GGML_TYPE_Q8_0, GGML_TYPE_Q6_K, GGML_TYPE_Q5_K, GGML_TYPE_Q4_K}) { -@@ -15435,6 +15462,9 @@ static std::vector> make_test_cases_perf() { +@@ -15442,6 +15469,9 @@ static std::vector> make_test_cases_perf() { test_cases.emplace_back(new test_glm_moe_q2_weight_roofline(false, "ROUTE_A")); test_cases.emplace_back(new test_glm_moe_q2_weight_roofline(true, "FIXED_IDS")); test_cases.emplace_back(new test_glm_moe_q2_weight_roofline(false, "ROUTE_B")); diff --git a/third_party/llama.cpp/patches/0033-ggml-add-active-count-Q3-GLM-MoE-down-kernels.patch b/third_party/llama.cpp/patches/0033-ggml-add-active-count-Q3-GLM-MoE-down-kernels.patch index 9137658be6..5494f2f67a 100644 --- a/third_party/llama.cpp/patches/0033-ggml-add-active-count-Q3-GLM-MoE-down-kernels.patch +++ b/third_party/llama.cpp/patches/0033-ggml-add-active-count-Q3-GLM-MoE-down-kernels.patch @@ -1,7 +1,7 @@ -From 83bd6794157379b2ac6fb6121d3940219a2ce5f3 Mon Sep 17 00:00:00 2001 +From b3c67b00ce793d1a1e5f7256b2a6be080804d7f3 Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Mon, 20 Jul 2026 01:02:36 +1000 -Subject: [PATCH 33/45] ggml: add active-count Q3 GLM MoE down kernels +Subject: [PATCH 33/47] ggml: add active-count Q3 GLM MoE down kernels --- ggml/src/ggml-metal/ggml-metal-device.cpp | 49 +++++++++++ @@ -12,7 +12,7 @@ Subject: [PATCH 33/45] ggml: add active-count Q3 GLM MoE down kernels 5 files changed, 222 insertions(+), 30 deletions(-) diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cpp -index 088089aa..870d5a92 100644 +index b6f3c494..baac9436 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.cpp +++ b/ggml/src/ggml-metal/ggml-metal-device.cpp @@ -2761,6 +2761,55 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_id_q3_wei @@ -72,7 +72,7 @@ index 088089aa..870d5a92 100644 assert(op->op == GGML_OP_MOE_WEIGHTED_SUM); GGML_ASSERT(op->src[0] != nullptr); diff --git a/ggml/src/ggml-metal/ggml-metal-device.h b/ggml/src/ggml-metal/ggml-metal-device.h -index 03567502..b8a48d13 100644 +index 61aa5d76..24228afc 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.h +++ b/ggml/src/ggml-metal/ggml-metal-device.h @@ -190,6 +190,9 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_id @@ -86,7 +86,7 @@ index 03567502..b8a48d13 100644 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_id_q3_weighted_reduce_slots_sg_r6_nb8_w0(ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_id_q3_weighted_reduce_slots_sg_r10_nb8_w0(ggml_metal_library_t lib, const struct ggml_tensor * op); diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp -index ab41e1ec..b4fc441a 100644 +index d2fdef79..c9a07642 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -507,6 +507,21 @@ static bool ggml_metal_glm_dsa_q3_down_slot_parallel_reduce_r8_nb8_w0_enabled() @@ -224,10 +224,10 @@ index ab41e1ec..b4fc441a 100644 ggml_metal_encoder_set_pipeline(enc, pipeline_slot_parallel); ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0); diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metal -index b5e37c11..000fe0b2 100644 +index c1e7e938..9bc4a0ff 100644 --- a/ggml/src/ggml-metal/ggml-metal.metal +++ b/ggml/src/ggml-metal/ggml-metal.metal -@@ -11754,7 +11754,7 @@ kernel void kernel_mul_mv_id_q3_K_weighted_reduce( +@@ -11816,7 +11816,7 @@ kernel void kernel_mul_mv_id_q3_K_weighted_reduce( } } @@ -236,7 +236,7 @@ index b5e37c11..000fe0b2 100644 void kernel_mul_mv_id_q3_K_weighted_reduce_slots_sg_impl( constant ggml_metal_kargs_mul_mv_id & args, device const char * src0s, -@@ -11767,7 +11767,7 @@ void kernel_mul_mv_id_q3_K_weighted_reduce_slots_sg_impl( +@@ -11829,7 +11829,7 @@ void kernel_mul_mv_id_q3_K_weighted_reduce_slots_sg_impl( uint3 tgpig[[threadgroup_position_in_grid]], ushort tiisg[[thread_index_in_simdgroup]], ushort sgitg[[simdgroup_index_in_threadgroup]]) { @@ -245,7 +245,7 @@ index b5e37c11..000fe0b2 100644 const int nb = NB_FIXED == 0 ? args.ne00/QK_K : NB_FIXED; const int r0 = tgpig.x; -@@ -11983,6 +11983,54 @@ kernel void kernel_mul_mv_id_q3_K_weighted_reduce_slots_sg_r8_nb8_w0( +@@ -12045,6 +12045,54 @@ kernel void kernel_mul_mv_id_q3_K_weighted_reduce_slots_sg_r8_nb8_w0( args, src0s, src1, dst, ids, weights, extra, partials, tgpig, tiisg, sgitg); } @@ -300,7 +300,7 @@ index b5e37c11..000fe0b2 100644 kernel void kernel_mul_mv_id_q3_K_weighted_reduce_slots_sg_r8_nb8_w0_f16( constant ggml_metal_kargs_mul_mv_id & args, device const char * src0s, -@@ -11995,7 +12043,7 @@ kernel void kernel_mul_mv_id_q3_K_weighted_reduce_slots_sg_r8_nb8_w0_f16( +@@ -12057,7 +12105,7 @@ kernel void kernel_mul_mv_id_q3_K_weighted_reduce_slots_sg_r8_nb8_w0_f16( uint3 tgpig[[threadgroup_position_in_grid]], ushort tiisg[[thread_index_in_simdgroup]], ushort sgitg[[simdgroup_index_in_threadgroup]]) { @@ -310,7 +310,7 @@ index b5e37c11..000fe0b2 100644 } diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp -index 9f566612..f0b94ded 100644 +index 67dbbdb6..153762c0 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -8922,6 +8922,15 @@ struct test_glm_moe_static_motif : public test_case { @@ -390,7 +390,7 @@ index 9f566612..f0b94ded 100644 data[token*n_expert_used + slot] = value; sum += value; } -@@ -15037,6 +15064,8 @@ static std::vector> make_test_cases_eval() { +@@ -15040,6 +15067,8 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_glm_moe_gate_up_swiglu_rowtile_q2(false, true)); test_cases.emplace_back(new test_glm_moe_static_motif(GGML_TYPE_Q2_K, GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, 256, 256, false)); test_cases.emplace_back(new test_glm_moe_static_motif(GGML_TYPE_Q2_K, GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, 256, 256, true)); @@ -399,7 +399,7 @@ index 9f566612..f0b94ded 100644 test_cases.emplace_back(new test_glm_moe_static_motif(GGML_TYPE_Q2_K, GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, 6144, 2048, true, true, false, 2)); test_cases.emplace_back(new test_glm_moe_static_motif(GGML_TYPE_Q2_K, GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, 6144, 2048, true, true, false, 4)); test_cases.emplace_back(new test_glm_moe_static_motif(GGML_TYPE_Q2_K, GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, 6144, 2048, true, true, false, 6)); -@@ -15057,6 +15086,9 @@ static std::vector> make_test_cases_eval() { +@@ -15060,6 +15089,9 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_glm_moe_down_reduce(GGML_TYPE_Q3_K, 256, 256)); test_cases.emplace_back(new test_glm_moe_down_reduce(GGML_TYPE_Q3_K, 256, 256, true)); test_cases.emplace_back(new test_glm_moe_down_reduce(GGML_TYPE_Q3_K, 6144, 2048)); @@ -409,7 +409,7 @@ index 9f566612..f0b94ded 100644 test_cases.emplace_back(new test_glm_moe_down_reduce(GGML_TYPE_Q3_K, 6144, 2048, true)); test_cases.emplace_back(new test_glm_moe_down_reduce(GGML_TYPE_Q2_K, 6144, 2048)); test_cases.emplace_back(new test_glm_moe_down_reduce(GGML_TYPE_Q2_K, 6144, 2048, true)); -@@ -15521,6 +15553,7 @@ static std::vector> make_test_cases_perf() { +@@ -15528,6 +15560,7 @@ static std::vector> make_test_cases_perf() { test_cases.emplace_back(new test_glm_moe_static_motif(GGML_TYPE_Q2_K, GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, 6144, 2048, false)); test_cases.emplace_back(new test_glm_moe_static_motif(GGML_TYPE_Q2_K, GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, 6144, 2048, true)); test_cases.emplace_back(new test_glm_moe_static_motif(GGML_TYPE_Q2_K, GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, 6144, 2048, false, true)); @@ -417,7 +417,7 @@ index 9f566612..f0b94ded 100644 test_cases.emplace_back(new test_glm_moe_static_motif(GGML_TYPE_Q2_K, GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, 6144, 2048, true, true, false, 2)); test_cases.emplace_back(new test_glm_moe_static_motif(GGML_TYPE_Q2_K, GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, 6144, 2048, true, true, false, 4)); test_cases.emplace_back(new test_glm_moe_static_motif(GGML_TYPE_Q2_K, GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, 6144, 2048, true, true, false, 6)); -@@ -15543,6 +15576,9 @@ static std::vector> make_test_cases_perf() { +@@ -15550,6 +15583,9 @@ static std::vector> make_test_cases_perf() { test_cases.emplace_back(new test_glm_moe_down_reduce(GGML_TYPE_Q3_K, 2048, 1024)); test_cases.emplace_back(new test_glm_moe_down_reduce(GGML_TYPE_Q3_K, 2048, 1024, true)); test_cases.emplace_back(new test_glm_moe_down_reduce(GGML_TYPE_Q3_K, 6144, 2048)); diff --git a/third_party/llama.cpp/patches/0034-ggml-use-GLM-max-active-policy-for-Q3-down-kernels.patch b/third_party/llama.cpp/patches/0034-ggml-use-GLM-max-active-policy-for-Q3-down-kernels.patch index 91829fed15..ccaba97ab6 100644 --- a/third_party/llama.cpp/patches/0034-ggml-use-GLM-max-active-policy-for-Q3-down-kernels.patch +++ b/third_party/llama.cpp/patches/0034-ggml-use-GLM-max-active-policy-for-Q3-down-kernels.patch @@ -1,7 +1,7 @@ -From d8a655fca4f47b48fc617b3302beb3403b743593 Mon Sep 17 00:00:00 2001 +From c28b48b4fae9418cd9f27845aff78dcbc6160788 Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Mon, 20 Jul 2026 05:19:37 +1000 -Subject: [PATCH 34/45] ggml: use GLM max-active policy for Q3 down kernels +Subject: [PATCH 34/47] ggml: use GLM max-active policy for Q3 down kernels --- ggml/src/ggml-metal/ggml-metal-ops.cpp | 30 ++++++++++++++++++++++++++ @@ -9,7 +9,7 @@ Subject: [PATCH 34/45] ggml: use GLM max-active policy for Q3 down kernels 2 files changed, 32 insertions(+) diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp -index b4fc441a..2e42a168 100644 +index c9a07642..38c485c5 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -522,6 +522,11 @@ static bool ggml_metal_glm_dsa_q3_down_slot_parallel_reduce_active2_enabled() { @@ -85,7 +85,7 @@ index b4fc441a..2e42a168 100644 pipeline_slot_parallel = ggml_metal_library_get_pipeline_mul_mv_id_q3_weighted_reduce_slots_sg_r8_nb8_w0_f16(lib, weighted_sum); diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp -index f0b94ded..12e2059f 100644 +index 153762c0..b268be2e 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -7333,6 +7333,7 @@ struct test_topk_moe_glm_component : public test_case { diff --git a/third_party/llama.cpp/patches/0035-ggml-honor-explicit-Q2-gate-up-Metal-variant-flags.patch b/third_party/llama.cpp/patches/0035-ggml-honor-explicit-Q2-gate-up-Metal-variant-flags.patch index 46dd3dcddf..788482f9f6 100644 --- a/third_party/llama.cpp/patches/0035-ggml-honor-explicit-Q2-gate-up-Metal-variant-flags.patch +++ b/third_party/llama.cpp/patches/0035-ggml-honor-explicit-Q2-gate-up-Metal-variant-flags.patch @@ -1,14 +1,14 @@ -From f8152becddf60982cc3a69f9db8baecd399bdacb Mon Sep 17 00:00:00 2001 +From 1cb7507b903b5f14d23662e3253e87893beada14 Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Mon, 20 Jul 2026 10:44:00 +1000 -Subject: [PATCH 35/45] ggml: honor explicit Q2 gate/up Metal variant flags +Subject: [PATCH 35/47] ggml: honor explicit Q2 gate/up Metal variant flags --- ggml/src/ggml-metal/ggml-metal-ops.cpp | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp -index 2e42a168..9469192d 100644 +index 38c485c5..b52204e6 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -4345,6 +4345,15 @@ static int ggml_metal_op_mul_mv_id_gate_up_swiglu( diff --git a/third_party/llama.cpp/patches/0036-ggml-make-GLM-MoE-roofline-honor-active-experts.patch b/third_party/llama.cpp/patches/0036-ggml-make-GLM-MoE-roofline-honor-active-experts.patch index 82b1789c4c..6bc31b0d8a 100644 --- a/third_party/llama.cpp/patches/0036-ggml-make-GLM-MoE-roofline-honor-active-experts.patch +++ b/third_party/llama.cpp/patches/0036-ggml-make-GLM-MoE-roofline-honor-active-experts.patch @@ -1,14 +1,14 @@ -From 36ee61b78bfa044c8701f3db79d8bc51bc1ca3d3 Mon Sep 17 00:00:00 2001 +From fd336839675aa17b31948afa26998881534bdb5f Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Mon, 20 Jul 2026 13:10:00 +1000 -Subject: [PATCH 36/45] ggml: make GLM MoE roofline honor active experts +Subject: [PATCH 36/47] ggml: make GLM MoE roofline honor active experts --- ggml/src/ggml-metal/ggml-metal-ops.cpp | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp -index 9469192d..2121075e 100644 +index b52204e6..370559cd 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -2026,7 +2026,12 @@ static int ggml_metal_op_glm_moe_q2_selected_weight_scan( diff --git a/third_party/llama.cpp/patches/0037-ggml-avoid-GLM-Q3-fused-tail-under-active-policy.patch b/third_party/llama.cpp/patches/0037-ggml-avoid-GLM-Q3-fused-tail-under-active-policy.patch index 89022fc2fa..33f68bb584 100644 --- a/third_party/llama.cpp/patches/0037-ggml-avoid-GLM-Q3-fused-tail-under-active-policy.patch +++ b/third_party/llama.cpp/patches/0037-ggml-avoid-GLM-Q3-fused-tail-under-active-policy.patch @@ -1,7 +1,7 @@ -From 3bcadd4535d6e48229e2edf4a10e849b23475083 Mon Sep 17 00:00:00 2001 +From 0566ecc45040b579f09693e3b2a670db26346259 Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Mon, 20 Jul 2026 14:05:00 +1000 -Subject: [PATCH 37/45] ggml: avoid GLM Q3 fused tail under active policy +Subject: [PATCH 37/47] ggml: avoid GLM Q3 fused tail under active policy --- ggml/src/ggml-metal/ggml-metal-ops.cpp | 3 ++ @@ -9,7 +9,7 @@ Subject: [PATCH 37/45] ggml: avoid GLM Q3 fused tail under active policy 2 files changed, 49 insertions(+) diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp -index 2121075e..91976954 100644 +index 370559cd..30efec9c 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -2738,6 +2738,9 @@ static bool ggml_metal_glm_moe_can_fuse_swiglu_q3_down(const ggml_metal_glm_moe_ @@ -23,7 +23,7 @@ index 2121075e..91976954 100644 motif.gate == nullptr || motif.up == nullptr || motif.glu == nullptr || motif.down == nullptr || motif.out == nullptr || diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp -index 12e2059f..43ce1a0e 100644 +index b268be2e..745c3814 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -7662,6 +7662,44 @@ struct test_glm_moe_routed_q2_final_output : public test_topk_moe_glm_component @@ -71,7 +71,7 @@ index 12e2059f..43ce1a0e 100644 struct test_glm_moe_q2_weight_roofline : public test_glm_moe_routed_q2_final_output { const bool bypass_route; const char * variant; -@@ -15000,6 +15038,10 @@ static std::vector> make_test_cases_eval() { +@@ -15003,6 +15041,10 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_glm_moe_routed_q2_final_output(true, false)); test_cases.emplace_back(new test_glm_moe_routed_q2_final_output(false, true)); test_cases.emplace_back(new test_glm_moe_routed_q2_final_output(true, true)); @@ -82,7 +82,7 @@ index 12e2059f..43ce1a0e 100644 test_cases.emplace_back(new test_glm_moe_q2_weight_roofline(false, "ROUTE_A")); test_cases.emplace_back(new test_glm_moe_q2_weight_roofline(true, "FIXED_IDS")); test_cases.emplace_back(new test_glm_moe_q2_weight_roofline(false, "ROUTE_B")); -@@ -15493,6 +15535,10 @@ static std::vector> make_test_cases_perf() { +@@ -15500,6 +15542,10 @@ static std::vector> make_test_cases_perf() { test_cases.emplace_back(new test_glm_moe_routed_q2_final_output(true, false)); test_cases.emplace_back(new test_glm_moe_routed_q2_final_output(false, true)); test_cases.emplace_back(new test_glm_moe_routed_q2_final_output(true, true)); diff --git a/third_party/llama.cpp/patches/0038-ggml-shrink-GLM-Q2-gate-up-active-dispatch.patch b/third_party/llama.cpp/patches/0038-ggml-shrink-GLM-Q2-gate-up-active-dispatch.patch index 5c293f158e..4993da2d99 100644 --- a/third_party/llama.cpp/patches/0038-ggml-shrink-GLM-Q2-gate-up-active-dispatch.patch +++ b/third_party/llama.cpp/patches/0038-ggml-shrink-GLM-Q2-gate-up-active-dispatch.patch @@ -1,7 +1,7 @@ -From 9fa1c53b208fec4bf4ac863c38135451a1fca6f3 Mon Sep 17 00:00:00 2001 +From aaa2838a8471e3455fa70f314e48b1c90ef7d170 Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Mon, 20 Jul 2026 03:51:17 +1000 -Subject: [PATCH 38/45] ggml: shrink GLM Q2 gate-up active dispatch +Subject: [PATCH 38/47] ggml: shrink GLM Q2 gate-up active dispatch --- ggml/src/ggml-metal/ggml-metal-ops.cpp | 6 ++++-- @@ -9,7 +9,7 @@ Subject: [PATCH 38/45] ggml: shrink GLM Q2 gate-up active dispatch 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp -index 91976954..646c983a 100644 +index 30efec9c..08f1d966 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -4556,9 +4556,11 @@ static int ggml_metal_op_mul_mv_id_gate_up_swiglu( @@ -27,10 +27,10 @@ index 91976954..646c983a 100644 const int grid_y = (1 + nr1 - 1)/nr1; const int grid_z = ne123; diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metal -index 000fe0b2..5dfeed66 100644 +index 9bc4a0ff..9502a0db 100644 --- a/ggml/src/ggml-metal/ggml-metal.metal +++ b/ggml/src/ggml-metal/ggml-metal.metal -@@ -10331,9 +10331,11 @@ void kernel_mul_mv_id_q2_K_gate_up_swiglu_pair_sg_impl( +@@ -10393,9 +10393,11 @@ void kernel_mul_mv_id_q2_K_gate_up_swiglu_pair_sg_impl( ushort sgitg[[simdgroup_index_in_threadgroup]]) { const int nb = args.ne00/QK_K; diff --git a/third_party/llama.cpp/patches/0039-ggml-skip-inactive-GLM-Q3-down-slots.patch b/third_party/llama.cpp/patches/0039-ggml-skip-inactive-GLM-Q3-down-slots.patch index 7ba0a89cd2..2c138a81a4 100644 --- a/third_party/llama.cpp/patches/0039-ggml-skip-inactive-GLM-Q3-down-slots.patch +++ b/third_party/llama.cpp/patches/0039-ggml-skip-inactive-GLM-Q3-down-slots.patch @@ -1,17 +1,17 @@ -From eb68fbbabf64915d6283683ae25fefd7475d46c2 Mon Sep 17 00:00:00 2001 +From b117244fe158b70caef4305818a738128e26317b Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Mon, 20 Jul 2026 05:30:42 +1000 -Subject: [PATCH 39/45] ggml: skip inactive GLM Q3 down slots +Subject: [PATCH 39/47] ggml: skip inactive GLM Q3 down slots --- ggml/src/ggml-metal/ggml-metal.metal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metal -index 5dfeed66..b098231d 100644 +index 9502a0db..d367da67 100644 --- a/ggml/src/ggml-metal/ggml-metal.metal +++ b/ggml/src/ggml-metal/ggml-metal.metal -@@ -11802,7 +11802,7 @@ void kernel_mul_mv_id_q3_K_weighted_reduce_slots_sg_impl( +@@ -11864,7 +11864,7 @@ void kernel_mul_mv_id_q3_K_weighted_reduce_slots_sg_impl( float sumf1[nr0] = {0.f}; float sumf2[nr0] = {0.f}; diff --git a/third_party/llama.cpp/patches/0040-Fix-GLM-DSA-native-MTP-execution.patch b/third_party/llama.cpp/patches/0040-Fix-GLM-DSA-native-MTP-execution.patch index 457cb41f9d..4ee00c5b29 100644 --- a/third_party/llama.cpp/patches/0040-Fix-GLM-DSA-native-MTP-execution.patch +++ b/third_party/llama.cpp/patches/0040-Fix-GLM-DSA-native-MTP-execution.patch @@ -1,7 +1,7 @@ -From 6b9709b217341e1229a3a56b1a6132b7391c1a49 Mon Sep 17 00:00:00 2001 +From d4d0e432507f497be6d2b5d443a525110daa04b4 Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Wed, 22 Jul 2026 08:41:09 +1000 -Subject: [PATCH 40/45] Fix GLM DSA native MTP execution +Subject: [PATCH 40/47] Fix GLM DSA native MTP execution --- src/llama-context.cpp | 16 ++ @@ -11,7 +11,7 @@ Subject: [PATCH 40/45] Fix GLM DSA native MTP execution 4 files changed, 356 insertions(+), 6 deletions(-) diff --git a/src/llama-context.cpp b/src/llama-context.cpp -index 9ab7be4b..4342e96e 100644 +index abfe34f9..c4e057ae 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -495,6 +495,22 @@ llama_context::~llama_context() { @@ -38,7 +38,7 @@ index 9ab7be4b..4342e96e 100644 if (!enabled) { return; diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp -index 244a400a..6cef391d 100644 +index 126a434b..a2467e66 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -661,15 +661,25 @@ bool llm_graph_input_attn_k::can_reuse(const llm_graph_params & params) { @@ -405,10 +405,10 @@ index 82c6a093..32996613 100644 llm_graph_context(params) { const bool is_mla = hparams.is_mla(); diff --git a/src/models/models.h b/src/models/models.h -index f3b576c0..ca471dcc 100644 +index f4fb02cc..aeeb7b22 100644 --- a/src/models/models.h +++ b/src/models/models.h -@@ -1221,10 +1221,13 @@ struct llama_model_glm_dsa : public llama_model_base { +@@ -1237,10 +1237,13 @@ struct llama_model_glm_dsa : public llama_model_base { void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; diff --git a/third_party/llama.cpp/patches/0041-Adapt-Skippy-model-loading-to-load-modes.patch b/third_party/llama.cpp/patches/0041-Adapt-Skippy-model-loading-to-load-modes.patch index d2a9becf94..fa3692d37e 100644 --- a/third_party/llama.cpp/patches/0041-Adapt-Skippy-model-loading-to-load-modes.patch +++ b/third_party/llama.cpp/patches/0041-Adapt-Skippy-model-loading-to-load-modes.patch @@ -1,7 +1,7 @@ -From 43fe1861b1b58d04444da0886fb5fddd1a14f05c Mon Sep 17 00:00:00 2001 +From ba1f26d0f7d6a8e8f7076d7f1f3b637cd5bc969a Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Sun, 26 Jul 2026 19:30:12 +1000 -Subject: [PATCH 41/45] Adapt Skippy model loading to load modes +Subject: [PATCH 41/47] Adapt Skippy model loading to load modes --- src/skippy.cpp | 7 +++---- diff --git a/third_party/llama.cpp/patches/0042-Harden-staged-session-and-sideband-bookkeeping.patch b/third_party/llama.cpp/patches/0042-Harden-staged-session-and-sideband-bookkeeping.patch index 91f747296b..899c64f26b 100644 --- a/third_party/llama.cpp/patches/0042-Harden-staged-session-and-sideband-bookkeeping.patch +++ b/third_party/llama.cpp/patches/0042-Harden-staged-session-and-sideband-bookkeeping.patch @@ -1,7 +1,7 @@ -From 66275c899d65c5fb9c8f44efaf2f4e423bac045c Mon Sep 17 00:00:00 2001 +From d474a2cf7f638e5155a83fb85624d47f6931e3d1 Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Sun, 26 Jul 2026 21:13:49 +1000 -Subject: [PATCH 42/45] Harden staged session and sideband bookkeeping +Subject: [PATCH 42/47] Harden staged session and sideband bookkeeping --- ggml/include/ggml-rpc.h | 2 +- diff --git a/third_party/llama.cpp/patches/0043-Correct-GLM-DSA-staged-graph-contracts.patch b/third_party/llama.cpp/patches/0043-Correct-GLM-DSA-staged-graph-contracts.patch index 96f20bd2b6..a9a4e5ac20 100644 --- a/third_party/llama.cpp/patches/0043-Correct-GLM-DSA-staged-graph-contracts.patch +++ b/third_party/llama.cpp/patches/0043-Correct-GLM-DSA-staged-graph-contracts.patch @@ -1,7 +1,7 @@ -From abb3456459c7535ec1e09286c8013a60c1170efe Mon Sep 17 00:00:00 2001 +From 2a38f4a874c8e0c6a49610957cebb6195d112610 Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Sun, 26 Jul 2026 21:14:11 +1000 -Subject: [PATCH 43/45] Correct GLM-DSA staged graph contracts +Subject: [PATCH 43/47] Correct GLM-DSA staged graph contracts --- src/models/glm-dsa.cpp | 21 ++++++++------------- diff --git a/third_party/llama.cpp/patches/0043-ggml-metal-make-small-batch-matmul-batch-invariant.patch b/third_party/llama.cpp/patches/0044-ggml-metal-make-small-batch-matmul-batch-invariant.patch similarity index 95% rename from third_party/llama.cpp/patches/0043-ggml-metal-make-small-batch-matmul-batch-invariant.patch rename to third_party/llama.cpp/patches/0044-ggml-metal-make-small-batch-matmul-batch-invariant.patch index 28276aae2a..f46b68d145 100644 --- a/third_party/llama.cpp/patches/0043-ggml-metal-make-small-batch-matmul-batch-invariant.patch +++ b/third_party/llama.cpp/patches/0044-ggml-metal-make-small-batch-matmul-batch-invariant.patch @@ -1,7 +1,7 @@ -From 0f7f31ef18355e7581e0e5a36f5a45e45274b3aa Mon Sep 17 00:00:00 2001 +From 2166a2f7ae1dfc8469a64a172a0e48b740267645 Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Sun, 26 Jul 2026 11:28:36 +1000 -Subject: [PATCH] ggml-metal: make small-batch matmul batch invariant +Subject: [PATCH 44/47] ggml-metal: make small-batch matmul batch invariant --- ggml/src/ggml-metal/ggml-metal-device.cpp | 8 +- @@ -11,7 +11,7 @@ Subject: [PATCH] ggml-metal: make small-batch matmul batch invariant 4 files changed, 176 insertions(+), 45 deletions(-) diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cpp -index 870d5a92e..b156d670c 100644 +index baac9436..39dcc313 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.cpp +++ b/ggml/src/ggml-metal/ggml-metal-device.cpp @@ -1186,6 +1186,12 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_ext(ggml_ @@ -37,7 +37,7 @@ index 870d5a92e..b156d670c 100644 smem = 32*sizeof(float)*nr0; suffix = ne00 % 4 == 0 ? "_4" : ""; diff --git a/ggml/src/ggml-metal/ggml-metal-impl.h b/ggml/src/ggml-metal/ggml-metal-impl.h -index f7112ee56..b8683e534 100644 +index 6d20ae26..5167d2df 100644 --- a/ggml/src/ggml-metal/ggml-metal-impl.h +++ b/ggml/src/ggml-metal/ggml-metal-impl.h @@ -44,6 +44,8 @@ @@ -50,10 +50,10 @@ index f7112ee56..b8683e534 100644 #define N_SG_MXFP4 2 diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp -index 646c983ad..a4b62a42e 100644 +index 08f1d966..5b41995c 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp -@@ -7704,6 +7704,41 @@ int ggml_metal_op_pool_2d(ggml_metal_op_t ctx, int idx) { +@@ -7744,6 +7744,41 @@ int ggml_metal_op_pool_2d(ggml_metal_op_t ctx, int idx) { return 1; } @@ -95,7 +95,7 @@ index 646c983ad..a4b62a42e 100644 int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); -@@ -7817,10 +7852,17 @@ int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) { +@@ -7869,10 +7904,17 @@ int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) { // find the break-even point where the matrix-matrix kernel becomes more efficient compared // to the matrix-vector kernel const int ne11_mm_min = 8; @@ -114,7 +114,7 @@ index 646c983ad..a4b62a42e 100644 ( ( ( -@@ -7836,7 +7878,10 @@ int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) { +@@ -7888,7 +7930,10 @@ int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) { op->src[0]->type == GGML_TYPE_Q8_0 || op->src[0]->type == GGML_TYPE_MXFP4 || op->src[0]->type == GGML_TYPE_IQ4_NL || @@ -126,7 +126,7 @@ index 646c983ad..a4b62a42e 100644 ) || ( ( -@@ -7845,7 +7890,10 @@ int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) { +@@ -7897,7 +7942,10 @@ int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) { op->src[0]->type == GGML_TYPE_Q6_K || op->src[0]->type == GGML_TYPE_Q2_K || op->src[0]->type == GGML_TYPE_Q3_K || @@ -138,7 +138,7 @@ index 646c983ad..a4b62a42e 100644 ) ) ) { -@@ -7857,8 +7905,6 @@ int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) { +@@ -7909,8 +7957,6 @@ int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) { // my current hypothesis is that the work grid is not evenly divisible for different nsg // values and there can be some tail effects when nsg is high. need to confirm this // @@ -147,7 +147,7 @@ index 646c983ad..a4b62a42e 100644 // num threads along row per simdgroup int16_t nxpsg = 0; if (ne00 % 256 == 0 && ne11 < 3) { -@@ -7869,26 +7915,33 @@ int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) { +@@ -7921,26 +7967,33 @@ int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) { nxpsg = 4; } @@ -199,7 +199,7 @@ index 646c983ad..a4b62a42e 100644 auto pipeline = ggml_metal_library_get_pipeline_mul_mv_ext(lib, op, nsg, nxpsg, r1ptg); -@@ -7919,7 +7972,14 @@ int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) { +@@ -7971,7 +8024,14 @@ int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) { ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2); ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3); @@ -216,7 +216,7 @@ index 646c983ad..a4b62a42e 100644 !ggml_is_transposed(op->src[0]) && !ggml_is_transposed(op->src[1]) && diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metal -index b098231dc..4e6d2b35f 100644 +index d367da67..7d155bd7 100644 --- a/ggml/src/ggml-metal/ggml-metal.metal +++ b/ggml/src/ggml-metal/ggml-metal.metal @@ -4778,11 +4778,6 @@ kernel void kernel_mul_mv_ext_q4x4_f32_disp( @@ -265,7 +265,7 @@ index b098231dc..4e6d2b35f 100644 template [[host_name("kernel_mul_mv_ext_q2_K_f32_r1_2")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<2, block_q2_K, 256, dequantize_q2_K>; template [[host_name("kernel_mul_mv_ext_q2_K_f32_r1_3")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<3, block_q2_K, 256, dequantize_q2_K>; template [[host_name("kernel_mul_mv_ext_q2_K_f32_r1_4")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<4, block_q2_K, 256, dequantize_q2_K>; -@@ -13429,6 +13409,89 @@ kernel void kernel_mul_mv_q6_K_f32( +@@ -13491,6 +13471,89 @@ kernel void kernel_mul_mv_q6_K_f32( kernel_mul_mv_q6_K_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); } @@ -356,5 +356,5 @@ index b098231dc..4e6d2b35f 100644 template -- -2.54.0 (Apple Git-157) +2.50.1 (Apple Git-155) diff --git a/third_party/llama.cpp/patches/0044-Fix-GLM-DSA-Metal-dispatch-safety.patch b/third_party/llama.cpp/patches/0045-Fix-GLM-DSA-Metal-dispatch-safety.patch similarity index 94% rename from third_party/llama.cpp/patches/0044-Fix-GLM-DSA-Metal-dispatch-safety.patch rename to third_party/llama.cpp/patches/0045-Fix-GLM-DSA-Metal-dispatch-safety.patch index 6ed3d91888..fa0495a228 100644 --- a/third_party/llama.cpp/patches/0044-Fix-GLM-DSA-Metal-dispatch-safety.patch +++ b/third_party/llama.cpp/patches/0045-Fix-GLM-DSA-Metal-dispatch-safety.patch @@ -1,7 +1,7 @@ -From e47c41e7d0e39fbc95f059678b01653d813f0e6a Mon Sep 17 00:00:00 2001 +From 104676d83b0fadf17f873e128144033a58bc6f5a Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Sun, 26 Jul 2026 21:15:27 +1000 -Subject: [PATCH 44/45] Fix GLM-DSA Metal dispatch safety +Subject: [PATCH 45/47] Fix GLM-DSA Metal dispatch safety --- ggml/src/ggml-metal/ggml-metal-device.cpp | 13 +++--- @@ -11,10 +11,10 @@ Subject: [PATCH 44/45] Fix GLM-DSA Metal dispatch safety 4 files changed, 33 insertions(+), 37 deletions(-) diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cpp -index 870d5a92..9ad67837 100644 +index 39dcc313..a4c38234 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.cpp +++ b/ggml/src/ggml-metal/ggml-metal-device.cpp -@@ -2779,15 +2779,14 @@ static ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_id +@@ -2785,15 +2785,14 @@ static ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_id constexpr int nr0 = 8; const int nsg = active_slots; @@ -37,10 +37,10 @@ index 870d5a92..9ad67837 100644 res.nr0 = nr0; diff --git a/ggml/src/ggml-metal/ggml-metal-impl.h b/ggml/src/ggml-metal/ggml-metal-impl.h -index f7112ee5..f8431715 100644 +index 5167d2df..4db4df53 100644 --- a/ggml/src/ggml-metal/ggml-metal-impl.h +++ b/ggml/src/ggml-metal/ggml-metal-impl.h -@@ -575,7 +575,7 @@ typedef struct { +@@ -577,7 +577,7 @@ typedef struct { uint64_t weights_nb1; uint64_t weights_nb2; int32_t weighted; @@ -50,7 +50,7 @@ index f7112ee5..f8431715 100644 typedef struct { diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp -index 646c983a..f6a287b1 100644 +index 5b41995c..7252aeb5 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -230,6 +230,11 @@ static int ggml_metal_glm_dsa_moe_max_active_experts() { @@ -166,7 +166,7 @@ index 646c983a..f6a287b1 100644 } else if (q3_f16_w0_shape) { pipeline_slot_parallel = ggml_metal_library_get_pipeline_mul_mv_id_q3_weighted_reduce_slots_sg_r8_nb8_w0_f16(lib, weighted_sum); -@@ -11666,8 +11663,8 @@ int ggml_metal_op_dsa_sparse_attn(ggml_metal_op_t ctx, int idx) { +@@ -11778,8 +11775,8 @@ int ggml_metal_op_dsa_sparse_attn(ggml_metal_op_t ctx, int idx) { ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[4]), ida++); // top_k ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), ida++); // dst @@ -178,10 +178,10 @@ index 646c983a..f6a287b1 100644 const int max_threads_per_group = std::max(1, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)/head_group); const int nth = std::min(nth_requested, max_threads_per_group); diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metal -index b098231d..726c02ad 100644 +index 7d155bd7..674216b8 100644 --- a/ggml/src/ggml-metal/ggml-metal.metal +++ b/ggml/src/ggml-metal/ggml-metal.metal -@@ -10331,7 +10331,7 @@ void kernel_mul_mv_id_q2_K_gate_up_swiglu_pair_sg_impl( +@@ -10373,7 +10373,7 @@ void kernel_mul_mv_id_q2_K_gate_up_swiglu_pair_sg_impl( ushort sgitg[[simdgroup_index_in_threadgroup]]) { const int nb = args.ne00/QK_K; @@ -190,7 +190,7 @@ index b098231d..726c02ad 100644 const int slot_groups = (active_slots + slots_per_tg - 1) / slots_per_tg; const int r0 = tgpig.x; const int token = tgpig.z / slot_groups; -@@ -10344,7 +10344,7 @@ void kernel_mul_mv_id_q2_K_gate_up_swiglu_pair_sg_impl( +@@ -10386,7 +10386,7 @@ void kernel_mul_mv_id_q2_K_gate_up_swiglu_pair_sg_impl( if (slot < args.nei0 && token < args.nei1) { const int32_t expert_id = ((device const int32_t *) (ids + token*args.nbi1))[slot]; @@ -199,7 +199,7 @@ index b098231d..726c02ad 100644 if (active_slot && expert_id >= 0 && expert_id < args.ne02) { device const char * src0_all = is_gate ? src0_gate_all : src0_up_all; const uint64_t offset0 = uint64_t(expert_id)*args.nb02 + uint64_t(first_row)*args.nb01; -@@ -10782,7 +10782,7 @@ void kernel_mul_mv_id_q2_K_gate_up_swiglu_pair_sg_dual_impl( +@@ -10824,7 +10824,7 @@ void kernel_mul_mv_id_q2_K_gate_up_swiglu_pair_sg_dual_impl( if (valid_output) { const int32_t expert_id = ((device const int32_t *) (ids + token*args.nbi1))[slot]; diff --git a/third_party/llama.cpp/patches/0044-Fix-empty-transposed-KV-page-export.patch b/third_party/llama.cpp/patches/0046-Fix-empty-transposed-KV-page-import-and-export.patch similarity index 94% rename from third_party/llama.cpp/patches/0044-Fix-empty-transposed-KV-page-export.patch rename to third_party/llama.cpp/patches/0046-Fix-empty-transposed-KV-page-import-and-export.patch index 1cf115e0ec..79782e6dc7 100644 --- a/third_party/llama.cpp/patches/0044-Fix-empty-transposed-KV-page-export.patch +++ b/third_party/llama.cpp/patches/0046-Fix-empty-transposed-KV-page-import-and-export.patch @@ -1,7 +1,7 @@ -From fe586db1fd61a8a8906d0bc571a65e05d12cc4ff Mon Sep 17 00:00:00 2001 +From 91eaf086ccb425da6c05af606f9ed20eb558cf3a Mon Sep 17 00:00:00 2001 From: James Dumay Date: Sun, 26 Jul 2026 15:39:58 +1000 -Subject: [PATCH] Fix empty transposed KV page import and export +Subject: [PATCH 46/47] Fix empty transposed KV page import and export --- src/llama-kv-cache.cpp | 6 + @@ -11,10 +11,10 @@ Subject: [PATCH] Fix empty transposed KV page import and export create mode 100644 tests/test-skippy-kv-page-export.cpp diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp -index 9df4984b3..1c4c2ddbc 100644 +index c4517d65..117855d6 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp -@@ -1505,6 +1505,9 @@ bool llama_kv_cache::stage_export_kv_page( +@@ -1622,6 +1622,9 @@ bool llama_kv_cache::stage_export_kv_page( } } else { for (const auto * layer : selected) { @@ -24,7 +24,7 @@ index 9df4984b3..1c4c2ddbc 100644 auto * v = layer->v_stream[strm]; const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(layer->il); for (uint32_t j = 0; j < n_embd_v_gqa; ++j) { -@@ -1679,6 +1682,9 @@ bool llama_kv_cache::stage_import_kv_page( +@@ -1804,6 +1807,9 @@ bool llama_kv_cache::stage_import_kv_page( } } else { for (const auto * layer : selected) { @@ -35,7 +35,7 @@ index 9df4984b3..1c4c2ddbc 100644 const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(layer->il); for (uint32_t j = 0; j < n_embd_v_gqa; ++j) { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt -index 7a93b19a0..158ab28df 100644 +index 7a93b19a..158ab28d 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -151,6 +151,7 @@ endif () @@ -48,7 +48,7 @@ index 7a93b19a0..158ab28df 100644 llama_build_and_test(test-reasoning-budget.cpp) diff --git a/tests/test-skippy-kv-page-export.cpp b/tests/test-skippy-kv-page-export.cpp new file mode 100644 -index 000000000..724835df7 +index 00000000..724835df --- /dev/null +++ b/tests/test-skippy-kv-page-export.cpp @@ -0,0 +1,165 @@ @@ -218,4 +218,5 @@ index 000000000..724835df7 + return 0; +} -- -2.54.0 (Apple Git-157) +2.50.1 (Apple Git-155) + diff --git a/third_party/llama.cpp/patches/0045-Scope-GLM-DSA-backend-test-environment.patch b/third_party/llama.cpp/patches/0047-Scope-GLM-DSA-backend-test-environment.patch similarity index 98% rename from third_party/llama.cpp/patches/0045-Scope-GLM-DSA-backend-test-environment.patch rename to third_party/llama.cpp/patches/0047-Scope-GLM-DSA-backend-test-environment.patch index df2d2f2df9..034295addb 100644 --- a/third_party/llama.cpp/patches/0045-Scope-GLM-DSA-backend-test-environment.patch +++ b/third_party/llama.cpp/patches/0047-Scope-GLM-DSA-backend-test-environment.patch @@ -1,14 +1,14 @@ -From 99a0fe861d955acc921d0f328f0b1e8ba95b7699 Mon Sep 17 00:00:00 2001 +From 5879e1ee1ce6d7a9a005a260f74acfd3dc72f2ef Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Sun, 26 Jul 2026 21:16:20 +1000 -Subject: [PATCH 45/45] Scope GLM-DSA backend test environment +Subject: [PATCH 47/47] Scope GLM-DSA backend test environment --- tests/test-backend-ops.cpp | 87 +++++++++++++++++++++++++++++--------- 1 file changed, 66 insertions(+), 21 deletions(-) diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp -index 43ce1a0e..7d47a4e0 100644 +index 745c3814..5af000a7 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -42,6 +42,7 @@ diff --git a/third_party/llama.cpp/upstream.txt b/third_party/llama.cpp/upstream.txt index 87b944fcc7..2fcaf1ab61 100644 --- a/third_party/llama.cpp/upstream.txt +++ b/third_party/llama.cpp/upstream.txt @@ -1 +1 @@ -ff067f76dd8e9e05f0528056f1274adf01a54d70 +9a3bf2b84923a85583b4ee8177b0cca13824bb03