diff --git a/.github/workflows/pr_builds.yml b/.github/workflows/pr_builds.yml index 39d985036..fc6e046b8 100644 --- a/.github/workflows/pr_builds.yml +++ b/.github/workflows/pr_builds.yml @@ -538,14 +538,14 @@ jobs: ${{ env.CACHE_NAMESPACE }}-${{ runner.os }}-skippy-ci-smoke-models- - name: SDK and API crate tests - if: ${{ matrix.group == 'sdk-api' && (needs.changes.outputs.all_rust == 'true' || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-client') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-api-client') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-api-server') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-config') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-commands') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-events') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-hardware-profile') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-runtime-install') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-sdk') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-cli') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-tui') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-embedded-runtime') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-console-server') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-ffi') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-nodejs')) }} + if: ${{ matrix.group == 'sdk-api' && (needs.changes.outputs.all_rust == 'true' || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-client') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-api-client') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-api-server') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-config') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-commands') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-events') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-hardware-profile') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-runtime-install') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-native-runtime') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-routing') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-types') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-sdk') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-cli') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-tui') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-embedded-runtime') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-console-server') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-ffi') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-nodejs')) }} run: | should_test() { local crate="$1" [ "$ALL_RUST" = "true" ] || jq -e --arg crate "$crate" 'index($crate) != null' <<<"$AFFECTED" >/dev/null } - for c in mesh-llm-client mesh-llm-api-client mesh-llm-api-server mesh-llm-config mesh-llm-commands mesh-llm-events mesh-llm-hardware-profile mesh-llm-runtime-install mesh-llm-cli mesh-llm-tui mesh-llm-embedded-runtime mesh-llm-sdk mesh-llm-console-server mesh-llm-ffi mesh-llm-nodejs; do + for c in mesh-llm-client mesh-llm-api-client mesh-llm-api-server mesh-llm-config mesh-llm-commands mesh-llm-events mesh-llm-hardware-profile mesh-llm-runtime-install mesh-llm-native-runtime mesh-llm-routing mesh-llm-types mesh-llm-cli mesh-llm-tui mesh-llm-embedded-runtime mesh-llm-sdk mesh-llm-console-server mesh-llm-ffi mesh-llm-nodejs; do if should_test "$c"; then cargo test -p "$c" else diff --git a/third_party/llama.cpp/patches/0001-Add-Skippy-ABI-and-package-writer-foundation.patch b/third_party/llama.cpp/patches/0001-Add-Skippy-ABI-and-package-writer-foundation.patch index 5c0b4b8c1..a989a5a68 100644 --- a/third_party/llama.cpp/patches/0001-Add-Skippy-ABI-and-package-writer-foundation.patch +++ b/third_party/llama.cpp/patches/0001-Add-Skippy-ABI-and-package-writer-foundation.patch @@ -1,4 +1,4 @@ -From 5c6e3bea6ac4126a8528a250bd270f72ee2cd61b Mon Sep 17 00:00:00 2001 +From 21cd005d8e0a4e5b1bb3bc0768969d4cb1bedafd Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Fri, 26 Jun 2026 13:47:52 +1000 Subject: [PATCH 01/10] Add Skippy ABI and package writer foundation @@ -334,10 +334,10 @@ index 00000000..7ab93d35 + +#endif // SKIPPY_ABI_H diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt -index d15ccfd9..2526f45c 100644 +index 320784c3..5298c337 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt -@@ -35,6 +35,7 @@ add_library(llama +@@ -36,6 +36,7 @@ add_library(llama llama-model.cpp llama-quant.cpp llama-sampler.cpp @@ -346,11 +346,11 @@ index d15ccfd9..2526f45c 100644 unicode-data.cpp unicode.cpp diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp -index 68c9e606..c39487b8 100644 +index 4c86e43c..118987b3 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp -@@ -19,6 +19,20 @@ - #include +@@ -21,6 +21,20 @@ + #include #include +static thread_local skippy_graph_filter g_skippy_graph_filter; @@ -370,7 +370,7 @@ index 68c9e606..c39487b8 100644 // dedup helpers static ggml_tensor * build_attn_inp_kq_mask( -@@ -1855,28 +1869,32 @@ ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd) const { +@@ -2164,28 +2178,32 @@ ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd) const { { auto & cur = inps[0]; @@ -421,10 +421,10 @@ index 68c9e606..c39487b8 100644 } diff --git a/src/llama-graph.h b/src/llama-graph.h -index a6e8c398..00170956 100644 +index 4b5b75c6..b943320e 100644 --- a/src/llama-graph.h +++ b/src/llama-graph.h -@@ -28,6 +28,18 @@ class llama_memory_recurrent_context; +@@ -30,6 +30,18 @@ class llama_memory_recurrent_context; class llama_memory_hybrid_context; class llama_memory_hybrid_iswa_context; @@ -444,7 +444,7 @@ index a6e8c398..00170956 100644 enum llm_graph_type { LLM_GRAPH_TYPE_DEFAULT, diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp -index 474cabdf..86909230 100644 +index e07b0d23..9b0e00ae 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -17,6 +17,16 @@ static const size_t kiB = 1024; @@ -464,7 +464,7 @@ index 474cabdf..86909230 100644 const char * llama_file_version_name(llama_fver version) { switch (version) { case GGUF_FILE_VERSION_V1: return "GGUF V1 (support until nov 2023)"; -@@ -1100,6 +1110,40 @@ struct ggml_tensor * llama_model_loader::create_tensor( +@@ -1103,6 +1113,40 @@ struct ggml_tensor * llama_model_loader::create_tensor( throw std::runtime_error(format("missing tensor info mapping for %s", tn.str().c_str())); } diff --git a/third_party/llama.cpp/patches/0002-Add-early-staged-model-family-and-chat-support.patch b/third_party/llama.cpp/patches/0002-Add-early-staged-model-family-and-chat-support.patch index d5fe106e9..4d67587f2 100644 --- a/third_party/llama.cpp/patches/0002-Add-early-staged-model-family-and-chat-support.patch +++ b/third_party/llama.cpp/patches/0002-Add-early-staged-model-family-and-chat-support.patch @@ -1,4 +1,4 @@ -From 459e05b46cc375894b3386f6f732f6028a9a6699 Mon Sep 17 00:00:00 2001 +From 5625209c975d14399629caa50adf3cbb810dbcf5 Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Fri, 26 Jun 2026 13:47:52 +1000 Subject: [PATCH 02/10] Add early staged model-family and chat support @@ -32,11 +32,11 @@ Squashes the first Qwen MoE staged-runtime support, native KV page basics, sessi create mode 100644 common/stage-chat.cpp diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt -index fc16b21c..4e5a28c0 100644 +index 4cf580a0..b06bfeda 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt -@@ -100,6 +100,7 @@ add_library(${TARGET} - regex-partial.h +@@ -98,6 +98,7 @@ add_library(${TARGET} + reasoning-budget.h sampling.cpp sampling.h + stage-chat.cpp @@ -319,10 +319,10 @@ index 7ab93d35..23a46e0f 100644 const char * path, struct skippy_model_info ** out_info, diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp -index c39487b8..126261f2 100644 +index 118987b3..dfde71c8 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp -@@ -20,6 +20,7 @@ +@@ -22,6 +22,7 @@ #include static thread_local skippy_graph_filter g_skippy_graph_filter; @@ -330,7 +330,7 @@ index c39487b8..126261f2 100644 void skippy_graph_set_filter(const skippy_graph_filter & filter) { g_skippy_graph_filter = filter; -@@ -33,6 +34,18 @@ const skippy_graph_filter & skippy_graph_get_filter() { +@@ -35,6 +36,18 @@ const skippy_graph_filter & skippy_graph_get_filter() { return g_skippy_graph_filter; } @@ -349,7 +349,7 @@ index c39487b8..126261f2 100644 // dedup helpers static ggml_tensor * build_attn_inp_kq_mask( -@@ -153,6 +166,18 @@ bool llm_graph_input_embd_h::can_reuse(const llm_graph_params & params) { +@@ -155,6 +168,18 @@ bool llm_graph_input_embd_h::can_reuse(const llm_graph_params & params) { return res; } @@ -368,7 +368,7 @@ index c39487b8..126261f2 100644 void llm_graph_input_pos::set_input(const llama_ubatch * ubatch) { if (ubatch->pos && pos) { const int64_t n_tokens = ubatch->n_tokens; -@@ -680,17 +705,22 @@ void llm_graph_input_attn_cross::set_input(const llama_ubatch * ubatch) { +@@ -967,17 +992,22 @@ void llm_graph_input_attn_cross::set_input(const llama_ubatch * ubatch) { } void llm_graph_input_mem_hybrid::set_input(const llama_ubatch * ubatch) { @@ -396,7 +396,7 @@ index c39487b8..126261f2 100644 } const int64_t n_rs = mctx->get_recr()->get_n_rs(); -@@ -733,9 +763,14 @@ bool llm_graph_input_mem_hybrid::can_reuse(const llm_graph_params & params) { +@@ -1020,9 +1050,14 @@ bool llm_graph_input_mem_hybrid::can_reuse(const llm_graph_params & params) { // Instead of creating a hybrid input, the graph can simply create 2 separate inputs. // Refactoring is required in the future. void llm_graph_input_mem_hybrid_k::set_input(const llama_ubatch * ubatch) { @@ -414,10 +414,10 @@ index c39487b8..126261f2 100644 const int64_t n_rs = mctx->get_recr()->get_n_rs(); diff --git a/src/llama-graph.h b/src/llama-graph.h -index 00170956..bc1e10e2 100644 +index b943320e..31083518 100644 --- a/src/llama-graph.h +++ b/src/llama-graph.h -@@ -36,9 +36,17 @@ struct skippy_graph_filter { +@@ -38,9 +38,17 @@ struct skippy_graph_filter { bool include_output = false; }; @@ -435,7 +435,7 @@ index 00170956..bc1e10e2 100644 // certain models (typically multi-modal) can produce different types of graphs enum llm_graph_type { -@@ -152,6 +160,18 @@ public: +@@ -154,6 +162,18 @@ public: const int64_t n_embd = 0; }; @@ -455,7 +455,7 @@ index 00170956..bc1e10e2 100644 public: llm_graph_input_pos(uint32_t n_pos_per_embd) : n_pos_per_embd(n_pos_per_embd) {} diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp -index 2802103b..400a9d29 100644 +index 12bf5c37..906e9eac 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp @@ -4,6 +4,7 @@ @@ -466,8 +466,8 @@ index 2802103b..400a9d29 100644 #include #include -@@ -1220,6 +1221,352 @@ ggml_type llama_kv_cache::type_v() const { - return layers[0].v->type; +@@ -1240,6 +1241,352 @@ ggml_tensor * llama_kv_cache::get_k_storage(int32_t il) const { + return layers[ikv].k; } +bool llama_kv_cache::stage_export_kv_page( @@ -820,7 +820,7 @@ index 2802103b..400a9d29 100644 uint32_t result = 0; diff --git a/src/llama-kv-cache.h b/src/llama-kv-cache.h -index 3d68f98c..83cee5f9 100644 +index 531d99db..d9301c19 100644 --- a/src/llama-kv-cache.h +++ b/src/llama-kv-cache.h @@ -6,12 +6,14 @@ @@ -838,9 +838,9 @@ index 3d68f98c..83cee5f9 100644 // // llama_kv_cache -@@ -161,6 +163,23 @@ public: - ggml_type type_k() const; - ggml_type type_v() const; +@@ -164,6 +166,23 @@ public: + std::vector get_layer_ids() const; + ggml_tensor * get_k_storage(int32_t il) const; + bool stage_export_kv_page( + int32_t layer_start, @@ -863,10 +863,10 @@ index 3d68f98c..83cee5f9 100644 // graph_build API // diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp -index 86909230..622b228e 100644 +index 9b0e00ae..561e9850 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp -@@ -1088,13 +1088,6 @@ struct ggml_tensor * llama_model_loader::create_tensor( +@@ -1091,13 +1091,6 @@ struct ggml_tensor * llama_model_loader::create_tensor( }; auto buft_for_tensor = [&](ggml_tensor * t_meta) -> ggml_backend_buffer_type_t { @@ -880,7 +880,7 @@ index 86909230..622b228e 100644 // some models use the token embedding tensor as the output, but since these are used in different layers and with different ops // the tensor is duplicated // to handle this, we check if the tensor is duplicated, and if so, we assume that it is being loaded as the output tensor -@@ -1128,22 +1121,38 @@ struct ggml_tensor * llama_model_loader::create_tensor( +@@ -1131,22 +1124,38 @@ struct ggml_tensor * llama_model_loader::create_tensor( break; } @@ -929,10 +929,10 @@ index 86909230..622b228e 100644 if (info.op == GGML_OP_NONE || (flags & TENSOR_SKIP)) { const size_t nbytes = ggml_nbytes(t_meta); diff --git a/src/llama-model.cpp b/src/llama-model.cpp -index d041a9ce..fc13d414 100644 +index d58ebac2..3196973f 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp -@@ -1360,13 +1360,13 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) { +@@ -1367,13 +1367,13 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) { } // MoE expert weight scales (per-expert, shape {n_expert}) diff --git a/third_party/llama.cpp/patches/0003-Add-staged-sampling-checkpoints-and-part-loading.patch b/third_party/llama.cpp/patches/0003-Add-staged-sampling-checkpoints-and-part-loading.patch index 2ac93fca3..a2886a569 100644 --- a/third_party/llama.cpp/patches/0003-Add-staged-sampling-checkpoints-and-part-loading.patch +++ b/third_party/llama.cpp/patches/0003-Add-staged-sampling-checkpoints-and-part-loading.patch @@ -1,4 +1,4 @@ -From 3bb375d5797de655cdee37f0a8eb272cfb609812 Mon Sep 17 00:00:00 2001 +From a4f23f17019419c4e1f7acd2bea9bd9ee18159f9 Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Fri, 26 Jun 2026 13:47:52 +1000 Subject: [PATCH 03/10] Add staged sampling, checkpoints, and part loading @@ -245,10 +245,10 @@ index 23a46e0f..4dd985da 100644 size_t output_text_capacity, size_t * out_text_bytes, diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp -index 622b228e..16cc0b06 100644 +index 561e9850..97372c4c 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp -@@ -525,6 +525,7 @@ llama_model_loader::llama_model_loader( +@@ -528,6 +528,7 @@ llama_model_loader::llama_model_loader( void * set_tensor_data_ud, const std::string & fname, std::vector & splits, @@ -256,7 +256,7 @@ index 622b228e..16cc0b06 100644 FILE * file, bool use_mmap, bool use_direct_io, -@@ -593,11 +594,50 @@ llama_model_loader::llama_model_loader( +@@ -596,11 +597,50 @@ llama_model_loader::llama_model_loader( n_bytes += ggml_nbytes(cur); weights_map.emplace(tensor_name, llama_tensor_weight(files.back().get(), 0, metadata, cur)); } diff --git a/third_party/llama.cpp/patches/0004-Add-lanes-external-media-and-chat-grammar-support.patch b/third_party/llama.cpp/patches/0004-Add-lanes-external-media-and-chat-grammar-support.patch index a431c1d61..e2f584250 100644 --- a/third_party/llama.cpp/patches/0004-Add-lanes-external-media-and-chat-grammar-support.patch +++ b/third_party/llama.cpp/patches/0004-Add-lanes-external-media-and-chat-grammar-support.patch @@ -1,4 +1,4 @@ -From 0e34ce3acca749ba8e949d9b3b9e24dfc1537933 Mon Sep 17 00:00:00 2001 +From 29baeca7b5f406c2df3966e98c1e5e4b4ef10258 Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Fri, 26 Jun 2026 13:47:52 +1000 Subject: [PATCH 04/10] Add lanes, external media, and chat grammar support @@ -8,11 +8,11 @@ Squashes Qwen3 MoE staged execution, KV cache type and generation signal ABI, ba common/stage-chat.cpp | 230 ++++++++ include/skippy-signals.h | 44 ++ include/skippy.h | 165 +++--- - src/llama-kv-cache.cpp | 346 ------------ + src/llama-kv-cache.cpp | 361 +------------ src/llama-kv-cache.h | 19 - src/models/qwen3moe.cpp | 18 +- src/skippy.cpp | 1079 +++++++++++++++++++++++--------------- - 7 files changed, 1013 insertions(+), 888 deletions(-) + 7 files changed, 1019 insertions(+), 897 deletions(-) create mode 100644 include/skippy-signals.h diff --git a/common/stage-chat.cpp b/common/stage-chat.cpp @@ -573,11 +573,40 @@ index 4dd985da..3ffdb879 100644 const char * path, struct skippy_model_info ** out_info, diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp -index 400a9d29..024bac7a 100644 +index 906e9eac..4bdefc22 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp -@@ -1221,352 +1221,6 @@ ggml_type llama_kv_cache::type_v() const { - return layers[0].v->type; +@@ -212,12 +212,10 @@ llama_kv_cache::llama_kv_cache( + n_embd_head_k_all = -1; + } + +- if (!is_mla) { +- if (n_embd_head_v_all == 0) { +- n_embd_head_v_all = (int32_t) hparams.n_embd_head_v(il); +- } else if (n_embd_head_v_all > 0 && n_embd_head_v_all != (int32_t) hparams.n_embd_head_v(il)) { +- n_embd_head_v_all = -1; +- } ++ if (n_embd_head_v_all == 0) { ++ n_embd_head_v_all = (int32_t) hparams.n_embd_head_v(il); ++ } else if (n_embd_head_v_all > 0 && n_embd_head_v_all != (int32_t) hparams.n_embd_head_v(il)) { ++ n_embd_head_v_all = -1; + } + + // [TAG_V_CACHE_VARIABLE] +@@ -339,9 +337,8 @@ llama_kv_cache::llama_kv_cache( + ggml_is_quantized(type_k) && + hparams.n_embd_head_k() % 64 == 0; + +- // always create Hadamard rotation tensors for DeepSeek lightning indexers +- if ((model.arch == LLM_ARCH_DEEPSEEK32 || model.arch == LLM_ARCH_DEEPSEEK4) && +- hparams.n_embd_head_k_full == hparams.indexer_head_size) { ++ // always create Hadamard rotation tensors for DeepSeek V3.2 DSA lightning indexer ++ if (model.arch == LLM_ARCH_DEEPSEEK32 && hparams.n_embd_head_k_full == hparams.indexer_head_size) { + attn_rot_k = true; + } + +@@ -1241,352 +1238,6 @@ ggml_tensor * llama_kv_cache::get_k_storage(int32_t il) const { + return layers[ikv].k; } -bool llama_kv_cache::stage_export_kv_page( @@ -930,7 +959,7 @@ index 400a9d29..024bac7a 100644 uint32_t result = 0; diff --git a/src/llama-kv-cache.h b/src/llama-kv-cache.h -index 83cee5f9..a3a1ec96 100644 +index d9301c19..92394d4c 100644 --- a/src/llama-kv-cache.h +++ b/src/llama-kv-cache.h @@ -13,8 +13,6 @@ struct llama_cparams; @@ -942,9 +971,9 @@ index 83cee5f9..a3a1ec96 100644 // // llama_kv_cache // -@@ -163,23 +161,6 @@ public: - ggml_type type_k() const; - ggml_type type_v() const; +@@ -166,23 +164,6 @@ public: + std::vector get_layer_ids() const; + ggml_tensor * get_k_storage(int32_t il) const; - bool stage_export_kv_page( - int32_t layer_start, diff --git a/third_party/llama.cpp/patches/0005-Add-resident-prefix-cache-and-session-refinements.patch b/third_party/llama.cpp/patches/0005-Add-resident-prefix-cache-and-session-refinements.patch index 6b5fa651c..96343b176 100644 --- a/third_party/llama.cpp/patches/0005-Add-resident-prefix-cache-and-session-refinements.patch +++ b/third_party/llama.cpp/patches/0005-Add-resident-prefix-cache-and-session-refinements.patch @@ -1,4 +1,4 @@ -From 54d9cc574e4e79f12cdb467fda805c0e1c50a89c Mon Sep 17 00:00:00 2001 +From a1a99269e1e3205bb9ba54fe5dcecb11bb5b0edb Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Fri, 26 Jun 2026 13:47:52 +1000 Subject: [PATCH 05/10] Add resident prefix cache and session refinements @@ -196,11 +196,11 @@ index 3ffdb879..deb5151e 100644 struct skippy_model * model, const char * text, diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp -index 024bac7a..6fe17286 100644 +index 4bdefc22..53b3b035 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp -@@ -1221,6 +1221,363 @@ ggml_type llama_kv_cache::type_v() const { - return layers[0].v->type; +@@ -1238,6 +1238,363 @@ ggml_tensor * llama_kv_cache::get_k_storage(int32_t il) const { + return layers[ikv].k; } +bool llama_kv_cache::stage_export_kv_page( @@ -564,7 +564,7 @@ index 024bac7a..6fe17286 100644 uint32_t result = 0; diff --git a/src/llama-kv-cache.h b/src/llama-kv-cache.h -index a3a1ec96..52a013fc 100644 +index 92394d4c..be51c3c7 100644 --- a/src/llama-kv-cache.h +++ b/src/llama-kv-cache.h @@ -13,6 +13,8 @@ struct llama_cparams; @@ -576,9 +576,9 @@ index a3a1ec96..52a013fc 100644 // // llama_kv_cache // -@@ -161,6 +163,25 @@ public: - ggml_type type_k() const; - ggml_type type_v() const; +@@ -164,6 +166,25 @@ public: + std::vector get_layer_ids() const; + ggml_tensor * get_k_storage(int32_t il) const; + bool stage_export_kv_page( + llama_seq_id seq_id, diff --git a/third_party/llama.cpp/patches/0006-Expand-staged-execution-across-dense-and-recurrent-f.patch b/third_party/llama.cpp/patches/0006-Expand-staged-execution-across-dense-and-recurrent-f.patch index 880e9aed7..eafd80d90 100644 --- a/third_party/llama.cpp/patches/0006-Expand-staged-execution-across-dense-and-recurrent-f.patch +++ b/third_party/llama.cpp/patches/0006-Expand-staged-execution-across-dense-and-recurrent-f.patch @@ -1,4 +1,4 @@ -From d0277a060caf50075309793f8766d659c0738945 Mon Sep 17 00:00:00 2001 +From e460fa7f64063a38265e145a9e27c81bdcedd044 Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Fri, 26 Jun 2026 13:47:53 +1000 Subject: [PATCH 06/10] Expand staged execution across dense and recurrent @@ -51,10 +51,10 @@ index deb5151e..84169398 100644 const struct skippy_session * session); diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp -index 126261f2..f975b84f 100644 +index dfde71c8..4945ab1e 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp -@@ -1951,10 +1951,15 @@ ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd) const { +@@ -2260,10 +2260,15 @@ ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd) const { res->t_inp_embd = cur; @@ -719,10 +719,10 @@ index dba160b0..2b69aaba 100644 cur = build_norm(inpL, model.output_norm, NULL, LLM_NORM_RMS, -1); diff --git a/src/models/lfm2.cpp b/src/models/lfm2.cpp -index 07b7346e..c497c3af 100644 +index 70e837d6..3534c956 100644 --- a/src/models/lfm2.cpp +++ b/src/models/lfm2.cpp -@@ -225,7 +225,20 @@ llama_model_lfm2::graph::graph(const llama_model & model, const llm_graph_ +@@ -226,7 +226,20 @@ llama_model_lfm2::graph::graph(const llama_model & model, const llm_graph_ }; // actual graph construction starts here @@ -744,7 +744,7 @@ index 07b7346e..c497c3af 100644 cb(cur, "model.embed_tokens", -1); ggml_build_forward_expand(gf, cur); -@@ -237,10 +250,10 @@ llama_model_lfm2::graph::graph(const llama_model & model, const llm_graph_ +@@ -238,10 +251,10 @@ llama_model_lfm2::graph::graph(const llama_model & model, const llm_graph_ inp_hybrid = build_inp_mem_hybrid(); } @@ -758,7 +758,7 @@ index 07b7346e..c497c3af 100644 const bool is_moe_layer = il >= static_cast(hparams.n_layer_dense_lead); auto * prev_cur = cur; -@@ -250,7 +263,7 @@ llama_model_lfm2::graph::graph(const llama_model & model, const llm_graph_ +@@ -251,7 +264,7 @@ llama_model_lfm2::graph::graph(const llama_model & model, const llm_graph_ cur = hparams.is_recr(il) ? build_shortconv_block(cur, inp_hybrid->get_recr(), il) : build_attn_block(cur, inp_pos, inp_hybrid->get_attn(), il); @@ -767,7 +767,7 @@ index 07b7346e..c497c3af 100644 cur = ggml_get_rows(ctx0, cur, inp_out_ids); prev_cur = ggml_get_rows(ctx0, prev_cur, inp_out_ids); } -@@ -270,6 +283,13 @@ llama_model_lfm2::graph::graph(const llama_model & model, const llm_graph_ +@@ -271,6 +284,13 @@ llama_model_lfm2::graph::graph(const llama_model & model, const llm_graph_ cb(cur, "l_out", il); } diff --git a/third_party/llama.cpp/patches/0007-Expand-staged-execution-across-VL-and-broad-model-fa.patch b/third_party/llama.cpp/patches/0007-Expand-staged-execution-across-VL-and-broad-model-fa.patch index f67875df9..8b7e7adce 100644 --- a/third_party/llama.cpp/patches/0007-Expand-staged-execution-across-VL-and-broad-model-fa.patch +++ b/third_party/llama.cpp/patches/0007-Expand-staged-execution-across-VL-and-broad-model-fa.patch @@ -1,4 +1,4 @@ -From 72f58d31ea15dd6b8bcdb846ce52b98120f2c026 Mon Sep 17 00:00:00 2001 +From e33129c171f99df9c7edc983078f5282d4a389d1 Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Fri, 26 Jun 2026 13:47:53 +1000 Subject: [PATCH 07/10] Expand staged execution across VL and broad model @@ -84,10 +84,10 @@ index 84169398..1c6d2c9f 100644 struct skippy_session; struct skippy_model_info; diff --git a/src/llama-context.cpp b/src/llama-context.cpp -index 220240ea..c10fa290 100644 +index 0465430d..d7f85fd4 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp -@@ -2335,6 +2335,10 @@ llm_graph_result * llama_context::get_gf_res_reserve() const { +@@ -2339,6 +2339,10 @@ llm_graph_result * llama_context::get_gf_res_reserve() const { return static_cast(gf_res_reserve.get()); } @@ -111,10 +111,10 @@ index f8b78058..b8fe101c 100644 // returns the result of ggml_backend_sched_graph_compute_async execution ggml_status graph_compute(ggml_cgraph * gf, bool batched); diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp -index f975b84f..57c02537 100644 +index 4945ab1e..58f644ab 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp -@@ -21,6 +21,7 @@ +@@ -23,6 +23,7 @@ static thread_local skippy_graph_filter g_skippy_graph_filter; static thread_local skippy_activation_tokens g_skippy_activation_tokens; @@ -122,7 +122,7 @@ index f975b84f..57c02537 100644 void skippy_graph_set_filter(const skippy_graph_filter & filter) { g_skippy_graph_filter = filter; -@@ -46,6 +47,18 @@ const skippy_activation_tokens & skippy_graph_get_activation_tokens() { +@@ -48,6 +49,18 @@ const skippy_activation_tokens & skippy_graph_get_activation_tokens() { return g_skippy_activation_tokens; } @@ -141,7 +141,7 @@ index f975b84f..57c02537 100644 // dedup helpers static ggml_tensor * build_attn_inp_kq_mask( -@@ -178,6 +191,19 @@ bool llm_graph_input_stage_tokens::can_reuse(const llm_graph_params & params) { +@@ -180,6 +193,19 @@ bool llm_graph_input_stage_tokens::can_reuse(const llm_graph_params & params) { return tokens && tokens->ne[0] == params.ubatch.n_tokens; } @@ -161,15 +161,15 @@ index f975b84f..57c02537 100644 void llm_graph_input_pos::set_input(const llama_ubatch * ubatch) { if (ubatch->pos && pos) { const int64_t n_tokens = ubatch->n_tokens; -@@ -953,6 +979,7 @@ void llm_graph_result::reset() { - t_logits = nullptr; +@@ -1241,6 +1267,7 @@ void llm_graph_result::reset() { t_embd = nullptr; t_embd_pooled = nullptr; + t_h_nextn = nullptr; + t_skippy_rwkv7_v_first = nullptr; t_layer_inp.resize(LLAMA_MAX_LAYERS); std::fill(t_layer_inp.begin(), t_layer_inp.end(), nullptr); -@@ -998,6 +1025,9 @@ void llm_graph_result::set_outputs(const llm_graph_params & params) { +@@ -1286,6 +1313,9 @@ void llm_graph_result::set_outputs(const llm_graph_params & params) { if (t_h_nextn != nullptr) { ggml_set_output(t_h_nextn); } @@ -180,10 +180,10 @@ index f975b84f..57c02537 100644 const auto & embeddings_layer_inp = params.cparams.embeddings_layer_inp; for (size_t il = 0; il < embeddings_layer_inp.size(); ++il) { diff --git a/src/llama-graph.h b/src/llama-graph.h -index bc1e10e2..2099ff40 100644 +index 31083518..d846f1ad 100644 --- a/src/llama-graph.h +++ b/src/llama-graph.h -@@ -41,12 +41,21 @@ struct skippy_activation_tokens { +@@ -43,12 +43,21 @@ struct skippy_activation_tokens { uint32_t token_count = 0; }; @@ -205,7 +205,7 @@ index bc1e10e2..2099ff40 100644 // certain models (typically multi-modal) can produce different types of graphs enum llm_graph_type { -@@ -172,6 +181,21 @@ public: +@@ -174,6 +183,21 @@ public: ggml_tensor * tokens = nullptr; // I32 [n_batch] }; @@ -227,7 +227,7 @@ index bc1e10e2..2099ff40 100644 class llm_graph_input_pos : public llm_graph_input_i { public: llm_graph_input_pos(uint32_t n_pos_per_embd) : n_pos_per_embd(n_pos_per_embd) {} -@@ -743,6 +767,7 @@ public: +@@ -818,6 +842,7 @@ public: ggml_tensor * get_embd() const { return t_embd; } ggml_tensor * get_embd_pooled() const { return t_embd_pooled; } ggml_tensor * get_h_nextn() const { return t_h_nextn; } @@ -235,7 +235,7 @@ index bc1e10e2..2099ff40 100644 ggml_tensor * get_layer_inp(int il) const { return t_layer_inp[il]; } -@@ -774,6 +799,7 @@ public: +@@ -849,6 +874,7 @@ public: ggml_tensor * t_embd = nullptr; ggml_tensor * t_embd_pooled = nullptr; ggml_tensor * t_h_nextn = nullptr; // [n_embd, n_outputs] hidden state before final output norm @@ -244,7 +244,7 @@ index bc1e10e2..2099ff40 100644 std::vector t_layer_inp; diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp -index 16cc0b06..38e0bc2a 100644 +index 97372c4c..b3c72a65 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -18,6 +18,7 @@ static const size_t MiB = 1024*kiB; @@ -266,7 +266,7 @@ index 16cc0b06..38e0bc2a 100644 const char * llama_file_version_name(llama_fver version) { switch (version) { case GGUF_FILE_VERSION_V1: return "GGUF V1 (support until nov 2023)"; -@@ -1180,6 +1185,9 @@ struct ggml_tensor * llama_model_loader::create_tensor( +@@ -1183,6 +1188,9 @@ struct ggml_tensor * llama_model_loader::create_tensor( if (!(flags & TENSOR_DUPLICATED)) { n_created++; } @@ -276,7 +276,7 @@ index 16cc0b06..38e0bc2a 100644 } return nullptr; -@@ -1187,12 +1195,15 @@ struct ggml_tensor * llama_model_loader::create_tensor( +@@ -1190,12 +1198,15 @@ struct ggml_tensor * llama_model_loader::create_tensor( } if (!t_meta) { @@ -305,10 +305,10 @@ index d889c99d..d85aa85d 100644 enum llama_fver { GGUF_FILE_VERSION_V1 = 1, diff --git a/src/llama-model.cpp b/src/llama-model.cpp -index fc13d414..eab46cf8 100644 +index 3196973f..95b22b7f 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp -@@ -2691,7 +2691,8 @@ void llama_model_base::create_tensor_qkv(llama_layer & layer, int bid, +@@ -2723,7 +2723,8 @@ void llama_model_base::create_tensor_qkv(llama_layer & layer, int bid, int flags) { const int64_t n_embd_qkv = n_embd_q_ + n_embd_k_ + n_embd_v_; layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", bid), {n_embd_, n_embd_qkv}, TENSOR_NOT_REQUIRED | TENSOR_SKIP_IF_VIRTUAL); diff --git a/third_party/llama.cpp/patches/0008-Add-external-decode-media-prefill-and-newer-family-s.patch b/third_party/llama.cpp/patches/0008-Add-external-decode-media-prefill-and-newer-family-s.patch index 95a51a2c1..71ebfefab 100644 --- a/third_party/llama.cpp/patches/0008-Add-external-decode-media-prefill-and-newer-family-s.patch +++ b/third_party/llama.cpp/patches/0008-Add-external-decode-media-prefill-and-newer-family-s.patch @@ -1,4 +1,4 @@ -From 83d3728b6e74381bced4ae728a8eeb66f912a24d Mon Sep 17 00:00:00 2001 +From 53c4c4f69edb33807ce4cc9b958bc97c15445eae Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Fri, 26 Jun 2026 13:47:53 +1000 Subject: [PATCH 08/10] Add external decode, media prefill, and newer family @@ -98,10 +98,10 @@ index 1c6d2c9f..f1ff7852 100644 struct skippy_session * session, llama_token token_id, diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp -index 57c02537..64d38fc4 100644 +index 58f644ab..af0e44e5 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp -@@ -22,6 +22,7 @@ +@@ -24,6 +24,7 @@ static thread_local skippy_graph_filter g_skippy_graph_filter; static thread_local skippy_activation_tokens g_skippy_activation_tokens; static thread_local skippy_activation_rwkv7_v_first g_skippy_rwkv7_v_first; @@ -109,7 +109,7 @@ index 57c02537..64d38fc4 100644 void skippy_graph_set_filter(const skippy_graph_filter & filter) { g_skippy_graph_filter = filter; -@@ -59,6 +60,18 @@ const skippy_activation_rwkv7_v_first & skippy_graph_get_rwkv7_v_first() { +@@ -61,6 +62,18 @@ const skippy_activation_rwkv7_v_first & skippy_graph_get_rwkv7_v_first() { return g_skippy_rwkv7_v_first; } @@ -128,7 +128,7 @@ index 57c02537..64d38fc4 100644 // dedup helpers static ggml_tensor * build_attn_inp_kq_mask( -@@ -204,6 +217,20 @@ bool llm_graph_input_rwkv7_v_first::can_reuse(const llm_graph_params & params) { +@@ -206,6 +219,20 @@ bool llm_graph_input_rwkv7_v_first::can_reuse(const llm_graph_params & params) { return values && values->ne[0] == n_embd && values->ne[1] == params.ubatch.n_tokens; } @@ -149,7 +149,7 @@ index 57c02537..64d38fc4 100644 void llm_graph_input_pos::set_input(const llama_ubatch * ubatch) { if (ubatch->pos && pos) { const int64_t n_tokens = ubatch->n_tokens; -@@ -647,19 +674,19 @@ void llm_graph_input_attn_kv_iswa::set_input(const llama_ubatch * ubatch) { +@@ -657,19 +684,19 @@ void llm_graph_input_attn_kv_iswa::set_input(const llama_ubatch * ubatch) { mctx->get_swa()->set_input_kq_mask(self_kq_mask_swa, ubatch, cparams.causal_attn); } @@ -173,7 +173,7 @@ index 57c02537..64d38fc4 100644 mctx->get_swa()->set_input_v_rot(self_v_rot_swa); } } -@@ -751,7 +778,7 @@ void llm_graph_input_mem_hybrid::set_input(const llama_ubatch * ubatch) { +@@ -1038,7 +1065,7 @@ void llm_graph_input_mem_hybrid::set_input(const llama_ubatch * ubatch) { const int64_t n_rs = mctx->get_recr()->get_n_rs(); @@ -182,7 +182,7 @@ index 57c02537..64d38fc4 100644 GGML_ASSERT(ggml_backend_buffer_is_host(inp_rs->s_copy->buffer)); int32_t * data = (int32_t *) inp_rs->s_copy->data; -@@ -774,10 +801,16 @@ bool llm_graph_input_mem_hybrid::can_reuse(const llm_graph_params & params) { +@@ -1061,10 +1088,16 @@ bool llm_graph_input_mem_hybrid::can_reuse(const llm_graph_params & params) { res &= can_reuse_kq_mask(inp_attn->self_kq_mask, mctx->get_attn(), params.ubatch, params.cparams); @@ -202,7 +202,7 @@ index 57c02537..64d38fc4 100644 res &= inp_rs->head == mctx->get_recr()->get_head(); res &= inp_rs->rs_z == mctx->get_recr()->get_rs_z(); -@@ -800,7 +833,7 @@ void llm_graph_input_mem_hybrid_k::set_input(const llama_ubatch * ubatch) { +@@ -1087,7 +1120,7 @@ void llm_graph_input_mem_hybrid_k::set_input(const llama_ubatch * ubatch) { const int64_t n_rs = mctx->get_recr()->get_n_rs(); @@ -211,7 +211,7 @@ index 57c02537..64d38fc4 100644 GGML_ASSERT(ggml_backend_buffer_is_host(inp_rs->s_copy->buffer)); int32_t * data = (int32_t *) inp_rs->s_copy->data; -@@ -822,10 +855,16 @@ bool llm_graph_input_mem_hybrid_k::can_reuse(const llm_graph_params & params) { +@@ -1109,10 +1142,16 @@ bool llm_graph_input_mem_hybrid_k::can_reuse(const llm_graph_params & params) { res &= can_reuse_kq_mask(inp_attn->self_kq_mask, mctx->get_attn(), params.ubatch, params.cparams); @@ -231,7 +231,7 @@ index 57c02537..64d38fc4 100644 res &= inp_rs->head == mctx->get_recr()->get_head(); res &= inp_rs->rs_z == mctx->get_recr()->get_rs_z(); -@@ -874,7 +913,7 @@ void llm_graph_input_mem_hybrid_iswa::set_input(const llama_ubatch * ubatch) { +@@ -1161,7 +1200,7 @@ void llm_graph_input_mem_hybrid_iswa::set_input(const llama_ubatch * ubatch) { const int64_t n_rs = mctx->get_recr()->get_n_rs(); @@ -240,7 +240,7 @@ index 57c02537..64d38fc4 100644 GGML_ASSERT(ggml_backend_buffer_is_host(inp_rs->s_copy->buffer)); int32_t * data = (int32_t *) inp_rs->s_copy->data; -@@ -910,10 +949,16 @@ bool llm_graph_input_mem_hybrid_iswa::can_reuse(const llm_graph_params & params) +@@ -1197,10 +1236,16 @@ bool llm_graph_input_mem_hybrid_iswa::can_reuse(const llm_graph_params & params) res &= can_reuse_kq_mask(inp_attn->self_kq_mask_swa, attn_ctx->get_swa(), params.ubatch, params.cparams); @@ -260,15 +260,15 @@ index 57c02537..64d38fc4 100644 res &= inp_rs->head == mctx->get_recr()->get_head(); res &= inp_rs->rs_z == mctx->get_recr()->get_rs_z(); -@@ -980,6 +1025,7 @@ void llm_graph_result::reset() { - t_embd = nullptr; +@@ -1268,6 +1313,7 @@ void llm_graph_result::reset() { t_embd_pooled = nullptr; + t_h_nextn = nullptr; t_skippy_rwkv7_v_first = nullptr; + t_skippy_gemma3n_altup = nullptr; t_layer_inp.resize(LLAMA_MAX_LAYERS); std::fill(t_layer_inp.begin(), t_layer_inp.end(), nullptr); -@@ -1028,6 +1074,9 @@ void llm_graph_result::set_outputs(const llm_graph_params & params) { +@@ -1316,6 +1362,9 @@ void llm_graph_result::set_outputs(const llm_graph_params & params) { if (t_skippy_rwkv7_v_first != nullptr) { ggml_set_output(t_skippy_rwkv7_v_first); } @@ -279,10 +279,10 @@ index 57c02537..64d38fc4 100644 const auto & embeddings_layer_inp = params.cparams.embeddings_layer_inp; for (size_t il = 0; il < embeddings_layer_inp.size(); ++il) { diff --git a/src/llama-graph.h b/src/llama-graph.h -index 2099ff40..416ae150 100644 +index d846f1ad..6838570c 100644 --- a/src/llama-graph.h +++ b/src/llama-graph.h -@@ -47,6 +47,13 @@ struct skippy_activation_rwkv7_v_first { +@@ -49,6 +49,13 @@ struct skippy_activation_rwkv7_v_first { uint32_t n_embd = 0; }; @@ -296,7 +296,7 @@ index 2099ff40..416ae150 100644 void skippy_graph_set_filter(const skippy_graph_filter & filter); void skippy_graph_clear_filter(); const skippy_graph_filter & skippy_graph_get_filter(); -@@ -56,6 +63,9 @@ const skippy_activation_tokens & skippy_graph_get_activation_tokens(); +@@ -58,6 +65,9 @@ const skippy_activation_tokens & skippy_graph_get_activation_tokens(); void skippy_graph_set_rwkv7_v_first(const skippy_activation_rwkv7_v_first & values); void skippy_graph_clear_rwkv7_v_first(); const skippy_activation_rwkv7_v_first & skippy_graph_get_rwkv7_v_first(); @@ -306,7 +306,7 @@ index 2099ff40..416ae150 100644 // certain models (typically multi-modal) can produce different types of graphs enum llm_graph_type { -@@ -196,6 +206,22 @@ private: +@@ -198,6 +208,22 @@ private: int64_t n_embd; }; @@ -329,7 +329,7 @@ index 2099ff40..416ae150 100644 class llm_graph_input_pos : public llm_graph_input_i { public: llm_graph_input_pos(uint32_t n_pos_per_embd) : n_pos_per_embd(n_pos_per_embd) {} -@@ -768,6 +794,7 @@ public: +@@ -843,6 +869,7 @@ public: ggml_tensor * get_embd_pooled() const { return t_embd_pooled; } ggml_tensor * get_h_nextn() const { return t_h_nextn; } ggml_tensor * get_skippy_rwkv7_v_first() const { return t_skippy_rwkv7_v_first; } @@ -337,7 +337,7 @@ index 2099ff40..416ae150 100644 ggml_tensor * get_layer_inp(int il) const { return t_layer_inp[il]; } -@@ -800,6 +827,7 @@ public: +@@ -875,6 +902,7 @@ public: ggml_tensor * t_embd_pooled = nullptr; ggml_tensor * t_h_nextn = nullptr; // [n_embd, n_outputs] hidden state before final output norm ggml_tensor * t_skippy_rwkv7_v_first = nullptr; @@ -346,10 +346,10 @@ index 2099ff40..416ae150 100644 std::vector t_layer_inp; diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp -index 6fe17286..8a3fb548 100644 +index 53b3b035..c29fe289 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp -@@ -1300,14 +1300,16 @@ bool llama_kv_cache::stage_export_kv_page( +@@ -1317,14 +1317,16 @@ bool llama_kv_cache::stage_export_kv_page( if (il < static_cast(layer_start) || il >= static_cast(layer_end)) { continue; } @@ -371,7 +371,7 @@ index 6fe17286..8a3fb548 100644 if (k_row > std::numeric_limits::max() || v_row > std::numeric_limits::max() || -@@ -1320,19 +1322,19 @@ bool llama_kv_cache::stage_export_kv_page( +@@ -1337,19 +1339,19 @@ bool llama_kv_cache::stage_export_kv_page( v_row_bytes = static_cast(v_row); v_element_bytes = static_cast(v_el); k_type = static_cast(layer.k_stream[strm]->type); @@ -396,7 +396,7 @@ index 6fe17286..8a3fb548 100644 const uint64_t k_bytes = token_count * k_row; if (payload_bytes > std::numeric_limits::max() - k_bytes || payload_bytes + k_bytes > std::numeric_limits::max() - v_bytes) { -@@ -1386,6 +1388,9 @@ bool llama_kv_cache::stage_export_kv_page( +@@ -1403,6 +1405,9 @@ bool llama_kv_cache::stage_export_kv_page( if (!v_trans) { for (const auto * layer : selected) { @@ -406,7 +406,7 @@ index 6fe17286..8a3fb548 100644 auto * v = layer->v_stream[strm]; for (uint32_t cell_idx : cell_idxs) { ggml_backend_tensor_get(v, dst, static_cast(cell_idx) * v_row_bytes, v_row_bytes); -@@ -1464,15 +1469,17 @@ bool llama_kv_cache::stage_import_kv_page( +@@ -1481,15 +1486,17 @@ bool llama_kv_cache::stage_import_kv_page( if (il < static_cast(desc.layer_start) || il >= static_cast(desc.layer_end)) { continue; } @@ -430,7 +430,7 @@ index 6fe17286..8a3fb548 100644 desc.k_row_bytes != k_row || desc.v_row_bytes != v_row || desc.v_element_bytes != v_el) { -@@ -1555,6 +1562,9 @@ bool llama_kv_cache::stage_import_kv_page( +@@ -1572,6 +1579,9 @@ bool llama_kv_cache::stage_import_kv_page( auto & cells = v_cells[strm]; if (!v_trans) { for (const auto * layer : selected) { @@ -441,10 +441,10 @@ index 6fe17286..8a3fb548 100644 for (uint32_t i = 0; i < n_tokens; ++i) { ggml_backend_tensor_set(v, src, static_cast(sinfo.idxs[0][i]) * desc.v_row_bytes, desc.v_row_bytes); diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp -index 38e0bc2a..1e7ec54c 100644 +index b3c72a65..4b4b2508 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp -@@ -1132,6 +1132,19 @@ struct ggml_tensor * llama_model_loader::create_tensor( +@@ -1135,6 +1135,19 @@ struct ggml_tensor * llama_model_loader::create_tensor( return it->second.get(); }; @@ -464,7 +464,7 @@ index 38e0bc2a..1e7ec54c 100644 auto buft_for_tensor = [&](ggml_tensor * t_meta) -> ggml_backend_buffer_type_t { // some models use the token embedding tensor as the output, but since these are used in different layers and with different ops // the tensor is duplicated -@@ -1172,18 +1185,34 @@ struct ggml_tensor * llama_model_loader::create_tensor( +@@ -1175,18 +1188,34 @@ struct ggml_tensor * llama_model_loader::create_tensor( tn.tensor == LLM_TENSOR_PER_LAYER_MODEL_PROJ || tn.tensor == LLM_TENSOR_PER_LAYER_PROJ_NORM; } diff --git a/third_party/llama.cpp/patches/0009-Add-chat-grammar-device-enumeration-and-runtime-even.patch b/third_party/llama.cpp/patches/0009-Add-chat-grammar-device-enumeration-and-runtime-even.patch index 23c935d9e..9888bcd6c 100644 --- a/third_party/llama.cpp/patches/0009-Add-chat-grammar-device-enumeration-and-runtime-even.patch +++ b/third_party/llama.cpp/patches/0009-Add-chat-grammar-device-enumeration-and-runtime-even.patch @@ -1,4 +1,4 @@ -From cb0d039fd5f045ee4ca9fce853bbcadb00c98fd4 Mon Sep 17 00:00:00 2001 +From be13ce7e46bea3e444fcf0aee3e611a7d9fbaabc Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Fri, 26 Jun 2026 13:47:53 +1000 Subject: [PATCH 09/10] Add chat grammar, device enumeration, and runtime @@ -446,7 +446,7 @@ index 00000000..382f1e69 + +#endif diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp -index 8a3fb548..0957c6c8 100644 +index c29fe289..3e2e8abb 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp @@ -751,11 +751,28 @@ llama_memory_context_ptr llama_kv_cache::init_full() { @@ -575,7 +575,7 @@ index 8a3fb548..0957c6c8 100644 bool llama_kv_cache::get_can_shift() const { // Step35 uses per-layer RoPE dims; K-shift assumes a single global n_rot. if (model.arch == LLM_ARCH_STEP35) { -@@ -2877,8 +2981,9 @@ llama_kv_cache_context::llama_kv_cache_context( +@@ -2894,8 +2998,9 @@ llama_kv_cache_context::llama_kv_cache_context( llama_kv_cache * kv, llama_context * lctx, bool do_shift, @@ -588,7 +588,7 @@ index 8a3fb548..0957c6c8 100644 } } diff --git a/src/llama-kv-cache.h b/src/llama-kv-cache.h -index 52a013fc..c1fdfbd5 100644 +index be51c3c7..f8569057 100644 --- a/src/llama-kv-cache.h +++ b/src/llama-kv-cache.h @@ -95,6 +95,12 @@ public: @@ -604,7 +604,7 @@ index 52a013fc..c1fdfbd5 100644 // TODO: refactor the memory instances to not depend on `llama_model` // instead pass all necessary info (e.g. hparams, dev layers, arch, etc.) directly // likely through `struct llama_memory_params` -@@ -214,6 +220,10 @@ public: +@@ -217,6 +223,10 @@ public: // emplace the ubatch context into slot: [sinfo.idxs[0...ubatch.n_tokens - 1]] void apply_ubatch(const slot_info & sinfo, const llama_ubatch & ubatch); @@ -615,7 +615,7 @@ index 52a013fc..c1fdfbd5 100644 // // input API // -@@ -356,7 +366,8 @@ public: +@@ -359,7 +369,8 @@ public: llama_kv_cache * kv, llama_context * lctx, bool do_shift, diff --git a/third_party/llama.cpp/patches/0010-Add-MTP-execution-support-and-sampling-cleanup.patch b/third_party/llama.cpp/patches/0010-Add-MTP-execution-support-and-sampling-cleanup.patch index 37262a718..406116bf5 100644 --- a/third_party/llama.cpp/patches/0010-Add-MTP-execution-support-and-sampling-cleanup.patch +++ b/third_party/llama.cpp/patches/0010-Add-MTP-execution-support-and-sampling-cleanup.patch @@ -1,4 +1,4 @@ -From 57b2b3d52428a53df6fe9dc406686ea53858f629 Mon Sep 17 00:00:00 2001 +From 4669175c128625867df3f7261db3834a5f9d81a5 Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Fri, 26 Jun 2026 13:47:53 +1000 Subject: [PATCH 10/10] Add MTP execution support and sampling cleanup @@ -19,7 +19,7 @@ Squashes GLM MTP graph support, batched stage execution, native MTP sidecar supp 11 files changed, 1341 insertions(+), 183 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp -index 0cee8043..8e95d5b0 100644 +index 6da59f4d..c028d210 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -663,6 +663,113 @@ std::string common_chat_format_example(const struct common_chat_templates * @@ -292,7 +292,7 @@ index a6bd7fd9..cfb4a8fa 100644 #ifdef __cplusplus diff --git a/src/llama-context.cpp b/src/llama-context.cpp -index c10fa290..2fcbd757 100644 +index d7f85fd4..a246f32a 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -1976,9 +1976,22 @@ int llama_context::decode(const llama_batch & batch_inp) { @@ -320,10 +320,10 @@ index c10fa290..2fcbd757 100644 } diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp -index 64d38fc4..d79bda76 100644 +index af0e44e5..a21230bd 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp -@@ -2639,6 +2639,8 @@ ggml_tensor * llm_graph_context::build_attn( +@@ -2948,6 +2948,8 @@ ggml_tensor * llm_graph_context::build_attn( const auto & kq_mask = inp->get_kq_mask_mla(); @@ -333,10 +333,10 @@ index 64d38fc4..d79bda76 100644 ggml_tensor * kq_mask_all = ggml_fill(ctx0, kq_mask, -INFINITY); diff --git a/src/llama-model.cpp b/src/llama-model.cpp -index eab46cf8..47a375e8 100644 +index 95b22b7f..ec4aa86f 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp -@@ -2153,6 +2153,14 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, +@@ -2160,6 +2160,14 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, } } @@ -348,8 +348,8 @@ index eab46cf8..47a375e8 100644 + } + } + - if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) { - GGML_ASSERT(hparams.is_swa_any()); + if (arch == LLM_ARCH_DEEPSEEK4) { + GGML_ASSERT(hparams.swa_type != LLAMA_SWA_TYPE_NONE); diff --git a/src/models/deepseek2.cpp b/src/models/deepseek2.cpp index d1b261b4..180fb082 100644 @@ -779,7 +779,7 @@ index 32fe6def..10bf235f 100644 } - diff --git a/src/models/models.h b/src/models/models.h -index 2ac8415a..6a7ec717 100644 +index 7a52e7bc..b40f9811 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -1068,6 +1068,10 @@ struct llama_model_deepseek2 : public llama_model_base { @@ -793,7 +793,7 @@ index 2ac8415a..6a7ec717 100644 std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; -@@ -1102,6 +1106,7 @@ struct llama_model_glm_dsa : public llama_model_base { +@@ -1217,6 +1221,7 @@ struct llama_model_glm_dsa : public llama_model_base { void load_arch_tensors(llama_model_loader & ml) override; using graph = llama_model_deepseek2::graph; diff --git a/third_party/llama.cpp/upstream.txt b/third_party/llama.cpp/upstream.txt index 5d6500cd3..a1a510b27 100644 --- a/third_party/llama.cpp/upstream.txt +++ b/third_party/llama.cpp/upstream.txt @@ -1 +1 @@ -27c8bb4f63ad9f20bf5901067810a4be5ffe20c4 +86b94708f22478f900b76ca02e316f4f3418faff