From 216ef8c49460e3ab99b0ae8ee4290173ea5589c3 Mon Sep 17 00:00:00 2001 From: Michael Neale <14976+michaelneale@users.noreply.github.com> Date: Mon, 27 Jul 2026 17:23:48 +1000 Subject: [PATCH 01/17] feat(skippy): add Laguna staged runtime candidate --- .../skippy-topology/src/family_capability.rs | 18 +++++ crates/skippy-topology/src/lib.rs | 6 +- crates/skippy-topology/src/tests.rs | 14 ++++ docs/skippy/llama-parity-candidates.json | 8 +++ ...0046-Support-Laguna-staged-execution.patch | 72 +++++++++++++++++++ 5 files changed, 115 insertions(+), 3 deletions(-) create mode 100644 third_party/llama.cpp/patches/0046-Support-Laguna-staged-execution.patch diff --git a/crates/skippy-topology/src/family_capability.rs b/crates/skippy-topology/src/family_capability.rs index 0b2aa63093..38d000e1f9 100644 --- a/crates/skippy-topology/src/family_capability.rs +++ b/crates/skippy-topology/src/family_capability.rs @@ -145,6 +145,11 @@ pub const STAGE_RUNTIME_LLAMA_FAMILY_EXPECTATIONS: &[StageRuntimeFamilyExpectati family_id: "jamba", recurrent_or_hybrid: true, }, + StageRuntimeFamilyExpectation { + llama_architecture: "laguna", + family_id: "laguna", + recurrent_or_hybrid: false, + }, StageRuntimeFamilyExpectation { llama_architecture: "lfm2", family_id: "lfm2", @@ -551,6 +556,16 @@ pub fn qwen3moe_capability(layer_count: u32, activation_width: u32) -> FamilyCap ) } +pub fn laguna_capability(layer_count: u32, activation_width: u32) -> FamilyCapabilityRecord { + dense_family_capability( + "laguna", + layer_count, + activation_width, + WireValidation::Untested, + ExactStateMobility::Untested, + ) +} + pub fn dense_family_capability( family_id: impl Into, layer_count: u32, @@ -1053,6 +1068,9 @@ fn infer_mistral_olmo_llama_capability( if compact.contains("olmo") { return Some(olmo_capability(layer_count, activation_width)); } + if compact.contains("laguna") { + return Some(laguna_capability(layer_count, activation_width)); + } if compact.contains("llama") { return Some(llama_capability(layer_count, activation_width)); } diff --git a/crates/skippy-topology/src/lib.rs b/crates/skippy-topology/src/lib.rs index ac023780cf..3df8b12bf6 100644 --- a/crates/skippy-topology/src/lib.rs +++ b/crates/skippy-topology/src/lib.rs @@ -12,9 +12,9 @@ pub use family_capability::{ deepseek3_capability, dense_attention_layers, dense_family_capability, falcon_h1_capability, falcon_h1_layers, gemma2_capability, gemma3_capability, gemma3n_capability, gemma4_a4b_capability, gemma4_e4b_capability, glm4_capability, glm47_flash_capability, - infer_family_capability, kimi_linear_capability, llama_capability, minimax_m27_capability, - olmo_capability, qwen2moe_capability, qwen3_dense_capability, qwen3moe_capability, - qwen3next_capability, qwen3next_layers, recurrent_family_capability, + infer_family_capability, kimi_linear_capability, laguna_capability, llama_capability, + minimax_m27_capability, olmo_capability, qwen2moe_capability, qwen3_dense_capability, + qwen3moe_capability, qwen3next_capability, qwen3next_layers, recurrent_family_capability, reviewed_capability_for_identity, reviewed_capability_records, rwkv6_capability, rwkv7_capability, }; diff --git a/crates/skippy-topology/src/tests.rs b/crates/skippy-topology/src/tests.rs index a317b78f75..39fbbfd2c6 100644 --- a/crates/skippy-topology/src/tests.rs +++ b/crates/skippy-topology/src/tests.rs @@ -1019,6 +1019,20 @@ fn infers_known_family_capabilities_from_model_identity() { .expect("qwen3moe"); assert_eq!(qwen3moe.family_id, "qwen3moe"); assert_eq!(qwen3moe.q8_wire_validation, WireValidation::Validated); + for identity in [ + "laguna", + "poolside/Laguna-XS-2.1-GGUF:Q4_K_M", + "poolside/Laguna-S-2.1-GGUF:Q4_K_M", + "poolside/Laguna-XS.2-GGUF:Q4_K_M", + "poolside/Laguna-M.1-GGUF:Q4_K_M", + ] { + let laguna = infer_family_capability(identity, 48, 3072) + .unwrap_or_else(|| panic!("failed to infer {identity}")); + assert_eq!(laguna.family_id, "laguna", "{identity}"); + assert_eq!(laguna.q8_wire_validation, WireValidation::Untested); + assert_eq!(laguna.exact_state_mobility, ExactStateMobility::Untested); + assert!(laguna.recurrent_ranges.is_empty()); + } let openai_moe = infer_family_capability("ggml-org/gpt-oss-20b-GGUF:gpt-oss-20b-mxfp4", 24, 2880) .expect("openai_moe/gpt-oss"); diff --git a/docs/skippy/llama-parity-candidates.json b/docs/skippy/llama-parity-candidates.json index 22796aefdc..92aa07738b 100644 --- a/docs/skippy/llama-parity-candidates.json +++ b/docs/skippy/llama-parity-candidates.json @@ -185,6 +185,14 @@ "include": "Qwen3-MOE-4x0.6B-2.4B-Writing-Thunder.Q4_K_M.gguf", "notes": "text lane passed in llama-parity-qwen3moe-runtime-slice-2; q8 activation wire validated; ResidentKv native-sequence remap cache smoke passed; MoE expert-stage smoke already passed" }, + { + "llama_model": "laguna", + "family": "laguna", + "status": "candidate", + "repo": "poolside/Laguna-S-2.1-GGUF", + "include": "laguna-s-2.1-Q4_K_M.gguf", + "notes": "P0 certification target from the official Poolside Q4_K_M GGUF; runtime-slice, package, dtype, state, cache, and mesh evidence remain pending until the Laguna certification run completes" + }, { "llama_model": "mistral3", "family": "mistral", diff --git a/third_party/llama.cpp/patches/0046-Support-Laguna-staged-execution.patch b/third_party/llama.cpp/patches/0046-Support-Laguna-staged-execution.patch new file mode 100644 index 0000000000..3ab8dcff07 --- /dev/null +++ b/third_party/llama.cpp/patches/0046-Support-Laguna-staged-execution.patch @@ -0,0 +1,72 @@ +From d94d179b554ff3de781b122b305a22d1a63f1b2d Mon Sep 17 00:00:00 2001 +From: Mesh-LLM CI +Date: Mon, 27 Jul 2026 17:19:02 +1000 +Subject: [PATCH] Support Laguna staged execution + +--- + src/models/laguna.cpp | 19 ++++++++++++++++--- + src/skippy.cpp | 1 + + 2 files changed, 17 insertions(+), 3 deletions(-) + +diff --git a/src/models/laguna.cpp b/src/models/laguna.cpp +index fb55ec12..7144a389 100644 +--- a/src/models/laguna.cpp ++++ b/src/models/laguna.cpp +@@ -157,7 +157,12 @@ llama_model_laguna::graph::graph(const llama_model & model, const llm_graph_para + ggml_tensor * cur; + ggml_tensor * inpL; + +- inpL = build_inp_embd(model.tok_embd); ++ const skippy_graph_filter & stage_filter = skippy_graph_get_filter(); ++ const bool stage_filtered = stage_filter.enabled; ++ const int il_start = stage_filtered ? stage_filter.layer_start : 0; ++ const int il_end = stage_filtered ? stage_filter.layer_end : n_layer; ++ ++ inpL = build_inp_embd(stage_filtered && il_start > 0 ? nullptr : model.tok_embd); + // No MuP embedding scale (laguna omits this; afmoe scales by sqrt(hidden)). + + ggml_tensor * inp_pos = build_inp_pos(); +@@ -166,11 +171,11 @@ llama_model_laguna::graph::graph(const llama_model & model, const llm_graph_para + const bool has_swa = hparams.swa_type != LLAMA_SWA_TYPE_NONE; + llm_graph_input_attn_kv * inp_attn_kv = has_swa ? nullptr : build_attn_inp_kv(); + llm_graph_input_attn_kv_iswa * inp_attn_iswa = has_swa ? build_attn_inp_kv_iswa() : nullptr; +- ggml_tensor * inp_out_ids = build_inp_out_ids(); ++ ggml_tensor * inp_out_ids = (!stage_filtered || stage_filter.include_output) ? build_inp_out_ids() : nullptr; + + const float kq_scale = 1.0f / sqrtf(float(n_embd_head)); + +- for (int il = 0; il < n_layer; ++il) { ++ for (int il = il_start; il < il_end; ++il) { + const bool is_swa_il = hparams.is_swa(il); + const int64_t n_head_il = hparams.n_head(il); + const int64_t n_head_kv_il = hparams.n_head_kv(il); +@@ -320,6 +325,14 @@ llama_model_laguna::graph::graph(const llama_model & model, const llm_graph_para + } + + cur = inpL; ++ ++ if (stage_filtered && !stage_filter.include_output) { ++ cb(cur, "stage_boundary", il_end - 1); ++ res->t_embd = cur; ++ ggml_build_forward_expand(gf, cur); ++ return; ++ } ++ + cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1); + cb(cur, "result_norm", -1); + res->t_embd = cur; +diff --git a/src/skippy.cpp b/src/skippy.cpp +index 47f0a549..da0d97bf 100644 +--- a/src/skippy.cpp ++++ b/src/skippy.cpp +@@ -4112,6 +4112,7 @@ static enum skippy_status skippy_finish_model_open( + model->arch != LLM_ARCH_JAIS && + model->arch != LLM_ARCH_JAIS2 && + model->arch != LLM_ARCH_JAMBA && ++ model->arch != LLM_ARCH_LAGUNA && + model->arch != LLM_ARCH_LFM2 && + model->arch != LLM_ARCH_LLADA && + model->arch != LLM_ARCH_LLADA_MOE && +-- +2.50.1 (Apple Git-155) + From 2c56bf98ad54af5e7802c7ee8d7f8b371002dc96 Mon Sep 17 00:00:00 2001 From: Michael Neale <14976+michaelneale@users.noreply.github.com> Date: Mon, 27 Jul 2026 17:27:23 +1000 Subject: [PATCH 02/17] fix(models): preserve HF source revision in package jobs --- crates/mesh-llm-commands/src/model_package.rs | 7 ++-- .../src/bin/queue-unsloth-layer-packages.rs | 4 ++- crates/model-package/src/jobs.rs | 35 +++++++++++++++++++ crates/model-package/src/prepare.rs | 19 ++++++++-- 4 files changed, 59 insertions(+), 6 deletions(-) diff --git a/crates/mesh-llm-commands/src/model_package.rs b/crates/mesh-llm-commands/src/model_package.rs index 15c52163cc..9a9a6bd13c 100644 --- a/crates/mesh-llm-commands/src/model_package.rs +++ b/crates/mesh-llm-commands/src/model_package.rs @@ -77,6 +77,7 @@ pub async fn dispatch_model_package(args: ModelPrepareArgs<'_>) -> Result<()> { let source_model_ref = model_ref::ModelRef::parse(source_ref) .with_context(|| format!("invalid source model ref: {source_ref}"))?; let source_repo = source_model_ref.repo.as_str(); + let source_revision = source_model_ref.revision.as_deref(); let source_quant = match (source_model_ref.selector.as_deref(), quant) { (Some(selector), Some(quant)) if selector != quant => { bail!( @@ -95,7 +96,7 @@ pub async fn dispatch_model_package(args: ModelPrepareArgs<'_>) -> Result<()> { // If no quant specified, list available quants and exit. // This path doesn't need HF_TOKEN — works for public repos. if source_quant.is_none() { - return run_list_quants(&hf_client, source_repo, json).await; + return run_list_quants(&hf_client, source_repo, source_revision, json).await; } let submitting = confirm && !dry_run; @@ -116,6 +117,7 @@ pub async fn dispatch_model_package(args: ModelPrepareArgs<'_>) -> Result<()> { eprintln!("🔍 Resolving source..."); let params = PrepareParams { source_repo: source_repo.to_string(), + source_revision: source_model_ref.revision.clone(), quant: source_quant.map(|s| s.to_string()), target: target.map(|s| s.to_string()), model_id: model_id.map(|s| s.to_string()), @@ -254,9 +256,10 @@ pub async fn dispatch_model_package(args: ModelPrepareArgs<'_>) -> Result<()> { async fn run_list_quants( client: &hf_hub::HFClient, source_repo: &str, + source_revision: Option<&str>, json_output: bool, ) -> Result<()> { - let quants = prepare::list_quants(client, source_repo).await?; + let quants = prepare::list_quants(client, source_repo, source_revision).await?; if json_output { println!( diff --git a/crates/model-package/src/bin/queue-unsloth-layer-packages.rs b/crates/model-package/src/bin/queue-unsloth-layer-packages.rs index ef2931d0da..0d17a29828 100644 --- a/crates/model-package/src/bin/queue-unsloth-layer-packages.rs +++ b/crates/model-package/src/bin/queue-unsloth-layer-packages.rs @@ -685,7 +685,7 @@ async fn build_candidate( return Ok(None); } - let quants = match prepare::list_quants(client, &model.repo_id).await { + let quants = match prepare::list_quants(client, &model.repo_id, None).await { Ok(quants) => quants, Err(err) => { eprintln!( @@ -1096,12 +1096,14 @@ fn job_spec_with_token( JobVolume { volume_type: "bucket".into(), source: "meshllm/layer-split-output".into(), + revision: None, mount_path: "/bucket".into(), read_only: None, }, JobVolume { volume_type: "model".into(), source: candidate.model.repo_id.clone(), + revision: Some("main".into()), mount_path: "/source".into(), read_only: Some(true), }, diff --git a/crates/model-package/src/jobs.rs b/crates/model-package/src/jobs.rs index 76eafd2b55..8c71c99722 100644 --- a/crates/model-package/src/jobs.rs +++ b/crates/model-package/src/jobs.rs @@ -37,6 +37,8 @@ pub struct JobVolume { #[serde(rename = "type")] pub volume_type: String, pub source: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub revision: Option, #[serde(rename = "mountPath")] pub mount_path: String, #[serde(rename = "readOnly", skip_serializing_if = "Option::is_none")] @@ -546,6 +548,39 @@ mod tests { } } + #[test] + fn model_volume_serializes_pinned_revision() { + let volume = JobVolume { + volume_type: "model".into(), + source: "poolside/Laguna-S-2.1-GGUF".into(), + revision: Some("edd093522473dc7313b0738d8b4116b7f8b9745f".into()), + mount_path: "/source".into(), + read_only: Some(true), + }; + + let value = serde_json::to_value(volume).unwrap(); + assert_eq!( + value["revision"], + "edd093522473dc7313b0738d8b4116b7f8b9745f" + ); + assert_eq!(value["mountPath"], "/source"); + assert_eq!(value["readOnly"], true); + } + + #[test] + fn bucket_volume_omits_revision() { + let volume = JobVolume { + volume_type: "bucket".into(), + source: "meshllm/layer-split-output".into(), + revision: None, + mount_path: "/bucket".into(), + read_only: None, + }; + + let value = serde_json::to_value(volume).unwrap(); + assert!(value.get("revision").is_none()); + } + #[test] fn estimates_minute_pricing() { let cost = estimate_cost_usd(2.0, "minute", 1800).unwrap(); diff --git a/crates/model-package/src/prepare.rs b/crates/model-package/src/prepare.rs index 4666b475c9..2f8b2221a7 100644 --- a/crates/model-package/src/prepare.rs +++ b/crates/model-package/src/prepare.rs @@ -16,6 +16,7 @@ use crate::permissions::PermissionCheck; /// Parameters for a model-package job. pub struct PrepareParams { pub source_repo: String, + pub source_revision: Option, pub quant: Option, pub target: Option, pub model_id: Option, @@ -51,12 +52,17 @@ pub struct DiscoveredQuant { } /// List all available GGUF quant variants in a HF model repo. -pub async fn list_quants(client: &HFClient, repo: &str) -> Result> { +pub async fn list_quants( + client: &HFClient, + repo: &str, + revision: Option<&str>, +) -> Result> { let (owner, name) = parse_repo(repo)?; let hf_repo = client.model(&owner, &name); let stream = hf_repo .list_tree() + .maybe_revision(revision.map(str::to_owned)) .recursive(true) .send() .context("list repo tree")?; @@ -122,7 +128,12 @@ pub async fn resolve( .as_deref() .context("--quant is required when submitting a job")?; - let quants = list_quants(client, ¶ms.source_repo).await?; + let source_revision = params + .source_revision + .as_deref() + .unwrap_or("main") + .to_string(); + let quants = list_quants(client, ¶ms.source_repo, Some(&source_revision)).await?; if quants.is_empty() { anyhow::bail!("No GGUF files found in {}", params.source_repo); @@ -194,7 +205,7 @@ pub async fn resolve( environment.insert("SOURCE_TOTAL_BYTES".into(), matched.total_bytes.to_string()); environment.insert("TARGET_REPO".into(), target_repo.clone()); environment.insert("MODEL_ID".into(), model_id.clone()); - environment.insert("SOURCE_REVISION".into(), "main".into()); + environment.insert("SOURCE_REVISION".into(), source_revision.clone()); environment.insert("MESH_LLM_REF".into(), params.mesh_llm_ref.clone()); environment.insert( "CATALOG_CREATE_PR".into(), @@ -217,12 +228,14 @@ pub async fn resolve( JobVolume { volume_type: "bucket".into(), source: "meshllm/layer-split-output".into(), + revision: None, mount_path: "/bucket".into(), read_only: None, }, JobVolume { volume_type: "model".into(), source: params.source_repo.clone(), + revision: Some(source_revision), mount_path: "/source".into(), read_only: Some(true), }, From d94cc296686232e1fb3af1a20a2c0817b44d9c54 Mon Sep 17 00:00:00 2001 From: Michael Neale <14976+michaelneale@users.noreply.github.com> Date: Mon, 27 Jul 2026 18:32:22 +1000 Subject: [PATCH 03/17] docs(skippy): record Laguna package certification --- crates/model-package/src/script.rs | 3 ++ .../src/scripts/split-model-job.sh | 46 ++++++++++++++++++- docs/skippy/LLAMA_PARITY.md | 1 + docs/skippy/llama-parity-candidates.json | 4 +- 4 files changed, 50 insertions(+), 4 deletions(-) diff --git a/crates/model-package/src/script.rs b/crates/model-package/src/script.rs index edd1946feb..40699d8310 100644 --- a/crates/model-package/src/script.rs +++ b/crates/model-package/src/script.rs @@ -130,6 +130,9 @@ mod tests { #[test] fn embedded_script_writes_rich_model_card() { assert!(EMBEDDED_SCRIPT.contains("pipeline_tag: text-generation")); + assert!(EMBEDDED_SCRIPT.contains("resolve_upstream_license")); + assert!(EMBEDDED_SCRIPT.contains("license_frontmatter")); + assert!(EMBEDDED_SCRIPT.contains("could not resolve upstream license metadata")); assert!(EMBEDDED_SCRIPT.contains("- openai-compatible")); assert!(EMBEDDED_SCRIPT.contains("## Model Overview")); assert!(EMBEDDED_SCRIPT.contains("## Highlights")); diff --git a/crates/model-package/src/scripts/split-model-job.sh b/crates/model-package/src/scripts/split-model-job.sh index 03541fc871..549dc7d36a 100755 --- a/crates/model-package/src/scripts/split-model-job.sh +++ b/crates/model-package/src/scripts/split-model-job.sh @@ -522,6 +522,7 @@ source_revision = os.environ.get("SOURCE_REVISION", "main") target_repo = os.environ["TARGET_REPO"] model_id = os.environ.get("MODEL_ID", manifest.get("model_id", target_repo)) mesh_llm_ref = os.environ.get("MESH_LLM_REF", "main") +api = HfApi(token=os.environ["HF_TOKEN"]) def sha256(path: Path) -> str: digest = hashlib.sha256() @@ -555,6 +556,38 @@ def code(value) -> str: def yaml_quote(value: str) -> str: return json.dumps(value) +def card_value(info, key: str): + card_data = getattr(info, "card_data", None) + if card_data is None: + return None + if isinstance(card_data, dict): + return card_data.get(key) + return getattr(card_data, key, None) + +def first_model_id(value): + if isinstance(value, list): + value = value[0] if value else None + if isinstance(value, dict): + return value.get("id") or value.get("modelId") + return value if isinstance(value, str) and value else None + +def resolve_upstream_license(): + try: + source_info = api.model_info(source_repo, revision=source_revision) + source_license = card_value(source_info, "license") + if source_license: + return str(source_license), source_repo + + base_repo = first_model_id(card_value(source_info, "base_model")) + if base_repo: + base_info = api.model_info(base_repo) + base_license = card_value(base_info, "license") + if base_license: + return str(base_license), base_repo + except Exception as error: + print(f" WARNING: could not resolve upstream license metadata: {error}") + return None, None + def infer_model_family(name: str) -> str: lowered = name.lower() for family in ["Qwen3", "Qwen2.5", "DeepSeek", "Kimi", "Gemma", "GLM", "Llama"]: @@ -601,6 +634,10 @@ activation_width = manifest.get("activation_width") or "not recorded" skippy_abi = manifest.get("skippy_abi_version") or "not recorded" source_sha = source_model.get("sha256") or "not recorded" canonical_ref = source_model.get("canonical_ref") or f"{source_repo}@{source_revision}/{source_file}" +upstream_license, license_source_repo = resolve_upstream_license() +license_frontmatter = ( + f"license: {yaml_quote(upstream_license)}\n" if upstream_license else "" +) file_rows = [ ("Manifest", "model-package.json", "Package schema, source identity, checksums", manifest_hash), @@ -647,10 +684,16 @@ rows = [ ("Source file", code(source_file)), ("Package repo", link(target_repo, f"https://huggingface.co/{target_repo}")), ] +if upstream_license and license_source_repo: + rows.append(( + "License", + f"{code(upstream_license)} from " + f"{link(license_source_repo, f'https://huggingface.co/{license_source_repo}')}", + )) readme = f"""--- library_name: mesh-llm -base_model: +{license_frontmatter}base_model: - {yaml_quote(source_repo)} pipeline_tag: text-generation tags: @@ -781,7 +824,6 @@ skippy-model-package write-package "{source_path}" --out-dir "{package_dir}" Path("/tmp/README.md").write_text(readme) -api = HfApi(token=os.environ["HF_TOKEN"]) api.upload_file( path_or_fileobj="/tmp/README.md", path_in_repo="README.md", diff --git a/docs/skippy/LLAMA_PARITY.md b/docs/skippy/LLAMA_PARITY.md index 4fdf1d0b75..d42940cb4f 100644 --- a/docs/skippy/LLAMA_PARITY.md +++ b/docs/skippy/LLAMA_PARITY.md @@ -370,6 +370,7 @@ themselves until the reviewed topology records are updated. | `deepseek` | `Morgen0052/deepseek-llm-7b-chat-Q4_K_M-GGUF` | `single-step`, `chain`, and f16 dtype matrix passed | rejected | accepted | `ResidentKv` borrowed-hit smoke passed, 64-token prefix, 1.58x cache-hit speedup | | `openai_moe` | `ggml-org/gpt-oss-20b-GGUF:gpt-oss-20b-mxfp4` | `single-step`, `chain`, and dtype matrix passed | rejected | accepted | `ResidentKv` state handoff passed; llama.cpp model file is `openai-moe`, GGUF architecture is `gpt-oss` | | `ernie4_5_moe` | `lmstudio-community/ERNIE-4.5-21B-A3B-PT-GGUF:Q4_K_M` | `single-step`, `chain`, and dtype matrix passed | validated | accepted | `ResidentKv` state handoff passed | +| `laguna` | `meshllm/laguna-s-2.1-Q4_K_M-layers@797f31cc813c0d643b7205118a901732f39de8ab` | package validated | untested | untested | package-only validation passed: 48 layers; `0..24` and `24..48` stages materialized at 33,500,313,312 and 34,752,130,336 bytes | | `llama4` | `ggml-org/Llama-4-Scout-17B-16E-Instruct-GGUF:Q4_K_M` | package validated | untested | untested | package-only validation passed: 48 layers, 627 owned tensors, 51 artifacts, no missing/duplicate tensors | | `mistral4` | `bartowski/mistralai_Mistral-Small-4-119B-2603-GGUF:IQ2_XXS` | package validated | untested | untested | package-only validation passed: 36 layers, 579 tensors, 39 artifacts, no missing/duplicate tensors | | `nemotron_h_moe` | `lmstudio-community/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-GGUF:Q4_K_M` | package validated | untested | rejected-too-large | package-only validation passed: 52 layers, 401 tensors, 55 artifacts; `KvRecurrent` target | diff --git a/docs/skippy/llama-parity-candidates.json b/docs/skippy/llama-parity-candidates.json index 92aa07738b..98e4aa8e2c 100644 --- a/docs/skippy/llama-parity-candidates.json +++ b/docs/skippy/llama-parity-candidates.json @@ -188,10 +188,10 @@ { "llama_model": "laguna", "family": "laguna", - "status": "candidate", + "status": "certified_package_only", "repo": "poolside/Laguna-S-2.1-GGUF", "include": "laguna-s-2.1-Q4_K_M.gguf", - "notes": "P0 certification target from the official Poolside Q4_K_M GGUF; runtime-slice, package, dtype, state, cache, and mesh evidence remain pending until the Laguna certification run completes" + "notes": "package-only certified from poolside/Laguna-S-2.1-GGUF:Q4_K_M at source revision edd093522473dc7313b0738d8b4116b7f8b9745f; HF job 6a6708667ef3c0846496a429 published meshllm/laguna-s-2.1-Q4_K_M-layers, and pinned package revision 797f31cc813c0d643b7205118a901732f39de8ab passed package materialization with 48 layers and two 24-layer stages of 33500313312 and 34752130336 bytes; runtime-slice, activation-wire dtype, state, cache, and mesh evidence remain pending" }, { "llama_model": "mistral3", From 8ea8b440d02d46ed8fe48fc5bf72f8e6e42e93a6 Mon Sep 17 00:00:00 2001 From: Michael Neale <14976+michaelneale@users.noreply.github.com> Date: Tue, 28 Jul 2026 10:52:53 +1000 Subject: [PATCH 04/17] docs(skippy): record Laguna M5 parity --- docs/skippy/FAMILY_STATUS.md | 9 +++++---- docs/skippy/LLAMA_PARITY.md | 2 +- docs/skippy/llama-parity-candidates.json | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/skippy/FAMILY_STATUS.md b/docs/skippy/FAMILY_STATUS.md index 953b2aca7d..f641e43ba4 100644 --- a/docs/skippy/FAMILY_STATUS.md +++ b/docs/skippy/FAMILY_STATUS.md @@ -8,7 +8,7 @@ Certification process lives in `docs/FAMILY_CERTIFY.md`. Payload measurements and topology constraints are summarized here so this file stays the only customer-facing source of truth. -Last updated: 2026-05-07. +Last updated: 2026-07-28. ## Customer Support Matrix @@ -88,12 +88,13 @@ Last updated: 2026-05-07. ## Text-Split Candidates -These families now pass the cheap runtime-slice text lane, but are not promoted -to the customer support matrix until the remaining cache smoke, reviewed -topology records, and family-specific policy notes are updated. +These families now pass a cheap runtime-slice or package-backed text lane, but +are not promoted to the customer support matrix until the remaining cache +smoke, reviewed topology records, and family-specific policy notes are updated. ```text Gemma text +Laguna S 2.1 (package-backed two-stage single-step; broader certification pending) ``` ## Exceptions diff --git a/docs/skippy/LLAMA_PARITY.md b/docs/skippy/LLAMA_PARITY.md index d42940cb4f..f162480807 100644 --- a/docs/skippy/LLAMA_PARITY.md +++ b/docs/skippy/LLAMA_PARITY.md @@ -370,7 +370,7 @@ themselves until the reviewed topology records are updated. | `deepseek` | `Morgen0052/deepseek-llm-7b-chat-Q4_K_M-GGUF` | `single-step`, `chain`, and f16 dtype matrix passed | rejected | accepted | `ResidentKv` borrowed-hit smoke passed, 64-token prefix, 1.58x cache-hit speedup | | `openai_moe` | `ggml-org/gpt-oss-20b-GGUF:gpt-oss-20b-mxfp4` | `single-step`, `chain`, and dtype matrix passed | rejected | accepted | `ResidentKv` state handoff passed; llama.cpp model file is `openai-moe`, GGUF architecture is `gpt-oss` | | `ernie4_5_moe` | `lmstudio-community/ERNIE-4.5-21B-A3B-PT-GGUF:Q4_K_M` | `single-step`, `chain`, and dtype matrix passed | validated | accepted | `ResidentKv` state handoff passed | -| `laguna` | `meshllm/laguna-s-2.1-Q4_K_M-layers@797f31cc813c0d643b7205118a901732f39de8ab` | package validated | untested | untested | package-only validation passed: 48 layers; `0..24` and `24..48` stages materialized at 33,500,313,312 and 34,752,130,336 bytes | +| `laguna` | `meshllm/laguna-s-2.1-Q4_K_M-layers@797f31cc813c0d643b7205118a901732f39de8ab` | package-backed `single-step` passed at split `24` | untested | untested | M5 Max f16 parity passed against pinned full-model revision `edd093522473dc7313b0738d8b4116b7f8b9745f`: baseline token `674` matched staged token `674`; width `3072`, 12,288-byte activation frame, 6,144-byte wire payload; chain, dtype matrix, state/cache, and mesh evidence remain pending | | `llama4` | `ggml-org/Llama-4-Scout-17B-16E-Instruct-GGUF:Q4_K_M` | package validated | untested | untested | package-only validation passed: 48 layers, 627 owned tensors, 51 artifacts, no missing/duplicate tensors | | `mistral4` | `bartowski/mistralai_Mistral-Small-4-119B-2603-GGUF:IQ2_XXS` | package validated | untested | untested | package-only validation passed: 36 layers, 579 tensors, 39 artifacts, no missing/duplicate tensors | | `nemotron_h_moe` | `lmstudio-community/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-GGUF:Q4_K_M` | package validated | untested | rejected-too-large | package-only validation passed: 52 layers, 401 tensors, 55 artifacts; `KvRecurrent` target | diff --git a/docs/skippy/llama-parity-candidates.json b/docs/skippy/llama-parity-candidates.json index 98e4aa8e2c..29e95b297c 100644 --- a/docs/skippy/llama-parity-candidates.json +++ b/docs/skippy/llama-parity-candidates.json @@ -191,7 +191,7 @@ "status": "certified_package_only", "repo": "poolside/Laguna-S-2.1-GGUF", "include": "laguna-s-2.1-Q4_K_M.gguf", - "notes": "package-only certified from poolside/Laguna-S-2.1-GGUF:Q4_K_M at source revision edd093522473dc7313b0738d8b4116b7f8b9745f; HF job 6a6708667ef3c0846496a429 published meshllm/laguna-s-2.1-Q4_K_M-layers, and pinned package revision 797f31cc813c0d643b7205118a901732f39de8ab passed package materialization with 48 layers and two 24-layer stages of 33500313312 and 34752130336 bytes; runtime-slice, activation-wire dtype, state, cache, and mesh evidence remain pending" + "notes": "package-only certification status retained while broader gates remain pending; source revision edd093522473dc7313b0738d8b4116b7f8b9745f (SHA-256 a34c74e46688122bef83122f4133031bababbefcf57436dde97048c91e2cc6ff) and package revision 797f31cc813c0d643b7205118a901732f39de8ab passed M5 Max package-backed single-step parity at split 24 with f16 activation transport: baseline token 674 matched staged token 674, activation width 3072, payload 12288 bytes, wire payload 6144 bytes; package materialization, production-shaped OpenAI smoke, and two-stage direct prediction return also passed; chain, dtype matrix, state, cache, and mesh evidence remain pending" }, { "llama_model": "mistral3", From 226fb5842a11809d6d7eb86ed572ede5dfa8e637 Mon Sep 17 00:00:00 2001 From: Michael Neale <14976+michaelneale@users.noreply.github.com> Date: Tue, 28 Jul 2026 11:08:55 +1000 Subject: [PATCH 05/17] test(skippy): certify Laguna three-stage parity --- crates/skippy-correctness/src/runner/mod.rs | 1 + .../src/runner/prediction_return.rs | 208 ++++++++++++++++++ .../src/runner/split_chain.rs | 26 ++- docs/skippy/FAMILY_STATUS.md | 2 +- docs/skippy/LLAMA_PARITY.md | 2 +- docs/skippy/llama-parity-candidates.json | 2 +- 6 files changed, 231 insertions(+), 10 deletions(-) create mode 100644 crates/skippy-correctness/src/runner/prediction_return.rs diff --git a/crates/skippy-correctness/src/runner/mod.rs b/crates/skippy-correctness/src/runner/mod.rs index 9ec2ef3a79..f64b2df962 100644 --- a/crates/skippy-correctness/src/runner/mod.rs +++ b/crates/skippy-correctness/src/runner/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod native_mtp; +mod prediction_return; mod single_step; mod split_chain; mod stage_execution; diff --git a/crates/skippy-correctness/src/runner/prediction_return.rs b/crates/skippy-correctness/src/runner/prediction_return.rs new file mode 100644 index 0000000000..f759f6c1a5 --- /dev/null +++ b/crates/skippy-correctness/src/runner/prediction_return.rs @@ -0,0 +1,208 @@ +use std::{ + env, io, + io::Write, + net::{SocketAddr, TcpListener, TcpStream}, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + mpsc, + }, + thread::{self, JoinHandle}, + time::Duration, +}; + +use anyhow::{Context, Result, anyhow, bail}; +use skippy_protocol::binary::{ + READY_MAGIC, StageReply, WireMessageKind, read_stage_message, recv_ready, recv_reply, + send_ready, +}; + +const CLIENT_READY_HELLO_ENV: &str = "SKIPPY_STAGE_CLIENT_READY_HELLO"; +const CLIENT_READY_HELLO_PEEK_TIMEOUT: Duration = Duration::from_millis(500); + +pub(super) struct PredictionReturnListener { + bind_addr: SocketAddr, + receiver: mpsc::Receiver>, + shutdown: Arc, + thread: Option>, +} + +impl PredictionReturnListener { + pub(super) fn start() -> Result { + let listener = TcpListener::bind("127.0.0.1:0") + .context("bind correctness prediction return listener")?; + let bind_addr = listener + .local_addr() + .context("read correctness prediction return listener address")?; + listener + .set_nonblocking(true) + .context("set correctness prediction return listener nonblocking")?; + let shutdown = Arc::new(AtomicBool::new(false)); + let thread_shutdown = shutdown.clone(); + let (sender, receiver) = mpsc::channel(); + let thread = thread::spawn(move || { + let result = accept_prediction_return(listener, &thread_shutdown, &sender); + if let Err(error) = result { + let _ = sender.send(Err(format!("{error:#}"))); + } + }); + Ok(Self { + bind_addr, + receiver, + shutdown, + thread: Some(thread), + }) + } + + pub(super) fn endpoint(&self) -> String { + format!("tcp://{}", self.bind_addr) + } + + pub(super) fn receive(&self, timeout: Duration) -> Result { + match self.receiver.recv_timeout(timeout) { + Ok(Ok(reply)) => Ok(reply), + Ok(Err(error)) => Err(anyhow!(error)), + Err(mpsc::RecvTimeoutError::Timeout) => { + bail!("timed out waiting for direct prediction return") + } + Err(mpsc::RecvTimeoutError::Disconnected) => { + bail!("direct prediction return listener disconnected") + } + } + } +} + +impl Drop for PredictionReturnListener { + fn drop(&mut self) { + self.shutdown.store(true, Ordering::SeqCst); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} + +fn accept_prediction_return( + listener: TcpListener, + shutdown: &AtomicBool, + sender: &mpsc::Sender>, +) -> Result<()> { + let mut stream = loop { + match listener.accept() { + Ok((stream, _)) => break stream, + Err(error) if error.kind() == io::ErrorKind::WouldBlock => { + if shutdown.load(Ordering::SeqCst) { + return Ok(()); + } + thread::sleep(Duration::from_millis(10)); + } + Err(error) if error.kind() == io::ErrorKind::Interrupted => {} + Err(error) => return Err(error).context("accept direct prediction return"), + } + }; + stream + .set_nonblocking(false) + .context("set direct prediction return stream blocking")?; + consume_optional_client_ready_hello(&mut stream)?; + send_ready(&mut stream).context("send direct prediction return ready")?; + stream.flush().ok(); + let open = + read_stage_message(&mut stream, 0).context("read direct prediction return open message")?; + if open.kind != WireMessageKind::PredictionReturnOpen { + bail!("expected direct prediction return open message"); + } + loop { + match recv_reply(&mut stream) { + Ok(reply) => { + if sender.send(Ok(reply)).is_err() { + return Ok(()); + } + } + Err(error) if error.kind() == io::ErrorKind::UnexpectedEof => return Ok(()), + Err(error) => return Err(error).context("read direct prediction return reply"), + } + } +} + +fn consume_optional_client_ready_hello(stream: &mut TcpStream) -> Result<()> { + if !client_ready_hello_enabled() { + return Ok(()); + } + let previous_timeout = stream + .read_timeout() + .context("read direct prediction return timeout")?; + stream + .set_read_timeout(Some(CLIENT_READY_HELLO_PEEK_TIMEOUT)) + .context("set direct prediction return hello timeout")?; + let mut bytes = [0_u8; 4]; + let peek_result = stream.peek(&mut bytes); + stream + .set_read_timeout(previous_timeout) + .context("restore direct prediction return timeout")?; + + match peek_result { + Ok(4) if i32::from_le_bytes(bytes) == READY_MAGIC => { + recv_ready(stream).context("consume direct prediction return client ready hello")?; + } + Ok(_) => {} + Err(error) + if matches!( + error.kind(), + io::ErrorKind::WouldBlock | io::ErrorKind::TimedOut + ) => {} + Err(error) => { + return Err(error).context("peek direct prediction return client ready hello"); + } + } + Ok(()) +} + +fn client_ready_hello_enabled() -> bool { + env::var(CLIENT_READY_HELLO_ENV) + .map(|value| matches!(value.as_str(), "1" | "true" | "TRUE" | "yes" | "on")) + .unwrap_or(false) +} + +#[cfg(test)] +mod tests { + use super::*; + use skippy_protocol::binary::{ + StageStateHeader, StageWireMessage, WireActivationDType, send_reply_predicted, + write_stage_message, + }; + + #[test] + fn receives_prediction_over_direct_return_endpoint() { + let listener = PredictionReturnListener::start().unwrap(); + let endpoint = listener.endpoint(); + let address = endpoint.strip_prefix("tcp://").unwrap().to_string(); + let client = thread::spawn(move || { + let mut stream = TcpStream::connect(address).unwrap(); + if client_ready_hello_enabled() { + send_ready(&mut stream).unwrap(); + } + recv_ready(&mut stream).unwrap(); + let kind = WireMessageKind::PredictionReturnOpen; + let open = StageWireMessage { + kind, + pos_start: 0, + token_count: 0, + state: StageStateHeader::new(kind, WireActivationDType::F32), + request_id: 11, + session_id: 13, + sampling: None, + chat_sampling_metadata: None, + tokens: Vec::new(), + positions: Vec::new(), + activation: Vec::new(), + raw_bytes: Vec::new(), + }; + write_stage_message(&mut stream, &open, WireActivationDType::F32).unwrap(); + send_reply_predicted(&mut stream, 674).unwrap(); + }); + + let reply = listener.receive(Duration::from_secs(1)).unwrap(); + + assert_eq!(reply.predicted, 674); + client.join().unwrap(); + } +} diff --git a/crates/skippy-correctness/src/runner/split_chain.rs b/crates/skippy-correctness/src/runner/split_chain.rs index c10667b9fd..52f4b9cbd8 100644 --- a/crates/skippy-correctness/src/runner/split_chain.rs +++ b/crates/skippy-correctness/src/runner/split_chain.rs @@ -1,9 +1,15 @@ -use std::{fs, net::SocketAddr, path::PathBuf, process::Command, time::Instant}; +use std::{ + fs, + net::SocketAddr, + path::PathBuf, + process::Command, + time::{Duration, Instant}, +}; use anyhow::{Context, Result, bail}; use model_artifact::ModelIdentity; use serde_json::json; -use skippy_protocol::binary::{StageWireMessage, WireReplyKind, recv_reply, write_stage_message}; +use skippy_protocol::binary::{StageWireMessage, WireReplyKind, write_stage_message}; use skippy_runtime::{GGML_TYPE_F16, RuntimeConfig, StageModel}; use crate::{ @@ -24,6 +30,7 @@ use super::{ native_mtp_satisfies_requirement, native_mtp_sideband_report, native_mtp_verification_report, native_mtp_verification_satisfies_requirement, }, + prediction_return::PredictionReturnListener, single_step::{SingleStepCase, run_full_model_decode, run_single_step_with_baseline}, stage_execution::{ BinaryDecodeMessageArgs, CorrectnessTopologyStage, FullModelResult, PackageStageSpec, @@ -347,6 +354,8 @@ fn run_binary_chain(args: BinaryChainConfig) -> Result { bail!("stage 0 produced an empty activation frame"); } let activation_width = activation_width(&boundary)?; + let prediction_return = PredictionReturnListener::start()?; + let stage0_endpoint = prediction_return.endpoint(); let run_id = generate_run_id(); let model_id = args.model_identity.model_id.clone(); @@ -407,7 +416,7 @@ fn run_binary_chain(args: BinaryChainConfig) -> Result { "upstream": { "stage_id": "stage-0", "stage_index": 0, - "endpoint": "driver" + "endpoint": stage0_endpoint }, "downstream": { "stage_id": "stage-2", @@ -422,7 +431,7 @@ fn run_binary_chain(args: BinaryChainConfig) -> Result { CorrectnessTopologyStage { stage_id: "stage-0", stage_index: 0, - endpoint: "driver".to_string(), + endpoint: stage0_endpoint, layer_start: 0, layer_end: args.split_layer_1, load_mode: protocol_load_mode(args.stage_load_mode), @@ -521,7 +530,9 @@ fn run_binary_chain(args: BinaryChainConfig) -> Result { session_id, })?; write_stage_message(&mut stream, &message, wire_dtype).context("send binary chain decode")?; - let reply = recv_reply(&mut stream).context("receive binary chain prediction reply")?; + let reply = prediction_return + .receive(Duration::from_secs(args.startup_timeout_secs)) + .context("receive binary chain direct prediction reply")?; ensure_reply_kind(&reply, WireReplyKind::PredictedToken)?; let native_mtp = native_mtp_sideband_report(&reply); let (second_predicted_token, native_mtp_verification_compute_us) = @@ -542,8 +553,9 @@ fn run_binary_chain(args: BinaryChainConfig) -> Result { })?; write_stage_message(&mut stream, &second_message, wire_dtype) .context("send second binary chain decode")?; - let second_reply = - recv_reply(&mut stream).context("receive second binary chain prediction reply")?; + let second_reply = prediction_return + .receive(Duration::from_secs(args.startup_timeout_secs)) + .context("receive second binary chain direct prediction reply")?; ensure_reply_kind(&second_reply, WireReplyKind::PredictedToken)?; ( Some(second_reply.predicted), diff --git a/docs/skippy/FAMILY_STATUS.md b/docs/skippy/FAMILY_STATUS.md index f641e43ba4..a6224e6869 100644 --- a/docs/skippy/FAMILY_STATUS.md +++ b/docs/skippy/FAMILY_STATUS.md @@ -94,7 +94,7 @@ smoke, reviewed topology records, and family-specific policy notes are updated. ```text Gemma text -Laguna S 2.1 (package-backed two-stage single-step; broader certification pending) +Laguna S 2.1 (package-backed M5 two-stage single-step and three-stage chain parity; broader certification pending) ``` ## Exceptions diff --git a/docs/skippy/LLAMA_PARITY.md b/docs/skippy/LLAMA_PARITY.md index f162480807..827305b3d6 100644 --- a/docs/skippy/LLAMA_PARITY.md +++ b/docs/skippy/LLAMA_PARITY.md @@ -370,7 +370,7 @@ themselves until the reviewed topology records are updated. | `deepseek` | `Morgen0052/deepseek-llm-7b-chat-Q4_K_M-GGUF` | `single-step`, `chain`, and f16 dtype matrix passed | rejected | accepted | `ResidentKv` borrowed-hit smoke passed, 64-token prefix, 1.58x cache-hit speedup | | `openai_moe` | `ggml-org/gpt-oss-20b-GGUF:gpt-oss-20b-mxfp4` | `single-step`, `chain`, and dtype matrix passed | rejected | accepted | `ResidentKv` state handoff passed; llama.cpp model file is `openai-moe`, GGUF architecture is `gpt-oss` | | `ernie4_5_moe` | `lmstudio-community/ERNIE-4.5-21B-A3B-PT-GGUF:Q4_K_M` | `single-step`, `chain`, and dtype matrix passed | validated | accepted | `ResidentKv` state handoff passed | -| `laguna` | `meshllm/laguna-s-2.1-Q4_K_M-layers@797f31cc813c0d643b7205118a901732f39de8ab` | package-backed `single-step` passed at split `24` | untested | untested | M5 Max f16 parity passed against pinned full-model revision `edd093522473dc7313b0738d8b4116b7f8b9745f`: baseline token `674` matched staged token `674`; width `3072`, 12,288-byte activation frame, 6,144-byte wire payload; chain, dtype matrix, state/cache, and mesh evidence remain pending | +| `laguna` | `meshllm/laguna-s-2.1-Q4_K_M-layers@797f31cc813c0d643b7205118a901732f39de8ab` | package-backed `single-step` passed at split `24`; three-stage `chain` passed at splits `16,32` | untested | untested | M5 Max f16 parity passed against pinned full-model revision `edd093522473dc7313b0738d8b4116b7f8b9745f`: baseline token `674` matched both two-stage and three-stage predictions; width `3072`, 12,288-byte activation frame, 6,144-byte wire payload; dtype matrix, state/cache, and M4+M5 mesh evidence remain pending | | `llama4` | `ggml-org/Llama-4-Scout-17B-16E-Instruct-GGUF:Q4_K_M` | package validated | untested | untested | package-only validation passed: 48 layers, 627 owned tensors, 51 artifacts, no missing/duplicate tensors | | `mistral4` | `bartowski/mistralai_Mistral-Small-4-119B-2603-GGUF:IQ2_XXS` | package validated | untested | untested | package-only validation passed: 36 layers, 579 tensors, 39 artifacts, no missing/duplicate tensors | | `nemotron_h_moe` | `lmstudio-community/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-GGUF:Q4_K_M` | package validated | untested | rejected-too-large | package-only validation passed: 52 layers, 401 tensors, 55 artifacts; `KvRecurrent` target | diff --git a/docs/skippy/llama-parity-candidates.json b/docs/skippy/llama-parity-candidates.json index 29e95b297c..0e20aa57e7 100644 --- a/docs/skippy/llama-parity-candidates.json +++ b/docs/skippy/llama-parity-candidates.json @@ -191,7 +191,7 @@ "status": "certified_package_only", "repo": "poolside/Laguna-S-2.1-GGUF", "include": "laguna-s-2.1-Q4_K_M.gguf", - "notes": "package-only certification status retained while broader gates remain pending; source revision edd093522473dc7313b0738d8b4116b7f8b9745f (SHA-256 a34c74e46688122bef83122f4133031bababbefcf57436dde97048c91e2cc6ff) and package revision 797f31cc813c0d643b7205118a901732f39de8ab passed M5 Max package-backed single-step parity at split 24 with f16 activation transport: baseline token 674 matched staged token 674, activation width 3072, payload 12288 bytes, wire payload 6144 bytes; package materialization, production-shaped OpenAI smoke, and two-stage direct prediction return also passed; chain, dtype matrix, state, cache, and mesh evidence remain pending" + "notes": "package-only certification status retained while broader gates remain pending; source revision edd093522473dc7313b0738d8b4116b7f8b9745f (SHA-256 a34c74e46688122bef83122f4133031bababbefcf57436dde97048c91e2cc6ff) and package revision 797f31cc813c0d643b7205118a901732f39de8ab passed M5 Max package-backed single-step parity at split 24 and three-stage chain parity at splits 16,32 with f16 activation transport: baseline token 674 matched both staged predictions, activation width 3072, payload 12288 bytes, wire payload 6144 bytes; package materialization, production-shaped OpenAI smoke, and direct prediction return also passed; dtype matrix, state, cache, and M4+M5 mesh evidence remain pending" }, { "llama_model": "mistral3", From 18b2128b37debf5f79008f9c0e8e125819c2d550 Mon Sep 17 00:00:00 2001 From: Michael Neale <14976+michaelneale@users.noreply.github.com> Date: Tue, 28 Jul 2026 12:48:49 +1000 Subject: [PATCH 06/17] docs(skippy): record Laguna distributed serving smoke --- docs/skippy/FAMILY_STATUS.md | 2 +- docs/skippy/LLAMA_PARITY.md | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/skippy/FAMILY_STATUS.md b/docs/skippy/FAMILY_STATUS.md index a6224e6869..74f3d4eee4 100644 --- a/docs/skippy/FAMILY_STATUS.md +++ b/docs/skippy/FAMILY_STATUS.md @@ -94,7 +94,7 @@ smoke, reviewed topology records, and family-specific policy notes are updated. ```text Gemma text -Laguna S 2.1 (package-backed M5 two-stage single-step and three-stage chain parity; broader certification pending) +Laguna S 2.1 (package-backed M5 two-stage single-step and three-stage chain parity, plus M5 Metal + Vast CUDA distributed OpenAI serving at split 24; broader certification pending) ``` ## Exceptions diff --git a/docs/skippy/LLAMA_PARITY.md b/docs/skippy/LLAMA_PARITY.md index 827305b3d6..a9a7c59659 100644 --- a/docs/skippy/LLAMA_PARITY.md +++ b/docs/skippy/LLAMA_PARITY.md @@ -360,9 +360,10 @@ implementation. ## Current Local Evidence -These rows were collected on the local Mac Studio against the Metal stage ABI. -They are cheap text-split and cache-smoke evidence, not full promotion by -themselves until the reviewed topology records are updated. +These rows were collected primarily on the local Mac Studio against the Metal +stage ABI. They are cheap text-split and cache-smoke evidence, not full +promotion by themselves until the reviewed topology records are updated. +Rows with distributed evidence call out the second backend explicitly. | Family | Artifact | Text Split | q8 Wire | Exact State | Cache | | --- | --- | --- | --- | --- | --- | @@ -370,7 +371,7 @@ themselves until the reviewed topology records are updated. | `deepseek` | `Morgen0052/deepseek-llm-7b-chat-Q4_K_M-GGUF` | `single-step`, `chain`, and f16 dtype matrix passed | rejected | accepted | `ResidentKv` borrowed-hit smoke passed, 64-token prefix, 1.58x cache-hit speedup | | `openai_moe` | `ggml-org/gpt-oss-20b-GGUF:gpt-oss-20b-mxfp4` | `single-step`, `chain`, and dtype matrix passed | rejected | accepted | `ResidentKv` state handoff passed; llama.cpp model file is `openai-moe`, GGUF architecture is `gpt-oss` | | `ernie4_5_moe` | `lmstudio-community/ERNIE-4.5-21B-A3B-PT-GGUF:Q4_K_M` | `single-step`, `chain`, and dtype matrix passed | validated | accepted | `ResidentKv` state handoff passed | -| `laguna` | `meshllm/laguna-s-2.1-Q4_K_M-layers@797f31cc813c0d643b7205118a901732f39de8ab` | package-backed `single-step` passed at split `24`; three-stage `chain` passed at splits `16,32` | untested | untested | M5 Max f16 parity passed against pinned full-model revision `edd093522473dc7313b0738d8b4116b7f8b9745f`: baseline token `674` matched both two-stage and three-stage predictions; width `3072`, 12,288-byte activation frame, 6,144-byte wire payload; dtype matrix, state/cache, and M4+M5 mesh evidence remain pending | +| `laguna` | `meshllm/laguna-s-2.1-Q4_K_M-layers@797f31cc813c0d643b7205118a901732f39de8ab` | package-backed `single-step` passed at split `24`; three-stage `chain` passed at splits `16,32`; distributed OpenAI serving passed at split `24` | untested | untested | M5 Max f16 parity passed against pinned full-model revision `edd093522473dc7313b0738d8b4116b7f8b9745f`: baseline token `674` matched both two-stage and three-stage predictions. A package-backed M5 Metal `0..24` + Australian Vast.ai RTX PRO 6000 Blackwell CUDA `24..48` run at commit `226fb584` returned HTTP 200 for a 43-token prompt plus one generated token (`19`); the decode activation used the expected 12,288-byte frame and 6,144-byte f16 wire payload. Dtype matrix, state/cache, and mesh-node orchestration remain pending. | | `llama4` | `ggml-org/Llama-4-Scout-17B-16E-Instruct-GGUF:Q4_K_M` | package validated | untested | untested | package-only validation passed: 48 layers, 627 owned tensors, 51 artifacts, no missing/duplicate tensors | | `mistral4` | `bartowski/mistralai_Mistral-Small-4-119B-2603-GGUF:IQ2_XXS` | package validated | untested | untested | package-only validation passed: 36 layers, 579 tensors, 39 artifacts, no missing/duplicate tensors | | `nemotron_h_moe` | `lmstudio-community/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-GGUF:Q4_K_M` | package validated | untested | rejected-too-large | package-only validation passed: 52 layers, 401 tensors, 55 artifacts; `KvRecurrent` target | From c4084517cda2286edcaa5aa30142289743ab5dfd Mon Sep 17 00:00:00 2001 From: Michael Neale <14976+michaelneale@users.noreply.github.com> Date: Tue, 28 Jul 2026 14:09:50 +1000 Subject: [PATCH 07/17] docs(skippy): record real Laguna mesh evidence --- docs/skippy/FAMILY_STATUS.md | 2 +- docs/skippy/LLAMA_PARITY.md | 2 +- docs/skippy/llama-parity-candidates.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/skippy/FAMILY_STATUS.md b/docs/skippy/FAMILY_STATUS.md index 74f3d4eee4..03fdb5b399 100644 --- a/docs/skippy/FAMILY_STATUS.md +++ b/docs/skippy/FAMILY_STATUS.md @@ -94,7 +94,7 @@ smoke, reviewed topology records, and family-specific policy notes are updated. ```text Gemma text -Laguna S 2.1 (package-backed M5 two-stage single-step and three-stage chain parity, plus M5 Metal + Vast CUDA distributed OpenAI serving at split 24; broader certification pending) +Laguna S 2.1 (package-backed M5 two-stage single-step and three-stage chain parity, plus ordinary M5 Metal 0..36 + Vast CUDA 36..48 Mesh serving at configured context 262144 with a 44,460-token prompt; broader certification pending) ``` ## Exceptions diff --git a/docs/skippy/LLAMA_PARITY.md b/docs/skippy/LLAMA_PARITY.md index a9a7c59659..e5a77050bd 100644 --- a/docs/skippy/LLAMA_PARITY.md +++ b/docs/skippy/LLAMA_PARITY.md @@ -371,7 +371,7 @@ Rows with distributed evidence call out the second backend explicitly. | `deepseek` | `Morgen0052/deepseek-llm-7b-chat-Q4_K_M-GGUF` | `single-step`, `chain`, and f16 dtype matrix passed | rejected | accepted | `ResidentKv` borrowed-hit smoke passed, 64-token prefix, 1.58x cache-hit speedup | | `openai_moe` | `ggml-org/gpt-oss-20b-GGUF:gpt-oss-20b-mxfp4` | `single-step`, `chain`, and dtype matrix passed | rejected | accepted | `ResidentKv` state handoff passed; llama.cpp model file is `openai-moe`, GGUF architecture is `gpt-oss` | | `ernie4_5_moe` | `lmstudio-community/ERNIE-4.5-21B-A3B-PT-GGUF:Q4_K_M` | `single-step`, `chain`, and dtype matrix passed | validated | accepted | `ResidentKv` state handoff passed | -| `laguna` | `meshllm/laguna-s-2.1-Q4_K_M-layers@797f31cc813c0d643b7205118a901732f39de8ab` | package-backed `single-step` passed at split `24`; three-stage `chain` passed at splits `16,32`; distributed OpenAI serving passed at split `24` | untested | untested | M5 Max f16 parity passed against pinned full-model revision `edd093522473dc7313b0738d8b4116b7f8b9745f`: baseline token `674` matched both two-stage and three-stage predictions. A package-backed M5 Metal `0..24` + Australian Vast.ai RTX PRO 6000 Blackwell CUDA `24..48` run at commit `226fb584` returned HTTP 200 for a 43-token prompt plus one generated token (`19`); the decode activation used the expected 12,288-byte frame and 6,144-byte f16 wire payload. Dtype matrix, state/cache, and mesh-node orchestration remain pending. | +| `laguna` | `meshllm/laguna-s-2.1-Q4_K_M-layers@797f31cc813c0d643b7205118a901732f39de8ab` | package-backed `single-step` passed at split `24`; three-stage `chain` passed at splits `16,32`; ordinary two-node Mesh OpenAI serving passed at `0..36 / 36..48` with configured context `262144` | untested | untested | M5 Max f16 parity passed against pinned full-model revision `edd093522473dc7313b0738d8b4116b7f8b9745f`: baseline token `674` matched both two-stage and three-stage predictions. The real Mesh run used M5 Max/Metal plus an Australian Vast RTX 6000 Ada/CUDA worker, normal planning and lifecycle, and a direct Iroh path with observed RTT samples from 24 ms to 320 ms. `doctor split` reported ready with no blockers. A 44,460-token prompt plus deterministic 65-token completion passed with exact output agreement. Suffix N-gram accepted 32 of 64 speculative tokens on the repeat/copy workload and raised decode from 5.72 to 9.61 tok/s; a 56-token low-overlap control produced no proposals. Exact-prefix reuse did not occur (`cached_tokens=0`), leaving streaming TTFT near 190 seconds. Equal `24/24` placement at full context exhausted the 48 GB CUDA worker; the working capacity-bounded topology was `36/12`. Dtype matrix, state/cache, tool-loop, and larger-context soak evidence remain pending. | | `llama4` | `ggml-org/Llama-4-Scout-17B-16E-Instruct-GGUF:Q4_K_M` | package validated | untested | untested | package-only validation passed: 48 layers, 627 owned tensors, 51 artifacts, no missing/duplicate tensors | | `mistral4` | `bartowski/mistralai_Mistral-Small-4-119B-2603-GGUF:IQ2_XXS` | package validated | untested | untested | package-only validation passed: 36 layers, 579 tensors, 39 artifacts, no missing/duplicate tensors | | `nemotron_h_moe` | `lmstudio-community/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-GGUF:Q4_K_M` | package validated | untested | rejected-too-large | package-only validation passed: 52 layers, 401 tensors, 55 artifacts; `KvRecurrent` target | diff --git a/docs/skippy/llama-parity-candidates.json b/docs/skippy/llama-parity-candidates.json index 0e20aa57e7..954ea5317b 100644 --- a/docs/skippy/llama-parity-candidates.json +++ b/docs/skippy/llama-parity-candidates.json @@ -191,7 +191,7 @@ "status": "certified_package_only", "repo": "poolside/Laguna-S-2.1-GGUF", "include": "laguna-s-2.1-Q4_K_M.gguf", - "notes": "package-only certification status retained while broader gates remain pending; source revision edd093522473dc7313b0738d8b4116b7f8b9745f (SHA-256 a34c74e46688122bef83122f4133031bababbefcf57436dde97048c91e2cc6ff) and package revision 797f31cc813c0d643b7205118a901732f39de8ab passed M5 Max package-backed single-step parity at split 24 and three-stage chain parity at splits 16,32 with f16 activation transport: baseline token 674 matched both staged predictions, activation width 3072, payload 12288 bytes, wire payload 6144 bytes; package materialization, production-shaped OpenAI smoke, and direct prediction return also passed; dtype matrix, state, cache, and M4+M5 mesh evidence remain pending" + "notes": "package-only certification status retained while broader gates remain pending; source revision edd093522473dc7313b0738d8b4116b7f8b9745f (SHA-256 a34c74e46688122bef83122f4133031bababbefcf57436dde97048c91e2cc6ff) and package revision 797f31cc813c0d643b7205118a901732f39de8ab passed M5 Max package-backed single-step parity at split 24 and three-stage chain parity at splits 16,32 with f16 activation transport: baseline token 674 matched both staged predictions, activation width 3072, payload 12288 bytes, wire payload 6144 bytes; an ordinary M5 Metal plus Vast RTX 6000 Ada CUDA private Mesh run passed normal 0..36/36..48 placement, direct Iroh transport, doctor readiness, configured context 262144, a 44460-token prompt, exact 65-token completion agreement, and suffix N-gram verification with 32/64 speculative tokens accepted; the 24/24 full-context CUDA stage exhausted 48 GB, exact-prefix reuse reported zero cached tokens, and dtype matrix, state/cache, tool-loop, and larger-context soak evidence remain pending" }, { "llama_model": "mistral3", From 75124302864f8cace115ebd97af94a2eb911775d Mon Sep 17 00:00:00 2001 From: Michael Neale <14976+michaelneale@users.noreply.github.com> Date: Tue, 28 Jul 2026 14:33:44 +1000 Subject: [PATCH 08/17] fix(skippy): make stage memory truly layer-local --- .../src/inference/skippy/deployment.rs | 2 +- .../src/inference/skippy/family_policy.rs | 101 +++++++++++++++++- .../src/inference/skippy/stage/mod.rs | 8 +- ...staged-runtime-memory-to-layer-range.patch | 72 +++++++++++++ 4 files changed, 178 insertions(+), 5 deletions(-) create mode 100644 third_party/llama.cpp/patches/0047-Filter-staged-runtime-memory-to-layer-range.patch diff --git a/crates/mesh-llm-host-runtime/src/inference/skippy/deployment.rs b/crates/mesh-llm-host-runtime/src/inference/skippy/deployment.rs index 572d36c14a..e11dca3249 100644 --- a/crates/mesh-llm-host-runtime/src/inference/skippy/deployment.rs +++ b/crates/mesh-llm-host-runtime/src/inference/skippy/deployment.rs @@ -125,7 +125,7 @@ pub(crate) fn stage0_config( }; config.kv_cache = context .family_policy - .stage_kv_cache_config_for_stage(&config); + .stage_kv_cache_config_for_package(&config, &context.package.package_dir); config } diff --git a/crates/mesh-llm-host-runtime/src/inference/skippy/family_policy.rs b/crates/mesh-llm-host-runtime/src/inference/skippy/family_policy.rs index 97b1fa9102..a65d0019aa 100644 --- a/crates/mesh-llm-host-runtime/src/inference/skippy/family_policy.rs +++ b/crates/mesh-llm-host-runtime/src/inference/skippy/family_policy.rs @@ -50,6 +50,24 @@ impl FamilyPolicy { pub(crate) fn stage_kv_cache_config_for_stage( &self, config: &StageConfig, + ) -> Option { + self.stage_kv_cache_config_for_stage_with_meta(config, None) + } + + pub(crate) fn stage_kv_cache_config_for_package( + &self, + config: &StageConfig, + package_dir: &Path, + ) -> Option { + let metadata_path = package_dir.join("shared/metadata.gguf"); + let metadata = scan_gguf_compact_meta(&metadata_path); + self.stage_kv_cache_config_for_stage_with_meta(config, metadata.as_ref()) + } + + fn stage_kv_cache_config_for_stage_with_meta( + &self, + config: &StageConfig, + package_meta: Option<&GgufCompactMeta>, ) -> Option { match self.prefix_cache { FamilyPrefixCachePolicy::Disabled { .. } => None, @@ -58,7 +76,7 @@ impl FamilyPolicy { min_tokens, max_entries, } => { - let max_bytes = derive_stage_cache_max_bytes(config)?; + let max_bytes = derive_stage_cache_max_bytes(config, package_meta)?; // The family policy's `max_entries` is a generous // upper bound on cache cardinality. The real ceiling // is the unified KV cell pool size: each resident @@ -316,7 +334,16 @@ fn wire_dtype_from_capability(dtype: WireDType) -> StageWireDType { } } -fn derive_stage_cache_max_bytes(config: &StageConfig) -> Option { +fn derive_stage_cache_max_bytes( + config: &StageConfig, + package_meta: Option<&GgufCompactMeta>, +) -> Option { + if let Some(max_bytes) = + package_meta.and_then(|meta| estimate_stage_cache_max_bytes(config, meta)) + { + return Some(max_bytes); + } + [ config.materialized_path.as_deref(), config.source_model_path.as_deref(), @@ -324,10 +351,15 @@ fn derive_stage_cache_max_bytes(config: &StageConfig) -> Option { ] .into_iter() .flatten() - .find_map(|path| scan_gguf_compact_meta(Path::new(path))) + .find_map(|path| scan_stage_cache_meta(Path::new(path))) .and_then(|meta| estimate_stage_cache_max_bytes(config, &meta)) } +fn scan_stage_cache_meta(path: &Path) -> Option { + scan_gguf_compact_meta(path) + .or_else(|| scan_gguf_compact_meta(&path.join("shared/metadata.gguf"))) +} + fn estimate_stage_cache_max_bytes(config: &StageConfig, meta: &GgufCompactMeta) -> Option { let stage_layers = config.layer_end.checked_sub(config.layer_start)?; if stage_layers == 0 { @@ -403,6 +435,8 @@ fn ggml_block_bytes(elements: u64, block_size: u64, type_size: u64) -> Option, value: &str) { + bytes.extend_from_slice(&(value.len() as u64).to_le_bytes()); + bytes.extend_from_slice(value.as_bytes()); + } + + fn push_gguf_u32(bytes: &mut Vec, key: &str, value: u32) { + push_gguf_string(bytes, key); + bytes.extend_from_slice(&4u32.to_le_bytes()); + bytes.extend_from_slice(&value.to_le_bytes()); + } + + fn push_gguf_string_kv(bytes: &mut Vec, key: &str, value: &str) { + push_gguf_string(bytes, key); + bytes.extend_from_slice(&8u32.to_le_bytes()); + push_gguf_string(bytes, value); + } + + fn write_package_metadata(package_dir: &Path) { + let mut bytes = Vec::new(); + bytes.extend_from_slice(b"GGUF"); + bytes.extend_from_slice(&2u32.to_le_bytes()); + bytes.extend_from_slice(&0i64.to_le_bytes()); + bytes.extend_from_slice(&8i64.to_le_bytes()); + push_gguf_string_kv(&mut bytes, "general.architecture", "llama"); + push_gguf_u32(&mut bytes, "llama.context_length", 8192); + push_gguf_u32(&mut bytes, "llama.embedding_length", 4096); + push_gguf_u32(&mut bytes, "llama.block_count", 32); + push_gguf_u32(&mut bytes, "llama.attention.head_count", 32); + push_gguf_u32(&mut bytes, "llama.attention.head_count_kv", 8); + push_gguf_u32(&mut bytes, "llama.attention.key_length", 128); + push_gguf_u32(&mut bytes, "llama.attention.value_length", 128); + let shared_dir = package_dir.join("shared"); + fs::create_dir_all(&shared_dir).expect("create package shared directory"); + fs::write(shared_dir.join("metadata.gguf"), bytes).expect("write package metadata"); + } + #[test] fn qwen_policy_comes_from_gguf_architecture() { let policy = family_policy_for_gguf_meta(&meta("qwen3"), None); @@ -761,4 +831,29 @@ mod tests { assert!(estimate_stage_cache_max_bytes(&config, &kv_meta()).is_none()); } + + #[test] + fn package_metadata_enables_cache_for_remote_package_paths() { + let package_dir = tempfile::tempdir().expect("package directory"); + write_package_metadata(package_dir.path()); + let mut config = stage_config(); + config.materialized_path = None; + config.source_model_path = Some("/source/not-downloaded/model.gguf".to_string()); + config.model_path = Some("hf://mesh-llm/laguna-layers".to_string()); + let policy = FamilyPolicy { + activation_wire_dtype: StageWireDType::F16, + prefix_cache: FamilyPrefixCachePolicy::Auto { + payload: FamilyPrefixCachePayload::ResidentKv, + min_tokens: 256, + max_entries: 16, + }, + }; + + let cache = policy + .stage_kv_cache_config_for_package(&config, package_dir.path()) + .expect("package metadata should provide the cache byte budget"); + + assert_eq!(cache.payload, StageKvCachePayload::ResidentKv); + assert_eq!(cache.max_bytes, 3_211_264); + } } diff --git a/crates/mesh-llm-host-runtime/src/inference/skippy/stage/mod.rs b/crates/mesh-llm-host-runtime/src/inference/skippy/stage/mod.rs index db2da2f8ff..8200e6ecfc 100644 --- a/crates/mesh-llm-host-runtime/src/inference/skippy/stage/mod.rs +++ b/crates/mesh-llm-host-runtime/src/inference/skippy/stage/mod.rs @@ -1,6 +1,7 @@ use std::{ collections::HashMap, net::SocketAddr, + path::Path, sync::{ Arc, atomic::{AtomicBool, Ordering}, @@ -738,7 +739,12 @@ fn stage_config( downstream: load.downstream.as_ref().map(peer_config), }; let family_policy = super::family_policy_for_stage_config(&config); - config.kv_cache = family_policy.stage_kv_cache_config_for_stage(&config); + config.kv_cache = package.map_or_else( + || family_policy.stage_kv_cache_config_for_stage(&config), + |package| { + family_policy.stage_kv_cache_config_for_package(&config, Path::new(&package.local_ref)) + }, + ); Ok(config) } diff --git a/third_party/llama.cpp/patches/0047-Filter-staged-runtime-memory-to-layer-range.patch b/third_party/llama.cpp/patches/0047-Filter-staged-runtime-memory-to-layer-range.patch new file mode 100644 index 0000000000..296b5c2d39 --- /dev/null +++ b/third_party/llama.cpp/patches/0047-Filter-staged-runtime-memory-to-layer-range.patch @@ -0,0 +1,72 @@ +From 3a5f43b7fe0e90ac28f0a2856f87c291f5990a29 Mon Sep 17 00:00:00 2001 +From: Mesh-LLM CI +Date: Tue, 28 Jul 2026 14:23:09 +1000 +Subject: [PATCH] Filter staged runtime memory to layer range + +Skippy graph execution already filters to the assigned layer range, but +llama_model::create_memory still allocated cache rows for the complete model. +Compose the active stage filter with each memory implementation's architecture +filter so split capacity estimates match the native allocation. Keep MTP +sidecar contexts on their existing next-token layer filter. + +--- + src/llama-model.cpp | 23 ++++++++++++++++++++++- + 1 file changed, 22 insertions(+), 1 deletion(-) + +diff --git a/src/llama-model.cpp b/src/llama-model.cpp +index 10633a51..7ad59b29 100644 +--- a/src/llama-model.cpp ++++ b/src/llama-model.cpp +@@ -2042,6 +2042,22 @@ ggml_tensor * llama_model::get_rope_factors(const llama_cparams & cparams, int i + return layers[il].rope_short; + } + ++static llama_memory_i::layer_filter_cb skippy_stage_memory_filter( ++ llama_memory_i::layer_filter_cb filter, ++ enum llama_context_type ctx_type) { ++ const skippy_graph_filter stage_filter = skippy_graph_get_filter(); ++ if (!stage_filter.enabled || ctx_type == LLAMA_CONTEXT_TYPE_MTP) { ++ return filter; ++ } ++ ++ const int32_t layer_start = stage_filter.layer_start; ++ const int32_t layer_end = stage_filter.layer_end; ++ return [filter = std::move(filter), layer_start, layer_end](int32_t il) { ++ const bool in_stage = il >= layer_start && il < layer_end; ++ return in_stage && (!filter || filter(il)); ++ }; ++} ++ + llama_memory_i * llama_model::create_memory(const llama_memory_params & params, const llama_cparams & cparams) const { + llama_memory_i * res; + +@@ -2102,7 +2118,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, + std::max((uint32_t) 1, cparams.n_seq_max), + cparams.n_seq_max, + cparams.n_rs_seq, +- nullptr); ++ skippy_stage_memory_filter(nullptr, params.ctx_type)); + } else if (llm_arch_is_hybrid(arch) && !mtp_on_hybrid_qwen35) { + // The main difference between hybrid architectures is the + // layer filters, so pick the right one here +@@ -2127,6 +2143,9 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, + }; + } + ++ filter_attn = skippy_stage_memory_filter(std::move(filter_attn), params.ctx_type); ++ filter_recr = skippy_stage_memory_filter(std::move(filter_recr), params.ctx_type); ++ + if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) { + // Use hybrid-iswa for hybrid models with SWA + res = new llama_memory_hybrid_iswa( +@@ -2204,6 +2223,8 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, + } + } + ++ filter = skippy_stage_memory_filter(std::move(filter), params.ctx_type); ++ + if (arch == LLM_ARCH_DEEPSEEK4) { + GGML_ASSERT(hparams.swa_type != LLAMA_SWA_TYPE_NONE); + +-- +2.50.1 (Apple Git-155) From a2e0d65cabbf9301a32b42f7c7616ea7e5f7df72 Mon Sep 17 00:00:00 2001 From: Michael Neale <14976+michaelneale@users.noreply.github.com> Date: Tue, 28 Jul 2026 16:07:52 +1000 Subject: [PATCH 09/17] fix(skippy): clean partial prefix restores --- .../src/frontend/prefix_cache.rs | 46 +++++++++++++++++-- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/crates/skippy-server/src/frontend/prefix_cache.rs b/crates/skippy-server/src/frontend/prefix_cache.rs index be25393b0e..e9741fb75c 100644 --- a/crates/skippy-server/src/frontend/prefix_cache.rs +++ b/crates/skippy-server/src/frontend/prefix_cache.rs @@ -108,6 +108,18 @@ fn estimated_activation_bytes( .unwrap_or(0) } +fn cleanup_partial_restore( + restored_tokens: usize, + required_tokens: usize, + cleanup: impl FnOnce(), +) -> bool { + if restored_tokens >= required_tokens { + return false; + } + cleanup(); + true +} + pub(super) fn request_allows_exact_replay(request: &EmbeddedStageZeroGeneration<'_>) -> bool { request.draft.is_none() && request.sampling.temperature <= 0.0 } @@ -719,7 +731,9 @@ impl StageOpenAiBackend { else { continue; }; - if restore.restored_tokens < checkpoint_tokens.len() { + if cleanup_partial_restore(restore.restored_tokens, checkpoint_tokens.len(), || { + self.drop_embedded_split_restore(request, session_key, downstream); + }) { continue; } let replay = replay_tokens[..replay_len].to_vec(); @@ -808,7 +822,11 @@ impl StageOpenAiBackend { else { return Ok(None); }; - if restore.restored_tokens < request.prompt_token_ids.len() { + if cleanup_partial_restore( + restore.restored_tokens, + request.prompt_token_ids.len(), + || self.drop_embedded_split_restore(request, session_key, downstream), + ) { return Ok(None); } let mut attrs = self.openai_attrs(request.ids); @@ -995,7 +1013,9 @@ impl StageOpenAiBackend { let Some(local_restore) = local_restore else { return Ok(None); }; - if local_restore.token_count < prefill_tokens.len() { + if cleanup_partial_restore(local_restore.token_count, prefill_tokens.len(), || { + self.drop_embedded_split_restore(request, session_key, downstream); + }) { return Ok(None); } reply_stats.kv_lookup_hits += 1; @@ -1232,4 +1252,24 @@ mod tests { assert_eq!(f32.stage0_activation_bytes_avoided, 5_242_880); assert_eq!(f32.interstage_activation_bytes_avoided_estimate, 5_242_880); } + + #[test] + fn partial_restore_runs_cleanup_before_another_probe() { + let mut cleaned = false; + + let rejected = cleanup_partial_restore(1_024, 1_030, || cleaned = true); + + assert!(rejected); + assert!(cleaned); + } + + #[test] + fn full_restore_keeps_the_session_active() { + let mut cleaned = false; + + let rejected = cleanup_partial_restore(1_030, 1_030, || cleaned = true); + + assert!(!rejected); + assert!(!cleaned); + } } From 25727d92e92b1f9dc8ede52b2b6096f651af9a9b Mon Sep 17 00:00:00 2001 From: Michael Neale <14976+michaelneale@users.noreply.github.com> Date: Tue, 28 Jul 2026 17:01:25 +1000 Subject: [PATCH 10/17] fix(skippy): cancel timed out nonstream generation --- crates/openai-frontend/src/backend.rs | 16 ++ .../openai-frontend/src/guardrails/compact.rs | 18 ++ crates/openai-frontend/src/guardrails/mod.rs | 44 ++++- crates/openai-frontend/src/hooks.rs | 20 ++ crates/openai-frontend/src/router.rs | 171 ++++++++++++++++-- crates/skippy-server/src/frontend/backend.rs | 78 +++++--- 6 files changed, 294 insertions(+), 53 deletions(-) diff --git a/crates/openai-frontend/src/backend.rs b/crates/openai-frontend/src/backend.rs index 632919b5fa..28fbf3fa03 100644 --- a/crates/openai-frontend/src/backend.rs +++ b/crates/openai-frontend/src/backend.rs @@ -74,6 +74,14 @@ pub trait OpenAiBackend: Send + Sync + 'static { request: ChatCompletionRequest, ) -> OpenAiResult; + async fn chat_completion_with_context( + &self, + request: ChatCompletionRequest, + _context: OpenAiRequestContext, + ) -> OpenAiResult { + self.chat_completion(request).await + } + async fn chat_completion_stream( &self, request: ChatCompletionRequest, @@ -86,6 +94,14 @@ pub trait OpenAiBackend: Send + Sync + 'static { )) } + async fn completion_with_context( + &self, + request: CompletionRequest, + _context: OpenAiRequestContext, + ) -> OpenAiResult { + self.completion(request).await + } + async fn completion_stream( &self, _request: CompletionRequest, diff --git a/crates/openai-frontend/src/guardrails/compact.rs b/crates/openai-frontend/src/guardrails/compact.rs index 093b29940e..263144144b 100644 --- a/crates/openai-frontend/src/guardrails/compact.rs +++ b/crates/openai-frontend/src/guardrails/compact.rs @@ -69,6 +69,16 @@ impl OpenAiBackend for CompactingOpenAiBackend { .await } + async fn chat_completion_with_context( + &self, + request: ChatCompletionRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend + .chat_completion_with_context(self.compact_request(request)?, context) + .await + } + async fn chat_completion_stream( &self, request: ChatCompletionRequest, @@ -83,6 +93,14 @@ impl OpenAiBackend for CompactingOpenAiBackend { self.backend.completion(request).await } + async fn completion_with_context( + &self, + request: CompletionRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.completion_with_context(request, context).await + } + async fn completion_stream( &self, request: CompletionRequest, diff --git a/crates/openai-frontend/src/guardrails/mod.rs b/crates/openai-frontend/src/guardrails/mod.rs index e6705b779f..6a1e6af45e 100644 --- a/crates/openai-frontend/src/guardrails/mod.rs +++ b/crates/openai-frontend/src/guardrails/mod.rs @@ -76,6 +76,7 @@ impl GuardedOpenAiBackend { async fn guarded_chat_completion( &self, request: ChatCompletionRequest, + context: Option<&OpenAiRequestContext>, ) -> OpenAiResult { let _guardrail_error_catalog = guardrail_error_catalog(); let policy = self.policy.snapshot(); @@ -92,13 +93,13 @@ impl GuardedOpenAiBackend { Some(GuardrailTelemetryParserStage::None), None, ); - self.backend.chat_completion(request).await + self.backend_chat_completion(request, context).await } GuardrailRequestOutcome::Reject { kind } => Err(errors::guardrail_error(*kind)), GuardrailRequestOutcome::Guarded { backend_request } => { if matches!(policy.mode, GuardrailMode::MetricsOnly) { return self - .metrics_only_chat_completion(request, &engine, &prepared) + .metrics_only_chat_completion(request, &engine, &prepared, context) .await; } @@ -108,8 +109,7 @@ impl GuardedOpenAiBackend { loop { let response = self - .backend - .chat_completion(attempt_request.clone()) + .backend_chat_completion(attempt_request.clone(), context) .await?; let classified = engine.classify_response(&prepared, &response); let parser_stage = telemetry_parser_stage(classified.parser_stage); @@ -178,8 +178,9 @@ impl GuardedOpenAiBackend { request: ChatCompletionRequest, engine: &GuardrailEngine, prepared: &state::PreparedGuardrailRequest, + context: Option<&OpenAiRequestContext>, ) -> OpenAiResult { - let response = self.backend.chat_completion(request).await?; + let response = self.backend_chat_completion(request, context).await?; let classified = engine.classify_response(prepared, &response); let parser_stage = telemetry_parser_stage(classified.parser_stage); self.record_outcome( @@ -192,6 +193,21 @@ impl GuardedOpenAiBackend { Ok(response) } + async fn backend_chat_completion( + &self, + request: ChatCompletionRequest, + context: Option<&OpenAiRequestContext>, + ) -> OpenAiResult { + match context { + Some(context) => { + self.backend + .chat_completion_with_context(request, context.clone()) + .await + } + None => self.backend.chat_completion(request).await, + } + } + fn record_decision(&self, prepared: &state::PreparedGuardrailRequest) { if let Some(telemetry) = &self.telemetry { telemetry.record_decision( @@ -336,7 +352,15 @@ impl OpenAiBackend for GuardedOpenAiBackend { &self, request: ChatCompletionRequest, ) -> OpenAiResult { - self.guarded_chat_completion(request).await + self.guarded_chat_completion(request, None).await + } + + async fn chat_completion_with_context( + &self, + request: ChatCompletionRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.guarded_chat_completion(request, Some(&context)).await } async fn chat_completion_stream( @@ -351,6 +375,14 @@ impl OpenAiBackend for GuardedOpenAiBackend { self.backend.completion(request).await } + async fn completion_with_context( + &self, + request: CompletionRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.completion_with_context(request, context).await + } + async fn completion_stream( &self, request: CompletionRequest, diff --git a/crates/openai-frontend/src/hooks.rs b/crates/openai-frontend/src/hooks.rs index 895149a53f..61f0be0cce 100644 --- a/crates/openai-frontend/src/hooks.rs +++ b/crates/openai-frontend/src/hooks.rs @@ -136,6 +136,18 @@ impl OpenAiBackend for HookedOpenAiBackend { self.backend.chat_completion(request).await } + async fn chat_completion_with_context( + &self, + mut request: ChatCompletionRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + let outcome = self.hooks.before_chat_completion(&mut request).await?; + apply_chat_hook_outcome(&mut request, &outcome); + self.backend + .chat_completion_with_context(request, context) + .await + } + async fn chat_completion_stream( &self, mut request: ChatCompletionRequest, @@ -150,6 +162,14 @@ impl OpenAiBackend for HookedOpenAiBackend { self.backend.completion(request).await } + async fn completion_with_context( + &self, + request: CompletionRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.completion_with_context(request, context).await + } + async fn completion_stream( &self, request: CompletionRequest, diff --git a/crates/openai-frontend/src/router.rs b/crates/openai-frontend/src/router.rs index 2085ac434e..19d44a53bc 100644 --- a/crates/openai-frontend/src/router.rs +++ b/crates/openai-frontend/src/router.rs @@ -157,10 +157,13 @@ async fn chat_completions( let model = request.model.clone(); let context = OpenAiRequestContext::new(); let cancellation = context.cancellation_token(); - let stream = backend_call( + let stream = backend_call_with_context( &state, "chat_completion_stream", - state.backend.chat_completion_stream(request, context), + &context, + state + .backend + .chat_completion_stream(request, context.clone()), ) .await?; let prelude = stream::once(async move { json_event(&ChatCompletionChunk::role(model)) }); @@ -175,11 +178,15 @@ async fn chat_completions( .chain(stream::once(async { done_event() })); Ok(sse_response(events, cancellation)) } else { + let context = OpenAiRequestContext::new(); Ok(Json( - backend_call( + backend_call_with_context( &state, "chat_completion", - state.backend.chat_completion(request), + &context, + state + .backend + .chat_completion_with_context(request, context.clone()), ) .await?, ) @@ -202,10 +209,13 @@ async fn responses( let context = OpenAiRequestContext::new(); let cancellation = context.cancellation_token(); let state_machine = Arc::new(Mutex::new(ResponseSseState::new(request.model.clone()))); - let stream = backend_call( + let stream = backend_call_with_context( &state, "responses_stream", - state.backend.chat_completion_stream(request, context), + &context, + state + .backend + .chat_completion_stream(request, context.clone()), ) .await?; let body_state = state_machine.clone(); @@ -396,8 +406,16 @@ async fn responses( Ok(sse_response(events, cancellation)) } _ => { - let response = - backend_call(&state, "responses", state.backend.chat_completion(request)).await?; + let context = OpenAiRequestContext::new(); + let response = backend_call_with_context( + &state, + "responses", + &context, + state + .backend + .chat_completion_with_context(request, context.clone()), + ) + .await?; let translated = translate_chat_completion_response_to_responses(&response)?; Ok(Json(translated).into_response()) } @@ -414,10 +432,11 @@ async fn completions( let include_usage = request.include_usage(); let context = OpenAiRequestContext::new(); let cancellation = context.cancellation_token(); - let stream = backend_call( + let stream = backend_call_with_context( &state, "completion_stream", - state.backend.completion_stream(request, context), + &context, + state.backend.completion_stream(request, context.clone()), ) .await?; let events = stream @@ -431,10 +450,19 @@ async fn completions( .chain(stream::once(async { done_event() })); Ok(sse_response(events, cancellation)) } else { - Ok( - Json(backend_call(&state, "completion", state.backend.completion(request)).await?) - .into_response(), + let context = OpenAiRequestContext::new(); + Ok(Json( + backend_call_with_context( + &state, + "completion", + &context, + state + .backend + .completion_with_context(request, context.clone()), + ) + .await?, ) + .into_response()) } } @@ -443,16 +471,46 @@ async fn backend_call( operation: &'static str, future: F, ) -> OpenAiResult +where + F: Future>, +{ + backend_call_inner(state, operation, None, future).await +} + +async fn backend_call_with_context( + state: &FrontendState, + operation: &'static str, + context: &OpenAiRequestContext, + future: F, +) -> OpenAiResult +where + F: Future>, +{ + backend_call_inner(state, operation, Some(context), future).await +} + +async fn backend_call_inner( + state: &FrontendState, + operation: &'static str, + context: Option<&OpenAiRequestContext>, + future: F, +) -> OpenAiResult where F: Future>, { match state.config.backend_timeout { - Some(timeout) => tokio::time::timeout(timeout, future).await.map_err(|_| { - OpenAiError::timeout(format!( - "{operation} timed out after {} ms", - timeout.as_millis() - )) - })?, + Some(timeout) => match tokio::time::timeout(timeout, future).await { + Ok(result) => result, + Err(_) => { + if let Some(context) = context { + context.cancel(); + } + Err(OpenAiError::timeout(format!( + "{operation} timed out after {} ms", + timeout.as_millis() + ))) + } + }, None => future.await, } } @@ -780,6 +838,10 @@ mod tests { token: Arc>>, } + struct TimeoutCancellationBackend { + token: Arc>>, + } + #[derive(Default)] struct GuardrailRescueBackend { seen_chat_requests: Arc>>, @@ -808,6 +870,37 @@ mod tests { } } + #[async_trait] + impl OpenAiBackend for TimeoutCancellationBackend { + async fn models(&self) -> OpenAiResult> { + Ok(vec![ModelObject::new("cancel-model")]) + } + + async fn chat_completion( + &self, + _request: ChatCompletionRequest, + ) -> OpenAiResult { + unreachable!("timeout cancellation must use the request context") + } + + async fn chat_completion_with_context( + &self, + _request: ChatCompletionRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + *self.token.lock().expect("token lock poisoned") = Some(context.cancellation_token()); + std::future::pending().await + } + + async fn chat_completion_stream( + &self, + _request: ChatCompletionRequest, + _context: OpenAiRequestContext, + ) -> OpenAiResult { + unreachable!("timeout cancellation backend only calls non-stream chat") + } + } + #[async_trait] impl OpenAiBackend for GuardrailRescueBackend { async fn models(&self) -> OpenAiResult> { @@ -1021,6 +1114,46 @@ mod tests { assert_eq!(body["error"]["code"], "timeout"); } + #[tokio::test] + async fn nonstream_backend_timeout_cancels_request_through_guardrails() { + let token = Arc::new(Mutex::new(None)); + let backend = Arc::new(TimeoutCancellationBackend { + token: token.clone(), + }); + let guarded = Arc::new(GuardedOpenAiBackend::new( + backend, + GuardrailPolicy::default(), + )); + let app = router_for_with_config( + guarded, + OpenAiFrontendConfig::default().with_backend_timeout(Duration::from_millis(1)), + ); + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri("/v1/chat/completions") + .header("content-type", "application/json") + .body(Body::from( + json!({ + "model": "cancel-model", + "messages": [{"role": "user", "content": "cancel me"}] + }) + .to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::GATEWAY_TIMEOUT); + let cancellation = token + .lock() + .expect("token lock poisoned") + .clone() + .expect("backend should receive cancellation token"); + assert!(cancellation.is_cancelled()); + } + #[tokio::test] async fn request_id_is_returned_on_success_and_errors() { let app = router_for(Arc::new(FakeBackend)); diff --git a/crates/skippy-server/src/frontend/backend.rs b/crates/skippy-server/src/frontend/backend.rs index d0d9d2c03f..61e3b404ed 100644 --- a/crates/skippy-server/src/frontend/backend.rs +++ b/crates/skippy-server/src/frontend/backend.rs @@ -51,6 +51,16 @@ use tokio::sync::OwnedSemaphorePermit; use tokio::sync::mpsc; use tokio::task; +struct NonstreamGeneration { + prompt: PreparedGenerationPrompt, + max_tokens: GenerationTokenLimit, + stop: Option, + sampling: SamplingConfig, + hook_request: Option, + cancellation: openai_frontend::CancellationToken, + ids: OpenAiGenerationIds, +} + #[async_trait] impl OpenAiBackend for StageOpenAiBackend { async fn models(&self) -> OpenAiResult> { @@ -58,8 +68,17 @@ impl OpenAiBackend for StageOpenAiBackend { } async fn chat_completion( + &self, + request: ChatCompletionRequest, + ) -> OpenAiResult { + self.chat_completion_with_context(request, OpenAiRequestContext::new()) + .await + } + + async fn chat_completion_with_context( &self, mut request: ChatCompletionRequest, + context: OpenAiRequestContext, ) -> OpenAiResult { let ids = OpenAiGenerationIds::new(OpenAiCacheHints::from_chat_request(&request)); let request_timer = PhaseTimer::start(); @@ -96,14 +115,15 @@ impl OpenAiBackend for StageOpenAiBackend { ); let chat_parse_metadata = prompt.chat_parse_metadata.clone(); let output = self - .run_generation( + .run_generation(NonstreamGeneration { prompt, max_tokens, - request.stop.clone(), + stop: request.stop.clone(), sampling, - Some(request.clone()), - ids.clone(), - ) + hook_request: Some(request.clone()), + cancellation: context.cancellation_token(), + ids: ids.clone(), + }) .await?; let response_timer = PhaseTimer::start(); let parsed_message = if parse_chat_output { @@ -214,7 +234,16 @@ impl OpenAiBackend for StageOpenAiBackend { }))) } - async fn completion(&self, mut request: CompletionRequest) -> OpenAiResult { + async fn completion(&self, request: CompletionRequest) -> OpenAiResult { + self.completion_with_context(request, OpenAiRequestContext::new()) + .await + } + + async fn completion_with_context( + &self, + mut request: CompletionRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { let ids = OpenAiGenerationIds::new(OpenAiCacheHints::from_completion_request(&request)); let request_timer = PhaseTimer::start(); self.ensure_model(&request.model)?; @@ -236,14 +265,15 @@ impl OpenAiBackend for StageOpenAiBackend { ); self.emit_openai_phase("stage.openai_prompt_prepare", prompt_timer, prompt_attrs); let output = self - .run_generation( + .run_generation(NonstreamGeneration { prompt, max_tokens, - request.stop.clone(), + stop: request.stop.clone(), sampling, - None, - ids.clone(), - ) + hook_request: None, + cancellation: context.cancellation_token(), + ids: ids.clone(), + }) .await?; let response_timer = PhaseTimer::start(); let response = completion_response_from_generated_text(request.model, &output); @@ -436,18 +466,10 @@ impl StageOpenAiBackend { Ok(()) } - async fn run_generation( - &self, - prompt: PreparedGenerationPrompt, - max_tokens: GenerationTokenLimit, - stop: Option, - sampling: SamplingConfig, - hook_request: Option, - ids: OpenAiGenerationIds, - ) -> OpenAiResult { + async fn run_generation(&self, request: NonstreamGeneration) -> OpenAiResult { let admit_timer = PhaseTimer::start(); let permit = self.acquire_generation_permit().await?; - let mut admit_attrs = self.openai_attrs(&ids); + let mut admit_attrs = self.openai_attrs(&request.ids); admit_attrs.insert( "llama_stage.openai_phase".to_string(), json!("generation_admit"), @@ -458,14 +480,14 @@ impl StageOpenAiBackend { task::spawn_blocking(move || { let _permit = permit; backend.generate_text( - prompt, - max_tokens, - stop.as_ref(), - sampling, - hook_request, + request.prompt, + request.max_tokens, + request.stop.as_ref(), + request.sampling, + request.hook_request, hook_runtime, - None, - ids, + Some(&request.cancellation), + request.ids, |_| Ok(()), ) }) From 28afde292a002e437c0b29859342c1db6cfb0d5e Mon Sep 17 00:00:00 2001 From: Michael Neale <14976+michaelneale@users.noreply.github.com> Date: Tue, 28 Jul 2026 19:15:01 +1000 Subject: [PATCH 11/17] skippy: recover hybrid verify windows without full KV copy Extract the recurrent checkpoint work from the Inkling branch into the common llama.cpp patch queue. Snapshot partial recurrent state, trim hybrid attention directly, retain pipelined checkpoints, and replay only the accepted prefix. This avoids copying the full long-context attention cache for each speculative verify window. Assisted-by: codex --- ...brid-verify-window-state-across-trim.patch | 423 ++++++++++++++++++ 1 file changed, 423 insertions(+) create mode 100644 third_party/llama.cpp/patches/0046-Preserve-hybrid-verify-window-state-across-trim.patch diff --git a/third_party/llama.cpp/patches/0046-Preserve-hybrid-verify-window-state-across-trim.patch b/third_party/llama.cpp/patches/0046-Preserve-hybrid-verify-window-state-across-trim.patch new file mode 100644 index 0000000000..d483f50a33 --- /dev/null +++ b/third_party/llama.cpp/patches/0046-Preserve-hybrid-verify-window-state-across-trim.patch @@ -0,0 +1,423 @@ +From 9ed92928e27968f8d6914e1b849566538d42af9a Mon Sep 17 00:00:00 2001 +From: Mesh-LLM CI +Date: Tue, 28 Jul 2026 19:12:14 +1000 +Subject: [PATCH] Preserve hybrid verify-window state across trim + +Checkpoint only partial recurrent state, trim hybrid attention directly, and replay the accepted prefix so pipelined speculative windows can recover without copying the full long-context attention cache. + +Based on the recurrent checkpoint work from the Inkling branch. + +Assisted-by: codex +--- + src/skippy.cpp | 303 ++++++++++++++++++++++++++++++++++++++++++++----- + 1 file changed, 277 insertions(+), 26 deletions(-) + +diff --git a/src/skippy.cpp b/src/skippy.cpp +index 47f0a549..18bcc92b 100644 +--- a/src/skippy.cpp ++++ b/src/skippy.cpp +@@ -124,6 +124,17 @@ struct skippy_model { + std::vector> lane_resident_prefix_tokens; + }; + ++struct skippy_verify_checkpoint { ++ bool valid = false; ++ int32_t token_start = 0; ++ size_t token_count = 0; ++ std::vector state; ++ std::vector token_ids; ++ bool has_activation_input = false; ++ skippy_activation_desc input_desc = {}; ++ std::vector input_payload; ++}; ++ + struct skippy_session { + skippy_model * stage_model = nullptr; + llama_context * ctx = nullptr; +@@ -149,6 +160,7 @@ struct skippy_session { + bool mtp_has_pending_draft = false; + llama_pos mtp_pending_draft_pos = 0; + llama_token mtp_pending_draft_token = -1; ++ std::vector verify_checkpoints; + }; + + struct skippy_tensor_meta { +@@ -1500,6 +1512,72 @@ static bool skippy_has_activation_payload(const skippy_activation_desc * desc, c + return desc != nullptr && desc->payload_bytes > 0 && payload != nullptr; + } + ++static bool skippy_memory_needs_verify_checkpoint(llama_memory_t memory) { ++ return dynamic_cast(memory) != nullptr || ++ dynamic_cast(memory) != nullptr || ++ dynamic_cast(memory) != nullptr; ++} ++ ++static enum skippy_status skippy_checkpoint_verify_window( ++ skippy_session * session, ++ const llama_token * token_ids, ++ size_t token_count, ++ const skippy_activation_desc * input_desc, ++ const void * input_payload, ++ skippy_error ** out_error) { ++ llama_memory_t memory = session->ctx->get_memory(); ++ if (!skippy_memory_needs_verify_checkpoint(memory)) { ++ return skippy_success(out_error); ++ } ++ ++ session->ctx->synchronize(); ++ const size_t state_size = llama_state_seq_get_size_ext( ++ session->ctx, ++ session->seq_id, ++ LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); ++ if (state_size == 0) { ++ skippy_set_error(out_error, SKIPPY_STATUS_RUNTIME_ERROR, "failed to size verify-window state checkpoint"); ++ return SKIPPY_STATUS_RUNTIME_ERROR; ++ } ++ ++ skippy_verify_checkpoint checkpoint; ++ checkpoint.state.resize(state_size); ++ const size_t written = llama_state_seq_get_data_ext( ++ session->ctx, ++ checkpoint.state.data(), ++ checkpoint.state.size(), ++ session->seq_id, ++ LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); ++ if (written != checkpoint.state.size()) { ++ skippy_set_error(out_error, SKIPPY_STATUS_RUNTIME_ERROR, "failed to checkpoint verify-window sequence state"); ++ return SKIPPY_STATUS_RUNTIME_ERROR; ++ } ++ ++ checkpoint.valid = true; ++ checkpoint.token_start = session->n_past; ++ checkpoint.token_count = token_count; ++ if (token_ids != nullptr) { ++ checkpoint.token_ids.assign(token_ids, token_ids + token_count); ++ } ++ checkpoint.has_activation_input = skippy_has_activation_payload(input_desc, input_payload); ++ if (checkpoint.has_activation_input) { ++ checkpoint.input_desc = *input_desc; ++ const uint8_t * bytes = static_cast(input_payload); ++ checkpoint.input_payload.assign(bytes, bytes + input_desc->payload_bytes); ++ } ++ ++ constexpr size_t max_verify_checkpoints = 64; ++ if (session->verify_checkpoints.size() >= max_verify_checkpoints) { ++ skippy_set_error( ++ out_error, ++ SKIPPY_STATUS_UNSUPPORTED, ++ "verify-window pipeline depth exceeds supported checkpoint retention"); ++ return SKIPPY_STATUS_UNSUPPORTED; ++ } ++ session->verify_checkpoints.push_back(std::move(checkpoint)); ++ return skippy_success(out_error); ++} ++ + static enum skippy_status skippy_validate_frame_input_sequences( + skippy_session * session, + const skippy_activation_desc * input_desc, +@@ -3956,6 +4034,139 @@ static enum skippy_status skippy_verify_activation_frame( + return status; + } + ++static std::vector skippy_verify_prefix_activation_payload( ++ const skippy_session * session, ++ const skippy_verify_checkpoint & checkpoint, ++ size_t token_count) { ++ const uint64_t flags = checkpoint.input_desc.flags; ++ if ((flags & SKIPPY_ACTIVATION_FLAG_GEMMA3N_ALTUP) != 0) { ++ const size_t prefix_bytes = skippy_activation_payload_bytes(session, token_count, flags); ++ return std::vector( ++ checkpoint.input_payload.begin(), ++ checkpoint.input_payload.begin() + static_cast(prefix_bytes)); ++ } ++ ++ const size_t source_hidden_bytes = skippy_activation_hidden_bytes(session, checkpoint.token_count); ++ const size_t prefix_hidden_bytes = skippy_activation_hidden_bytes(session, token_count); ++ const uint32_t glm_dsa_top_k = skippy_glm_dsa_top_k_count_from_desc(session, &checkpoint.input_desc); ++ const size_t prefix_glm_dsa_bytes = (flags & SKIPPY_ACTIVATION_FLAG_GLM_DSA_TOP_K) != 0 ? ++ skippy_glm_dsa_top_k_bytes_for_count(token_count, glm_dsa_top_k) : 0; ++ const size_t prefix_rwkv7_bytes = (flags & SKIPPY_ACTIVATION_FLAG_RWKV7_V_FIRST) != 0 ? ++ prefix_hidden_bytes : 0; ++ std::vector payload(prefix_hidden_bytes + prefix_glm_dsa_bytes + prefix_rwkv7_bytes); ++ std::memcpy(payload.data(), checkpoint.input_payload.data(), prefix_hidden_bytes); ++ ++ size_t source_offset = source_hidden_bytes; ++ size_t prefix_offset = prefix_hidden_bytes; ++ if (prefix_glm_dsa_bytes > 0) { ++ std::memcpy( ++ payload.data() + prefix_offset, ++ checkpoint.input_payload.data() + source_offset, ++ prefix_glm_dsa_bytes); ++ source_offset += skippy_glm_dsa_top_k_bytes_for_count(checkpoint.token_count, glm_dsa_top_k); ++ prefix_offset += prefix_glm_dsa_bytes; ++ } ++ if (prefix_rwkv7_bytes > 0) { ++ std::memcpy( ++ payload.data() + prefix_offset, ++ checkpoint.input_payload.data() + source_offset, ++ prefix_rwkv7_bytes); ++ } ++ return payload; ++} ++ ++static bool skippy_trim_verify_attention( ++ llama_memory_t memory, ++ llama_seq_id seq_id, ++ llama_pos token_start) { ++ if (auto * hybrid = dynamic_cast(memory)) { ++ return hybrid->get_mem_attn()->seq_rm(seq_id, token_start, -1); ++ } ++ if (auto * hybrid_iswa = dynamic_cast(memory)) { ++ return hybrid_iswa->get_mem_attn()->seq_rm(seq_id, token_start, -1); ++ } ++ return true; ++} ++ ++static enum skippy_status skippy_restore_verify_prefix( ++ skippy_session * session, ++ uint64_t token_count, ++ bool * out_restored, ++ skippy_error ** out_error) { ++ *out_restored = false; ++ const skippy_verify_checkpoint * checkpoint = nullptr; ++ for (auto it = session->verify_checkpoints.rbegin(); it != session->verify_checkpoints.rend(); ++it) { ++ const uint64_t token_start = static_cast(it->token_start); ++ const uint64_t token_end = token_start + it->token_count; ++ if (it->valid && token_count >= token_start && token_count < token_end) { ++ checkpoint = &*it; ++ break; ++ } ++ } ++ if (checkpoint == nullptr) { ++ return skippy_success(out_error); ++ } ++ ++ const size_t read = llama_state_seq_set_data_ext( ++ session->ctx, ++ checkpoint->state.data(), ++ checkpoint->state.size(), ++ session->seq_id, ++ LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); ++ if (read != checkpoint->state.size()) { ++ skippy_set_error(out_error, SKIPPY_STATUS_RUNTIME_ERROR, "failed to restore verify-window sequence state"); ++ return SKIPPY_STATUS_RUNTIME_ERROR; ++ } ++ ++ llama_memory_t memory = session->ctx->get_memory(); ++ if (!skippy_trim_verify_attention(memory, session->seq_id, checkpoint->token_start)) { ++ skippy_set_error(out_error, SKIPPY_STATUS_RUNTIME_ERROR, "failed to trim verify-window attention suffix"); ++ return SKIPPY_STATUS_RUNTIME_ERROR; ++ } ++ ++ const uint64_t token_start = static_cast(checkpoint->token_start); ++ session->n_past = checkpoint->token_start; ++ const size_t accepted_count = static_cast(token_count - token_start); ++ if (accepted_count > 0 && checkpoint->token_ids.empty()) { ++ skippy_set_error(out_error, SKIPPY_STATUS_RUNTIME_ERROR, "verify-window checkpoint is missing token ids"); ++ return SKIPPY_STATUS_RUNTIME_ERROR; ++ } ++ ++ enum skippy_status status = SKIPPY_STATUS_OK; ++ if (accepted_count > 0 && checkpoint->has_activation_input) { ++ skippy_activation_desc input_desc = checkpoint->input_desc; ++ std::vector input_payload = skippy_verify_prefix_activation_payload( ++ session, ++ *checkpoint, ++ accepted_count); ++ input_desc.token_count = static_cast(accepted_count); ++ input_desc.payload_bytes = input_payload.size(); ++ status = skippy_decode_activation_frame( ++ session, ++ &input_desc, ++ input_payload.data(), ++ checkpoint->token_ids.data(), ++ nullptr, ++ 0, ++ accepted_count, ++ false, ++ SKIPPY_GLM_DSA_PHASE_HINT_VERIFY, ++ out_error); ++ } else if (accepted_count > 0) { ++ status = skippy_decode_tokens( ++ session, ++ checkpoint->token_ids.data(), ++ accepted_count, ++ false, ++ SKIPPY_GLM_DSA_PHASE_HINT_VERIFY, ++ out_error); ++ } ++ if (status == SKIPPY_STATUS_OK) { ++ *out_restored = true; ++ } ++ return status; ++} ++ + extern "C" { + + struct skippy_abi_version skippy_abi_version(void) { +@@ -4722,6 +4933,7 @@ enum skippy_status skippy_session_set_position( + session->signal_history.resize(static_cast(n_past)); + } + skippy_mtp_clear_session_state(session); ++ session->verify_checkpoints.clear(); + return skippy_success(out_error); + } + +@@ -4815,6 +5027,7 @@ enum skippy_status skippy_session_reset( + session->signal_history.clear(); + skippy_clear_chat_sampling(session); + skippy_mtp_clear_session_state(session); ++ session->verify_checkpoints.clear(); + session->ctx->synchronize(); + return skippy_success(out_error); + } +@@ -5143,7 +5356,17 @@ enum skippy_status skippy_verify_tokens( + return SKIPPY_STATUS_INVALID_ARGUMENT; + } + +- enum skippy_status status = skippy_verify_token_batch(session, token_ids, token_count, out_error); ++ enum skippy_status status = skippy_checkpoint_verify_window( ++ session, ++ token_ids, ++ token_count, ++ nullptr, ++ nullptr, ++ out_error); ++ if (status != SKIPPY_STATUS_OK) { ++ return status; ++ } ++ status = skippy_verify_token_batch(session, token_ids, token_count, out_error); + if (status == SKIPPY_STATUS_OK) { + const int32_t n_tokens = static_cast(token_count); + for (int32_t i = 0; i < n_tokens; ++i) { +@@ -5204,7 +5427,8 @@ static enum skippy_status skippy_prefill_chunk_frame_impl( + return status; + } + +- if (skippy_is_filtered(session) && session->stage_model->config.layer_start > 0) { ++ const bool activation_input = skippy_is_filtered(session) && session->stage_model->config.layer_start > 0; ++ if (activation_input) { + status = skippy_decode_activation_frame( + session, + input_desc, +@@ -5840,7 +6064,19 @@ enum skippy_status skippy_verify_tokens_frame_sampled( + } + } + +- if (skippy_is_filtered(session) && session->stage_model->config.layer_start > 0) { ++ const bool activation_input = skippy_is_filtered(session) && session->stage_model->config.layer_start > 0; ++ status = skippy_checkpoint_verify_window( ++ session, ++ token_ids, ++ token_count, ++ activation_input ? input_desc : nullptr, ++ activation_input ? input_payload : nullptr, ++ out_error); ++ if (status != SKIPPY_STATUS_OK) { ++ return status; ++ } ++ ++ if (activation_input) { + status = session->stage_model->config.include_output ? + skippy_verify_activation_frame(session, input_desc, input_payload, token_ids, token_count, out_error) : + skippy_decode_activation_frame( +@@ -5968,6 +6204,7 @@ static void skippy_update_session_state_after_import( + session->signal_history.resize(static_cast(session->n_past)); + } + skippy_mtp_clear_session_state(session); ++ session->verify_checkpoints.clear(); + } + + enum skippy_status skippy_export_state( +@@ -6386,6 +6623,7 @@ enum skippy_status skippy_import_kv_page( + desc->token_start + desc->token_count, + static_cast(std::numeric_limits::max())))); + skippy_mtp_clear_session_state(session); ++ session->verify_checkpoints.clear(); + session->ctx->synchronize(); + + return skippy_success(out_error); +@@ -6414,30 +6652,41 @@ enum skippy_status skippy_trim_session( + skippy_set_error(out_error, SKIPPY_STATUS_RUNTIME_ERROR, "runtime memory is unavailable"); + return SKIPPY_STATUS_RUNTIME_ERROR; + } +- const llama_pos p0 = static_cast(token_count); +- if (auto * hybrid = dynamic_cast(memory)) { +- if (!hybrid->seq_rm(session->seq_id, p0, -1)) { +- skippy_set_error(out_error, SKIPPY_STATUS_RUNTIME_ERROR, "failed to trim hybrid memory suffix"); +- return SKIPPY_STATUS_RUNTIME_ERROR; +- } +- } else if (auto * hybrid_iswa = dynamic_cast(memory)) { +- if (!hybrid_iswa->seq_rm(session->seq_id, p0, -1)) { +- skippy_set_error(out_error, SKIPPY_STATUS_RUNTIME_ERROR, "failed to trim hybrid ISWA memory suffix"); +- return SKIPPY_STATUS_RUNTIME_ERROR; +- } +- } else if (auto * dsa = dynamic_cast(memory)) { +- if (!dsa->seq_rm(session->seq_id, p0, -1)) { +- skippy_set_error(out_error, SKIPPY_STATUS_RUNTIME_ERROR, "failed to trim GLM-DSA KV suffix"); +- return SKIPPY_STATUS_RUNTIME_ERROR; +- } +- } else if (auto * kv = dynamic_cast(memory)) { +- if (!kv->seq_rm(session->seq_id, p0, -1)) { +- skippy_set_error(out_error, SKIPPY_STATUS_RUNTIME_ERROR, "failed to trim native KV suffix"); +- return SKIPPY_STATUS_RUNTIME_ERROR; ++ bool restored_verify_prefix = false; ++ enum skippy_status status = skippy_restore_verify_prefix( ++ session, ++ token_count, ++ &restored_verify_prefix, ++ out_error); ++ if (status != SKIPPY_STATUS_OK) { ++ return status; ++ } ++ if (!restored_verify_prefix) { ++ const llama_pos p0 = static_cast(token_count); ++ if (auto * hybrid = dynamic_cast(memory)) { ++ if (!hybrid->seq_rm(session->seq_id, p0, -1)) { ++ skippy_set_error(out_error, SKIPPY_STATUS_RUNTIME_ERROR, "failed to trim hybrid memory suffix"); ++ return SKIPPY_STATUS_RUNTIME_ERROR; ++ } ++ } else if (auto * hybrid_iswa = dynamic_cast(memory)) { ++ if (!hybrid_iswa->seq_rm(session->seq_id, p0, -1)) { ++ skippy_set_error(out_error, SKIPPY_STATUS_RUNTIME_ERROR, "failed to trim hybrid ISWA memory suffix"); ++ return SKIPPY_STATUS_RUNTIME_ERROR; ++ } ++ } else if (auto * dsa = dynamic_cast(memory)) { ++ if (!dsa->seq_rm(session->seq_id, p0, -1)) { ++ skippy_set_error(out_error, SKIPPY_STATUS_RUNTIME_ERROR, "failed to trim GLM-DSA KV suffix"); ++ return SKIPPY_STATUS_RUNTIME_ERROR; ++ } ++ } else if (auto * kv = dynamic_cast(memory)) { ++ if (!kv->seq_rm(session->seq_id, p0, -1)) { ++ skippy_set_error(out_error, SKIPPY_STATUS_RUNTIME_ERROR, "failed to trim native KV suffix"); ++ return SKIPPY_STATUS_RUNTIME_ERROR; ++ } ++ } else if (dynamic_cast(memory) == nullptr) { ++ skippy_set_error(out_error, SKIPPY_STATUS_UNSUPPORTED, "runtime memory type is not supported for trim"); ++ return SKIPPY_STATUS_UNSUPPORTED; + } +- } else if (dynamic_cast(memory) == nullptr) { +- skippy_set_error(out_error, SKIPPY_STATUS_UNSUPPORTED, "runtime memory type is not supported for trim"); +- return SKIPPY_STATUS_UNSUPPORTED; + } + session->n_past = static_cast(token_count); + if (session->token_history.size() > token_count) { +@@ -6448,6 +6697,7 @@ enum skippy_status skippy_trim_session( + session->signal_history.resize(static_cast(token_count)); + } + skippy_mtp_clear_session_state(session); ++ session->verify_checkpoints.clear(); + session->ctx->synchronize(); + + return skippy_success(out_error); +@@ -6600,6 +6850,7 @@ enum skippy_status skippy_session_restore_prefix( + } + session->signal_history.clear(); + skippy_mtp_clear_session_state(session); ++ session->verify_checkpoints.clear(); + session->ctx->synchronize(); + return skippy_success(out_error); + } +-- +2.50.1 (Apple Git-155) + From c21ccdd05406506b34c4bbea8eaf4ab9beaf3f2f Mon Sep 17 00:00:00 2001 From: Michael Neale Date: Fri, 24 Jul 2026 03:31:47 +1000 Subject: [PATCH 12/17] skippy: reclaim stage lanes after mid-request errors A generation that failed mid-decode on a split topology (e.g. a runtime trim error) tore down the binary stage connection before the graceful Stop message, so the stage never called drop_session_timed for that session. The leaked RuntimeState lane stayed 'active' forever; retried requests used fresh session ids, so leaked lanes accumulated until every admission wedged (EAGAIN fast-fails / reply timeouts) and only a process restart recovered. Track sessions created per binary connection and drop any that never saw a graceful Stop when the connection handler exits, returning their execution lanes to the pool so the next request gets a fresh lane. Assisted-by: goose --- .../binary_messaging/connection.rs | 153 ++++++++++++++++++ 1 file changed, 153 insertions(+) diff --git a/crates/skippy-server/src/binary_transport/binary_messaging/connection.rs b/crates/skippy-server/src/binary_transport/binary_messaging/connection.rs index e114fb17a8..e790cb2e99 100644 --- a/crates/skippy-server/src/binary_transport/binary_messaging/connection.rs +++ b/crates/skippy-server/src/binary_transport/binary_messaging/connection.rs @@ -72,8 +72,129 @@ use std::time::Instant; static BINARY_SESSION_COUNTER: AtomicU64 = AtomicU64::new(1); +/// Tracks runtime session keys created by one binary stage connection so +/// that a mid-request failure (connection error, protocol violation, +/// runtime error) releases the execution lanes those sessions hold. +/// +/// Without this, a generation that errors before the graceful `Stop` +/// message leaks its lane session in [`RuntimeState`]: each retried +/// request uses a fresh session id, so leaked lanes accumulate until +/// every admission fails with "all execution lanes are busy" and only a +/// process restart recovers. +#[derive(Default)] +pub(super) struct ConnectionSessionTracker { + active: std::collections::BTreeSet, +} + +impl ConnectionSessionTracker { + pub(super) fn touch(&mut self, session_key: &str) { + if !self.active.contains(session_key) { + self.active.insert(session_key.to_string()); + } + } + + pub(super) fn stopped(&mut self, session_key: &str) { + self.active.remove(session_key); + } + + pub(super) fn drain(&mut self) -> Vec { + std::mem::take(&mut self.active).into_iter().collect() + } +} + #[allow(clippy::too_many_arguments)] pub(super) fn handle_binary_connection( + config: &StageConfig, + topology: Option<&StageTopology>, + runtime: &Arc>, + decode_frame_batcher: &DecodeFrameBatcher, + kv: Option<&Arc>, + telemetry: &Telemetry, + upstream: &mut TcpStream, + downstream: Option, + activation_width: i32, + wire_dtype: WireActivationDType, + max_inflight: usize, + reply_credit_limit: Option, + async_prefill_forward: bool, + downstream_wire_condition: WireCondition, + downstream_connect_timeout_secs: u64, + native_mtp_enabled: bool, + prediction_return_sinks: &PredictionReturnSinks, + first_message: StageWireMessage, +) -> Result<()> { + let mut session_tracker = ConnectionSessionTracker::default(); + let result = handle_binary_connection_messages( + config, + topology, + runtime, + decode_frame_batcher, + kv, + telemetry, + upstream, + downstream, + activation_width, + wire_dtype, + max_inflight, + reply_credit_limit, + async_prefill_forward, + downstream_wire_condition, + downstream_connect_timeout_secs, + native_mtp_enabled, + prediction_return_sinks, + first_message, + &mut session_tracker, + ); + release_tracked_connection_sessions(config, runtime, telemetry, &mut session_tracker); + result +} + +/// Drops any runtime sessions this connection created but never stopped +/// gracefully, returning their execution lanes to the pool. +fn release_tracked_connection_sessions( + config: &StageConfig, + runtime: &Arc>, + telemetry: &Telemetry, + session_tracker: &mut ConnectionSessionTracker, +) { + let orphaned = session_tracker.drain(); + if orphaned.is_empty() { + return; + } + let Ok(mut runtime) = runtime.lock() else { + return; + }; + for session_key in orphaned { + match runtime.drop_session_timed(&session_key) { + Ok(drop_stats) => { + let mut attrs = crate::telemetry::lifecycle_attrs(config); + attrs.insert("llama_stage.session_key".to_string(), json!(session_key)); + attrs.insert( + "llama_stage.session_reset".to_string(), + json!(drop_stats.reset_session), + ); + attrs.insert( + "llama_stage.lane_discarded".to_string(), + json!(drop_stats.lane_discarded), + ); + insert_runtime_session_stats( + &mut attrs, + "llama_stage.runtime_sessions_after", + &drop_stats.stats_after, + ); + telemetry.emit("stage.binary_session_orphan_reclaimed", attrs); + } + Err(error) => { + eprintln!( + "failed to reclaim orphaned binary stage session {session_key}: {error:#}" + ); + } + } + } +} + +#[allow(clippy::too_many_arguments)] +fn handle_binary_connection_messages( config: &StageConfig, topology: Option<&StageTopology>, runtime: &Arc>, @@ -92,6 +213,7 @@ pub(super) fn handle_binary_connection( native_mtp_enabled: bool, prediction_return_sinks: &PredictionReturnSinks, first_message: StageWireMessage, + session_tracker: &mut ConnectionSessionTracker, ) -> Result<()> { if let Some(downstream) = downstream.as_mut() { send_client_ready_hello_if_enabled(&mut *downstream) @@ -146,6 +268,7 @@ pub(super) fn handle_binary_connection( let message_started = Instant::now(); let session_id = binary_message_session_id(connection_session_id, &message); let session_key = session_id.to_string(); + session_tracker.touch(&session_key); if message.kind == WireMessageKind::VerifyWindow && !positional_speculation_supported { bail!( "stage-state v10 positional speculation requires an attention-only stage; {} contains recurrent state", @@ -278,6 +401,7 @@ pub(super) fn handle_binary_connection( reset_start_unix_nanos, reset_end_unix_nanos, ); + session_tracker.stopped(&session_key); prediction_return_streams.remove(&(message.request_id, message.session_id)); prediction_return_sinks.remove(message.request_id, message.session_id); send_reply_ack_with_stats(&mut *upstream, stop_stats).context("send stop ACK")?; @@ -1293,3 +1417,32 @@ pub(super) fn handle_binary_connection( } } } + +#[cfg(test)] +mod tests { + use super::ConnectionSessionTracker; + + #[test] + fn tracker_drains_sessions_that_never_saw_a_stop() { + let mut tracker = ConnectionSessionTracker::default(); + tracker.touch("session-a"); + tracker.touch("session-a"); + tracker.touch("session-b"); + + // Simulate a mid-request stage error: session-a errored before its + // graceful Stop, session-b completed normally. + tracker.stopped("session-b"); + + assert_eq!(tracker.drain(), vec!["session-a".to_string()]); + // Idempotent: a second drain reclaims nothing. + assert!(tracker.drain().is_empty()); + } + + #[test] + fn tracker_reclaims_nothing_after_graceful_stop() { + let mut tracker = ConnectionSessionTracker::default(); + tracker.touch("session-a"); + tracker.stopped("session-a"); + assert!(tracker.drain().is_empty()); + } +} From 505cb97f51763bf46e00e9ef2f2dfa4e4c22ea9a Mon Sep 17 00:00:00 2001 From: Michael Neale <14976+michaelneale@users.noreply.github.com> Date: Tue, 28 Jul 2026 19:17:00 +1000 Subject: [PATCH 13/17] refactor(skippy): isolate binary connection session tracking Keep orphaned-lane reclamation separate from the already oversized binary connection message loop, with its focused unit tests beside the tracker. Assisted-by: codex --- .../src/binary_transport/binary_messaging.rs | 1 + .../binary_messaging/connection.rs | 104 +----------------- .../binary_messaging/session_tracker.rs | 102 +++++++++++++++++ 3 files changed, 104 insertions(+), 103 deletions(-) create mode 100644 crates/skippy-server/src/binary_transport/binary_messaging/session_tracker.rs diff --git a/crates/skippy-server/src/binary_transport/binary_messaging.rs b/crates/skippy-server/src/binary_transport/binary_messaging.rs index a9f1dcdf47..aa9df775b6 100644 --- a/crates/skippy-server/src/binary_transport/binary_messaging.rs +++ b/crates/skippy-server/src/binary_transport/binary_messaging.rs @@ -36,6 +36,7 @@ pub(in crate::binary_transport) mod async_forwarder; mod connection; mod prefill_recording; pub(in crate::binary_transport) mod reply; +mod session_tracker; mod summary; mod telemetry; diff --git a/crates/skippy-server/src/binary_transport/binary_messaging/connection.rs b/crates/skippy-server/src/binary_transport/binary_messaging/connection.rs index e790cb2e99..c6ba5a9f9d 100644 --- a/crates/skippy-server/src/binary_transport/binary_messaging/connection.rs +++ b/crates/skippy-server/src/binary_transport/binary_messaging/connection.rs @@ -2,6 +2,7 @@ use super::async_forwarder::AsyncForwarder; use super::reply::drain_deferred_prefill_replies; use super::reply::send_stage_reply; use super::reply::{configure_prediction_return_stream, reply_window_for_message}; +use super::session_tracker::{ConnectionSessionTracker, release_tracked_connection_sessions}; use super::summary::BinaryMessageObservation; use super::summary::BinaryRequestSummary; use super::telemetry::UpstreamReplyWriteSpan; @@ -72,36 +73,6 @@ use std::time::Instant; static BINARY_SESSION_COUNTER: AtomicU64 = AtomicU64::new(1); -/// Tracks runtime session keys created by one binary stage connection so -/// that a mid-request failure (connection error, protocol violation, -/// runtime error) releases the execution lanes those sessions hold. -/// -/// Without this, a generation that errors before the graceful `Stop` -/// message leaks its lane session in [`RuntimeState`]: each retried -/// request uses a fresh session id, so leaked lanes accumulate until -/// every admission fails with "all execution lanes are busy" and only a -/// process restart recovers. -#[derive(Default)] -pub(super) struct ConnectionSessionTracker { - active: std::collections::BTreeSet, -} - -impl ConnectionSessionTracker { - pub(super) fn touch(&mut self, session_key: &str) { - if !self.active.contains(session_key) { - self.active.insert(session_key.to_string()); - } - } - - pub(super) fn stopped(&mut self, session_key: &str) { - self.active.remove(session_key); - } - - pub(super) fn drain(&mut self) -> Vec { - std::mem::take(&mut self.active).into_iter().collect() - } -} - #[allow(clippy::too_many_arguments)] pub(super) fn handle_binary_connection( config: &StageConfig, @@ -149,50 +120,6 @@ pub(super) fn handle_binary_connection( result } -/// Drops any runtime sessions this connection created but never stopped -/// gracefully, returning their execution lanes to the pool. -fn release_tracked_connection_sessions( - config: &StageConfig, - runtime: &Arc>, - telemetry: &Telemetry, - session_tracker: &mut ConnectionSessionTracker, -) { - let orphaned = session_tracker.drain(); - if orphaned.is_empty() { - return; - } - let Ok(mut runtime) = runtime.lock() else { - return; - }; - for session_key in orphaned { - match runtime.drop_session_timed(&session_key) { - Ok(drop_stats) => { - let mut attrs = crate::telemetry::lifecycle_attrs(config); - attrs.insert("llama_stage.session_key".to_string(), json!(session_key)); - attrs.insert( - "llama_stage.session_reset".to_string(), - json!(drop_stats.reset_session), - ); - attrs.insert( - "llama_stage.lane_discarded".to_string(), - json!(drop_stats.lane_discarded), - ); - insert_runtime_session_stats( - &mut attrs, - "llama_stage.runtime_sessions_after", - &drop_stats.stats_after, - ); - telemetry.emit("stage.binary_session_orphan_reclaimed", attrs); - } - Err(error) => { - eprintln!( - "failed to reclaim orphaned binary stage session {session_key}: {error:#}" - ); - } - } - } -} - #[allow(clippy::too_many_arguments)] fn handle_binary_connection_messages( config: &StageConfig, @@ -1417,32 +1344,3 @@ fn handle_binary_connection_messages( } } } - -#[cfg(test)] -mod tests { - use super::ConnectionSessionTracker; - - #[test] - fn tracker_drains_sessions_that_never_saw_a_stop() { - let mut tracker = ConnectionSessionTracker::default(); - tracker.touch("session-a"); - tracker.touch("session-a"); - tracker.touch("session-b"); - - // Simulate a mid-request stage error: session-a errored before its - // graceful Stop, session-b completed normally. - tracker.stopped("session-b"); - - assert_eq!(tracker.drain(), vec!["session-a".to_string()]); - // Idempotent: a second drain reclaims nothing. - assert!(tracker.drain().is_empty()); - } - - #[test] - fn tracker_reclaims_nothing_after_graceful_stop() { - let mut tracker = ConnectionSessionTracker::default(); - tracker.touch("session-a"); - tracker.stopped("session-a"); - assert!(tracker.drain().is_empty()); - } -} diff --git a/crates/skippy-server/src/binary_transport/binary_messaging/session_tracker.rs b/crates/skippy-server/src/binary_transport/binary_messaging/session_tracker.rs new file mode 100644 index 0000000000..5f395e2d15 --- /dev/null +++ b/crates/skippy-server/src/binary_transport/binary_messaging/session_tracker.rs @@ -0,0 +1,102 @@ +use super::telemetry::insert_runtime_session_stats; +use crate::{ + runtime_state::RuntimeState, + telemetry::{Telemetry, lifecycle_attrs}, +}; +use serde_json::json; +use skippy_protocol::StageConfig; +use std::{ + collections::BTreeSet, + sync::{Arc, Mutex}, +}; + +/// Runtime session keys created by one binary stage connection. +/// +/// A connection that fails before its graceful `Stop` message would otherwise +/// leave those sessions holding execution lanes indefinitely. +#[derive(Default)] +pub(super) struct ConnectionSessionTracker { + active: BTreeSet, +} + +impl ConnectionSessionTracker { + pub(super) fn touch(&mut self, session_key: &str) { + self.active.insert(session_key.to_string()); + } + + pub(super) fn stopped(&mut self, session_key: &str) { + self.active.remove(session_key); + } + + fn drain(&mut self) -> Vec { + std::mem::take(&mut self.active).into_iter().collect() + } +} + +/// Returns lanes held by sessions that never reached a graceful `Stop`. +pub(super) fn release_tracked_connection_sessions( + config: &StageConfig, + runtime: &Arc>, + telemetry: &Telemetry, + session_tracker: &mut ConnectionSessionTracker, +) { + let orphaned = session_tracker.drain(); + if orphaned.is_empty() { + return; + } + let Ok(mut runtime) = runtime.lock() else { + return; + }; + for session_key in orphaned { + match runtime.drop_session_timed(&session_key) { + Ok(drop_stats) => { + let mut attrs = lifecycle_attrs(config); + attrs.insert("llama_stage.session_key".to_string(), json!(session_key)); + attrs.insert( + "llama_stage.session_reset".to_string(), + json!(drop_stats.reset_session), + ); + attrs.insert( + "llama_stage.lane_discarded".to_string(), + json!(drop_stats.lane_discarded), + ); + insert_runtime_session_stats( + &mut attrs, + "llama_stage.runtime_sessions_after", + &drop_stats.stats_after, + ); + telemetry.emit("stage.binary_session_orphan_reclaimed", attrs); + } + Err(error) => { + eprintln!( + "failed to reclaim orphaned binary stage session {session_key}: {error:#}" + ); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::ConnectionSessionTracker; + + #[test] + fn tracker_drains_sessions_that_never_saw_a_stop() { + let mut tracker = ConnectionSessionTracker::default(); + tracker.touch("session-a"); + tracker.touch("session-a"); + tracker.touch("session-b"); + tracker.stopped("session-b"); + + assert_eq!(tracker.drain(), vec!["session-a"]); + assert!(tracker.drain().is_empty()); + } + + #[test] + fn tracker_reclaims_nothing_after_graceful_stop() { + let mut tracker = ConnectionSessionTracker::default(); + tracker.touch("session-a"); + tracker.stopped("session-a"); + assert!(tracker.drain().is_empty()); + } +} From 1d3ee4a065d820603d5343abcefbde96182ea005 Mon Sep 17 00:00:00 2001 From: Michael Neale <14976+michaelneale@users.noreply.github.com> Date: Tue, 28 Jul 2026 19:51:52 +1000 Subject: [PATCH 14/17] skippy: trim interleaved-SWA KV sessions --- ...brid-verify-window-state-across-trim.patch | 54 ++++++++++++------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/third_party/llama.cpp/patches/0046-Preserve-hybrid-verify-window-state-across-trim.patch b/third_party/llama.cpp/patches/0046-Preserve-hybrid-verify-window-state-across-trim.patch index d483f50a33..1d9e09fdc3 100644 --- a/third_party/llama.cpp/patches/0046-Preserve-hybrid-verify-window-state-across-trim.patch +++ b/third_party/llama.cpp/patches/0046-Preserve-hybrid-verify-window-state-across-trim.patch @@ -1,7 +1,7 @@ -From 9ed92928e27968f8d6914e1b849566538d42af9a Mon Sep 17 00:00:00 2001 +From cf0c069381520c0098fbac525b0b95d1f73b6816 Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI Date: Tue, 28 Jul 2026 19:12:14 +1000 -Subject: [PATCH] Preserve hybrid verify-window state across trim +Subject: [PATCH 46/46] Preserve hybrid verify-window state across trim Checkpoint only partial recurrent state, trim hybrid attention directly, and replay the accepted prefix so pipelined speculative windows can recover without copying the full long-context attention cache. @@ -9,14 +9,22 @@ Based on the recurrent checkpoint work from the Inkling branch. Assisted-by: codex --- - src/skippy.cpp | 303 ++++++++++++++++++++++++++++++++++++++++++++----- - 1 file changed, 277 insertions(+), 26 deletions(-) + src/skippy.cpp | 312 ++++++++++++++++++++++++++++++++++++++++++++----- + 1 file changed, 286 insertions(+), 26 deletions(-) diff --git a/src/skippy.cpp b/src/skippy.cpp -index 47f0a549..18bcc92b 100644 +index 47f0a549..b68ab87b 100644 --- a/src/skippy.cpp +++ b/src/skippy.cpp -@@ -124,6 +124,17 @@ struct skippy_model { +@@ -9,6 +9,7 @@ + #include "llama-ext.h" + #include "llama-graph.h" + #include "llama-kv-cache.h" ++#include "llama-kv-cache-iswa.h" + #include "llama-kv-cache-dsa.h" + #include "llama-memory-hybrid.h" + #include "llama-memory-hybrid-iswa.h" +@@ -124,6 +125,17 @@ struct skippy_model { std::vector> lane_resident_prefix_tokens; }; @@ -34,7 +42,7 @@ index 47f0a549..18bcc92b 100644 struct skippy_session { skippy_model * stage_model = nullptr; llama_context * ctx = nullptr; -@@ -149,6 +160,7 @@ struct skippy_session { +@@ -149,6 +161,7 @@ struct skippy_session { bool mtp_has_pending_draft = false; llama_pos mtp_pending_draft_pos = 0; llama_token mtp_pending_draft_token = -1; @@ -42,7 +50,7 @@ index 47f0a549..18bcc92b 100644 }; struct skippy_tensor_meta { -@@ -1500,6 +1512,72 @@ static bool skippy_has_activation_payload(const skippy_activation_desc * desc, c +@@ -1500,6 +1513,72 @@ static bool skippy_has_activation_payload(const skippy_activation_desc * desc, c return desc != nullptr && desc->payload_bytes > 0 && payload != nullptr; } @@ -115,7 +123,7 @@ index 47f0a549..18bcc92b 100644 static enum skippy_status skippy_validate_frame_input_sequences( skippy_session * session, const skippy_activation_desc * input_desc, -@@ -3956,6 +4034,139 @@ static enum skippy_status skippy_verify_activation_frame( +@@ -3956,6 +4035,142 @@ static enum skippy_status skippy_verify_activation_frame( return status; } @@ -164,6 +172,9 @@ index 47f0a549..18bcc92b 100644 + llama_memory_t memory, + llama_seq_id seq_id, + llama_pos token_start) { ++ if (auto * iswa = dynamic_cast(memory)) { ++ return iswa->seq_rm(seq_id, token_start, -1); ++ } + if (auto * hybrid = dynamic_cast(memory)) { + return hybrid->get_mem_attn()->seq_rm(seq_id, token_start, -1); + } @@ -255,7 +266,7 @@ index 47f0a549..18bcc92b 100644 extern "C" { struct skippy_abi_version skippy_abi_version(void) { -@@ -4722,6 +4933,7 @@ enum skippy_status skippy_session_set_position( +@@ -4722,6 +4937,7 @@ enum skippy_status skippy_session_set_position( session->signal_history.resize(static_cast(n_past)); } skippy_mtp_clear_session_state(session); @@ -263,7 +274,7 @@ index 47f0a549..18bcc92b 100644 return skippy_success(out_error); } -@@ -4815,6 +5027,7 @@ enum skippy_status skippy_session_reset( +@@ -4815,6 +5031,7 @@ enum skippy_status skippy_session_reset( session->signal_history.clear(); skippy_clear_chat_sampling(session); skippy_mtp_clear_session_state(session); @@ -271,7 +282,7 @@ index 47f0a549..18bcc92b 100644 session->ctx->synchronize(); return skippy_success(out_error); } -@@ -5143,7 +5356,17 @@ enum skippy_status skippy_verify_tokens( +@@ -5143,7 +5360,17 @@ enum skippy_status skippy_verify_tokens( return SKIPPY_STATUS_INVALID_ARGUMENT; } @@ -290,7 +301,7 @@ index 47f0a549..18bcc92b 100644 if (status == SKIPPY_STATUS_OK) { const int32_t n_tokens = static_cast(token_count); for (int32_t i = 0; i < n_tokens; ++i) { -@@ -5204,7 +5427,8 @@ static enum skippy_status skippy_prefill_chunk_frame_impl( +@@ -5204,7 +5431,8 @@ static enum skippy_status skippy_prefill_chunk_frame_impl( return status; } @@ -300,7 +311,7 @@ index 47f0a549..18bcc92b 100644 status = skippy_decode_activation_frame( session, input_desc, -@@ -5840,7 +6064,19 @@ enum skippy_status skippy_verify_tokens_frame_sampled( +@@ -5840,7 +6068,19 @@ enum skippy_status skippy_verify_tokens_frame_sampled( } } @@ -321,7 +332,7 @@ index 47f0a549..18bcc92b 100644 status = session->stage_model->config.include_output ? skippy_verify_activation_frame(session, input_desc, input_payload, token_ids, token_count, out_error) : skippy_decode_activation_frame( -@@ -5968,6 +6204,7 @@ static void skippy_update_session_state_after_import( +@@ -5968,6 +6208,7 @@ static void skippy_update_session_state_after_import( session->signal_history.resize(static_cast(session->n_past)); } skippy_mtp_clear_session_state(session); @@ -329,7 +340,7 @@ index 47f0a549..18bcc92b 100644 } enum skippy_status skippy_export_state( -@@ -6386,6 +6623,7 @@ enum skippy_status skippy_import_kv_page( +@@ -6386,6 +6627,7 @@ enum skippy_status skippy_import_kv_page( desc->token_start + desc->token_count, static_cast(std::numeric_limits::max())))); skippy_mtp_clear_session_state(session); @@ -337,7 +348,7 @@ index 47f0a549..18bcc92b 100644 session->ctx->synchronize(); return skippy_success(out_error); -@@ -6414,30 +6652,41 @@ enum skippy_status skippy_trim_session( +@@ -6414,30 +6656,46 @@ enum skippy_status skippy_trim_session( skippy_set_error(out_error, SKIPPY_STATUS_RUNTIME_ERROR, "runtime memory is unavailable"); return SKIPPY_STATUS_RUNTIME_ERROR; } @@ -382,6 +393,11 @@ index 47f0a549..18bcc92b 100644 + skippy_set_error(out_error, SKIPPY_STATUS_RUNTIME_ERROR, "failed to trim hybrid ISWA memory suffix"); + return SKIPPY_STATUS_RUNTIME_ERROR; + } ++ } else if (auto * iswa = dynamic_cast(memory)) { ++ if (!iswa->seq_rm(session->seq_id, p0, -1)) { ++ skippy_set_error(out_error, SKIPPY_STATUS_RUNTIME_ERROR, "failed to trim interleaved-SWA KV suffix"); ++ return SKIPPY_STATUS_RUNTIME_ERROR; ++ } + } else if (auto * dsa = dynamic_cast(memory)) { + if (!dsa->seq_rm(session->seq_id, p0, -1)) { + skippy_set_error(out_error, SKIPPY_STATUS_RUNTIME_ERROR, "failed to trim GLM-DSA KV suffix"); @@ -402,7 +418,7 @@ index 47f0a549..18bcc92b 100644 } session->n_past = static_cast(token_count); if (session->token_history.size() > token_count) { -@@ -6448,6 +6697,7 @@ enum skippy_status skippy_trim_session( +@@ -6448,6 +6706,7 @@ enum skippy_status skippy_trim_session( session->signal_history.resize(static_cast(token_count)); } skippy_mtp_clear_session_state(session); @@ -410,7 +426,7 @@ index 47f0a549..18bcc92b 100644 session->ctx->synchronize(); return skippy_success(out_error); -@@ -6600,6 +6850,7 @@ enum skippy_status skippy_session_restore_prefix( +@@ -6600,6 +6859,7 @@ enum skippy_status skippy_session_restore_prefix( } session->signal_history.clear(); skippy_mtp_clear_session_state(session); From 28e66d1562cd0f5c6bcd3f7c6d064cfd703252c7 Mon Sep 17 00:00:00 2001 From: Michael Neale <14976+michaelneale@users.noreply.github.com> Date: Tue, 28 Jul 2026 20:35:27 +1000 Subject: [PATCH 15/17] fix(packaging): preserve source revisions --- crates/mesh-llm-commands/src/model_package.rs | 28 ++++++++++- crates/model-package/src/script.rs | 7 +++ .../src/scripts/split-model-job.sh | 21 ++++---- .../src/runner/prediction_return.rs | 49 +++++++++++++++++-- 4 files changed, 90 insertions(+), 15 deletions(-) diff --git a/crates/mesh-llm-commands/src/model_package.rs b/crates/mesh-llm-commands/src/model_package.rs index 9a9a6bd13c..baa9f7cdda 100644 --- a/crates/mesh-llm-commands/src/model_package.rs +++ b/crates/mesh-llm-commands/src/model_package.rs @@ -266,6 +266,7 @@ async fn run_list_quants( "{}", serde_json::to_string_pretty(&json!({ "sourceRepo": source_repo, + "sourceRevision": source_revision, "quants": quants, }))? ); @@ -283,13 +284,20 @@ async fn run_list_quants( eprintln!(); eprintln!("Specify one as a model ref, e.g.:"); eprintln!( - " mesh-llm models package {}:{}", - source_repo, quants[0].name + " mesh-llm models package {}", + source_quant_ref(source_repo, source_revision, &quants[0].name) ); Ok(()) } +fn source_quant_ref(source_repo: &str, source_revision: Option<&str>, quant: &str) -> String { + source_revision.map_or_else( + || format!("{source_repo}:{quant}"), + |revision| format!("{source_repo}@{revision}:{quant}"), + ) +} + fn print_quant_table(quants: &[DiscoveredQuant]) { // Find the longest name for alignment. let max_name = quants.iter().map(|q| q.name.len()).max().unwrap_or(0); @@ -642,4 +650,20 @@ mod tests { fn parse_timeout_mixed() { assert_eq!(parse_timeout("1h30m45s").unwrap(), 5445); } + + #[test] + fn source_quant_ref_preserves_revision() { + assert_eq!( + source_quant_ref("poolside/Laguna-S-2.1-GGUF", Some("abc123"), "Q4_K_M"), + "poolside/Laguna-S-2.1-GGUF@abc123:Q4_K_M" + ); + } + + #[test] + fn source_quant_ref_omits_absent_revision() { + assert_eq!( + source_quant_ref("poolside/Laguna-S-2.1-GGUF", None, "Q4_K_M"), + "poolside/Laguna-S-2.1-GGUF:Q4_K_M" + ); + } } diff --git a/crates/model-package/src/script.rs b/crates/model-package/src/script.rs index 40699d8310..7b795baddc 100644 --- a/crates/model-package/src/script.rs +++ b/crates/model-package/src/script.rs @@ -204,4 +204,11 @@ mod tests { assert!(EMBEDDED_SCRIPT.contains(r#"time "$SLICER" write-package "$WRITE_PACKAGE_INPUT""#)); assert!(!EMBEDDED_SCRIPT.contains(r#"time $SLICER write-package "$SOURCE_PATH""#)); } + + #[test] + fn embedded_script_preserves_catalog_source_revision() { + assert!(EMBEDDED_SCRIPT.contains(r#""source_revision": source_revision"#)); + assert!(EMBEDDED_SCRIPT.contains(r#"variants[variant_name]["source"] = source_entry"#)); + assert!(EMBEDDED_SCRIPT.contains(r#"existing_variant["source"] = source_entry"#)); + } } diff --git a/crates/model-package/src/scripts/split-model-job.sh b/crates/model-package/src/scripts/split-model-job.sh index 549dc7d36a..6b86099dc5 100755 --- a/crates/model-package/src/scripts/split-model-job.sh +++ b/crates/model-package/src/scripts/split-model-job.sh @@ -429,6 +429,13 @@ package_entry = { "type": "layer-package", "repo": target_repo, "layer_count": layer_count, + "source_revision": source_revision, +} + +source_entry = { + "repo": source_repo, + "file": source_file, + "revision": source_revision, } # Handle both dict-style and list-style variants @@ -439,14 +446,11 @@ if isinstance(variants, dict): packages = variants[variant_name].get("packages", []) packages = [p for p in packages if p.get("repo") != target_repo] packages.append(package_entry) + variants[variant_name]["source"] = source_entry variants[variant_name]["packages"] = packages else: variants[variant_name] = { - "source": { - "repo": source_repo, - "file": source_file, - "revision": source_revision, - }, + "source": source_entry, "curated": { "name": variant_name, "size": f"{layer_count} layers", @@ -466,14 +470,11 @@ else: packages = existing_variant.get("packages", []) packages = [p for p in packages if p.get("repo") != target_repo] packages.append(package_entry) + existing_variant["source"] = source_entry existing_variant["packages"] = packages else: variants.append({ - "source": { - "repo": source_repo, - "file": source_file, - "revision": source_revision, - }, + "source": source_entry, "curated": { "name": variant_name, "size": f"{layer_count} layers", diff --git a/crates/skippy-correctness/src/runner/prediction_return.rs b/crates/skippy-correctness/src/runner/prediction_return.rs index f759f6c1a5..c58f82236b 100644 --- a/crates/skippy-correctness/src/runner/prediction_return.rs +++ b/crates/skippy-correctness/src/runner/prediction_return.rs @@ -1,9 +1,9 @@ use std::{ env, io, io::Write, - net::{SocketAddr, TcpListener, TcpStream}, + net::{Shutdown, SocketAddr, TcpListener, TcpStream}, sync::{ - Arc, + Arc, Mutex, atomic::{AtomicBool, Ordering}, mpsc, }, @@ -24,6 +24,7 @@ pub(super) struct PredictionReturnListener { bind_addr: SocketAddr, receiver: mpsc::Receiver>, shutdown: Arc, + connection: Arc>>, thread: Option>, } @@ -39,9 +40,12 @@ impl PredictionReturnListener { .context("set correctness prediction return listener nonblocking")?; let shutdown = Arc::new(AtomicBool::new(false)); let thread_shutdown = shutdown.clone(); + let connection = Arc::new(Mutex::new(None)); + let thread_connection = connection.clone(); let (sender, receiver) = mpsc::channel(); let thread = thread::spawn(move || { - let result = accept_prediction_return(listener, &thread_shutdown, &sender); + let result = + accept_prediction_return(listener, &thread_shutdown, &thread_connection, &sender); if let Err(error) = result { let _ = sender.send(Err(format!("{error:#}"))); } @@ -50,6 +54,7 @@ impl PredictionReturnListener { bind_addr, receiver, shutdown, + connection, thread: Some(thread), }) } @@ -75,6 +80,11 @@ impl PredictionReturnListener { impl Drop for PredictionReturnListener { fn drop(&mut self) { self.shutdown.store(true, Ordering::SeqCst); + if let Ok(connection) = self.connection.lock() + && let Some(stream) = connection.as_ref() + { + let _ = stream.shutdown(Shutdown::Both); + } if let Some(thread) = self.thread.take() { let _ = thread.join(); } @@ -84,6 +94,7 @@ impl Drop for PredictionReturnListener { fn accept_prediction_return( listener: TcpListener, shutdown: &AtomicBool, + connection: &Mutex>, sender: &mpsc::Sender>, ) -> Result<()> { let mut stream = loop { @@ -102,6 +113,18 @@ fn accept_prediction_return( stream .set_nonblocking(false) .context("set direct prediction return stream blocking")?; + let shutdown_stream = stream + .try_clone() + .context("clone direct prediction return stream for shutdown")?; + { + let mut connection = connection + .lock() + .map_err(|_| anyhow!("direct prediction return connection lock poisoned"))?; + if shutdown.load(Ordering::SeqCst) { + return Ok(()); + } + *connection = Some(shutdown_stream); + } consume_optional_client_ready_hello(&mut stream)?; send_ready(&mut stream).context("send direct prediction return ready")?; stream.flush().ok(); @@ -205,4 +228,24 @@ mod tests { assert_eq!(reply.predicted, 674); client.join().unwrap(); } + + #[test] + fn drop_stops_when_connected_peer_stalls() { + let listener = PredictionReturnListener::start().unwrap(); + let address = listener + .endpoint() + .strip_prefix("tcp://") + .unwrap() + .to_string(); + let mut stream = TcpStream::connect(address).unwrap(); + if client_ready_hello_enabled() { + send_ready(&mut stream).unwrap(); + } + recv_ready(&mut stream).unwrap(); + + let started = std::time::Instant::now(); + drop(listener); + + assert!(started.elapsed() < Duration::from_secs(1)); + } } From 624fc7b97a0c1125fd8bfa7f9f0e93e1cf25b149 Mon Sep 17 00:00:00 2001 From: Michael Neale <14976+michaelneale@users.noreply.github.com> Date: Tue, 28 Jul 2026 21:47:31 +1000 Subject: [PATCH 16/17] feat(skippy): honor package verification depth --- .../skippy/resolver/native_mtp_tests.rs | 14 +++-- .../inference/skippy/resolver/speculative.rs | 2 +- crates/skippy-model-package/src/package.rs | 3 + crates/skippy-model-package/src/preflight.rs | 61 ++++++++++++++++++- crates/skippy-runtime/src/package.rs | 4 ++ .../reviewed-family-capabilities.json | 20 ++++++ crates/skippy-topology/src/tests.rs | 10 +++ docs/LAYER_PACKAGE_REPOS.md | 5 +- docs/skippy/FAMILY_STATUS.md | 2 +- docs/skippy/LLAMA_PARITY.md | 2 +- docs/skippy/SUFFIX_NGRAM_PROPOSER.md | 14 ++++- docs/skippy/llama-parity-candidates.json | 2 +- docs/specs/layer-package-repos.md | 7 ++- 13 files changed, 130 insertions(+), 16 deletions(-) diff --git a/crates/mesh-llm-host-runtime/src/inference/skippy/resolver/native_mtp_tests.rs b/crates/mesh-llm-host-runtime/src/inference/skippy/resolver/native_mtp_tests.rs index df1b58fe67..172acf54de 100644 --- a/crates/mesh-llm-host-runtime/src/inference/skippy/resolver/native_mtp_tests.rs +++ b/crates/mesh-llm-host-runtime/src/inference/skippy/resolver/native_mtp_tests.rs @@ -34,6 +34,7 @@ fn native_mtp_generation() -> PackageGenerationInfo { initial_window: 1, min_window: 1, max_window: 1, + pipeline_depth: None, }), proposer: Some("mtp".to_string()), primary: None, @@ -89,6 +90,7 @@ fn native_mtp_cache_generation() -> PackageGenerationInfo { initial_window: 2, min_window: 1, max_window: 6, + pipeline_depth: None, }), proposer: None, primary: Some("mtp".to_string()), @@ -131,6 +133,7 @@ fn ngram_cache_generation() -> PackageGenerationInfo { initial_window: 6, min_window: 1, max_window: 6, + pipeline_depth: None, }), proposer: Some("cache".to_string()), primary: None, @@ -173,6 +176,7 @@ fn ngram_suffix_generation() -> PackageGenerationInfo { initial_window: 32, min_window: 1, max_window: 32, + pipeline_depth: Some(2), }), proposer: Some("suffix".to_string()), primary: None, @@ -239,6 +243,7 @@ fn speculative_strategy_auto_detects_direct_gguf_native_mtp_tensors() { assert_eq!(resolved.speculative.strategy, "auto"); assert!(resolved.speculative.native_mtp_enabled); + assert_eq!(resolved.speculative.decode.verify_window.pipeline_depth, 1); let load_options = resolved .to_model_load_options(SkippyTelemetryOptions::off()) .expect("model load options should build"); @@ -451,12 +456,7 @@ strategy = "ngram-cache" #[test] fn package_suffix_strategy_resolves_as_a_standalone_proposer() { - let mesh_config = parse_config( - r#" -[defaults.speculative] -strategy = "ngram-suffix" -"#, - ); + let mesh_config = parse_config(""); let model_file = temp_model_file(); let generation = ngram_suffix_generation(); @@ -476,10 +476,12 @@ strategy = "ngram-suffix" resolved.speculative.decode.effective_strategy, "ngram-suffix" ); + assert_eq!(resolved.speculative.decode.verify_window.pipeline_depth, 2); let openai = resolved .to_embedded_openai_args(4096, true) .expect("package suffix strategy should build OpenAI args"); assert_eq!(openai.speculative_window, 48); + assert_eq!(openai.speculative.verify_window.pipeline_depth, 2); assert_eq!( openai.speculative.ngram.as_ref().map(|ngram| ngram.kind), Some(skippy_server::NgramProposerKind::Suffix) diff --git a/crates/mesh-llm-host-runtime/src/inference/skippy/resolver/speculative.rs b/crates/mesh-llm-host-runtime/src/inference/skippy/resolver/speculative.rs index 0320d219f5..dbacb3ee90 100644 --- a/crates/mesh-llm-host-runtime/src/inference/skippy/resolver/speculative.rs +++ b/crates/mesh-llm-host-runtime/src/inference/skippy/resolver/speculative.rs @@ -563,7 +563,7 @@ fn verify_window_config(policy: &PackageWindowPolicyInfo) -> VerifyWindowConfig VerifyWindowConfig { min_tokens: policy.min_window as usize, max_tokens: policy.max_window as usize, - pipeline_depth: 1, + pipeline_depth: policy.pipeline_depth.unwrap_or(1) as usize, } } diff --git a/crates/skippy-model-package/src/package.rs b/crates/skippy-model-package/src/package.rs index 49f954a1ff..5b67c5cd38 100644 --- a/crates/skippy-model-package/src/package.rs +++ b/crates/skippy-model-package/src/package.rs @@ -124,6 +124,8 @@ pub(crate) struct PackageWindowPolicy { pub(crate) initial_window: u32, pub(crate) min_window: u32, pub(crate) max_window: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) pipeline_depth: Option, } #[derive(Debug, Deserialize, Serialize)] @@ -663,6 +665,7 @@ pub(crate) fn package_generation(tensors: &[TensorInfo]) -> Option, } #[derive(Debug, Deserialize)] @@ -273,6 +275,8 @@ struct PackageWindowPolicy { initial_window: u32, min_window: u32, max_window: u32, + #[serde(default)] + pipeline_depth: Option, } #[derive(Debug, Deserialize)] @@ -1014,6 +1018,16 @@ fn validate_window_policy( "use positive window sizes", ); } + if window.pipeline_depth == Some(0) { + report.error( + "invalid_window_policy_pipeline_depth", + format!( + "speculative strategy {name} window_policy.pipeline_depth must be greater than zero" + ), + Some("model-package.json".to_string()), + "set pipeline_depth to a positive in-flight verification-window capacity", + ); + } if window.min_window > window.max_window { report.error( "invalid_window_policy_bounds", @@ -1135,6 +1149,7 @@ fn preflight_window_policy(window: &PackageWindowPolicy) -> PreflightWindowPolic initial_window: window.initial_window, min_window: window.min_window, max_window: window.max_window, + pipeline_depth: window.pipeline_depth, } } @@ -1735,7 +1750,8 @@ mod tests { "default": "fixed", "initial_window": 1, "min_window": 1, - "max_window": 1 + "max_window": 1, + "pipeline_depth": 2 } } } @@ -1770,6 +1786,7 @@ mod tests { assert_eq!(window_policy.initial_window, 1); assert_eq!(window_policy.min_window, 1); assert_eq!(window_policy.max_window, 1); + assert_eq!(window_policy.pipeline_depth, Some(2)); fs::remove_dir_all(dir).unwrap(); } @@ -2104,6 +2121,48 @@ mod tests { fs::remove_dir_all(dir).unwrap(); } + #[test] + fn preflight_rejects_zero_verify_window_pipeline_depth() { + let dir = unique_test_dir("zero-window-pipeline-depth"); + let package = write_package_fixture(&dir, true); + write_generation_to_manifest( + &package, + serde_json::json!({ + "speculative_decoding": { + "default": "ngram-suffix", + "proposers": { + "suffix": { + "type": "ngram-suffix", + "ngram_min": 5, + "ngram_max": 32, + "max_proposal_tokens": 48, + "history_scope": "request" + } + }, + "strategies": { + "ngram-suffix": { + "type": "ngram-suffix", + "proposer": "suffix", + "window_policy": { + "default": "fixed", + "initial_window": 32, + "min_window": 1, + "max_window": 32, + "pipeline_depth": 0 + } + } + } + } + }), + ); + + let report = preflight_package(&package, &PackagePreflightOptions::default()); + + assert!(!report.valid); + assert_issue(&report, "invalid_window_policy_pipeline_depth"); + fs::remove_dir_all(dir).unwrap(); + } + fn assert_issue(report: &PackagePreflightReport, code: &str) { assert!( report.issues.iter().any(|issue| issue.code == code), diff --git a/crates/skippy-runtime/src/package.rs b/crates/skippy-runtime/src/package.rs index 69a13be16b..efbcc6eea2 100644 --- a/crates/skippy-runtime/src/package.rs +++ b/crates/skippy-runtime/src/package.rs @@ -104,6 +104,7 @@ pub struct PackageWindowPolicyInfo { pub initial_window: u32, pub min_window: u32, pub max_window: u32, + pub pipeline_depth: Option, } #[derive(Debug, Clone)] @@ -288,6 +289,8 @@ struct PackageWindowPolicy { initial_window: u32, min_window: u32, max_window: u32, + #[serde(default)] + pipeline_depth: Option, } #[derive(Debug, Deserialize)] @@ -651,6 +654,7 @@ fn package_speculative_strategy_info( initial_window: window.initial_window, min_window: window.min_window, max_window: window.max_window, + pipeline_depth: window.pipeline_depth, }), proposer: strategy.proposer, primary: strategy.primary, diff --git a/crates/skippy-topology/capabilities/reviewed-family-capabilities.json b/crates/skippy-topology/capabilities/reviewed-family-capabilities.json index 35e5cabfae..823a5a540f 100644 --- a/crates/skippy-topology/capabilities/reviewed-family-capabilities.json +++ b/crates/skippy-topology/capabilities/reviewed-family-capabilities.json @@ -2070,5 +2070,25 @@ "split_constraints": [], "sidebands": [] } + }, + { + "model_id": "poolside/Laguna-S-2.1-GGUF:Q4_K_M", + "source_repo": "poolside/Laguna-S-2.1-GGUF", + "source_revision": "edd093522473dc7313b0738d8b4116b7f8b9745f", + "source_file": "laguna-s-2.1-Q4_K_M.gguf", + "canonical_ref": "poolside/Laguna-S-2.1-GGUF@edd093522473dc7313b0738d8b4116b7f8b9745f/laguna-s-2.1-Q4_K_M.gguf", + "distribution_id": "laguna-s-2.1-Q4_K_M", + "selector": "Q4_K_M", + "capability": { + "family_id": "laguna", + "layer_count": 48, + "activation_width": 3072, + "default_wire_dtype": "f16", + "q8_wire_validation": "untested", + "exact_state_mobility": "untested", + "recurrent_ranges": [], + "split_constraints": [], + "sidebands": [] + } } ] diff --git a/crates/skippy-topology/src/tests.rs b/crates/skippy-topology/src/tests.rs index 39fbbfd2c6..1c49f50f95 100644 --- a/crates/skippy-topology/src/tests.rs +++ b/crates/skippy-topology/src/tests.rs @@ -921,6 +921,16 @@ fn infers_known_family_capabilities_from_model_identity() { assert_eq!(llama.q8_wire_validation, WireValidation::Validated); assert_eq!(llama.exact_state_mobility, ExactStateMobility::Accepted); + let laguna = infer_family_capability( + "poolside/Laguna-S-2.1-GGUF@edd093522473dc7313b0738d8b4116b7f8b9745f/laguna-s-2.1-Q4_K_M.gguf", + 48, + 3072, + ) + .expect("reviewed Poolside Laguna S 2.1 Q4_K_M"); + assert_eq!(laguna.family_id, "laguna"); + assert_eq!(laguna.default_wire_dtype, WireDType::F16); + assert_eq!(laguna.q8_wire_validation, WireValidation::Untested); + let gemma4_e4b = infer_family_capability( "unsloth/gemma-4-E4B-it-GGUF@315e03409eb1cdde302488d66e586dea1e82aad1/gemma-4-E4B-it-Q4_K_M.gguf", 42, diff --git a/docs/LAYER_PACKAGE_REPOS.md b/docs/LAYER_PACKAGE_REPOS.md index 73411f2643..f1ba583e8b 100644 --- a/docs/LAYER_PACKAGE_REPOS.md +++ b/docs/LAYER_PACKAGE_REPOS.md @@ -96,7 +96,8 @@ Minimal GLM-DSA shape: "default": "fixed", "initial_window": 1, "min_window": 1, - "max_window": 1 + "max_window": 1, + "pipeline_depth": 1 } } } @@ -112,7 +113,7 @@ Authoring rule of thumb: | `generation.policy` | Stable semantic execution choices validated for the package. | `profile`, `decode`, `short_prefill`, `long_prefill`, `verify`, `indexshare` | | `generation.policy.experimental` | Named opt-in paths that need package/backend evidence before becoming defaults. | `selected_row_flash`, `moe_weighted_down`, `moe_merged_shared_gate_up` | | `generation.thresholds` | Numeric resolver inputs used to accept, reject, or fall back from a policy. | `short_prefill_max_tokens`, `compact_flash_min_kv`, `dense_mask_max_bytes` | -| `generation.speculative_decoding` | Package-owned native or draft speculation strategy defaults. | `native-mtp-n1`, `prediction_depth`, `layer_indices`, `window_policy` | +| `generation.speculative_decoding` | Package-owned native, N-gram, or draft speculation strategy defaults. | strategy id, proposer bounds, `window_policy`, optional positive `pipeline_depth` | | GGUF metadata | Architecture correctness and tensor layout requirements. | GLM-DSA q/k/v split dimensions, IndexShare roles, MTP tensor presence | Writers should emit a profile only after the artifact actually matches that diff --git a/docs/skippy/FAMILY_STATUS.md b/docs/skippy/FAMILY_STATUS.md index 03fdb5b399..8ef5c9c683 100644 --- a/docs/skippy/FAMILY_STATUS.md +++ b/docs/skippy/FAMILY_STATUS.md @@ -94,7 +94,7 @@ smoke, reviewed topology records, and family-specific policy notes are updated. ```text Gemma text -Laguna S 2.1 (package-backed M5 two-stage single-step and three-stage chain parity, plus ordinary M5 Metal 0..36 + Vast CUDA 36..48 Mesh serving at configured context 262144 with a 44,460-token prompt; broader certification pending) +Laguna S 2.1 Q4_K_M (pinned package-backed M5 two-stage single-step and three-stage chain parity, plus ordinary M5 Metal 0..36 + Australian Vast CUDA 36..48 Mesh serving with a 44,460-token prompt; suffix N-gram depth 2 is published as the package default, with a fresh live no-override confirmation still pending; broader family certification remains pending) ``` ## Exceptions diff --git a/docs/skippy/LLAMA_PARITY.md b/docs/skippy/LLAMA_PARITY.md index e5a77050bd..4804ac475c 100644 --- a/docs/skippy/LLAMA_PARITY.md +++ b/docs/skippy/LLAMA_PARITY.md @@ -371,7 +371,7 @@ Rows with distributed evidence call out the second backend explicitly. | `deepseek` | `Morgen0052/deepseek-llm-7b-chat-Q4_K_M-GGUF` | `single-step`, `chain`, and f16 dtype matrix passed | rejected | accepted | `ResidentKv` borrowed-hit smoke passed, 64-token prefix, 1.58x cache-hit speedup | | `openai_moe` | `ggml-org/gpt-oss-20b-GGUF:gpt-oss-20b-mxfp4` | `single-step`, `chain`, and dtype matrix passed | rejected | accepted | `ResidentKv` state handoff passed; llama.cpp model file is `openai-moe`, GGUF architecture is `gpt-oss` | | `ernie4_5_moe` | `lmstudio-community/ERNIE-4.5-21B-A3B-PT-GGUF:Q4_K_M` | `single-step`, `chain`, and dtype matrix passed | validated | accepted | `ResidentKv` state handoff passed | -| `laguna` | `meshllm/laguna-s-2.1-Q4_K_M-layers@797f31cc813c0d643b7205118a901732f39de8ab` | package-backed `single-step` passed at split `24`; three-stage `chain` passed at splits `16,32`; ordinary two-node Mesh OpenAI serving passed at `0..36 / 36..48` with configured context `262144` | untested | untested | M5 Max f16 parity passed against pinned full-model revision `edd093522473dc7313b0738d8b4116b7f8b9745f`: baseline token `674` matched both two-stage and three-stage predictions. The real Mesh run used M5 Max/Metal plus an Australian Vast RTX 6000 Ada/CUDA worker, normal planning and lifecycle, and a direct Iroh path with observed RTT samples from 24 ms to 320 ms. `doctor split` reported ready with no blockers. A 44,460-token prompt plus deterministic 65-token completion passed with exact output agreement. Suffix N-gram accepted 32 of 64 speculative tokens on the repeat/copy workload and raised decode from 5.72 to 9.61 tok/s; a 56-token low-overlap control produced no proposals. Exact-prefix reuse did not occur (`cached_tokens=0`), leaving streaming TTFT near 190 seconds. Equal `24/24` placement at full context exhausted the 48 GB CUDA worker; the working capacity-bounded topology was `36/12`. Dtype matrix, state/cache, tool-loop, and larger-context soak evidence remain pending. | +| `laguna` | `meshllm/laguna-s-2.1-Q4_K_M-layers@0c467ad441ee94cb5a76f626294d963c4048507d` | package-backed `single-step` passed at split `24`; three-stage `chain` passed at splits `16,32`; ordinary two-node Mesh OpenAI serving passed at `0..36 / 36..48` with configured context `262144` | untested | untested | M5 Max f16 parity passed against pinned full-model revision `edd093522473dc7313b0738d8b4116b7f8b9745f`: baseline token `674` matched both two-stage and three-stage predictions. The real Mesh run used M5 Max/Metal plus an Australian Vast RTX 6000 Ada/CUDA worker, normal planning and lifecycle, and direct Iroh transport. A final three-request recovery probe used a 44,460-token prompt plus deterministic 65-token completion and returned byte-identical content on every request; decode measured 19.74, 18.24, and 15.42 tok/s. Each request proposed 64 suffix N-gram tokens and accepted 32 at verify pipeline depth `2`; the exact-prefix hit reused 44,416 prompt tokens. A 63-request stability harness returned 63 HTTP 200 responses with no unsupported-trim, all-lanes-busy, reset, decode, slot, or proactive-eviction fatal errors. Structured tool use remains uncertified: the formal harness passed 6/13 checks because textual `` output was not consistently projected into OpenAI `tool_calls`. Package manifest SHA-256 `0250cfb54ceeb94a9c71e48df447f780e32fc625553844d6403770f315be0237` now publishes suffix N-gram (`min=5`, `max=32`, proposal cap `48`) and verify pipeline depth `2` as the default. Resolver coverage proves a no-config launch consumes that package policy; a fresh live M5 + Vast no-override confirmation remains pending. Equal `24/24` placement at full context exhausted the 48 GB CUDA worker; the working capacity-bounded topology was `36/12`. Q8 wire, state handoff, structured tool use, and full native-context saturation remain unproven. | | `llama4` | `ggml-org/Llama-4-Scout-17B-16E-Instruct-GGUF:Q4_K_M` | package validated | untested | untested | package-only validation passed: 48 layers, 627 owned tensors, 51 artifacts, no missing/duplicate tensors | | `mistral4` | `bartowski/mistralai_Mistral-Small-4-119B-2603-GGUF:IQ2_XXS` | package validated | untested | untested | package-only validation passed: 36 layers, 579 tensors, 39 artifacts, no missing/duplicate tensors | | `nemotron_h_moe` | `lmstudio-community/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-GGUF:Q4_K_M` | package validated | untested | rejected-too-large | package-only validation passed: 52 layers, 401 tensors, 55 artifacts; `KvRecurrent` target | diff --git a/docs/skippy/SUFFIX_NGRAM_PROPOSER.md b/docs/skippy/SUFFIX_NGRAM_PROPOSER.md index ba1bbd7fc4..396d84d964 100644 --- a/docs/skippy/SUFFIX_NGRAM_PROPOSER.md +++ b/docs/skippy/SUFFIX_NGRAM_PROPOSER.md @@ -165,7 +165,14 @@ strategy types. A suffix package proposer must declare request-local history: "strategies": { "ngram-suffix": { "type": "ngram-suffix", - "proposer": "suffix" + "proposer": "suffix", + "window_policy": { + "default": "fixed", + "initial_window": 32, + "min_window": 1, + "max_window": 32, + "pipeline_depth": 2 + } } } } @@ -173,6 +180,11 @@ strategy types. A suffix package proposer must declare request-local history: } ``` +`window_policy.pipeline_depth` is optional and defaults to `1` for older +packages. A package may select a deeper verification pipeline only after the +specific artifact and serving topology have passed workload and stability +gates at that depth. + The cache and suffix limits intentionally differ. Cache uses llama.cpp's stateful lookup with a match window no larger than four tokens. Suffix may use a much longer exact match. For both proposers, proposal output length is diff --git a/docs/skippy/llama-parity-candidates.json b/docs/skippy/llama-parity-candidates.json index 954ea5317b..9b027c6c97 100644 --- a/docs/skippy/llama-parity-candidates.json +++ b/docs/skippy/llama-parity-candidates.json @@ -191,7 +191,7 @@ "status": "certified_package_only", "repo": "poolside/Laguna-S-2.1-GGUF", "include": "laguna-s-2.1-Q4_K_M.gguf", - "notes": "package-only certification status retained while broader gates remain pending; source revision edd093522473dc7313b0738d8b4116b7f8b9745f (SHA-256 a34c74e46688122bef83122f4133031bababbefcf57436dde97048c91e2cc6ff) and package revision 797f31cc813c0d643b7205118a901732f39de8ab passed M5 Max package-backed single-step parity at split 24 and three-stage chain parity at splits 16,32 with f16 activation transport: baseline token 674 matched both staged predictions, activation width 3072, payload 12288 bytes, wire payload 6144 bytes; an ordinary M5 Metal plus Vast RTX 6000 Ada CUDA private Mesh run passed normal 0..36/36..48 placement, direct Iroh transport, doctor readiness, configured context 262144, a 44460-token prompt, exact 65-token completion agreement, and suffix N-gram verification with 32/64 speculative tokens accepted; the 24/24 full-context CUDA stage exhausted 48 GB, exact-prefix reuse reported zero cached tokens, and dtype matrix, state/cache, tool-loop, and larger-context soak evidence remain pending" + "notes": "package-only certification status retained while broader gates remain pending; source revision edd093522473dc7313b0738d8b4116b7f8b9745f (SHA-256 a34c74e46688122bef83122f4133031bababbefcf57436dde97048c91e2cc6ff) and package revision 0c467ad441ee94cb5a76f626294d963c4048507d (manifest SHA-256 0250cfb54ceeb94a9c71e48df447f780e32fc625553844d6403770f315be0237) passed M5 Max package-backed single-step parity at split 24 and three-stage chain parity at splits 16,32 with f16 activation transport: baseline token 674 matched both staged predictions, activation width 3072, payload 12288 bytes, wire payload 6144 bytes; an ordinary M5 Metal plus Australian Vast RTX 6000 Ada CUDA private Mesh run passed normal 0..36/36..48 placement, direct Iroh transport, configured context 262144, a 44460-token prompt, exact 65-token completion agreement, and suffix N-gram verification at depth 2 with 32/64 speculative tokens accepted per request; the final three-request recovery probe decoded at 19.74, 18.24, and 15.42 tok/s and an exact-prefix hit reused 44416 prompt tokens; 63/63 stability requests returned HTTP 200; suffix N-gram min 5, max 32, proposal cap 48, and verify pipeline depth 2 are now the package default and no-config resolver coverage consumes that policy, while a fresh live M5 plus Vast no-override confirmation remains pending; 24/24 full-context placement exhausted the 48 GB CUDA worker, the structured tool harness passed 6/13, and Q8 wire, state handoff, structured tool use, and full native-context saturation remain unproven" }, { "llama_model": "mistral3", diff --git a/docs/specs/layer-package-repos.md b/docs/specs/layer-package-repos.md index 4f4b0beaf1..03e6a92cd3 100644 --- a/docs/specs/layer-package-repos.md +++ b/docs/specs/layer-package-repos.md @@ -214,7 +214,8 @@ Minimal shape: "default": "fixed", "initial_window": 1, "min_window": 1, - "max_window": 1 + "max_window": 1, + "pipeline_depth": 1 } } } @@ -688,7 +689,8 @@ The current native MTP strategy shape is: "default": "fixed", "initial_window": 1, "min_window": 1, - "max_window": 1 + "max_window": 1, + "pipeline_depth": 1 } } ``` @@ -746,6 +748,7 @@ The package schema separates a proposer match length from its output budget: | `ngram_min` / `ngram_max` | N-gram proposers | Define the historical token match range. Both are required and `ngram_min <= ngram_max`. | | `max_proposal_tokens` | N-gram proposers | Caps how many continuation tokens the proposer may return. It is independent of `ngram_max`. | | `history_scope` | `ngram-cache`, `ngram-suffix` | Must be `"request"`; a history proposer never observes another request's tokens. | +| `window_policy.pipeline_depth` | All strategies | Optional positive per-request capacity for in-flight verification windows. Omission preserves the legacy depth of `1`; package defaults above `1` require topology/workload-specific evidence. | | `initial_tokens` / `max_tokens` | composite extension policy | Bound the adaptive N-gram tail after an MTP prefix. | | `tail_backoff_proposals` | composite extension policy | Sets how many proposals to back off after an unhelpful tail. | From 6ee85338063f3968285777064360b0cce31bfb97 Mon Sep 17 00:00:00 2001 From: Michael Neale <14976+michaelneale@users.noreply.github.com> Date: Wed, 29 Jul 2026 13:16:33 +1000 Subject: [PATCH 17/17] test(skippy): assert Laguna cache policy --- .../mesh-llm-host-runtime/src/inference/skippy/family_policy.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/mesh-llm-host-runtime/src/inference/skippy/family_policy.rs b/crates/mesh-llm-host-runtime/src/inference/skippy/family_policy.rs index a65d0019aa..d5483920eb 100644 --- a/crates/mesh-llm-host-runtime/src/inference/skippy/family_policy.rs +++ b/crates/mesh-llm-host-runtime/src/inference/skippy/family_policy.rs @@ -678,7 +678,7 @@ mod tests { | "minicpm3" | "plamo" | "plamo3" | "plm" | "refact" | "smallthinker" | "smollm3" | "arcee" | "chatglm" | "codeshell" | "deci" | "xverse" | "apertus" | "bitnet" | "command_r" | "starcoder" | "ernie4_5" | "ernie4_5_moe" | "qwen" - | "jais" | "jais2" | "nemotron" | "llama4" | "mistral4" | "seed_oss" => { + | "jais" | "jais2" | "nemotron" | "llama4" | "mistral4" | "seed_oss" | "laguna" => { assert_eq!( policy.prefix_cache, FamilyPrefixCachePolicy::Auto {