From 6dd31f0dc04caef5a7217e82fde2c27bf8ba9f7a Mon Sep 17 00:00:00 2001 From: hardcoreerik Date: Tue, 14 Jul 2026 14:13:30 -0700 Subject: [PATCH 1/3] Fix NoKvSlot/OOM crash on recurrent-architecture models (Qwen3.5) ModelParams.SeqMax defaulted to 1, which plain-transformer llama.cpp paths tolerated but recurrent/hybrid architectures (Qwen3.5's Gated Delta Net layers) validate strictly -- every second-ever conversation on a role's executor failed find_slot/init_batch outright, 100% reproducible. Wiring SeqMax to AdapterManager.SequenceHardLimit fixes that, but the old hard limit (240, calibrated when native n_seq_max was effectively unlimited) turned into a live ~12GB rs-cache VRAM reservation for hybrid models and OOM-crashed the process instead. Lowered the limit to 40, keeping the reservation affordable (~2GB) while preserving recycle-threshold headroom. Verified live against Qwen3.5-9B-Q8_0 on a 16GB GPU: zero NoKvSlot errors, zero OOM crashes, full CF-7 smoke run completes in the expected timeframe (was previously either recycle-storming on every call or crashing native with 0xC0000005). --- .../Core/Runtime/AdapterManager.cs | 20 ++++++++++++++++--- .../Core/Runtime/LLamaSharpRuntime.cs | 10 ++++++++++ 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/OrchestratorIDE/Core/Runtime/AdapterManager.cs b/OrchestratorIDE/Core/Runtime/AdapterManager.cs index b530b98b..1cbf7384 100644 --- a/OrchestratorIDE/Core/Runtime/AdapterManager.cs +++ b/OrchestratorIDE/Core/Runtime/AdapterManager.cs @@ -68,9 +68,23 @@ public sealed class AdapterManager : IAsyncDisposable // Absolute refusal point: if outstanding conversations have kept the executor from recycling // and it is now approaching the native slot cap, minting another conversation would trade a - // recoverable managed exception for a process-killing native assert. Below 256 so the throw - // always wins the race against the assert. - internal const int SequenceHardLimit = 240; + // recoverable managed exception for a process-killing native assert. + // + // Also used directly as ModelParams.SeqMax (LLamaSharpRuntime.cs) — the default n_seq_max=1 + // means the second-ever conversation on a fresh executor fails find_slot/init_batch outright + // on recurrent/hybrid architectures (e.g. Qwen3.5's Gated Delta Net layers validate seq_id + // strictly against n_seq_max; plain-transformer llama.cpp paths happened to tolerate seq_id + // >= n_seq_max, which is why this was never noticed before). That makes this constant a + // direct native VRAM reservation, not just a managed counter: llama.cpp allocates a + // per-sequence recurrent-state ("rs cache") buffer sized by n_seq_max on hybrid models. + // Confirmed live on a 16GB GPU with Qwen3.5-9B-Q8_0 (8.86GB weights): SeqMax=240 reserved + // ~12GB for the rs cache alone (~50MB/slot) and OOM-crashed the native process + // (cudaMalloc failed, 0xC0000005) before the managed hard-limit check ever got a chance to + // throw. Lowered from 240 to keep the rs-cache reservation (~2GB at this value) affordable + // on consumer GPUs while still giving SequenceRecycleThreshold plenty of grace for the + // active-conversations-outstanding path below. Do not raise this without checking rs-cache + // size against available VRAM for the largest hybrid-architecture model in use. + internal const int SequenceHardLimit = 40; // Opt-in, zero-cost-by-default diagnostic for the open KV-cache exhaustion investigation // (docs/CONTEXT_FABRIC_TEST_HARNESS.md §7): a threshold change alone was tried and had no diff --git a/OrchestratorIDE/Core/Runtime/LLamaSharpRuntime.cs b/OrchestratorIDE/Core/Runtime/LLamaSharpRuntime.cs index a8f00a49..b663ef24 100644 --- a/OrchestratorIDE/Core/Runtime/LLamaSharpRuntime.cs +++ b/OrchestratorIDE/Core/Runtime/LLamaSharpRuntime.cs @@ -228,6 +228,16 @@ public async Task LoadModelAsync( { ContextSize = (uint)_options.ContextLength, GpuLayerCount = _options.GpuLayers, + // Default n_seq_max is 1. AdapterManager mints a fresh, never-recycled sequence + // id per conversation (see AdapterManager.SequenceHardLimit) and only tears down + // the executor once minted count approaches that cap -- an assumption that only + // held by accident for plain-transformer architectures, where llama.cpp tolerates + // seq_id >= n_seq_max in the unified KV-cache path. Recurrent/hybrid architectures + // (e.g. Qwen3.5's Gated Delta Net layers) validate seq_id strictly against + // n_seq_max and fail find_slot/init_batch on literally the second conversation + // (seq_id=1) otherwise -- 100% reproducible, not load-dependent. Matching SeqMax to + // the hard limit makes the recycle contract hold for every architecture. + SeqMax = (uint)AdapterManager.SequenceHardLimit, // SwaFull deliberately left at its native default (true). false was tried during // the NoKvSlot investigation to shrink SWA-layer cache 6x on Gemma-3-class // architectures, but that shrinks the SWA cache to min(fullContext, n_swa + From e03f8240ca0d70e396bc29d0a6cc2172b994087e Mon Sep 17 00:00:00 2001 From: hardcoreerik Date: Tue, 14 Jul 2026 14:17:16 -0700 Subject: [PATCH 2/3] docs: record Qwen3.5 SeqMax fix and update stale SequenceHardLimit=240 reference MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Grok review (PR #56) flagged docs/CONTEXT_FABRIC_TEST_HARNESS.md still citing the old SequenceHardLimit=240 value without noting the 2026-07-14 change. Added §7b documenting the recurrent/hybrid-architecture failure mode and fix. --- docs/CONTEXT_FABRIC_TEST_HARNESS.md | 35 ++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/docs/CONTEXT_FABRIC_TEST_HARNESS.md b/docs/CONTEXT_FABRIC_TEST_HARNESS.md index affc73c3..17709e83 100644 --- a/docs/CONTEXT_FABRIC_TEST_HARNESS.md +++ b/docs/CONTEXT_FABRIC_TEST_HARNESS.md @@ -260,7 +260,8 @@ IDs are minted monotonically and never recycled, even after a `Conversation` is crash "at exactly the 257th reader conversation"). The existing fix, `SequenceRecycleThreshold = 128` (rebuild the role's executor — a fresh native context — every 128 minted conversations, at an idle point) and -`SequenceHardLimit = 240` (fail closed with a managed exception rather than let +`SequenceHardLimit = 240` (as of 2026-07-04; lowered to 40 on 2026-07-14, see +§7b) (fail closed with a managed exception rather than let the native assert kill the process), protects against exhausting the *count* of sequence IDs. **It does not protect against exhausting actual KV-cache *memory*, which is a function of prompt size × live-but-unrecycled sequences, @@ -421,6 +422,38 @@ reason not yet identified (only a short raw-output excerpt is persisted, not the response, so this needs a fresh instrumented run to pin down further). Not yet fixed; tracked here as a known, low-priority, separate gap. +### 7b. Second architecture-specific failure mode (2026-07-14): recurrent/hybrid models (Qwen3.5) need explicit `SeqMax` + +Distinct from §7a's Gemma-4 shared-KV-cache issue. `Qwen3.5-9B-Q8_0` (a hybrid +attention + Gated Delta Net / recurrent architecture) hit `NoKvSlot` on +**literally the second conversation ever minted** on any role's executor — not +a slow climb toward `SequenceRecycleThreshold`, 100% reproducible from the +first recycle. Native log showed `find_slot: seq_id=1 >= n_seq_max=1` and +`init_batch: failed to prepare recurrent ubatches`. + +Root cause: `LLamaSharpRuntime.cs`'s `ModelParams` never set `SeqMax`, so it +defaulted to 1. `AdapterManager` mints a fresh, monotonically-increasing +sequence id per conversation and only tears the executor down at +`SequenceRecycleThreshold`/`SequenceHardLimit` — an assumption that happened +to hold for plain-transformer architectures (llama.cpp's unified KV-cache path +tolerates `seq_id >= n_seq_max` there) but recurrent/hybrid architectures +validate `seq_id` strictly against `n_seq_max`, so every executor's second +conversation failed outright regardless of load. + +Fix: set `ModelParams.SeqMax = AdapterManager.SequenceHardLimit`. This alone +traded one bug for another — llama.cpp allocates a per-sequence recurrent- +state ("rs cache") buffer sized by `n_seq_max` on hybrid models, and the old +`SequenceHardLimit = 240` (calibrated when native `n_seq_max` was effectively +unlimited) became a live ~12GB VRAM reservation, OOM-crashing the native +process (`cudaMalloc failed`, exit `0xC0000005`) on a 16GB GPU. Lowered +`SequenceHardLimit` to 40 (~2GB rs-cache reservation, still well above +`SequenceRecycleThreshold = 24`). Verified live: zero `NoKvSlot`, zero OOM, +full CF-7 smoke run (3 questions) completes cleanly in ~58 minutes. See PR #56. + +**Practical guidance:** any future `SequenceHardLimit` change must be checked +against rs-cache VRAM cost for the largest hybrid-architecture model in use, +not just against the native `LLAMA_MAX_SEQ` sequence-count cap. + ## 8. Known fleet/environment issues (not scoring-logic bugs) These are infrastructure problems observed while running the gate on specific From 1bf38f3208e9f08613e0e66aa018d84ca57603b3 Mon Sep 17 00:00:00 2001 From: hardcoreerik Date: Tue, 14 Jul 2026 14:19:35 -0700 Subject: [PATCH 3/3] Isolate SeqMax for StatelessExecutor to avoid VRAM waste (CodeRabbit) StreamCompletionAsync's StatelessExecutor reused _modelParams, which now carries SeqMax=SequenceHardLimit(40) for the persistent AdapterManager executors. StatelessExecutor only ever runs one sequence per call, so reusing those params made every stateless call reserve the same ~2GB hybrid-architecture rs-cache budget for no benefit, risking OOM under concurrent calls. Split into a separate _statelessModelParams instance with the native default SeqMax=1. --- .../Core/Runtime/LLamaSharpRuntime.cs | 33 +++++++++++++++---- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/OrchestratorIDE/Core/Runtime/LLamaSharpRuntime.cs b/OrchestratorIDE/Core/Runtime/LLamaSharpRuntime.cs index b663ef24..f765be13 100644 --- a/OrchestratorIDE/Core/Runtime/LLamaSharpRuntime.cs +++ b/OrchestratorIDE/Core/Runtime/LLamaSharpRuntime.cs @@ -32,6 +32,7 @@ public sealed class LLamaSharpRuntime : ILocalModelRuntime { private LLamaWeights? _weights; private ModelParams? _modelParams; + private ModelParams? _statelessModelParams; private string? _activeModelPath; private string? _activeAdapterPath; private RuntimeOptions _options = new(); @@ -106,7 +107,7 @@ public async IAsyncEnumerable StreamCompletionAsync( Action? onUsage = null, [EnumeratorCancellation] CancellationToken ct = default) { - if (_weights is null || _modelParams is null) + if (_weights is null || _statelessModelParams is null) throw new InvalidOperationException( "No model loaded. Call LoadModelAsync before streaming."); @@ -115,7 +116,12 @@ public async IAsyncEnumerable StreamCompletionAsync( // itself isn't IDisposable, but the native context it owns (.Context) is — confirmed // via reflection during the Stage 1 smoke test (RUNTIME_SWITCH_PLAN.md) after Grok // caught that this was never disposed, leaking native KV-cache memory on every call. - var executor = new StatelessExecutor(_weights, _modelParams); + // + // Uses _statelessModelParams (SeqMax=1), NOT _modelParams (SeqMax=SequenceHardLimit) -- + // this executor only ever runs a single sequence per call, so reusing the persistent- + // executor params would reserve the same hybrid-architecture rs-cache VRAM budget on + // every stateless call for no benefit (CodeRabbit review, PR #56). + var executor = new StatelessExecutor(_weights, _statelessModelParams); try { // Build the raw prompt using the GGUF's embedded chat template. @@ -253,6 +259,19 @@ public async Task LoadModelAsync( // see docs/CONTEXT_FABRIC_TEST_HARNESS.md §7. }; + // StreamCompletionAsync's StatelessExecutor only ever runs one sequence per call + // (CodeRabbit review, PR #56) -- it must NOT reuse `mp` above. llama.cpp allocates + // its per-sequence recurrent-state ("rs cache") buffer sized by n_seq_max on hybrid + // architectures, so reusing the persistent-executor params (SeqMax = + // SequenceHardLimit = 40) would make every single stateless call reserve the same + // ~2GB the persistent AdapterManager executors reserve once, wasting VRAM and + // risking OOM under concurrent calls. Separate instance, native default SeqMax (1). + var statelessMp = new ModelParams(baseGgufPath) + { + ContextSize = (uint)_options.ContextLength, + GpuLayerCount = _options.GpuLayers, + }; + if (!string.IsNullOrEmpty(adapterPath)) { if (!File.Exists(adapterPath)) @@ -263,10 +282,11 @@ public async Task LoadModelAsync( // The adapter path is stored so the UI can surface it, but it is NOT applied. } - _weights = await LLamaWeights.LoadFromFileAsync(mp, ct); - _modelParams = mp; - _activeModelPath = baseGgufPath; - _activeAdapterPath = adapterPath; + _weights = await LLamaWeights.LoadFromFileAsync(mp, ct); + _modelParams = mp; + _statelessModelParams = statelessMp; + _activeModelPath = baseGgufPath; + _activeAdapterPath = adapterPath; Interlocked.Increment(ref _weightsGeneration); // Fix #3: do not claim adapter is active — LoRA is not applied in Phase 2. @@ -313,6 +333,7 @@ public ValueTask DisposeAsync() _weights?.Dispose(); _weights = null; _modelParams = null; + _statelessModelParams = null; _activeModelPath = null; _activeAdapterPath = null; _lastTokensPerSecond = null;