CF-7 test-harness diagnostics + native CUDA backend fixes (dev + release + installer)

.github/workflows/release.yml

@@ -100,6 +100,27 @@ jobs:
       - name: Restore
         run: dotnet restore OrchestratorIDE.slnx --runtime ${{ matrix.rid }}
 
+      # ── Fetch CUDA redistributables (Windows only) ────────────────────────────
+      # This runner has no CUDA Toolkit installed (stock windows-latest, no GPU), so
+      # OrchestratorIDE.NativeRuntime.csproj's TheOrcCudaRedistDir default (derived from
+      # $(CUDA_PATH)) resolves to nothing here -- meaning every past release build has
+      # silently shipped WITHOUT cudart64_12.dll/cublas64_12.dll/cublasLt64_12.dll, so the
+      # official Windows build CPU-falls-back for every real end user with an NVIDIA GPU
+      # (root-caused 2026-07-04, see docs/CONTEXT_FABRIC_TEST_HARNESS.md and
+      # NativeBackendBootstrap.cs's class doc). Fetches just the ~3 files needed directly
+      # from NVIDIA's own official redistributable feed (SHA-256 verified), not a full
+      # toolkit install and not a third-party NuGet repackaging.
+      - name: Fetch CUDA redistributables (Windows)
+        if: runner.os == 'Windows'
+        shell: pwsh
+        run: |
+          $cudaRedistDir = & Tools\Get-CudaRedistributables.ps1 | Select-Object -Last 1
+          if ($LASTEXITCODE -ne 0) {
+            Write-Host "Failed to fetch CUDA redistributables - release build would silently CPU-fallback for GPU users." -ForegroundColor Red
+            exit 1
+          }
+          echo "THEORC_CUDA_REDIST_DIR=$cudaRedistDir\" >> $env:GITHUB_ENV
+
       # ── Publish OrchestratorIDE (Avalonia shell, self-contained single-file) ──
       # AssemblyName is NOT overridden via -p: here -- OrchestratorIDE.Avalonia.csproj
       # references OrchestratorIDE.NativeRuntime, which has its own explicit
@@ -136,6 +157,7 @@ jobs:
             -p:AssemblyVersion=$ver `
             -p:FileVersion=$ver `
             -p:OutputType=WinExe `
+            -p:TheOrcCudaRedistDir="$env:THEORC_CUDA_REDIST_DIR" `
             --output publish/app
           Move-Item publish/app/OrchestratorIDE.Avalonia.exe publish/app/OrchestratorIDE.exe -Force
 

OrchestratorIDE/Core/Runtime/AdapterManager.cs

@@ -53,14 +53,45 @@ public sealed class AdapterManager : IAsyncDisposable
     // live on the first 1.8M-token unattended benchmark run, at exactly the 257th reader
     // conversation (~45 min in). Recycle the role's executor at a safe idle point well before
     // the cap; rebuilding costs one context allocation, not a weights reload.
-    internal const int SequenceRecycleThreshold = 128;
+    //
+    // This threshold bounds sequence-ID *count*, not KV-cache *memory* — a distinct exhaustion
+    // mode that shares the same "disposed conversations aren't reclaimed" root cause. The
+    // 2026-07-04 CF-7 gate run hit native NoKvSlot decode failures (docs/CONTEXT_FABRIC_TEST_HARNESS.md
+    // §7) well under the old threshold of 128, because BuildEvidencePack's uncapped evidence
+    // packs (up to ~26 segments/6.3K tokens per question, versus 1-4 before) consume far more of
+    // the shared KV pool per conversation than this threshold was calibrated for. Lowered as a
+    // conservative stopgap pending a real fix (recycling by cumulative prompt tokens instead of
+    // conversation count, which is what actually correlates with KV-cache pressure now that
+    // evidence-pack size varies per question). Do not raise this back toward 128 until that
+    // token-based recycle trigger lands and is validated against a full gate run.
+    internal const int SequenceRecycleThreshold = 24;
 
     // Absolute refusal point: if outstanding conversations have kept the executor from recycling
     // and it is now approaching the native slot cap, minting another conversation would trade a
     // recoverable managed exception for a process-killing native assert. Below 256 so the throw
     // always wins the race against the assert.
     internal const int SequenceHardLimit = 240;
 
+    // Opt-in, zero-cost-by-default diagnostic for the open KV-cache exhaustion investigation
+    // (docs/CONTEXT_FABRIC_TEST_HARNESS.md §7): a threshold change alone was tried and had no
+    // measurable effect on the failure trace, so the next step is confirming or ruling out
+    // whether ActiveCount is ever actually reaching zero (which would explain why recycling
+    // never engages regardless of the threshold value). Set THEORC_KVCACHE_DIAGNOSTICS=1 to
+    // print one line per recycle-eligibility check to stderr; unset, this is a single cached
+    // bool read with no other behavior change.
+    private static readonly bool s_kvDiagnosticsEnabled =
+        Environment.GetEnvironmentVariable("THEORC_KVCACHE_DIAGNOSTICS") == "1";
+
+    private static void LogKvDiagnostic(string message)
+    {
+        if (s_kvDiagnosticsEnabled)
+            // stdout, not stderr: Run-CF7GateExpanded.ps1 pipes the benchmark exe through
+            // `2>&1 | Tee-Object`, and PowerShell treats any native-process stderr output as a
+            // NativeCommandError under $ErrorActionPreference = 'Stop', aborting the whole run
+            // after the first diagnostic line (observed directly — fixed same session).
+            Console.WriteLine($"[KvCacheDiag] {message}");
+    }
+
     public AdapterManager(LLamaSharpRuntime runtime) =>
         _runtime = runtime ?? throw new ArgumentNullException(nameof(runtime));
 
@@ -113,9 +144,16 @@ private async Task<TrackedConversation> GetOrCreateConversationAsync(
                             $"while {existing.ActiveCount} conversation(s) remain active, so it cannot " +
                             "recycle and is about to exhaust the native sequence-slot cap. Dispose " +
                             "outstanding conversations for this role and retry.");
+                    LogKvDiagnostic(
+                        $"role={binding.Role} served-without-recycle minted={minted} " +
+                        $"activeCount={existing.ActiveCount} threshold={SequenceRecycleThreshold} " +
+                        $"reason={(minted < SequenceRecycleThreshold ? "under-threshold" : "active-conversations-outstanding")}");
                     return existing.CreateTrackedConversation();
                 }
 
+                LogKvDiagnostic(
+                    $"role={binding.Role} RECYCLING minted={minted} activeCount={existing.ActiveCount} " +
+                    $"threshold={SequenceRecycleThreshold}");
                 _entries.Remove(binding.Role);
                 // Best-effort, same contract as the stale-binding teardown below: the entry is
                 // already untracked, so a disposal fault must not block the replacement build.

OrchestratorIDE/Core/Runtime/LLamaSharpRuntime.cs

@@ -201,7 +201,9 @@ public async Task<ModelLoadResult> LoadModelAsync(
     {
         // Pin backend selection (CUDA preference on driver-only machines) before the first
         // NativeApi touch. Idempotent — callers that already surfaced the report pay nothing.
-        NativeBackendBootstrap.EnsureConfigured();
+        // Captured (not discarded) so a load failure below can report exactly what the backend
+        // pre-flight found/tried, instead of only the generic NativeApi TypeInitializationException.
+        var backendReport = NativeBackendBootstrap.EnsureConfigured();
 
         await DisposeAsync();  // unload previous model
 
@@ -248,7 +250,15 @@ public async Task<ModelLoadResult> LoadModelAsync(
         }
         catch (Exception ex)
         {
-            return new ModelLoadResult(false, RuntimeName, baseGgufPath, FormatLoadFailure(ex));
+            // Append the backend-selection report (CUDA-driver detection, cuda12 DLL pre-flight
+            // results, which backend was actually selected) — it was already computed above and
+            // previously discarded. On a native-load failure this is exactly the detail needed
+            // to tell "no CUDA-capable driver" from "packaged cuda12 DLL chain rejected" from
+            // "selection succeeded but the real load still failed anyway".
+            var backendDetail = $"backend: {backendReport.Verdict}" +
+                (backendReport.Log.Count > 0 ? $" [{string.Join("; ", backendReport.Log)}]" : "");
+            return new ModelLoadResult(false, RuntimeName, baseGgufPath,
+                $"{FormatLoadFailure(ex)} | {backendDetail}");
         }
     }
 
@@ -385,11 +395,16 @@ private string ApplyEmbeddedTemplate(IEnumerable<AgentMessage> messages)
 
     private static string FormatLoadFailure(Exception ex)
     {
-        var message = $"{ex.GetType().Name}: {ex.Message}";
-        if (ex.InnerException is null)
-            return message;
-
-        return $"{message} | Inner: {ex.InnerException.GetType().Name}: {ex.InnerException.Message}";
+        // Walk the FULL chain, not just one level. A TypeInitializationException's
+        // InnerException is often itself a wrapper (e.g. LLamaSharp's RuntimeError) with its
+        // own InnerException carrying the actual root cause — truncating at one level silently
+        // dropped exactly the detail needed to diagnose a native-library load failure (observed
+        // 2026-07-04 on HARDCOREPC: every failure showed only "RuntimeError: Failed to load the
+        // native library. Please check the log for more information." with the real reason cut off).
+        var parts = new List<string>();
+        for (var current = ex; current is not null; current = current.InnerException)
+            parts.Add($"{current.GetType().Name}: {current.Message}");
+        return string.Join(" | Inner: ", parts);
     }
 
     private static List<ToolCall> ParseToolCalls(string text) => ToolCallTextParser.Parse(text);

OrchestratorIDE/Core/Runtime/NativeBackendBootstrap.cs

@@ -1,10 +1,53 @@
 // Copyright (C) 2025-present hardcoreerik / TheOrc contributors
 // SPDX-License-Identifier: AGPL-3.0-or-later
 using System.Runtime.InteropServices;
+using LLama.Abstractions;
 using LLama.Native;
 
 namespace OrchestratorIDE.Core.Runtime;
 
+/// <summary>
+/// Overrides LLamaSharp's own CUDA-candidate construction so it always falls back to the
+/// packaged cuda12 backend (then cuda11) instead of only trying a folder matching whatever
+/// CUDA *toolkit* version <see cref="SystemInfo.CudaMajorVersion"/> happens to detect via
+/// CUDA_PATH/version.json.
+///
+/// Why this exists: <c>NativeLibraryWithCuda.Prepare</c> only takes the "try cuda12, then
+/// cuda11" fallback path when its majorCudaVersion is exactly -1 (the driver-only, no-toolkit
+/// case this fleet was built around — see <see cref="NativeBackendBootstrap"/>'s own class
+/// doc). When a real CUDA toolkit is installed, LLamaSharp's toolkit detection succeeds and
+/// returns that toolkit's major version instead, and the class then ONLY tries that one exact
+/// version's folder with no fallback at all. We only ever ship a cuda12 backend, so a machine
+/// with e.g. CUDA 13.3 installed (toolkit present, but only for other work — the driver alone
+/// is sufficient for a statically-linked cuda12 backend) ends up trying a nonexistent cuda13
+/// folder and failing outright, even though the working cuda12 folder is sitting right there.
+/// Confirmed live on HARDCOREPC (RTX 3050, driver-only originally; installing the CUDA 13.3
+/// SDK the same night broke native library loading entirely — LLamaSharp's own log showed it
+/// trying "runtimes\win-x64\native\cuda13\ggml-base.dll" and failing, never attempting cuda12).
+///
+/// Forcing majorCudaVersion back to -1 for CUDA candidates makes toolkit version irrelevant —
+/// exactly the "no user interaction, works regardless of what's installed" behavior wanted.
+/// If a genuine cuda13 (or other version) backend is ever packaged, extend this policy to try
+/// it first and fall back to cuda12, rather than relying on LLamaSharp's toolkit sniffing.
+/// </summary>
+internal sealed class Cuda12FallbackSelectingPolicy : INativeLibrarySelectingPolicy
+{
+    private readonly DefaultNativeLibrarySelectingPolicy _default = new();
+
+    public IEnumerable<INativeLibrary> Apply(
+        NativeLibraryConfig.Description description,
+        SystemInfo systemInfo,
+        NativeLogConfig.LLamaLogCallback? logCallback = null)
+    {
+        foreach (var library in _default.Apply(description, systemInfo, logCallback))
+        {
+            yield return library is NativeLibraryWithCuda
+                ? new NativeLibraryWithCuda(-1, description.Library, description.AvxLevel, description.SkipCheck)
+                : library;
+        }
+    }
+}
+
 /// <summary>
 /// Result of the one-time native backend selection. <see cref="SelectedCuda"/> false while
 /// <see cref="CudaCapableGpu"/> true is the loud "you are silently on CPU" signal every
@@ -46,6 +89,17 @@ public sealed record NativeBackendReport(
 /// </summary>
 public static class NativeBackendBootstrap
 {
+    // Must match OrchestratorSetup/Services/CudaRedistributableInstaller.cs's install target
+    // exactly -- the two projects don't reference each other (the installer stays lightweight,
+    // no LLamaSharp dependency), so this path is duplicated by design, not shared code. Outside
+    // any app-install or self-extraction directory on purpose: the installer runs before the
+    // app has ever launched once, so it cannot predict where a self-extracting single-file
+    // bundle will land its own temp extraction dir, and %LOCALAPPDATA% is guaranteed
+    // per-user-writable without elevation, unlike Program Files-style install locations.
+    public static readonly string StableRedistDir = Path.Combine(
+        Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),
+        "TheOrc", "CudaRedist");
+
     private static readonly object Gate = new();
     private static NativeBackendReport? _report;
     private static Action<string>? _ongoingSink;
@@ -92,7 +146,9 @@ public static NativeBackendReport EnsureConfigured(Action<string>? nativeLogSink
                     });
 
                 if (forceCuda)
-                    NativeLibraryConfig.All.WithCuda(true).SkipCheck(true).WithAutoFallback(false);
+                    NativeLibraryConfig.All
+                        .WithCuda(true).SkipCheck(true).WithAutoFallback(false)
+                        .WithSelectingPolicy(new Cuda12FallbackSelectingPolicy());
                 else
                     NativeLibraryConfig.All.WithAutoFallback(true);
             }
@@ -182,6 +238,33 @@ private static bool PreflightCudaBackend(List<string> log)
             return false;
         }
 
+        // The three CUDA runtime redistributables (cudart64_12/cublas64_12/cublasLt64_12) are
+        // not part of LLamaSharp.Backend.Cuda12.Windows's own NuGet content -- see this
+        // project's csproj comment on TheOrcCudaRedistDir. A published app bundle only has them
+        // in cudaDir if the machine that ran `dotnet publish` had a CUDA Toolkit (build-time
+        // fix landed 2026-07-04 in Tools/Get-CudaRedistributables.ps1 for the release CI
+        // build). For an *installed* app, OrchestratorSetup's CudaRedistributableInstaller
+        // fetches the same three files (from NVIDIA's own official redistributable feed) into
+        // StableRedistDir at install time instead -- a location outside wherever this
+        // single-file bundle's own self-extraction happens to land, which an installer running
+        // before the app has ever launched cannot predict or write into. Pre-loading them here
+        // from an absolute path (Windows-only; Linux never hits this branch, see the
+        // RID switch above) works regardless of which of the two directories actually has them:
+        // once a DLL is loaded into the process under a given name, any other code's later load
+        // of that same name (here, ggml-cuda.dll's own import of cudart64_12.dll) resolves to
+        // the already-loaded module, not wherever ggml-cuda.dll's own directory-relative search
+        // would have looked. A miss in both locations is not fatal here -- ggml-cuda.dll's own
+        // load attempt below will fail with a clear reason if these are genuinely unavailable.
+        foreach (var redist in new[] { "cudart64_12.dll", "cublas64_12.dll", "cublasLt64_12.dll" })
+        {
+            if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) &&
+                !File.Exists(Path.Combine(cudaDir, redist)) &&
+                File.Exists(Path.Combine(StableRedistDir, redist)))
+            {
+                TryLoadFrom(StableRedistDir, redist, log);
+            }
+        }
+
         if (!TryLoadFrom(cudaDir, $"{prefix}ggml-base{ext}", log))
             return false;