diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 14bbec75..0ef6d95a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -100,6 +100,27 @@ jobs: - name: Restore run: dotnet restore OrchestratorIDE.slnx --runtime ${{ matrix.rid }} + # ── Fetch CUDA redistributables (Windows only) ──────────────────────────── + # This runner has no CUDA Toolkit installed (stock windows-latest, no GPU), so + # OrchestratorIDE.NativeRuntime.csproj's TheOrcCudaRedistDir default (derived from + # $(CUDA_PATH)) resolves to nothing here -- meaning every past release build has + # silently shipped WITHOUT cudart64_12.dll/cublas64_12.dll/cublasLt64_12.dll, so the + # official Windows build CPU-falls-back for every real end user with an NVIDIA GPU + # (root-caused 2026-07-04, see docs/CONTEXT_FABRIC_TEST_HARNESS.md and + # NativeBackendBootstrap.cs's class doc). Fetches just the ~3 files needed directly + # from NVIDIA's own official redistributable feed (SHA-256 verified), not a full + # toolkit install and not a third-party NuGet repackaging. + - name: Fetch CUDA redistributables (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + $cudaRedistDir = & Tools\Get-CudaRedistributables.ps1 | Select-Object -Last 1 + if ($LASTEXITCODE -ne 0) { + Write-Host "Failed to fetch CUDA redistributables - release build would silently CPU-fallback for GPU users." -ForegroundColor Red + exit 1 + } + echo "THEORC_CUDA_REDIST_DIR=$cudaRedistDir\" >> $env:GITHUB_ENV + # ── Publish OrchestratorIDE (Avalonia shell, self-contained single-file) ── # AssemblyName is NOT overridden via -p: here -- OrchestratorIDE.Avalonia.csproj # references OrchestratorIDE.NativeRuntime, which has its own explicit @@ -136,6 +157,7 @@ jobs: -p:AssemblyVersion=$ver ` -p:FileVersion=$ver ` -p:OutputType=WinExe ` + -p:TheOrcCudaRedistDir="$env:THEORC_CUDA_REDIST_DIR" ` --output publish/app Move-Item publish/app/OrchestratorIDE.Avalonia.exe publish/app/OrchestratorIDE.exe -Force diff --git a/OrchestratorIDE/Core/Runtime/AdapterManager.cs b/OrchestratorIDE/Core/Runtime/AdapterManager.cs index 5de34723..a1ea305d 100644 --- a/OrchestratorIDE/Core/Runtime/AdapterManager.cs +++ b/OrchestratorIDE/Core/Runtime/AdapterManager.cs @@ -53,7 +53,18 @@ public sealed class AdapterManager : IAsyncDisposable // live on the first 1.8M-token unattended benchmark run, at exactly the 257th reader // conversation (~45 min in). Recycle the role's executor at a safe idle point well before // the cap; rebuilding costs one context allocation, not a weights reload. - internal const int SequenceRecycleThreshold = 128; + // + // This threshold bounds sequence-ID *count*, not KV-cache *memory* — a distinct exhaustion + // mode that shares the same "disposed conversations aren't reclaimed" root cause. The + // 2026-07-04 CF-7 gate run hit native NoKvSlot decode failures (docs/CONTEXT_FABRIC_TEST_HARNESS.md + // §7) well under the old threshold of 128, because BuildEvidencePack's uncapped evidence + // packs (up to ~26 segments/6.3K tokens per question, versus 1-4 before) consume far more of + // the shared KV pool per conversation than this threshold was calibrated for. Lowered as a + // conservative stopgap pending a real fix (recycling by cumulative prompt tokens instead of + // conversation count, which is what actually correlates with KV-cache pressure now that + // evidence-pack size varies per question). Do not raise this back toward 128 until that + // token-based recycle trigger lands and is validated against a full gate run. + internal const int SequenceRecycleThreshold = 24; // Absolute refusal point: if outstanding conversations have kept the executor from recycling // and it is now approaching the native slot cap, minting another conversation would trade a @@ -61,6 +72,26 @@ public sealed class AdapterManager : IAsyncDisposable // always wins the race against the assert. internal const int SequenceHardLimit = 240; + // Opt-in, zero-cost-by-default diagnostic for the open KV-cache exhaustion investigation + // (docs/CONTEXT_FABRIC_TEST_HARNESS.md §7): a threshold change alone was tried and had no + // measurable effect on the failure trace, so the next step is confirming or ruling out + // whether ActiveCount is ever actually reaching zero (which would explain why recycling + // never engages regardless of the threshold value). Set THEORC_KVCACHE_DIAGNOSTICS=1 to + // print one line per recycle-eligibility check to stderr; unset, this is a single cached + // bool read with no other behavior change. + private static readonly bool s_kvDiagnosticsEnabled = + Environment.GetEnvironmentVariable("THEORC_KVCACHE_DIAGNOSTICS") == "1"; + + private static void LogKvDiagnostic(string message) + { + if (s_kvDiagnosticsEnabled) + // stdout, not stderr: Run-CF7GateExpanded.ps1 pipes the benchmark exe through + // `2>&1 | Tee-Object`, and PowerShell treats any native-process stderr output as a + // NativeCommandError under $ErrorActionPreference = 'Stop', aborting the whole run + // after the first diagnostic line (observed directly — fixed same session). + Console.WriteLine($"[KvCacheDiag] {message}"); + } + public AdapterManager(LLamaSharpRuntime runtime) => _runtime = runtime ?? throw new ArgumentNullException(nameof(runtime)); @@ -113,9 +144,16 @@ private async Task GetOrCreateConversationAsync( $"while {existing.ActiveCount} conversation(s) remain active, so it cannot " + "recycle and is about to exhaust the native sequence-slot cap. Dispose " + "outstanding conversations for this role and retry."); + LogKvDiagnostic( + $"role={binding.Role} served-without-recycle minted={minted} " + + $"activeCount={existing.ActiveCount} threshold={SequenceRecycleThreshold} " + + $"reason={(minted < SequenceRecycleThreshold ? "under-threshold" : "active-conversations-outstanding")}"); return existing.CreateTrackedConversation(); } + LogKvDiagnostic( + $"role={binding.Role} RECYCLING minted={minted} activeCount={existing.ActiveCount} " + + $"threshold={SequenceRecycleThreshold}"); _entries.Remove(binding.Role); // Best-effort, same contract as the stale-binding teardown below: the entry is // already untracked, so a disposal fault must not block the replacement build. diff --git a/OrchestratorIDE/Core/Runtime/LLamaSharpRuntime.cs b/OrchestratorIDE/Core/Runtime/LLamaSharpRuntime.cs index d0460500..b9cb3ab9 100644 --- a/OrchestratorIDE/Core/Runtime/LLamaSharpRuntime.cs +++ b/OrchestratorIDE/Core/Runtime/LLamaSharpRuntime.cs @@ -201,7 +201,9 @@ public async Task LoadModelAsync( { // Pin backend selection (CUDA preference on driver-only machines) before the first // NativeApi touch. Idempotent — callers that already surfaced the report pay nothing. - NativeBackendBootstrap.EnsureConfigured(); + // Captured (not discarded) so a load failure below can report exactly what the backend + // pre-flight found/tried, instead of only the generic NativeApi TypeInitializationException. + var backendReport = NativeBackendBootstrap.EnsureConfigured(); await DisposeAsync(); // unload previous model @@ -248,7 +250,15 @@ public async Task LoadModelAsync( } catch (Exception ex) { - return new ModelLoadResult(false, RuntimeName, baseGgufPath, FormatLoadFailure(ex)); + // Append the backend-selection report (CUDA-driver detection, cuda12 DLL pre-flight + // results, which backend was actually selected) — it was already computed above and + // previously discarded. On a native-load failure this is exactly the detail needed + // to tell "no CUDA-capable driver" from "packaged cuda12 DLL chain rejected" from + // "selection succeeded but the real load still failed anyway". + var backendDetail = $"backend: {backendReport.Verdict}" + + (backendReport.Log.Count > 0 ? $" [{string.Join("; ", backendReport.Log)}]" : ""); + return new ModelLoadResult(false, RuntimeName, baseGgufPath, + $"{FormatLoadFailure(ex)} | {backendDetail}"); } } @@ -385,11 +395,16 @@ private string ApplyEmbeddedTemplate(IEnumerable messages) private static string FormatLoadFailure(Exception ex) { - var message = $"{ex.GetType().Name}: {ex.Message}"; - if (ex.InnerException is null) - return message; - - return $"{message} | Inner: {ex.InnerException.GetType().Name}: {ex.InnerException.Message}"; + // Walk the FULL chain, not just one level. A TypeInitializationException's + // InnerException is often itself a wrapper (e.g. LLamaSharp's RuntimeError) with its + // own InnerException carrying the actual root cause — truncating at one level silently + // dropped exactly the detail needed to diagnose a native-library load failure (observed + // 2026-07-04 on HARDCOREPC: every failure showed only "RuntimeError: Failed to load the + // native library. Please check the log for more information." with the real reason cut off). + var parts = new List(); + for (var current = ex; current is not null; current = current.InnerException) + parts.Add($"{current.GetType().Name}: {current.Message}"); + return string.Join(" | Inner: ", parts); } private static List ParseToolCalls(string text) => ToolCallTextParser.Parse(text); diff --git a/OrchestratorIDE/Core/Runtime/NativeBackendBootstrap.cs b/OrchestratorIDE/Core/Runtime/NativeBackendBootstrap.cs index 28d3e586..b9c91c9e 100644 --- a/OrchestratorIDE/Core/Runtime/NativeBackendBootstrap.cs +++ b/OrchestratorIDE/Core/Runtime/NativeBackendBootstrap.cs @@ -1,10 +1,53 @@ // Copyright (C) 2025-present hardcoreerik / TheOrc contributors // SPDX-License-Identifier: AGPL-3.0-or-later using System.Runtime.InteropServices; +using LLama.Abstractions; using LLama.Native; namespace OrchestratorIDE.Core.Runtime; +/// +/// Overrides LLamaSharp's own CUDA-candidate construction so it always falls back to the +/// packaged cuda12 backend (then cuda11) instead of only trying a folder matching whatever +/// CUDA *toolkit* version happens to detect via +/// CUDA_PATH/version.json. +/// +/// Why this exists: NativeLibraryWithCuda.Prepare only takes the "try cuda12, then +/// cuda11" fallback path when its majorCudaVersion is exactly -1 (the driver-only, no-toolkit +/// case this fleet was built around — see 's own class +/// doc). When a real CUDA toolkit is installed, LLamaSharp's toolkit detection succeeds and +/// returns that toolkit's major version instead, and the class then ONLY tries that one exact +/// version's folder with no fallback at all. We only ever ship a cuda12 backend, so a machine +/// with e.g. CUDA 13.3 installed (toolkit present, but only for other work — the driver alone +/// is sufficient for a statically-linked cuda12 backend) ends up trying a nonexistent cuda13 +/// folder and failing outright, even though the working cuda12 folder is sitting right there. +/// Confirmed live on HARDCOREPC (RTX 3050, driver-only originally; installing the CUDA 13.3 +/// SDK the same night broke native library loading entirely — LLamaSharp's own log showed it +/// trying "runtimes\win-x64\native\cuda13\ggml-base.dll" and failing, never attempting cuda12). +/// +/// Forcing majorCudaVersion back to -1 for CUDA candidates makes toolkit version irrelevant — +/// exactly the "no user interaction, works regardless of what's installed" behavior wanted. +/// If a genuine cuda13 (or other version) backend is ever packaged, extend this policy to try +/// it first and fall back to cuda12, rather than relying on LLamaSharp's toolkit sniffing. +/// +internal sealed class Cuda12FallbackSelectingPolicy : INativeLibrarySelectingPolicy +{ + private readonly DefaultNativeLibrarySelectingPolicy _default = new(); + + public IEnumerable Apply( + NativeLibraryConfig.Description description, + SystemInfo systemInfo, + NativeLogConfig.LLamaLogCallback? logCallback = null) + { + foreach (var library in _default.Apply(description, systemInfo, logCallback)) + { + yield return library is NativeLibraryWithCuda + ? new NativeLibraryWithCuda(-1, description.Library, description.AvxLevel, description.SkipCheck) + : library; + } + } +} + /// /// Result of the one-time native backend selection. false while /// true is the loud "you are silently on CPU" signal every @@ -46,6 +89,17 @@ public sealed record NativeBackendReport( /// public static class NativeBackendBootstrap { + // Must match OrchestratorSetup/Services/CudaRedistributableInstaller.cs's install target + // exactly -- the two projects don't reference each other (the installer stays lightweight, + // no LLamaSharp dependency), so this path is duplicated by design, not shared code. Outside + // any app-install or self-extraction directory on purpose: the installer runs before the + // app has ever launched once, so it cannot predict where a self-extracting single-file + // bundle will land its own temp extraction dir, and %LOCALAPPDATA% is guaranteed + // per-user-writable without elevation, unlike Program Files-style install locations. + public static readonly string StableRedistDir = Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), + "TheOrc", "CudaRedist"); + private static readonly object Gate = new(); private static NativeBackendReport? _report; private static Action? _ongoingSink; @@ -92,7 +146,9 @@ public static NativeBackendReport EnsureConfigured(Action? nativeLogSink }); if (forceCuda) - NativeLibraryConfig.All.WithCuda(true).SkipCheck(true).WithAutoFallback(false); + NativeLibraryConfig.All + .WithCuda(true).SkipCheck(true).WithAutoFallback(false) + .WithSelectingPolicy(new Cuda12FallbackSelectingPolicy()); else NativeLibraryConfig.All.WithAutoFallback(true); } @@ -182,6 +238,33 @@ private static bool PreflightCudaBackend(List log) return false; } + // The three CUDA runtime redistributables (cudart64_12/cublas64_12/cublasLt64_12) are + // not part of LLamaSharp.Backend.Cuda12.Windows's own NuGet content -- see this + // project's csproj comment on TheOrcCudaRedistDir. A published app bundle only has them + // in cudaDir if the machine that ran `dotnet publish` had a CUDA Toolkit (build-time + // fix landed 2026-07-04 in Tools/Get-CudaRedistributables.ps1 for the release CI + // build). For an *installed* app, OrchestratorSetup's CudaRedistributableInstaller + // fetches the same three files (from NVIDIA's own official redistributable feed) into + // StableRedistDir at install time instead -- a location outside wherever this + // single-file bundle's own self-extraction happens to land, which an installer running + // before the app has ever launched cannot predict or write into. Pre-loading them here + // from an absolute path (Windows-only; Linux never hits this branch, see the + // RID switch above) works regardless of which of the two directories actually has them: + // once a DLL is loaded into the process under a given name, any other code's later load + // of that same name (here, ggml-cuda.dll's own import of cudart64_12.dll) resolves to + // the already-loaded module, not wherever ggml-cuda.dll's own directory-relative search + // would have looked. A miss in both locations is not fatal here -- ggml-cuda.dll's own + // load attempt below will fail with a clear reason if these are genuinely unavailable. + foreach (var redist in new[] { "cudart64_12.dll", "cublas64_12.dll", "cublasLt64_12.dll" }) + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && + !File.Exists(Path.Combine(cudaDir, redist)) && + File.Exists(Path.Combine(StableRedistDir, redist))) + { + TryLoadFrom(StableRedistDir, redist, log); + } + } + if (!TryLoadFrom(cudaDir, $"{prefix}ggml-base{ext}", log)) return false; diff --git a/OrchestratorSetup/Services/CudaRedistributableInstaller.cs b/OrchestratorSetup/Services/CudaRedistributableInstaller.cs new file mode 100644 index 00000000..95fc97fe --- /dev/null +++ b/OrchestratorSetup/Services/CudaRedistributableInstaller.cs @@ -0,0 +1,149 @@ +// Copyright (C) 2025-present hardcoreerik / TheOrc contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +using System.Net.Http; +using System.Text.Json; + +namespace OrchestratorSetup.Services; + +/// +/// Installs cudart64_12.dll, cublas64_12.dll, and cublasLt64_12.dll -- the CUDA runtime +/// redistributables OrchestratorIDE's in-process LLamaSharp backend +/// (OrchestratorIDE.NativeRuntime's NativeBackendBootstrap/LLamaSharpRuntime) needs to load its +/// cuda12 backend, but which LLamaSharp.Backend.Cuda12.Windows's NuGet package does not itself +/// ship (see that project's own csproj comment). Every fleet dev machine sourced these from a +/// locally installed CUDA Toolkit; a real end user has neither a toolkit nor a reason to install +/// one just for this, so the installer fetches them directly instead -- conditionally, only for +/// detected NVIDIA hardware, so AMD/Intel/CPU-only installs never pay for a download they can't +/// use. +/// +/// Source: NVIDIA's own official CUDA redistributable manifest feed -- the same channel +/// conda/pip use to build their nvidia-cuda-runtime-cu12/nvidia-cublas-cu12 packages, not a +/// third-party NuGet repackaging and not a full multi-GB Toolkit installer. Mirrors +/// Tools/Get-CudaRedistributables.ps1 (used by the release CI build to fix the SAME gap for +/// the build machine's own published artifact) but reuses this project's own DownloadService +/// (resumable, SHA-256-verified, retry-on-failure -- already exercised by every other download +/// this installer performs) and ZipExtractService (zip-slip-guarded extraction) rather than +/// hand-rolling HTTP/hashing/extraction a second time. +/// +public sealed class CudaRedistributableInstaller +{ + private const string ManifestVersion = "12.4.0"; + private const string RedistBaseUrl = "https://developer.download.nvidia.com/compute/cuda/redist"; + + // Only the two NVIDIA redistributable components that contain the three DLLs we need. + private static readonly string[] Components = ["cuda_cudart", "libcublas"]; + private static readonly string[] NeededDlls = ["cudart64_12.dll", "cublas64_12.dll", "cublasLt64_12.dll"]; + + private readonly DownloadService _dl; + private readonly ZipExtractService _zip; + + /// Log line for the scrolling install log -- same event shape InstallOrchestrator already relays. + public event Action? OnLog; + + public CudaRedistributableInstaller(DownloadService downloadService, ZipExtractService zipExtractService) + { + _dl = downloadService; + _zip = zipExtractService; + } + + /// + /// Ensures all three redistributable DLLs exist in . Callers + /// pass the stable CUDA redistributable cache directory (InstallOrchestrator's + /// StableCudaRedistDir, %LOCALAPPDATA%\TheOrc\CudaRedist) -- deliberately NOT the app's own + /// "runtimes/win-x64/native/cuda12" bundle folder, since a self-extracting single-file + /// install can't predict that path before the app has ever launched. See + /// NativeBackendBootstrap.StableRedistDir/PreflightCudaBackend for how the in-process + /// runtime finds files placed here. Idempotent: a re-run (repair install, upgrade) with all + /// three DLLs already present skips the network entirely. Returns false (non-fatal to the + /// overall install -- callers should log a warning and continue, matching how model + /// download failures are already handled) if the manifest, download, or extraction fails. + /// + public async Task InstallAsync(string targetDir, CancellationToken ct) + { + if (NeededDlls.All(dll => File.Exists(Path.Combine(targetDir, dll)))) + { + Log("CUDA runtime redistributables already present -- skipping."); + return true; + } + + Directory.CreateDirectory(targetDir); + var workDir = Path.Combine(Path.GetTempPath(), $"theorc-cuda-redist-{Guid.NewGuid():N}"); + Directory.CreateDirectory(workDir); + + try + { + string manifestJson; + using (var http = new HttpClient()) + { + http.DefaultRequestHeaders.UserAgent.ParseAdd("OrchestratorSetup/1.0"); + var manifestUrl = $"{RedistBaseUrl}/redistrib_{ManifestVersion}.json"; + Log($"Fetching NVIDIA CUDA redistributable manifest ({ManifestVersion})..."); + manifestJson = await http.GetStringAsync(manifestUrl, ct); + } + + using var manifestDoc = JsonDocument.Parse(manifestJson); + var manifest = manifestDoc.RootElement; + + foreach (var component in Components) + { + if (!manifest.TryGetProperty(component, out var comp) || + !comp.TryGetProperty("windows-x86_64", out var entry)) + { + Log($"NVIDIA manifest is missing a windows-x86_64 entry for '{component}' -- CUDA acceleration will not be available."); + return false; + } + + var relativePath = entry.GetProperty("relative_path").GetString() + ?? throw new InvalidOperationException($"Manifest entry for '{component}' has no relative_path."); + var sha256 = entry.GetProperty("sha256").GetString(); + var sizeStr = entry.TryGetProperty("size", out var sizeProp) ? sizeProp.GetString() : null; + var size = long.TryParse(sizeStr, out var s) ? s : (long?)null; + + var downloadUrl = $"{RedistBaseUrl}/{relativePath}"; + var zipPath = Path.Combine(workDir, Path.GetFileName(relativePath)); + + Log($"Downloading {component}..."); + await _dl.DownloadFileAsync(downloadUrl, zipPath, component, size, sha256, ct); + + var extractDir = Path.Combine(workDir, Path.GetFileNameWithoutExtension(relativePath)); + await _zip.ExtractAsync(zipPath, extractDir, ct); + + var binDir = Directory.GetDirectories(extractDir, "bin", SearchOption.AllDirectories) + .FirstOrDefault(); + if (binDir is null) + { + Log($"Could not find a 'bin' directory inside the extracted {component} archive -- CUDA acceleration will not be available."); + return false; + } + + foreach (var dllPath in Directory.GetFiles(binDir, "*.dll")) + { + var name = Path.GetFileName(dllPath); + if (!NeededDlls.Contains(name)) continue; + File.Copy(dllPath, Path.Combine(targetDir, name), overwrite: true); + Log($" Installed {name}"); + } + } + + var missing = NeededDlls.Where(dll => !File.Exists(Path.Combine(targetDir, dll))).ToList(); + if (missing.Count > 0) + { + Log($"Missing expected DLL(s) after extraction: {string.Join(", ", missing)} -- CUDA acceleration will not be available."); + return false; + } + + return true; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + Log($"CUDA redistributable install failed: {ex.Message} -- CUDA acceleration will not be available, but the rest of the install can continue."); + return false; + } + finally + { + try { Directory.Delete(workDir, recursive: true); } catch { /* best-effort cleanup */ } + } + } + + private void Log(string msg) => OnLog?.Invoke(msg); +} diff --git a/OrchestratorSetup/Services/InstallOrchestrator.cs b/OrchestratorSetup/Services/InstallOrchestrator.cs index 7d4137af..04399568 100644 --- a/OrchestratorSetup/Services/InstallOrchestrator.cs +++ b/OrchestratorSetup/Services/InstallOrchestrator.cs @@ -34,10 +34,11 @@ public sealed class InstallOrchestrator : IDisposable // ── State ───────────────────────────────────────────────────────────────── - private readonly InstallerState _state; - private readonly InstallerViewModel _vm; - private readonly DownloadService _dl; - private readonly ZipExtractService _zip; + private readonly InstallerState _state; + private readonly InstallerViewModel _vm; + private readonly DownloadService _dl; + private readonly ZipExtractService _zip; + private readonly CudaRedistributableInstaller _cudaRedist; private int _totalSteps; private int _stepsDone; @@ -45,10 +46,12 @@ public sealed class InstallOrchestrator : IDisposable public InstallOrchestrator(InstallerViewModel vm) { - _vm = vm; - _state = vm.State; - _dl = new DownloadService(); - _zip = new ZipExtractService(); + _vm = vm; + _state = vm.State; + _dl = new DownloadService(); + _zip = new ZipExtractService(); + _cudaRedist = new CudaRedistributableInstaller(_dl, _zip); + _cudaRedist.OnLog += msg => Log($" {msg}"); _dl.OnProgress += p => { @@ -58,6 +61,26 @@ public InstallOrchestrator(InstallerViewModel vm) }; } + /// + /// True only when this install actually needs OrchestratorIDE's in-process cuda12 backend + /// working -- an NVIDIA GPU was detected AND we're installing on Windows (the only OS the + /// packaged cuda12 backend and NVIDIA's redistributable feed both target). Gates the new + /// step so AMD/Intel/CPU-only/macOS/Linux installs never pay for a download they can't use. + /// + private bool NeedsCudaRedistributables => + OperatingSystem.IsWindows() && _state.DetectedGpuVendor == "nvidia"; + + /// + /// Must match OrchestratorIDE.Core.Runtime.NativeBackendBootstrap.StableRedistDir exactly -- + /// this project deliberately does not reference OrchestratorIDE.NativeRuntime (no LLamaSharp + /// dependency in the installer), so the path is duplicated here rather than shared via a + /// project reference. See that class's own doc comment for why this location (outside any + /// app-install or self-extraction directory) was chosen. + /// + private static readonly string StableCudaRedistDir = Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), + "TheOrc", "CudaRedist"); + // ── Main entry point ────────────────────────────────────────────────────── public async Task RunAsync(CancellationToken ct = default) @@ -234,6 +257,24 @@ await _dl.DownloadFileAsync( Log("⚠ App download URL not found in manifest — exe must be placed manually."); } + // ── CUDA runtime redistributables (in-process backend, NVIDIA only) ─ + // Independent of the Ollama/llama.cpp backend choice below -- this is specifically + // for OrchestratorIDE.exe's own in-process LLamaSharp cuda12 backend (Context + // Fabric and other native-runtime features), which needs cudart64_12.dll/ + // cublas64_12.dll/cublasLt64_12.dll regardless of which external inference backend + // the user also sets up. Non-fatal: a failure here disables CUDA acceleration for + // the in-process backend only (it falls back to CPU) and does not abort the + // install, matching how model download failures are already handled below. + if (NeedsCudaRedistributables) + { + await Step("Fetching CUDA runtime redistributables", async () => + { + var ok = await _cudaRedist.InstallAsync(StableCudaRedistDir, ct); + if (!ok) + Log(" ⚠ CUDA redistributables unavailable — in-process native features will use CPU."); + }, ct); + } + // ── Backend-specific steps ───────────────────────────────────── if (_state.InstallOllama) @@ -495,6 +536,9 @@ private int ComputeTotalSteps() if (File.Exists(_state.PortableAppExePath) || !string.IsNullOrEmpty(_state.AppDownloadUrl)) n += 1; + if (NeedsCudaRedistributables) + n += 1; // Fetching CUDA runtime redistributables + if (_state.InstallOllama) { n += 1; // Install Ollama (includes model pull) diff --git a/Tools/Get-CudaRedistributables.ps1 b/Tools/Get-CudaRedistributables.ps1 new file mode 100644 index 00000000..b78c9763 --- /dev/null +++ b/Tools/Get-CudaRedistributables.ps1 @@ -0,0 +1,138 @@ +# Get-CudaRedistributables.ps1 - fetch NVIDIA's official CUDA runtime redistributables +# (cudart64_12.dll, cublas64_12.dll, cublasLt64_12.dll) without installing the full CUDA +# Toolkit or depending on a third-party NuGet repackaging. +# +# Source: NVIDIA's own redistributable manifest feed, the same official channel conda/pip +# use to build their nvidia-cuda-runtime-cu12/nvidia-cublas-cu12 packages: +# https://developer.download.nvidia.com/compute/cuda/redist/redistrib_.json +# +# Why this exists: LLamaSharp.Backend.Cuda12.Windows's ggml-cuda.dll dynamically imports +# cudart64_12.dll and cublas64_12.dll (which itself needs cublasLt64_12.dll) at runtime, but +# the NuGet package does not ship them - see OrchestratorIDE.NativeRuntime.csproj's +# TheOrcCudaRedistDir property. Every fleet dev machine sources these from a locally installed +# CUDA Toolkit; the release CI runner (a stock GitHub-hosted windows-latest box) has no toolkit +# at all, so the official release build has been silently missing these DLLs and CPU-falling- +# back for every real end user with an NVIDIA GPU. This script gives CI (or any machine) a way +# to fetch just the ~3 files actually needed, verified against NVIDIA's published SHA-256, in a +# fraction of the time/bandwidth a full toolkit install would cost. +# +# Usage: +# Tools\Get-CudaRedistributables.ps1 # default version, default output dir +# Tools\Get-CudaRedistributables.ps1 -Version 12.4.0 -OutputDir F:\CudaRedist12 +# +# Exit codes: 0 = success (or already present with matching hash), 1 = download/verify/extract +# failure. On success, prints the resolved output directory on its own final line so a CI step +# can capture it (e.g. into $env:TheOrcCudaRedistDir). +param( + [string]$Version = "12.4.0", + [string]$OutputDir = "", + [int] $TimeoutSec = 300 +) + +$ErrorActionPreference = "Stop" +# Invoke-WebRequest renders a progress bar by default, which materially slows large downloads +# on some PowerShell hosts (noticeable here: multi-hundred-MB CUDA archives on every cache-miss +# CI run). +$ProgressPreference = "SilentlyContinue" + +if (-not $OutputDir) { + $OutputDir = Join-Path $env:TEMP "theorc-cuda-redist-$Version" +} + +$manifestUrl = "https://developer.download.nvidia.com/compute/cuda/redist/redistrib_$Version.json" +$components = @("cuda_cudart", "libcublas") +$neededDlls = @("cudart64_12.dll", "cublas64_12.dll", "cublasLt64_12.dll") + +function Test-Sha256 { + param([string]$FilePath, [string]$ExpectedHash) + $actual = (Get-FileHash -Path $FilePath -Algorithm SHA256).Hash + return $actual -ieq $ExpectedHash +} + +# Idempotent: skip everything if all three DLLs are already present. Re-run to force a refresh +# by pointing -OutputDir at a fresh/empty directory. +$allPresent = $true +foreach ($dll in $neededDlls) { + if (-not (Test-Path (Join-Path $OutputDir $dll))) { $allPresent = $false; break } +} +if ($allPresent) { + Write-Host "All CUDA redistributables already present in '$OutputDir' - skipping download." -ForegroundColor DarkGray + Write-Output $OutputDir + exit 0 +} + +New-Item -ItemType Directory -Force -Path $OutputDir | Out-Null +$workDir = Join-Path $OutputDir "_download" +New-Item -ItemType Directory -Force -Path $workDir | Out-Null + +Write-Host "Fetching NVIDIA CUDA redistributable manifest for version $Version..." -ForegroundColor Cyan +try { + $manifest = Invoke-RestMethod -Uri $manifestUrl -TimeoutSec $TimeoutSec +} catch { + Write-Host "Failed to fetch manifest from $manifestUrl : $($_.Exception.Message)" -ForegroundColor Red + exit 1 +} + +foreach ($component in $components) { + if (-not $manifest.$component) { + Write-Host "Manifest is missing expected component '$component' - NVIDIA may have restructured the feed." -ForegroundColor Red + exit 1 + } + $entry = $manifest.$component.'windows-x86_64' + if (-not $entry) { + Write-Host "Component '$component' has no windows-x86_64 entry in this manifest." -ForegroundColor Red + exit 1 + } + + $relativePath = $entry.relative_path + $expectedSha = $entry.sha256 + $downloadUrl = "https://developer.download.nvidia.com/compute/cuda/redist/$relativePath" + $zipPath = Join-Path $workDir ([System.IO.Path]::GetFileName($relativePath)) + + Write-Host "Downloading $component ($($entry.size) bytes) from $downloadUrl..." -ForegroundColor Cyan + try { + Invoke-WebRequest -Uri $downloadUrl -OutFile $zipPath -TimeoutSec $TimeoutSec + } catch { + Write-Host "Failed to download $component : $($_.Exception.Message)" -ForegroundColor Red + exit 1 + } + + if (-not (Test-Sha256 -FilePath $zipPath -ExpectedHash $expectedSha)) { + Write-Host "SHA-256 mismatch for $component - refusing to use a corrupted/tampered download." -ForegroundColor Red + exit 1 + } + Write-Host " Verified SHA-256 for $component." -ForegroundColor DarkGray + + $extractDir = Join-Path $workDir ([System.IO.Path]::GetFileNameWithoutExtension($relativePath)) + try { + Expand-Archive -Path $zipPath -DestinationPath $extractDir -Force + + $binDir = Get-ChildItem -Path $extractDir -Directory -Recurse -Filter "bin" | Select-Object -First 1 + if (-not $binDir) { + Write-Host "Could not find a 'bin' directory inside the extracted $component archive." -ForegroundColor Red + exit 1 + } + + foreach ($dll in Get-ChildItem -Path $binDir.FullName -Filter "*.dll") { + if ($neededDlls -contains $dll.Name) { + Copy-Item -Path $dll.FullName -Destination (Join-Path $OutputDir $dll.Name) -Force + Write-Host " Extracted $($dll.Name)" -ForegroundColor DarkGray + } + } + } catch { + Write-Host "Failed to extract or copy $component : $($_.Exception.Message)" -ForegroundColor Red + exit 1 + } +} + +Remove-Item -Path $workDir -Recurse -Force -ErrorAction SilentlyContinue + +$missing = $neededDlls | Where-Object { -not (Test-Path (Join-Path $OutputDir $_)) } +if ($missing.Count -gt 0) { + Write-Host "Missing expected DLLs after extraction: $($missing -join ', ')" -ForegroundColor Red + exit 1 +} + +Write-Host "All CUDA redistributables ready in '$OutputDir'." -ForegroundColor Green +Write-Output $OutputDir +exit 0 diff --git a/docs/CONTEXT_FABRIC_BENCHMARK_MANIFEST.md b/docs/CONTEXT_FABRIC_BENCHMARK_MANIFEST.md index 8a899c41..202246e9 100644 --- a/docs/CONTEXT_FABRIC_BENCHMARK_MANIFEST.md +++ b/docs/CONTEXT_FABRIC_BENCHMARK_MANIFEST.md @@ -101,6 +101,12 @@ The `systems` array must include B0 through B4. Missing artifacts are explicit ` The initial CF-7 slice may emit a `NO-GO` report with only B3 plus diagnostics populated. That is valid progress: it freezes the report shape and prevents partial benchmark evidence from being mistaken for an architecture pass. +For a full walkthrough of how an answer gets built and graded — evidence +selection, JSON recovery, verification rules, and the known residual risks in +each — see [CONTEXT_FABRIC_TEST_HARNESS.md](CONTEXT_FABRIC_TEST_HARNESS.md). +That document exists specifically so the scoring logic can be reviewed +independently of any one run's result. + ### Re-Running The Expanded 120-Question Gate [`Tools/ContextFabricBench/Run-CF7GateExpanded.ps1`](../Tools/ContextFabricBench/Run-CF7GateExpanded.ps1) diff --git a/docs/CONTEXT_FABRIC_TEST_HARNESS.md b/docs/CONTEXT_FABRIC_TEST_HARNESS.md new file mode 100644 index 00000000..f517c95d --- /dev/null +++ b/docs/CONTEXT_FABRIC_TEST_HARNESS.md @@ -0,0 +1,393 @@ +# Context Fabric CF-7 Test Harness — How It Grades Answers + +This document explains, end to end, how the `cf7-gate-expanded` benchmark decides +whether an answer is right or wrong. It exists so the scoring logic itself can be +reviewed independently of any particular run's result — a NO-GO should mean "the +model got it wrong," not "the harness has a bug." + +Companion docs: [CONTEXT_FABRIC_BENCHMARK_MANIFEST.md](CONTEXT_FABRIC_BENCHMARK_MANIFEST.md) +(report schema, re-run recipe), [CONTEXT_FABRIC_BENCHMARK_CORPUS.md](CONTEXT_FABRIC_BENCHMARK_CORPUS.md) +(public/private corpus rules). + +## 1. What's being tested + +Four live systems plus one frozen artifact answer the same 120 held-out questions +against the same 128-segment, un-marked expanded corpus (43,968 estimated source +tokens): + +| System | What it is | Code | +|---|---|---| +| B0 | Closed-book — no corpus access at all | `ContextFabricBaselineRunner` | +| B1 | Truncated prompt — corpus crammed in until it runs out of budget, no ranking | `ContextFabricBaselineRunner` | +| B2 | Conventional top-k RAG — IDF-ranked segment retrieval | `ContextFabricBaselineRunner.BuildTopKText` | +| B3 | Single-node Context Fabric — the actual product answering path | `ContextFabricFeasibilityRunner` | +| B4 | HIVE Context Fabric — frozen multi-node acceptance artifact, not re-run per gate | `cf6-acceptance-*.json` | + +The corpus is deliberately **un-marked**: facts are embedded in ordinary prose, not +flagged with an `EVIDENCE:` line. That's a load-bearing property — an earlier +"GO" verdict was invalidated by an adversarial review that found the old fixture +let the model pattern-match markup instead of reading. See `DeterministicExpandedFabricCorpus` +and its `OpenExtractionReading` reader-prompt mode. + +The held-out set is 120 of a 150-question suite (30 held back as a dev set for +prompt tuning — see `docs/The Orc Context Fabric.md:963`). Categories and minimum +counts: Needle/local fact (40), Unanswerable (20), Multi-hop — two-hop + three-to- +five-hop chains (30), Exhaustive enumeration (15), Contradiction/change (10), +Global synthesis (15), Paraphrased retrieval (20). Every question was mechanically +verified against the *rendered* corpus text before being frozen — the verifier +checks that every `ExpectedTerm` actually appears in its claimed `ExpectedSegmentId`, +which caught a real generator bug during authoring (see commit `dcffd05e`). + +## 2. How an answer gets built (the part that can introduce false failures) + +This is the part worth reviewing hardest, because a bug here produces a wrong +*grade*, not a wrong *answer* — the model could be right and the harness could +still mark it failed, or vice versa. + +### B3 — `BuildEvidencePack` (`ContextFabricFeasibilityRunner.cs`) + +This is not benchmark-only code — it's the same evidence selection used by +`FabricNativeReaderService` and `HiveNativeRoleExecutorAdapter` in the real +product. Given a question and the corpus's evidence cards: + +1. Compute IDF (inverse document frequency) per term across the supplied cards, + after tokenizing with a 2-character minimum (`TokenizeForScoring`) — short + enough to keep 2-digit identifiers like `01` in `case-ledger-01`, since the + 3-character-minimum `Tokenize` would silently split that on the hyphen and + destroy the exact signal needed to tell `ledger-01` from `ledger-09`. +2. Exclude English stopwords entirely from scoring (`the`, `and`, `this`, ...), + so common words don't dilute the ranking signal that should come from + distinctive terms. +3. Score every card via `ScoreTextIdf` and greedily fill the evidence budget + (6,144 tokens by default, 3,072 for HIVE) in ranked order — **no fixed card + count cap**. Cards scoring 0 are excluded outright. + +**What was wrong before (fixed in commit `c68e01cf`):** `BuildEvidencePack` used +to hard-cap at 1/2/4 cards by question kind, with no documented cost/latency +justification. Global-synthesis questions need evidence from up to 8 segments — +capped at 4, the method was *structurally* incapable of answering them correctly +regardless of how good the ranking was. Comparing `ExpectedSegmentIds` against +`IncludedSegmentIds` on failing questions in the NO-GO run showed this was +exactly what was happening: CF frequently never gathered the segment containing +the answer. That's an evidence-*selection* bug, not a reasoning failure — and it +was inflating the failure count with cases where the model was never given a +chance to be right. + +### B2 — `BuildTopKText` (`ContextFabricBaselineRunner.cs`) + +Same fix, same reasoning, applied to the "conventional RAG" comparison baseline +(commit `c55e5058`). Before the fix, B2 used `Take(4)` with raw term-overlap +counting and no stopword filtering — it actually scored *worse* (21%) than the +dumber truncated-prompt baseline B1 (26%), which was itself a strong signal the +implementation was broken rather than that top-k RAG is inherently worse than +truncation. If B2 isn't fixed too, "B3 beats B2" isn't a fair claim — B2 would be +losing by construction, not by a real retrieval contest. + +### Exhaustive-category answers — `BuildExhaustiveAnswer` (`ContextFabricFeasibilityRunner.cs:~740`) + +Exhaustive questions ("list every case-file ID under ledger X") do **not** go +through `BuildEvidencePack` — they hit this separate method, because the goal +isn't "the top-N most relevant cards," it's "every card that actually belongs to +the named category." All 12 Exhaustive failures in the NO-GO run hit the same +error: the answer over-included claims from unrelated categories because the old +filter accepted a claim if it shared *any* word with the question — and corpus- +idiomatic filler words ("ledger", "recorded") appear in nearly every claim across +all 15 ledgers. + +Current logic (commit `3ef5fb0b`, line ~763): + +1. Tokenize the question, find which of its terms are actually present in the + corpus's cards, and compute each one's document frequency. +2. Classify the question as **entity-scoped** if its rarest present term appears + in fewer than half the cards (`minDocumentFrequency < cards.Count / 2.0`) — + e.g. `"case-ledger-01"` is genuinely rare relative to the corpus, so hard- + require that term. +3. Otherwise classify as **category-wide** (e.g. `"archive token"`, where every + segment is genuinely relevant) and fall back to "any non-stopword term + matches." + +This went through two earlier failed attempts (a pure IDF aggregate score +couldn't discriminate between two equally-rare ledger IDs; hard-requiring the +single rarest term broke a case where *every* segment is relevant) before landing +on the entity-scoped/category-wide split — both failure modes now have dedicated +regression tests. + +**Known residual risk, explicitly not fixed:** this classification is a +heuristic (`minDocumentFrequency < cards.Count / 2.0`), not a proof. A genuinely +category-wide question whose real content terms happen to have <50% document +frequency by corpus coincidence would still be mis-classified as entity-scoped. +Grok's adversarial review of this fix (`.orc/reviews/grok_20260703_185505.md`) +flagged this explicitly. Both real scenarios uncovered so far (ledger-scoped, +archive-token-wide) have tests; the boundary case does not. **If a future run +produces a new Exhaustive-category failure, check this heuristic first before +assuming it's a model capability gap.** + +## 3. How an answer gets graded — `FabricAnswerVerifier.NormalizeAndVerify` + +(`ContextFabricValidation.cs:838`) + +Given the model's raw JSON answer, corpus, and the question's ground truth: + +- **Structural sanity caps**, scaled to the question's own ground truth rather + than fixed globally — `maxAnswerChars = max(12000, 80 * ExpectedTerms.Count)`, + `maxCitationsPerClaim = max(32, ExpectedSegmentIds.Count)`. These exist to + reject genuine model garbage (runaway repetition, hallucinated citation + floods) without penalizing a legitimately large exhaustive enumeration, which + scales with the question's own expected-term count. +- Every citation must reference a real segment ID and pass + `FabricEvidenceProcessor.NormalizeCitation` (the quote must actually appear in + that segment — this is what makes `citation_precision` meaningful rather than + just "the model said a segment ID"). +- For non-abstention questions: every term in `question.ExpectedTerms` must + appear somewhere in the answer text or a citation quote, and every segment in + `question.ExpectedSegmentIds` must have been actually cited + (`verifiedSegments`) — not just any correct-sounding text, but evidence from + the *specific* segments the question was authored against. +- For `ExpectAbstention` questions: the model must abstain and say the corpus + doesn't establish the answer, and must not smuggle in factual claims anyway. + +`citation_precision` = valid citations / total citations attempted. A question +only "passes" (`Verification.Passed`) if `errors.Count == 0` — all of the above +in one gate, not a partial-credit score. + +## 4. JSON recovery — why answers don't get graded "wrong" for formatting noise + +(`FabricJson.ParseModelObject` in `ContextFabricValidation.cs`) + +Autoregressive models emit two specific token-boundary artifacts that would +otherwise turn a correct answer into an unparseable one and grade it as failed +for the wrong reason: + +1. **Keyword-suffix runs** — `falseC`, `trueX`, `nullValue` — where a JSON + keyword token runs directly into the next word token with no boundary. + `TrySanitizeLiteralSuffixes` walks the string state-aware and strips only + out-of-string garbage suffixes. +2. **Unescaped inner quotes** — a model quotes a term inline (`called it + "Chapter Alpha" a fitting name`) without escaping it, which otherwise + terminates the JSON string early and corrupts everything after. This + sanitizer tracks whether the current string is an object key vs. a value + before deciding whether a `:` or `,`/`}`/`]` is a real terminator — a value + string containing a quoted term immediately followed by `:` must not be cut + there (only key strings terminate on `:`). + +The parser tries, in order: strict parse → lenient parse (trailing +commas/comments) → both sanitizer orders composed together (`keyword→quote` and +`quote→keyword`, since either artifact can appear first and partially block the +other's own internal validity check) → throw. Composing both orders required +splitting each sanitizer into a raw scanning core (no internal validation) plus +a validated public wrapper, because a partially-repaired intermediate result +(quotes fixed, keyword suffix still broken) would otherwise be rejected by the +quote-sanitizer's own `JsonDocument.Parse` check before the keyword-fix pass ever +got a chance to run on it. + +Separately, `ContextFabricBaselineRunner` splits its catch into `JsonException` +(counts as `Succeeded=true`, incorrect-answer-recorded) vs. any other `Exception` +(counts as `Succeeded=false`, a genuine runtime failure) — so a run of B0/B1/B2 +always reaches `RunCompleted=true` unless the executor itself actually crashes, +rather than an unparseable answer masquerading as an infrastructure failure. + +## 5. The gate report — `ContextFabricBenchmarkGateEvaluator` + +Five metrics, each with a hardcoded target: + +| Metric | Target | What it means if it fails | +|---|---|---| +| `segment_terminal_coverage` | 1.0 | Not every segment was accepted during ingestion — an ingestion bug, not a model problem | +| `question_pass_rate` | **1.0** | At least one held-out question failed verification | +| `citation_precision` | 0.90 | The model is citing segments that don't actually support its claims | +| `max_prompt_tokens` | ≤ context limit | The evidence pack overflowed the context budget | +| `boundary_stitch_pass_rate` | 1.0 | A question spanning a segment boundary wasn't stitched correctly | + +**Important interpretation note:** `question_pass_rate`'s target is 1.0 — literal +100%. As configured, the gate reports `NO-GO` unless *every one* of 120 +held-out questions passes verification exactly. This is a deliberate fail-closed +design (see `docs/CONTEXT_FABRIC_BENCHMARK_MANIFEST.md`'s "explicit `Missing` +entries, not omitted rows" philosophy for the same pattern elsewhere), but it +also means B3 can substantially outscore every baseline (56/120 vs. B1's 31/120, +B2's 25/120 in the last run) and the gate will still say `NO-GO`. When reviewing +a gate report, look at the `systems` table's raw pass counts, not just the +top-line verdict, to judge whether a NO-GO reflects "close but not perfect" or +"still fundamentally broken." + +## 6. Change history relevant to grading correctness + +| Date | Commit / PR | What changed | +|---|---|---| +| 2026-07-03 | `01f3fd09` (PR #34) | JSON recovery pipeline (keyword-suffix sanitizer), `ModelAdmissionGate` 3B floor | +| 2026-07-04 00:21 | `c55e5058` (PR #34) | B2 `BuildTopKText` rewrite: IDF-weighted, budget-fill, no fixed `Take(4)` | +| 2026-07-04 01:23 | `c68e01cf` (PR #34) | B3 `BuildEvidencePack` fix: same IDF-weighted/budget-fill approach, removes the 1/2/4 `maxCards` cap — **the diagnosed root cause of the 56/120 NO-GO** | +| 2026-07-04 01:54 | `3ef5fb0b` (PR #34) | `BuildExhaustiveAnswer` entity-scoped vs. category-wide term filtering — fixes all 12 Exhaustive-category failures from the NO-GO run | +| 2026-07-04 02:00 | `40d79e1b` (PR #34) | Grok adversarial review of the Exhaustive fix: fixed a segment-lookup crash risk, documented the heuristic's known residual risk (section 2 above) | +| 2026-07-04 04:25 | PR #37 | Unescaped-inner-quote JSON sanitizer + key-vs-value colon handling, composed with the keyword-suffix sanitizer | +| 2026-07-04 04:46 | PR #38 | PowerShell 5.1 compatibility fixes in `Run-CF7GateExpanded.ps1` (re-run tooling only, not scoring logic) | +| 2026-07-04 | Grok review, `.orc/reviews/grok_20260703_223402.md` | Independent review of the full fix set above (36 files, 5,099 insertions), focused specifically on false-failure/false-pass risk in the scoring/parsing paths. Verdict: **CLEAN**, no findings. | + +**The 120-question NO-GO run on record (2026-07-04T00:36:28Z, B3 56/120) predates +the `BuildEvidencePack` and `BuildExhaustiveAnswer` fixes** — it measured the old, +known-buggy evidence selection. It is not yet known what B3 scores with the +current, fixed code; that is the open item this document supports reviewing +before the next run. + +## 7. Open bug: KV-cache exhaustion invalidates most of a full 120-question run + +**Status as of 2026-07-04: unresolved, high priority.** This is not a fleet +quirk like section 8 below — it reproduced on NEWCOREPC, the machine with the +most headroom, and it likely invalidates most B0/B3 results from any full run +since the `BuildEvidencePack`/`BuildTopKText` fixes landed. + +The 2026-07-04 02:44:29-elapsed full 120-question run on NEWCOREPC (Gemma-4-12B, +8192 context) reported B3 at 12/120 — *worse* than the pre-fix NO-GO's 56/120. +Inspecting the raw result JSON showed why: 216 of B3's 223 failed +question-attempts had `verification.errors: ["Native inference failed while +draining a prompt batch: NoKvSlot."]` — a native KV-cache exhaustion, not a +wrong answer. Only 7 failures were genuine (6 "reducer output references +claims outside its children", 1 unterminated-JSON) — these are the +`ContextFabricFeasibilityRunner.cs:515` reducer-validation gate correctly +catching Gemma-4-12B inventing a claim ID not present in its supplied +children, which the reducer prompt explicitly forbids ("claimIds may contain +only IDs present in the input"). That's the harness's honesty check working as +designed, not a harness bug — a real, if small, model hallucination rate worth +tracking separately from the infrastructure noise below, but not something to +"fix" in the scoring logic. B0 (closed-book) then failed near-identically once +B3 had already burned through the shared KV pool. +**The 12/120 and B0's failure are not meaningful capability measurements** — +they're an infrastructure crash wearing a NO-GO costume. + +Root cause, traced through the code: `AdapterManager.cs` already documents and +guards against a *related* but distinct problem — llama.cpp's KV-cache sequence +IDs are minted monotonically and never recycled, even after a `Conversation` is +`Dispose()`d (see the comment at `AdapterManager.cs:48-56`, referencing a prior +crash "at exactly the 257th reader conversation"). The existing fix, +`SequenceRecycleThreshold = 128` (rebuild the role's executor — a fresh native +context — every 128 minted conversations, at an idle point) and +`SequenceHardLimit = 240` (fail closed with a managed exception rather than let +the native assert kill the process), protects against exhausting the *count* of +sequence IDs. **It does not protect against exhausting actual KV-cache +*memory*, which is a function of prompt size × live-but-unrecycled sequences, +not conversation count.** Since `BuildEvidencePack`'s fix removed the +`maxCards` cap, a single LocalFact question observed in this run's JSON pulled +in **26 segments** (6,309 prompt tokens) where the old, capped code would have +used 1-4 cards — meaning each conversation now reserves far more of the shared +KV pool before being abandoned. `NoKvSlot` is reachable well before the +128-conversation recycle point fires, and indeed did: the managed hard-limit +exception (which has its own distinct message, "has minted N native sequence +slots...") never appeared in the log — only the native `NoKvSlot` — confirming +the existing protection's own counters never tripped even though the native +pool was already exhausted. + +**Stopgap attempted 2026-07-04, empirically DID NOT WORK — root cause is +narrower than first diagnosed.** `SequenceRecycleThreshold` was lowered from +128 to 24 (`AdapterManager.cs`) on the theory that conversation *count* was the +gating factor. A second full 120-question run with this change produced a +**byte-for-byte identical** question-pass/fail trace to the pre-fix run — same +33 failures, same 12 successes, same 75 failures after, in the exact same +positions. A 5x lower threshold changing literally nothing about the outcome +means conversation-count-based recycling was never the actual mechanism in +play here, or recycling isn't firing at all regardless of the threshold value. + +Re-reading `AdapterManager.GetOrCreateConversationAsync`: the recycle-eligible +branch only runs when `existing.ActiveCount == 0` — the check is +`if (minted < SequenceRecycleThreshold || existing.ActiveCount > 0) { serve +without recycling }`. If `ActiveCount` is stuck above zero for some reason +(a `TrackedConversation` not being disposed/decremented correctly somewhere in +the call chain), this condition is true unconditionally regardless of `minted` +or the threshold — recycling would never trigger no matter how low the +threshold is set, which fully explains the null result observed. **This has +not been proven, only inferred from the identical-trace result** — it's the +most defensible next hypothesis, not a confirmed diagnosis. The original +"recycle by tokens not count" idea may still be correct as a longer-term +design, but it's moot until whatever is keeping `ActiveCount` from reaching +zero (if that's really what's happening) is found and fixed; a threshold +adjustment of any kind cannot help if the recycle branch is never reached. + +**Next investigation step:** instrument or step through +`ActiveCount`/`ConversationsCreated` for the shared role executor across a +run — confirmed via `Program.cs:194` that B0-B3 all share one +`NativeRoleRuntime`/`AdapterManager` instance for the whole `cf7-gate-expanded` +suite, so the cumulative-pressure theory itself still holds; what's now in +question is only why recycling isn't relieving that pressure. Two +single-constant changes have now been tried and evidence suggests the recycle +path may not run at all — that needs actual data, not another guess. + +Added an opt-in diagnostic for exactly this (`AdapterManager.cs`, purely +additive, zero behavior change unless enabled): set +`THEORC_KVCACHE_DIAGNOSTICS=1` before a run and every recycle-eligibility check +prints one line to **stdout** (not stderr — `Run-CF7GateExpanded.ps1` pipes the +benchmark exe through `2>&1 | Tee-Object`, and PowerShell treats native stderr +output as a terminating `NativeCommandError`, which killed the run on first use +before this was caught) — `role=... served-without-recycle +minted=... activeCount=... threshold=... reason=under-threshold| +active-conversations-outstanding` or `role=... RECYCLING minted=... +activeCount=...`. Grep the run's console log for +`reason=active-conversations-outstanding` — if that's the reason on every +single check (never `under-threshold`), it would confirm `ActiveCount` never +reaches zero and recycling truly never fires, regardless of the threshold. + +**Result from the first real run with this enabled:** `ActiveCount` was 0 on +every single check (hundreds of checks, zero `active-conversations-outstanding` +occurrences) and recycling fired correctly at every threshold crossing — yet +`NoKvSlot` still occurred. **This rules out the stuck-`ActiveCount` hypothesis +entirely.** The recycle mechanism (both the count threshold and the +`ActiveCount` gate) works exactly as designed; the actual root cause is +something recycling doesn't address at all — most likely that rebuilding the +executor doesn't fully reclaim the previous one's native KV-cache memory +before the new one starts allocating, or that a single oversized evidence pack +can exhaust the pool on its own regardless of recycling frequency. Still +unresolved; this narrows the next investigation to executor-disposal memory +reclamation rather than conversation-count bookkeeping. + +**What this means for reading any prior or future run's B3/B0 numbers:** check +`verification.errors` in the raw JSON, not just the summary line, before +trusting a low pass count as a real capability result — grep for `NoKvSlot` +across `cf0_*.json` and `cf7_baseline_b0_*.json`. If present in more than a +handful of entries, the run needs to be redone once the above is actually +root-caused and fixed, not interpreted as-is — the `SequenceRecycleThreshold` +change alone is confirmed **not** to resolve this. + +## 8. Known fleet/environment issues (not scoring-logic bugs) + +These are infrastructure problems observed while running the gate on specific +machines. They affect whether a run *executes*, not whether the grading logic +above is correct — recorded here so a future NO-GO or crash isn't mistaken for +a scoring bug or a model capability gap. + +- **HARDCOREPC (RTX 3050, 6GB VRAM) native-library load regression, 2026-07-04.** + After a clean rebuild (`rmdir` of `bin`/`obj`/`publish` followed by + `dotnet publish -r win-x64 --self-contained true`), every model load on this + machine fails immediately with `TypeInitializationException: The type + initializer for 'LLama.Native.NativeApi' threw an exception. | Inner: + RuntimeError: Failed to load the native library.` — before any inference is + attempted (`segments 0/128, questions 0/N`). Confirmed **not** model-specific: + reproduced identically with both `Qwen3.5-4B-Q8_0.gguf` and + `qwen2.5-coder-7b-instruct-q5_k_m.gguf` (the latter had loaded and run + successfully on this same machine earlier in the same session, before the + clean rebuild). Native DLLs in `publish/runtimes/win-x64/native/*` are + present at expected file sizes across all variants (avx/avx2/avx512/cuda12/ + noavx), so this isn't a missing- or truncated-file problem — the underlying + first-chance exception is being swallowed by .NET's cached + `TypeInitializationException` behavior (a static constructor's exception is + saved and rethrown verbatim on every later access), so the *real* root cause + is not yet visible from application logs alone. **Not yet resolved** — + needs investigation with a debugger attached or `COMPlus_LegacyExceptionHandling`/ + first-chance-exception logging enabled, ideally comparing against + NEWCOREPC and HARDCORELAPTOPMSI where the identical `dotnet publish -r + win-x64 --self-contained true` recipe succeeded the same night. HARDCOREPC + was left idle (no benchmark process running) pending this investigation. + +- **Windows/OpenSSH process detachment.** A benchmark launched via + `ssh host "start /b ... "` does **not** survive the SSH session closing — + Windows' OpenSSH server tears down the whole console process tree when the + channel closes, killing detached children too. Two working alternatives: + keep the `ssh host "long-running command"` invocation itself running under + the orchestrating side's own background-task mechanism (simplest, used for + NEWCOREPC/HARDCOREPC runs), or register a Task Scheduler job + (`schtasks /create ... /tr `) and trigger it with + `schtasks /run` (works even if the orchestrating side disconnects, used for + the HARDCORELAPTOPMSI run). When using `schtasks`, the `/tr` command runs + via `CreateProcess`, not a shell — `>`/`2>&1` redirection syntax is silently + ignored unless wrapped in a `.bat` file or `cmd /c "..."`. + +## 9. Re-running + +See [CONTEXT_FABRIC_BENCHMARK_MANIFEST.md § Re-Running The Expanded 120-Question Gate](CONTEXT_FABRIC_BENCHMARK_MANIFEST.md#re-running-the-expanded-120-question-gate) +for the canonical recipe (`Tools/ContextFabricBench/Run-CF7GateExpanded.ps1`). diff --git a/docs/README.md b/docs/README.md index e1fe8f4c..babb5d63 100644 --- a/docs/README.md +++ b/docs/README.md @@ -85,6 +85,13 @@ adversarial-review context and may contain deeper implementation notes. the gate sits in the run lifecycle and how Off/Advisory/Gated modes behave - [REVIEWER_ADAPTER_GUIDE.md](REVIEWER_ADAPTER_GUIDE.md) — plan to train a local reviewer model (**parked** since 2026-06-13; see [reviewer-adapter/00-index.md](reviewer-adapter/00-index.md)) +- [CONTEXT_FABRIC_TEST_HARNESS.md](CONTEXT_FABRIC_TEST_HARNESS.md) — how the CF-7 benchmark grades + answers: evidence selection, JSON recovery, verification rules, and known residual risks, kept + independent of any single run's result so the scoring logic itself can be reviewed +- [CONTEXT_FABRIC_BENCHMARK_MANIFEST.md](CONTEXT_FABRIC_BENCHMARK_MANIFEST.md) — pinned fixture + manifest shape and the CF-7 gate report schema, plus the re-run recipe +- [CONTEXT_FABRIC_BENCHMARK_CORPUS.md](CONTEXT_FABRIC_BENCHMARK_CORPUS.md) — public benchmark shelf, + private/licensed corpus rules, phase-to-corpus mapping ---