From c54ea52f111e6fac78342bc2aa8817146f749bad Mon Sep 17 00:00:00 2001 From: hardcoreerik Date: Tue, 21 Jul 2026 07:03:30 -0700 Subject: [PATCH 1/4] feat: HV-2 scheduling driver + native admission telemetry endpoint Adds Tools/Hv2SchedulingRunner for HV-2 (docs/ NATIVE_RUNTIME_HIVE_VALIDATION_PLAN.md): proves capability/resource- aware scheduling across the fleet. Since NativeContextSize is a per-worker-process startup config rather than a per-job HIVE parameter, the two HV-2 checks (large-context denial on the low-VRAM box, small-context admission everywhere) run as two separate fleet configurations of the same three machines against the same driver, using --phase large|small. Also exposes RuntimeReservationSnapshot (RejectedAdmissionCount, LastRejectionReason, Total/Reserved/AvailableBytes) over a new GET /hive/native-telemetry endpoint on HiveNodeServer, wired from HiveService.cs after building the native runtime. This telemetry existed in-process but had no remote observability surface on a headless worker -- only the Avalonia GUI's diagnostics panel could see it. Unauthenticated read-only, same posture as the existing /hive/info endpoint. Co-Authored-By: Claude Sonnet 5 --- OrchestratorIDE.Daemon/HiveService.cs | 5 + .../Services/Hive/HiveNodeServer.cs | 21 ++ .../Hv2SchedulingRunner.csproj | 17 ++ Tools/Hv2SchedulingRunner/Program.cs | 282 ++++++++++++++++++ 4 files changed, 325 insertions(+) create mode 100644 Tools/Hv2SchedulingRunner/Hv2SchedulingRunner.csproj create mode 100644 Tools/Hv2SchedulingRunner/Program.cs diff --git a/OrchestratorIDE.Daemon/HiveService.cs b/OrchestratorIDE.Daemon/HiveService.cs index 8bd63c73..91b2a1b7 100644 --- a/OrchestratorIDE.Daemon/HiveService.cs +++ b/OrchestratorIDE.Daemon/HiveService.cs @@ -197,6 +197,11 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) roleBindings: roleBindings); nativeExecutor = new HiveNativeRoleExecutorAdapter(nativeRuntime, _cfg.WorkspaceRoot); + // HV-2 (docs/NATIVE_RUNTIME_HIVE_VALIDATION_PLAN.md): RejectedAdmissionCount/ + // LastRejectionReason existed in-process (RuntimeOrchestrator) but had no remote + // observability surface on a headless worker -- only the Avalonia GUI's own + // diagnostics panel could see them. Exposed read-only over GET /hive/native-telemetry. + _nodeServer.NativeTelemetryProvider = () => nativeRuntime.GetReservationSnapshot(); } _worker = new HiveWorkerAgent diff --git a/OrchestratorIDE/Services/Hive/HiveNodeServer.cs b/OrchestratorIDE/Services/Hive/HiveNodeServer.cs index 32cc9e02..71b63bb9 100644 --- a/OrchestratorIDE/Services/Hive/HiveNodeServer.cs +++ b/OrchestratorIDE/Services/Hive/HiveNodeServer.cs @@ -65,6 +65,16 @@ public sealed class HiveNodeServer : IDisposable /// public Action? ShutdownCallback { get; set; } + /// + /// Injected by the app (Daemon: HiveService, after building its NativeRoleRuntime) so + /// GET /hive/native-telemetry can report this node's own admission/reservation state + /// (RuntimeReservationSnapshot: Reservations, Total/Reserved/AvailableBytes, + /// RejectedAdmissionCount, LastRejectionReason) without this class taking a hard + /// dependency on the native runtime types. Null when native execution isn't configured + /// on this node (e.g. Ollama-only) or before the runtime has been built. + /// + public Func? NativeTelemetryProvider { get; set; } + // Pending pairing sessions: sessionId → (request, expiry, initiator-remote-ip) private readonly Dictionary _pendingPairings = []; // Completed results: sessionId → (response, stored-at). Pruned after 10 min. @@ -435,6 +445,17 @@ private async Task HandleAsync(HttpListenerContext ctx, CancellationToken ct) Ok(resp, JsonSerializer.Serialize(_info)); return; } + // Read-only native-runtime admission telemetry — same unauthenticated posture as + // /hive/info (operational status, not a secret or control action). {} when native + // execution isn't configured on this node, rather than 404, so a fleet-wide poller + // doesn't need to special-case Ollama-only boxes. + if (method == "GET" && path == "/hive/native-telemetry") + { + var snapshot = NativeTelemetryProvider?.Invoke(); + Ok(resp, JsonSerializer.Serialize(snapshot ?? new { }, _jsonOut)); + return; + } + if (method == "POST" && path == "/hive/pair") { var remoteIp = req.RemoteEndPoint?.Address?.ToString() ?? ""; diff --git a/Tools/Hv2SchedulingRunner/Hv2SchedulingRunner.csproj b/Tools/Hv2SchedulingRunner/Hv2SchedulingRunner.csproj new file mode 100644 index 00000000..626a37d7 --- /dev/null +++ b/Tools/Hv2SchedulingRunner/Hv2SchedulingRunner.csproj @@ -0,0 +1,17 @@ + + + + Exe + net10.0 + enable + enable + hv2-scheduling-runner + Hv2SchedulingRunner + + + + + + + + diff --git a/Tools/Hv2SchedulingRunner/Program.cs b/Tools/Hv2SchedulingRunner/Program.cs new file mode 100644 index 00000000..06fc8e24 --- /dev/null +++ b/Tools/Hv2SchedulingRunner/Program.cs @@ -0,0 +1,282 @@ +// Copyright (C) 2025-present hardcoreerik / TheOrc contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +using System.Net.Http.Json; +using System.Text.Json; +using OrchestratorIDE.Services.Hive; + +namespace Hv2SchedulingRunner; + +/// +/// HV-2 driver (docs/NATIVE_RUNTIME_HIVE_VALIDATION_PLAN.md): proves capability/resource-aware +/// scheduling across the fleet. NativeContextSize is a per-worker-process startup config, not a +/// per-job parameter the HIVE contract exposes, so the two HV-2 checks map to two separate fleet +/// configurations of the same three machines rather than two job shapes against one config: +/// +/// --phase large: every worker started with a context size whose estimated footprint exceeds +/// the low-VRAM box's budget. That box must deny with a real RuntimeAdmissionDeniedException +/// (correct numbers, observable via GET /hive/native-telemetry's RejectedAdmissionCount/ +/// LastRejectionReason) while the higher-VRAM boxes complete normally -- and must never fall +/// back to Ollama. +/// --phase small: every worker started with a context size that fits everywhere. All three +/// must complete -- proving the large-phase denial was footprint-driven, not "that box +/// always fails." +/// +/// Run this once per phase against a fleet already reconfigured (NativeContextSize env var) and +/// restarted for that phase. +/// +internal static class Program +{ + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, + }; + + public static async Task Main(string[] args) + { + var warchief = GetArg(args, "--warchief") ?? "http://localhost:7079"; + var outDir = GetArg(args, "--out") ?? Path.Combine(Environment.CurrentDirectory, ".orc", "hv-2-lane"); + var phase = GetArg(args, "--phase") + ?? throw new InvalidOperationException("--phase large|small is required."); + if (phase is not ("large" or "small")) + throw new InvalidOperationException("--phase must be 'large' or 'small'."); + var role = GetArg(args, "--role") ?? "Coder"; + var timeoutMs = int.TryParse(GetArg(args, "--timeout-ms"), out var t) ? t : 300_000; + + var workers = new List<(string Id, string NodeUrl, bool ExpectDenied)>(); + foreach (var slot in new[] { "a", "b", "c" }) + { + var id = GetArg(args, $"--worker-{slot}"); + var nodeUrl = GetArg(args, $"--worker-{slot}-node"); + if (id is null || nodeUrl is null) continue; + var isLowVram = string.Equals(GetArg(args, "--low-vram-worker"), id, StringComparison.OrdinalIgnoreCase); + workers.Add((id, nodeUrl, isLowVram && phase == "large")); + } + if (workers.Count == 0) + throw new InvalidOperationException( + "No workers configured. Pass --worker-a --worker-a-node (and -b/-c), " + + "plus --low-vram-worker to mark which one is expected to deny in --phase large."); + + Directory.CreateDirectory(outDir); + using var http = new HttpClient { BaseAddress = new Uri(warchief), Timeout = TimeSpan.FromMinutes(10) }; + + var report = new Hv2Report { Warchief = warchief, Phase = phase, StartedAt = DateTimeOffset.UtcNow }; + + try + { + Console.WriteLine($"HV-2 scheduling run, phase={phase}, against {warchief}"); + + // Baseline each worker's native telemetry BEFORE dispatch, so a denial's effect on + // RejectedAdmissionCount can be verified as an increase, not just a nonzero value + // that might already be nonzero from an earlier run. + var baselineTelemetry = new Dictionary(); + foreach (var w in workers) + baselineTelemetry[w.Id] = await TryFetchTelemetryAsync(w.NodeUrl); + + var otherIds = workers.Select(w => w.Id).ToArray(); + var workUnits = workers.Select(w => new WorkUnit + { + WorkUnitId = $"hv2-{phase}-{w.Id}", + Title = $"HV-2 {phase}-context scheduling proof targeting {w.Id}", + Role = role, + ExecutionKind = HiveExecutionKinds.NativeAgent, + Requirements = new ResourceRequirements + { + ExcludedWorkerIds = otherIds.Where(id => !string.Equals(id, w.Id, StringComparison.OrdinalIgnoreCase)).ToArray(), + }, + Spec = $"Create a file named hv2_proof.txt in the workspace root containing exactly " + + $"this single line and nothing else: HV2-PROOF {phase} {w.Id}", + TimeoutMs = timeoutMs, + }).ToList(); + + var campaign = new CampaignDefinition { Name = $"hv2-{phase}", WorkUnits = workUnits }; + Console.WriteLine($"Submitting campaign {campaign.CampaignId}: {workUnits.Count} work unit(s)..."); + using (var resp = await http.PostAsJsonAsync("/hive/campaigns", campaign, JsonOptions)) + resp.EnsureSuccessStatusCode(); + report.CampaignId = campaign.CampaignId; + + foreach (var unit in workUnits) + { + var target = workers.First(w => unit.WorkUnitId.EndsWith(w.Id, StringComparison.Ordinal)); + var taskId = $"{campaign.CampaignId}-{unit.WorkUnitId}"; + var deadline = DateTime.UtcNow.AddMilliseconds(unit.TimeoutMs + 60_000); + HiveTaskStatusResponse? last = null; + var seenClaimed = false; + var consecutiveNotFound = 0; + while (DateTime.UtcNow < deadline) + { + using var statusResp = await http.GetAsync($"/hive/tasks/{taskId}"); + if (statusResp.IsSuccessStatusCode) + { + consecutiveNotFound = 0; + var body = await statusResp.Content.ReadFromJsonAsync(JsonOptions); + if (body is not null) + { + last = body; + if (body.Status is "claimed" or "running") seenClaimed = true; + if (body.Status is "completed" or "failed" or "timeout" or "cancelled") break; + } + } + else if (statusResp.StatusCode == System.Net.HttpStatusCode.NotFound && seenClaimed) + { + if (++consecutiveNotFound >= 3) { if (last is not null) last.Status = "swept-unknown"; break; } + } + await Task.Delay(2000); + } + + var evidence = BuildJobEvidence(unit.WorkUnitId, target.Id, target.ExpectDenied, last); + report.Jobs.Add(evidence); + Console.WriteLine($" [{evidence.Status}] {unit.WorkUnitId} -> worker={target.Id} " + + $"expectDenied={target.ExpectDenied} runtime={evidence.RuntimeName ?? "-"} " + + $"errorMsg={evidence.ErrorMsg ?? "-"} matchesExpectation={evidence.MatchesExpectation}"); + } + + // Post-dispatch telemetry: only meaningful for the worker(s) expected to deny. + foreach (var w in workers.Where(w => w.ExpectDenied)) + { + var after = await TryFetchTelemetryAsync(w.NodeUrl); + var before = baselineTelemetry[w.Id]; + var rejectedDelta = (after?.RejectedAdmissionCount ?? 0) - (before?.RejectedAdmissionCount ?? 0); + report.TelemetryChecks.Add(new Hv2TelemetryCheck + { + WorkerId = w.Id, + NodeUrl = w.NodeUrl, + RejectedAdmissionCountBefore = before?.RejectedAdmissionCount, + RejectedAdmissionCountAfter = after?.RejectedAdmissionCount, + RejectedAdmissionCountIncreased = rejectedDelta > 0, + LastRejectionReason = after?.LastRejectionReason, + }); + Console.WriteLine($" telemetry[{w.Id}]: rejected {(before is null ? "?" : before.RejectedAdmissionCount.ToString())} -> " + + $"{(after is null ? "?" : after.RejectedAdmissionCount.ToString())}, reason: {after?.LastRejectionReason ?? "(none)"}"); + } + + report.FinishedAt = DateTimeOffset.UtcNow; + var allJobsMatchedExpectation = report.Jobs.All(j => j.MatchesExpectation); + var allTelemetryConfirmed = report.TelemetryChecks.All(c => c.RejectedAdmissionCountIncreased); + report.Passed = allJobsMatchedExpectation && allTelemetryConfirmed; + + var outPath = Path.Combine(outDir, $"hv2_{phase}_{DateTime.UtcNow:yyyyMMdd_HHmmss}.json"); + await File.WriteAllTextAsync(outPath, + JsonSerializer.Serialize(report, new JsonSerializerOptions(JsonOptions) { WriteIndented = true })); + + Console.WriteLine(); + Console.WriteLine($"All jobs matched expectation: {allJobsMatchedExpectation}"); + Console.WriteLine($"All expected-denial telemetry confirmed: {allTelemetryConfirmed}"); + Console.WriteLine($"Verdict: {(report.Passed ? "PASS" : "FAIL")}"); + Console.WriteLine($"Evidence written: {outPath}"); + return report.Passed ? 0 : 2; + } + catch (Exception ex) + { + report.Error = ex.ToString(); + report.FinishedAt = DateTimeOffset.UtcNow; + var outPath = Path.Combine(outDir, $"hv2_{phase}_FAILED_{DateTime.UtcNow:yyyyMMdd_HHmmss}.json"); + await File.WriteAllTextAsync(outPath, + JsonSerializer.Serialize(report, new JsonSerializerOptions(JsonOptions) { WriteIndented = true })); + Console.Error.WriteLine($"Run failed: {ex.Message}"); + Console.Error.WriteLine($"Partial evidence written: {outPath}"); + return 1; + } + } + + private static Hv2JobEvidence BuildJobEvidence( + string workUnitId, string workerId, bool expectDenied, HiveTaskStatusResponse? last) + { + var status = last?.Status ?? "unknown"; + var attestation = last?.Attestation; + var isNativeRuntime = attestation is not null && + string.Equals(attestation.RuntimeName, "NativeRoleRuntime", StringComparison.Ordinal); + var errorMsg = last?.ErrorMsg; + // A real, fail-closed denial surfaces as a "failed" task whose error text is the + // RuntimeAdmissionDeniedException's own message (wrapped by HiveWorkerAgent's "Phase 3B + // does not fall back" guard) -- never a silent reroute to Ollama, and never a 200 with + // Attestation.RuntimeName pointing at anything other than NativeRoleRuntime. + var wasDenied = status == "failed" && + (errorMsg?.Contains("admission", StringComparison.OrdinalIgnoreCase) ?? false); + var wasAdmitted = status == "completed" && isNativeRuntime; + var matchesExpectation = expectDenied ? wasDenied : wasAdmitted; + + return new Hv2JobEvidence + { + WorkUnitId = workUnitId, + WorkerId = workerId, + ExpectDenied = expectDenied, + Status = status, + RuntimeName = attestation?.RuntimeName, + Backend = attestation?.Backend, + Stats = last?.Metrics ?? [], + ErrorMsg = errorMsg, + WasDenied = wasDenied, + WasAdmitted = wasAdmitted, + MatchesExpectation = matchesExpectation, + }; + } + + private static async Task TryFetchTelemetryAsync(string nodeUrl) + { + try + { + using var http = new HttpClient { Timeout = TimeSpan.FromSeconds(15) }; + using var resp = await http.GetAsync($"{nodeUrl.TrimEnd('/')}/hive/native-telemetry"); + if (!resp.IsSuccessStatusCode) return null; + return await resp.Content.ReadFromJsonAsync(JsonOptions); + } + catch + { + return null; + } + } + + private static string? GetArg(string[] args, string name) + { + var idx = Array.IndexOf(args, name); + return idx >= 0 && idx + 1 < args.Length ? args[idx + 1] : null; + } +} + +internal sealed class NativeTelemetry +{ + public long RejectedAdmissionCount { get; set; } + public string? LastRejectionReason { get; set; } + public long TotalBytes { get; set; } + public long ReservedBytes { get; set; } + public long AvailableBytes { get; set; } +} + +internal sealed class Hv2Report +{ + public string Warchief { get; set; } = ""; + public string Phase { get; set; } = ""; + public string CampaignId { get; set; } = ""; + public DateTimeOffset StartedAt { get; set; } + public DateTimeOffset? FinishedAt { get; set; } + public List Jobs { get; set; } = []; + public List TelemetryChecks { get; set; } = []; + public bool Passed { get; set; } + public string? Error { get; set; } +} + +internal sealed class Hv2JobEvidence +{ + public string WorkUnitId { get; set; } = ""; + public string WorkerId { get; set; } = ""; + public bool ExpectDenied { get; set; } + public string Status { get; set; } = ""; + public string? RuntimeName { get; set; } + public string? Backend { get; set; } + public Dictionary Stats { get; set; } = []; + public string? ErrorMsg { get; set; } + public bool WasDenied { get; set; } + public bool WasAdmitted { get; set; } + public bool MatchesExpectation { get; set; } +} + +internal sealed class Hv2TelemetryCheck +{ + public string WorkerId { get; set; } = ""; + public string NodeUrl { get; set; } = ""; + public long? RejectedAdmissionCountBefore { get; set; } + public long? RejectedAdmissionCountAfter { get; set; } + public bool RejectedAdmissionCountIncreased { get; set; } + public string? LastRejectionReason { get; set; } +} From d6078b8b2d757792aed450a0aa7be069abfb17b4 Mon Sep 17 00:00:00 2001 From: hardcoreerik Date: Tue, 21 Jul 2026 07:20:39 -0700 Subject: [PATCH 2/4] fix: HV-2 driver denial classification -- task ErrorMsg is a generic wrapper Confirmed empirically during real-fleet calibration: a genuine admission denial with correct numbers ("Requires ~6.8 GB, only 5.6 GB available") still surfaces as HiveWorkerAgent's generic "native role runtime failed. Phase 3B does not fall back." wrapper text at the task level -- the detailed RuntimeAdmissionDeniedException message never reaches HiveTaskResult.ErrorMsg, only the worker's own local log. Classify denial by task status alone (this execution kind can structurally never fall back instead) and let the separate /hive/native-telemetry check be the authority on whether it was specifically an admission denial with correct numbers. Co-Authored-By: Claude Sonnet 5 --- Tools/Hv2SchedulingRunner/Program.cs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/Tools/Hv2SchedulingRunner/Program.cs b/Tools/Hv2SchedulingRunner/Program.cs index 06fc8e24..e8477441 100644 --- a/Tools/Hv2SchedulingRunner/Program.cs +++ b/Tools/Hv2SchedulingRunner/Program.cs @@ -187,12 +187,16 @@ private static Hv2JobEvidence BuildJobEvidence( var isNativeRuntime = attestation is not null && string.Equals(attestation.RuntimeName, "NativeRoleRuntime", StringComparison.Ordinal); var errorMsg = last?.ErrorMsg; - // A real, fail-closed denial surfaces as a "failed" task whose error text is the - // RuntimeAdmissionDeniedException's own message (wrapped by HiveWorkerAgent's "Phase 3B - // does not fall back" guard) -- never a silent reroute to Ollama, and never a 200 with - // Attestation.RuntimeName pointing at anything other than NativeRoleRuntime. - var wasDenied = status == "failed" && - (errorMsg?.Contains("admission", StringComparison.OrdinalIgnoreCase) ?? false); + // A real, fail-closed denial surfaces as a "failed" task -- but the task-level ErrorMsg + // the Warchief actually sees is HiveWorkerAgent's generic wrapper text ("native role + // runtime failed. Phase 3B does not fall back."), NOT the RuntimeAdmissionDeniedException's + // own detailed message (which only reaches the worker's own local log). Confirmed + // empirically during HV-2 calibration: a genuine admission denial with correct numbers + // ("Requires ~6.8 GB, only 5.6 GB available...") still produced this exact generic + // wrapper as ErrorMsg. So "failed" here (this shape can structurally never fall back + // instead) is what proves fail-closed; the SEPARATE /hive/native-telemetry check below + // is what proves it was specifically an admission denial with correct numbers. + var wasDenied = status == "failed"; var wasAdmitted = status == "completed" && isNativeRuntime; var matchesExpectation = expectDenied ? wasDenied : wasAdmitted; From b62a8f845ba636759ba55cb8bbcc790b0fe86049 Mon Sep 17 00:00:00 2001 From: hardcoreerik Date: Tue, 21 Jul 2026 07:22:01 -0700 Subject: [PATCH 3/4] docs: HV-2 real-fleet run results -- closed for 6GB/8GB spread HardcorePC (6GB) denied at ctx=40000 with a real RuntimeAdmissionDeniedException and correct telemetry (RejectedAdmissionCount 3->6, "Requires ~6.8 GB, only 5.6 GB available"), then admitted cleanly at ctx=8192 -- same box, same model, proving denial is footprint-driven not box-driven. HardcoreLaptopMSI (8GB) admitted at both. The 16GB leg (NewcorePC) is blocked on a separate, genuine finding: running the Daemon locally on NewcorePC to add it as a worker regenerated its own HIVE identity (same DPAPI/AES-GCM collision as HV-1, now on the warchief side), and unlike HV-1's workers this can't be re-paired -- the Daemon can never approve incoming pairing by its own design. Filed as an open follow-up rather than worked around. Also documents a driver classification bug found and fixed mid-campaign, and a real inbound- firewall gap worked around via SSH rather than asking to change system settings. Co-Authored-By: Claude Sonnet 5 --- docs/NATIVE_RUNTIME_HIVE_VALIDATION_PLAN.md | 82 +++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/docs/NATIVE_RUNTIME_HIVE_VALIDATION_PLAN.md b/docs/NATIVE_RUNTIME_HIVE_VALIDATION_PLAN.md index 9eef23dc..c0478e56 100644 --- a/docs/NATIVE_RUNTIME_HIVE_VALIDATION_PLAN.md +++ b/docs/NATIVE_RUNTIME_HIVE_VALIDATION_PLAN.md @@ -241,6 +241,88 @@ estimator, large `ContextLength`) fits 16 GB and 8 GB but must be **denied** on - Also run the inverse: a small job admitted on all three, proving denial is footprint-driven, not box-driven. +**2026-07-21 — Driver built (`Tools/Hv2SchedulingRunner`, PR #88), real run against the fleet. +CLOSED for the two-machine spread (6 GB deny / 8 GB admit); NewcorePC (16 GB) excluded — a +genuine Daemon-architecture constraint, not a scheduling gap.** + +`NativeContextSize` is a per-worker-process startup config, not a per-job HIVE parameter, so +the "large footprint" and "small footprint" checks run as two separate fleet configurations of +the same machines rather than two job shapes against one running config (`--phase large|small`). +Also added `GET /hive/native-telemetry` on `HiveNodeServer` (`RejectedAdmissionCount`, +`LastRejectionReason`, VRAM totals) — this existed in-process but had no remote observability +surface on a headless worker before this. + +**Calibration, computed before touching real hardware:** `OrcScheduler.EstimateRequiredBytes` +is `legacy(base+adapter file size) + 256 MB (CUDA overhead) + 384 MB (compute buffer) + kvBytes`, +where `kvBytes` scales linearly with `ContextLength`. Back-solving from HV-1's own observed +figures (base ≈1.881 GB, ctx=8192 → ~3.4 GB total ⇒ kv(8192) ≈ 894 MB) gave `ctx=40000` ⇒ +≈6.77 GB total — denied on 6 GB, comfortable margin under 8 GB. Confirmed near-exact on real +hardware: the actual denial read **"Requires ~6.8 GB, only 5.6 GB available."** + +**Large-context phase (ctx=40000), pinned per-worker via `ExcludedWorkerIds`:** +- **HardcorePC (6 GB): DENIED**, a real `RuntimeAdmissionDeniedException` surfaced as `status: + "failed"` (this execution kind is structurally fail-closed — no Ollama fallback path is even + reachable). Confirmed via `/hive/native-telemetry`: `RejectedAdmissionCount` 3→6 (exactly the + 3 retry attempts this run made), `LastRejectionReason: "Requires ~6.8 GB, only 5.6 GB + available."` — the "correct numbers in the reason" bar, met. +- **HardcoreLaptopMSI (8 GB): ADMITTED**, completed normally, `Attestation.RuntimeName == + "NativeRoleRuntime"`. +- Evidence: `.orc/hv-2-lane/hv2_large_20260721_141745.json`. + +**Small-context (inverse) phase (ctx=8192, already proven safe from HV-1), same two boxes:** +both completed normally — **the same HardcorePC that just denied at ctx=40000 admitted cleanly +at ctx=8192**, the direct proof that the denial above was footprint-driven, not "HardcorePC +always fails." Evidence: `.orc/hv-2-lane/hv2_small_20260721_141958.json`. + +**Driver bug found and fixed mid-campaign:** the task-level `HiveTaskResult.ErrorMsg` the +Warchief actually sees is `HiveWorkerAgent`'s generic wrapper text ("native role runtime failed. +Phase 3B does not fall back.") — the `RuntimeAdmissionDeniedException`'s own detailed message +never reaches it, only the worker's local log does. The driver's first pass tried to classify +denial by matching "admission" in that wrapper text and got it wrong (`matchesExpectation: +false` on a genuinely-correct denial). Fixed: classify denial by task status alone (this +execution kind can't fall back instead of failing), and let the separate +`/hive/native-telemetry` check be the sole authority on whether it was specifically an admission +denial with correct numbers — which is exactly why that endpoint needed to exist in the first +place, not just as a nice-to-have. + +**Real infrastructure gap found, not fixed (system-settings change, correctly out of scope for +an agent to make unilaterally): HardcorePC's inbound Windows Firewall doesn't allow port 7078 +from NewcorePC's LAN address**, so the driver's own remote telemetry fetch times out — confirmed +this is general (even the pre-existing `/hive/info` times out the same way remotely, works fine +over loopback) and not a bug in the new endpoint. Worked around by fetching telemetry via `ssh +HardcorePC curl http://localhost:7078/hive/native-telemetry` instead and splicing it into the +evidence file with a note. A future HV-2+ run should either open that inbound rule (an explicit, +user-authorized action) or teach the driver an SSH-fetch fallback. + +**NewcorePC (16 GB) excluded from this run — a real, separate finding, not a scheduling gap.** +Attempted to run `OrchestratorIDE.Daemon` locally on NewcorePC as Warchief+self-worker (to prove +the "fits 16 GB" case); this **regenerated NewcorePC's own HIVE identity** (the same DPAPI/ +AES-GCM protector collision from the HV-1 campaign, this time on the box that had never run the +Daemon binary before — NewcorePC's warchief role had only ever run via `swarmcli`, whose +identity uses a different protector). Confirmed via `--show-identity`: new nodeId `e5333a93...` +vs. the `f083b993...` both remote workers still had on file. Unlike the HV-1 recovery, **this +one has no clean fix**: `OrchestratorIDE.Daemon`'s `HiveService.cs` never subscribes to +`OnPairingRequestReceived` and never calls `HiveNodeServer.EnableDevAutoApprove` — by design +(`Program.cs`'s own comment: "this daemon must always be the INITIATOR, never the responder, +until a headless approval path exists"), so a Daemon-hosted Warchief can **never approve an +incoming pairing request** the way `swarmcli --warchief --allow-fingerprint` can. The Daemon +architecture assumes it is always a remote headless *worker* managed by an interactively-running +GUI/swarmcli elsewhere, not something that can host the Warchief role for peers to pair against +unattended. Reverted: killed the Daemon, restarted NewcorePC's Warchief via +`swarmcli --warchief --no-run --allow-fingerprint` (unaffected — its identity was never +touched), which the workers already trusted from the HV-1 fix, and the two-machine run above +completed cleanly on the first real attempt afterward. **Filed as an open follow-up**: either +give the Daemon a headless pairing-approval mode (env-var-gated auto-approve, mirroring +`EnableDevAutoApprove`) or find another way to get a 16 GB box into the worker fleet without +running the Daemon as its own Warchief. + +**HV-2 verdict: CLOSED for the 6 GB / 8 GB spread** (the decisive comparison — denial vs. +admission on genuinely different VRAM classes, with correct real numbers and real telemetry). +**The 16 GB "fits" leg is not yet run**, blocked on the Daemon pairing-approval gap above, not +on any scheduling defect — NewcorePC's own native execution was never in question (proven +extensively across Phase A-D). Cleaned up: both Daemon processes, local Warchief, remote scratch +workspaces and logs all stopped/removed. + ### HV-3 — Model/adapter lifecycle across machines - Sequential load → generate → dispose cycles per worker; residency (`ActiveCount`) returns to From a99fbd1e666e96b7a33674b1ac5456055985715b Mon Sep 17 00:00:00 2001 From: hardcoreerik Date: Tue, 21 Jul 2026 22:16:16 -0700 Subject: [PATCH 4/4] fix: HV-2 driver rejects --phase large without a matching --low-vram-worker Prevents a silent vacuous PASS: without this guard, an omitted or mistyped --low-vram-worker left ExpectDenied false for every worker, so a misconfigured large-context run could report success without ever exercising the admission-denial path it exists to prove. Co-Authored-By: Claude Sonnet 5 --- Tools/Hv2SchedulingRunner/Program.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Tools/Hv2SchedulingRunner/Program.cs b/Tools/Hv2SchedulingRunner/Program.cs index e8477441..e17d2a19 100644 --- a/Tools/Hv2SchedulingRunner/Program.cs +++ b/Tools/Hv2SchedulingRunner/Program.cs @@ -56,6 +56,10 @@ public static async Task Main(string[] args) throw new InvalidOperationException( "No workers configured. Pass --worker-a --worker-a-node (and -b/-c), " + "plus --low-vram-worker to mark which one is expected to deny in --phase large."); + if (phase == "large" && !workers.Any(w => w.ExpectDenied)) + throw new InvalidOperationException( + "--phase large requires --low-vram-worker to match one of the configured --worker-* ids " + + "(otherwise the run can vacuously PASS without ever exercising a denial)."); Directory.CreateDirectory(outDir); using var http = new HttpClient { BaseAddress = new Uri(warchief), Timeout = TimeSpan.FromMinutes(10) };