diff --git a/OrchestratorIDE.Daemon/HiveService.cs b/OrchestratorIDE.Daemon/HiveService.cs index 8bd63c73..91b2a1b7 100644 --- a/OrchestratorIDE.Daemon/HiveService.cs +++ b/OrchestratorIDE.Daemon/HiveService.cs @@ -197,6 +197,11 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) roleBindings: roleBindings); nativeExecutor = new HiveNativeRoleExecutorAdapter(nativeRuntime, _cfg.WorkspaceRoot); + // HV-2 (docs/NATIVE_RUNTIME_HIVE_VALIDATION_PLAN.md): RejectedAdmissionCount/ + // LastRejectionReason existed in-process (RuntimeOrchestrator) but had no remote + // observability surface on a headless worker -- only the Avalonia GUI's own + // diagnostics panel could see them. Exposed read-only over GET /hive/native-telemetry. + _nodeServer.NativeTelemetryProvider = () => nativeRuntime.GetReservationSnapshot(); } _worker = new HiveWorkerAgent diff --git a/OrchestratorIDE/Services/Hive/HiveNodeServer.cs b/OrchestratorIDE/Services/Hive/HiveNodeServer.cs index 32cc9e02..71b63bb9 100644 --- a/OrchestratorIDE/Services/Hive/HiveNodeServer.cs +++ b/OrchestratorIDE/Services/Hive/HiveNodeServer.cs @@ -65,6 +65,16 @@ public sealed class HiveNodeServer : IDisposable /// public Action? ShutdownCallback { get; set; } + /// + /// Injected by the app (Daemon: HiveService, after building its NativeRoleRuntime) so + /// GET /hive/native-telemetry can report this node's own admission/reservation state + /// (RuntimeReservationSnapshot: Reservations, Total/Reserved/AvailableBytes, + /// RejectedAdmissionCount, LastRejectionReason) without this class taking a hard + /// dependency on the native runtime types. Null when native execution isn't configured + /// on this node (e.g. Ollama-only) or before the runtime has been built. + /// + public Func? NativeTelemetryProvider { get; set; } + // Pending pairing sessions: sessionId → (request, expiry, initiator-remote-ip) private readonly Dictionary _pendingPairings = []; // Completed results: sessionId → (response, stored-at). Pruned after 10 min. @@ -435,6 +445,17 @@ private async Task HandleAsync(HttpListenerContext ctx, CancellationToken ct) Ok(resp, JsonSerializer.Serialize(_info)); return; } + // Read-only native-runtime admission telemetry — same unauthenticated posture as + // /hive/info (operational status, not a secret or control action). {} when native + // execution isn't configured on this node, rather than 404, so a fleet-wide poller + // doesn't need to special-case Ollama-only boxes. + if (method == "GET" && path == "/hive/native-telemetry") + { + var snapshot = NativeTelemetryProvider?.Invoke(); + Ok(resp, JsonSerializer.Serialize(snapshot ?? new { }, _jsonOut)); + return; + } + if (method == "POST" && path == "/hive/pair") { var remoteIp = req.RemoteEndPoint?.Address?.ToString() ?? ""; diff --git a/Tools/Hv2SchedulingRunner/Hv2SchedulingRunner.csproj b/Tools/Hv2SchedulingRunner/Hv2SchedulingRunner.csproj new file mode 100644 index 00000000..626a37d7 --- /dev/null +++ b/Tools/Hv2SchedulingRunner/Hv2SchedulingRunner.csproj @@ -0,0 +1,17 @@ + + + + Exe + net10.0 + enable + enable + hv2-scheduling-runner + Hv2SchedulingRunner + + + + + + + + diff --git a/Tools/Hv2SchedulingRunner/Program.cs b/Tools/Hv2SchedulingRunner/Program.cs new file mode 100644 index 00000000..e17d2a19 --- /dev/null +++ b/Tools/Hv2SchedulingRunner/Program.cs @@ -0,0 +1,290 @@ +// Copyright (C) 2025-present hardcoreerik / TheOrc contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +using System.Net.Http.Json; +using System.Text.Json; +using OrchestratorIDE.Services.Hive; + +namespace Hv2SchedulingRunner; + +/// +/// HV-2 driver (docs/NATIVE_RUNTIME_HIVE_VALIDATION_PLAN.md): proves capability/resource-aware +/// scheduling across the fleet. NativeContextSize is a per-worker-process startup config, not a +/// per-job parameter the HIVE contract exposes, so the two HV-2 checks map to two separate fleet +/// configurations of the same three machines rather than two job shapes against one config: +/// +/// --phase large: every worker started with a context size whose estimated footprint exceeds +/// the low-VRAM box's budget. That box must deny with a real RuntimeAdmissionDeniedException +/// (correct numbers, observable via GET /hive/native-telemetry's RejectedAdmissionCount/ +/// LastRejectionReason) while the higher-VRAM boxes complete normally -- and must never fall +/// back to Ollama. +/// --phase small: every worker started with a context size that fits everywhere. All three +/// must complete -- proving the large-phase denial was footprint-driven, not "that box +/// always fails." +/// +/// Run this once per phase against a fleet already reconfigured (NativeContextSize env var) and +/// restarted for that phase. +/// +internal static class Program +{ + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, + }; + + public static async Task Main(string[] args) + { + var warchief = GetArg(args, "--warchief") ?? "http://localhost:7079"; + var outDir = GetArg(args, "--out") ?? Path.Combine(Environment.CurrentDirectory, ".orc", "hv-2-lane"); + var phase = GetArg(args, "--phase") + ?? throw new InvalidOperationException("--phase large|small is required."); + if (phase is not ("large" or "small")) + throw new InvalidOperationException("--phase must be 'large' or 'small'."); + var role = GetArg(args, "--role") ?? "Coder"; + var timeoutMs = int.TryParse(GetArg(args, "--timeout-ms"), out var t) ? t : 300_000; + + var workers = new List<(string Id, string NodeUrl, bool ExpectDenied)>(); + foreach (var slot in new[] { "a", "b", "c" }) + { + var id = GetArg(args, $"--worker-{slot}"); + var nodeUrl = GetArg(args, $"--worker-{slot}-node"); + if (id is null || nodeUrl is null) continue; + var isLowVram = string.Equals(GetArg(args, "--low-vram-worker"), id, StringComparison.OrdinalIgnoreCase); + workers.Add((id, nodeUrl, isLowVram && phase == "large")); + } + if (workers.Count == 0) + throw new InvalidOperationException( + "No workers configured. Pass --worker-a --worker-a-node (and -b/-c), " + + "plus --low-vram-worker to mark which one is expected to deny in --phase large."); + if (phase == "large" && !workers.Any(w => w.ExpectDenied)) + throw new InvalidOperationException( + "--phase large requires --low-vram-worker to match one of the configured --worker-* ids " + + "(otherwise the run can vacuously PASS without ever exercising a denial)."); + + Directory.CreateDirectory(outDir); + using var http = new HttpClient { BaseAddress = new Uri(warchief), Timeout = TimeSpan.FromMinutes(10) }; + + var report = new Hv2Report { Warchief = warchief, Phase = phase, StartedAt = DateTimeOffset.UtcNow }; + + try + { + Console.WriteLine($"HV-2 scheduling run, phase={phase}, against {warchief}"); + + // Baseline each worker's native telemetry BEFORE dispatch, so a denial's effect on + // RejectedAdmissionCount can be verified as an increase, not just a nonzero value + // that might already be nonzero from an earlier run. + var baselineTelemetry = new Dictionary(); + foreach (var w in workers) + baselineTelemetry[w.Id] = await TryFetchTelemetryAsync(w.NodeUrl); + + var otherIds = workers.Select(w => w.Id).ToArray(); + var workUnits = workers.Select(w => new WorkUnit + { + WorkUnitId = $"hv2-{phase}-{w.Id}", + Title = $"HV-2 {phase}-context scheduling proof targeting {w.Id}", + Role = role, + ExecutionKind = HiveExecutionKinds.NativeAgent, + Requirements = new ResourceRequirements + { + ExcludedWorkerIds = otherIds.Where(id => !string.Equals(id, w.Id, StringComparison.OrdinalIgnoreCase)).ToArray(), + }, + Spec = $"Create a file named hv2_proof.txt in the workspace root containing exactly " + + $"this single line and nothing else: HV2-PROOF {phase} {w.Id}", + TimeoutMs = timeoutMs, + }).ToList(); + + var campaign = new CampaignDefinition { Name = $"hv2-{phase}", WorkUnits = workUnits }; + Console.WriteLine($"Submitting campaign {campaign.CampaignId}: {workUnits.Count} work unit(s)..."); + using (var resp = await http.PostAsJsonAsync("/hive/campaigns", campaign, JsonOptions)) + resp.EnsureSuccessStatusCode(); + report.CampaignId = campaign.CampaignId; + + foreach (var unit in workUnits) + { + var target = workers.First(w => unit.WorkUnitId.EndsWith(w.Id, StringComparison.Ordinal)); + var taskId = $"{campaign.CampaignId}-{unit.WorkUnitId}"; + var deadline = DateTime.UtcNow.AddMilliseconds(unit.TimeoutMs + 60_000); + HiveTaskStatusResponse? last = null; + var seenClaimed = false; + var consecutiveNotFound = 0; + while (DateTime.UtcNow < deadline) + { + using var statusResp = await http.GetAsync($"/hive/tasks/{taskId}"); + if (statusResp.IsSuccessStatusCode) + { + consecutiveNotFound = 0; + var body = await statusResp.Content.ReadFromJsonAsync(JsonOptions); + if (body is not null) + { + last = body; + if (body.Status is "claimed" or "running") seenClaimed = true; + if (body.Status is "completed" or "failed" or "timeout" or "cancelled") break; + } + } + else if (statusResp.StatusCode == System.Net.HttpStatusCode.NotFound && seenClaimed) + { + if (++consecutiveNotFound >= 3) { if (last is not null) last.Status = "swept-unknown"; break; } + } + await Task.Delay(2000); + } + + var evidence = BuildJobEvidence(unit.WorkUnitId, target.Id, target.ExpectDenied, last); + report.Jobs.Add(evidence); + Console.WriteLine($" [{evidence.Status}] {unit.WorkUnitId} -> worker={target.Id} " + + $"expectDenied={target.ExpectDenied} runtime={evidence.RuntimeName ?? "-"} " + + $"errorMsg={evidence.ErrorMsg ?? "-"} matchesExpectation={evidence.MatchesExpectation}"); + } + + // Post-dispatch telemetry: only meaningful for the worker(s) expected to deny. + foreach (var w in workers.Where(w => w.ExpectDenied)) + { + var after = await TryFetchTelemetryAsync(w.NodeUrl); + var before = baselineTelemetry[w.Id]; + var rejectedDelta = (after?.RejectedAdmissionCount ?? 0) - (before?.RejectedAdmissionCount ?? 0); + report.TelemetryChecks.Add(new Hv2TelemetryCheck + { + WorkerId = w.Id, + NodeUrl = w.NodeUrl, + RejectedAdmissionCountBefore = before?.RejectedAdmissionCount, + RejectedAdmissionCountAfter = after?.RejectedAdmissionCount, + RejectedAdmissionCountIncreased = rejectedDelta > 0, + LastRejectionReason = after?.LastRejectionReason, + }); + Console.WriteLine($" telemetry[{w.Id}]: rejected {(before is null ? "?" : before.RejectedAdmissionCount.ToString())} -> " + + $"{(after is null ? "?" : after.RejectedAdmissionCount.ToString())}, reason: {after?.LastRejectionReason ?? "(none)"}"); + } + + report.FinishedAt = DateTimeOffset.UtcNow; + var allJobsMatchedExpectation = report.Jobs.All(j => j.MatchesExpectation); + var allTelemetryConfirmed = report.TelemetryChecks.All(c => c.RejectedAdmissionCountIncreased); + report.Passed = allJobsMatchedExpectation && allTelemetryConfirmed; + + var outPath = Path.Combine(outDir, $"hv2_{phase}_{DateTime.UtcNow:yyyyMMdd_HHmmss}.json"); + await File.WriteAllTextAsync(outPath, + JsonSerializer.Serialize(report, new JsonSerializerOptions(JsonOptions) { WriteIndented = true })); + + Console.WriteLine(); + Console.WriteLine($"All jobs matched expectation: {allJobsMatchedExpectation}"); + Console.WriteLine($"All expected-denial telemetry confirmed: {allTelemetryConfirmed}"); + Console.WriteLine($"Verdict: {(report.Passed ? "PASS" : "FAIL")}"); + Console.WriteLine($"Evidence written: {outPath}"); + return report.Passed ? 0 : 2; + } + catch (Exception ex) + { + report.Error = ex.ToString(); + report.FinishedAt = DateTimeOffset.UtcNow; + var outPath = Path.Combine(outDir, $"hv2_{phase}_FAILED_{DateTime.UtcNow:yyyyMMdd_HHmmss}.json"); + await File.WriteAllTextAsync(outPath, + JsonSerializer.Serialize(report, new JsonSerializerOptions(JsonOptions) { WriteIndented = true })); + Console.Error.WriteLine($"Run failed: {ex.Message}"); + Console.Error.WriteLine($"Partial evidence written: {outPath}"); + return 1; + } + } + + private static Hv2JobEvidence BuildJobEvidence( + string workUnitId, string workerId, bool expectDenied, HiveTaskStatusResponse? last) + { + var status = last?.Status ?? "unknown"; + var attestation = last?.Attestation; + var isNativeRuntime = attestation is not null && + string.Equals(attestation.RuntimeName, "NativeRoleRuntime", StringComparison.Ordinal); + var errorMsg = last?.ErrorMsg; + // A real, fail-closed denial surfaces as a "failed" task -- but the task-level ErrorMsg + // the Warchief actually sees is HiveWorkerAgent's generic wrapper text ("native role + // runtime failed. Phase 3B does not fall back."), NOT the RuntimeAdmissionDeniedException's + // own detailed message (which only reaches the worker's own local log). Confirmed + // empirically during HV-2 calibration: a genuine admission denial with correct numbers + // ("Requires ~6.8 GB, only 5.6 GB available...") still produced this exact generic + // wrapper as ErrorMsg. So "failed" here (this shape can structurally never fall back + // instead) is what proves fail-closed; the SEPARATE /hive/native-telemetry check below + // is what proves it was specifically an admission denial with correct numbers. + var wasDenied = status == "failed"; + var wasAdmitted = status == "completed" && isNativeRuntime; + var matchesExpectation = expectDenied ? wasDenied : wasAdmitted; + + return new Hv2JobEvidence + { + WorkUnitId = workUnitId, + WorkerId = workerId, + ExpectDenied = expectDenied, + Status = status, + RuntimeName = attestation?.RuntimeName, + Backend = attestation?.Backend, + Stats = last?.Metrics ?? [], + ErrorMsg = errorMsg, + WasDenied = wasDenied, + WasAdmitted = wasAdmitted, + MatchesExpectation = matchesExpectation, + }; + } + + private static async Task TryFetchTelemetryAsync(string nodeUrl) + { + try + { + using var http = new HttpClient { Timeout = TimeSpan.FromSeconds(15) }; + using var resp = await http.GetAsync($"{nodeUrl.TrimEnd('/')}/hive/native-telemetry"); + if (!resp.IsSuccessStatusCode) return null; + return await resp.Content.ReadFromJsonAsync(JsonOptions); + } + catch + { + return null; + } + } + + private static string? GetArg(string[] args, string name) + { + var idx = Array.IndexOf(args, name); + return idx >= 0 && idx + 1 < args.Length ? args[idx + 1] : null; + } +} + +internal sealed class NativeTelemetry +{ + public long RejectedAdmissionCount { get; set; } + public string? LastRejectionReason { get; set; } + public long TotalBytes { get; set; } + public long ReservedBytes { get; set; } + public long AvailableBytes { get; set; } +} + +internal sealed class Hv2Report +{ + public string Warchief { get; set; } = ""; + public string Phase { get; set; } = ""; + public string CampaignId { get; set; } = ""; + public DateTimeOffset StartedAt { get; set; } + public DateTimeOffset? FinishedAt { get; set; } + public List Jobs { get; set; } = []; + public List TelemetryChecks { get; set; } = []; + public bool Passed { get; set; } + public string? Error { get; set; } +} + +internal sealed class Hv2JobEvidence +{ + public string WorkUnitId { get; set; } = ""; + public string WorkerId { get; set; } = ""; + public bool ExpectDenied { get; set; } + public string Status { get; set; } = ""; + public string? RuntimeName { get; set; } + public string? Backend { get; set; } + public Dictionary Stats { get; set; } = []; + public string? ErrorMsg { get; set; } + public bool WasDenied { get; set; } + public bool WasAdmitted { get; set; } + public bool MatchesExpectation { get; set; } +} + +internal sealed class Hv2TelemetryCheck +{ + public string WorkerId { get; set; } = ""; + public string NodeUrl { get; set; } = ""; + public long? RejectedAdmissionCountBefore { get; set; } + public long? RejectedAdmissionCountAfter { get; set; } + public bool RejectedAdmissionCountIncreased { get; set; } + public string? LastRejectionReason { get; set; } +} diff --git a/docs/NATIVE_RUNTIME_HIVE_VALIDATION_PLAN.md b/docs/NATIVE_RUNTIME_HIVE_VALIDATION_PLAN.md index 9eef23dc..c0478e56 100644 --- a/docs/NATIVE_RUNTIME_HIVE_VALIDATION_PLAN.md +++ b/docs/NATIVE_RUNTIME_HIVE_VALIDATION_PLAN.md @@ -241,6 +241,88 @@ estimator, large `ContextLength`) fits 16 GB and 8 GB but must be **denied** on - Also run the inverse: a small job admitted on all three, proving denial is footprint-driven, not box-driven. +**2026-07-21 — Driver built (`Tools/Hv2SchedulingRunner`, PR #88), real run against the fleet. +CLOSED for the two-machine spread (6 GB deny / 8 GB admit); NewcorePC (16 GB) excluded — a +genuine Daemon-architecture constraint, not a scheduling gap.** + +`NativeContextSize` is a per-worker-process startup config, not a per-job HIVE parameter, so +the "large footprint" and "small footprint" checks run as two separate fleet configurations of +the same machines rather than two job shapes against one running config (`--phase large|small`). +Also added `GET /hive/native-telemetry` on `HiveNodeServer` (`RejectedAdmissionCount`, +`LastRejectionReason`, VRAM totals) — this existed in-process but had no remote observability +surface on a headless worker before this. + +**Calibration, computed before touching real hardware:** `OrcScheduler.EstimateRequiredBytes` +is `legacy(base+adapter file size) + 256 MB (CUDA overhead) + 384 MB (compute buffer) + kvBytes`, +where `kvBytes` scales linearly with `ContextLength`. Back-solving from HV-1's own observed +figures (base ≈1.881 GB, ctx=8192 → ~3.4 GB total ⇒ kv(8192) ≈ 894 MB) gave `ctx=40000` ⇒ +≈6.77 GB total — denied on 6 GB, comfortable margin under 8 GB. Confirmed near-exact on real +hardware: the actual denial read **"Requires ~6.8 GB, only 5.6 GB available."** + +**Large-context phase (ctx=40000), pinned per-worker via `ExcludedWorkerIds`:** +- **HardcorePC (6 GB): DENIED**, a real `RuntimeAdmissionDeniedException` surfaced as `status: + "failed"` (this execution kind is structurally fail-closed — no Ollama fallback path is even + reachable). Confirmed via `/hive/native-telemetry`: `RejectedAdmissionCount` 3→6 (exactly the + 3 retry attempts this run made), `LastRejectionReason: "Requires ~6.8 GB, only 5.6 GB + available."` — the "correct numbers in the reason" bar, met. +- **HardcoreLaptopMSI (8 GB): ADMITTED**, completed normally, `Attestation.RuntimeName == + "NativeRoleRuntime"`. +- Evidence: `.orc/hv-2-lane/hv2_large_20260721_141745.json`. + +**Small-context (inverse) phase (ctx=8192, already proven safe from HV-1), same two boxes:** +both completed normally — **the same HardcorePC that just denied at ctx=40000 admitted cleanly +at ctx=8192**, the direct proof that the denial above was footprint-driven, not "HardcorePC +always fails." Evidence: `.orc/hv-2-lane/hv2_small_20260721_141958.json`. + +**Driver bug found and fixed mid-campaign:** the task-level `HiveTaskResult.ErrorMsg` the +Warchief actually sees is `HiveWorkerAgent`'s generic wrapper text ("native role runtime failed. +Phase 3B does not fall back.") — the `RuntimeAdmissionDeniedException`'s own detailed message +never reaches it, only the worker's local log does. The driver's first pass tried to classify +denial by matching "admission" in that wrapper text and got it wrong (`matchesExpectation: +false` on a genuinely-correct denial). Fixed: classify denial by task status alone (this +execution kind can't fall back instead of failing), and let the separate +`/hive/native-telemetry` check be the sole authority on whether it was specifically an admission +denial with correct numbers — which is exactly why that endpoint needed to exist in the first +place, not just as a nice-to-have. + +**Real infrastructure gap found, not fixed (system-settings change, correctly out of scope for +an agent to make unilaterally): HardcorePC's inbound Windows Firewall doesn't allow port 7078 +from NewcorePC's LAN address**, so the driver's own remote telemetry fetch times out — confirmed +this is general (even the pre-existing `/hive/info` times out the same way remotely, works fine +over loopback) and not a bug in the new endpoint. Worked around by fetching telemetry via `ssh +HardcorePC curl http://localhost:7078/hive/native-telemetry` instead and splicing it into the +evidence file with a note. A future HV-2+ run should either open that inbound rule (an explicit, +user-authorized action) or teach the driver an SSH-fetch fallback. + +**NewcorePC (16 GB) excluded from this run — a real, separate finding, not a scheduling gap.** +Attempted to run `OrchestratorIDE.Daemon` locally on NewcorePC as Warchief+self-worker (to prove +the "fits 16 GB" case); this **regenerated NewcorePC's own HIVE identity** (the same DPAPI/ +AES-GCM protector collision from the HV-1 campaign, this time on the box that had never run the +Daemon binary before — NewcorePC's warchief role had only ever run via `swarmcli`, whose +identity uses a different protector). Confirmed via `--show-identity`: new nodeId `e5333a93...` +vs. the `f083b993...` both remote workers still had on file. Unlike the HV-1 recovery, **this +one has no clean fix**: `OrchestratorIDE.Daemon`'s `HiveService.cs` never subscribes to +`OnPairingRequestReceived` and never calls `HiveNodeServer.EnableDevAutoApprove` — by design +(`Program.cs`'s own comment: "this daemon must always be the INITIATOR, never the responder, +until a headless approval path exists"), so a Daemon-hosted Warchief can **never approve an +incoming pairing request** the way `swarmcli --warchief --allow-fingerprint` can. The Daemon +architecture assumes it is always a remote headless *worker* managed by an interactively-running +GUI/swarmcli elsewhere, not something that can host the Warchief role for peers to pair against +unattended. Reverted: killed the Daemon, restarted NewcorePC's Warchief via +`swarmcli --warchief --no-run --allow-fingerprint` (unaffected — its identity was never +touched), which the workers already trusted from the HV-1 fix, and the two-machine run above +completed cleanly on the first real attempt afterward. **Filed as an open follow-up**: either +give the Daemon a headless pairing-approval mode (env-var-gated auto-approve, mirroring +`EnableDevAutoApprove`) or find another way to get a 16 GB box into the worker fleet without +running the Daemon as its own Warchief. + +**HV-2 verdict: CLOSED for the 6 GB / 8 GB spread** (the decisive comparison — denial vs. +admission on genuinely different VRAM classes, with correct real numbers and real telemetry). +**The 16 GB "fits" leg is not yet run**, blocked on the Daemon pairing-approval gap above, not +on any scheduling defect — NewcorePC's own native execution was never in question (proven +extensively across Phase A-D). Cleaned up: both Daemon processes, local Warchief, remote scratch +workspaces and logs all stopped/removed. + ### HV-3 — Model/adapter lifecycle across machines - Sequential load → generate → dispose cycles per worker; residency (`ActiveCount`) returns to