diff --git a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx index 7fa0288a17..5367f72c3b 100644 --- a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx +++ b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx @@ -262,7 +262,7 @@ $$nemoclaw rebuild ### Resolve Rebuild Preflight Stops -Before it backs up or deletes the existing sandbox, `rebuild` validates the recorded sandbox, gateway, policy, MCP, agent, and operation-lock state. +Before it backs up or deletes the existing sandbox, `rebuild` validates the recorded sandbox, gateway, inference route, policy, MCP, agent, and operation-lock state. When one of these checks fails, NemoClaw prints `Rebuild preflight failed`, explains how to recover, and ends with `Aborting rebuild`. At this boundary, the existing sandbox is unchanged and no sandbox data has been removed. @@ -274,6 +274,10 @@ Use the recovery guidance that matches the reported check: - Resolve an incomplete MCP destroy transaction before retrying. - Back up the sandbox state and recreate it with `$$nemoclaw onboard` when the record contains multiple agents. Transactional multi-agent rebuild is not supported. - Wait for another onboarding or rebuild operation to finish before retrying. If verified stale-lock cleanup is still in progress, wait briefly and rerun the command. Do not delete the lock manually. +- Set the live OpenShell inference route to the sandbox's recorded provider and model when rebuild reports route drift. + +A gateway that reports no live inference route does not stop the rebuild. +Replacement onboarding configures and verifies the recorded route before it recreates the sandbox. The rebuild command preserves the mounted workspace and registered policies while recreating the container. diff --git a/docs/reference/system-readiness.mdx b/docs/reference/system-readiness.mdx index 100dac2a78..a91d80a406 100644 --- a/docs/reference/system-readiness.mdx +++ b/docs/reference/system-readiness.mdx @@ -289,8 +289,13 @@ Fresh onboarding and authoritative rebuilds use this order: 8. Revalidate gateway authority immediately before gateway selection, recovery, reconciliation, or other lifecycle effects. The readiness gate runs before model-router cleanup, provider selection, credential registration, policy changes, image builds, or sandbox lifecycle effects. -Host and gateway observations have a 30-second reuse window. -If collection itself takes too long, onboarding rejects the stale composite instead of assigning a fresh timestamp to old facts. +Host and gateway observations have a 30-second reuse window that starts when collection finishes. +The readiness gate does not reject a collection for the time its own probes take. +The `gateway.owner` evidence records the gateway collection duration as `collectionMs`. +The readiness gate rejects an observation set that waits past the window for another collection. +Onboarding then collects that set again instead of assigning a fresh timestamp to old facts. +Bounded evidence for a rejected set appears under `host.probe.stale` or `gateway.probe.stale` with the applied `windowMs` and the measured `ageMs`. +`ageMs` is `null` when the recorded time cannot be parsed or is later than the current time. The policy permits only these narrow exceptions: diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 1cae38016e..728bb986ff 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -929,6 +929,7 @@ const providerExistsInGateway = (name: string, gatewayName: string = GATEWAY_NAM const { verifyInferenceRoute, isInferenceRouteReady, + readInferenceRouteState, checkGatewayRouteCompatibility, preflightGatewayRouteDiscovery, } = inferenceRouteHelpers.createInferenceRouteHelpers(runCaptureOpenshell); @@ -944,8 +945,6 @@ const { inspectSandboxForCreate, confirmRecreateForSelectionDrift, isOpenclawRea const { ensureValidatedWebSearchCredential, ensureValidatedBraveSearchCredential, configureWebSearch, verifyWebSearchInsideSandbox, webSearchProviderForConfig } = createWebSearchFlowHelpers({ prompt, note, isNonInteractive, cliName, runCaptureOpenshell }); -// getSandboxInferenceConfig — moved to onboard-providers.ts -// Inference probes — moved to inference/onboard-probes.ts const { hasResponsesToolCall, hasChatCompletionsToolCall, @@ -3020,7 +3019,7 @@ async function preflightAuthoritativeRebuildTarget( fail(`OpenShell component preflight exited with code ${String(code)}`), ), assertGatewayReadiness: onboardPreflightGatewayAuthority.collectGatewayReadiness, - inferenceRouteReady: (p, m) => isInferenceRouteReady(authoritativeGateway.name, p, m), + inferenceRouteState: (p, m) => readInferenceRouteState(authoritativeGateway.name, p, m), captureForwardList: () => runCaptureOpenshell(["forward", "list"], { ignoreError: true }), checkPort: (port) => checkPortAvailable(port), }, diff --git a/src/lib/onboard/authoritative-rebuild-target.test.ts b/src/lib/onboard/authoritative-rebuild-target.test.ts index 76448c7ff5..d2aad30250 100644 --- a/src/lib/onboard/authoritative-rebuild-target.test.ts +++ b/src/lib/onboard/authoritative-rebuild-target.test.ts @@ -11,6 +11,7 @@ import { rebuildProviderFlowOptions, resolveAuthoritativeOnboardGatewayBinding, } from "./authoritative-rebuild-target"; +import type { InferenceRouteState } from "./inference-route"; import { mintProviderRecoveryReceipt, type ProviderRecoveryReceiptTarget, @@ -65,7 +66,7 @@ function deps(overrides: Partial = {}) { runFatalRuntimePreflight: vi.fn(), ensureOpenshell: vi.fn(), assertGatewayReadiness: vi.fn(), - inferenceRouteReady: vi.fn(() => true), + inferenceRouteState: vi.fn((): InferenceRouteState => "matched"), captureForwardList: vi.fn(() => "alpha 127.0.0.1 18789 42 active"), checkPort: vi.fn(async () => ({ ok: true })), ...overrides, @@ -265,7 +266,7 @@ describe("authoritative rebuild target preflight", () => { expect(targetDeps.bindGatewayAuthority).not.toHaveBeenCalled(); expect(targetDeps.ensureOpenshell).not.toHaveBeenCalled(); expect(targetDeps.assertGatewayReadiness).not.toHaveBeenCalled(); - expect(targetDeps.inferenceRouteReady).not.toHaveBeenCalled(); + expect(targetDeps.inferenceRouteState).not.toHaveBeenCalled(); }); it("pins the requested gateway for route and forward checks, then restores it", async () => { @@ -275,9 +276,9 @@ describe("authoritative rebuild target preflight", () => { await preflightAuthoritativeRebuildTarget( target, deps({ - inferenceRouteReady: vi.fn(() => { + inferenceRouteState: vi.fn((): InferenceRouteState => { seen.push(`route:${process.env.OPENSHELL_GATEWAY}`); - return true; + return "matched"; }), captureForwardList: vi.fn(() => { seen.push(`forward:${process.env.OPENSHELL_GATEWAY}`); @@ -296,13 +297,25 @@ describe("authoritative rebuild target preflight", () => { await expect( preflightAuthoritativeRebuildTarget( target, - deps({ inferenceRouteReady: vi.fn(() => false) }), + deps({ inferenceRouteState: vi.fn((): InferenceRouteState => "mismatched") }), ), ).rejects.toThrow("inference route does not match"); }); + it("proceeds when the gateway cannot answer the route query (#9310)", async () => { + const targetDeps = deps({ + inferenceRouteState: vi.fn((): InferenceRouteState => "unanswered"), + }); + + await expect(preflightAuthoritativeRebuildTarget(target, targetDeps)).resolves.toBeUndefined(); + + expect(targetDeps.inferenceRouteState).toHaveBeenCalledOnce(); + }); + it("defers route validation for prepared recovery until authoritative onboard (#6114)", async () => { - const targetDeps = deps({ inferenceRouteReady: vi.fn(() => false) }); + const targetDeps = deps({ + inferenceRouteState: vi.fn((): InferenceRouteState => "mismatched"), + }); await expect( preflightAuthoritativeRebuildTarget( @@ -311,7 +324,7 @@ describe("authoritative rebuild target preflight", () => { ), ).resolves.toBeUndefined(); - expect(targetDeps.inferenceRouteReady).not.toHaveBeenCalled(); + expect(targetDeps.inferenceRouteState).not.toHaveBeenCalled(); expect(targetDeps.runFatalRuntimePreflight).toHaveBeenCalledOnce(); expect(targetDeps.ensureOpenshell).toHaveBeenCalledOnce(); }); @@ -366,9 +379,9 @@ describe("authoritative rebuild target preflight", () => { }), ensureOpenshell: vi.fn(() => calls.push("openshell")), assertGatewayReadiness: vi.fn(() => calls.push("gateway")), - inferenceRouteReady: vi.fn(() => { + inferenceRouteState: vi.fn((): InferenceRouteState => { calls.push("route"); - return true; + return "matched"; }), }); @@ -393,6 +406,6 @@ describe("authoritative rebuild target preflight", () => { ); expect(targetDeps.ensureOpenshell).not.toHaveBeenCalled(); expect(targetDeps.assertGatewayReadiness).not.toHaveBeenCalled(); - expect(targetDeps.inferenceRouteReady).not.toHaveBeenCalled(); + expect(targetDeps.inferenceRouteState).not.toHaveBeenCalled(); }); }); diff --git a/src/lib/onboard/authoritative-rebuild-target.ts b/src/lib/onboard/authoritative-rebuild-target.ts index 796545bc9f..b50f45336d 100644 --- a/src/lib/onboard/authoritative-rebuild-target.ts +++ b/src/lib/onboard/authoritative-rebuild-target.ts @@ -3,6 +3,7 @@ import { findDashboardForwardOwner } from "./dashboard-port"; import { resolveGatewayName } from "./gateway-binding"; +import type { InferenceRouteState } from "./inference-route"; import type { PortProbeResult } from "./preflight"; import { assertDashboardPortNotReserved } from "./preflight-ports"; import { @@ -195,7 +196,7 @@ export type AuthoritativeRebuildTargetDeps = { runFatalRuntimePreflight(): unknown | Promise; ensureOpenshell(): unknown; assertGatewayReadiness(): unknown | Promise; - inferenceRouteReady(provider: string, model: string): boolean; + inferenceRouteState(provider: string, model: string): InferenceRouteState; captureForwardList(): string | null; checkPort(port: number): Promise; env?: NodeJS.ProcessEnv; @@ -223,10 +224,12 @@ export async function preflightAuthoritativeRebuildTarget( // Prepared-backup recovery can run after the installer has replaced a // legacy gateway. That fresh gateway has no inference route to validate // yet; authoritative onboarding configures and verifies the pinned route - // before recreating the sandbox. Normal rebuilds must still match here. + // before recreating the sandbox. A gateway that cannot answer at all leaves + // the route unknown, which onboarding resolves the same way. Only a gateway + // that answers with a different route contradicts the rebuild target. if ( target.deferInferenceRouteUntilOnboard !== true && - !deps.inferenceRouteReady(target.provider, target.model) + deps.inferenceRouteState(target.provider, target.model) === "mismatched" ) { fail( `OpenShell inference route does not match provider '${target.provider}' and model '${target.model}'.`, diff --git a/src/lib/onboard/fatal-runtime-preflight.test.ts b/src/lib/onboard/fatal-runtime-preflight.test.ts index 04cc1dac2e..e58eaed3d5 100644 --- a/src/lib/onboard/fatal-runtime-preflight.test.ts +++ b/src/lib/onboard/fatal-runtime-preflight.test.ts @@ -10,13 +10,14 @@ vi.mock("./experimental/portable-host-preparation", () => ({ })); import type { DetectGpuDeps, GpuDetection } from "../inference/nim"; -import type { GatewayReadinessProjection } from "../readiness/gateway"; +import type { GatewayObservationSnapshot, GatewayReadinessProjection } from "../readiness/gateway"; import type { SystemReadinessReport } from "../readiness/types"; import { isLinuxDockerDriverGatewayEnabled } from "./docker-driver-platform"; import { assertOnboardGatewayReadiness, assertOnboardHostReadiness, assertOnboardSystemReadiness, + type CollectedGatewayReadiness, runFatalOnboardRuntimePreflight, runOnboardRuntimeEffectfulPreflightChecks, runReadinessGatedRuntimePreflight, @@ -85,6 +86,37 @@ function managedGatewayReadiness( }; } +function managedGatewaySnapshot( + completedAt = new Date().toISOString(), +): GatewayObservationSnapshot { + return { + observedAt: completedAt, + completedAt, + observations: { + owner: { + gatewayName: "nemoclaw", + gatewayPort: 8080, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + supervisor: null, + requiredCapabilities: [], + }, + attachmentState: "not-applicable", + reuseState: "healthy", + driftState: "not-detected", + portConflictState: "none", + }, + }; +} + +function collectedGatewayReadiness( + projection: GatewayReadinessProjection = managedGatewayReadiness(), + completedAt?: string, +): CollectedGatewayReadiness { + return { projection, snapshot: managedGatewaySnapshot(completedAt) }; +} + afterEach(() => { vi.unstubAllEnvs(); vi.restoreAllMocks(); @@ -271,7 +303,7 @@ describe("report-backed runtime readiness (#7411)", () => { {}, { nonInteractive: true, - collectGatewayReadiness: async () => managedGatewayReadiness(), + collectGatewayReadiness: async () => collectedGatewayReadiness(), assessHost: () => ({ ...hostWithRuntime("docker"), dockerHostInvalid: true, @@ -457,34 +489,78 @@ describe("runFatalOnboardRuntimePreflight", () => { }); describe("readiness-gated runtime preflight", () => { - it("rejects a host assessment that exceeds the freshness window before effects (#7411)", async () => { + it("recollects host facts after a gateway collection exceeds the freshness window (#7411)", async () => { let currentTime = Date.parse("2026-08-07T12:00:00.000Z"); const bridge = vi.fn(); const validateGpu = vi.fn(); - const exitProcess = vi.fn((_code: number): never => { - throw new Error("stale host blocked"); - }); + const assessHost = vi.fn(() => hostWithRuntime("docker")); + const gatewayCollectionDelays = [0, 0, 30_001]; - await expect( - runReadinessGatedRuntimePreflight( - {}, - { - nonInteractive: true, - now: () => new Date(currentTime), - collectGatewayReadiness: async () => managedGatewayReadiness(), - assessHost: () => { - currentTime += 30_001; - return hostWithRuntime("docker"); - }, - detectGpu: () => null, - assertDockerBridgeAndContainerDnsHealthy: bridge, - validateSandboxGpuPreflight: validateGpu, - exitProcess, + const result = await runReadinessGatedRuntimePreflight( + {}, + { + nonInteractive: true, + now: () => new Date(currentTime), + collectGatewayReadiness: async () => { + currentTime += gatewayCollectionDelays.shift() ?? 0; + return collectedGatewayReadiness( + managedGatewayReadiness(), + new Date(currentTime).toISOString(), + ); }, - ), - ).rejects.toThrow("stale host blocked"); - expect(bridge).not.toHaveBeenCalled(); - expect(validateGpu).not.toHaveBeenCalled(); + assessHost, + detectGpu: () => null, + assertDockerBridgeAndContainerDnsHealthy: bridge, + validateSandboxGpuPreflight: validateGpu, + }, + ); + + expect(assessHost).toHaveBeenCalledTimes(3); + expect(result.readinessReport.evidence).not.toContainEqual( + expect.objectContaining({ id: "host.probe.stale" }), + ); + expect(bridge).toHaveBeenCalledOnce(); + expect(validateGpu).toHaveBeenCalledOnce(); + }); + + it("recollects gateway facts when refreshing the host expires the paired snapshot (#7411)", async () => { + let currentTime = Date.parse("2026-08-07T12:00:00.000Z"); + const bridge = vi.fn(); + const validateGpu = vi.fn(); + const gatewayCollectionDelays = [0, 0, 30_001]; + const hostCollectionDelays = [0, 0, 30_001]; + const collectGatewayReadiness = vi.fn(async () => { + currentTime += gatewayCollectionDelays.shift() ?? 0; + return collectedGatewayReadiness( + managedGatewayReadiness(), + new Date(currentTime).toISOString(), + ); + }); + const assessHost = vi.fn(() => { + currentTime += hostCollectionDelays.shift() ?? 0; + return hostWithRuntime("docker"); + }); + + const result = await runReadinessGatedRuntimePreflight( + {}, + { + nonInteractive: true, + now: () => new Date(currentTime), + collectGatewayReadiness, + assessHost, + detectGpu: () => null, + assertDockerBridgeAndContainerDnsHealthy: bridge, + validateSandboxGpuPreflight: validateGpu, + }, + ); + + expect(assessHost).toHaveBeenCalledTimes(3); + expect(collectGatewayReadiness).toHaveBeenCalledTimes(5); + expect(result.gatewayReadiness.evidence).not.toContainEqual( + expect.objectContaining({ id: "gateway.probe.stale" }), + ); + expect(bridge).toHaveBeenCalledOnce(); + expect(validateGpu).toHaveBeenCalledOnce(); }); it("rejects the initial gateway snapshot before collecting host facts", async () => { @@ -515,7 +591,7 @@ describe("readiness-gated runtime preflight", () => { {}, { nonInteractive: true, - collectGatewayReadiness: async () => blocked, + collectGatewayReadiness: async () => collectedGatewayReadiness(blocked), assessHost, detectGpu, exitProcess: exitProcess as never, @@ -549,7 +625,7 @@ describe("readiness-gated runtime preflight", () => { nonInteractive: true, collectGatewayReadiness: async () => { calls.push("gateway-admission"); - return managedGatewayReadiness(); + return collectedGatewayReadiness(); }, assessHost: () => { calls.push("host-observation"); @@ -569,9 +645,10 @@ describe("readiness-gated runtime preflight", () => { "gateway-admission", "host-observation", "gpu-observation", - "gpu-runtime-proof", "gateway-admission", + "gpu-runtime-proof", "host-observation", + "gateway-admission", "gpu-validation", "bridge-dns", ]); @@ -585,7 +662,7 @@ describe("readiness-gated runtime preflight", () => { {}, { nonInteractive: true, - collectGatewayReadiness: async () => managedGatewayReadiness(), + collectGatewayReadiness: async () => collectedGatewayReadiness(), assessHost: wslDockerDesktopHost, detectGpu, warnIfHostProxyMissesLoopback: vi.fn(), @@ -615,7 +692,7 @@ describe("readiness-gated runtime preflight", () => { { sandboxGpu: "enable" }, { nonInteractive: true, - collectGatewayReadiness: async () => managedGatewayReadiness(), + collectGatewayReadiness: async () => collectedGatewayReadiness(), assessHost: wslDockerDesktopHost, detectGpu: () => null, warnIfHostProxyMissesLoopback: vi.fn(), @@ -635,7 +712,7 @@ describe("readiness-gated runtime preflight", () => { const calls: string[] = []; const collectGatewayReadiness = vi.fn(async () => { calls.push("gateway"); - return managedGatewayReadiness(); + return collectedGatewayReadiness(); }); await runReadinessGatedRuntimePreflight( @@ -654,7 +731,16 @@ describe("readiness-gated runtime preflight", () => { }, ); - expect(calls).toEqual(["gateway", "host", "gateway", "host", "gpu", "bridge"]); + expect(calls).toEqual([ + "gateway", + "host", + "gateway", + "host", + "gateway", + "gateway", + "gpu", + "bridge", + ]); }); it("uses the already-qualified portable host facts for runtime probe effects", async () => { @@ -667,7 +753,7 @@ describe("readiness-gated runtime preflight", () => { nonInteractive: true, collectGatewayReadiness: async () => { calls.push("gateway"); - return managedGatewayReadiness(); + return collectedGatewayReadiness(); }, assessHost: () => { calls.push("host"); @@ -680,7 +766,16 @@ describe("readiness-gated runtime preflight", () => { }, ); - expect(calls).toEqual(["gateway", "host", "gateway", "host", "gpu", "bridge"]); + expect(calls).toEqual([ + "gateway", + "host", + "gateway", + "host", + "gateway", + "gateway", + "gpu", + "bridge", + ]); expect(mocks.preparePortableExperimentalHost).not.toHaveBeenCalled(); }); @@ -707,9 +802,9 @@ describe("readiness-gated runtime preflight", () => { ], }); const collectGatewayReadiness = vi - .fn<() => Promise>() - .mockResolvedValueOnce(managedGatewayReadiness()) - .mockResolvedValueOnce(blocked); + .fn<() => Promise>() + .mockResolvedValueOnce(collectedGatewayReadiness()) + .mockResolvedValueOnce(collectedGatewayReadiness(blocked)); await expect( runReadinessGatedRuntimePreflight( @@ -733,9 +828,12 @@ describe("readiness-gated runtime preflight", () => { }); describe("GPU trust-gate rejection reason propagation (#9000)", () => { - const gatedContext = (detectGpu: (deps?: DetectGpuDeps) => GpuDetection | null, host: HostAssessment) => ({ + const gatedContext = ( + detectGpu: (deps?: DetectGpuDeps) => GpuDetection | null, + host: HostAssessment, + ) => ({ nonInteractive: true, - collectGatewayReadiness: async () => managedGatewayReadiness(), + collectGatewayReadiness: async () => collectedGatewayReadiness(), assessHost: () => host, detectGpu, warnIfHostProxyMissesLoopback: vi.fn(), diff --git a/src/lib/onboard/fatal-runtime-preflight.ts b/src/lib/onboard/fatal-runtime-preflight.ts index 237229ce26..c69098d554 100644 --- a/src/lib/onboard/fatal-runtime-preflight.ts +++ b/src/lib/onboard/fatal-runtime-preflight.ts @@ -4,15 +4,20 @@ import { getBuildIdentity } from "../core/version"; import { detectGpu, type GpuDetection } from "../inference/nim"; import { - createGatewayReadinessProjection, + collectGatewayObservations, + type GatewayObservationSnapshot, type GatewayReadinessProjection, - refreshGatewayReadinessProjection, + projectGatewayReadiness, } from "../readiness/gateway"; import { createProductionGatewayReadinessDependencies, type ProductionGatewayReadinessOptions, } from "../readiness/gateway-production"; -import { collectHostObservations, projectHostReadiness } from "../readiness/host"; +import { + collectHostObservations, + type HostObservationSnapshot, + projectHostReadiness, +} from "../readiness/host"; import { evaluateOnboardGatewayReadinessAdmission, evaluateOnboardReadinessAdmission, @@ -85,9 +90,14 @@ export type ReadinessGatedRuntimePreflightContext = Omit< FatalRuntimePreflightContext, "allowStorageRemediation" | "deferEffectfulChecks" > & { - collectGatewayReadiness(): Promise; + collectGatewayReadiness(): Promise; }; +export interface CollectedGatewayReadiness { + projection: GatewayReadinessProjection; + snapshot: GatewayObservationSnapshot; +} + export interface ReadinessGatedRuntimePreflightResult extends FatalRuntimePreflightResult { gatewayReadiness: GatewayReadinessProjection; } @@ -203,15 +213,16 @@ export function assertOnboardGatewayReadiness( throw new Error("Onboarding continued after an unsafe gateway readiness result."); } -/** Collect and admit the production gateway projection before onboarding effects. */ +/** Collect and admit production gateway facts before onboarding effects. */ export async function collectOnboardGatewayReadiness( options: ProductionGatewayReadinessOptions, -): Promise { - const gatewayReadiness = await createGatewayReadinessProjection( +): Promise { + const snapshot = await collectGatewayObservations( createProductionGatewayReadinessDependencies(options), ); - assertOnboardGatewayReadiness(gatewayReadiness); - return gatewayReadiness; + const projection = projectGatewayReadiness(snapshot); + assertOnboardGatewayReadiness(projection); + return { projection, snapshot }; } function isManagedGatewayReadiness(gateway: GatewayReadinessProjection): boolean { @@ -236,17 +247,24 @@ function requiresRuntimeGpuProof( ); } -function refreshOnboardHostReadiness( +interface RuntimeGpuReadiness { + value: GpuDetection | null; + wslDockerDesktopGpuProofPassed?: boolean; + gpuTrustGateRejection?: string; +} + +interface CollectedOnboardHostReadiness { + result: FatalRuntimePreflightResult; + snapshot: HostObservationSnapshot; +} + +function collectOnboardHostReadiness( options: FatalRuntimePreflightOptions, context: FatalRuntimePreflightContext, allowStorageRemediation: boolean, - runtimeGpu?: { - value: GpuDetection | null; - wslDockerDesktopGpuProofPassed?: boolean; - }, -): FatalRuntimePreflightResult { + runtimeGpu?: RuntimeGpuReadiness, +): CollectedOnboardHostReadiness { const now = context.now ?? (() => new Date()); - const observedAt = now().toISOString(); const host = (context.assessHost ?? assessHost)(); let gpuTrustGateRejection: string | undefined; const gpu = runtimeGpu @@ -261,7 +279,14 @@ function refreshOnboardHostReadiness( flag: resolveSandboxGpuFlagFromOptions(options), device: options.sandboxGpuDevice ?? null, }); - const readinessReport = assertOnboardHostReadiness(host, gpu, { + const snapshot = collectHostObservations({ + assess: () => host, + detectGpu: () => gpu, + wslDockerDesktopGpuProofPassed: runtimeGpu?.wslDockerDesktopGpuProofPassed, + now, + }); + const readinessReport = projectHostReadiness(snapshot, { ...getBuildIdentity(), now }); + assertOnboardSystemReadiness(readinessReport, host, { explicitlyOptedOutGpuPassthrough: sandboxGpuConfig.mode === "0" || options.optedOutGpuPassthrough === true, wslDockerDesktopGpuProofPassed: runtimeGpu?.wslDockerDesktopGpuProofPassed, @@ -269,18 +294,97 @@ function refreshOnboardHostReadiness( allowStorageRemediation, allowDeferredN1xManagedVllm: options.allowDeferredN1xManagedVllm, exitProcess: context.exitProcess, - observedAt, - now, }); return { - gpu, - host, - readinessReport, - sandboxGpuConfig, - ...(gpuTrustGateRejection ? { gpuTrustGateRejection } : {}), + result: { + gpu, + host, + readinessReport, + sandboxGpuConfig, + ...(runtimeGpu?.gpuTrustGateRejection || gpuTrustGateRejection + ? { gpuTrustGateRejection: runtimeGpu?.gpuTrustGateRejection ?? gpuTrustGateRejection } + : {}), + }, + snapshot, }; } +function refreshOnboardHostReadiness( + options: FatalRuntimePreflightOptions, + context: FatalRuntimePreflightContext, + allowStorageRemediation: boolean, + runtimeGpu?: RuntimeGpuReadiness, +): FatalRuntimePreflightResult { + return collectOnboardHostReadiness(options, context, allowStorageRemediation, runtimeGpu).result; +} + +function projectCollectedHostReadiness( + collected: CollectedOnboardHostReadiness, + evaluatedAt: Date, +): CollectedOnboardHostReadiness { + return { + ...collected, + result: { + ...collected.result, + readinessReport: projectHostReadiness(collected.snapshot, { + ...getBuildIdentity(), + now: () => evaluatedAt, + }), + }, + }; +} + +function hasStaleHostEvidence(report: SystemReadinessReport): boolean { + return report.evidence.some(({ id }) => id === "host.probe.stale"); +} + +async function collectAdmittedReadinessPair( + collectedHost: CollectedOnboardHostReadiness, + options: FatalRuntimePreflightOptions, + context: ReadinessGatedRuntimePreflightContext, + runtimeGpu?: RuntimeGpuReadiness, +): Promise<{ + host: CollectedOnboardHostReadiness; + gateway: GatewayReadinessProjection; + report: SystemReadinessReport; +}> { + const exitProcess = context.exitProcess ?? exitProcessByDefault; + const now = context.now ?? (() => new Date()); + let collectedGateway = await context.collectGatewayReadiness(); + assertOnboardGatewayReadiness(collectedGateway.projection, exitProcess); + + let evaluatedAt = now(); + let gateway = projectGatewayReadiness(collectedGateway.snapshot, { now: () => evaluatedAt }); + assertOnboardGatewayReadiness(gateway, exitProcess); + let host = projectCollectedHostReadiness(collectedHost, evaluatedAt); + + if (hasStaleHostEvidence(host.result.readinessReport)) { + host = collectOnboardHostReadiness( + options, + context, + isManagedGatewayReadiness(gateway), + runtimeGpu, + ); + collectedGateway = await context.collectGatewayReadiness(); + assertOnboardGatewayReadiness(collectedGateway.projection, exitProcess); + evaluatedAt = now(); + gateway = projectGatewayReadiness(collectedGateway.snapshot, { now: () => evaluatedAt }); + assertOnboardGatewayReadiness(gateway, exitProcess); + host = projectCollectedHostReadiness(host, evaluatedAt); + } + + const report = composeSystemReadinessReport(host.result.readinessReport, gateway); + assertOnboardSystemReadiness(report, host.result.host, { + explicitlyOptedOutGpuPassthrough: + host.result.sandboxGpuConfig.mode === "0" || options.optedOutGpuPassthrough === true, + resuming: context.resuming, + allowStorageRemediation: isManagedGatewayReadiness(gateway), + allowDeferredN1xManagedVllm: options.allowDeferredN1xManagedVllm, + exitProcess, + }); + return { host, gateway, report }; +} + /** Resolve the bounded WSL GPU proof only after canonical readiness admission. */ function resolveRuntimeGpuProof( result: FatalRuntimePreflightResult, @@ -356,51 +460,35 @@ export async function runReadinessGatedRuntimePreflight( context: ReadinessGatedRuntimePreflightContext, ): Promise { const exitProcess = context.exitProcess ?? exitProcessByDefault; - const gatewayBeforePreparation = await context.collectGatewayReadiness(); + const gatewayBeforePreparation = (await context.collectGatewayReadiness()).projection; assertOnboardGatewayReadiness(gatewayBeforePreparation, exitProcess); runFatalOnboardRuntimePreflight(options, { ...context, allowStorageRemediation: isManagedGatewayReadiness(gatewayBeforePreparation), deferEffectfulChecks: true, }); - let gatewayReadiness = await context.collectGatewayReadiness(); + let gatewayReadiness = (await context.collectGatewayReadiness()).projection; assertOnboardGatewayReadiness(gatewayReadiness, exitProcess); let managedGatewayReadiness = isManagedGatewayReadiness(gatewayReadiness); - // Gateway collection can be slow. Replace the earlier host observation so - // the composite gate never stamps an old assessment with a fresh timestamp. - let refreshedResult = refreshOnboardHostReadiness(options, context, managedGatewayReadiness); - gatewayReadiness = refreshGatewayReadinessProjection(gatewayReadiness); - assertOnboardGatewayReadiness(gatewayReadiness, exitProcess); + let collectedHost = collectOnboardHostReadiness(options, context, managedGatewayReadiness); + let admitted = await collectAdmittedReadinessPair(collectedHost, options, context); + collectedHost = admitted.host; + let refreshedResult = collectedHost.result; + gatewayReadiness = admitted.gateway; managedGatewayReadiness = isManagedGatewayReadiness(gatewayReadiness); - let readinessReport = composeSystemReadinessReport( - refreshedResult.readinessReport, - gatewayReadiness, - ); - assertOnboardSystemReadiness(readinessReport, refreshedResult.host, { - explicitlyOptedOutGpuPassthrough: - refreshedResult.sandboxGpuConfig.mode === "0" || options.optedOutGpuPassthrough === true, - resuming: context.resuming, - allowStorageRemediation: managedGatewayReadiness, - allowDeferredN1xManagedVllm: options.allowDeferredN1xManagedVllm, - exitProcess, - }); + // The only GPU detection path that may pull or start a container is delayed // until both canonical host and gateway reports have admitted the run. const runtimeGpu = resolveRuntimeGpuProof(refreshedResult, options, { ...context, allowStorageRemediation: managedGatewayReadiness, }); - refreshedResult = runtimeGpu.result; + let runtimeGpuReadiness: RuntimeGpuReadiness | undefined; if (runtimeGpu.proofRan) { // An explicit GPU request cannot fall back to CPU after a failed proof. // Reject that known configuration error before any later container probe. - exitOnSandboxGpuConfigErrors(refreshedResult.sandboxGpuConfig, exitProcess); - // The bounded proof may pull an image or start a short-lived container. - // Replace both host and gateway observations again before later probes. - gatewayReadiness = await context.collectGatewayReadiness(); - assertOnboardGatewayReadiness(gatewayReadiness, exitProcess); - managedGatewayReadiness = isManagedGatewayReadiness(gatewayReadiness); - refreshedResult = refreshOnboardHostReadiness(options, context, managedGatewayReadiness, { + exitOnSandboxGpuConfigErrors(runtimeGpu.result.sandboxGpuConfig, exitProcess); + runtimeGpuReadiness = { value: runtimeGpu.result.gpu, // `detectGpu()` rejects a failed bounded proof by returning null. Keep // that negative outcome distinct from the observation-only phase's @@ -410,28 +498,25 @@ export async function runReadinessGatedRuntimePreflight( runtimeGpu.result.gpu === null ? false : runtimeGpu.result.gpu.wslDockerDesktopGpuProofPassed, - }); - // The refresh reuses the proof-phase GPU value without re-detecting, so - // carry the proof-phase rejection reason alongside it (#9000). - refreshedResult = { - ...refreshedResult, - ...(runtimeGpu.result.gpuTrustGateRejection - ? { gpuTrustGateRejection: runtimeGpu.result.gpuTrustGateRejection } - : {}), + gpuTrustGateRejection: runtimeGpu.result.gpuTrustGateRejection, }; + collectedHost = collectOnboardHostReadiness( + options, + context, + managedGatewayReadiness, + runtimeGpuReadiness, + ); } - gatewayReadiness = refreshGatewayReadinessProjection(gatewayReadiness); - assertOnboardGatewayReadiness(gatewayReadiness, exitProcess); - managedGatewayReadiness = isManagedGatewayReadiness(gatewayReadiness); - readinessReport = composeSystemReadinessReport(refreshedResult.readinessReport, gatewayReadiness); - assertOnboardSystemReadiness(readinessReport, refreshedResult.host, { - explicitlyOptedOutGpuPassthrough: - refreshedResult.sandboxGpuConfig.mode === "0" || options.optedOutGpuPassthrough === true, - resuming: context.resuming, - allowStorageRemediation: managedGatewayReadiness, - allowDeferredN1xManagedVllm: options.allowDeferredN1xManagedVllm, - exitProcess, - }); + + admitted = await collectAdmittedReadinessPair( + collectedHost, + options, + context, + runtimeGpuReadiness, + ); + refreshedResult = admitted.host.result; + gatewayReadiness = admitted.gateway; + const readinessReport = admitted.report; const gatedResult = { ...refreshedResult, readinessReport, diff --git a/src/lib/onboard/inference-route.test.ts b/src/lib/onboard/inference-route.test.ts index b7f90c7055..5d307cc07e 100644 --- a/src/lib/onboard/inference-route.test.ts +++ b/src/lib/onboard/inference-route.test.ts @@ -50,3 +50,37 @@ describe("verifyInferenceRoute", () => { ); }); }); + +describe("readInferenceRouteState", () => { + it("reports a matched route", () => { + const helpers = createInferenceRouteHelpers(() => + gatewayRoute("compatible-endpoint", "test-model"), + ); + + expect(helpers.readInferenceRouteState("nemoclaw", "compatible-endpoint", "test-model")).toBe( + "matched", + ); + }); + + it.each([ + ["openai-api", "test-model"], + ["compatible-endpoint", "other-model"], + ])("reports a route answered as %s/%s as mismatched", (provider, model) => { + const helpers = createInferenceRouteHelpers(() => gatewayRoute(provider, model)); + + expect(helpers.readInferenceRouteState("nemoclaw", "compatible-endpoint", "test-model")).toBe( + "mismatched", + ); + }); + + it("separates a gateway that cannot answer from a mismatched route (#9310)", () => { + const helpers = createInferenceRouteHelpers(() => null); + + expect(helpers.readInferenceRouteState("nemoclaw", "compatible-endpoint", "test-model")).toBe( + "unanswered", + ); + expect(helpers.isInferenceRouteReady("nemoclaw", "compatible-endpoint", "test-model")).toBe( + false, + ); + }); +}); diff --git a/src/lib/onboard/inference-route.ts b/src/lib/onboard/inference-route.ts index 55b78b08d2..2d3fa42e5d 100644 --- a/src/lib/onboard/inference-route.ts +++ b/src/lib/onboard/inference-route.ts @@ -16,6 +16,9 @@ import { listSandboxes } from "../state/registry"; type RunCaptureOpenshell = (args: string[], options?: { ignoreError?: boolean }) => string | null; +/** A gateway that cannot answer is distinct from one that answers with another route. */ +export type InferenceRouteState = "matched" | "mismatched" | "unanswered"; + /** Resolve the exact portable inference route used by managed clone preparation. */ export function resolveManagedStartupInferenceRoute( agentName: string, @@ -50,11 +53,20 @@ export function createInferenceRouteHelpers( } } - function isInferenceRouteReady(gatewayName: string, provider: string, model: string): boolean { + function readInferenceRouteState( + gatewayName: string, + provider: string, + model: string, + ): InferenceRouteState { const live = parseGatewayInference( runCaptureOpenshell(["inference", "get", "-g", gatewayName], { ignoreError: true }), ); - return Boolean(live && live.provider === provider && live.model === model); + if (!live) return "unanswered"; + return live.provider === provider && live.model === model ? "matched" : "mismatched"; + } + + function isInferenceRouteReady(gatewayName: string, provider: string, model: string): boolean { + return readInferenceRouteState(gatewayName, provider, model) === "matched"; } const checkGatewayRouteCompatibility: CurrentGatewayRouteCompatibilityCheck = (request) => @@ -72,6 +84,7 @@ export function createInferenceRouteHelpers( return { verifyInferenceRoute, isInferenceRouteReady, + readInferenceRouteState, checkGatewayRouteCompatibility, preflightGatewayRouteDiscovery, }; diff --git a/src/lib/onboard/machine/handlers/sandbox-messaging.test.ts b/src/lib/onboard/machine/handlers/sandbox-messaging.test.ts index ce7e784b1c..eceece82f4 100644 --- a/src/lib/onboard/machine/handlers/sandbox-messaging.test.ts +++ b/src/lib/onboard/machine/handlers/sandbox-messaging.test.ts @@ -390,13 +390,13 @@ describe("reconcileReusedSandboxMessaging", () => { it("omits a retired host-backed channel from a reused sandbox selection (#9283)", () => { const plan = discordPlan(hashCredential("previous-discord-token") ?? ""); - const clearPlanEnv = vi.fn(); + const deps = { clearPlanEnv: vi.fn(), note: vi.fn(), writePlanToEnv: vi.fn() }; vi.stubEnv("DISCORD_BOT_TOKEN", ""); const result = reconcileReusedSandboxMessaging( structuredClone(plan), { name: "openclaw" }, - { clearPlanEnv, note: vi.fn(), writePlanToEnv: vi.fn() }, + deps, plan, ); @@ -407,7 +407,7 @@ describe("reconcileReusedSandboxMessaging", () => { selectedChannels: [], changed: true, }); - expect(clearPlanEnv).not.toHaveBeenCalled(); + expect(deps.clearPlanEnv).not.toHaveBeenCalled(); }); it("keeps a still-configured channel in a reused sandbox selection (#9283)", () => { diff --git a/src/lib/onboard/machine/preflight-gateway-authority.test.ts b/src/lib/onboard/machine/preflight-gateway-authority.test.ts index 9b908d9f28..5767033754 100644 --- a/src/lib/onboard/machine/preflight-gateway-authority.test.ts +++ b/src/lib/onboard/machine/preflight-gateway-authority.test.ts @@ -2,7 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 import { afterEach, describe, expect, it, vi } from "vitest"; -import type { GatewayReadinessProjection } from "../../readiness/gateway"; +import type { + GatewayObservationSnapshot, + GatewayReadinessProjection, +} from "../../readiness/gateway"; import type { Session } from "../../state/onboard-session"; import * as fatalRuntimePreflight from "../fatal-runtime-preflight"; import type { GatewayOwner } from "../gateway-ownership"; @@ -42,12 +45,31 @@ describe("preflight gateway authority", () => { findings: [], evidence: [], }; + const gatewaySnapshot: GatewayObservationSnapshot = { + observedAt: "2026-08-17T12:00:00.000Z", + completedAt: "2026-08-17T12:00:00.000Z", + observations: { + owner: { + gatewayName: "nemoclaw", + gatewayPort: 8080, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + supervisor: null, + requiredCapabilities: [], + }, + attachmentState: "not-applicable", + reuseState: "healthy", + driftState: "not-detected", + portConflictState: "none", + }, + }; const collectReadiness = vi.fn(async (collectorDeps) => { events.push("collect readiness"); expect(collectorDeps.gatewayName?.()).toBe("nemoclaw"); expect(collectorDeps.gatewayPort?.()).toBe(8080); expect(collectorDeps.resolveOwner?.()).toBe(owner); - return gatewayReadiness; + return { projection: gatewayReadiness, snapshot: gatewaySnapshot }; }); const session = {} as Session; const deps = { @@ -117,10 +139,15 @@ describe("preflight gateway authority", () => { {}, { nonInteractive: true, - collectGatewayReadiness: authority.collectGatewayReadiness, + collectGatewayReadiness: expect.any(Function), exitProcess, }, ); + const passedCollector = runRuntimePreflight.mock.calls[0]![1].collectGatewayReadiness; + await expect(passedCollector()).resolves.toEqual({ + projection: gatewayReadiness, + snapshot: gatewaySnapshot, + }); await expect(authority.prepareGatewayAuthority()).resolves.toEqual({ externallySupervised: false, @@ -128,6 +155,8 @@ describe("preflight gateway authority", () => { }); expect(events).toEqual([ + "collect readiness", + "read gateway port", "read gateway port", "ensure openshell", "update session", @@ -139,8 +168,9 @@ describe("preflight gateway authority", () => { "select named gateway", "refresh reuse state", ]); - expect(deps.getGatewayOwnerDeps).toHaveBeenCalledOnce(); - expect(deps.gatewayPort).toHaveBeenCalledTimes(2); + expect(deps.getGatewayOwnerDeps).toHaveBeenCalledTimes(2); + expect(deps.gatewayPort).toHaveBeenCalledTimes(3); + expect(collectReadiness).toHaveBeenCalledTimes(2); expect(collectReadiness).toHaveBeenCalledWith( expect.objectContaining({ gatewayName: deps.gatewayName, gatewayPort: deps.gatewayPort }), ); diff --git a/src/lib/onboard/machine/preflight-gateway-authority.ts b/src/lib/onboard/machine/preflight-gateway-authority.ts index 9bb3f0d5ba..7bb71bc998 100644 --- a/src/lib/onboard/machine/preflight-gateway-authority.ts +++ b/src/lib/onboard/machine/preflight-gateway-authority.ts @@ -42,7 +42,7 @@ export interface OnboardPreflightGatewayAuthorityDeps extends Pick { collectGatewayReadiness( deps: OnboardGatewayReadinessCollectorDeps, - ): Promise; + ): Promise; getGatewayOwnerDeps(): { resolveGatewayOwner(): GatewayOwner; probeGatewayAttachment: OnboardGatewayReadinessCollectorDeps["probeAttachment"]; @@ -65,7 +65,7 @@ export interface OnboardPreflightGatewayAuthorityDeps } export function createOnboardPreflightGatewayAuthority(deps: OnboardPreflightGatewayAuthorityDeps) { - const collectGatewayReadiness = () => { + const collectGateway = () => { const ownerDeps = deps.getGatewayOwnerDeps(); return deps.collectGatewayReadiness({ gatewayName: deps.gatewayName, @@ -74,6 +74,7 @@ export function createOnboardPreflightGatewayAuthority(deps: OnboardPreflightGat probeAttachment: ownerDeps.probeGatewayAttachment, }); }; + const collectGatewayReadiness = async () => (await collectGateway()).projection; return { collectGatewayReadiness, runRuntimePreflight: ( @@ -84,7 +85,7 @@ export function createOnboardPreflightGatewayAuthority(deps: OnboardPreflightGat ) => fatalRuntimePreflight.runReadinessGatedRuntimePreflight(options, { nonInteractive: deps.isNonInteractive(), - collectGatewayReadiness, + collectGatewayReadiness: collectGateway, ...(exitProcess ? { exitProcess } : {}), }), prepareGatewayAuthority: () => @@ -113,7 +114,7 @@ export function createOnboardPreflightGatewayAuthority(deps: OnboardPreflightGat export function collectOnboardGatewayReadiness( deps: OnboardGatewayReadinessCollectorDeps, -): Promise { +): Promise { return fatalRuntimePreflight.collectOnboardGatewayReadiness(deps); } diff --git a/src/lib/readiness/gateway.test.ts b/src/lib/readiness/gateway.test.ts index 6091c1b801..71f3cfe650 100644 --- a/src/lib/readiness/gateway.test.ts +++ b/src/lib/readiness/gateway.test.ts @@ -74,7 +74,7 @@ function dependencies(owner: GatewayOwner): GatewayReadinessDependencies { } describe("gateway readiness projection (#7411)", () => { - it("rejects a snapshot made stale by a slow onboarding collection", async () => { + it("admits a slow onboarding collection and records how long it took (#9310)", async () => { let currentTime = NOW.getTime(); const deps = dependencies(managedOwner()); vi.mocked(deps.observeManagedGateway).mockImplementationOnce(async () => { @@ -90,10 +90,16 @@ describe("gateway readiness projection (#7411)", () => { now: () => new Date(currentTime), }); - expect(projection.capabilities.every(({ state }) => state === "unknown")).toBe(true); - expect(projection.evidence).toContainEqual( + expect(projection.capabilities.every(({ state }) => state === "unknown")).toBe(false); + expect(projection.evidence).not.toContainEqual( expect.objectContaining({ id: "gateway.probe.stale" }), ); + expect(projection.evidence).toContainEqual( + expect.objectContaining({ + id: "gateway.owner", + details: expect.objectContaining({ collectionMs: 30_001 }), + }), + ); }); it("collects managed reuse, drift, and port ownership without attachment effects", async () => { @@ -140,27 +146,30 @@ describe("gateway readiness projection (#7411)", () => { it.each([ [{ listenerPids: [4242, 4343] }, "gateway.ownership.multiple", "multiple-owners"], [{ listenerSupervisorMatch: false }, "gateway.ownership.mismatch", "owner-mismatch"], - ] as const)("rejects ambiguous external ownership before any managed operation", async (probeOverrides, findingId, conflictState) => { - const owner = externalOwner(); - const deps = dependencies(owner); - vi.mocked(deps.probeAttachment).mockResolvedValueOnce(attachment(probeOverrides)); + ] as const)( + "rejects ambiguous external ownership before any managed operation", + async (probeOverrides, findingId, conflictState) => { + const owner = externalOwner(); + const deps = dependencies(owner); + vi.mocked(deps.probeAttachment).mockResolvedValueOnce(attachment(probeOverrides)); - const projection = projectGatewayReadiness( - await collectGatewayObservations(deps, { now: () => NOW }), - { now: () => NOW }, - ); + const projection = projectGatewayReadiness( + await collectGatewayObservations(deps, { now: () => NOW }), + { now: () => NOW }, + ); - expect(deps.observeManagedGateway).not.toHaveBeenCalled(); - expect(projection.findings).toContainEqual( - expect.objectContaining({ id: findingId, severity: "blocking" }), - ); - expect(projection.observations).toContainEqual( - expect.objectContaining({ id: "gateway.port_conflict", value: conflictState }), - ); - expect(projection.capabilities).toContainEqual( - expect.objectContaining({ id: "gateway.attachment.valid", state: "absent" }), - ); - }); + expect(deps.observeManagedGateway).not.toHaveBeenCalled(); + expect(projection.findings).toContainEqual( + expect.objectContaining({ id: findingId, severity: "blocking" }), + ); + expect(projection.observations).toContainEqual( + expect.objectContaining({ id: "gateway.port_conflict", value: conflictState }), + ); + expect(projection.capabilities).toContainEqual( + expect.objectContaining({ id: "gateway.attachment.valid", state: "absent" }), + ); + }, + ); it("redacts probe failures and omits private gateway state", async () => { const token = `nvapi-${"a".repeat(24)}`; @@ -205,17 +214,20 @@ describe("gateway readiness projection (#7411)", () => { expect(projection.capabilities.every(({ state }) => state === "unknown")).toBe(true); }); - it("rejects observations older than 30 seconds unless collection marked them reusable", async () => { + it("rejects observations held longer than 30 seconds and reports the measured age", async () => { const snapshot = await collectGatewayObservations(dependencies(managedOwner()), { now: () => NOW, }); - const stale = { ...snapshot, observedAt: "2026-08-07T11:00:00.000Z", reusable: false }; + const stale = { ...snapshot, completedAt: "2026-08-07T11:00:00.000Z" }; const projection = projectGatewayReadiness(stale, { now: () => NOW }); expect(projection.capabilities.every(({ state }) => state === "unknown")).toBe(true); expect(projection.evidence).toContainEqual( - expect.objectContaining({ id: "gateway.probe.stale" }), + expect.objectContaining({ + id: "gateway.probe.stale", + details: expect.objectContaining({ ageMs: expect.any(Number), windowMs: 30_000 }), + }), ); }); diff --git a/src/lib/readiness/gateway.ts b/src/lib/readiness/gateway.ts index f08900f12f..b16ab05675 100644 --- a/src/lib/readiness/gateway.ts +++ b/src/lib/readiness/gateway.ts @@ -12,6 +12,7 @@ import { isExternallySupervised, } from "../onboard/gateway-ownership"; import type { GatewayReuseState } from "../state/gateway"; +import { measureObservationAge, type ObservationAge, staleEvidence } from "./observation-age"; import { sanitizeReadinessText } from "./sanitize"; import type { EvidenceScalar, @@ -24,10 +25,6 @@ import type { const DEFAULT_MAX_AGE_MS = 30_000; const MAX_REPORT_TEXT_LENGTH = 1024; -const projectionSnapshots = new WeakMap< - GatewayReadinessProjection, - Readonly ->(); export type GatewayAttachmentState = "verified" | "rejected" | "not-applicable" | "unknown"; export type GatewayDriftState = "detected" | "not-detected" | "not-applicable" | "unknown"; @@ -65,10 +62,10 @@ export interface GatewayObservations { export interface GatewayObservationSnapshot { observedAt: string; + completedAt: string; observations?: Readonly; failure?: string; authorityFailure?: boolean; - reusable?: boolean; } export interface CollectGatewayObservationsOptions { @@ -125,11 +122,20 @@ function rejectedAttachment( }; } +/** Stamp the completion of a collection so its own duration cannot age it out. */ export async function collectGatewayObservations( deps: GatewayReadinessDependencies, options: CollectGatewayObservationsOptions = {}, ): Promise { - const observedAt = (options.now ?? (() => new Date()))().toISOString(); + const now = options.now ?? (() => new Date()); + const observed = await observeGateway(deps, now().toISOString()); + return { ...observed, completedAt: now().toISOString() }; +} + +async function observeGateway( + deps: GatewayReadinessDependencies, + observedAt: string, +): Promise> { let owner: GatewayOwner; try { owner = deps.resolveOwner(); @@ -138,7 +144,6 @@ export async function collectGatewayObservations( observedAt, failure: "Gateway lifecycle authority could not be resolved before lifecycle effects.", authorityFailure: true, - reusable: false, }; } @@ -149,7 +154,6 @@ export async function collectGatewayObservations( return { observedAt, observations: rejectedAttachment(owner, result.code, result.message), - reusable: false, }; } return { @@ -161,14 +165,12 @@ export async function collectGatewayObservations( driftState: "not-applicable", portConflictState: "none", }, - reusable: false, }; } catch (error) { if (error instanceof GatewayOwnershipError) { return { observedAt, observations: rejectedAttachment(owner, error.code, error.message), - reusable: false, }; } return { @@ -181,7 +183,6 @@ export async function collectGatewayObservations( portConflictState: "unknown", }, failure: "The externally supervised gateway attachment probe failed safely.", - reusable: false, }; } } @@ -200,7 +201,6 @@ export async function collectGatewayObservations( ? safeReportText(managed.portConflictDetail) : undefined, }, - reusable: false, }; } catch { return { @@ -213,7 +213,6 @@ export async function collectGatewayObservations( portConflictState: "unknown", }, failure: "Managed gateway observations could not be collected safely.", - reusable: false, }; } } @@ -260,7 +259,7 @@ const ATTACHMENT_FINDING_IDS: Record = { function unknownProjection( snapshot: Readonly, - stale = false, + stale?: ObservationAge, ): GatewayReadinessProjection { const evidenceIds = [ ...(snapshot.failure ? ["gateway.probe.failure"] : []), @@ -301,24 +300,22 @@ function unknownProjection( ? [{ id: "gateway.probe.failure", summary: safeReportText(snapshot.failure) }] : []), ...(stale - ? [ - { - id: "gateway.probe.stale", - summary: "Gateway observations exceeded their safe reuse window.", - }, - ] + ? [staleEvidence("gateway.probe.stale", "Gateway", snapshot.completedAt, stale)] : []), ], }; - projectionSnapshots.set(projection, snapshot); return projection; } -function ownerEvidence(owner: GatewayOwnerDescription): ReadinessEvidence { +function ownerEvidence( + owner: GatewayOwnerDescription, + snapshot: Readonly, +): ReadinessEvidence { return { id: "gateway.owner", summary: "Resolved gateway lifecycle authority.", details: { + collectionMs: Date.parse(snapshot.completedAt) - Date.parse(snapshot.observedAt), gatewayName: safeReportText(owner.gatewayName), gatewayPort: owner.gatewayPort, mode: owner.mode, @@ -332,29 +329,22 @@ function ownerEvidence(owner: GatewayOwnerDescription): ReadinessEvidence { }; } -/** Re-evaluate the original snapshot after another readiness collection. */ -export function refreshGatewayReadinessProjection( - projection: GatewayReadinessProjection, - options: ProjectGatewayReadinessOptions = {}, -): GatewayReadinessProjection { - const snapshot = projectionSnapshots.get(projection); - return snapshot ? projectGatewayReadiness(snapshot, options) : projection; -} - export function projectGatewayReadiness( snapshot: Readonly, options: ProjectGatewayReadinessOptions = {}, ): GatewayReadinessProjection { const now = (options.now ?? (() => new Date()))(); - const age = now.getTime() - Date.parse(snapshot.observedAt); - const stale = - !Number.isFinite(age) || age < 0 || age > (options.maxObservationAgeMs ?? DEFAULT_MAX_AGE_MS); - if (stale && snapshot.reusable !== true) return unknownProjection(snapshot, true); + const stale = measureObservationAge( + snapshot.completedAt, + now, + options.maxObservationAgeMs ?? DEFAULT_MAX_AGE_MS, + ); + if (stale) return unknownProjection(snapshot, stale); const gateway = snapshot.observations; if (!gateway) return unknownProjection(snapshot); - const evidence: ReadinessEvidence[] = [ownerEvidence(gateway.owner)]; + const evidence: ReadinessEvidence[] = [ownerEvidence(gateway.owner, snapshot)]; const evidenceIds = ["gateway.owner"]; if (snapshot.failure) { evidence.push({ id: "gateway.probe.failure", summary: safeReportText(snapshot.failure) }); @@ -529,7 +519,6 @@ export function projectGatewayReadiness( findings, evidence, }; - projectionSnapshots.set(projection, snapshot); return projection; } diff --git a/src/lib/readiness/host.test.ts b/src/lib/readiness/host.test.ts index e04cd12f2a..6f8c06436b 100644 --- a/src/lib/readiness/host.test.ts +++ b/src/lib/readiness/host.test.ts @@ -3,9 +3,7 @@ import Ajv2020, { type AnySchema } from "ajv/dist/2020.js"; import { beforeEach, describe, expect, it, vi } from "vitest"; -import systemReadinessSchema from "../../../schemas/system-readiness.schema.json" with { - type: "json", -}; +import systemReadinessSchema from "../../../schemas/system-readiness.schema.json" with { type: "json" }; import type { GpuDetection, NvidiaPlatform } from "../inference/nim"; import type { HostAssessment } from "../onboard/preflight"; import { collectHostObservations, createHostReadinessReport, projectHostReadiness } from "./host"; @@ -171,12 +169,15 @@ describe("host readiness projection (#7408)", () => { "absent", "host.gpu.cdi_stale", ], - ] as const)("returns stable results for %s", (overrides, capabilityId, expectedState, findingId) => { - const result = report(overrides); - - expect(state(result, capabilityId)).toBe(expectedState); - expect(findingIds(result)).toContain(findingId); - }); + ] as const)( + "returns stable results for %s", + (overrides, capabilityId, expectedState, findingId) => { + const result = report(overrides); + + expect(state(result, capabilityId)).toBe(expectedState); + expect(findingIds(result)).toContain(findingId); + }, + ); it("blocks a reachable but unsupported DOCKER_HOST before using daemon evidence (#7411)", () => { const result = report({ dockerHostInvalid: true, dockerReachable: true }); @@ -487,7 +488,7 @@ describe("host readiness projection (#7408)", () => { collectPlatformIdentity: emptyPlatformIdentity, now: () => NOW, }); - const snapshot = { ...current, observedAt: "2026-06-01T11:00:00Z", reusable: false }; + const snapshot = { ...current, completedAt: "2026-06-01T11:00:00Z" }; const result = projectHostReadiness(snapshot, { nemoclawVersion: "0.1.0", sourceRevision: SOURCE_REVISION, @@ -501,19 +502,38 @@ describe("host readiness projection (#7408)", () => { ).toBe(true); }); - it.each([ - ["2026-06-01T11:00:00Z", true], - ["2026-06-01T11:59:30Z", false], - ] as const)("projects safe snapshot reuse at %s", (observedAt, reusable) => { - const current = collectHostObservations({ + it.each(["2026-06-01T11:59:30Z", "2026-06-01T12:00:00Z"] as const)( + "projects safe snapshot reuse at %s", + (completedAt) => { + const current = collectHostObservations({ + assess: () => host(), + collectPlatformIdentity: emptyPlatformIdentity, + now: () => NOW, + }); + const result = projectHostReadiness( + { ...current, completedAt }, + { nemoclawVersion: "0.1.0", sourceRevision: SOURCE_REVISION, now: () => NOW }, + ); + + expect(result.status).toBe("supported"); + expect(result.evidence.map(({ id }) => id)).not.toContain("host.probe.stale"); + }, + ); + + it("admits a collection that was itself slower than the reuse window (#9310)", () => { + const clock = [new Date(NOW.getTime() - 45_000), NOW]; + let index = 0; + + const snapshot = collectHostObservations({ assess: () => host(), collectPlatformIdentity: emptyPlatformIdentity, + now: () => clock[Math.min(index++, clock.length - 1)] ?? NOW, + }); + const result = projectHostReadiness(snapshot, { + nemoclawVersion: "0.1.0", + sourceRevision: SOURCE_REVISION, now: () => NOW, }); - const result = projectHostReadiness( - { ...current, observedAt, reusable }, - { nemoclawVersion: "0.1.0", sourceRevision: SOURCE_REVISION, now: () => NOW }, - ); expect(result.status).toBe("supported"); expect(result.evidence.map(({ id }) => id)).not.toContain("host.probe.stale"); diff --git a/src/lib/readiness/host.ts b/src/lib/readiness/host.ts index 5464b1d163..811d31c56e 100644 --- a/src/lib/readiness/host.ts +++ b/src/lib/readiness/host.ts @@ -17,6 +17,7 @@ import { type PlatformIdentity, projectPlatformQualification, } from "./platform-qualification.js"; +import { measureObservationAge, staleEvidence } from "./observation-age.js"; import { buildSystemReadinessProbeEnv, createSystemReadinessCapture } from "./probe-env.js"; import { sanitizeReadinessText } from "./sanitize.js"; import { @@ -69,9 +70,9 @@ export interface HostObservations { export interface HostObservationSnapshot { observedAt: string; + completedAt: string; observations?: Readonly; failure?: string; - reusable?: boolean; } export interface CollectHostObservationsOptions { @@ -146,10 +147,19 @@ function adaptHostAssessment( }; } +/** Stamp the completion of a collection so its own duration cannot age it out. */ export function collectHostObservations( options: CollectHostObservationsOptions = {}, ): HostObservationSnapshot { - const observedAt = (options.now ?? (() => new Date()))().toISOString(); + const now = options.now ?? (() => new Date()); + const observed = observeHost(options, now().toISOString()); + return { ...observed, completedAt: now().toISOString() }; +} + +function observeHost( + options: CollectHostObservationsOptions, + observedAt: string, +): Omit { try { const probeEnv = buildSystemReadinessProbeEnv(); const runCaptureImpl = createSystemReadinessCapture(probeEnv); @@ -212,13 +222,11 @@ export function collectHostObservations( )(), wslDockerDesktopGpuProofPassed, ), - reusable: false, }; } catch (error) { return { observedAt, failure: safeReportText(error instanceof Error ? error.message : String(error)), - reusable: false, }; } } @@ -320,19 +328,17 @@ export function projectHostReadiness( options: CreateHostReadinessReportOptions, ): SystemReadinessReport { const now = (options.now ?? (() => new Date()))(); - const age = now.getTime() - Date.parse(snapshot.observedAt); - const stale = - !Number.isFinite(age) || age < 0 || age > (options.maxObservationAgeMs ?? DEFAULT_MAX_AGE_MS); - const unsafeReuse = stale && snapshot.reusable !== true; + const unsafeReuse = measureObservationAge( + snapshot.completedAt, + now, + options.maxObservationAgeMs ?? DEFAULT_MAX_AGE_MS, + ); const evidence: ReadinessEvidence[] = []; if (snapshot.failure) { evidence.push({ id: "host.probe.failure", summary: safeReportText(snapshot.failure) }); } if (unsafeReuse) { - evidence.push({ - id: "host.probe.stale", - summary: "Host observations exceeded their safe reuse window.", - }); + evidence.push(staleEvidence("host.probe.stale", "Host", snapshot.completedAt, unsafeReuse)); } let observations: ReadinessObservation[]; diff --git a/src/lib/readiness/index.ts b/src/lib/readiness/index.ts index 8a3c0c1c46..0a92776fd8 100644 --- a/src/lib/readiness/index.ts +++ b/src/lib/readiness/index.ts @@ -21,7 +21,6 @@ export { collectGatewayObservations, createGatewayReadinessProjection, projectGatewayReadiness, - refreshGatewayReadinessProjection, } from "./gateway.js"; export type { CollectHostObservationsOptions, diff --git a/src/lib/readiness/observation-age.ts b/src/lib/readiness/observation-age.ts new file mode 100644 index 0000000000..fc78b000b7 --- /dev/null +++ b/src/lib/readiness/observation-age.ts @@ -0,0 +1,37 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { ReadinessEvidence } from "./types"; + +export interface ObservationAge { + ageMs: number | null; + windowMs: number; +} + +/** Report the age of an observation only when it falls outside the safe reuse window. */ +export function measureObservationAge( + observedAt: string, + now: Date, + windowMs: number, +): ObservationAge | null { + const ageMs = now.getTime() - Date.parse(observedAt); + if (!Number.isFinite(ageMs) || ageMs < 0) return { ageMs: null, windowMs }; + return ageMs > windowMs ? { ageMs, windowMs } : null; +} + +export function staleEvidence( + id: string, + subject: "Gateway" | "Host", + completedAt: string, + { ageMs, windowMs }: ObservationAge, +): ReadinessEvidence { + const measured = + ageMs === null + ? `completed at ${completedAt}, which is not usable against the ${String(windowMs)}ms window` + : `${String(ageMs)}ms old against a ${String(windowMs)}ms window`; + return { + id, + summary: `${subject} observations exceeded their safe reuse window: ${measured}.`, + details: { completedAt, ageMs, windowMs }, + }; +} diff --git a/src/lib/readiness/system.test.ts b/src/lib/readiness/system.test.ts index b837ab12d5..aead299385 100644 --- a/src/lib/readiness/system.test.ts +++ b/src/lib/readiness/system.test.ts @@ -3,9 +3,7 @@ import Ajv2020, { type AnySchema } from "ajv/dist/2020.js"; import { describe, expect, it } from "vitest"; -import systemReadinessSchema from "../../../schemas/system-readiness.schema.json" with { - type: "json", -}; +import systemReadinessSchema from "../../../schemas/system-readiness.schema.json" with { type: "json" }; import type { HostAssessment } from "../onboard/preflight"; import { type GatewayObservationSnapshot, projectGatewayReadiness } from "./gateway"; import { createPublicReadinessReport } from "./presentation"; @@ -37,7 +35,7 @@ function hostReport(): SystemReadinessReport { function gatewaySnapshot(): GatewayObservationSnapshot { return { observedAt: NOW.toISOString(), - reusable: false, + completedAt: NOW.toISOString(), observations: { owner: { gatewayName: "nemoclaw", @@ -123,7 +121,7 @@ describe("composite system readiness (#7411)", () => { ); }); - it("marks host and gateway facts stale when collection exceeds the reuse window", async () => { + it("marks facts held across a slow gateway probe stale, but not the probe itself (#9310)", async () => { let currentTime = NOW.getTime(); const report = await createSystemReadinessReport( { @@ -167,14 +165,15 @@ describe("composite system readiness (#7411)", () => { }, ); + // The host facts were collected before the probe and held across it, so + // they age out. The gateway facts are as fresh as collection can make them. expect(report.status).toBe("inconclusive"); expect(report.evidence).toEqual( - expect.arrayContaining([ - expect.objectContaining({ id: "host.probe.stale" }), - expect.objectContaining({ id: "gateway.probe.stale" }), - ]), + expect.arrayContaining([expect.objectContaining({ id: "host.probe.stale" })]), + ); + expect(report.evidence).not.toContainEqual( + expect.objectContaining({ id: "gateway.probe.stale" }), ); expect(report.capabilities.length).toBeGreaterThan(0); - expect(report.capabilities.every(({ state }) => state === "unknown")).toBe(true); }); }); diff --git a/test/e2e/support/e2e-collaborator-permission-retry.test.ts b/test/e2e/support/e2e-collaborator-permission-retry.test.ts index 635d785e2a..fd352addd7 100644 --- a/test/e2e/support/e2e-collaborator-permission-retry.test.ts +++ b/test/e2e/support/e2e-collaborator-permission-retry.test.ts @@ -36,9 +36,9 @@ const AUTHORIZATION_STEPS: AuthorizationStep[] = [ name: "Authorize release qualification waiver", }, { - deniedMessage: "Launchable image publication requires a repository maintainer or administrator", - mismatchMessage: "Launchable image publication permission response did not match the actor", - name: "Authorize Launchable image publication", + deniedMessage: "Launchable E2E requires a repository maintainer or administrator", + mismatchMessage: "Launchable E2E permission response did not match the actor", + name: "Authorize Launchable E2E maintainer dispatch", }, ]; diff --git a/test/onboard-inference-reconciliation.test.ts b/test/onboard-inference-reconciliation.test.ts index 41ce65124b..06b7799fbc 100644 --- a/test/onboard-inference-reconciliation.test.ts +++ b/test/onboard-inference-reconciliation.test.ts @@ -312,18 +312,44 @@ gatewayState.isGatewayHealthy = () => true; dockerDriverPlatform.isLinuxDockerDriverGatewayEnabled = () => false; gatewayGpuPassthrough.reconcileGatewayGpuReuseForGpuIntent = ({ gatewayReuseState }) => gatewayReuseState; onboardProbes.verifyOnboardInferenceSmoke = () => {}; -preflightGatewayAuthority.collectOnboardGatewayReadiness = async () => ({ - observations: [{ id: "gateway.management.mode", state: "present", value: "nemoclaw-managed" }], - capabilities: [ - "gateway.authority.resolved", - "gateway.attachment.valid", - "gateway.reuse.ready", - "gateway.version.compatible", - "gateway.port.uncontested", - ].map((id) => ({ id, state: "present" })), - findings: [], - evidence: [], -}); +preflightGatewayAuthority.collectOnboardGatewayReadiness = async () => { + const completedAt = new Date().toISOString(); + return { + projection: { + observations: [ + { id: "gateway.management.mode", state: "present", value: "nemoclaw-managed" }, + ], + capabilities: [ + "gateway.authority.resolved", + "gateway.attachment.valid", + "gateway.reuse.ready", + "gateway.version.compatible", + "gateway.port.uncontested", + ].map((id) => ({ id, state: "present" })), + findings: [], + evidence: [], + }, + snapshot: { + observedAt: completedAt, + completedAt, + observations: { + owner: { + gatewayName: "nemoclaw", + gatewayPort: 8080, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + supervisor: null, + requiredCapabilities: [], + }, + attachmentState: "not-applicable", + reuseState: "healthy", + driftState: "not-detected", + portConflictState: "none", + }, + }, + }; +}; const complete = () => ({ status: "complete",