diff --git a/ci/full-e2e-cold-path-calibration.json b/ci/full-e2e-cold-path-calibration.json index dab55d06d96..1bf2181b48a 100644 --- a/ci/full-e2e-cold-path-calibration.json +++ b/ci/full-e2e-cold-path-calibration.json @@ -1,5 +1,5 @@ { - "$comment": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\nSPDX-License-Identifier: Apache-2.0\n\nFive independent workflow_dispatch full-e2e samples of current-main baseline 1a74b8348c2182fbf806726341186f008444e28e with the phase-measurement changes at 4544d07c8bfd500c3b64a74380ef5cd0e62089f5. All runs completed install, BuildKit prebuild without fallback, the silence assertion, and the expected first turn. Each budget is derived independently so phase caps diagnose regressions; they are not portions that must sum to the root-start budget.", + "$comment": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\nSPDX-License-Identifier: Apache-2.0\n\nFive independent workflow_dispatch full-e2e samples of current-main baseline 1a74b8348c2182fbf806726341186f008444e28e with the phase-measurement changes at 4544d07c8bfd500c3b64a74380ef5cd0e62089f5. All runs completed install, BuildKit prebuild without fallback, the silence assertion, and the expected first turn. Each baseline budget is derived independently so phase caps diagnose regressions; they are not portions that must sum to the root-start budget. The separate validation adjustment records four existing full-e2e job observations from three descendant heads after f62c278bd737f4f47be2e85436f65b270d5b4280 added the reviewed WeChat runtime graph to the supported sandbox image. The relevant image-building inputs listed in imageInputPaths were unchanged through 2adc8481ff3053a5a7be37d130cb183e222934ff; repository tests enforce that ancestry and unchanged-input boundary. All four jobs completed install, BuildKit prebuild without fallback, the silence assertion, and the expected first turn; two exceeded the prior root-start and/or sandbox-phase caps. The conclusion fields record the full-e2e job conclusions, not aggregate workflow conclusions. The adjustment raises only those two caps from the maximum observed value plus the existing headroom, rounded up to one second. This is a bounded post-change validation adjustment, not a replacement five-run exact-head calibration. Retire it by replacing the baseline and removing validationAdjustment after five successful full-e2e samples from one exact head that contains the image change.", "schemaVersion": 1, "calibratedAt": "2026-07-13", "baselineMainSha": "1a74b8348c2182fbf806726341186f008444e28e", @@ -150,6 +150,183 @@ } } ], + "validationAdjustment": { + "validatedAt": "2026-07-14", + "imageChangeSha": "f62c278bd737f4f47be2e85436f65b270d5b4280", + "imageInputsVerifiedThroughSha": "2adc8481ff3053a5a7be37d130cb183e222934ff", + "imageInputPaths": [ + "Dockerfile", + "tsconfig.runtime-preloads.json", + "agents/openclaw/mcporter-runtime", + "agents/openclaw/wechat-runtime", + "nemoclaw/package.json", + "nemoclaw/package-lock.json", + "nemoclaw/tsconfig.json", + "nemoclaw/openclaw.plugin.json", + "nemoclaw/src", + "nemoclaw-blueprint/blueprint.yaml", + "nemoclaw-blueprint/policies", + "nemoclaw-blueprint/scripts", + "nemoclaw-blueprint/openclaw-plugins", + "nemoclaw-blueprint/model-specific-setup", + "scripts/checks/verify-openshell-policy-boundary-dependencies.mts", + "scripts/nemoclaw-start.sh", + "scripts/gateway-control.sh", + "scripts/managed-gateway-control.py", + "scripts/state-dir-guard.py", + "scripts/openclaw-config-guard.py", + "scripts/codex-acp-wrapper.sh", + "scripts/generate-openclaw-config.mts", + "scripts/validate-openclaw-tool-search.mts", + "scripts/lib/sandbox-init.sh", + "scripts/lib/gateway-supervisor.sh", + "scripts/lib/sandbox-rlimits.sh", + "scripts/lib/openclaw_device_approval_policy.py", + "scripts/lib/clean_runtime_shell_env_shim.py", + "scripts/lib/normalize_mutable_config_perms.py", + "src/lib/messaging", + "src/lib/tool-disclosure.ts", + "scripts/patch-openclaw-tool-catalog.js", + "scripts/patch-openclaw-chat-send.js", + "scripts/patch-openclaw-mcp-npx.mts", + "scripts/patch-openclaw-issue-4434-diagnostics.ts", + "scripts/patch-openclaw-device-self-approval.ts", + "scripts/verify-wechat-runtime-lock.mts", + "scripts/lib/reviewed-npm-archive.mts", + "src/lib/sandbox/build-context.ts" + ], + "adjustedMetrics": [ + "rootStartToFirstTurnCompletion", + "nemoclaw.onboard.phase.sandbox" + ], + "derivation": { + "statistic": "maximum", + "minimumHeadroomMs": 5000, + "relativeHeadroomPercent": 10, + "roundUpMs": 1000 + }, + "retirement": { + "trigger": "successful-exact-head-calibration", + "minimumSampleCount": 5, + "allSamplesSameHead": true, + "imageChangeMustBeAncestor": true, + "action": "replace-baseline-and-remove-adjustment" + }, + "runs": [ + { + "runId": 29296660267, + "runUrl": "https://github.com/NVIDIA/NemoClaw/actions/runs/29296660267", + "headSha": "b634f011e78aa6c31537a70ff76d57bfa8ad7ee6", + "conclusion": "failure", + "installExitCode": 0, + "firstTurnExitCode": 0, + "performancePassed": false, + "usedBuildKitPrebuild": true, + "buildKitFallback": false, + "maxSilenceSecs": 32, + "responseChars": 23, + "measurementsMs": { + "onboardRoot": 191509, + "rootStartToFirstTurnCompletion": 199164, + "rootEndToInstallCompletion": 181, + "firstTurnCommand": 7470, + "rootEndToFirstTurnCompletion": 7655, + "phases": { + "nemoclaw.onboard.phase.preflight": 3526, + "nemoclaw.onboard.phase.gateway": 66, + "nemoclaw.onboard.phase.provider_selection": 1257, + "nemoclaw.onboard.phase.inference": 1309, + "nemoclaw.onboard.phase.sandbox": 149492 + } + } + }, + { + "runId": 29307095519, + "runUrl": "https://github.com/NVIDIA/NemoClaw/actions/runs/29307095519", + "headSha": "4f64b0ac19d321c6a85c88799c5675e33c3c1a7e", + "conclusion": "success", + "installExitCode": 0, + "firstTurnExitCode": 0, + "performancePassed": true, + "usedBuildKitPrebuild": true, + "buildKitFallback": false, + "maxSilenceSecs": 33, + "responseChars": 23, + "measurementsMs": { + "onboardRoot": 187160, + "rootStartToFirstTurnCompletion": 195237, + "rootEndToInstallCompletion": 168, + "firstTurnCommand": 7905, + "rootEndToFirstTurnCompletion": 8077, + "phases": { + "nemoclaw.onboard.phase.preflight": 1204, + "nemoclaw.onboard.phase.gateway": 63, + "nemoclaw.onboard.phase.provider_selection": 2187, + "nemoclaw.onboard.phase.inference": 1467, + "nemoclaw.onboard.phase.sandbox": 139921 + } + } + }, + { + "runId": 29313003676, + "runUrl": "https://github.com/NVIDIA/NemoClaw/actions/runs/29313003676", + "headSha": "2adc8481ff3053a5a7be37d130cb183e222934ff", + "conclusion": "failure", + "installExitCode": 0, + "firstTurnExitCode": 0, + "performancePassed": false, + "usedBuildKitPrebuild": true, + "buildKitFallback": false, + "maxSilenceSecs": 39, + "responseChars": 23, + "measurementsMs": { + "onboardRoot": 199724, + "rootStartToFirstTurnCompletion": 206895, + "rootEndToInstallCompletion": 139, + "firstTurnCommand": 7028, + "rootEndToFirstTurnCompletion": 7171, + "phases": { + "nemoclaw.onboard.phase.preflight": 1222, + "nemoclaw.onboard.phase.gateway": 64, + "nemoclaw.onboard.phase.provider_selection": 2634, + "nemoclaw.onboard.phase.inference": 1072, + "nemoclaw.onboard.phase.sandbox": 155080 + } + } + }, + { + "runId": 29313408425, + "runUrl": "https://github.com/NVIDIA/NemoClaw/actions/runs/29313408425", + "headSha": "2adc8481ff3053a5a7be37d130cb183e222934ff", + "conclusion": "success", + "installExitCode": 0, + "firstTurnExitCode": 0, + "performancePassed": true, + "usedBuildKitPrebuild": true, + "buildKitFallback": false, + "maxSilenceSecs": 31, + "responseChars": 23, + "measurementsMs": { + "onboardRoot": 185646, + "rootStartToFirstTurnCompletion": 193112, + "rootEndToInstallCompletion": 180, + "firstTurnCommand": 7282, + "rootEndToFirstTurnCompletion": 7466, + "phases": { + "nemoclaw.onboard.phase.preflight": 1262, + "nemoclaw.onboard.phase.gateway": 51, + "nemoclaw.onboard.phase.provider_selection": 1367, + "nemoclaw.onboard.phase.inference": 708, + "nemoclaw.onboard.phase.sandbox": 141977 + } + } + } + ], + "derivedCapsMs": { + "rootStartToFirstTurnCompletionBudgetMs": 228000, + "sandboxPhaseBudgetMs": 171000 + } + }, "derivedBudgetsMs": { "rootStartToFirstTurnCompletionBudgetMs": 205000, "rootEndToFirstTurnCompletionBudgetMs": 14000, diff --git a/ci/onboard-performance-budget.json b/ci/onboard-performance-budget.json index 4b3bc235c68..2519461126d 100644 --- a/ci/onboard-performance-budget.json +++ b/ci/onboard-performance-budget.json @@ -1,5 +1,5 @@ { - "$comment": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\nSPDX-License-Identifier: Apache-2.0\n\nInitial advisory budget for the cloud-onboard-e2e warm-system trace signal. Profiling traces from #3769 were not available in durable CI artifacts when #3776 was implemented. The latest three release tags only exposed one tag-matching trace artifact, from a failed v0.0.66 nightly run, so this cap is calibrated from the latest ten distinct successful main full-trace samples available on 2026-06-23. Those samples had total durations of 298250 ms, 296926 ms, 304190 ms, 294859 ms, 305013 ms, 316147 ms, 300843 ms, 292702 ms, 201332 ms, and 206250 ms; the cap uses p95 via linear interpolation (index 8.55 between samples 8 and 9) plus 25 percent, rounded up to the nearest 30 seconds.\n\nThe full-e2e cold-path budgets are derived from the five current-main samples recorded in ci/full-e2e-cold-path-calibration.json. For each interval and phase independently, the budget is nearest-rank p95 plus the larger of 5 seconds or 10 percent, rounded up to the nearest second. Independent phase caps are diagnostic regression gates, not additive portions of the root-start interval.", + "$comment": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\nSPDX-License-Identifier: Apache-2.0\n\nInitial advisory budget for the cloud-onboard-e2e warm-system trace signal. Profiling traces from #3769 were not available in durable CI artifacts when #3776 was implemented. The latest three release tags only exposed one tag-matching trace artifact, from a failed v0.0.66 nightly run, so this cap is calibrated from the latest ten distinct successful main full-trace samples available on 2026-06-23. Those samples had total durations of 298250 ms, 296926 ms, 304190 ms, 294859 ms, 305013 ms, 316147 ms, 300843 ms, 292702 ms, 201332 ms, and 206250 ms; the cap uses p95 via linear interpolation (index 8.55 between samples 8 and 9) plus 25 percent, rounded up to the nearest 30 seconds.\n\nThe full-e2e cold-path baseline budgets are derived from the five current-main samples recorded in ci/full-e2e-cold-path-calibration.json. For each interval and phase independently, the baseline is nearest-rank p95 plus the larger of 5 seconds or 10 percent, rounded up to the nearest second. Four later functional full-e2e jobs completed install, BuildKit prebuild without fallback, and the expected first turn on their listed heads after the supported sandbox image changed; their separately recorded bounded validation adjustment raises only the root-start and sandbox-phase caps to the observed maximum plus 10 percent, rounded up to one second. The adjustment is retired after five successful samples from one head containing the image change, when that calibration replaces the baseline and the adjustment is removed. Independent phase caps are diagnostic regression gates, not additive portions of the root-start interval.", "schemaVersion": 1, "mode": "advisory", "scope": "cloud-onboard-e2e warm-system", @@ -13,14 +13,14 @@ "minPercent": 30 }, "fullE2eColdPath": { - "rootStartToFirstTurnCompletionBudgetMs": 205000, + "rootStartToFirstTurnCompletionBudgetMs": 228000, "rootEndToFirstTurnCompletionBudgetMs": 14000, "phaseBudgetsMs": { "nemoclaw.onboard.phase.preflight": 11000, "nemoclaw.onboard.phase.gateway": 6000, "nemoclaw.onboard.phase.provider_selection": 8000, "nemoclaw.onboard.phase.inference": 6000, - "nemoclaw.onboard.phase.sandbox": 146000 + "nemoclaw.onboard.phase.sandbox": 171000 } } } diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index 0d32cce497a..e229bba1eef 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -283,7 +283,12 @@ }, { "file": "test/onboard-performance-config-schema.test.ts", - "test": "keeps configured budgets derived from the checked-in samples", + "test": "keeps baseline budgets derived from the checked-in samples", + "category": "compatibility" + }, + { + "file": "test/onboard-performance-config-schema.test.ts", + "test": "keeps interim cap adjustments tied to functional post-change evidence", "category": "compatibility" }, { diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 035cec8f89b..a8f7a146394 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1059,8 +1059,11 @@ When the root-owned managed controller attests two unchanged zero-supervisor pro The recreation uses a credential-free managed startup command, pins the registered container identity, retains the previous container for rollback, and commits only after managed gateway health and the settle check pass. The recreation preserves mounted sandbox state, but a committed swap does not retain changes stored only in the previous container's writable layer. It is idempotent. -When `recover` repairs a stopped built-in OpenClaw or Hermes gateway, it retries only when stdout is empty and stderr is exactly one `SUPERVISOR_BUSY` or `SUPERVISOR_UNAVAILABLE` line, with at most three controller attempts. -Other controller failures stop immediately. +When `recover` repairs a stopped built-in OpenClaw or Hermes gateway, it repeats the recovery action only for an exit status of `1` with blank stdout and a sole nonblank stderr line equal to `SUPERVISOR_BUSY`, with at most three controller attempts. +The same result is inconclusive during managed settle confirmation and can be probed again only within the configured settle window. +NemoClaw treats `SUPERVISOR_UNAVAILABLE` as terminal because the managed controller uses it for integrity refusals, ambiguous discovery, and process-identity changes. +It does not repeat the recovery action or treat the settle probe as inconclusive, and instead prints host-side restart and rebuild guidance. +Other controller failures also stop immediately. If the gateway is already running, the command exits zero without force-restarting it; it can still re-evaluate supported safety checks and check or recover host-side forwards. Use [`$$nemoclaw gateway restart`](#$$nemoclaw-name-gateway-restart) when you deliberately need a running gateway to reload runtime configuration or plugins. diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 1f4f2f9b803..e5c3a9de50c 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -973,15 +973,19 @@ On a direct-container deployment, first confirm that the sandbox is running: $$nemoclaw status ``` -When `recover` repairs a stopped built-in gateway, NemoClaw automatically retries only when stdout is empty and stderr is exactly one `SUPERVISOR_BUSY` or `SUPERVISOR_UNAVAILABLE` line, with at most three controller attempts. +When `recover` repairs a stopped built-in gateway, NemoClaw repeats the recovery action only for an exit status of `1` with blank stdout and a sole nonblank stderr line equal to `SUPERVISOR_BUSY`, with at most three controller attempts. +The same result is inconclusive during managed settle confirmation and can be probed again only within the configured settle window. +NemoClaw treats `SUPERVISOR_UNAVAILABLE` as terminal because it can report unreadable or untrusted supervisor state, ambiguous discovery, or a process-identity change. It does not retry other status or output combinations. `SUPERVISOR_NOT_RUNNING` is a separate result that requires two zero-supervisor scans with a stable PID 1 and does not enter that retry loop. On a supported local Docker-driver sandbox with the legacy keepalive startup, it can authorize a container-identity-pinned recreation that commits only after managed health and settle checks pass. To bypass that trusted recreation while troubleshooting, run `NEMOCLAW_DISABLE_SUPERVISOR_RELAUNCH=1 $$nemoclaw recover`; NemoClaw leaves the container unchanged and returns rebuild or re-onboard guidance. If that bounded retry is exhausted, or if `gateway restart` reports `SUPERVISOR_BUSY`, wait for the active request to finish and retry the command. -If the error mentions `SUPERVISOR_NOT_RUNNING` and trusted recreation could not proceed, `SUPERVISOR_REBUILD_REQUIRED`, `SUPERVISOR_UNAVAILABLE`, a missing `nemoclaw-gateway-control` helper, or a missing managed controller, the sandbox image may predate the current lifecycle contract. -The managed controller also refuses an ambiguous or changing process tree instead of guessing which same-UID process is the gateway. -Rebuild the image before retrying: +If the error mentions `SUPERVISOR_NOT_RUNNING` and trusted recreation could not proceed, `SUPERVISOR_REBUILD_REQUIRED`, a missing `nemoclaw-gateway-control` helper, or a missing managed controller, the sandbox image may predate the current lifecycle contract. +An exact `SUPERVISOR_UNAVAILABLE` result instead means the managed controller refused the current supervisor state rather than guessing which same-UID process is the gateway. +The current recovery action and any managed settle confirmation stop immediately. +If `recover` reports this result, follow its host-side `gateway restart` guidance. +If restart also reports `SUPERVISOR_UNAVAILABLE`, or the image is incompatible, rebuild the image: ```bash $$nemoclaw rebuild --yes diff --git a/src/lib/actions/sandbox/gateway-restart.test.ts b/src/lib/actions/sandbox/gateway-restart.test.ts index db05d0400ff..58093a4bb32 100644 --- a/src/lib/actions/sandbox/gateway-restart.test.ts +++ b/src/lib/actions/sandbox/gateway-restart.test.ts @@ -121,7 +121,7 @@ describe("restartSandboxGateway — host-mediated gateway restart", () => { } }); - it("uses the injected supervisor action for the managed settle probe", () => { + it("uses the injected supervisor action for managed settle probes", () => { const restore = silenceConsole(); const previousSettleSeconds = process.env.NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS; process.env.NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS = "0.001"; @@ -147,6 +147,7 @@ describe("restartSandboxGateway — host-mediated gateway restart", () => { expect(requestGatewaySupervisorAction.mock.calls).toEqual([ ["alpha", "restart", 210000], ["alpha", "probe"], + ["alpha", "probe"], ]); } finally { previousSettleSeconds === undefined diff --git a/src/lib/actions/sandbox/process-recovery.test.ts b/src/lib/actions/sandbox/process-recovery.test.ts index 96bba814c93..bfab0cdd343 100644 --- a/src/lib/actions/sandbox/process-recovery.test.ts +++ b/src/lib/actions/sandbox/process-recovery.test.ts @@ -199,6 +199,18 @@ describe("confirmRecoveredSandboxGatewayManaged scope", () => { }), ).toBe(false); }); + + it("keeps unavailable supervisor results terminal while lease contention stays transient", () => { + const confirm = (stderr: string) => + confirmRecoveredSandboxGatewayManaged("my-sandbox", { + getSandboxImpl: () => openClawEntry, + getSessionAgentImpl: () => null, + requestGatewaySupervisorActionImpl: () => ({ status: 1, stdout: "", stderr }), + }); + + expect(confirm("SUPERVISOR_UNAVAILABLE")).toBe(false); + expect(confirm("SUPERVISOR_BUSY")).toBeNull(); + }); }); describe("waitForRecoveredSandboxGateway settle-window confirmation (#4710)", () => { @@ -234,7 +246,7 @@ describe("waitForRecoveredSandboxGateway settle-window confirmation (#4710)", () expect(sleeps).toEqual([25]); }); - it("uses one authenticated managed probe after the settle window", () => { + it("uses authenticated managed probes inside and at the settle deadline", () => { const sleeps: number[] = []; const managedProbe = vi.fn(() => true); const ordinaryProbe = vi.fn(() => false); @@ -245,9 +257,39 @@ describe("waitForRecoveredSandboxGateway settle-window confirmation (#4710)", () sleepImpl: (seconds: number) => sleeps.push(seconds), }); expect(ok).toBe(true); - expect(managedProbe).toHaveBeenCalledOnce(); + expect(managedProbe).toHaveBeenCalledTimes(2); expect(ordinaryProbe).not.toHaveBeenCalled(); - expect(sleeps).toEqual([25]); + expect(sleeps).toEqual([22, 3]); + }); + + it("retries one transient managed result without extending the settle window", () => { + process.env.NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS = "5"; + process.env.NEMOCLAW_GATEWAY_RECOVERY_POLL_INTERVAL_SECONDS = "2"; + const sleeps: number[] = []; + const managedProbe = vi.fn(makeProbe([null, true])); + const ok = waitForRecoveredSandboxGateway("my-sandbox", { + initialManagedHealthPassed: true, + managedProbeImpl: managedProbe, + sleepImpl: (seconds: number) => sleeps.push(seconds), + }); + expect(ok).toBe(true); + expect(managedProbe).toHaveBeenCalledTimes(2); + expect(sleeps).toEqual([3, 2]); + }); + + it("keeps a recent authenticated result when only the deadline probe is transient", () => { + process.env.NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS = "5"; + process.env.NEMOCLAW_GATEWAY_RECOVERY_POLL_INTERVAL_SECONDS = "2"; + const sleeps: number[] = []; + const managedProbe = vi.fn(makeProbe([true, null])); + const ok = waitForRecoveredSandboxGateway("my-sandbox", { + initialManagedHealthPassed: true, + managedProbeImpl: managedProbe, + sleepImpl: (seconds: number) => sleeps.push(seconds), + }); + expect(ok).toBe(true); + expect(managedProbe).toHaveBeenCalledTimes(2); + expect(sleeps).toEqual([3, 2]); }); it("does not let ordinary outer-namespace health override a managed probe failure", () => { @@ -263,7 +305,7 @@ describe("waitForRecoveredSandboxGateway settle-window confirmation (#4710)", () expect(ok).toBe(false); expect(managedProbe).toHaveBeenCalledOnce(); expect(ordinaryProbe).not.toHaveBeenCalled(); - expect(sleeps).toEqual([25]); + expect(sleeps).toEqual([22]); }); it("accepts the initial managed proof without another probe when settling is disabled", () => { @@ -326,7 +368,7 @@ describe("waitForRecoveredSandboxGateway settle-window confirmation (#4710)", () sleepImpl: (seconds: number) => sleeps.push(seconds), }); expect(ok).toBe(false); - expect(sleeps).toEqual([25]); + expect(sleeps).toEqual([22]); }); it("skips the settle confirm when NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS=0", () => { diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 54174854729..9024fc530cb 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -361,6 +361,13 @@ function hasGatewayRecoveryMarker(result: SandboxCommandResult | null): boolean ); } +// Source contract: scripts/gateway-control.sh and its installed managed helper +// emit SUPERVISOR_BUSY while another request owns the controller lease or +// publication marker. SUPERVISOR_UNAVAILABLE also covers integrity refusals, +// ambiguous discovery, and process-identity changes, so it must remain +// definitive. Retry only the exact lease-contention marker within the +// existing bounded window. Removal condition: delete this classifier and its +// retry cases once the installed controller waits through contention itself. function isExactlyRetryableManagedRecoveryFailure(result: SandboxCommandResult | null): boolean { if (result === null) return false; if (result.status !== 1) return false; @@ -369,7 +376,7 @@ function isExactlyRetryableManagedRecoveryFailure(result: SandboxCommandResult | .split(/\r?\n/) .map((line) => line.trim()) .filter(Boolean); - return lines.length === 1 && ["SUPERVISOR_UNAVAILABLE", "SUPERVISOR_BUSY"].includes(lines[0]); + return lines.length === 1 && lines[0] === "SUPERVISOR_BUSY"; } function isExactlyMissingManagedSupervisor(result: SandboxCommandResult | null): boolean { @@ -406,7 +413,8 @@ export function confirmRecoveredSandboxGatewayManaged( options.requestGatewaySupervisorActionImpl ?? executeGatewaySupervisorAction; const result = requestGatewaySupervisorAction(sandboxName, "probe"); if (hasGatewayRecoveryMarker(result)) return true; - return result === null ? null : false; + if (result === null || isExactlyRetryableManagedRecoveryFailure(result)) return null; + return false; } export async function isSandboxGatewayRunningForStatus( @@ -710,6 +718,26 @@ function recoveryAgentDisplayName( return agentRuntime.getAgentDisplayName(null); } +function confirmManagedGatewayWithinSettleWindow( + sandboxName: string, + managedProbe: (sandboxName: string) => boolean | null, + sleep: (seconds: number) => void, + settleSeconds: number, + intervalSeconds: number, +): boolean { + const retryLeadSeconds = + intervalSeconds > 0 ? Math.min(intervalSeconds, settleSeconds) : settleSeconds; + const beforeDeadlineSeconds = settleSeconds - retryLeadSeconds; + if (beforeDeadlineSeconds > 0) sleep(beforeDeadlineSeconds); + + const beforeDeadlineResult = managedProbe(sandboxName); + if (beforeDeadlineResult === false) return false; + if (retryLeadSeconds > 0) sleep(retryLeadSeconds); + const atDeadlineResult = managedProbe(sandboxName); + if (atDeadlineResult !== null) return atDeadlineResult; + return beforeDeadlineResult === true; +} + export function waitForRecoveredSandboxGateway( sandboxName: string, options: { @@ -784,14 +812,24 @@ export function waitForRecoveredSandboxGateway( if (!options.quiet) { console.log(` Confirming the gateway stays responsive (~${settleSeconds}s)...`); } - sleep(settleSeconds); if (initialManagedHealthPassed) { // The managed probe is a read-only, authenticated point check in the exact - // gateway network namespace. Its typed failure is authoritative: never let - // an outer-namespace HTTP response override it or extend this settle check - // beyond the controller's single bounded probe. - return managedProbe?.(sandboxName) === true; + // gateway network namespace. Probe once inside the final poll interval and + // again at the settle deadline, so one authenticated controller race can + // clear without extending the configured settle window. A recent + // authenticated success remains authoritative when only the deadline + // attempt is transient; a definitive failure is authoritative, and an + // outer-namespace HTTP response must never override either result. + if (!managedProbe) return false; + return confirmManagedGatewayWithinSettleWindow( + sandboxName, + managedProbe, + sleep, + settleSeconds, + intervalSeconds, + ); } + sleep(settleSeconds); // A stopped HTTP probe is still only a point-in-time observation. PID 1 can // have respawned the gateway while OpenClaw is still finishing its startup // transition, so multiple stopped results may precede a healthy listener. diff --git a/src/lib/onboard/docker-gpu-patch-constants.ts b/src/lib/onboard/docker-gpu-patch-constants.ts index c01febdaee1..b51378a6529 100644 --- a/src/lib/onboard/docker-gpu-patch-constants.ts +++ b/src/lib/onboard/docker-gpu-patch-constants.ts @@ -3,3 +3,10 @@ /** Default timeout for one Docker CLI operation in the compatibility GPU patch path. */ export const DOCKER_GPU_PATCH_TIMEOUT_MS = 30_000; + +/** + * Docker may still be flushing a just-built image when the compatibility path stops the + * provisioning container. Give that state transition a longer client deadline while keeping + * every other mutation on the short default timeout. + */ +export const DOCKER_GPU_PATCH_STOP_TIMEOUT_MS = 90_000; diff --git a/src/lib/onboard/docker-gpu-patch-recreate.test.ts b/src/lib/onboard/docker-gpu-patch-recreate.test.ts index b48080bca2c..ee15aeba57d 100644 --- a/src/lib/onboard/docker-gpu-patch-recreate.test.ts +++ b/src/lib/onboard/docker-gpu-patch-recreate.test.ts @@ -43,6 +43,10 @@ describe("Docker GPU recreate orchestration", () => { expect(result.newContainerId).toBe("new-container-id"); expect(result.mode.kind).toBe("gpus"); + expect(dockerStop).toHaveBeenCalledWith( + "old-container-id", + expect.objectContaining({ timeout: 90_000 }), + ); expect(dockerRunDetached).toHaveBeenCalledWith( expect.arrayContaining([ "--name", diff --git a/src/lib/onboard/docker-gpu-patch-recreate.ts b/src/lib/onboard/docker-gpu-patch-recreate.ts index 22513f45fba..f58818ce81b 100644 --- a/src/lib/onboard/docker-gpu-patch-recreate.ts +++ b/src/lib/onboard/docker-gpu-patch-recreate.ts @@ -20,7 +20,10 @@ import { parseDockerInspectJson, sameContainerId, } from "./docker-gpu-patch-clone"; -import { DOCKER_GPU_PATCH_TIMEOUT_MS } from "./docker-gpu-patch-constants"; +import { + DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + DOCKER_GPU_PATCH_TIMEOUT_MS, +} from "./docker-gpu-patch-constants"; import { reconcileSupervisorReconnect } from "./docker-gpu-patch-finalize"; import { selectDockerGpuPatchMode } from "./docker-gpu-patch-mode"; import { restoreDockerGpuPatchBackupAfterRecreateFailure } from "./docker-gpu-patch-rollback"; @@ -249,7 +252,10 @@ export function recreateOpenShellDockerSandboxContainer( suppressOutput: true, timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, }; - const stopResult = d.dockerStop(oldContainerId, containerMutationOptions); + const stopResult = d.dockerStop(oldContainerId, { + ...containerMutationOptions, + timeout: DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + }); if (!hasZeroDockerExitStatus(stopResult)) { context.rolledBack = hasZeroDockerExitStatus( d.dockerStart(oldContainerId, containerMutationOptions), diff --git a/test/e2e/fixtures/clients/gateway.ts b/test/e2e/fixtures/clients/gateway.ts index dcb677a4f62..4bd3721e024 100644 --- a/test/e2e/fixtures/clients/gateway.ts +++ b/test/e2e/fixtures/clients/gateway.ts @@ -166,36 +166,40 @@ export class GatewayClient { // ─── Guard-chain recovery probes (#2478, #2701) ──────────────────── /** - * Resolve the running openclaw gateway PID inside the sandbox by parsing - * `ps`. Returns the lowest matching PID, or null if no gateway process is - * running. Mirrors the legacy bash `gateway_pid()` helper. - * - * Two-pass match: first prefer rows whose argv contains "gateway" alongside - * comm "openclaw"; fall back to any "openclaw" comm. The two-pass shape - * tolerates older builds that exposed gateway under a slightly different - * argv but the same comm. + * Resolve the supervisor-owned gateway PID record inside the sandbox. + * The PID is accepted only while the process exists and its `/proc` start + * identity still matches the second field recorded by the supervisor. */ async resolveGatewayPid(instance: NemoClawInstance): Promise { const script = "set -e; " + - // Primary: argv contains "gateway" and comm is "openclaw". - 'pid="$(ps -eo pid=,comm=,args= 2>/dev/null | ' + - "awk '($2 == \"openclaw\" && $0 ~ /gateway/) || $0 ~ /openclaw[ -]gateway/ { print $1 }' | " + - 'sort -n | head -n 1)"; ' + - // Fallback: any process with comm "openclaw". - 'if [ -z "$pid" ]; then ' + - 'pid="$(ps -eo pid=,comm=,args= 2>/dev/null | ' + - 'awk \'$2 == "openclaw" { print $1 }\' | sort -n | head -n 1)"; ' + - "fi; " + - 'printf "%s\\n" "$pid"'; + 'record="$(cat /tmp/nemoclaw-gateway.pid 2>/dev/null || true)"; ' + + 'set -- $record; [ "$#" -eq 2 ] || exit 0; ' + + 'pid="$1"; expected_start="$2"; ' + + 'case "$pid" in ""|*[!0-9]*) exit 0 ;; esac; ' + + 'case "$expected_start" in ""|*[!0-9]*) exit 0 ;; esac; ' + + 'kill -0 "$pid" 2>/dev/null || exit 0; ' + + 'stat="$(cat "/proc/$pid/stat" 2>/dev/null || true)"; ' + + '[ -n "$stat" ] || exit 0; rest="${stat##*) }"; ' + + '[ "$rest" != "$stat" ] || exit 0; set -- $rest; ' + + 'case "$1" in Z|X) exit 0 ;; esac; state="$1"; ' + + '[ "$#" -ge 20 ] || exit 0; actual_start="${20}"; ' + + 'printf "%s %s %s %s\\n" "$pid" "$expected_start" "$actual_start" "$state"'; const result = await this.sandbox.exec(instance.sandboxName, ["sh", "-c", script], { artifactName: `gateway-resolve-pid-${instance.sandboxName}`, env: probeEnv(), }); - const trimmed = result.stdout.trim(); - if (!/^[0-9]+$/.test(trimmed)) return null; - const pid = Number(trimmed); + const identity = result.stdout.trim().match(/^([0-9]+) ([0-9]+) ([0-9]+) ([A-Za-z])$/); + if ( + result.exitCode !== 0 || + !identity || + identity[2] !== identity[3] || + /^(?:X|Z)$/.test(identity[4]) + ) { + return null; + } + const pid = Number(identity[1]); return Number.isSafeInteger(pid) && pid > 0 ? pid : null; } @@ -285,10 +289,10 @@ export class GatewayClient { options: ExpectPidStableOptions, ): Promise { const pollIntervalSeconds = options.pollIntervalSeconds ?? 3; - if (options.durationSeconds <= 0) { + if (!Number.isFinite(options.durationSeconds) || options.durationSeconds <= 0) { throw new Error("expectPidStable: durationSeconds must be > 0"); } - if (pollIntervalSeconds <= 0) { + if (!Number.isFinite(pollIntervalSeconds) || pollIntervalSeconds <= 0) { throw new Error("expectPidStable: pollIntervalSeconds must be > 0"); } diff --git a/test/e2e/live/gpu-e2e.test.ts b/test/e2e/live/gpu-e2e.test.ts index 8c629771c1c..f6e0f548b5a 100644 --- a/test/e2e/live/gpu-e2e.test.ts +++ b/test/e2e/live/gpu-e2e.test.ts @@ -244,6 +244,8 @@ test("GPU Ollama onboard enables CUDA, auth proxy, and sandbox inference", { { model, messages: [{ role: "user", content: "Reply with exactly one word: PONG" }], + // Keep this assertion about routed inference, not the model's reasoning-token budget. + reasoning_effort: "none", max_tokens: 32, }, )}'`, diff --git a/test/e2e/live/rebuild-hermes-image-state.ts b/test/e2e/live/rebuild-hermes-image-state.ts new file mode 100644 index 00000000000..1c6f8f21957 --- /dev/null +++ b/test/e2e/live/rebuild-hermes-image-state.ts @@ -0,0 +1,46 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { extractBuiltImageRef } from "../../../src/lib/build-context"; +import { + LOCAL_SANDBOX_IMAGE_REPO, + SANDBOX_FROM_IMAGE_REPO, +} from "../../../src/lib/domain/sandbox/image-tag"; + +export interface RebuildHermesRegistryImageState { + imageTag: string; + fromDockerfile: null; +} + +export async function cleanupTrackedRebuildHermesImage( + imageTag: string | null, + remove: (imageTag: string) => Promise, +): Promise { + if (imageTag !== null) await remove(imageTag); +} + +export function requireRebuildHermesInitialImageTag(value: unknown, sandboxName: string): string { + const prefix = `${LOCAL_SANDBOX_IMAGE_REPO}:${sandboxName}-`; + const imageTag = typeof value === "string" ? value : ""; + const buildPart = imageTag.startsWith(prefix) ? imageTag.slice(prefix.length) : ""; + if (!/^\d+$/.test(buildPart)) { + throw new Error( + `initial Hermes fixture imageTag must be an owned ${prefix} tag; got ${imageTag || ""}`, + ); + } + return imageTag; +} + +export function rebuildHermesRegistryImageState( + createOutput: string, +): RebuildHermesRegistryImageState { + const imageTag = extractBuiltImageRef(createOutput); + const prefix = `${SANDBOX_FROM_IMAGE_REPO}:`; + const buildId = imageTag?.startsWith(prefix) ? imageTag.slice(prefix.length) : ""; + if (!imageTag || !/^\d+$/.test(buildId)) { + throw new Error( + `old Hermes sandbox create must report an exact ${prefix} image tag; got ${imageTag ?? ""}`, + ); + } + return { imageTag, fromDockerfile: null }; +} diff --git a/test/e2e/live/rebuild-hermes.test.ts b/test/e2e/live/rebuild-hermes.test.ts index a95f9cc1415..a46dae51434 100644 --- a/test/e2e/live/rebuild-hermes.test.ts +++ b/test/e2e/live/rebuild-hermes.test.ts @@ -23,6 +23,12 @@ import { REPO_ROOT } from "../fixtures/paths.ts"; import { listCredentialLeakPaths } from "../fixtures/phases/state-validation.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; import { buildRebuildHermesChildEnv } from "./rebuild-hermes-env.ts"; +import { + cleanupTrackedRebuildHermesImage, + type RebuildHermesRegistryImageState, + rebuildHermesRegistryImageState, + requireRebuildHermesInitialImageTag, +} from "./rebuild-hermes-image-state.ts"; import { startRebuildHermesProgress } from "./rebuild-hermes-progress.ts"; // The migrated scope is the legacy non-interactive shell regression: install.sh, @@ -231,16 +237,28 @@ async function cleanupOldHermesBaseImage( host: HostCliClient, apiKey: string | undefined, ): Promise { - const result = await host.command("docker", ["rmi", OLD_BASE_TAG], { + await removeHermesFixtureImage(host, apiKey, OLD_BASE_TAG, { artifactName: "cleanup-hermes-rebuild-resources-docker-rmi-old-base", + label: `cleanup old Hermes base image ${OLD_BASE_TAG}`, + }); +} + +async function removeHermesFixtureImage( + host: HostCliClient, + apiKey: string | undefined, + imageTag: string, + options: { artifactName: string; label: string }, +): Promise { + const result = await host.command("docker", ["image", "rm", imageTag], { + artifactName: options.artifactName, env: hermesCleanupEnv(apiKey), redactionValues: hermesCleanupRedactions(apiKey), timeoutMs: 3 * 60_000, }); assertCleanupSucceededOrAbsent( result, - /No such image|image .* not found/iu, - `cleanup old Hermes base image ${OLD_BASE_TAG}`, + /No such image|No such object|image .* not found/iu, + options.label, ); } @@ -292,7 +310,10 @@ async function waitForSandboxReady(host: HostCliClient, apiKey: string): Promise throw new Error(`sandbox ${SANDBOX_NAME} did not become Ready`); } -function seedRegistryAndSession(dashboardPort: number): SessionArtifactSummary { +function seedRegistryAndSession( + dashboardPort: number, + imageState: RebuildHermesRegistryImageState, +): SessionArtifactSummary { const registry = readJsonFileOr(REGISTRY_FILE, {}); registry.sandboxes = registry.sandboxes ?? {}; @@ -352,7 +373,7 @@ function seedRegistryAndSession(dashboardPort: number): SessionArtifactSummary { // This curated old-version fixture is still a NemoClaw-managed image. // Preserve that provenance explicitly; an absent value must remain // fail-closed because it could represent a custom `--from` image. - fromDockerfile: null, + ...imageState, messaging: { schemaVersion: 1, plan: messagingPlan }, }; expect( @@ -467,6 +488,8 @@ test(STALE_BASE_REBUILD await bestEffortPrecleanHermesResources(host, apiKey, "pre-cleanup-hermes-rebuild-resources"); + let phase1ImageTag: string | null = null; + let oldSandboxImageState: RebuildHermesRegistryImageState | null = null; cleanup.trackDisposable(`remove old Hermes base image ${OLD_BASE_TAG}`, () => cleanupOldHermesBaseImage(host, apiKey), ); @@ -485,6 +508,24 @@ test(STALE_BASE_REBUILD redactionValues: hermesCleanupRedactions(apiKey), timeoutMs: 3 * 60_000, }); + // Cleanup is LIFO: remove the sandbox before reclaiming its exact image tags, + // while the gateway/provider/forward remain available for sandbox teardown. + cleanup.trackDisposable("remove initial Hermes fixture image", () => + cleanupTrackedRebuildHermesImage(phase1ImageTag, (imageTag) => + removeHermesFixtureImage(host, apiKey, imageTag, { + artifactName: "cleanup-hermes-rebuild-resources-docker-rmi-initial-image", + label: `cleanup initial Hermes fixture image ${imageTag}`, + }), + ), + ); + cleanup.trackDisposable("remove old derived Hermes fixture image", () => + cleanupTrackedRebuildHermesImage(oldSandboxImageState?.imageTag ?? null, (imageTag) => + removeHermesFixtureImage(host, apiKey, imageTag, { + artifactName: "cleanup-hermes-rebuild-resources-docker-rmi-old-derived-image", + label: `cleanup old derived Hermes fixture image ${imageTag}`, + }), + ), + ); cleanup.trackDisposable(`delete Hermes rebuild OpenShell sandbox ${SANDBOX_NAME}`, () => sandbox.cleanupSandbox(SANDBOX_NAME, { artifactName: "cleanup-hermes-rebuild-resources-openshell-sandbox-delete", @@ -538,22 +579,19 @@ test(STALE_BASE_REBUILD phase1DashboardPort <= 65535, "initial Hermes onboard must persist the dashboard port used by authoritative rebuild", ).toBe(true); + phase1ImageTag = requireRebuildHermesInitialImageTag(registrySandbox().imageTag, SANDBOX_NAME); + await artifacts.writeJson("phase-1-owned-image.json", { imageTag: phase1ImageTag }); - const deleteCurrentSandbox = await host.command( - "openshell", - ["sandbox", "delete", SANDBOX_NAME], - { - artifactName: "phase-1-delete-current-sandbox", - env: testEnv(apiKey), - redactionValues, - timeoutMs: OPENSHELL_TIMEOUT_MS, - }, - ); - deleteCurrentSandbox.exitCode === 0 || - (await artifacts.writeText( - "phase-1-delete-current-sandbox-note.txt", - resultText(deleteCurrentSandbox), - )); + await sandbox.cleanupSandbox(SANDBOX_NAME, { + artifactName: "phase-1-delete-current-sandbox", + env: testEnv(apiKey), + redactionValues, + timeoutMs: OPENSHELL_TIMEOUT_MS, + }); + await removeHermesFixtureImage(host, apiKey, phase1ImageTag, { + artifactName: "phase-1-remove-initial-hermes-image", + label: `remove initial Hermes fixture image ${phase1ImageTag}`, + }); await host.command("openshell", ["forward", "stop", "8642"], { artifactName: "phase-1-stop-hermes-forward", env: testEnv(apiKey), @@ -664,10 +702,17 @@ test(STALE_BASE_REBUILD }, ); expectExitZero(createOldSandbox, "create old Hermes sandbox"); + oldSandboxImageState = rebuildHermesRegistryImageState(resultText(createOldSandbox)); } finally { fs.rmSync(oldDockerfileDir, { recursive: true, force: true }); } + const seededOldSandboxImageState = + oldSandboxImageState ?? fail("old Hermes sandbox create did not produce managed image state"); await waitForSandboxReady(host, apiKey); + await removeHermesFixtureImage(host, apiKey, OLD_BASE_TAG, { + artifactName: "phase-3-release-old-hermes-base-tag", + label: `release old Hermes base tag ${OLD_BASE_TAG}`, + }); progress.phase("phase 4 seed rebuild state"); const writeMarker = await host.command( @@ -717,11 +762,19 @@ test(STALE_BASE_REBUILD expectExitZero(preConfig, "read pre-rebuild Hermes config.yaml"); expect(preConfig.stdout).toContain("discord:"); - const sessionSummary = seedRegistryAndSession(phase1DashboardPort as number); + const sessionSummary = seedRegistryAndSession( + phase1DashboardPort as number, + seededOldSandboxImageState, + ); const seededRegistry = registrySandbox(); + expect( + seededRegistry.imageTag, + "curated rebuild registry must retain the exact old derived image tag for cleanup", + ).toBe(seededOldSandboxImageState.imageTag); await artifacts.writeJson("phase-4-registry-session-summary.json", { registryVersion: seededRegistry.agentVersion, dashboardPort: seededRegistry.dashboardPort, + imageTag: seededRegistry.imageTag, registryInference: { provider: seededRegistry.provider, endpointUrl: seededRegistry.endpointUrl, @@ -775,6 +828,22 @@ test(STALE_BASE_REBUILD }); expectExitZero(rebuild, "nemoclaw rebuild Hermes sandbox"); + const oldImageInspect = await host.command( + "docker", + ["image", "inspect", seededOldSandboxImageState.imageTag], + { + artifactName: "phase-6-old-derived-image-removed", + env: testEnv(apiKey), + redactionValues, + timeoutMs: OPENSHELL_TIMEOUT_MS, + }, + ); + expect( + typeof oldImageInspect.exitCode === "number" && oldImageInspect.exitCode > 0, + resultText(oldImageInspect), + ).toBe(true); + expect(resultText(oldImageInspect)).toMatch(/No such (?:image|object)(?::|\s)/iu); + progress.phase("phase 7 verification"); const restoredMarker = await host.command( "openshell", diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index ea305ac9cd7..0d5c9cb8344 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -391,6 +391,7 @@ "fast": [ "test/e2e/support/e2e-cleanup-resources.test.ts", "test/e2e/support/e2e-clients.test.ts", + "test/e2e/support/rebuild-hermes-image-state.test.ts", "test/e2e/support/rebuild-hermes-progress.test.ts" ] }, diff --git a/test/e2e/support/e2e-recovery-helpers.test.ts b/test/e2e/support/e2e-recovery-helpers.test.ts index 5c57096f305..9a6548585f9 100644 --- a/test/e2e/support/e2e-recovery-helpers.test.ts +++ b/test/e2e/support/e2e-recovery-helpers.test.ts @@ -187,14 +187,38 @@ describe("GatewayClient recovery helpers (#2701)", () => { }); describe("resolveGatewayPid", () => { - it("returns the parsed PID when the script prints a number", async () => { + it("accepts the recorded PID when its process start identity still matches", async () => { const runner = new ScriptedRunner(); - runner.queue({ stdout: "1234\n" }); + runner.queue({ stdout: "1234 987654 987654 S\n" }); const gateway = buildGateway(runner); await expect(gateway.resolveGatewayPid(fakeInstance())).resolves.toBe(1234); }); + it("returns null when the PID probe fails despite valid-looking output", async () => { + const runner = new ScriptedRunner(); + runner.queue({ exitCode: 1, stdout: "1234 987654 987654 S\n" }); + const gateway = buildGateway(runner); + + await expect(gateway.resolveGatewayPid(fakeInstance())).resolves.toBeNull(); + }); + + it("rejects a reused PID whose process start identity no longer matches", async () => { + const runner = new ScriptedRunner(); + runner.queue({ stdout: "1234 987654 123456 S\n" }); + const gateway = buildGateway(runner); + + await expect(gateway.resolveGatewayPid(fakeInstance())).resolves.toBeNull(); + }); + + it.each(["Z", "X"])("rejects a gateway process in terminal state %s", async (state) => { + const runner = new ScriptedRunner(); + runner.queue({ stdout: `1234 987654 987654 ${state}\n` }); + const gateway = buildGateway(runner); + + await expect(gateway.resolveGatewayPid(fakeInstance())).resolves.toBeNull(); + }); + it("returns null when the script prints non-numeric output", async () => { const runner = new ScriptedRunner(); runner.queue({ stdout: "" }); @@ -212,10 +236,10 @@ describe("GatewayClient recovery helpers (#2701)", () => { const runner = new ScriptedRunner(); // initial sample + 3 stable samples runner.queue( - { stdout: "100\n" }, - { stdout: "100\n" }, - { stdout: "100\n" }, - { stdout: "100\n" }, + { stdout: "100 111 111 S\n" }, + { stdout: "100 111 111 S\n" }, + { stdout: "100 111 111 S\n" }, + { stdout: "100 111 111 S\n" }, ); const gateway = buildGateway(runner); @@ -229,7 +253,7 @@ describe("GatewayClient recovery helpers (#2701)", () => { it("throws when the PID changes (crash-loop)", async () => { const runner = new ScriptedRunner(); - runner.queue({ stdout: "100\n" }, { stdout: "201\n" }); + runner.queue({ stdout: "100 111 111 S\n" }, { stdout: "201 222 222 S\n" }); const gateway = buildGateway(runner); const observation = expect( @@ -244,7 +268,7 @@ describe("GatewayClient recovery helpers (#2701)", () => { it("throws when the gateway disappears mid-window", async () => { const runner = new ScriptedRunner(); - runner.queue({ stdout: "100\n" }, { stdout: "" }); + runner.queue({ stdout: "100 111 111 S\n" }, { stdout: "" }); const gateway = buildGateway(runner); const observation = expect( @@ -270,16 +294,52 @@ describe("GatewayClient recovery helpers (#2701)", () => { ).rejects.toThrow(/no gateway process.*at start/); }); - it("rejects non-positive durations", async () => { + it.each([ + { + name: "zero duration", + durationSeconds: 0, + pollIntervalSeconds: 1, + message: /durationSeconds must be > 0/, + }, + { + name: "NaN duration", + durationSeconds: Number.NaN, + pollIntervalSeconds: 1, + message: /durationSeconds must be > 0/, + }, + { + name: "infinite duration", + durationSeconds: Number.POSITIVE_INFINITY, + pollIntervalSeconds: 1, + message: /durationSeconds must be > 0/, + }, + { + name: "zero poll interval", + durationSeconds: 1, + pollIntervalSeconds: 0, + message: /pollIntervalSeconds must be > 0/, + }, + { + name: "NaN poll interval", + durationSeconds: 1, + pollIntervalSeconds: Number.NaN, + message: /pollIntervalSeconds must be > 0/, + }, + { + name: "infinite poll interval", + durationSeconds: 1, + pollIntervalSeconds: Number.POSITIVE_INFINITY, + message: /pollIntervalSeconds must be > 0/, + }, + ])("rejects $name before any probe or timer", async (options) => { const runner = new ScriptedRunner(); const gateway = buildGateway(runner); - await expect( - gateway.expectPidStable(fakeInstance(), { - durationSeconds: 0, - pollIntervalSeconds: 1, - }), - ).rejects.toThrow(/durationSeconds must be > 0/); + await expect(gateway.expectPidStable(fakeInstance(), options)).rejects.toThrow( + options.message, + ); + expect(runner.calls).toHaveLength(0); + expect(vi.getTimerCount()).toBe(0); }); }); }); diff --git a/test/e2e/support/rebuild-hermes-image-state.test.ts b/test/e2e/support/rebuild-hermes-image-state.test.ts new file mode 100644 index 00000000000..85ff9194a8a --- /dev/null +++ b/test/e2e/support/rebuild-hermes-image-state.test.ts @@ -0,0 +1,65 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; +import { + cleanupTrackedRebuildHermesImage, + rebuildHermesRegistryImageState, + requireRebuildHermesInitialImageTag, +} from "../live/rebuild-hermes-image-state.ts"; + +describe("Hermes rebuild fixture image ownership", () => { + it("runs exact-tag cleanup only after a fixture image is tracked", async () => { + const remove = vi.fn(async (_imageTag: string) => undefined); + + await cleanupTrackedRebuildHermesImage(null, remove); + expect(remove).not.toHaveBeenCalled(); + + await cleanupTrackedRebuildHermesImage("openshell/sandbox-from:1784010200", remove); + expect(remove).toHaveBeenCalledExactlyOnceWith("openshell/sandbox-from:1784010200"); + }); + + it("accepts only the initial local image owned by the fixture sandbox", () => { + const sandboxName = "e2e-rebuild-hermes-123"; + const imageTag = `nemoclaw-sandbox-local:${sandboxName}-1784010000`; + + expect(requireRebuildHermesInitialImageTag(imageTag, sandboxName)).toBe(imageTag); + expect(() => requireRebuildHermesInitialImageTag(undefined, sandboxName)).toThrow(""); + expect(() => + requireRebuildHermesInitialImageTag( + "nemoclaw-sandbox-local:another-sandbox-1784010000", + sandboxName, + ), + ).toThrow("owned"); + expect(() => + requireRebuildHermesInitialImageTag( + `nemoclaw-sandbox-local:${sandboxName}-base-1784010000`, + sandboxName, + ), + ).toThrow("owned"); + }); + + it("retains the exact OpenShell-derived tag in managed rebuild state", () => { + expect( + rebuildHermesRegistryImageState( + [ + "Successfully tagged openshell/sandbox-from:1784010200", + " Built image openshell/sandbox-from:1784010200", + ].join("\n"), + ), + ).toEqual({ + imageTag: "openshell/sandbox-from:1784010200", + fromDockerfile: null, + }); + }); + + it("rejects missing, fabricated, or non-fixture create tags", () => { + expect(() => rebuildHermesRegistryImageState("Created sandbox fixture")).toThrow(""); + expect(() => + rebuildHermesRegistryImageState("Successfully tagged openshell/sandbox-from:latest"), + ).toThrow("exact"); + expect(() => + rebuildHermesRegistryImageState("Successfully tagged unrelated/image:1784010200"), + ).toThrow("exact"); + }); +}); diff --git a/test/onboard-performance-config-schema.test.ts b/test/onboard-performance-config-schema.test.ts index 7bef1798ab7..d3a2f57c24e 100644 --- a/test/onboard-performance-config-schema.test.ts +++ b/test/onboard-performance-config-schema.test.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { execFileSync, spawnSync } from "node:child_process"; import { readFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -57,6 +58,31 @@ interface Calibration { roundUpMs: number; }; samples: CalibrationSample[]; + validationAdjustment?: { + validatedAt: string; + imageChangeSha: string; + imageInputsVerifiedThroughSha: string; + imageInputPaths: string[]; + adjustedMetrics: string[]; + derivation: { + statistic: string; + minimumHeadroomMs: number; + relativeHeadroomPercent: number; + roundUpMs: number; + }; + retirement: { + trigger: string; + minimumSampleCount: number; + allSamplesSameHead: boolean; + imageChangeMustBeAncestor: boolean; + action: string; + }; + runs: CalibrationSample[]; + derivedCapsMs: { + rootStartToFirstTurnCompletionBudgetMs: number; + sandboxPhaseBudgetMs: number; + }; + }; derivedBudgetsMs: ColdPathBudget; } @@ -175,6 +201,105 @@ function deriveBudgets(input: Calibration): ColdPathBudget { }; } +function validationThreshold( + values: number[], + derivation: NonNullable["derivation"], +): number { + const maximum = Math.max(...values); + const headroom = Math.max( + derivation.minimumHeadroomMs, + maximum * (derivation.relativeHeadroomPercent / 100), + ); + return Math.ceil((maximum + headroom) / derivation.roundUpMs) * derivation.roundUpMs; +} + +function effectiveBudgets(input: Calibration): ColdPathBudget { + const baseline = input.derivedBudgetsMs; + const adjustment = input.validationAdjustment?.derivedCapsMs; + return { + ...baseline, + rootStartToFirstTurnCompletionBudgetMs: Math.max( + baseline.rootStartToFirstTurnCompletionBudgetMs, + adjustment?.rootStartToFirstTurnCompletionBudgetMs ?? + baseline.rootStartToFirstTurnCompletionBudgetMs, + ), + phaseBudgetsMs: { + ...baseline.phaseBudgetsMs, + "nemoclaw.onboard.phase.sandbox": Math.max( + baseline.phaseBudgetsMs["nemoclaw.onboard.phase.sandbox"], + adjustment?.sandboxPhaseBudgetMs ?? + baseline.phaseBudgetsMs["nemoclaw.onboard.phase.sandbox"], + ), + }, + }; +} + +function gitIsAncestor(ancestor: string, descendant: string): boolean { + const result = spawnSync("git", ["merge-base", "--is-ancestor", ancestor, descendant], { + cwd: REPO_ROOT, + encoding: "utf8", + }); + switch (result.status) { + case 0: + return true; + case 1: + return false; + default: + throw new Error( + `git merge-base could not verify calibration ancestry; ensure the checkout has full history (status ${String(result.status)}): ${result.error?.message ?? result.stderr.trim()}`, + ); + } +} + +function gitRevision(revision: string): string { + return execFileSync("git", ["rev-parse", "--verify", revision], { + cwd: REPO_ROOT, + encoding: "utf8", + }).trim(); +} + +function changedImageInputs( + fromSha: string, + throughSha: string, + imageInputPaths: string[], +): string[] { + const output = execFileSync( + "git", + ["diff", "--name-only", fromSha, throughSha, "--", ...imageInputPaths], + { cwd: REPO_ROOT, encoding: "utf8" }, + ).trim(); + return output === "" ? [] : output.split(/\r?\n/u); +} + +function validationProvenanceViolations( + validation: NonNullable, +) { + const runHeadsWithChangedImageInputs = validation.runs + .map((run) => ({ + headSha: run.headSha, + changedPaths: changedImageInputs( + validation.imageChangeSha, + run.headSha, + validation.imageInputPaths, + ), + })) + .filter((run) => run.changedPaths.length > 0); + return { + nonDescendantRunHeads: validation.runs + .map((run) => run.headSha) + .filter((headSha) => !gitIsAncestor(validation.imageChangeSha, headSha)), + runHeadsBeyondVerifiedInputs: validation.runs + .map((run) => run.headSha) + .filter((headSha) => !gitIsAncestor(headSha, validation.imageInputsVerifiedThroughSha)), + runHeadsWithChangedImageInputs, + changedImageInputsThroughBoundary: changedImageInputs( + validation.imageChangeSha, + validation.imageInputsVerifiedThroughSha, + validation.imageInputPaths, + ), + }; +} + describe("full-E2E cold-path calibration", () => { // source-shape-contract: compatibility -- Exact-head provenance is durable evidence for the hosted-run budget calibration it("records five independent successful samples for current main", () => { @@ -213,9 +338,103 @@ describe("full-E2E cold-path calibration", () => { }); // source-shape-contract: compatibility -- Recomputed thresholds keep enforced budgets tied to the reviewed calibration evidence - it("keeps configured budgets derived from the checked-in samples", () => { + it("keeps baseline budgets derived from the checked-in samples", () => { const derived = deriveBudgets(calibration); expect(calibration.derivedBudgetsMs).toEqual(derived); - expect(checkedInConfig.fullE2eColdPath).toEqual(derived); + }); + + // source-shape-contract: compatibility -- Post-image-growth validation may adjust only observed stale cold-path caps without pretending to replace the five-run calibration + it("keeps interim cap adjustments tied to functional post-change evidence", () => { + const validation = calibration.validationAdjustment!; + expect(validation.validatedAt).toMatch(/^\d{4}-\d{2}-\d{2}$/u); + expect(validation.imageChangeSha).toMatch(/^[0-9a-f]{40}$/u); + expect(validation.imageInputsVerifiedThroughSha).toMatch(/^[0-9a-f]{40}$/u); + expect(validation.imageInputPaths.length).toBeGreaterThan(0); + expect(validationProvenanceViolations(validation)).toEqual({ + nonDescendantRunHeads: [], + runHeadsBeyondVerifiedInputs: [], + runHeadsWithChangedImageInputs: [], + changedImageInputsThroughBoundary: [], + }); + expect( + validationProvenanceViolations({ + ...validation, + runs: [{ ...validation.runs[0], headSha: calibration.baselineMainSha }], + }).nonDescendantRunHeads, + ).toEqual([calibration.baselineMainSha]); + const currentHeadSha = gitRevision("HEAD"); + expect( + validationProvenanceViolations({ + ...validation, + runs: [{ ...validation.runs[0], headSha: currentHeadSha }], + }).runHeadsBeyondVerifiedInputs, + ).toEqual([currentHeadSha]); + const staleImageReference = validationProvenanceViolations({ + ...validation, + imageChangeSha: calibration.baselineMainSha, + }); + expect(staleImageReference.runHeadsWithChangedImageInputs.map((run) => run.headSha)).toEqual( + validation.runs.map((run) => run.headSha), + ); + expect( + staleImageReference.runHeadsWithChangedImageInputs.flatMap((run) => run.changedPaths), + ).toContain("agents/openclaw/wechat-runtime/package.json"); + expect(staleImageReference.changedImageInputsThroughBoundary).toContain( + "agents/openclaw/wechat-runtime/package.json", + ); + expect(validation.adjustedMetrics).toEqual([ + "rootStartToFirstTurnCompletion", + "nemoclaw.onboard.phase.sandbox", + ]); + expect(validation.derivation.statistic).toBe("maximum"); + expect(validation.retirement).toEqual({ + trigger: "successful-exact-head-calibration", + minimumSampleCount: 5, + allSamplesSameHead: true, + imageChangeMustBeAncestor: true, + action: "replace-baseline-and-remove-adjustment", + }); + expect(validation.runs).toHaveLength(4); + expect(new Set(validation.runs.map((run) => run.runId)).size).toBe(4); + expect(validation.runs.map((run) => run.conclusion).sort()).toEqual([ + "failure", + "failure", + "success", + "success", + ]); + expect(validation.runs.map((run) => run.performancePassed).sort()).toEqual([ + false, + false, + true, + true, + ]); + + for (const run of validation.runs) { + expect(run.runUrl).toBe(`https://github.com/NVIDIA/NemoClaw/actions/runs/${run.runId}`); + expect(run.headSha).toMatch(/^[0-9a-f]{40}$/u); + expect(run).toMatchObject({ + installExitCode: 0, + firstTurnExitCode: 0, + usedBuildKitPrebuild: true, + buildKitFallback: false, + }); + expect(run.maxSilenceSecs).toBeLessThanOrEqual(60); + expect(run.responseChars).toBeGreaterThan(0); + } + + expect(validation.derivedCapsMs).toEqual({ + rootStartToFirstTurnCompletionBudgetMs: validationThreshold( + validation.runs.map((run) => run.measurementsMs.rootStartToFirstTurnCompletion), + validation.derivation, + ), + sandboxPhaseBudgetMs: validationThreshold( + validation.runs.map((run) => run.measurementsMs.phases["nemoclaw.onboard.phase.sandbox"]), + validation.derivation, + ), + }); + expect(checkedInConfig.fullE2eColdPath).toEqual(effectiveBudgets(calibration)); + expect(effectiveBudgets({ ...calibration, validationAdjustment: undefined })).toEqual( + calibration.derivedBudgetsMs, + ); }); }); diff --git a/test/process-recovery-managed-controller.test.ts b/test/process-recovery-managed-controller.test.ts index 1c53b604b46..a5edc9dbe4d 100644 --- a/test/process-recovery-managed-controller.test.ts +++ b/test/process-recovery-managed-controller.test.ts @@ -77,13 +77,40 @@ describe("managed gateway recovery controller", () => { recoverResults: [successfulControl], managedProbeResult: successfulProbe, expectedResult: recoveredGateway, - expectedActions: ["recover", "probe"], + expectedActions: ["recover", "probe", "probe"], + settleSeconds: "1", + }, + { + label: "transient post-settle controller contention", + recoverResults: [successfulControl], + managedProbeResults: [{ status: 1, stdout: "", stderr: "SUPERVISOR_BUSY" }, successfulProbe], + expectedResult: recoveredGateway, + expectedActions: ["recover", "probe", "probe"], + settleSeconds: "1", + }, + { + label: "persistent post-settle controller contention", + recoverResults: [successfulControl], + managedProbeResults: [{ status: 1, stdout: "", stderr: "SUPERVISOR_BUSY" }], + expectedResult: unrecoveredGateway, + expectedActions: ["recover", "probe", "probe"], settleSeconds: "1", }, { - label: "two transient controller races followed by authenticated recovery", + label: "post-settle controller contention followed by terminal failure", + recoverResults: [successfulControl], + managedProbeResults: [ + { status: 1, stdout: "", stderr: "SUPERVISOR_BUSY" }, + { status: 1, stdout: "", stderr: "GATEWAY_HEALTH_TIMEOUT" }, + ], + expectedResult: unrecoveredGateway, + expectedActions: ["recover", "probe", "probe"], + settleSeconds: "1", + }, + { + label: "two transient controller contentions followed by authenticated recovery", recoverResults: [ - { status: 1, stdout: "", stderr: "SUPERVISOR_UNAVAILABLE" }, + { status: 1, stdout: "", stderr: "SUPERVISOR_BUSY" }, { status: 1, stdout: "", stderr: "SUPERVISOR_BUSY" }, successfulControl, ], @@ -92,10 +119,10 @@ describe("managed gateway recovery controller", () => { settleSeconds: "0", }, { - label: "persistent exact unavailable controller result", + label: "exact unavailable controller result", recoverResults: [{ status: 1, stdout: "", stderr: "SUPERVISOR_UNAVAILABLE" }], expectedResult: unrecoveredGateway, - expectedActions: ["recover", "recover", "recover"], + expectedActions: ["recover"], settleSeconds: "0", }, { @@ -114,22 +141,22 @@ describe("managed gateway recovery controller", () => { settleSeconds: "1", }, { - label: "non-exact unavailable marker", - recoverResults: [{ status: 1, stdout: "", stderr: "prefix SUPERVISOR_UNAVAILABLE suffix" }], + label: "non-exact busy marker", + recoverResults: [{ status: 1, stdout: "", stderr: "prefix SUPERVISOR_BUSY suffix" }], expectedResult: unrecoveredGateway, expectedActions: ["recover"], settleSeconds: "0", }, { - label: "unavailable marker with another error line", - recoverResults: [{ status: 1, stdout: "", stderr: "SUPERVISOR_UNAVAILABLE\nGATEWAY_FAILED" }], + label: "busy marker with another error line", + recoverResults: [{ status: 1, stdout: "", stderr: "SUPERVISOR_BUSY\nGATEWAY_FAILED" }], expectedResult: unrecoveredGateway, expectedActions: ["recover"], settleSeconds: "0", }, { - label: "unavailable marker with a nonstandard status", - recoverResults: [{ status: 2, stdout: "", stderr: "SUPERVISOR_UNAVAILABLE" }], + label: "busy marker with a nonstandard status", + recoverResults: [{ status: 2, stdout: "", stderr: "SUPERVISOR_BUSY" }], expectedResult: unrecoveredGateway, expectedActions: ["recover"], settleSeconds: "0", @@ -167,6 +194,7 @@ describe("managed gateway recovery controller", () => { expectedResult, expectedActions, managedProbeResult, + managedProbeResults, settleSeconds, }) => { const openshellRuntime = requireSource("../src/lib/adapters/openshell/runtime.js"); @@ -179,13 +207,16 @@ beta 127.0.0.1 18789 12345 running`; const previousPollInterval = process.env.NEMOCLAW_GATEWAY_RECOVERY_POLL_INTERVAL_SECONDS; const previousSettleSeconds = process.env.NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS; let recoveryActionCalls = 0; + let managedProbeCalls = 0; const requestGatewaySupervisorAction = vi.fn( (_sandboxName: string, action: "restart" | "recover" | "probe") => { const isProbe = action === "probe"; + const probeResults = managedProbeResults ?? [managedProbeResult ?? successfulProbe]; const result = isProbe - ? (managedProbeResult ?? successfulProbe) + ? probeResults[Math.min(managedProbeCalls, probeResults.length - 1)] : recoverResults[Math.min(recoveryActionCalls, recoverResults.length - 1)]; recoveryActionCalls += Number(!isProbe); + managedProbeCalls += Number(isProbe); return result; }, ); diff --git a/test/process-recovery-supervisor-relaunch.test.ts b/test/process-recovery-supervisor-relaunch.test.ts index 945cfbdc629..ef262ce6aef 100644 --- a/test/process-recovery-supervisor-relaunch.test.ts +++ b/test/process-recovery-supervisor-relaunch.test.ts @@ -55,7 +55,7 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { }); expect(result).toMatchObject({ checked: true, wasRunning: false, recovered: false }); - expect(requestGatewaySupervisorAction).toHaveBeenCalledTimes(3); + expect(requestGatewaySupervisorAction).toHaveBeenCalledOnce(); expect(relaunchManagedSupervisorSessionImpl).not.toHaveBeenCalled(); }); diff --git a/test/process-recovery.test.ts b/test/process-recovery.test.ts index 64c4b09506e..0e4b2df6a83 100644 --- a/test/process-recovery.test.ts +++ b/test/process-recovery.test.ts @@ -523,8 +523,6 @@ hermes-box 127.0.0.1 18789 12345 running`; stderr, })); - // Preserve managed recovery retries without sleeping between mocked supervisor attempts. - vi.stubEnv("NEMOCLAW_GATEWAY_RECOVERY_POLL_INTERVAL_SECONDS", "0"); vi.spyOn(childProcess, "spawnSync").mockImplementation( (_command: unknown, rawArgs: unknown) => { const shellCommand = getSandboxExecShellCommand(rawArgs); @@ -569,8 +567,7 @@ hermes-box 127.0.0.1 18789 12345 running`; recovered: false, forwardRecovered: false, }); - const transientFailure = stdout === "" && stderr === "SUPERVISOR_UNAVAILABLE"; - expect(requestGatewaySupervisorAction).toHaveBeenCalledTimes(transientFailure ? 3 : 1); + expect(requestGatewaySupervisorAction).toHaveBeenCalledOnce(); expect(requestGatewaySupervisorAction).toHaveBeenCalledWith("hermes-box", "recover"); });