diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 7468c15e23..3e9c34d8a1 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1241,7 +1241,8 @@ Run `$$nemoclaw shields down` before a Hermes config or inference change; -The command can fail at these layers: unsupported agent, privileged control unavailable, supervisor not running, secret-boundary refusal, unsafe config path, config hash mismatch when a strict hash is available, launch failure, health timeout, or forward recovery failure. +The command can fail at these layers: unsupported agent, privileged control unavailable, supervisor not running, secret-boundary refusal, unsafe config path, config hash mismatch when a strict hash is available, MCP reconciliation refusal, relaunch quarantined, launch failure, health timeout, or forward recovery failure. +`relaunch quarantined` means the in-sandbox supervisor stopped attempting relaunch after a startup refusal or repeated gateway exits, so restart and recovery report the supported repair, `$$nemoclaw rebuild --yes`, instead of a retry. An older direct-container image without the matching supervisor or managed controller helper reports `privileged control unavailable` and requires `$$nemoclaw rebuild --yes`. Ordinary OpenShell exec and manual in-sandbox relaunch are not fallback paths. Terminal agents do not have a gateway runtime and fail as unsupported. diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 6cad47f36f..3243c021d7 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -2713,6 +2713,21 @@ The Hermes entrypoint supervisor remains responsible for the gateway, dashboard, In the OpenShell-managed topology, that nonroot supervisor repairs failed auxiliaries continuously, recovers a gateway after four consecutive failed listener or HTTP health checks, and quarantines relaunch after five exits within 60 seconds until the sandbox is recreated. The host only repairs the host-side OpenShell forwards after the supervised processes pass health checks. +### Restart or recovery reports `relaunch quarantined` + +In the OpenShell-managed topology the strict root-owned hash is not a trust anchor for mutable config, so a direct edit of `/sandbox/.hermes/config.yaml` or `/sandbox/.hermes/.env` is not refused by the host controller. +The in-sandbox supervisor still refuses to start a gateway on configuration it cannot match to the persisted managed state, and it stops attempting relaunch once that refusal or repeated gateway exits exhaust its crash budget. +`$$nemoclaw gateway restart`, `$$nemoclaw recover`, and `$$nemoclaw connect` then report the `relaunch quarantined` failure layer. + +The refusal is deterministic, so retrying any of those commands cannot clear it. +Restore the registered configuration and refresh its integrity metadata in one transaction: + +```bash +nemohermes rebuild --yes +``` + +After the rebuild, make the intended change through a supported command such as `$$nemoclaw config set` or `$$nemoclaw inference set`, which update the configuration and its hashes together. + ### Port 8642 in a browser shows a blank page or `Cannot GET /` `nemohermes onboard` forwards port `8642`, but Hermes serves an OpenAI-compatible API at that port, not a chat dashboard. diff --git a/src/lib/actions/sandbox/connect-boundary-refusal.ts b/src/lib/actions/sandbox/connect-boundary-refusal.ts index dc9e3a62f9..6beb5a4868 100644 --- a/src/lib/actions/sandbox/connect-boundary-refusal.ts +++ b/src/lib/actions/sandbox/connect-boundary-refusal.ts @@ -1,6 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { + type GatewayRestartFailureLayer, + gatewayIntegrityRepairLines, + isGatewayIntegrityRepairLayer, +} from "./gateway-restart"; import type { SecretBoundaryRefusalReason } from "./hermes-secret-boundary-recovery"; import { hermesMcpReconciliationRemediationLines, @@ -9,6 +14,25 @@ import { type ConnectBoundaryContext = "Probe" | "Connect"; +/** + * A managed recovery that failed on a deterministic integrity refusal cannot be + * retried: every relaunch re-reads the same drifted protected configuration. + * The probe path recovers quietly, so without this the operator only sees the + * generic "check the gateway log" and never learns the supported repair (#7801). + * Returns false when the layer is a retryable failure, leaving the caller's + * existing wedge diagnostics in charge. + */ +export function printGatewayIntegrityRepairGuidance( + sandboxName: string, + layer: GatewayRestartFailureLayer | null | undefined, +): boolean { + if (!isGatewayIntegrityRepairLayer(layer)) return false; + for (const line of gatewayIntegrityRepairLines(sandboxName, layer)) { + console.error(` ${line}`); + } + return true; +} + export function exitOnSecretBoundaryRefusal( sandboxName: string, agentName: string, diff --git a/src/lib/actions/sandbox/connect-flow.test.ts b/src/lib/actions/sandbox/connect-flow.test.ts index 630fa51530..afee834c18 100644 --- a/src/lib/actions/sandbox/connect-flow.test.ts +++ b/src/lib/actions/sandbox/connect-flow.test.ts @@ -460,7 +460,10 @@ describe("connectSandbox flow", () => { await expect(harness.connectSandbox("alpha", { probeOnly: true })).resolves.toBeUndefined(); - expect(harness.checkAndRecoverSpy).toHaveBeenCalledWith("alpha", { quiet: true }); + expect(harness.checkAndRecoverSpy).toHaveBeenCalledWith( + "alpha", + expect.objectContaining({ quiet: true }), + ); expect(harness.runAutoPairSpy).toHaveBeenCalledWith("alpha", expect.any(Object)); expect(harness.spawnSyncSpy).not.toHaveBeenCalledWith( "openshell", @@ -489,6 +492,30 @@ describe("connectSandbox flow", () => { ); expect(exitSpy).toHaveBeenCalledWith(1); }); + it("probe-only mode reports the supported repair when relaunch is quarantined (#7801)", async () => { + const harness = createConnectHarness({ + processCheck: { checked: true, wasRunning: false, recovered: false }, + }); + // Managed recovery runs quiet on this path, so the classified layer only + // reaches the operator through the callback the probe passes in. + harness.checkAndRecoverSpy.mockImplementation((_sandboxName: unknown, options: unknown) => { + ( + options as { onRecoveryFailureLayer?: (layer: string) => void } | undefined + )?.onRecoveryFailureLayer?.("relaunch quarantined"); + return { checked: true, wasRunning: false, recovered: false }; + }); + + await expect(harness.connectSandbox("alpha", { probeOnly: true })).rejects.toThrow( + "process.exit(1)", + ); + + const errorOutput = harness.errorSpy.mock.calls.map((call) => String(call[0] ?? "")).join("\n"); + expect(errorOutput).toContain("quarantined gateway relaunch"); + expect(errorOutput).toContain("nemoclaw alpha rebuild --yes"); + expect(errorOutput).not.toContain("Check /tmp/gateway.log inside the sandbox for details."); + expect(exitSpy).toHaveBeenCalledWith(1); + }); + it("probe-only mode exits when primary dashboard/API forward recovery fails", async () => { const harness = createConnectHarness({ processCheck: { diff --git a/src/lib/actions/sandbox/connect.ts b/src/lib/actions/sandbox/connect.ts index 6b8c97c336..e40a5e83f9 100644 --- a/src/lib/actions/sandbox/connect.ts +++ b/src/lib/actions/sandbox/connect.ts @@ -62,6 +62,7 @@ import { import { exitOnMcpReconciliationRefusal, exitOnSecretBoundaryRefusal, + printGatewayIntegrityRepairGuidance, } from "./connect-boundary-refusal"; import { prepareHermesLightTerminalSkin } from "./connect-hermes-light-skin"; import { @@ -82,6 +83,7 @@ import { printGatewayWedgeDiagnostics } from "./gateway-wedge-diagnostics"; import { checkAndRecoverSandboxProcesses, executeSandboxExecCommand, + type GatewayRestartFailureLayer, resolveSandboxDashboardPort, } from "./process-recovery"; import { runTerminalAgentConnectProbe } from "./terminal-connect-probe"; @@ -245,7 +247,16 @@ async function runSandboxConnectProbe(sandboxName: string): Promise { return; } - const processCheck = checkAndRecoverSandboxProcesses(sandboxName, { quiet: true }); + // Managed recovery runs quiet here, so its classified failure layer is the + // only way this path can tell a retryable wedge apart from a deterministic + // integrity refusal that no restart, recover, or connect can clear (#7801). + let recoveryFailureLayer: GatewayRestartFailureLayer | null = null; + const processCheck = checkAndRecoverSandboxProcesses(sandboxName, { + quiet: true, + onRecoveryFailureLayer: (layer) => { + recoveryFailureLayer = layer; + }, + }); if (!processCheck.checked) { console.error( ` Probe failed: could not inspect the ${agentName} gateway inside sandbox '${sandboxName}'.`, @@ -296,6 +307,9 @@ async function runSandboxConnectProbe(sandboxName: string): Promise { console.error( ` Probe failed: ${agentName} gateway is not running in '${sandboxName}' and automatic recovery failed.`, ); + if (printGatewayIntegrityRepairGuidance(sandboxName, recoveryFailureLayer)) { + process.exit(1); + } // Surface the #4710 wedge signature: recovery ran with quiet=true, so this // is the operator's only window into a gateway that served briefly and // then dropped its listener. diff --git a/src/lib/actions/sandbox/gateway-restart-quarantine-repair.test.ts b/src/lib/actions/sandbox/gateway-restart-quarantine-repair.test.ts new file mode 100644 index 0000000000..a6600f053b --- /dev/null +++ b/src/lib/actions/sandbox/gateway-restart-quarantine-repair.test.ts @@ -0,0 +1,149 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + classifyGatewayRestartFailure, + gatewayIntegrityRepairLines, + isGatewayIntegrityRepairLayer, + printGatewayRestartFailure, +} from "./gateway-restart"; + +// The exact lines the in-sandbox Hermes supervisor emits when it stops +// attempting relaunch. `scripts/managed-gateway-control.py` allowlists these +// before forwarding them to the host as `NEMOCLAW_START_LOG=` lines. +const QUARANTINE_LINES = [ + "[gateway] CRITICAL: 5 exits in 60s window — Hermes relaunch is quarantined until sandbox recreation; check /tmp/gateway.log", + "[SECURITY] Hermes automatic respawn is quarantined until MCP integrity is restored by rebuilding the sandbox", + "[gateway] CRITICAL: exact Hermes replacement could not be stopped; managed supervisor is quarantined without another launch", + "[CRITICAL] Unproven Hermes gateway child exited; managed supervisor remains quarantined until sandbox recreation", + "[CRITICAL] Newly launched Hermes gateway pid 4242 failed exact role identity capture; quarantining the managed startup supervisor without signaling the unproven child", +] as const; + +// Verbatim controller output captured on a Hermes sandbox whose protected +// `config.yaml` was edited outside a supported command, then restarted. +const REPORTED_RESTART_OUTPUT = [ + "GATEWAY_HEALTH_TIMEOUT", + "NEMOCLAW_CONTROL_STAGE=await-replacement", + "NEMOCLAW_SUPERVISOR_PID=42", + "NEMOCLAW_GATEWAY_PID=0", + "NEMOCLAW_START_LOG=[gateway] Hermes gateway respawned (pid 18424)", + "NEMOCLAW_START_LOG=[SECURITY] Hermes automatic respawn is quarantined until MCP integrity is restored by rebuilding the sandbox", +].join("\n"); + +function classify(stdout: string) { + return classifyGatewayRestartFailure({ status: 1, stdout, stderr: "" }); +} + +function captureStderr(run: () => void): string[] { + const lines: string[] = []; + const spy = vi.spyOn(console, "error").mockImplementation((value?: unknown) => { + lines.push(String(value)); + }); + run(); + spy.mockRestore(); + return lines; +} + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("supervisor relaunch quarantine classification (#7801)", () => { + it.each(QUARANTINE_LINES)("classifies %s as a relaunch quarantine", (line) => { + expect(classify(line)).toMatchObject({ layer: "relaunch quarantined" }); + }); + + it("classifies the reported restart output as a quarantine, not a health timeout", () => { + expect(classify(REPORTED_RESTART_OUTPUT)).toMatchObject({ layer: "relaunch quarantined" }); + }); + + it("prefers the quarantine over the MCP drift it is reported through", () => { + const output = [ + "HERMES_MCP_CONFIG_DRIFT", + "[SECURITY] Hermes automatic respawn is quarantined until MCP integrity is restored by rebuilding the sandbox", + ].join("\n"); + expect(classify(output)).toMatchObject({ layer: "relaunch quarantined" }); + }); + + it("keeps the pre-existing layers for output without a quarantine line", () => { + expect(classify("GATEWAY_HEALTH_TIMEOUT")).toMatchObject({ layer: "health timeout" }); + expect(classify("HERMES_MCP_CONFIG_DRIFT")).toMatchObject({ + layer: "MCP reconciliation refusal", + }); + expect(classify("GATEWAY_CONFIG_HASH_MISMATCH")).toMatchObject({ + layer: "config hash mismatch", + }); + expect(classify("SUPERVISOR_NOT_RUNNING")).toMatchObject({ layer: "supervisor not running" }); + }); + + it("ignores an unrelated line that merely mentions the supervisor", () => { + expect(classify("[gateway] Hermes gateway respawned (pid 18424)")).toMatchObject({ + layer: "launch failure", + }); + }); +}); + +describe("integrity repair guidance (#7801)", () => { + it("treats both deterministic integrity refusals as repairable layers", () => { + expect(isGatewayIntegrityRepairLayer("relaunch quarantined")).toBe(true); + expect(isGatewayIntegrityRepairLayer("config hash mismatch")).toBe(true); + expect(isGatewayIntegrityRepairLayer("health timeout")).toBe(false); + expect(isGatewayIntegrityRepairLayer("launch failure")).toBe(false); + expect(isGatewayIntegrityRepairLayer(null)).toBe(false); + expect(isGatewayIntegrityRepairLayer(undefined)).toBe(false); + }); + + it.each([ + "relaunch quarantined", + "config hash mismatch", + ] as const)("names the supported repair command for %s", (layer) => { + const lines = gatewayIntegrityRepairLines("repro-7801", layer).join("\n"); + expect(lines).toContain("nemoclaw repro-7801 rebuild --yes"); + expect(lines).toContain("Retrying the restart cannot clear it."); + expect(lines).toContain("nemoclaw repro-7801 config set"); + }); + + it("describes the two refusals differently", () => { + const quarantined = gatewayIntegrityRepairLines("alpha", "relaunch quarantined")[0]; + const drifted = gatewayIntegrityRepairLines("alpha", "config hash mismatch")[0]; + expect(quarantined).not.toEqual(drifted); + expect(drifted).toContain("integrity hash"); + expect(quarantined).toContain("quarantined"); + }); +}); + +describe("printGatewayRestartFailure repair guidance (#7801)", () => { + it("appends the repair to a quarantined restart failure", () => { + const lines = captureStderr(() => + printGatewayRestartFailure("repro-7801", "relaunch quarantined", REPORTED_RESTART_OUTPUT), + ).join("\n"); + expect(lines).toContain("Failure layer: relaunch quarantined"); + expect(lines).toContain("nemoclaw repro-7801 rebuild --yes"); + }); + + it("still prints the repair when the controller returned no detail", () => { + const lines = captureStderr(() => + printGatewayRestartFailure("repro-7801", "config hash mismatch", ""), + ).join("\n"); + expect(lines).toContain("nemoclaw repro-7801 rebuild --yes"); + }); + + it("leaves retryable failure layers without a rebuild instruction", () => { + const timeout = captureStderr(() => + printGatewayRestartFailure("repro-7801", "health timeout", "GATEWAY_HEALTH_TIMEOUT"), + ).join("\n"); + const launch = captureStderr(() => + printGatewayRestartFailure("repro-7801", "launch failure", "GATEWAY_FAILED"), + ).join("\n"); + expect(timeout).not.toContain("rebuild --yes"); + expect(launch).not.toContain("rebuild --yes"); + }); + + it("keeps the MCP reconciliation remediation it already emitted", () => { + const lines = captureStderr(() => + printGatewayRestartFailure("repro-7801", "MCP reconciliation refusal", "mcp-integrity"), + ).join("\n"); + expect(lines).toContain("nemoclaw repro-7801 mcp restart"); + }); +}); diff --git a/src/lib/actions/sandbox/gateway-restart.ts b/src/lib/actions/sandbox/gateway-restart.ts index 728334f626..f2026ce053 100644 --- a/src/lib/actions/sandbox/gateway-restart.ts +++ b/src/lib/actions/sandbox/gateway-restart.ts @@ -24,6 +24,7 @@ export type GatewayRestartFailureLayer = | "unsafe config path" | "config hash mismatch" | "MCP reconciliation refusal" + | "relaunch quarantined" | "launch failure" | "health timeout" | "forward recovery failure"; @@ -57,6 +58,18 @@ type SandboxExec = ( const GATEWAY_RESTART_SUPPORTED_AGENTS = ["openclaw", "hermes"] as const; +// Substrings of the in-sandbox supervisor's quarantine lines. The supervisor +// only forwards allowlisted lines to the host, so matching them is what tells +// the host that no further relaunch will be attempted until the sandbox is +// rebuilt. Keep in sync with the quarantine messages in agents/hermes/start.sh +// and their allowlist in scripts/managed-gateway-control.py. +const GATEWAY_RELAUNCH_QUARANTINE_MARKERS = [ + "quarantined until sandbox recreation", + "quarantined until MCP integrity is restored", + "quarantined without another launch", + "quarantining the managed startup supervisor", +] as const; + export type GatewayRestartDeps = { getSessionAgent: typeof agentRuntime.getSessionAgent; getSandbox: SandboxAgentLookup; @@ -175,6 +188,18 @@ export function classifyGatewayRestartFailure(result: GatewayRestartCommandResul ) { return { layer: "unsafe config path", detail: detail || "unsafe config path" }; } + // A quarantined supervisor is the strictly more specific and terminal fact: + // it stops attempting relaunch entirely, so the controller then reports the + // generic health timeout it would report for any unresponsive gateway, and a + // config refusal that tripped the crash budget is reported as MCP drift by the + // non-root startup guard. Classify the quarantine ahead of both so the host + // names the state that actually blocks recovery instead of its side effect. + if (GATEWAY_RELAUNCH_QUARANTINE_MARKERS.some((marker) => output.includes(marker))) { + return { + layer: "relaunch quarantined", + detail: detail || "the in-sandbox supervisor quarantined gateway relaunch", + }; + } if ( output.includes("mcp-integrity") || output.includes("mcp-reconcile-required") || @@ -201,26 +226,63 @@ export function classifyGatewayRestartFailure(result: GatewayRestartCommandResul return { layer: "launch failure", detail: detail || `restart exited ${result.status}` }; } +export function isGatewayIntegrityRepairLayer( + layer: GatewayRestartFailureLayer | null | undefined, +): layer is "config hash mismatch" | "relaunch quarantined" { + return layer === "config hash mismatch" || layer === "relaunch quarantined"; +} + +/** + * The supported repair for a sandbox whose protected configuration drifted away + * from its recorded integrity metadata. Both layers are deterministic refusals: + * every relaunch re-reads the same drifted file, so retrying a restart or a + * recover only burns the supervisor's crash budget. `rebuild` is the documented + * command that restores the registered configuration, refreshes the integrity + * hashes, and brings the gateway back in one transaction (#7801). + */ +export function gatewayIntegrityRepairLines( + sandboxName: string, + layer: "config hash mismatch" | "relaunch quarantined", +): readonly string[] { + const cause = + layer === "config hash mismatch" + ? "A protected configuration file no longer matches its recorded integrity hash." + : "The in-sandbox supervisor quarantined gateway relaunch after a startup refusal."; + return [ + `${cause} Retrying the restart cannot clear it.`, + `Restore the registered configuration and refresh its integrity metadata with \`nemoclaw ${sandboxName} rebuild --yes\`.`, + `Then make intended changes through supported commands such as \`nemoclaw ${sandboxName} config set\` or \`nemoclaw inference set --sandbox ${sandboxName}\`, which update the configuration and its hashes together.`, + ]; +} + export function printGatewayRestartFailure( sandboxName: string, layer: GatewayRestartFailureLayer, detail: string, ): void { console.error(` Failure layer: ${layer} - gateway restart failed for '${sandboxName}'.`); - if (!detail.trim()) return; - const lines = detail - .split(/\r?\n/) - .map((line) => sanitizeGatewayRestartFailureLine(line.trim())) - .filter(Boolean) - .slice(-12); - for (const line of lines) { - console.error(` ${line}`); + if (detail.trim()) { + const lines = detail + .split(/\r?\n/) + .map((line) => sanitizeGatewayRestartFailureLine(line.trim())) + .filter(Boolean) + .slice(-12); + for (const line of lines) { + console.error(` ${line}`); + } } + // Remediation is emitted outside the detail guard: an empty controller detail + // is exactly the case where the operator has nothing else to go on. if (layer === "MCP reconciliation refusal") { for (const line of hermesMcpReconciliationRemediationLines(sandboxName)) { console.error(` ${line}`); } } + if (isGatewayIntegrityRepairLayer(layer)) { + for (const line of gatewayIntegrityRepairLines(sandboxName, layer)) { + console.error(` ${line}`); + } + } } function unsupportedGatewayRestartAgentDetail(agentName: string, reason: string): string { diff --git a/src/lib/actions/sandbox/mcp-bridge-adapter-hermes.ts b/src/lib/actions/sandbox/mcp-bridge-adapter-hermes.ts index f7369f2381..105addf9ab 100644 --- a/src/lib/actions/sandbox/mcp-bridge-adapter-hermes.ts +++ b/src/lib/actions/sandbox/mcp-bridge-adapter-hermes.ts @@ -188,6 +188,7 @@ export function assertHermesMcpMutationRuntimeCapability(sandboxName: string): v classification.layer === "secret-boundary refusal" || classification.layer === "unsafe config path" || classification.layer === "config hash mismatch" || + classification.layer === "relaunch quarantined" || classification.layer === "health timeout" || recoveryFailureDetail.includes("SUPERVISOR_REBUILD_REQUIRED") || recoveryFailureDetail.includes("SUPERVISOR_UNSAFE_CONTROL_DIR") || diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 9bc28614ae..b6f8afa13e 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -39,6 +39,8 @@ import { type GatewayRestartDeps, type GatewayRestartFailureLayer, type GatewayRestartResult, + gatewayIntegrityRepairLines, + isGatewayIntegrityRepairLayer, printGatewayRestartFailure, type RestartSandboxGatewayOptions, restartSandboxGatewayWithDeps, @@ -851,6 +853,15 @@ function printHostManagedGatewayRecoveryHints( console.error(" If rebuild is blocked, destroy and re-onboard the sandbox to restore it."); return; } + // A drifted protected config and a quarantined supervisor both refuse every + // relaunch deterministically, so the generic "retry the managed restart" hint + // below would send the operator into a loop that cannot succeed (#7801). + if (isGatewayIntegrityRepairLayer(failureLayer)) { + for (const line of gatewayIntegrityRepairLines(quotedSandboxName, failureLayer)) { + console.error(` ${line}`); + } + return; + } let agentName = agent?.name ?? null; if (!agentName) { try { @@ -1024,6 +1035,10 @@ function isHermesAgent( * host-side dashboard port-forward when it has gone dead independently * of the gateway. Returns an object describing the outcome: * `{ checked, wasRunning, recovered, forwardRecovered, forwardRecoveryFailed?, secretBoundaryRefused?, secretBoundaryReason? }`. + * `onRecoveryFailureLayer` reports the classified managed-restart failure so a + * quiet caller (`recover`, `connect --probe-only`) can still explain why + * recovery is not retryable instead of printing a generic "check the gateway + * log". The result shape is unchanged so existing callers keep their contract. */ function checkAndRecoverSandboxProcessesWithoutHostLock( sandboxName: string, @@ -1035,6 +1050,7 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( isSandboxGatewayRunningImpl = isSandboxGatewayRunning, waitForRecreatedSandboxOpenShellReadyImpl = waitForRecreatedSandboxOpenShellReady, isWsl: isWslOverride, + onRecoveryFailureLayer, }: { quiet?: boolean; requestGatewaySupervisorAction?: typeof executeGatewaySupervisorAction; @@ -1043,6 +1059,7 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( isSandboxGatewayRunningImpl?: typeof isSandboxGatewayRunning; waitForRecreatedSandboxOpenShellReadyImpl?: typeof waitForRecreatedSandboxOpenShellReady; isWsl?: boolean; + onRecoveryFailureLayer?: (layer: GatewayRestartFailureLayer | null) => void; } = {}, ) { const recoveryAgent = agentRuntime.getSessionAgent(sandboxName); @@ -1297,6 +1314,7 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( managedRecoveryFailureLayer, ); } + onRecoveryFailureLayer?.(managedRecoveryFailureLayer); return { checked: true, wasRunning: false, recovered: false, forwardRecovered: false }; } if (relaunch) { @@ -1419,6 +1437,7 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( printHostManagedGatewayRecoveryHints(sandboxName, recoveryAgent, managedRecoveryFailureLayer); } + onRecoveryFailureLayer?.(managedRecoveryFailureLayer); return { checked: true, wasRunning: false, recovered: false, forwardRecovered: false }; } @@ -1432,6 +1451,7 @@ export function checkAndRecoverSandboxProcesses( isSandboxGatewayRunningImpl?: typeof isSandboxGatewayRunning; waitForRecreatedSandboxOpenShellReadyImpl?: typeof waitForRecreatedSandboxOpenShellReady; isWsl?: boolean; + onRecoveryFailureLayer?: (layer: GatewayRestartFailureLayer | null) => void; } = {}, ) { return withTimerBoundShieldsMutationLock(sandboxName, "gateway process recovery", () =>