Skip to content
Merged
50 changes: 42 additions & 8 deletions src/lib/actions/sandbox/process-recovery.ts
Original file line number Diff line number Diff line change
Expand Up @@ -688,6 +688,7 @@ type RecreatedSandboxOpenShellReadyOptions = {
function recreatedSandboxOpenShellReadinessFailureDetail(
failure: RecreatedSandboxOpenShellReadinessFailure,
openshellError?: string,
managedHealthFailureDetail?: string,
): string {
const detail = (() => {
switch (failure) {
Expand All @@ -699,7 +700,13 @@ function recreatedSandboxOpenShellReadinessFailureDetail(
return "the recreated sandbox did not become ready in OpenShell, so the primary dashboard/API host forward was not started";
}
})();
return openshellError ? `${detail} Last OpenShell readiness error: ${openshellError}` : detail;
const managedHealthResult = managedHealthFailureDetail
? ` Managed health result: ${managedHealthFailureDetail}`
: "";
const openshellResult = openshellError
? ` Last OpenShell readiness error: ${openshellError}`
: "";
return `${detail}${managedHealthResult}${openshellResult}`;
}

// Default seconds to wait for OpenShell to re-register a recreated sandbox as
Expand Down Expand Up @@ -1249,17 +1256,29 @@ function checkAndRecoverSandboxProcessesWithoutHostLock(
requestPinnedGatewaySupervisorAction(name, action, timeout, relaunch.containerId)
: requestGatewaySupervisorAction;
let relaunchedIdentityRejected = false;
let relaunchedManagedHealthFailureDetail: string | null = null;
const confirmRelaunchedManagedHealth = relaunch
? (timeout = OPENSHELL_PROBE_TIMEOUT_MS) => {
let probeResult: SandboxCommandResult | null = null;
try {
const confirmed = confirmRecoveredSandboxGatewayManaged(sandboxName, {
requestGatewaySupervisorActionImpl: (name, action) =>
requestManagedProbe(name, action, timeout),
requestGatewaySupervisorActionImpl: (name, action) => {
probeResult = requestManagedProbe(name, action, timeout);
return probeResult;
},
});
if (confirmed === false) relaunchedIdentityRejected = true;
if (confirmed === false) {
relaunchedIdentityRejected = true;
const failure = classifyGatewayRestartFailure(probeResult);
relaunchedManagedHealthFailureDetail = `${failure.layer}: ${failure.detail}`;
} else if (confirmed === true) {
relaunchedManagedHealthFailureDetail = null;
}
return confirmed;
} catch {
relaunchedIdentityRejected = true;
relaunchedManagedHealthFailureDetail =
"the pinned replacement sandbox identity changed during the managed probe";
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
return false;
}
}
Expand All @@ -1276,10 +1295,12 @@ function checkAndRecoverSandboxProcessesWithoutHostLock(
initialManagedHealthPassed: recovery.kind === "managed",
requireManagedProbe: recovery.kind === "relaunched",
timeoutSeconds: gatewayRecoveryTimeoutSeconds(recoveryAgent),
managedProbeImpl: (name) =>
confirmRecoveredSandboxGatewayManaged(name, {
requestGatewaySupervisorActionImpl: requestManagedProbe,
}),
managedProbeImpl: relaunch
? () => confirmRelaunchedManagedHealth?.(210000) ?? null
: (name) =>
confirmRecoveredSandboxGatewayManaged(name, {
requestGatewaySupervisorActionImpl: requestManagedProbe,
}),
});
} catch (error) {
try {
Expand Down Expand Up @@ -1315,6 +1336,16 @@ function checkAndRecoverSandboxProcessesWithoutHostLock(
);
}
onRecoveryFailureLayer?.(managedRecoveryFailureLayer);
if (relaunchedManagedHealthFailureDetail) {
return {
checked: true,
wasRunning: false,
recovered: false,
forwardRecovered: false,
forwardRecoveryFailed: true,
forwardRecoveryFailureDetail: `the recreated sandbox failed the managed health guard while waiting for its gateway. Managed health result: ${relaunchedManagedHealthFailureDetail}`,
};
}
return { checked: true, wasRunning: false, recovered: false, forwardRecovered: false };
}
if (relaunch) {
Expand Down Expand Up @@ -1380,6 +1411,9 @@ function checkAndRecoverSandboxProcessesWithoutHostLock(
: recreatedSandboxOpenShellReadinessFailureDetail(
readiness.failure,
"openshellError" in readiness ? readiness.openshellError : undefined,
readiness.failure === "managed-health-definitive-failure"
? (relaunchedManagedHealthFailureDetail ?? undefined)
: undefined,
);
})()
: null;
Expand Down
56 changes: 56 additions & 0 deletions src/lib/state/state-file-restore-mode.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { spawnSync } from "node:child_process";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";

import { afterEach, describe, expect, it } from "vitest";

import { buildStateFileRestoreCommand } from "./state-file-restore";

const STATE_FILE = { path: "openclaw.json", strategy: "copy" } as const;
const fixtures: string[] = [];

function runRestore(refreshOpenClawConfigHash: boolean): {
configPath: string;
stateDir: string;
} {
const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-state-file-mode-"));
fixtures.push(fixture);
const stateDir = path.join(fixture, ".openclaw");
fs.mkdirSync(stateDir);
const command = buildStateFileRestoreCommand(stateDir, STATE_FILE, refreshOpenClawConfigHash);
const result = spawnSync("bash", ["-c", command], {
input: Buffer.from('{"gateway":{"mode":"local"}}\n'),
});
expect(result.status, result.stderr.toString()).toBe(0);
return { configPath: path.join(stateDir, STATE_FILE.path), stateDir };
}

function mode(filePath: string): number {
return fs.statSync(filePath).mode & 0o777;
}

afterEach(() => {
for (const fixture of fixtures.splice(0)) {
fs.rmSync(fixture, { force: true, recursive: true });
}
});

describe("state-file restore modes", () => {
it("restores an OpenClaw config with the mutable managed-guard mode", () => {
const { configPath, stateDir } = runRestore(true);

expect(mode(configPath)).toBe(0o660);
expect(mode(`${configPath}.last-good`)).toBe(0o660);
expect(mode(path.join(stateDir, ".config-hash"))).toBe(0o660);
});

it("keeps the restricted mode for ordinary copied state files", () => {
const { configPath } = runRestore(false);

expect(mode(configPath)).toBe(0o640);
});
Comment on lines +51 to +55

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🎯 Functional Correctness | 🟡 Minor | ⚡ Quick win

Test a non-managed copied state file.

runRestore(false) still restores openclaw.json from STATE_FILE. This does not verify the 0640 mode for an ordinary copied state file. Add a second fixture with a non-managed path and assert its mode is 0640.

Based on PR objectives, ordinary copied state files must retain mode 0640. As per path instructions, review tests for behavioral confidence rather than implementation lock-in.

🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@src/lib/state/state-file-restore-mode.test.ts` around lines 51 - 55, Add a
separate fixture for an ordinary copied state file with a non-managed path, then
invoke the restore flow against that fixture and assert its resulting mode is
0o640. Keep the existing runRestore(false) coverage intact, and focus the new
assertion on behavior rather than the specific implementation.

Source: Path instructions

});
7 changes: 6 additions & 1 deletion src/lib/state/state-file-restore.ts
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,12 @@ export function buildStateFileRestoreCommand(
'tmp="$(mktemp "${parent}/.nemoclaw-restore.XXXXXX")"',
'trap \'rm -f "$tmp" "${anchor_tmp:-}"\' EXIT',
'cat > "$tmp"',
'chmod 640 "$tmp"',
// The managed OpenClaw restart preflight accepts only the exact mutable
// sandbox:sandbox 0660 configuration posture. Apply that mode to the
// staged inode before the atomic swap so the gateway and its trusted
// controller never observe the restored config with the generic 0640
// state-file mode.
refreshOpenClawConfigHash ? 'chmod 660 "$tmp"' : 'chmod 640 "$tmp"',
];

if (refreshOpenClawConfigHash) {
Expand Down
98 changes: 98 additions & 0 deletions test/e2e/live/gateway-guard-legacy-keepalive-fixture.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import path from "node:path";
import { pathToFileURL } from "node:url";

import * as startupCommandPatch from "../../../src/lib/onboard/docker-startup-command-patch.ts";
import { redactString } from "../fixtures/redaction.ts";

const LEGACY_KEEPALIVE_COMMAND = ["sleep", "infinity"] as const;
const DEFAULT_RECREATE_TIMEOUT_SECS = 180;
const DOCKER_CONTAINER_ID_PATTERN = /^[0-9a-f]{64}$/i;
const { recreateOpenShellDockerSandboxWithStartupCommand } = startupCommandPatch;

type StartupCommandRecreate = typeof recreateOpenShellDockerSandboxWithStartupCommand;

export type LegacyKeepaliveFixtureOptions = {
sandboxName: string;
expectedContainerId: string;
timeoutSecs?: number;
};

export type LegacyKeepaliveFixtureDeps = {
recreate: StartupCommandRecreate;
};

const defaultDeps: LegacyKeepaliveFixtureDeps = {
recreate: recreateOpenShellDockerSandboxWithStartupCommand,
};

function requireFixtureInput(condition: boolean, message: string): asserts condition {
if (!condition) throw new Error(message);
}

export function createLegacyKeepaliveFixture(
options: LegacyKeepaliveFixtureOptions,
deps: LegacyKeepaliveFixtureDeps = defaultDeps,
): ReturnType<StartupCommandRecreate> {
requireFixtureInput(options.sandboxName.trim() !== "", "sandbox name is required");
requireFixtureInput(
DOCKER_CONTAINER_ID_PATTERN.test(options.expectedContainerId),
"expected container ID must be a full Docker container ID",
);

const result = deps.recreate({
sandboxName: options.sandboxName,
expectedOldContainerId: options.expectedContainerId,
openshellSandboxCommand: LEGACY_KEEPALIVE_COMMAND,
timeoutSecs: options.timeoutSecs ?? DEFAULT_RECREATE_TIMEOUT_SECS,
});

requireFixtureInput(
result.oldContainerId === options.expectedContainerId,
"legacy keepalive recreation changed an unpinned container",
);
requireFixtureInput(
result.mode.kind === "startup-command",
"legacy keepalive recreation did not use startup-command mode",
);
requireFixtureInput(
result.newContainerId !== result.oldContainerId,
"legacy keepalive recreation did not replace the container",
);
requireFixtureInput(
result.backupRemoved,
"legacy keepalive recreation left the original container backup in place",
);
return result;
}

function main(): void {
const [sandboxName, expectedContainerId, ...extraArgs] = process.argv.slice(2);
requireFixtureInput(
sandboxName !== undefined && expectedContainerId !== undefined && extraArgs.length === 0,
"usage: gateway-guard-legacy-keepalive-fixture <sandbox-name> <container-id>",
);
const result = createLegacyKeepaliveFixture({ sandboxName, expectedContainerId });
process.stdout.write(
`${JSON.stringify({
oldContainerId: result.oldContainerId,
newContainerId: result.newContainerId,
startupCommand: LEGACY_KEEPALIVE_COMMAND.join(" "),
})}\n`,
);
}

const invokedAsScript =
process.argv[1] !== undefined &&
import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href;
if (invokedAsScript) {
try {
main();
} catch (error: unknown) {
const detail = error instanceof Error ? error.message : String(error);
process.stderr.write(`${redactString(detail)}\n`);
process.exitCode = 1;
}
}
44 changes: 32 additions & 12 deletions test/e2e/live/gateway-guard-recovery.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@
* #2701 guard-chain assertion.
*/

import { fileURLToPath } from "node:url";

import { containsInteger42Answer } from "../../helpers/e2e-answer-assertions.ts";
import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts";
import { resultText } from "../fixtures/clients/command.ts";
Expand All @@ -60,6 +62,9 @@ import { ubuntuRepoDocker } from "../registry/matrix.ts";
const ENVIRONMENT = ubuntuRepoDocker("cloud-openclaw");

const SANDBOX_NAME = "e2e-2701";
const LEGACY_KEEPALIVE_FIXTURE = fileURLToPath(
new URL("./gateway-guard-legacy-keepalive-fixture.ts", import.meta.url),
);

const STARTUP_COMMAND_INSPECT_SCRIPT = String.raw`
const { spawnSync } = require("node:child_process");
Expand Down Expand Up @@ -147,7 +152,7 @@ test("gateway recovery restores /tmp guard chain after pod-recreate wipe (#2701)
"wipe guard chain and gateway tree",
"recover gateway through connect probe",
"validate recovered guard and stable PID",
"restart legacy Docker sandbox",
"recreate and restart sandbox container with legacy keepalive",
"recover managed supervisor and inference",
],
},
Expand Down Expand Up @@ -282,22 +287,37 @@ test("gateway recovery restores /tmp guard chain after pod-recreate wipe (#2701)

expect(stablePid).toBeGreaterThan(0);

progress.phase("restart legacy Docker sandbox");
progress.phase("recreate and restart sandbox container with legacy keepalive");
// ── Assert #6635 legacy Docker restart recovery ────────────────
// Fresh non-GPU OpenClaw containers on this OpenShell floor still carry the
// legacy keepalive. Restarting the container therefore kills the initial
// OpenShell workload session and deterministically leaves no managed
// supervisor. Recovery must upgrade that container through the host-side
// transaction and commit only after managed control accepts the new tree.
const originalContainerId = await findSandboxContainer(host, "legacy-restart-container-before");
// Current onboarding persists nemoclaw-start. Recreate the sandbox container
// with the exact legacy keepalive before restarting it so this target
// continues to prove the compatibility migration rather than asserting
// stale defaults.
const modernContainerId = await findSandboxContainer(host, "legacy-restart-modern-container");
expect(
await inspectStartupCommand(host, originalContainerId, "legacy-restart-command-before"),
).toBe("sleep infinity");
await inspectStartupCommand(host, modernContainerId, "legacy-restart-modern-command"),
).toMatch(/(?:^| )nemoclaw-start$/);
await host.cleanupForward(18789, {
artifactName: "legacy-restart-stop-dashboard-forward",
env: buildAvailabilityProbeEnv(),
});
const restart = await host.command("docker", ["restart", originalContainerId], {
const createLegacyKeepalive = await host.command(
"npx",
["--no-install", "tsx", LEGACY_KEEPALIVE_FIXTURE, instance.sandboxName, modernContainerId],
{
artifactName: "legacy-restart-create-keepalive",
env: buildAvailabilityProbeEnv(),
timeoutMs: 240_000,
},
);
expect(createLegacyKeepalive.exitCode, resultText(createLegacyKeepalive)).toBe(0);

const legacyContainerId = await findSandboxContainer(host, "legacy-restart-legacy-container");
expect(legacyContainerId).not.toBe(modernContainerId);
expect(
await inspectStartupCommand(host, legacyContainerId, "legacy-restart-legacy-command"),
).toBe("sleep infinity");
const restart = await host.command("docker", ["restart", legacyContainerId], {
artifactName: "legacy-restart-docker-restart",
env: buildAvailabilityProbeEnv(),
timeoutMs: 120_000,
Expand All @@ -321,7 +341,7 @@ test("gateway recovery restores /tmp guard chain after pod-recreate wipe (#2701)
expect(resultText(trustedRecovery)).toContain("Probe complete: recovered OpenClaw gateway");

const recoveredContainerId = await findSandboxContainer(host, "legacy-restart-container-after");
expect(recoveredContainerId).not.toBe(originalContainerId);
expect(recoveredContainerId).not.toBe(legacyContainerId);
const recoveredStartupCommand = await inspectStartupCommand(
host,
recoveredContainerId,
Expand Down
1 change: 1 addition & 0 deletions test/e2e/mock-parity.json
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@
"src/lib/sandbox/privileged-exec.test.ts",
"test/managed-gateway-control.test.ts",
"test/nemoclaw-start-guard-recovery.test.ts",
"test/e2e/support/gateway-guard-legacy-keepalive-fixture.test.ts",
"test/process-recovery-supervisor-relaunch.test.ts"
]
},
Expand Down
Loading
Loading