Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions src/lib/onboard/docker-startup-command-agent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,10 @@ export function resolveDockerStartupCommandPatch(
if (dockerDriverGateway !== true) {
return { persistStartupCommand: false, requiredUlimits: null };
}
const agentName = agent?.name;
const agentName = agent?.name ?? "openclaw";
return {
persistStartupCommand: agentName === "hermes" || agentName === DCODE_AGENT_NAME,
persistStartupCommand:
agentName === "openclaw" || agentName === "hermes" || agentName === DCODE_AGENT_NAME,
requiredUlimits: agentName === DCODE_AGENT_NAME ? DCODE_DOCKER_ULIMITS : null,
};
}
9 changes: 7 additions & 2 deletions src/lib/onboard/sandbox-create-step.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,12 @@ describe("runSandboxCreateStep", () => {
});
});

it("persists the Hermes startup command for Docker-driver container restarts", async () => {
it.each([
{ label: "OpenClaw", agent: null },
{ label: "Hermes", agent: { name: "hermes" } as SandboxCreateStepContext["agent"] },
])("persists the $label startup command for Docker-driver container restarts", async ({
agent,
}) => {
const launch = makeLaunch({
sandboxStartupCommand: ["env", "CHAT_UI_URL=http://127.0.0.1:8642", "nemoclaw-start"],
});
Expand All @@ -143,7 +148,7 @@ describe("runSandboxCreateStep", () => {

await runSandboxCreateStep(
makeContext({
agent: { name: "hermes" } as SandboxCreateStepContext["agent"],
agent,
prebuild: {
buildCtx: "/tmp/ctx",
buildId: "b1",
Expand Down
119 changes: 116 additions & 3 deletions test/e2e/live/sandbox-survival.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,12 @@ import {
cleanupWhenCommandAvailable,
cleanupWhenOpenShellAvailable,
} from "../fixtures/cleanup-resources.ts";
import { assertExitZero, resultText, sandboxAccessEnv } from "../fixtures/clients/index.ts";
import {
assertExitZero,
type HostCliClient,
resultText,
sandboxAccessEnv,
} from "../fixtures/clients/index.ts";
import { trustedProviderEndpoint } from "../fixtures/clients/provider.ts";
import { expect, test } from "../fixtures/e2e-test.ts";
import { requireHostedInferenceConfig } from "../fixtures/hosted-inference.ts";
Expand All @@ -29,6 +34,86 @@ const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-survival";
const MIN_OPENSHELL_VERSION = "0.0.24";
const MODEL = process.env.NEMOCLAW_MODEL ?? "nvidia/nemotron-3-super-120b-a12b";

const SURVIVAL_DIAGNOSTICS_SCRIPT = String.raw`
set +e
sandbox_name="$1"

printf '%s\n' '== OpenShell sandbox status =='
openshell sandbox get "$sandbox_name" 2>&1
printf '%s\n' '== OpenShell forwards =='
openshell forward list 2>&1
printf '%s\n' '== OpenShell gateway service =='
systemctl --user status nemoclaw-openshell-gateway --no-pager -l 2>&1
printf '%s\n' '== OpenShell gateway journal =='
journalctl --user -u nemoclaw-openshell-gateway -n 200 --no-pager 2>&1

container_ids="$(docker ps -aq \
--filter label=openshell.ai/managed-by=openshell \
--filter "label=openshell.ai/sandbox-name=$sandbox_name")"
printf '%s\n' '== matching containers =='
if [ -n "$container_ids" ]; then
docker ps -a --no-trunc \
--filter label=openshell.ai/managed-by=openshell \
--filter "label=openshell.ai/sandbox-name=$sandbox_name" \
--format '{{.ID}} {{.Names}} {{.Status}}'
else
printf '%s\n' 'none'
fi

for container_id in $container_ids; do
printf '%s\n' "== container $container_id inspect =="
docker inspect "$container_id" 2>&1 | node -e '
const fs = require("node:fs");
const row = JSON.parse(fs.readFileSync(0, "utf8"))[0] || {};
const prefix = "OPENSHELL_SANDBOX_COMMAND=";
const matches = (row.Config?.Env || []).filter((entry) => entry.startsWith(prefix));
const command = matches.length === 1 ? matches[0].slice(prefix.length) : "";
const tokens = command.trim().split(/\s+/).filter(Boolean);
process.stdout.write(JSON.stringify({
name: row.Name || "",
configUser: row.Config?.User || "",
state: {
status: row.State?.Status || "",
running: Boolean(row.State?.Running),
restarting: Boolean(row.State?.Restarting),
pid: row.State?.Pid || 0,
exitCode: row.State?.ExitCode ?? null,
error: row.State?.Error || "",
startedAt: row.State?.StartedAt || "",
finishedAt: row.State?.FinishedAt || "",
health: row.State?.Health?.Status || "",
},
restartPolicy: row.HostConfig?.RestartPolicy?.Name || "",
startupCommandCount: matches.length,
startupCommandIsSleepInfinity: tokens.length === 2
&& tokens[0] === "sleep" && tokens[1] === "infinity",
startupCommandEndsWithNemoclawStart: tokens.length > 0
&& ["nemoclaw-start", "/usr/local/bin/nemoclaw-start"].includes(tokens.at(-1)),
}) + "\n");
'
printf '%s\n' "== container $container_id host process tree =="
docker top "$container_id" -eo pid,ppid,user,stat,comm 2>&1
printf '%s\n' "== container $container_id runtime state =="
docker exec "$container_id" sh -lc '
printf "%s\n" "== pid 1 =="
cat /proc/1/comm 2>/dev/null || true
printf "\n%s\n" "== process tree =="
ps -eo user=,pid=,ppid=,stat=,comm= 2>&1 || true
printf "%s\n" "== direct gateway health =="
curl -q --noproxy "*" -sS -o /dev/null -w "HTTP %{http_code}\n" \
--connect-timeout 2 --max-time 5 http://127.0.0.1:18789/health 2>&1 || true
printf "%s\n" "== managed controller status =="
cat /run/nemoclaw/gateway-control/status 2>&1 || true
printf "%s\n" "== startup log =="
tail -n 300 /tmp/nemoclaw-start.log 2>&1 || true
printf "%s\n" "== gateway log =="
tail -n 300 /tmp/gateway.log 2>&1 || true
' 2>&1
printf '%s\n' "== container $container_id logs =="
docker logs --tail 300 "$container_id" 2>&1
done
`;
Comment thread
sandl99 marked this conversation as resolved.
Resolved

function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
Expand Down Expand Up @@ -61,6 +146,23 @@ function installEnv(hostedEnv: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
};
}

async function captureSurvivalDiagnostics(
host: HostCliClient,
stage: string,
redactionValues: string[],
): Promise<void> {
await host.command(
"sh",
["-lc", SURVIVAL_DIAGNOSTICS_SCRIPT, "sandbox-survival-diagnostics", SANDBOX_NAME],
{
artifactName: `sandbox-survival-${stage}-diagnostics`,
env: buildAvailabilityProbeEnv(),
redactionValues,
timeoutMs: 60_000,
},
);
}

async function expectSandboxExecAlive(
sandboxName: string,
exec: (
Expand All @@ -85,7 +187,7 @@ test(
"install and register the OpenClaw sandbox",
"prove baseline sandbox access and inference",
"write persistent OpenClaw markers",
"restart the gateway and reconnect the sandbox",
"restart the gateway and recover the sandbox",
"recheck state and inference after restart",
"destroy the sandbox and confirm registry removal",
],
Expand Down Expand Up @@ -302,15 +404,25 @@ test(
await stateValidation.writeSandboxMarkers(instance, markers);
await stateValidation.expectSandboxMarkers(instance, markers, "pre-restart-marker-read");

progress.phase("restart the gateway and reconnect the sandbox");
progress.phase("restart the gateway and recover the sandbox");
await captureSurvivalDiagnostics(host, "before-gateway-restart", [apiKey]);
await lifecycle.restartGatewayRuntime({
delayMs: 5_000,
sandboxName: SANDBOX_NAME,
});
await captureSurvivalDiagnostics(host, "after-gateway-restart", [apiKey]);
await lifecycle.waitForGatewayConnected({
attempts: 60,
intervalMs: 5_000,
});
await captureSurvivalDiagnostics(host, "before-recover", [apiKey]);
const recovery = await host.nemoclaw([SANDBOX_NAME, "recover"], {
artifactName: "post-restart-nemoclaw-recover",
env: buildAvailabilityProbeEnv(),
timeoutMs: 240_000,
});
await captureSurvivalDiagnostics(host, "after-recover", [apiKey]);
assertExitZero(recovery, `recover sandbox ${SANDBOX_NAME}`);

progress.phase("recheck state and inference after restart");
await sandbox.expectListed(SANDBOX_NAME, {
Expand All @@ -324,6 +436,7 @@ test(
artifactName: "post-restart-nemoclaw-status",
timeoutMs: 120_000,
});
await stateValidation.from("cloud-openclaw-ready", instance);
await expectSandboxExecAlive(SANDBOX_NAME, execShell, "post-restart-sandbox-exec-alive");
await stateValidation.expectSandboxMarkers(instance, markers, "post-restart-marker-read");
await stateValidation.expectSandboxDirectoryPopulated(
Expand Down
37 changes: 37 additions & 0 deletions test/helpers/onboard-script-mocks.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,30 @@ const OPENCLAW_SECURITY_INVENTORY_PROBE = [
`printf '%s\\n' "nemoclaw-security-inventory-ok"`,
].join("; ");

const ONBOARD_SANDBOX_OLD_CONTAINER_ID = "a".repeat(64);
const ONBOARD_SANDBOX_NEW_CONTAINER_ID = "b".repeat(64);
const ONBOARD_SANDBOX_INSPECT = {
Id: ONBOARD_SANDBOX_OLD_CONTAINER_ID,
Image: `sha256:${"c".repeat(64)}`,
Name: "/openshell-my-assistant",
Config: {
Image: "openshell/sandbox:test",
Env: ["OPENSHELL_SANDBOX_COMMAND=sleep infinity"],
Labels: {
"openshell.ai/managed-by": "openshell",
"openshell.ai/sandbox-name": "my-assistant",
},
Entrypoint: ["/opt/openshell/bin/openshell-sandbox"],
Cmd: [],
User: "0",
WorkingDir: "/sandbox",
},
HostConfig: {
NetworkMode: "openshell-docker",
RestartPolicy: { Name: "unless-stopped" },
},
};

function isOpenClawSecurityInventoryProbe(command) {
const commandArgs = Array.isArray(command) ? command.map(String) : [];
const dockerArgs = commandArgs[0] === "docker" ? commandArgs.slice(1) : commandArgs;
Expand Down Expand Up @@ -113,6 +137,19 @@ function mockSandboxExecCurl(command, options = {}) {

function mockOnboardRunCapture(command, options = {}) {
const normalized = normalizeCommand(command);
if (
normalized.startsWith("docker ps -a --no-trunc ") &&
normalized.includes("label=openshell.ai/sandbox-name=my-assistant") &&
normalized.endsWith("--format {{.ID}}")
) {
return `${ONBOARD_SANDBOX_OLD_CONTAINER_ID}\n${ONBOARD_SANDBOX_NEW_CONTAINER_ID}\n`;
}
if (
normalized ===
`docker inspect --type container ${ONBOARD_SANDBOX_OLD_CONTAINER_ID}`
) {
return JSON.stringify([ONBOARD_SANDBOX_INSPECT]);
}
if (isOpenClawSecurityInventoryProbe(command)) {
return "nemoclaw-security-inventory-ok";
}
Expand Down
9 changes: 9 additions & 0 deletions test/onboard-sandbox-build.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -216,6 +216,15 @@ const { createSandbox } = require(${onboardPath});
),
"expected dashboard forward (loopback or WSL 0.0.0.0)",
);
assert.ok(
payload.commands.some(
(entry: CommandEntry) =>
entry.command.includes("docker run -d") &&
entry.command.includes("OPENSHELL_SANDBOX_COMMAND=") &&
entry.command.includes("nemoclaw-start"),
),
"expected the default OpenClaw startup command to be persisted in the recreated container",
);
});

it("skips OpenClaw sandbox-base resolution for agent-staged Dockerfiles", async () => {
Expand Down
Loading