Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions scripts/nemoclaw-start.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2358,6 +2358,21 @@ try:
if cleaned == lines:
sys.exit(0)

# When the rc file is not owned by us (and we are not root) we cannot
# safely rewrite it: fchmod would raise EPERM without CAP_FOWNER, and
# the in-place reopen via /proc/self/fd would fail anyway. The legacy
# shim line we would have removed is functionally inert (it sources
# /tmp/nemoclaw-proxy-env.sh only if that file exists), so leaving it
# in place is safer than crashing the container under errexit. A later
# root-mode boot can finish the cleanup.
if uid != 0 and st.st_uid != uid:
print(
f"[SECURITY] skipping rc cleanup for {rc_path}: not owned by uid={uid} "
f"(file uid={st.st_uid}); legacy shim left in place",
file=sys.stderr,
)
sys.exit(0)

try:
rewrite_open_rc_file(fd, st, cleaned)
except OSError as exc:
Expand Down
4 changes: 4 additions & 0 deletions src/lib/adapters/docker/container.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,10 @@ export function dockerStop(containerName: string, opts: DockerRunOptions = {}):
return dockerRun(["stop", containerName], opts);
}

export function dockerStart(containerName: string, opts: DockerRunOptions = {}): DockerRunResult {
return dockerRun(["start", containerName], opts);
}

// NIM (and most Python services) write errors to stderr, so a stdout-only
// capture would silently lose the auth-error tail. Use dockerRun so we can
// read both streams from the SpawnResult and concatenate them. Bounded by a
Expand Down
100 changes: 99 additions & 1 deletion src/lib/onboard/docker-gpu-patch.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -579,11 +579,109 @@ describe("docker-gpu-patch", () => {
String(call[0]).includes("nemoclaw-gpu-backup"),
);
expect(backupRmCall).toBeGreaterThanOrEqual(0);
expect(dockerRm.mock.invocationCallOrder[backupRmCall]).toBeLessThan(
// Backup container is removed only AFTER supervisor reconnect confirms
// the GPU container is reachable. If reconnect fails the rollback path
// restores the backup under the original name (see the rollback test
// below), so the backup must outlive the supervisor probe.
expect(dockerRm.mock.invocationCallOrder[backupRmCall]).toBeGreaterThan(
runOpenshell.mock.invocationCallOrder[0],
);
});

it("rolls back to the backup container when supervisor reconnect fails", () => {
const dockerCapture = vi.fn((args: readonly string[]) => {
if (args[0] === "ps") return "old-container-id\n";
if (args[0] === "inspect") return JSON.stringify([inspectFixture()]);
if (args[0] === "info") return "";
return "";
});
const dockerRun = vi.fn(() => ({ status: 0, stdout: "probe-id\n" }));
const dockerRunDetached = vi.fn(() => ({ status: 0, stdout: "new-container-id\n" }));
const dockerRename = vi.fn((_old: string, _next: string) => ({ status: 0 }));
const dockerStop = vi.fn(() => ({ status: 0 }));
const dockerStart = vi.fn(() => ({ status: 0 }));
const dockerRm = vi.fn((_name: string) => ({ status: 0 }));
const runOpenshell = vi.fn(() => ({ status: 1, stderr: "supervisor unreachable" }));
const runCaptureOpenshell = vi.fn(() => "");

expect(() =>
recreateOpenShellDockerSandboxWithGpu(
{ sandboxName: "alpha", timeoutSecs: 1 },
{
dockerCapture,
dockerRun,
dockerRunDetached,
dockerRename,
dockerStop,
dockerStart,
dockerRm,
runOpenshell,
runCaptureOpenshell,
sleep: vi.fn(),
now: () => new Date("2026-05-12T00:00:00Z"),
errorPhaseDebouncePolls: 1,
},
),
).toThrow(/pre-patch sandbox restored/);

const restoreRename = dockerRename.mock.calls.find(
(call) =>
String(call[0]).includes("nemoclaw-gpu-backup") && call[1] === "openshell-alpha",
);
expect(restoreRename).toBeDefined();
expect(dockerStart).toHaveBeenCalledWith(
"openshell-alpha",
expect.objectContaining({ ignoreError: true }),
);
expect(
dockerRm.mock.calls.some((call) => String(call[0]).includes("nemoclaw-gpu-backup")),
).toBe(false);
});

it("reports rollback failure when restoring the backup container fails", () => {
const dockerCapture = vi.fn((args: readonly string[]) => {
if (args[0] === "ps") return "old-container-id\n";
if (args[0] === "inspect") return JSON.stringify([inspectFixture()]);
if (args[0] === "info") return "";
return "";
});
const dockerRun = vi.fn(() => ({ status: 0, stdout: "probe-id\n" }));
const dockerRunDetached = vi.fn(() => ({ status: 0, stdout: "new-container-id\n" }));
const dockerRename = vi.fn((oldName: string) => {
if (String(oldName).includes("nemoclaw-gpu-backup")) {
return { status: 1, stderr: "no such container" };
}
return { status: 0 };
});
const dockerStop = vi.fn(() => ({ status: 0 }));
const dockerStart = vi.fn(() => ({ status: 0 }));
const dockerRm = vi.fn(() => ({ status: 0 }));
const runOpenshell = vi.fn(() => ({ status: 1, stderr: "supervisor unreachable" }));
const runCaptureOpenshell = vi.fn(() => "");

expect(() =>
recreateOpenShellDockerSandboxWithGpu(
{ sandboxName: "alpha", timeoutSecs: 1 },
{
dockerCapture,
dockerRun,
dockerRunDetached,
dockerRename,
dockerStop,
dockerStart,
dockerRm,
runOpenshell,
runCaptureOpenshell,
sleep: vi.fn(),
now: () => new Date("2026-05-12T00:00:00Z"),
errorPhaseDebouncePolls: 1,
},
),
).toThrow(/rollback to backup container failed/);

expect(dockerStart).not.toHaveBeenCalled();
});

it("can recreate during sandbox create before supervisor exec is allowed", () => {
const dockerCapture = vi.fn((args: readonly string[]) => {
if (args[0] === "ps") return "old-container-id\n";
Expand Down
70 changes: 59 additions & 11 deletions src/lib/onboard/docker-gpu-patch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import {
dockerRm,
dockerRun,
dockerRunDetached,
dockerStart,
dockerStop,
} from "../adapters/docker";
import {
Expand Down Expand Up @@ -70,6 +71,7 @@ export type DockerGpuPatchDeps = {
dockerRunDetached?: DockerRunFn;
dockerRename?: DockerRenameFn;
dockerRm?: DockerContainerFn;
dockerStart?: DockerContainerFn;
dockerStop?: DockerContainerFn;
dockerLogs?: DockerLogsFn;
runOpenshell?: (args: string[], opts?: Record<string, unknown>) => DockerRunResult;
Expand Down Expand Up @@ -112,6 +114,7 @@ export type DockerGpuPatchFailureContext = {
backupContainerName?: string | null;
selectedMode?: DockerGpuPatchMode | null;
modeAttempts?: DockerGpuPatchModeAttempt[];
rolledBack?: boolean;
};

export type DockerGpuPatchResult = {
Expand Down Expand Up @@ -245,6 +248,7 @@ function depsWithDefaults(deps: DockerGpuPatchDeps): Required<
| "dockerRunDetached"
| "dockerRename"
| "dockerRm"
| "dockerStart"
| "dockerStop"
| "dockerLogs"
| "sleep"
Expand All @@ -260,6 +264,7 @@ function depsWithDefaults(deps: DockerGpuPatchDeps): Required<
dockerRunDetached,
dockerRename,
dockerRm,
dockerStart,
dockerStop,
dockerLogs,
sleep: (seconds: number) => {
Expand Down Expand Up @@ -867,6 +872,37 @@ export function getDockerGpuPatchFailureContext(
return null;
}

// Tear down the newly-created GPU container and put the pre-patch backup
// container back in its place. Used when supervisor reconnect cannot confirm
// the GPU container is reachable — keeps the user on the CPU-only sandbox
// they had before the patch attempt rather than a deleted-backup / failed-new
// state that requires manual cleanup.
function rollbackToBackupContainer(
refs: { newContainerId: string; backupContainerName: string; originalName: string },
d: {
dockerStop: DockerContainerFn;
dockerRm: DockerContainerFn;
dockerRename: DockerRenameFn;
dockerStart: DockerContainerFn;
},
): boolean {
const containerOpts = {
ignoreError: true,
suppressOutput: true,
timeout: DOCKER_GPU_PATCH_TIMEOUT_MS,
};
d.dockerStop(refs.newContainerId, containerOpts);
d.dockerRm(refs.newContainerId, containerOpts);
const restored = d.dockerRename(
refs.backupContainerName,
refs.originalName,
containerOpts,
);
if (!isZeroStatus(restored)) return false;
d.dockerStart(refs.originalName, containerOpts);
return true;
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
}

export function recreateOpenShellDockerSandboxWithGpu(
options: {
sandboxName: string;
Expand Down Expand Up @@ -961,23 +997,34 @@ export function recreateOpenShellDockerSandboxWithGpu(
}
context.newContainerId = newContainerId;

const execReady =
options.waitForSupervisor === false
? true
: waitForOpenShellSupervisorReconnect(
options.sandboxName,
options.timeoutSecs ?? DOCKER_GPU_PATCH_WAIT_SECS,
deps,
);

if (!execReady) {
const rolledBack = rollbackToBackupContainer(
{ newContainerId, backupContainerName, originalName },
d,
);
context.rolledBack = rolledBack;
throw new Error(
rolledBack
? "OpenShell supervisor did not reconnect to the GPU-enabled container; pre-patch sandbox restored."
: "OpenShell supervisor did not reconnect to the GPU-enabled container; rollback to backup container failed.",
);
}

d.dockerRm(backupContainerName, {
ignoreError: true,
suppressOutput: true,
timeout: DOCKER_GPU_PATCH_TIMEOUT_MS,
});

if (options.waitForSupervisor !== false) {
const execReady = waitForOpenShellSupervisorReconnect(
options.sandboxName,
options.timeoutSecs ?? DOCKER_GPU_PATCH_WAIT_SECS,
deps,
);
if (!execReady) {
throw new Error("OpenShell supervisor did not reconnect to the GPU-enabled container.");
}
}

return {
applied: true,
oldContainerId,
Expand Down Expand Up @@ -1498,6 +1545,7 @@ export function collectDockerGpuPatchDiagnostics(
`old_container_id=${context?.oldContainerId ?? "unknown"}`,
`new_container_id=${context?.newContainerId ?? "unknown"}`,
`backup_container_name=${context?.backupContainerName ?? "none"}`,
`rolled_back=${context?.rolledBack === true ? "yes" : context?.rolledBack === false ? "failed" : "no"}`,
"cleanup_commands:",
...cleanupCommands.map((command) => ` ${command}`),
];
Expand Down
10 changes: 5 additions & 5 deletions src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -98,8 +98,8 @@ describe("docker-gpu-supervisor-reconnect Error-phase debounce", () => {
expect(runOpenshell).toHaveBeenCalledTimes(6);
});

it("defaults the debounce to 5 polls and honors the env override", () => {
expect(getDockerGpuSupervisorReconnectErrorDebouncePolls({})).toBe(5);
it("defaults the debounce to 15 polls and honors the env override", () => {
expect(getDockerGpuSupervisorReconnectErrorDebouncePolls({})).toBe(15);
expect(
getDockerGpuSupervisorReconnectErrorDebouncePolls({
NEMOCLAW_DOCKER_GPU_SUPERVISOR_RECONNECT_ERROR_DEBOUNCE: "2",
Expand Down Expand Up @@ -151,9 +151,9 @@ describe("docker-gpu-supervisor-reconnect Error-phase debounce", () => {
});

expect(ok).toBe(false);
// Default K=5 from the env-backed helper: 5 polls + 4 sleeps before fast-fail.
expect(runOpenshell).toHaveBeenCalledTimes(5);
expect(sleep).toHaveBeenCalledTimes(4);
// Default K=15 from the env-backed helper: 15 polls + 14 sleeps before fast-fail.
expect(runOpenshell).toHaveBeenCalledTimes(15);
expect(sleep).toHaveBeenCalledTimes(14);
}
});
});
10 changes: 6 additions & 4 deletions src/lib/onboard/docker-gpu-supervisor-reconnect.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,12 @@ import { envInt } from "./env";
const DOCKER_GPU_PATCH_TIMEOUT_MS = 30_000;
const DOCKER_GPU_SUPERVISOR_RECONNECT_MIN_SECS = 900;
// Default consecutive Error-phase polls required before fast-fail. With a
// 2-second poll interval this is ~10s of sustained Error, which absorbs the
// transient Error reported during container recreation while still bailing
// fast on a patched container that crashed on startup.
const DOCKER_GPU_SUPERVISOR_RECONNECT_ERROR_PHASE_DEFAULT_DEBOUNCE_POLLS = 5;
// 2-second poll interval this is ~30s of sustained Error, leaving headroom
// for slower hosts (Docker Desktop on WSL2, DGX Spark cached re-onboard)
// whose sandbox list cache divergence was observed at ~12s in the original
// repro. Hosts that genuinely crashed on startup hit the rollback path in
// applyDockerGpuPatch rather than waiting out the full window.
const DOCKER_GPU_SUPERVISOR_RECONNECT_ERROR_PHASE_DEFAULT_DEBOUNCE_POLLS = 15;

export const DOCKER_GPU_SUPERVISOR_RECONNECT_TIMEOUT_ENV =
"NEMOCLAW_DOCKER_GPU_SUPERVISOR_RECONNECT_TIMEOUT";
Expand Down
78 changes: 78 additions & 0 deletions test/nemoclaw-start.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4694,3 +4694,81 @@ describe("direct-root entrypoint composition under CAP_DAC_OVERRIDE drop", () =>
}
});
});

describe("ensure_runtime_shell_env_shim rc cleanup ownership guard", () => {
function extractRcCleanupPython(src: string): string {
const heredocStart = src.indexOf("<<'PY'");
expect(heredocStart).toBeGreaterThan(-1);
const bodyStart = src.indexOf("\n", heredocStart) + 1;
const bodyEnd = src.indexOf("\nPY\n", bodyStart);
expect(bodyEnd).toBeGreaterThan(bodyStart);
return src.slice(bodyStart, bodyEnd);
}

it("skips rewrite and exits 0 when the rc file is not owned by the current uid", () => {
const src = fs.readFileSync(START_SCRIPT, "utf-8");
const py = extractRcCleanupPython(src);

const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-rc-cleanup-"));
try {
const rcPath = path.join(tmpDir, ".bashrc");
const shim = "[ -f /tmp/nemoclaw-proxy-env.sh ] && . /tmp/nemoclaw-proxy-env.sh";
const original = `# user content\nexport FOO=bar\n${shim}\n`;
fs.writeFileSync(rcPath, original);

const pyPath = path.join(tmpDir, "cleanup.py");
fs.writeFileSync(pyPath, py);

// Lie about the running uid so the script sees uid != file owner.
// The on-disk file is owned by the test runner; passing an arbitrary
// non-zero uid that differs from the file's uid drives the new
// ownership-guard branch.
const fakeUid = (process.getuid?.() ?? 1000) + 7777;
const result = spawnSync(
"python3",
[pyPath, rcPath, shim, String(fakeUid)],
{ encoding: "utf-8" },
);

expect(result.status).toBe(0);
expect(result.stderr).toContain("skipping rc cleanup");
expect(result.stderr).toContain("not owned by uid=");
expect(fs.readFileSync(rcPath, "utf-8")).toBe(original);
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true });
}
});

it("still rewrites the rc file when the current uid owns it", () => {
const src = fs.readFileSync(START_SCRIPT, "utf-8");
const py = extractRcCleanupPython(src);

const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-rc-cleanup-"));
try {
const rcPath = path.join(tmpDir, ".bashrc");
const shim = "[ -f /tmp/nemoclaw-proxy-env.sh ] && . /tmp/nemoclaw-proxy-env.sh";
const original = `# user content\nexport FOO=bar\n${shim}\n`;
fs.writeFileSync(rcPath, original);

const pyPath = path.join(tmpDir, "cleanup.py");
fs.writeFileSync(pyPath, py);

const realUid = process.getuid?.() ?? 0;
const result = spawnSync(
"python3",
[pyPath, rcPath, shim, String(realUid)],
{ encoding: "utf-8" },
);

expect(result.status).toBe(0);
expect(result.stderr).not.toContain("could not safely clean runtime env shim");
expect(result.stderr).not.toContain("skipping rc cleanup");
const cleaned = fs.readFileSync(rcPath, "utf-8");
expect(cleaned).not.toContain(shim);
expect(cleaned).toContain("# user content");
expect(cleaned).toContain("export FOO=bar");
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true });
}
});
});
Loading