diff --git a/docs/security/best-practices.mdx b/docs/security/best-practices.mdx index 8274edd7130..cac58db0f63 100644 --- a/docs/security/best-practices.mdx +++ b/docs/security/best-practices.mdx @@ -516,6 +516,8 @@ If the container runtime restricts `ulimit` modification, the entrypoint logs a Managed Deep Agents uses stricter enforcement than the best-effort entrypoint behavior described above. The managed Deep Agents image applies the 512-process and 65,536-file-descriptor caps to the long-running sandbox entrypoint tree and to direct managed `dcode` launches. +During Docker-driver onboarding, NemoClaw also configures the container with exact `nproc=512:512` and `nofile=65536:65536` limits so the managed entrypoint and fresh exec or connect processes start under the same hard caps. +The container-level limits and managed startup command remain in effect when you stop and start the sandbox or restart the OpenShell gateway. It also applies and verifies the caps in Bash login and interactive shells because fresh `openshell sandbox exec` and connect shells do not inherit the dcode entrypoint child's lowered limits. For managed Deep Agents, successful verification requires both the soft and hard limits to equal 512 processes and 65,536 file descriptors; a lower inherited file-descriptor default such as 1,024 is not accepted as successful hardening. @@ -524,7 +526,7 @@ The entrypoint and launcher complete this verification before proxy setup or use A login or interactive shell remains available when the helper is missing or its limits cannot be verified, but it prints `[SECURITY] Sandbox resource limits were NOT hardened for this shell.` OpenShell creates those fresh exec and connect processes outside the entrypoint tree, so an image layer cannot make them inherit the entrypoint child's lowered limits. The shell compatibility exception can be removed when OpenShell guarantees that every exec and connect process starts under enforced caps or exposes a fail-closed resource-limit contract. -Set hard `nproc` and `nofile` limits at the container runtime when you require fail-closed enforcement. +For custom container launches outside NemoClaw's managed Docker-driver onboarding path, set hard `nproc` and `nofile` limits at the container runtime when you require fail-closed enforcement. ### Deep Agents Thread Auto-Approval Capability diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 958cb648a6b..2265f703181 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -2801,7 +2801,7 @@ async function createSandboxWithBaseImageResolution( prebuild, restoreBackupPath, terminalAgent: agentDefs.isTerminalAgent(agent), - persistStartupCommand: dockerDriverGateway === true && agent?.name === "hermes", + ...sandboxGpuCreateFlow.resolveDockerStartupCommandPatch(agent, dockerDriverGateway), }, { runOpenshell, diff --git a/src/lib/onboard/docker-gpu-patch-clone.test.ts b/src/lib/onboard/docker-gpu-patch-clone.test.ts index e21d31f7ba9..b49d77f970c 100644 --- a/src/lib/onboard/docker-gpu-patch-clone.test.ts +++ b/src/lib/onboard/docker-gpu-patch-clone.test.ts @@ -73,6 +73,33 @@ describe("Docker GPU clone envelope", () => { ); }); + it("preserves inspected ulimits and overrides DCode's exact required limits", () => { + const inspect = inspectFixture(); + inspect.HostConfig!.Ulimits = [ + { Name: "core", Soft: 0, Hard: -1 }, + { Name: "nofile", Soft: 1024, Hard: 1024 }, + ]; + + const args = buildDockerGpuCloneRunArgs(inspect, buildDockerGpuMode("startup-command"), { + requiredUlimits: [ + { name: "nproc", soft: 512, hard: 512 }, + { name: "nofile", soft: 65_536, hard: 65_536 }, + ], + }); + + expect(args).toEqual( + expect.arrayContaining([ + "--ulimit", + "core=0:-1", + "--ulimit", + "nofile=65536:65536", + "--ulimit", + "nproc=512:512", + ]), + ); + expect(args).not.toContain("nofile=1024:1024"); + }); + it("adds SYS_PTRACE to the GPU clone when the baseline container lacks it", () => { const inspect = inspectFixture(); inspect.HostConfig!.CapAdd = ["SYS_ADMIN", "NET_ADMIN"]; diff --git a/src/lib/onboard/docker-gpu-patch-clone.ts b/src/lib/onboard/docker-gpu-patch-clone.ts index 59087d10b4c..cb904068318 100644 --- a/src/lib/onboard/docker-gpu-patch-clone.ts +++ b/src/lib/onboard/docker-gpu-patch-clone.ts @@ -5,6 +5,7 @@ import type { DockerContainerInspect, DockerGpuCloneRunOptions, DockerGpuPatchMode, + DockerUlimit, } from "./docker-gpu-patch-types"; import { openshellSandboxCommandEnvValue } from "./docker-startup-command-env"; @@ -64,6 +65,56 @@ function pushStringFlag(args: string[], flag: string, value: unknown): void { if (normalized) args.push(flag, normalized); } +function normalizeRequiredUlimit(ulimit: DockerUlimit): DockerUlimit { + const name = String(ulimit.name).trim(); + if (!/^[a-z][a-z0-9_]*$/u.test(name)) { + throw new Error(`Invalid Docker ulimit name '${name}'.`); + } + if ( + !Number.isSafeInteger(ulimit.soft) || + ulimit.soft < 0 || + !Number.isSafeInteger(ulimit.hard) || + ulimit.hard < ulimit.soft + ) { + throw new Error(`Invalid Docker ulimit values for '${name}'.`); + } + return { name, soft: ulimit.soft, hard: ulimit.hard }; +} + +export function validateRequiredDockerUlimits( + required: readonly DockerUlimit[] | null | undefined, +): void { + for (const ulimit of required ?? []) normalizeRequiredUlimit(ulimit); +} + +function dockerUlimits( + inspect: DockerContainerInspect, + required: readonly DockerUlimit[] | null | undefined, +): DockerUlimit[] { + const merged = new Map(); + for (const ulimit of inspect.HostConfig?.Ulimits ?? []) { + const name = String(ulimit.Name ?? "").trim(); + const soft = ulimit.Soft; + const hard = ulimit.Hard; + if ( + !name || + !Number.isSafeInteger(soft) || + (soft as number) < -1 || + !Number.isSafeInteger(hard) || + (hard as number) < -1 || + ((hard as number) !== -1 && (soft as number) > (hard as number)) + ) { + continue; + } + merged.set(name, { name, soft: soft as number, hard: hard as number }); + } + for (const ulimit of required ?? []) { + const normalized = normalizeRequiredUlimit(ulimit); + merged.set(normalized.name, normalized); + } + return [...merged.values()]; +} + function pushNumberFlag(args: string[], flag: string, value: unknown): void { if (typeof value === "number" && Number.isFinite(value) && value > 0) { args.push(flag, String(value)); @@ -217,6 +268,9 @@ export function buildDockerGpuCloneRunArgs( args.push("--group-add", normalized); } } + for (const ulimit of dockerUlimits(inspect, options.requiredUlimits)) { + args.push("--ulimit", `${ulimit.name}=${ulimit.soft}:${ulimit.hard}`); + } if (networkMode !== "host") { const dnsServers = stringArray(host.Dns); for (const dns of dnsServers) args.push("--dns", dns); diff --git a/src/lib/onboard/docker-gpu-patch-recreate.ts b/src/lib/onboard/docker-gpu-patch-recreate.ts index f58818ce81b..2584a9b42c0 100644 --- a/src/lib/onboard/docker-gpu-patch-recreate.ts +++ b/src/lib/onboard/docker-gpu-patch-recreate.ts @@ -19,6 +19,7 @@ import { dockerContainerName, parseDockerInspectJson, sameContainerId, + validateRequiredDockerUlimits, } from "./docker-gpu-patch-clone"; import { DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, @@ -151,6 +152,7 @@ export function recreateOpenShellDockerSandboxContainer( timeoutSecs?: number; waitForSupervisor?: boolean; openshellSandboxCommand?: readonly string[] | null; + requiredUlimits?: readonly import("./docker-gpu-patch-types").DockerUlimit[] | null; expectedOldContainerId?: string | null; backend?: "generic" | "jetson"; dockerDesktopWsl?: boolean; @@ -164,6 +166,7 @@ export function recreateOpenShellDockerSandboxContainer( modeAttempts: [], }; try { + validateRequiredDockerUlimits(options.requiredUlimits); const containerIds = findOpenShellDockerSandboxContainerIds(options.sandboxName, deps); const oldContainerId = containerIds[0]; if (!oldContainerId) { @@ -228,6 +231,7 @@ export function recreateOpenShellDockerSandboxContainer( const cloneOptions = buildDockerGpuCloneRunOptions(inspect); cloneOptions.image = image; cloneOptions.openshellSandboxCommand = options.openshellSandboxCommand ?? null; + cloneOptions.requiredUlimits = options.requiredUlimits ?? null; const sandboxFallbackDns = d.detectSandboxFallbackDns(); if (sandboxFallbackDns) cloneOptions.sandboxFallbackDns = sandboxFallbackDns; if (selection.mode.kind !== "startup-command" && options.backend === "jetson") { diff --git a/src/lib/onboard/docker-gpu-patch-types.ts b/src/lib/onboard/docker-gpu-patch-types.ts index b0ac01836a7..723dec4bd5f 100644 --- a/src/lib/onboard/docker-gpu-patch-types.ts +++ b/src/lib/onboard/docker-gpu-patch-types.ts @@ -94,12 +94,19 @@ export type DockerGpuPatchResult = { backupRemoved: boolean; }; +export type DockerUlimit = { + name: string; + soft: number; + hard: number; +}; + export type DockerGpuCloneRunOptions = { image?: string | null; networkMode?: string | null; openshellEndpoint?: string | null; sandboxFallbackDns?: string | null; openshellSandboxCommand?: readonly string[] | null; + requiredUlimits?: readonly DockerUlimit[] | null; /** * Extra supplementary group IDs to add to the recreated container via * `--group-add`. On Jetson these are the host group(s) owning the Tegra GPU @@ -201,6 +208,11 @@ export type DockerContainerInspect = { IpcMode?: string; PidMode?: string; GroupAdd?: string[] | null; + Ulimits?: Array<{ + Name?: string; + Soft?: number; + Hard?: number; + }> | null; Dns?: string[] | null; DnsSearch?: string[] | null; DeviceRequests?: Array<{ diff --git a/src/lib/onboard/docker-gpu-patch-validation.test.ts b/src/lib/onboard/docker-gpu-patch-validation.test.ts index 0d66d97e2e5..e1a270b3b9f 100644 --- a/src/lib/onboard/docker-gpu-patch-validation.test.ts +++ b/src/lib/onboard/docker-gpu-patch-validation.test.ts @@ -185,4 +185,40 @@ describe("Docker GPU startup command validation (#6110)", () => { expect(dockerRename).not.toHaveBeenCalled(); expect(dockerRunDetached).not.toHaveBeenCalled(); }); + + it("rejects malformed required ulimits before touching the original container", () => { + const dockerStop = vi.fn(() => ({ status: 0 })); + const dockerRun = vi.fn(() => ({ status: 0, stdout: "probe-id\n" })); + const dockerRunDetached = vi.fn(() => ({ status: 0, stdout: "new-container-id\n" })); + + expect(() => + recreateOpenShellDockerSandboxWithGpu( + { + sandboxName: "alpha", + timeoutSecs: 1, + requiredUlimits: [{ name: "nofile;id", soft: 65_536, hard: 65_536 }], + }, + { + dockerCapture: vi.fn((args: readonly string[]) => + args[0] === "ps" + ? "old-container-id\n" + : args[0] === "inspect" + ? JSON.stringify([inspectFixture()]) + : "", + ), + detectSandboxFallbackDns: vi.fn(() => null), + dockerRun, + dockerRunDetached, + dockerRename: vi.fn(() => ({ status: 0 })), + dockerRm: vi.fn(() => ({ status: 0 })), + dockerStop, + readDir: vi.fn(() => null), + readFile: vi.fn(() => null), + }, + ), + ).toThrow("Invalid Docker ulimit name"); + expect(dockerRun).not.toHaveBeenCalled(); + expect(dockerStop).not.toHaveBeenCalled(); + expect(dockerRunDetached).not.toHaveBeenCalled(); + }); }); diff --git a/src/lib/onboard/docker-gpu-patch.ts b/src/lib/onboard/docker-gpu-patch.ts index eb9dfa9fda9..d54847a7af6 100644 --- a/src/lib/onboard/docker-gpu-patch.ts +++ b/src/lib/onboard/docker-gpu-patch.ts @@ -15,6 +15,7 @@ import type { DockerGpuPatchMode, DockerGpuPatchResult, DockerGpuPatchSandboxSnapshot, + DockerUlimit, } from "./docker-gpu-patch-types"; export { detectSandboxFallbackDns } from "./docker-gpu-dns-fallback"; @@ -77,6 +78,7 @@ export type { DockerGpuPatchModeKind, DockerGpuPatchResult, DockerGpuPatchSandboxSnapshot, + DockerUlimit, } from "./docker-gpu-patch-types"; export { findOpenShellDockerSandboxContainerIds, diff --git a/src/lib/onboard/docker-gpu-sandbox-create.ts b/src/lib/onboard/docker-gpu-sandbox-create.ts index dd3c80408af..b683cbb1c77 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create.ts @@ -64,6 +64,7 @@ type DockerGpuSandboxCreatePatchOptions = { sandboxName: string; gpuDevice?: string | null; openshellSandboxCommand?: readonly string[] | null; + requiredUlimits?: Parameters[0]["requiredUlimits"]; timeoutSecs: number; backend?: DockerGpuPatchBackend; /** @@ -139,6 +140,7 @@ export function createDockerGpuSandboxCreatePatch( sandboxName: options.sandboxName, gpuDevice: options.gpuDevice, openshellSandboxCommand: options.openshellSandboxCommand ?? null, + requiredUlimits: options.requiredUlimits ?? null, timeoutSecs: options.timeoutSecs, backend: options.backend, dockerDesktopWsl: options.dockerDesktopWsl ?? isDockerDesktopWslRuntime(), @@ -149,6 +151,7 @@ export function createDockerGpuSandboxCreatePatch( gpuEnabled: routeAdapter.enabled, gpuOptions: applyOptions, startupCommand: options.openshellSandboxCommand, + requiredUlimits: options.requiredUlimits, recreateGpu: recreatePatch, recreateStartup: recreateStartupPatch, }); diff --git a/src/lib/onboard/docker-startup-command-agent.ts b/src/lib/onboard/docker-startup-command-agent.ts new file mode 100644 index 00000000000..23968f21d4c --- /dev/null +++ b/src/lib/onboard/docker-startup-command-agent.ts @@ -0,0 +1,32 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { AgentDefinition } from "../agent/defs"; +import type { DockerUlimit } from "./docker-gpu-patch-types"; + +const DCODE_AGENT_NAME = "langchain-deepagents-code"; + +// DCode's managed entrypoint fails closed unless both limits are exact. Set +// them on the Docker container so the OpenShell supervisor and every child +// inherit the contract, including after container and gateway restarts. +export const DCODE_DOCKER_ULIMITS: readonly DockerUlimit[] = [ + { name: "nproc", soft: 512, hard: 512 }, + { name: "nofile", soft: 65_536, hard: 65_536 }, +]; + +export function resolveDockerStartupCommandPatch( + agent: AgentDefinition | null | undefined, + dockerDriverGateway: boolean | null | undefined, +): { + persistStartupCommand: boolean; + requiredUlimits: readonly DockerUlimit[] | null; +} { + if (dockerDriverGateway !== true) { + return { persistStartupCommand: false, requiredUlimits: null }; + } + const agentName = agent?.name; + return { + persistStartupCommand: agentName === "hermes" || agentName === DCODE_AGENT_NAME, + requiredUlimits: agentName === DCODE_AGENT_NAME ? DCODE_DOCKER_ULIMITS : null, + }; +} diff --git a/src/lib/onboard/docker-startup-command-patch.ts b/src/lib/onboard/docker-startup-command-patch.ts index f9f48ba41c0..2b5b6f761ef 100644 --- a/src/lib/onboard/docker-startup-command-patch.ts +++ b/src/lib/onboard/docker-startup-command-patch.ts @@ -16,6 +16,7 @@ export function recreateOpenShellDockerSandboxWithStartupCommand( timeoutSecs?: number; waitForSupervisor?: boolean; openshellSandboxCommand: readonly string[]; + requiredUlimits?: readonly import("./docker-gpu-patch-types").DockerUlimit[] | null; expectedOldContainerId?: string | null; }, deps: DockerGpuPatchDeps = {}, diff --git a/src/lib/onboard/docker-startup-command-sandbox-create.test.ts b/src/lib/onboard/docker-startup-command-sandbox-create.test.ts index b9a166c9e5d..4c444253ff0 100644 --- a/src/lib/onboard/docker-startup-command-sandbox-create.test.ts +++ b/src/lib/onboard/docker-startup-command-sandbox-create.test.ts @@ -67,7 +67,7 @@ describe("Docker startup-command sandbox creation", () => { vi.restoreAllMocks(); }); - it("uses the default startup-command recreation path for non-GPU Hermes containers", () => { + it("uses the startup-command recreation path with DCode's exact resource limits", () => { const dockerCaptureOutput: Record = { ps: "old-container-id\n", inspect: JSON.stringify([inspectFixture()]), @@ -90,6 +90,10 @@ describe("Docker startup-command sandbox creation", () => { persistStartupCommand: true, sandboxName: "alpha", openshellSandboxCommand: ["env", "nemoclaw-start"], + requiredUlimits: [ + { name: "nproc", soft: 512, hard: 512 }, + { name: "nofile", soft: 65_536, hard: 65_536 }, + ], timeoutSecs: 60, deps, overrides: { @@ -101,7 +105,14 @@ describe("Docker startup-command sandbox creation", () => { expect(recreatePatch).not.toHaveBeenCalled(); expect(dockerRunDetached.mock.calls[0]?.[0]).toEqual( - expect.arrayContaining(["--env", "OPENSHELL_SANDBOX_COMMAND=env nemoclaw-start"]), + expect.arrayContaining([ + "--env", + "OPENSHELL_SANDBOX_COMMAND=env nemoclaw-start", + "--ulimit", + "nproc=512:512", + "--ulimit", + "nofile=65536:65536", + ]), ); expect(patch.selectedMode()?.kind).toBe("startup-command"); }); diff --git a/src/lib/onboard/docker-startup-command-sandbox-create.ts b/src/lib/onboard/docker-startup-command-sandbox-create.ts index 129603c6750..dd8df2e17b0 100644 --- a/src/lib/onboard/docker-startup-command-sandbox-create.ts +++ b/src/lib/onboard/docker-startup-command-sandbox-create.ts @@ -15,6 +15,7 @@ export function createDockerSandboxRecreator(options: { gpuEnabled: boolean; gpuOptions: Parameters[0]; startupCommand: readonly string[] | null | undefined; + requiredUlimits?: Parameters[0]["requiredUlimits"]; recreateGpu?: RecreateGpuPatchFn; recreateStartup?: RecreateStartupPatchFn; }): (waitForSupervisor: boolean, deps: DockerGpuPatchDeps) => DockerGpuPatchResult { @@ -29,6 +30,7 @@ export function createDockerSandboxRecreator(options: { { sandboxName: options.gpuOptions.sandboxName, openshellSandboxCommand: options.startupCommand || [], + requiredUlimits: options.requiredUlimits, timeoutSecs: options.gpuOptions.timeoutSecs, waitForSupervisor, }, diff --git a/src/lib/onboard/sandbox-create-step.test.ts b/src/lib/onboard/sandbox-create-step.test.ts index fe22a3f3003..ba26917eb04 100644 --- a/src/lib/onboard/sandbox-create-step.test.ts +++ b/src/lib/onboard/sandbox-create-step.test.ts @@ -163,6 +163,39 @@ describe("runSandboxCreateStep", () => { ); }); + it("persists DCode startup with its exact Docker resource limits", async () => { + const launch = makeLaunch({ + sandboxStartupCommand: ["env", "nemoclaw-start"], + }); + const patch = makePatch(); + const deps = makeDeps(launch, patch, { status: 0, output: "created" }); + + await runSandboxCreateStep( + makeContext({ + agent: { + name: "langchain-deepagents-code", + } as SandboxCreateStepContext["agent"], + prebuild: { + buildCtx: "/tmp/ctx", + buildId: "b1", + dockerDriverGateway: true, + origin: "generated", + }, + }), + deps, + ); + + expect(deps.createDockerGpuPatch).toHaveBeenCalledWith( + expect.objectContaining({ + persistStartupCommand: true, + requiredUlimits: [ + { name: "nproc", soft: 512, hard: 512 }, + { name: "nofile", soft: 65_536, hard: 65_536 }, + ], + }), + ); + }); + it("separates readiness detection from GPU patch polling", async () => { const launch = makeLaunch(); const patch = makePatch(); diff --git a/src/lib/onboard/sandbox-create-step.ts b/src/lib/onboard/sandbox-create-step.ts index be97e726201..04609be02c3 100644 --- a/src/lib/onboard/sandbox-create-step.ts +++ b/src/lib/onboard/sandbox-create-step.ts @@ -12,6 +12,7 @@ import type { createDockerGpuSandboxCreatePatch, DockerGpuSandboxCreatePatch, } from "./docker-gpu-sandbox-create"; +import { resolveDockerStartupCommandPatch } from "./docker-startup-command-agent"; import type { prepareSandboxCreateLaunchWithPrebuild, SandboxCreateLaunchWithPrebuild, @@ -93,10 +94,14 @@ export async function runSandboxCreateStep( openshellArgv: context.openshellArgv, prebuild: context.prebuild, }); + const startupCommandPatch = resolveDockerStartupCommandPatch( + context.agent, + context.prebuild.dockerDriverGateway, + ); const dockerGpuCreatePatch = deps.createDockerGpuPatch({ route: context.useDockerGpuPatch ? "compatibility" : "native", - persistStartupCommand: - context.prebuild.dockerDriverGateway === true && context.agent?.name === "hermes", + persistStartupCommand: startupCommandPatch.persistStartupCommand, + requiredUlimits: startupCommandPatch.requiredUlimits, sandboxName: context.sandboxName, gpuDevice: context.gpuDevice, openshellSandboxCommand: sandboxStartupCommand, diff --git a/src/lib/onboard/sandbox-gpu-create-flow.test.ts b/src/lib/onboard/sandbox-gpu-create-flow.test.ts index dc0a63b01d7..f11e1e21142 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.test.ts @@ -265,7 +265,7 @@ describe("runSandboxGpuCreateFlow proof authorization", () => { }); describe("runSandboxGpuCreateFlow native failure and readiness", () => { - it("persists the Hermes startup command on the no-GPU Docker route", async () => { + it("threads restart-safe startup resource limits on the no-GPU Docker route", async () => { const input = createInput(); input.sandboxGpuConfig = { ...input.sandboxGpuConfig, @@ -276,13 +276,21 @@ describe("runSandboxGpuCreateFlow native failure and readiness", () => { input.initialGpuRoute = "none"; input.createArgv = ["openshell", "sandbox", "create"]; input.persistStartupCommand = true; + input.requiredUlimits = [ + { name: "nproc", soft: 512, hard: 512 }, + { name: "nofile", soft: 65_536, hard: 65_536 }, + ]; await expect(runSandboxGpuCreateFlow(input, createDeps())).resolves.toMatchObject({ route: "none", }); expect(mocks.createDockerGpuSandboxCreatePatch).toHaveBeenCalledWith( - expect.objectContaining({ route: "none", persistStartupCommand: true }), + expect.objectContaining({ + route: "none", + persistStartupCommand: true, + requiredUlimits: input.requiredUlimits, + }), ); }); @@ -299,6 +307,27 @@ describe("runSandboxGpuCreateFlow native failure and readiness", () => { ); }); + it("applies exact required limits while preserving the native GPU route", async () => { + const input = createInput(); + input.persistStartupCommand = true; + input.requiredUlimits = [ + { name: "nproc", soft: 512, hard: 512 }, + { name: "nofile", soft: 65_536, hard: 65_536 }, + ]; + + await expect(runSandboxGpuCreateFlow(input, createDeps())).resolves.toMatchObject({ + route: "native", + }); + + expect(mocks.createDockerGpuSandboxCreatePatch).toHaveBeenCalledWith( + expect.objectContaining({ + route: "native", + persistStartupCommand: true, + requiredUlimits: input.requiredUlimits, + }), + ); + }); + it.each([ { failure: "image build", diff --git a/src/lib/onboard/sandbox-gpu-create-flow.ts b/src/lib/onboard/sandbox-gpu-create-flow.ts index 0bcd3c647d0..98574891e0b 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.ts @@ -6,7 +6,7 @@ import { redactFull } from "../security/redact"; import type { SandboxGpuProofResult } from "../state/registry"; import * as dockerGpuLocalInference from "./docker-gpu-local-inference"; import { collectDockerGpuPatchDiagnostics } from "./docker-gpu-patch"; -import type { DockerGpuPatchDeps } from "./docker-gpu-patch-types"; +import type { DockerGpuPatchDeps, DockerUlimit } from "./docker-gpu-patch-types"; import type { SelectedDockerGpuRoute } from "./docker-gpu-route"; import { renderCompatibilityFallbackCreateArgs } from "./docker-gpu-route"; import { adaptDockerGpuRouteForPatch } from "./docker-gpu-route-patch-adapter"; @@ -18,6 +18,8 @@ import type { SandboxGpuConfig } from "./sandbox-gpu-mode"; import type { SandboxPrebuildResult } from "./sandbox-prebuild"; import { addTraceEvent } from "./tracing"; +export { resolveDockerStartupCommandPatch } from "./docker-startup-command-agent"; + type RunOpenshell = NonNullable; type RunCaptureOpenshell = NonNullable; type Sleep = NonNullable; @@ -39,6 +41,7 @@ export interface SandboxGpuCreateFlowInput { restoreBackupPath: string | null; terminalAgent: boolean; persistStartupCommand?: boolean; + requiredUlimits?: readonly DockerUlimit[] | null; } export interface SandboxGpuCreateFlowDeps { diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index 0cf98a659ad..726758cc0ac 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -69,14 +69,18 @@ export function createSandboxGpuCreateAttemptRunner( " This compatibility container swap may relax container confinement compared with native injection. The retry is running only because NEMOCLAW_DOCKER_GPU_PATCH=fallback explicitly authorized it.", ); } + const hasRequiredUlimits = (input.requiredUlimits?.length ?? 0) > 0; const dockerGpuCreatePatch = createDockerGpuSandboxCreatePatch({ route, - // Native attachment cannot be reproduced by a startup-only swap. The - // compatibility route owns its GPU envelope; no-GPU can persist startup. - persistStartupCommand: input.persistStartupCommand === true && route !== "native", + // The startup clone preserves native CDI devices, so DCode can apply its + // exact required limits without replacing the native GPU envelope. + // Other native routes are not swapped solely to persist a command. + persistStartupCommand: + input.persistStartupCommand === true && (route !== "native" || hasRequiredUlimits), sandboxName: input.sandboxName, gpuDevice: input.sandboxGpuConfig.sandboxGpuDevice, openshellSandboxCommand: input.sandboxStartupCommand, + requiredUlimits: input.requiredUlimits, timeoutSecs: input.sandboxReadyTimeoutSecs, backend: input.sandboxGpuConfig.hostGpuPlatform === "jetson" ? "jetson" : "generic", deps, diff --git a/test/onboard-prepared-build-context.test.ts b/test/onboard-prepared-build-context.test.ts index ba3593bfddb..0d91d612ae4 100644 --- a/test/onboard-prepared-build-context.test.ts +++ b/test/onboard-prepared-build-context.test.ts @@ -64,6 +64,9 @@ function runPreparedContextScenario(scenario: PreparedContextScenario): Prepared const imageTagPath = JSON.stringify( path.join(repoRoot, "src", "lib", "domain", "sandbox", "image-tag.ts"), ); + const dockerGpuSandboxCreatePath = JSON.stringify( + path.join(repoRoot, "src", "lib", "onboard", "docker-gpu-sandbox-create.ts"), + ); const script = String.raw` const fs = require("node:fs"); @@ -77,6 +80,7 @@ const buildContextStage = require(${buildContextStagePath}); const dockerfilePatchFlow = require(${dockerfilePatchFlowPath}); const sandboxCreatePlanMaterialization = require(${sandboxCreatePlanPath}); const imageTag = require(${imageTagPath}); +const dockerGpuSandboxCreate = require(${dockerGpuSandboxCreatePath}); const { loadAgent } = require(${agentDefsPath}); const scenario = ${JSON.stringify(scenario)}; @@ -91,6 +95,17 @@ let cleanupCalls = 0; let patchCalls = 0; let stageCalls = 0; +dockerGpuSandboxCreate.createDockerGpuSandboxCreatePatch = () => ({ + maybeApplyDuringCreate: () => {}, + createFailureMessage: () => null, + exitOnPatchError: () => {}, + ensureApplied: () => {}, + waitForSupervisorReconnectIfNeeded: () => {}, + selectedMode: () => null, + printReadinessFailureIfEnabled: () => {}, + verifyGpuOrExit: (verify) => verify(sandboxName), +}); + buildContextStage.stageCreateSandboxBuildContext = () => { stageCalls += 1; throw new Error("prepared context was unexpectedly restaged"); diff --git a/test/onboard-terminal-dashboard.test.ts b/test/onboard-terminal-dashboard.test.ts index 54b58375205..e45a9b5b17e 100644 --- a/test/onboard-terminal-dashboard.test.ts +++ b/test/onboard-terminal-dashboard.test.ts @@ -38,6 +38,9 @@ function runTerminalDashboardScenario(scenario: "create" | "reuse") { const registryPath = JSON.stringify(path.join(repoRoot, "src", "lib", "state", "registry.ts")); const agentDefsPath = JSON.stringify(path.join(repoRoot, "src", "lib", "agent", "defs.ts")); const agentOnboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "agent", "onboard.ts")); + const dockerGpuSandboxCreatePath = JSON.stringify( + path.join(repoRoot, "src", "lib", "onboard", "docker-gpu-sandbox-create.ts"), + ); fs.mkdirSync(fakeBin, { recursive: true }); writeExecutable(path.join(fakeBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n"); @@ -50,6 +53,7 @@ const runner = require(${runnerPath}); const registry = require(${registryPath}); const agentDefs = require(${agentDefsPath}); const agentOnboard = require(${agentOnboardPath}); +const dockerGpuSandboxCreate = require(${dockerGpuSandboxCreatePath}); const childProcess = require("node:child_process"); const { EventEmitter } = require("node:events"); const scenario = ${JSON.stringify(scenario)}; @@ -60,6 +64,17 @@ const updateCalls = []; const keepAlive = setInterval(() => {}, 1000); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +dockerGpuSandboxCreate.createDockerGpuSandboxCreatePatch = () => ({ + maybeApplyDuringCreate: () => {}, + createFailureMessage: () => null, + exitOnPatchError: () => {}, + ensureApplied: () => {}, + waitForSupervisorReconnectIfNeeded: () => {}, + selectedMode: () => null, + printReadinessFailureIfEnabled: () => {}, + verifyGpuOrExit: (verify) => verify(sandboxName), +}); + agentOnboard.createAgentSandbox = () => { const buildCtx = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-terminal-agent-")); const stagedDockerfile = path.join(buildCtx, "Dockerfile");