diff --git a/src/lib/actions/sandbox/snapshot-auto-create-failure.test.ts b/src/lib/actions/sandbox/snapshot-auto-create-failure.test.ts index 65612e0ae3d..9dc200a96e1 100644 --- a/src/lib/actions/sandbox/snapshot-auto-create-failure.test.ts +++ b/src/lib/actions/sandbox/snapshot-auto-create-failure.test.ts @@ -28,13 +28,22 @@ const streamSandboxCreateMock = vi.fn(async () forcedReady: false, })); -vi.mock("../../adapters/docker", () => ({ dockerCapture: vi.fn(() => "") })); +vi.mock("../../adapters/docker", () => ({ + dockerCapture: vi.fn(() => ""), + dockerForceRm: vi.fn(), + dockerRunDetached: vi.fn(), +})); vi.mock("../../adapters/openshell/runtime", () => ({ captureOpenshell: captureOpenshellMock, getOpenshellBinary: vi.fn(() => "openshell"), runOpenshell: vi.fn(() => ({ status: 0, output: "" })), })); -vi.mock("../../credentials/store", () => ({ prompt: vi.fn() })); +vi.mock("../../credentials/store", () => ({ + deleteCredential: vi.fn(), + getCredential: vi.fn(() => null), + prompt: vi.fn(), + saveCredential: vi.fn(), +})); vi.mock("../../domain/sandbox/destroy", () => ({ getSandboxDeleteOutcome: vi.fn(() => ({ alreadyGone: false, gatewayUnreachable: false })), })); @@ -50,6 +59,11 @@ vi.mock("../../inference/nim", () => ({ stopNimContainerByName: vi.fn(), })); vi.mock("../../messaging/channels", () => ({ + BUILT_IN_CHANNEL_MANIFESTS: [], + getMessagingConfigEnvAliases: vi.fn(() => ({})), + getMessagingCredentialEnvKeysByChannel: vi.fn(() => ({})), + getMessagingProviderSuffixesByChannel: vi.fn(() => ({})), + listBuiltInMessagingChannelManifests: vi.fn(() => []), listMessagingProviderSuffixes: vi.fn(() => []), listMessagingCredentialMetadata: vi.fn(() => []), })); diff --git a/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts b/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts index 66819eb0951..0d99c458141 100644 --- a/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts +++ b/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts @@ -213,7 +213,9 @@ export { lifecycleMock, shieldsMock }; vi.mock("../../adapters/docker", () => ({ dockerCapture: vi.fn(() => ""), + dockerForceRm: vi.fn(), dockerInspect: dockerInspectMock, + dockerRunDetached: vi.fn(), })); vi.mock("../../agent/defs", () => ({ @@ -227,7 +229,10 @@ vi.mock("../../adapters/openshell/runtime", () => ({ })); vi.mock("../../credentials/store", () => ({ + deleteCredential: vi.fn(), + getCredential: vi.fn(() => null), prompt: vi.fn(), + saveCredential: vi.fn(), })); vi.mock("../../domain/sandbox/destroy", () => ({ diff --git a/src/lib/actions/sandbox/snapshot.test.ts b/src/lib/actions/sandbox/snapshot.test.ts index 977a2f49822..19cf6428de0 100644 --- a/src/lib/actions/sandbox/snapshot.test.ts +++ b/src/lib/actions/sandbox/snapshot.test.ts @@ -144,19 +144,21 @@ const latestBackupFixture = { vi.mock("../../adapters/docker", () => ({ dockerCapture: vi.fn(() => ""), + dockerForceRm: vi.fn(), dockerInspect: dockerInspectMock, + dockerRunDetached: vi.fn(), })); - vi.mock("../../adapters/openshell/runtime", () => ({ captureOpenshell: captureOpenshellMock, getOpenshellBinary: vi.fn(() => "openshell"), runOpenshell: runOpenshellMock, })); - vi.mock("../../credentials/store", () => ({ + deleteCredential: vi.fn(), + getCredential: vi.fn(() => null), prompt: vi.fn(), + saveCredential: vi.fn(), })); - vi.mock("../../domain/sandbox/destroy", () => ({ getSandboxDeleteOutcome: vi.fn(() => ({ alreadyGone: false, gatewayUnreachable: false })), })); @@ -165,7 +167,6 @@ vi.mock("../../inference/nim", () => ({ stopNimContainer: vi.fn(), stopNimContainerByName: vi.fn(), })); - vi.mock("../../policy", async (importOriginal) => ({ ...(await importOriginal()), applyPreset: applyPresetMock, @@ -176,7 +177,6 @@ vi.mock("../../policy", async (importOriginal) => ({ removePreset: removePresetMock, resolveAgentBaselinePolicy: resolveTestAgentBaselinePolicy, })); - vi.mock("../../runner", () => ({ ROOT: "/repo", run: vi.fn(() => ({ status: 0 })), diff --git a/src/lib/adapters/openshell/sandbox-identity.test.ts b/src/lib/adapters/openshell/sandbox-identity.test.ts new file mode 100644 index 00000000000..422bf71ccd8 --- /dev/null +++ b/src/lib/adapters/openshell/sandbox-identity.test.ts @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { parseOpenShellSandboxId } from "./sandbox-identity"; + +describe("OpenShell sandbox identity parsing", () => { + it("accepts one exact durable ID with optional terminal color", () => { + expect(parseOpenShellSandboxId("Name: alpha\nID: sandbox-alpha\n")).toBe("sandbox-alpha"); + expect(parseOpenShellSandboxId("\u001b[32mId: sandbox.alpha_2\u001b[0m\n")).toBe( + "sandbox.alpha_2", + ); + }); + + it("rejects ambiguous or non-canonical IDs", () => { + expect(parseOpenShellSandboxId("ID: first\nID: second\n")).toBeNull(); + expect(parseOpenShellSandboxId("ID: sandbox/alpha\n")).toBeNull(); + expect(parseOpenShellSandboxId("id: sandbox-alpha\n")).toBeNull(); + }); +}); diff --git a/src/lib/adapters/openshell/sandbox-identity.ts b/src/lib/adapters/openshell/sandbox-identity.ts new file mode 100644 index 00000000000..1820a8f8f7d --- /dev/null +++ b/src/lib/adapters/openshell/sandbox-identity.ts @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +const ANSI_RE = /\x1b\[[0-9;]*m/gu; +const SANDBOX_ID_RE = /^[A-Za-z0-9._-]+$/u; + +export function parseOpenShellSandboxId(output: string): string | null { + const matches = [ + ...String(output) + .replace(ANSI_RE, "") + .matchAll(/^\s*(?:Id|ID):\s*(\S+)\s*$/gm), + ].map((match) => match[1] ?? ""); + return matches.length === 1 && SANDBOX_ID_RE.test(matches[0] as string) + ? (matches[0] as string) + : null; +} diff --git a/src/lib/onboard/docker-gpu-patch-clone.test.ts b/src/lib/onboard/docker-gpu-patch-clone.test.ts index 8ad39b30162..cbdbfb1e252 100644 --- a/src/lib/onboard/docker-gpu-patch-clone.test.ts +++ b/src/lib/onboard/docker-gpu-patch-clone.test.ts @@ -122,6 +122,41 @@ describe("Docker GPU clone envelope", () => { expect(args).not.toContain("nofile=1024:1024"); }); + it("uses exact managed-bootstrap container, entrypoint, and command overrides", () => { + const args = buildDockerGpuCloneRunArgs( + inspectFixture(), + buildDockerGpuMode("startup-command"), + { + containerName: "openshell-alpha-bootstrap-stage", + containerEntrypoint: "/usr/local/bin/nemoclaw-managed-bootstrap", + containerCommand: ["--request", "/run/nemoclaw/bootstrap-request.json"], + }, + ); + + expect(args.slice(0, 2)).toEqual(["--name", "openshell-alpha-bootstrap-stage"]); + expect(args).toEqual( + expect.arrayContaining(["--entrypoint", "/usr/local/bin/nemoclaw-managed-bootstrap"]), + ); + expect(args.slice(args.indexOf("openshell/sandbox:abc"))).toEqual([ + "openshell/sandbox:abc", + "--request", + "/run/nemoclaw/bootstrap-request.json", + ]); + }); + + it.each([ + "", + "-starts-with-dash", + "contains/slash", + "a".repeat(254), + ])("rejects invalid managed-bootstrap container name %j", (containerName) => { + expect(() => + buildDockerGpuCloneRunArgs(inspectFixture(), buildDockerGpuMode("startup-command"), { + containerName, + }), + ).toThrow("Docker clone container name is invalid."); + }); + it("adds SYS_PTRACE to the GPU clone when the baseline container lacks it", () => { const inspect = inspectFixture(); inspect.HostConfig!.CapAdd = ["SYS_ADMIN", "NET_ADMIN"]; diff --git a/src/lib/onboard/docker-gpu-patch-clone.ts b/src/lib/onboard/docker-gpu-patch-clone.ts index c92767adad0..828ce0c540b 100644 --- a/src/lib/onboard/docker-gpu-patch-clone.ts +++ b/src/lib/onboard/docker-gpu-patch-clone.ts @@ -329,7 +329,15 @@ export function buildDockerGpuCloneRunArgs( const image = String(options.image || config.Image || "").trim(); if (!image) throw new Error("Docker inspect output did not include Config.Image."); - const args: string[] = ["--name", dockerContainerName(inspect), ...mode.args]; + const containerName = String(options.containerName ?? dockerContainerName(inspect)).trim(); + if ( + containerName.length === 0 || + containerName.length > 253 || + !/^[A-Za-z0-9][A-Za-z0-9_.-]*$/u.test(containerName) + ) { + throw new Error("Docker clone container name is invalid."); + } + const args: string[] = ["--name", containerName, ...mode.args]; const gpuAugment = mode.kind !== "startup-command"; // Startup-command recreation must retain OpenShell's native CDI attachment. @@ -435,8 +443,17 @@ export function buildDockerGpuCloneRunArgs( if (host.Init) args.push("--init"); const entrypoint = stringArray(config.Entrypoint); - if (entrypoint.length > 0) args.push("--entrypoint", entrypoint[0]); - const commandArgs = sandboxCommand ? [] : [...entrypoint.slice(1), ...stringArray(config.Cmd)]; + const replacementEntrypoint = String(options.containerEntrypoint ?? "").trim(); + if (replacementEntrypoint) { + args.push("--entrypoint", replacementEntrypoint); + } else if (entrypoint.length > 0) { + args.push("--entrypoint", entrypoint[0]); + } + const commandArgs = options.containerCommand + ? [...options.containerCommand] + : sandboxCommand + ? [] + : [...entrypoint.slice(1), ...stringArray(config.Cmd)]; args.push(image, ...commandArgs); return args; } diff --git a/src/lib/onboard/docker-gpu-patch-types.ts b/src/lib/onboard/docker-gpu-patch-types.ts index 32be0afe46f..d72046bd320 100644 --- a/src/lib/onboard/docker-gpu-patch-types.ts +++ b/src/lib/onboard/docker-gpu-patch-types.ts @@ -112,6 +112,14 @@ export type DockerGpuCloneRunOptions = { sandboxFallbackDns?: string | null; openshellSandboxCommand?: readonly string[] | null; requiredUlimits?: readonly DockerUlimit[] | null; + /** + * Exact replacement process boundary used only by dormant managed bootstrap. + * Ordinary recreation leaves both fields unset. + */ + containerEntrypoint?: string | null; + containerCommand?: readonly string[] | null; + /** Stopped staging name used before exact-name cutover. */ + containerName?: string | null; /** * Extra supplementary group IDs to add to the recreated container via * `--group-add`. On Jetson these are the host group(s) owning the Tegra GPU @@ -190,6 +198,14 @@ export type DockerContainerInspect = { Hostname?: string; Tty?: boolean; OpenStdin?: boolean; + StopTimeout?: number | null; + Volumes?: Record | null; + } | null; + State?: { + Running?: boolean; + Paused?: boolean; + Restarting?: boolean; + Dead?: boolean; } | null; HostConfig?: { Binds?: string[] | null; @@ -244,6 +260,7 @@ export type DockerContainerInspect = { DeviceIDs?: string[] | null; }> | null; ShmSize?: number; + ReadonlyRootfs?: boolean; ReadonlyPaths?: string[] | null; MaskedPaths?: string[] | null; } | null; diff --git a/src/lib/onboard/managed-bootstrap/README.md b/src/lib/onboard/managed-bootstrap/README.md index adde636fa56..a102dc031e7 100644 --- a/src/lib/onboard/managed-bootstrap/README.md +++ b/src/lib/onboard/managed-bootstrap/README.md @@ -3,9 +3,9 @@ # Managed bootstrap protocol -This directory defines a dormant, driver-neutral transaction contract. It does -not register a runtime provider or change sandbox creation, onboarding, -snapshot, clone, or restore behavior. +This directory defines a dormant, driver-neutral transaction contract and its +first driver adapter. It does not register a runtime provider, activate managed +bootstrap, or change the current user-visible lifecycle paths. The protocol binds one random bootstrap identity to: @@ -71,6 +71,22 @@ journal and a canonical launch-spec normalizer. Each surface is independently validated and remains dormant: no registered runtime provider imports either module, and neither changes sandbox creation or lifecycle behavior. +The Docker adapter creates and validates a stopped replacement under an +identity-derived staging name while the original remains running. It stages the +0400 envelope and returns exact cleanup authority without quiescing, renaming, +or otherwise mutating the original. Only after the coordinator durably records +that complete prepared authority may activation journal both full runtime IDs, +all three names, both launch-spec hashes, image identity, profile fingerprint, +and sandbox ID and then enter the destructive cutover. Post-cutover rollback +publishes `rollback-authorized` before exact replacement deletion; pre-cutover +staged cleanup removes only the exact prepared replacement without that journal +transition. Commit publishes `shared-state-committed` before exact backup +deletion. Cleanup is bound to full runtime IDs. Mutable OpenShell names are read +only to detect ownership reuse, and unsafe name-only deletion returns a typed +retention error. The dormant adapter assumes the protocol's single coordinator; +multi-process lease/arbitration remains an explicit production-activation gate. +Activation must also inject the selected gateway's canonical state root. + ## Architectural disposition The coordinator deliberately lands as a dormant trust-boundary slice before a @@ -86,22 +102,22 @@ This is executable, bounded groundwork rather than an untested placeholder. failure rollback for OpenClaw, Hermes, and LangChain Deep Agents Code through an MXC-named fake driver. `runtime-provider-source-shape.test.ts` separately inventories the protocol, provider, and image-packaging surfaces and proves that -production activation cannot import or package the protocol yet. The later -activation slice must add a registered-provider contract test for the same -transaction before removing those dormancy assertions. +production activation does not import or install the protocol into a runtime +image yet. The later activation slice must add a registered-provider contract +test for the same transaction before removing those dormancy assertions. The native entrypoint source is intentionally not compiled into production -artifacts, and neither source is packaged or selected yet. No production -TypeScript module imports this protocol. The current image definitions do not -package `nemoclaw-managed-startup-hold` or -`managed-startup-image-runtime.cjs`. A later -provider integration must compile and verify the freestanding entrypoint -natively for amd64 and arm64 in every agent image. It must add those -prerequisites together with their image-runtime bootstrap modes, implement -driver-specific prepare, durable-record, activate, exact cleanup, and rollback, -and only then wire the coordinator into create. The same contract is exercised -for OpenClaw, Hermes, and Deep Agents Code without a provider-specific central -switch. The remaining integration and qualification work is tracked in -[epic #7744](https://github.com/NVIDIA/NemoClaw/issues/7744) and its linked -implementation stack. Until that complete boundary lands, every registered -runtime provider keeps its bootstrap surface unsupported. +artifacts, and neither image-owned source is installed or selected in a runtime +image yet. No production activation or provider module outside this dormant +directory imports the protocol or Docker adapter. The current image definitions +do not package `nemoclaw-managed-startup-hold`, +`managed-startup-image-runtime.cjs`, or the shared-state bootstrap modes consumed +by the adapter. A later provider integration must compile and verify the +freestanding entrypoint natively for amd64 and arm64 in every agent image. It +must add those prerequisites together with their image-runtime bootstrap modes +and wire the coordinator and Docker adapter into create as one boundary. The +same contract is exercised for OpenClaw, Hermes, and Deep Agents Code without a +provider-specific central switch. The remaining integration and qualification +work is tracked in [epic #7744](https://github.com/NVIDIA/NemoClaw/issues/7744) +and its linked implementation stack. Until that complete boundary lands, every +registered runtime provider keeps its bootstrap surface unsupported. diff --git a/src/lib/onboard/managed-bootstrap/adapter.test.ts b/src/lib/onboard/managed-bootstrap/adapter.test.ts index 7941dc8686b..9968171289d 100644 --- a/src/lib/onboard/managed-bootstrap/adapter.test.ts +++ b/src/lib/onboard/managed-bootstrap/adapter.test.ts @@ -909,13 +909,16 @@ describe("managed bootstrap adapter contract", () => { }); it.each([ + "BASHOPTS=extdebug", "BASH_ENV=/sandbox/attacker", "ENV=/sandbox/attacker", - "LD_PRELOAD=/sandbox/attacker.so", "LD_AUDIT=/sandbox/attacker.so", "LD_LIBRARY_PATH=/sandbox/lib", - "SHELLOPTS=xtrace", + "LD_PRELOAD=/sandbox/attacker.so", + "NODE_OPTIONS=--require=/sandbox/attacker.cjs", + "NODE_PATH=/sandbox/attacker-modules", "PS4=$(touch /sandbox/bypass)", + "SHELLOPTS=xtrace", "BASH_FUNC_attacker%%=() { touch /sandbox/bypass; }", ])("rejects a process-control assignment before rendering the held command: %s", (assignment) => { const request = requestFor("hermes"); diff --git a/src/lib/onboard/managed-bootstrap/adapter.ts b/src/lib/onboard/managed-bootstrap/adapter.ts index c29fa7a01d7..415fced86cd 100644 --- a/src/lib/onboard/managed-bootstrap/adapter.ts +++ b/src/lib/onboard/managed-bootstrap/adapter.ts @@ -25,6 +25,8 @@ const PROCESS_INJECTION_ENV_KEYS = new Set([ "LD_AUDIT", "LD_LIBRARY_PATH", "LD_PRELOAD", + "NODE_OPTIONS", + "NODE_PATH", "PS4", "SHELLOPTS", ]); @@ -931,7 +933,7 @@ function normalizePreparedReplacement( }); } -function createPreparedAuthority( +export function createManagedBootstrapPreparedAuthority( transaction: ManagedBootstrapPreparedTransaction, ): ManagedBootstrapPreparedAuthority { const { handle, snapshot, prepared } = transaction; @@ -1358,7 +1360,7 @@ export async function activateManagedBootstrapSequence( let durablePreparation: ManagedBootstrapDurablePreparationReceipt | null = null; let replacement: ManagedBootstrapReplacementHandle | null = null; try { - const authority = createPreparedAuthority(input.transaction); + const authority = createManagedBootstrapPreparedAuthority(input.transaction); durablePreparation = normalizeDurablePreparationReceipt( await input.authorityStore.recordPreparedAuthority(authority), authority, diff --git a/src/lib/onboard/managed-bootstrap/docker-shared-state.test.ts b/src/lib/onboard/managed-bootstrap/docker-shared-state.test.ts new file mode 100644 index 00000000000..ae965865eb7 --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker-shared-state.test.ts @@ -0,0 +1,129 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import type { DockerGpuPatchDeps } from "../docker-gpu-patch-types"; +import { + clearDockerManagedStartupSharedStateCommitReceipt, + type DockerManagedBootstrapSharedStateTransaction, + finalizeDockerManagedStartupSharedState, +} from "./docker-shared-state"; +import { authority, fixture, IDENTITY, NEW_ID } from "./docker-test-fixture"; + +const CLEAN_NODE_COMMAND = [ + "/usr/bin/env", + "-i", + "HOME=/root", + "LANG=C.UTF-8", + "LC_ALL=C.UTF-8", + "NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=1", + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "/usr/local/bin/node", +] as const; +const PRE_ENTRYPOINT_ENV_OVERRIDES = [ + "--env", + "LD_AUDIT=", + "--env", + "LD_LIBRARY_PATH=", + "--env", + "LD_PRELOAD=", + "--env", + "NODE_OPTIONS=", + "--env", + "NODE_PATH=", +] as const; + +function sharedStateTransaction(): DockerManagedBootstrapSharedStateTransaction { + const { handle } = authority("hermes"); + return { + agent: "hermes", + bootstrapIdentity: IDENTITY, + containerId: NEW_ID, + image: `sha256:${"4".repeat(64)}`, + profileFingerprint: handle.plan.profile.fingerprint, + }; +} + +function nodeHelperCalls(deps: DockerGpuPatchDeps): readonly (readonly string[])[] { + return vi + .mocked(deps.dockerRun!) + .mock.calls.map(([args]) => args) + .filter((args) => args[0] === "run" || args[0] === "exec") + .filter((args) => args.includes("/usr/local/bin/node")); +} + +function expectPreEntrypointEnvironmentNeutralized(args: readonly string[]): void { + expect(args).toEqual(expect.arrayContaining([...PRE_ENTRYPOINT_ENV_OVERRIDES])); + expect(args).not.toContain("BASH_FUNC_*"); +} + +function expectCleanRunNodeHelper(args: readonly string[]): void { + expectPreEntrypointEnvironmentNeutralized(args); + const nodeIndex = args.indexOf("/usr/local/bin/node"); + expect(nodeIndex).toBeGreaterThan(0); + const entrypointIndex = args.indexOf("--entrypoint"); + expect(args[entrypointIndex + 1]).toBe(CLEAN_NODE_COMMAND[0]); + expect(args.slice(nodeIndex - CLEAN_NODE_COMMAND.length + 2, nodeIndex + 1)).toEqual( + CLEAN_NODE_COMMAND.slice(1), + ); +} + +function expectCleanExecNodeHelper(args: readonly string[]): void { + expectPreEntrypointEnvironmentNeutralized(args); + const nodeIndex = args.indexOf("/usr/local/bin/node"); + expect(nodeIndex).toBeGreaterThan(0); + expect(args.slice(nodeIndex - CLEAN_NODE_COMMAND.length + 1, nodeIndex + 1)).toEqual( + CLEAN_NODE_COMMAND, + ); +} + +describe("Docker managed-bootstrap shared-state helper environment", () => { + it("clears arbitrary image and container environment before every verification and commit helper", () => { + const fake = fixture({ sharedState: "pending" }); + const outcome = finalizeDockerManagedStartupSharedState( + { + transaction: sharedStateTransaction(), + retainContainerAfterRollback: true, + supervisorReady: true, + }, + fake.deps, + ); + + expect(outcome).toEqual({ supervisorReady: true, failure: null }); + const helpers = nodeHelperCalls(fake.deps); + expect(helpers.some((args) => args.includes("--shared-state-transaction-status"))).toBe(true); + expect(helpers.some((args) => args.includes("--commit-shared-state-transaction"))).toBe(true); + expect(helpers).not.toHaveLength(0); + helpers.filter((args) => args[0] === "run").forEach(expectCleanRunNodeHelper); + helpers.filter((args) => args[0] === "exec").forEach(expectCleanExecNodeHelper); + }); + + it("clears arbitrary image environment before the immutable rollback helper", () => { + const fake = fixture({ sharedState: "pending" }); + const outcome = finalizeDockerManagedStartupSharedState( + { + transaction: sharedStateTransaction(), + retainContainerAfterRollback: true, + supervisorReady: false, + }, + fake.deps, + ); + + expect(outcome).toEqual({ supervisorReady: false, failure: null }); + const helpers = nodeHelperCalls(fake.deps); + expect(helpers).toHaveLength(1); + expect(helpers[0]).toContain("--rollback-shared-state-transaction"); + expectCleanRunNodeHelper(helpers[0]!); + }); + + it("clears arbitrary container environment before the durable receipt-clear helper", () => { + const fake = fixture({ sharedState: "committed" }); + clearDockerManagedStartupSharedStateCommitReceipt(sharedStateTransaction(), fake.deps); + + const helpers = nodeHelperCalls(fake.deps); + expect(helpers).toHaveLength(1); + expect(helpers[0]).toContain("--clear-shared-state-commit-receipt"); + expectCleanExecNodeHelper(helpers[0]!); + }); +}); diff --git a/src/lib/onboard/managed-bootstrap/docker-shared-state.ts b/src/lib/onboard/managed-bootstrap/docker-shared-state.ts new file mode 100644 index 00000000000..415fe56fb82 --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker-shared-state.ts @@ -0,0 +1,640 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import path from "node:path"; + +import { + dockerRm as defaultDockerRm, + dockerStop as defaultDockerStop, +} from "../../adapters/docker/container"; +import { dockerRun as defaultDockerRun } from "../../adapters/docker/run"; +import { hasZeroDockerExitStatus } from "../docker-command-result"; +import { + DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + DOCKER_GPU_PATCH_TIMEOUT_MS, +} from "../docker-gpu-patch-constants"; +import type { DockerGpuPatchDeps, DockerGpuPatchResult } from "../docker-gpu-patch-types"; +import { MANAGED_STARTUP_RUNTIME_EXECUTABLE } from "../managed-startup/image-runtime"; +import { MANAGED_STARTUP_AGENTS, type ManagedStartupAgent } from "../managed-startup/profile"; +import { + MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY, + MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, +} from "../managed-startup/shared-state-transaction"; +import { isImmutableDockerImageId } from "../openshell-docker-sandbox-containers"; +import { cleanupTempDir, secureTempFile } from "../temp-files"; + +const RECEIPT_TEMP_PREFIX = "nemoclaw-managed-startup-receipt"; +const MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY = + "/var/lib/nemoclaw/managed-startup-shared-state-commit-v1"; +const FULL_CONTAINER_ID_RE = /^[a-f0-9]{64}$/u; +const DURABLE_IDENTITY_RE = /^[a-f0-9]{64}$/u; +const DOCKER_MUTATION_OPTIONS = { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, +} as const; +/** + * The dynamic loader consumes LD_* before `/usr/bin/env -i` can clear the + * inherited image or container environment. NODE_* is also cleared at this + * boundary as defense in depth; the clean-Node argv below removes every other + * variable before Node starts. + */ +const NEUTRALIZED_PRE_ENTRYPOINT_ENV = [ + "--env", + "LD_AUDIT=", + "--env", + "LD_LIBRARY_PATH=", + "--env", + "LD_PRELOAD=", + "--env", + "NODE_OPTIONS=", + "--env", + "NODE_PATH=", +] as const; +const CLEAN_NODE_ENTRYPOINT = "/usr/bin/env"; +const CLEAN_NODE_ARGV = [ + "-i", + "HOME=/root", + "LANG=C.UTF-8", + "LC_ALL=C.UTF-8", + "NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=1", + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "/usr/local/bin/node", +] as const; + +export interface DockerManagedBootstrapSharedStateTransaction { + readonly agent: ManagedStartupAgent; + readonly bootstrapIdentity: string; + readonly containerId: string; + readonly image: string; + readonly profileFingerprint: string; +} + +export interface DockerManagedStartupSharedStateOutcome { + /** + * True only when the new supervisor is still eligible for successful + * container cutover. A commit failure forces shared-state rollback first. + */ + readonly supervisorReady: boolean; + /** Original commit failure after a successful shared-state rollback. */ + readonly failure: Error | null; +} + +export class DockerManagedStartupSharedStateCommitIndeterminateError extends Error { + constructor(detail: string, options?: ErrorOptions) { + super( + `Managed-startup shared-state commit may have completed, but immutable status is unavailable: ${detail}`, + options, + ); + this.name = "DockerManagedStartupSharedStateCommitIndeterminateError"; + } +} + +export function probeDockerManagedStartupSharedState( + input: { + readonly transaction: DockerManagedBootstrapSharedStateTransaction; + readonly profileFingerprint: string; + }, + deps: DockerGpuPatchDeps = {}, +): "committed" | "none" | "pending" { + const transaction = input.transaction; + assertValidManagedStartupTransaction(transaction); + if (input.profileFingerprint !== transaction.profileFingerprint) { + throw new Error("Managed bootstrap shared-state status fingerprint does not match."); + } + const committedReceiptPath = copyManagedStartupReceiptAt( + transaction, + MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY, + deps, + true, + ); + if (committedReceiptPath) { + let verified = false; + try { + verifyCopiedManagedStartupReceipt( + transaction, + input.profileFingerprint, + committedReceiptPath, + MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY, + "committed", + deps, + ); + verified = true; + return "committed"; + } finally { + if (verified) cleanupReceiptBestEffort(committedReceiptPath); + } + } + const receiptPath = copyManagedStartupReceipt(transaction, deps, true); + if (!receiptPath) return "none"; + let verified = false; + try { + verifyCopiedManagedStartupReceipt( + transaction, + input.profileFingerprint, + receiptPath, + MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, + "pending", + deps, + ); + verified = true; + return "pending"; + } finally { + if (verified) cleanupReceiptBestEffort(receiptPath); + } +} + +function verifyCopiedManagedStartupReceipt( + transaction: DockerManagedBootstrapSharedStateTransaction, + profileFingerprint: string, + receiptPath: string, + receiptDirectory: string, + expectedStatus: "committed" | "pending", + deps: DockerGpuPatchDeps, +): void { + if (!transaction.bootstrapIdentity || !/^[a-f0-9]{64}$/u.test(profileFingerprint)) { + throw new Error("Managed bootstrap copied-receipt identity is incomplete."); + } + const dockerRun = deps.dockerRun ?? defaultDockerRun; + const result = dockerRun( + [ + "run", + "--rm", + "--pull", + "never", + "--network", + "none", + "--read-only", + "--user", + "0:0", + "--security-opt", + "no-new-privileges", + "--cap-drop", + "ALL", + ...NEUTRALIZED_PRE_ENTRYPOINT_ENV, + "--mount", + transactionReceiptMount(receiptPath, receiptDirectory), + "--entrypoint", + CLEAN_NODE_ENTRYPOINT, + transaction.image, + ...CLEAN_NODE_ARGV, + MANAGED_STARTUP_RUNTIME_EXECUTABLE, + "--shared-state-transaction-status", + "--agent", + transaction.agent, + "--profile-fingerprint", + profileFingerprint, + "--bootstrap-identity", + transaction.bootstrapIdentity, + ], + DOCKER_MUTATION_OPTIONS, + ); + if (!hasZeroDockerExitStatus(result)) { + throw new Error( + `Immutable managed-startup helper could not verify shared-state status: ${commandDetail(result)}. ` + + `Protected receipt retained at ${receiptPath}`, + ); + } + if (String(result.stdout ?? "").trim() !== expectedStatus) { + throw new Error( + `Immutable managed-startup helper returned an invalid copied transaction status. Protected receipt retained at ${receiptPath}`, + ); + } +} + +function commandDetail(result: { + readonly stderr?: string | Buffer | null; + readonly stdout?: string | Buffer | null; + readonly error?: Error | null; +}): string { + return `${String(result.stderr ?? "")} ${String(result.stdout ?? "")} ${String( + result.error?.message ?? "", + )}` + .trim() + .slice(-800); +} + +function cleanupReceiptBestEffort(receiptPath: string): void { + try { + cleanupTempDir(receiptPath, RECEIPT_TEMP_PREFIX); + } catch (error) { + console.warn( + ` ⚠ Managed-startup shared state is finalized, but its protected host receipt could not be removed (${receiptPath}): ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } +} + +function assertValidManagedStartupTransaction( + transaction: DockerManagedBootstrapSharedStateTransaction, +): asserts transaction is DockerManagedBootstrapSharedStateTransaction & { + readonly bootstrapIdentity: string; + readonly profileFingerprint: string; +} { + if (!(MANAGED_STARTUP_AGENTS as readonly string[]).includes(transaction.agent)) { + throw new Error("Managed bootstrap shared-state transaction agent is invalid."); + } + if (!FULL_CONTAINER_ID_RE.test(transaction.containerId)) { + throw new Error("Managed bootstrap shared-state transaction container identity is invalid."); + } + if (!isImmutableDockerImageId(transaction.image)) { + throw new Error("Managed bootstrap shared-state transaction image identity is not immutable."); + } + if (!transaction.bootstrapIdentity || !DURABLE_IDENTITY_RE.test(transaction.bootstrapIdentity)) { + throw new Error("Managed bootstrap shared-state transaction identity is missing or invalid."); + } + if ( + !transaction.profileFingerprint || + !DURABLE_IDENTITY_RE.test(transaction.profileFingerprint) + ) { + throw new Error( + "Managed bootstrap shared-state transaction profile fingerprint is missing or invalid.", + ); + } +} + +function transactionCommand( + action: "clear-shared-state-commit-receipt" | "commit" | "rollback", + transaction: DockerManagedBootstrapSharedStateTransaction, +): string[] { + assertValidManagedStartupTransaction(transaction); + return [ + MANAGED_STARTUP_RUNTIME_EXECUTABLE, + action === "clear-shared-state-commit-receipt" + ? "--clear-shared-state-commit-receipt" + : `--${action}-shared-state-transaction`, + "--agent", + transaction.agent, + "--bootstrap-identity", + transaction.bootstrapIdentity, + ]; +} + +export function clearDockerManagedStartupSharedStateCommitReceipt( + transaction: DockerManagedBootstrapSharedStateTransaction, + deps: DockerGpuPatchDeps = {}, +): void { + const dockerRun = deps.dockerRun ?? defaultDockerRun; + assertValidManagedStartupTransaction(transaction); + const command = transactionCommand("clear-shared-state-commit-receipt", transaction); + const cleared = dockerRun( + [ + "exec", + "--user", + "0:0", + "--workdir", + "/", + ...NEUTRALIZED_PRE_ENTRYPOINT_ENV, + transaction.containerId, + CLEAN_NODE_ENTRYPOINT, + ...CLEAN_NODE_ARGV, + ...command, + ], + DOCKER_MUTATION_OPTIONS, + ); + // Accept a lost Docker acknowledgement only when both exact image-owned + // receipt paths are independently proven absent by the immutable helper. + let status: "committed" | "none" | "pending"; + try { + status = probeDockerManagedStartupSharedState( + { + transaction, + profileFingerprint: transaction.profileFingerprint, + }, + deps, + ); + } catch (error) { + throw new DockerManagedStartupSharedStateCommitIndeterminateError( + error instanceof Error ? error.message : String(error), + { cause: error }, + ); + } + if (status === "none") return; + if (!hasZeroDockerExitStatus(cleared)) { + throw new Error( + `Managed-startup durable commit receipt cleanup failed and exact absence was not proven (status=${status}): ${commandDetail(cleared)}`, + ); + } + throw new Error( + `Managed-startup durable commit receipt cleanup returned success, but exact absence was not proven (status=${status}).`, + ); +} + +function commitManagedStartupSharedState( + transaction: DockerManagedBootstrapSharedStateTransaction, + deps: DockerGpuPatchDeps, +): void { + const dockerRun = deps.dockerRun ?? defaultDockerRun; + assertValidManagedStartupTransaction(transaction); + const command = transactionCommand("commit", transaction); + const commit = dockerRun( + [ + "exec", + "--user", + "0:0", + "--workdir", + "/", + ...NEUTRALIZED_PRE_ENTRYPOINT_ENV, + transaction.containerId, + CLEAN_NODE_ENTRYPOINT, + ...CLEAN_NODE_ARGV, + ...command, + ], + DOCKER_MUTATION_OPTIONS, + ); + // The commit helper atomically renames the rollback receipt into a compact + // identity-bound commit receipt before Docker returns. Always probe it + // afterward so a lost daemon acknowledgement is accepted only when durable + // commit state is independently proven. + let status: "committed" | "none" | "pending"; + try { + status = probeDockerManagedStartupSharedState( + { + transaction, + profileFingerprint: transaction.profileFingerprint, + }, + deps, + ); + } catch (error) { + throw new DockerManagedStartupSharedStateCommitIndeterminateError( + error instanceof Error ? error.message : String(error), + { cause: error }, + ); + } + if (status === "committed") return; + if (!hasZeroDockerExitStatus(commit)) { + throw new Error( + `Managed-startup shared-state commit helper failed and durable commit was not proven (status=${status}): ${commandDetail(commit)}`, + ); + } + throw new Error( + `Managed-startup shared-state commit helper returned success, but durable commit was not proven (status=${status}).`, + ); +} + +function quiesceManagedStartupContainer( + transaction: DockerManagedBootstrapSharedStateTransaction, + deps: DockerGpuPatchDeps, +): void { + const dockerStop = deps.dockerStop ?? defaultDockerStop; + const stopped = dockerStop(transaction.containerId, { + ...DOCKER_MUTATION_OPTIONS, + timeout: DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + }); + if (!hasZeroDockerExitStatus(stopped)) { + throw new Error( + `Could not quiesce the failed managed-startup container before shared-state rollback: ${commandDetail(stopped)}`, + ); + } +} + +function isExactMissingReceiptCopy( + transaction: DockerManagedBootstrapSharedStateTransaction, + sourcePath: string, + result: { + readonly stderr?: string | Buffer | null; + readonly stdout?: string | Buffer | null; + readonly error?: Error | null; + }, +): boolean { + const detail = commandDetail(result); + const escapedPath = sourcePath.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); + const escapedContainer = transaction.containerId.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); + return [ + new RegExp( + `^(?:Error response from daemon: )?Could not find the file ${escapedPath} in container ${escapedContainer}$`, + "u", + ), + new RegExp(`^(?:lstat|stat) ${escapedPath}: no such file or directory$`, "u"), + ].some((pattern) => pattern.test(detail)); +} + +function transactionReceiptMount(receiptPath: string, receiptDirectory: string): string { + return `type=bind,src=${receiptPath},dst=${receiptDirectory},readonly`; +} + +function copyManagedStartupReceiptAt( + transaction: DockerManagedBootstrapSharedStateTransaction, + sourcePath: string, + deps: DockerGpuPatchDeps, + allowAbsent = false, +): string | null { + const dockerRun = deps.dockerRun ?? defaultDockerRun; + const tempSeed = secureTempFile(RECEIPT_TEMP_PREFIX); + const receiptPath = path.join(path.dirname(tempSeed), path.basename(sourcePath)); + try { + const copy = dockerRun( + ["cp", "-a", `${transaction.containerId}:${sourcePath}`, receiptPath], + DOCKER_MUTATION_OPTIONS, + ); + if (!hasZeroDockerExitStatus(copy)) { + if (allowAbsent && isExactMissingReceiptCopy(transaction, sourcePath, copy)) { + cleanupReceiptBestEffort(receiptPath); + return null; + } + throw new Error( + `Could not copy the managed-startup rollback receipt from the failed container: ${commandDetail(copy)}`, + ); + } + if (receiptPath.includes(",") || /[\r\n\0]/u.test(receiptPath)) { + throw new Error("Managed-startup rollback receipt path is unsafe for a Docker bind mount"); + } + return receiptPath; + } catch (error) { + cleanupReceiptBestEffort(receiptPath); + throw error; + } +} + +function copyManagedStartupReceipt( + transaction: DockerManagedBootstrapSharedStateTransaction, + deps: DockerGpuPatchDeps, + allowAbsent = false, +): string | null { + return copyManagedStartupReceiptAt( + transaction, + MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, + deps, + allowAbsent, + ); +} + +function rollbackManagedStartupSharedState( + transaction: DockerManagedBootstrapSharedStateTransaction, + receiptPath: string, + deps: DockerGpuPatchDeps, +): void { + const dockerRun = deps.dockerRun ?? defaultDockerRun; + let restored = false; + try { + // The immutable image owns the canonical receipt parser and exact + // post-restore verification. Keep the host receipt opaque so that its + // validator cannot drift from the image contract. After dropping every + // capability, rollback retains only CHOWN for original ownership, + // DAC_OVERRIDE for owner-restricted state, and FOWNER for original modes. + const helper = dockerRun( + [ + "run", + "--rm", + "--pull", + "never", + "--network", + "none", + "--read-only", + "--user", + "0:0", + "--security-opt", + "no-new-privileges", + "--cap-drop", + "ALL", + "--cap-add", + "CHOWN", + "--cap-add", + "DAC_OVERRIDE", + "--cap-add", + "FOWNER", + ...NEUTRALIZED_PRE_ENTRYPOINT_ENV, + "--volumes-from", + transaction.containerId, + "--mount", + `type=bind,src=${receiptPath},dst=${MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY},readonly`, + "--entrypoint", + CLEAN_NODE_ENTRYPOINT, + transaction.image, + ...CLEAN_NODE_ARGV, + ...transactionCommand("rollback", transaction), + "--read-only-receipt", + ], + DOCKER_MUTATION_OPTIONS, + ); + if (!hasZeroDockerExitStatus(helper)) { + throw new Error( + `Immutable managed-startup helper could not restore and verify shared state: ${commandDetail(helper)}. ` + + `Protected receipt retained at ${receiptPath}`, + ); + } + restored = true; + } finally { + if (restored) { + cleanupReceiptBestEffort(receiptPath); + } + } +} + +function removeFailedUnbackedContainer( + transaction: DockerManagedBootstrapSharedStateTransaction, + deps: DockerGpuPatchDeps, +): void { + const dockerRm = deps.dockerRm ?? defaultDockerRm; + const removed = dockerRm(transaction.containerId, DOCKER_MUTATION_OPTIONS); + if (!hasZeroDockerExitStatus(removed)) { + throw new Error( + `Could not remove the failed managed-startup container after shared-state rollback: ${commandDetail(removed)}`, + ); + } +} + +/** + * Finalize the shared-state half of managed container cutover before generic + * backup removal or rollback. A shared-state rollback failure deliberately + * throws so callers cannot remove the new container or restart the old one + * while `/sandbox` remains partially applied. + */ +export function finalizeDockerManagedStartupSharedState( + input: { + readonly transaction: DockerManagedBootstrapSharedStateTransaction | null; + readonly patchResult?: DockerGpuPatchResult | null; + /** + * The managed-bootstrap journal owns exact replacement removal. Retaining + * it lets the caller publish rollback authorization after shared-state + * restoration and before the first runtime deletion. + */ + readonly retainContainerAfterRollback?: boolean; + readonly supervisorReady: boolean; + }, + deps: DockerGpuPatchDeps = {}, +): DockerManagedStartupSharedStateOutcome { + const transaction = input.transaction; + if (!transaction) { + return { supervisorReady: input.supervisorReady, failure: null }; + } + assertValidManagedStartupTransaction(transaction); + if (input.supervisorReady) { + // Preserve and validate an explicit writable-layer receipt before logical + // commit. The helper receives the copy read-only and does not delete it; + // this keeps rollback possible when Docker loses the helper acknowledgement. + // --volumes-from exposes shared mounts only; it cannot expose this + // container-local transaction directory to an immutable helper. + let receiptPath: string; + try { + const copiedReceipt = copyManagedStartupReceipt(transaction, deps); + if (!copiedReceipt) { + throw new Error("Managed-startup pending receipt disappeared before commit."); + } + receiptPath = copiedReceipt; + } catch (error) { + try { + quiesceManagedStartupContainer(transaction, deps); + } catch (stopError) { + throw new Error( + `Managed-startup receipt preservation failed and the new workload could not be quiesced: ${ + error instanceof Error ? error.message : String(error) + }; ${stopError instanceof Error ? stopError.message : String(stopError)}`, + ); + } + throw error; + } + let commitFailure: Error | null = null; + try { + verifyCopiedManagedStartupReceipt( + transaction, + transaction.profileFingerprint, + receiptPath, + MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, + "pending", + deps, + ); + commitManagedStartupSharedState(transaction, deps); + cleanupReceiptBestEffort(receiptPath); + return { supervisorReady: true, failure: null }; + } catch (error) { + if (error instanceof DockerManagedStartupSharedStateCommitIndeterminateError) { + throw error; + } + commitFailure = error instanceof Error ? error : new Error(String(error)); + } + const failure = new Error( + `OpenShell supervisor reconnected, but managed shared-state logical commit validation failed: ${commitFailure.message}`, + ); + try { + quiesceManagedStartupContainer(transaction, deps); + } catch (stopError) { + throw new Error( + `${failure.message}; the new workload could not be quiesced: ${ + stopError instanceof Error ? stopError.message : String(stopError) + }`, + { cause: stopError }, + ); + } + rollbackManagedStartupSharedState(transaction, receiptPath, deps); + if (!input.patchResult && !input.retainContainerAfterRollback) { + removeFailedUnbackedContainer(transaction, deps); + } + return { supervisorReady: false, failure }; + } + + quiesceManagedStartupContainer(transaction, deps); + const receiptPath = copyManagedStartupReceipt(transaction, deps, true); + if (!receiptPath) { + if (!input.patchResult && !input.retainContainerAfterRollback) { + removeFailedUnbackedContainer(transaction, deps); + } + return { supervisorReady: false, failure: null }; + } + rollbackManagedStartupSharedState(transaction, receiptPath, deps); + if (!input.patchResult && !input.retainContainerAfterRollback) { + removeFailedUnbackedContainer(transaction, deps); + } + return { supervisorReady: false, failure: null }; +} diff --git a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts new file mode 100644 index 00000000000..cb069cb5bca --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts @@ -0,0 +1,484 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; + +import { expect, vi } from "vitest"; + +import { managedStartupE2eProfile } from "../../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import type { DockerContainerInspect } from "../docker-gpu-patch-types"; +import { encodeManagedStartupProfile, type ManagedStartupAgent } from "../managed-startup/profile"; +import { createManagedStartupRootApplyRequest } from "../managed-startup/root-apply"; +import { + createManagedBootstrapPreparedAuthority, + MANAGED_BOOTSTRAP_SCHEMA_VERSION, + type ManagedBootstrapCompletionReceipt, + type ManagedBootstrapDurablePreparationReceipt, + type ManagedBootstrapHeldWorkloadHandle, + type ManagedBootstrapObservedSnapshot, + type ManagedBootstrapPreparedReplacementHandle, + type ManagedBootstrapReplacementHandle, +} from "./adapter"; +import type { DockerManagedBootstrapDeps } from "./docker"; +import { + type DockerManagedBootstrapJournal, + DockerManagedBootstrapJournalAcknowledgementLostError, + type DockerManagedBootstrapJournalPhase, + type DockerManagedBootstrapJournalStore, +} from "./docker-journal"; +import { normalizeDockerManagedBootstrapLaunchSpec } from "./docker-spec"; +import { parseManagedBootstrapEnvelope } from "./envelope"; + +export const IDENTITY = "1".repeat(64); +export const OLD_ID = "2".repeat(64); +export const NEW_ID = "3".repeat(64); +const CONFIG_ID = `sha256:${"4".repeat(64)}`; +const MANIFEST = `sha256:${"5".repeat(64)}` as const; +const REPOSITORY = "registry.example/nemoclaw/hermes"; +const IMAGE = `${REPOSITORY}@${MANIFEST}`; +const SUPERVISOR = ["/opt/openshell/bin/openshell-sandbox", "supervise"] as const; +export const SUPPORTED_AGENTS = ["openclaw", "hermes", "langchain-deepagents-code"] as const; + +type FixtureCommandResult = { + readonly status: number; + readonly stdout?: string; + readonly stderr?: string; +}; + +export type DockerFixtureAcknowledgement = + | "container:create" + | "container:remove" + | "container:rename" + | "container:start" + | "container:stop" + | "journal:create" + | "journal:cutover" + | "journal:remove" + | "journal:rollback-authorized" + | "journal:staged" + | "journal:shared-state-committed"; + +export type DockerFixtureOptions = { + readonly agent?: ManagedStartupAgent; + readonly dockerStartResults?: Readonly>; + readonly journalTransitionFailures?: Partial< + Readonly> + >; + readonly lostAcknowledgements?: readonly DockerFixtureAcknowledgement[]; + readonly ownerId?: string; + readonly sharedState?: "committed" | "none" | "pending"; + readonly sharedStateCommitResult?: FixtureCommandResult; +}; + +function agentInputs(agent: ManagedStartupAgent = "hermes") { + const request = createManagedStartupRootApplyRequest({ + agent, + encodedProfile: encodeManagedStartupProfile(managedStartupE2eProfile(agent, false, false)), + }); + const heldArgv = [ + "env", + "A=1", + "/usr/local/bin/nemoclaw-managed-startup-hold", + "--agent", + agent, + "--profile-fingerprint", + request.profileFingerprint, + "--bootstrap-identity", + IDENTITY, + ] as const; + return { + request, + heldArgv, + metadata: { "nemoclaw.ai/managed-profile": request.profileFingerprint }, + }; +} + +export const { heldArgv } = agentInputs(); +export const sandbox = { + sandboxName: "alpha", + sandboxId: "sandbox-alpha", + driverId: "docker", +}; + +function shellArgv(argv: readonly string[]): string { + return argv.join(" "); +} + +function originalInspect(inputs = agentInputs()): DockerContainerInspect { + return { + Id: OLD_ID, + Image: CONFIG_ID, + Name: "/openshell-alpha", + Config: { + Image: IMAGE, + Env: ["A=1", `OPENSHELL_SANDBOX_COMMAND=${shellArgv(inputs.heldArgv)}`], + Labels: { + "openshell.ai/managed-by": "openshell", + "openshell.ai/sandbox-name": "alpha", + "openshell.ai/sandbox-id": "sandbox-alpha", + ...inputs.metadata, + }, + Entrypoint: [SUPERVISOR[0]], + Cmd: SUPERVISOR.slice(1), + User: "root", + WorkingDir: "/sandbox", + Hostname: "alpha", + }, + State: { Running: true, Paused: false, Restarting: false, Dead: false }, + HostConfig: { + Binds: ["/host/workspace:/sandbox:rw"], + NetworkMode: "openshell", + RestartPolicy: { Name: "unless-stopped" }, + CapDrop: ["NET_RAW"], + SecurityOpt: ["no-new-privileges"], + Ulimits: [{ Name: "nofile", Soft: 65_536, Hard: 65_536 }], + }, + NetworkSettings: { Networks: { openshell: { Aliases: ["openshell-alpha"] } } }, + }; +} + +export function authority(agent: ManagedStartupAgent = "hermes") { + const inputs = agentInputs(agent); + const inspect = originalInspect(inputs); + const normalized = normalizeDockerManagedBootstrapLaunchSpec(inspect); + const plan = { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandboxName: "alpha", + driverId: "docker", + image: { repository: REPOSITORY, manifestDigest: MANIFEST }, + profile: { agent, fingerprint: inputs.request.profileFingerprint }, + agentIdentity: { uid: 1000, gid: 1000, workdir: "/sandbox" }, + intendedWorkloadArgv: ["env", "A=1", "nemoclaw-start"], + expectedSupervisorArgv: SUPERVISOR, + metadata: inputs.metadata, + }; + const handle: ManagedBootstrapHeldWorkloadHandle = { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox, + bootstrapIdentity: IDENTITY, + heldWorkloadArgv: inputs.heldArgv, + intendedWorkloadArgv: plan.intendedWorkloadArgv, + plan, + createReceipt: { sandbox, ready: true, readyAt: "2026-07-31T12:00:00.000Z" }, + }; + const snapshot: ManagedBootstrapObservedSnapshot = { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox, + runtimeId: OLD_ID, + bootstrapIdentity: IDENTITY, + image: plan.image, + runtimeImageContentId: CONFIG_ID, + specHash: normalized.hash, + specCanonicalJson: normalized.canonicalJson, + agentIdentity: plan.agentIdentity, + supervisorArgv: SUPERVISOR, + heldWorkloadArgv: inputs.heldArgv, + metadata: inputs.metadata, + }; + return { handle, plan, request: inputs.request, snapshot }; +} + +function failFixture(message: string): never { + throw new Error(message); +} + +function readProtectedEnvelope(source: string): ReturnType { + const noFollow = fs.constants.O_NOFOLLOW; + if (typeof noFollow !== "number") throw new Error("test requires O_NOFOLLOW"); + const descriptor = fs.openSync(source, fs.constants.O_RDONLY | noFollow); + try { + const before = fs.fstatSync(descriptor, { bigint: true }); + expect(Number(before.mode & 0o777n)).toBe(0o400); + const parsed = parseManagedBootstrapEnvelope(fs.readFileSync(descriptor, "utf8")); + const after = fs.fstatSync(descriptor, { bigint: true }); + expect(after.dev).toBe(before.dev); + expect(after.ino).toBe(before.ino); + expect(after.size).toBe(before.size); + expect(after.mtimeNs).toBe(before.mtimeNs); + expect(after.ctimeNs).toBe(before.ctimeNs); + return parsed; + } finally { + fs.closeSync(descriptor); + } +} + +export function fixture(options: DockerFixtureOptions = {}) { + let original: DockerContainerInspect | null = originalInspect(agentInputs(options.agent)); + let replacement: DockerContainerInspect | null = null; + let journal: DockerManagedBootstrapJournal | null = null; + let sharedState: "committed" | "none" | "pending" = options.sharedState ?? "none"; + const events: string[] = []; + const lostAcknowledgements = new Set(options.lostAcknowledgements ?? []); + const losesAcknowledgement = (operation: DockerFixtureAcknowledgement) => + lostAcknowledgements.has(operation); + const ok = (stdout = ""): FixtureCommandResult => ({ status: 0, stdout, stderr: "" }); + const copyJournal = () => (journal ? structuredClone(journal) : null); + const store: DockerManagedBootstrapJournalStore = { + create(value) { + journal = structuredClone(value); + events.push("journal:staged"); + if (losesAcknowledgement("journal:create")) { + throw new DockerManagedBootstrapJournalAcknowledgementLostError( + "lost journal create acknowledgement", + ); + } + }, + load: () => copyJournal(), + transition(_identity, expected, next) { + const current = + journal !== null && journal.phase === expected + ? journal + : failFixture("stale journal transition"); + journal = { ...current, phase: next }; + events.push(`journal:${next}`); + const injectedFailure = options.journalTransitionFailures?.[next]; + if (injectedFailure) throw injectedFailure; + if (losesAcknowledgement(`journal:${next}`)) { + throw new DockerManagedBootstrapJournalAcknowledgementLostError( + "lost journal transition acknowledgement", + ); + } + return structuredClone(journal); + }, + remove(_identity, expected) { + const current = journal; + void (current !== null && expected.includes(current.phase) + ? current + : failFixture("stale journal remove")); + journal = null; + events.push("journal:removed"); + if (losesAcknowledgement("journal:remove")) { + throw new DockerManagedBootstrapJournalAcknowledgementLostError( + "lost journal remove acknowledgement", + ); + } + }, + }; + const inspect = (reference: string): DockerContainerInspect => { + const candidates = [original, replacement].filter( + (value): value is DockerContainerInspect => value !== null, + ); + const found = candidates.find( + (value) => + value.Id === reference || String(value.Name ?? "").replace(/^\/+/u, "") === reference, + ); + return found ? structuredClone(found) : failFixture(`No such container: ${reference}`); + }; + const dockerCapture: NonNullable = vi.fn((args) => { + switch (args[0]) { + case "image": + return JSON.stringify([{ Id: CONFIG_ID, RepoDigests: [IMAGE] }]); + default: + return JSON.stringify([inspect(String(args[3] ?? ""))]); + } + }); + const dockerRun: NonNullable = vi.fn( + (args: readonly string[]) => { + switch (args[0]) { + case "create": { + events.push("create:replacement"); + const source = + original ?? failFixture("original disappeared before replacement creation"); + const name = String(args[args.indexOf("--name") + 1] ?? ""); + const entrypoint = String(args[args.indexOf("--entrypoint") + 1] ?? ""); + const imageIndex = args.indexOf(IMAGE); + const env = args.flatMap((value, index) => + value === "--env" ? [String(args[index + 1] ?? "")] : [], + ); + replacement = { + ...structuredClone(source), + Id: NEW_ID, + Name: `/${name}`, + Config: { + ...structuredClone(source.Config), + Image: IMAGE, + Env: env, + Entrypoint: [entrypoint], + Cmd: args.slice(imageIndex + 1), + }, + State: { Running: false, Paused: false, Restarting: false, Dead: false }, + }; + return losesAcknowledgement("container:create") + ? { status: 1, stdout: "", stderr: "lost create acknowledgement" } + : ok(NEW_ID); + } + case "ps": + return ok(original ? OLD_ID : ""); + case "inspect": { + const id = String(args[3] ?? ""); + try { + inspect(id); + return ok(`[{"Id":"${id}"}]`); + } catch { + return { status: 1, stderr: `Error response from daemon: No such container: ${id}` }; + } + } + case "cp": { + const sourceIndex = args[1] === "-a" ? 2 : 1; + const source = String(args[sourceIndex] ?? ""); + const destination = String(args[sourceIndex + 1] ?? ""); + const copyIntoContainer = () => { + events.push("stage:envelope"); + expect(readProtectedEnvelope(source).bootstrapIdentity).toBe(IDENTITY); + return ok(); + }; + const copyFromContainer = () => { + const receipt = source.split(":")[1]; + const expected = receipt?.includes("shared-state-commit") ? "committed" : "pending"; + return sharedState === expected + ? (() => { + fs.mkdirSync(destination, { recursive: true }); + return ok(); + })() + : { + status: 1, + stderr: `Error response from daemon: Could not find the file ${receipt} in container ${NEW_ID}`, + }; + }; + return source.includes(":") ? copyFromContainer() : copyIntoContainer(); + } + case "run": + switch (true) { + case args.includes("--shared-state-transaction-status"): + return ok(`${sharedState}\n`); + case args.includes("--rollback-shared-state-transaction"): + sharedState = "none"; + events.push("shared:rollback"); + return ok(); + } + break; + case "exec": + switch (true) { + case args.includes("--commit-shared-state-transaction"): { + const result = options.sharedStateCommitResult ?? ok(); + sharedState = result.status === 0 ? "committed" : sharedState; + events.push("shared:commit"); + return result; + } + case args.includes("--clear-shared-state-commit-receipt"): + sharedState = "none"; + events.push("shared:clear"); + return ok(); + } + break; + } + throw new Error(`unexpected Docker command: ${args.join(" ")}`); + }, + ); + const deps: DockerManagedBootstrapDeps = { + journalStore: store, + dockerCapture, + dockerRun, + dockerStop: vi.fn((id) => { + events.push(`stop:${id}`); + const target = id === OLD_ID ? original : replacement; + [target] + .filter((value): value is DockerContainerInspect => value?.State !== undefined) + .forEach((value) => { + value.State = { ...value.State, Running: false }; + }); + return losesAcknowledgement("container:stop") + ? { status: 1, stderr: "lost stop acknowledgement" } + : ok(); + }), + dockerRename: vi.fn((id, name) => { + events.push(`rename:${id}:${name}`); + const target = id === OLD_ID ? original : replacement; + [target] + .filter((value): value is DockerContainerInspect => value !== null) + .forEach((value) => { + value.Name = `/${name}`; + }); + return losesAcknowledgement("container:rename") + ? { status: 1, stderr: "lost rename acknowledgement" } + : ok(); + }), + dockerStart: vi.fn((id) => { + events.push(`start:${id}`); + const result = options.dockerStartResults?.[id] ?? ok(); + const target = id === OLD_ID ? original : replacement; + [target] + .filter( + (value): value is DockerContainerInspect => + value?.State !== undefined && result.status === 0, + ) + .forEach((value) => { + value.State = { ...value.State, Running: true }; + }); + return losesAcknowledgement("container:start") + ? { status: 1, stderr: "lost start acknowledgement" } + : result; + }), + dockerRm: vi.fn((id) => { + events.push(`rm:${id}`); + switch (id) { + case OLD_ID: + original = null; + break; + case NEW_ID: + replacement = null; + break; + } + return losesAcknowledgement("container:remove") + ? { status: 1, stderr: "lost rm acknowledgement" } + : ok(); + }), + runCaptureOpenshell: vi.fn(() => `Name: alpha\nID: ${options.ownerId ?? "sandbox-alpha"}\n`), + runOpenshell: vi.fn(() => ok()), + now: () => new Date("2026-07-31T12:30:00.000Z"), + }; + return { + deps, + events, + get journal() { + return journal; + }, + get original() { + return original; + }, + get replacement() { + return replacement; + }, + get sharedState() { + return sharedState; + }, + }; +} + +export function completion( + replacement: ManagedBootstrapReplacementHandle, +): ManagedBootstrapCompletionReceipt { + return { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox, + runtimeId: replacement.replacementRuntimeId, + image: replacement.image, + runtimeImageContentId: replacement.runtimeImageContentId, + originalSpecHash: replacement.originalSpecHash, + replacementSpecHash: replacement.replacementSpecHash, + profileFingerprint: replacement.profileFingerprint, + bootstrapIdentity: replacement.bootstrapIdentity, + transactionPending: true, + completedAt: "2026-07-31T12:15:00.000Z", + }; +} + +export function durablePreparation( + handle: ManagedBootstrapHeldWorkloadHandle, + snapshot: ManagedBootstrapObservedSnapshot, + prepared: ManagedBootstrapPreparedReplacementHandle, +): ManagedBootstrapDurablePreparationReceipt { + const preparedAuthority = createManagedBootstrapPreparedAuthority({ + handle, + snapshot, + prepared, + }); + return { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + authorityFingerprint: preparedAuthority.authorityFingerprint, + recordId: `test-authority-${handle.plan.profile.agent}`, + recordedAt: "2026-07-31T12:10:00.000Z", + }; +} diff --git a/src/lib/onboard/managed-bootstrap/docker.test.ts b/src/lib/onboard/managed-bootstrap/docker.test.ts new file mode 100644 index 00000000000..7979330b180 --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker.test.ts @@ -0,0 +1,433 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { assert, describe, expect, it, vi } from "vitest"; + +import { ManagedBootstrapOwnerCleanupRequiredError } from "./adapter"; +import { createDockerManagedBootstrapAdapter } from "./docker"; +import { + normalizeDockerManagedBootstrapLaunchSpec, + parseDockerManagedBootstrapLaunchSpec, +} from "./docker-spec"; +import { + authority, + completion, + durablePreparation, + fixture, + heldArgv, + IDENTITY, + NEW_ID, + OLD_ID, + SUPPORTED_AGENTS, +} from "./docker-test-fixture"; + +describe("Docker managed bootstrap adapter", () => { + it("publishes durable commit authority before deleting the rollback backup after lost acknowledgements", async () => { + const fake = fixture({ + lostAcknowledgements: [ + "container:create", + "container:remove", + "container:rename", + "container:start", + "container:stop", + "journal:create", + "journal:cutover", + "journal:remove", + "journal:shared-state-committed", + ], + sharedState: "pending", + }); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request: rootRequest, snapshot } = authority(); + const prepared = await adapter.prepareBootstrapReplacement({ + handle, + snapshot, + request: rootRequest, + replacementOptions: { values: {} }, + }); + expect(fake.journal).toBeNull(); + expect(fake.events).not.toContain(`stop:${OLD_ID}`); + fake.events.push("authority:recorded"); + const durable = durablePreparation(handle, snapshot, prepared); + const replacement = await adapter.activateBootstrapReplacement({ + handle, + snapshot, + prepared, + durablePreparation: durable, + }); + const order = fake.events; + expect(order).toContain("authority:recorded"); + expect(order).toContain("journal:staged"); + expect(order.indexOf("journal:staged")).toBeGreaterThan(order.indexOf("authority:recorded")); + expect(order).toContain("journal:cutover"); + expect(order).toContain(`stop:${OLD_ID}`); + expect(order.indexOf("journal:cutover")).toBeLessThan(order.indexOf(`stop:${OLD_ID}`)); + expect(fake.journal).toMatchObject({ + phase: "cutover", + originalRuntimeId: OLD_ID, + replacementRuntimeId: NEW_ID, + }); + + await expect( + adapter.finalizeBootstrap({ + outcome: "commit", + handle, + snapshot, + prepared, + durablePreparation: durable, + replacement, + completion: completion(replacement), + }), + ).resolves.toMatchObject({ outcome: "committed" }); + expect(fake.events).toContain("journal:shared-state-committed"); + expect(fake.events).toContain(`rm:${OLD_ID}`); + expect(fake.events.indexOf("journal:shared-state-committed")).toBeLessThan( + fake.events.indexOf(`rm:${OLD_ID}`), + ); + expect(fake.journal).toBeNull(); + expect(fake.sharedState).toBe("none"); + expect(fake.replacement?.Id).toBe(NEW_ID); + }); + + it("preserves commit validation failure details when the replacement cannot be quiesced", async () => { + const fake = fixture({ + sharedState: "pending", + sharedStateCommitResult: { status: 1, stderr: "injected commit failure" }, + }); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request, snapshot } = authority(); + const prepared = await adapter.prepareBootstrapReplacement({ + handle, + snapshot, + request, + replacementOptions: { values: {} }, + }); + const durable = durablePreparation(handle, snapshot, prepared); + const replacement = await adapter.activateBootstrapReplacement({ + handle, + snapshot, + prepared, + durablePreparation: durable, + }); + vi.mocked(fake.deps.dockerStop!).mockReturnValue({ + status: 1, + stderr: "injected quiesce failure", + }); + + await expect( + adapter.finalizeBootstrap({ + outcome: "commit", + handle, + snapshot, + prepared, + durablePreparation: durable, + replacement, + completion: completion(replacement), + }), + ).rejects.toThrow( + /logical commit validation failed: Managed-startup shared-state commit helper failed.*injected commit failure.*new workload could not be quiesced.*injected quiesce failure/u, + ); + expect(fake.events).not.toContain("shared:rollback"); + }); + + it("publishes durable rollback authority before deleting the replacement after restart", async () => { + const fake = fixture({ + dockerStartResults: { + [NEW_ID]: { status: 1, stderr: "injected start failure" }, + }, + }); + const first = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request: rootRequest, snapshot } = authority(); + const prepared = await first.prepareBootstrapReplacement({ + handle, + snapshot, + request: rootRequest, + replacementOptions: { values: {} }, + }); + const durable = durablePreparation(handle, snapshot, prepared); + await expect( + first.activateBootstrapReplacement({ + handle, + snapshot, + prepared, + durablePreparation: durable, + }), + ).rejects.toThrow("could not prove its exact replacement running"); + expect(fake.journal?.phase).toBe("cutover"); + + const restarted = createDockerManagedBootstrapAdapter(fake.deps); + await expect( + restarted.finalizeBootstrap({ + outcome: "rollback", + handle, + snapshot, + prepared, + durablePreparation: durable, + replacement: null, + completion: null, + }), + ).rejects.toBeInstanceOf(ManagedBootstrapOwnerCleanupRequiredError); + expect(fake.events).toContain("journal:rollback-authorized"); + expect(fake.events).toContain(`rm:${NEW_ID}`); + expect(fake.events.indexOf("journal:rollback-authorized")).toBeLessThan( + fake.events.indexOf(`rm:${NEW_ID}`), + ); + expect(fake.journal).toBeNull(); + expect(fake.replacement).toBeNull(); + expect(fake.original).not.toBeNull(); + expect(fake.original?.Name).toBe("/openshell-alpha"); + expect(fake.original?.State?.Running).toBe(false); + }); + + it("recovers the pre-stop cutover crash state after adapter restart", async () => { + const fake = fixture({ + journalTransitionFailures: { + cutover: new Error("injected crash after durable cutover fence"), + }, + }); + const { handle, request: rootRequest, snapshot } = authority(); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const prepared = await adapter.prepareBootstrapReplacement({ + handle, + snapshot, + request: rootRequest, + replacementOptions: { values: {} }, + }); + const durable = durablePreparation(handle, snapshot, prepared); + await expect( + adapter.activateBootstrapReplacement({ + handle, + snapshot, + prepared, + durablePreparation: durable, + }), + ).rejects.toThrow("crash after durable cutover fence"); + expect(fake.events).not.toContain(`stop:${OLD_ID}`); + await expect( + createDockerManagedBootstrapAdapter(fake.deps).finalizeBootstrap({ + outcome: "rollback", + handle, + snapshot, + prepared, + durablePreparation: durable, + replacement: null, + completion: null, + }), + ).rejects.toBeInstanceOf(ManagedBootstrapOwnerCleanupRequiredError); + expect(fake.events).toContain("journal:rollback-authorized"); + expect(fake.events).toContain(`rm:${NEW_ID}`); + expect(fake.events.indexOf("journal:rollback-authorized")).toBeLessThan( + fake.events.indexOf(`rm:${NEW_ID}`), + ); + expect(fake.journal).toBeNull(); + }); + + it("fences rollback when image-owned shared state is already committed", async () => { + const fake = fixture({ sharedState: "committed" }); + const { handle, request: rootRequest, snapshot } = authority(); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const prepared = await adapter.prepareBootstrapReplacement({ + handle, + snapshot, + request: rootRequest, + replacementOptions: { values: {} }, + }); + const durable = durablePreparation(handle, snapshot, prepared); + const replacement = await adapter.activateBootstrapReplacement({ + handle, + snapshot, + prepared, + durablePreparation: durable, + }); + const eventCount = fake.events.length; + await expect( + adapter.finalizeBootstrap({ + outcome: "rollback", + handle, + snapshot, + prepared, + durablePreparation: durable, + replacement, + completion: null, + }), + ).rejects.toMatchObject({ name: "ManagedBootstrapDurableCommitCleanupPendingError" }); + expect(fake.journal?.phase).toBe("shared-state-committed"); + expect(fake.events.slice(eventCount)).toEqual(["journal:shared-state-committed"]); + }); + + it("rejects cutover before the exact durable authority receipt", async () => { + const fake = fixture(); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request: rootRequest, snapshot } = authority(); + const prepared = await adapter.prepareBootstrapReplacement({ + handle, + snapshot, + request: rootRequest, + replacementOptions: { values: {} }, + }); + const invalid = { + ...durablePreparation(handle, snapshot, prepared), + authorityFingerprint: "f".repeat(64), + }; + await expect( + adapter.activateBootstrapReplacement({ + handle, + snapshot, + prepared, + durablePreparation: invalid, + }), + ).rejects.toThrow("exact durable prepared-authority receipt"); + expect(fake.journal).toBeNull(); + expect(fake.events).not.toContain(`stop:${OLD_ID}`); + await expect( + adapter.finalizeBootstrap({ + outcome: "rollback", + handle, + snapshot, + prepared, + durablePreparation: null, + replacement: null, + completion: null, + }), + ).rejects.toBeInstanceOf(ManagedBootstrapOwnerCleanupRequiredError); + expect(fake.replacement).toBeNull(); + }); + + it.each( + SUPPORTED_AGENTS, + )("prepares, activates, and exactly rolls back the %s agent without a central switch", async (agent) => { + const fake = fixture({ agent, sharedState: "pending" }); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request: rootRequest, snapshot } = authority(agent); + const prepared = await adapter.prepareBootstrapReplacement({ + handle, + snapshot, + request: rootRequest, + replacementOptions: { values: {} }, + }); + const durable = durablePreparation(handle, snapshot, prepared); + const replacement = await adapter.activateBootstrapReplacement({ + handle, + snapshot, + prepared, + durablePreparation: durable, + }); + await expect( + adapter.finalizeBootstrap({ + outcome: "rollback", + handle, + snapshot, + prepared, + durablePreparation: durable, + replacement, + completion: null, + }), + ).rejects.toBeInstanceOf(ManagedBootstrapOwnerCleanupRequiredError); + expect(fake.journal).toBeNull(); + expect(fake.replacement).toBeNull(); + expect( + vi.mocked(fake.deps.dockerRun!).mock.calls.some(([args]) => { + const agentIndex = args.indexOf("--agent"); + return ( + args.includes("--shared-state-transaction-status") && + agentIndex >= 0 && + args[agentIndex + 1] === agent + ); + }), + ).toBe(true); + }); + + it("rejects an empty intended workload argv with a precise boundary error", async () => { + const fake = fixture(); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request, snapshot } = authority(); + const plan = { ...handle.plan, intendedWorkloadArgv: [] }; + const emptyArgvHandle = { ...handle, intendedWorkloadArgv: [], plan }; + await expect( + adapter.prepareBootstrapReplacement({ + handle: emptyArgvHandle, + snapshot, + request, + replacementOptions: { values: {} }, + }), + ).rejects.toThrow( + "Managed bootstrap Docker replacement requires one bounded intended workload argv.", + ); + expect(fake.events).toContain("create:replacement"); + expect(fake.events).not.toContain(`stop:${OLD_ID}`); + }); + + it.each([ + "NODE_OPTIONS", + "NODE_PATH", + "LD_PRELOAD", + "BASH_ENV", + ])("rejects hostile %s from the launch snapshot before replacement creation", async (key) => { + const fake = fixture(); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request, snapshot } = authority(); + const parsed = parseDockerManagedBootstrapLaunchSpec(snapshot.specCanonicalJson); + const hostileInspect = structuredClone(parsed.inspect); + hostileInspect.Config!.Env = [...(hostileInspect.Config!.Env ?? []), `${key}=/tmp/hostile`]; + const hostileSpec = normalizeDockerManagedBootstrapLaunchSpec(hostileInspect); + + await expect( + adapter.prepareBootstrapReplacement({ + handle, + snapshot: { + ...snapshot, + specHash: hostileSpec.hash, + specCanonicalJson: hostileSpec.canonicalJson, + }, + request, + replacementOptions: { values: {} }, + }), + ).rejects.toThrow(`Managed bootstrap refuses root-process injection environment '${key}'.`); + expect(fake.events).not.toContain("create:replacement"); + expect(fake.replacement).toBeNull(); + }); + + it("quiesces and retains an exact incomplete create when its mutable name is reused", async () => { + const fake = fixture({ ownerId: "sandbox-alpha-recreated" }); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, plan } = authority(); + await expect( + adapter.cleanupIncompleteCreate({ + plan, + bootstrapIdentity: IDENTITY, + heldWorkloadArgv: heldArgv, + createReceipt: handle.createReceipt, + }), + ).rejects.toMatchObject({ + name: "ManagedBootstrapOwnerCleanupRequiredError", + sandboxId: "sandbox-alpha", + runtimeId: OLD_ID, + }); + expect(fake.original).not.toBeNull(); + expect(fake.original?.State?.Running).toBe(false); + expect(fake.events).not.toContain(`rm:${OLD_ID}`); + expect(vi.mocked(fake.deps.runOpenshell!)).not.toHaveBeenCalled(); + }); + + it("retains a same-name workload that differs from the validated create receipt", async () => { + const replacementSandboxId = "sandbox-alpha-recreated"; + const fake = fixture({ ownerId: replacementSandboxId }); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, plan } = authority(); + expect(fake.original).not.toBeNull(); + const labels = fake.original?.Config?.Labels; + assert(labels, "fixture labels are required"); + labels["openshell.ai/sandbox-id"] = replacementSandboxId; + + await expect( + adapter.cleanupIncompleteCreate({ + plan, + bootstrapIdentity: IDENTITY, + heldWorkloadArgv: heldArgv, + createReceipt: handle.createReceipt, + }), + ).rejects.toThrow(/does not match the exact validated create receipt/u); + expect(fake.events).not.toContain(`stop:${OLD_ID}`); + expect(fake.events).not.toContain(`rm:${OLD_ID}`); + }); +}); diff --git a/src/lib/onboard/managed-bootstrap/docker.ts b/src/lib/onboard/managed-bootstrap/docker.ts new file mode 100644 index 00000000000..03afb2ba280 --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker.ts @@ -0,0 +1,2949 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; + +import { + dockerRename as defaultDockerRename, + dockerRm as defaultDockerRm, + dockerStart as defaultDockerStart, + dockerStop as defaultDockerStop, +} from "../../adapters/docker/container"; +import { + dockerCapture as defaultDockerCapture, + dockerRun as defaultDockerRun, +} from "../../adapters/docker/run"; +import { parseOpenShellSandboxId } from "../../adapters/openshell/sandbox-identity"; +import { hasZeroDockerExitStatus } from "../docker-command-result"; +import { buildDockerGpuCloneRunArgs, dockerContainerName } from "../docker-gpu-patch-clone"; +import { + DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + DOCKER_GPU_PATCH_TIMEOUT_MS, +} from "../docker-gpu-patch-constants"; +import type { + DockerContainerInspect, + DockerGpuPatchDeps, + DockerGpuPatchMode, + DockerGpuPatchModeKind, + DockerUlimit, +} from "../docker-gpu-patch-types"; +import { waitForOpenShellSupervisorReconnect } from "../docker-gpu-supervisor-reconnect"; +import { openshellSandboxCommandEnvValue } from "../docker-startup-command-env"; +import { + OPENSHELL_MANAGED_BY_LABEL, + OPENSHELL_MANAGED_BY_VALUE, + OPENSHELL_SANDBOX_ID_LABEL, + OPENSHELL_SANDBOX_NAME_LABEL, + queryOpenShellDockerSandboxContainers, +} from "../openshell-docker-sandbox-containers"; +import { cleanupTempDir, secureTempFile } from "../temp-files"; +import { + assertManagedBootstrapIdentity, + assertManagedBootstrapSafeProcessEnvironmentKey, + attachManagedBootstrapRollbackError, + createManagedBootstrapIdentity, + createManagedBootstrapPreparedAuthority, + MANAGED_BOOTSTRAP_SCHEMA_VERSION, + type ManagedBootstrapAdapter, + ManagedBootstrapCommitStateIndeterminateError, + type ManagedBootstrapCompletionReceipt, + type ManagedBootstrapDiscoveredWorkload, + type ManagedBootstrapDiscoveryInput, + ManagedBootstrapDurableCommitCleanupPendingError, + type ManagedBootstrapDurablePreparationReceipt, + type ManagedBootstrapFinalizationReceipt, + type ManagedBootstrapHeldWorkloadHandle, + type ManagedBootstrapIncompleteCreateCleanupInput, + type ManagedBootstrapObservedSnapshot, + ManagedBootstrapOwnerCleanupRequiredError, + type ManagedBootstrapPreparedReplacementHandle, + type ManagedBootstrapReplacementHandle, + type ManagedBootstrapReplacementOptions, + type ManagedBootstrapSandboxIdentity, + renderManagedBootstrapHeldCommand, +} from "./adapter"; +import { + createFileDockerManagedBootstrapJournalStore, + DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, + type DockerManagedBootstrapJournal, + DockerManagedBootstrapJournalAcknowledgementLostError, + type DockerManagedBootstrapJournalStore, + parseDockerManagedBootstrapJournal, + serializeDockerManagedBootstrapJournal, +} from "./docker-journal"; +import { + clearDockerManagedStartupSharedStateCommitReceipt, + DockerManagedStartupSharedStateCommitIndeterminateError, + finalizeDockerManagedStartupSharedState, + probeDockerManagedStartupSharedState, +} from "./docker-shared-state"; +import { + normalizeDockerManagedBootstrapLaunchSpec, + parseDockerManagedBootstrapLaunchSpec, + parseExactDockerContainerInspect, +} from "./docker-spec"; +import { + MANAGED_BOOTSTRAP_COMPLETION_FILE, + MANAGED_BOOTSTRAP_REQUEST_FILE, + parseManagedBootstrapImageCompletion, + serializeManagedBootstrapEnvelope, +} from "./envelope"; + +const FULL_CONTAINER_ID_RE = /^[a-f0-9]{64}$/u; +const FULL_SHA256_RE = /^sha256:[a-f0-9]{64}$/u; +const MAX_ARGV_BYTES = 128 * 1024; +const MAX_CONTAINER_NAME_LENGTH = 253; +const REQUEST_TEMP_PREFIX = "nemoclaw-managed-bootstrap-request"; +const COMPLETION_TEMP_PREFIX = "nemoclaw-managed-bootstrap-completion"; +const COMPLETION_MAX_BYTES = 4096; +const DOCKER_DRIVER_ID = "docker"; + +export const MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE = "/usr/local/bin/nemoclaw-managed-bootstrap"; + +type DockerCommandResult = { + readonly status?: number | null; + readonly stdout?: string | Buffer | null; + readonly stderr?: string | Buffer | null; + readonly error?: Error | null; +}; + +export type DockerManagedBootstrapDeps = Pick< + DockerGpuPatchDeps, + | "dockerCapture" + | "dockerRename" + | "dockerRm" + | "dockerRun" + | "dockerStart" + | "dockerStop" + | "runCaptureOpenshell" + | "runOpenshell" + | "sleep" + | "now" +> & { + readonly createBootstrapIdentity?: () => string; + readonly journalStore?: DockerManagedBootstrapJournalStore; + /** Canonical gateway-scoped state root; required when no store is injected. */ + readonly stateRoot?: string; +}; + +type ResolvedDeps = Required< + Pick< + DockerManagedBootstrapDeps, + | "dockerCapture" + | "dockerRename" + | "dockerRm" + | "dockerRun" + | "dockerStart" + | "dockerStop" + | "journalStore" + | "now" + | "createBootstrapIdentity" + > +> & + DockerManagedBootstrapDeps; + +type DockerBootstrapTransaction = DockerManagedBootstrapJournal; + +interface DockerBootstrapRollbackTombstone { + readonly profileFingerprint: string; + readonly imageReference: string; + readonly receipt: ManagedBootstrapFinalizationReceipt; +} + +export interface DockerManagedBootstrapAdapter extends ManagedBootstrapAdapter {} + +function resolveDeps(deps: DockerManagedBootstrapDeps): ResolvedDeps { + const journalStore = + deps.journalStore ?? + (deps.stateRoot ? createFileDockerManagedBootstrapJournalStore(deps.stateRoot) : null); + if (!journalStore) { + throw new Error( + "Managed bootstrap Docker requires its canonical state root or an injected journal store.", + ); + } + return { + dockerCapture: defaultDockerCapture, + dockerRename: defaultDockerRename, + dockerRm: defaultDockerRm, + dockerRun: defaultDockerRun, + dockerStart: defaultDockerStart, + dockerStop: defaultDockerStop, + journalStore, + now: () => new Date(), + createBootstrapIdentity: createManagedBootstrapIdentity, + ...deps, + }; +} + +function commandDetail(result: DockerCommandResult): string { + return `${String(result.stderr ?? "")} ${String(result.stdout ?? "")} ${String( + result.error?.message ?? "", + )}` + .trim() + .slice(-1200); +} + +function isExactMissingDockerContainer(containerId: string, result: DockerCommandResult): boolean { + const escapedContainerId = containerId.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); + const patterns = [ + new RegExp( + `^(?:Error response from daemon: )?No such (?:container|object): ${escapedContainerId}$`, + "u", + ), + new RegExp(`^Error: No such (?:container|object): ${escapedContainerId}$`, "u"), + ]; + return [result.stderr, result.stdout, result.error?.message] + .map((value) => String(value ?? "").trim()) + .filter(Boolean) + .some((detail) => patterns.some((pattern) => pattern.test(detail))); +} + +function probeExactDockerContainerAbsence( + containerId: string, + deps: ResolvedDeps, +): "absent" | "present" | "unknown" { + let result: DockerCommandResult; + try { + result = deps.dockerRun(["inspect", "--type", "container", containerId], { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + } catch { + return "unknown"; + } + if (hasZeroDockerExitStatus(result)) return "present"; + return isExactMissingDockerContainer(containerId, result) ? "absent" : "unknown"; +} + +function assertZero(result: DockerCommandResult, message: string): void { + if (!hasZeroDockerExitStatus(result)) { + throw new Error(`${message}: ${commandDetail(result) || "Docker command failed"}`); + } +} + +function exactStringArray(value: unknown, label: string): string[] { + if (value === null || value === undefined) return []; + const values = typeof value === "string" ? [value] : value; + if ( + !Array.isArray(values) || + values.some( + (item) => + typeof item !== "string" || + item.length === 0 || + item.includes("\0") || + Buffer.byteLength(item, "utf8") > 64 * 1024, + ) + ) { + throw new Error(`Managed bootstrap Docker ${label} is not an exact bounded argv.`); + } + const result = [...values]; + if (Buffer.byteLength(JSON.stringify(result), "utf8") > MAX_ARGV_BYTES) { + throw new Error(`Managed bootstrap Docker ${label} exceeds its bounded argv transport.`); + } + return result; +} + +function exactSupervisorArgv(inspect: DockerContainerInspect): readonly string[] { + const argv = [ + ...exactStringArray(inspect.Config?.Entrypoint, "entrypoint"), + ...exactStringArray(inspect.Config?.Cmd, "command"), + ]; + if (argv.length === 0 || !argv[0]?.startsWith("/")) { + throw new Error( + "Managed bootstrap requires one bounded absolute supervisor argv from Docker inspect.", + ); + } + return Object.freeze(argv); +} + +function exactArrayEqual(left: readonly string[], right: readonly string[]): boolean { + return left.length === right.length && left.every((value, index) => value === right[index]); +} + +function envValue(env: readonly string[] | null | undefined, key: string): string | null { + const prefix = `${key}=`; + const matches = (env ?? []).filter((value) => value.startsWith(prefix)); + return matches.length === 1 ? (matches[0]?.slice(prefix.length) ?? null) : null; +} + +function assertNoRootProcessInjectionEnvironment(env: readonly string[] | null | undefined): void { + for (const entry of env ?? []) { + const separator = entry.indexOf("="); + const key = separator < 0 ? entry : entry.slice(0, separator); + try { + assertManagedBootstrapSafeProcessEnvironmentKey(key); + } catch { + throw new Error(`Managed bootstrap refuses root-process injection environment '${key}'.`); + } + } +} + +function assertRootSupervisor(inspect: DockerContainerInspect): void { + const user = String(inspect.Config?.User ?? "") + .trim() + .toLowerCase(); + if (!["", "0", "0:0", "root", "root:root"].includes(user)) { + throw new Error("Managed bootstrap Docker workload must retain a root supervisor user."); + } +} + +function isStableRunning(inspect: DockerContainerInspect): boolean { + return inspect.State?.Running !== true || + inspect.State.Paused === true || + inspect.State.Restarting === true || + inspect.State.Dead === true + ? false + : true; +} + +function assertStableRunning(inspect: DockerContainerInspect, label: string): void { + if (!isStableRunning(inspect)) { + throw new Error(`Managed bootstrap Docker ${label} is not stably running.`); + } +} + +function isExplicitlyStopped(inspect: DockerContainerInspect): boolean { + return ( + inspect.State?.Running === false && + inspect.State.Paused === false && + inspect.State.Restarting === false && + inspect.State.Dead === false + ); +} + +function assertExplicitlyStopped(inspect: DockerContainerInspect, label: string): void { + if (!isExplicitlyStopped(inspect)) { + throw new Error(`Managed bootstrap Docker ${label} is not explicitly stopped.`); + } +} + +function expectedImageReference(repository: string, manifestDigest: string): string { + if ( + repository.length === 0 || + repository !== repository.trim() || + repository.includes("@") || + repository.includes("\0") || + !FULL_SHA256_RE.test(manifestDigest) + ) { + throw new Error("Managed bootstrap image repository/manifest identity is invalid."); + } + return `${repository}@${manifestDigest}`; +} + +function assertImage( + inspect: DockerContainerInspect, + image: ManagedBootstrapHeldWorkloadHandle["plan"]["image"], + deps: ResolvedDeps, +): string { + const runtimeContentId = String(inspect.Image ?? "").toLowerCase(); + if (!FULL_SHA256_RE.test(runtimeContentId)) { + throw new Error("Managed bootstrap Docker image does not have an immutable local content ID."); + } + const expectedReference = expectedImageReference(image.repository, image.manifestDigest); + const configuredImage = String(inspect.Config?.Image ?? "").trim(); + if (configuredImage !== expectedReference) { + throw new Error( + "Managed bootstrap Docker configured image is not the exact repository@manifestDigest.", + ); + } + const imageOutput = deps.dockerCapture(["image", "inspect", expectedReference], { + ignoreError: false, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + let parsed: unknown; + try { + parsed = JSON.parse(imageOutput); + } catch { + throw new Error("Managed bootstrap Docker image evidence is malformed."); + } + if (!Array.isArray(parsed) || parsed.length !== 1) { + throw new Error("Managed bootstrap Docker image evidence is not exact."); + } + const evidence = parsed[0] as { + readonly Id?: unknown; + readonly RepoDigests?: unknown; + }; + const evidenceId = String(evidence.Id ?? "").toLowerCase(); + const repoDigests = Array.isArray(evidence.RepoDigests) + ? evidence.RepoDigests.filter((value): value is string => typeof value === "string") + : []; + if (evidenceId !== runtimeContentId || !repoDigests.includes(expectedReference)) { + throw new Error( + "Managed bootstrap Docker image manifest evidence does not match its local content ID.", + ); + } + return runtimeContentId; +} + +function assertMetadata( + inspect: DockerContainerInspect, + sandbox: ManagedBootstrapHeldWorkloadHandle["sandbox"], + metadata: Readonly>, +): void { + const labels = inspect.Config?.Labels ?? {}; + if ( + labels[OPENSHELL_MANAGED_BY_LABEL] !== OPENSHELL_MANAGED_BY_VALUE || + labels[OPENSHELL_SANDBOX_NAME_LABEL] !== sandbox.sandboxName || + labels[OPENSHELL_SANDBOX_ID_LABEL] !== sandbox.sandboxId + ) { + throw new Error( + "Managed bootstrap Docker workload does not match the durable OpenShell sandbox identity.", + ); + } + for (const [key, value] of Object.entries(metadata)) { + if (labels[key] !== value) { + throw new Error(`Managed bootstrap Docker metadata label '${key}' changed.`); + } + } +} + +function assertHeldCommand( + inspect: DockerContainerInspect, + heldWorkloadArgv: readonly string[], + bootstrapIdentity: string, +): void { + assertManagedBootstrapIdentity(bootstrapIdentity); + const expected = openshellSandboxCommandEnvValue(heldWorkloadArgv); + const observed = envValue(inspect.Config?.Env, "OPENSHELL_SANDBOX_COMMAND"); + if (!expected || observed !== expected) { + throw new Error( + "Managed bootstrap Docker workload does not contain the exact identity-bound hold.", + ); + } + const identityIndexes = heldWorkloadArgv + .map((value, index) => (value === bootstrapIdentity ? index : -1)) + .filter((index) => index >= 0); + if (identityIndexes.length !== 1) { + throw new Error("Managed bootstrap hold does not contain exactly one bootstrap identity."); + } +} + +function assertBootstrapIdentityInObservedHold( + inspect: DockerContainerInspect, + bootstrapIdentity: string, +): void { + assertManagedBootstrapIdentity(bootstrapIdentity); + const observed = envValue(inspect.Config?.Env, "OPENSHELL_SANDBOX_COMMAND"); + if (!observed) { + throw new Error("Managed bootstrap Docker workload is missing its held command."); + } + const occurrences = observed.split(bootstrapIdentity).length - 1; + if (occurrences !== 1) { + throw new Error( + "Managed bootstrap Docker held command does not contain one exact bootstrap identity.", + ); + } +} + +function inspectExact(containerId: string, deps: ResolvedDeps): DockerContainerInspect { + if (!FULL_CONTAINER_ID_RE.test(containerId)) { + throw new Error("Managed bootstrap requires one full lowercase Docker container ID."); + } + const output = deps.dockerCapture(["inspect", "--type", "container", containerId], { + ignoreError: false, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + const inspect = parseExactDockerContainerInspect(output); + if (String(inspect.Id ?? "").toLowerCase() !== containerId) { + throw new Error("Managed bootstrap Docker workload identity changed during inspection."); + } + return inspect; +} + +function inspectDockerContainerReference( + reference: string, + deps: ResolvedDeps, +): DockerContainerInspect { + if ( + reference.length === 0 || + reference !== reference.trim() || + reference.includes("\0") || + Buffer.byteLength(reference, "utf8") > MAX_CONTAINER_NAME_LENGTH + ) { + throw new Error("Managed bootstrap Docker lookup reference is invalid."); + } + const output = deps.dockerCapture(["inspect", "--type", "container", reference], { + ignoreError: false, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + const inspect = parseExactDockerContainerInspect(output); + const runtimeId = String(inspect.Id ?? "").toLowerCase(); + if (!FULL_CONTAINER_ID_RE.test(runtimeId)) { + throw new Error("Managed bootstrap Docker lookup did not resolve one full runtime ID."); + } + return inspect; +} + +function tryInspectExact(containerId: string, deps: ResolvedDeps): DockerContainerInspect | null { + try { + return inspectExact(containerId, deps); + } catch { + return null; + } +} + +function backupName(originalName: string, bootstrapIdentity: string): string { + const suffix = `-nemoclaw-bootstrap-${bootstrapIdentity.slice(0, 20)}`; + return `${originalName.slice(0, Math.max(1, MAX_CONTAINER_NAME_LENGTH - suffix.length))}${suffix}`; +} + +function replacementStagingName(originalName: string, bootstrapIdentity: string): string { + const suffix = `-nemoclaw-staged-${bootstrapIdentity.slice(0, 20)}`; + return `${originalName.slice(0, Math.max(1, MAX_CONTAINER_NAME_LENGTH - suffix.length))}${suffix}`; +} + +function writeProtectedEnvelope( + bootstrapIdentity: string, + request: Parameters[0]["rootApplyRequest"], +): string { + const file = secureTempFile(REQUEST_TEMP_PREFIX, ".json"); + try { + fs.writeFileSync( + file, + serializeManagedBootstrapEnvelope({ bootstrapIdentity, rootApplyRequest: request }), + { encoding: "utf8", flag: "wx", mode: 0o400 }, + ); + fs.chmodSync(file, 0o400); + const stat = fs.lstatSync(file); + if ( + !stat.isFile() || + stat.isSymbolicLink() || + stat.nlink !== 1 || + (stat.mode & 0o777) !== 0o400 + ) { + throw new Error("Managed bootstrap request source is not one protected 0400 file."); + } + return file; + } catch (error) { + cleanupTempDir(file, REQUEST_TEMP_PREFIX); + throw error; + } +} + +function readProtectedImageCompletion( + replacementRuntimeId: string, + deps: ResolvedDeps, +): ReturnType { + const file = secureTempFile(COMPLETION_TEMP_PREFIX, ".json"); + let descriptor: number | undefined; + try { + const copied = deps.dockerRun( + ["cp", `${replacementRuntimeId}:${MANAGED_BOOTSTRAP_COMPLETION_FILE}`, file], + { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }, + ); + assertZero(copied, "Managed bootstrap could not retrieve its image completion receipt"); + descriptor = fs.openSync(file, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + const before = fs.fstatSync(descriptor, { bigint: true }); + if ( + !before.isFile() || + before.isSymbolicLink() || + before.nlink !== 1n || + Number(before.mode & 0o777n) !== 0o444 || + before.size < 1n || + before.size > BigInt(COMPLETION_MAX_BYTES) + ) { + throw new Error("Managed bootstrap image completion is not one protected bounded 0444 file."); + } + const bytes = Buffer.alloc(Number(before.size)); + let offset = 0; + while (offset < bytes.length) { + const count = fs.readSync(descriptor, bytes, offset, bytes.length - offset, offset); + if (count === 0) break; + offset += count; + } + const after = fs.fstatSync(descriptor, { bigint: true }); + if ( + offset !== bytes.length || + after.dev !== before.dev || + after.ino !== before.ino || + after.size !== before.size || + after.mtimeNs !== before.mtimeNs || + after.ctimeNs !== before.ctimeNs || + after.mode !== before.mode || + after.nlink !== before.nlink + ) { + throw new Error("Managed bootstrap image completion changed during stable read."); + } + return parseManagedBootstrapImageCompletion(bytes.toString("utf8")); + } finally { + if (descriptor !== undefined) fs.closeSync(descriptor); + cleanupTempDir(file, COMPLETION_TEMP_PREFIX); + } +} + +function parseRequiredUlimits(value: unknown): DockerUlimit[] { + if (value === undefined) return []; + if ( + !Array.isArray(value) || + value.some((entry) => typeof entry !== "string" || entry.includes("\0")) + ) { + throw new Error("Managed bootstrap Docker requiredUlimits must be string entries."); + } + return value.map((entry) => { + const match = /^([a-z][a-z0-9_]*)=(\d+):(\d+)$/u.exec(entry); + if (!match) { + throw new Error(`Managed bootstrap Docker ulimit '${entry}' is invalid.`); + } + const soft = Number(match[2]); + const hard = Number(match[3]); + if (!Number.isSafeInteger(soft) || !Number.isSafeInteger(hard) || hard < soft) { + throw new Error(`Managed bootstrap Docker ulimit '${entry}' is invalid.`); + } + return { name: match[1] as string, soft, hard }; + }); +} + +function replacementPlan(options: ManagedBootstrapReplacementOptions): { + readonly mode: DockerGpuPatchMode; + readonly requiredUlimits: readonly DockerUlimit[]; + readonly extraGroupGids: readonly string[]; +} { + const allowed = new Set([ + "gpuModeArgs", + "gpuModeDevice", + "gpuModeKind", + "gpuModeLabel", + "extraGroupGids", + "requiredUlimits", + ]); + const unknown = Object.keys(options.values).filter((key) => !allowed.has(key)); + if (unknown.length > 0) { + throw new Error( + `Managed bootstrap Docker replacement options are unsupported: ${unknown.sort().join(", ")}.`, + ); + } + const kind = String(options.values.gpuModeKind ?? "startup-command") as DockerGpuPatchModeKind; + if (!["gpus", "nvidia-runtime", "cdi", "startup-command"].includes(kind)) { + throw new Error(`Managed bootstrap Docker GPU mode '${kind}' is invalid.`); + } + const args = exactStringArray(options.values.gpuModeArgs ?? [], "GPU mode arguments"); + return { + mode: { + kind, + label: String(options.values.gpuModeLabel ?? "managed bootstrap"), + device: String(options.values.gpuModeDevice ?? ""), + args, + }, + extraGroupGids: exactStringArray(options.values.extraGroupGids ?? [], "extra group GIDs").map( + (value) => { + if (!/^\d+$/u.test(value)) { + throw new Error(`Managed bootstrap Docker supplementary group '${value}' is invalid.`); + } + return value; + }, + ), + requiredUlimits: parseRequiredUlimits(options.values.requiredUlimits), + }; +} + +function replacementCommand( + handle: ManagedBootstrapHeldWorkloadHandle, + snapshot: ManagedBootstrapObservedSnapshot, +): readonly string[] { + return Object.freeze([ + "--agent", + handle.plan.profile.agent, + "--profile-fingerprint", + handle.plan.profile.fingerprint, + "--bootstrap-identity", + handle.bootstrapIdentity, + "--agent-uid", + String(snapshot.agentIdentity.uid), + "--agent-gid", + String(snapshot.agentIdentity.gid), + "--agent-workdir", + snapshot.agentIdentity.workdir, + "--request-file", + MANAGED_BOOTSTRAP_REQUEST_FILE, + "--", + ...snapshot.supervisorArgv, + ]); +} + +function assertReplacementBoundary( + inspect: DockerContainerInspect, + handle: ManagedBootstrapHeldWorkloadHandle, + snapshot: ManagedBootstrapObservedSnapshot, +): void { + const entrypoint = exactStringArray(inspect.Config?.Entrypoint, "replacement entrypoint"); + const command = exactStringArray(inspect.Config?.Cmd, "replacement command"); + if ( + !exactArrayEqual(entrypoint, [MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE]) || + !exactArrayEqual(command, replacementCommand(handle, snapshot)) + ) { + throw new Error("Managed bootstrap Docker replacement process boundary changed."); + } + const intended = openshellSandboxCommandEnvValue(handle.intendedWorkloadArgv); + if (envValue(inspect.Config?.Env, "OPENSHELL_SANDBOX_COMMAND") !== intended) { + throw new Error( + "Managed bootstrap Docker replacement did not restore the intended sandbox command.", + ); + } +} + +const REPLACED_GPU_ENV_KEYS = new Set([ + "NVIDIA_DISABLE_REQUIRE", + "NVIDIA_DRIVER_CAPABILITIES", + "NVIDIA_REQUIRE_CUDA", + "NVIDIA_VISIBLE_DEVICES", +]); + +function canonicalObject(text: string): Record { + const value = JSON.parse(text) as unknown; + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error("Managed bootstrap normalized Docker spec is not an object."); + } + return value as Record; +} + +function objectField(record: Record, key: string): Record { + const value = record[key]; + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error(`Managed bootstrap normalized Docker spec is missing ${key}.`); + } + return value as Record; +} + +function exactJson(value: unknown): string { + return JSON.stringify(value ?? null); +} + +function stringSet(value: unknown, label: string): string[] { + const values = exactStringArray(value ?? [], label); + if (new Set(values).size !== values.length) { + throw new Error(`Managed bootstrap Docker ${label} contains duplicate entries.`); + } + return values.sort(); +} + +function assertExactStringSet(observed: unknown, expected: readonly string[], label: string): void { + if (!exactArrayEqual(stringSet(observed, label), [...expected].sort())) { + throw new Error(`Managed bootstrap Docker ${label} changed outside declared deltas.`); + } +} + +function modeEnvironment(mode: DockerGpuPatchMode): string[] { + const values: string[] = []; + for (let index = 0; index < mode.args.length; index += 1) { + if (mode.args[index] === "--env") { + const value = mode.args[index + 1]; + if (!value || !value.includes("=")) { + throw new Error("Managed bootstrap Docker GPU mode has an invalid environment delta."); + } + values.push(value); + index += 1; + } + } + return values; +} + +function assertExactEnvironmentDelta( + original: Record, + replacement: Record, + mode: DockerGpuPatchMode, + intendedSandboxCommand: string, +): void { + const gpuAugment = mode.kind !== "startup-command"; + const originalEnv = exactStringArray(original.Env ?? [], "original environment"); + const expected = [ + ...modeEnvironment(mode), + ...originalEnv + .filter((entry) => !gpuAugment || !REPLACED_GPU_ENV_KEYS.has(entry.split("=", 1)[0] ?? "")) + .map((entry) => + entry.startsWith("OPENSHELL_SANDBOX_COMMAND=") + ? `OPENSHELL_SANDBOX_COMMAND=${intendedSandboxCommand}` + : entry, + ), + ]; + const observed = exactStringArray(replacement.Env ?? [], "replacement environment"); + if (!exactArrayEqual(observed, expected)) { + throw new Error( + "Managed bootstrap Docker replacement environment changed outside declared deltas.", + ); + } +} + +function canonicalUlimits(value: unknown, label: string): string { + if (!Array.isArray(value)) { + if (value === undefined || value === null) return "[]"; + throw new Error(`Managed bootstrap Docker ${label} is invalid.`); + } + const normalized = value.map((entry) => { + if (typeof entry !== "object" || entry === null || Array.isArray(entry)) { + throw new Error(`Managed bootstrap Docker ${label} is invalid.`); + } + const record = entry as Record; + const name = String(record.Name ?? ""); + const soft = record.Soft; + const hard = record.Hard; + if (!name || !Number.isSafeInteger(soft) || !Number.isSafeInteger(hard)) { + throw new Error(`Managed bootstrap Docker ${label} is invalid.`); + } + return { Hard: hard, Name: name, Soft: soft }; + }); + if (new Set(normalized.map((entry) => entry.Name)).size !== normalized.length) { + throw new Error(`Managed bootstrap Docker ${label} contains duplicate entries.`); + } + return JSON.stringify(normalized.sort((left, right) => left.Name.localeCompare(right.Name))); +} + +function expectedUlimits(original: unknown, required: readonly DockerUlimit[]): string { + const existing = JSON.parse(canonicalUlimits(original, "original ulimits")) as Array<{ + Hard: number; + Name: string; + Soft: number; + }>; + const merged = new Map(existing.map((entry) => [entry.Name, entry])); + for (const requiredEntry of required) { + merged.set(requiredEntry.name, { + Name: requiredEntry.name, + Soft: requiredEntry.soft, + Hard: requiredEntry.hard, + }); + } + return JSON.stringify( + [...merged.values()].sort((left, right) => left.Name.localeCompare(right.Name)), + ); +} + +function assertExactDeviceRequests( + original: unknown, + observed: unknown, + mode: DockerGpuPatchMode, +): void { + if (mode.kind === "startup-command") { + if (exactJson(observed) !== exactJson(original)) { + throw new Error("Managed bootstrap Docker device requests were not preserved exactly."); + } + return; + } + if (Array.isArray(original) && original.length > 0) { + throw new Error( + "Managed bootstrap Docker GPU augmentation cannot replace an existing device request.", + ); + } + const requests = Array.isArray(observed) ? observed : []; + if (mode.kind === "nvidia-runtime") { + if (requests.length !== 0) { + throw new Error( + "Managed bootstrap Docker NVIDIA runtime added an undeclared device request.", + ); + } + return; + } + if (requests.length !== 1 || typeof requests[0] !== "object" || requests[0] === null) { + throw new Error("Managed bootstrap Docker GPU mode did not add one exact device request."); + } + const request = requests[0] as Record; + if (mode.kind === "gpus") { + const all = mode.device === "all"; + const expectedIds = all ? [] : [mode.device]; + const ids = Array.isArray(request.DeviceIDs) ? request.DeviceIDs : []; + if ( + String(request.Driver ?? "") !== "" || + Number(request.Count) !== (all ? -1 : 0) || + !exactArrayEqual(ids.map(String), expectedIds) || + exactJson(request.Capabilities) !== JSON.stringify([["gpu"]]) || + exactJson(request.Options ?? {}) !== "{}" + ) { + throw new Error("Managed bootstrap Docker --gpus request changed outside its exact delta."); + } + return; + } + const ids = Array.isArray(request.DeviceIDs) ? request.DeviceIDs.map(String) : []; + if ( + request.Driver !== "cdi" || + ![-1, 0].includes(Number(request.Count ?? 0)) || + !exactArrayEqual(ids, [mode.device]) || + (request.Capabilities != null && + (!Array.isArray(request.Capabilities) || request.Capabilities.length > 0)) || + exactJson(request.Options ?? {}) !== "{}" + ) { + throw new Error("Managed bootstrap Docker CDI request changed outside its exact delta."); + } +} + +function scrubVerifiedReplacementDeltas(canonicalJson: string): string { + const root = canonicalObject(canonicalJson); + const inspect = objectField(root, "inspect"); + const config = objectField(inspect, "Config"); + const host = objectField(inspect, "HostConfig"); + config.Image = ""; + config.Entrypoint = [""]; + config.Cmd = [""]; + config.Env = ""; + for (const key of [ + "CapAdd", + "DeviceRequests", + "Devices", + "GroupAdd", + "Runtime", + "SecurityOpt", + "Ulimits", + ]) { + host[key] = ``; + } + return JSON.stringify(root); +} + +function assertReplacementMatchesIntent( + originalCanonicalJson: string, + replacement: DockerContainerInspect, + authoritativeName: string, + plan: { + readonly mode: DockerGpuPatchMode; + readonly requiredUlimits: readonly DockerUlimit[]; + readonly extraGroupGids: readonly string[]; + }, + intendedSandboxCommand: string, +): string { + const original = canonicalObject(originalCanonicalJson); + const originalInspect = objectField(original, "inspect"); + const originalConfig = objectField(originalInspect, "Config"); + const originalHost = objectField(originalInspect, "HostConfig"); + const replacementSpec = normalizeDockerManagedBootstrapLaunchSpec({ + ...replacement, + Name: `/${authoritativeName}`, + }); + const observed = canonicalObject(replacementSpec.canonicalJson); + const observedInspect = objectField(observed, "inspect"); + const observedConfig = objectField(observedInspect, "Config"); + const observedHost = objectField(observedInspect, "HostConfig"); + const gpuAugment = plan.mode.kind !== "startup-command"; + assertExactEnvironmentDelta(originalConfig, observedConfig, plan.mode, intendedSandboxCommand); + assertExactStringSet( + observedHost.CapAdd, + [ + ...stringSet(originalHost.CapAdd, "original capability additions"), + ...(gpuAugment ? ["SYS_PTRACE"] : []), + ].filter((value, index, values) => values.indexOf(value) === index), + "capability additions", + ); + const originalSecurity = stringSet(originalHost.SecurityOpt, "original security options"); + assertExactStringSet( + observedHost.SecurityOpt, + [ + ...originalSecurity, + ...(gpuAugment && !originalSecurity.some((value) => value.startsWith("apparmor")) + ? ["apparmor=unconfined"] + : []), + ], + "security options", + ); + if (exactJson(observedHost.Devices) !== exactJson(originalHost.Devices)) { + throw new Error("Managed bootstrap Docker non-GPU devices were not preserved exactly."); + } + assertExactDeviceRequests(originalHost.DeviceRequests, observedHost.DeviceRequests, plan.mode); + const expectedRuntime = plan.mode.kind === "nvidia-runtime" ? "nvidia" : originalHost.Runtime; + if (exactJson(observedHost.Runtime) !== exactJson(expectedRuntime)) { + throw new Error("Managed bootstrap Docker runtime changed outside its selected GPU delta."); + } + assertExactStringSet( + observedHost.GroupAdd, + [ + ...stringSet(originalHost.GroupAdd, "original supplementary groups"), + ...plan.extraGroupGids, + ].filter((value, index, values) => values.indexOf(value) === index), + "supplementary groups", + ); + if ( + canonicalUlimits(observedHost.Ulimits, "replacement ulimits") !== + expectedUlimits(originalHost.Ulimits, plan.requiredUlimits) + ) { + throw new Error("Managed bootstrap Docker ulimits changed outside declared requirements."); + } + const expectedPreserved = scrubVerifiedReplacementDeltas(originalCanonicalJson); + const observedPreserved = scrubVerifiedReplacementDeltas(replacementSpec.canonicalJson); + if (observedPreserved !== expectedPreserved) { + throw new Error( + "Managed bootstrap Docker replacement normalized spec changed outside declared deltas.", + ); + } + return replacementSpec.hash; +} + +function inspectTransactionRuntime( + transaction: DockerBootstrapTransaction, + runtimeId: string, + deps: ResolvedDeps, +): DockerContainerInspect | null { + const presence = probeExactDockerContainerAbsence(runtimeId, deps); + if (presence === "unknown") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId, + detail: "exact Docker runtime presence could not be proven before mutation", + }); + } + if (presence === "absent") return null; + try { + return inspectExact(runtimeId, deps); + } catch (error) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId, + detail: `exact Docker runtime inspection became unavailable: ${ + error instanceof Error ? error.message : String(error) + }`, + }); + } +} + +function assertTransactionOriginal( + transaction: DockerBootstrapTransaction, + inspect: DockerContainerInspect, +): void { + const name = dockerContainerName(inspect); + if (name !== transaction.originalName && name !== transaction.backupName) { + throw new Error("Managed bootstrap original container has an unexpected transaction name."); + } + const normalized = normalizeDockerManagedBootstrapLaunchSpec({ + ...inspect, + Name: `/${transaction.originalName}`, + }); + if (normalized.hash !== transaction.originalSpecHash) { + throw new Error( + "Managed bootstrap refused mutation because the exact original launch spec changed.", + ); + } +} + +function assertTransactionReplacement( + transaction: DockerBootstrapTransaction, + inspect: DockerContainerInspect, +): void { + const name = dockerContainerName(inspect); + if (name !== transaction.replacementStagingName && name !== transaction.originalName) { + throw new Error("Managed bootstrap replacement container has an unexpected transaction name."); + } + const normalized = normalizeDockerManagedBootstrapLaunchSpec({ + ...inspect, + Name: `/${transaction.originalName}`, + }); + if (normalized.hash !== transaction.replacementSpecHash) { + throw new Error( + "Managed bootstrap refused mutation because the exact replacement launch spec changed.", + ); + } +} + +function assertCompletedCutoverRuntimeState( + transaction: DockerBootstrapTransaction, + deps: ResolvedDeps, +): void { + const original = inspectTransactionRuntime(transaction, transaction.originalRuntimeId, deps); + const replacement = inspectTransactionRuntime( + transaction, + transaction.replacementRuntimeId, + deps, + ); + if (!original || !replacement) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId: original ? transaction.replacementRuntimeId : transaction.originalRuntimeId, + detail: "completed cutover requires both exact transaction runtimes", + }); + } + assertTransactionOriginal(transaction, original); + assertTransactionReplacement(transaction, replacement); + assertExplicitlyStopped(original, "rollback backup"); + assertStableRunning(replacement, "replacement"); + if ( + dockerContainerName(original) !== transaction.backupName || + dockerContainerName(replacement) !== transaction.originalName + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId: transaction.replacementRuntimeId, + detail: "completed cutover runtime names do not match durable authority", + }); + } +} + +function removeExactReplacement( + transaction: DockerBootstrapTransaction, + replacement: DockerContainerInspect, + deps: ResolvedDeps, +): void { + assertTransactionReplacement(transaction, replacement); + const options = { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }; + if (replacement.State?.Running === true) { + const stopped = deps.dockerStop(transaction.replacementRuntimeId, { + ...options, + timeout: DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + }); + if (!hasZeroDockerExitStatus(stopped)) { + const afterStop = inspectTransactionRuntime( + transaction, + transaction.replacementRuntimeId, + deps, + ); + if (!afterStop || afterStop.State?.Running === true) { + throw new Error( + `Managed bootstrap could not quiesce its exact replacement: ${ + commandDetail(stopped) || "Docker stop failed" + }`, + ); + } + assertTransactionReplacement(transaction, afterStop); + } + } + const removed = deps.dockerRm(transaction.replacementRuntimeId, options); + if ( + !hasZeroDockerExitStatus(removed) && + probeExactDockerContainerAbsence(transaction.replacementRuntimeId, deps) !== "absent" + ) { + throw new Error( + `Managed bootstrap could not remove its exact replacement: ${ + commandDetail(removed) || "Docker removal failed" + }`, + ); + } +} + +function restoreOriginal(transaction: DockerBootstrapTransaction, deps: ResolvedDeps): void { + const options = { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }; + const originalBeforeReplacementRemoval = inspectTransactionRuntime( + transaction, + transaction.originalRuntimeId, + deps, + ); + if (!originalBeforeReplacementRemoval) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId: transaction.originalRuntimeId, + detail: "the exact rollback original is absent", + }); + } + assertTransactionOriginal(transaction, originalBeforeReplacementRemoval); + const replacement = inspectTransactionRuntime( + transaction, + transaction.replacementRuntimeId, + deps, + ); + if (replacement) { + removeExactReplacement(transaction, replacement, deps); + } + const original = inspectExact(transaction.originalRuntimeId, deps); + assertTransactionOriginal(transaction, original); + const currentName = dockerContainerName(original); + if (currentName !== transaction.originalName) { + if (currentName !== transaction.backupName) { + throw new Error("Managed bootstrap original container has an unexpected rollback name."); + } + const renamed = deps.dockerRename( + transaction.originalRuntimeId, + transaction.originalName, + options, + ); + if (!hasZeroDockerExitStatus(renamed)) { + const afterRename = inspectTransactionRuntime( + transaction, + transaction.originalRuntimeId, + deps, + ); + if (!afterRename || dockerContainerName(afterRename) !== transaction.originalName) { + throw new Error( + `Managed bootstrap could not restore the original container name: ${ + commandDetail(renamed) || "Docker rename failed" + }`, + ); + } + assertTransactionOriginal(transaction, afterRename); + } + } + const restoredBeforeStart = inspectExact(transaction.originalRuntimeId, deps); + if (restoredBeforeStart.State?.Running !== true) { + const started = deps.dockerStart(transaction.originalRuntimeId, options); + if (!hasZeroDockerExitStatus(started)) { + const afterStart = inspectTransactionRuntime( + transaction, + transaction.originalRuntimeId, + deps, + ); + if (!afterStart || afterStart.State?.Running !== true) { + throw new Error( + `Managed bootstrap could not restart the original container: ${ + commandDetail(started) || "Docker start failed" + }`, + ); + } + assertTransactionOriginal(transaction, afterStart); + } + } + const restored = inspectExact(transaction.originalRuntimeId, deps); + assertStableRunning(restored, "restored workload"); + const normalized = normalizeDockerManagedBootstrapLaunchSpec(restored); + if (normalized.hash !== transaction.originalSpecHash) { + throw new Error("Managed bootstrap rollback did not restore the exact launch spec."); + } +} + +function retainOwnedWorkloadForOwnerCleanup( + sandbox: ManagedBootstrapSandboxIdentity, + deps: ResolvedDeps, + expectedRuntimeId?: string, +): never { + const expectedIdentity = + expectedRuntimeId === undefined + ? `sandbox ${sandbox.sandboxId} with no previously resolved runtime ID` + : `sandbox ${sandbox.sandboxId} expected runtime ${expectedRuntimeId}`; + let containers: DockerCommandResult; + try { + containers = deps.dockerRun( + [ + "ps", + "-a", + "--no-trunc", + "--filter", + `label=${OPENSHELL_MANAGED_BY_LABEL}=${OPENSHELL_MANAGED_BY_VALUE}`, + "--filter", + `label=${OPENSHELL_SANDBOX_ID_LABEL}=${sandbox.sandboxId}`, + "--format", + "{{.ID}}", + ], + { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }, + ); + } catch (error) { + throw new Error( + `Managed bootstrap owner cleanup could not enumerate the exact held runtime for ${expectedIdentity}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + if (Number(containers.status ?? 1) !== 0) { + throw new Error( + `Managed bootstrap owner cleanup could not verify the exact held runtime for ${expectedIdentity}: ${ + commandDetail(containers) || "Docker enumeration failed" + }`, + ); + } + const runtimeIds = String(containers.stdout ?? "") + .trim() + .split(/\r?\n/u) + .map((value) => value.trim().toLowerCase()) + .filter(Boolean); + if ( + runtimeIds.length !== 1 || + !FULL_CONTAINER_ID_RE.test(runtimeIds[0] ?? "") || + (expectedRuntimeId !== undefined && runtimeIds[0] !== expectedRuntimeId) + ) { + throw new Error( + `Managed bootstrap owner cleanup could not bind retention for ${expectedIdentity}; resolved runtime IDs: ${ + runtimeIds.length === 0 ? "none" : runtimeIds.join(", ") + }.`, + ); + } + const runtimeId = runtimeIds[0] as string; + let inspect: DockerContainerInspect; + try { + inspect = inspectExact(runtimeId, deps); + } catch (error) { + throw new Error( + `Managed bootstrap could not inspect retained sandbox ${sandbox.sandboxId} exact runtime ${runtimeId}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + const labels = inspect.Config?.Labels ?? {}; + if ( + labels[OPENSHELL_MANAGED_BY_LABEL] !== OPENSHELL_MANAGED_BY_VALUE || + labels[OPENSHELL_SANDBOX_NAME_LABEL] !== sandbox.sandboxName || + labels[OPENSHELL_SANDBOX_ID_LABEL] !== sandbox.sandboxId + ) { + throw new Error( + `Managed bootstrap owner cleanup refused retention after exact runtime ${runtimeId} ownership changed for sandbox ${sandbox.sandboxId}.`, + ); + } + let stopped: DockerCommandResult; + try { + stopped = deps.dockerStop(runtimeId, { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + }); + } catch (error) { + throw new Error( + `Managed bootstrap could not quiesce retained sandbox ${sandbox.sandboxId} exact runtime ${runtimeId}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + assertZero( + stopped, + `Managed bootstrap could not quiesce retained sandbox ${sandbox.sandboxId} exact runtime ${runtimeId}`, + ); + let retained: DockerContainerInspect; + try { + retained = inspectExact(runtimeId, deps); + } catch (error) { + throw new Error( + `Managed bootstrap could not re-inspect quiesced sandbox ${sandbox.sandboxId} exact runtime ${runtimeId}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + if ( + retained.State?.Running !== false || + retained.State.Paused !== false || + retained.State.Restarting !== false + ) { + throw new Error( + `Managed bootstrap retained sandbox ${sandbox.sandboxId} exact runtime ${runtimeId} did not prove an explicitly quiescent state.`, + ); + } + if (!deps.runCaptureOpenshell) { + throw new ManagedBootstrapOwnerCleanupRequiredError({ + sandboxName: sandbox.sandboxName, + sandboxId: sandbox.sandboxId, + runtimeId, + }); + } + let getBeforeDelete: string; + try { + getBeforeDelete = deps.runCaptureOpenshell(["sandbox", "get", sandbox.sandboxName], { + ignoreError: false, + }); + } catch (error) { + throw new ManagedBootstrapOwnerCleanupRequiredError({ + sandboxName: sandbox.sandboxName, + sandboxId: sandbox.sandboxId, + runtimeId, + detail: `OpenShell owner lookup also failed: ${ + error instanceof Error ? error.message : String(error) + }.`, + }); + } + const sandboxIdBeforeDelete = parseOpenShellSandboxId(getBeforeDelete); + if (sandboxIdBeforeDelete !== sandbox.sandboxId) { + throw new ManagedBootstrapOwnerCleanupRequiredError({ + sandboxName: sandbox.sandboxName, + sandboxId: sandbox.sandboxId, + runtimeId, + detail: `The same mutable name now resolves to durable sandbox ID ${ + sandboxIdBeforeDelete ?? "unknown" + } instead of ${sandbox.sandboxId}.`, + }); + } + throw new ManagedBootstrapOwnerCleanupRequiredError({ + sandboxName: sandbox.sandboxName, + sandboxId: sandbox.sandboxId, + runtimeId, + }); +} + +function resolveIncompleteCreateSandbox( + input: ManagedBootstrapIncompleteCreateCleanupInput, + deps: ResolvedDeps, +): { + readonly sandbox: ManagedBootstrapSandboxIdentity; + readonly runtimeId: string; +} { + if ( + input.plan.schemaVersion !== MANAGED_BOOTSTRAP_SCHEMA_VERSION || + input.plan.driverId !== DOCKER_DRIVER_ID + ) { + throw new Error("Managed bootstrap Docker incomplete-create cleanup received another driver."); + } + assertManagedBootstrapIdentity(input.bootstrapIdentity); + const query = queryOpenShellDockerSandboxContainers(input.plan.sandboxName, deps); + if (!query.ok) { + throw new Error(`Managed bootstrap Docker incomplete-create discovery failed: ${query.error}`); + } + if (query.ids.length !== 1) { + throw new Error( + `Managed bootstrap incomplete-create cleanup requires exactly one labeled Docker workload; found ${String( + query.ids.length, + )}.`, + ); + } + const runtimeId = String(query.ids[0] ?? "").toLowerCase(); + const inspect = inspectExact(runtimeId, deps); + const sandboxId = String(inspect.Config?.Labels?.[OPENSHELL_SANDBOX_ID_LABEL] ?? ""); + if (parseOpenShellSandboxId(`ID: ${sandboxId}\n`) !== sandboxId) { + throw new Error( + "Managed bootstrap Docker incomplete-create workload has no exact durable sandbox ID.", + ); + } + const sandbox = Object.freeze({ + sandboxName: input.plan.sandboxName, + sandboxId, + driverId: input.plan.driverId, + }); + if ( + input.createReceipt.ready !== true || + input.createReceipt.sandbox.sandboxName !== sandbox.sandboxName || + input.createReceipt.sandbox.sandboxId !== sandbox.sandboxId || + input.createReceipt.sandbox.driverId !== sandbox.driverId + ) { + throw new Error( + "Managed bootstrap Docker incomplete-create workload does not match the exact validated create receipt.", + ); + } + assertImage(inspect, input.plan.image, deps); + assertMetadata(inspect, sandbox, input.plan.metadata); + assertHeldCommand(inspect, input.heldWorkloadArgv, input.bootstrapIdentity); + return { sandbox, runtimeId }; +} + +function managedSharedStateTransaction( + handle: ManagedBootstrapHeldWorkloadHandle, + containerId: string, + image: string, +) { + return { + agent: handle.plan.profile.agent, + bootstrapIdentity: handle.bootstrapIdentity, + containerId, + image, + profileFingerprint: handle.plan.profile.fingerprint, + } as const; +} + +function sameDockerBootstrapJournal( + left: DockerBootstrapTransaction, + right: DockerBootstrapTransaction, +): boolean { + return ( + serializeDockerManagedBootstrapJournal(left) === serializeDockerManagedBootstrapJournal(right) + ); +} + +function createDockerBootstrapJournalDurably( + journal: DockerBootstrapTransaction, + deps: ResolvedDeps, +): DockerBootstrapTransaction { + try { + deps.journalStore.create(journal); + } catch (error) { + if (!(error instanceof DockerManagedBootstrapJournalAcknowledgementLostError)) throw error; + const recovered = deps.journalStore.load(journal.bootstrapIdentity); + if (!recovered || !sameDockerBootstrapJournal(recovered, journal)) throw error; + return recovered; + } + const persisted = deps.journalStore.load(journal.bootstrapIdentity); + if (!persisted || !sameDockerBootstrapJournal(persisted, journal)) { + throw new Error("Managed bootstrap Docker staged journal was not durably re-readable."); + } + return persisted; +} + +function transitionDockerBootstrapJournalDurably( + journal: DockerBootstrapTransaction, + next: "cutover" | "rollback-authorized" | "shared-state-committed", + deps: ResolvedDeps, +): DockerBootstrapTransaction { + try { + deps.journalStore.transition(journal.bootstrapIdentity, journal.phase, next); + } catch (error) { + if (!(error instanceof DockerManagedBootstrapJournalAcknowledgementLostError)) throw error; + const recovered = deps.journalStore.load(journal.bootstrapIdentity); + const expected = Object.freeze({ ...journal, phase: next }); + if (!recovered || !sameDockerBootstrapJournal(recovered, expected)) throw error; + return recovered; + } + const persisted = deps.journalStore.load(journal.bootstrapIdentity); + const expected = Object.freeze({ ...journal, phase: next }); + if (!persisted || !sameDockerBootstrapJournal(persisted, expected)) { + throw new Error(`Managed bootstrap Docker journal transition to ${next} was not durable.`); + } + return persisted; +} + +function removeDockerBootstrapJournalDurably( + journal: DockerBootstrapTransaction, + deps: ResolvedDeps, +): void { + try { + deps.journalStore.remove(journal.bootstrapIdentity, [journal.phase]); + } catch (error) { + if (!(error instanceof DockerManagedBootstrapJournalAcknowledgementLostError)) throw error; + const recovered = deps.journalStore.load(journal.bootstrapIdentity); + if (recovered !== null) throw error; + return; + } + if (deps.journalStore.load(journal.bootstrapIdentity) !== null) { + throw new Error("Managed bootstrap Docker journal removal was not durable."); + } +} + +function assertDockerBootstrapTransactionAuthority( + transaction: DockerBootstrapTransaction, + handle: ManagedBootstrapHeldWorkloadHandle, + snapshot: ManagedBootstrapObservedSnapshot, + prepared?: ManagedBootstrapPreparedReplacementHandle | null, + replacement?: ManagedBootstrapReplacementHandle | null, +): void { + const originalName = dockerContainerName( + parseDockerManagedBootstrapLaunchSpec(snapshot.specCanonicalJson).inspect, + ); + const expectedSandbox = handle.sandbox; + if ( + transaction.schemaVersion !== DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION || + transaction.bootstrapIdentity !== handle.bootstrapIdentity || + transaction.sandbox.sandboxName !== expectedSandbox.sandboxName || + transaction.sandbox.sandboxId !== expectedSandbox.sandboxId || + transaction.sandbox.driverId !== expectedSandbox.driverId || + transaction.profileFingerprint !== handle.plan.profile.fingerprint || + transaction.imageReference !== + expectedImageReference(snapshot.image.repository, snapshot.image.manifestDigest) || + transaction.runtimeImageContentId !== snapshot.runtimeImageContentId || + transaction.originalRuntimeId !== snapshot.runtimeId || + transaction.originalName !== originalName || + transaction.replacementStagingName !== + replacementStagingName(originalName, handle.bootstrapIdentity) || + transaction.backupName !== backupName(originalName, handle.bootstrapIdentity) || + transaction.originalSpecHash !== snapshot.specHash || + (prepared !== undefined && + prepared !== null && + (transaction.originalRuntimeId !== prepared.originalRuntimeId || + transaction.replacementRuntimeId !== prepared.preparedRuntimeId || + transaction.replacementSpecHash !== prepared.expectedActivatedSpecHash)) || + (replacement !== undefined && + replacement !== null && + (transaction.originalRuntimeId !== replacement.originalRuntimeId || + transaction.replacementRuntimeId !== replacement.replacementRuntimeId || + transaction.replacementSpecHash !== replacement.replacementSpecHash)) + ) { + throw new Error( + "Managed bootstrap receipts do not match the durable Docker transaction authority.", + ); + } +} + +function transactionFromPreparedAuthority( + handle: ManagedBootstrapHeldWorkloadHandle, + snapshot: ManagedBootstrapObservedSnapshot, + prepared: ManagedBootstrapPreparedReplacementHandle, +): DockerBootstrapTransaction { + const transaction = parseDockerManagedBootstrapJournal(prepared.rollbackAuthority); + if (transaction.phase !== "staged") { + throw new Error("Managed bootstrap Docker prepared authority must describe a staged runtime."); + } + assertDockerBootstrapTransactionAuthority(transaction, handle, snapshot, prepared); + return transaction; +} + +function assertDurablePreparationAuthority( + handle: ManagedBootstrapHeldWorkloadHandle, + snapshot: ManagedBootstrapObservedSnapshot, + prepared: ManagedBootstrapPreparedReplacementHandle, + receipt: ManagedBootstrapDurablePreparationReceipt, +): void { + const authority = createManagedBootstrapPreparedAuthority({ handle, snapshot, prepared }); + const recordedAt = new Date(receipt.recordedAt); + if ( + receipt.schemaVersion !== MANAGED_BOOTSTRAP_SCHEMA_VERSION || + receipt.sandbox.sandboxName !== authority.sandbox.sandboxName || + receipt.sandbox.sandboxId !== authority.sandbox.sandboxId || + receipt.sandbox.driverId !== authority.sandbox.driverId || + receipt.bootstrapIdentity !== authority.bootstrapIdentity || + receipt.authorityFingerprint !== authority.authorityFingerprint || + typeof receipt.recordId !== "string" || + receipt.recordId.length === 0 || + receipt.recordId.includes("\0") || + typeof receipt.recordedAt !== "string" || + !Number.isFinite(recordedAt.getTime()) || + recordedAt.toISOString() !== receipt.recordedAt + ) { + throw new Error( + "Managed bootstrap Docker activation requires the exact durable prepared-authority receipt.", + ); + } +} + +function reconstructDockerBootstrapTransaction( + handle: ManagedBootstrapHeldWorkloadHandle, + snapshot: ManagedBootstrapObservedSnapshot, + replacement: ManagedBootstrapReplacementHandle, + deps: ResolvedDeps, +): DockerBootstrapTransaction { + if ( + replacement.bootstrapIdentity !== handle.bootstrapIdentity || + replacement.originalRuntimeId !== snapshot.runtimeId || + replacement.originalSpecHash !== snapshot.specHash || + replacement.replacementRuntimeId === replacement.originalRuntimeId + ) { + throw new Error( + "Managed bootstrap finalization receipts do not reconstruct one exact Docker transaction.", + ); + } + const transaction = deps.journalStore.load(handle.bootstrapIdentity); + if (!transaction) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: replacement.replacementRuntimeId, + detail: "the durable Docker cutover journal is absent", + }); + } + assertDockerBootstrapTransactionAuthority(transaction, handle, snapshot, null, replacement); + return transaction; +} + +function cleanupUnjournaledPreparedContainer( + input: { + readonly snapshot: ManagedBootstrapObservedSnapshot; + readonly preparedRuntimeId: string; + readonly stagingName: string; + }, + deps: ResolvedDeps, +): void { + if (!FULL_CONTAINER_ID_RE.test(input.preparedRuntimeId)) return; + const original = inspectExact(input.snapshot.runtimeId, deps); + const originalName = dockerContainerName( + parseDockerManagedBootstrapLaunchSpec(input.snapshot.specCanonicalJson).inspect, + ); + if ( + !isStableRunning(original) || + dockerContainerName(original) !== originalName || + normalizeDockerManagedBootstrapLaunchSpec(original).hash !== input.snapshot.specHash + ) { + throw new Error( + "Managed bootstrap cannot clean an unjournaled replacement after original drift.", + ); + } + const prepared = tryInspectExact(input.preparedRuntimeId, deps); + if (!prepared) return; + if ( + String(prepared.Id ?? "").toLowerCase() !== input.preparedRuntimeId || + dockerContainerName(prepared) !== input.stagingName || + !isExplicitlyStopped(prepared) + ) { + throw new Error( + "Managed bootstrap refused cleanup because the unjournaled prepared runtime changed.", + ); + } + const removed = deps.dockerRm(input.preparedRuntimeId, { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + if ( + !hasZeroDockerExitStatus(removed) && + probeExactDockerContainerAbsence(input.preparedRuntimeId, deps) !== "absent" + ) { + throw new Error( + `Managed bootstrap could not remove its unjournaled prepared runtime: ${ + commandDetail(removed) || "Docker removal failed" + }`, + ); + } +} + +function resolvePreparedRollbackAuthority(input: { + readonly handle: ManagedBootstrapHeldWorkloadHandle; + readonly snapshot: ManagedBootstrapObservedSnapshot; + readonly prepared: ManagedBootstrapPreparedReplacementHandle | null; + readonly durablePreparation: ManagedBootstrapDurablePreparationReceipt | null; +}): DockerBootstrapTransaction | null { + if (input.durablePreparation && !input.prepared) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: input.handle.bootstrapIdentity, + runtimeId: input.snapshot.runtimeId, + detail: "durable prepared authority is present without its exact prepared handle", + }); + } + if (!input.prepared) return null; + const authority = transactionFromPreparedAuthority(input.handle, input.snapshot, input.prepared); + if (input.durablePreparation) { + assertDurablePreparationAuthority( + input.handle, + input.snapshot, + input.prepared, + input.durablePreparation, + ); + } + return authority; +} + +export function createDockerManagedBootstrapAdapter( + dependencies: DockerManagedBootstrapDeps = {}, +): DockerManagedBootstrapAdapter { + const deps = resolveDeps(dependencies); + const committedTransactions = new Set(); + const rollbackTombstones = new Map(); + const completedRollback = ( + handle: ManagedBootstrapHeldWorkloadHandle, + alreadyRolledBack: boolean, + ): ManagedBootstrapFinalizationReceipt => { + const receipt = Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + outcome: "rolled-back", + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: true, + alreadyRolledBack, + finalizedAt: deps.now().toISOString(), + } satisfies ManagedBootstrapFinalizationReceipt); + rollbackTombstones.set(handle.bootstrapIdentity, { + profileFingerprint: handle.plan.profile.fingerprint, + imageReference: expectedImageReference( + handle.plan.image.repository, + handle.plan.image.manifestDigest, + ), + receipt, + }); + return receipt; + }; + const priorRollback = ( + handle: ManagedBootstrapHeldWorkloadHandle, + ): ManagedBootstrapFinalizationReceipt | null => { + const tombstone = rollbackTombstones.get(handle.bootstrapIdentity); + if (!tombstone) return null; + const receipt = tombstone.receipt; + if ( + receipt.sandbox.sandboxName !== handle.sandbox.sandboxName || + receipt.sandbox.sandboxId !== handle.sandbox.sandboxId || + receipt.sandbox.driverId !== handle.sandbox.driverId || + tombstone.profileFingerprint !== handle.plan.profile.fingerprint || + tombstone.imageReference !== + expectedImageReference(handle.plan.image.repository, handle.plan.image.manifestDigest) + ) { + throw new Error("Managed bootstrap rollback tombstone does not match its durable identity."); + } + return Object.freeze({ + ...receipt, + alreadyRolledBack: true, + }); + }; + const rollbackBootstrapNow = ({ + handle, + snapshot, + prepared, + durablePreparation, + replacement, + sharedStateAlreadyRolledBack = false, + }: { + readonly handle: ManagedBootstrapHeldWorkloadHandle; + readonly snapshot: ManagedBootstrapObservedSnapshot | null; + readonly prepared: ManagedBootstrapPreparedReplacementHandle | null; + readonly durablePreparation: ManagedBootstrapDurablePreparationReceipt | null; + readonly replacement: ManagedBootstrapReplacementHandle | null; + readonly sharedStateAlreadyRolledBack?: boolean; + }): ManagedBootstrapFinalizationReceipt => { + const finalized = priorRollback(handle); + if (finalized) return finalized; + const journal = deps.journalStore.load(handle.bootstrapIdentity); + if ( + committedTransactions.has(handle.bootstrapIdentity) || + journal?.phase === "shared-state-committed" + ) { + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: handle.bootstrapIdentity, + cleanupRuntimeId: journal?.originalRuntimeId ?? snapshot?.runtimeId ?? "unknown", + detail: + "rollback is no longer legal after the durable Docker commit fence; retry commit finalization", + }); + } + if (!snapshot) { + if (journal || prepared || durablePreparation || replacement) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: journal?.originalRuntimeId ?? prepared?.originalRuntimeId ?? "unknown", + detail: "Docker replacement authority exists without its observed snapshot", + }); + } + retainOwnedWorkloadForOwnerCleanup(handle.sandbox, deps); + return completedRollback(handle, false); + } + + const preparedAuthority = resolvePreparedRollbackAuthority({ + handle, + snapshot, + prepared, + durablePreparation, + }); + + if (!journal) { + const originalPresence = probeExactDockerContainerAbsence(snapshot.runtimeId, deps); + if (originalPresence === "unknown") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: snapshot.runtimeId, + detail: "the original runtime presence is unknown and no durable journal is available", + }); + } + if (originalPresence === "absent") { + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: handle.bootstrapIdentity, + cleanupRuntimeId: snapshot.runtimeId, + detail: + "rollback is forbidden because the exact original is absent after journal retirement", + }); + } + const original = inspectExact(snapshot.runtimeId, deps); + const expectedOriginalName = dockerContainerName( + parseDockerManagedBootstrapLaunchSpec(snapshot.specCanonicalJson).inspect, + ); + const normalized = normalizeDockerManagedBootstrapLaunchSpec(original); + if ( + dockerContainerName(original) !== expectedOriginalName || + original.State?.Running !== true || + normalized.hash !== snapshot.specHash + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: snapshot.runtimeId, + detail: "the journal is absent and the exact original is not a proven restored workload", + }); + } + if (preparedAuthority) { + const observedPrepared = inspectTransactionRuntime( + preparedAuthority, + preparedAuthority.replacementRuntimeId, + deps, + ); + if (observedPrepared) { + assertExplicitlyStopped(observedPrepared, "prepared replacement"); + if ( + dockerContainerName(observedPrepared) !== preparedAuthority.replacementStagingName || + normalizeDockerManagedBootstrapLaunchSpec(observedPrepared).canonicalJson !== + prepared?.preparedSpecCanonicalJson + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: preparedAuthority.replacementRuntimeId, + detail: "the unjournaled prepared runtime changed before exact cleanup", + }); + } + removeExactReplacement(preparedAuthority, observedPrepared, deps); + } + } else if (replacement) { + const replacementPresence = probeExactDockerContainerAbsence( + replacement.replacementRuntimeId, + deps, + ); + if (replacementPresence !== "absent") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: replacement.replacementRuntimeId, + detail: + replacementPresence === "present" + ? "the replacement still exists without durable journal authority" + : "replacement absence is unknown without durable journal authority", + }); + } + } + retainOwnedWorkloadForOwnerCleanup(handle.sandbox, deps, snapshot.runtimeId); + return completedRollback(handle, true); + } + + if (!preparedAuthority || !durablePreparation) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "durable Docker cutover lacks its coordinator-recorded prepared authority", + }); + } + const stagedJournal = Object.freeze({ ...journal, phase: "staged" as const }); + if (!sameDockerBootstrapJournal(stagedJournal, preparedAuthority)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "durable Docker cutover changed its prepared rollback authority", + }); + } + assertDockerBootstrapTransactionAuthority(journal, handle, snapshot, prepared, replacement); + const original = inspectTransactionRuntime(journal, journal.originalRuntimeId, deps); + if (!original) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: "the exact rollback original is absent", + }); + } + assertTransactionOriginal(journal, original); + const observedReplacement = inspectTransactionRuntime( + journal, + journal.replacementRuntimeId, + deps, + ); + + if (journal.phase === "staged") { + assertStableRunning(original, "staged original"); + if (observedReplacement) { + assertExplicitlyStopped(observedReplacement, "staged replacement"); + } + if ( + dockerContainerName(original) !== journal.originalName || + (observedReplacement !== null && + dockerContainerName(observedReplacement) !== journal.replacementStagingName) + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: "staged transaction runtime state does not match its pre-cutover fence", + }); + } + if (observedReplacement) { + removeExactReplacement(journal, observedReplacement, deps); + } + removeDockerBootstrapJournalDurably(journal, deps); + retainOwnedWorkloadForOwnerCleanup(handle.sandbox, deps, journal.originalRuntimeId); + return completedRollback(handle, false); + } + + if (journal.phase !== "cutover" && journal.phase !== "rollback-authorized") { + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: journal.bootstrapIdentity, + cleanupRuntimeId: journal.originalRuntimeId, + detail: "rollback is forbidden by the durable Docker commit phase", + }); + } + + const originalNameNow = dockerContainerName(original); + const replacementNameNow = observedReplacement + ? dockerContainerName(observedReplacement) + : null; + const originalAtTargetRecoverable = + originalNameNow === journal.originalName && + (isStableRunning(original) || isExplicitlyStopped(original)); + const originalAtBackupRecoverable = + originalNameNow === journal.backupName && isExplicitlyStopped(original); + const replacementAtStagingRecoverable = + replacementNameNow === journal.replacementStagingName && + observedReplacement !== null && + isExplicitlyStopped(observedReplacement); + const replacementAtTargetRecoverable = + replacementNameNow === journal.originalName && + observedReplacement !== null && + (isStableRunning(observedReplacement) || isExplicitlyStopped(observedReplacement)); + const validCutoverState = + (originalAtTargetRecoverable && replacementAtStagingRecoverable) || + (originalAtBackupRecoverable && replacementAtStagingRecoverable) || + (originalAtBackupRecoverable && replacementAtTargetRecoverable); + let activeJournal = journal; + + if (journal.phase === "cutover") { + if ( + (!sharedStateAlreadyRolledBack && (!observedReplacement || !validCutoverState)) || + (sharedStateAlreadyRolledBack && + (observedReplacement !== null || + !( + (originalNameNow === journal.backupName && isExplicitlyStopped(original)) || + originalAtTargetRecoverable + ))) + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: + observedReplacement === null && !sharedStateAlreadyRolledBack + ? "the exact replacement disappeared before rollback authorization was durable" + : "cutover runtime names or states do not match a recoverable phase", + }); + } + const current = deps.journalStore.load(journal.bootstrapIdentity); + if (!current || !sameDockerBootstrapJournal(current, journal)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "durable transaction authority changed before rollback authorization", + }); + } + + let sharedStatus: "committed" | "none" | "pending" = "none"; + const sharedTransaction = managedSharedStateTransaction( + handle, + journal.replacementRuntimeId, + journal.runtimeImageContentId, + ); + if (!sharedStateAlreadyRolledBack) { + sharedStatus = probeDockerManagedStartupSharedState( + { + transaction: sharedTransaction, + profileFingerprint: journal.profileFingerprint, + }, + deps, + ); + if (sharedStatus === "committed") { + const committedJournal = transitionDockerBootstrapJournalDurably( + journal, + "shared-state-committed", + deps, + ); + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: committedJournal.bootstrapIdentity, + cleanupRuntimeId: committedJournal.originalRuntimeId, + detail: "image-owned shared state is durably committed; rollback is no longer legal", + }); + } + } + activeJournal = transitionDockerBootstrapJournalDurably(journal, "rollback-authorized", deps); + if (!sharedStateAlreadyRolledBack && sharedStatus === "pending") { + finalizeDockerManagedStartupSharedState( + { + transaction: sharedTransaction, + supervisorReady: false, + retainContainerAfterRollback: true, + }, + deps, + ); + } + } else { + if (!originalAtTargetRecoverable && !originalAtBackupRecoverable) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: "rollback-authorized original runtime state is not recoverable", + }); + } + if ( + observedReplacement && + originalNameNow === journal.originalName && + replacementNameNow === journal.originalName + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "both transaction runtimes claim the authoritative workload name", + }); + } + if (observedReplacement) { + const current = deps.journalStore.load(journal.bootstrapIdentity); + if (!current || !sameDockerBootstrapJournal(current, journal)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "rollback authorization changed before replacement cleanup", + }); + } + const sharedTransaction = managedSharedStateTransaction( + handle, + journal.replacementRuntimeId, + journal.runtimeImageContentId, + ); + const sharedStatus = probeDockerManagedStartupSharedState( + { + transaction: sharedTransaction, + profileFingerprint: journal.profileFingerprint, + }, + deps, + ); + if (sharedStatus === "committed") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: + "shared state became committed after rollback authorization; no mutation was attempted", + }); + } + if (sharedStatus === "pending") { + finalizeDockerManagedStartupSharedState( + { + transaction: sharedTransaction, + supervisorReady: false, + retainContainerAfterRollback: true, + }, + deps, + ); + } + } + } + + const beforeRestore = deps.journalStore.load(activeJournal.bootstrapIdentity); + if (!beforeRestore || !sameDockerBootstrapJournal(beforeRestore, activeJournal)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: "durable transaction authority changed before original restoration", + }); + } + restoreOriginal(activeJournal, deps); + const restored = inspectExact(activeJournal.originalRuntimeId, deps); + assertStableRunning(restored, "restored workload"); + if ( + dockerContainerName(restored) !== activeJournal.originalName || + normalizeDockerManagedBootstrapLaunchSpec(restored).hash !== activeJournal.originalSpecHash + ) { + throw new Error("Managed bootstrap Docker rollback did not restore its exact original."); + } + removeDockerBootstrapJournalDurably(activeJournal, deps); + retainOwnedWorkloadForOwnerCleanup(handle.sandbox, deps, activeJournal.originalRuntimeId); + return completedRollback(handle, false); + }; + const commitBootstrapNow = ( + receipt: ManagedBootstrapCompletionReceipt, + transaction: DockerBootstrapTransaction, + input: { + readonly sharedStateStatus: "committed" | "none"; + readonly sharedStateTransaction: ReturnType; + }, + ): void => { + if (committedTransactions.has(receipt.bootstrapIdentity)) return; + if ( + transaction.phase !== "shared-state-committed" || + transaction.replacementRuntimeId !== receipt.runtimeId || + transaction.originalSpecHash !== receipt.originalSpecHash || + transaction.replacementSpecHash !== receipt.replacementSpecHash + ) { + throw new Error("Managed bootstrap Docker commit receipt does not match its commit fence."); + } + const current = deps.journalStore.load(transaction.bootstrapIdentity); + if (!current || !sameDockerBootstrapJournal(current, transaction)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId: transaction.originalRuntimeId, + detail: "durable commit authority changed before exact cleanup", + }); + } + + const replacement = inspectTransactionRuntime( + transaction, + transaction.replacementRuntimeId, + deps, + ); + if (!replacement) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId: transaction.replacementRuntimeId, + detail: "the exact committed replacement is absent", + }); + } + assertTransactionReplacement(transaction, replacement); + if ( + dockerContainerName(replacement) !== transaction.originalName || + replacement.State?.Running !== true + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId: transaction.replacementRuntimeId, + detail: "the exact replacement is not running under the authoritative workload name", + }); + } + + const original = inspectTransactionRuntime(transaction, transaction.originalRuntimeId, deps); + if (original) { + assertTransactionOriginal(transaction, original); + if (dockerContainerName(original) !== transaction.backupName) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId: transaction.originalRuntimeId, + detail: "the exact rollback backup is not quiescent under its durable backup name", + }); + } + assertExplicitlyStopped(original, "commit rollback backup"); + const beforeRemove = deps.journalStore.load(transaction.bootstrapIdentity); + if (!beforeRemove || !sameDockerBootstrapJournal(beforeRemove, transaction)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId: transaction.originalRuntimeId, + detail: "durable commit authority changed before exact rollback-backup removal", + }); + } + const removed = deps.dockerRm(transaction.originalRuntimeId, { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + if ( + !hasZeroDockerExitStatus(removed) && + probeExactDockerContainerAbsence(transaction.originalRuntimeId, deps) !== "absent" + ) { + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: receipt.bootstrapIdentity, + cleanupRuntimeId: transaction.originalRuntimeId, + detail: `${commandDetail(removed) || "Docker removal failed"}; exact backup absence was not proven`, + }); + } + } + + if (input.sharedStateStatus === "committed") { + try { + clearDockerManagedStartupSharedStateCommitReceipt(input.sharedStateTransaction, deps); + } catch (error) { + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: receipt.bootstrapIdentity, + cleanupRuntimeId: transaction.replacementRuntimeId, + detail: `exact rollback backup is absent, but its image-owned commit receipt could not be retired: ${ + error instanceof Error ? error.message : String(error) + }`, + }); + } + } + removeDockerBootstrapJournalDurably(transaction, deps); + committedTransactions.add(receipt.bootstrapIdentity); + }; + const finalizeBootstrap = async ( + input: Parameters[0], + ): Promise => { + if (input.outcome === "rollback") { + return rollbackBootstrapNow(input); + } + const { completion, durablePreparation, handle, prepared, replacement, snapshot } = input; + if (!completion || !snapshot || !prepared || !durablePreparation || !replacement) { + throw new Error("Managed bootstrap commit requires one complete cutover receipt."); + } + const preparedAuthority = transactionFromPreparedAuthority(handle, snapshot, prepared); + assertDurablePreparationAuthority(handle, snapshot, prepared, durablePreparation); + const sharedTransaction = managedSharedStateTransaction( + handle, + replacement.replacementRuntimeId, + replacement.runtimeImageContentId, + ); + let sharedStatus = probeDockerManagedStartupSharedState( + { + transaction: sharedTransaction, + profileFingerprint: completion.profileFingerprint, + }, + deps, + ); + let journal = deps.journalStore.load(handle.bootstrapIdentity); + + if (!journal) { + if (committedTransactions.has(completion.bootstrapIdentity)) { + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + outcome: "committed", + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: false, + alreadyRolledBack: false, + finalizedAt: deps.now().toISOString(), + }); + } + const originalPresence = probeExactDockerContainerAbsence(snapshot.runtimeId, deps); + if (originalPresence === "unknown") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: completion.bootstrapIdentity, + runtimeId: snapshot.runtimeId, + detail: "the retired-journal commit cannot prove exact backup absence", + }); + } + if (originalPresence !== "absent" || sharedStatus !== "none") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: completion.bootstrapIdentity, + runtimeId: + originalPresence === "absent" ? replacement.replacementRuntimeId : snapshot.runtimeId, + detail: + "the durable journal is absent before both exact backup and shared commit receipt retirement were proven", + }); + } + const committedReplacement = inspectExact(replacement.replacementRuntimeId, deps); + assertStableRunning(committedReplacement, "committed replacement"); + const originalName = dockerContainerName( + parseDockerManagedBootstrapLaunchSpec(snapshot.specCanonicalJson).inspect, + ); + if ( + dockerContainerName(committedReplacement) !== originalName || + normalizeDockerManagedBootstrapLaunchSpec(committedReplacement).hash !== + replacement.replacementSpecHash + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: completion.bootstrapIdentity, + runtimeId: replacement.replacementRuntimeId, + detail: "the retired-journal replacement does not match the exact completion receipt", + }); + } + committedTransactions.add(completion.bootstrapIdentity); + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + outcome: "committed", + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: false, + alreadyRolledBack: false, + finalizedAt: deps.now().toISOString(), + }); + } + + if ( + !sameDockerBootstrapJournal( + Object.freeze({ ...journal, phase: "staged" as const }), + preparedAuthority, + ) + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "durable Docker commit changed its prepared rollback authority", + }); + } + assertDockerBootstrapTransactionAuthority(journal, handle, snapshot, prepared, replacement); + if (journal.phase === "staged" || journal.phase === "rollback-authorized") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: `commit is forbidden from durable journal phase ${journal.phase}`, + }); + } + if (!completion.transactionPending && sharedStatus !== "none") { + throw new Error( + "Managed bootstrap image completion disagrees with shared-state transaction status.", + ); + } + + if (journal.phase === "cutover") { + if (completion.transactionPending && sharedStatus === "none") { + throw new Error( + "Managed bootstrap image completion lost its shared-state receipt before the durable commit fence.", + ); + } + if (sharedStatus === "pending") { + let outcome; + try { + outcome = finalizeDockerManagedStartupSharedState( + { + transaction: sharedTransaction, + supervisorReady: true, + retainContainerAfterRollback: true, + }, + deps, + ); + } catch (error) { + if (error instanceof DockerManagedStartupSharedStateCommitIndeterminateError) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: completion.bootstrapIdentity, + runtimeId: replacement.replacementRuntimeId, + detail: error.message, + }); + } + throw error; + } + if (!outcome.supervisorReady) { + const failure = + outcome.failure ?? new Error("Managed bootstrap shared-state commit did not complete."); + try { + const current = deps.journalStore.load(journal.bootstrapIdentity); + if (!current || !sameDockerBootstrapJournal(current, journal)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: + "durable authority changed after shared-state rollback and before restoration", + }); + } + transitionDockerBootstrapJournalDurably(journal, "rollback-authorized", deps); + await rollbackBootstrapNow({ + handle, + snapshot, + prepared, + durablePreparation, + replacement, + sharedStateAlreadyRolledBack: true, + }); + } catch (rollbackError) { + attachManagedBootstrapRollbackError(failure, rollbackError); + } + throw failure; + } + sharedStatus = "committed"; + } + journal = transitionDockerBootstrapJournalDurably(journal, "shared-state-committed", deps); + } else if (completion.transactionPending && sharedStatus === "pending") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "shared state is pending after the durable Docker commit fence", + }); + } + + commitBootstrapNow(completion, journal, { + sharedStateStatus: sharedStatus === "committed" ? "committed" : "none", + sharedStateTransaction: sharedTransaction, + }); + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + outcome: "committed", + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: false, + alreadyRolledBack: false, + finalizedAt: deps.now().toISOString(), + }); + }; + return { + async createHeldWorkload(input) { + if ( + input.plan.schemaVersion !== MANAGED_BOOTSTRAP_SCHEMA_VERSION || + input.plan.driverId !== DOCKER_DRIVER_ID || + input.request.agent !== input.plan.profile.agent || + input.request.profileFingerprint !== input.plan.profile.fingerprint + ) { + throw new Error("Managed bootstrap Docker create plan does not match its root request."); + } + const bootstrapIdentity = input.bootstrapIdentity ?? deps.createBootstrapIdentity(); + assertManagedBootstrapIdentity(bootstrapIdentity); + const heldWorkloadArgv = renderManagedBootstrapHeldCommand( + input.request, + bootstrapIdentity, + input.plan.intendedWorkloadArgv, + ); + const createReceipt = await input.launch({ heldWorkloadArgv, bootstrapIdentity }); + if ( + createReceipt.ready !== true || + createReceipt.sandbox.sandboxName !== input.plan.sandboxName || + createReceipt.sandbox.driverId !== input.plan.driverId || + !createReceipt.sandbox.sandboxId + ) { + throw new Error( + "Managed bootstrap Docker create did not return one Ready durable sandbox identity.", + ); + } + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: Object.freeze({ ...createReceipt.sandbox }), + bootstrapIdentity, + heldWorkloadArgv, + intendedWorkloadArgv: Object.freeze([...input.plan.intendedWorkloadArgv]), + plan: input.plan, + createReceipt, + }); + }, + + async cleanupIncompleteCreate(input) { + const { sandbox, runtimeId } = resolveIncompleteCreateSandbox(input, deps); + retainOwnedWorkloadForOwnerCleanup(sandbox, deps, runtimeId); + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox, + bootstrapIdentity: input.bootstrapIdentity, + outcome: "rolled-back", + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: true, + alreadyRolledBack: false, + finalizedAt: deps.now().toISOString(), + }); + }, + + async discoverHeldWorkload( + input: ManagedBootstrapDiscoveryInput, + ): Promise { + if (input.sandbox.driverId !== DOCKER_DRIVER_ID) { + throw new Error("Managed bootstrap Docker adapter received another runtime driver."); + } + const query = queryOpenShellDockerSandboxContainers(input.sandbox.sandboxName, deps); + if (!query.ok) { + throw new Error(`Managed bootstrap Docker discovery failed: ${query.error}`); + } + if (query.ids.length !== 1) { + throw new Error( + `Managed bootstrap requires exactly one labeled Docker workload after Ready; found ${String( + query.ids.length, + )}.`, + ); + } + const runtimeId = String(query.ids[0] ?? "").toLowerCase(); + const inspect = inspectExact(runtimeId, deps); + assertStableRunning(inspect, "held workload"); + assertRootSupervisor(inspect); + assertImage(inspect, input.expectedImage, deps); + assertMetadata(inspect, input.sandbox, input.metadata); + assertBootstrapIdentityInObservedHold(inspect, input.bootstrapIdentity); + return Object.freeze({ + sandbox: input.sandbox, + runtimeId, + bootstrapIdentity: input.bootstrapIdentity, + }); + }, + + async inspectHeldWorkload({ handle, discovered }) { + if ( + discovered.bootstrapIdentity !== handle.bootstrapIdentity || + discovered.sandbox.sandboxId !== handle.sandbox.sandboxId || + discovered.sandbox.driverId !== handle.sandbox.driverId + ) { + throw new Error("Managed bootstrap Docker identity changed before inspection."); + } + const first = inspectExact(discovered.runtimeId, deps); + assertStableRunning(first, "held workload"); + assertRootSupervisor(first); + assertNoRootProcessInjectionEnvironment(first.Config?.Env); + const runtimeImageContentId = assertImage(first, handle.plan.image, deps); + assertMetadata(first, handle.sandbox, handle.plan.metadata); + assertHeldCommand(first, handle.heldWorkloadArgv, handle.bootstrapIdentity); + const firstNormalized = normalizeDockerManagedBootstrapLaunchSpec(first); + const inspect = inspectExact(discovered.runtimeId, deps); + assertStableRunning(inspect, "held workload"); + assertRootSupervisor(inspect); + assertNoRootProcessInjectionEnvironment(inspect.Config?.Env); + if (assertImage(inspect, handle.plan.image, deps) !== runtimeImageContentId) { + throw new Error("Managed bootstrap Docker image content changed during stable capture."); + } + assertMetadata(inspect, handle.sandbox, handle.plan.metadata); + assertHeldCommand(inspect, handle.heldWorkloadArgv, handle.bootstrapIdentity); + const normalized = normalizeDockerManagedBootstrapLaunchSpec(inspect); + if ( + normalized.hash !== firstNormalized.hash || + normalized.canonicalJson !== firstNormalized.canonicalJson + ) { + throw new Error("Managed bootstrap Docker launch spec changed during stable capture."); + } + const supervisorArgv = exactSupervisorArgv(inspect); + if (!exactArrayEqual(supervisorArgv, handle.plan.expectedSupervisorArgv)) { + throw new Error("Managed bootstrap Docker supervisor argv changed before replacement."); + } + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + runtimeId: discovered.runtimeId, + bootstrapIdentity: handle.bootstrapIdentity, + image: handle.plan.image, + runtimeImageContentId, + specHash: normalized.hash, + specCanonicalJson: normalized.canonicalJson, + agentIdentity: Object.freeze({ ...handle.plan.agentIdentity }), + supervisorArgv, + heldWorkloadArgv: handle.heldWorkloadArgv, + metadata: handle.plan.metadata, + }); + }, + + async prepareBootstrapReplacement({ handle, snapshot, request, replacementOptions }) { + if ( + snapshot.bootstrapIdentity !== handle.bootstrapIdentity || + !FULL_CONTAINER_ID_RE.test(snapshot.runtimeId) || + request.agent !== handle.plan.profile.agent || + request.profileFingerprint !== handle.plan.profile.fingerprint + ) { + throw new Error("Managed bootstrap Docker replacement identities do not match."); + } + const parsed = parseDockerManagedBootstrapLaunchSpec(snapshot.specCanonicalJson); + const normalizedOriginal = normalizeDockerManagedBootstrapLaunchSpec(parsed.inspect); + if (normalizedOriginal.hash !== snapshot.specHash) { + throw new Error("Managed bootstrap Docker replacement snapshot is not exact."); + } + assertNoRootProcessInjectionEnvironment(parsed.inspect.Config?.Env); + if (parsed.inspect.HostConfig?.ReadonlyRootfs === true) { + throw new Error( + "Managed bootstrap cannot stage its root-owned request in a read-only root filesystem.", + ); + } + const plan = replacementPlan(replacementOptions); + const originalName = dockerContainerName(parsed.inspect); + const backupContainerName = backupName(originalName, handle.bootstrapIdentity); + const stagingName = replacementStagingName(originalName, handle.bootstrapIdentity); + const existingJournal = deps.journalStore.load(handle.bootstrapIdentity); + if (existingJournal) { + assertDockerBootstrapTransactionAuthority(existingJournal, handle, snapshot); + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: existingJournal.bootstrapIdentity, + runtimeId: existingJournal.replacementRuntimeId, + detail: `preparation requires rollback or commit from durable phase ${existingJournal.phase}`, + }); + } + const trampolineCommand = replacementCommand(handle, snapshot); + const cloneArgs = buildDockerGpuCloneRunArgs(parsed.inspect, plan.mode, { + image: expectedImageReference(snapshot.image.repository, snapshot.image.manifestDigest), + openshellSandboxCommand: handle.intendedWorkloadArgv, + requiredUlimits: plan.requiredUlimits, + extraGroupGids: plan.extraGroupGids, + containerEntrypoint: MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE, + containerCommand: trampolineCommand, + containerName: stagingName, + }); + const options = { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }; + + let requestFile = ""; + let replacementRuntimeId = ""; + let stagedAuthority: DockerBootstrapTransaction | null = null; + try { + const created = deps.dockerRun(["create", ...cloneArgs], options); + const returnedRuntimeId = String(created.stdout ?? "") + .trim() + .toLowerCase(); + let createdInspect: DockerContainerInspect; + if (FULL_CONTAINER_ID_RE.test(returnedRuntimeId)) { + replacementRuntimeId = returnedRuntimeId; + createdInspect = inspectExact(replacementRuntimeId, deps); + } else { + try { + createdInspect = inspectDockerContainerReference(stagingName, deps); + } catch (lookupError) { + throw new Error( + "Managed bootstrap could not prove a stopped Docker replacement after create: " + + (commandDetail(created) || + (lookupError instanceof Error ? lookupError.message : String(lookupError))), + ); + } + replacementRuntimeId = String(createdInspect.Id ?? "").toLowerCase(); + } + if ( + !FULL_CONTAINER_ID_RE.test(replacementRuntimeId) || + dockerContainerName(createdInspect) !== stagingName + ) { + throw new Error( + "Managed bootstrap Docker create did not resolve one stopped identity-bound staging container.", + ); + } + assertExplicitlyStopped(createdInspect, "created replacement"); + const createdImageContentId = assertImage(createdInspect, snapshot.image, deps); + if (createdImageContentId !== snapshot.runtimeImageContentId) { + throw new Error( + "Managed bootstrap Docker replacement resolved a different image content ID.", + ); + } + assertMetadata(createdInspect, handle.sandbox, snapshot.metadata); + assertRootSupervisor(createdInspect); + const intendedSandboxCommand = openshellSandboxCommandEnvValue(handle.intendedWorkloadArgv); + if (!intendedSandboxCommand) { + throw new Error( + "Managed bootstrap Docker replacement requires one bounded intended workload argv.", + ); + } + assertReplacementBoundary(createdInspect, handle, snapshot); + const expectedActivatedSpecHash = assertReplacementMatchesIntent( + snapshot.specCanonicalJson, + createdInspect, + originalName, + plan, + intendedSandboxCommand, + ); + const preparedSpec = normalizeDockerManagedBootstrapLaunchSpec(createdInspect); + const expectedActivatedSpec = normalizeDockerManagedBootstrapLaunchSpec({ + ...createdInspect, + Name: `/${originalName}`, + }); + if (expectedActivatedSpec.hash !== expectedActivatedSpecHash) { + throw new Error("Managed bootstrap Docker expected activation spec is inconsistent."); + } + stagedAuthority = Object.freeze({ + schemaVersion: DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, + phase: "staged", + bootstrapIdentity: handle.bootstrapIdentity, + sandbox: Object.freeze({ ...handle.sandbox }), + profileFingerprint: handle.plan.profile.fingerprint, + imageReference: expectedImageReference( + snapshot.image.repository, + snapshot.image.manifestDigest, + ), + runtimeImageContentId: snapshot.runtimeImageContentId, + originalRuntimeId: snapshot.runtimeId, + replacementRuntimeId, + originalName, + replacementStagingName: stagingName, + backupName: backupContainerName, + originalSpecHash: snapshot.specHash, + replacementSpecHash: expectedActivatedSpecHash, + }); + + requestFile = writeProtectedEnvelope(handle.bootstrapIdentity, request); + const copied = deps.dockerRun( + ["cp", requestFile, replacementRuntimeId + ":" + MANAGED_BOOTSTRAP_REQUEST_FILE], + options, + ); + assertZero( + copied, + "Managed bootstrap could not stage its protected root-owned 0400 envelope", + ); + + const originalBeforeJournal = inspectExact(snapshot.runtimeId, deps); + assertStableRunning(originalBeforeJournal, "pre-journal original"); + if ( + dockerContainerName(originalBeforeJournal) !== originalName || + normalizeDockerManagedBootstrapLaunchSpec(originalBeforeJournal).hash !== + snapshot.specHash + ) { + throw new Error( + "Managed bootstrap Docker original changed while the replacement was staged.", + ); + } + const replacementBeforeJournal = inspectExact(replacementRuntimeId, deps); + assertTransactionReplacement(stagedAuthority, replacementBeforeJournal); + const observedPreparedSpec = + normalizeDockerManagedBootstrapLaunchSpec(replacementBeforeJournal); + const observedActivatedSpec = normalizeDockerManagedBootstrapLaunchSpec({ + ...replacementBeforeJournal, + Name: `/${originalName}`, + }); + if ( + dockerContainerName(replacementBeforeJournal) !== stagingName || + observedPreparedSpec.canonicalJson !== preparedSpec.canonicalJson || + observedActivatedSpec.canonicalJson !== expectedActivatedSpec.canonicalJson + ) { + throw new Error("Managed bootstrap Docker replacement changed before durable staging."); + } + assertExplicitlyStopped(replacementBeforeJournal, "pre-journal replacement"); + + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + originalRuntimeId: snapshot.runtimeId, + preparedRuntimeId: replacementRuntimeId, + image: snapshot.image, + runtimeImageContentId: snapshot.runtimeImageContentId, + originalSpecHash: snapshot.specHash, + preparedSpecHash: preparedSpec.hash, + preparedSpecCanonicalJson: preparedSpec.canonicalJson, + expectedActivatedSpecHash, + expectedActivatedSpecCanonicalJson: expectedActivatedSpec.canonicalJson, + profileFingerprint: handle.plan.profile.fingerprint, + rollbackAuthority: serializeDockerManagedBootstrapJournal(stagedAuthority), + }); + } catch (error) { + let rollbackError: unknown = null; + try { + const durable = deps.journalStore.load(handle.bootstrapIdentity); + if (!durable) { + cleanupUnjournaledPreparedContainer( + { snapshot, preparedRuntimeId: replacementRuntimeId, stagingName }, + deps, + ); + } + } catch (cleanupError) { + rollbackError = cleanupError; + } + const failure = error instanceof Error ? error : new Error(String(error)); + if (rollbackError) attachManagedBootstrapRollbackError(failure, rollbackError); + throw failure; + } finally { + if (requestFile) cleanupTempDir(requestFile, REQUEST_TEMP_PREFIX); + } + }, + async activateBootstrapReplacement({ handle, snapshot, prepared, durablePreparation }) { + const authority = transactionFromPreparedAuthority(handle, snapshot, prepared); + assertDurablePreparationAuthority(handle, snapshot, prepared, durablePreparation); + const existingJournal = deps.journalStore.load(handle.bootstrapIdentity); + if (existingJournal) { + assertDockerBootstrapTransactionAuthority(existingJournal, handle, snapshot, prepared); + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: existingJournal.bootstrapIdentity, + runtimeId: existingJournal.replacementRuntimeId, + detail: `activation requires rollback or commit from durable phase ${existingJournal.phase}`, + }); + } + const options = { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }; + try { + const originalBeforeJournal = inspectExact(snapshot.runtimeId, deps); + const preparedBeforeJournal = inspectExact(prepared.preparedRuntimeId, deps); + assertTransactionOriginal(authority, originalBeforeJournal); + assertTransactionReplacement(authority, preparedBeforeJournal); + assertStableRunning(originalBeforeJournal, "pre-activation original"); + assertExplicitlyStopped(preparedBeforeJournal, "pre-activation replacement"); + if ( + dockerContainerName(originalBeforeJournal) !== authority.originalName || + dockerContainerName(preparedBeforeJournal) !== authority.replacementStagingName || + normalizeDockerManagedBootstrapLaunchSpec(preparedBeforeJournal).canonicalJson !== + prepared.preparedSpecCanonicalJson + ) { + throw new Error( + "Managed bootstrap Docker prepared runtimes changed before durable activation.", + ); + } + + let journal = createDockerBootstrapJournalDurably(authority, deps); + const originalAtFence = inspectExact(snapshot.runtimeId, deps); + const replacementAtFence = inspectExact(prepared.preparedRuntimeId, deps); + assertTransactionOriginal(journal, originalAtFence); + assertTransactionReplacement(journal, replacementAtFence); + if ( + dockerContainerName(originalAtFence) !== journal.originalName || + originalAtFence.State?.Running !== true || + dockerContainerName(replacementAtFence) !== journal.replacementStagingName || + normalizeDockerManagedBootstrapLaunchSpec(replacementAtFence).canonicalJson !== + prepared.preparedSpecCanonicalJson + ) { + throw new Error( + "Managed bootstrap Docker staged runtimes changed before the cutover fence.", + ); + } + assertExplicitlyStopped(replacementAtFence, "staged replacement"); + journal = transitionDockerBootstrapJournalDurably(journal, "cutover", deps); + + const stopped = deps.dockerStop(snapshot.runtimeId, { + ...options, + timeout: DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + }); + const afterStop = inspectExact(snapshot.runtimeId, deps); + assertTransactionOriginal(journal, afterStop); + if (dockerContainerName(afterStop) !== journal.originalName) { + throw new Error( + "Managed bootstrap could not prove its exact original stopped after Docker stop: " + + (commandDetail(stopped) || "state did not reach stopped"), + ); + } + assertExplicitlyStopped(afterStop, "stopped original"); + + const renamedOriginal = deps.dockerRename(snapshot.runtimeId, journal.backupName, options); + const afterOriginalRename = inspectExact(snapshot.runtimeId, deps); + assertTransactionOriginal(journal, afterOriginalRename); + if (dockerContainerName(afterOriginalRename) !== journal.backupName) { + throw new Error( + "Managed bootstrap could not prove its exact original backup rename: " + + (commandDetail(renamedOriginal) || "name did not reach backup"), + ); + } + assertExplicitlyStopped(afterOriginalRename, "renamed rollback backup"); + + const renamedReplacement = deps.dockerRename( + prepared.preparedRuntimeId, + journal.originalName, + options, + ); + const afterReplacementRename = inspectExact(prepared.preparedRuntimeId, deps); + assertTransactionReplacement(journal, afterReplacementRename); + if (dockerContainerName(afterReplacementRename) !== journal.originalName) { + throw new Error( + "Managed bootstrap could not prove its exact replacement cutover rename: " + + (commandDetail(renamedReplacement) || "name did not reach target"), + ); + } + + const started = deps.dockerStart(prepared.preparedRuntimeId, options); + const running = inspectExact(prepared.preparedRuntimeId, deps); + assertTransactionReplacement(journal, running); + const runningSpec = normalizeDockerManagedBootstrapLaunchSpec(running); + if ( + dockerContainerName(running) !== journal.originalName || + running.State?.Running !== true || + running.State.Paused === true || + running.State.Restarting === true || + running.State.Dead === true || + runningSpec.canonicalJson !== prepared.expectedActivatedSpecCanonicalJson + ) { + throw new Error( + "Managed bootstrap could not prove its exact replacement running after Docker start: " + + (commandDetail(started) || "state did not reach running"), + ); + } + assertReplacementBoundary(running, handle, snapshot); + const preservedOriginal = inspectExact(snapshot.runtimeId, deps); + assertTransactionOriginal(journal, preservedOriginal); + if (dockerContainerName(preservedOriginal) !== journal.backupName) { + throw new Error("Managed bootstrap Docker rollback backup changed during cutover."); + } + assertExplicitlyStopped(preservedOriginal, "preserved rollback backup"); + + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + originalRuntimeId: snapshot.runtimeId, + replacementRuntimeId: prepared.preparedRuntimeId, + image: snapshot.image, + runtimeImageContentId: snapshot.runtimeImageContentId, + originalSpecHash: snapshot.specHash, + replacementSpecHash: prepared.expectedActivatedSpecHash, + replacementSpecCanonicalJson: prepared.expectedActivatedSpecCanonicalJson, + profileFingerprint: handle.plan.profile.fingerprint, + }); + } catch (error) { + let rollbackError: unknown = null; + try { + if (!deps.journalStore.load(handle.bootstrapIdentity)) { + cleanupUnjournaledPreparedContainer( + { + snapshot, + preparedRuntimeId: prepared.preparedRuntimeId, + stagingName: authority.replacementStagingName, + }, + deps, + ); + } + } catch (cleanupError) { + rollbackError = cleanupError; + } + const failure = error instanceof Error ? error : new Error(String(error)); + if (rollbackError) attachManagedBootstrapRollbackError(failure, rollbackError); + throw failure; + } + }, + async awaitBootstrap({ handle, snapshot, replacement, timeoutSecs }) { + if ( + replacement.bootstrapIdentity !== handle.bootstrapIdentity || + replacement.originalRuntimeId !== snapshot.runtimeId || + replacement.profileFingerprint !== handle.plan.profile.fingerprint + ) { + throw new Error("Managed bootstrap Docker completion identities do not match."); + } + const journal = reconstructDockerBootstrapTransaction(handle, snapshot, replacement, deps); + if (journal.phase !== "cutover") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: `bootstrap completion is invalid from durable journal phase ${journal.phase}`, + }); + } + assertCompletedCutoverRuntimeState(journal, deps); + const before = inspectExact(replacement.replacementRuntimeId, deps); + assertStableRunning(before, "replacement"); + const beforeImageContentId = assertImage(before, replacement.image, deps); + if (beforeImageContentId !== replacement.runtimeImageContentId) { + throw new Error("Managed bootstrap Docker replacement image content changed."); + } + assertReplacementBoundary(before, handle, snapshot); + if (!waitForOpenShellSupervisorReconnect(handle.sandbox.sandboxName, timeoutSecs, deps)) { + throw new Error("Managed bootstrap Docker supervisor did not reconnect."); + } + const afterWaitJournal = deps.journalStore.load(journal.bootstrapIdentity); + if (!afterWaitJournal || !sameDockerBootstrapJournal(afterWaitJournal, journal)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "durable transaction authority changed while awaiting bootstrap", + }); + } + assertCompletedCutoverRuntimeState(afterWaitJournal, deps); + const after = inspectExact(replacement.replacementRuntimeId, deps); + assertStableRunning(after, "completed replacement"); + if (assertImage(after, replacement.image, deps) !== replacement.runtimeImageContentId) { + throw new Error("Managed bootstrap Docker completed image content changed."); + } + assertReplacementBoundary(after, handle, snapshot); + const normalized = normalizeDockerManagedBootstrapLaunchSpec(after); + if (normalized.hash !== replacement.replacementSpecHash) { + throw new Error("Managed bootstrap Docker replacement changed during bootstrap."); + } + const imageCompletion = readProtectedImageCompletion(replacement.replacementRuntimeId, deps); + if ( + imageCompletion.bootstrapIdentity !== replacement.bootstrapIdentity || + imageCompletion.agent !== handle.plan.profile.agent || + imageCompletion.profileFingerprint !== replacement.profileFingerprint + ) { + throw new Error( + "Managed bootstrap Docker image completion identities do not match the transaction.", + ); + } + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + runtimeId: replacement.replacementRuntimeId, + image: replacement.image, + runtimeImageContentId: replacement.runtimeImageContentId, + originalSpecHash: replacement.originalSpecHash, + replacementSpecHash: replacement.replacementSpecHash, + profileFingerprint: replacement.profileFingerprint, + bootstrapIdentity: replacement.bootstrapIdentity, + transactionPending: imageCompletion.transactionPending, + completedAt: deps.now().toISOString(), + }); + }, + + finalizeBootstrap, + }; +} diff --git a/src/lib/onboard/managed-startup-shared-state-transaction.test.ts b/src/lib/onboard/managed-startup-shared-state-transaction.test.ts index 0d28be2cbe2..3022fc71850 100644 --- a/src/lib/onboard/managed-startup-shared-state-transaction.test.ts +++ b/src/lib/onboard/managed-startup-shared-state-transaction.test.ts @@ -313,6 +313,25 @@ describe("managed startup shared-state transaction", () => { expect(rollbackManagedStartupSharedStateTransaction("openclaw", options)).toBe(true); }); + it("rejects a malformed rollback receipt before restoring shared state", () => { + const root = agentRoot("openclaw"); + fs.mkdirSync(root); + const config = path.join(root, "openclaw.json"); + fs.writeFileSync(config, "before\n"); + beginManagedStartupSharedStateTransaction(managedStartupE2eProfile("openclaw"), options); + fs.writeFileSync(config, "changed\n"); + const manifest = path.join(transactionDirectory, "manifest.json"); + fs.chmodSync(manifest, 0o600); + fs.writeFileSync(manifest, "{malformed\n"); + fs.chmodSync(manifest, 0o400); + + expect(() => rollbackManagedStartupSharedStateTransaction("openclaw", options)).toThrow( + /manifest is not valid JSON/u, + ); + expect(fs.readFileSync(config, "utf8")).toBe("changed\n"); + expect(fs.existsSync(transactionDirectory)).toBe(true); + }); + it("rejects an oversized managed output before creating a receipt", () => { const root = agentRoot("openclaw"); fs.mkdirSync(root); diff --git a/src/lib/onboard/openshell-docker-sandbox-containers.ts b/src/lib/onboard/openshell-docker-sandbox-containers.ts index a8934027fd6..94c067f027c 100644 --- a/src/lib/onboard/openshell-docker-sandbox-containers.ts +++ b/src/lib/onboard/openshell-docker-sandbox-containers.ts @@ -7,6 +7,7 @@ import type { DockerGpuPatchDeps } from "./docker-gpu-patch-types"; export const OPENSHELL_MANAGED_BY_LABEL = "openshell.ai/managed-by"; export const OPENSHELL_MANAGED_BY_VALUE = "openshell"; export const OPENSHELL_SANDBOX_NAME_LABEL = "openshell.ai/sandbox-name"; +export const OPENSHELL_SANDBOX_ID_LABEL = "openshell.ai/sandbox-id"; const DOCKER_SANDBOX_QUERY_TIMEOUT_MS = 30_000; diff --git a/src/lib/onboard/temp-files.test.ts b/src/lib/onboard/temp-files.test.ts index 8e36513c998..6643be81160 100644 --- a/src/lib/onboard/temp-files.test.ts +++ b/src/lib/onboard/temp-files.test.ts @@ -27,6 +27,29 @@ describe("onboard temp file helpers", () => { expect(path.basename(filePath)).toBe("nemoclaw-test.txt"); }); + it.skipIf(process.platform === "win32")( + "creates an owner-only real parent and rejects a planted target with exclusive creation", + () => { + const filePath = secureTempFile("nemoclaw-exclusive", ".txt"); + const parent = path.dirname(filePath); + createdParents.push(parent); + const parentStat = fs.lstatSync(parent); + const plantedTarget = path.join(parent, "planted-target.txt"); + fs.writeFileSync(plantedTarget, "untouched\n", { mode: 0o600 }); + + expect(parentStat.isDirectory()).toBe(true); + expect(parentStat.isSymbolicLink()).toBe(false); + expect(parentStat.mode & 0o777).toBe(0o700); + expect(fs.existsSync(filePath)).toBe(false); + + fs.symlinkSync(plantedTarget, filePath); + expect(() => + fs.writeFileSync(filePath, "replacement\n", { flag: "wx", mode: 0o400 }), + ).toThrowError(expect.objectContaining({ code: "EEXIST" })); + expect(fs.readFileSync(plantedTarget, "utf8")).toBe("untouched\n"); + }, + ); + it("rejects temp prefixes with path separators", () => { expect(() => secureTempFile("../nemoclaw-test", ".txt")).toThrow("Invalid temp file prefix"); expect(() => secureTempFile("nested/nemoclaw-test", ".txt")).toThrow( diff --git a/test/package-contract/openshell-policy-boundary.test.ts b/test/package-contract/openshell-policy-boundary.test.ts index 5d24e319db3..5cb1809f531 100644 --- a/test/package-contract/openshell-policy-boundary.test.ts +++ b/test/package-contract/openshell-policy-boundary.test.ts @@ -212,7 +212,7 @@ describe("OpenShell policy boundary package contract", () => { expect(validation).toEqual([true, false]); }); - it("ships an out-of-tree runtime sandbox-policy schema validator", { timeout: 90_000 }, () => { + it("ships an out-of-tree runtime sandbox-policy schema validator", { timeout: 240_000 }, () => { const productionDependencyTree = spawnSync( "npm", ["ls", "ajv", "--omit=dev", "--all", "--json"], diff --git a/test/runtime-provider-source-shape.test.ts b/test/runtime-provider-source-shape.test.ts index 94d235cad52..48e889d78c1 100644 --- a/test/runtime-provider-source-shape.test.ts +++ b/test/runtime-provider-source-shape.test.ts @@ -122,7 +122,10 @@ describe("runtime provider central source boundary", () => { expect(bootstrapProtocolPaths).toEqual([ "src/lib/onboard/managed-bootstrap/adapter.ts", "src/lib/onboard/managed-bootstrap/docker-journal.ts", + "src/lib/onboard/managed-bootstrap/docker-shared-state.ts", "src/lib/onboard/managed-bootstrap/docker-spec.ts", + "src/lib/onboard/managed-bootstrap/docker-test-fixture.ts", + "src/lib/onboard/managed-bootstrap/docker.ts", "src/lib/onboard/managed-bootstrap/envelope.ts", "src/lib/onboard/managed-bootstrap/index.ts", ]); diff --git a/tsconfig.src.json b/tsconfig.src.json index da1287ae436..13c12fb1310 100644 --- a/tsconfig.src.json +++ b/tsconfig.src.json @@ -16,5 +16,10 @@ "types": ["node"] }, "include": ["src/**/*.ts"], - "exclude": ["node_modules", "nemoclaw", "src/**/*.test.ts"] + "exclude": [ + "node_modules", + "nemoclaw", + "src/**/*.test.ts", + "src/**/*-test-fixture.ts" + ] }