Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions src/lib/onboard/managed-bootstrap/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,13 @@ commit atomically moves its pending manifest and backups into a durable receipt
namespace, compacts that state to an exact commit receipt, and rejects rollback
after a restart. The provider may retire that receipt only after it proves the
external rollback backup is gone, leaving the next bootstrap attempt unblocked.
The parser accepts the exact canonical schema-v1 manifest written before
`bootstrapIdentity` was added only for the legacy null-identity path. It rejects
additional fields, missing historical fields, and legacy state presented as
identity-bound authority. Before rollback, the Docker adapter stops the
replacement and copies its writable-layer commit receipt to a protected host
path for verification. The immutable helper cannot obtain that receipt through
`--volumes-from`, which exposes volumes but not the replacement writable layer.
At managed create-lifecycle startup, the driver-neutral coordinator asks the
selected provider to reconcile every unfinished record before a new sandbox
create begins. The Docker provider then resumes the durable phase monotonically:
Expand Down
279 changes: 279 additions & 0 deletions src/lib/onboard/managed-bootstrap/docker-shared-state.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,279 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import fs from "node:fs";
import path from "node:path";

import { afterEach, describe, expect, it, vi } from "vitest";

import type { DockerGpuPatchDeps } from "../docker-gpu-patch-types";
import {
MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY,
MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY,
MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY,
} from "../managed-startup/shared-state-transaction";
import {
type DockerManagedBootstrapSharedStateTransaction,
finalizeDockerManagedStartupSharedState,
} from "./docker-shared-state";

const CONTAINER_ID = "c".repeat(64);
const TRANSACTION: DockerManagedBootstrapSharedStateTransaction = {
agent: "openclaw",
bootstrapIdentity: "b".repeat(64),
containerId: CONTAINER_ID,
image: `sha256:${"a".repeat(64)}`,
profileFingerprint: "d".repeat(64),
};

interface SharedStateFixture {
readonly commands: readonly (readonly string[])[];
readonly deps: DockerGpuPatchDeps;
readonly events: readonly string[];
readonly state: () => "committed" | "none" | "pending";
}

interface SharedStateFixtureOptions {
readonly stateAfterCommitFailure?: "committed" | "none" | "pending";
readonly stateAfterStop?: "committed" | "none" | "pending";
}

const copiedReceiptPaths: string[] = [];

afterEach(() => {
vi.restoreAllMocks();
for (const receiptPath of copiedReceiptPaths.splice(0)) {
fs.rmSync(path.dirname(receiptPath), { force: true, recursive: true });
}
});

function fixture(
initialState: "committed" | "none" | "pending",
options: SharedStateFixtureOptions = {},
): SharedStateFixture {
let state = initialState;
const commands: string[][] = [];
const events: string[] = [];
const copyPresentReceipt = (destination: string) => {
fs.mkdirSync(destination, { recursive: true });
copiedReceiptPaths.push(destination);
return { status: 0 };
};
const copyMissingReceipt = (sourcePath: string) => ({
status: 1,
stderr: `Error response from daemon: Could not find the file ${sourcePath} in container ${CONTAINER_ID}`,
});
const dockerRun = vi.fn((args: readonly string[]) => {
commands.push([...args]);
switch (args[0]) {
case "cp": {
const source = String(args[2] ?? "");
const destination = String(args[3] ?? "");
const sourcePath = source.slice(`${CONTAINER_ID}:`.length);
const present =
(sourcePath === MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY &&
state === "committed") ||
(sourcePath === MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY && state === "pending");
events.push(`copy:${path.basename(sourcePath)}:${present ? "present" : "absent"}`);
return present ? copyPresentReceipt(destination) : copyMissingReceipt(sourcePath);
}
case "run": {
const action = args.includes("--shared-state-transaction-status")
? "status"
: args.includes("--rollback-shared-state-transaction")
? "rollback"
: "unexpected";
switch (action) {
case "status":
events.push(`status:${state}`);
return { status: 0, stdout: `${state}\n` };
case "rollback":
events.push("rollback");
state = "none";
return { status: 0 };
default:
throw new Error(`Unexpected Docker command: ${args.join(" ")}`);
}
}
case "exec":
switch (args.includes("--commit-shared-state-transaction")) {
case true:
events.push("commit:failed");
state = options.stateAfterCommitFailure ?? state;
return { status: 1, stderr: "commit helper failed" };
default:
throw new Error("Unexpected Docker commit command");
}
default:
throw new Error(`Unexpected Docker command: ${args.join(" ")}`);
}
});
return {
commands,
deps: {
dockerRm: vi.fn(() => ({ status: 0 })),
dockerRun,
dockerStop: vi.fn(() => {
events.push("stop");
state = options.stateAfterStop ?? state;
return { status: 0 };
}),
},
events,
state: () => state,
};
}

describe("Docker managed-bootstrap shared-state rollback authority", () => {
it("copies and verifies writable-layer commit authority before rollback", () => {
const fake = fixture("committed");

expect(() =>
finalizeDockerManagedStartupSharedState(
{ transaction: TRANSACTION, supervisorReady: false },
fake.deps,
),
).toThrow(/durably committed and cannot be rolled back/u);

expect(fake.events).toEqual([
"stop",
`copy:${path.basename(MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY)}:present`,
"status:committed",
]);
const statusCommand = fake.commands.find((args) =>
args.includes("--shared-state-transaction-status"),
);
expect(statusCommand).toContainEqual(
expect.stringMatching(
new RegExp(
`^type=bind,src=.+,dst=${MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY},readonly$`,
"u",
),
),
);
expect(fake.commands.some((args) => args.includes("--rollback-shared-state-transaction"))).toBe(
false,
);
});

it("proves pending authority after quiescence before starting the rollback helper", () => {
const fake = fixture("pending");

expect(
finalizeDockerManagedStartupSharedState(
{ transaction: TRANSACTION, supervisorReady: false },
fake.deps,
),
).toEqual({ supervisorReady: false, failure: null });

expect(fake.events[0]).toBe("stop");
expect(fake.events.indexOf("status:pending")).toBeLessThan(fake.events.indexOf("rollback"));
expect(fake.state()).toBe("none");
const rollbackCommand = fake.commands.find((args) =>
args.includes("--rollback-shared-state-transaction"),
);
expect(rollbackCommand).toContainEqual(
expect.stringMatching(
new RegExp(
`^type=bind,src=.+,dst=${MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY},readonly$`,
"u",
),
),
);
});

it("removes the exact failed container without rollback when both receipts are absent", () => {
const fake = fixture("none");

expect(
finalizeDockerManagedStartupSharedState(
{ transaction: TRANSACTION, supervisorReady: false },
fake.deps,
),
).toEqual({ supervisorReady: false, failure: null });

expect(fake.events).toEqual([
"stop",
`copy:${path.basename(MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY)}:absent`,
`copy:${path.basename(MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY)}:absent`,
]);
expect(fake.commands.some((args) => args.includes("--rollback-shared-state-transaction"))).toBe(
false,
);
expect(fake.deps.dockerRm).toHaveBeenCalledTimes(1);
expect(fake.deps.dockerRm).toHaveBeenCalledWith(CONTAINER_ID, expect.any(Object));
expect(fake.state()).toBe("none");
});

it("reuses one preserved pending receipt when commit validation fails", () => {
const fake = fixture("pending", { stateAfterCommitFailure: "none" });

const outcome = finalizeDockerManagedStartupSharedState(
{
retainContainerAfterRollback: true,
transaction: TRANSACTION,
supervisorReady: true,
},
fake.deps,
);

expect(outcome.supervisorReady).toBe(false);
expect(outcome.failure).toEqual(
expect.objectContaining({
message: expect.stringContaining("commit helper failed"),
}),
);
const pendingSource = CONTAINER_ID + ":" + MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY;
const pendingCopies = fake.commands.filter(
(args) => args[0] === "cp" && args[2] === pendingSource,
);
expect(
fake.events.filter(
(event) =>
event ===
"copy:" + path.basename(MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY) + ":present",
),
).toHaveLength(1);
const preservedReceiptPath = String(pendingCopies[0]?.[3] ?? "");
const rollbackCommand = fake.commands.find((args) =>
args.includes("--rollback-shared-state-transaction"),
);
expect(rollbackCommand).toContain(
"type=bind,src=" +
preservedReceiptPath +
",dst=" +
MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY +
",readonly",
);
expect(fake.deps.dockerRm).not.toHaveBeenCalled();
});

it("rejects rollback when a failed commit becomes durable during quiescence", () => {
const fake = fixture("pending", {
stateAfterCommitFailure: "none",
stateAfterStop: "committed",
});

expect(() =>
finalizeDockerManagedStartupSharedState(
{ transaction: TRANSACTION, supervisorReady: true },
fake.deps,
),
).toThrow(/durably committed and cannot be rolled back/u);

expect(
fake.events.filter(
(event) =>
event ===
"copy:" + path.basename(MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY) + ":present",
),
).toHaveLength(1);
expect(fake.events).toContain("commit:failed");
expect(fake.events).toContain("status:committed");
expect(fake.events).not.toContain("rollback");
expect(fake.commands.some((args) => args.includes("--rollback-shared-state-transaction"))).toBe(
false,
);
expect(fake.deps.dockerRm).not.toHaveBeenCalled();
});
});
27 changes: 16 additions & 11 deletions src/lib/onboard/managed-bootstrap/docker-shared-state.ts
Original file line number Diff line number Diff line change
Expand Up @@ -465,10 +465,21 @@ function copyManagedStartupReceipt(

function rollbackManagedStartupSharedState(
transaction: DockerManagedBootstrapSharedStateTransaction,
receiptPath: string,
deps: DockerGpuPatchDeps,
): void {
preservedReceiptPath?: string,
): boolean {
const dockerRun = deps.dockerRun ?? defaultDockerRun;
const status = probeDockerManagedStartupSharedState(
{ transaction, profileFingerprint: transaction.profileFingerprint },
deps,
);
if (status === "committed") {
throw new Error("Managed-startup shared state is durably committed and cannot be rolled back.");
}
const receiptPath =
preservedReceiptPath ??
(status === "pending" ? copyManagedStartupReceipt(transaction, deps) : null);
if (!receiptPath) return false;
let restored = false;
try {
const helper = dockerRun(
Expand Down Expand Up @@ -517,6 +528,7 @@ function rollbackManagedStartupSharedState(
cleanupReceiptBestEffort(receiptPath);
}
}
return true;
}

function removeFailedUnbackedContainer(
Expand Down Expand Up @@ -605,22 +617,15 @@ export function finalizeDockerManagedStartupSharedState(
`OpenShell supervisor reconnected, but managed shared-state logical commit validation failed: ${commitFailure.message}`,
);
quiesceManagedStartupContainer(transaction, deps);
rollbackManagedStartupSharedState(transaction, receiptPath, deps);
rollbackManagedStartupSharedState(transaction, deps, receiptPath);
if (!input.patchResult && !input.retainContainerAfterRollback) {
removeFailedUnbackedContainer(transaction, deps);
}
return { supervisorReady: false, failure };
}

quiesceManagedStartupContainer(transaction, deps);
const receiptPath = copyManagedStartupReceipt(transaction, deps, true);
if (!receiptPath) {
if (!input.patchResult && !input.retainContainerAfterRollback) {
removeFailedUnbackedContainer(transaction, deps);
}
return { supervisorReady: false, failure: null };
}
rollbackManagedStartupSharedState(transaction, receiptPath, deps);
rollbackManagedStartupSharedState(transaction, deps);
if (!input.patchResult && !input.retainContainerAfterRollback) {
removeFailedUnbackedContainer(transaction, deps);
}
Expand Down
Loading
Loading