Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions src/lib/onboard/docker-gpu-local-inference.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -425,6 +425,27 @@ describe("verifyGpuSandboxLocalInferenceAndCommitAfterReady", () => {
expect(runtimePatch.rollbackManagedStartupAfterCreateFailure).toHaveBeenCalledOnce();
expect(runtimePatch.commitAfterReady).not.toHaveBeenCalled();
});

it("treats a failed commit as terminal without attempting rollback", async () => {
const runtimePatch = {
commitAfterReady: vi.fn(async () => {
throw new Error("durable commit acknowledgement failed");
}),
rollbackManagedStartupAfterCreateFailure: vi.fn(),
};
await expect(
verifyGpuSandboxLocalInferenceAndCommitAfterReady(
GPU_CONFIG,
"ollama-local",
{
...options(),
deps: { execInSandbox: execEmitting("HTTP_200"), sleep: vi.fn() },
},
runtimePatch,
),
).rejects.toThrow("durable commit acknowledgement failed");
expect(runtimePatch.rollbackManagedStartupAfterCreateFailure).not.toHaveBeenCalled();
});
});

describe("printDockerGpuSandboxInferenceVerificationFailure", () => {
Expand Down
2 changes: 1 addition & 1 deletion src/lib/onboard/docker-gpu-local-inference.ts
Original file line number Diff line number Diff line change
Expand Up @@ -512,7 +512,6 @@ export async function verifyGpuSandboxLocalInferenceAndCommitAfterReady(
): Promise<void> {
try {
verifyGpuSandboxLocalInferenceAfterReady(config, provider, options);
await runtimePatch.commitAfterReady();
} catch (error) {
const failure = error instanceof Error ? error : new Error(String(error));
try {
Expand All @@ -524,4 +523,5 @@ export async function verifyGpuSandboxLocalInferenceAndCommitAfterReady(
}
throw failure;
}
await runtimePatch.commitAfterReady();
}
33 changes: 32 additions & 1 deletion src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,8 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => {
patch.waitForSupervisorReconnectIfNeeded();
expect(onPatchFailureExit).not.toHaveBeenCalled();

await patch.commitAfterReady();
await expect(patch.commitAfterReady()).rejects.toThrow("rollback backup");
await expect(patch.commitAfterReady()).rejects.toThrow("rollback backup");

expect(onPatchFailureExit).toHaveBeenCalledOnce();
expect(onPatchFailureExit.mock.calls[0]?.[1]).toEqual(
Expand All @@ -177,6 +178,36 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => {
);
});

it("rejects an early commit after rolling back before supervisor reconnect", async () => {
const deps = makeDeps();
const result = deferredCreateResult();
const finalizeBackup = vi.fn(() => ({ backupRemoved: false, rolledBack: true }));
const onPatchFailureExit = vi.fn();
const patch = createDockerGpuSandboxCreatePatch({
route: "compatibility",
sandboxName: "alpha",
timeoutSecs: 60,
deps,
overrides: {
findContainerIds: vi.fn(() => ["existing-container"]),
recreatePatch: vi.fn(() => result),
finalizeBackup,
onPatchFailureExit,
},
});

patch.maybeApplyDuringCreate();

await expect(patch.commitAfterReady()).rejects.toThrow(
"cannot commit before the recreated OpenShell supervisor reconnects",
);
await expect(patch.commitAfterReady()).rejects.toThrow(
"cannot commit before the recreated OpenShell supervisor reconnects",
);
expect(finalizeBackup).toHaveBeenCalledWith({ result, supervisorReady: false }, deps);
expect(onPatchFailureExit).toHaveBeenCalledOnce();
});

it("rolls back to the backup container and surfaces rolledBack=true diagnostics when supervisorReady=false", () => {
const deps = makeDeps();
const result = deferredCreateResult();
Expand Down
46 changes: 24 additions & 22 deletions src/lib/onboard/docker-gpu-sandbox-create.ts
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,7 @@ export function createDockerGpuSandboxCreatePatch(
let cutoverFinalized = false;
let cutoverFinalization: Promise<void> | null = null;
let cutoverFinalizationOutcome: "commit" | "rollback" | null = null;
let cutoverFinalizationFailure: Error | null = null;

const findContainerIds =
options.overrides?.findContainerIds ?? findOpenShellDockerSandboxContainerIds;
Expand Down Expand Up @@ -374,24 +375,23 @@ export function createDockerGpuSandboxCreatePatch(
},

async commitAfterReady() {
if (cutoverFinalizationFailure) throw cutoverFinalizationFailure;
if (cutoverFinalized || (!managedBootstrapCutover && !result)) return;
if (needsSupervisorWait) {
const error = new Error(
"Managed startup cannot commit before the recreated OpenShell supervisor reconnects.",
);
const rollbackError = await rollbackAfterFailure();
onPatchFailureExit(
options.sandboxName,
rollbackError
? new Error(`${error.message} Rollback failed: ${rollbackError.message}`)
: error,
{
runCaptureOpenshell: options.deps.runCaptureOpenshell,
dockerCapture: options.deps.dockerCapture,
additionalSummaryLines: routeAdapter.additionalSummaryLines,
},
);
return;
const failure = rollbackError
? new Error(`${error.message} Rollback failed: ${rollbackError.message}`)
: error;
cutoverFinalizationFailure = failure;
onPatchFailureExit(options.sandboxName, failure, {
runCaptureOpenshell: options.deps.runCaptureOpenshell,
dockerCapture: options.deps.dockerCapture,
additionalSummaryLines: routeAdapter.additionalSummaryLines,
});
throw failure;
Comment thread
coderabbitai[bot] marked this conversation as resolved.
}
if (cutoverFinalization) {
if (cutoverFinalizationOutcome !== "commit") {
Expand Down Expand Up @@ -420,6 +420,7 @@ export function createDockerGpuSandboxCreatePatch(
failure as Error & { managedBootstrapRollbackError?: unknown }
).managedBootstrapRollbackError = rollbackError;
}
cutoverFinalizationFailure = failure;
onPatchFailureExit(options.sandboxName, failure, {
runCaptureOpenshell: options.deps.runCaptureOpenshell,
dockerCapture: options.deps.dockerCapture,
Expand All @@ -429,24 +430,25 @@ export function createDockerGpuSandboxCreatePatch(
rolledBack: rollbackError === null,
},
});
return;
throw failure;
}
}
const finalizeOutcome = result
? finalizeBackup({ result, supervisorReady: true }, options.deps)
: null;
cutoverFinalized = true;
if (!finalizeOutcome || finalizeOutcome.backupRemoved) return;
onPatchFailureExit(
options.sandboxName,
new Error("Managed startup passed Ready, but its rollback backup could not be removed."),
{
runCaptureOpenshell: options.deps.runCaptureOpenshell,
dockerCapture: options.deps.dockerCapture,
additionalSummaryLines: routeAdapter.additionalSummaryLines,
context: failureContext(),
},
const failure = new Error(
"Managed startup passed Ready, but its rollback backup could not be removed.",
);
cutoverFinalizationFailure = failure;
onPatchFailureExit(options.sandboxName, failure, {
runCaptureOpenshell: options.deps.runCaptureOpenshell,
dockerCapture: options.deps.dockerCapture,
additionalSummaryLines: routeAdapter.additionalSummaryLines,
context: failureContext(),
});
throw failure;
})();
cutoverFinalization = finalization;
cutoverFinalizationOutcome = "commit";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -233,7 +233,7 @@ describe("Docker startup-command sandbox creation", () => {
rollback,
});

await patch.commitAfterReady();
await expect(patch.commitAfterReady()).rejects.toThrow("receipt validation failed");

expect(events).toEqual(["commit", "rollback", "exit"]);
expect(onPatchFailureExit).toHaveBeenCalledWith(
Expand Down
16 changes: 13 additions & 3 deletions src/lib/onboard/managed-bootstrap/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -85,9 +85,12 @@ and sandbox ID and then enter the destructive cutover. Post-cutover rollback
publishes `rollback-authorized` before exact replacement deletion; pre-cutover
staged cleanup removes only the exact prepared replacement without that journal
transition. Commit publishes `shared-state-committed` before exact backup
deletion. Cleanup is bound to full runtime IDs. Its private state root retains
versioned, identity-addressed transaction records containing the provider and
sandbox identities, plan and profile
deletion. Cleanup is bound to full runtime IDs. Commit or rollback is claimed
synchronously before asynchronous finalization begins. Repeated calls for the
claimed outcome share its one pending result, while the opposite outcome remains
invalid even if acknowledgement of the first finalization is lost. Its private
state root retains versioned, identity-addressed transaction records containing
the provider and sandbox identities, plan and profile
fingerprints, exact original and replacement IDs, rollback target, and phase.
Exact commit and cleanup receipts are durable terminal records, so adapter
recreation does not depend on process-local transaction sets or tombstone maps.
Expand All @@ -102,6 +105,13 @@ commit atomically moves its pending manifest and backups into a durable receipt
namespace, compacts that state to an exact commit receipt, and rejects rollback
after a restart. The provider may retire that receipt only after it proves the
external rollback backup is gone, leaving the next bootstrap attempt unblocked.
The parser accepts the exact canonical schema-v1 manifest written before
`bootstrapIdentity` was added only for the legacy null-identity path. It rejects
additional fields, missing historical fields, and legacy state presented as
identity-bound authority. Before rollback, the Docker adapter stops the
replacement and copies its writable-layer commit receipt to a protected host
path for verification. The immutable helper cannot obtain that receipt through
`--volumes-from`, which exposes volumes but not the replacement writable layer.
Direct identity lookup reconstructs one known transaction record, while managed
create-lifecycle startup uses unfinished-record enumeration to ask the selected
provider to reconcile every identity-addressed record before a new sandbox
Expand Down
43 changes: 43 additions & 0 deletions src/lib/onboard/managed-bootstrap/adapter.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import {
type ManagedBootstrapAuthorityStore,
type ManagedBootstrapCompletionReceipt,
type ManagedBootstrapCreateReceipt,
type ManagedBootstrapDurablePreparationReceipt,
type ManagedBootstrapFinalizationReceipt,
type ManagedBootstrapHeldWorkloadHandle,
type ManagedBootstrapObservedSnapshot,
Expand All @@ -29,7 +30,10 @@ import {
prepareManagedBootstrapSequence,
recoverManagedBootstrapTransactions,
renderManagedBootstrapHeldCommand,
sameManagedBootstrapCompletionReceipt,
sameManagedBootstrapDurablePreparationReceipt,
} from "./adapter";
import { reverseKeys } from "./managed-bootstrap-test-fixture";

const IDENTITY = "1".repeat(64);
const CONFIG_ID = `sha256:${"2".repeat(64)}`;
Expand Down Expand Up @@ -356,6 +360,45 @@ async function captureFailure<T>(promise: Promise<T>) {
}

describe("managed bootstrap adapter contract", () => {
it("compares provider-neutral durable receipts by canonical value", () => {
const handle = handleFor(requestFor("hermes"));
const preparation: ManagedBootstrapDurablePreparationReceipt = {
schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION,
sandbox: handle.sandbox,
bootstrapIdentity: handle.bootstrapIdentity,
authorityFingerprint: "a".repeat(64),
recordId: "mxc-durable-authority",
recordedAt: "2026-07-29T12:00:30.000Z",
};
const reorderedPreparation = reverseKeys({
...preparation,
sandbox: reverseKeys({ ...preparation.sandbox }),
});
expect(sameManagedBootstrapDurablePreparationReceipt(preparation, reorderedPreparation)).toBe(
true,
);
expect(
sameManagedBootstrapDurablePreparationReceipt(preparation, {
...reorderedPreparation,
recordId: "changed-authority",
}),
).toBe(false);

const completion = completionFor(requestFor("hermes"), handle);
const reorderedCompletion = reverseKeys({
...completion,
image: reverseKeys({ ...completion.image }),
sandbox: reverseKeys({ ...completion.sandbox }),
});
expect(sameManagedBootstrapCompletionReceipt(completion, reorderedCompletion)).toBe(true);
expect(
sameManagedBootstrapCompletionReceipt(completion, {
...reorderedCompletion,
transactionPending: false,
}),
).toBe(false);
});

it.each(
MANAGED_STARTUP_AGENTS,
)("prepares, durably records, and only then activates %s through a provider-neutral adapter", async (agent) => {
Expand Down
16 changes: 16 additions & 0 deletions src/lib/onboard/managed-bootstrap/adapter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -905,6 +905,22 @@ function canonicalJson(value: unknown): string {
.join(",")}}`;
}

/** Compare durable provider receipts by canonical value, independent of object key order. */
export function sameManagedBootstrapDurablePreparationReceipt(
left: ManagedBootstrapDurablePreparationReceipt,
right: ManagedBootstrapDurablePreparationReceipt,
): boolean {
return canonicalJson(left) === canonicalJson(right);
}

/** Compare completion receipts by canonical value, independent of object key order. */
export function sameManagedBootstrapCompletionReceipt(
left: ManagedBootstrapCompletionReceipt,
right: ManagedBootstrapCompletionReceipt,
): boolean {
return canonicalJson(left) === canonicalJson(right);
}

export function assertManagedBootstrapIdentity(value: string): void {
if (!SHA256_RE.test(value)) {
protocolFail("identity must be 32 random bytes encoded as lowercase hex");
Expand Down
Loading
Loading