diff --git a/src/lib/onboard/managed-bootstrap/docker.test.ts b/src/lib/onboard/managed-bootstrap/docker.test.ts index 4618989b5c3..ebb3d149c0c 100644 --- a/src/lib/onboard/managed-bootstrap/docker.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker.test.ts @@ -316,6 +316,10 @@ describe("Docker managed bootstrap adapter", () => { metadata: handle.plan.metadata, }), ).rejects.toThrow("one exact persisted bootstrap identity"); + expect(fake.replacement).toBeNull(); + expect(fake.journal).toBeNull(); + expect(fake.events).not.toContain("create:replacement"); + expect(fake.events).not.toContain(`stop:${OLD_ID}`); }); it("publishes durable commit authority before deleting the rollback backup after lost acknowledgements", async () => { @@ -511,7 +515,11 @@ describe("Docker managed bootstrap adapter", () => { return { status: 1 }; }); fake.deps.runCaptureOpenshell = vi.fn(() => "alpha Error"); - fake.deps.dockerLogs = vi.fn(() => `managed startup failed with NVIDIA_API_KEY=${secret}`); + fake.deps.dockerLogs = vi.fn((id, options) => { + expect(id).toBe(NEW_ID); + expect(options).toEqual({ tail: 120, timeout: 2_000 }); + return `${"oversized diagnostic context ".repeat(60)}managed startup failed with NVIDIA_API_KEY=${secret}`; + }); const adapter = createDockerManagedBootstrapAdapter(fake.deps); const { handle, request, snapshot } = authority(); const prepared = await adapter.prepareBootstrapReplacement({ @@ -539,6 +547,92 @@ describe("Docker managed bootstrap adapter", () => { "managed startup failed with NVIDIA_API_KEY=", ); expect((failure as Error).message).not.toContain(secret); + const redactedLogTail = (failure as Error).message.split("Redacted replacement log tail:\n")[1]; + expect(redactedLogTail).toBeDefined(); + expect(redactedLogTail!.length).toBeLessThanOrEqual(1_200); + }); + + it("reports an exited replacement through exact authorized cleanup (#9465)", async () => { + const fake = fixture({ agent: "openclaw" }); + const secret = "replacement-diagnostic-secret"; + fake.deps.dockerLogs = vi.fn((id, options) => { + expect(id).toBe(NEW_ID); + expect(options).toEqual({ tail: 120, timeout: 2_000 }); + return `managed startup rejected SLACK_BOT_TOKEN=${secret}`; + }); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request, snapshot } = authority("openclaw"); + const prepared = await adapter.prepareBootstrapReplacement({ + handle, + snapshot, + request, + replacementOptions: { values: {} }, + }); + const durable = durablePreparation(handle, snapshot, prepared); + const replacement = await adapter.activateBootstrapReplacement({ + handle, + snapshot, + prepared, + durablePreparation: durable, + }); + assert(fake.replacement?.State); + Object.assign(fake.replacement.State, { + Status: "exited", + Running: false, + ExitCode: 23, + FinishedAt: "2026-08-18T12:00:00.000Z", + }); + + const failure = await adapter + .awaitBootstrap({ handle, snapshot, replacement, timeoutSecs: 1 }) + .catch((error: unknown) => error); + + expect(failure).toBeInstanceOf(Error); + expect((failure as Error).message).toContain(`Replacement runtime ID: ${NEW_ID}.`); + expect((failure as Error).message).toContain( + "Replacement state: status=exited running=false exit_code=23 finished_at=2026-08-18T12:00:00.000Z.", + ); + expect((failure as Error).message).toContain( + "managed startup rejected SLACK_BOT_TOKEN=", + ); + expect((failure as Error).message).not.toContain(secret); + + await expect( + adapter.finalizeBootstrap({ + outcome: "rollback", + handle, + snapshot, + prepared, + durablePreparation: durable, + replacement, + completion: null, + }), + ).rejects.toMatchObject({ + name: "ManagedBootstrapOwnerCleanupRequiredError", + runtimeId: OLD_ID, + }); + expect(fake.replacement).toBeNull(); + expect(fake.original).toMatchObject({ + Id: OLD_ID, + Name: "/openshell-alpha", + State: { Running: false }, + }); + + fake.removeOriginalExternally(); + await expect(adapter.recoverUnfinishedTransactions()).resolves.toMatchObject({ + receipts: [ + { + bootstrapIdentity: IDENTITY, + sourcePhase: "owner-cleanup-required", + outcome: "rolled-back", + }, + ], + failures: [], + }); + expect(fake.original).toBeNull(); + expect(fake.replacement).toBeNull(); + expect(fake.journal).toBeNull(); + expect(fake.finalization?.phase).toBe("rolled-back"); }); it("preserves commit validation failure details when the replacement cannot be quiesced", async () => { @@ -589,11 +683,34 @@ describe("Docker managed bootstrap adapter", () => { }); it("publishes durable rollback authority before deleting the replacement after restart", async () => { - const fake = fixture({ - dockerStartResults: { - [NEW_ID]: { status: 1, stderr: "injected start failure" }, - }, + const fake = fixture(); + const secret = "post-start-provider-secret"; + fake.deps.dockerLogs = vi.fn((id, options) => { + expect(id).toBe(NEW_ID); + expect(options).toEqual({ tail: 120, timeout: 2_000 }); + return `startup failed with NVIDIA_API_KEY=${secret}`; }); + const dockerStarts: Record< + string, + () => { status: number; stdout?: string; stderr: string } + > = { + [NEW_ID]: () => { + assert(fake.replacement?.State); + Object.assign(fake.replacement.State, { + Status: "exited", + Running: false, + ExitCode: 31, + FinishedAt: "2026-08-18T12:30:00.000Z", + }); + return { status: 1, stderr: "injected start failure" }; + }, + [OLD_ID]: () => { + assert(fake.original?.State); + Object.assign(fake.original.State, { Running: true }); + return { status: 0, stdout: "", stderr: "" }; + }, + }; + fake.deps.dockerStart = vi.fn((id) => dockerStarts[id]!()); const first = createDockerManagedBootstrapAdapter(fake.deps); const { handle, request: rootRequest, snapshot } = authority(); const prepared = await first.prepareBootstrapReplacement({ @@ -603,14 +720,26 @@ describe("Docker managed bootstrap adapter", () => { replacementOptions: { values: {} }, }); const durable = durablePreparation(handle, snapshot, prepared); - await expect( - first.activateBootstrapReplacement({ + const activationFailure = await first + .activateBootstrapReplacement({ handle, snapshot, prepared, durablePreparation: durable, - }), - ).rejects.toThrow("could not prove its exact replacement running"); + }) + .catch((error: unknown) => error); + expect(activationFailure).toBeInstanceOf(Error); + expect((activationFailure as Error).message).toContain( + "replacement after Docker start is not stably running", + ); + expect((activationFailure as Error).message).toContain(`Replacement runtime ID: ${NEW_ID}.`); + expect((activationFailure as Error).message).toContain( + "Replacement state: status=exited running=false exit_code=31 finished_at=2026-08-18T12:30:00.000Z.", + ); + expect((activationFailure as Error).message).toContain( + "startup failed with NVIDIA_API_KEY=", + ); + expect((activationFailure as Error).message).not.toContain(secret); expect(fake.journal?.phase).toBe("cutover"); const restarted = createDockerManagedBootstrapAdapter(fake.deps); diff --git a/src/lib/onboard/managed-bootstrap/docker.ts b/src/lib/onboard/managed-bootstrap/docker.ts index 9294e2140a2..2f5c9c72c5f 100644 --- a/src/lib/onboard/managed-bootstrap/docker.ts +++ b/src/lib/onboard/managed-bootstrap/docker.ts @@ -264,6 +264,21 @@ function supervisorReconnectFailureDetail(runtimeId: string, deps: ResolvedDeps) .join(" "); } +function replacementNotStableError(runtimeId: string, label: string, deps: ResolvedDeps): Error { + const evidence = captureDockerContainerFailureEvidence(runtimeId, deps); + const stateDetail = formatDockerContainerState(evidence.state).join(" "); + return new Error( + [ + `Managed bootstrap Docker ${label} is not stably running.`, + `Replacement runtime ID: ${runtimeId}.`, + stateDetail ? `Replacement state: ${stateDetail}.` : "", + evidence.redactedLogTail ? `Redacted replacement log tail:\n${evidence.redactedLogTail}` : "", + ] + .filter(Boolean) + .join(" "), + ); +} + function isExactMissingDockerContainer(containerId: string, result: DockerCommandResult): boolean { const escapedContainerId = containerId.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); const patterns = [ @@ -1201,7 +1216,9 @@ function assertCompletedCutoverRuntimeState( assertTransactionOriginal(transaction, original); assertTransactionReplacement(transaction, replacement); assertExplicitlyStopped(original, "rollback backup"); - assertStableRunning(replacement, "replacement"); + if (!isStableRunning(replacement)) { + throw replacementNotStableError(transaction.replacementRuntimeId, "replacement", deps); + } if ( dockerContainerName(original) !== transaction.backupName || dockerContainerName(replacement) !== transaction.originalName @@ -3575,13 +3592,16 @@ export function createDockerManagedBootstrapAdapter( const started = deps.dockerStart(prepared.preparedRuntimeId, options); const running = inspectExact(prepared.preparedRuntimeId, deps); assertTransactionReplacement(journal, running); + if (!isStableRunning(running)) { + throw replacementNotStableError( + journal.replacementRuntimeId, + "replacement after Docker start", + deps, + ); + } const runningSpec = normalizeDockerManagedBootstrapLaunchSpec(running); if ( dockerContainerName(running) !== journal.originalName || - running.State?.Running !== true || - running.State.Paused === true || - running.State.Restarting === true || - running.State.Dead === true || runningSpec.canonicalJson !== prepared.expectedActivatedSpecCanonicalJson ) { throw new Error( @@ -3649,13 +3669,16 @@ export function createDockerManagedBootstrapAdapter( } assertCompletedCutoverRuntimeState(journal, deps); const before = inspectExact(replacement.replacementRuntimeId, deps); - assertStableRunning(before, "replacement"); + if (!isStableRunning(before)) { + throw replacementNotStableError(replacement.replacementRuntimeId, "replacement", deps); + } const beforeImageContentId = assertImage(before, replacement.image, deps); if (beforeImageContentId !== replacement.runtimeImageContentId) { throw new Error("Managed bootstrap Docker replacement image content changed."); } assertReplacementBoundary(before, handle, snapshot); - const supervisorReconnectTimeoutSecs = getDockerGpuSupervisorReconnectTimeoutSecs(timeoutSecs); + const supervisorReconnectTimeoutSecs = + getDockerGpuSupervisorReconnectTimeoutSecs(timeoutSecs); if ( !waitForOpenShellSupervisorReconnect( handle.sandbox.sandboxName, @@ -3675,7 +3698,13 @@ export function createDockerManagedBootstrapAdapter( } assertCompletedCutoverRuntimeState(afterWaitJournal, deps); const after = inspectExact(replacement.replacementRuntimeId, deps); - assertStableRunning(after, "completed replacement"); + if (!isStableRunning(after)) { + throw replacementNotStableError( + replacement.replacementRuntimeId, + "completed replacement", + deps, + ); + } if (assertImage(after, replacement.image, deps) !== replacement.runtimeImageContentId) { throw new Error("Managed bootstrap Docker completed image content changed."); }