diff --git a/src/lib/actions/sandbox/connect-hermes-accepted-readiness.test.ts b/src/lib/actions/sandbox/connect-hermes-accepted-readiness.test.ts index 35b6bb7a500..1a676531567 100644 --- a/src/lib/actions/sandbox/connect-hermes-accepted-readiness.test.ts +++ b/src/lib/actions/sandbox/connect-hermes-accepted-readiness.test.ts @@ -390,7 +390,32 @@ describe("Hermes accepted launch-readiness probe", () => { snapshot: {}, assertCurrent: assertRequalifiedReceiptCurrent, } as never); - harness.recoverPortableDemoLifecycleSpy.mockReturnValue({ kind: "recovered" }); + harness.recoverPortableDemoLifecycleSpy.mockImplementation((...args) => { + args[4]?.onComplete({ + entryQualificationMs: 101, + containerStartMs: 102, + postStartCurrentnessMs: 103, + execReadyMs: 104, + preHealthCurrentnessMs: 105, + authenticatedHealthMs: 106, + startupLaunchMs: 107, + healthPollCurrentnessMs: 108, + finalQualificationMs: 109, + rollbackMs: 0, + qualificationCount: 2, + transactionCurrentnessCount: 20, + containerInspectionCount: 8, + containerStartCount: 1, + execReadyAttempts: 1, + authenticatedHealthCount: 1, + startupLaunchCount: 0, + rollbackCount: 0, + totalMs: 938, + containerAction: "started", + result: "recovered", + }); + return { kind: "recovered" }; + }); await expect(harness.connectSandbox("alpha", { probeOnly: true })).resolves.toBeUndefined(); @@ -406,6 +431,83 @@ describe("Hermes accepted launch-readiness probe", () => { expect(harness.inspectLaunchReadinessSpy).toHaveBeenCalledOnce(); expect(harness.publishLaunchReadinessSpy).not.toHaveBeenCalled(); expect(harness.logSpy.mock.calls.flat().join("\n")).toMatch(/result=ready/); + expect(harness.logSpy.mock.calls.flat().join("\n")).toContain( + "Hermes Portable lifecycle recovery timing: entryQualification=101ms containerStart=102ms postStartCurrentness=103ms execReady=104ms preHealthCurrentness=105ms authenticatedHealth=106ms startupLaunch=107ms healthPollCurrentness=108ms finalQualification=109ms rollback=0ms qualificationCount=2 transactionCurrentnessCount=20 containerInspectionCount=8 containerStartCount=1 execReadyAttempts=1 authenticatedHealthCount=1 startupLaunchCount=0 rollbackCount=0 total=938ms containerAction=started result=recovered", + ); + }); + + it("reuses one recovered lifecycle when missing readiness routes to stopped inference", async () => { + const harness = missingHermesHarness("stopped"); + harness.qualifyHermesPortableAcceptedReadinessAuthoritySpy + .mockImplementationOnce(() => { + throw new Error("stopped container has no current operating authority"); + }) + .mockReturnValue({ + kind: "current", + commandAuthority: { + assertCurrent: harness.assertHermesPortableOperatingCommandCurrentSpy, + assertTransactionCurrent: harness.assertHermesPortableOperatingCommandCurrentSpy, + receipt: {} as never, + env: {}, + executablePath: "/usr/bin/openshell", + }, + }); + const assertRequalifiedReceiptCurrent = vi.fn(); + harness.requalifyPortableAgentAuthoritySpy.mockReturnValue({ + kind: "already-current", + snapshot: {}, + assertCurrent: assertRequalifiedReceiptCurrent, + } as never); + harness.recoverPortableDemoLifecycleSpy.mockReturnValue({ kind: "recovered" }); + + await expect(harness.connectSandbox("alpha", { probeOnly: true })).resolves.toBeUndefined(); + + expect(harness.recoverPortableDemoLifecycleSpy).toHaveBeenCalledOnce(); + expect(harness.inspectHermesPortableOllamaReadinessRuntimeSpy).toHaveBeenCalledOnce(); + expect(harness.recoverHermesPortableOllamaInferenceSpy).toHaveBeenCalledOnce(); + expect(harness.publishLaunchReadinessSpy).toHaveBeenCalledOnce(); + expect(assertRequalifiedReceiptCurrent.mock.calls.length).toBeGreaterThanOrEqual(2); + expect(harness.logSpy.mock.calls.flat().join("\n")).toMatch( + /lifecycleAction=recovered forwardAction=verified result=ready/, + ); + }); + + it("rejects recovered lifecycle drift before stopped inference recovery", async () => { + const harness = missingHermesHarness("stopped"); + harness.qualifyHermesPortableAcceptedReadinessAuthoritySpy + .mockImplementationOnce(() => { + throw new Error("stopped container has no current operating authority"); + }) + .mockReturnValue({ + kind: "current", + commandAuthority: { + assertCurrent: harness.assertHermesPortableOperatingCommandCurrentSpy, + assertTransactionCurrent: harness.assertHermesPortableOperatingCommandCurrentSpy, + receipt: {} as never, + env: {}, + executablePath: "/usr/bin/openshell", + }, + }); + const assertRequalifiedReceiptCurrent = vi + .fn() + .mockImplementationOnce(() => undefined) + .mockImplementation(() => { + throw new Error("recovered receipt authority changed"); + }); + harness.requalifyPortableAgentAuthoritySpy.mockReturnValue({ + kind: "already-current", + snapshot: {}, + assertCurrent: assertRequalifiedReceiptCurrent, + } as never); + harness.recoverPortableDemoLifecycleSpy.mockReturnValue({ kind: "recovered" }); + + await expect(harness.connectSandbox("alpha", { probeOnly: true })).rejects.toThrow( + "process.exit(1)", + ); + + expect(harness.recoverPortableDemoLifecycleSpy).toHaveBeenCalledOnce(); + expect(harness.recoverHermesPortableOllamaInferenceSpy).not.toHaveBeenCalled(); + expect(harness.publishLaunchReadinessSpy).not.toHaveBeenCalled(); }); it("does not recover when stopped schema-6 requalification fails", async () => { diff --git a/src/lib/actions/sandbox/connect.ts b/src/lib/actions/sandbox/connect.ts index 27d335858de..d4fd754dd53 100644 --- a/src/lib/actions/sandbox/connect.ts +++ b/src/lib/actions/sandbox/connect.ts @@ -108,6 +108,7 @@ import { settlePortableOpenClawPairing, withLaunchReadinessMutationGate, } from "./launch-readiness"; +import type { HermesPortableLifecycleRecoveryTimingEvidence } from "../../onboard/experimental/hermes-portable-lifecycle"; import { checkAndRecoverSandboxProcesses, executeSandboxExecCommand, @@ -724,7 +725,15 @@ function writeHermesPortableForwardRecoveryTiming( evidence: HermesPortableForwardRecoveryTimingEvidence, ): void { console.log( - ` Hermes Portable forward recovery timing: list=${String(evidence.listMs)}ms listCount=${String(evidence.listCount)} stop=${String(evidence.stopMs)}ms stopCount=${String(evidence.stopCount)} start=${String(evidence.startMs)}ms startCount=${String(evidence.startCount)} settle=${String(evidence.settleMs)}ms settleCount=${String(evidence.settleCount)} total=${String(evidence.totalMs)}ms result=proved`, + ` Hermes Portable forward recovery timing: list=${String(evidence.listMs)}ms listCount=${String(evidence.listCount)} stop=${String(evidence.stopMs)}ms stopCount=${String(evidence.stopCount)} start=${String(evidence.startMs)}ms startCount=${String(evidence.startCount)} settle=${String(evidence.settleMs)}ms settleCount=${String(evidence.settleCount)} total=${String(evidence.totalMs)}ms result=${evidence.result}`, + ); +} + +function writeHermesPortableLifecycleRecoveryTiming( + evidence: HermesPortableLifecycleRecoveryTimingEvidence, +): void { + console.log( + ` Hermes Portable lifecycle recovery timing: entryQualification=${String(evidence.entryQualificationMs)}ms containerStart=${String(evidence.containerStartMs)}ms postStartCurrentness=${String(evidence.postStartCurrentnessMs)}ms execReady=${String(evidence.execReadyMs)}ms preHealthCurrentness=${String(evidence.preHealthCurrentnessMs)}ms authenticatedHealth=${String(evidence.authenticatedHealthMs)}ms startupLaunch=${String(evidence.startupLaunchMs)}ms healthPollCurrentness=${String(evidence.healthPollCurrentnessMs)}ms finalQualification=${String(evidence.finalQualificationMs)}ms rollback=${String(evidence.rollbackMs)}ms qualificationCount=${String(evidence.qualificationCount)} transactionCurrentnessCount=${String(evidence.transactionCurrentnessCount)} containerInspectionCount=${String(evidence.containerInspectionCount)} containerStartCount=${String(evidence.containerStartCount)} execReadyAttempts=${String(evidence.execReadyAttempts)} authenticatedHealthCount=${String(evidence.authenticatedHealthCount)} startupLaunchCount=${String(evidence.startupLaunchCount)} rollbackCount=${String(evidence.rollbackCount)} total=${String(evidence.totalMs)}ms containerAction=${evidence.containerAction} result=${evidence.result}`, ); } @@ -1846,11 +1855,16 @@ async function runConnectEntryPreflight( probeOnly, probeTiming, hermesPortableCommandAuthority, + retainedHermesLifecycleRecovery, withinLifecycleFence, }: { probeOnly: boolean; probeTiming?: ProbeTimingRecorder; hermesPortableCommandAuthority?: HermesPortableReadinessCommandAuthority; + retainedHermesLifecycleRecovery?: { + readonly kind: "already-running" | "recovered"; + readonly assertCurrent: () => void; + }; withinLifecycleFence?: (route: { readonly hermesPortable: boolean; readonly hermesPortableCommandAuthority?: HermesPortableReadinessCommandAuthority; @@ -1901,16 +1915,36 @@ async function runConnectEntryPreflight( assertSandboxGatewayRouteCompatible(sandboxName, registered, gatewayName), ); } - const initialRecovery = measure("lifecycle", () => - hermesPortableCommandAuthority - ? recoverPortableDemoSandboxLifecycleForConnect( - sandboxName, - registered, - gatewayName, - hermesPortableCommandAuthority, - ) - : recoverPortableDemoSandboxLifecycleForConnect(sandboxName, registered, gatewayName), - ); + const initialRecovery = retainedHermesLifecycleRecovery + ? measure("authority", () => { + retainedHermesLifecycleRecovery.assertCurrent(); + return { kind: retainedHermesLifecycleRecovery.kind } as const; + }) + : measure("lifecycle", () => + hermesPortableCommandAuthority + ? recoverPortableDemoSandboxLifecycleForConnect( + sandboxName, + registered, + gatewayName, + hermesPortableCommandAuthority, + hermesPortable && probeTiming + ? { onComplete: writeHermesPortableLifecycleRecoveryTiming } + : undefined, + ) + : hermesPortable && probeTiming + ? recoverPortableDemoSandboxLifecycleForConnect( + sandboxName, + registered, + gatewayName, + undefined, + { onComplete: writeHermesPortableLifecycleRecoveryTiming }, + ) + : recoverPortableDemoSandboxLifecycleForConnect( + sandboxName, + registered, + gatewayName, + ), + ); probeTiming?.setLifecycleAction( initialRecovery.kind === "recovered" ? "recovered" @@ -1969,12 +2003,23 @@ async function runConnectEntryPreflight( activeAuthority.entry, currentGateway, hermesPortableCommandAuthority, + probeTiming + ? { onComplete: writeHermesPortableLifecycleRecoveryTiming } + : undefined, ) - : recoverPortableDemoSandboxLifecycleForConnect( - sandboxName, - activeAuthority.entry, - currentGateway, - ), + : hermesPortable && probeTiming + ? recoverPortableDemoSandboxLifecycleForConnect( + sandboxName, + activeAuthority.entry, + currentGateway, + undefined, + { onComplete: writeHermesPortableLifecycleRecoveryTiming }, + ) + : recoverPortableDemoSandboxLifecycleForConnect( + sandboxName, + activeAuthority.entry, + currentGateway, + ), ); if (recovery.kind === "not-installed") { probeTiming?.setLifecycleAction("failed"); @@ -2235,13 +2280,14 @@ async function prepareConnectSandboxWithinLifecycleFence( readonly active: HermesPortableActiveLifecycleAuthority; readonly command: HermesPortableReadinessCommandAuthority; } | null = null; + let retainedHermesLifecycleRecovery: { + readonly kind: "already-running" | "recovered"; + readonly assertCurrent: () => void; + } | null = null; let initialPortableAuthority: ReturnType; try { initialPortableAuthority = probeTiming!.measure("authority", () => - qualifyPortableAgentLifecycleAuthority( - sandboxName, - portableAgentLifecycleAuthorityDeps(), - ), + qualifyPortableAgentLifecycleAuthority(sandboxName, portableAgentLifecycleAuthorityDeps()), ); } catch { probeTiming!.markFailureStage("authority"); @@ -2257,6 +2303,12 @@ async function prepareConnectSandboxWithinLifecycleFence( ), ); let qualified; + let recoveredLifecycle: + | { + readonly kind: "already-running" | "recovered"; + readonly assertReceiptCurrent: () => void; + } + | undefined; try { qualified = probeTiming!.measure("authority", () => qualifyHermesPortableAcceptedReadinessAuthority(sandboxName), @@ -2292,6 +2344,8 @@ async function prepareConnectSandboxWithinLifecycleFence( sandboxName, registered, resolveSandboxGatewayName(registered), + undefined, + { onComplete: writeHermesPortableLifecycleRecoveryTiming }, ), ); if (recovery.kind === "not-installed") { @@ -2299,6 +2353,10 @@ async function prepareConnectSandboxWithinLifecycleFence( throw new Error("Hermes portable lifecycle authority disappeared during probe"); } probeTiming!.setLifecycleAction(recovery.kind === "recovered" ? "recovered" : "reused"); + recoveredLifecycle = { + kind: recovery.kind, + assertReceiptCurrent: requalified.assertCurrent, + }; active = probeTiming!.measure("authority", () => requireHermesPortableActiveLifecycleAuthority( sandboxName, @@ -2311,6 +2369,8 @@ async function prepareConnectSandboxWithinLifecycleFence( priorReceiptAuthority: requalified, }), ); + hermesMissingFastPathEligible = + requalified.kind === "already-current" && qualified.kind === "current"; } if (qualified.kind === "requalification-required") { const requalified = probeTiming!.measure("authority", () => @@ -2344,6 +2404,25 @@ async function prepareConnectSandboxWithinLifecycleFence( active, command: qualified.commandAuthority, }; + if (recoveredLifecycle) { + const retainedActive = active; + const retainedCommand = qualified.commandAuthority; + retainedHermesLifecycleRecovery = { + kind: recoveredLifecycle.kind, + assertCurrent: () => { + recoveredLifecycle.assertReceiptCurrent(); + retainedCommand.assertCurrent(); + const current = requireHermesPortableActiveLifecycleAuthority( + sandboxName, + retainedActive, + portableAgentLifecycleAuthorityDeps(), + ); + if (!isDeepStrictEqual(current.entry, retainedActive.entry)) { + throw new Error("Hermes portable lifecycle authority changed after recovery"); + } + }, + }; + } } catch { probeTiming!.markFailureStage("authority"); failHermesPortableReadinessAuthority(sandboxName); @@ -2550,6 +2629,7 @@ async function prepareConnectSandboxWithinLifecycleFence( ...(hermesReadinessAuthority ? { hermesPortableCommandAuthority: hermesReadinessAuthority.command } : {}), + ...(retainedHermesLifecycleRecovery ? { retainedHermesLifecycleRecovery } : {}), withinLifecycleFence: async ({ hermesPortable, hermesPortableCommandAuthority, diff --git a/src/lib/actions/sandbox/gateway-state-observe-mode.test.ts b/src/lib/actions/sandbox/gateway-state-observe-mode.test.ts index c0582cf97ea..f784f3bfbac 100644 --- a/src/lib/actions/sandbox/gateway-state-observe-mode.test.ts +++ b/src/lib/actions/sandbox/gateway-state-observe-mode.test.ts @@ -4,11 +4,14 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import * as gatewayRuntime from "../../gateway-runtime-action"; +import * as openshellRuntime from "../../adapters/openshell/runtime"; +import * as portableAgentLifecycle from "../../onboard/experimental/portable-agent-lifecycle"; import * as registry from "../../state/registry"; import * as gatewaySelect from "./gateway-select"; import { captureHermesPortableInferenceRecoveryGateway, getReconciledSandboxGatewayState, + recoverPortableDemoSandboxLifecycleForConnect, } from "./gateway-state"; describe("getReconciledSandboxGatewayState observe mode", () => { @@ -126,3 +129,98 @@ describe("Hermes Portable inference recovery gateway", () => { ).toThrow("rejected command environment drift"); }); }); + +describe("Hermes Portable lifecycle recovery command authority", () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("uses transaction currentness for intermediate captures and full currentness at recovery boundaries", () => { + const assertCurrent = vi.fn(); + const assertTransactionCurrent = vi.fn(); + const capture = vi.spyOn(openshellRuntime, "captureResolvedOpenshell").mockReturnValue({ + status: 0, + output: "", + stdout: "", + stderr: "", + } as never); + const recover = vi + .spyOn(portableAgentLifecycle, "recoverPortableAgentSandboxLifecycle") + .mockImplementation((_sandboxName, _context, deps) => { + const recoveryDeps = deps!; + recoveryDeps.assertOpenShellExecutableAuthority?.({} as never, {}, {}); + recoveryDeps.captureOpenshell?.(["sandbox", "exec", "--", "true"], 1_000); + recoveryDeps.captureOpenshell?.(["sandbox", "exec", "--", "health"], 1_000); + recoveryDeps.assertOpenShellExecutableAuthority?.({} as never, {}, {}); + return { kind: "recovered" }; + }); + + expect( + recoverPortableDemoSandboxLifecycleForConnect( + "alpha", + { + name: "alpha", + agent: "hermes", + gatewayName: "nemoclaw", + openshellDriver: "docker", + } as never, + "nemoclaw", + { + assertCurrent, + assertTransactionCurrent, + receipt: {} as never, + env: { HOME: "/home/test" }, + executablePath: "/usr/bin/openshell", + }, + ), + ).toEqual({ kind: "recovered" }); + + expect(recover).toHaveBeenCalledOnce(); + expect(capture).toHaveBeenCalledTimes(2); + expect(assertCurrent).toHaveBeenCalledTimes(4); + expect(assertTransactionCurrent).toHaveBeenCalledTimes(4); + }); + + it("rejects transaction drift around an intermediate capture", () => { + const assertCurrent = vi.fn(); + const assertTransactionCurrent = vi + .fn() + .mockImplementationOnce(() => undefined) + .mockImplementation(() => { + throw new Error("transaction authority changed"); + }); + vi.spyOn(openshellRuntime, "captureResolvedOpenshell").mockReturnValue({ + status: 0, + output: "", + stdout: "", + stderr: "", + } as never); + vi.spyOn(portableAgentLifecycle, "recoverPortableAgentSandboxLifecycle").mockImplementation( + (_sandboxName, _context, deps) => { + deps!.captureOpenshell?.(["sandbox", "exec", "--", "true"], 1_000); + return { kind: "recovered" }; + }, + ); + + expect(() => + recoverPortableDemoSandboxLifecycleForConnect( + "alpha", + { + name: "alpha", + agent: "hermes", + gatewayName: "nemoclaw", + openshellDriver: "docker", + } as never, + "nemoclaw", + { + assertCurrent, + assertTransactionCurrent, + receipt: {} as never, + env: { HOME: "/home/test" }, + executablePath: "/usr/bin/openshell", + }, + ), + ).toThrow("transaction authority changed"); + expect(assertCurrent).toHaveBeenCalledTimes(2); + }); +}); diff --git a/src/lib/actions/sandbox/gateway-state.ts b/src/lib/actions/sandbox/gateway-state.ts index f39257f6d92..9fd8afcfd1d 100644 --- a/src/lib/actions/sandbox/gateway-state.ts +++ b/src/lib/actions/sandbox/gateway-state.ts @@ -79,6 +79,7 @@ import { recoverPortableAgentSandboxLifecycle, requireHermesPortableActiveLifecycleAuthority, } from "../../onboard/experimental/portable-agent-lifecycle"; +import type { HermesPortableLifecycleRecoveryTiming } from "../../onboard/experimental/hermes-portable-lifecycle"; import type { PortableDemoLifecycleRecoveryResult } from "../../onboard/experimental/portable-demo-lifecycle"; import { compareAndSetLegacySandboxLifecycleGeneration } from "../../state/registry/lifecycle-generation"; import type { SandboxEntry } from "../../state/registry/types"; @@ -205,9 +206,10 @@ export function recoverPortableDemoSandboxLifecycleForConnect( sandbox: SandboxEntry | null, gatewayName: string, commandAuthority?: ReturnType, + lifecycleTiming?: HermesPortableLifecycleRecoveryTiming, ): PortableDemoLifecycleRecoveryResult { const capture = (args: readonly string[], timeoutMs: number) => { - commandAuthority?.assertCurrent(); + commandAuthority?.assertTransactionCurrent(); try { const result = commandAuthority ? captureResolvedOpenshell([...args], { @@ -230,7 +232,7 @@ export function recoverPortableDemoSandboxLifecycleForConnect( error: result.error, }; } finally { - commandAuthority?.assertCurrent(); + commandAuthority?.assertTransactionCurrent(); } }; commandAuthority?.assertCurrent(); @@ -263,6 +265,7 @@ export function recoverPortableDemoSandboxLifecycleForConnect( : {}), captureOpenshell: capture, readRegistry: (name) => (sandbox?.name === name ? sandbox : null), + ...(lifecycleTiming ? { recoveryTiming: lifecycleTiming } : {}), }, ); } finally { diff --git a/src/lib/actions/sandbox/probe/hermes-portable-forward-recovery.test.ts b/src/lib/actions/sandbox/probe/hermes-portable-forward-recovery.test.ts index b4fd741cd9a..0b872426d7d 100644 --- a/src/lib/actions/sandbox/probe/hermes-portable-forward-recovery.test.ts +++ b/src/lib/actions/sandbox/probe/hermes-portable-forward-recovery.test.ts @@ -200,6 +200,7 @@ describe("Hermes Portable probe-only forward recovery", () => { settleMs: 2, settleCount: 1, totalMs: 9, + result: "proved", }); }); @@ -226,11 +227,23 @@ describe("Hermes Portable probe-only forward recovery", () => { it("rolls back a possibly started forward when detached mutation transport throws", () => { const fixture = createRecoveryFixture(); const runMutation = fixture.input.deps.runCurrentMutation; + const captureRollbackList = fixture.input.deps.captureRollbackList; + const rollbackSequence: string[] = []; + const onComplete = vi.fn( + (evidence: { readonly result: "proved" | "failed" }) => + rollbackSequence.push(`timing:${evidence.result}`), + ); Object.assign(fixture.input.deps, { runCurrentMutation: runThen(runMutation, "start", () => { throw new Error("detached mutation transport canary"); }), + captureRollbackList: (args: readonly string[], timeout: number) => { + const result = captureRollbackList(args, timeout); + rollbackSequence.push("rollback-list"); + return result; + }, }); + Object.assign(fixture.input, { timing: { onComplete } }); expect(() => recoverHermesPortableLaunchForwards(fixture.input)).toThrow( expect.objectContaining({ failure: "recovery-failed" }), @@ -244,6 +257,10 @@ describe("Hermes Portable probe-only forward recovery", () => { "nemoclaw", ]); expect(fixture.records.has(18_789)).toBe(false); + expect(onComplete).toHaveBeenCalledWith( + expect.objectContaining({ result: "failed", startCount: 1 }), + ); + expect(rollbackSequence.at(-1)).toBe("timing:failed"); }); it("rejects a returned nonzero start without the exact settled owner", () => { @@ -533,6 +550,7 @@ describe("Hermes Portable connect composition", () => { expect.objectContaining({ assertCurrent: harness.assertHermesPortableOperatingCommandCurrentSpy, }), + expect.objectContaining({ onComplete: expect.any(Function) }), ); const startCall = harness.runOpenshellSpy.mock.calls.find( ([args]) => Array.isArray(args) && args[0] === "forward" && args[1] === "start", @@ -679,6 +697,9 @@ describe("Hermes Portable connect composition", () => { "Probe complete: launch readiness is healthy", ); expect(harness.publishLaunchReadinessSpy).not.toHaveBeenCalled(); + expect(harness.logSpy.mock.calls.flat().join("\n")).toMatch( + /Hermes Portable forward recovery timing: .*listCount=1 .*result=failed/u, + ); expect( harness.runOpenshellSpy.mock.calls.some( ([args]) => Array.isArray(args) && ["start", "stop"].includes(String(args[1])), diff --git a/src/lib/actions/sandbox/probe/hermes-portable-forward-recovery.ts b/src/lib/actions/sandbox/probe/hermes-portable-forward-recovery.ts index fd2d069e8eb..30d384c3581 100644 --- a/src/lib/actions/sandbox/probe/hermes-portable-forward-recovery.ts +++ b/src/lib/actions/sandbox/probe/hermes-portable-forward-recovery.ts @@ -35,6 +35,7 @@ export interface HermesPortableForwardRecoveryTimingEvidence { readonly settleMs: number; readonly settleCount: number; readonly totalMs: number; + readonly result: "proved" | "failed"; } export interface HermesPortableForwardRecoveryTiming { @@ -113,13 +114,14 @@ function safeTimingNow(now: () => number): number | null { } function createForwardTimingRecorder(timing?: HermesPortableForwardRecoveryTiming): { - readonly finish: () => void; + readonly finish: (result: HermesPortableForwardRecoveryTimingEvidence["result"]) => void; readonly measure: (stage: ForwardTimingStage, operation: () => T) => T; } { const now = timing?.now ?? (() => performance.now()); const startedAt = safeTimingNow(now); const durations = new Map(); const counts = new Map(); + let finished = false; const elapsed = (start: number | null, end: number | null): number => { if (start === null || end === null) return 0; const value = Math.round(end - start); @@ -141,7 +143,9 @@ function createForwardTimingRecorder(timing?: HermesPortableForwardRecoveryTimin ); } }, - finish(): void { + finish(result): void { + if (finished) return; + finished = true; if (!timing) return; try { timing.onComplete( @@ -155,6 +159,7 @@ function createForwardTimingRecorder(timing?: HermesPortableForwardRecoveryTimin settleMs: durations.get("settle") ?? 0, settleCount: counts.get("settle") ?? 0, totalMs: elapsed(startedAt, safeTimingNow(now)), + result, }), ); } catch { @@ -425,16 +430,16 @@ function retainForwardRecovery( export function prepareHermesPortableLaunchForwards( input: HermesPortableForwardRecoveryInput, ): PreparedHermesPortableForwardRecovery { - validatePorts(input); const timing = createForwardTimingRecorder(input.timing); const touchedPorts: number[] = []; try { + validatePorts(input); const initial = observeForwards(input, false, timing); requireNoOccupied(initial); const missing = input.ports.filter((port) => initial.get(port) !== "healthy"); if (missing.length === 0) { requireCurrent(input, false); - timing.finish(); + timing.finish("proved"); return retainForwardRecovery(input, touchedPorts, { kind: "verified", restoredPorts: [] }); } @@ -473,20 +478,23 @@ export function prepareHermesPortableLaunchForwards( failure("recovery-failed"); } requireCurrent(input, false); - timing.finish(); + timing.finish("proved"); return retainForwardRecovery(input, touchedPorts, { kind: "restored", restoredPorts: [...missing], }); } catch (error) { - if (touchedPorts.length > 0) { - try { + let normalized = normalizeFailure(error); + try { + if (touchedPorts.length > 0) { rollbackTouchedPorts(input, touchedPorts); - } catch { - failure("restoration-unproved"); } + } catch { + normalized = new HermesPortableForwardRecoveryError("restoration-unproved"); + } finally { + timing.finish("failed"); } - throw normalizeFailure(error); + throw normalized; } } diff --git a/src/lib/onboard/experimental/hermes-portable-lifecycle.test.ts b/src/lib/onboard/experimental/hermes-portable-lifecycle.test.ts index 622ddc16dce..7c4ccc6191c 100644 --- a/src/lib/onboard/experimental/hermes-portable-lifecycle.test.ts +++ b/src/lib/onboard/experimental/hermes-portable-lifecycle.test.ts @@ -329,6 +329,13 @@ function lifecycleDeps( ); }); const launchOpenShell = vi.fn(); + const captureSocketAuthority = vi.fn(() => ({ ...receipt.socketAuthority, inode: "102" })); + const captureOpenShellExecutableAuthority = vi.fn(() => receipt.openshellExecutableAuthority); + const capturePodmanExecutableAuthority = vi.fn(() => receipt.podmanExecutableAuthority); + const assertOpenShellExecutableFileAuthority = vi.fn( + () => receipt.openshellExecutableAuthority.executable.executablePath, + ); + const capturePodmanExecutableFileAuthority = vi.fn(() => receipt.podmanExecutableAuthority); return { deps: { stateDir, @@ -359,9 +366,11 @@ function lifecycleDeps( XDG_CONFIG_HOME: receipt.runtimeAuthority.configHome, XDG_RUNTIME_DIR: receipt.runtimeAuthority.runtimeDir, }, - captureSocketAuthority: () => ({ ...receipt.socketAuthority, inode: "102" }), - captureOpenShellExecutableAuthority: () => receipt.openshellExecutableAuthority, - capturePodmanExecutableAuthority: () => receipt.podmanExecutableAuthority, + captureSocketAuthority, + captureOpenShellExecutableAuthority, + capturePodmanExecutableAuthority, + assertOpenShellExecutableFileAuthority, + capturePodmanExecutableFileAuthority, }, container: { podman, assertSocketAuthority: vi.fn() }, sleep: vi.fn(), @@ -369,9 +378,22 @@ function lifecycleDeps( podman, captureOpenShell, launchOpenShell, + captureSocketAuthority, + captureOpenShellExecutableAuthority, + capturePodmanExecutableAuthority, + assertOpenShellExecutableFileAuthority, + capturePodmanExecutableFileAuthority, }; } +function publishSuccessor(): void { + withMcpLifecycleLockSync( + SANDBOX, + () => publishHermesPortableSuccessorReceipt(SANDBOX, stateDir), + { stateDir: path.join(stateDir, "state") }, + ); +} + function lifecycleContext() { return { agent: "hermes", @@ -394,6 +416,220 @@ afterEach(() => { }); describe("Hermes portable lifecycle", () => { + it("uses one entry and final qualification when the timing callback fails (#10423)", () => { + const receipt = activeReceipt(); + publishSuccessor(); + const fixture = lifecycleDeps(receipt, false); + const evidence = vi.fn(() => { + throw new Error("timing sink unavailable"); + }); + let timingNow = 0; + + const result = withMcpLifecycleLockSync( + SANDBOX, + () => + recoverHermesPortableSandboxLifecycle(SANDBOX, lifecycleContext(), { + ...fixture.deps, + recoveryTiming: { + now: () => (timingNow += 1), + onComplete: evidence, + }, + }), + { stateDir: path.join(stateDir, "state") }, + ); + + expect(result).toEqual({ kind: "recovered" }); + expect(evidence).toHaveBeenCalledOnce(); + expect(evidence).toHaveBeenCalledWith( + expect.objectContaining({ + qualificationCount: 2, + containerStartCount: 1, + execReadyAttempts: 1, + authenticatedHealthCount: 1, + startupLaunchCount: 0, + rollbackCount: 0, + containerAction: "started", + result: "recovered", + }), + ); + const operations = fixture.captureOpenShell.mock.calls.map(([args]) => + args.slice(0, 2).join(":"), + ); + expect(operations.filter((operation) => operation === "sandbox:list")).toHaveLength(2); + expect(operations.filter((operation) => operation === "sandbox:get")).toHaveLength(2); + expect(operations.filter((operation) => operation === "policy:get")).toHaveLength(2); + expect(fixture.podman.mock.calls.filter(([args]) => args[1] === "start")).toHaveLength(1); + expect(fixture.podman.mock.calls.filter(([args]) => args[1] === "stop")).toHaveLength(0); + expect(fixture.assertOpenShellExecutableFileAuthority).toHaveBeenCalled(); + expect(fixture.capturePodmanExecutableFileAuthority).toHaveBeenCalled(); + }); + + it("emits failed timing when entry qualification rejects socket authority (#10423)", () => { + const receipt = activeReceipt(); + publishSuccessor(); + const fixture = lifecycleDeps(receipt, false); + const entryError = new Error("socket authority changed during entry qualification"); + const evidence = vi.fn(); + fixture.captureSocketAuthority.mockImplementation(() => { + throw entryError; + }); + + expect(() => + withMcpLifecycleLockSync( + SANDBOX, + () => + recoverHermesPortableSandboxLifecycle(SANDBOX, lifecycleContext(), { + ...fixture.deps, + recoveryTiming: { onComplete: evidence }, + }), + { stateDir: path.join(stateDir, "state") }, + ), + ).toThrow(entryError); + expect(fixture.podman.mock.calls.some(([args]) => args[1] === "start")).toBe(false); + expect(evidence).toHaveBeenCalledOnce(); + expect(evidence).toHaveBeenCalledWith( + expect.objectContaining({ + qualificationCount: 1, + containerStartCount: 0, + result: "failed", + }), + ); + }); + + it("fails before post-start work when retained socket authority drifts (#10423)", () => { + const receipt = activeReceipt(); + publishSuccessor(); + const fixture = lifecycleDeps(receipt, false); + const stableCapture = fixture.captureSocketAuthority.getMockImplementation()!; + fixture.captureSocketAuthority.mockImplementation(() => { + const socket = stableCapture(); + const started = fixture.podman.mock.calls.some(([args]) => args[1] === "start"); + return started ? { ...socket, inode: "changed-after-start" } : socket; + }); + + expect(() => + withMcpLifecycleLockSync( + SANDBOX, + () => recoverHermesPortableSandboxLifecycle(SANDBOX, lifecycleContext(), fixture.deps), + { stateDir: path.join(stateDir, "state") }, + ), + ).toThrow( + "Hermes portable lifecycle recovery failed and exact container rollback was not proven", + ); + expect(fixture.captureOpenShell).not.toHaveBeenCalledWith( + expect.arrayContaining(["true"]), + expect.any(Number), + ); + expect(fixture.podman.mock.calls.filter(([args]) => args[1] === "stop")).toHaveLength(0); + }); + + it("records polling failure and proves exact stopped rollback under retained authority (#10423)", () => { + const receipt = activeReceipt(); + publishSuccessor(); + const fixture = lifecycleDeps(receipt, false); + const defaultCapture = fixture.captureOpenShell.getMockImplementation()!; + fixture.captureOpenShell.mockImplementation((args: readonly string[]) => + args.includes("python3") + ? { status: 0, stdout: "unavailable\n", stderr: "" } + : defaultCapture(args), + ); + const evidence = vi.fn(); + let now = 0; + + expect(() => + withMcpLifecycleLockSync( + SANDBOX, + () => + recoverHermesPortableSandboxLifecycle(SANDBOX, lifecycleContext(), { + ...fixture.deps, + now: () => now, + sleep: (milliseconds) => { + now += milliseconds; + }, + recoveryTiming: { onComplete: evidence }, + }), + { stateDir: path.join(stateDir, "state") }, + ), + ).toThrow("managed startup did not pass authenticated health"); + expect(fixture.podman.mock.calls.filter(([args]) => args[1] === "stop")).toHaveLength(1); + expect(evidence).toHaveBeenCalledOnce(); + expect(evidence).toHaveBeenCalledWith( + expect.objectContaining({ + qualificationCount: 2, + containerStartCount: 1, + startupLaunchCount: 1, + rollbackCount: 1, + containerAction: "started", + result: "failed", + }), + ); + const recorded = evidence.mock.calls[0]?.[0]; + expect(recorded.authenticatedHealthCount).toBeGreaterThan(1); + expect(recorded.transactionCurrentnessCount).toBeGreaterThan(1); + }); + + it("rejects live sandbox rebind before the name-addressed startup launch (#10423)", () => { + const receipt = activeReceipt(); + publishSuccessor(); + const fixture = lifecycleDeps(receipt, false); + const defaultCapture = fixture.captureOpenShell.getMockImplementation()!; + fixture.captureOpenShell + .mockImplementationOnce(defaultCapture) + .mockImplementationOnce(defaultCapture) + .mockImplementationOnce(defaultCapture) + .mockImplementationOnce(defaultCapture) + .mockReturnValueOnce({ status: 0, stdout: "unavailable\n", stderr: "" }) + .mockImplementationOnce(defaultCapture) + .mockReturnValueOnce({ + status: 0, + stdout: sandboxListJson("rebound-sandbox-id", "Ready"), + stderr: "", + }) + .mockImplementation(defaultCapture); + + expect(() => + withMcpLifecycleLockSync( + SANDBOX, + () => recoverHermesPortableSandboxLifecycle(SANDBOX, lifecycleContext(), fixture.deps), + { stateDir: path.join(stateDir, "state") }, + ), + ).toThrow("OpenShell sandbox identity disagrees with the receipt container"); + + expect(fixture.launchOpenShell).not.toHaveBeenCalled(); + expect(fixture.podman.mock.calls.filter(([args]) => args[1] === "start")).toHaveLength(1); + expect(fixture.podman.mock.calls.filter(([args]) => args[1] === "stop")).toHaveLength(1); + }); + + it("rolls back when the final full qualification detects registry drift (#10423)", () => { + const receipt = activeReceipt(); + publishSuccessor(); + const fixture = lifecycleDeps(receipt, false); + const stableReadRegistry: NonNullable = + fixture.deps.readRegistry!; + let registryReads = 0; + + expect(() => + withMcpLifecycleLockSync( + SANDBOX, + () => + recoverHermesPortableSandboxLifecycle(SANDBOX, lifecycleContext(), { + ...fixture.deps, + readRegistry: (sandboxName) => { + registryReads += 1; + const entry = stableReadRegistry(sandboxName); + return registryReads === 2 && entry + ? { ...entry, lifecycleGeneration: "f".repeat(64) } + : entry; + }, + }), + { stateDir: path.join(stateDir, "state") }, + ), + ).toThrow("registry authority disagrees with the active receipt"); + expect(fixture.podman.mock.calls.filter(([args]) => args[1] === "start")).toHaveLength(1); + expect(fixture.podman.mock.calls.filter(([args]) => args[1] === "stop")).toHaveLength(1); + expect(registryReads).toBe(3); + }); + it("reconciles an interrupted schema-8 publication inside both probe fences (#10423)", async () => { const receipt = activeReceipt(stateDir); expect(() => diff --git a/src/lib/onboard/experimental/hermes-portable-lifecycle.ts b/src/lib/onboard/experimental/hermes-portable-lifecycle.ts index f46786e4358..9c701020d64 100644 --- a/src/lib/onboard/experimental/hermes-portable-lifecycle.ts +++ b/src/lib/onboard/experimental/hermes-portable-lifecycle.ts @@ -107,6 +107,151 @@ export interface HermesPortableLifecycleDeps { readonly now?: () => number; readonly sleep?: (milliseconds: number) => void; readonly log?: (message: string) => void; + readonly recoveryTiming?: HermesPortableLifecycleRecoveryTiming; +} + +const HERMES_PORTABLE_LIFECYCLE_TIMING_STAGES = [ + "entryQualification", + "containerStart", + "postStartCurrentness", + "execReady", + "preHealthCurrentness", + "authenticatedHealth", + "startupLaunch", + "healthPollCurrentness", + "finalQualification", + "rollback", +] as const; + +type HermesPortableLifecycleTimingStage = (typeof HERMES_PORTABLE_LIFECYCLE_TIMING_STAGES)[number]; +type HermesPortableLifecycleTimingCounter = + | "qualification" + | "transactionCurrentness" + | "containerInspection" + | "containerStart" + | "execReadyAttempt" + | "authenticatedHealth" + | "startupLaunch" + | "rollback"; + +export interface HermesPortableLifecycleRecoveryTimingEvidence { + readonly entryQualificationMs: number; + readonly containerStartMs: number; + readonly postStartCurrentnessMs: number; + readonly execReadyMs: number; + readonly preHealthCurrentnessMs: number; + readonly authenticatedHealthMs: number; + readonly startupLaunchMs: number; + readonly healthPollCurrentnessMs: number; + readonly finalQualificationMs: number; + readonly rollbackMs: number; + readonly qualificationCount: number; + readonly transactionCurrentnessCount: number; + readonly containerInspectionCount: number; + readonly containerStartCount: number; + readonly execReadyAttempts: number; + readonly authenticatedHealthCount: number; + readonly startupLaunchCount: number; + readonly rollbackCount: number; + readonly totalMs: number; + readonly containerAction: "reused" | "started"; + readonly result: "already-running" | "recovered" | "failed"; +} + +export interface HermesPortableLifecycleRecoveryTiming { + readonly now?: () => number; + readonly onComplete: (evidence: HermesPortableLifecycleRecoveryTimingEvidence) => void; +} + +type HermesPortableLifecycleTimingRecorder = { + readonly measure: (stage: HermesPortableLifecycleTimingStage, operation: () => T) => T; + readonly increment: (counter: HermesPortableLifecycleTimingCounter) => void; + readonly setContainerAction: (action: "reused" | "started") => void; + readonly finish: (result: HermesPortableLifecycleRecoveryTimingEvidence["result"]) => void; +}; + +function safeTimingNow(now: () => number): number | null { + try { + const value = now(); + return Number.isFinite(value) ? value : null; + } catch { + return null; + } +} + +function createHermesPortableLifecycleTimingRecorder( + timing: HermesPortableLifecycleRecoveryTiming | undefined, +): HermesPortableLifecycleTimingRecorder { + if (!timing) { + return Object.freeze({ + measure: (_stage: HermesPortableLifecycleTimingStage, operation: () => T): T => + operation(), + increment: (_counter: HermesPortableLifecycleTimingCounter): void => undefined, + setContainerAction: (_action: "reused" | "started"): void => undefined, + finish: (_result: HermesPortableLifecycleRecoveryTimingEvidence["result"]): void => undefined, + }); + } + const now = timing.now ?? (() => performance.now()); + const startedAt = safeTimingNow(now); + const durations = new Map(); + const counts = new Map(); + let containerAction: HermesPortableLifecycleRecoveryTimingEvidence["containerAction"] = "reused"; + let finished = false; + const elapsed = (start: number | null, end: number | null): number => { + if (start === null || end === null) return 0; + const duration = Math.round(end - start); + return Number.isFinite(duration) ? Math.min(9_999_999, Math.max(0, duration)) : 0; + }; + return Object.freeze({ + measure(stage: HermesPortableLifecycleTimingStage, operation: () => T): T { + const stageStartedAt = safeTimingNow(now); + try { + return operation(); + } finally { + const duration = elapsed(stageStartedAt, safeTimingNow(now)); + durations.set(stage, Math.min(9_999_999, (durations.get(stage) ?? 0) + duration)); + } + }, + increment(counter: HermesPortableLifecycleTimingCounter): void { + counts.set(counter, Math.min(9_999_999, (counts.get(counter) ?? 0) + 1)); + }, + setContainerAction(action): void { + containerAction = action; + }, + finish(result): void { + if (finished) return; + finished = true; + try { + timing.onComplete( + Object.freeze({ + entryQualificationMs: durations.get("entryQualification") ?? 0, + containerStartMs: durations.get("containerStart") ?? 0, + postStartCurrentnessMs: durations.get("postStartCurrentness") ?? 0, + execReadyMs: durations.get("execReady") ?? 0, + preHealthCurrentnessMs: durations.get("preHealthCurrentness") ?? 0, + authenticatedHealthMs: durations.get("authenticatedHealth") ?? 0, + startupLaunchMs: durations.get("startupLaunch") ?? 0, + healthPollCurrentnessMs: durations.get("healthPollCurrentness") ?? 0, + finalQualificationMs: durations.get("finalQualification") ?? 0, + rollbackMs: durations.get("rollback") ?? 0, + qualificationCount: counts.get("qualification") ?? 0, + transactionCurrentnessCount: counts.get("transactionCurrentness") ?? 0, + containerInspectionCount: counts.get("containerInspection") ?? 0, + containerStartCount: counts.get("containerStart") ?? 0, + execReadyAttempts: counts.get("execReadyAttempt") ?? 0, + authenticatedHealthCount: counts.get("authenticatedHealth") ?? 0, + startupLaunchCount: counts.get("startupLaunch") ?? 0, + rollbackCount: counts.get("rollback") ?? 0, + totalMs: elapsed(startedAt, safeTimingNow(now)), + containerAction, + result, + }), + ); + } catch { + // Timing output must not change lifecycle recovery. + } + }, + }); } interface QualifiedHermesPortableLifecycle { @@ -117,7 +262,10 @@ interface QualifiedHermesPortableLifecycle { readonly containerDeps: HermesPortableContainerDeps; readonly container: HermesPortableContainerInspection; readonly capture: NonNullable; + readonly rawCapture: NonNullable; readonly openShellPhase: string; + readonly hasTransactionAuthority: boolean; + readonly assertTransactionCurrent: () => void; readonly assertOperatingAuthority: () => void; } @@ -219,8 +367,12 @@ function createContainerDeps( ); return { podman: (args, timeoutMs): HermesPortablePodmanResult => { - authority.assertCurrent(); - return authority.engine.capture(args, timeoutMs); + authority.assertTransactionCurrent(); + try { + return authority.engine.capture(args, timeoutMs); + } finally { + authority.assertTransactionCurrent(); + } }, assertSocketAuthority: () => authority.engine.assertAuthority(), }; @@ -396,6 +548,7 @@ function qualify( deps.operatingAuthority, options, ); + operatingAuthority.assertCurrent(); const receipt = operatingAuthority.receipt; if (!contextMatches(receipt, context)) fail("registry context disagrees with the active receipt"); assertCurrentHermesPortableStoredStartupContract(receipt.startup, sandboxName); @@ -440,13 +593,25 @@ function qualify( fail("container state or restart policy disagrees with active authority"); } operatingAuthority.assertCurrent(); + const hasTransactionAuthority = snapshot.successor !== undefined; + const assertTransactionCurrent = hasTransactionAuthority + ? retainRequalifiedOperatingAuthority( + sandboxName, + stateDir, + snapshot, + operatingAuthority.assertTransactionCurrent, + ) + : operatingAuthority.assertCurrent; return { snapshot: snapshot as QualifiedHermesPortableLifecycle["snapshot"], receipt, containerDeps, container, capture, + rawCapture, openShellPhase: liveIdentity.phase, + hasTransactionAuthority, + assertTransactionCurrent, assertOperatingAuthority: operatingAuthority.assertCurrent, }; } @@ -545,35 +710,141 @@ function rollbackStartedHermesPortableRecovery( context: PortableDemoLifecycleContext, deps: HermesPortableLifecycleDeps, qualified: QualifiedHermesPortableLifecycle, + timing: HermesPortableLifecycleTimingRecorder, ): void { + if (qualified.hasTransactionAuthority) { + timing.increment("transactionCurrentness"); + qualified.assertTransactionCurrent(); + } stopHermesPortableContainer(qualified.receipt, { ...qualified.containerDeps, ...(deps.now ? { now: deps.now } : {}), ...(deps.sleep ? { sleep: deps.sleep } : {}), }); + timing.increment("qualification"); const stopped = qualify(sandboxName, context, deps, qualified.snapshot, ["Error", "Stopped"]); if (stopped.container.authority.running || stopped.container.status !== "exited") { fail("failed recovery did not restore the exact stopped container"); } } +function assertLifecycleTransactionCurrent( + qualified: QualifiedHermesPortableLifecycle, + timing: HermesPortableLifecycleTimingRecorder, + expectedRunning: boolean, +): HermesPortableContainerInspection { + timing.increment("transactionCurrentness"); + qualified.assertTransactionCurrent(); + timing.increment("containerInspection"); + const current = assertCurrentHermesPortableContainer(qualified.receipt, qualified.containerDeps); + timing.increment("transactionCurrentness"); + qualified.assertTransactionCurrent(); + if ( + current.authority.running !== expectedRunning || + current.paused || + current.authority.restartPolicy !== "unless-stopped" || + (expectedRunning ? current.status !== "running" : current.status !== "exited") + ) { + fail("container state changed during retained lifecycle authority"); + } + return current; +} + +function refreshLifecycleCurrentness( + sandboxName: string, + context: PortableDemoLifecycleContext, + deps: HermesPortableLifecycleDeps, + qualified: QualifiedHermesPortableLifecycle, + timing: HermesPortableLifecycleTimingRecorder, + expectedRunning: boolean, + acceptedPhases: readonly string[] = ["Ready"], +): QualifiedHermesPortableLifecycle { + if (!qualified.hasTransactionAuthority) { + timing.increment("qualification"); + return qualify(sandboxName, context, deps, qualified.snapshot, acceptedPhases); + } + return { + ...qualified, + container: assertLifecycleTransactionCurrent(qualified, timing, expectedRunning), + }; +} + +function captureRetainedLifecycleCommand( + qualified: QualifiedHermesPortableLifecycle, + timing: HermesPortableLifecycleTimingRecorder, + args: readonly string[], + timeoutMs: number, +): HermesPortableLifecycleCommandResult { + if (!qualified.hasTransactionAuthority) return qualified.capture(args, timeoutMs); + timing.increment("transactionCurrentness"); + qualified.assertTransactionCurrent(); + try { + return qualified.rawCapture(args, timeoutMs); + } finally { + timing.increment("transactionCurrentness"); + qualified.assertTransactionCurrent(); + } +} + +/** Rebind the live target immediately before the name-addressed startup command. */ +function assertLiveHermesPortableStartupBinding( + qualified: QualifiedHermesPortableLifecycle, + deps: HermesPortableLifecycleDeps, + timing: HermesPortableLifecycleTimingRecorder, +): void { + if (!qualified.hasTransactionAuthority) return; + const capture: NonNullable = (args, timeoutMs) => + captureRetainedLifecycleCommand(qualified, timing, args, timeoutMs); + proveHermesPortableLivePolicy({ + gatewayName: qualified.receipt.gatewayName, + sandboxName: qualified.receipt.sandboxName, + capture: policyCapture(capture), + }); + const liveIdentity = observeOpenShellIdentity(qualified.receipt, capture); + requireRegistry(qualified.receipt, liveIdentity.liveIdentityFingerprint, deps); +} + /** Recover the exact receipt-owned container and manifest-owned Hermes startup. */ export function recoverHermesPortableSandboxLifecycle( sandboxName: string, context: PortableDemoLifecycleContext, deps: HermesPortableLifecycleDeps = {}, ): PortableDemoLifecycleRecoveryResult { - let qualified = qualify(sandboxName, context, deps, undefined, ["Ready", "Error", "Stopped"]); + const timing = createHermesPortableLifecycleTimingRecorder(deps.recoveryTiming); + timing.increment("qualification"); + let qualified: QualifiedHermesPortableLifecycle; + try { + qualified = timing.measure("entryQualification", () => + qualify(sandboxName, context, deps, undefined, ["Ready", "Error", "Stopped"]), + ); + } catch (error) { + timing.finish("failed"); + throw error; + } const wasRunning = qualified.container.authority.running; + timing.setContainerAction(wasRunning ? "reused" : "started"); const rollbackAuthority = qualified; let startedByRecovery = false; try { if (!wasRunning) { try { - startedByRecovery = - startHermesPortableContainer(qualified.receipt, qualified.containerDeps) === "started"; + if (qualified.hasTransactionAuthority) { + timing.increment("transactionCurrentness"); + qualified.assertTransactionCurrent(); + } + timing.increment("containerStart"); + startedByRecovery = timing.measure( + "containerStart", + () => + startHermesPortableContainer(qualified.receipt, qualified.containerDeps) === "started", + ); + if (qualified.hasTransactionAuthority) { + timing.increment("transactionCurrentness"); + qualified.assertTransactionCurrent(); + } } catch (startError) { try { + timing.increment("containerInspection"); const current = assertCurrentHermesPortableContainer( qualified.receipt, qualified.containerDeps, @@ -587,84 +858,144 @@ export function recoverHermesPortableSandboxLifecycle( } throw startError; } - qualified = qualify(sandboxName, context, deps, qualified.snapshot, [ - "Ready", - "Error", - "Stopped", - ]); + qualified = timing.measure("postStartCurrentness", () => + refreshLifecycleCurrentness(sandboxName, context, deps, qualified, timing, true, [ + "Ready", + "Error", + "Stopped", + ]), + ); } const commandEnv = deps.env ?? process.env; - const assertExecutable = - deps.assertOpenShellExecutableAuthority ?? assertHermesPortableOpenShellExecutableAuthority; - const commandAuthority = buildHermesPortableOpenShellCommandAuthority( - qualified.receipt, - commandEnv, - assertExecutable, - ); - const rawCapture = - deps.captureOpenShell ?? - defaultCaptureOpenShell( - commandAuthority.executablePath, - commandEnv, - qualified.receipt.runtimeAuthority, - ); const capture: NonNullable = ( args, timeoutMs, - ) => { - buildHermesPortableOpenShellCommandAuthority(qualified.receipt, commandEnv, assertExecutable); - return rawCapture(args, timeoutMs); + ) => captureRetainedLifecycleCommand(qualified, timing, args, timeoutMs); + const execReady = timing.measure("execReady", () => + waitFor(EXEC_READY_TIMEOUT_MS, deps, (remainingMs) => { + timing.increment("execReadyAttempt"); + if (qualified.hasTransactionAuthority) { + assertLifecycleTransactionCurrent(qualified, timing, true); + } + const result = capture( + openshellExecArgs(qualified.receipt, ["true"]), + Math.min(COMMAND_TIMEOUT_MS, remainingMs), + ); + if (qualified.hasTransactionAuthority) { + assertLifecycleTransactionCurrent(qualified, timing, true); + } + return result.status === 0 && !result.error; + }), + ); + if (!execReady) fail("did not reconnect to the selected OpenShell gateway"); + qualified = timing.measure("preHealthCurrentness", () => + refreshLifecycleCurrentness(sandboxName, context, deps, qualified, timing, true), + ); + const transactionContainerDeps: HermesPortableContainerDeps = { + ...qualified.containerDeps, + authenticatedHealth: createAuthenticatedHealthCapture(qualified.receipt, capture), }; - const execReady = waitFor(EXEC_READY_TIMEOUT_MS, deps, (remainingMs) => { - const result = capture( - openshellExecArgs(qualified.receipt, ["true"]), - Math.min(COMMAND_TIMEOUT_MS, remainingMs), + timing.increment("authenticatedHealth"); + const initialHealth = timing.measure("authenticatedHealth", () => + observeHermesPortableAuthenticatedHealth(qualified.receipt, transactionContainerDeps), + ); + if (qualified.hasTransactionAuthority) { + timing.measure("healthPollCurrentness", () => + assertLifecycleTransactionCurrent(qualified, timing, true), ); - return result.status === 0 && !result.error; - }); - if (!execReady) fail("did not reconnect to the selected OpenShell gateway"); - qualified = qualify(sandboxName, context, deps, qualified.snapshot); - if ( - observeHermesPortableAuthenticatedHealth(qualified.receipt, qualified.containerDeps) === - "ready" - ) { - qualify(sandboxName, context, deps, qualified.snapshot); - return wasRunning ? { kind: "already-running" } : { kind: "recovered" }; + } + if (initialHealth === "ready") { + timing.increment("qualification"); + timing.measure("finalQualification", () => + qualify(sandboxName, context, deps, qualified.snapshot), + ); + const result = wasRunning + ? { kind: "already-running" as const } + : { kind: "recovered" as const }; + timing.finish(result.kind); + return result; } if (startedByRecovery) { - qualified = qualify(sandboxName, context, deps, qualified.snapshot); + qualified = timing.measure("healthPollCurrentness", () => + refreshLifecycleCurrentness(sandboxName, context, deps, qualified, timing, true), + ); + const assertExecutable = + deps.assertOpenShellExecutableAuthority ?? assertHermesPortableOpenShellExecutableAuthority; + const executablePath = qualified.hasTransactionAuthority + ? qualified.receipt.openshellExecutableAuthority.executable.executablePath + : buildHermesPortableOpenShellCommandAuthority( + qualified.receipt, + commandEnv, + assertExecutable, + ).executablePath; const rawLaunch = deps.launchOpenShell ?? - defaultLaunchOpenShell( - commandAuthority.executablePath, - commandEnv, - qualified.receipt.runtimeAuthority, - ); - buildHermesPortableOpenShellCommandAuthority(qualified.receipt, commandEnv, assertExecutable); - rawLaunch(openshellExecArgs(qualified.receipt, qualified.receipt.startup.argv)); + defaultLaunchOpenShell(executablePath, commandEnv, qualified.receipt.runtimeAuthority); + timing.measure("preHealthCurrentness", () => + assertLiveHermesPortableStartupBinding(qualified, deps, timing), + ); + timing.increment("startupLaunch"); + timing.measure("startupLaunch", () => + rawLaunch(openshellExecArgs(qualified.receipt, qualified.receipt.startup.argv)), + ); + if (qualified.hasTransactionAuthority) { + timing.increment("transactionCurrentness"); + qualified.assertTransactionCurrent(); + } } const recovered = waitFor(STARTUP_TIMEOUT_MS, deps, () => { - const current = qualify(sandboxName, context, deps, qualified.snapshot); - return ( - observeHermesPortableAuthenticatedHealth(current.receipt, current.containerDeps) === "ready" + qualified = timing.measure("healthPollCurrentness", () => + refreshLifecycleCurrentness(sandboxName, context, deps, qualified, timing, true), + ); + const currentContainerDeps: HermesPortableContainerDeps = { + ...qualified.containerDeps, + authenticatedHealth: createAuthenticatedHealthCapture(qualified.receipt, capture), + }; + timing.increment("authenticatedHealth"); + const health = timing.measure("authenticatedHealth", () => + observeHermesPortableAuthenticatedHealth(qualified.receipt, currentContainerDeps), ); + if (qualified.hasTransactionAuthority) { + timing.measure("healthPollCurrentness", () => + assertLifecycleTransactionCurrent(qualified, timing, true), + ); + } + return health === "ready"; }); if (!recovered) fail("managed startup did not pass authenticated health"); - qualify(sandboxName, context, deps, qualified.snapshot); - if (wasRunning) return { kind: "already-running" }; + timing.increment("qualification"); + timing.measure("finalQualification", () => + qualify(sandboxName, context, deps, qualified.snapshot), + ); + if (wasRunning) { + timing.finish("already-running"); + return { kind: "already-running" }; + } (deps.log ?? console.log)(` Hermes portable lifecycle recovered sandbox '${sandboxName}'.`); + timing.finish("recovered"); return { kind: "recovered" }; } catch (error) { if (startedByRecovery) { try { - rollbackStartedHermesPortableRecovery(sandboxName, context, deps, rollbackAuthority); + timing.increment("rollback"); + timing.measure("rollback", () => + rollbackStartedHermesPortableRecovery( + sandboxName, + context, + deps, + rollbackAuthority, + timing, + ), + ); } catch (rollbackError) { + timing.finish("failed"); throw new AggregateError( [error, rollbackError], "Hermes portable lifecycle recovery failed and exact container rollback was not proven", ); } } + timing.finish("failed"); throw error; } } diff --git a/src/lib/onboard/experimental/hermes-portable-ollama-authority.ts b/src/lib/onboard/experimental/hermes-portable-ollama-authority.ts index 622ccf895d2..8c13271aea3 100644 --- a/src/lib/onboard/experimental/hermes-portable-ollama-authority.ts +++ b/src/lib/onboard/experimental/hermes-portable-ollama-authority.ts @@ -330,6 +330,7 @@ function inspectPortableNetworkSnapshot( export interface PreparedPortableRegistryRecovery { readonly started: boolean; + readonly assertRetainedCurrent: () => void; readonly assertTransactionCurrent: () => void; readonly assertCurrent: () => void; readonly rollback: () => void; @@ -579,8 +580,14 @@ export function preparePortableRegistryRecovery( transactionCurrent.assertEngineCurrent(); authority.assertCurrent(); }; + const assertRetainedCurrent = () => { + if (released) throw new Error("Hermes Portable inference registry recovery was released."); + transactionCurrent.assertCallerCurrent(); + transactionCurrent.assertEngineCurrent(); + }; return Object.freeze({ started: false, + assertRetainedCurrent, assertTransactionCurrent, assertCurrent, rollback: assertCurrent, @@ -637,6 +644,11 @@ export function preparePortableRegistryRecovery( transactionCurrent.assertEngineCurrent(); authority.assertCurrent(); }; + const assertRetainedCurrent = () => { + if (released) throw new Error("Hermes Portable inference registry recovery was released."); + transactionCurrent.assertCallerCurrent(); + transactionCurrent.assertEngineCurrent(); + }; const rollback = () => { if (released) throw new Error("Hermes Portable inference registry recovery was released."); assertEngineCurrent(); @@ -655,6 +667,7 @@ export function preparePortableRegistryRecovery( }; return Object.freeze({ started: true, + assertRetainedCurrent, assertTransactionCurrent, assertCurrent, rollback, diff --git a/src/lib/onboard/experimental/hermes-portable-ollama-inference.ts b/src/lib/onboard/experimental/hermes-portable-ollama-inference.ts index 6a61167499c..aa696a71520 100644 --- a/src/lib/onboard/experimental/hermes-portable-ollama-inference.ts +++ b/src/lib/onboard/experimental/hermes-portable-ollama-inference.ts @@ -23,6 +23,7 @@ import { } from "../runtime-provider/host-local-inference"; import { assertHermesPortableHostLocalInferencePublishedRecoveryAuthorityCurrent, + assertHermesPortableHostLocalInferencePublishedRecoveryTransactionCurrent, prepareHermesPortableHostLocalInferencePublishedRecoveryAuthority, type HostLocalInferenceLifecycleSandbox, } from "../runtime-provider/host-local-inference-lifecycle"; @@ -154,6 +155,7 @@ export interface HermesPortableOllamaRuntimeAuthority { readonly bundle: RuntimeProviderBundle; readonly inferenceStateDir: string; readonly network: ReturnType; + readonly assertRetainedCurrent: () => void; readonly assertTransactionCurrent: () => void; readonly assertCurrent: () => void; } @@ -170,8 +172,12 @@ export interface HermesPortableOllamaRecoveryTimingEvidence { readonly routeMs: number; readonly dependencyMs: number; readonly finalCurrentnessMs: number; + readonly retainedCurrentnessCount: number; + readonly fullCurrentnessCount: number; + readonly preparedAuthorityInspectionCount: number; readonly totalMs: number; - readonly runtimeAction: "reused" | "recovered"; + readonly runtimeAction: "unknown" | "reused" | "recovered"; + readonly result: "proved" | "failed"; } export interface HermesPortableOllamaRecoveryTiming { @@ -197,7 +203,7 @@ function writeHermesPortableOllamaRecoveryTiming( evidence: HermesPortableOllamaRecoveryTimingEvidence, ): void { console.log( - ` Hermes Portable Ollama recovery timing: entryAuthority=${String(evidence.entryAuthorityMs)}ms operatingAuthority=${String(evidence.operatingAuthorityMs)}ms registryPreparation=${String(evidence.registryPreparationMs)}ms privatePublication=${String(evidence.privatePublicationMs)}ms runtimeAuthority=${String(evidence.runtimeAuthorityMs)}ms preparedInferenceAuthority=${String(evidence.preparedInferenceAuthorityMs)}ms exactRuntimeInspection=${String(evidence.exactRuntimeInspectionMs)}ms preRouteCurrentness=${String(evidence.preRouteCurrentnessMs)}ms route=${String(evidence.routeMs)}ms dependency=${String(evidence.dependencyMs)}ms finalCurrentness=${String(evidence.finalCurrentnessMs)}ms total=${String(evidence.totalMs)}ms runtimeAction=${evidence.runtimeAction} result=proved`, + ` Hermes Portable Ollama recovery timing: entryAuthority=${String(evidence.entryAuthorityMs)}ms operatingAuthority=${String(evidence.operatingAuthorityMs)}ms registryPreparation=${String(evidence.registryPreparationMs)}ms privatePublication=${String(evidence.privatePublicationMs)}ms runtimeAuthority=${String(evidence.runtimeAuthorityMs)}ms preparedInferenceAuthority=${String(evidence.preparedInferenceAuthorityMs)}ms exactRuntimeInspection=${String(evidence.exactRuntimeInspectionMs)}ms preRouteCurrentness=${String(evidence.preRouteCurrentnessMs)}ms route=${String(evidence.routeMs)}ms dependency=${String(evidence.dependencyMs)}ms finalCurrentness=${String(evidence.finalCurrentnessMs)}ms retainedCurrentnessCount=${String(evidence.retainedCurrentnessCount)} fullCurrentnessCount=${String(evidence.fullCurrentnessCount)} preparedAuthorityInspectionCount=${String(evidence.preparedAuthorityInspectionCount)} total=${String(evidence.totalMs)}ms runtimeAction=${evidence.runtimeAction} result=${evidence.result}`, ); } @@ -216,6 +222,11 @@ function createHermesPortableOllamaRecoveryTimingRecorder( readonly measure: (stage: HermesPortableOllamaRecoveryTimingStage, operation: () => T) => T; readonly finish: ( runtimeAction: HermesPortableOllamaRecoveryTimingEvidence["runtimeAction"], + counts: Pick< + HermesPortableOllamaRecoveryTimingEvidence, + "retainedCurrentnessCount" | "fullCurrentnessCount" | "preparedAuthorityInspectionCount" + >, + result?: HermesPortableOllamaRecoveryTimingEvidence["result"], ) => void; } { const now = timing.now ?? (() => performance.now()); @@ -278,7 +289,7 @@ function createHermesPortableOllamaRecoveryTimingRecorder( onComplete: (durationMs: number) => recordExternalStage(stage, durationMs), }), measure: measureStage, - finish(runtimeAction): void { + finish(runtimeAction, counts, result = "proved"): void { if (finished) return; finished = true; try { @@ -295,8 +306,10 @@ function createHermesPortableOllamaRecoveryTimingRecorder( routeMs: durations.get("route") ?? 0, dependencyMs: durations.get("dependency") ?? 0, finalCurrentnessMs: durations.get("finalCurrentness") ?? 0, + ...counts, totalMs: elapsed(startedAt, safeTimingNow(now)), runtimeAction, + result, }), ); } catch { @@ -323,6 +336,9 @@ function writeHermesPortablePublishedResumeTiming( ); } +const DEFAULT_HERMES_PORTABLE_PUBLISHED_RESUME_TIMING: PodmanPublishedResumeTiming = + Object.freeze({ onComplete: writeHermesPortablePublishedResumeTiming }); + interface PreparedHermesPortableOllamaRecoveryEntry { readonly registryRecovery: PreparedPortableRegistryRecovery; readonly createRuntimeAuthority: (options: { @@ -416,7 +432,7 @@ function prepareHermesPortableOllamaRecoveryEntry(options: { }, publishedResumeTiming: runtimeOptions.publishedResumeTiming ?? - Object.freeze({ onComplete: writeHermesPortablePublishedResumeTiming }), + DEFAULT_HERMES_PORTABLE_PUBLISHED_RESUME_TIMING, onFailureEvidence: (evidence) => { const message = redactOnboardDiagnosticText(evidence.message); if (message) console.error(` Podman inference ${evidence.phase}: ${message}`); @@ -444,7 +460,7 @@ function prepareHermesPortableOllamaRecoveryEntry(options: { }, publishedResumeTiming: runtimeOptions.publishedResumeTiming ?? - Object.freeze({ onComplete: writeHermesPortablePublishedResumeTiming }), + DEFAULT_HERMES_PORTABLE_PUBLISHED_RESUME_TIMING, onFailureEvidence: (evidence) => { const message = redactOnboardDiagnosticText(evidence.message); if (message) console.error(` Podman inference ${evidence.phase}: ${message}`); @@ -458,6 +474,10 @@ function prepareHermesPortableOllamaRecoveryEntry(options: { operationAuthority.assertTransactionCurrent(); network.assertCurrent(); }; + const assertRetainedCurrent = (): void => { + runtimeOptions.assertForwardAuthority(); + operationAuthority.assertTransactionCurrent(); + }; const assertCurrent = (): void => { runtimeOptions.assertForwardAuthority(); operationAuthority.assertCurrent(); @@ -469,6 +489,7 @@ function prepareHermesPortableOllamaRecoveryEntry(options: { inferenceStateDir, network, operation, + assertRetainedCurrent, assertTransactionCurrent, assertCurrent, }); @@ -521,6 +542,10 @@ export function createHermesPortableOllamaRuntimeAuthority(options: { engines.assertTransactionCurrent(); network.assertCurrent(); }; + const assertRetainedCurrent = (): void => { + options.publishedRecovery?.assertForwardAuthority(); + engines.assertTransactionCurrent(); + }; const assertCurrent = (): void => { options.publishedRecovery?.assertForwardAuthority(); revalidatePodmanInferenceAuthority(engines.hostLocalInference, authority, qualification); @@ -551,7 +576,7 @@ export function createHermesPortableOllamaRuntimeAuthority(options: { }, publishedResumeTiming: options.publishedResumeTiming ?? - Object.freeze({ onComplete: writeHermesPortablePublishedResumeTiming }), + DEFAULT_HERMES_PORTABLE_PUBLISHED_RESUME_TIMING, } : {}), authorityStore: openFilePersistedEngineAuthorityStore(inferenceStateDir), @@ -570,6 +595,7 @@ export function createHermesPortableOllamaRuntimeAuthority(options: { bundle, inferenceStateDir, network, + assertRetainedCurrent, assertTransactionCurrent, assertCurrent, }); @@ -665,6 +691,7 @@ interface HermesPortableOllamaRecoveryDeps { readonly prepareRecoveryEntry: typeof prepareHermesPortableOllamaRecoveryEntry; readonly prepareInferenceAuthority: typeof prepareHermesPortableHostLocalInferencePublishedRecoveryAuthority; readonly assertPreparedInferenceAuthorityCurrent: typeof assertHermesPortableHostLocalInferencePublishedRecoveryAuthorityCurrent; + readonly assertPreparedInferenceAuthorityTransactionCurrent: typeof assertHermesPortableHostLocalInferencePublishedRecoveryTransactionCurrent; readonly preparePublishedAuthority: typeof prepareHermesPortableOllamaPublishedInferenceAuthority; readonly prepareStartup: typeof prepareHermesPortablePublishedHostLocalInferenceStartup; readonly recoveryTiming: HermesPortableOllamaRecoveryTiming; @@ -677,6 +704,8 @@ const DEFAULT_RECOVERY_DEPS: HermesPortableOllamaRecoveryDeps = Object.freeze({ prepareInferenceAuthority: prepareHermesPortableHostLocalInferencePublishedRecoveryAuthority, assertPreparedInferenceAuthorityCurrent: assertHermesPortableHostLocalInferencePublishedRecoveryAuthorityCurrent, + assertPreparedInferenceAuthorityTransactionCurrent: + assertHermesPortableHostLocalInferencePublishedRecoveryTransactionCurrent, preparePublishedAuthority: prepareHermesPortableOllamaPublishedInferenceAuthority, prepareStartup: prepareHermesPortablePublishedHostLocalInferenceStartup, recoveryTiming: Object.freeze({ onComplete: writeHermesPortableOllamaRecoveryTiming }), @@ -929,62 +958,98 @@ export function recoverHermesPortableOllamaInference( const recoveryTiming = createHermesPortableOllamaRecoveryTimingRecorder(deps.recoveryTiming); const env = input.env ?? process.env; const stateDir = input.stateDir ?? defaultPortableDemoStateDir(env); - input.assertCallerCurrent?.(); - const snapshot = deps.readReceipt(input.sandboxName, stateDir); - if (!snapshot || snapshot.receipt.phase !== "active" || !snapshot.successor) { - failRecovery("active schema-6 lifecycle authority is missing"); - } - const operating = recoveryTiming.measureEntry("operatingAuthority", () => - deps.qualifyOperatingAuthority( - snapshot as typeof snapshot & { readonly receipt: HermesPortableConfiguredReceipt }, - ), - ); - operating.assertTransactionCurrent(); - if (!isDeepStrictEqual(input.readRegistry(input.sandboxName), input.entry)) { - failRecovery("sandbox registry authority changed before recovery"); - } - const serializedRegistryReceipt = input.entry.hostLocalInferenceReceipt; - if (typeof serializedRegistryReceipt !== "string") { - failRecovery("sandbox registry host-local inference receipt is missing"); - } - const receipt = parseHostLocalInferenceReceipt(serializedRegistryReceipt); - requirePublishedOllamaRecoveryReceipt(receipt); - const providerEntry = inferenceLifecycleRow(input.entry, receipt.providerId); - const assertCallerCurrent = (): void => { - input.assertCallerCurrent?.(); - try { - operating.assertCurrent(); - } catch { - failRecovery("schema-6 operating authority changed during recovery"); - } - if (!isDeepStrictEqual(input.readRegistry(input.sandboxName), input.entry)) { - failRecovery("sandbox registry authority changed during recovery"); - } - input.assertCallerCurrent?.(); - }; - const assertCallerTransactionCurrent = (): void => { - input.assertCallerTransactionCurrent?.(); + let runtimeAction: HermesPortableOllamaRecoveryTimingEvidence["runtimeAction"] = "unknown"; + let retainedCurrentnessCount = 0; + let fullCurrentnessCount = 0; + let preparedAuthorityInspectionCount = 0; + const timingCounts = () => + Object.freeze({ + retainedCurrentnessCount, + fullCurrentnessCount, + preparedAuthorityInspectionCount, + }); + const entry = (() => { try { + input.assertCallerCurrent?.(); + const snapshot = deps.readReceipt(input.sandboxName, stateDir); + if (!snapshot || snapshot.receipt.phase !== "active" || !snapshot.successor) { + failRecovery("active schema-6 lifecycle authority is missing"); + } + const operating = recoveryTiming.measureEntry("operatingAuthority", () => + deps.qualifyOperatingAuthority( + snapshot as typeof snapshot & { readonly receipt: HermesPortableConfiguredReceipt }, + ), + ); operating.assertTransactionCurrent(); - } catch { - failRecovery("schema-6 operating authority changed during recovery"); - } - if (!isDeepStrictEqual(input.readRegistry(input.sandboxName), input.entry)) { - failRecovery("sandbox registry authority changed during recovery"); - } - input.assertCallerTransactionCurrent?.(); - }; - const recoveryEntry = recoveryTiming.measureEntry("registryPreparation", () => - atOllamaRecoveryPhase("REGISTRY_PREPARATION_POSTCONDITION", () => - deps.prepareRecoveryEntry({ - receipt: operating.receipt, - inferenceReceipt: receipt, - stateDir, - env, + if (!isDeepStrictEqual(input.readRegistry(input.sandboxName), input.entry)) { + failRecovery("sandbox registry authority changed before recovery"); + } + const serializedRegistryReceipt = input.entry.hostLocalInferenceReceipt; + if (typeof serializedRegistryReceipt !== "string") { + failRecovery("sandbox registry host-local inference receipt is missing"); + } + const receipt = parseHostLocalInferenceReceipt(serializedRegistryReceipt); + requirePublishedOllamaRecoveryReceipt(receipt); + const providerEntry = inferenceLifecycleRow(input.entry, receipt.providerId); + const assertCallerCurrent = (): void => { + input.assertCallerCurrent?.(); + try { + operating.assertCurrent(); + } catch { + failRecovery("schema-6 operating authority changed during recovery"); + } + if (!isDeepStrictEqual(input.readRegistry(input.sandboxName), input.entry)) { + failRecovery("sandbox registry authority changed during recovery"); + } + input.assertCallerCurrent?.(); + }; + const assertCallerTransactionCurrent = (): void => { + input.assertCallerTransactionCurrent?.(); + try { + operating.assertTransactionCurrent(); + } catch { + failRecovery("schema-6 operating authority changed during recovery"); + } + if (!isDeepStrictEqual(input.readRegistry(input.sandboxName), input.entry)) { + failRecovery("sandbox registry authority changed during recovery"); + } + input.assertCallerTransactionCurrent?.(); + }; + const recoveryEntry = recoveryTiming.measureEntry("registryPreparation", () => + atOllamaRecoveryPhase("REGISTRY_PREPARATION_POSTCONDITION", () => + deps.prepareRecoveryEntry({ + receipt: operating.receipt, + inferenceReceipt: receipt, + stateDir, + env, + assertCallerTransactionCurrent, + }), + ), + ); + return { + assertCallerCurrent, assertCallerTransactionCurrent, - }), - ), - ); + operating, + providerEntry, + receipt, + recoveryEntry, + serializedRegistryReceipt, + }; + } catch (error) { + recoveryTiming.finishEntryAuthority(); + recoveryTiming.finish(runtimeAction, timingCounts(), "failed"); + throw error; + } + })(); + const { + assertCallerCurrent, + assertCallerTransactionCurrent, + operating, + providerEntry, + receipt, + recoveryEntry, + serializedRegistryReceipt, + } = entry; const { registryRecovery } = recoveryEntry; let ollamaStateRestored = true; try { @@ -1053,12 +1118,25 @@ export function recoverHermesPortableOllamaInference( }); }), ); + preparedAuthorityInspectionCount = 1; + const assertPreparedAuthorityTransactionCurrent = (): void => { + const currentEntry = input.readRegistry(input.sandboxName); + if (!currentEntry || !isDeepStrictEqual(currentEntry, input.entry)) { + failRecovery("sandbox registry authority changed during recovery"); + } + deps.assertPreparedInferenceAuthorityTransactionCurrent( + runtimeAuthority.bundle, + inferenceLifecycleRow(currentEntry, runtimeAuthority.bundle.identity.id), + preparedAuthority, + ); + }; const assertPreparedAuthorityCurrent = (expectedRunning: boolean): void => { const currentEntry = input.readRegistry(input.sandboxName); if (!currentEntry || !isDeepStrictEqual(currentEntry, input.entry)) { failRecovery("sandbox registry authority changed during recovery"); } try { + preparedAuthorityInspectionCount += 1; const current = deps.assertPreparedInferenceAuthorityCurrent( runtimeAuthority.bundle, inferenceLifecycleRow(currentEntry, runtimeAuthority.bundle.identity.id), @@ -1071,28 +1149,24 @@ export function recoverHermesPortableOllamaInference( failRecovery("host-local inference authority changed during recovery"); } }; - const requireTransactionCurrent = (expectedRunning: boolean): void => { + const requireRetainedCurrent = (): void => { + retainedCurrentnessCount += 1; assertCallerTransactionCurrent(); - registryRecovery.assertTransactionCurrent(); - runtimeAuthority.assertTransactionCurrent(); + registryRecovery.assertRetainedCurrent(); + runtimeAuthority.assertRetainedCurrent(); published.assertTransactionCurrent(); - assertPreparedAuthorityCurrent(expectedRunning); + assertPreparedAuthorityTransactionCurrent(); assertCallerTransactionCurrent(); }; const requireCompletionCurrent = (): void => { - assertCallerTransactionCurrent(); - registryRecovery.assertTransactionCurrent(); - runtimeAuthority.assertTransactionCurrent(); - published.assertTransactionCurrent(); - assertCallerTransactionCurrent(); + fullCurrentnessCount += 1; + requireRetainedCurrent(); assertCallerCurrent(); + registryRecovery.assertCurrent(); runtimeAuthority.assertCurrent(); published.assertCurrent(); assertPreparedAuthorityCurrent(true); - registryRecovery.assertTransactionCurrent(); - runtimeAuthority.assertTransactionCurrent(); - published.assertTransactionCurrent(); - assertCallerTransactionCurrent(); + requireRetainedCurrent(); }; const verifyFinalRoute = (): void => { const verified = input.verifyRoute(); @@ -1113,6 +1187,7 @@ export function recoverHermesPortableOllamaInference( ); return current; }); + runtimeAction = inspected.running ? "reused" : "recovered"; recoveryTiming.finishEntryAuthority(); if (inspected.running) { let preparedDependency: HermesPortableOllamaPreparedProbeDependency | null = null; @@ -1127,7 +1202,7 @@ export function recoverHermesPortableOllamaInference( validatePublishedResume(receipt), "running runtime validation changed receipt", ); - requireTransactionCurrent(true); + requireRetainedCurrent(); }); recoveryTiming.measure("route", verifyFinalRoute); preparedDependency = recoveryTiming.measure( @@ -1137,7 +1212,7 @@ export function recoverHermesPortableOllamaInference( recoveryTiming.measure("finalCurrentness", requireCompletionCurrent); registryRecovery.release(); preparedDependency?.release(); - recoveryTiming.finish("reused"); + recoveryTiming.finish("reused", timingCounts()); return "reused"; } catch (error) { if (preparedDependency) { @@ -1156,7 +1231,7 @@ export function recoverHermesPortableOllamaInference( let preparedDependency: HermesPortableOllamaPreparedProbeDependency | null = null; try { prepared = recoveryTiming.measure("preRouteCurrentness", () => { - requireTransactionCurrent(false); + requireRetainedCurrent(); return deps.prepareStartup( operation, createPublishedResumeRequest(receipt, published.receiptWriter), @@ -1197,7 +1272,7 @@ export function recoverHermesPortableOllamaInference( prepared.validateBeforeCommit(), "pre-commit recovery validation changed receipt", ); - requireTransactionCurrent(true); + requireRetainedCurrent(); }); recoveryTiming.measure("route", verifyFinalRoute); preparedDependency = recoveryTiming.measure( @@ -1218,7 +1293,7 @@ export function recoverHermesPortableOllamaInference( ollamaStateRestored = true; registryRecovery.release(); preparedDependency?.release(); - recoveryTiming.finish("recovered"); + recoveryTiming.finish("recovered", timingCounts()); return "recovered"; } catch (error) { let dependencyRollbackError: unknown = null; @@ -1242,22 +1317,27 @@ export function recoverHermesPortableOllamaInference( throw error; } } catch (error) { - if (!ollamaStateRestored) { - failRecovery( - "recovery failed before dependent runtime restoration was proved", - "runtime-restoration-unproved", - ); - } try { - registryRecovery.rollback(); - } catch { - failRecovery( - "recovery failed and exact stopped-registry restoration was not proved", - "registry-restoration-unproved", - ); + if (!ollamaStateRestored) { + failRecovery( + "recovery failed before dependent runtime restoration was proved", + "runtime-restoration-unproved", + ); + } + try { + registryRecovery.rollback(); + } catch { + failRecovery( + "recovery failed and exact stopped-registry restoration was not proved", + "registry-restoration-unproved", + ); + } + rethrowNestedHermesPortableRecoveryError(error); + throw error; + } finally { + recoveryTiming.finishEntryAuthority(); + recoveryTiming.finish(runtimeAction, timingCounts(), "failed"); } - rethrowNestedHermesPortableRecoveryError(error); - throw error; } } diff --git a/src/lib/onboard/experimental/hermes-portable-ollama-published-engine-recovery.test.ts b/src/lib/onboard/experimental/hermes-portable-ollama-published-engine-recovery.test.ts index 9019019a4ee..d018e3428e4 100644 --- a/src/lib/onboard/experimental/hermes-portable-ollama-published-engine-recovery.test.ts +++ b/src/lib/onboard/experimental/hermes-portable-ollama-published-engine-recovery.test.ts @@ -303,6 +303,7 @@ function composedRecovery(fixture: ReturnType, assertPublished = v }); const registryRecovery = { started: true, + assertRetainedCurrent: vi.fn(), assertTransactionCurrent: vi.fn(), assertCurrent: vi.fn(), rollback: vi.fn(), @@ -317,6 +318,7 @@ function composedRecovery(fixture: ReturnType, assertPublished = v inferenceStateDir: "/state/portable-inference/alpha", network: fixture.externalNetwork, operation: fixture.publishedOperation, + assertRetainedCurrent: assertRuntimeTransaction, assertTransactionCurrent: assertRuntimeTransaction, assertCurrent: assertRuntime, }; diff --git a/src/lib/onboard/experimental/hermes-portable-ollama-recovery.test.ts b/src/lib/onboard/experimental/hermes-portable-ollama-recovery.test.ts index fc8d0f8be4c..cae638c088c 100644 --- a/src/lib/onboard/experimental/hermes-portable-ollama-recovery.test.ts +++ b/src/lib/onboard/experimental/hermes-portable-ollama-recovery.test.ts @@ -158,7 +158,14 @@ function createHarness(initiallyRunning = false, registryInitiallyRunning = fals return { prepared, receipt }; }); const assertOperating = vi.fn(() => events.push("operating")); - const assertRuntime = vi.fn(() => events.push("runtime")); + const assertRuntimeRetainedCurrent = vi.fn(() => events.push("runtime-retained-current")); + const assertRuntimeTransactionCurrent = vi.fn(() => + events.push("runtime-transaction-current"), + ); + const assertRuntimeCurrent = vi.fn(() => { + events.push("runtime-current"); + expect(running).toBe(true); + }); const assertPublished = vi.fn(() => { events.push("publication"); }); @@ -187,6 +194,7 @@ function createHarness(initiallyRunning = false, registryInitiallyRunning = fals sandboxAuthoritySha256: "6".repeat(64), managedInspection, managedOperation, + assertPublishedRecoveryTransactionCurrent: managedOperation.assertTransactionCurrent, }; }, ); @@ -194,8 +202,9 @@ function createHarness(initiallyRunning = false, registryInitiallyRunning = fals bundle: { identity: { id: "podman" } }, inferenceStateDir: "/state/portable-inference/alpha", operation: managedOperation, - assertTransactionCurrent: assertRuntime, - assertCurrent: assertRuntime, + assertRetainedCurrent: assertRuntimeRetainedCurrent, + assertTransactionCurrent: assertRuntimeTransactionCurrent, + assertCurrent: assertRuntimeCurrent, })); const prepareRegistryRecovery = vi.fn(() => { const started = !registryRunning; @@ -211,6 +220,10 @@ function createHarness(initiallyRunning = false, registryInitiallyRunning = fals events.push("registry-transaction-current"); expect(registryRunning).toBe(true); }), + assertRetainedCurrent: vi.fn(() => { + events.push("registry-retained-current"); + expect(registryRunning).toBe(true); + }), rollback: vi.fn(() => { events.push("registry-rollback"); registryRunning = started ? false : registryRunning; @@ -232,6 +245,7 @@ function createHarness(initiallyRunning = false, registryInitiallyRunning = fals })), prepareInferenceAuthority, assertPreparedInferenceAuthorityCurrent: vi.fn(() => ({ running, receipt })), + assertPreparedInferenceAuthorityTransactionCurrent: vi.fn(), preparePublishedAuthority: vi.fn(() => ({ receipt, serializedReceipt, @@ -260,6 +274,9 @@ function createHarness(initiallyRunning = false, registryInitiallyRunning = fals }), }; return { + assertRuntimeCurrent, + assertRuntimeRetainedCurrent, + assertRuntimeTransactionCurrent, events, input, managedOperation, @@ -447,6 +464,46 @@ describe("Hermes Portable Ollama inference recovery", () => { expect(harness.events.at(-1)).toBe("registry-release"); }); + it("uses retained inference currentness until one final full qualification", () => { + const harness = createHarness(); + const retained = vi.fn(); + const full = vi.fn(() => ({ running: true, receipt: harness.receipt })); + harness.overrides.assertPreparedInferenceAuthorityTransactionCurrent = retained; + harness.overrides.assertPreparedInferenceAuthorityCurrent = full; + + expect(recoverHermesPortableOllamaInference(harness.input, harness.overrides as never)).toBe( + "recovered", + ); + + expect(retained).toHaveBeenCalledTimes(4); + expect(full).toHaveBeenCalledOnce(); + expect(harness.assertRuntimeRetainedCurrent).toHaveBeenCalled(); + expect(harness.assertRuntimeTransactionCurrent).toHaveBeenCalledOnce(); + expect(harness.assertRuntimeCurrent).toHaveBeenCalledOnce(); + expect(harness.overrides.prepareInferenceAuthority).toHaveBeenCalledOnce(); + expect(harness.writeExact).not.toHaveBeenCalled(); + }); + + it("rolls the exact stopped runtime back when retained inference authority drifts", () => { + const harness = createHarness(); + const drift = new Error("retained inference authority changed"); + harness.overrides.assertPreparedInferenceAuthorityTransactionCurrent = vi + .fn() + .mockImplementationOnce(() => undefined) + .mockImplementationOnce(() => { + throw drift; + }); + + expect(() => + recoverHermesPortableOllamaInference(harness.input, harness.overrides as never), + ).toThrow(drift); + + expect(harness.prepared.rollback).toHaveBeenCalledOnce(); + expect(harness.input.verifyRoute).not.toHaveBeenCalled(); + expect(harness.writeExact).not.toHaveBeenCalled(); + expect(harness.running()).toBe(false); + }); + it("releases a prepared probe dependency only after stopped-runtime finalization", () => { const harness = createHarness(); const dependency = { @@ -720,6 +777,7 @@ describe("Hermes Portable Ollama inference recovery", () => { }; harness.overrides.prepareRegistryRecovery.mockReturnValue({ started: false, + assertRetainedCurrent: vi.fn(), assertTransactionCurrent: vi.fn(), assertCurrent: vi.fn(), rollback: vi.fn(() => { @@ -757,16 +815,29 @@ describe("Hermes Portable Ollama inference recovery", () => { expect(harness.events.at(-1)).toBe("registry-release"); }); - it("restores the exact stopped state when final route verification fails", () => { + it("emits failed timing after final route failure restores the exact stopped state", () => { const harness = createHarness(); + let now = 0; + const routeError = new Error("route unavailable"); + const onComplete = vi.fn(() => harness.events.push("timing")); + Object.assign(harness.overrides, { + recoveryTiming: { + now: () => ++now, + onComplete, + }, + }); harness.input.verifyRoute.mockImplementation(() => { - throw new Error("route unavailable"); + throw routeError; }); - expect(() => - recoverHermesPortableOllamaInference(harness.input, harness.overrides as never), - ).toThrow("route unavailable"); + let caught: unknown; + try { + recoverHermesPortableOllamaInference(harness.input, harness.overrides as never); + } catch (error) { + caught = error; + } + expect(caught).toBe(routeError); expect(harness.prepared.rollback).toHaveBeenCalledOnce(); expect(harness.prepared.commit).not.toHaveBeenCalled(); expect(harness.running()).toBe(false); @@ -774,6 +845,19 @@ describe("Hermes Portable Ollama inference recovery", () => { expect(harness.events.indexOf("rollback")).toBeLessThan( harness.events.indexOf("registry-rollback"), ); + expect(harness.events.indexOf("registry-rollback")).toBeLessThan( + harness.events.indexOf("timing"), + ); + expect(onComplete).toHaveBeenCalledOnce(); + expect(onComplete).toHaveBeenCalledWith( + expect.objectContaining({ + dependencyMs: 0, + finalCurrentnessMs: 0, + result: "failed", + routeMs: 1, + runtimeAction: "recovered", + }), + ); }); it("restores the exact stopped state when provider revalidation fails", () => { @@ -888,11 +972,15 @@ describe("Hermes Portable Ollama inference recovery", () => { "entryAuthorityMs", "exactRuntimeInspectionMs", "finalCurrentnessMs", + "fullCurrentnessCount", "operatingAuthorityMs", "preRouteCurrentnessMs", + "preparedAuthorityInspectionCount", "preparedInferenceAuthorityMs", "privatePublicationMs", "registryPreparationMs", + "result", + "retainedCurrentnessCount", "routeMs", "runtimeAction", "runtimeAuthorityMs", @@ -904,6 +992,10 @@ describe("Hermes Portable Ollama inference recovery", () => { preparedInferenceAuthorityMs: 2, privatePublicationMs: 1, registryPreparationMs: 1, + retainedCurrentnessCount: action === "recovered" ? 4 : 3, + fullCurrentnessCount: 1, + preparedAuthorityInspectionCount: 2, + result: "proved", runtimeAction: action, runtimeAuthorityMs: 1, }); @@ -977,6 +1069,8 @@ describe("Hermes Portable Ollama inference recovery", () => { (owner, phase, registryRollbackCount) => { const harness = createHarness(); const canary = "nested recovery diagnostic canary"; + const onComplete = vi.fn(); + Object.assign(harness.overrides, { recoveryTiming: { onComplete } }); switch (owner) { case "registry": harness.overrides.prepareRegistryRecovery.mockImplementation(() => { @@ -1028,6 +1122,10 @@ describe("Hermes Portable Ollama inference recovery", () => { registryRollbackCount, ); expect(harness.prepareStartup).not.toHaveBeenCalled(); + expect(onComplete).toHaveBeenCalledOnce(); + expect(onComplete).toHaveBeenCalledWith( + expect.objectContaining({ result: "failed", runtimeAction: "unknown" }), + ); }, ); @@ -1069,6 +1167,7 @@ describe("Hermes Portable Ollama inference recovery", () => { const canary = "nested recovery diagnostic canary"; harness.overrides.prepareRegistryRecovery.mockReturnValue({ started: true, + assertRetainedCurrent: vi.fn(), assertTransactionCurrent: vi.fn(), assertCurrent: vi.fn(), rollback: vi.fn(() => { diff --git a/src/lib/onboard/experimental/hermes-portable-ollama-registry-recovery.test.ts b/src/lib/onboard/experimental/hermes-portable-ollama-registry-recovery.test.ts index 8930258f07c..fd9ab406c05 100644 --- a/src/lib/onboard/experimental/hermes-portable-ollama-registry-recovery.test.ts +++ b/src/lib/onboard/experimental/hermes-portable-ollama-registry-recovery.test.ts @@ -290,6 +290,34 @@ describe("Hermes Portable registry recovery", () => { expect(harness.calls.some((args) => args[0] === "start" || args[0] === "stop")).toBe(false); }); + it("uses retained transaction fences without another network or registry inspection", () => { + const harness = createRegistryHarness(true); + const assertEngineCurrent = vi.fn(); + const assertCallerCurrent = vi.fn(); + const transactionEngineCurrent = vi.fn(); + const transactionCallerCurrent = vi.fn(); + const prepared = preparePortableRegistryRecovery( + harness.engine as never, + harness.expectedAuthoritySha256, + assertEngineCurrent, + assertCallerCurrent, + {}, + { + assertEngineCurrent: transactionEngineCurrent, + assertCallerCurrent: transactionCallerCurrent, + }, + ); + const before = harness.calls.length; + + prepared.assertRetainedCurrent(); + + expect(transactionCallerCurrent).toHaveBeenCalledOnce(); + expect(transactionEngineCurrent).toHaveBeenCalledOnce(); + expect(assertCallerCurrent).toHaveBeenCalledOnce(); + expect(assertEngineCurrent).toHaveBeenCalledOnce(); + expect(harness.calls).toHaveLength(before); + }); + it("rejects canonical authority drift against the independently fixed receipt digest", () => { const harness = createRegistryHarness(true); expect(harness.expectedAuthoritySha256).toBe(EXPECTED_AUTHORITY_SHA256); diff --git a/src/lib/onboard/runtime-provider/host-local-inference-lifecycle.test.ts b/src/lib/onboard/runtime-provider/host-local-inference-lifecycle.test.ts index e92e69acfbf..1689208dc83 100644 --- a/src/lib/onboard/runtime-provider/host-local-inference-lifecycle.test.ts +++ b/src/lib/onboard/runtime-provider/host-local-inference-lifecycle.test.ts @@ -16,6 +16,7 @@ import type { } from "./host-local-inference"; import { serializeHostLocalInferenceReceipt } from "./host-local-inference"; import { + assertHermesPortableHostLocalInferencePublishedRecoveryTransactionCurrent, assertPreparedHostLocalInferenceRuntimePresent, confirmHostLocalInferenceAuthority, type ManagedHostLocalInferenceService, @@ -732,6 +733,33 @@ describe("host-local inference lifecycle authority", () => { expect(onTimingComplete).toHaveBeenCalledWith(4); }); + it("retains the published recovery operation between full runtime inspections", () => { + const value = receipt("ollama"); + const entry = sandbox("alpha", value, { agent: "hermes" }); + const runtimeProvider = provider({ + preparePublishedRecoveryEntry: (current) => ({ running: false, receipt: current }), + }); + const prepared = requiredPrepared( + prepareHermesPortableHostLocalInferencePublishedRecoveryAuthority( + runtimeProvider.bundle, + entry, + {}, + undefined, + runtimeProvider.operation, + ), + ); + + assertHermesPortableHostLocalInferencePublishedRecoveryTransactionCurrent( + runtimeProvider.bundle, + entry, + prepared, + ); + + expect(runtimeProvider.operation.assertTransactionCurrent).toHaveBeenCalledOnce(); + expect(runtimeProvider.runtime.inspectManaged).not.toHaveBeenCalled(); + expect(runtimeProvider.prepareDestroy).not.toHaveBeenCalled(); + }); + it("fails closed when published recovery has no dedicated entry proof", () => { const runtimeProvider = provider(); diff --git a/src/lib/onboard/runtime-provider/host-local-inference-lifecycle.ts b/src/lib/onboard/runtime-provider/host-local-inference-lifecycle.ts index 1a47ac22ae4..f33f9a2d48f 100644 --- a/src/lib/onboard/runtime-provider/host-local-inference-lifecycle.ts +++ b/src/lib/onboard/runtime-provider/host-local-inference-lifecycle.ts @@ -78,6 +78,8 @@ export interface PreparedHostLocalInferenceAuthority { readonly managedOperation?: HostLocalInferenceOperation; /** Exact managed state observed after the unchanged full published entry proof. */ readonly managedInspection?: HostLocalManagedInferenceInspection; + /** Rechecks the retained registry row and operation endpoint without inspecting the runtime. */ + readonly assertPublishedRecoveryTransactionCurrent?: () => void; } export type HostLocalInferenceRetirementResult = @@ -383,7 +385,21 @@ function prepare( ? "provider authority changed during destroy preflight" : "provider authority changed while it was being preserved", ); - return Object.freeze({ + const assertPublishedRecoveryTransactionCurrent = + authorityMode === "published-recovery" + ? () => { + if ( + !required.operation.assertTransactionCurrent || + required.operation.providerId !== provider.identity.id || + required.operation.engine.engineId !== receipt.engineAuthority.engineId || + required.operation.engine.operation !== "host-local-inference" + ) { + fail("published recovery operation authority is missing or changed"); + } + required.operation.assertTransactionCurrent(); + } + : undefined; + const prepared: PreparedHostLocalInferenceAuthority = { providerId: provider.identity.id, sandboxName: sandboxAuthority.sandboxName, serializedReceipt: serialized, @@ -396,11 +412,16 @@ function prepare( destroyRuntime: runtime, assertDestroyRuntimeAuthority: required.assertAuthority, ...(authorityMode === "published-recovery" - ? { managedInspection, managedOperation: required.operation } + ? { + managedInspection, + managedOperation: required.operation, + assertPublishedRecoveryTransactionCurrent, + } : {}), } : {}), - }); + }; + return Object.freeze(prepared); } /** Re-prove a canonical receipt only while it remains bound to the complete sandbox row. */ @@ -532,6 +553,20 @@ export function assertHermesPortableHostLocalInferencePublishedRecoveryAuthority return current; } +/** Recheck the retained published-recovery row and command endpoint without runtime inspection. */ +export function assertHermesPortableHostLocalInferencePublishedRecoveryTransactionCurrent( + provider: RuntimeProviderBundle, + sandbox: HostLocalInferenceLifecycleSandbox, + prepared: PreparedHostLocalInferenceAuthority, +): void { + requireCurrentSandboxAuthority(provider, sandbox, prepared, "destroy"); + const assertTransactionCurrent = prepared.assertPublishedRecoveryTransactionCurrent; + if (!assertTransactionCurrent) { + fail("published recovery transaction currentness is missing"); + } + assertTransactionCurrent(); +} + export function confirmHostLocalInferenceAuthority( provider: RuntimeProviderBundle, sandbox: HostLocalInferenceLifecycleSandbox, diff --git a/src/lib/onboard/runtime-provider/podman-host-local-inference.ts b/src/lib/onboard/runtime-provider/podman-host-local-inference.ts index bbf93fc100b..015c70cc601 100644 --- a/src/lib/onboard/runtime-provider/podman-host-local-inference.ts +++ b/src/lib/onboard/runtime-provider/podman-host-local-inference.ts @@ -4166,8 +4166,11 @@ export function createPodmanHostLocalInferenceRuntime( const current = inspectPublishedResumeTransaction(receipt); container = current.container; }; + const assertPublishedResumeTransactionCurrent = () => { + requirePublishedResumeTransactionCurrent(receipt); + }; const assertResumeForwardAuthority = publishedEngineAuthority - ? assertReceiptTransactionAuthority + ? assertPublishedResumeTransactionCurrent : assertReceiptAuthority; const assertRollbackReceiptAuthority = () => { assertReceiptExecutionAuthority("rollback"); @@ -4306,7 +4309,9 @@ export function createPodmanHostLocalInferenceRuntime( validateReceipt(receipt, true, resumeTiming); } }, - validatePublication: assertReceiptAuthority, + validatePublication: publishedEngineAuthority + ? assertPublishedResumeTransactionCurrent + : assertReceiptAuthority, publishedResume: true, onPublishedResumeFinalized: () => { resumeTiming.finish(wasRunning ? "reused" : "started");