diff --git a/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.test.ts b/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.test.ts index b3075757d9b..a9a5c935513 100644 --- a/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.test.ts +++ b/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.test.ts @@ -26,6 +26,7 @@ import type { HostLocalCreateJournalStore, } from "./host-local-create-journal"; import { + type HostLocalInferenceReceiptWriter, parseHostLocalInferenceReceipt, serializeHostLocalInferenceReceipt, } from "./host-local-inference"; @@ -37,6 +38,7 @@ const PROBE_IMAGE = `quay.io/curl/curl@sha256:${"d".repeat(64)}`; const RUNTIME_ID = "e".repeat(64); const NETWORK_ID = "7".repeat(64); const TRANSACTION_ID = "9".repeat(64); +const RECEIPT_TARGET_SHA256 = "8".repeat(64); const MODEL_CONTENT = Buffer.alloc(64, 0x61); const MODEL_FILENAME = "Nemotron-3-Nano-30B-A3B-UD-Q4_K_XL.gguf"; const REVISION = "f".repeat(40); @@ -77,6 +79,17 @@ function rawDigest(value: unknown): string { .digest("hex"); } +function receiptWriter( + writeExact: (serializedReceipt: string) => string = (serializedReceipt) => serializedReceipt, + overrides: Partial> = {}, +): HostLocalInferenceReceiptWriter & { readonly writeExact: ReturnType } { + return { + transactionId: overrides.transactionId ?? TRANSACTION_ID, + targetSha256: overrides.targetSha256 ?? RECEIPT_TARGET_SHA256, + writeExact: vi.fn(writeExact), + }; +} + beforeEach(() => { temporaryRoot = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-llama-life-"))); cacheRoot = path.join(temporaryRoot, "cache"); @@ -255,11 +268,17 @@ function authorityStore(): PersistedEngineAuthorityStore { interface TestJournalStore extends HostLocalCreateJournalStore { readonly abandonExecution: () => void; readonly hasExecution: () => boolean; + readonly failNextPrepareReceipt: () => void; + readonly failNextPrepareReceiptAfterCommit: () => void; + readonly failNextFinalize: () => void; } function journalStore(): TestJournalStore { const records = new Map(); let activeLease: HostLocalCreateJournalExecutionLease | null = null; + let prepareReceiptFails = false; + let prepareReceiptFailsAfterCommit = false; + let finalizeFails = false; const update = ( id: string, mutate: (value: HostLocalCreateJournalRecord) => HostLocalCreateJournalRecord, @@ -282,12 +301,36 @@ function journalStore(): TestJournalStore { recordCreated: (id, runtimeId) => update(id, (record) => ({ ...record, phase: "created", runtimeId })), recordStarted: (id) => update(id, (record) => ({ ...record, phase: "started" })), - finalize: (id, receiptSha256) => - update(id, (record) => ({ + prepareReceipt: (id, serializedReceipt) => { + switch (prepareReceiptFails) { + case true: + prepareReceiptFails = false; + throw new Error("prepare receipt failed"); + } + const prepared = update(id, (record) => ({ + ...record, + phase: "receipt-prepared", + serializedReceipt, + receiptSha256: createHash("sha256").update(serializedReceipt).digest("hex"), + })); + switch (prepareReceiptFailsAfterCommit) { + case true: + prepareReceiptFailsAfterCommit = false; + throw new Error("prepare receipt outcome unknown"); + } + return prepared; + }, + finalize: (id) => { + switch (finalizeFails) { + case true: + finalizeFails = false; + throw new Error("finalize failed"); + } + return update(id, (record) => ({ ...record, phase: "finalized", - receiptSha256, - })), + })); + }, retire: (id) => void records.delete(id), acquireExecution: (transactionId) => { invariant(activeLease === null, "execution is already owned by a live process"); @@ -308,6 +351,9 @@ function journalStore(): TestJournalStore { }, abandonExecution: () => (activeLease = null), hasExecution: () => activeLease !== null, + failNextPrepareReceipt: () => (prepareReceiptFails = true), + failNextPrepareReceiptAfterCommit: () => (prepareReceiptFailsAfterCommit = true), + failNextFinalize: () => (finalizeFails = true), }; } @@ -584,7 +630,6 @@ function options( function controller(fixture: DockerFixture, store = journalStore(), now: () => number = Date.now) { return createDockerLlamaCppManagedLifecycle(options(fixture, store), { - createTransactionId: () => TRANSACTION_ID, now, }); } @@ -616,6 +661,7 @@ function preparedJournal(): HostLocalCreateJournalRecord { value: "gateway.primary", }, probeImageReference: PROBE_IMAGE, + receiptTargetSha256: RECEIPT_TARGET_SHA256, runtimeGid: 1001, runtimeUid: 1001, }), @@ -630,6 +676,8 @@ function preparedJournal(): HostLocalCreateJournalRecord { }, apiKeyIdentitySha256: "3".repeat(64), apiKeyRootIdentitySha256: keyRootIdentitySha256(), + receiptTargetSha256: RECEIPT_TARGET_SHA256, + serializedReceipt: null, receiptSha256: null, }; } @@ -639,8 +687,8 @@ describe("dormant Docker llama.cpp managed lifecycle", () => { const fixture = dockerFixture(); const store = journalStore(); const lifecycle = controller(fixture, store); - const persist = vi.fn(); - const receipt = lifecycle.start(persist); + const writer = receiptWriter(); + const receipt = lifecycle.start(writer); const serialized = serializeHostLocalInferenceReceipt(receipt); expect(receipt.endpoint.port).toBe(49152); @@ -654,7 +702,7 @@ describe("dormant Docker llama.cpp managed lifecycle", () => { runtimeId: RUNTIME_ID, networkId: NETWORK_ID, }); - expect(persist).toHaveBeenCalledExactlyOnceWith(serialized); + expect(writer.writeExact).toHaveBeenCalledExactlyOnceWith(serialized); expect(serialized).not.toContain(modelPath); expect(serialized).not.toContain(apiKeyPath); expect(serialized).not.toContain("filesystemIdentity"); @@ -673,7 +721,7 @@ describe("dormant Docker llama.cpp managed lifecycle", () => { it("keeps already-absent destroy idempotent after its Docker network is removed (#8395)", () => { const fixture = dockerFixture(); const lifecycle = controller(fixture); - const receipt = lifecycle.start(vi.fn()); + const receipt = lifecycle.start(receiptWriter()); expect(lifecycle.runtime.destroy(receipt).status).toBe("removed"); fixture.removeNetwork(); expect(lifecycle.runtime.destroy(receipt).status).toBe("already-absent"); @@ -694,7 +742,7 @@ describe("dormant Docker llama.cpp managed lifecycle", () => { it("rejects a self-consistent plan for another GGUF before any mutation (#8395)", () => { const fixture = dockerFixture(); const store = journalStore(); - const persist = vi.fn(); + const writer = receiptWriter(); const original = plan(); const changedPayload = { schemaVersion: original.schemaVersion, @@ -716,17 +764,17 @@ describe("dormant Docker llama.cpp managed lifecycle", () => { ...changedPayload, planDigest: digest(changedPayload), }; - const lifecycle = createDockerLlamaCppManagedLifecycle( - { ...options(fixture, store), plan: changedPlan }, - { createTransactionId: () => TRANSACTION_ID }, - ); + const lifecycle = createDockerLlamaCppManagedLifecycle({ + ...options(fixture, store), + plan: changedPlan, + }); - expect(() => lifecycle.start(persist)).toThrow( + expect(() => lifecycle.start(writer)).toThrow( "plan, launch contract, and verified artifact disagree", ); expect(fixture.capture).not.toHaveBeenCalled(); expect(store.list()).toEqual([]); - expect(persist).not.toHaveBeenCalled(); + expect(writer.writeExact).not.toHaveBeenCalled(); }); it("accepts the canonical blob resolved by the plan's exact snapshot entry (#8395)", () => { @@ -739,7 +787,7 @@ describe("dormant Docker llama.cpp managed lifecycle", () => { const fixture = dockerFixture(); const lifecycle = controller(fixture); - const receipt = lifecycle.start(vi.fn()); + const receipt = lifecycle.start(receiptWriter()); expect(receipt.runtime).toMatchObject({ kind: "container", runtimeId: RUNTIME_ID, @@ -749,14 +797,18 @@ describe("dormant Docker llama.cpp managed lifecycle", () => { it("rejects writable cache authority and non-private API-key authority (#8395)", () => { fs.chmodSync(path.dirname(modelPath), 0o777); - expect(() => controller(dockerFixture()).start(vi.fn())).toThrow("owner-controlled"); + expect(() => controller(dockerFixture()).start(receiptWriter())).toThrow("owner-controlled"); fs.chmodSync(path.dirname(modelPath), 0o700); fs.chmodSync(apiKeyPath, 0o644); - expect(() => controller(dockerFixture()).start(vi.fn())).toThrow("private-file authority"); + expect(() => controller(dockerFixture()).start(receiptWriter())).toThrow( + "private-file authority", + ); fs.chmodSync(apiKeyPath, 0o600); fs.chmodSync(apiKeyRoot, 0o777); const unsafeParentFixture = dockerFixture(); - expect(() => controller(unsafeParentFixture).start(vi.fn())).toThrow("owner-controlled"); + expect(() => controller(unsafeParentFixture).start(receiptWriter())).toThrow( + "owner-controlled", + ); expect(unsafeParentFixture.capture).not.toHaveBeenCalled(); }); @@ -768,7 +820,7 @@ describe("dormant Docker llama.cpp managed lifecycle", () => { const future = new Date(Date.now() + 10_000); fs.utimesSync(modelPath, future, future); }); - expect(() => controller(fixture, store).start(vi.fn())).toThrow("filesystem identity"); + expect(() => controller(fixture, store).start(receiptWriter())).toThrow("filesystem identity"); expect(store.list()).toEqual([]); expect(fixture.capture.mock.calls.map((call) => call[0]?.slice(0, 2))).toContainEqual([ "rm", @@ -779,13 +831,13 @@ describe("dormant Docker llama.cpp managed lifecycle", () => { it("rolls back pathname replacement from inside Docker create capture before persistence (#8395)", () => { const fixture = dockerFixture(); const store = journalStore(); - const persist = vi.fn(); + const writer = receiptWriter(); fixture.onCreate(() => { fs.renameSync(modelPath, `${modelPath}.verified`); fs.writeFileSync(modelPath, MODEL_CONTENT, { mode: 0o600 }); }); - expect(() => controller(fixture, store).start(persist)).toThrow("filesystem identity"); - expect(persist).not.toHaveBeenCalled(); + expect(() => controller(fixture, store).start(writer)).toThrow("filesystem identity"); + expect(writer.writeExact).not.toHaveBeenCalled(); expect(store.list()).toEqual([]); expect(fixture.capture.mock.calls.map((call) => call[0]?.slice(0, 2))).toContainEqual([ "rm", @@ -796,7 +848,7 @@ describe("dormant Docker llama.cpp managed lifecycle", () => { it("rolls back an API-key root swap-and-restore inside Docker create capture (#8395)", () => { const fixture = dockerFixture(); const store = journalStore(); - const persist = vi.fn(); + const writer = receiptWriter(); fixture.onCreate(() => { const retained = `${apiKeyRoot}.retained`; fs.renameSync(apiKeyRoot, retained); @@ -809,8 +861,8 @@ describe("dormant Docker llama.cpp managed lifecycle", () => { const future = new Date(Date.now() + 10_000); fs.utimesSync(apiKeyRoot, future, future); }); - expect(() => controller(fixture, store).start(persist)).toThrow("API-key file changed"); - expect(persist).not.toHaveBeenCalled(); + expect(() => controller(fixture, store).start(writer)).toThrow("API-key file changed"); + expect(writer.writeExact).not.toHaveBeenCalled(); expect(store.list()).toEqual([]); expect(fixture.capture.mock.calls.map((call) => call[0]?.slice(0, 2))).toContainEqual([ "rm", @@ -818,23 +870,16 @@ describe("dormant Docker llama.cpp managed lifecycle", () => { ]); }); - it("rolls back malformed create output, readiness failure, and receipt persistence failure (#8395)", () => { + it("rolls back malformed create output and readiness failure before receipt prepare (#8395)", () => { const arrangeFailure = { stdout: (fixture: DockerFixture) => fixture.setCreateStdout("short-id\n"), probe: (fixture: DockerFixture) => fixture.failProbe(), - persist: (_fixture: DockerFixture) => undefined, } as const; - for (const failure of ["stdout", "probe", "persist"] as const) { + for (const failure of ["stdout", "probe"] as const) { const fixture = dockerFixture(); const store = journalStore(); arrangeFailure[failure](fixture); - const persist = - failure === "persist" - ? () => { - throw new Error("persist failed"); - } - : vi.fn(); - expect(() => controller(fixture, store).start(persist)).toThrow(); + expect(() => controller(fixture, store).start(receiptWriter())).toThrow(); expect(store.list()).toEqual([]); expect(fixture.capture.mock.calls.map((call) => call[0]?.slice(0, 2))).toContainEqual([ "rm", @@ -843,6 +888,144 @@ describe("dormant Docker llama.cpp managed lifecycle", () => { } }); + it("rolls back when durable receipt preparation fails before publication is possible (#8414)", () => { + const fixture = dockerFixture(); + const store = journalStore(); + store.failNextPrepareReceipt(); + + expect(() => controller(fixture, store).start(receiptWriter())).toThrow( + "prepare receipt failed", + ); + expect(store.list()).toEqual([]); + expect(fixture.capture.mock.calls.map((call) => call[0]?.slice(0, 2))).toContainEqual([ + "rm", + "--force", + ]); + }); + + it("preserves and replays when receipt preparation commits then throws (#8414)", () => { + const fixture = dockerFixture(); + const store = journalStore(); + const writer = receiptWriter(); + const lifecycle = controller(fixture, store); + store.failNextPrepareReceiptAfterCommit(); + + expect(() => lifecycle.start(writer)).toThrow("prepare receipt outcome unknown"); + expect(store.load(TRANSACTION_ID)?.phase).toBe("receipt-prepared"); + expect(writer.writeExact).not.toHaveBeenCalled(); + expect(fixture.capture.mock.calls.map((call) => call[0]?.slice(0, 2))).not.toContainEqual([ + "rm", + "--force", + ]); + expect(lifecycle.recoverUnfinished(writer)).toEqual({ + recovered: [TRANSACTION_ID], + failures: [], + }); + expect(store.load(TRANSACTION_ID)?.phase).toBe("finalized"); + expect(writer.writeExact).toHaveBeenCalledTimes(1); + }); + + it("preserves and replays a receipt when the exact writer commits then throws (#8414)", () => { + const fixture = dockerFixture(); + const store = journalStore(); + let committed: string | null = null; + const writer = receiptWriter((serializedReceipt) => { + switch (committed) { + case null: + committed = serializedReceipt; + throw new Error("writer outcome unknown"); + default: + invariant(committed === serializedReceipt, "different receipt"); + return committed; + } + }); + const lifecycle = controller(fixture, store); + + expect(() => lifecycle.start(writer)).toThrow("writer outcome unknown"); + expect(store.load(TRANSACTION_ID)).toMatchObject({ + phase: "receipt-prepared", + serializedReceipt: committed, + }); + expect(fixture.capture.mock.calls.map((call) => call[0]?.slice(0, 2))).not.toContainEqual([ + "rm", + "--force", + ]); + expect(lifecycle.recoverUnfinished(writer)).toEqual({ + recovered: [TRANSACTION_ID], + failures: [], + }); + expect(store.load(TRANSACTION_ID)?.phase).toBe("finalized"); + expect(writer.writeExact).toHaveBeenCalledTimes(2); + }); + + it("replays an exact committed receipt after journal finalization fails (#8414)", () => { + const fixture = dockerFixture(); + const store = journalStore(); + const writer = receiptWriter(); + const lifecycle = controller(fixture, store); + store.failNextFinalize(); + + expect(() => lifecycle.start(writer)).toThrow("finalize failed"); + expect(store.load(TRANSACTION_ID)?.phase).toBe("receipt-prepared"); + expect(lifecycle.recoverUnfinished(writer)).toEqual({ + recovered: [TRANSACTION_ID], + failures: [], + }); + expect(store.load(TRANSACTION_ID)?.phase).toBe("finalized"); + expect(writer.writeExact).toHaveBeenCalledTimes(2); + }); + + it("fails closed on receipt writer target or existing-value drift (#8414)", () => { + for (const drift of ["transaction", "target", "value"] as const) { + const fixture = dockerFixture(); + const store = journalStore(); + const initialWriter = receiptWriter(() => { + throw new Error("writer unavailable"); + }); + const lifecycle = controller(fixture, store); + expect(() => lifecycle.start(initialWriter)).toThrow("writer unavailable"); + const writesBeforeRecovery = fixture.capture.mock.calls.length; + const recoveryWriter = + drift === "transaction" + ? receiptWriter(undefined, { transactionId: "5".repeat(64) }) + : drift === "target" + ? receiptWriter(undefined, { targetSha256: "6".repeat(64) }) + : receiptWriter(() => { + throw new Error("different existing receipt"); + }); + + const recovery = lifecycle.recoverUnfinished(recoveryWriter); + expect(recovery.recovered).toEqual([]); + expect(recovery.failures[0]?.message).toContain( + drift === "value" ? "different existing receipt" : "publication authority", + ); + expect(store.load(TRANSACTION_ID)?.phase).toBe("receipt-prepared"); + switch (drift) { + case "transaction": + case "target": + expect(fixture.capture.mock.calls).toHaveLength(writesBeforeRecovery); + } + } + }); + + it("re-proves the verified model before replaying a prepared receipt (#8414)", () => { + const fixture = dockerFixture(); + const store = journalStore(); + const unavailableWriter = receiptWriter(() => { + throw new Error("writer unavailable"); + }); + const lifecycle = controller(fixture, store); + expect(() => lifecycle.start(unavailableWriter)).toThrow("writer unavailable"); + fs.writeFileSync(modelPath, Buffer.alloc(MODEL_CONTENT.length, 0x62)); + const replayWriter = receiptWriter(); + + const recovery = lifecycle.recoverUnfinished(replayWriter); + expect(recovery.recovered).toEqual([]); + expect(recovery.failures[0]?.message).toContain("filesystem identity"); + expect(replayWriter.writeExact).not.toHaveBeenCalled(); + expect(store.load(TRANSACTION_ID)?.phase).toBe("receipt-prepared"); + }); + it("holds execution authority after an uncertain create and recovers a late exact container (#8395)", () => { const fixture = dockerFixture(); const store = journalStore(); @@ -850,7 +1033,8 @@ describe("dormant Docker llama.cpp managed lifecycle", () => { const lifecycle = controller(fixture, store, () => now); fixture.failCreateUncertain(); - expect(() => lifecycle.start(vi.fn())).toThrow("container create failed"); + const writer = receiptWriter(); + expect(() => lifecycle.start(writer)).toThrow("container create failed"); const creating = store.load(TRANSACTION_ID); expect(creating).toMatchObject({ phase: "creating", @@ -859,13 +1043,13 @@ describe("dormant Docker llama.cpp managed lifecycle", () => { }); expect(store.hasExecution()).toBe(true); - const concurrent = lifecycle.recoverUnfinished(); + const concurrent = lifecycle.recoverUnfinished(writer); expect(concurrent.recovered).toEqual([]); expect(concurrent.failures[0]?.message).toContain("already owned"); expect(store.load(TRANSACTION_ID)).not.toBeNull(); store.abandonExecution(); - const insideGrace = lifecycle.recoverUnfinished(); + const insideGrace = lifecycle.recoverUnfinished(writer); expect(insideGrace.recovered).toEqual([]); expect(insideGrace.failures[0]?.message).toContain("absence grace period"); expect(store.load(TRANSACTION_ID)).not.toBeNull(); @@ -880,7 +1064,7 @@ describe("dormant Docker llama.cpp managed lifecycle", () => { fixture.seed(creating, false); } }); - expect(lifecycle.recoverUnfinished()).toEqual({ + expect(lifecycle.recoverUnfinished(writer)).toEqual({ recovered: [TRANSACTION_ID], failures: [], }); @@ -913,7 +1097,7 @@ describe("dormant Docker llama.cpp managed lifecycle", () => { const recovery = createDockerLlamaCppManagedLifecycle( options(fixture, store, bindings(), persistedAuthority), { now: () => 31 * 60 * 1_000 }, - ).recoverUnfinished(); + ).recoverUnfinished(receiptWriter()); expect(recovery).toEqual({ recovered: [TRANSACTION_ID], failures: [] }); expect(store.list()).toEqual([]); } @@ -940,7 +1124,7 @@ describe("dormant Docker llama.cpp managed lifecycle", () => { arrangeAuthority[state](); const recovery = createDockerLlamaCppManagedLifecycle( options(fixture, store, bindings(), persistedAuthority), - ).recoverUnfinished(); + ).recoverUnfinished(receiptWriter()); expect(recovery.recovered).toEqual([]); expect(recovery.failures).toHaveLength(1); expect(store.load(TRANSACTION_ID)).not.toBeNull(); @@ -954,7 +1138,7 @@ describe("dormant Docker llama.cpp managed lifecycle", () => { it("fails re-prove on Docker network identity drift (#8395)", () => { const fixture = dockerFixture(); const lifecycle = controller(fixture); - const receipt = lifecycle.start(vi.fn()); + const receipt = lifecycle.start(receiptWriter()); fixture.setNetworkId("8".repeat(64)); expect(() => lifecycle.runtime.preserveForRebuild(receipt)).toThrow( "internal network identity changed", @@ -964,7 +1148,7 @@ describe("dormant Docker llama.cpp managed lifecycle", () => { it("rejects effective hardening drift after creation (#8395)", () => { const fixture = dockerFixture(); const lifecycle = controller(fixture); - const receipt = lifecycle.start(vi.fn()); + const receipt = lifecycle.start(receiptWriter()); fixture.driftHardening(); expect(() => lifecycle.runtime.inspectManaged(receipt)).toThrow("exact journal authority"); @@ -977,7 +1161,7 @@ describe("dormant Docker llama.cpp managed lifecycle", () => { ]) { const candidate = dockerFixture(); const candidateLifecycle = controller(candidate); - const candidateReceipt = candidateLifecycle.start(vi.fn()); + const candidateReceipt = candidateLifecycle.start(receiptWriter()); mutate(candidate); expect(() => candidateLifecycle.runtime.inspectManaged(candidateReceipt)).toThrow( "exact journal authority", @@ -989,7 +1173,7 @@ describe("dormant Docker llama.cpp managed lifecycle", () => { const fixture = dockerFixture(); const store = journalStore(); const lifecycle = controller(fixture, store); - const receipt = lifecycle.start(vi.fn()); + const receipt = lifecycle.start(receiptWriter()); invariant(receipt.runtime.kind === "container", "expected container receipt"); const crafted = { ...receipt, @@ -1001,7 +1185,7 @@ describe("dormant Docker llama.cpp managed lifecycle", () => { const unavailable = dockerFixture(); const unavailableStore = journalStore(); unavailable.failInspectWithDaemonError(); - expect(() => controller(unavailable, unavailableStore).start(vi.fn())).toThrow( + expect(() => controller(unavailable, unavailableStore).start(receiptWriter())).toThrow( "container inspection failed", ); expect(unavailableStore.list()).toEqual([]); diff --git a/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.ts b/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.ts index 82ff80d49e4..c69ff020de5 100644 --- a/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.ts +++ b/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { createHash, randomUUID } from "node:crypto"; +import { createHash } from "node:crypto"; import fs, { type BigIntStats } from "node:fs"; import path from "node:path"; @@ -27,10 +27,15 @@ import { type HostLocalCreateJournalStore, normalizeHostLocalCreateJournalRecord, } from "./host-local-create-journal"; -import type { HostLocalInferenceReceipt, HostLocalInferenceRuntime } from "./host-local-inference"; +import type { + HostLocalInferenceReceipt, + HostLocalInferenceReceiptWriter, + HostLocalInferenceRuntime, +} from "./host-local-inference"; import { normalizeHostLocalInferenceImageRef, normalizeHostLocalInferenceReceipt, + parseHostLocalInferenceReceipt, serializeHostLocalInferenceReceipt, } from "./host-local-inference"; import { @@ -71,7 +76,6 @@ export interface DockerLlamaCppManagedLifecycleOptions { } export interface DockerLlamaCppManagedLifecycleDependencies { - readonly createTransactionId?: () => string; readonly now?: () => number; } @@ -85,8 +89,8 @@ export interface DockerLlamaCppRecoveryResult { export interface DockerLlamaCppManagedLifecycle { readonly runtime: HostLocalInferenceRuntime; - start(persistReceipt: (serializedReceipt: string) => void): HostLocalInferenceReceipt; - recoverUnfinished(): DockerLlamaCppRecoveryResult; + start(writer: HostLocalInferenceReceiptWriter): HostLocalInferenceReceipt; + recoverUnfinished(writer: HostLocalInferenceReceiptWriter): DockerLlamaCppRecoveryResult; } interface DockerNetworkAuthority { @@ -165,6 +169,10 @@ function sha256(value: unknown): string { .digest("hex"); } +function sha256Text(value: string): string { + return createHash("sha256").update(value).digest("hex"); +} + function record(value: unknown, label: string): Record { if (typeof value !== "object" || value === null || Array.isArray(value)) { throw new Error(`${label} must be an object.`); @@ -660,6 +668,7 @@ function specificationDigest( options: DockerLlamaCppManagedLifecycleOptions, network: DockerNetworkAuthority, apiKeyRootIdentity: string, + receiptTargetSha256: string, ): string { return sha256({ apiKeyRootIdentitySha256: apiKeyRootIdentity, @@ -675,6 +684,7 @@ function specificationDigest( network, ownerLabel: options.bindings.ownerLabel, probeImageReference: options.probeImageReference, + receiptTargetSha256, runtimeGid: options.bindings.runtimeGid, runtimeUid: options.bindings.runtimeUid, }); @@ -898,6 +908,52 @@ function operationTime(dependencies: DockerLlamaCppManagedLifecycleDependencies) return value; } +function requireReceiptWriter( + value: HostLocalInferenceReceiptWriter, +): HostLocalInferenceReceiptWriter { + const transactionId = value?.transactionId; + const targetSha256 = value?.targetSha256; + const writeExact = value?.writeExact; + if ( + typeof value !== "object" || + value === null || + typeof transactionId !== "string" || + !SHA256.test(transactionId) || + typeof targetSha256 !== "string" || + !SHA256.test(targetSha256) || + typeof writeExact !== "function" + ) { + throw new Error("Docker llama.cpp receipt writer authority is malformed."); + } + return Object.freeze({ + transactionId, + targetSha256, + writeExact: (serializedReceipt: string) => writeExact.call(value, serializedReceipt), + }); +} + +function writePreparedReceipt( + writerValue: HostLocalInferenceReceiptWriter, + journalValue: HostLocalCreateJournalRecord, +): string { + const writer = requireReceiptWriter(writerValue); + const journal = normalizeHostLocalCreateJournalRecord(journalValue); + if ( + journal.phase !== "receipt-prepared" || + journal.serializedReceipt === null || + journal.receiptSha256 === null || + writer.transactionId !== journal.transactionId || + writer.targetSha256 !== journal.receiptTargetSha256 + ) { + throw new Error("Docker llama.cpp receipt writer differs from prepared publication authority."); + } + const committed = writer.writeExact(journal.serializedReceipt); + if (committed !== journal.serializedReceipt) { + throw new Error("Docker llama.cpp receipt writer did not acknowledge the exact receipt."); + } + return committed; +} + export function createDockerLlamaCppManagedLifecycle( options: DockerLlamaCppManagedLifecycleOptions, dependencies: DockerLlamaCppManagedLifecycleDependencies = {}, @@ -962,7 +1018,11 @@ export function createDockerLlamaCppManagedLifecycle( options, { id: journal.networkId, name: options.bindings.network.name }, journal.apiKeyRootIdentitySha256, + journal.receiptTargetSha256, ); + const serializedReceipt = serializeHostLocalInferenceReceipt(receipt); + const publicationPrepared = + journal?.phase === "receipt-prepared" || journal?.phase === "finalized"; if ( journal === null || journal.providerId !== PROVIDER_ID || @@ -972,9 +1032,11 @@ export function createDockerLlamaCppManagedLifecycle( journal.specSha256 !== expectedSpecSha256 || journal.specSha256 !== receipt.runtime.specSha256 || JSON.stringify(journal.engineAuthority) !== JSON.stringify(qualifiedAuthority) || + !publicationPrepared || + journal.serializedReceipt !== serializedReceipt || + journal.receiptSha256 !== sha256Text(serializedReceipt) || (requireFinalized && journal.phase !== "finalized") || - (journal.phase === "finalized" && - journal.receiptSha256 !== sha256(JSON.parse(serializeHostLocalInferenceReceipt(receipt)))) + (journal.phase !== "receipt-prepared" && journal.phase !== "finalized") ) { throw new Error("Docker llama.cpp receipt does not match its durable create journal."); } @@ -1178,25 +1240,25 @@ export function createDockerLlamaCppManagedLifecycle( return Object.freeze({ runtime, - start(persistReceipt: (serializedReceipt: string) => void) { - if (typeof persistReceipt !== "function") { - throw new Error("Docker llama.cpp start requires an operation-scoped receipt writer."); - } + start(writerValue: HostLocalInferenceReceiptWriter) { + const writer = requireReceiptWriter(writerValue); assertLlamaCppGgufCachePlanDigest(options.plan); assertModelFilesystemAuthority(options); const startingApiKeyRootIdentitySha256 = apiKeyRootIdentitySha256(options); const startingKeyIdentity = apiKeyIdentity(options); const network = inspectNetwork(options.engine, options.bindings.network.name); const authority = authorizeEngine(options, qualifiedAuthority, true); - const specSha256 = specificationDigest(options, network, startingApiKeyRootIdentitySha256); - const transactionId = - dependencies.createTransactionId?.() ?? sha256({ generation: randomUUID() }); - if (!SHA256.test(transactionId)) { - throw new Error("Docker llama.cpp transaction identity is malformed."); - } + const transactionId = writer.transactionId; + const specSha256 = specificationDigest( + options, + network, + startingApiKeyRootIdentitySha256, + writer.targetSha256, + ); const lease = options.journalStore.acquireExecution(transactionId); const execution: MutationExecutionState = { unknown: false }; let journal: HostLocalCreateJournalRecord | null = null; + let receiptPublicationPossible = false; try { if ( inspectContainer( @@ -1223,6 +1285,8 @@ export function createDockerLlamaCppManagedLifecycle( engineAuthority: authority, apiKeyIdentitySha256: apiKeyIdentitySha256(startingKeyIdentity), apiKeyRootIdentitySha256: startingApiKeyRootIdentitySha256, + receiptTargetSha256: writer.targetSha256, + serializedReceipt: null, receiptSha256: null, }); options.journalStore.assertExecution(lease); @@ -1287,17 +1351,39 @@ export function createDockerLlamaCppManagedLifecycle( requireExactNetwork(options, network.id); const receipt = receiptFor(options, authority, journal, started); const serialized = serializeHostLocalInferenceReceipt(receipt); - // Schema-v1 llama.cpp receipt persistence remains rejected by the production registry. - // Activation must make this persist/finalize boundary atomically durable first; #8414 - // tracks that commit boundary: https://github.com/NVIDIA/NemoClaw/issues/8414 - persistReceipt(serialized); options.journalStore.assertExecution(lease); - options.journalStore.finalize(transactionId, sha256(JSON.parse(serialized))); + try { + journal = options.journalStore.prepareReceipt(transactionId, serialized); + receiptPublicationPossible = true; + } catch (error) { + try { + const persisted = options.journalStore.load(transactionId); + if (persisted === null) { + receiptPublicationPossible = true; + } else { + journal = normalizeHostLocalCreateJournalRecord(persisted); + receiptPublicationPossible = journal.phase !== "started"; + } + } catch { + receiptPublicationPossible = true; + } + throw error; + } + options.journalStore.assertExecution(lease); + writePreparedReceipt(writer, journal); + options.journalStore.assertExecution(lease); + journal = options.journalStore.finalize(transactionId); options.journalStore.assertExecution(lease); return receipt; } catch (error) { let rollbackFailure: unknown; - if (journal !== null && !execution.unknown) { + if ( + journal !== null && + !receiptPublicationPossible && + journal.phase !== "receipt-prepared" && + journal.phase !== "finalized" && + !execution.unknown + ) { try { rollbackExact(options, journal, lease, execution); } catch (rollbackError) { @@ -1314,7 +1400,8 @@ export function createDockerLlamaCppManagedLifecycle( if (!execution.unknown) options.journalStore.releaseExecution(lease); } }, - recoverUnfinished() { + recoverUnfinished(writerValue: HostLocalInferenceReceiptWriter) { + const writer = requireReceiptWriter(writerValue); const recovered: string[] = []; const failures: { transactionId: string; message: string }[] = []; for (const candidate of options.journalStore.list()) { @@ -1334,6 +1421,15 @@ export function createDockerLlamaCppManagedLifecycle( if (active === null) continue; journal = normalizeHostLocalCreateJournalRecord(active); if (journal.phase === "finalized") continue; + if ( + journal.phase === "receipt-prepared" && + (writer.transactionId !== journal.transactionId || + writer.targetSha256 !== journal.receiptTargetSha256) + ) { + throw new Error( + "Docker llama.cpp receipt writer differs from prepared publication authority.", + ); + } const currentAuthority = authorizeEngine(options, qualifiedAuthority, false); const journalAuthority = requirePersistedEngineAuthority( journal.engineAuthority, @@ -1348,6 +1444,7 @@ export function createDockerLlamaCppManagedLifecycle( options, { id: journal.networkId, name: options.bindings.network.name }, journal.apiKeyRootIdentitySha256, + journal.receiptTargetSha256, ); if (journal.specSha256 !== expectedSpecSha256) { throw new Error( @@ -1355,13 +1452,39 @@ export function createDockerLlamaCppManagedLifecycle( ); } requireExactNetwork(options, journal.networkId); - rollbackExact( - options, - journal, - lease, - execution, - journal.phase === "creating" ? operationTime(dependencies) : undefined, - ); + if (journal.phase === "receipt-prepared") { + if (journal.serializedReceipt === null) { + throw new Error("Docker llama.cpp prepared receipt is missing."); + } + const receipt = parseHostLocalInferenceReceipt(journal.serializedReceipt); + const activeKeyIdentity = apiKeyIdentity(options); + if (apiKeyIdentitySha256(activeKeyIdentity) !== journal.apiKeyIdentitySha256) { + throw new Error("Docker llama.cpp API-key identity differs from its create journal."); + } + assertModelFilesystemAuthority(options); + assertApiKeyIdentity(options, activeKeyIdentity, journal.apiKeyRootIdentitySha256); + const inspected = inspectAuthorized(receipt, false); + if (!inspected.container.running) { + throw new Error("Docker llama.cpp receipt publication requires a running runtime."); + } + probeReady(options, lease, execution); + assertModelFilesystemAuthority(options); + assertApiKeyIdentity(options, activeKeyIdentity, journal.apiKeyRootIdentitySha256); + requireExactNetwork(options, journal.networkId); + options.journalStore.assertExecution(lease); + writePreparedReceipt(writer, journal); + options.journalStore.assertExecution(lease); + options.journalStore.finalize(journal.transactionId); + options.journalStore.assertExecution(lease); + } else { + rollbackExact( + options, + journal, + lease, + execution, + journal.phase === "creating" ? operationTime(dependencies) : undefined, + ); + } recovered.push(journal.transactionId); } catch (error) { failures.push({ diff --git a/src/lib/onboard/runtime-provider/host-local-create-journal.test.ts b/src/lib/onboard/runtime-provider/host-local-create-journal.test.ts index 975b4d327f7..e327612fa54 100644 --- a/src/lib/onboard/runtime-provider/host-local-create-journal.test.ts +++ b/src/lib/onboard/runtime-provider/host-local-create-journal.test.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { createHash } from "node:crypto"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -13,10 +14,10 @@ import { type HostLocalCreateJournalRecord, serializeHostLocalCreateJournalRecord, } from "./host-local-create-journal"; +import { serializeHostLocalInferenceReceipt } from "./host-local-inference"; const TRANSACTION_ID = "a".repeat(64); const RUNTIME_ID = "b".repeat(64); -const RECEIPT_SHA256 = "c".repeat(64); const CREATE_INTENT_UNIX_MS = 1_786_000_000_000; const OWNER_ONE = "11111111-1111-4111-8111-111111111111"; const OWNER_TWO = "22222222-2222-4222-8222-222222222222"; @@ -56,10 +57,39 @@ function prepared(): HostLocalCreateJournalRecord { authorityId: "docker:local", bindingSha256: "f".repeat(64), }, + receiptTargetSha256: "3".repeat(64), + serializedReceipt: null, receiptSha256: null, }; } +function serializedReceipt(): string { + const authority = prepared(); + return serializeHostLocalInferenceReceipt({ + schemaVersion: 1, + providerId: authority.providerId, + service: "llama-cpp", + engineAuthority: authority.engineAuthority, + endpoint: { host: "host.openshell.internal", port: 49152, networkName: "internal" }, + runtime: { + kind: "container", + runtimeId: RUNTIME_ID, + name: authority.containerName, + imageRef: `ghcr.io/nvidia/llama-cpp@sha256:${"4".repeat(64)}`, + probeImageRef: `quay.io/curl/curl@sha256:${"5".repeat(64)}`, + specSha256: authority.specSha256, + model: { + planDigest: `sha256:${"6".repeat(64)}`, + recipeId: "llama-cpp.nemotron.spark.v1", + generation: TRANSACTION_ID, + digest: `sha256:${"7".repeat(64)}`, + sizeBytes: 64, + }, + gpu: { vendor: "nvidia", count: 1 }, + }, + }); +} + function journalPath(): string { return path.join(stateDirectory, HOST_LOCAL_CREATE_JOURNAL_DIRECTORY, `${TRANSACTION_ID}.json`); } @@ -73,7 +103,7 @@ function executionSource(transactionId: string, ownerId: string, ownerPid: numbe } describe("host-local create journal", () => { - it("durably resumes every create phase without persisting executor paths or secrets (#8395)", () => { + it("durably resumes every create and receipt-publication phase without secrets (#8414)", () => { const fsync = vi.spyOn(fs, "fsyncSync"); const first = createHostLocalCreateJournalStore(stateDirectory); expect(first.create(prepared()).phase).toBe("prepared"); @@ -82,8 +112,16 @@ describe("host-local create journal", () => { const restarted = createHostLocalCreateJournalStore(stateDirectory); expect(restarted.recordStarted(TRANSACTION_ID).phase).toBe("started"); - const finalized = restarted.finalize(TRANSACTION_ID, RECEIPT_SHA256); + const receipt = serializedReceipt(); + const preparedReceipt = restarted.prepareReceipt(TRANSACTION_ID, receipt); + expect(preparedReceipt).toMatchObject({ + phase: "receipt-prepared", + serializedReceipt: receipt, + receiptSha256: createHash("sha256").update(receipt).digest("hex"), + }); + const finalized = restarted.finalize(TRANSACTION_ID); expect(finalized.phase).toBe("finalized"); + expect(restarted.finalize(TRANSACTION_ID)).toEqual(finalized); expect(restarted.list()).toEqual([finalized]); const serialized = fs.readFileSync(journalPath(), "utf8"); @@ -106,8 +144,52 @@ describe("host-local create journal", () => { expect(() => store.recordStarted(TRANSACTION_ID)).toThrow( "only a created transaction can record start", ); - expect(() => store.finalize(TRANSACTION_ID, RECEIPT_SHA256)).toThrow( - "only a started transaction can finalize", + expect(() => store.prepareReceipt(TRANSACTION_ID, serializedReceipt())).toThrow( + "only a started transaction can prepare receipt publication", + ); + expect(() => store.finalize(TRANSACTION_ID)).toThrow( + "only a receipt-prepared transaction can finalize", + ); + }); + + it("rejects noncanonical receipt bytes before publication intent is durable (#8414)", () => { + const store = createHostLocalCreateJournalStore(stateDirectory); + store.create(prepared()); + store.recordCreating(TRANSACTION_ID, CREATE_INTENT_UNIX_MS); + store.recordCreated(TRANSACTION_ID, RUNTIME_ID); + store.recordStarted(TRANSACTION_ID); + + expect(() => store.prepareReceipt(TRANSACTION_ID, serializedReceipt().trim())).toThrow( + "Host-local create journal is invalid: prepared receipt is invalid", + ); + expect(store.load(TRANSACTION_ID)?.phase).toBe("started"); + }); + + it.each([ + ["receipt without its digest", { serializedReceipt: "hostPath=/secret\n" }], + ["digest without its receipt", { receiptSha256: "9".repeat(64) }], + ])("rejects partial %s journal state (#8414)", (_name, partial) => { + expect(() => + serializeHostLocalCreateJournalRecord({ + ...prepared(), + ...partial, + }), + ).toThrow("Host-local create journal is invalid: phase and receipt fields disagree"); + }); + + it("rejects receipt or digest tampering after a durable publication prepare (#8414)", () => { + const store = createHostLocalCreateJournalStore(stateDirectory); + store.create(prepared()); + store.recordCreating(TRANSACTION_ID, CREATE_INTENT_UNIX_MS); + store.recordCreated(TRANSACTION_ID, RUNTIME_ID); + store.recordStarted(TRANSACTION_ID); + store.prepareReceipt(TRANSACTION_ID, serializedReceipt()); + + const record = JSON.parse(fs.readFileSync(journalPath(), "utf8")); + record.receiptSha256 = "8".repeat(64); + fs.writeFileSync(journalPath(), `${JSON.stringify(record)}\n`, { mode: 0o600 }); + expect(() => store.load(TRANSACTION_ID)).toThrow( + "Host-local create journal is invalid: prepared receipt digest changed", ); }); diff --git a/src/lib/onboard/runtime-provider/host-local-create-journal.ts b/src/lib/onboard/runtime-provider/host-local-create-journal.ts index 7f65c1e2e41..22bd7781315 100644 --- a/src/lib/onboard/runtime-provider/host-local-create-journal.ts +++ b/src/lib/onboard/runtime-provider/host-local-create-journal.ts @@ -1,10 +1,13 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { randomUUID } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import fs, { type BigIntStats } from "node:fs"; import path from "node:path"; - +import { + parseHostLocalInferenceReceipt, + serializeHostLocalInferenceReceipt, +} from "./host-local-inference"; import { normalizePersistedEngineAuthority, type PersistedEngineAuthority, @@ -18,6 +21,7 @@ export type HostLocalCreateJournalPhase = | "creating" | "created" | "started" + | "receipt-prepared" | "finalized"; export interface HostLocalCreateJournalRecord { @@ -37,6 +41,10 @@ export interface HostLocalCreateJournalRecord { /** Path-free identity of the private directory chain that owns the API-key pathname. */ readonly apiKeyRootIdentitySha256: string; readonly engineAuthority: PersistedEngineAuthority; + /** Path- and value-free identity of the operation-scoped external state target. */ + readonly receiptTargetSha256: string; + /** Canonical secret-free receipt retained for exact crash replay. */ + readonly serializedReceipt: string | null; readonly receiptSha256: string | null; } @@ -60,7 +68,11 @@ export interface HostLocalCreateJournalStore { runtimeId: string, ) => HostLocalCreateJournalRecord; readonly recordStarted: (transactionId: string) => HostLocalCreateJournalRecord; - readonly finalize: (transactionId: string, receiptSha256: string) => HostLocalCreateJournalRecord; + readonly prepareReceipt: ( + transactionId: string, + serializedReceipt: string, + ) => HostLocalCreateJournalRecord; + readonly finalize: (transactionId: string) => HostLocalCreateJournalRecord; readonly retire: (transactionId: string) => void; readonly acquireExecution: (transactionId: string) => HostLocalCreateJournalExecutionLease; readonly assertExecution: (lease: HostLocalCreateJournalExecutionLease) => void; @@ -75,7 +87,7 @@ export interface HostLocalCreateJournalStoreDependencies { const DIRECTORY_MODE = 0o700; const FILE_MODE = 0o600; -const MAX_BYTES = 32 * 1024; +const MAX_BYTES = 64 * 1024; const SHA256 = /^[a-f0-9]{64}$/u; const PROVIDER = /^[a-z][a-z0-9-]{0,62}$/u; const SERVICE = /^[a-z][a-z0-9.-]{0,62}$/u; @@ -89,6 +101,7 @@ const PHASES = new Set([ "creating", "created", "started", + "receipt-prepared", "finalized", ]); @@ -115,6 +128,14 @@ function exactUnixMs(value: unknown): number { return Number(value); } +function parsePreparedReceipt(serialized: string) { + try { + return parseHostLocalInferenceReceipt(serialized); + } catch (error) { + fail(`prepared receipt is invalid: ${error instanceof Error ? error.message : String(error)}`); + } +} + function normalizeExecutionLease(value: unknown): HostLocalCreateJournalExecutionLease { if (typeof value !== "object" || value === null || Array.isArray(value)) { fail("execution lease must be an object"); @@ -155,8 +176,10 @@ export function normalizeHostLocalCreateJournalRecord( "phase", "providerId", "receiptSha256", + "receiptTargetSha256", "runtimeId", "schemaVersion", + "serializedReceipt", "service", "specSha256", "transactionId", @@ -170,6 +193,9 @@ export function normalizeHostLocalCreateJournalRecord( fail("record schema is unsupported"); } const phase = record.phase as HostLocalCreateJournalPhase; + const transactionId = exactText(record.transactionId, SHA256, "transaction identity"); + const providerId = exactText(record.providerId, PROVIDER, "provider identity"); + const service = exactText(record.service, SERVICE, "service identity"); const runtimeId = record.runtimeId === null ? null : exactText(record.runtimeId, RUNTIME_ID, "runtime identity"); const createIntentUnixMs = @@ -178,25 +204,54 @@ export function normalizeHostLocalCreateJournalRecord( record.receiptSha256 === null ? null : exactText(record.receiptSha256, SHA256, "receipt digest"); + const serializedReceipt = + record.serializedReceipt === null + ? null + : typeof record.serializedReceipt === "string" + ? record.serializedReceipt + : fail("serialized receipt must be canonical text or null"); if ((phase === "prepared" || phase === "creating") !== (runtimeId === null)) { fail("phase and runtime identity disagree"); } if ((phase === "prepared") !== (createIntentUnixMs === null)) { fail("phase and create intent timestamp disagree"); } - if ((phase === "finalized") !== (receiptSha256 !== null)) { - fail("phase and receipt digest disagree"); + const hasPreparedReceipt = phase === "receipt-prepared" || phase === "finalized"; + if ( + hasPreparedReceipt !== (receiptSha256 !== null) || + hasPreparedReceipt !== (serializedReceipt !== null) + ) { + fail("phase and receipt fields disagree"); } const authority = normalizePersistedEngineAuthority(record.engineAuthority); if (authority.operation !== "host-local-inference") { fail("engine authority has the wrong operation"); } + if (serializedReceipt !== null && receiptSha256 !== null) { + const receipt = parsePreparedReceipt(serializedReceipt); + if ( + serializeHostLocalInferenceReceipt(receipt) !== serializedReceipt || + receipt.providerId !== providerId || + receipt.service !== service || + receipt.runtime.kind !== "container" || + receipt.runtime.runtimeId !== runtimeId || + receipt.runtime.specSha256 !== record.specSha256 || + receipt.runtime.model?.generation !== transactionId || + JSON.stringify(receipt.engineAuthority) !== JSON.stringify(authority) + ) { + fail("prepared receipt differs from create authority"); + } + const expectedReceiptSha256 = createHash("sha256").update(serializedReceipt).digest("hex"); + if (receiptSha256 !== expectedReceiptSha256) { + fail("prepared receipt digest changed"); + } + } return Object.freeze({ schemaVersion: HOST_LOCAL_CREATE_JOURNAL_SCHEMA_VERSION, - transactionId: exactText(record.transactionId, SHA256, "transaction identity"), + transactionId, phase, - providerId: exactText(record.providerId, PROVIDER, "provider identity"), - service: exactText(record.service, SERVICE, "service identity"), + providerId, + service, containerName: exactText(record.containerName, NAME, "container name"), runtimeId, createIntentUnixMs, @@ -213,6 +268,8 @@ export function normalizeHostLocalCreateJournalRecord( "API-key directory authority digest", ), engineAuthority: authority, + receiptTargetSha256: exactText(record.receiptTargetSha256, SHA256, "receipt target identity"), + serializedReceipt, receiptSha256, }); } @@ -568,10 +625,30 @@ export function createHostLocalCreateJournalStore( return { ...current, phase: "started" }; }); }, - finalize(transactionId, receiptSha256) { + prepareReceipt(transactionId, serializedReceipt) { return replace(transactionId, (current) => { - if (current.phase !== "started") fail("only a started transaction can finalize"); - return { ...current, phase: "finalized", receiptSha256 }; + if (current.phase !== "started") { + fail("only a started transaction can prepare receipt publication"); + } + const receipt = parsePreparedReceipt(serializedReceipt); + if (serializeHostLocalInferenceReceipt(receipt) !== serializedReceipt) { + fail("prepared receipt bytes are not canonical"); + } + return { + ...current, + phase: "receipt-prepared", + serializedReceipt, + receiptSha256: createHash("sha256").update(serializedReceipt).digest("hex"), + }; + }); + }, + finalize(transactionId) { + return replace(transactionId, (current) => { + if (current.phase === "finalized") return current; + if (current.phase !== "receipt-prepared") { + fail("only a receipt-prepared transaction can finalize"); + } + return { ...current, phase: "finalized" }; }); }, retire(transactionId) { diff --git a/src/lib/onboard/runtime-provider/host-local-inference.ts b/src/lib/onboard/runtime-provider/host-local-inference.ts index b9a492dca2e..2c143114bd3 100644 --- a/src/lib/onboard/runtime-provider/host-local-inference.ts +++ b/src/lib/onboard/runtime-provider/host-local-inference.ts @@ -129,6 +129,19 @@ export interface HostLocalInferenceRouteAuthorityStore { ) => HostLocalInferenceRouteAuthority; } +/** + * Operation-scoped durable writer for one exact receipt target. Implementations + * must atomically retain an absent target or the exact same canonical receipt, + * reject a different current value, and return the committed canonical bytes. + */ +export interface HostLocalInferenceReceiptWriter { + /** Exact create transaction this writer may publish. */ + readonly transactionId: string; + /** Path- and value-free identity of the one external state target. */ + readonly targetSha256: string; + readonly writeExact: (serializedReceipt: string) => string; +} + export interface HostLocalInferenceRuntime { readonly providerId: string; /** Exact opaque endpoint identity shared with the operation-scoped engine. */