diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index c5d00279b51..0f307cfca1a 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -561,6 +561,11 @@ "test": "keeps migrated provider identities and implementations behind the one bundle composition", "category": "compatibility" }, + { + "file": "test/runtime-provider-source-shape.test.ts", + "test": "keeps Docker llama.cpp lifecycle authority dormant (#8395)", + "category": "security" + }, { "file": "test/source-architecture.test.ts", "test": "keeps removed step mutation APIs out of production source (#7703)", diff --git a/src/lib/adapters/container-engine.test.ts b/src/lib/adapters/container-engine.test.ts new file mode 100644 index 00000000000..3bd190ed5d1 --- /dev/null +++ b/src/lib/adapters/container-engine.test.ts @@ -0,0 +1,146 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { createContainerEngineCommand } from "./container-engine"; + +describe("operation-scoped container engine command", () => { + it("binds endpoint arguments without changing host-only commands", () => { + const capture = vi.fn(() => ({ status: 0, stdout: "ok", stderr: "" })); + const engine = createContainerEngineCommand({ + operation: "sandbox-lifecycle", + engineId: "podman", + displayName: "Podman", + authorityId: "test:podman-socket", + executable: "podman", + endpointArgs: ["--url", "unix:///runtime/podman.sock"], + capture, + }); + + expect(engine.capture(["container", "inspect", "abc"], 1234)).toEqual({ + status: 0, + stdout: "ok", + stderr: "", + }); + engine.captureHost(["unshare", "cat", "/proc/self/uid_map"], 2345); + + expect(capture.mock.calls).toEqual([ + ["podman", ["--url", "unix:///runtime/podman.sock", "container", "inspect", "abc"], 1234], + ["podman", ["unshare", "cat", "/proc/self/uid_map"], 2345], + ]); + expect(Object.isFrozen(engine)).toBe(true); + }); + + it("keeps separately scoped engines isolated", () => { + const doctorCapture = vi.fn(() => ({ status: 0, stdout: "doctor", stderr: "" })); + const lifecycleCapture = vi.fn(() => ({ status: 0, stdout: "lifecycle", stderr: "" })); + const doctor = createContainerEngineCommand({ + operation: "host-doctor", + engineId: "podman", + displayName: "Podman", + authorityId: "test:doctor", + executable: "podman-doctor", + capture: doctorCapture, + }); + const lifecycle = createContainerEngineCommand({ + operation: "sandbox-lifecycle", + engineId: "podman", + displayName: "Podman", + authorityId: "test:lifecycle", + executable: "podman-lifecycle", + capture: lifecycleCapture, + }); + + expect(doctor.capture(["info"]).stdout).toBe("doctor"); + expect(lifecycle.capture(["start", "abc"]).stdout).toBe("lifecycle"); + expect(doctorCapture).toHaveBeenCalledExactlyOnceWith("podman-doctor", ["info"], 15_000); + expect(lifecycleCapture).toHaveBeenCalledExactlyOnceWith( + "podman-lifecycle", + ["start", "abc"], + 15_000, + ); + }); + + it("guards before and after commands while preserving command failures", () => { + const commandFailure = new Error("command failed"); + const guardFailure = new Error("authority changed"); + const guard = vi + .fn() + .mockImplementationOnce(() => {}) + .mockImplementationOnce(() => { + throw guardFailure; + }); + const engine = createContainerEngineCommand({ + operation: "sandbox-lifecycle", + engineId: "podman", + displayName: "Podman", + authorityId: "test:podman-socket", + executable: "podman", + capture: () => { + throw commandFailure; + }, + guard, + }); + + expect(() => engine.capture(["stop", "abc"])).toThrow(commandFailure); + expect(guard).toHaveBeenCalledTimes(2); + }); + + it("rejects endpoint rotation observed after a successful command", () => { + const authorityChanged = new Error("authority changed"); + const guard = vi + .fn() + .mockImplementationOnce(() => undefined) + .mockImplementationOnce(() => { + throw authorityChanged; + }); + const capture = vi.fn(() => ({ status: 0, stdout: "ok", stderr: "" })); + const engine = createContainerEngineCommand({ + operation: "sandbox-lifecycle", + engineId: "podman", + displayName: "Podman", + authorityId: "test:podman-socket", + executable: "podman", + capture, + guard, + }); + + expect(() => engine.capture(["start", "a".repeat(64)])).toThrow(authorityChanged); + expect(capture).toHaveBeenCalledOnce(); + expect(guard).toHaveBeenCalledTimes(2); + }); + + it("rejects invalid identities, timeouts, and command arguments before capture", () => { + const capture = vi.fn(() => ({ status: 0, stdout: "", stderr: "" })); + expect(() => + createContainerEngineCommand({ + operation: "host-doctor", + engineId: "Podman", + displayName: "Podman", + authorityId: "test:podman-socket", + executable: "podman", + }), + ).toThrow("identity is invalid"); + expect(() => + createContainerEngineCommand({ + operation: "host-doctor", + engineId: "podman", + displayName: "Podman", + authorityId: "unsafe/socket", + executable: "podman", + }), + ).toThrow("authority identity is invalid"); + const engine = createContainerEngineCommand({ + operation: "host-doctor", + engineId: "podman", + displayName: "Podman", + authorityId: "test:podman-socket", + executable: "podman", + capture, + }); + expect(() => engine.capture(["info"], 0)).toThrow("positive safe integer"); + expect(() => engine.capture(["bad\0argument"])).toThrow("arguments[0] is invalid"); + expect(capture).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/adapters/container-engine.ts b/src/lib/adapters/container-engine.ts new file mode 100644 index 00000000000..842ab2fa5ce --- /dev/null +++ b/src/lib/adapters/container-engine.ts @@ -0,0 +1,245 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import path from "node:path"; + +export type ContainerEngineOperationScope = + | "host-doctor" + | "host-local-inference" + | "gateway-inspection" + | "managed-bootstrap" + | "sandbox-lifecycle" + | "workload-cleanup"; + +export interface ContainerEngineCommandResult { + readonly status: number; + readonly stdout: string; + readonly stderr: string; + readonly error?: Error; +} + +export type ContainerEngineCommandCapture = ( + executable: string, + args: readonly string[], + timeoutMs: number, +) => ContainerEngineCommandResult; + +/** + * Immutable command boundary injected into one provider operation. The + * executable and endpoint prefix cannot change after construction, and no + * process-global engine selection is consulted by a command. + */ +export interface ContainerEngine { + readonly operation: ContainerEngineOperationScope; + readonly engineId: string; + readonly displayName: string; + /** Opaque identity for the exact endpoint authority bound to this command. */ + readonly authorityId: string; + readonly capture: (args: readonly string[], timeoutMs?: number) => ContainerEngineCommandResult; + readonly captureHost: ( + args: readonly string[], + timeoutMs?: number, + ) => ContainerEngineCommandResult; +} + +export interface ContainerEngineCommandOptions { + readonly operation: ContainerEngineOperationScope; + readonly engineId: string; + readonly displayName: string; + readonly authorityId: string; + readonly executable: string; + readonly endpointArgs?: readonly string[]; + readonly capture?: ContainerEngineCommandCapture; + readonly guard?: () => void; +} + +const DEFAULT_TIMEOUT_MS = 15_000; +const MAX_ARGUMENTS = 512; +const MAX_ARGUMENT_BYTES = 16 * 1024; +const MAX_OUTPUT_BYTES = 1024 * 1024; +const ENGINE_ID_PATTERN = /^[a-z][a-z0-9-]{0,62}$/u; +const AUTHORITY_ID_PATTERN = /^[a-z][a-z0-9-]{0,62}:[A-Za-z0-9._:-]{1,255}$/u; +const EXECUTABLE_NAME_PATTERN = /^[A-Za-z0-9._-]+$/u; +const CONTROL_CHARACTERS = /[\u0000-\u001f\u007f-\u009f]/u; +const COMMAND_ENV_NAMES = new Set([ + "HOME", + "USER", + "LOGNAME", + "SHELL", + "PATH", + "TERM", + "HOSTNAME", + "LANG", + "TMPDIR", + "TMP", + "TEMP", + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "http_proxy", + "https_proxy", + "no_proxy", + "SSL_CERT_FILE", + "SSL_CERT_DIR", + "GIT_SSL_CAINFO", + "GIT_SSL_CAPATH", + "CURL_CA_BUNDLE", +]); +const COMMAND_ENV_PREFIXES = ["LC_", "XDG_"] as const; + +function containerEngineCommandEnvironment(): NodeJS.ProcessEnv { + return Object.fromEntries( + Object.entries(process.env).filter( + ([name, value]) => + value !== undefined && + (COMMAND_ENV_NAMES.has(name) || + COMMAND_ENV_PREFIXES.some((prefix) => name.startsWith(prefix))), + ), + ); +} + +function boundedText(value: string, label: string, allowPath = false): string { + const normalized = value.trim(); + if ( + normalized === "" || + Buffer.byteLength(normalized, "utf8") > MAX_ARGUMENT_BYTES || + CONTROL_CHARACTERS.test(normalized) || + (!allowPath && /[\\/]/u.test(normalized)) + ) { + throw new Error(`${label} is invalid.`); + } + return normalized; +} + +function normalizedArguments(args: readonly string[], label: string): readonly string[] { + if (!Array.isArray(args) || args.length > MAX_ARGUMENTS) { + throw new Error(`${label} has too many arguments.`); + } + return Object.freeze( + args.map((value, index) => { + if ( + typeof value !== "string" || + Buffer.byteLength(value, "utf8") > MAX_ARGUMENT_BYTES || + value.includes("\0") + ) { + throw new Error(`${label}[${String(index)}] is invalid.`); + } + return value; + }), + ); +} + +function positiveTimeout(value: number): number { + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error("Container engine command timeout must be a positive safe integer."); + } + return value; +} + +function normalizedExecutable(value: string): string { + const executable = boundedText(value, "Container engine executable", true); + if (!path.isAbsolute(executable) && !EXECUTABLE_NAME_PATTERN.test(executable)) { + throw new Error("Container engine executable is invalid."); + } + return executable; +} + +function normalizedResult(value: ContainerEngineCommandResult): ContainerEngineCommandResult { + if ( + typeof value !== "object" || + value === null || + !Number.isSafeInteger(value.status) || + value.status < 0 || + typeof value.stdout !== "string" || + typeof value.stderr !== "string" || + (value.error !== undefined && !(value.error instanceof Error)) + ) { + throw new Error("Container engine command returned an invalid result."); + } + return Object.freeze({ + status: value.status, + stdout: value.stdout, + stderr: value.stderr, + ...(value.error ? { error: value.error } : {}), + }); +} + +function defaultCapture( + executable: string, + args: readonly string[], + timeoutMs: number, +): ContainerEngineCommandResult { + const result = spawnSync(executable, [...args], { + cwd: process.cwd(), + env: containerEngineCommandEnvironment(), + encoding: "utf8", + maxBuffer: MAX_OUTPUT_BYTES, + shell: false, + stdio: ["ignore", "pipe", "pipe"], + timeout: timeoutMs, + }); + return { + status: result.status ?? (result.error || result.signal ? 1 : 0), + stdout: String(result.stdout || ""), + stderr: String(result.stderr || ""), + ...(result.error ? { error: result.error } : {}), + }; +} + +function invokeGuarded( + guard: (() => void) | undefined, + capture: () => ContainerEngineCommandResult, +): ContainerEngineCommandResult { + guard?.(); + let result: ContainerEngineCommandResult | undefined; + let failure: unknown; + try { + result = capture(); + } catch (error) { + failure = error; + } + try { + guard?.(); + } catch (error) { + if (failure === undefined) failure = error; + } + if (failure !== undefined) throw failure; + return normalizedResult(result as ContainerEngineCommandResult); +} + +export function createContainerEngineCommand( + options: ContainerEngineCommandOptions, +): ContainerEngine { + if (!ENGINE_ID_PATTERN.test(options.engineId)) { + throw new Error("Container engine identity is invalid."); + } + if (!AUTHORITY_ID_PATTERN.test(options.authorityId)) { + throw new Error("Container engine authority identity is invalid."); + } + const executable = normalizedExecutable(options.executable); + const displayName = boundedText(options.displayName, "Container engine display name"); + const endpointArgs = normalizedArguments( + options.endpointArgs ?? [], + "Container engine endpoint arguments", + ); + const capture = options.capture ?? defaultCapture; + const run = (args: readonly string[], timeoutMs: number, endpoint: boolean) => { + const normalized = normalizedArguments(args, "Container engine command arguments"); + const commandArgs = endpoint ? [...endpointArgs, ...normalized] : [...normalized]; + return invokeGuarded(options.guard, () => + capture(executable, commandArgs, positiveTimeout(timeoutMs)), + ); + }; + + return Object.freeze({ + operation: options.operation, + engineId: options.engineId, + displayName, + authorityId: options.authorityId, + capture: (args: readonly string[], timeoutMs = DEFAULT_TIMEOUT_MS) => + run(args, timeoutMs, true), + captureHost: (args: readonly string[], timeoutMs = DEFAULT_TIMEOUT_MS) => + run(args, timeoutMs, false), + }); +} diff --git a/src/lib/inference/llama-cpp/gguf-cache-plan.ts b/src/lib/inference/llama-cpp/gguf-cache-plan.ts index 0dd33ef1b73..c35a0d9bbee 100644 --- a/src/lib/inference/llama-cpp/gguf-cache-plan.ts +++ b/src/lib/inference/llama-cpp/gguf-cache-plan.ts @@ -71,6 +71,16 @@ function digest(value: unknown): string { return `sha256:${createHash("sha256").update(canonicalJson).digest("hex")}`; } +/** Reject a plan whose recorded digest does not match its canonical declarative payload. */ +export function assertLlamaCppGgufCachePlanDigest(plan: LlamaCppGgufCachePlan): void { + const { planDigest, ...payload } = plan; + if (planDigest !== digest(payload)) { + throw new LlamaCppGgufCachePlanError( + "The llama.cpp GGUF cache plan digest does not match its canonical payload.", + ); + } +} + function assertDeclarativeContract(recipe: LlamaCppServingRecipe): void { const { acquisition, cache, files } = recipe.spec.model; if (recipe.spec.backend !== "install-llama-cpp" || recipe.spec.providerId !== "llama-cpp-local") { diff --git a/src/lib/inference/llama-cpp/host-local-runtime.test.ts b/src/lib/inference/llama-cpp/host-local-runtime.test.ts index 226fc4181be..76a1f1e3a67 100644 --- a/src/lib/inference/llama-cpp/host-local-runtime.test.ts +++ b/src/lib/inference/llama-cpp/host-local-runtime.test.ts @@ -133,7 +133,7 @@ describe("llama.cpp host-local runtime materializer", () => { `type=bind,source=${runtime.apiKeyHostPath},target=/run/secrets/llama-cpp-api-key,readonly`, ]); expect(valuesAfter(argv, "--publish")).toEqual(["127.0.0.1::8081"]); - expect(valuesAfter(argv, "--gpus")).toEqual(["1"]); + expect(valuesAfter(argv, "--gpus")).toEqual(["driver=nvidia,count=1"]); expect(valuesAfter(argv, "--gpu-layers")).toEqual(["all"]); expect(valuesAfter(argv, "--ctx-size")).toEqual([String(input.serve.contextSize)]); expect(valuesAfter(argv, "--batch-size")).toEqual([String(input.serve.batchSize)]); @@ -206,7 +206,7 @@ describe("llama.cpp host-local runtime materializer", () => { "--pids-limit", "256", "--gpus", - "1", + "driver=nvidia,count=1", "--tmpfs", "/tmp:rw,noexec,nosuid,nodev,size=42949672960,uid=1001,gid=1001,mode=1777", "--mount", diff --git a/src/lib/inference/llama-cpp/host-local-runtime.ts b/src/lib/inference/llama-cpp/host-local-runtime.ts index ff1d2014e71..4bb9163b757 100644 --- a/src/lib/inference/llama-cpp/host-local-runtime.ts +++ b/src/lib/inference/llama-cpp/host-local-runtime.ts @@ -171,15 +171,15 @@ function validateContract(contract: LlamaCppHostLocalLaunchContract): void { } } -function validateBindings( +/** Re-prove executor-only filesystem identity before a container-engine mutation. */ +export function assertLlamaCppVerifiedLocalModelArtifact( contract: LlamaCppHostLocalLaunchContract, - bindings: LlamaCppHostLocalRuntimeBindings, + model: VerifiedLocalModelArtifact, ): void { - safeHostPath(bindings.model.hostPath, "llama.cpp model path"); - safeHostPath(bindings.apiKeyHostPath, "llama.cpp API-key path"); + safeHostPath(model.hostPath, "llama.cpp model path"); if ( - bindings.model.digest !== contract.model.file.digest || - bindings.model.sizeBytes !== contract.model.file.sizeBytes + model.digest !== contract.model.file.digest || + model.sizeBytes !== contract.model.file.sizeBytes ) { throw new Error( "llama.cpp verified model artifact does not match the declarative GGUF identity", @@ -188,12 +188,12 @@ function validateBindings( let canonicalModelPath: string; let modelStatus: BigIntStats; try { - canonicalModelPath = realpathSync(bindings.model.hostPath); - modelStatus = lstatSync(bindings.model.hostPath, { bigint: true }); + canonicalModelPath = realpathSync(model.hostPath); + modelStatus = lstatSync(model.hostPath, { bigint: true }); } catch { throw new Error("llama.cpp verified model artifact is unavailable"); } - const identity = bindings.model.filesystemIdentity; + const identity = model.filesystemIdentity; if ( !identity || typeof identity.dev !== "bigint" || @@ -201,19 +201,27 @@ function validateBindings( typeof identity.size !== "bigint" || typeof identity.mtimeNs !== "bigint" || typeof identity.ctimeNs !== "bigint" || - canonicalModelPath !== bindings.model.hostPath || + canonicalModelPath !== model.hostPath || !modelStatus.isFile() || modelStatus.dev !== identity.dev || modelStatus.ino !== identity.ino || modelStatus.size !== identity.size || modelStatus.mtimeNs !== identity.mtimeNs || modelStatus.ctimeNs !== identity.ctimeNs || - modelStatus.size !== BigInt(bindings.model.sizeBytes) + modelStatus.size !== BigInt(model.sizeBytes) ) { throw new Error( "llama.cpp verified model artifact does not match its verified filesystem identity", ); } +} + +function validateBindings( + contract: LlamaCppHostLocalLaunchContract, + bindings: LlamaCppHostLocalRuntimeBindings, +): void { + assertLlamaCppVerifiedLocalModelArtifact(contract, bindings.model); + safeHostPath(bindings.apiKeyHostPath, "llama.cpp API-key path"); if ( !SAFE_NAME.test(bindings.containerName) || bindings.network.isolation !== "docker-internal" || @@ -268,7 +276,7 @@ export function buildLlamaCppHostLocalDockerArgv( "--pids-limit", String(resources.pidsLimit), "--gpus", - "1", + "driver=nvidia,count=1", "--tmpfs", `/tmp:rw,noexec,nosuid,nodev,size=${String(resources.writableStorageBytes)},uid=${String(bindings.runtimeUid)},gid=${String(bindings.runtimeGid)},mode=1777`, "--mount", @@ -276,6 +284,18 @@ export function buildLlamaCppHostLocalDockerArgv( "--mount", `type=bind,source=${bindings.apiKeyHostPath},target=${LLAMA_CPP_HOST_LOCAL_CONTAINER_API_KEY_PATH},readonly`, bindings.imageReference, + ...buildLlamaCppHostLocalServerArgv(contract), + ]; +} + +/** Reconstruct the immutable in-container server command without host filesystem state. */ +export function buildLlamaCppHostLocalServerArgv( + contract: LlamaCppHostLocalLaunchContract, +): readonly string[] { + validateContract(contract); + const { serve } = contract; + const containerModelPath = `/models/${contract.model.file.path}`; + return Object.freeze([ "--model", containerModelPath, "--alias", @@ -311,5 +331,5 @@ export function buildLlamaCppHostLocalDockerArgv( "--no-slots", "--no-mmproj", "--no-agent", - ]; + ]); } diff --git a/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.test.ts b/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.test.ts new file mode 100644 index 00000000000..b3075757d9b --- /dev/null +++ b/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.test.ts @@ -0,0 +1,1009 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import type { ContainerEngine } from "../../adapters/container-engine"; +import type { LlamaCppGgufCachePlan } from "../../inference/llama-cpp/gguf-cache-plan"; +/* Test-only reconstruction of the exact immutable command for recovery fixtures. */ +import { + buildLlamaCppHostLocalServerArgv, + type LlamaCppHostLocalLaunchContract, + type LlamaCppHostLocalRuntimeBindings, +} from "../../inference/llama-cpp/host-local-runtime"; +import { + createDockerLlamaCppManagedLifecycle, + type DockerLlamaCppManagedLifecycleOptions, +} from "./docker-llama-cpp-managed-lifecycle"; +import type { + HostLocalCreateJournalExecutionLease, + HostLocalCreateJournalRecord, + HostLocalCreateJournalStore, +} from "./host-local-create-journal"; +import { + parseHostLocalInferenceReceipt, + serializeHostLocalInferenceReceipt, +} from "./host-local-inference"; +import type { PersistedEngineAuthorityStore } from "./persisted-engine-authority"; + +const MODEL_DIGEST = `sha256:${"a".repeat(64)}`; +const IMAGE = `ghcr.io/nvidia/nemoclaw/llama-cpp-server@sha256:${"c".repeat(64)}`; +const PROBE_IMAGE = `quay.io/curl/curl@sha256:${"d".repeat(64)}`; +const RUNTIME_ID = "e".repeat(64); +const NETWORK_ID = "7".repeat(64); +const TRANSACTION_ID = "9".repeat(64); +const MODEL_CONTENT = Buffer.alloc(64, 0x61); +const MODEL_FILENAME = "Nemotron-3-Nano-30B-A3B-UD-Q4_K_XL.gguf"; +const REVISION = "f".repeat(40); +let temporaryRoot = ""; +let cacheRoot = ""; +let modelPath = ""; +let apiKeyRoot = ""; +let apiKeyPath = ""; + +function canonical(value: unknown): unknown { + return Array.isArray(value) + ? value.map(canonical) + : value !== null && typeof value === "object" + ? Object.fromEntries( + Object.entries(value) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, nested]) => [key, canonical(nested)]), + ) + : value; +} + +function invariant(condition: unknown, message: string): asserts condition { + switch (Boolean(condition)) { + case false: + throw new Error(message); + } +} + +function digest(value: unknown): string { + return `sha256:${createHash("sha256") + .update(JSON.stringify(canonical(value))) + .digest("hex")}`; +} + +function rawDigest(value: unknown): string { + return createHash("sha256") + .update(JSON.stringify(canonical(value))) + .digest("hex"); +} + +beforeEach(() => { + temporaryRoot = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-llama-life-"))); + cacheRoot = path.join(temporaryRoot, "cache"); + modelPath = path.join( + cacheRoot, + "hub", + "models--example--model", + "snapshots", + REVISION, + MODEL_FILENAME, + ); + fs.mkdirSync(path.dirname(modelPath), { recursive: true, mode: 0o700 }); + fs.writeFileSync(modelPath, MODEL_CONTENT, { mode: 0o600 }); + apiKeyRoot = path.join(temporaryRoot, "key-root"); + fs.mkdirSync(apiKeyRoot, { mode: 0o700 }); + apiKeyPath = path.join(apiKeyRoot, "api-key"); + fs.writeFileSync(apiKeyPath, "test-only-secret\n", { mode: 0o600 }); +}); + +afterEach(() => fs.rmSync(temporaryRoot, { force: true, recursive: true })); + +function contract(): LlamaCppHostLocalLaunchContract { + return { + model: { + servedName: "nvidia-nemotron-3-nano-30b-a3b", + file: { + digest: MODEL_DIGEST, + path: MODEL_FILENAME, + sizeBytes: MODEL_CONTENT.length, + }, + }, + policy: { + egress: "disabled", + modelDownloads: "disabled", + modelSource: "verified-local", + }, + runtime: { + gpu: { + count: 1, + cpuFallback: "reject", + offload: "full", + vendor: "nvidia", + }, + resources: { + memoryBytes: 51_539_607_552, + pidsLimit: 256, + writableStorageBytes: 1024, + }, + }, + serve: { + authentication: "bearer", + batchSize: 2048, + contextSize: 262_144, + flashAttention: "enabled", + idleSleepSeconds: -1, + kvCache: { key: "f16", value: "f16" }, + limits: { requestTimeoutSeconds: 900 }, + microBatchSize: 512, + port: 8081, + protocol: "openai-completions", + slots: 1, + speculativeDecoding: "disabled", + }, + surfaces: { + agentMode: "disabled", + mcpProxy: "disabled", + multimodalProjection: "disabled", + router: "disabled", + serverTools: "disabled", + slotInspection: "disabled", + ui: "disabled", + }, + }; +} + +function plan(): LlamaCppGgufCachePlan { + const payload = { + schemaVersion: 1 as const, + recipeId: "llama-cpp.nemotron.spark.v1", + acquisition: { + ref: "hugging-face-exact-file/v1" as const, + url: `https://huggingface.co/example/model/resolve/${REVISION}/${MODEL_FILENAME}`, + authentication: { + mode: "optional" as const, + environment: "HF_TOKEN" as const, + }, + source: { + repository: "example/model", + revision: REVISION, + file: { + path: MODEL_FILENAME, + digest: MODEL_DIGEST, + sizeBytes: MODEL_CONTENT.length, + }, + }, + }, + cache: { + ref: "llama-cpp.gguf-content-addressed/v1" as const, + receiptRef: "llama-cpp.gguf-cache-entry.receipt/v1" as const, + root: "user-cache" as const, + key: "sha256-model", + quotaBytes: 1024, + stagingHeadroomBytes: 128, + staging: "same-filesystem" as const, + publication: "atomic-no-clobber" as const, + reuse: "verified-only-offline" as const, + sharing: "owner-only" as const, + cleanup: "receipt-owner-only" as const, + }, + }; + return { ...payload, planDigest: digest(payload) }; +} + +function identity() { + const status = fs.lstatSync(modelPath, { bigint: true }); + return { + ctimeNs: status.ctimeNs, + dev: status.dev, + ino: status.ino, + mtimeNs: status.mtimeNs, + size: status.size, + }; +} + +function keyRootIdentitySha256(): string { + const status = fs.lstatSync(apiKeyRoot, { bigint: true }); + return rawDigest({ + schemaVersion: 1, + identities: [ + { + dev: status.dev.toString(), + ino: status.ino.toString(), + uid: status.uid.toString(), + gid: status.gid.toString(), + nlink: status.nlink.toString(), + mode: (status.mode & 0o777n).toString(8), + mtimeNs: status.mtimeNs.toString(), + ctimeNs: status.ctimeNs.toString(), + }, + ], + }); +} + +function bindings(): LlamaCppHostLocalRuntimeBindings { + return { + apiKeyHostPath: apiKeyPath, + containerName: "nemoclaw-llama-cpp", + imageReference: IMAGE, + model: { + digest: MODEL_DIGEST, + filesystemIdentity: identity(), + hostPath: modelPath, + sizeBytes: MODEL_CONTENT.length, + }, + network: { + isolation: "docker-internal", + name: "nemoclaw-llama-cpp-internal", + }, + ownerLabel: { + name: "io.nvidia.nemoclaw.llama-cpp-owner", + value: "gateway.primary", + }, + runtimeGid: 1001, + runtimeUid: 1001, + }; +} + +function authorityStore(): PersistedEngineAuthorityStore { + let authority: ReturnType | null = null; + return { + load: () => authority, + record: (next) => (authority = next), + }; +} + +interface TestJournalStore extends HostLocalCreateJournalStore { + readonly abandonExecution: () => void; + readonly hasExecution: () => boolean; +} + +function journalStore(): TestJournalStore { + const records = new Map(); + let activeLease: HostLocalCreateJournalExecutionLease | null = null; + const update = ( + id: string, + mutate: (value: HostLocalCreateJournalRecord) => HostLocalCreateJournalRecord, + ) => { + const current = records.get(id); + invariant(current, "missing journal"); + const next = Object.freeze(mutate(current)); + records.set(id, next); + return next; + }; + return { + load: (id) => records.get(id) ?? null, + list: () => [...records.values()], + create: (record) => { + records.set(record.transactionId, Object.freeze(record)); + return record; + }, + recordCreating: (id, createIntentUnixMs) => + update(id, (record) => ({ ...record, phase: "creating", createIntentUnixMs })), + recordCreated: (id, runtimeId) => + update(id, (record) => ({ ...record, phase: "created", runtimeId })), + recordStarted: (id) => update(id, (record) => ({ ...record, phase: "started" })), + finalize: (id, receiptSha256) => + update(id, (record) => ({ + ...record, + phase: "finalized", + receiptSha256, + })), + retire: (id) => void records.delete(id), + acquireExecution: (transactionId) => { + invariant(activeLease === null, "execution is already owned by a live process"); + activeLease = Object.freeze({ + schemaVersion: 1, + transactionId, + ownerId: "12345678-1234-4123-8123-123456789abc", + ownerPid: process.pid, + }); + return activeLease; + }, + assertExecution: (lease) => { + invariant(activeLease === lease, "execution ownership changed"); + }, + releaseExecution: (lease) => { + invariant(activeLease === lease, "execution ownership changed"); + activeLease = null; + }, + abandonExecution: () => (activeLease = null), + hasExecution: () => activeLease !== null, + }; +} + +interface DockerFixture { + readonly engine: ContainerEngine; + readonly capture: ReturnType; + readonly setNetworkId: (value: string) => void; + readonly removeNetwork: () => void; + readonly setCreateStdout: (value: string) => void; + readonly failCreateUncertain: () => void; + readonly failProbe: () => void; + readonly driftHardening: () => void; + readonly dropTmpfs: () => void; + readonly driftGpuRequest: (driver: string | undefined, count: number) => void; + readonly driftExtraDeviceAuthority: (kind: "cap-add" | "legacy-device") => void; + readonly failInspectWithDaemonError: () => void; + readonly onAbsentInspect: (callback: () => void) => void; + readonly onStart: (callback: () => void) => void; + readonly onCreate: (callback: () => void) => void; + readonly seed: (journal: HostLocalCreateJournalRecord, running: boolean) => void; +} + +function dockerFixture(): DockerFixture { + let networkId = NETWORK_ID; + let networkPresent = true; + let createStdout = `${RUNTIME_ID}\n`; + let createUncertain = false; + let probeFails = false; + let hardeningDrift = false; + let tmpfs: Record | null = { + "/tmp": "rw,noexec,nosuid,nodev,size=1024,uid=1001,gid=1001,mode=1777", + }; + let gpuDriver: string | undefined = "nvidia"; + let gpuCount = 1; + let capAdd: null | string[] = null; + let legacyDevices: null | object[] = null; + let inspectDaemonError = false; + let absentInspectHook: (() => void) | undefined; + let startHook: (() => void) | undefined; + let createHook: (() => void) | undefined; + let startedOnce = false; + let container: + | { + labels: Record; + running: boolean; + status: string; + transactionId: string; + command: string[]; + } + | undefined; + + const inspection = () => [ + { + Id: RUNTIME_ID, + Name: "/nemoclaw-llama-cpp", + Config: { + Image: IMAGE, + User: "1001:1001", + Cmd: container?.command ?? [], + Labels: container?.labels ?? {}, + }, + HostConfig: { + NetworkMode: "nemoclaw-llama-cpp-internal", + PortBindings: { "8081/tcp": [{ HostIp: "127.0.0.1", HostPort: "" }] }, + ReadonlyRootfs: !hardeningDrift, + CapDrop: ["ALL"], + SecurityOpt: ["no-new-privileges:true"], + Memory: 51_539_607_552, + MemorySwap: 51_539_607_552, + PidsLimit: 256, + DeviceRequests: [ + { + ...(gpuDriver === undefined ? {} : { Driver: gpuDriver }), + Count: gpuCount, + DeviceIDs: null, + Capabilities: [["gpu"]], + Options: {}, + }, + ], + CapAdd: capAdd, + Devices: legacyDevices, + Privileged: false, + Tmpfs: tmpfs, + }, + State: { + Running: container?.running ?? false, + Status: container?.status ?? "created", + }, + NetworkSettings: { + Networks: { "nemoclaw-llama-cpp-internal": { NetworkID: networkId } }, + Ports: { + "8081/tcp": startedOnce ? [{ HostIp: "127.0.0.1", HostPort: "49152" }] : null, + }, + }, + Mounts: [ + { + Type: "bind", + Source: modelPath, + Destination: `/models/${MODEL_FILENAME}`, + RW: false, + }, + { + Type: "bind", + Source: apiKeyPath, + Destination: "/run/secrets/llama-cpp-api-key", + RW: false, + }, + ], + }, + ]; + + const capture = vi.fn((args: readonly string[]) => { + const unexpected = `unexpected Docker command: ${args.join(" ")}`; + switch (args[0]) { + case "network": + invariant(args[1] === "inspect", unexpected); + return networkPresent + ? { + status: 0, + stdout: JSON.stringify([{ Id: networkId, Name: args[2], Internal: true }]), + stderr: "", + } + : { + status: 1, + stdout: "", + stderr: `Error response from daemon: No such network: ${String(args[2])}`, + }; + case "container": { + invariant(args[1] === "inspect", unexpected); + switch (inspectDaemonError) { + case true: + return { status: 1, stdout: "", stderr: "daemon unavailable" }; + } + const target = args[2]; + switch (Boolean(container && (target === RUNTIME_ID || target === "nemoclaw-llama-cpp"))) { + case true: + return { status: 0, stdout: JSON.stringify(inspection()), stderr: "" }; + } + absentInspectHook?.(); + return { + status: 1, + stdout: "", + stderr: `Error response from daemon: No such container: ${String(target)}`, + }; + } + case "create": { + switch (createUncertain) { + case true: + return { + status: 1, + stdout: "", + stderr: "", + error: new Error("Docker create capture timed out"), + }; + } + const labels = Object.fromEntries( + args + .flatMap((argument, index) => + argument === "--label" ? [String(args[index + 1]).split("=")] : [], + ) + .filter(([name, value]) => Boolean(name && value)), + ); + container = { + labels, + running: false, + status: "created", + transactionId: labels["io.nvidia.nemoclaw.host-local-inference.transaction-sha256"] ?? "", + command: args.slice(args.indexOf(IMAGE) + 1), + }; + createHook?.(); + return { status: 0, stdout: createStdout, stderr: "" }; + } + case "start": + startHook?.(); + switch (container) { + case undefined: + break; + default: + startedOnce = true; + container.running = true; + container.status = "running"; + } + return { status: 0, stdout: `${RUNTIME_ID}\n`, stderr: "" }; + case "stop": + switch (container) { + case undefined: + break; + default: + container.running = false; + container.status = "exited"; + } + return { status: 0, stdout: RUNTIME_ID, stderr: "" }; + case "rm": + invariant(args[1] === "--force", unexpected); + container = undefined; + return { status: 0, stdout: RUNTIME_ID, stderr: "" }; + case "run": + invariant(args[1] === "--rm", unexpected); + return probeFails + ? { status: 1, stdout: "", stderr: "not ready" } + : { status: 0, stdout: "ok", stderr: "" }; + default: + throw new Error(unexpected); + } + }); + return { + engine: { + operation: "host-local-inference", + engineId: "docker", + displayName: "Docker", + authorityId: "docker:local", + capture, + captureHost: capture, + }, + capture, + setNetworkId: (value) => (networkId = value), + removeNetwork: () => (networkPresent = false), + setCreateStdout: (value) => (createStdout = value), + failCreateUncertain: () => (createUncertain = true), + failProbe: () => (probeFails = true), + driftHardening: () => (hardeningDrift = true), + dropTmpfs: () => (tmpfs = null), + driftGpuRequest: (driver, count) => { + gpuDriver = driver; + gpuCount = count; + }, + driftExtraDeviceAuthority: (kind) => { + kind === "cap-add" + ? (capAdd = ["SYS_ADMIN"]) + : (legacyDevices = [{ PathOnHost: "/dev/nvidia0" }]); + }, + failInspectWithDaemonError: () => (inspectDaemonError = true), + onAbsentInspect: (callback) => (absentInspectHook = callback), + onStart: (callback) => (startHook = callback), + onCreate: (callback) => (createHook = callback), + seed: (journal, running) => { + container = { + labels: { + "io.nvidia.nemoclaw.host-local-inference.managed": "true", + "io.nvidia.nemoclaw.host-local-inference.provider": "docker", + "io.nvidia.nemoclaw.host-local-inference.service": "llama-cpp", + "io.nvidia.nemoclaw.host-local-inference.spec-sha256": journal.specSha256, + "io.nvidia.nemoclaw.host-local-inference.transaction-sha256": journal.transactionId, + "io.nvidia.nemoclaw.llama-cpp-owner": "gateway.primary", + }, + running, + status: running ? "running" : "created", + transactionId: journal.transactionId, + command: [...buildLlamaCppHostLocalServerArgv(contract())], + }; + }, + }; +} + +function options( + fixture: DockerFixture, + store = journalStore(), + runtimeBindings = bindings(), + persistedAuthorityStore = authorityStore(), +): DockerLlamaCppManagedLifecycleOptions { + return { + authorityStore: persistedAuthorityStore, + apiKeyRootHostPath: apiKeyRoot, + bindingSha256: "1".repeat(64), + bindings: runtimeBindings, + cacheRootHostPath: cacheRoot, + contract: contract(), + engine: fixture.engine, + journalStore: store, + plan: plan(), + probeImageReference: PROBE_IMAGE, + }; +} + +function controller(fixture: DockerFixture, store = journalStore(), now: () => number = Date.now) { + return createDockerLlamaCppManagedLifecycle(options(fixture, store), { + createTransactionId: () => TRANSACTION_ID, + now, + }); +} + +function preparedJournal(): HostLocalCreateJournalRecord { + return { + schemaVersion: 1, + transactionId: TRANSACTION_ID, + phase: "prepared", + providerId: "docker", + service: "llama-cpp", + containerName: "nemoclaw-llama-cpp", + runtimeId: null, + createIntentUnixMs: null, + specSha256: rawDigest({ + contract: contract(), + apiKeyRootIdentitySha256: keyRootIdentitySha256(), + containerName: "nemoclaw-llama-cpp", + imageReference: IMAGE, + model: { + planDigest: plan().planDigest, + recipeId: plan().recipeId, + digest: MODEL_DIGEST, + sizeBytes: MODEL_CONTENT.length, + }, + network: { id: NETWORK_ID, name: "nemoclaw-llama-cpp-internal" }, + ownerLabel: { + name: "io.nvidia.nemoclaw.llama-cpp-owner", + value: "gateway.primary", + }, + probeImageReference: PROBE_IMAGE, + runtimeGid: 1001, + runtimeUid: 1001, + }), + networkId: NETWORK_ID, + engineAuthority: { + schemaVersion: 1, + providerId: "docker", + operation: "host-local-inference", + engineId: "docker", + authorityId: "docker:local", + bindingSha256: "1".repeat(64), + }, + apiKeyIdentitySha256: "3".repeat(64), + apiKeyRootIdentitySha256: keyRootIdentitySha256(), + receiptSha256: null, + }; +} + +describe("dormant Docker llama.cpp managed lifecycle", () => { + it("journals create/start/finalize and serves the provider-neutral lifecycle in a test-only bundle (#8395)", () => { + const fixture = dockerFixture(); + const store = journalStore(); + const lifecycle = controller(fixture, store); + const persist = vi.fn(); + const receipt = lifecycle.start(persist); + const serialized = serializeHostLocalInferenceReceipt(receipt); + + expect(receipt.endpoint.port).toBe(49152); + expect(receipt.runtime).toMatchObject({ + kind: "container", + runtimeId: RUNTIME_ID, + model: { generation: TRANSACTION_ID, planDigest: plan().planDigest }, + }); + expect(store.load(TRANSACTION_ID)).toMatchObject({ + phase: "finalized", + runtimeId: RUNTIME_ID, + networkId: NETWORK_ID, + }); + expect(persist).toHaveBeenCalledExactlyOnceWith(serialized); + expect(serialized).not.toContain(modelPath); + expect(serialized).not.toContain(apiKeyPath); + expect(serialized).not.toContain("filesystemIdentity"); + expect(serialized).not.toContain("test-only-secret"); + + expect(serializeHostLocalInferenceReceipt(parseHostLocalInferenceReceipt(serialized))).toBe( + serialized, + ); + expect(lifecycle.runtime.inspectManaged(receipt).running).toBe(true); + expect(lifecycle.runtime.stopManaged(receipt).running).toBe(false); + expect(lifecycle.runtime.prepareDestroy(receipt)).toEqual(receipt); + expect(lifecycle.runtime.destroy(receipt).status).toBe("removed"); + expect(lifecycle.runtime.destroy(receipt).status).toBe("already-absent"); + }); + + it("keeps already-absent destroy idempotent after its Docker network is removed (#8395)", () => { + const fixture = dockerFixture(); + const lifecycle = controller(fixture); + const receipt = lifecycle.start(vi.fn()); + expect(lifecycle.runtime.destroy(receipt).status).toBe("removed"); + fixture.removeNetwork(); + expect(lifecycle.runtime.destroy(receipt).status).toBe("already-absent"); + }); + + it("rejects canonical plan-digest drift before Docker or journal mutation (#8395)", () => { + const fixture = dockerFixture(); + const store = journalStore(); + const invalid = { + ...options(fixture, store), + plan: { ...plan(), planDigest: `sha256:${"0".repeat(64)}` }, + }; + expect(() => createDockerLlamaCppManagedLifecycle(invalid)).toThrow("canonical payload"); + expect(store.list()).toEqual([]); + expect(fixture.capture).not.toHaveBeenCalled(); + }); + + it("rejects a self-consistent plan for another GGUF before any mutation (#8395)", () => { + const fixture = dockerFixture(); + const store = journalStore(); + const persist = vi.fn(); + const original = plan(); + const changedPayload = { + schemaVersion: original.schemaVersion, + recipeId: original.recipeId, + acquisition: { + ...original.acquisition, + source: { + ...original.acquisition.source, + file: { + path: "Different-Nemotron.gguf", + digest: `sha256:${"6".repeat(64)}`, + sizeBytes: MODEL_CONTENT.length + 1, + }, + }, + }, + cache: original.cache, + }; + const changedPlan: LlamaCppGgufCachePlan = { + ...changedPayload, + planDigest: digest(changedPayload), + }; + const lifecycle = createDockerLlamaCppManagedLifecycle( + { ...options(fixture, store), plan: changedPlan }, + { createTransactionId: () => TRANSACTION_ID }, + ); + + expect(() => lifecycle.start(persist)).toThrow( + "plan, launch contract, and verified artifact disagree", + ); + expect(fixture.capture).not.toHaveBeenCalled(); + expect(store.list()).toEqual([]); + expect(persist).not.toHaveBeenCalled(); + }); + + it("accepts the canonical blob resolved by the plan's exact snapshot entry (#8395)", () => { + const snapshotEntry = modelPath; + const blobPath = path.join(cacheRoot, "hub", "models--example--model", "blobs", "model-blob"); + fs.mkdirSync(path.dirname(blobPath), { recursive: true, mode: 0o700 }); + fs.renameSync(snapshotEntry, blobPath); + fs.symlinkSync(path.relative(path.dirname(snapshotEntry), blobPath), snapshotEntry); + modelPath = fs.realpathSync(snapshotEntry); + + const fixture = dockerFixture(); + const lifecycle = controller(fixture); + const receipt = lifecycle.start(vi.fn()); + expect(receipt.runtime).toMatchObject({ + kind: "container", + runtimeId: RUNTIME_ID, + }); + expect(lifecycle.runtime.destroy(receipt).status).toBe("removed"); + }); + + it("rejects writable cache authority and non-private API-key authority (#8395)", () => { + fs.chmodSync(path.dirname(modelPath), 0o777); + expect(() => controller(dockerFixture()).start(vi.fn())).toThrow("owner-controlled"); + fs.chmodSync(path.dirname(modelPath), 0o700); + fs.chmodSync(apiKeyPath, 0o644); + expect(() => controller(dockerFixture()).start(vi.fn())).toThrow("private-file authority"); + fs.chmodSync(apiKeyPath, 0o600); + fs.chmodSync(apiKeyRoot, 0o777); + const unsafeParentFixture = dockerFixture(); + expect(() => controller(unsafeParentFixture).start(vi.fn())).toThrow("owner-controlled"); + expect(unsafeParentFixture.capture).not.toHaveBeenCalled(); + }); + + it("rolls back exact ownership when the GGUF changes inside Docker start capture (#8395)", () => { + const fixture = dockerFixture(); + const store = journalStore(); + fixture.onStart(() => { + fs.writeFileSync(modelPath, Buffer.alloc(MODEL_CONTENT.length, 0x62)); + const future = new Date(Date.now() + 10_000); + fs.utimesSync(modelPath, future, future); + }); + expect(() => controller(fixture, store).start(vi.fn())).toThrow("filesystem identity"); + expect(store.list()).toEqual([]); + expect(fixture.capture.mock.calls.map((call) => call[0]?.slice(0, 2))).toContainEqual([ + "rm", + "--force", + ]); + }); + + it("rolls back pathname replacement from inside Docker create capture before persistence (#8395)", () => { + const fixture = dockerFixture(); + const store = journalStore(); + const persist = vi.fn(); + fixture.onCreate(() => { + fs.renameSync(modelPath, `${modelPath}.verified`); + fs.writeFileSync(modelPath, MODEL_CONTENT, { mode: 0o600 }); + }); + expect(() => controller(fixture, store).start(persist)).toThrow("filesystem identity"); + expect(persist).not.toHaveBeenCalled(); + expect(store.list()).toEqual([]); + expect(fixture.capture.mock.calls.map((call) => call[0]?.slice(0, 2))).toContainEqual([ + "rm", + "--force", + ]); + }); + + it("rolls back an API-key root swap-and-restore inside Docker create capture (#8395)", () => { + const fixture = dockerFixture(); + const store = journalStore(); + const persist = vi.fn(); + fixture.onCreate(() => { + const retained = `${apiKeyRoot}.retained`; + fs.renameSync(apiKeyRoot, retained); + fs.mkdirSync(apiKeyRoot, { mode: 0o700 }); + fs.writeFileSync(path.join(apiKeyRoot, "api-key"), "attacker-key\n", { + mode: 0o600, + }); + fs.rmSync(apiKeyRoot, { recursive: true }); + fs.renameSync(retained, apiKeyRoot); + const future = new Date(Date.now() + 10_000); + fs.utimesSync(apiKeyRoot, future, future); + }); + expect(() => controller(fixture, store).start(persist)).toThrow("API-key file changed"); + expect(persist).not.toHaveBeenCalled(); + expect(store.list()).toEqual([]); + expect(fixture.capture.mock.calls.map((call) => call[0]?.slice(0, 2))).toContainEqual([ + "rm", + "--force", + ]); + }); + + it("rolls back malformed create output, readiness failure, and receipt persistence failure (#8395)", () => { + const arrangeFailure = { + stdout: (fixture: DockerFixture) => fixture.setCreateStdout("short-id\n"), + probe: (fixture: DockerFixture) => fixture.failProbe(), + persist: (_fixture: DockerFixture) => undefined, + } as const; + for (const failure of ["stdout", "probe", "persist"] as const) { + const fixture = dockerFixture(); + const store = journalStore(); + arrangeFailure[failure](fixture); + const persist = + failure === "persist" + ? () => { + throw new Error("persist failed"); + } + : vi.fn(); + expect(() => controller(fixture, store).start(persist)).toThrow(); + expect(store.list()).toEqual([]); + expect(fixture.capture.mock.calls.map((call) => call[0]?.slice(0, 2))).toContainEqual([ + "rm", + "--force", + ]); + } + }); + + it("holds execution authority after an uncertain create and recovers a late exact container (#8395)", () => { + const fixture = dockerFixture(); + const store = journalStore(); + let now = 1_000; + const lifecycle = controller(fixture, store, () => now); + fixture.failCreateUncertain(); + + expect(() => lifecycle.start(vi.fn())).toThrow("container create failed"); + const creating = store.load(TRANSACTION_ID); + expect(creating).toMatchObject({ + phase: "creating", + runtimeId: null, + createIntentUnixMs: now, + }); + expect(store.hasExecution()).toBe(true); + + const concurrent = lifecycle.recoverUnfinished(); + expect(concurrent.recovered).toEqual([]); + expect(concurrent.failures[0]?.message).toContain("already owned"); + expect(store.load(TRANSACTION_ID)).not.toBeNull(); + + store.abandonExecution(); + const insideGrace = lifecycle.recoverUnfinished(); + expect(insideGrace.recovered).toEqual([]); + expect(insideGrace.failures[0]?.message).toContain("absence grace period"); + expect(store.load(TRANSACTION_ID)).not.toBeNull(); + + invariant(creating, "expected creating journal"); + now += 31 * 60 * 1_000; + let appeared = false; + fixture.onAbsentInspect(() => { + switch (appeared) { + case false: + appeared = true; + fixture.seed(creating, false); + } + }); + expect(lifecycle.recoverUnfinished()).toEqual({ + recovered: [TRANSACTION_ID], + failures: [], + }); + expect(appeared).toBe(true); + expect(store.list()).toEqual([]); + }); + + it("recovers prepared and exact creating/created/started journals without touching finalized ownership (#8395)", () => { + for (const phase of ["prepared", "creating", "created", "started"] as const) { + const fixture = dockerFixture(); + const store = journalStore(); + const base = preparedJournal(); + store.create(base); + const arrangePhase = { + prepared: () => undefined, + creating: () => void store.recordCreating(TRANSACTION_ID, 1_000), + created: () => { + store.recordCreating(TRANSACTION_ID, 1_000); + fixture.seed(store.recordCreated(TRANSACTION_ID, RUNTIME_ID), false); + }, + started: () => { + store.recordCreating(TRANSACTION_ID, 1_000); + store.recordCreated(TRANSACTION_ID, RUNTIME_ID); + fixture.seed(store.recordStarted(TRANSACTION_ID), true); + }, + } as const; + arrangePhase[phase](); + const persistedAuthority = authorityStore(); + persistedAuthority.record(base.engineAuthority); + const recovery = createDockerLlamaCppManagedLifecycle( + options(fixture, store, bindings(), persistedAuthority), + { now: () => 31 * 60 * 1_000 }, + ).recoverUnfinished(); + expect(recovery).toEqual({ recovered: [TRANSACTION_ID], failures: [] }); + expect(store.list()).toEqual([]); + } + }); + + it("refuses unfinished recovery when protected engine authority is missing or drifted (#8395)", () => { + for (const state of ["missing", "drifted"] as const) { + const fixture = dockerFixture(); + const store = journalStore(); + const base = preparedJournal(); + store.create(base); + store.recordCreating(base.transactionId, 1_000); + const created = store.recordCreated(base.transactionId, RUNTIME_ID); + fixture.seed(created, false); + const persistedAuthority = authorityStore(); + const arrangeAuthority = { + missing: () => undefined, + drifted: () => + persistedAuthority.record({ + ...base.engineAuthority, + bindingSha256: "2".repeat(64), + }), + } as const; + arrangeAuthority[state](); + const recovery = createDockerLlamaCppManagedLifecycle( + options(fixture, store, bindings(), persistedAuthority), + ).recoverUnfinished(); + expect(recovery.recovered).toEqual([]); + expect(recovery.failures).toHaveLength(1); + expect(store.load(TRANSACTION_ID)).not.toBeNull(); + expect(fixture.capture.mock.calls.map((call) => call[0]?.slice(0, 2))).not.toContainEqual([ + "rm", + "--force", + ]); + } + }); + + it("fails re-prove on Docker network identity drift (#8395)", () => { + const fixture = dockerFixture(); + const lifecycle = controller(fixture); + const receipt = lifecycle.start(vi.fn()); + fixture.setNetworkId("8".repeat(64)); + expect(() => lifecycle.runtime.preserveForRebuild(receipt)).toThrow( + "internal network identity changed", + ); + }); + + it("rejects effective hardening drift after creation (#8395)", () => { + const fixture = dockerFixture(); + const lifecycle = controller(fixture); + const receipt = lifecycle.start(vi.fn()); + fixture.driftHardening(); + expect(() => lifecycle.runtime.inspectManaged(receipt)).toThrow("exact journal authority"); + + for (const mutate of [ + (candidate: DockerFixture) => candidate.driftGpuRequest(undefined, 1), + (candidate: DockerFixture) => candidate.driftGpuRequest("nvidia", 2), + (candidate: DockerFixture) => candidate.driftExtraDeviceAuthority("cap-add"), + (candidate: DockerFixture) => candidate.driftExtraDeviceAuthority("legacy-device"), + (candidate: DockerFixture) => candidate.dropTmpfs(), + ]) { + const candidate = dockerFixture(); + const candidateLifecycle = controller(candidate); + const candidateReceipt = candidateLifecycle.start(vi.fn()); + mutate(candidate); + expect(() => candidateLifecycle.runtime.inspectManaged(candidateReceipt)).toThrow( + "exact journal authority", + ); + } + }); + + it("fails closed on crafted absent destroy authority and status-one daemon errors (#8395)", () => { + const fixture = dockerFixture(); + const store = journalStore(); + const lifecycle = controller(fixture, store); + const receipt = lifecycle.start(vi.fn()); + invariant(receipt.runtime.kind === "container", "expected container receipt"); + const crafted = { + ...receipt, + runtime: { ...receipt.runtime, runtimeId: "a".repeat(64) }, + }; + expect(() => lifecycle.runtime.destroy(crafted)).toThrow("durable create journal"); + expect(store.load(TRANSACTION_ID)).not.toBeNull(); + + const unavailable = dockerFixture(); + const unavailableStore = journalStore(); + unavailable.failInspectWithDaemonError(); + expect(() => controller(unavailable, unavailableStore).start(vi.fn())).toThrow( + "container inspection failed", + ); + expect(unavailableStore.list()).toEqual([]); + }); +}); diff --git a/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.ts b/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.ts new file mode 100644 index 00000000000..82ff80d49e4 --- /dev/null +++ b/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.ts @@ -0,0 +1,1383 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash, randomUUID } from "node:crypto"; +import fs, { type BigIntStats } from "node:fs"; +import path from "node:path"; + +import type { + ContainerEngine, + ContainerEngineCommandResult, +} from "../../adapters/container-engine"; +import { + assertLlamaCppGgufCachePlanDigest, + type LlamaCppGgufCachePlan, +} from "../../inference/llama-cpp/gguf-cache-plan"; +import { + assertLlamaCppVerifiedLocalModelArtifact, + buildLlamaCppHostLocalDockerArgv, + buildLlamaCppHostLocalServerArgv, + LLAMA_CPP_HOST_LOCAL_CONTAINER_API_KEY_PATH, + type LlamaCppHostLocalLaunchContract, + type LlamaCppHostLocalRuntimeBindings, +} from "../../inference/llama-cpp/host-local-runtime"; +import { + type HostLocalCreateJournalExecutionLease, + type HostLocalCreateJournalRecord, + type HostLocalCreateJournalStore, + normalizeHostLocalCreateJournalRecord, +} from "./host-local-create-journal"; +import type { HostLocalInferenceReceipt, HostLocalInferenceRuntime } from "./host-local-inference"; +import { + normalizeHostLocalInferenceImageRef, + normalizeHostLocalInferenceReceipt, + serializeHostLocalInferenceReceipt, +} from "./host-local-inference"; +import { + createPersistedEngineAuthority, + type PersistedEngineAuthority, + type PersistedEngineAuthorityStore, + requirePersistedEngineAuthority, +} from "./persisted-engine-authority"; + +const PROVIDER_ID = "docker"; +const SERVICE = "llama-cpp"; +const ENDPOINT_HOST = "host.openshell.internal"; +const FULL_ID = /^[a-f0-9]{64}$/u; +const SHA256 = /^[a-f0-9]{64}$/u; +const SAFE_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u; +const MANAGED_LABEL = "io.nvidia.nemoclaw.host-local-inference.managed"; +const PROVIDER_LABEL = "io.nvidia.nemoclaw.host-local-inference.provider"; +const SERVICE_LABEL = "io.nvidia.nemoclaw.host-local-inference.service"; +const SPEC_LABEL = "io.nvidia.nemoclaw.host-local-inference.spec-sha256"; +const TRANSACTION_LABEL = "io.nvidia.nemoclaw.host-local-inference.transaction-sha256"; +const INSPECT_TIMEOUT_MS = 15_000; +const MUTATION_TIMEOUT_MS = 30 * 60 * 1000; +const UNCERTAIN_CREATE_ABSENCE_GRACE_MS = MUTATION_TIMEOUT_MS + INSPECT_TIMEOUT_MS; +const STOP_GRACE_SECONDS = 30; +const AT_REST = new Set(["created", "dead", "exited"]); + +export interface DockerLlamaCppManagedLifecycleOptions { + readonly authorityStore: PersistedEngineAuthorityStore; + readonly apiKeyRootHostPath: string; + readonly bindingSha256: string; + readonly bindings: LlamaCppHostLocalRuntimeBindings; + readonly cacheRootHostPath: string; + readonly contract: LlamaCppHostLocalLaunchContract; + readonly engine: ContainerEngine; + readonly journalStore: HostLocalCreateJournalStore; + readonly plan: LlamaCppGgufCachePlan; + readonly probeImageReference: string; +} + +export interface DockerLlamaCppManagedLifecycleDependencies { + readonly createTransactionId?: () => string; + readonly now?: () => number; +} + +export interface DockerLlamaCppRecoveryResult { + readonly recovered: readonly string[]; + readonly failures: readonly { + readonly transactionId: string; + readonly message: string; + }[]; +} + +export interface DockerLlamaCppManagedLifecycle { + readonly runtime: HostLocalInferenceRuntime; + start(persistReceipt: (serializedReceipt: string) => void): HostLocalInferenceReceipt; + recoverUnfinished(): DockerLlamaCppRecoveryResult; +} + +interface DockerNetworkAuthority { + readonly id: string; + readonly name: string; +} + +interface DockerContainerInspection { + readonly id: string; + readonly name: string; + readonly imageRef: string; + readonly labels: Readonly>; + readonly running: boolean; + readonly status: string; + readonly networkId: string; + readonly networkName: string; + readonly hostPort: number | null; + readonly mounts: readonly { + readonly type: string; + readonly source: string; + readonly destination: string; + readonly readOnly: boolean; + }[]; + readonly hardening: { + readonly user: string; + readonly networkMode: string; + readonly readOnlyRootfs: boolean; + readonly capDrop: readonly string[]; + readonly securityOpt: readonly string[]; + readonly memory: number; + readonly memorySwap: number; + readonly pidsLimit: number; + readonly gpuCount: number; + readonly deviceAuthorityExact: boolean; + readonly capAddEmpty: boolean; + readonly legacyDevicesEmpty: boolean; + readonly privileged: boolean; + readonly command: readonly string[]; + readonly tmpfs: Readonly>; + }; +} + +interface StableFileIdentity { + readonly dev: bigint; + readonly ino: bigint; + readonly size: bigint; + readonly mtimeNs: bigint; + readonly ctimeNs: bigint; +} + +interface MutationExecutionState { + unknown: boolean; +} + +function compareText(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + +function normalizeForCanonicalJson(value: unknown): unknown { + if (Array.isArray(value)) return value.map(normalizeForCanonicalJson); + if (value !== null && typeof value === "object") { + return Object.fromEntries( + Object.entries(value) + .sort(([left], [right]) => compareText(left, right)) + .map(([key, nested]) => [key, normalizeForCanonicalJson(nested)]), + ); + } + return value; +} + +function sha256(value: unknown): string { + // This hashes non-secret canonical lifecycle identity metadata for drift detection, + // not passwords, credentials, or secret material. + return createHash("sha256") + .update(JSON.stringify(normalizeForCanonicalJson(value))) // codeql[js/insufficient-password-hash] + .digest("hex"); +} + +function record(value: unknown, label: string): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error(`${label} must be an object.`); + } + return value as Record; +} + +function requireSuccess(operation: string, result: ContainerEngineCommandResult): string { + if (result.error || result.status !== 0) { + throw new Error(`Docker llama.cpp ${operation} failed (exit ${String(result.status)}).`); + } + return result.stdout; +} + +function exactId(value: unknown, label: string): string { + if (typeof value !== "string" || !FULL_ID.test(value)) { + throw new Error(`${label} must be one full immutable Docker ID.`); + } + return value; +} + +function exactPort(value: unknown): number { + const port = typeof value === "string" && /^[0-9]{1,5}$/u.test(value) ? Number(value) : -1; + if (!Number.isSafeInteger(port) || port < 1 || port > 65_535) { + throw new Error("Docker llama.cpp inspection returned an invalid host port."); + } + return port; +} + +function inspectNetwork(engine: ContainerEngine, name: string): DockerNetworkAuthority { + const output = requireSuccess( + "network inspection", + engine.capture(["network", "inspect", name], INSPECT_TIMEOUT_MS), + ); + let parsed: unknown; + try { + parsed = JSON.parse(output); + } catch { + throw new Error("Docker llama.cpp network inspection returned unreadable JSON."); + } + if (!Array.isArray(parsed) || parsed.length !== 1) { + throw new Error("Docker llama.cpp network inspection must identify exactly one network."); + } + const source = record(parsed[0], "Docker llama.cpp network inspection"); + if (source.Name !== name || source.Internal !== true) { + throw new Error("Docker llama.cpp requires the exact internal Docker network."); + } + return Object.freeze({ + id: exactId(source.Id, "Docker network identity"), + name, + }); +} + +function parseLabels(value: unknown): Readonly> { + const source = record(value, "Docker llama.cpp labels"); + const labels: Record = Object.create(null); + for (const [key, candidate] of Object.entries(source)) { + if (typeof candidate !== "string" || candidate.includes("\0")) { + throw new Error("Docker llama.cpp inspection returned malformed labels."); + } + labels[key] = candidate; + } + return Object.freeze(labels); +} + +function parseInspection( + output: string, + contract: LlamaCppHostLocalLaunchContract, + networkName: string, +): DockerContainerInspection { + let parsed: unknown; + try { + parsed = JSON.parse(output); + } catch { + throw new Error("Docker llama.cpp container inspection returned unreadable JSON."); + } + if (!Array.isArray(parsed) || parsed.length !== 1) { + throw new Error("Docker llama.cpp inspection must identify exactly one container."); + } + const source = record(parsed[0], "Docker llama.cpp inspection"); + const config = record(source.Config, "Docker llama.cpp container configuration"); + const hostConfig = record(source.HostConfig, "Docker llama.cpp host configuration"); + const state = record(source.State, "Docker llama.cpp container state"); + const networkSettings = record(source.NetworkSettings, "Docker llama.cpp network settings"); + const networks = record(networkSettings.Networks, "Docker llama.cpp attached networks"); + const networkNames = Object.keys(networks); + if (networkNames.length !== 1 || networkNames[0] !== networkName) { + throw new Error("Docker llama.cpp container has unexpected network attachments."); + } + const attached = record(networks[networkName], "Docker llama.cpp network attachment"); + const ports = record(networkSettings.Ports, "Docker llama.cpp published ports"); + const portKey = `${String(contract.serve.port)}/tcp`; + const configuredPorts = record(hostConfig.PortBindings, "Docker llama.cpp configured ports"); + if (Object.keys(configuredPorts).length !== 1) { + throw new Error("Docker llama.cpp container has extra configured ports."); + } + const configuredBindings = configuredPorts[portKey]; + if (!Array.isArray(configuredBindings) || configuredBindings.length !== 1) { + throw new Error("Docker llama.cpp container has unexpected configured ports."); + } + const configuredPort = record(configuredBindings[0], "Docker llama.cpp configured port"); + if (configuredPort.HostIp !== "127.0.0.1" || configuredPort.HostPort !== "") { + throw new Error("Docker llama.cpp configured host port is not loopback-only."); + } + const bindings = ports[portKey]; + const published = + Array.isArray(bindings) && bindings.length === 1 + ? record(bindings[0], "Docker llama.cpp published port") + : null; + if (published !== null && published.HostIp !== "127.0.0.1") { + throw new Error("Docker llama.cpp host port is not loopback-only."); + } + if (!Array.isArray(source.Mounts)) { + throw new Error("Docker llama.cpp inspection returned malformed mounts."); + } + const mounts = source.Mounts.map((candidate) => { + const mount = record(candidate, "Docker llama.cpp mount"); + if (typeof mount.Source !== "string" || typeof mount.Destination !== "string") { + throw new Error("Docker llama.cpp inspection returned malformed mount paths."); + } + return Object.freeze({ + type: String(mount.Type ?? ""), + source: mount.Source, + destination: mount.Destination, + readOnly: mount.RW === false, + }); + }); + const rawName = typeof source.Name === "string" ? source.Name.replace(/^\//u, "") : ""; + if (!SAFE_NAME.test(rawName) || typeof state.Running !== "boolean") { + throw new Error("Docker llama.cpp inspection returned malformed runtime state."); + } + const stateStatus = String(state.Status ?? "").toLowerCase(); + if ( + (state.Running && stateStatus !== "running") || + (!state.Running && !AT_REST.has(stateStatus)) || + typeof hostConfig.Privileged !== "boolean" + ) { + throw new Error("Docker llama.cpp inspection returned inconsistent runtime state."); + } + const deviceRequests = Array.isArray(hostConfig.DeviceRequests) ? hostConfig.DeviceRequests : []; + const gpuRequest = + deviceRequests.length === 1 ? record(deviceRequests[0], "Docker GPU request") : {}; + const gpuCapabilities = Array.isArray(gpuRequest.Capabilities) ? gpuRequest.Capabilities : []; + const gpuDeviceIds = gpuRequest.DeviceIDs; + const gpuOptions = gpuRequest.Options; + const gpuCount = + gpuRequest.Driver === "nvidia" && + gpuRequest.Count === 1 && + gpuCapabilities.some((group) => Array.isArray(group) && group.includes("gpu")) + ? 1 + : 0; + const deviceAuthorityExact = + deviceRequests.length === 1 && + gpuRequest.Driver === "nvidia" && + gpuRequest.Count === 1 && + (gpuDeviceIds === null || (Array.isArray(gpuDeviceIds) && gpuDeviceIds.length === 0)) && + typeof gpuOptions === "object" && + gpuOptions !== null && + !Array.isArray(gpuOptions) && + Object.keys(gpuOptions).length === 0 && + gpuCapabilities.length === 1 && + Array.isArray(gpuCapabilities[0]) && + gpuCapabilities[0].length === 1 && + gpuCapabilities[0][0] === "gpu"; + const capAddEmpty = + hostConfig.CapAdd === null || + (Array.isArray(hostConfig.CapAdd) && hostConfig.CapAdd.length === 0); + const legacyDevicesEmpty = + hostConfig.Devices === null || + (Array.isArray(hostConfig.Devices) && hostConfig.Devices.length === 0); + return Object.freeze({ + id: exactId(source.Id, "Docker container identity"), + name: rawName, + imageRef: String(config.Image ?? ""), + labels: parseLabels(config.Labels), + running: state.Running, + status: stateStatus, + networkId: exactId(attached.NetworkID, "Docker attached network identity"), + networkName, + hostPort: published === null ? null : exactPort(published.HostPort), + mounts: Object.freeze(mounts), + hardening: Object.freeze({ + user: String(config.User ?? ""), + networkMode: String(hostConfig.NetworkMode ?? ""), + readOnlyRootfs: hostConfig.ReadonlyRootfs === true, + capDrop: Object.freeze( + Array.isArray(hostConfig.CapDrop) ? hostConfig.CapDrop.map(String) : [], + ), + securityOpt: Object.freeze( + Array.isArray(hostConfig.SecurityOpt) ? hostConfig.SecurityOpt.map(String) : [], + ), + memory: Number(hostConfig.Memory), + memorySwap: Number(hostConfig.MemorySwap), + pidsLimit: Number(hostConfig.PidsLimit), + gpuCount, + deviceAuthorityExact, + capAddEmpty, + legacyDevicesEmpty, + privileged: hostConfig.Privileged, + command: Object.freeze(Array.isArray(config.Cmd) ? config.Cmd.map(String) : []), + tmpfs: Object.freeze( + Object.fromEntries( + Object.entries( + hostConfig.Tmpfs === null ? {} : record(hostConfig.Tmpfs, "Docker llama.cpp tmpfs"), + ), + ), + ) as Readonly>, + }), + }); +} + +function inspectContainer( + engine: ContainerEngine, + target: string, + contract: LlamaCppHostLocalLaunchContract, + networkName: string, +): DockerContainerInspection | null { + const result = engine.capture(["container", "inspect", target], INSPECT_TIMEOUT_MS); + const escapedTarget = target.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); + const exactAbsent = new RegExp( + `^(?:Error response from daemon:\\s*)?(?:No such container|No such object): ${escapedTarget}$`, + "iu", + ); + if (!result.error && result.status === 1 && exactAbsent.test(result.stderr.trim())) { + return null; + } + return parseInspection(requireSuccess("container inspection", result), contract, networkName); +} + +function currentUid(): bigint { + if (typeof process.getuid !== "function") { + throw new Error("Docker llama.cpp requires current-user filesystem identity."); + } + return BigInt(process.getuid()); +} + +function secureOwner(status: BigIntStats): boolean { + return status.uid === 0n || status.uid === currentUid(); +} + +function requireSecureNode(target: string, kind: "directory" | "file"): BigIntStats { + const status = fs.lstatSync(target, { bigint: true }); + if ( + (kind === "directory" ? !status.isDirectory() : !status.isFile()) || + status.isSymbolicLink() || + (kind === "directory" ? !secureOwner(status) : status.uid !== currentUid()) || + (status.mode & 0o022n) !== 0n + ) { + throw new Error(`Docker llama.cpp ${kind} authority is not owner-controlled.`); + } + return status; +} + +function isWithin(root: string, target: string): boolean { + const relative = path.relative(root, target); + return ( + relative === "" || + (!path.isAbsolute(relative) && relative !== ".." && !relative.startsWith(`..${path.sep}`)) + ); +} + +function assertModelFilesystemAuthority(options: DockerLlamaCppManagedLifecycleOptions): void { + const plannedFile = options.plan.acquisition.source.file; + if ( + plannedFile.path !== options.contract.model.file.path || + plannedFile.digest !== options.contract.model.file.digest || + plannedFile.sizeBytes !== options.contract.model.file.sizeBytes || + options.bindings.model.digest !== plannedFile.digest || + options.bindings.model.sizeBytes !== plannedFile.sizeBytes + ) { + throw new Error("Docker llama.cpp GGUF plan, launch contract, and verified artifact disagree."); + } + assertLlamaCppVerifiedLocalModelArtifact(options.contract, options.bindings.model); + const cacheRoot = fs.realpathSync(options.cacheRootHostPath); + const modelRoot = fs.realpathSync( + path.join( + cacheRoot, + "hub", + `models--${options.plan.acquisition.source.repository.replaceAll("/", "--")}`, + ), + ); + const modelPath = fs.realpathSync(options.bindings.model.hostPath); + const snapshotEntry = path.join( + cacheRoot, + "hub", + `models--${options.plan.acquisition.source.repository.replaceAll("/", "--")}`, + "snapshots", + options.plan.acquisition.source.revision, + plannedFile.path, + ); + let snapshotTarget: string; + try { + snapshotTarget = fs.realpathSync(snapshotEntry); + } catch { + throw new Error("Docker llama.cpp exact GGUF snapshot entry is unavailable."); + } + if ( + cacheRoot !== options.cacheRootHostPath || + modelPath !== options.bindings.model.hostPath || + !isWithin(cacheRoot, modelRoot) || + !isWithin(modelRoot, modelPath) || + snapshotTarget !== modelPath + ) { + throw new Error("Docker llama.cpp GGUF resolves outside its canonical model cache."); + } + requireSecureNode(cacheRoot, "directory"); + requireSecureNode(modelRoot, "directory"); + const relative = path.relative(cacheRoot, path.dirname(modelPath)); + let current = cacheRoot; + for (const component of relative.split(path.sep).filter(Boolean)) { + current = path.join(current, component); + requireSecureNode(current, "directory"); + } + requireSecureNode(modelPath, "file"); +} + +function sameFileIdentity(left: StableFileIdentity, right: StableFileIdentity): boolean { + return ( + left.dev === right.dev && + left.ino === right.ino && + left.size === right.size && + left.mtimeNs === right.mtimeNs && + left.ctimeNs === right.ctimeNs + ); +} + +function apiKeyIdentitySha256(identity: StableFileIdentity): string { + return sha256({ + dev: identity.dev.toString(), + ino: identity.ino.toString(), + size: identity.size.toString(), + mtimeNs: identity.mtimeNs.toString(), + ctimeNs: identity.ctimeNs.toString(), + }); +} + +function apiKeyRootIdentitySha256(options: DockerLlamaCppManagedLifecycleOptions): string { + const root = fs.realpathSync(options.apiKeyRootHostPath); + const declaredParent = path.dirname(options.bindings.apiKeyHostPath); + const parent = fs.realpathSync(declaredParent); + if (root !== options.apiKeyRootHostPath || parent !== declaredParent || !isWithin(root, parent)) { + throw new Error("Docker llama.cpp API-key path resolves outside its canonical private root."); + } + const identities: { + dev: string; + ino: string; + uid: string; + gid: string; + nlink: string; + mode: string; + mtimeNs: string; + ctimeNs: string; + }[] = []; + const relative = path.relative(root, parent); + let current = root; + for (const component of ["", ...relative.split(path.sep).filter(Boolean)]) { + if (component !== "") current = path.join(current, component); + const status = requireSecureNode(current, "directory"); + identities.push({ + dev: status.dev.toString(), + ino: status.ino.toString(), + uid: status.uid.toString(), + gid: status.gid.toString(), + nlink: status.nlink.toString(), + mode: (status.mode & 0o777n).toString(8), + mtimeNs: status.mtimeNs.toString(), + ctimeNs: status.ctimeNs.toString(), + }); + } + return sha256({ schemaVersion: 1, identities }); +} + +function apiKeyIdentity(options: DockerLlamaCppManagedLifecycleOptions): StableFileIdentity { + apiKeyRootIdentitySha256(options); + const apiKeyPath = options.bindings.apiKeyHostPath; + if (typeof fs.constants.O_NOFOLLOW !== "number" || typeof fs.constants.O_NONBLOCK !== "number") { + throw new Error("Docker llama.cpp API-key validation requires O_NOFOLLOW and O_NONBLOCK."); + } + let descriptor: number; + try { + descriptor = fs.openSync( + apiKeyPath, + fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | fs.constants.O_NONBLOCK, + ); + } catch { + throw new Error("Docker llama.cpp API-key file is unavailable."); + } + try { + const opened = fs.fstatSync(descriptor, { bigint: true }); + const linked = fs.lstatSync(apiKeyPath, { bigint: true }); + if ( + !opened.isFile() || + !linked.isFile() || + opened.uid !== currentUid() || + opened.nlink !== 1n || + opened.size < 1n || + opened.size > 64n * 1024n || + (opened.mode & 0o777n) !== 0o600n || + opened.dev !== linked.dev || + opened.ino !== linked.ino || + fs.realpathSync(apiKeyPath) !== apiKeyPath + ) { + throw new Error("Docker llama.cpp API-key file lacks exact private-file authority."); + } + return Object.freeze({ + dev: opened.dev, + ino: opened.ino, + size: opened.size, + mtimeNs: opened.mtimeNs, + ctimeNs: opened.ctimeNs, + }); + } finally { + fs.closeSync(descriptor); + } +} + +function assertApiKeyIdentity( + options: DockerLlamaCppManagedLifecycleOptions, + expected: StableFileIdentity, + expectedRootIdentitySha256: string, +): void { + if ( + apiKeyRootIdentitySha256(options) !== expectedRootIdentitySha256 || + !sameFileIdentity(apiKeyIdentity(options), expected) + ) { + throw new Error("Docker llama.cpp API-key file changed during lifecycle mutation."); + } +} + +function qualifyEngine(options: DockerLlamaCppManagedLifecycleOptions): PersistedEngineAuthority { + if ( + options.engine.operation !== "host-local-inference" || + options.engine.engineId !== PROVIDER_ID || + !SHA256.test(options.bindingSha256) + ) { + throw new Error("Docker llama.cpp requires an exact host-local inference engine."); + } + return createPersistedEngineAuthority(PROVIDER_ID, options.engine, options.bindingSha256); +} + +function authorizeEngine( + options: DockerLlamaCppManagedLifecycleOptions, + qualified: PersistedEngineAuthority, + recordIfMissing: boolean, +): PersistedEngineAuthority { + const persisted = options.authorityStore.load("host-local-inference"); + if (persisted === null) { + if (!recordIfMissing) throw new Error("Docker llama.cpp engine authority is missing."); + return options.authorityStore.record(qualified); + } + return requirePersistedEngineAuthority( + persisted, + PROVIDER_ID, + options.engine, + options.bindingSha256, + ); +} + +function createArguments( + options: DockerLlamaCppManagedLifecycleOptions, + specSha256: string, + transactionId: string, +): readonly string[] { + const run = buildLlamaCppHostLocalDockerArgv(options.contract, options.bindings); + if (run[0] !== "run" || run[1] !== "--detach") { + throw new Error("Docker llama.cpp materializer returned an unsupported launch operation."); + } + const networkIndex = run.indexOf("--network"); + if (networkIndex < 0) throw new Error("Docker llama.cpp materializer omitted its network."); + return Object.freeze([ + "create", + "--pull=never", + ...run.slice(2, networkIndex), + "--label", + `${MANAGED_LABEL}=true`, + "--label", + `${PROVIDER_LABEL}=${PROVIDER_ID}`, + "--label", + `${SERVICE_LABEL}=${SERVICE}`, + "--label", + `${SPEC_LABEL}=${specSha256}`, + "--label", + `${TRANSACTION_LABEL}=${transactionId}`, + ...run.slice(networkIndex), + ]); +} + +function expectedCommand(options: DockerLlamaCppManagedLifecycleOptions): readonly string[] { + return buildLlamaCppHostLocalServerArgv(options.contract); +} + +function specificationDigest( + options: DockerLlamaCppManagedLifecycleOptions, + network: DockerNetworkAuthority, + apiKeyRootIdentity: string, +): string { + return sha256({ + apiKeyRootIdentitySha256: apiKeyRootIdentity, + contract: options.contract, + containerName: options.bindings.containerName, + imageReference: options.bindings.imageReference, + model: { + planDigest: options.plan.planDigest, + recipeId: options.plan.recipeId, + digest: options.plan.acquisition.source.file.digest, + sizeBytes: options.plan.acquisition.source.file.sizeBytes, + }, + network, + ownerLabel: options.bindings.ownerLabel, + probeImageReference: options.probeImageReference, + runtimeGid: options.bindings.runtimeGid, + runtimeUid: options.bindings.runtimeUid, + }); +} + +function requireOwnedContainer( + container: DockerContainerInspection, + options: DockerLlamaCppManagedLifecycleOptions, + recordValue: HostLocalCreateJournalRecord, +): DockerContainerInspection { + const record = normalizeHostLocalCreateJournalRecord(recordValue); + const modelDestination = `/models/${options.contract.model.file.path}`; + const expectedMounts = [ + `${options.bindings.model.hostPath}\0${modelDestination}`, + `${options.bindings.apiKeyHostPath}\0${LLAMA_CPP_HOST_LOCAL_CONTAINER_API_KEY_PATH}`, + ].sort(); + const actualMounts = container.mounts + .filter((mount) => mount.type === "bind" && mount.readOnly) + .map((mount) => `${mount.source}\0${mount.destination}`) + .sort(); + if ( + (record.runtimeId !== null && container.id !== record.runtimeId) || + container.name !== record.containerName || + container.imageRef !== options.bindings.imageReference || + container.networkId !== record.networkId || + container.networkName !== options.bindings.network.name || + container.labels[MANAGED_LABEL] !== "true" || + container.labels[PROVIDER_LABEL] !== PROVIDER_ID || + container.labels[SERVICE_LABEL] !== SERVICE || + container.labels[SPEC_LABEL] !== record.specSha256 || + container.labels[TRANSACTION_LABEL] !== record.transactionId || + container.labels[options.bindings.ownerLabel.name] !== options.bindings.ownerLabel.value || + container.hardening.user !== + `${String(options.bindings.runtimeUid)}:${String(options.bindings.runtimeGid)}` || + container.hardening.networkMode !== options.bindings.network.name || + !container.hardening.readOnlyRootfs || + container.hardening.capDrop.length !== 1 || + container.hardening.capDrop[0] !== "ALL" || + container.hardening.securityOpt.length !== 1 || + (container.hardening.securityOpt[0] !== "no-new-privileges=true" && + container.hardening.securityOpt[0] !== "no-new-privileges:true") || + container.hardening.memory !== options.contract.runtime.resources.memoryBytes || + container.hardening.memorySwap !== options.contract.runtime.resources.memoryBytes || + container.hardening.pidsLimit !== options.contract.runtime.resources.pidsLimit || + container.hardening.gpuCount !== 1 || + !container.hardening.deviceAuthorityExact || + !container.hardening.capAddEmpty || + !container.hardening.legacyDevicesEmpty || + container.hardening.privileged || + container.hardening.command.join("\0") !== expectedCommand(options).join("\0") || + Object.keys(container.hardening.tmpfs).length !== 1 || + container.hardening.tmpfs["/tmp"] !== + `rw,noexec,nosuid,nodev,size=${String(options.contract.runtime.resources.writableStorageBytes)},uid=${String(options.bindings.runtimeUid)},gid=${String(options.bindings.runtimeGid)},mode=1777` || + container.mounts.length !== expectedMounts.length || + actualMounts.join("\n") !== expectedMounts.join("\n") + ) { + throw new Error("Docker llama.cpp container does not match its exact journal authority."); + } + return container; +} + +function captureMutation( + options: DockerLlamaCppManagedLifecycleOptions, + lease: HostLocalCreateJournalExecutionLease, + execution: MutationExecutionState, + args: readonly string[], + timeoutMs: number, +): ContainerEngineCommandResult { + options.journalStore.assertExecution(lease); + execution.unknown = true; + const result = options.engine.capture(args, timeoutMs); + if (!result.error) execution.unknown = false; + options.journalStore.assertExecution(lease); + return result; +} + +function probeReady( + options: DockerLlamaCppManagedLifecycleOptions, + lease: HostLocalCreateJournalExecutionLease, + execution: MutationExecutionState, +): void { + requireSuccess( + "readiness probe", + captureMutation( + options, + lease, + execution, + [ + "run", + "--rm", + "--pull=never", + "--network", + options.bindings.network.name, + options.probeImageReference, + "--fail", + "--silent", + "--show-error", + "--retry", + "10", + "--retry-delay", + "1", + "--retry-connrefused", + `http://${options.bindings.containerName}:${String(options.contract.serve.port)}/health`, + ], + INSPECT_TIMEOUT_MS, + ), + ); +} + +function rollbackExact( + options: DockerLlamaCppManagedLifecycleOptions, + record: HostLocalCreateJournalRecord, + lease: HostLocalCreateJournalExecutionLease, + execution: MutationExecutionState, + uncertainRecoveryUnixMs?: number, +): void { + options.journalStore.assertExecution(lease); + const target = record.runtimeId ?? record.containerName; + let container = inspectContainer( + options.engine, + target, + options.contract, + options.bindings.network.name, + ); + if (container === null && record.phase === "creating" && uncertainRecoveryUnixMs !== undefined) { + if ( + record.createIntentUnixMs === null || + uncertainRecoveryUnixMs < record.createIntentUnixMs || + uncertainRecoveryUnixMs - record.createIntentUnixMs < UNCERTAIN_CREATE_ABSENCE_GRACE_MS + ) { + throw new Error("Docker llama.cpp uncertain create remains inside its absence grace period."); + } + options.journalStore.assertExecution(lease); + container = inspectContainer( + options.engine, + target, + options.contract, + options.bindings.network.name, + ); + } + if (container !== null) { + const owned = requireOwnedContainer(container, options, record); + requireSuccess( + "exact rollback", + captureMutation(options, lease, execution, ["rm", "--force", owned.id], MUTATION_TIMEOUT_MS), + ); + if ( + inspectContainer( + options.engine, + owned.id, + options.contract, + options.bindings.network.name, + ) !== null + ) { + throw new Error("Docker llama.cpp exact rollback left the owned runtime present."); + } + } + options.journalStore.assertExecution(lease); + options.journalStore.retire(record.transactionId); + options.journalStore.assertExecution(lease); +} + +function requireExactNetwork( + options: DockerLlamaCppManagedLifecycleOptions, + expectedId: string, +): DockerNetworkAuthority { + const network = inspectNetwork(options.engine, options.bindings.network.name); + if (network.id !== expectedId) { + throw new Error("Docker llama.cpp internal network identity changed."); + } + return network; +} + +function receiptFor( + options: DockerLlamaCppManagedLifecycleOptions, + authority: PersistedEngineAuthority, + journal: HostLocalCreateJournalRecord, + container: DockerContainerInspection, +): HostLocalInferenceReceipt { + if (container.hostPort === null) { + throw new Error("Docker llama.cpp did not publish one loopback host port."); + } + return normalizeHostLocalInferenceReceipt({ + schemaVersion: 1, + providerId: PROVIDER_ID, + service: SERVICE, + engineAuthority: authority, + endpoint: { + host: ENDPOINT_HOST, + port: container.hostPort, + networkName: options.bindings.network.name, + }, + runtime: { + kind: "container", + runtimeId: container.id, + name: options.bindings.containerName, + imageRef: options.bindings.imageReference, + probeImageRef: options.probeImageReference, + specSha256: journal.specSha256, + model: { + planDigest: options.plan.planDigest, + recipeId: options.plan.recipeId, + generation: journal.transactionId, + digest: options.plan.acquisition.source.file.digest, + sizeBytes: options.plan.acquisition.source.file.sizeBytes, + }, + gpu: { vendor: "nvidia", count: 1 }, + }, + }); +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function operationTime(dependencies: DockerLlamaCppManagedLifecycleDependencies): number { + const value = (dependencies.now ?? Date.now)(); + if (!Number.isSafeInteger(value) || value < 1) { + throw new Error("Docker llama.cpp operation clock is invalid."); + } + return value; +} + +export function createDockerLlamaCppManagedLifecycle( + options: DockerLlamaCppManagedLifecycleOptions, + dependencies: DockerLlamaCppManagedLifecycleDependencies = {}, +): DockerLlamaCppManagedLifecycle { + assertLlamaCppGgufCachePlanDigest(options.plan); + normalizeHostLocalInferenceImageRef(options.probeImageReference); + const qualifiedAuthority = qualifyEngine(options); + + const authorizeStaticReceipt = (value: HostLocalInferenceReceipt) => { + const receipt = normalizeHostLocalInferenceReceipt(value); + if ( + receipt.providerId !== PROVIDER_ID || + receipt.service !== SERVICE || + receipt.runtime.kind !== "container" || + receipt.runtime.model === undefined || + receipt.engineAuthority.authorityId !== qualifiedAuthority.authorityId + ) { + throw new Error("Docker llama.cpp receipt belongs to another lifecycle authority."); + } + authorizeEngine(options, qualifiedAuthority, false); + requirePersistedEngineAuthority( + receipt.engineAuthority, + PROVIDER_ID, + options.engine, + options.bindingSha256, + ); + if ( + receipt.runtime.name !== options.bindings.containerName || + receipt.runtime.imageRef !== options.bindings.imageReference || + receipt.runtime.probeImageRef !== options.probeImageReference || + receipt.runtime.model.planDigest !== options.plan.planDigest || + receipt.runtime.model.recipeId !== options.plan.recipeId || + receipt.runtime.model.digest !== options.plan.acquisition.source.file.digest || + receipt.runtime.model.sizeBytes !== options.plan.acquisition.source.file.sizeBytes || + receipt.endpoint.host !== ENDPOINT_HOST || + receipt.endpoint.networkName !== options.bindings.network.name || + receipt.runtime.gpu.vendor !== "nvidia" || + !("count" in receipt.runtime.gpu) || + receipt.runtime.gpu.count !== 1 + ) { + throw new Error("Docker llama.cpp receipt differs from current declarative authority."); + } + return receipt; + }; + + const authorizeReceipt = ( + value: HostLocalInferenceReceipt, + requireFinalized: boolean, + ): { + readonly receipt: HostLocalInferenceReceipt; + readonly journal: HostLocalCreateJournalRecord; + } => { + const receipt = authorizeStaticReceipt(value); + if (receipt.runtime.kind !== "container" || receipt.runtime.model === undefined) { + throw new Error("Docker llama.cpp receipt lacks container model authority."); + } + const journal = options.journalStore.load(receipt.runtime.model.generation); + const expectedSpecSha256 = + journal === null + ? null + : specificationDigest( + options, + { id: journal.networkId, name: options.bindings.network.name }, + journal.apiKeyRootIdentitySha256, + ); + if ( + journal === null || + journal.providerId !== PROVIDER_ID || + journal.service !== SERVICE || + journal.containerName !== options.bindings.containerName || + journal.runtimeId !== receipt.runtime.runtimeId || + journal.specSha256 !== expectedSpecSha256 || + journal.specSha256 !== receipt.runtime.specSha256 || + JSON.stringify(journal.engineAuthority) !== JSON.stringify(qualifiedAuthority) || + (requireFinalized && journal.phase !== "finalized") || + (journal.phase === "finalized" && + journal.receiptSha256 !== sha256(JSON.parse(serializeHostLocalInferenceReceipt(receipt)))) + ) { + throw new Error("Docker llama.cpp receipt does not match its durable create journal."); + } + return { receipt, journal }; + }; + + const inspectAuthorized = ( + value: HostLocalInferenceReceipt, + requireFinalized = true, + ): { + readonly receipt: HostLocalInferenceReceipt; + readonly journal: HostLocalCreateJournalRecord; + readonly container: DockerContainerInspection; + } => { + const authorized = authorizeReceipt(value, requireFinalized); + requireExactNetwork(options, authorized.journal.networkId); + if (authorized.receipt.runtime.kind !== "container") { + throw new Error("Docker llama.cpp receipt is not a container authority."); + } + const inspected = inspectContainer( + options.engine, + authorized.receipt.runtime.runtimeId, + options.contract, + options.bindings.network.name, + ); + if (inspected === null) throw new Error("Docker llama.cpp owned runtime is absent."); + const container = requireOwnedContainer(inspected, options, authorized.journal); + if ( + container.hostPort !== authorized.receipt.endpoint.port || + authorized.receipt.endpoint.host !== ENDPOINT_HOST || + authorized.receipt.endpoint.networkName !== options.bindings.network.name + ) { + throw new Error("Docker llama.cpp endpoint authority changed."); + } + return { ...authorized, container }; + }; + + const runtime: HostLocalInferenceRuntime = Object.freeze({ + providerId: PROVIDER_ID, + authorityId: options.engine.authorityId, + services: Object.freeze([SERVICE] as const), + translateContainerArgs() { + throw new Error("Docker llama.cpp translation remains dormant behind its controller."); + }, + qualifyOllama() { + throw new Error("Docker llama.cpp does not qualify Ollama routes."); + }, + startManaged() { + throw new Error("Docker llama.cpp creation requires its declarative controller."); + }, + inspectManaged(receipt: HostLocalInferenceReceipt) { + const inspected = inspectAuthorized(receipt); + return Object.freeze({ + running: inspected.container.running, + receipt: inspected.receipt, + }); + }, + stopManaged(receipt: HostLocalInferenceReceipt) { + const authorized = authorizeReceipt(receipt, true); + const lease = options.journalStore.acquireExecution(authorized.journal.transactionId); + const execution: MutationExecutionState = { unknown: false }; + try { + let inspected = inspectAuthorized(receipt); + if (!inspected.container.running) { + if (!AT_REST.has(inspected.container.status)) { + throw new Error("Docker llama.cpp container is not in an exact stoppable state."); + } + return Object.freeze({ running: false, receipt: inspected.receipt }); + } + requireSuccess( + "container stop", + captureMutation( + options, + lease, + execution, + ["stop", "--time", String(STOP_GRACE_SECONDS), inspected.container.id], + MUTATION_TIMEOUT_MS, + ), + ); + inspected = inspectAuthorized(inspected.receipt); + if (inspected.container.running || !AT_REST.has(inspected.container.status)) { + throw new Error("Docker llama.cpp stop did not leave the exact runtime at rest."); + } + return Object.freeze({ running: false, receipt: inspected.receipt }); + } finally { + if (!execution.unknown) options.journalStore.releaseExecution(lease); + } + }, + preserveForRebuild(receipt: HostLocalInferenceReceipt) { + const initial = authorizeReceipt(receipt, true); + const lease = options.journalStore.acquireExecution(initial.journal.transactionId); + const execution: MutationExecutionState = { unknown: false }; + try { + const authorized = authorizeReceipt(receipt, true); + const activeKeyIdentity = apiKeyIdentity(options); + if (apiKeyIdentitySha256(activeKeyIdentity) !== authorized.journal.apiKeyIdentitySha256) { + throw new Error("Docker llama.cpp API-key identity differs from its create journal."); + } + assertModelFilesystemAuthority(options); + assertApiKeyIdentity( + options, + activeKeyIdentity, + authorized.journal.apiKeyRootIdentitySha256, + ); + const inspected = inspectAuthorized(receipt); + if (!inspected.container.running) { + throw new Error("Docker llama.cpp cannot preserve a stopped runtime."); + } + probeReady(options, lease, execution); + assertModelFilesystemAuthority(options); + assertApiKeyIdentity( + options, + activeKeyIdentity, + authorized.journal.apiKeyRootIdentitySha256, + ); + requireExactNetwork(options, inspected.journal.networkId); + return inspected.receipt; + } finally { + if (!execution.unknown) options.journalStore.releaseExecution(lease); + } + }, + prepareDestroy(receipt: HostLocalInferenceReceipt) { + const normalized = authorizeStaticReceipt(receipt); + if (normalized.runtime.kind !== "container" || normalized.runtime.model === undefined) + throw new Error("Docker llama.cpp destroy requires container authority."); + const existing = inspectContainer( + options.engine, + normalized.runtime.runtimeId, + options.contract, + options.bindings.network.name, + ); + const journal = options.journalStore.load(normalized.runtime.model.generation); + if (existing !== null || journal !== null) authorizeReceipt(normalized, true); + if (existing !== null) inspectAuthorized(normalized); + return normalized; + }, + destroy(receipt: HostLocalInferenceReceipt) { + const normalized = authorizeStaticReceipt(receipt); + if (normalized.runtime.kind !== "container" || normalized.runtime.model === undefined) + throw new Error("Docker llama.cpp destroy receipt is invalid."); + const existing = inspectContainer( + options.engine, + normalized.runtime.runtimeId, + options.contract, + options.bindings.network.name, + ); + if (existing === null) { + const journal = options.journalStore.load(normalized.runtime.model.generation); + if (journal !== null) { + const lease = options.journalStore.acquireExecution(journal.transactionId); + try { + const authorized = authorizeReceipt(normalized, true); + options.journalStore.assertExecution(lease); + options.journalStore.retire(authorized.journal.transactionId); + options.journalStore.assertExecution(lease); + } finally { + options.journalStore.releaseExecution(lease); + } + } + return Object.freeze({ + status: "already-absent" as const, + receipt: normalized, + }); + } + const lease = options.journalStore.acquireExecution(normalized.runtime.model.generation); + const execution: MutationExecutionState = { unknown: false }; + try { + const inspected = inspectAuthorized(normalized); + requireSuccess( + "container removal", + captureMutation( + options, + lease, + execution, + ["rm", "--force", inspected.container.id], + MUTATION_TIMEOUT_MS, + ), + ); + if ( + inspectContainer( + options.engine, + inspected.container.id, + options.contract, + options.bindings.network.name, + ) !== null + ) { + throw new Error("Docker llama.cpp removal left the exact runtime present."); + } + options.journalStore.assertExecution(lease); + options.journalStore.retire(inspected.journal.transactionId); + options.journalStore.assertExecution(lease); + return Object.freeze({ + status: "removed" as const, + receipt: inspected.receipt, + }); + } finally { + if (!execution.unknown) options.journalStore.releaseExecution(lease); + } + }, + }); + + return Object.freeze({ + runtime, + start(persistReceipt: (serializedReceipt: string) => void) { + if (typeof persistReceipt !== "function") { + throw new Error("Docker llama.cpp start requires an operation-scoped receipt writer."); + } + assertLlamaCppGgufCachePlanDigest(options.plan); + assertModelFilesystemAuthority(options); + const startingApiKeyRootIdentitySha256 = apiKeyRootIdentitySha256(options); + const startingKeyIdentity = apiKeyIdentity(options); + const network = inspectNetwork(options.engine, options.bindings.network.name); + const authority = authorizeEngine(options, qualifiedAuthority, true); + const specSha256 = specificationDigest(options, network, startingApiKeyRootIdentitySha256); + const transactionId = + dependencies.createTransactionId?.() ?? sha256({ generation: randomUUID() }); + if (!SHA256.test(transactionId)) { + throw new Error("Docker llama.cpp transaction identity is malformed."); + } + const lease = options.journalStore.acquireExecution(transactionId); + const execution: MutationExecutionState = { unknown: false }; + let journal: HostLocalCreateJournalRecord | null = null; + try { + if ( + inspectContainer( + options.engine, + options.bindings.containerName, + options.contract, + options.bindings.network.name, + ) !== null + ) { + throw new Error("Docker llama.cpp container name is already in use."); + } + options.journalStore.assertExecution(lease); + journal = options.journalStore.create({ + schemaVersion: 1, + transactionId, + phase: "prepared", + providerId: PROVIDER_ID, + service: SERVICE, + containerName: options.bindings.containerName, + runtimeId: null, + createIntentUnixMs: null, + specSha256, + networkId: network.id, + engineAuthority: authority, + apiKeyIdentitySha256: apiKeyIdentitySha256(startingKeyIdentity), + apiKeyRootIdentitySha256: startingApiKeyRootIdentitySha256, + receiptSha256: null, + }); + options.journalStore.assertExecution(lease); + requireExactNetwork(options, network.id); + assertModelFilesystemAuthority(options); + assertApiKeyIdentity(options, startingKeyIdentity, startingApiKeyRootIdentitySha256); + options.journalStore.assertExecution(lease); + journal = options.journalStore.recordCreating(transactionId, operationTime(dependencies)); + options.journalStore.assertExecution(lease); + const create = captureMutation( + options, + lease, + execution, + createArguments(options, specSha256, transactionId), + MUTATION_TIMEOUT_MS, + ); + const created = inspectContainer( + options.engine, + options.bindings.containerName, + options.contract, + options.bindings.network.name, + ); + if (create.error || create.status !== 0 || created === null) { + throw new Error( + `Docker llama.cpp container create failed (exit ${String(create.status)}).`, + ); + } + const createId = exactId(create.stdout.trim(), "Docker create result"); + if (created.id !== createId) { + throw new Error("Docker llama.cpp create result disagrees with exact name inspection."); + } + requireOwnedContainer(created, options, journal); + options.journalStore.assertExecution(lease); + journal = options.journalStore.recordCreated(transactionId, created.id); + options.journalStore.assertExecution(lease); + requireExactNetwork(options, network.id); + assertModelFilesystemAuthority(options); + assertApiKeyIdentity(options, startingKeyIdentity, startingApiKeyRootIdentitySha256); + requireSuccess( + "container start", + captureMutation(options, lease, execution, ["start", created.id], MUTATION_TIMEOUT_MS), + ); + const started = inspectContainer( + options.engine, + created.id, + options.contract, + options.bindings.network.name, + ); + if (started === null || !started.running) { + throw new Error("Docker llama.cpp start did not leave the exact runtime running."); + } + requireOwnedContainer(started, options, journal); + options.journalStore.assertExecution(lease); + journal = options.journalStore.recordStarted(transactionId); + options.journalStore.assertExecution(lease); + assertModelFilesystemAuthority(options); + assertApiKeyIdentity(options, startingKeyIdentity, startingApiKeyRootIdentitySha256); + requireExactNetwork(options, network.id); + probeReady(options, lease, execution); + assertModelFilesystemAuthority(options); + assertApiKeyIdentity(options, startingKeyIdentity, startingApiKeyRootIdentitySha256); + requireExactNetwork(options, network.id); + const receipt = receiptFor(options, authority, journal, started); + const serialized = serializeHostLocalInferenceReceipt(receipt); + // Schema-v1 llama.cpp receipt persistence remains rejected by the production registry. + // Activation must make this persist/finalize boundary atomically durable first; #8414 + // tracks that commit boundary: https://github.com/NVIDIA/NemoClaw/issues/8414 + persistReceipt(serialized); + options.journalStore.assertExecution(lease); + options.journalStore.finalize(transactionId, sha256(JSON.parse(serialized))); + options.journalStore.assertExecution(lease); + return receipt; + } catch (error) { + let rollbackFailure: unknown; + if (journal !== null && !execution.unknown) { + try { + rollbackExact(options, journal, lease, execution); + } catch (rollbackError) { + rollbackFailure = rollbackError; + } + } + if (rollbackFailure !== undefined) { + throw new Error( + `${errorMessage(error)} Exact rollback also failed: ${errorMessage(rollbackFailure)}`, + ); + } + throw error; + } finally { + if (!execution.unknown) options.journalStore.releaseExecution(lease); + } + }, + recoverUnfinished() { + const recovered: string[] = []; + const failures: { transactionId: string; message: string }[] = []; + for (const candidate of options.journalStore.list()) { + let journal = normalizeHostLocalCreateJournalRecord(candidate); + if ( + journal.providerId !== PROVIDER_ID || + journal.service !== SERVICE || + journal.phase === "finalized" + ) { + continue; + } + let lease: HostLocalCreateJournalExecutionLease | null = null; + const execution: MutationExecutionState = { unknown: false }; + try { + lease = options.journalStore.acquireExecution(journal.transactionId); + const active = options.journalStore.load(journal.transactionId); + if (active === null) continue; + journal = normalizeHostLocalCreateJournalRecord(active); + if (journal.phase === "finalized") continue; + const currentAuthority = authorizeEngine(options, qualifiedAuthority, false); + const journalAuthority = requirePersistedEngineAuthority( + journal.engineAuthority, + PROVIDER_ID, + options.engine, + options.bindingSha256, + ); + if (JSON.stringify(journalAuthority) !== JSON.stringify(currentAuthority)) { + throw new Error("Docker llama.cpp recovery engine authority changed."); + } + const expectedSpecSha256 = specificationDigest( + options, + { id: journal.networkId, name: options.bindings.network.name }, + journal.apiKeyRootIdentitySha256, + ); + if (journal.specSha256 !== expectedSpecSha256) { + throw new Error( + "Docker llama.cpp recovery journal differs from declarative authority.", + ); + } + requireExactNetwork(options, journal.networkId); + rollbackExact( + options, + journal, + lease, + execution, + journal.phase === "creating" ? operationTime(dependencies) : undefined, + ); + recovered.push(journal.transactionId); + } catch (error) { + failures.push({ + transactionId: journal.transactionId, + message: errorMessage(error), + }); + } finally { + if (lease !== null && !execution.unknown) { + options.journalStore.releaseExecution(lease); + } + } + } + return Object.freeze({ + recovered: Object.freeze(recovered), + failures: Object.freeze(failures), + }); + }, + }); +} diff --git a/src/lib/onboard/runtime-provider/host-local-create-journal.test.ts b/src/lib/onboard/runtime-provider/host-local-create-journal.test.ts new file mode 100644 index 00000000000..975b4d327f7 --- /dev/null +++ b/src/lib/onboard/runtime-provider/host-local-create-journal.test.ts @@ -0,0 +1,344 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { + createHostLocalCreateJournalStore, + HOST_LOCAL_CREATE_JOURNAL_DIRECTORY, + type HostLocalCreateJournalRecord, + serializeHostLocalCreateJournalRecord, +} from "./host-local-create-journal"; + +const TRANSACTION_ID = "a".repeat(64); +const RUNTIME_ID = "b".repeat(64); +const RECEIPT_SHA256 = "c".repeat(64); +const CREATE_INTENT_UNIX_MS = 1_786_000_000_000; +const OWNER_ONE = "11111111-1111-4111-8111-111111111111"; +const OWNER_TWO = "22222222-2222-4222-8222-222222222222"; +const OWNER_THREE = "33333333-3333-4333-8333-333333333333"; +let stateDirectory = ""; + +beforeEach(() => { + stateDirectory = fs.realpathSync( + fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-host-local-create-journal-")), + ); +}); + +afterEach(() => { + vi.restoreAllMocks(); + fs.rmSync(stateDirectory, { force: true, recursive: true }); +}); + +function prepared(): HostLocalCreateJournalRecord { + return { + schemaVersion: 1, + transactionId: TRANSACTION_ID, + phase: "prepared", + providerId: "docker", + service: "llama-cpp", + containerName: "nemoclaw-llama-cpp", + runtimeId: null, + createIntentUnixMs: null, + specSha256: "d".repeat(64), + networkId: "e".repeat(64), + apiKeyIdentitySha256: "1".repeat(64), + apiKeyRootIdentitySha256: "2".repeat(64), + engineAuthority: { + schemaVersion: 1, + providerId: "docker", + operation: "host-local-inference", + engineId: "docker", + authorityId: "docker:local", + bindingSha256: "f".repeat(64), + }, + receiptSha256: null, + }; +} + +function journalPath(): string { + return path.join(stateDirectory, HOST_LOCAL_CREATE_JOURNAL_DIRECTORY, `${TRANSACTION_ID}.json`); +} + +function executionPath(name: ".execution-lease.json" | ".execution-recovery.json"): string { + return path.join(stateDirectory, HOST_LOCAL_CREATE_JOURNAL_DIRECTORY, name); +} + +function executionSource(transactionId: string, ownerId: string, ownerPid: number): string { + return `${JSON.stringify({ schemaVersion: 1, transactionId, ownerId, ownerPid })}\n`; +} + +describe("host-local create journal", () => { + it("durably resumes every create phase without persisting executor paths or secrets (#8395)", () => { + const fsync = vi.spyOn(fs, "fsyncSync"); + const first = createHostLocalCreateJournalStore(stateDirectory); + expect(first.create(prepared()).phase).toBe("prepared"); + expect(first.recordCreating(TRANSACTION_ID, CREATE_INTENT_UNIX_MS).phase).toBe("creating"); + expect(first.recordCreated(TRANSACTION_ID, RUNTIME_ID).phase).toBe("created"); + + const restarted = createHostLocalCreateJournalStore(stateDirectory); + expect(restarted.recordStarted(TRANSACTION_ID).phase).toBe("started"); + const finalized = restarted.finalize(TRANSACTION_ID, RECEIPT_SHA256); + expect(finalized.phase).toBe("finalized"); + expect(restarted.list()).toEqual([finalized]); + + const serialized = fs.readFileSync(journalPath(), "utf8"); + expect(serialized).toBe(serializeHostLocalCreateJournalRecord(finalized)); + expect(serialized).not.toMatch( + /hostPath|apiKeyHostPath|apiKeyValue|HF_TOKEN|filesystemIdentity/u, + ); + expect(fs.statSync(path.dirname(journalPath())).mode & 0o777).toBe(0o700); + expect(fs.statSync(journalPath()).mode & 0o777).toBe(0o600); + expect(fsync.mock.calls.length).toBeGreaterThanOrEqual(9); + }); + + it("rejects duplicate creation and out-of-order transitions (#8395)", () => { + const store = createHostLocalCreateJournalStore(stateDirectory); + store.create(prepared()); + expect(() => store.create(prepared())).toThrow("transaction already exists"); + expect(() => store.recordCreated(TRANSACTION_ID, RUNTIME_ID)).toThrow( + "only a creating transaction can record creation", + ); + expect(() => store.recordStarted(TRANSACTION_ID)).toThrow( + "only a created transaction can record start", + ); + expect(() => store.finalize(TRANSACTION_ID, RECEIPT_SHA256)).toThrow( + "only a started transaction can finalize", + ); + }); + + it("rejects a symlinked journal record instead of following it (#8395)", () => { + const store = createHostLocalCreateJournalStore(stateDirectory); + store.create(prepared()); + const outside = path.join(stateDirectory, "outside.json"); + fs.renameSync(journalPath(), outside); + fs.symlinkSync(outside, journalPath()); + + expect(() => store.load(TRANSACTION_ID)).toThrow("ELOOP"); + }); + + it("rejects a non-private journal directory (#8395)", () => { + const root = path.join(stateDirectory, HOST_LOCAL_CREATE_JOURNAL_DIRECTORY); + fs.mkdirSync(root, { mode: 0o700 }); + fs.chmodSync(root, 0o770); + + expect(() => createHostLocalCreateJournalStore(stateDirectory).list()).toThrow( + "private current-user-owned directory", + ); + }); + + it("retires an exact generation idempotently and persists the deletion (#8395)", () => { + const fsync = vi.spyOn(fs, "fsyncSync"); + const store = createHostLocalCreateJournalStore(stateDirectory); + store.create(prepared()); + fsync.mockClear(); + + store.retire(TRANSACTION_ID); + expect(store.load(TRANSACTION_ID)).toBeNull(); + expect(fsync).toHaveBeenCalled(); + store.retire(TRANSACTION_ID); + }); + + it("gives one live process exclusive durable execution ownership (#8395)", () => { + const first = createHostLocalCreateJournalStore(stateDirectory, { + createOwnerId: () => OWNER_ONE, + ownerPid: 101, + processIsAlive: () => true, + }); + const lease = first.acquireExecution(TRANSACTION_ID); + first.assertExecution(lease); + + const contender = createHostLocalCreateJournalStore(stateDirectory, { + createOwnerId: () => OWNER_TWO, + ownerPid: 202, + processIsAlive: () => true, + }); + expect(() => contender.acquireExecution("3".repeat(64))).toThrow( + "already owned by a live process", + ); + expect(() => contender.releaseExecution({ ...lease, ownerId: OWNER_TWO })).toThrow( + "execution ownership changed", + ); + + first.releaseExecution(lease); + const replacement = contender.acquireExecution("3".repeat(64)); + contender.assertExecution(replacement); + contender.releaseExecution(replacement); + }); + + it("recovers only a dead execution owner through the durable recovery marker (#8395)", () => { + const abandonedStore = createHostLocalCreateJournalStore(stateDirectory, { + createOwnerId: () => OWNER_ONE, + ownerPid: 101, + processIsAlive: (pid) => pid === 101, + }); + const abandoned = abandonedStore.acquireExecution(TRANSACTION_ID); + + const recoveryStore = createHostLocalCreateJournalStore(stateDirectory, { + createOwnerId: () => OWNER_TWO, + ownerPid: 202, + processIsAlive: (pid) => pid === 202, + }); + const recovered = recoveryStore.acquireExecution("3".repeat(64)); + expect(recovered).toMatchObject({ ownerId: OWNER_TWO, ownerPid: 202 }); + expect(() => abandonedStore.assertExecution(abandoned)).toThrow("execution ownership changed"); + recoveryStore.releaseExecution(recovered); + }); + + it("reclaims a dead recovery marker with and without an abandoned lease (#8395)", () => { + for (const withLease of [false, true]) { + fs.rmSync(stateDirectory, { force: true, recursive: true }); + fs.mkdirSync(stateDirectory, { mode: 0o700 }); + const setup = createHostLocalCreateJournalStore(stateDirectory, { + createOwnerId: () => OWNER_ONE, + ownerPid: 101, + processIsAlive: () => true, + }); + setup.list(); + withLease ? setup.acquireExecution(TRANSACTION_ID) : undefined; + fs.writeFileSync( + executionPath(".execution-recovery.json"), + executionSource("2".repeat(64), OWNER_TWO, 202), + { mode: 0o600 }, + ); + + const recovered = createHostLocalCreateJournalStore(stateDirectory, { + createOwnerId: () => OWNER_THREE, + ownerPid: 303, + processIsAlive: (pid) => pid === 303, + }); + const lease = recovered.acquireExecution("3".repeat(64)); + expect(lease).toMatchObject({ ownerId: OWNER_THREE, ownerPid: 303 }); + expect(fs.existsSync(executionPath(".execution-recovery.json"))).toBe(false); + recovered.releaseExecution(lease); + } + }); + + it("preserves live recovery-marker exclusion (#8395)", () => { + const setup = createHostLocalCreateJournalStore(stateDirectory); + setup.list(); + fs.writeFileSync( + executionPath(".execution-recovery.json"), + executionSource("2".repeat(64), OWNER_TWO, 202), + { mode: 0o600 }, + ); + const contender = createHostLocalCreateJournalStore(stateDirectory, { + createOwnerId: () => OWNER_THREE, + ownerPid: 303, + processIsAlive: (pid) => pid === 202 || pid === 303, + }); + expect(() => contender.acquireExecution("3".repeat(64))).toThrow( + "recovery is already owned by a live process", + ); + }); + + it("reconciles the exact orphan hard link from a crashed exclusive publish (#8395)", () => { + const store = createHostLocalCreateJournalStore(stateDirectory, { + createOwnerId: () => OWNER_ONE, + }); + store.create(prepared()); + const journalOrphan = path.join( + path.dirname(journalPath()), + `.${path.basename(journalPath())}.${OWNER_ONE}.tmp`, + ); + fs.linkSync(journalPath(), journalOrphan); + expect(fs.statSync(journalPath()).nlink).toBe(2); + expect(store.load(TRANSACTION_ID)).toEqual(prepared()); + expect(fs.existsSync(journalOrphan)).toBe(false); + expect(fs.statSync(journalPath()).nlink).toBe(1); + + const lease = store.acquireExecution(TRANSACTION_ID); + const leasePath = executionPath(".execution-lease.json"); + const leaseOrphan = path.join( + path.dirname(leasePath), + `.${path.basename(leasePath)}.${OWNER_TWO}.tmp`, + ); + fs.linkSync(leasePath, leaseOrphan); + store.assertExecution(lease); + expect(fs.existsSync(leaseOrphan)).toBe(false); + expect(fs.statSync(leasePath).nlink).toBe(1); + store.releaseExecution(lease); + }); + + it.each([ + [ + "before identity inspection", + (orphan: string) => { + const lstatSync = fs.lstatSync.bind(fs); + vi.spyOn(fs, "lstatSync").mockImplementation(((target, options) => { + switch (target === orphan) { + case true: + fs.unlinkSync(orphan); + throw Object.assign(new Error("candidate disappeared"), { code: "ENOENT" }); + default: + return lstatSync(target, options); + } + }) as typeof fs.lstatSync); + }, + ], + [ + "before unlink", + (orphan: string) => { + const unlinkSync = fs.unlinkSync.bind(fs); + vi.spyOn(fs, "unlinkSync").mockImplementation((target) => { + switch (target === orphan) { + case true: + unlinkSync(orphan); + throw Object.assign(new Error("candidate disappeared"), { code: "ENOENT" }); + default: + unlinkSync(target); + } + }); + }, + ], + ])("accepts the exact orphan disappearing %s (#8395)", (_name, arrangeRace) => { + const store = createHostLocalCreateJournalStore(stateDirectory, { + createOwnerId: () => OWNER_ONE, + }); + store.create(prepared()); + const journalOrphan = path.join( + path.dirname(journalPath()), + `.${path.basename(journalPath())}.${OWNER_ONE}.tmp`, + ); + fs.linkSync(journalPath(), journalOrphan); + arrangeRace(journalOrphan); + + expect(store.load(TRANSACTION_ID)).toEqual(prepared()); + expect(fs.existsSync(journalOrphan)).toBe(false); + }); + + it("preserves a publication error when temporary cleanup also fails (#8395)", () => { + const store = createHostLocalCreateJournalStore(stateDirectory); + vi.spyOn(fs, "writeFileSync").mockImplementationOnce(() => { + throw new Error("publication failed"); + }); + vi.spyOn(fs, "unlinkSync").mockImplementationOnce(() => { + throw Object.assign(new Error("cleanup failed"), { code: "EIO" }); + }); + + expect(() => store.create(prepared())).toThrow("publication failed"); + }); + + it("lets a live publisher finish when a concurrent reader reconciles its hard link (#8395)", () => { + const writer = createHostLocalCreateJournalStore(stateDirectory); + const reader = createHostLocalCreateJournalStore(stateDirectory); + const linkSync = fs.linkSync.bind(fs); + let reconciled = false; + vi.spyOn(fs, "linkSync").mockImplementation((existingPath, newPath) => { + linkSync(existingPath, newPath); + switch (!reconciled && newPath === journalPath()) { + case true: + reconciled = true; + expect(reader.load(TRANSACTION_ID)).toEqual(prepared()); + } + }); + + expect(writer.create(prepared())).toEqual(prepared()); + expect(reconciled).toBe(true); + expect(writer.load(TRANSACTION_ID)).toEqual(prepared()); + }); +}); diff --git a/src/lib/onboard/runtime-provider/host-local-create-journal.ts b/src/lib/onboard/runtime-provider/host-local-create-journal.ts new file mode 100644 index 00000000000..7f65c1e2e41 --- /dev/null +++ b/src/lib/onboard/runtime-provider/host-local-create-journal.ts @@ -0,0 +1,647 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { randomUUID } from "node:crypto"; +import fs, { type BigIntStats } from "node:fs"; +import path from "node:path"; + +import { + normalizePersistedEngineAuthority, + type PersistedEngineAuthority, +} from "./persisted-engine-authority"; + +export const HOST_LOCAL_CREATE_JOURNAL_SCHEMA_VERSION = 1 as const; +export const HOST_LOCAL_CREATE_JOURNAL_DIRECTORY = "host-local-create-journal"; + +export type HostLocalCreateJournalPhase = + | "prepared" + | "creating" + | "created" + | "started" + | "finalized"; + +export interface HostLocalCreateJournalRecord { + readonly schemaVersion: typeof HOST_LOCAL_CREATE_JOURNAL_SCHEMA_VERSION; + readonly transactionId: string; + readonly phase: HostLocalCreateJournalPhase; + readonly providerId: string; + readonly service: string; + readonly containerName: string; + readonly runtimeId: string | null; + /** Durable wall-clock boundary for an issued container-create mutation. */ + readonly createIntentUnixMs: number | null; + readonly specSha256: string; + readonly networkId: string; + /** Path- and value-free identity of the API-key file captured before create. */ + readonly apiKeyIdentitySha256: string; + /** Path-free identity of the private directory chain that owns the API-key pathname. */ + readonly apiKeyRootIdentitySha256: string; + readonly engineAuthority: PersistedEngineAuthority; + readonly receiptSha256: string | null; +} + +export interface HostLocalCreateJournalExecutionLease { + readonly schemaVersion: typeof HOST_LOCAL_CREATE_JOURNAL_SCHEMA_VERSION; + readonly transactionId: string; + readonly ownerId: string; + readonly ownerPid: number; +} + +export interface HostLocalCreateJournalStore { + readonly load: (transactionId: string) => HostLocalCreateJournalRecord | null; + readonly list: () => readonly HostLocalCreateJournalRecord[]; + readonly create: (record: HostLocalCreateJournalRecord) => HostLocalCreateJournalRecord; + readonly recordCreating: ( + transactionId: string, + createIntentUnixMs: number, + ) => HostLocalCreateJournalRecord; + readonly recordCreated: ( + transactionId: string, + runtimeId: string, + ) => HostLocalCreateJournalRecord; + readonly recordStarted: (transactionId: string) => HostLocalCreateJournalRecord; + readonly finalize: (transactionId: string, receiptSha256: string) => HostLocalCreateJournalRecord; + readonly retire: (transactionId: string) => void; + readonly acquireExecution: (transactionId: string) => HostLocalCreateJournalExecutionLease; + readonly assertExecution: (lease: HostLocalCreateJournalExecutionLease) => void; + readonly releaseExecution: (lease: HostLocalCreateJournalExecutionLease) => void; +} + +export interface HostLocalCreateJournalStoreDependencies { + readonly createOwnerId?: () => string; + readonly ownerPid?: number; + readonly processIsAlive?: (pid: number) => boolean; +} + +const DIRECTORY_MODE = 0o700; +const FILE_MODE = 0o600; +const MAX_BYTES = 32 * 1024; +const SHA256 = /^[a-f0-9]{64}$/u; +const PROVIDER = /^[a-z][a-z0-9-]{0,62}$/u; +const SERVICE = /^[a-z][a-z0-9.-]{0,62}$/u; +const NAME = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u; +const RUNTIME_ID = /^[A-Za-z0-9][A-Za-z0-9._:/=+-]{0,511}$/u; +const UUID = /^[a-f0-9]{8}-[a-f0-9]{4}-[1-5][a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/u; +const EXECUTION_LEASE_FILE = ".execution-lease.json"; +const EXECUTION_RECOVERY_FILE = ".execution-recovery.json"; +const PHASES = new Set([ + "prepared", + "creating", + "created", + "started", + "finalized", +]); + +function fail(message: string): never { + throw new Error(`Host-local create journal is invalid: ${message}`); +} + +function exactText(value: unknown, pattern: RegExp, label: string): string { + if (typeof value !== "string" || !pattern.test(value)) fail(`${label} is malformed`); + return value; +} + +function exactPid(value: unknown): number { + if (!Number.isSafeInteger(value) || Number(value) < 1 || Number(value) > 0x7fffffff) { + fail("execution owner process ID is malformed"); + } + return Number(value); +} + +function exactUnixMs(value: unknown): number { + if (!Number.isSafeInteger(value) || Number(value) < 1) { + fail("create intent timestamp is malformed"); + } + return Number(value); +} + +function normalizeExecutionLease(value: unknown): HostLocalCreateJournalExecutionLease { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + fail("execution lease must be an object"); + } + const lease = value as Record; + if ( + Object.keys(lease).sort().join(",") !== "ownerId,ownerPid,schemaVersion,transactionId" || + lease.schemaVersion !== HOST_LOCAL_CREATE_JOURNAL_SCHEMA_VERSION + ) { + fail("execution lease schema is unsupported"); + } + return Object.freeze({ + schemaVersion: HOST_LOCAL_CREATE_JOURNAL_SCHEMA_VERSION, + transactionId: exactText(lease.transactionId, SHA256, "execution transaction identity"), + ownerId: exactText(lease.ownerId, UUID, "execution owner identity"), + ownerPid: exactPid(lease.ownerPid), + }); +} + +function serializeExecutionLease(value: HostLocalCreateJournalExecutionLease): string { + return `${JSON.stringify(normalizeExecutionLease(value))}\n`; +} + +export function normalizeHostLocalCreateJournalRecord( + value: unknown, +): HostLocalCreateJournalRecord { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + fail("record must be an object"); + } + const record = value as Record; + const keys = [ + "apiKeyIdentitySha256", + "apiKeyRootIdentitySha256", + "containerName", + "createIntentUnixMs", + "engineAuthority", + "networkId", + "phase", + "providerId", + "receiptSha256", + "runtimeId", + "schemaVersion", + "service", + "specSha256", + "transactionId", + ]; + if ( + Object.keys(record).sort().join(",") !== keys.sort().join(",") || + record.schemaVersion !== HOST_LOCAL_CREATE_JOURNAL_SCHEMA_VERSION || + typeof record.phase !== "string" || + !PHASES.has(record.phase as HostLocalCreateJournalPhase) + ) { + fail("record schema is unsupported"); + } + const phase = record.phase as HostLocalCreateJournalPhase; + const runtimeId = + record.runtimeId === null ? null : exactText(record.runtimeId, RUNTIME_ID, "runtime identity"); + const createIntentUnixMs = + record.createIntentUnixMs === null ? null : exactUnixMs(record.createIntentUnixMs); + const receiptSha256 = + record.receiptSha256 === null + ? null + : exactText(record.receiptSha256, SHA256, "receipt digest"); + if ((phase === "prepared" || phase === "creating") !== (runtimeId === null)) { + fail("phase and runtime identity disagree"); + } + if ((phase === "prepared") !== (createIntentUnixMs === null)) { + fail("phase and create intent timestamp disagree"); + } + if ((phase === "finalized") !== (receiptSha256 !== null)) { + fail("phase and receipt digest disagree"); + } + const authority = normalizePersistedEngineAuthority(record.engineAuthority); + if (authority.operation !== "host-local-inference") { + fail("engine authority has the wrong operation"); + } + return Object.freeze({ + schemaVersion: HOST_LOCAL_CREATE_JOURNAL_SCHEMA_VERSION, + transactionId: exactText(record.transactionId, SHA256, "transaction identity"), + phase, + providerId: exactText(record.providerId, PROVIDER, "provider identity"), + service: exactText(record.service, SERVICE, "service identity"), + containerName: exactText(record.containerName, NAME, "container name"), + runtimeId, + createIntentUnixMs, + specSha256: exactText(record.specSha256, SHA256, "specification digest"), + networkId: exactText(record.networkId, RUNTIME_ID, "network identity"), + apiKeyIdentitySha256: exactText( + record.apiKeyIdentitySha256, + SHA256, + "API-key file identity digest", + ), + apiKeyRootIdentitySha256: exactText( + record.apiKeyRootIdentitySha256, + SHA256, + "API-key directory authority digest", + ), + engineAuthority: authority, + receiptSha256, + }); +} + +export function serializeHostLocalCreateJournalRecord(value: HostLocalCreateJournalRecord): string { + const serialized = `${JSON.stringify(normalizeHostLocalCreateJournalRecord(value))}\n`; + if (Buffer.byteLength(serialized, "utf8") > MAX_BYTES) fail("record exceeds its size limit"); + return serialized; +} + +function currentUid(): number { + if (typeof process.getuid !== "function") fail("current-user identity is unavailable"); + return process.getuid(); +} + +function fsyncDirectory(directory: string): void { + const descriptor = fs.openSync(directory, "r"); + try { + fs.fsyncSync(descriptor); + } finally { + fs.closeSync(descriptor); + } +} + +function requireDirectory(root: string): void { + const existed = fs.existsSync(root); + fs.mkdirSync(root, { recursive: true, mode: DIRECTORY_MODE }); + const status = fs.lstatSync(root); + if ( + !status.isDirectory() || + status.isSymbolicLink() || + status.uid !== currentUid() || + (status.mode & 0o077) !== 0 || + fs.realpathSync(root) !== root + ) { + fail("journal directory must be a private current-user-owned directory"); + } + if (!existed) fsyncDirectory(path.dirname(root)); +} + +function recordPath(root: string, transactionId: string): string { + return path.join(root, `${exactText(transactionId, SHA256, "transaction identity")}.json`); +} + +function sameStableMetadata(left: BigIntStats, right: BigIntStats): boolean { + return ( + left.dev === right.dev && + left.ino === right.ino && + left.mode === right.mode && + left.nlink === right.nlink && + left.uid === right.uid && + left.gid === right.gid && + left.size === right.size && + left.mtimeNs === right.mtimeNs && + left.ctimeNs === right.ctimeNs + ); +} + +function reconcileExclusivePublishOrphan( + target: string, + descriptor: number, + initial: BigIntStats, +): BigIntStats { + if (initial.nlink === 1n) return initial; + if (initial.nlink !== 2n) fail("journal file link authority is invalid"); + const root = path.dirname(target); + const prefix = `.${path.basename(target)}.`; + const candidates = fs.readdirSync(root).filter((entry) => { + if (!entry.startsWith(prefix) || !entry.endsWith(".tmp")) return false; + const ownerId = entry.slice(prefix.length, -".tmp".length); + if (!UUID.test(ownerId)) return false; + let status: BigIntStats; + try { + status = fs.lstatSync(path.join(root, entry), { bigint: true }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return false; + throw error; + } + return ( + status.isFile() && + !status.isSymbolicLink() && + status.dev === initial.dev && + status.ino === initial.ino + ); + }); + const afterScan = fs.fstatSync(descriptor, { bigint: true }); + if (afterScan.dev !== initial.dev || afterScan.ino !== initial.ino) { + fail("exclusive journal publication changed during recovery"); + } + if (afterScan.nlink === 1n) { + fsyncDirectory(root); + return afterScan; + } + if (candidates.length !== 1) fail("exclusive journal publication is not recoverable"); + try { + fs.unlinkSync(path.join(root, candidates[0])); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + fsyncDirectory(root); + const reconciled = fs.fstatSync(descriptor, { bigint: true }); + if (reconciled.nlink !== 1n || reconciled.dev !== initial.dev || reconciled.ino !== initial.ino) { + fail("exclusive journal publication changed during recovery"); + } + return reconciled; +} + +function readPrivateFile(target: string): string | null { + if (typeof fs.constants.O_NOFOLLOW !== "number") fail("O_NOFOLLOW is unavailable"); + const nonblock = typeof fs.constants.O_NONBLOCK === "number" ? fs.constants.O_NONBLOCK : 0; + let descriptor: number; + try { + descriptor = fs.openSync(target, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | nonblock); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } + try { + const before = reconcileExclusivePublishOrphan( + target, + descriptor, + fs.fstatSync(descriptor, { bigint: true }), + ); + if ( + !before.isFile() || + before.uid !== BigInt(currentUid()) || + before.nlink !== 1n || + (before.mode & 0o077n) !== 0n || + before.size < 1n || + before.size > BigInt(MAX_BYTES) + ) { + fail("journal file authority is invalid"); + } + const contents = Buffer.alloc(Number(before.size)); + let offset = 0; + while (offset < contents.length) { + const count = fs.readSync(descriptor, contents, offset, contents.length - offset, offset); + if (count === 0) break; + offset += count; + } + const overflow = Buffer.alloc(1); + const overflowCount = fs.readSync(descriptor, overflow, 0, 1, offset); + const after = fs.fstatSync(descriptor, { bigint: true }); + if (offset !== contents.length || overflowCount !== 0 || !sameStableMetadata(before, after)) { + fail("journal file changed during its stable read"); + } + return contents.toString("utf8"); + } finally { + fs.closeSync(descriptor); + } +} + +function readRecord(target: string): HostLocalCreateJournalRecord | null { + const source = readPrivateFile(target); + if (source === null) return null; + let parsed: unknown; + try { + parsed = JSON.parse(source); + } catch { + fail("journal record is not valid JSON"); + } + const record = normalizeHostLocalCreateJournalRecord(parsed); + if (serializeHostLocalCreateJournalRecord(record) !== source) { + fail("journal record is not canonical"); + } + return record; +} + +function readExecutionLease(target: string): HostLocalCreateJournalExecutionLease | null { + const source = readPrivateFile(target); + if (source === null) return null; + let parsed: unknown; + try { + parsed = JSON.parse(source); + } catch { + fail("execution lease is not valid JSON"); + } + const lease = normalizeExecutionLease(parsed); + if (serializeExecutionLease(lease) !== source) fail("execution lease is not canonical"); + return lease; +} + +class HostLocalCreateJournalEntryExistsError extends Error {} + +function publish(root: string, target: string, source: string, exclusive: boolean): void { + if (typeof fs.constants.O_NOFOLLOW !== "number") fail("O_NOFOLLOW is unavailable"); + const temporary = path.join(root, `.${path.basename(target)}.${randomUUID()}.tmp`); + let descriptor: number | null = null; + let operationFailed = false; + let operationFailure: unknown; + try { + descriptor = fs.openSync( + temporary, + fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_NOFOLLOW, + FILE_MODE, + ); + fs.writeFileSync(descriptor, source, "utf8"); + fs.fsyncSync(descriptor); + fs.closeSync(descriptor); + descriptor = null; + if (exclusive) { + try { + fs.linkSync(temporary, target); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "EEXIST") { + throw new HostLocalCreateJournalEntryExistsError( + "Host-local create journal is invalid: transaction already exists", + ); + } + throw error; + } + try { + fs.unlinkSync(temporary); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + } else { + fs.renameSync(temporary, target); + } + fs.chmodSync(target, FILE_MODE); + fsyncDirectory(root); + } catch (error) { + operationFailed = true; + operationFailure = error; + } + let cleanupFailure: unknown; + try { + if (descriptor !== null) fs.closeSync(descriptor); + } catch (error) { + cleanupFailure = error; + } + try { + fs.unlinkSync(temporary); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT" && cleanupFailure === undefined) { + cleanupFailure = error; + } + } + if (operationFailed) throw operationFailure; + if (cleanupFailure !== undefined) throw cleanupFailure; +} + +function publishExclusive(root: string, target: string, source: string): boolean { + try { + publish(root, target, source, true); + return true; + } catch (error) { + if (error instanceof HostLocalCreateJournalEntryExistsError) return false; + throw error; + } +} + +function sameExecutionLease( + left: HostLocalCreateJournalExecutionLease, + right: HostLocalCreateJournalExecutionLease, +): boolean { + return serializeExecutionLease(left) === serializeExecutionLease(right); +} + +function removeExactExecutionLease( + root: string, + target: string, + expected: HostLocalCreateJournalExecutionLease, +): void { + const current = readExecutionLease(target); + if (current === null || !sameExecutionLease(current, expected)) { + fail("execution ownership changed"); + } + fs.unlinkSync(target); + fsyncDirectory(root); +} + +function defaultProcessIsAlive(pid: number): boolean { + if (pid === process.pid) return true; + try { + process.kill(pid, 0); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code !== "ESRCH"; + } +} + +export function createHostLocalCreateJournalStore( + stateDirectory: string, + dependencies: HostLocalCreateJournalStoreDependencies = {}, +): HostLocalCreateJournalStore { + const root = path.join(stateDirectory, HOST_LOCAL_CREATE_JOURNAL_DIRECTORY); + const executionLeasePath = path.join(root, EXECUTION_LEASE_FILE); + const executionRecoveryPath = path.join(root, EXECUTION_RECOVERY_FILE); + const ownerPid = exactPid(dependencies.ownerPid ?? process.pid); + const processIsAlive = dependencies.processIsAlive ?? defaultProcessIsAlive; + const load = (transactionId: string) => { + requireDirectory(root); + return readRecord(recordPath(root, transactionId)); + }; + const replace = ( + transactionId: string, + update: (current: HostLocalCreateJournalRecord) => HostLocalCreateJournalRecord, + ) => { + const current = load(transactionId); + if (!current) fail("transaction is missing"); + const next = normalizeHostLocalCreateJournalRecord(update(current)); + publish( + root, + recordPath(root, transactionId), + serializeHostLocalCreateJournalRecord(next), + false, + ); + return next; + }; + const store: HostLocalCreateJournalStore = { + load, + list() { + requireDirectory(root); + return Object.freeze( + fs + .readdirSync(root) + .filter((entry) => SHA256.test(entry.replace(/\.json$/u, "")) && entry.endsWith(".json")) + .sort() + .flatMap((entry) => { + const record = readRecord(path.join(root, entry)); + return record === null ? [] : [record]; + }), + ); + }, + create(record) { + requireDirectory(root); + const normalized = normalizeHostLocalCreateJournalRecord(record); + if (normalized.phase !== "prepared") fail("new transaction must be prepared"); + publish( + root, + recordPath(root, normalized.transactionId), + serializeHostLocalCreateJournalRecord(normalized), + true, + ); + return normalized; + }, + recordCreating(transactionId, createIntentUnixMs) { + return replace(transactionId, (current) => { + if (current.phase !== "prepared") fail("only a prepared transaction can record intent"); + return { ...current, phase: "creating", createIntentUnixMs }; + }); + }, + recordCreated(transactionId, runtimeId) { + return replace(transactionId, (current) => { + if (current.phase !== "creating") fail("only a creating transaction can record creation"); + return { ...current, phase: "created", runtimeId }; + }); + }, + recordStarted(transactionId) { + return replace(transactionId, (current) => { + if (current.phase !== "created") fail("only a created transaction can record start"); + return { ...current, phase: "started" }; + }); + }, + finalize(transactionId, receiptSha256) { + return replace(transactionId, (current) => { + if (current.phase !== "started") fail("only a started transaction can finalize"); + return { ...current, phase: "finalized", receiptSha256 }; + }); + }, + retire(transactionId) { + requireDirectory(root); + try { + fs.unlinkSync(recordPath(root, transactionId)); + fsyncDirectory(root); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + }, + acquireExecution(transactionId) { + requireDirectory(root); + const lease = normalizeExecutionLease({ + schemaVersion: HOST_LOCAL_CREATE_JOURNAL_SCHEMA_VERSION, + transactionId, + ownerId: (dependencies.createOwnerId ?? randomUUID)(), + ownerPid, + }); + for (let attempt = 0; attempt < 3; attempt += 1) { + const recovery = readExecutionLease(executionRecoveryPath); + if (recovery !== null) { + if (processIsAlive(recovery.ownerPid)) { + fail("execution recovery is already owned by a live process"); + } + removeExactExecutionLease(root, executionRecoveryPath, recovery); + continue; + } + if (publishExclusive(root, executionLeasePath, serializeExecutionLease(lease))) { + if (readExecutionLease(executionRecoveryPath) === null) return lease; + removeExactExecutionLease(root, executionLeasePath, lease); + continue; + } + const existing = readExecutionLease(executionLeasePath); + if (existing === null) continue; + if (processIsAlive(existing.ownerPid)) { + fail("execution is already owned by a live process"); + } + if (!publishExclusive(root, executionRecoveryPath, serializeExecutionLease(lease))) { + fail("execution recovery is already in progress"); + } + try { + const abandoned = readExecutionLease(executionLeasePath); + if (abandoned !== null && processIsAlive(abandoned.ownerPid)) { + fail("execution owner became live during recovery"); + } + if (abandoned !== null) { + removeExactExecutionLease(root, executionLeasePath, abandoned); + } + } finally { + const marker = readExecutionLease(executionRecoveryPath); + if (marker !== null && sameExecutionLease(marker, lease)) { + removeExactExecutionLease(root, executionRecoveryPath, lease); + } + } + } + fail("execution ownership could not be acquired"); + }, + assertExecution(lease) { + requireDirectory(root); + const expected = normalizeExecutionLease(lease); + const current = readExecutionLease(executionLeasePath); + if (current === null || !sameExecutionLease(current, expected)) { + fail("execution ownership changed"); + } + }, + releaseExecution(lease) { + requireDirectory(root); + removeExactExecutionLease(root, executionLeasePath, normalizeExecutionLease(lease)); + }, + }; + return Object.freeze(store); +} diff --git a/src/lib/onboard/runtime-provider/host-local-inference.test.ts b/src/lib/onboard/runtime-provider/host-local-inference.test.ts new file mode 100644 index 00000000000..b2b602566bb --- /dev/null +++ b/src/lib/onboard/runtime-provider/host-local-inference.test.ts @@ -0,0 +1,226 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + type HostLocalInferenceReceipt, + normalizeHostLocalInferenceReceipt, + parseHostLocalInferenceReceipt, + serializeHostLocalInferenceReceipt, +} from "./host-local-inference"; + +const ENGINE_AUTHORITY = { + schemaVersion: 1, + providerId: "mxc", + operation: "host-local-inference", + engineId: "mxc", + authorityId: `mxc-endpoint:${"a".repeat(64)}`, + bindingSha256: "b".repeat(64), +} as const; + +function receipt( + service: "ollama" | "nim" | "vllm" | "llama-cpp" = "vllm", +): HostLocalInferenceReceipt { + return { + schemaVersion: 1, + providerId: "mxc", + service, + engineAuthority: ENGINE_AUTHORITY, + endpoint: { + host: "host.openshell.internal", + port: service === "ollama" ? 11435 : 8000, + networkName: "openshell", + }, + runtime: + service === "ollama" + ? { + kind: "host", + probeImageRef: `quay.io/curl/curl@sha256:${"d".repeat(64)}`, + } + : { + kind: "container", + runtimeId: "mxc-runtime:alpha", + name: `nemoclaw-${service}-alpha`, + imageRef: `nvcr.io/nvidia/${service}@sha256:${"c".repeat(64)}`, + probeImageRef: `quay.io/curl/curl@sha256:${"e".repeat(64)}`, + specSha256: "d".repeat(64), + ...(service === "llama-cpp" + ? { + model: { + planDigest: `sha256:${"f".repeat(64)}`, + recipeId: "llama-cpp.nemotron.spark.v1", + generation: "9".repeat(64), + digest: `sha256:${"a".repeat(64)}`, + sizeBytes: 22_833_947_424, + }, + } + : {}), + gpu: + service === "llama-cpp" + ? { vendor: "nvidia", count: 1 } + : { vendor: "nvidia", devices: ["nvidia.com/gpu=all"] }, + }, + }; +} + +function containerRuntime(value: HostLocalInferenceReceipt) { + expect(value.runtime.kind).toBe("container"); + return value.runtime as Extract; +} + +describe("host-local inference receipt contract", () => { + it.each([ + "ollama", + "nim", + "vllm", + "llama-cpp", + ] as const)("round-trips %s authority without provider-specific state", (service) => { + const expected = normalizeHostLocalInferenceReceipt(receipt(service)); + const serialized = serializeHostLocalInferenceReceipt(expected); + + expect(parseHostLocalInferenceReceipt(serialized)).toEqual(expected); + expect(expected.providerId).toBe("mxc"); + expect(Object.isFrozen(expected)).toBe(true); + expect(Object.isFrozen(expected.endpoint)).toBe(true); + expect(Object.isFrozen(expected.runtime)).toBe(true); + }); + + it("requires declarative model authority only for llama.cpp receipts (#8395)", () => { + const llamaCpp = receipt("llama-cpp"); + const llamaRuntime = containerRuntime(llamaCpp); + expect(() => + normalizeHostLocalInferenceReceipt({ + ...llamaCpp, + runtime: { ...llamaRuntime, model: undefined }, + }), + ).toThrow("model authority must be an object"); + expect(() => + normalizeHostLocalInferenceReceipt({ + ...llamaCpp, + runtime: { ...llamaRuntime, gpu: { vendor: "nvidia", count: 2 } }, + }), + ).toThrow("exactly one GPU"); + expect(() => + normalizeHostLocalInferenceReceipt({ + ...llamaCpp, + runtime: { + ...llamaRuntime, + gpu: { vendor: "nvidia", devices: ["nvidia.com/gpu=all"] }, + }, + }), + ).toThrow("GPU authority schema is unsupported"); + + for (const service of ["nim", "vllm"] as const) { + const managed = receipt(service); + expect(() => + normalizeHostLocalInferenceReceipt({ + ...managed, + runtime: { + ...managed.runtime, + model: { + planDigest: `sha256:${"f".repeat(64)}`, + recipeId: "unexpected", + generation: "9".repeat(64), + digest: `sha256:${"a".repeat(64)}`, + sizeBytes: 1, + }, + }, + }), + ).toThrow("container authority schema is unsupported"); + } + }); + + it("rejects malformed llama.cpp model identity without exposing executor state (#8395)", () => { + const llamaCpp = receipt("llama-cpp"); + const llamaRuntime = containerRuntime(llamaCpp); + expect(() => + normalizeHostLocalInferenceReceipt({ + ...llamaCpp, + runtime: { + ...llamaRuntime, + model: { ...llamaRuntime.model, digest: "mutable" }, + }, + }), + ).toThrow("model digest is malformed"); + expect(JSON.stringify(normalizeHostLocalInferenceReceipt(llamaCpp))).not.toContain("hostPath"); + }); + + it("rejects provider, operation, endpoint, image, and device authority drift", () => { + const base = receipt(); + expect(() => normalizeHostLocalInferenceReceipt({ ...base, providerId: "other" })).toThrow( + "does not match engine authority", + ); + expect(() => + normalizeHostLocalInferenceReceipt({ + ...base, + engineAuthority: { ...ENGINE_AUTHORITY, operation: "sandbox-lifecycle" }, + }), + ).toThrow("wrong operation scope"); + expect(() => + normalizeHostLocalInferenceReceipt({ + ...base, + endpoint: { ...base.endpoint, port: 0 }, + }), + ).toThrow("endpoint port is malformed"); + expect(() => + normalizeHostLocalInferenceReceipt({ + ...base, + runtime: { ...base.runtime, imageRef: "nvcr.io/nvidia/vllm:latest" }, + }), + ).toThrow("runtime image reference is malformed"); + expect(() => + normalizeHostLocalInferenceReceipt({ + ...base, + runtime: { + ...base.runtime, + gpu: { vendor: "nvidia", devices: ["/dev/nvidia0"] }, + }, + }), + ).toThrow("GPU device is malformed"); + }); + + it("rejects a host runtime for managed services and container runtime for Ollama", () => { + expect(() => + normalizeHostLocalInferenceReceipt({ + ...receipt("nim"), + runtime: { + kind: "host", + probeImageRef: `quay.io/curl/curl@sha256:${"d".repeat(64)}`, + }, + }), + ).toThrow("only Ollama"); + expect(() => + normalizeHostLocalInferenceReceipt({ + ...receipt("ollama"), + runtime: receipt("vllm").runtime, + }), + ).toThrow("Ollama must use host-process authority"); + }); + + it("rejects mutable probe images and malformed managed specification digests", () => { + const ollama = receipt("ollama"); + expect(() => + normalizeHostLocalInferenceReceipt({ + ...ollama, + runtime: { kind: "host", probeImageRef: "curlimages/curl:latest" }, + }), + ).toThrow("runtime image reference is malformed"); + + const vllm = receipt("vllm"); + expect(() => + normalizeHostLocalInferenceReceipt({ + ...vllm, + runtime: { ...vllm.runtime, specSha256: "mutable" }, + }), + ).toThrow("runtime specification digest is malformed"); + }); + + it("rejects extensions and noncanonical serialized receipts", () => { + const base = receipt(); + expect(() => normalizeHostLocalInferenceReceipt({ ...base, extra: true })).toThrow( + "receipt schema is unsupported", + ); + expect(() => parseHostLocalInferenceReceipt(JSON.stringify(base))).toThrow("not canonical"); + }); +}); diff --git a/src/lib/onboard/runtime-provider/host-local-inference.ts b/src/lib/onboard/runtime-provider/host-local-inference.ts new file mode 100644 index 00000000000..b9a492dca2e --- /dev/null +++ b/src/lib/onboard/runtime-provider/host-local-inference.ts @@ -0,0 +1,364 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + normalizePersistedEngineAuthority, + type PersistedEngineAuthority, +} from "./persisted-engine-authority"; + +export const HOST_LOCAL_INFERENCE_RECEIPT_SCHEMA_VERSION = 1 as const; + +export type HostLocalInferenceService = "ollama" | "nim" | "vllm" | "llama-cpp"; + +export interface HostLocalInferenceModelAuthority { + /** Digest of the complete YAML-compiled acquisition plan. */ + readonly planDigest: string; + readonly recipeId: string; + /** Provider-owned create transaction generation; never a filesystem or secret identity. */ + readonly generation: string; + readonly digest: string; + readonly sizeBytes: number; +} + +export interface HostLocalInferenceEndpointInput { + readonly networkName: string; + readonly hostPort: number; + readonly probeImageRef: string; +} + +export interface HostLocalInferenceMount { + readonly source: string; + readonly target: string; + readonly readOnly?: boolean; +} + +export interface HostLocalManagedInferenceInput extends HostLocalInferenceEndpointInput { + readonly service: "nim" | "vllm"; + readonly containerName: string; + readonly containerPort: number; + readonly imageRef: string; + readonly gpuDevices: readonly string[]; + /** Environment variable names forwarded from the current process; values are never persisted. */ + readonly environment?: readonly string[]; + readonly mounts?: readonly HostLocalInferenceMount[]; + readonly sharedMemory?: string; + readonly ipc?: "host" | "private"; + readonly command?: readonly string[]; +} + +export interface HostLocalInferenceEndpointAuthority { + readonly host: string; + readonly port: number; + readonly networkName: string; +} + +export type HostLocalInferenceRuntimeAuthority = + | { + readonly kind: "host"; + /** Immutable utility image used to prove endpoint reachability from the runtime network. */ + readonly probeImageRef: string; + } + | { + readonly kind: "container"; + readonly runtimeId: string; + readonly name: string; + readonly imageRef: string; + /** Immutable utility image used to re-prove service readiness from the runtime network. */ + readonly probeImageRef: string; + /** Secret-free digest of the complete provider-owned container specification. */ + readonly specSha256: string; + /** + * Declarative model identity for runtimes that bind one verified local + * artifact. Host paths and executor-only filesystem identity never enter + * durable provider state. + */ + readonly model?: HostLocalInferenceModelAuthority; + readonly gpu: + | { readonly vendor: "nvidia"; readonly devices: readonly string[] } + | { readonly vendor: "nvidia"; readonly count: 1 }; + }; + +/** + * Secret-free durable proof for one host-local inference route. The injected + * provider owns command reconstruction; central consumers retain only this + * normalized endpoint and runtime authority. + */ +export interface HostLocalInferenceReceipt { + readonly schemaVersion: typeof HOST_LOCAL_INFERENCE_RECEIPT_SCHEMA_VERSION; + readonly providerId: string; + readonly service: HostLocalInferenceService; + readonly engineAuthority: PersistedEngineAuthority; + readonly endpoint: HostLocalInferenceEndpointAuthority; + readonly runtime: HostLocalInferenceRuntimeAuthority; +} + +export interface HostLocalManagedInferenceInspection { + readonly running: boolean; + readonly receipt: HostLocalInferenceReceipt; +} + +export type HostLocalInferenceDestroyResult = + | { + readonly status: "retained"; + readonly reason: "host-process"; + readonly receipt: HostLocalInferenceReceipt; + } + | { + readonly status: "removed" | "already-absent"; + readonly receipt: HostLocalInferenceReceipt; + }; + +export interface HostLocalInferenceRouteAuthority { + readonly schemaVersion: 1; + readonly providerId: string; + readonly service: "ollama"; + readonly authorityId: string; + /** Digest of the provider-owned route and probe authority, excluding secrets. */ + readonly receiptSha256: string; +} + +/** + * Provider-owned protected storage for host-process route identity. A runtime + * must inject a durable implementation before production activation; tests + * use a write-once memory implementation. + */ +export interface HostLocalInferenceRouteAuthorityStore { + readonly load: (service: "ollama") => HostLocalInferenceRouteAuthority | null; + readonly record: ( + authority: HostLocalInferenceRouteAuthority, + ) => HostLocalInferenceRouteAuthority; +} + +export interface HostLocalInferenceRuntime { + readonly providerId: string; + /** Exact opaque endpoint identity shared with the operation-scoped engine. */ + readonly authorityId: string; + readonly services: readonly HostLocalInferenceService[]; + translateContainerArgs(args: readonly string[]): readonly string[]; + qualifyOllama(input: HostLocalInferenceEndpointInput): HostLocalInferenceReceipt; + startManaged(input: HostLocalManagedInferenceInput): HostLocalInferenceReceipt; + inspectManaged(receipt: HostLocalInferenceReceipt): HostLocalManagedInferenceInspection; + stopManaged(receipt: HostLocalInferenceReceipt): HostLocalManagedInferenceInspection; + /** + * Re-prove the same out-of-sandbox service before carrying it across a + * lifecycle boundary. Every invocation must perform a fresh provider-native + * identity inspection and network health probe; cached or receipt-only + * validation does not satisfy this contract. + */ + preserveForRebuild(receipt: HostLocalInferenceReceipt): HostLocalInferenceReceipt; + /** Prove exact ownership for teardown without requiring the service to be healthy. */ + prepareDestroy(receipt: HostLocalInferenceReceipt): HostLocalInferenceReceipt; + /** + * Retire only the exact provider-owned runtime; host processes remain + * externally owned. Managed cleanup must remain idempotent across retries and + * revalidate exact runtime authority before each deletion so a retained + * ownership journal can resume teardown after a process crash or provider + * failure. + */ + destroy(receipt: HostLocalInferenceReceipt): HostLocalInferenceDestroyResult; +} + +const PROVIDER_ID = /^[a-z][a-z0-9-]{0,62}$/u; +const SAFE_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u; +const SAFE_HOST = /^[A-Za-z0-9](?:[A-Za-z0-9.-]{0,251}[A-Za-z0-9])?$/u; +const RUNTIME_ID = /^[A-Za-z0-9][A-Za-z0-9._:/=+-]{0,511}$/u; +const OCI_DIGEST_REFERENCE = + /^(?:[A-Za-z0-9._-]+(?::[0-9]+)?\/)*(?:[A-Za-z0-9._-]+)@sha256:[a-f0-9]{64}$/u; +const CDI_DEVICE = /^nvidia\.com\/gpu=[A-Za-z0-9][A-Za-z0-9_.:/-]{0,255}$/u; +const SHA256 = /^[a-f0-9]{64}$/u; +const SHA256_DIGEST = /^sha256:[a-f0-9]{64}$/u; +const RECIPE_ID = /^[a-z0-9][a-z0-9._-]{0,159}$/u; +const SERVICES = new Set(["ollama", "nim", "vllm", "llama-cpp"]); +const CONTROL_CHARACTERS = /[\u0000-\u001f\u007f-\u009f]/u; +const MAX_SERIALIZED_BYTES = 32 * 1024; + +function fail(message: string): never { + throw new Error(`Host-local inference receipt is invalid: ${message}`); +} + +function exactRecord(value: unknown, label: string): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + fail(`${label} must be an object`); + } + return value as Record; +} + +function exactKeys(value: Record, keys: readonly string[], label: string): void { + if (Object.keys(value).sort().join(",") !== [...keys].sort().join(",")) { + fail(`${label} schema is unsupported`); + } +} + +function exactText(value: unknown, pattern: RegExp, label: string): string { + if ( + typeof value !== "string" || + value !== value.trim() || + CONTROL_CHARACTERS.test(value) || + !pattern.test(value) + ) { + fail(`${label} is malformed`); + } + return value; +} + +function exactPort(value: unknown, label: string): number { + if (!Number.isSafeInteger(value) || Number(value) < 1 || Number(value) > 65_535) { + fail(`${label} is malformed`); + } + return Number(value); +} + +export function normalizeHostLocalInferenceImageRef(value: unknown): string { + return exactText(value, OCI_DIGEST_REFERENCE, "runtime image reference"); +} + +function normalizeEndpoint(value: unknown): HostLocalInferenceEndpointAuthority { + const endpoint = exactRecord(value, "endpoint authority"); + exactKeys(endpoint, ["host", "networkName", "port"], "endpoint authority"); + return Object.freeze({ + host: exactText(endpoint.host, SAFE_HOST, "endpoint host"), + port: exactPort(endpoint.port, "endpoint port"), + networkName: exactText(endpoint.networkName, SAFE_NAME, "endpoint network"), + }); +} + +function normalizeRuntime( + service: HostLocalInferenceService, + value: unknown, +): HostLocalInferenceRuntimeAuthority { + const runtime = exactRecord(value, "runtime authority"); + if (runtime.kind === "host") { + exactKeys(runtime, ["kind", "probeImageRef"], "host runtime authority"); + if (service !== "ollama") fail("only Ollama may use host-process authority"); + return Object.freeze({ + kind: "host" as const, + probeImageRef: normalizeHostLocalInferenceImageRef(runtime.probeImageRef), + }); + } + if (runtime.kind !== "container") fail("runtime kind is unsupported"); + exactKeys( + runtime, + service === "llama-cpp" + ? ["gpu", "imageRef", "kind", "model", "name", "probeImageRef", "runtimeId", "specSha256"] + : ["gpu", "imageRef", "kind", "name", "probeImageRef", "runtimeId", "specSha256"], + "container authority", + ); + if (service === "ollama") fail("Ollama must use host-process authority"); + const gpu = exactRecord(runtime.gpu, "GPU authority"); + exactKeys( + gpu, + service === "llama-cpp" ? ["count", "vendor"] : ["devices", "vendor"], + "GPU authority", + ); + if (gpu.vendor !== "nvidia") fail("GPU authority must identify NVIDIA devices"); + let normalizedGpu: + | { readonly vendor: "nvidia"; readonly devices: readonly string[] } + | { readonly vendor: "nvidia"; readonly count: 1 }; + if (service === "llama-cpp") { + if (gpu.count !== 1) fail("llama.cpp GPU authority must identify exactly one GPU"); + normalizedGpu = Object.freeze({ vendor: "nvidia" as const, count: 1 as const }); + } else { + if (!Array.isArray(gpu.devices) || gpu.devices.length === 0) { + fail("GPU authority must identify NVIDIA devices"); + } + const devices = gpu.devices.map((device) => exactText(device, CDI_DEVICE, "GPU device")); + if (new Set(devices).size !== devices.length) fail("GPU devices must be unique"); + normalizedGpu = Object.freeze({ + vendor: "nvidia" as const, + devices: Object.freeze(devices), + }); + } + let model: HostLocalInferenceModelAuthority | undefined; + if (service === "llama-cpp") { + const source = exactRecord(runtime.model, "model authority"); + exactKeys( + source, + ["digest", "generation", "planDigest", "recipeId", "sizeBytes"], + "model authority", + ); + if (!Number.isSafeInteger(source.sizeBytes) || Number(source.sizeBytes) < 1) { + fail("model size is malformed"); + } + model = Object.freeze({ + planDigest: exactText(source.planDigest, SHA256_DIGEST, "model plan digest"), + recipeId: exactText(source.recipeId, RECIPE_ID, "model recipe identity"), + generation: exactText(source.generation, SHA256, "model lifecycle generation"), + digest: exactText(source.digest, SHA256_DIGEST, "model digest"), + sizeBytes: Number(source.sizeBytes), + }); + } + return Object.freeze({ + kind: "container" as const, + runtimeId: exactText(runtime.runtimeId, RUNTIME_ID, "runtime identity"), + name: exactText(runtime.name, SAFE_NAME, "runtime name"), + imageRef: normalizeHostLocalInferenceImageRef(runtime.imageRef), + probeImageRef: normalizeHostLocalInferenceImageRef(runtime.probeImageRef), + specSha256: exactText(runtime.specSha256, SHA256, "runtime specification digest"), + ...(model ? { model } : {}), + gpu: normalizedGpu, + }); +} + +export function normalizeHostLocalInferenceReceipt(value: unknown): HostLocalInferenceReceipt { + const receipt = exactRecord(value, "receipt"); + exactKeys( + receipt, + ["endpoint", "engineAuthority", "providerId", "runtime", "schemaVersion", "service"], + "receipt", + ); + if (receipt.schemaVersion !== HOST_LOCAL_INFERENCE_RECEIPT_SCHEMA_VERSION) { + fail("schema version is unsupported"); + } + if ( + typeof receipt.service !== "string" || + !SERVICES.has(receipt.service as HostLocalInferenceService) + ) { + fail("service is unsupported"); + } + const service = receipt.service as HostLocalInferenceService; + const engineAuthority = normalizePersistedEngineAuthority(receipt.engineAuthority); + if (engineAuthority.operation !== "host-local-inference") { + fail("engine authority has the wrong operation scope"); + } + const providerId = exactText(receipt.providerId, PROVIDER_ID, "provider identity"); + if (engineAuthority.providerId !== providerId) { + fail("provider identity does not match engine authority"); + } + return Object.freeze({ + schemaVersion: HOST_LOCAL_INFERENCE_RECEIPT_SCHEMA_VERSION, + providerId, + service, + engineAuthority, + endpoint: normalizeEndpoint(receipt.endpoint), + runtime: normalizeRuntime(service, receipt.runtime), + }); +} + +export function serializeHostLocalInferenceReceipt(receipt: HostLocalInferenceReceipt): string { + const serialized = `${JSON.stringify(normalizeHostLocalInferenceReceipt(receipt))}\n`; + if (Buffer.byteLength(serialized, "utf8") > MAX_SERIALIZED_BYTES) { + fail("serialized receipt exceeds its bounded transport"); + } + return serialized; +} + +export function parseHostLocalInferenceReceipt(serialized: string): HostLocalInferenceReceipt { + if ( + serialized.length === 0 || + serialized.includes("\0") || + Buffer.byteLength(serialized, "utf8") > MAX_SERIALIZED_BYTES + ) { + fail("serialized receipt is empty or too large"); + } + let parsed: unknown; + try { + parsed = JSON.parse(serialized); + } catch { + fail("serialized receipt is not valid JSON"); + } + const receipt = normalizeHostLocalInferenceReceipt(parsed); + if (serializeHostLocalInferenceReceipt(receipt) !== serialized) { + fail("serialized receipt is not canonical"); + } + return receipt; +} diff --git a/src/lib/onboard/runtime-provider/persisted-engine-authority.test.ts b/src/lib/onboard/runtime-provider/persisted-engine-authority.test.ts new file mode 100644 index 00000000000..e63e30d47c4 --- /dev/null +++ b/src/lib/onboard/runtime-provider/persisted-engine-authority.test.ts @@ -0,0 +1,228 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { + type ContainerEngineOperationScope, + createContainerEngineCommand, +} from "../../adapters/container-engine"; +import { + createFilePersistedEngineAuthorityStore, + createPersistedEngineAuthority, + normalizePersistedEngineAuthority, + PERSISTED_ENGINE_AUTHORITY_DIRECTORY, + parsePersistedEngineAuthority, + persistedEngineAuthorityPath, + requirePersistedEngineAuthority, + serializePersistedEngineAuthority, +} from "./persisted-engine-authority"; + +const BINDING_SHA256 = "1".repeat(64); +const roots: string[] = []; + +function temporaryRoot(): string { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-engine-authority-")); + roots.push(root); + return root; +} + +function engine( + operation: ContainerEngineOperationScope = "sandbox-lifecycle", + authorityId = `mxc-endpoint:${"2".repeat(64)}`, +) { + return createContainerEngineCommand({ + operation, + engineId: "mxc", + displayName: "MXC test engine", + authorityId, + executable: "mxcctl", + endpointArgs: ["--endpoint", "unix:///run/mxc/runtime.sock"], + capture: () => ({ status: 0, stdout: "", stderr: "" }), + }); +} + +afterEach(() => { + for (const root of roots.splice(0)) fs.rmSync(root, { force: true, recursive: true }); +}); + +describe("persisted engine authority", () => { + it("round-trips an MXC-style provider without a Podman-specific switch", () => { + const qualified = engine(); + const authority = createPersistedEngineAuthority("mxc", qualified, BINDING_SHA256); + const serialized = serializePersistedEngineAuthority(authority); + + expect(parsePersistedEngineAuthority(serialized)).toEqual(authority); + expect(requirePersistedEngineAuthority(authority, "mxc", qualified, BINDING_SHA256)).toEqual( + authority, + ); + expect(authority).toEqual({ + schemaVersion: 1, + providerId: "mxc", + operation: "sandbox-lifecycle", + engineId: "mxc", + authorityId: `mxc-endpoint:${"2".repeat(64)}`, + bindingSha256: BINDING_SHA256, + }); + }); + + it("writes one private canonical record and accepts an exact retry", () => { + const root = temporaryRoot(); + const authority = createPersistedEngineAuthority("mxc", engine(), BINDING_SHA256); + const store = createFilePersistedEngineAuthorityStore(root); + + expect(store.record(authority)).toEqual(authority); + expect(store.record(authority)).toEqual(authority); + expect(store.load("sandbox-lifecycle")).toEqual(authority); + + const directory = path.join(root, PERSISTED_ENGINE_AUTHORITY_DIRECTORY); + const target = persistedEngineAuthorityPath(root, "sandbox-lifecycle"); + expect(fs.statSync(directory).mode & 0o777).toBe(0o700); + const descriptor = fs.openSync(target, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + try { + expect(fs.fstatSync(descriptor).mode & 0o777).toBe(0o600); + expect(fs.readFileSync(descriptor, "utf8")).toBe( + serializePersistedEngineAuthority(authority), + ); + } finally { + fs.closeSync(descriptor); + } + }); + + it("persists a host-local inference engine independently of lifecycle authority", () => { + const store = createFilePersistedEngineAuthorityStore(temporaryRoot()); + const inference = createPersistedEngineAuthority( + "mxc", + engine("host-local-inference"), + BINDING_SHA256, + ); + const lifecycle = createPersistedEngineAuthority( + "mxc", + engine("sandbox-lifecycle"), + BINDING_SHA256, + ); + + expect(store.record(inference)).toEqual(inference); + expect(store.record(lifecycle)).toEqual(lifecycle); + expect(store.load("host-local-inference")).toEqual(inference); + expect(store.load("sandbox-lifecycle")).toEqual(lifecycle); + }); + + it.each([ + { + label: "provider", + providerId: "other", + operation: "sandbox-lifecycle" as const, + engineId: "mxc", + authorityId: `mxc-endpoint:${"2".repeat(64)}`, + bindingSha256: BINDING_SHA256, + message: "provider does not match", + }, + { + label: "operation", + providerId: "mxc", + operation: "workload-cleanup" as const, + engineId: "mxc", + authorityId: `mxc-endpoint:${"2".repeat(64)}`, + bindingSha256: BINDING_SHA256, + message: "operation does not match", + }, + { + label: "engine", + providerId: "mxc", + operation: "sandbox-lifecycle" as const, + engineId: "other", + authorityId: `mxc-endpoint:${"2".repeat(64)}`, + bindingSha256: BINDING_SHA256, + message: "identity does not match", + }, + { + label: "endpoint authority", + providerId: "mxc", + operation: "sandbox-lifecycle" as const, + engineId: "mxc", + authorityId: `mxc-endpoint:${"3".repeat(64)}`, + bindingSha256: BINDING_SHA256, + message: "endpoint does not match", + }, + { + label: "binding", + providerId: "mxc", + operation: "sandbox-lifecycle" as const, + engineId: "mxc", + authorityId: `mxc-endpoint:${"2".repeat(64)}`, + bindingSha256: "4".repeat(64), + message: "binding does not match", + }, + ])("fails closed when qualified $label differs", (candidate) => { + const persisted = createPersistedEngineAuthority("mxc", engine(), BINDING_SHA256); + const qualified = createContainerEngineCommand({ + operation: candidate.operation, + engineId: candidate.engineId, + displayName: "candidate", + authorityId: candidate.authorityId, + executable: "mxcctl", + capture: () => ({ status: 0, stdout: "", stderr: "" }), + }); + + expect(() => + requirePersistedEngineAuthority( + persisted, + candidate.providerId, + qualified, + candidate.bindingSha256, + ), + ).toThrow(candidate.message); + }); + + it("rejects malformed, extended, and noncanonical records", () => { + const authority = createPersistedEngineAuthority("mxc", engine(), BINDING_SHA256); + expect(() => normalizePersistedEngineAuthority({ ...authority, extra: true })).toThrow( + "schema is unsupported", + ); + expect(() => + normalizePersistedEngineAuthority({ ...authority, authorityId: "not-an-authority" }), + ).toThrow("endpoint authority identity is malformed"); + expect(() => parsePersistedEngineAuthority(JSON.stringify(authority))).toThrow("not canonical"); + expect(() => + parsePersistedEngineAuthority(`{\"value\":\"${"x".repeat(17 * 1024)}\"}\n`), + ).toThrow("too large"); + }); + + it("rejects a conflicting record for the same operation", () => { + const store = createFilePersistedEngineAuthorityStore(temporaryRoot()); + const authority = createPersistedEngineAuthority("mxc", engine(), BINDING_SHA256); + store.record(authority); + + expect(() => + store.record( + createPersistedEngineAuthority( + "mxc", + engine("sandbox-lifecycle", `mxc-endpoint:${"5".repeat(64)}`), + BINDING_SHA256, + ), + ), + ).toThrow("already exists for 'sandbox-lifecycle'"); + expect(store.load("sandbox-lifecycle")).toEqual(authority); + }); + + it("rejects a symlink or shared-permission authority file", () => { + const root = temporaryRoot(); + const store = createFilePersistedEngineAuthorityStore(root); + const target = persistedEngineAuthorityPath(root, "sandbox-lifecycle"); + const outside = path.join(root, "outside.json"); + const authority = createPersistedEngineAuthority("mxc", engine(), BINDING_SHA256); + fs.writeFileSync(outside, serializePersistedEngineAuthority(authority), { mode: 0o600 }); + fs.symlinkSync(outside, target); + expect(() => store.load("sandbox-lifecycle")).toThrow("must not be a symbolic link"); + + fs.unlinkSync(target); + fs.writeFileSync(target, serializePersistedEngineAuthority(authority), { mode: 0o644 }); + fs.chmodSync(target, 0o644); + expect(() => store.load("sandbox-lifecycle")).toThrow("failed ownership, mode, link, or size"); + }); +}); diff --git a/src/lib/onboard/runtime-provider/persisted-engine-authority.ts b/src/lib/onboard/runtime-provider/persisted-engine-authority.ts new file mode 100644 index 00000000000..e4300d84d45 --- /dev/null +++ b/src/lib/onboard/runtime-provider/persisted-engine-authority.ts @@ -0,0 +1,331 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { randomUUID } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; + +import type { + ContainerEngine, + ContainerEngineOperationScope, +} from "../../adapters/container-engine"; + +export const PERSISTED_ENGINE_AUTHORITY_SCHEMA_VERSION = 1 as const; +export const PERSISTED_ENGINE_AUTHORITY_DIRECTORY = "runtime-provider-authority"; + +const DIRECTORY_MODE = 0o700; +const FILE_MODE = 0o600; +const MAX_RECORD_BYTES = 16 * 1024; +const PROVIDER_ID = /^[a-z][a-z0-9-]{0,62}$/u; +const ENGINE_ID = /^[a-z][a-z0-9-]{0,62}$/u; +const AUTHORITY_ID = /^[a-z][a-z0-9-]{0,62}:[A-Za-z0-9._:-]{1,255}$/u; +const SHA256 = /^[a-f0-9]{64}$/u; +const OPERATIONS = new Set([ + "host-doctor", + "host-local-inference", + "gateway-inspection", + "managed-bootstrap", + "sandbox-lifecycle", + "workload-cleanup", +]); + +/** + * Immutable, provider-neutral identity for one qualified container-engine + * operation. The record carries no executable, endpoint, environment, or + * credential material; those remain owned by the provider that reconstructs + * and qualifies the injected ContainerEngine. + */ +export interface PersistedEngineAuthority { + readonly schemaVersion: typeof PERSISTED_ENGINE_AUTHORITY_SCHEMA_VERSION; + readonly providerId: string; + readonly operation: ContainerEngineOperationScope; + readonly engineId: string; + readonly authorityId: string; + readonly bindingSha256: string; +} + +export interface PersistedEngineAuthorityStore { + readonly load: (operation: ContainerEngineOperationScope) => PersistedEngineAuthority | null; + /** Write once, or prove an existing record is byte-for-byte identical. */ + readonly record: (authority: PersistedEngineAuthority) => PersistedEngineAuthority; +} + +function fail(message: string): never { + throw new Error(`Persisted engine authority is invalid: ${message}`); +} + +function exactIdentity(value: unknown, pattern: RegExp, label: string): string { + if (typeof value !== "string" || !pattern.test(value)) { + fail(`${label} is malformed`); + } + return value; +} + +function exactOperation(value: unknown): ContainerEngineOperationScope { + if (typeof value !== "string" || !OPERATIONS.has(value as ContainerEngineOperationScope)) { + fail("operation is unsupported"); + } + return value as ContainerEngineOperationScope; +} + +export function normalizePersistedEngineAuthority(value: unknown): PersistedEngineAuthority { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + fail("record must be an object"); + } + const record = value as Record; + const expectedKeys = [ + "authorityId", + "bindingSha256", + "engineId", + "operation", + "providerId", + "schemaVersion", + ]; + if ( + Object.keys(record).sort().join(",") !== expectedKeys.join(",") || + record.schemaVersion !== PERSISTED_ENGINE_AUTHORITY_SCHEMA_VERSION + ) { + fail("record schema is unsupported"); + } + return Object.freeze({ + schemaVersion: PERSISTED_ENGINE_AUTHORITY_SCHEMA_VERSION, + providerId: exactIdentity(record.providerId, PROVIDER_ID, "provider identity"), + operation: exactOperation(record.operation), + engineId: exactIdentity(record.engineId, ENGINE_ID, "engine identity"), + authorityId: exactIdentity(record.authorityId, AUTHORITY_ID, "endpoint authority identity"), + bindingSha256: exactIdentity(record.bindingSha256, SHA256, "binding digest"), + }); +} + +export function serializePersistedEngineAuthority(authority: PersistedEngineAuthority): string { + const serialized = `${JSON.stringify(normalizePersistedEngineAuthority(authority))}\n`; + if (Buffer.byteLength(serialized, "utf8") > MAX_RECORD_BYTES) { + fail("serialized record exceeds its bounded transport"); + } + return serialized; +} + +export function parsePersistedEngineAuthority(serialized: string): PersistedEngineAuthority { + if ( + serialized.length === 0 || + serialized.includes("\0") || + Buffer.byteLength(serialized, "utf8") > MAX_RECORD_BYTES + ) { + fail("serialized record is empty or too large"); + } + let parsed: unknown; + try { + parsed = JSON.parse(serialized); + } catch { + fail("serialized record is not valid JSON"); + } + const authority = normalizePersistedEngineAuthority(parsed); + if (serializePersistedEngineAuthority(authority) !== serialized) { + fail("serialized record is not canonical"); + } + return authority; +} + +export function createPersistedEngineAuthority( + providerId: string, + engine: ContainerEngine, + bindingSha256: string, +): PersistedEngineAuthority { + return normalizePersistedEngineAuthority({ + schemaVersion: PERSISTED_ENGINE_AUTHORITY_SCHEMA_VERSION, + providerId, + operation: engine.operation, + engineId: engine.engineId, + authorityId: engine.authorityId, + bindingSha256, + }); +} + +/** + * Reject a freshly provider-qualified engine unless it matches the recorded + * identity. Callers must perform this check before lifecycle or destructive + * mutation. + */ +export function requirePersistedEngineAuthority( + persisted: PersistedEngineAuthority, + providerId: string, + engine: ContainerEngine, + bindingSha256: string, +): PersistedEngineAuthority { + const authority = normalizePersistedEngineAuthority(persisted); + const qualified = createPersistedEngineAuthority(providerId, engine, bindingSha256); + if (authority.providerId !== qualified.providerId) { + throw new Error("Qualified runtime provider does not match persisted engine authority."); + } + if (authority.operation !== qualified.operation) { + throw new Error("Qualified container-engine operation does not match persisted authority."); + } + if (authority.engineId !== qualified.engineId) { + throw new Error("Qualified container-engine identity does not match persisted authority."); + } + if (authority.authorityId !== qualified.authorityId) { + throw new Error("Qualified container endpoint does not match persisted authority."); + } + if (authority.bindingSha256 !== qualified.bindingSha256) { + throw new Error("Qualified runtime binding does not match persisted engine authority."); + } + return authority; +} + +function operationFileName(operation: ContainerEngineOperationScope): string { + return `${exactOperation(operation)}.json`; +} + +export function persistedEngineAuthorityPath( + stateDir: string, + operation: ContainerEngineOperationScope, +): string { + return path.join(stateDir, PERSISTED_ENGINE_AUTHORITY_DIRECTORY, operationFileName(operation)); +} + +function currentUid(fallback: number | bigint): bigint { + return BigInt(typeof process.getuid === "function" ? process.getuid() : fallback); +} + +function requirePrivateDirectory(directory: string): void { + fs.mkdirSync(directory, { recursive: true, mode: DIRECTORY_MODE }); + const metadata = fs.lstatSync(directory); + if ( + !metadata.isDirectory() || + metadata.isSymbolicLink() || + BigInt(metadata.uid) !== currentUid(metadata.uid) || + (metadata.mode & 0o077) !== 0 + ) { + fail("authority directory must be a private real directory owned by the current user"); + } +} + +function sameStableMetadata(left: fs.BigIntStats, right: fs.BigIntStats): boolean { + return ( + left.dev === right.dev && + left.ino === right.ino && + left.mode === right.mode && + left.nlink === right.nlink && + left.uid === right.uid && + left.gid === right.gid && + left.size === right.size && + left.mtimeNs === right.mtimeNs && + left.ctimeNs === right.ctimeNs + ); +} + +function readPrivateRecord(target: string): string | null { + const noFollow = fs.constants.O_NOFOLLOW; + if (typeof noFollow !== "number") { + fail("O_NOFOLLOW is unavailable for persisted authority reads"); + } + const nonblock = typeof fs.constants.O_NONBLOCK === "number" ? fs.constants.O_NONBLOCK : 0; + let descriptor: number; + try { + descriptor = fs.openSync(target, fs.constants.O_RDONLY | noFollow | nonblock); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code === "ENOENT") return null; + if (code === "ELOOP") fail("authority file must not be a symbolic link"); + throw error; + } + try { + const before = fs.fstatSync(descriptor, { bigint: true }); + if ( + !before.isFile() || + before.nlink !== 1n || + before.uid !== currentUid(before.uid) || + (before.mode & 0o077n) !== 0n || + before.size <= 0n || + before.size > BigInt(MAX_RECORD_BYTES) + ) { + fail("authority file failed ownership, mode, link, or size checks"); + } + const contents = Buffer.alloc(Number(before.size)); + let offset = 0; + while (offset < contents.length) { + const count = fs.readSync(descriptor, contents, offset, contents.length - offset, offset); + if (count === 0) break; + offset += count; + } + const overflow = Buffer.alloc(1); + const overflowCount = fs.readSync(descriptor, overflow, 0, 1, offset); + const after = fs.fstatSync(descriptor, { bigint: true }); + if (offset !== contents.length || overflowCount !== 0 || !sameStableMetadata(before, after)) { + fail("authority file changed during its stable read"); + } + return contents.toString("utf8"); + } finally { + fs.closeSync(descriptor); + } +} + +function fsyncDirectory(directory: string): void { + const descriptor = fs.openSync(directory, "r"); + try { + fs.fsyncSync(descriptor); + } finally { + fs.closeSync(descriptor); + } +} + +function writeExclusiveRecord(directory: string, target: string, serialized: string): boolean { + const noFollow = fs.constants.O_NOFOLLOW; + if (typeof noFollow !== "number") { + fail("O_NOFOLLOW is unavailable for persisted authority writes"); + } + const temporary = path.join(directory, `.${path.basename(target)}.${randomUUID()}.tmp`); + let descriptor: number | null = null; + try { + descriptor = fs.openSync( + temporary, + fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_WRONLY | noFollow, + FILE_MODE, + ); + fs.writeFileSync(descriptor, serialized, { encoding: "utf8" }); + fs.fsyncSync(descriptor); + fs.closeSync(descriptor); + descriptor = null; + try { + fs.linkSync(temporary, target); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "EEXIST") return false; + throw error; + } + fs.unlinkSync(temporary); + fsyncDirectory(directory); + return true; + } finally { + if (descriptor !== null) fs.closeSync(descriptor); + fs.rmSync(temporary, { force: true }); + } +} + +export function createFilePersistedEngineAuthorityStore( + stateDir: string, +): PersistedEngineAuthorityStore { + const directory = path.join(stateDir, PERSISTED_ENGINE_AUTHORITY_DIRECTORY); + requirePrivateDirectory(directory); + const load = (operation: ContainerEngineOperationScope): PersistedEngineAuthority | null => { + requirePrivateDirectory(directory); + const serialized = readPrivateRecord(path.join(directory, operationFileName(operation))); + return serialized === null ? null : parsePersistedEngineAuthority(serialized); + }; + return Object.freeze({ + load, + record(authority: PersistedEngineAuthority) { + const normalized = normalizePersistedEngineAuthority(authority); + const serialized = serializePersistedEngineAuthority(normalized); + const target = path.join(directory, operationFileName(normalized.operation)); + const existing = load(normalized.operation); + if (existing !== null) { + if (serializePersistedEngineAuthority(existing) === serialized) return existing; + throw new Error(`Persisted engine authority already exists for '${normalized.operation}'.`); + } + if (writeExclusiveRecord(directory, target, serialized)) return normalized; + const raced = load(normalized.operation); + if (raced !== null && serializePersistedEngineAuthority(raced) === serialized) return raced; + throw new Error(`Persisted engine authority already exists for '${normalized.operation}'.`); + }, + }); +} diff --git a/test/runtime-provider-source-shape.test.ts b/test/runtime-provider-source-shape.test.ts index a71f8a8645e..ac2d80b869e 100644 --- a/test/runtime-provider-source-shape.test.ts +++ b/test/runtime-provider-source-shape.test.ts @@ -153,13 +153,39 @@ describe("runtime provider central source boundary", () => { "src/lib/onboard/runtime-provider/access.ts", "src/lib/onboard/runtime-provider/contract.ts", "src/lib/onboard/runtime-provider/current.ts", + "src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.ts", "src/lib/onboard/runtime-provider/docker.ts", + "src/lib/onboard/runtime-provider/host-local-create-journal.ts", + "src/lib/onboard/runtime-provider/host-local-inference.ts", "src/lib/onboard/runtime-provider/mxc.ts", + "src/lib/onboard/runtime-provider/persisted-engine-authority.ts", "src/lib/onboard/runtime-provider/registry.ts", "src/lib/onboard/runtime-provider/snapshot.ts", ]); }); + // source-shape-contract: security -- Production provider composition must keep the dormant llama.cpp controller unreachable until its activation boundary is crash safe + it("keeps Docker llama.cpp lifecycle authority dormant (#8395)", () => { + const docker = read("src/lib/onboard/runtime-provider/docker.ts"); + const adapter = read("src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.ts"); + const productionComposition = trackedPaths(".") + .filter( + (path) => + /\.[cm]?ts$/u.test(path) && + !path.endsWith(".test.ts") && + !path.includes("/test/") && + !path.startsWith("test/") && + path !== "src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.ts", + ) + .map(read) + .join("\n"); + expect(docker).not.toContain("docker-llama-cpp-managed-lifecycle"); + expect(docker).not.toMatch(/operation:\s*["']host-local-inference["']/u); + expect(adapter).toContain("createDockerLlamaCppManagedLifecycle"); + expect(productionComposition).not.toContain("createDockerLlamaCppManagedLifecycle"); + expect(productionComposition).not.toContain("docker-llama-cpp-managed-lifecycle"); + }); + it("inventories every production Dockerfile", () => { expect(dockerfilePaths).toEqual([ "Dockerfile", @@ -233,8 +259,13 @@ describe("runtime provider central source boundary", () => { // source-shape-contract: security -- Registered runtime providers must remain bootstrap-unsupported until their complete transaction implementations are qualified it("keeps registered providers bootstrap-unsupported", () => { const dockerProvider = read("src/lib/onboard/runtime-provider/docker.ts"); + // Neutral contracts may name an operation but cannot activate a provider implementation. const providerImplementationSource = providerPaths - .filter((path) => path !== "src/lib/onboard/runtime-provider/contract.ts") + .filter( + (path) => + path !== "src/lib/onboard/runtime-provider/contract.ts" && + path !== "src/lib/onboard/runtime-provider/persisted-engine-authority.ts", + ) .map(read) .join("\n"); expect(dockerProvider).not.toMatch(