diff --git a/docs/inference/set-up-llama-cpp.mdx b/docs/inference/set-up-llama-cpp.mdx index fe79833d3e3..9add8e5e69d 100644 --- a/docs/inference/set-up-llama-cpp.mdx +++ b/docs/inference/set-up-llama-cpp.mdx @@ -135,7 +135,7 @@ Onboarding performs these actions: - Reuses a verified GGUF from the shared `~/.cache/huggingface/` cache, or acquires the exact file through the existing Hugging Face mechanism when it is absent or invalid. - Verifies the GGUF size and SHA-256 digest before it starts the runtime. - Runs the authenticated container as the current non-root user with one NVIDIA GPU and no CPU fallback. -- Publishes port `8081` on `127.0.0.1` and connects the container to a Docker internal network. +- Starts a private host bridge on `127.0.0.1:8081` and connects the container to a Docker internal network without publishing a container port. - Registers `llama-cpp-local` and routes agent traffic through `https://inference.local/v1`. The runtime cannot download a model and has no egress on its internal Docker network. @@ -147,16 +147,14 @@ Selecting either runtime fails before model acquisition or runtime mutation and ## Verify the Managed Runtime -Run the managed runtime, route, and host-port checks: +Run the managed runtime and route checks: ```bash $$nemoclaw my-assistant status $$nemoclaw my-assistant doctor -docker port nemoclaw-llama-cpp ``` Accept the result when `status` reports `Managed llama.cpp: running`, the inference route is `reachable`, and `doctor` exits with status `0`. -The Docker port output must contain `127.0.0.1:8081`. These checks do not establish agent and model qualification. `status` reports the recipe ID, model digest, image reference, endpoint, and lifecycle state without exposing the API key. @@ -165,6 +163,44 @@ Refer to [CLI Commands](../../reference/commands) for complete command behavior. ## Recover the Managed Runtime +During managed installation, NemoClaw first checks internal runtime readiness and host-loopback health. +It repeats these checks when onboarding resumes. +It then runs the authoritative OpenShell Docker bridge probe on fixed port `8081`. +If that bridge probe cannot connect or times out after the earlier checks pass, onboarding stops without changing UFW. +When the bridge reports a valid narrow subnet and a gateway IP address inside that subnet, the error reports these values: + +- The detected Docker network. +- The source subnet. +- The gateway IP address. +- The fixed port `8081`. +- The exact narrow UFW command. + +If the bridge topology is missing or invalid, onboarding fails closed without printing topology or a UFW command. +Inspect the OpenShell Docker bridge configuration before you retry onboarding. + + +The printed command changes UFW. +Confirm that the reported Docker network, source subnet, and gateway IP address belong to the OpenShell Docker bridge before you run it. +The rule allows TCP traffic only from that subnet to port `8081` on that gateway IP address. + + +Run the exact command from the onboarding error. +It has this form: + +```bash +sudo ufw allow from to port 8081 proto tcp +``` + +After you apply the rule, rerun the same onboarding selection. + +If onboarding still reports the bridge failure, or when the managed runtime no longer needs the rule, remove the exact rule that you added: + +```bash +sudo ufw --force delete allow from to port 8081 proto tcp +``` + +Use the same subnet and gateway IP address that appeared in the original command. + If onboarding stops, rerun the same provider and recipe selection. NemoClaw resumes only the exact persisted Docker authority, image, recipe, model digest, network, and runtime identity. It reconciles an unfinished create journal before it starts a new runtime. diff --git a/src/lib/onboard/host-service-reachability.test.ts b/src/lib/onboard/host-service-reachability.test.ts index cc76797cc31..2e6aa3b9bcb 100644 --- a/src/lib/onboard/host-service-reachability.test.ts +++ b/src/lib/onboard/host-service-reachability.test.ts @@ -106,6 +106,27 @@ describe("formatHostServiceUnreachableMessage", () => { expect(msg).toContain("nemoclaw onboard"); }); + it.each([ + "nemohermes", + "nemo-deepagents", + ])("uses the invoked %s CLI in the recovery command (#8712)", (invokedAs) => { + vi.stubEnv("NEMOCLAW_INVOKED_AS", invokedAs); + + const msg = formatHostServiceUnreachableMessage( + { + ok: false, + reason: "tcp_failed", + port: 8081, + networkName: "openshell-docker", + subnet: "172.18.0.0/16", + gatewayIp: "172.18.0.1", + }, + { serviceLabel: "managed llama.cpp server" }, + ); + + expect(msg).toContain(`Then rerun \`${invokedAs} onboard\`.`); + }); + it("falls back to result.port when no explicit port option is given", () => { const msg = formatHostServiceUnreachableMessage( { diff --git a/src/lib/onboard/host-service-reachability.ts b/src/lib/onboard/host-service-reachability.ts index 648a9a33635..f5aacc36e27 100644 --- a/src/lib/onboard/host-service-reachability.ts +++ b/src/lib/onboard/host-service-reachability.ts @@ -18,6 +18,7 @@ */ import { dockerCapture, dockerRun } from "../adapters/docker/run"; +import { cliName } from "./branding"; export const DEFAULT_PROBE_NETWORK = "openshell-docker"; const HOST_INTERNAL_NAME = "host.openshell.internal"; @@ -252,7 +253,7 @@ export function formatHostServiceUnreachableMessage( " A host firewall may be blocking traffic from the OpenShell Docker bridge.", " To allow it:", allowCmd, - " Then re-run `nemoclaw onboard`.", + ` Then rerun \`${cliName()} onboard\`.`, ].join("\n"); } diff --git a/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle-engine.test-support.ts b/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle-engine.test-support.ts new file mode 100644 index 00000000000..b4b07f8b2b1 --- /dev/null +++ b/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle-engine.test-support.ts @@ -0,0 +1,417 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { vi } from "vitest"; + +import type { ContainerEngine } from "../../adapters/container-engine"; +import { buildLlamaCppHostLocalServerArgv } from "../../inference/llama-cpp/host-local-runtime"; +import { + contract, + IMAGE, + invariant, + MODEL_FILENAME, + NETWORK_ID, + RUNTIME_ID, + TRANSACTION_ID, +} from "./docker-llama-cpp-managed-lifecycle.test-support"; +import type { HostLocalCreateJournalRecord } from "./host-local-create-journal"; + +export interface DockerFixture { + readonly engine: ContainerEngine; + readonly capture: ReturnType; + readonly setNetworkId: (value: string) => void; + readonly setNetworkTransactionId: (value: string) => void; + readonly removeNetwork: () => void; + readonly failNetworkCreateUncertain: (networkAppears: boolean) => void; + readonly setCreateStdout: (value: string) => void; + readonly failCreateUncertain: () => void; + readonly failProbe: () => void; + readonly failSandboxBridgeProbe: (result: { + readonly status: number; + readonly stderr?: string; + }) => void; + readonly setOpenShellBridgeSubnet: (value: string) => void; + readonly driftHardening: () => void; + readonly dropTmpfs: () => void; + readonly driftGpuRequest: (driver: string | undefined, count: number) => void; + readonly driftExtraDeviceAuthority: (kind: "cap-add" | "legacy-device") => void; + readonly failInspectWithDaemonError: () => void; + readonly setAbsentNetworkInspectError: (value: string) => void; + readonly onAbsentInspect: (callback: () => void) => void; + readonly onAbsentNetworkInspect: (callback: () => void) => void; + readonly onNetworkCreate: (callback: () => void) => void; + readonly onStart: (callback: () => void) => void; + readonly onProbe: (callback: () => void) => void; + readonly onCreate: (callback: () => void) => void; + readonly setContainerState: (running: boolean, status: string) => void; + readonly seedNetwork: (journal: HostLocalCreateJournalRecord) => void; + readonly seed: (journal: HostLocalCreateJournalRecord, running: boolean) => void; +} + +interface DockerFixturePaths { + readonly apiKeyPath: string; + readonly modelPath: string; + readonly networkName: string; +} + +export function createDockerFixture( + paths: DockerFixturePaths, + configuredHostPort = "", + publishedHostPort?: string, + publishedHostIp = "127.0.0.1", + publishedBindingCount = 0, +): DockerFixture { + const effectivePublishedHostPort = publishedHostPort ?? (configuredHostPort || "49152"); + let networkId = NETWORK_ID; + let networkPresent = false; + let networkTransactionId = TRANSACTION_ID; + let networkCreateUncertain = false; + let uncertainNetworkAppears = false; + let createStdout = `${RUNTIME_ID}\n`; + let createUncertain = false; + let probeFails = false; + let sandboxBridgeProbeFailure: { + readonly status: number; + readonly stderr?: string; + } | null = null; + let openShellBridgeSubnet = "172.29.0.0/16"; + let hardeningDrift = false; + let tmpfs: Record | null = { + "/tmp": "rw,noexec,nosuid,nodev,size=1024,uid=1001,gid=1001,mode=1777", + }; + let gpuDriver: string | undefined = "nvidia"; + let gpuCount = 1; + let capAdd: null | string[] = null; + let legacyDevices: null | object[] = null; + let inspectDaemonError = false; + let absentNetworkInspectError: string | null = null; + let absentInspectHook: (() => void) | undefined; + let absentNetworkInspectHook: (() => void) | undefined; + let networkCreateHook: (() => void) | undefined; + let startHook: (() => void) | undefined; + let probeHook: (() => void) | undefined; + let createHook: (() => void) | undefined; + let startedOnce = false; + let container: + | { + labels: Record; + running: boolean; + status: string; + transactionId: string; + command: string[]; + } + | undefined; + const inspection = () => [ + { + Id: RUNTIME_ID, + Name: "/nemoclaw-llama-cpp", + Config: { + Image: IMAGE, + User: "1001:1001", + Cmd: container?.command ?? [], + Labels: container?.labels ?? {}, + }, + HostConfig: { + NetworkMode: paths.networkName, + RestartPolicy: { Name: "unless-stopped", MaximumRetryCount: 0 }, + PortBindings: configuredHostPort + ? { "8081/tcp": [{ HostIp: "127.0.0.1", HostPort: configuredHostPort }] } + : {}, + ReadonlyRootfs: !hardeningDrift, + CapDrop: ["ALL"], + SecurityOpt: ["no-new-privileges:true"], + Memory: 51_539_607_552, + MemorySwap: 51_539_607_552, + PidsLimit: 256, + DeviceRequests: [ + { + ...(gpuDriver === undefined ? {} : { Driver: gpuDriver }), + Count: gpuCount, + DeviceIDs: null, + Capabilities: [["gpu"]], + Options: {}, + }, + ], + CapAdd: capAdd, + Devices: legacyDevices, + Privileged: false, + Tmpfs: tmpfs, + }, + State: { + Running: container?.running ?? false, + Status: container?.status ?? "created", + }, + NetworkSettings: { + Networks: { + [paths.networkName]: { + NetworkID: startedOnce ? networkId : "", + IPAddress: container?.running ? "172.30.0.2" : "", + }, + }, + Ports: { + "8081/tcp": + startedOnce && publishedBindingCount > 0 + ? Array.from({ length: publishedBindingCount }, () => ({ + HostIp: publishedHostIp, + HostPort: effectivePublishedHostPort, + })) + : null, + }, + }, + Mounts: [ + { + Type: "bind", + Source: paths.modelPath, + Destination: `/models/${MODEL_FILENAME}`, + RW: false, + }, + { + Type: "bind", + Source: paths.apiKeyPath, + Destination: "/run/secrets/llama-cpp-api-key", + RW: false, + }, + ], + }, + ]; + const capture = vi.fn((args: readonly string[]) => { + const unexpected = `unexpected Docker command: ${args.join(" ")}`; + switch (args[0]) { + case "network": + switch (args[1]) { + case "inspect": + switch (args[2]) { + case "openshell-docker": + return { + status: 0, + stdout: JSON.stringify([ + { + Name: "openshell-docker", + Internal: false, + Driver: "bridge", + Scope: "local", + IPAM: { + Config: [{ Subnet: openShellBridgeSubnet, Gateway: "172.29.0.1" }], + }, + }, + ]), + stderr: "", + }; + } + switch (networkPresent) { + case false: + absentNetworkInspectHook?.(); + } + return networkPresent + ? { + status: 0, + stdout: JSON.stringify([ + { + Id: networkId, + Name: args[2], + Internal: true, + Driver: "bridge", + Scope: "local", + Labels: { + "io.nvidia.nemoclaw.llama-cpp-owner": "gateway.primary", + "io.nvidia.nemoclaw.host-local-inference.network-transaction-sha256": + networkTransactionId, + }, + }, + ]), + stderr: "", + } + : { + status: 1, + stdout: "", + stderr: + absentNetworkInspectError ?? + `Error response from daemon: No such network: ${String(args[2])}`, + }; + case "create": { + networkCreateHook?.(); + const labelIndex = args.lastIndexOf("--label"); + networkTransactionId = String(args[labelIndex + 1]).split("=")[1] ?? ""; + networkPresent = !networkCreateUncertain || uncertainNetworkAppears; + switch (networkCreateUncertain) { + case true: + return { + status: 1, + stdout: "", + stderr: "", + error: new Error("Docker network create capture timed out"), + }; + } + return { status: 0, stdout: `${networkId}\n`, stderr: "" }; + } + case "rm": + invariant(args[2] === networkId, unexpected); + networkPresent = false; + return { status: 0, stdout: `${networkId}\n`, stderr: "" }; + default: + throw new Error(unexpected); + } + case "container": { + invariant(args[1] === "inspect", unexpected); + switch (inspectDaemonError) { + case true: + return { status: 1, stdout: "", stderr: "daemon unavailable" }; + } + const target = args[2]; + switch (Boolean(container && (target === RUNTIME_ID || target === "nemoclaw-llama-cpp"))) { + case true: + return { status: 0, stdout: JSON.stringify(inspection()), stderr: "" }; + } + absentInspectHook?.(); + return { + status: 1, + stdout: "", + stderr: `Error response from daemon: No such container: ${String(target)}`, + }; + } + case "create": { + switch (createUncertain) { + case true: + return { + status: 1, + stdout: "", + stderr: "", + error: new Error("Docker create capture timed out"), + }; + } + const labels = Object.fromEntries( + args + .flatMap((argument, index) => + argument === "--label" ? [String(args[index + 1]).split("=")] : [], + ) + .filter(([name, value]) => Boolean(name && value)), + ); + container = { + labels, + running: false, + status: "created", + transactionId: labels["io.nvidia.nemoclaw.host-local-inference.transaction-sha256"] ?? "", + command: args.slice(args.indexOf(IMAGE) + 1), + }; + createHook?.(); + return { status: 0, stdout: createStdout, stderr: "" }; + } + case "start": + startHook?.(); + switch (container) { + case undefined: + break; + default: + startedOnce = true; + container.running = true; + container.status = "running"; + } + return { status: 0, stdout: `${RUNTIME_ID}\n`, stderr: "" }; + case "stop": + switch (container) { + case undefined: + break; + default: + container.running = false; + container.status = "exited"; + } + return { status: 0, stdout: RUNTIME_ID, stderr: "" }; + case "rm": + invariant(args[1] === "--force", unexpected); + container = undefined; + return { status: 0, stdout: RUNTIME_ID, stderr: "" }; + case "run": + invariant(args[1] === "--rm", unexpected); + probeHook?.(); + if ( + sandboxBridgeProbeFailure !== null && + args[args.indexOf("--network") + 1] === "openshell-docker" + ) { + return { + ...sandboxBridgeProbeFailure, + stdout: "", + stderr: sandboxBridgeProbeFailure.stderr ?? "", + }; + } + return probeFails + ? { status: 1, stdout: "", stderr: "not ready" } + : { status: 0, stdout: "ok", stderr: "" }; + default: + throw new Error(unexpected); + } + }); + return { + engine: { + operation: "host-local-inference", + engineId: "docker", + displayName: "Docker", + authorityId: "docker:local", + capture, + captureHost: capture, + }, + capture, + setNetworkId: (value) => (networkId = value), + setNetworkTransactionId: (value) => (networkTransactionId = value), + removeNetwork: () => (networkPresent = false), + failNetworkCreateUncertain: (networkAppears) => { + networkCreateUncertain = true; + uncertainNetworkAppears = networkAppears; + }, + setCreateStdout: (value) => (createStdout = value), + failCreateUncertain: () => (createUncertain = true), + failProbe: () => (probeFails = true), + failSandboxBridgeProbe: (result) => (sandboxBridgeProbeFailure = result), + setOpenShellBridgeSubnet: (value) => (openShellBridgeSubnet = value), + driftHardening: () => (hardeningDrift = true), + dropTmpfs: () => (tmpfs = null), + driftGpuRequest: (driver, count) => { + gpuDriver = driver; + gpuCount = count; + }, + driftExtraDeviceAuthority: (kind) => { + kind === "cap-add" + ? (capAdd = ["SYS_ADMIN"]) + : (legacyDevices = [{ PathOnHost: "/dev/nvidia0" }]); + }, + failInspectWithDaemonError: () => (inspectDaemonError = true), + setAbsentNetworkInspectError: (value) => (absentNetworkInspectError = value), + onAbsentInspect: (callback) => (absentInspectHook = callback), + onAbsentNetworkInspect: (callback) => (absentNetworkInspectHook = callback), + onNetworkCreate: (callback) => (networkCreateHook = callback), + onStart: (callback) => (startHook = callback), + onProbe: (callback) => (probeHook = callback), + onCreate: (callback) => (createHook = callback), + setContainerState: (running, status) => { + invariant(container !== undefined, "cannot change an absent fixture container"); + container.running = running; + container.status = status; + }, + seedNetwork: (journal) => { + invariant(journal.networkId !== null, "seeded network identity is missing"); + networkId = journal.networkId; + networkTransactionId = journal.transactionId; + networkPresent = true; + }, + seed: (journal, running) => { + invariant(journal.networkId !== null, "seeded network identity is missing"); + networkId = journal.networkId; + networkTransactionId = journal.transactionId; + networkPresent = true; + startedOnce = journal.phase !== "creating" && journal.phase !== "created"; + container = { + labels: { + "io.nvidia.nemoclaw.host-local-inference.managed": "true", + "io.nvidia.nemoclaw.host-local-inference.provider": "docker", + "io.nvidia.nemoclaw.host-local-inference.service": "llama-cpp", + "io.nvidia.nemoclaw.host-local-inference.spec-sha256": journal.specSha256, + "io.nvidia.nemoclaw.host-local-inference.transaction-sha256": journal.transactionId, + "io.nvidia.nemoclaw.llama-cpp-owner": "gateway.primary", + }, + running, + status: running ? "running" : "created", + transactionId: journal.transactionId, + command: [...buildLlamaCppHostLocalServerArgv(contract())], + }; + }, + }; +} diff --git a/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.test.ts b/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.test.ts index bd183b71121..b51bffbd819 100644 --- a/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.test.ts +++ b/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.test.ts @@ -8,10 +8,8 @@ import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import type { ContainerEngine } from "../../adapters/container-engine"; import { LLAMA_CPP_PORT } from "../../inference/llama-cpp/contract"; import type { LlamaCppGgufCachePlan } from "../../inference/llama-cpp/gguf-cache-plan"; -import { buildLlamaCppHostLocalServerArgv } from "../../inference/llama-cpp/host-local-runtime"; import type { DockerLlamaCppManagedLifecycleOptions } from "./docker-llama-cpp-managed-lifecycle"; import { contract, @@ -29,6 +27,10 @@ import { rawDigest, TRANSACTION_ID, } from "./docker-llama-cpp-managed-lifecycle.test-support"; +import { + createDockerFixture, + type DockerFixture, +} from "./docker-llama-cpp-managed-lifecycle-engine.test-support"; import { createTestDockerLlamaCppManagedLifecycle as createLifecycle, privateBridgeFixture, @@ -278,375 +280,20 @@ function journalStore(): TestJournalStore { }; } -interface DockerFixture { - readonly engine: ContainerEngine; - readonly capture: ReturnType; - readonly setNetworkId: (value: string) => void; - readonly setNetworkTransactionId: (value: string) => void; - readonly removeNetwork: () => void; - readonly failNetworkCreateUncertain: (networkAppears: boolean) => void; - readonly setCreateStdout: (value: string) => void; - readonly failCreateUncertain: () => void; - readonly failProbe: () => void; - readonly driftHardening: () => void; - readonly dropTmpfs: () => void; - readonly driftGpuRequest: (driver: string | undefined, count: number) => void; - readonly driftExtraDeviceAuthority: (kind: "cap-add" | "legacy-device") => void; - readonly failInspectWithDaemonError: () => void; - readonly setAbsentNetworkInspectError: (value: string) => void; - readonly onAbsentInspect: (callback: () => void) => void; - readonly onAbsentNetworkInspect: (callback: () => void) => void; - readonly onNetworkCreate: (callback: () => void) => void; - readonly onStart: (callback: () => void) => void; - readonly onProbe: (callback: () => void) => void; - readonly onCreate: (callback: () => void) => void; - readonly setContainerState: (running: boolean, status: string) => void; - readonly seedNetwork: (journal: HostLocalCreateJournalRecord) => void; - readonly seed: (journal: HostLocalCreateJournalRecord, running: boolean) => void; -} - function dockerFixture( configuredHostPort = "", publishedHostPort?: string, publishedHostIp = "127.0.0.1", publishedBindingCount = 0, ): DockerFixture { - const effectivePublishedHostPort = publishedHostPort ?? (configuredHostPort || "49152"); - let networkId = NETWORK_ID; - let networkPresent = false; - let networkTransactionId = TRANSACTION_ID; - let networkCreateUncertain = false; - let uncertainNetworkAppears = false; - let createStdout = `${RUNTIME_ID}\n`; - let createUncertain = false; - let probeFails = false; - let hardeningDrift = false; - let tmpfs: Record | null = { - "/tmp": "rw,noexec,nosuid,nodev,size=1024,uid=1001,gid=1001,mode=1777", - }; - let gpuDriver: string | undefined = "nvidia"; - let gpuCount = 1; - let capAdd: null | string[] = null; - let legacyDevices: null | object[] = null; - let inspectDaemonError = false; - let absentNetworkInspectError: string | null = null; - let absentInspectHook: (() => void) | undefined; - let absentNetworkInspectHook: (() => void) | undefined; - let networkCreateHook: (() => void) | undefined; - let startHook: (() => void) | undefined; - let probeHook: (() => void) | undefined; - let createHook: (() => void) | undefined; - let startedOnce = false; - let container: - | { - labels: Record; - running: boolean; - status: string; - transactionId: string; - command: string[]; - } - | undefined; - const inspection = () => [ - { - Id: RUNTIME_ID, - Name: "/nemoclaw-llama-cpp", - Config: { - Image: IMAGE, - User: "1001:1001", - Cmd: container?.command ?? [], - Labels: container?.labels ?? {}, - }, - HostConfig: { - NetworkMode: "nemoclaw-llama-cpp-internal", - RestartPolicy: { Name: "unless-stopped", MaximumRetryCount: 0 }, - PortBindings: configuredHostPort - ? { "8081/tcp": [{ HostIp: "127.0.0.1", HostPort: configuredHostPort }] } - : {}, - ReadonlyRootfs: !hardeningDrift, - CapDrop: ["ALL"], - SecurityOpt: ["no-new-privileges:true"], - Memory: 51_539_607_552, - MemorySwap: 51_539_607_552, - PidsLimit: 256, - DeviceRequests: [ - { - ...(gpuDriver === undefined ? {} : { Driver: gpuDriver }), - Count: gpuCount, - DeviceIDs: null, - Capabilities: [["gpu"]], - Options: {}, - }, - ], - CapAdd: capAdd, - Devices: legacyDevices, - Privileged: false, - Tmpfs: tmpfs, - }, - State: { - Running: container?.running ?? false, - Status: container?.status ?? "created", - }, - NetworkSettings: { - Networks: { - [bindings().network.name]: { - NetworkID: startedOnce ? networkId : "", - IPAddress: container?.running ? "172.30.0.2" : "", - }, - }, - Ports: { - "8081/tcp": - startedOnce && publishedBindingCount > 0 - ? Array.from({ length: publishedBindingCount }, () => ({ - HostIp: publishedHostIp, - HostPort: effectivePublishedHostPort, - })) - : null, - }, - }, - Mounts: [ - { - Type: "bind", - Source: modelPath, - Destination: `/models/${MODEL_FILENAME}`, - RW: false, - }, - { - Type: "bind", - Source: apiKeyPath, - Destination: "/run/secrets/llama-cpp-api-key", - RW: false, - }, - ], - }, - ]; - const capture = vi.fn((args: readonly string[]) => { - const unexpected = `unexpected Docker command: ${args.join(" ")}`; - switch (args[0]) { - case "network": - switch (args[1]) { - case "inspect": - switch (args[2]) { - case "openshell-docker": - return { - status: 0, - stdout: JSON.stringify([ - { - Name: "openshell-docker", - Internal: false, - Driver: "bridge", - Scope: "local", - IPAM: { Config: [{ Subnet: "172.29.0.0/16", Gateway: "172.29.0.1" }] }, - }, - ]), - stderr: "", - }; - } - switch (networkPresent) { - case false: - absentNetworkInspectHook?.(); - } - return networkPresent - ? { - status: 0, - stdout: JSON.stringify([ - { - Id: networkId, - Name: args[2], - Internal: true, - Driver: "bridge", - Scope: "local", - Labels: { - "io.nvidia.nemoclaw.llama-cpp-owner": "gateway.primary", - "io.nvidia.nemoclaw.host-local-inference.network-transaction-sha256": - networkTransactionId, - }, - }, - ]), - stderr: "", - } - : { - status: 1, - stdout: "", - stderr: - absentNetworkInspectError ?? - `Error response from daemon: No such network: ${String(args[2])}`, - }; - case "create": { - networkCreateHook?.(); - const labelIndex = args.lastIndexOf("--label"); - networkTransactionId = String(args[labelIndex + 1]).split("=")[1] ?? ""; - networkPresent = !networkCreateUncertain || uncertainNetworkAppears; - switch (networkCreateUncertain) { - case true: - return { - status: 1, - stdout: "", - stderr: "", - error: new Error("Docker network create capture timed out"), - }; - } - return { status: 0, stdout: `${networkId}\n`, stderr: "" }; - } - case "rm": - invariant(args[2] === networkId, unexpected); - networkPresent = false; - return { status: 0, stdout: `${networkId}\n`, stderr: "" }; - default: - throw new Error(unexpected); - } - case "container": { - invariant(args[1] === "inspect", unexpected); - switch (inspectDaemonError) { - case true: - return { status: 1, stdout: "", stderr: "daemon unavailable" }; - } - const target = args[2]; - switch (Boolean(container && (target === RUNTIME_ID || target === "nemoclaw-llama-cpp"))) { - case true: - return { status: 0, stdout: JSON.stringify(inspection()), stderr: "" }; - } - absentInspectHook?.(); - return { - status: 1, - stdout: "", - stderr: `Error response from daemon: No such container: ${String(target)}`, - }; - } - case "create": { - switch (createUncertain) { - case true: - return { - status: 1, - stdout: "", - stderr: "", - error: new Error("Docker create capture timed out"), - }; - } - const labels = Object.fromEntries( - args - .flatMap((argument, index) => - argument === "--label" ? [String(args[index + 1]).split("=")] : [], - ) - .filter(([name, value]) => Boolean(name && value)), - ); - container = { - labels, - running: false, - status: "created", - transactionId: labels["io.nvidia.nemoclaw.host-local-inference.transaction-sha256"] ?? "", - command: args.slice(args.indexOf(IMAGE) + 1), - }; - createHook?.(); - return { status: 0, stdout: createStdout, stderr: "" }; - } - case "start": - startHook?.(); - switch (container) { - case undefined: - break; - default: - startedOnce = true; - container.running = true; - container.status = "running"; - } - return { status: 0, stdout: `${RUNTIME_ID}\n`, stderr: "" }; - case "stop": - switch (container) { - case undefined: - break; - default: - container.running = false; - container.status = "exited"; - } - return { status: 0, stdout: RUNTIME_ID, stderr: "" }; - case "rm": - invariant(args[1] === "--force", unexpected); - container = undefined; - return { status: 0, stdout: RUNTIME_ID, stderr: "" }; - case "run": - invariant(args[1] === "--rm", unexpected); - probeHook?.(); - return probeFails - ? { status: 1, stdout: "", stderr: "not ready" } - : { status: 0, stdout: "ok", stderr: "" }; - default: - throw new Error(unexpected); - } - }); - return { - engine: { - operation: "host-local-inference", - engineId: "docker", - displayName: "Docker", - authorityId: "docker:local", - capture, - captureHost: capture, - }, - capture, - setNetworkId: (value) => (networkId = value), - setNetworkTransactionId: (value) => (networkTransactionId = value), - removeNetwork: () => (networkPresent = false), - failNetworkCreateUncertain: (networkAppears) => { - networkCreateUncertain = true; - uncertainNetworkAppears = networkAppears; - }, - setCreateStdout: (value) => (createStdout = value), - failCreateUncertain: () => (createUncertain = true), - failProbe: () => (probeFails = true), - driftHardening: () => (hardeningDrift = true), - dropTmpfs: () => (tmpfs = null), - driftGpuRequest: (driver, count) => { - gpuDriver = driver; - gpuCount = count; - }, - driftExtraDeviceAuthority: (kind) => { - kind === "cap-add" - ? (capAdd = ["SYS_ADMIN"]) - : (legacyDevices = [{ PathOnHost: "/dev/nvidia0" }]); - }, - failInspectWithDaemonError: () => (inspectDaemonError = true), - setAbsentNetworkInspectError: (value) => (absentNetworkInspectError = value), - onAbsentInspect: (callback) => (absentInspectHook = callback), - onAbsentNetworkInspect: (callback) => (absentNetworkInspectHook = callback), - onNetworkCreate: (callback) => (networkCreateHook = callback), - onStart: (callback) => (startHook = callback), - onProbe: (callback) => (probeHook = callback), - onCreate: (callback) => (createHook = callback), - setContainerState: (running, status) => { - invariant(container !== undefined, "cannot change an absent fixture container"); - container.running = running; - container.status = status; - }, - seedNetwork: (journal) => { - invariant(journal.networkId !== null, "seeded network identity is missing"); - networkId = journal.networkId; - networkTransactionId = journal.transactionId; - networkPresent = true; - }, - seed: (journal, running) => { - invariant(journal.networkId !== null, "seeded network identity is missing"); - networkId = journal.networkId; - networkTransactionId = journal.transactionId; - networkPresent = true; - startedOnce = journal.phase !== "creating" && journal.phase !== "created"; - container = { - labels: { - "io.nvidia.nemoclaw.host-local-inference.managed": "true", - "io.nvidia.nemoclaw.host-local-inference.provider": "docker", - "io.nvidia.nemoclaw.host-local-inference.service": "llama-cpp", - "io.nvidia.nemoclaw.host-local-inference.spec-sha256": journal.specSha256, - "io.nvidia.nemoclaw.host-local-inference.transaction-sha256": journal.transactionId, - "io.nvidia.nemoclaw.llama-cpp-owner": "gateway.primary", - }, - running, - status: running ? "running" : "created", - transactionId: journal.transactionId, - command: [...buildLlamaCppHostLocalServerArgv(contract())], - }; - }, - }; + return createDockerFixture( + { apiKeyPath, modelPath, networkName: bindings().network.name }, + configuredHostPort, + publishedHostPort, + publishedHostIp, + publishedBindingCount, + ); } - function dockerCommandPrefixes(fixture: DockerFixture): unknown[] { return fixture.capture.mock.calls.map((call) => call[0]?.slice(0, 2)); } @@ -775,6 +422,93 @@ describe("dormant Docker llama.cpp managed lifecycle", () => { expect(lifecycle.runtime.destroy(receipt).status).toBe("already-absent"); }); + it("accepts managed reachability through the authoritative OpenShell bridge (#8712)", () => { + const fixture = dockerFixture(); + createLifecycle(options(fixture)).start(receiptWriter()); + + expect(fixture.capture.mock.calls.map((call) => call[0])).toContainEqual( + expect.arrayContaining([ + "--network", + "openshell-docker", + "--add-host", + "host.openshell.internal:172.29.0.1", + "http://host.openshell.internal:8081/health", + ]), + ); + }); + + it.each([ + { failure: "connection refusal", status: 7, stderr: "curl: (7) failed to connect" }, + { failure: "timeout", status: 28, stderr: "curl: (28) timed out" }, + ])("reports a narrow UFW remediation and rolls back after bridge $failure (#8712)", ({ + status, + stderr, + }) => { + const fixture = dockerFixture(); + const store = journalStore(); + const privateBridge = privateBridgeFixture(); + fixture.failSandboxBridgeProbe({ status, stderr }); + + let failure: Error | undefined; + try { + createLifecycle(options(fixture, store), {}, privateBridge).start(receiptWriter()); + } catch (error) { + failure = error instanceof Error ? error : new Error(String(error)); + } + + expect(failure?.message).toContain( + "Managed llama.cpp host-loopback health check passed, but the OpenShell Docker bridge health check failed.", + ); + expect(failure?.message).toContain("OpenShell Docker network: openshell-docker"); + expect(failure?.message).toContain("Source subnet: 172.29.0.0/16"); + expect(failure?.message).toContain("Gateway IP address: 172.29.0.1"); + expect(failure?.message).toContain("TCP port: 8081"); + expect(failure?.message).toContain( + "sudo ufw allow from 172.29.0.0/16 to 172.29.0.1 port 8081 proto tcp", + ); + expect(failure?.message).not.toContain("test-only-secret"); + expect(privateBridge.stopTransaction).toHaveBeenCalledWith(TRANSACTION_ID); + expect(store.list()).toEqual([]); + expect(dockerCommandPrefixes(fixture)).toContainEqual(["rm", "--force"]); + expect(dockerCommandPrefixes(fixture)).toContainEqual(["network", "rm"]); + }); + + it("does not report UFW remediation for a Docker probe command failure (#8712)", () => { + const fixture = dockerFixture(); + fixture.failSandboxBridgeProbe({ status: 125, stderr: "docker run failed" }); + + let failure: Error | undefined; + try { + createLifecycle(options(fixture)).start(receiptWriter()); + } catch (error) { + failure = error instanceof Error ? error : new Error(String(error)); + } + + expect(failure?.message).toBe( + "Docker llama.cpp private sandbox bridge probe failed (exit 125).", + ); + expect(failure?.message).not.toContain("sudo ufw"); + }); + + it("does not print unvalidated bridge topology in UFW remediation (#8712)", () => { + const fixture = dockerFixture(); + fixture.setOpenShellBridgeSubnet("0.0.0.0/0"); + fixture.failSandboxBridgeProbe({ status: 28, stderr: "curl: (28) timed out" }); + + let failure: Error | undefined; + try { + createLifecycle(options(fixture)).start(receiptWriter()); + } catch (error) { + failure = error instanceof Error ? error : new Error(String(error)); + } + + expect(failure?.message).toBe( + "Docker llama.cpp private sandbox bridge probe failed (exit 28).", + ); + expect(failure?.message).not.toContain("sudo ufw"); + expect(failure?.message).not.toContain("0.0.0.0/0"); + }); + it("resumes an already-running receipt without creating or starting resources (#8144)", () => { const fixture = dockerFixture(); const lifecycle = controller(fixture); @@ -805,6 +539,41 @@ describe("dormant Docker llama.cpp managed lifecycle", () => { expect(calls).not.toContainEqual(expect.arrayContaining(["network", "create"])); }); + it("preserves receipt-bound resources after bridge refusal during resume (#8712)", () => { + const fixture = dockerFixture(); + const store = journalStore(); + const privateBridge = privateBridgeFixture(); + const lifecycle = createLifecycle(options(fixture, store), {}, privateBridge); + const receipt = lifecycle.start(receiptWriter()); + lifecycle.runtime.stopManaged(receipt); + fixture.capture.mockClear(); + privateBridge.stopTransaction.mockClear(); + fixture.failSandboxBridgeProbe({ status: 7, stderr: "curl: (7) failed to connect" }); + + let failure: Error | undefined; + try { + lifecycle.resume(receipt); + } catch (error) { + failure = error instanceof Error ? error : new Error(String(error)); + } + + expect(failure?.message).toContain( + "Managed llama.cpp host-loopback health check passed, but the OpenShell Docker bridge health check failed.", + ); + expect(failure?.message).toContain( + "sudo ufw allow from 172.29.0.0/16 to 172.29.0.1 port 8081 proto tcp", + ); + expect(privateBridge.stopTransaction).not.toHaveBeenCalled(); + expect(store.load(TRANSACTION_ID)).toMatchObject({ + phase: "finalized", + runtimeId: RUNTIME_ID, + networkId: NETWORK_ID, + }); + expect(lifecycle.runtime.inspectManaged(receipt).running).toBe(true); + expect(dockerCommandPrefixes(fixture)).not.toContainEqual(["rm", "--force"]); + expect(dockerCommandPrefixes(fixture)).not.toContainEqual(["network", "rm"]); + }); + it("rejects a non-resumable exact runtime without lifecycle mutation (#8144)", () => { const fixture = dockerFixture(); const lifecycle = controller(fixture); diff --git a/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.ts b/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.ts index 4a857bb730b..55833e120e2 100644 --- a/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.ts +++ b/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.ts @@ -21,6 +21,8 @@ import { type LlamaCppHostLocalLaunchContract, type LlamaCppHostLocalRuntimeBindings, } from "../../inference/llama-cpp/host-local-runtime"; +import { formatHostServiceUnreachableMessage } from "../host-service-reachability"; +import { validateUfwRuleOperands } from "../ufw-auto-apply"; import { createDockerLlamaCppPrivateBridgeController, type DockerLlamaCppPrivateBridgeAuthority, @@ -72,6 +74,7 @@ const MUTATION_TIMEOUT_MS = 30 * 60 * 1000; const UNCERTAIN_CREATE_ABSENCE_GRACE_MS = MUTATION_TIMEOUT_MS + INSPECT_TIMEOUT_MS; const STOP_GRACE_SECONDS = 30; const AT_REST = new Set(["created", "dead", "exited"]); +const CURL_CONNECTIVITY_FAILURE_EXIT_CODES = new Set([7, 28]); export type DockerLlamaCppManagedLifecycleOptions = HostLocalLlamaCppLifecycleInput; @@ -92,6 +95,7 @@ interface DockerNetworkAuthority { interface DockerGatewayBridgeAuthority { readonly gatewayIp: string; readonly name: typeof OPENSHELL_DOCKER_NETWORK; + readonly subnet?: string; } interface DockerContainerInspection { @@ -298,19 +302,20 @@ function inspectGatewayBridge(engine: ContainerEngine): DockerGatewayBridgeAutho ) { throw new Error("Docker llama.cpp requires the native OpenShell Docker bridge."); } - const gatewayIps = configs.flatMap((candidate) => { + const gateways = configs.flatMap((candidate) => { const config = record(candidate, "Docker llama.cpp OpenShell bridge IPAM entry"); - return typeof config.Gateway === "string" && - /^172\.(?:1[6-9]|2[0-9]|3[01])\.(?:[0-9]{1,3})\.(?:[0-9]{1,3})$/u.test(config.Gateway) - ? [config.Gateway] - : typeof config.Gateway === "string" && /^(?:10\.|192\.168\.)/u.test(config.Gateway) - ? [config.Gateway] - : []; + const gatewayIp = config.Gateway; + const subnet = config.Subnet; + return typeof gatewayIp === "string" && + (/^172\.(?:1[6-9]|2[0-9]|3[01])\.(?:[0-9]{1,3})\.(?:[0-9]{1,3})$/u.test(gatewayIp) || + /^(?:10\.|192\.168\.)/u.test(gatewayIp)) + ? [{ gatewayIp, ...(typeof subnet === "string" ? { subnet } : {}) }] + : []; }); - if (gatewayIps.length !== 1) { + if (gateways.length !== 1) { throw new Error("Docker llama.cpp requires one private IPv4 OpenShell bridge gateway."); } - return Object.freeze({ gatewayIp: gatewayIps[0]!, name: OPENSHELL_DOCKER_NETWORK }); + return Object.freeze({ ...gateways[0]!, name: OPENSHELL_DOCKER_NETWORK }); } function inspectNetwork( @@ -1018,28 +1023,58 @@ function probePrivateBridge( timeoutSeconds * 1_000 + INSPECT_TIMEOUT_MS, ), ); - requireSuccess( - "private sandbox bridge probe", - captureMutation( - options, - lease, - execution, - [ - "run", - "--rm", - "--pull=never", - "--network", - gateway.name, - "--add-host", - `${ENDPOINT_HOST}:${gateway.gatewayIp}`, - "--entrypoint", - "curl", - options.probeImageReference, - ...curlArguments(`http://${ENDPOINT_HOST}:${String(options.bindings.hostPort)}/health`), - ], - timeoutSeconds * 1_000 + INSPECT_TIMEOUT_MS, - ), + const sandboxProbe = captureMutation( + options, + lease, + execution, + [ + "run", + "--rm", + "--pull=never", + "--network", + gateway.name, + "--add-host", + `${ENDPOINT_HOST}:${gateway.gatewayIp}`, + "--entrypoint", + "curl", + options.probeImageReference, + ...curlArguments(`http://${ENDPOINT_HOST}:${String(options.bindings.hostPort)}/health`), + ], + timeoutSeconds * 1_000 + INSPECT_TIMEOUT_MS, ); + if (sandboxProbe.error || sandboxProbe.status !== 0) { + const port = options.bindings.hostPort; + const subnet = gateway.subnet; + if ( + !sandboxProbe.error && + CURL_CONNECTIVITY_FAILURE_EXIT_CODES.has(sandboxProbe.status) && + subnet && + !validateUfwRuleOperands(subnet, gateway.gatewayIp, port) + ) { + const remediation = formatHostServiceUnreachableMessage( + { + ok: false, + reason: "tcp_failed", + port, + networkName: gateway.name, + subnet, + gatewayIp: gateway.gatewayIp, + }, + { serviceLabel: "managed llama.cpp server", port }, + ); + throw new Error( + [ + "Managed llama.cpp host-loopback health check passed, but the OpenShell Docker bridge health check failed.", + ` OpenShell Docker network: ${gateway.name}`, + ` Source subnet: ${subnet}`, + ` Gateway IP address: ${gateway.gatewayIp}`, + ` TCP port: ${String(port)}`, + remediation, + ].join("\n"), + ); + } + requireSuccess("private sandbox bridge probe", sandboxProbe); + } bridge.assertRunning(authority); options.journalStore.assertExecution(lease); } diff --git a/src/lib/onboard/ufw-auto-apply.ts b/src/lib/onboard/ufw-auto-apply.ts index 9dab81b9d28..76ac1954a59 100644 --- a/src/lib/onboard/ufw-auto-apply.ts +++ b/src/lib/onboard/ufw-auto-apply.ts @@ -95,7 +95,7 @@ function parseDockerBridgeCidr( return { network, prefix, mask }; } -function validateUfwRuleOperands( +export function validateUfwRuleOperands( subnet: string, gatewayIp: string, port: number,