diff --git a/src/lib/onboard/experimental/hermes-portable-build-context.test.ts b/src/lib/onboard/experimental/hermes-portable-build-context.test.ts index 02328071aaf..d54d2411fdf 100644 --- a/src/lib/onboard/experimental/hermes-portable-build-context.test.ts +++ b/src/lib/onboard/experimental/hermes-portable-build-context.test.ts @@ -112,9 +112,18 @@ describe("Hermes portable staged build context", testTimeoutOptions(30_000), () const plan = createHermesPortableBuildContextPlan(ROOT, BUILD_SETTINGS); const first = plan.materialize(contextInput()); - expect(first.dockerfilePath).toBe( - path.join(first.buildContextPath, "agents/hermes/Dockerfile"), - ); + expect(first.dockerfilePath).toBe(path.join(first.buildContextPath, "Dockerfile")); + const inferredContext = path.dirname(first.dockerfilePath); + expect(inferredContext).toBe(first.buildContextPath); + expect( + fs.existsSync( + path.join( + inferredContext, + "tools/mcp-tool-discovery-runtime/reviewed-runtime-bundle/mcp-tool-discovery/BUNDLED_PACKAGES.json", + ), + ), + ).toBe(true); + expect(fs.existsSync(path.join(inferredContext, "agents/hermes/Dockerfile"))).toBe(false); expect(plan.authority.sourceRevision).toMatch(/^[a-f0-9]{40,64}$/u); expect(plan.authority.contextManifestSha256).toMatch(/^[a-f0-9]{64}$/u); const stagedDockerfile = fs.readFileSync(first.dockerfilePath, "utf8"); diff --git a/src/lib/onboard/experimental/hermes-portable-build-context.ts b/src/lib/onboard/experimental/hermes-portable-build-context.ts index 3796568d694..1931b2209bb 100644 --- a/src/lib/onboard/experimental/hermes-portable-build-context.ts +++ b/src/lib/onboard/experimental/hermes-portable-build-context.ts @@ -26,6 +26,8 @@ const OPEN_READ_FLAGS = fs.constants.O_RDONLY | (typeof fs.constants.O_NOFOLLOW === "number" ? fs.constants.O_NOFOLLOW : 0) | (typeof fs.constants.O_NONBLOCK === "number" ? fs.constants.O_NONBLOCK : 0); +const SOURCE_DOCKERFILE_RELATIVE_PATH = "agents/hermes/Dockerfile" as const; +const CONTEXT_DOCKERFILE_RELATIVE_PATH = "Dockerfile" as const; const LOCAL_COPY_SOURCES = [ "agents/hermes/build-mcp-digest.py", @@ -128,7 +130,7 @@ type DirectoryEvidence = { export interface HermesPortableBuildContextAuthority { readonly schemaVersion: typeof CONTEXT_SCHEMA_VERSION; readonly sourceRevision: string; - readonly dockerfileRelativePath: "agents/hermes/Dockerfile"; + readonly dockerfileRelativePath: typeof CONTEXT_DOCKERFILE_RELATIVE_PATH; readonly sourceManifestSha256: string; readonly contextManifestSha256: string; } @@ -640,7 +642,7 @@ function captureSourceEntries( entries.push(entry); }; - visit("agents/hermes/Dockerfile"); + visit(SOURCE_DOCKERFILE_RELATIVE_PATH); for (const token of LOCAL_COPY_SOURCES) { if (token.endsWith("/")) { visit(token.slice(0, -1)); @@ -676,7 +678,7 @@ function sourceAuthority( }); const contextManifestSha256 = canonicalDigest({ schemaVersion: CONTEXT_SCHEMA_VERSION, - dockerfileRelativePath: "agents/hermes/Dockerfile", + dockerfileRelativePath: CONTEXT_DOCKERFILE_RELATIVE_PATH, entries: contextEntries.map((entry) => ({ kind: entry.kind, relativePath: entry.relativePath, @@ -687,7 +689,7 @@ function sourceAuthority( return { schemaVersion: CONTEXT_SCHEMA_VERSION, sourceRevision: revision.revision, - dockerfileRelativePath: "agents/hermes/Dockerfile", + dockerfileRelativePath: CONTEXT_DOCKERFILE_RELATIVE_PATH, sourceManifestSha256, contextManifestSha256, }; @@ -698,12 +700,20 @@ function renderContextEntries( settings: HermesPortableBuildContextSettings, ): readonly SourceEntry[] { return sourceEntries.map((entry) => { - if (entry.kind !== "file" || entry.relativePath !== "agents/hermes/Dockerfile") return entry; + if (entry.kind !== "file" || entry.relativePath !== SOURCE_DOCKERFILE_RELATIVE_PATH) { + return entry; + } const bytes = Buffer.from( renderHermesPortableDockerfileBuildSettings(UTF8.decode(entry.bytes!), settings), "utf8", ); - return { ...entry, bytes, size: bytes.byteLength, sha256: digest(bytes) }; + return { + ...entry, + relativePath: CONTEXT_DOCKERFILE_RELATIVE_PATH, + bytes, + size: bytes.byteLength, + sha256: digest(bytes), + }; }); } @@ -724,7 +734,7 @@ function capture( ); const sourceEntries = captureSourceEntries(rootPath, tracked); const dockerfile = sourceEntries.find( - (entry) => entry.relativePath === "agents/hermes/Dockerfile", + (entry) => entry.relativePath === SOURCE_DOCKERFILE_RELATIVE_PATH, ); if (dockerfile?.kind !== "file" || !dockerfile.bytes) fail("Dockerfile source is unavailable"); parseDockerfileSources(dockerfile.bytes); @@ -1375,7 +1385,7 @@ export function createHermesPortableBuildContextPlan( }; return { authority: captured.authority, - sourceDockerfilePath: path.join(rootPath, captured.authority.dockerfileRelativePath), + sourceDockerfilePath: path.join(rootPath, SOURCE_DOCKERFILE_RELATIVE_PATH), assertCurrentSource, materialize: (input) => { assertCurrentSource(); diff --git a/src/lib/onboard/experimental/hermes-portable-onboarding.test.ts b/src/lib/onboard/experimental/hermes-portable-onboarding.test.ts index 058989df415..da92040c110 100644 --- a/src/lib/onboard/experimental/hermes-portable-onboarding.test.ts +++ b/src/lib/onboard/experimental/hermes-portable-onboarding.test.ts @@ -724,20 +724,29 @@ describe("Hermes portable onboarding transaction", () => { expect(fixture.events).not.toContain("create"); }); - it("classifies absence only after a reachable gateway returns exact sandbox-not-found evidence (#9203)", () => { - const capture = vi - .fn() - .mockReturnValueOnce({ status: 0, stdout: "[]", stderr: "" }) - .mockReturnValueOnce({ status: 1, stdout: "", stderr: "sandbox 'alpha' not found" }); - - expect(observeHermesPortableSandbox("alpha", "nemoclaw", capture)).toEqual({ - kind: "absent", - }); - expect(capture.mock.calls).toEqual([ - [["sandbox", "list", "-g", "nemoclaw"]], - [["sandbox", "get", "-g", "nemoclaw", "alpha"]], - ]); - }); + it.each([ + ["named sandbox-not-found", "sandbox 'alpha' not found"], + [ + "OpenShell 0.0.106 entity-not-found", + "Error: × code: 'Some requested entity was not found', message: \"sandbox not found\"", + ], + ])( + "classifies absence after a reachable gateway returns %s evidence (#9203)", + (_label, stderr) => { + const capture = vi + .fn() + .mockReturnValueOnce({ status: 0, stdout: "[]", stderr: "" }) + .mockReturnValueOnce({ status: 1, stdout: "", stderr }); + + expect(observeHermesPortableSandbox("alpha", "nemoclaw", capture)).toEqual({ + kind: "absent", + }); + expect(capture.mock.calls).toEqual([ + [["sandbox", "list", "-g", "nemoclaw"]], + [["sandbox", "get", "-g", "nemoclaw", "alpha"]], + ]); + }, + ); it("accepts Ready identity only from the exact receipt gateway (#9203)", () => { const capture = vi @@ -796,6 +805,16 @@ describe("Hermes portable onboarding transaction", () => { ["gateway missing", { status: 1, stdout: "", stderr: "gateway not found" }, false], ["transport failure", { status: null, stdout: "", stderr: "transport unavailable" }, true], ["unnamed sandbox", { status: 1, stdout: "", stderr: "unknown sandbox" }, true], + [ + "entity-not-found response for a different resource", + { + status: 1, + stdout: "", + stderr: + "Error: × code: 'Some requested entity was not found', message: \"gateway not found\"", + }, + true, + ], ["ambiguous absence", { status: 1, stdout: "", stderr: "no sandbox connection" }, true], ])( "keeps %s fail-closed instead of treating it as sandbox absence (#9203)", diff --git a/src/lib/onboard/experimental/hermes-portable-onboarding.ts b/src/lib/onboard/experimental/hermes-portable-onboarding.ts index a0a8c1756c3..05fce1dc9c8 100644 --- a/src/lib/onboard/experimental/hermes-portable-onboarding.ts +++ b/src/lib/onboard/experimental/hermes-portable-onboarding.ts @@ -567,8 +567,11 @@ export function observeHermesPortableSandbox( `^(?:Error:\\s*)?sandbox ['\"]?${escapedRegExp(sandboxName)}['\"]? not found\\.?$`, "u", ); - const coded = /^Error: code: 'NotFound', message: "sandbox not found"$/u; - return named.test(output) || coded.test(output) + const coded = + output === `Error: code: 'NotFound', message: "sandbox not found"` || + output === + `Error: × code: 'Some requested entity was not found', message: "sandbox not found"`; + return named.test(output) || coded ? { kind: "absent" } : { kind: "ambiguous", detail: "sandbox get did not prove exact sandbox absence" }; } diff --git a/src/lib/onboard/runtime-provider/podman-host-local-inference-probe-inspect.test.ts b/src/lib/onboard/runtime-provider/podman-host-local-inference-probe-inspect.test.ts new file mode 100644 index 00000000000..f8adefbf8d1 --- /dev/null +++ b/src/lib/onboard/runtime-provider/podman-host-local-inference-probe-inspect.test.ts @@ -0,0 +1,90 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { createPodmanHostLocalInferenceTestHarness } from "../../../../test/helpers/podman-host-local-inference-test-harness"; +import type { HostLocalInferenceRuntime } from "./host-local-inference"; +import { createPodmanHostLocalInferenceOperation } from "./podman-host-local-inference"; + +function operationRuntime( + harness: ReturnType, +): HostLocalInferenceRuntime { + const operation = createPodmanHostLocalInferenceOperation({ + engine: harness.engine, + env: harness.env, + acceleration: harness.operationAcceleration, + authorityStore: harness.authorityStore, + routeAuthorityStore: harness.routeAuthorityStore, + onFailureEvidence: harness.onFailureEvidence, + redactSensitive: harness.redactSensitive, + }); + return ( + operation.managedRuntime ?? + (() => { + throw new Error("test operation lacks managed runtime"); + })() + ); +} + +const probeId = "c".repeat(64); +const inspectEvent = `podman:container inspect ${probeId}`; + +describe("Podman host-local inference post-create probe inspection", () => { + it("retries one timeout using the same full probe ID", () => { + const harness = createPodmanHostLocalInferenceTestHarness(); + harness.state.probePostCreateInspectTimeoutsRemaining = 1; + + const prepared = operationRuntime(harness).startManaged(harness.input, harness.writer); + + const waitIndex = harness.events.indexOf(`podman:wait ${probeId}`); + expect(waitIndex).toBeGreaterThan(0); + expect( + harness.events.slice(0, waitIndex).filter((event) => event === inspectEvent), + ).toHaveLength(2); + expect(prepared.receipt.service).toBe("nim"); + expect(harness.probe()).toBeNull(); + }); + + it("fails closed after two timeouts without acting on the probe", () => { + const harness = createPodmanHostLocalInferenceTestHarness(); + harness.state.probePostCreateInspectTimeoutsRemaining = 2; + harness.state.probeForbiddenActions = ["wait", "logs", "rm"]; + + expect(() => operationRuntime(harness).startManaged(harness.input, harness.writer)).toThrow( + "probe identity is indeterminate after create", + ); + + expect(harness.events.filter((event) => event === inspectEvent)).toHaveLength(2); + expect(harness.probe()).toMatchObject({ id: probeId }); + expect(harness.written).toHaveLength(0); + }); + + it("does not retry a non-timeout failure", () => { + const harness = createPodmanHostLocalInferenceTestHarness(); + harness.state.probePostCreateInspectFailuresRemaining = 1; + harness.state.probeForbiddenActions = ["wait", "logs", "rm"]; + + expect(() => operationRuntime(harness).startManaged(harness.input, harness.writer)).toThrow( + "probe identity is indeterminate after create", + ); + + expect(harness.events.filter((event) => event === inspectEvent)).toHaveLength(1); + expect(harness.probe()).toMatchObject({ id: probeId }); + expect(harness.written).toHaveLength(0); + }); + + it("does not retry an inspect result with a mismatched container ID", () => { + const harness = createPodmanHostLocalInferenceTestHarness(); + harness.state.probeInspectRuntimeIdMismatchAt = 1; + harness.state.probeForbiddenActions = ["wait", "logs", "rm"]; + + expect(() => operationRuntime(harness).startManaged(harness.input, harness.writer)).toThrow( + "probe identity is indeterminate after create", + ); + + expect(harness.events.filter((event) => event === inspectEvent)).toHaveLength(1); + expect(harness.probe()).toMatchObject({ id: probeId }); + expect(harness.written).toHaveLength(0); + }); +}); diff --git a/src/lib/onboard/runtime-provider/podman-host-local-inference.ts b/src/lib/onboard/runtime-provider/podman-host-local-inference.ts index db50c3e9f9a..bb4ddd3a9b1 100644 --- a/src/lib/onboard/runtime-provider/podman-host-local-inference.ts +++ b/src/lib/onboard/runtime-provider/podman-host-local-inference.ts @@ -374,13 +374,21 @@ function sanitizeFailureResult( redactor: PodmanInferenceRedactor, ): ContainerEngineCommandResult { const failed = result.status !== 0 || result.error !== undefined; + const error = + result.error === undefined + ? undefined + : Object.assign(new Error(redactEvidence(redactor, result.error.message)), { + ...((result.error as NodeJS.ErrnoException).code === "ETIMEDOUT" + ? { code: "ETIMEDOUT" } + : {}), + }); return Object.freeze({ status: result.status, // Successful stdout remains intact because it carries provider JSON and // exact runtime identities. Stderr is never parsed and is always redacted. stdout: failed ? redactEvidence(redactor, result.stdout) : result.stdout, stderr: redactEvidence(redactor, result.stderr), - ...(result.error ? { error: new Error(redactEvidence(redactor, result.error.message)) } : {}), + ...(error ? { error } : {}), }); } @@ -425,6 +433,10 @@ function commandEvidence(result: ContainerEngineCommandResult): string { ); } +function commandTimedOut(result: ContainerEngineCommandResult): boolean { + return (result.error as NodeJS.ErrnoException | undefined)?.code === "ETIMEDOUT"; +} + function redactedCommandEvidence( redactor: PodmanInferenceRedactor, result: ContainerEngineCommandResult, @@ -1164,11 +1176,17 @@ function inspectContainer(engine: ContainerEngine, runtimeId: string): ManagedCo }); } -function inspectProbeContainer(engine: ContainerEngine, runtimeId: string): ProbeContainer { - const output = requireSuccess( - "probe container inspection", - engine.capture(["container", "inspect", runtimeId], PROBE_TIMEOUT_MS), - ); +function inspectProbeContainer( + engine: ContainerEngine, + runtimeId: string, + retryTimeoutOnce = false, +): ProbeContainer { + const args = ["container", "inspect", runtimeId] as const; + let result = engine.capture(args, PROBE_TIMEOUT_MS); + if (retryTimeoutOnce && commandTimedOut(result)) { + result = engine.capture(args, PROBE_TIMEOUT_MS); + } + const output = requireSuccess("probe container inspection", result); let parsed: unknown; try { parsed = JSON.parse(output); @@ -1957,7 +1975,11 @@ function executeExactProbe( } let container: ProbeContainer; try { - container = requireProbeIdentity(inspectProbeContainer(engine, runtimeId), spec, runtimeId); + container = requireProbeIdentity( + inspectProbeContainer(engine, runtimeId, true), + spec, + runtimeId, + ); } catch (error) { captureFailure(error); throw new PodmanInferenceIndeterminateCleanupError( diff --git a/test/helpers/hermes-portable-onboarding-fixture.ts b/test/helpers/hermes-portable-onboarding-fixture.ts index ab822f0b071..21b97c27f97 100644 --- a/test/helpers/hermes-portable-onboarding-fixture.ts +++ b/test/helpers/hermes-portable-onboarding-fixture.ts @@ -221,7 +221,7 @@ export function createHermesPortableTestInput(stateDir: string, policyPath: stri authority: { schemaVersion: 1, sourceRevision: "1".repeat(40), - dockerfileRelativePath: "agents/hermes/Dockerfile", + dockerfileRelativePath: "Dockerfile", sourceManifestSha256: "2".repeat(64), contextManifestSha256: "3".repeat(64), }, @@ -229,7 +229,7 @@ export function createHermesPortableTestInput(stateDir: string, policyPath: stri assertCurrentSource: vi.fn(), materialize: vi.fn(() => ({ buildContextPath: "/private/staged-hermes", - dockerfilePath: "/private/staged-hermes/agents/hermes/Dockerfile", + dockerfilePath: "/private/staged-hermes/Dockerfile", assertCurrent: vi.fn(), })), retire: vi.fn(() => true), @@ -390,7 +390,7 @@ export function createHermesPortableTransactionFixture( const policyIndex = argv.indexOf("--policy"); expect(argv[policyIndex + 1]).toContain("policy."); expect(argv[argv.indexOf("--from") + 1]).toBe( - options.expectedDockerfilePath ?? "/private/staged-hermes/agents/hermes/Dockerfile", + options.expectedDockerfilePath ?? "/private/staged-hermes/Dockerfile", ); expect(buildContextPath).toBe(options.expectedBuildContextPath ?? "/private/staged-hermes"); present = true; diff --git a/test/helpers/podman-host-local-inference-test-harness.ts b/test/helpers/podman-host-local-inference-test-harness.ts index f3fb16a8d83..a94fb08550a 100644 --- a/test/helpers/podman-host-local-inference-test-harness.ts +++ b/test/helpers/podman-host-local-inference-test-harness.ts @@ -122,6 +122,8 @@ export interface PodmanHostLocalInferenceHarness { probeRunLostAcknowledgement: boolean; probeRunAcknowledgementText: string | null; probePostCreateNameLookupTimeout: boolean; + probePostCreateInspectFailuresRemaining: number; + probePostCreateInspectTimeoutsRemaining: number; probeInspectRuntimeIdMismatchAt: number | null; probeForbiddenActions: Array<"logs" | "rm" | "wait">; probeWaitTimeouts: number[]; @@ -424,6 +426,8 @@ export function createPodmanHostLocalInferenceTestHarness( probeRunLostAcknowledgement: false, probeRunAcknowledgementText: null as string | null, probePostCreateNameLookupTimeout: false, + probePostCreateInspectFailuresRemaining: 0, + probePostCreateInspectTimeoutsRemaining: 0, probeInspectRuntimeIdMismatchAt: null as number | null, probeForbiddenActions: [] as Array<"logs" | "rm" | "wait">, probeWaitTimeouts: [] as number[], @@ -584,6 +588,18 @@ export function createPodmanHostLocalInferenceTestHarness( if (currentContainer?.id === args[2]) return result(0, inspectPayload(currentContainer)); if (currentProbe?.id === args[2]) { probeInspectCount += 1; + if (state.probePostCreateInspectFailuresRemaining > 0) { + state.probePostCreateInspectFailuresRemaining -= 1; + const error = Object.assign(new Error("permission denied"), { code: "EACCES" }); + return result(1, "", "permission denied", error); + } + if (state.probePostCreateInspectTimeoutsRemaining > 0) { + state.probePostCreateInspectTimeoutsRemaining -= 1; + const error = Object.assign(new Error("spawnSync /usr/local/bin/podman ETIMEDOUT"), { + code: "ETIMEDOUT", + }); + return result(1, "", "spawnSync /usr/local/bin/podman ETIMEDOUT", error); + } const inspectedProbe = state.probeInspectRuntimeIdMismatchAt === probeInspectCount ? { ...currentProbe, id: REUSED_PROBE_CONTAINER_ID }