fix(hermes): recognize OpenShell sandbox absence

src/lib/onboard/experimental/hermes-portable-build-context.test.ts

@@ -112,9 +112,18 @@ describe("Hermes portable staged build context", testTimeoutOptions(30_000), ()
     const plan = createHermesPortableBuildContextPlan(ROOT, BUILD_SETTINGS);
     const first = plan.materialize(contextInput());
 
-    expect(first.dockerfilePath).toBe(
-      path.join(first.buildContextPath, "agents/hermes/Dockerfile"),
-    );
+    expect(first.dockerfilePath).toBe(path.join(first.buildContextPath, "Dockerfile"));
+    const inferredContext = path.dirname(first.dockerfilePath);
+    expect(inferredContext).toBe(first.buildContextPath);
+    expect(
+      fs.existsSync(
+        path.join(
+          inferredContext,
+          "tools/mcp-tool-discovery-runtime/reviewed-runtime-bundle/mcp-tool-discovery/BUNDLED_PACKAGES.json",
+        ),
+      ),
+    ).toBe(true);
+    expect(fs.existsSync(path.join(inferredContext, "agents/hermes/Dockerfile"))).toBe(false);
     expect(plan.authority.sourceRevision).toMatch(/^[a-f0-9]{40,64}$/u);
     expect(plan.authority.contextManifestSha256).toMatch(/^[a-f0-9]{64}$/u);
     const stagedDockerfile = fs.readFileSync(first.dockerfilePath, "utf8");

src/lib/onboard/experimental/hermes-portable-build-context.ts

@@ -26,6 +26,8 @@ const OPEN_READ_FLAGS =
   fs.constants.O_RDONLY |
   (typeof fs.constants.O_NOFOLLOW === "number" ? fs.constants.O_NOFOLLOW : 0) |
   (typeof fs.constants.O_NONBLOCK === "number" ? fs.constants.O_NONBLOCK : 0);
+const SOURCE_DOCKERFILE_RELATIVE_PATH = "agents/hermes/Dockerfile" as const;
+const CONTEXT_DOCKERFILE_RELATIVE_PATH = "Dockerfile" as const;
 
 const LOCAL_COPY_SOURCES = [
   "agents/hermes/build-mcp-digest.py",
@@ -128,7 +130,7 @@ type DirectoryEvidence = {
 export interface HermesPortableBuildContextAuthority {
   readonly schemaVersion: typeof CONTEXT_SCHEMA_VERSION;
   readonly sourceRevision: string;
-  readonly dockerfileRelativePath: "agents/hermes/Dockerfile";
+  readonly dockerfileRelativePath: typeof CONTEXT_DOCKERFILE_RELATIVE_PATH;
   readonly sourceManifestSha256: string;
   readonly contextManifestSha256: string;
 }
@@ -640,7 +642,7 @@ function captureSourceEntries(
     entries.push(entry);
   };
 
-  visit("agents/hermes/Dockerfile");
+  visit(SOURCE_DOCKERFILE_RELATIVE_PATH);
   for (const token of LOCAL_COPY_SOURCES) {
     if (token.endsWith("/")) {
       visit(token.slice(0, -1));
@@ -676,7 +678,7 @@ function sourceAuthority(
   });
   const contextManifestSha256 = canonicalDigest({
     schemaVersion: CONTEXT_SCHEMA_VERSION,
-    dockerfileRelativePath: "agents/hermes/Dockerfile",
+    dockerfileRelativePath: CONTEXT_DOCKERFILE_RELATIVE_PATH,
     entries: contextEntries.map((entry) => ({
       kind: entry.kind,
       relativePath: entry.relativePath,
@@ -687,7 +689,7 @@ function sourceAuthority(
   return {
     schemaVersion: CONTEXT_SCHEMA_VERSION,
     sourceRevision: revision.revision,
-    dockerfileRelativePath: "agents/hermes/Dockerfile",
+    dockerfileRelativePath: CONTEXT_DOCKERFILE_RELATIVE_PATH,
     sourceManifestSha256,
     contextManifestSha256,
   };
@@ -698,12 +700,20 @@ function renderContextEntries(
   settings: HermesPortableBuildContextSettings,
 ): readonly SourceEntry[] {
   return sourceEntries.map((entry) => {
-    if (entry.kind !== "file" || entry.relativePath !== "agents/hermes/Dockerfile") return entry;
+    if (entry.kind !== "file" || entry.relativePath !== SOURCE_DOCKERFILE_RELATIVE_PATH) {
+      return entry;
+    }
     const bytes = Buffer.from(
       renderHermesPortableDockerfileBuildSettings(UTF8.decode(entry.bytes!), settings),
       "utf8",
     );
-    return { ...entry, bytes, size: bytes.byteLength, sha256: digest(bytes) };
+    return {
+      ...entry,
+      relativePath: CONTEXT_DOCKERFILE_RELATIVE_PATH,
+      bytes,
+      size: bytes.byteLength,
+      sha256: digest(bytes),
+    };
   });
 }
 
@@ -724,7 +734,7 @@ function capture(
   );
   const sourceEntries = captureSourceEntries(rootPath, tracked);
   const dockerfile = sourceEntries.find(
-    (entry) => entry.relativePath === "agents/hermes/Dockerfile",
+    (entry) => entry.relativePath === SOURCE_DOCKERFILE_RELATIVE_PATH,
   );
   if (dockerfile?.kind !== "file" || !dockerfile.bytes) fail("Dockerfile source is unavailable");
   parseDockerfileSources(dockerfile.bytes);
@@ -1375,7 +1385,7 @@ export function createHermesPortableBuildContextPlan(
   };
   return {
     authority: captured.authority,
-    sourceDockerfilePath: path.join(rootPath, captured.authority.dockerfileRelativePath),
+    sourceDockerfilePath: path.join(rootPath, SOURCE_DOCKERFILE_RELATIVE_PATH),
     assertCurrentSource,
     materialize: (input) => {
       assertCurrentSource();

src/lib/onboard/experimental/hermes-portable-onboarding.test.ts

@@ -724,20 +724,29 @@ describe("Hermes portable onboarding transaction", () => {
     expect(fixture.events).not.toContain("create");
   });
 
-  it("classifies absence only after a reachable gateway returns exact sandbox-not-found evidence (#9203)", () => {
-    const capture = vi
-      .fn()
-      .mockReturnValueOnce({ status: 0, stdout: "[]", stderr: "" })
-      .mockReturnValueOnce({ status: 1, stdout: "", stderr: "sandbox 'alpha' not found" });
-
-    expect(observeHermesPortableSandbox("alpha", "nemoclaw", capture)).toEqual({
-      kind: "absent",
-    });
-    expect(capture.mock.calls).toEqual([
-      [["sandbox", "list", "-g", "nemoclaw"]],
-      [["sandbox", "get", "-g", "nemoclaw", "alpha"]],
-    ]);
-  });
+  it.each([
+    ["named sandbox-not-found", "sandbox 'alpha' not found"],
+    [
+      "OpenShell 0.0.106 entity-not-found",
+      "Error:   × code: 'Some requested entity was not found', message: \"sandbox not found\"",
+    ],
+  ])(
+    "classifies absence after a reachable gateway returns %s evidence (#9203)",
+    (_label, stderr) => {
+      const capture = vi
+        .fn()
+        .mockReturnValueOnce({ status: 0, stdout: "[]", stderr: "" })
+        .mockReturnValueOnce({ status: 1, stdout: "", stderr });
+
+      expect(observeHermesPortableSandbox("alpha", "nemoclaw", capture)).toEqual({
+        kind: "absent",
+      });
+      expect(capture.mock.calls).toEqual([
+        [["sandbox", "list", "-g", "nemoclaw"]],
+        [["sandbox", "get", "-g", "nemoclaw", "alpha"]],
+      ]);
+    },
+  );
 
   it("accepts Ready identity only from the exact receipt gateway (#9203)", () => {
     const capture = vi
@@ -796,6 +805,16 @@ describe("Hermes portable onboarding transaction", () => {
     ["gateway missing", { status: 1, stdout: "", stderr: "gateway not found" }, false],
     ["transport failure", { status: null, stdout: "", stderr: "transport unavailable" }, true],
     ["unnamed sandbox", { status: 1, stdout: "", stderr: "unknown sandbox" }, true],
+    [
+      "entity-not-found response for a different resource",
+      {
+        status: 1,
+        stdout: "",
+        stderr:
+          "Error:   × code: 'Some requested entity was not found', message: \"gateway not found\"",
+      },
+      true,
+    ],
     ["ambiguous absence", { status: 1, stdout: "", stderr: "no sandbox connection" }, true],
   ])(
     "keeps %s fail-closed instead of treating it as sandbox absence (#9203)",

src/lib/onboard/experimental/hermes-portable-onboarding.ts

@@ -567,8 +567,11 @@ export function observeHermesPortableSandbox(
     `^(?:Error:\\s*)?sandbox ['\"]?${escapedRegExp(sandboxName)}['\"]? not found\\.?$`,
     "u",
   );
-  const coded = /^Error: code: 'NotFound', message: "sandbox not found"$/u;
-  return named.test(output) || coded.test(output)
+  const coded =
+    output === `Error: code: 'NotFound', message: "sandbox not found"` ||
+    output ===
+      `Error:   × code: 'Some requested entity was not found', message: "sandbox not found"`;
+  return named.test(output) || coded
     ? { kind: "absent" }
     : { kind: "ambiguous", detail: "sandbox get did not prove exact sandbox absence" };
 }

src/lib/onboard/runtime-provider/podman-host-local-inference-probe-inspect.test.ts

@@ -0,0 +1,90 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { describe, expect, it } from "vitest";
+
+import { createPodmanHostLocalInferenceTestHarness } from "../../../../test/helpers/podman-host-local-inference-test-harness";
+import type { HostLocalInferenceRuntime } from "./host-local-inference";
+import { createPodmanHostLocalInferenceOperation } from "./podman-host-local-inference";
+
+function operationRuntime(
+  harness: ReturnType<typeof createPodmanHostLocalInferenceTestHarness>,
+): HostLocalInferenceRuntime {
+  const operation = createPodmanHostLocalInferenceOperation({
+    engine: harness.engine,
+    env: harness.env,
+    acceleration: harness.operationAcceleration,
+    authorityStore: harness.authorityStore,
+    routeAuthorityStore: harness.routeAuthorityStore,
+    onFailureEvidence: harness.onFailureEvidence,
+    redactSensitive: harness.redactSensitive,
+  });
+  return (
+    operation.managedRuntime ??
+    (() => {
+      throw new Error("test operation lacks managed runtime");
+    })()
+  );
+}
+
+const probeId = "c".repeat(64);
+const inspectEvent = `podman:container inspect ${probeId}`;
+
+describe("Podman host-local inference post-create probe inspection", () => {
+  it("retries one timeout using the same full probe ID", () => {
+    const harness = createPodmanHostLocalInferenceTestHarness();
+    harness.state.probePostCreateInspectTimeoutsRemaining = 1;
+
+    const prepared = operationRuntime(harness).startManaged(harness.input, harness.writer);
+
+    const waitIndex = harness.events.indexOf(`podman:wait ${probeId}`);
+    expect(waitIndex).toBeGreaterThan(0);
+    expect(
+      harness.events.slice(0, waitIndex).filter((event) => event === inspectEvent),
+    ).toHaveLength(2);
+    expect(prepared.receipt.service).toBe("nim");
+    expect(harness.probe()).toBeNull();
+  });
+
+  it("fails closed after two timeouts without acting on the probe", () => {
+    const harness = createPodmanHostLocalInferenceTestHarness();
+    harness.state.probePostCreateInspectTimeoutsRemaining = 2;
+    harness.state.probeForbiddenActions = ["wait", "logs", "rm"];
+
+    expect(() => operationRuntime(harness).startManaged(harness.input, harness.writer)).toThrow(
+      "probe identity is indeterminate after create",
+    );
+
+    expect(harness.events.filter((event) => event === inspectEvent)).toHaveLength(2);
+    expect(harness.probe()).toMatchObject({ id: probeId });
+    expect(harness.written).toHaveLength(0);
+  });
+
+  it("does not retry a non-timeout failure", () => {
+    const harness = createPodmanHostLocalInferenceTestHarness();
+    harness.state.probePostCreateInspectFailuresRemaining = 1;
+    harness.state.probeForbiddenActions = ["wait", "logs", "rm"];
+
+    expect(() => operationRuntime(harness).startManaged(harness.input, harness.writer)).toThrow(
+      "probe identity is indeterminate after create",
+    );
+
+    expect(harness.events.filter((event) => event === inspectEvent)).toHaveLength(1);
+    expect(harness.probe()).toMatchObject({ id: probeId });
+    expect(harness.written).toHaveLength(0);
+  });
+
+  it("does not retry an inspect result with a mismatched container ID", () => {
+    const harness = createPodmanHostLocalInferenceTestHarness();
+    harness.state.probeInspectRuntimeIdMismatchAt = 1;
+    harness.state.probeForbiddenActions = ["wait", "logs", "rm"];
+
+    expect(() => operationRuntime(harness).startManaged(harness.input, harness.writer)).toThrow(
+      "probe identity is indeterminate after create",
+    );
+
+    expect(harness.events.filter((event) => event === inspectEvent)).toHaveLength(1);
+    expect(harness.probe()).toMatchObject({ id: probeId });
+    expect(harness.written).toHaveLength(0);
+  });
+});

src/lib/onboard/runtime-provider/podman-host-local-inference.ts

@@ -374,13 +374,21 @@ function sanitizeFailureResult(
   redactor: PodmanInferenceRedactor,
 ): ContainerEngineCommandResult {
   const failed = result.status !== 0 || result.error !== undefined;
+  const error =
+    result.error === undefined
+      ? undefined
+      : Object.assign(new Error(redactEvidence(redactor, result.error.message)), {
+          ...((result.error as NodeJS.ErrnoException).code === "ETIMEDOUT"
+            ? { code: "ETIMEDOUT" }
+            : {}),
+        });
   return Object.freeze({
     status: result.status,
     // Successful stdout remains intact because it carries provider JSON and
     // exact runtime identities. Stderr is never parsed and is always redacted.
     stdout: failed ? redactEvidence(redactor, result.stdout) : result.stdout,
     stderr: redactEvidence(redactor, result.stderr),
-    ...(result.error ? { error: new Error(redactEvidence(redactor, result.error.message)) } : {}),
+    ...(error ? { error } : {}),
   });
 }
 
@@ -425,6 +433,10 @@ function commandEvidence(result: ContainerEngineCommandResult): string {
   );
 }
 
+function commandTimedOut(result: ContainerEngineCommandResult): boolean {
+  return (result.error as NodeJS.ErrnoException | undefined)?.code === "ETIMEDOUT";
+}
+
 function redactedCommandEvidence(
   redactor: PodmanInferenceRedactor,
   result: ContainerEngineCommandResult,
@@ -1164,11 +1176,17 @@ function inspectContainer(engine: ContainerEngine, runtimeId: string): ManagedCo
   });
 }
 
-function inspectProbeContainer(engine: ContainerEngine, runtimeId: string): ProbeContainer {
-  const output = requireSuccess(
-    "probe container inspection",
-    engine.capture(["container", "inspect", runtimeId], PROBE_TIMEOUT_MS),
-  );
+function inspectProbeContainer(
+  engine: ContainerEngine,
+  runtimeId: string,
+  retryTimeoutOnce = false,
+): ProbeContainer {
+  const args = ["container", "inspect", runtimeId] as const;
+  let result = engine.capture(args, PROBE_TIMEOUT_MS);
+  if (retryTimeoutOnce && commandTimedOut(result)) {
+    result = engine.capture(args, PROBE_TIMEOUT_MS);
+  }
+  const output = requireSuccess("probe container inspection", result);
   let parsed: unknown;
   try {
     parsed = JSON.parse(output);
@@ -1957,7 +1975,11 @@ function executeExactProbe(
   }
   let container: ProbeContainer;
   try {
-    container = requireProbeIdentity(inspectProbeContainer(engine, runtimeId), spec, runtimeId);
+    container = requireProbeIdentity(
+      inspectProbeContainer(engine, runtimeId, true),
+      spec,
+      runtimeId,
+    );
   } catch (error) {
     captureFailure(error);
     throw new PodmanInferenceIndeterminateCleanupError(

test/helpers/hermes-portable-onboarding-fixture.ts

@@ -221,15 +221,15 @@ export function createHermesPortableTestInput(stateDir: string, policyPath: stri
       authority: {
         schemaVersion: 1,
         sourceRevision: "1".repeat(40),
-        dockerfileRelativePath: "agents/hermes/Dockerfile",
+        dockerfileRelativePath: "Dockerfile",
         sourceManifestSha256: "2".repeat(64),
         contextManifestSha256: "3".repeat(64),
       },
       sourceDockerfilePath,
       assertCurrentSource: vi.fn(),
       materialize: vi.fn(() => ({
         buildContextPath: "/private/staged-hermes",
-        dockerfilePath: "/private/staged-hermes/agents/hermes/Dockerfile",
+        dockerfilePath: "/private/staged-hermes/Dockerfile",
         assertCurrent: vi.fn(),
       })),
       retire: vi.fn(() => true),
@@ -390,7 +390,7 @@ export function createHermesPortableTransactionFixture(
       const policyIndex = argv.indexOf("--policy");
       expect(argv[policyIndex + 1]).toContain("policy.");
       expect(argv[argv.indexOf("--from") + 1]).toBe(
-        options.expectedDockerfilePath ?? "/private/staged-hermes/agents/hermes/Dockerfile",
+        options.expectedDockerfilePath ?? "/private/staged-hermes/Dockerfile",
       );
       expect(buildContextPath).toBe(options.expectedBuildContextPath ?? "/private/staged-hermes");
       present = true;