fix(hermes): recognize OpenShell sandbox absence

src/lib/onboard/experimental/hermes-portable-onboarding.test.ts

@@ -724,20 +724,29 @@ describe("Hermes portable onboarding transaction", () => {
     expect(fixture.events).not.toContain("create");
   });
 
-  it("classifies absence only after a reachable gateway returns exact sandbox-not-found evidence (#9203)", () => {
-    const capture = vi
-      .fn()
-      .mockReturnValueOnce({ status: 0, stdout: "[]", stderr: "" })
-      .mockReturnValueOnce({ status: 1, stdout: "", stderr: "sandbox 'alpha' not found" });
-
-    expect(observeHermesPortableSandbox("alpha", "nemoclaw", capture)).toEqual({
-      kind: "absent",
-    });
-    expect(capture.mock.calls).toEqual([
-      [["sandbox", "list", "-g", "nemoclaw"]],
-      [["sandbox", "get", "-g", "nemoclaw", "alpha"]],
-    ]);
-  });
+  it.each([
+    ["named sandbox-not-found", "sandbox 'alpha' not found"],
+    [
+      "OpenShell 0.0.106 entity-not-found",
+      "Error:   × code: 'Some requested entity was not found', message: \"sandbox not found\"",
+    ],
+  ])(
+    "classifies absence after a reachable gateway returns %s evidence (#9203)",
+    (_label, stderr) => {
+      const capture = vi
+        .fn()
+        .mockReturnValueOnce({ status: 0, stdout: "[]", stderr: "" })
+        .mockReturnValueOnce({ status: 1, stdout: "", stderr });
+
+      expect(observeHermesPortableSandbox("alpha", "nemoclaw", capture)).toEqual({
+        kind: "absent",
+      });
+      expect(capture.mock.calls).toEqual([
+        [["sandbox", "list", "-g", "nemoclaw"]],
+        [["sandbox", "get", "-g", "nemoclaw", "alpha"]],
+      ]);
+    },
+  );
 
   it("accepts Ready identity only from the exact receipt gateway (#9203)", () => {
     const capture = vi
@@ -796,6 +805,16 @@ describe("Hermes portable onboarding transaction", () => {
     ["gateway missing", { status: 1, stdout: "", stderr: "gateway not found" }, false],
     ["transport failure", { status: null, stdout: "", stderr: "transport unavailable" }, true],
     ["unnamed sandbox", { status: 1, stdout: "", stderr: "unknown sandbox" }, true],
+    [
+      "entity-not-found response for a different resource",
+      {
+        status: 1,
+        stdout: "",
+        stderr:
+          "Error:   × code: 'Some requested entity was not found', message: \"gateway not found\"",
+      },
+      true,
+    ],
     ["ambiguous absence", { status: 1, stdout: "", stderr: "no sandbox connection" }, true],
   ])(
     "keeps %s fail-closed instead of treating it as sandbox absence (#9203)",

src/lib/onboard/experimental/hermes-portable-onboarding.ts

@@ -567,8 +567,11 @@ export function observeHermesPortableSandbox(
     `^(?:Error:\\s*)?sandbox ['\"]?${escapedRegExp(sandboxName)}['\"]? not found\\.?$`,
     "u",
   );
-  const coded = /^Error: code: 'NotFound', message: "sandbox not found"$/u;
-  return named.test(output) || coded.test(output)
+  const coded =
+    output === `Error: code: 'NotFound', message: "sandbox not found"` ||
+    output ===
+      `Error:   × code: 'Some requested entity was not found', message: "sandbox not found"`;
+  return named.test(output) || coded
     ? { kind: "absent" }
     : { kind: "ambiguous", detail: "sandbox get did not prove exact sandbox absence" };
 }

src/lib/onboard/runtime-provider/podman-host-local-inference-probe-inspect.test.ts

@@ -0,0 +1,90 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { describe, expect, it } from "vitest";
+
+import { createPodmanHostLocalInferenceTestHarness } from "../../../../test/helpers/podman-host-local-inference-test-harness";
+import type { HostLocalInferenceRuntime } from "./host-local-inference";
+import { createPodmanHostLocalInferenceOperation } from "./podman-host-local-inference";
+
+function operationRuntime(
+  harness: ReturnType<typeof createPodmanHostLocalInferenceTestHarness>,
+): HostLocalInferenceRuntime {
+  const operation = createPodmanHostLocalInferenceOperation({
+    engine: harness.engine,
+    env: harness.env,
+    acceleration: harness.operationAcceleration,
+    authorityStore: harness.authorityStore,
+    routeAuthorityStore: harness.routeAuthorityStore,
+    onFailureEvidence: harness.onFailureEvidence,
+    redactSensitive: harness.redactSensitive,
+  });
+  return (
+    operation.managedRuntime ??
+    (() => {
+      throw new Error("test operation lacks managed runtime");
+    })()
+  );
+}
+
+const probeId = "c".repeat(64);
+const inspectEvent = `podman:container inspect ${probeId}`;
+
+describe("Podman host-local inference post-create probe inspection", () => {
+  it("retries one timeout using the same full probe ID", () => {
+    const harness = createPodmanHostLocalInferenceTestHarness();
+    harness.state.probePostCreateInspectTimeoutsRemaining = 1;
+
+    const prepared = operationRuntime(harness).startManaged(harness.input, harness.writer);
+
+    const waitIndex = harness.events.indexOf(`podman:wait ${probeId}`);
+    expect(waitIndex).toBeGreaterThan(0);
+    expect(
+      harness.events.slice(0, waitIndex).filter((event) => event === inspectEvent),
+    ).toHaveLength(2);
+    expect(prepared.receipt.service).toBe("nim");
+    expect(harness.probe()).toBeNull();
+  });
+
+  it("fails closed after two timeouts without acting on the probe", () => {
+    const harness = createPodmanHostLocalInferenceTestHarness();
+    harness.state.probePostCreateInspectTimeoutsRemaining = 2;
+    harness.state.probeForbiddenActions = ["wait", "logs", "rm"];
+
+    expect(() => operationRuntime(harness).startManaged(harness.input, harness.writer)).toThrow(
+      "probe identity is indeterminate after create",
+    );
+
+    expect(harness.events.filter((event) => event === inspectEvent)).toHaveLength(2);
+    expect(harness.probe()).toMatchObject({ id: probeId });
+    expect(harness.written).toHaveLength(0);
+  });
+
+  it("does not retry a non-timeout failure", () => {
+    const harness = createPodmanHostLocalInferenceTestHarness();
+    harness.state.probePostCreateInspectFailuresRemaining = 1;
+    harness.state.probeForbiddenActions = ["wait", "logs", "rm"];
+
+    expect(() => operationRuntime(harness).startManaged(harness.input, harness.writer)).toThrow(
+      "probe identity is indeterminate after create",
+    );
+
+    expect(harness.events.filter((event) => event === inspectEvent)).toHaveLength(1);
+    expect(harness.probe()).toMatchObject({ id: probeId });
+    expect(harness.written).toHaveLength(0);
+  });
+
+  it("does not retry an inspect result with a mismatched container ID", () => {
+    const harness = createPodmanHostLocalInferenceTestHarness();
+    harness.state.probeInspectRuntimeIdMismatchAt = 1;
+    harness.state.probeForbiddenActions = ["wait", "logs", "rm"];
+
+    expect(() => operationRuntime(harness).startManaged(harness.input, harness.writer)).toThrow(
+      "probe identity is indeterminate after create",
+    );
+
+    expect(harness.events.filter((event) => event === inspectEvent)).toHaveLength(1);
+    expect(harness.probe()).toMatchObject({ id: probeId });
+    expect(harness.written).toHaveLength(0);
+  });
+});

src/lib/onboard/runtime-provider/podman-host-local-inference.ts

@@ -374,13 +374,21 @@ function sanitizeFailureResult(
   redactor: PodmanInferenceRedactor,
 ): ContainerEngineCommandResult {
   const failed = result.status !== 0 || result.error !== undefined;
+  const error =
+    result.error === undefined
+      ? undefined
+      : Object.assign(new Error(redactEvidence(redactor, result.error.message)), {
+          ...((result.error as NodeJS.ErrnoException).code === "ETIMEDOUT"
+            ? { code: "ETIMEDOUT" }
+            : {}),
+        });
   return Object.freeze({
     status: result.status,
     // Successful stdout remains intact because it carries provider JSON and
     // exact runtime identities. Stderr is never parsed and is always redacted.
     stdout: failed ? redactEvidence(redactor, result.stdout) : result.stdout,
     stderr: redactEvidence(redactor, result.stderr),
-    ...(result.error ? { error: new Error(redactEvidence(redactor, result.error.message)) } : {}),
+    ...(error ? { error } : {}),
   });
 }
 
@@ -425,6 +433,10 @@ function commandEvidence(result: ContainerEngineCommandResult): string {
   );
 }
 
+function commandTimedOut(result: ContainerEngineCommandResult): boolean {
+  return (result.error as NodeJS.ErrnoException | undefined)?.code === "ETIMEDOUT";
+}
+
 function redactedCommandEvidence(
   redactor: PodmanInferenceRedactor,
   result: ContainerEngineCommandResult,
@@ -1164,11 +1176,17 @@ function inspectContainer(engine: ContainerEngine, runtimeId: string): ManagedCo
   });
 }
 
-function inspectProbeContainer(engine: ContainerEngine, runtimeId: string): ProbeContainer {
-  const output = requireSuccess(
-    "probe container inspection",
-    engine.capture(["container", "inspect", runtimeId], PROBE_TIMEOUT_MS),
-  );
+function inspectProbeContainer(
+  engine: ContainerEngine,
+  runtimeId: string,
+  retryTimeoutOnce = false,
+): ProbeContainer {
+  const args = ["container", "inspect", runtimeId] as const;
+  let result = engine.capture(args, PROBE_TIMEOUT_MS);
+  if (retryTimeoutOnce && commandTimedOut(result)) {
+    result = engine.capture(args, PROBE_TIMEOUT_MS);
+  }
+  const output = requireSuccess("probe container inspection", result);
   let parsed: unknown;
   try {
     parsed = JSON.parse(output);
@@ -1957,7 +1975,11 @@ function executeExactProbe(
   }
   let container: ProbeContainer;
   try {
-    container = requireProbeIdentity(inspectProbeContainer(engine, runtimeId), spec, runtimeId);
+    container = requireProbeIdentity(
+      inspectProbeContainer(engine, runtimeId, true),
+      spec,
+      runtimeId,
+    );
   } catch (error) {
     captureFailure(error);
     throw new PodmanInferenceIndeterminateCleanupError(

test/helpers/podman-host-local-inference-test-harness.ts

@@ -122,6 +122,8 @@ export interface PodmanHostLocalInferenceHarness {
     probeRunLostAcknowledgement: boolean;
     probeRunAcknowledgementText: string | null;
     probePostCreateNameLookupTimeout: boolean;
+    probePostCreateInspectFailuresRemaining: number;
+    probePostCreateInspectTimeoutsRemaining: number;
     probeInspectRuntimeIdMismatchAt: number | null;
     probeForbiddenActions: Array<"logs" | "rm" | "wait">;
     probeWaitTimeouts: number[];
@@ -424,6 +426,8 @@ export function createPodmanHostLocalInferenceTestHarness(
     probeRunLostAcknowledgement: false,
     probeRunAcknowledgementText: null as string | null,
     probePostCreateNameLookupTimeout: false,
+    probePostCreateInspectFailuresRemaining: 0,
+    probePostCreateInspectTimeoutsRemaining: 0,
     probeInspectRuntimeIdMismatchAt: null as number | null,
     probeForbiddenActions: [] as Array<"logs" | "rm" | "wait">,
     probeWaitTimeouts: [] as number[],
@@ -584,6 +588,18 @@ export function createPodmanHostLocalInferenceTestHarness(
         if (currentContainer?.id === args[2]) return result(0, inspectPayload(currentContainer));
         if (currentProbe?.id === args[2]) {
           probeInspectCount += 1;
+          if (state.probePostCreateInspectFailuresRemaining > 0) {
+            state.probePostCreateInspectFailuresRemaining -= 1;
+            const error = Object.assign(new Error("permission denied"), { code: "EACCES" });
+            return result(1, "", "permission denied", error);
+          }
+          if (state.probePostCreateInspectTimeoutsRemaining > 0) {
+            state.probePostCreateInspectTimeoutsRemaining -= 1;
+            const error = Object.assign(new Error("spawnSync /usr/local/bin/podman ETIMEDOUT"), {
+              code: "ETIMEDOUT",
+            });
+            return result(1, "", "spawnSync /usr/local/bin/podman ETIMEDOUT", error);
+          }
           const inspectedProbe =
             state.probeInspectRuntimeIdMismatchAt === probeInspectCount
               ? { ...currentProbe, id: REUSED_PROBE_CONTAINER_ID }