fix(onboard): verify the agent API host forward before reporting ready

docs/reference/troubleshooting.mdx

@@ -4893,6 +4893,26 @@ Expected output:
 Point an OpenAI-compatible client at `http://127.0.0.1:8642/v1` for chat completions.
 For terminal use, run `nemohermes launch <name>`.
 
+### Onboarding reports "Hermes is not ready" with an unreachable API port
+
+Deployment verification probes the OpenAI-compatible API on its host forward, not only inside the sandbox.
+When the sandbox gateway is healthy but the host forward never came up, verification reports the API link as failed and onboarding finishes with `Hermes is not ready` instead of a successful handoff:
+
+```text
+✗ api: port forward not working (connection refused)
+  The OpenAI-compatible API on port 8642 is not reachable from the host. Run: openshell forward start --background 8642 <name>
+```
+
+The sandbox itself is intact, so start the forward and re-check:
+
+```bash
+openshell forward list                        # confirm no row owns the API port
+openshell forward start --background 8642 <name>
+curl -sf http://127.0.0.1:8642/health
+```
+
+If the forward still refuses connections, a foreign host listener usually holds the port; stop it, then rerun `$$nemoclaw onboard` to complete verification.
+
 ### `docker port` shows no mapping for 8642 even though forwarding works
 
 OpenShell port forwards are host-side relays managed by the OpenShell gateway process, not Docker `-p` publish mappings on the sandbox container.

src/lib/onboard.ts

@@ -2906,6 +2906,7 @@ const setupOpenclaw = createOpenclawSetup({
 
 const {
   buildChain,
+  resolveVerifyAgentApiPort,
   buildControlUiUrls,
   buildOrphanedSandboxRollbackMessage,
   ensureDashboardForward,
@@ -3644,8 +3645,8 @@ async function runOnboard(opts: OnboardOptions = {}): Promise<void> {
         removeLegacyCredentialsFile,
         cleanupStaleHostFiles,
         getChatUiUrl: () => process.env.CHAT_UI_URL || `http://127.0.0.1:${DASHBOARD_PORT}`,
-        buildVerifyChain: (chatUiUrl) =>
-          buildChain({ chatUiUrl, isWsl: isWsl(), wslHostAddress: getWslHostAddress(), dashboardHealthEndpoint: agent?.dashboard.healthPath, gatewayPort: agent?.healthProbe?.port, gatewayHealthEndpoint: agent?.healthProbe?.url }),
+        buildVerifyChain: (chatUiUrl, verifiedSandboxName) =>
+          buildChain({ chatUiUrl, isWsl: isWsl(), wslHostAddress: getWslHostAddress(), dashboardHealthEndpoint: agent?.dashboard.healthPath, gatewayPort: resolveVerifyAgentApiPort(verifiedSandboxName, agent), gatewayHealthEndpoint: agent?.healthProbe?.url }),
         verifyDeployment: async (name, chain) => {
           const verifyDeploymentModule: typeof import("./verify-deployment") =
             require("./verify-deployment");

src/lib/onboard/agent-dashboard-forward.ts

@@ -11,6 +11,11 @@ import {
 } from "./dashboard-runtime";
 import { resolveOnboardHermesApiPort } from "./hermes-api-port";
 
+// The port deployment verification must probe lives with the rest of this
+// module's "which host port does this agent publish" logic, so onboarding
+// reaches it through the dashboard helpers it already consumes (#9290).
+export { resolveVerifyAgentApiPort } from "./hermes-api-port";
+
 export type EnsureDashboardForward = (
   sandboxName: string,
   chatUiUrl?: string,

src/lib/onboard/dashboard.ts

@@ -14,6 +14,7 @@ import { runCapture as defaultRunCapture } from "../runner";
 import {
   ensureAgentDashboardForward as ensureAgentDashboardForwardForAgent,
   replaceUrlPort,
+  resolveVerifyAgentApiPort,
 } from "./agent-dashboard-forward";
 import { ensureAgentFixedForward as ensureFixedAgentForward } from "./agent-fixed-forward";
 import { fetchAgentWebAuthTokenFromSandbox as fetchAgentWebAuthToken } from "./agent-web-auth-token";
@@ -88,6 +89,7 @@ export interface OnboardDashboardDeps {
 
 export interface OnboardDashboardHelpers {
   buildChain: typeof buildChain;
+  resolveVerifyAgentApiPort: typeof resolveVerifyAgentApiPort;
   buildControlUiUrls: typeof buildControlUiUrls;
   buildOrphanedSandboxRollbackMessage(
     sandboxName: string,
@@ -630,6 +632,7 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa
 
   return {
     buildChain,
+    resolveVerifyAgentApiPort,
     buildControlUiUrls,
     buildOrphanedSandboxRollbackMessage,
     ensureDashboardForward,

src/lib/onboard/hermes-api-port.test.ts

@@ -13,6 +13,7 @@ import {
   reserveCreateSandboxHermesApiPort,
   resolveOnboardHermesApiPort,
   resolveSandboxHermesApiPort,
+  resolveVerifyAgentApiPort,
   retargetHermesApiPortInUrl,
   withHermesApiPortReservationScope,
 } from "./hermes-api-port";
@@ -323,3 +324,34 @@ describe("retargetHermesApiPortInUrl", () => {
     );
   });
 });
+
+describe("resolveVerifyAgentApiPort (#9290)", () => {
+  const hermes = { name: "hermes", healthProbe: { port: 8642 } };
+
+  it("targets the port this Hermes sandbox actually owns", () => {
+    // A second Hermes sandbox serves its API on a reallocated port; probing the
+    // manifest default would report a sibling sandbox's port as unreachable.
+    expect(
+      resolveVerifyAgentApiPort("second", hermes, { getSandbox: () => ({ hermesApiPort: 8643 }) }),
+    ).toBe(8643);
+  });
+
+  it("falls back to the manifest default when the sandbox is not registered yet", () => {
+    expect(resolveVerifyAgentApiPort("fresh", hermes, { getSandbox: () => null })).toBe(8642);
+  });
+
+  it("keeps a non-Hermes agent's declared probe port", () => {
+    expect(
+      resolveVerifyAgentApiPort("sb", { name: "other", healthProbe: { port: 9000 } }, {
+        getSandbox: () => ({ hermesApiPort: 8643 }),
+      }),
+    ).toBe(9000);
+  });
+
+  it("returns undefined when the agent declares no health probe port", () => {
+    expect(resolveVerifyAgentApiPort("sb", { name: "openclaw" }, { getSandbox: () => null })).toBe(
+      undefined,
+    );
+    expect(resolveVerifyAgentApiPort("sb", null, { getSandbox: () => null })).toBe(undefined);
+  });
+});

src/lib/onboard/hermes-api-port.ts

@@ -388,3 +388,27 @@ export function resolveOnboardHermesApiPort(
   }
   return publish(port);
 }
+
+/**
+ * Resolve the API port deployment verification must probe for `agent`.
+ *
+ * Returns the agent's declared health-probe port, except for Hermes, whose
+ * per-sandbox allocation from the 8642-8652 range means the manifest default
+ * would name a sibling sandbox's port. Returns undefined when the agent
+ * declares no health-probe port, which leaves `buildChain` on its dashboard-port
+ * fallback so agents without a separate API surface keep their existing single
+ * host probe (#9290).
+ */
+export function resolveVerifyAgentApiPort(
+  sandboxName: string,
+  agent: { name?: string; healthProbe?: { port?: number } | null } | null | undefined,
+  options: {
+    getSandbox?: (name: string) => { hermesApiPort?: number | null } | null | undefined;
+  } = {},
+): number | undefined {
+  const declared = agent?.healthProbe?.port;
+  if (!Number.isInteger(declared)) return undefined;
+  if (agent?.name !== "hermes" || declared !== HERMES_OPENAI_API_PORT) return declared;
+  const getSandbox = options.getSandbox ?? registry.getSandbox;
+  return resolveSandboxHermesApiPort(getSandbox(sandboxName) ?? {});
+}

src/lib/onboard/machine/final-flow-phases.runtime.test.ts

@@ -21,6 +21,7 @@ function deploymentResult(healthy: boolean): VerifyDeploymentResult {
       gatewayVersion: "test",
       inferenceRouteWorking: healthy,
       dashboardReachable: true,
+      agentApiReachable: null,
       messagingBridgesHealthy: true,
       messagingRuntimeChannelsMissing: null,
       messagingConfigChannelsMissing: null,
@@ -269,6 +270,7 @@ describe("final onboard flow runtime boundary", () => {
             gatewayVersion: "test",
             inferenceRouteWorking: true,
             dashboardReachable: true,
+            agentApiReachable: null,
             messagingBridgesHealthy: true,
             messagingRuntimeChannelsMissing: null,
             messagingConfigChannelsMissing: null,

src/lib/onboard/machine/handlers/finalization.test.ts

@@ -133,7 +133,9 @@ describe("finalization handlers", () => {
     );
     expect(calls.cleanupHost).toHaveBeenCalledOnce();
     expect(calls.recoverProcesses).toHaveBeenCalledWith("my-assistant", { quiet: true });
-    expect(calls.buildChain).toHaveBeenCalledWith("http://127.0.0.1:18789");
+    // The sandbox name lets the chain resolve this sandbox's own agent API
+    // port rather than the agent manifest default (#9290).
+    expect(calls.buildChain).toHaveBeenCalledWith("http://127.0.0.1:18789", "my-assistant");
     expect(calls.verify).toHaveBeenCalledWith("my-assistant", { port: 18789 });
     expect(calls.log).toHaveBeenCalledWith("  ✓ verified");
     expect(calls.dashboard).toHaveBeenCalledWith(

src/lib/onboard/machine/handlers/finalization.ts

@@ -61,7 +61,12 @@ export interface FinalizationStateOptions<Agent, VerifyChain, VerificationResult
      */
     warmupScopeUpgrade(sandboxName: string): void;
     getChatUiUrl(): string;
-    buildVerifyChain(chatUiUrl: string): VerifyChain;
+    /**
+     * `sandboxName` lets the chain target the API port this sandbox actually
+     * owns: Hermes allocates a per-sandbox port from the 8642-8652 range, so
+     * the manifest default would probe a sibling sandbox's port (#9290).
+     */
+    buildVerifyChain(chatUiUrl: string, sandboxName: string): VerifyChain;
     verifyDeployment(sandboxName: string, chain: VerifyChain): Promise<VerificationResult>;
     formatVerificationDiagnostics(result: VerificationResult): string[];
     isDeploymentHealthy(result: VerificationResult): boolean;
@@ -231,7 +236,7 @@ export async function handlePostVerifyState<Agent, VerifyChain, VerificationResu
       (webSearchProvider !== null &&
         deps.verifyWebSearchInsideSandbox(sandboxName, agent, webSearchProvider));
     // Confirm the delivered sandbox is reachable before printing the live dashboard (#2342).
-    const verifyChain = deps.buildVerifyChain(deps.getChatUiUrl());
+    const verifyChain = deps.buildVerifyChain(deps.getChatUiUrl(), sandboxName);
     const verificationResult = await deps.verifyDeployment(sandboxName, verifyChain);
     deploymentHealthy =
       webSearchCredentialBoundarySafe && deps.isDeploymentHealthy(verificationResult);

src/lib/verify-deployment-agent.test.ts

@@ -54,3 +54,88 @@ describe("verifyDeployment agent dashboard probes", () => {
     expect(hostProbes).toContainEqual({ port: 18789, path: "/api/status" });
   });
 });
+
+describe("verifyDeployment agent OpenAI-compatible API host forward (#9290)", () => {
+  const agentChain = buildChain({
+    chatUiUrl: "http://127.0.0.1:18789",
+    dashboardHealthEndpoint: "/api/status",
+    gatewayPort: 8642,
+    gatewayHealthEndpoint: "/health",
+  });
+
+  function runWithApiHostCode(apiCode: number) {
+    const hostProbes: Array<{ port: number; path: string }> = [];
+    const deps = makeDeps({
+      probeHostPort: (port: number, path: string) => {
+        hostProbes.push({ port, path });
+        return port === 8642 ? apiCode : 200;
+      },
+    });
+    return { hostProbes, run: () => verifyDeployment("my-sandbox", agentChain, deps, NO_RETRY) };
+  }
+
+  it("probes the API port on the host, not just inside the sandbox", async () => {
+    const { hostProbes, run } = runWithApiHostCode(200);
+
+    const result = await run();
+
+    // The in-sandbox gateway probe only proves the API answers inside the
+    // sandbox; the host forward is what operators actually connect through.
+    expect(hostProbes).toContainEqual({ port: 8642, path: "/health" });
+    expect(result.healthy).toBe(true);
+    expect(result.verification.agentApiReachable).toBe(true);
+  });
+
+  it("fails verification when the API host forward refuses connections", async () => {
+    // The reported regression: the sandbox gateway is healthy and the dashboard
+    // forward is up, but port 8642 never became reachable on the host.
+    const { run } = runWithApiHostCode(0);
+
+    const result = await run();
+
+    expect(result.healthy).toBe(false);
+    expect(result.verification.agentApiReachable).toBe(false);
+    expect(result.verification.gatewayReachable).toBe(true);
+    expect(result.verification.dashboardReachable).toBe(true);
+    const api = result.diagnostics.find((d) => d.link === "api");
+    expect(api?.status).toBe("fail");
+    expect(api?.hint).toContain("openshell forward start --background 8642 my-sandbox");
+  });
+
+  it("fails verification when the API host forward answers with a server error", async () => {
+    const { run } = runWithApiHostCode(502);
+
+    const result = await run();
+
+    expect(result.healthy).toBe(false);
+    expect(result.diagnostics.find((d) => d.link === "api")?.detail).toContain("502");
+  });
+
+  it("accepts an authenticated API host forward (HTTP 401)", async () => {
+    // Bearer auth is enabled on the Hermes API; 401 still proves the forward
+    // reaches a live listener.
+    const { run } = runWithApiHostCode(401);
+
+    expect((await run()).healthy).toBe(true);
+  });
+
+  it("keeps a single host probe for agents without a separate API port", async () => {
+    // Regression lock: OpenClaw declares no gateway port, so buildChain falls
+    // back to the dashboard port and no API probe or diagnostic is added.
+    const openClawChain = buildChain({ chatUiUrl: "http://127.0.0.1:18789" });
+    const hostProbes: Array<{ port: number; path: string }> = [];
+    const deps = makeDeps({
+      probeHostPort: (port: number, path: string) => {
+        hostProbes.push({ port, path });
+        return 200;
+      },
+    });
+
+    const result = await verifyDeployment("my-sandbox", openClawChain, deps, NO_RETRY);
+
+    expect(hostProbes).toEqual([{ port: 18789, path: "/health" }]);
+    expect(result.diagnostics.some((d) => d.link === "api")).toBe(false);
+    expect(result.verification.agentApiReachable).toBeNull();
+    expect(result.healthy).toBe(true);
+  });
+});