diff --git a/src/lib/actions/sandbox/gateway-state-drift.test.ts b/src/lib/actions/sandbox/gateway-state-drift.test.ts index 8bc9d0d027d..5fe24aae3f1 100644 --- a/src/lib/actions/sandbox/gateway-state-drift.test.ts +++ b/src/lib/actions/sandbox/gateway-state-drift.test.ts @@ -194,4 +194,67 @@ describe("sandbox gateway state drift guard", () => { ); expect(recoverNamedGatewayRuntimeSpy).toHaveBeenCalledWith({ gatewayName: "nemoclaw-8090" }); }); + + it("classifies the `sandbox has no spec` gRPC reply as a missing sandbox so the named-gateway reconciler can retry on the owning gateway", () => { + detectPreflightIssueSpy.mockReturnValue(null); + captureOpenshellSpy.mockReturnValue({ + status: 1, + output: + 'status: Internal, message: "sandbox has no spec", details: [], metadata: MetadataMap {}', + }); + + const lookup = gatewayState.getSandboxGatewayState("alpha"); + + expect(lookup.state).toBe("missing"); + expect(lookup.output).toContain("sandbox has no spec"); + }); + + it("classifies the same gRPC reply as `missing` on the async status-probe path so the live `nemoclaw status` lookup goes through the named-gateway reconciler too", async () => { + detectPreflightIssueSpy.mockReturnValue(null); + captureOpenshellForStatusSpy.mockResolvedValue({ + status: 1, + output: + 'status: Internal, message: "sandbox has no spec", details: [], metadata: MetadataMap {}', + }); + + const lookup = await gatewayState.getSandboxGatewayStateForStatus("alpha"); + + expect(lookup.state).toBe("missing"); + expect(lookup.output).toContain("sandbox has no spec"); + }); + + it("selects the sandbox's owning gateway and retries when the active gateway is a sibling that has no spec for it", () => { + detectPreflightIssueSpy.mockReturnValue(null); + getSandboxSpy.mockReturnValue({ + name: "instance-a", + gatewayName: "nemoclaw", + gatewayPort: 8080, + }); + getNamedGatewayLifecycleStateSpy.mockReturnValue({ + state: "connected_other", + activeGateway: "nemoclaw-8081", + status: "Gateway: nemoclaw-8081\nStatus: Connected", + }); + captureOpenshellSpy.mockReturnValueOnce({ + status: 0, + output: "Sandbox:\n Name: instance-a\n Phase: Ready", + }); + + const retry = gatewayState.reconcileMissingAgainstNamedGateway("instance-a", { + state: "missing", + output: 'status: Internal, message: "sandbox has no spec"', + }); + + expect(retry).toEqual( + expect.objectContaining({ + state: "present", + recoveredGateway: true, + recoveryVia: "select", + }), + ); + expect(runOpenshellSpy).toHaveBeenCalledWith( + ["gateway", "select", "nemoclaw"], + expect.objectContaining({ ignoreError: true }), + ); + }); }); diff --git a/src/lib/actions/sandbox/gateway-state.ts b/src/lib/actions/sandbox/gateway-state.ts index 5e036ffe689..b7ee96da27e 100644 --- a/src/lib/actions/sandbox/gateway-state.ts +++ b/src/lib/actions/sandbox/gateway-state.ts @@ -144,7 +144,13 @@ export function getSandboxGatewayState(sandboxName: string): SandboxGatewayState } return { state: "present", output }; } - if (/\bNotFound\b|\bNot Found\b|sandbox not found/i.test(output)) { + // `sandbox has no spec` is the gRPC reply when the active OpenShell gateway + // is reachable but does not know about this sandbox — the multi-instance + // case where the active gateway is a sibling of the one the sandbox was + // onboarded against. Classify as `missing` so the named-gateway reconciler + // selects the sandbox's owning gateway and retries; without this the lookup + // would fall to `unknown_error` and exit with a hint instead of recovering. + if (/\bNotFound\b|\bNot Found\b|sandbox not found|sandbox has no spec/i.test(output)) { return { state: "missing", output }; } if ( @@ -202,7 +208,7 @@ export async function getSandboxGatewayStateForStatus( } return { state: "present", output }; } - if (/\bNotFound\b|\bNot Found\b|sandbox not found/i.test(output)) { + if (/\bNotFound\b|\bNot Found\b|sandbox not found|sandbox has no spec/i.test(output)) { return { state: "missing", output }; } if ( diff --git a/src/lib/onboard/dashboard-port.ts b/src/lib/onboard/dashboard-port.ts index 8dc7a144e11..cd26fc4f45b 100644 --- a/src/lib/onboard/dashboard-port.ts +++ b/src/lib/onboard/dashboard-port.ts @@ -30,7 +30,7 @@ type SandboxRegistryEntry = { dashboardPort?: number | null; }; -type ListSandboxesFn = () => { sandboxes: SandboxRegistryEntry[] }; +export type ListSandboxesFn = () => { sandboxes: SandboxRegistryEntry[] }; // Match the broader pattern used by onboard.ts (covers CSI, OSC, and Fe escapes) // so colorised `openshell forward list` output parses correctly. diff --git a/src/lib/onboard/dashboard.ts b/src/lib/onboard/dashboard.ts index 8c719f852c7..bc0d36533ec 100644 --- a/src/lib/onboard/dashboard.ts +++ b/src/lib/onboard/dashboard.ts @@ -23,6 +23,7 @@ import { getOccupiedPorts, getRegistryOccupiedDashboardPorts, isLiveForwardStatus, + type ListSandboxesFn, } from "./dashboard-port"; import { bestEffortForwardStop } from "./forward-cleanup"; import { @@ -52,6 +53,11 @@ export interface OnboardDashboardDeps { isWsl(): boolean; redact(value: unknown): string; sleep(seconds: number): void; + // Sandbox-registry lookup used by `ensureDashboardForward` for the + // cross-gateway dashboard port view. Tests inject a stub so the allocator + // never reads the runner's real `~/.nemoclaw/sandboxes.json`; production + // callers leave it unset and the helper falls back to the live registry. + listSandboxes?: ListSandboxesFn; printAgentDashboardUi( sandboxName: string, token: string | null, @@ -255,7 +261,7 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa preferredPort, existingForwards, undefined, - getRegistryOccupiedDashboardPorts(sandboxName), + getRegistryOccupiedDashboardPorts(sandboxName, deps.listSandboxes), ); } catch (err) { if (!rollbackSandboxOnFailure) throw err; diff --git a/test/onboard-dashboard.test.ts b/test/onboard-dashboard.test.ts index 271499617a1..23678ae709b 100644 --- a/test/onboard-dashboard.test.ts +++ b/test/onboard-dashboard.test.ts @@ -45,6 +45,7 @@ describe("onboard dashboard helpers", () => { redact: (value: unknown) => String(value), sleep: vi.fn(), printAgentDashboardUi: vi.fn(), + listSandboxes: () => ({ sandboxes: [] }), }); expect(helpers.ensureDashboardForward("my-sandbox", "http://127.0.0.1:18789")).toBe(18789); @@ -87,6 +88,7 @@ describe("onboard dashboard helpers", () => { redact: (value: unknown) => String(value), sleep: vi.fn(), printAgentDashboardUi: vi.fn(), + listSandboxes: () => ({ sandboxes: [] }), }); expect(helpers.ensureDashboardForward("my-sandbox", "http://127.0.0.1:18789")).toBe(18789); @@ -121,6 +123,7 @@ describe("onboard dashboard helpers", () => { redact: (value: unknown) => String(value), sleep: vi.fn(), printAgentDashboardUi: vi.fn(), + listSandboxes: () => ({ sandboxes: [] }), }); expect( @@ -168,6 +171,7 @@ describe("onboard dashboard helpers", () => { redact: (value: unknown) => String(value), sleep: vi.fn(), printAgentDashboardUi: vi.fn(), + listSandboxes: () => ({ sandboxes: [] }), }); let output = ""; @@ -208,6 +212,7 @@ describe("onboard dashboard helpers", () => { redact: (value: unknown) => String(value), sleep: vi.fn(), printAgentDashboardUi: vi.fn(), + listSandboxes: () => ({ sandboxes: [] }), }); let output = "";