diff --git a/src/lib/readiness/gateway-production.test.ts b/src/lib/readiness/gateway-production.test.ts index 498a07bfbb9..d353c846207 100644 --- a/src/lib/readiness/gateway-production.test.ts +++ b/src/lib/readiness/gateway-production.test.ts @@ -21,6 +21,7 @@ vi.mock("node:child_process", async (importOriginal) => ({ afterEach(() => { resetTraceForTests(); + subprocess.spawnSync.mockReset(); vi.unstubAllEnvs(); vi.restoreAllMocks(); }); @@ -37,8 +38,8 @@ import { parseDarwinLsofExecutable, } from "./gateway-production"; -function commandResult(stdout = "", status = 1) { - return { status, stdout, stderr: "", signal: null, pid: 1, output: [] }; +function commandResult(stdout = "", status = 1, stderr = "") { + return { status, stdout, stderr, signal: null, pid: 1, output: [] }; } function managedOwner(gatewayPort: number): GatewayOwner { @@ -290,14 +291,27 @@ describe("managed gateway port readiness (#7411)", () => { }); it.each([ - ["https://127.0.0.1:8080", 8080, "match"], - ["http://localhost:8080", 8080, "match"], - ["https://127.0.0.1:9090", 8080, "mismatch"], - ["https://gateway.example:8080", 8080, "mismatch"], - ] as const)("binds managed endpoint %s to port %s as %s", (endpoint, port, expected) => { - expect(classifyManagedGatewayEndpointBinding([`Gateway endpoint: ${endpoint}`], port)).toBe( - expected, - ); + ["Gateway endpoint: https://127.0.0.1:8080", 8080, "match"], + ["Gateway endpoint: http://localhost:8080", 8080, "match"], + ["Server: https://127.0.0.1:8080", 8080, "match"], + ["Server: https://127.0.0.1:9090", 8080, "mismatch"], + ["Server: https://gateway.example:8080", 8080, "mismatch"], + ["Server: ftp://127.0.0.1:8080", 8080, "mismatch"], + ["Server: not-a-url", 8080, "mismatch"], + ["Gateway endpoint:", 8080, "mismatch"], + ["Server: https://127.0.0.1:8080 trailing-data", 8080, "mismatch"], + ["DNS Server: https://127.0.0.1:8080", 8080, "unknown"], + ] as const)("classifies managed endpoint output %s for port %s as %s", (output, port, expected) => { + expect(classifyManagedGatewayEndpointBinding([output], port)).toBe(expected); + }); + + it("rejects conflicting managed endpoint output across OpenShell probes", () => { + expect( + classifyManagedGatewayEndpointBinding( + ["Gateway endpoint: https://127.0.0.1:8080", "Server: https://127.0.0.1:9090"], + 8080, + ), + ).toBe("mismatch"); }); it("rejects healthy managed metadata bound to another endpoint", () => { @@ -366,6 +380,68 @@ describe("managed gateway port readiness (#7411)", () => { ); }); + it("preserves scoped stale gateway state from OpenShell connection errors", async () => { + const statusConnectionRefused = [ + "Error: × client error (Connect)", + " ├─▶ tcp connect error", + " ╰─▶ Connection refused (os error 111)", + ].join("\n"); + const infoConnectionRefused = [ + "Error: × transport error", + " ╰─▶ Connection refused (os error 111)", + ].join("\n"); + const resultByInvocation = new Map([ + [ + ["sh", "-c", 'command -v "$1"', "--", "openshell"].join("\0"), + commandResult("/usr/local/bin/openshell\n", 0), + ], + [ + ["/usr/local/bin/openshell", "status", "-g", "nemoclaw-readiness-test"].join("\0"), + commandResult("", 1, statusConnectionRefused), + ], + [ + ["/usr/local/bin/openshell", "gateway", "info", "-g", "nemoclaw-readiness-test"].join("\0"), + commandResult("", 1, infoConnectionRefused), + ], + [ + ["/usr/local/bin/openshell", "gateway", "info"].join("\0"), + commandResult("", 1, infoConnectionRefused), + ], + ]); + subprocess.spawnSync.mockImplementation((command: string, args: readonly string[] = []) => { + return resultByInvocation.get([command, ...args].join("\0")) ?? commandResult(); + }); + + const gatewayPort = 0; + const deps = createProductionGatewayReadinessDependencies({ + gatewayName: () => "nemoclaw-readiness-test", + gatewayPort: () => gatewayPort, + }); + + await expect(deps.observeManagedGateway(managedOwner(gatewayPort))).resolves.toMatchObject({ + reuseState: "stale", + driftState: "not-detected", + portConflictState: "none", + }); + expect(subprocess.spawnSync).toHaveBeenCalledWith( + "/usr/local/bin/openshell", + ["status", "-g", "nemoclaw-readiness-test"], + expect.objectContaining({ + env: expect.objectContaining({ OPENSHELL_GATEWAY: "nemoclaw-readiness-test" }), + }), + ); + expect(subprocess.spawnSync).toHaveBeenCalledWith( + "/usr/local/bin/openshell", + ["gateway", "info", "-g", "nemoclaw-readiness-test"], + expect.any(Object), + ); + expect(subprocess.spawnSync).toHaveBeenCalledWith( + "/usr/local/bin/openshell", + ["gateway", "info"], + expect.any(Object), + ); + }); + it("collects production port evidence without attempting sudo", async () => { vi.stubEnv("GITHUB_TOKEN", "github-secret"); vi.stubEnv("OPENSHELL_GATEWAY_AUTH_TOKEN", "gateway-secret"); diff --git a/src/lib/readiness/gateway-production.ts b/src/lib/readiness/gateway-production.ts index d78db04c70d..d1eee202868 100644 --- a/src/lib/readiness/gateway-production.ts +++ b/src/lib/readiness/gateway-production.ts @@ -303,24 +303,31 @@ export function classifyManagedGatewayEndpointBinding( outputs: readonly string[], expectedGatewayPort: number, ): Exclude { + let observedEndpoint = false; for (const output of outputs) { - const match = stripAnsi(output).match(/^\s*Gateway endpoint:\s+(\S+)\s*$/m); - if (!match?.[1]) continue; - try { - const endpoint = new URL(match[1]); - const localHost = - endpoint.hostname === "127.0.0.1" || - endpoint.hostname === "localhost" || - endpoint.hostname === "[::1]"; - const endpointPort = - endpoint.port || - (endpoint.protocol === "https:" ? "443" : endpoint.protocol === "http:" ? "80" : ""); - return localHost && endpointPort === String(expectedGatewayPort) ? "match" : "mismatch"; - } catch { - return "mismatch"; + for (const match of stripAnsi(output).matchAll(/^\s*(?:Gateway endpoint|Server):(.*)$/gm)) { + observedEndpoint = true; + const endpointText = match[1]?.trim() ?? ""; + if (!endpointText || /\s/u.test(endpointText)) return "mismatch"; + try { + const endpoint = new URL(endpointText); + const localProtocol = endpoint.protocol === "https:" || endpoint.protocol === "http:"; + const localHost = + endpoint.hostname === "127.0.0.1" || + endpoint.hostname === "localhost" || + endpoint.hostname === "[::1]"; + const endpointPort = + endpoint.port || + (endpoint.protocol === "https:" ? "443" : endpoint.protocol === "http:" ? "80" : ""); + if (!localProtocol || !localHost || endpointPort !== String(expectedGatewayPort)) { + return "mismatch"; + } + } catch { + return "mismatch"; + } } } - return "unknown"; + return observedEndpoint ? "match" : "unknown"; } function observeReuseState( @@ -331,7 +338,7 @@ function observeReuseState( ): { endpointBinding: ManagedGatewayEndpointBinding; reuseState: GatewayReuseState | "unknown" } { if (!openshell) return { endpointBinding: "not-applicable", reuseState: "missing" }; - const status = captureReadonly([openshell, "status"], env); + const status = captureReadonly([openshell, "status", "-g", gatewayName], env); const named = captureReadonly([openshell, "gateway", "info", "-g", gatewayName], env); const active = captureReadonly([openshell, "gateway", "info"], env); if ([status, named, active].some(({ exitCode, timedOut }) => timedOut || exitCode === null)) { @@ -344,6 +351,7 @@ function observeReuseState( combinedOutput(named), combinedOutput(active), gatewayName, + gatewayName, ); if (status.exitCode !== 0 && reuseState === "missing") { reuseState = /\bNo active gateway\b|\bNo gateway metadata found\b/i.test(statusOutput) @@ -369,7 +377,7 @@ function inspectLegacyCluster( env: NodeJS.ProcessEnv, ): { active: boolean; imageRef: string | null } { if (!openshell) return { active: false, imageRef: null }; - const status = captureReadonly([openshell, "status"], env); + const status = captureReadonly([openshell, "status", "-g", gatewayName], env); const named = captureReadonly([openshell, "gateway", "info", "-g", gatewayName], env); const active = captureReadonly([openshell, "gateway", "info"], env); if ( diff --git a/test/e2e/live/double-onboard.test.ts b/test/e2e/live/double-onboard.test.ts index d25a4528b32..9f15e0e0544 100644 --- a/test/e2e/live/double-onboard.test.ts +++ b/test/e2e/live/double-onboard.test.ts @@ -260,6 +260,10 @@ function gatewayNameFromOutput(output: string): string | undefined { return stripAnsi(output).match(/^\s*Gateway:\s+([^\s]+)/m)?.[1]; } +function gatewayServerEndpointFromOutput(output: string): string | undefined { + return stripAnsi(output).match(/^\s*Server:\s+(\S+)\s*$/m)?.[1]; +} + function dashboardPortFromList(output: string, sandboxName: string): string | undefined { let current: string | undefined; for (const line of output.split("\n")) { @@ -524,6 +528,7 @@ test("double-onboard: reuses gateway, preserves sibling sandbox, and recovers st boundary: "direct-cli-openshell-lifecycle", contract: [ "first onboard creates a sandbox and NemoClaw gateway", + "OpenShell status reports the managed gateway through its Server endpoint line", "same-name recreate reuses the healthy gateway without port conflicts", "different-name onboard preserves the first sandbox and allocates distinct dashboard forwards", "stopping one sandbox releases only its dashboard forward and reports the container stopped", @@ -548,6 +553,25 @@ test("double-onboard: reuses gateway, preserves sibling sandbox, and recovers st }); expect(resultText(gatewayInfo)).toContain("nemoclaw"); + const gatewayStatus = await sandbox.openshell(["status"], { + artifactName: "phase-2-openshell-status", + env: commandEnv(), + timeoutMs: 30_000, + }); + const gatewayStatusText = resultText(gatewayStatus); + expect(gatewayStatus.exitCode, gatewayStatusText).toBe(0); + const gatewayServerEndpoint = gatewayServerEndpointFromOutput(gatewayStatusText); + expect(gatewayServerEndpoint, gatewayStatusText).toBeDefined(); + const parsedGatewayServerEndpoint = new URL(gatewayServerEndpoint as string); + const gatewayServerPort = + parsedGatewayServerEndpoint.port || + (parsedGatewayServerEndpoint.protocol === "https:" + ? "443" + : parsedGatewayServerEndpoint.protocol === "http:" + ? "80" + : ""); + expect(gatewayServerPort).toBe(process.env.NEMOCLAW_GATEWAY_PORT ?? "8080"); + const sandboxAAfterFirst = await sandbox.openshell(["sandbox", "get", SANDBOX_A], { artifactName: "phase-2-openshell-sandbox-a-get", env: commandEnv(), @@ -566,6 +590,7 @@ test("double-onboard: reuses gateway, preserves sibling sandbox, and recovers st const gatewayAfterSecond = await gatewayRuntimeId(host, "phase-3-gateway-id-after"); expect(gatewayBeforeSecond, "gateway runtime id before second onboard").not.toBe(""); expect(gatewayAfterSecond).toBe(gatewayBeforeSecond); + expect(secondText).toContain("Reusing healthy NemoClaw gateway."); expect(secondText).not.toContain("Port 8080 is not available"); expect(secondText).not.toContain("Port 18789 is not available"); const sandboxAAfterSecond = await sandbox.openshell(["sandbox", "get", SANDBOX_A], { @@ -844,7 +869,10 @@ test("double-onboard: reuses gateway, preserves sibling sandbox, and recovers st fakeOpenAiRequests: fake.requests(), assertions: { firstOnboard: first.exitCode === 0, - secondOnboardReusedGateway: gatewayAfterSecond === gatewayBeforeSecond, + gatewayStatusReportedServerEndpoint: Boolean(gatewayServerEndpoint), + secondOnboardReusedGateway: + gatewayAfterSecond === gatewayBeforeSecond && + secondText.includes("Reusing healthy NemoClaw gateway."), thirdOnboardPreservedSibling: sandboxAAfterThird.exitCode === 0 && sandboxBAfterThird.exitCode === 0, distinctDashboardPorts: Boolean(portA && portB && portA !== portB), diff --git a/test/onboard-gateway-port-conflict-fast-fail.test.ts b/test/onboard-gateway-port-conflict-fast-fail.test.ts index d1b06fb504d..89f941f947c 100644 --- a/test/onboard-gateway-port-conflict-fast-fail.test.ts +++ b/test/onboard-gateway-port-conflict-fast-fail.test.ts @@ -5,7 +5,10 @@ import type { AddressInfo } from "node:net"; import net from "node:net"; import path from "node:path"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { getGatewayClusterContainerName } from "../src/lib/adapters/openshell/gateway-drift"; +import { resolveGatewayName } from "../src/lib/onboard/gateway-binding"; +import { createProductionGatewayReadinessDependencies } from "../src/lib/readiness/gateway-production"; import { createOnboardProcessWorkspace, type OnboardProcessWorkspace, @@ -77,6 +80,7 @@ describe("onboard gateway port conflict readiness (#6752)", () => { }); afterEach(async () => { + vi.unstubAllEnvs(); await new Promise((resolve) => gatewayServer.close(() => resolve())); workspace.remove(); }); @@ -114,4 +118,97 @@ describe("onboard gateway port conflict readiness (#6752)", () => { ); }, ); + + it( + "accepts Server endpoint evidence on repeated production readiness probes", + testTimeoutOptions(30_000), + async () => { + const gatewayName = resolveGatewayName(gatewayPort); + const gatewayEndpoint = `https://127.0.0.1:${String(gatewayPort)}/`; + const gatewayStatus = [ + "Server Status", + "", + `Gateway: ${gatewayName}`, + `Server: ${gatewayEndpoint}`, + "Status: Connected", + "", + ].join("\n"); + const gatewayInfo = [ + "Gateway Info", + "", + `Gateway: ${gatewayName}`, + `Server: ${gatewayEndpoint}`, + "", + ].join("\n"); + + for (const component of ["openshell", "openshell-gateway", "openshell-sandbox"]) { + workspace.writeExecutable( + component, + [ + "#!/usr/bin/env bash", + "# openshell capabilities: request-body-credential-rewrite websocket-credential-rewrite allow_all_known_mcp_methods", + 'case "$*" in', + ' --version|-V) printf "%s 0.0.101\\n" "${0##*/}"; exit 0;;', + ` status|"status -g ${gatewayName}") printf ${JSON.stringify(gatewayStatus)}; exit 0;;`, + ` "gateway info"|"gateway info -g ${gatewayName}") printf ${JSON.stringify(gatewayInfo)}; exit 0;;`, + "esac", + "exit 1", + ].join("\n"), + ); + } + + const containerName = getGatewayClusterContainerName(gatewayName); + const portBindings = JSON.stringify({ + [`${String(gatewayPort)}/tcp`]: [{ HostPort: String(gatewayPort) }], + }); + workspace.writeExecutable( + "docker", + [ + "#!/usr/bin/env bash", + `if [ "$1" = info ]; then printf '%s\\n' ${JSON.stringify( + JSON.stringify({ + ServerVersion: "24.0.0", + OperatingSystem: "Docker Desktop", + NCPU: 8, + MemTotal: 17_179_869_184, + }), + )}; exit 0; fi`, + 'if [ "$1" = ps ]; then exit 0; fi', + 'if [ "$1" = inspect ] && [ "$4" = ' + JSON.stringify(containerName) + " ]; then", + ' case "$3" in', + ' "{{.State.Running}}") printf "true\\n";;', + ` "{{json .NetworkSettings.Ports}}") printf '%s\\n' ${JSON.stringify(portBindings)};;`, + ' "{{.Config.Image}}") printf "nvcr.io/nvidia/openshell/cluster:0.0.101\\n";;', + " *) exit 1;;", + " esac", + " exit 0", + "fi", + "exit 0", + ].join("\n"), + ); + workspace.writeExecutable("lsof", "#!/usr/bin/env bash\nexit 1\n"); + + vi.stubEnv("HOME", workspace.homeDir); + vi.stubEnv("PATH", `${workspace.binDir}:${process.env.PATH || ""}`); + vi.stubEnv("NEMOCLAW_GATEWAY_PORT", String(gatewayPort)); + vi.stubEnv( + "NEMOCLAW_OPENSHELL_GATEWAY_BIN", + path.join(workspace.binDir, "openshell-gateway"), + ); + workspace.writeExecutable("sudo", "#!/usr/bin/env bash\nexit 1\n"); + + const readiness = createProductionGatewayReadinessDependencies({ + gatewayName: () => gatewayName, + gatewayPort: () => gatewayPort, + }); + const owner = readiness.resolveOwner(); + for (const result of [ + await readiness.observeManagedGateway(owner), + await readiness.observeManagedGateway(owner), + ]) { + expect(result.reuseState).toBe("healthy"); + expect(result.portConflictState).toBe("none"); + } + }, + ); });