diff --git a/.github/workflows/nightly-e2e.yaml b/.github/workflows/nightly-e2e.yaml index 47dde6a9e7e..843f2e7551b 100644 --- a/.github/workflows/nightly-e2e.yaml +++ b/.github/workflows/nightly-e2e.yaml @@ -68,6 +68,14 @@ # openclaw-anthropic-inference-switch-e2e # Switches a running OpenClaw sandbox to a compatible # Anthropic Messages provider and verifies agent traffic. +# openclaw-skill-cli-e2e Validates workspace-installed OpenClaw skills survive sandbox +# lifecycle through OPENCLAW_HOME/STATE_DIR/WORKSPACE_DIR pinning +# (#4766 / #4709). Seven-phase deterministic skill-CLI exercise +# inside a real onboarded sandbox (install, list, info, check). +# channels-add-remove-e2e Telegram/Discord/Slack channel add/remove lifecycle plus +# gateway-credential reuse on rebuild (#4745 / #3895). Exercises +# the path where the host env credential is empty but the +# gateway already holds the provider credential. # issue-4434-tui-unreachable-inference-e2e # Recreates #4434's NVIDIA endpoint firewall block and verifies # OpenClaw TUI shows a visible error and stops the active spinner. @@ -77,6 +85,10 @@ # launchable-smoke-e2e Community install path (brev-launchable-ci-cpu.sh) on ubuntu-latest. # gpu-e2e Local Ollama inference on an NVKS ephemeral GPU runner. # gpu-double-onboard-e2e Ollama proxy token consistency after re-onboard (#2553). +# concurrent-gateway-ports-e2e +# Two sandboxes coexisting on the same host with distinct +# NEMOCLAW_GATEWAY_PORT values; verifies per-instance +# gateway and dashboard segregation. # notify-on-failure Auto-creates a GitHub issue when any E2E job fails. # # Runs directly on the runner (not inside Docker) because OpenShell bootstraps @@ -126,7 +138,8 @@ on: rebuild-hermes-stale-base-e2e, double-onboard-e2e, onboard-repair-e2e, onboard-resume-e2e, onboard-negative-paths-e2e, runtime-overrides-e2e, credential-sanitization-e2e, telegram-injection-e2e, overlayfs-autofix-e2e, - device-auth-health-e2e, launchable-smoke-e2e, gpu-e2e, gpu-double-onboard-e2e + device-auth-health-e2e, launchable-smoke-e2e, gpu-e2e, gpu-double-onboard-e2e, + concurrent-gateway-ports-e2e required: false type: string default: "" @@ -2063,6 +2076,58 @@ jobs: path: /tmp/nemoclaw-gpu-double-onboard-test.log if-no-files-found: ignore + concurrent-gateway-ports-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',concurrent-gateway-ports-e2e,')) + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_E2E_PHASE_TIMEOUT: "1200" + steps: + - *target-ref-checkout + - *dockerhub-auth-step + - name: Install NemoClaw + env: + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + run: bash install.sh --non-interactive --yes-i-accept-third-party-software + - name: Run concurrent gateway ports E2E test + env: + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} + run: | + [ -f "$HOME/.bashrc" ] && source "$HOME/.bashrc" 2>/dev/null || true + export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" + [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" + [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]] && export PATH="$HOME/.local/bin:$PATH" + bash test/e2e/test-concurrent-gateway-ports.sh + - name: Upload sandbox A onboard log on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: concurrent-gateway-ports-sandbox-a-onboard-log + path: /tmp/e2e-cgp-a-onboard.log + if-no-files-found: ignore + - name: Upload sandbox B onboard log on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: concurrent-gateway-ports-sandbox-b-onboard-log + path: /tmp/e2e-cgp-b-onboard.log + if-no-files-found: ignore + - name: Upload sandbox B destroy log on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: concurrent-gateway-ports-sandbox-b-destroy-log + path: /tmp/e2e-cgp-b-destroy.log + if-no-files-found: ignore + notify-on-failure: runs-on: ubuntu-latest needs: @@ -2131,6 +2196,7 @@ jobs: launchable-smoke-e2e, gpu-e2e, gpu-double-onboard-e2e, + concurrent-gateway-ports-e2e, ] if: ${{ always() && github.event_name == 'schedule' && (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) }} permissions: @@ -2245,6 +2311,7 @@ jobs: launchable-smoke-e2e, gpu-e2e, gpu-double-onboard-e2e, + concurrent-gateway-ports-e2e, ] if: ${{ always() && github.event_name == 'workflow_dispatch' }} permissions: @@ -2416,6 +2483,7 @@ jobs: launchable-smoke-e2e, gpu-e2e, gpu-double-onboard-e2e, + concurrent-gateway-ports-e2e, ] if: ${{ always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }} permissions: diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index ec5e4c31a4d..a94a5381c4a 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -439,6 +439,8 @@ const { findAvailableDashboardPort, preflightDashboardPortRangeAvailability, } = require("./onboard/dashboard-port") as typeof import("./onboard/dashboard-port"); +const { tryCleanupOrphanedDashboardForward } = + require("./onboard/orphaned-dashboard-forward") as typeof import("./onboard/orphaned-dashboard-forward"); const { destroyGatewayForReuse } = require("./onboard/gateway-cleanup") as typeof import("./onboard/gateway-cleanup"); const { applyPreflightGatewayCleanup } = require("./onboard/preflight-gateway-cleanup-decision") as typeof import("./onboard/preflight-gateway-cleanup-decision"); @@ -2079,20 +2081,12 @@ async function preflight( // if its command line contains "openshell" to avoid killing unrelated SSH // tunnels the user may have set up on the same port. (#1950) if (port === DASHBOARD_PORT && portCheck.process === "ssh" && portCheck.pid) { - // Use `ps` to get the command line — works on Linux, macOS, and WSL. - const cmdline = captureProcessArgs(portCheck.pid); - if (cmdline.includes("openshell")) { - console.log( - ` Cleaning up orphaned SSH port-forward on port ${port} (PID ${portCheck.pid})...`, - ); - run(["kill", String(portCheck.pid)], { ignoreError: true }); - sleepSeconds(1); - portCheck = await checkPortAvailable(port, portCheckOptions); - if (portCheck.ok) { - console.log(` ✓ Port ${port} available after orphaned forward cleanup (${label})`); - continue; - } - } + const outcome = await tryCleanupOrphanedDashboardForward({ + port, pid: portCheck.pid, label, portCheckOptions, + captureProcessArgs, runCaptureOpenshell, run, sleepSeconds, checkPortAvailable, + }); + if (outcome.kind === "killed-still-blocked") portCheck = outcome.portCheck; + else if (outcome.kind !== "not-openshell") continue; } console.error(""); console.error(` !! Port ${port} is not available.`); diff --git a/src/lib/onboard/machine/handlers/gateway.test.ts b/src/lib/onboard/machine/handlers/gateway.test.ts index b184fdb8263..ca69d2dbc3f 100644 --- a/src/lib/onboard/machine/handlers/gateway.test.ts +++ b/src/lib/onboard/machine/handlers/gateway.test.ts @@ -314,4 +314,22 @@ describe("handleGatewayState", () => { " Replacing legacy OpenShell gateway metadata with Docker-driver gateway.", ); }); + + it("does not retire a foreign-active Docker-driver gateway (concurrent instances)", async () => { + const { deps, calls } = createDeps({ + isLinuxDockerDriverGatewayEnabled: vi.fn(() => true), + reconcileGatewayGpuReuseForGpuIntent: vi.fn( + () => "foreign-active" as GatewayReuseState, + ), + }); + + const result = await handleGatewayState(baseOptions(deps, "foreign-active")); + + expect(calls.retireLegacy).not.toHaveBeenCalled(); + expect(calls.note).not.toHaveBeenCalledWith( + " Replacing legacy OpenShell gateway metadata with Docker-driver gateway.", + ); + expect(calls.startGateway).toHaveBeenCalledOnce(); + expect(result.gatewayReuseState).toBe("missing"); + }); }); diff --git a/src/lib/onboard/machine/handlers/gateway.ts b/src/lib/onboard/machine/handlers/gateway.ts index 6589db29cda..54ec2e73d57 100644 --- a/src/lib/onboard/machine/handlers/gateway.ts +++ b/src/lib/onboard/machine/handlers/gateway.ts @@ -204,10 +204,16 @@ export async function handleGatewayState({ } } await deps.startRecordedStep("gateway"); - if (deps.isLinuxDockerDriverGatewayEnabled() && gatewayReuseState !== "missing") { + if ( + deps.isLinuxDockerDriverGatewayEnabled() && + gatewayReuseState !== "missing" && + gatewayReuseState !== "foreign-active" + ) { deps.note(" Replacing legacy OpenShell gateway metadata with Docker-driver gateway."); deps.retireLegacyGatewayForDockerDriverUpgrade(); gatewayReuseState = "missing"; + } else if (gatewayReuseState === "foreign-active") { + gatewayReuseState = "missing"; } await withGatewayTrace(gatewayReuseState, gpuPassthrough, () => deps.startGateway(gpu, { gpuPassthrough }), diff --git a/src/lib/onboard/orphaned-dashboard-forward.test.ts b/src/lib/onboard/orphaned-dashboard-forward.test.ts new file mode 100644 index 00000000000..b99b4a494dc --- /dev/null +++ b/src/lib/onboard/orphaned-dashboard-forward.test.ts @@ -0,0 +1,124 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { + tryCleanupOrphanedDashboardForward, + type OrphanedDashboardForwardDeps, +} from "../../../dist/lib/onboard/orphaned-dashboard-forward"; + +function forwardListWith( + entries: Array<{ sandbox: string; port: number; status?: string }>, +): string { + const header = "SANDBOX BIND PORT PID STATUS"; + const rows = entries.map( + (e) => `${e.sandbox} 127.0.0.1 ${e.port} 1234 ${e.status ?? "running"}`, + ); + return [header, ...rows].join("\n"); +} + +interface MakeDepsOverrides { + cmdline?: string; + listFn?: () => string; + portCheckResult?: { ok: boolean; process?: string; pid?: number | null; reason?: string }; +} + +function makeDeps(overrides: MakeDepsOverrides = {}) { + const calls = { + captureProcessArgs: vi.fn((_pid: number) => overrides.cmdline ?? "ssh -L openshell-forward 18789:..."), + runCaptureOpenshell: vi.fn( + overrides.listFn ?? (() => forwardListWith([])), + ) as OrphanedDashboardForwardDeps["runCaptureOpenshell"], + run: vi.fn() as unknown as OrphanedDashboardForwardDeps["run"], + sleepSeconds: vi.fn() as OrphanedDashboardForwardDeps["sleepSeconds"], + checkPortAvailable: vi.fn(async () => overrides.portCheckResult ?? { ok: true }) as unknown as OrphanedDashboardForwardDeps["checkPortAvailable"], + log: vi.fn(), + }; + const deps: OrphanedDashboardForwardDeps = { + port: 18789, + pid: 4321, + label: "Test dashboard", + captureProcessArgs: calls.captureProcessArgs, + runCaptureOpenshell: calls.runCaptureOpenshell, + run: calls.run, + sleepSeconds: calls.sleepSeconds, + checkPortAvailable: calls.checkPortAvailable, + log: calls.log, + }; + return { deps, calls }; +} + +describe("tryCleanupOrphanedDashboardForward", () => { + it("returns not-openshell when the listener is unrelated SSH", async () => { + const { deps, calls } = makeDeps({ cmdline: "ssh -L 18789:remote-host:80 user@bastion" }); + const outcome = await tryCleanupOrphanedDashboardForward(deps); + expect(outcome).toEqual({ kind: "not-openshell" }); + expect(calls.runCaptureOpenshell).not.toHaveBeenCalled(); + expect(calls.run).not.toHaveBeenCalled(); + }); + + it("returns list-failed and skips the kill when forward list throws", async () => { + const { deps, calls } = makeDeps({ + listFn: () => { + throw new Error("gateway probe timed out"); + }, + }); + const outcome = await tryCleanupOrphanedDashboardForward(deps); + expect(outcome).toEqual({ kind: "list-failed" }); + expect(calls.run).not.toHaveBeenCalled(); + expect(calls.checkPortAvailable).not.toHaveBeenCalled(); + expect(calls.log).toHaveBeenCalledWith( + expect.stringContaining("Could not enumerate OpenShell forwards"), + ); + }); + + it("does not pass ignoreError to runCaptureOpenshell (failures must throw to be classified list-failed)", async () => { + const { deps, calls } = makeDeps(); + await tryCleanupOrphanedDashboardForward(deps); + expect(calls.runCaptureOpenshell).toHaveBeenCalledWith( + ["forward", "list"], + expect.objectContaining({ timeout: 10_000, suppressOutput: true }), + ); + expect(calls.runCaptureOpenshell).not.toHaveBeenCalledWith( + ["forward", "list"], + expect.objectContaining({ ignoreError: true }), + ); + }); + + it("returns owned-by-live when another live sandbox owns the port", async () => { + const { deps, calls } = makeDeps({ + listFn: () => forwardListWith([{ sandbox: "other-sandbox", port: 18789 }]), + }); + const outcome = await tryCleanupOrphanedDashboardForward(deps); + expect(outcome).toEqual({ kind: "owned-by-live", owner: "other-sandbox" }); + expect(calls.run).not.toHaveBeenCalled(); + expect(calls.checkPortAvailable).not.toHaveBeenCalled(); + }); + + it("returns killed-cleared when the kill frees the port", async () => { + const { deps, calls } = makeDeps({ portCheckResult: { ok: true } }); + const outcome = await tryCleanupOrphanedDashboardForward(deps); + expect(outcome).toEqual({ kind: "killed-cleared" }); + expect(calls.run).toHaveBeenCalledWith(["kill", "4321"], { ignoreError: true }); + expect(calls.sleepSeconds).toHaveBeenCalledWith(1); + expect(calls.checkPortAvailable).toHaveBeenCalledWith(18789, undefined); + }); + + it("returns killed-still-blocked when the kill ran but the port stayed blocked", async () => { + const refreshedCheck = { ok: false, process: "ssh", pid: 4321, reason: "still busy" }; + const { deps, calls } = makeDeps({ portCheckResult: refreshedCheck }); + const outcome = await tryCleanupOrphanedDashboardForward(deps); + expect(outcome).toEqual({ kind: "killed-still-blocked", portCheck: refreshedCheck }); + expect(calls.run).toHaveBeenCalledTimes(1); + }); + + it("ignores non-live forward statuses when deciding ownership", async () => { + const { deps, calls } = makeDeps({ + listFn: () => forwardListWith([{ sandbox: "other-sandbox", port: 18789, status: "stopped" }]), + }); + const outcome = await tryCleanupOrphanedDashboardForward(deps); + expect(outcome.kind).toBe("killed-cleared"); + expect(calls.run).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/lib/onboard/orphaned-dashboard-forward.ts b/src/lib/onboard/orphaned-dashboard-forward.ts new file mode 100644 index 00000000000..b7343a75205 --- /dev/null +++ b/src/lib/onboard/orphaned-dashboard-forward.ts @@ -0,0 +1,109 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { CheckPortOpts, PortProbeResult } from "./preflight"; + +import { getOccupiedPorts } from "./dashboard-port"; + +export type ListForwardsRunner = ( + args: string[], + opts: { timeout?: number; suppressOutput?: boolean }, +) => string; + +export type KillRunner = (args: string[], opts: { ignoreError?: boolean }) => unknown; + +export type CheckPortAvailableFn = ( + port: number, + opts?: CheckPortOpts, +) => Promise; + +export type SleepFn = (seconds: number) => void; + +export interface OrphanedDashboardForwardDeps { + port: number; + pid: number; + label: string; + portCheckOptions?: CheckPortOpts; + captureProcessArgs(pid: number): string; + runCaptureOpenshell: ListForwardsRunner; + run: KillRunner; + sleepSeconds: SleepFn; + checkPortAvailable: CheckPortAvailableFn; + log?: (message: string) => void; +} + +export type OrphanedDashboardForwardOutcome = + | { kind: "not-openshell" } + | { kind: "list-failed" } + | { kind: "owned-by-live"; owner: string } + | { kind: "killed-cleared" } + | { kind: "killed-still-blocked"; portCheck: PortProbeResult }; + +/** + * Decide whether an orphaned SSH port-forward sitting on the dashboard port + * can be killed to free the port. The caller has already detected that the + * port is blocked by an `ssh` listener (typical signature of a stale + * `openshell forward start` left behind after a previous session). + * + * Cross-instance safety is enforced by consulting `openshell forward list` + * for the live owner: + * - `not-openshell` — listener is unrelated SSH; caller should fall + * through to the generic port-blocked error path. + * - `list-failed` — could not enumerate forwards; the kill is + * SKIPPED. With no ownership data, a kill could + * collateral-damage a concurrent live sandbox's + * dashboard forward. Caller continues — the runtime + * allocator will pick a different dashboard port. + * - `owned-by-live` — another live sandbox holds the forward; kill is + * skipped, caller continues with auto-allocation. + * - `killed-cleared` — kill succeeded and the port is now free. + * - `killed-still-blocked` — kill ran but the port stayed blocked; the + * refreshed `portCheck` is returned so the caller + * can fall through to the generic port-blocked + * error path with up-to-date diagnostics. + * + * The `forward list` call is intentionally allowed to throw — `ignoreError` + * would swallow the failure into an empty string, which `getOccupiedPorts` + * parses as an empty map, and the "no entry → kill" branch would still run + * with no ownership data. + */ +export async function tryCleanupOrphanedDashboardForward( + deps: OrphanedDashboardForwardDeps, +): Promise { + const log = deps.log ?? ((message: string) => console.log(message)); + const cmdline = deps.captureProcessArgs(deps.pid); + if (!cmdline.includes("openshell")) { + return { kind: "not-openshell" }; + } + + let listOutput: string; + try { + listOutput = deps.runCaptureOpenshell(["forward", "list"], { + suppressOutput: true, + timeout: 10_000, + }); + } catch { + log( + ` Could not enumerate OpenShell forwards while checking port ${deps.port}; leaving its forward intact to avoid killing a live sandbox.`, + ); + return { kind: "list-failed" }; + } + + const owner = getOccupiedPorts(listOutput).get(String(deps.port)) ?? null; + if (owner) { + log( + ` Port ${deps.port} held by live sandbox '${owner}'; leaving its forward intact (this sandbox will auto-allocate a different dashboard port).`, + ); + return { kind: "owned-by-live", owner }; + } + + log(` Cleaning up orphaned SSH port-forward on port ${deps.port} (PID ${deps.pid})...`); + deps.run(["kill", String(deps.pid)], { ignoreError: true }); + deps.sleepSeconds(1); + const portCheck = await deps.checkPortAvailable(deps.port, deps.portCheckOptions); + if (portCheck.ok) { + log(` ✓ Port ${deps.port} available after orphaned forward cleanup (${deps.label})`); + return { kind: "killed-cleared" }; + } + return { kind: "killed-still-blocked", portCheck }; +} diff --git a/test/e2e/test-concurrent-gateway-ports.sh b/test/e2e/test-concurrent-gateway-ports.sh new file mode 100755 index 00000000000..af3db85d7cd --- /dev/null +++ b/test/e2e/test-concurrent-gateway-ports.sh @@ -0,0 +1,370 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Concurrent gateway ports — exercises multiple NemoClaw-managed sandboxes on a +# single host with fully segregated gateways, dashboards, and registries. A +# second onboard with NEMOCLAW_GATEWAY_PORT set to a non-default port must not +# touch the first sandbox's gateway process, dashboard SSH forward, or sandbox +# container. +# +# Scenario shape: +# 1. Onboard sandbox A on the default gateway port (8080) + default dashboard +# port (18789). +# 2. Onboard sandbox B with NEMOCLAW_GATEWAY_PORT set to a non-default port +# that drives the per-port binding path. The dashboard port should +# auto-allocate from the 18789-18799 range without colliding with A. +# 3. Verify both sandboxes coexist: distinct gateways, distinct dashboards, +# distinct sandbox containers, no SIGKILL of A during B's onboard, and +# nemoclaw list reports two entries with two distinct dashboard URLs. +# 4. Destroy B and verify A remains healthy. +# +# This script intentionally uses a local fake OpenAI-compatible endpoint so it +# does not depend on real NVIDIA endpoints, matching the pattern in +# test-double-onboard.sh. + +# ShellCheck cannot see EXIT trap invocations of cleanup helpers in this E2E script. +# shellcheck disable=SC2317 +set -uo pipefail + +export NEMOCLAW_E2E_DEFAULT_TIMEOUT=4800 +SCRIPT_DIR_TIMEOUT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +# shellcheck source=test/e2e/e2e-timeout.sh +source "${SCRIPT_DIR_TIMEOUT}/e2e-timeout.sh" + +PASS=0 +FAIL=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +PHASE_TIMEOUT="${NEMOCLAW_E2E_PHASE_TIMEOUT:-1200}" + +SANDBOX_A="e2e-cgp-a" +SANDBOX_B="e2e-cgp-b" +GATEWAY_PORT_A=8080 +GATEWAY_PORT_B="${NEMOCLAW_E2E_GATEWAY_PORT_B:-18080}" +DASHBOARD_PORT_A=18789 + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +FAKE_HOST="127.0.0.1" +FAKE_PORT="${NEMOCLAW_E2E_FAKE_PORT:-18180}" +FAKE_BASE_URL="http://${FAKE_HOST}:${FAKE_PORT}/v1" +FAKE_LOG="$(mktemp)" +FAKE_PID="" + +if command -v node >/dev/null 2>&1 && [ -f "$REPO_ROOT/bin/nemoclaw.js" ]; then + NEMOCLAW_CMD=(node "$REPO_ROOT/bin/nemoclaw.js") +else + NEMOCLAW_CMD=(nemoclaw) +fi + +# shellcheck disable=SC2329 +cleanup() { + if [ -n "$FAKE_PID" ] && kill -0 "$FAKE_PID" 2>/dev/null; then + kill "$FAKE_PID" 2>/dev/null || true + wait "$FAKE_PID" 2>/dev/null || true + fi + rm -f "$FAKE_LOG" +} +trap cleanup EXIT + +start_fake_openai() { + python3 - "$FAKE_HOST" "$FAKE_PORT" >"$FAKE_LOG" 2>&1 <<'PY' & +import json +import sys +from http.server import BaseHTTPRequestHandler, HTTPServer + +HOST = sys.argv[1] +PORT = int(sys.argv[2]) + + +class Handler(BaseHTTPRequestHandler): + def _send(self, status, payload): + body = json.dumps(payload).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format, *args): + return + + def do_GET(self): + if self.path in ("/v1/models", "/models"): + self._send(200, {"data": [{"id": "test-model", "object": "model"}]}) + return + self._send(404, {"error": {"message": "not found"}}) + + def do_POST(self): + length = int(self.headers.get("Content-Length", "0")) + if length: + self.rfile.read(length) + if self.path in ("/v1/chat/completions", "/chat/completions"): + self._send( + 200, + { + "id": "chatcmpl-test", + "object": "chat.completion", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + }, + ) + return + if self.path in ("/v1/responses", "/responses"): + self._send( + 200, + { + "id": "resp-test", + "object": "response", + "output": [{"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "ok"}]}], + }, + ) + return + self._send(404, {"error": {"message": "not found"}}) + + +HTTPServer((HOST, PORT), Handler).serve_forever() +PY + FAKE_PID=$! + + for _ in $(seq 1 20); do + if curl -sf "${FAKE_BASE_URL}/models" >/dev/null 2>&1; then + info "Fake OpenAI server up on ${FAKE_BASE_URL} (pid ${FAKE_PID})" + return 0 + fi + sleep 1 + done + + fail "Fake OpenAI server did not become ready on ${FAKE_BASE_URL}; see ${FAKE_LOG}" + cat "$FAKE_LOG" + exit 1 +} + +dashboard_port_from_list() { + local sandbox="$1" + "${NEMOCLAW_CMD[@]}" list 2>/dev/null \ + | awk -v want="${sandbox}" ' + /^[[:space:]]+[A-Za-z0-9_-]+( \*)?[[:space:]]*$/ { + name=$1 + inblock=(name == want) ? 1 : 0 + next + } + inblock && /dashboard:[[:space:]]*http:\/\/[0-9.]+:[0-9]+/ { + match($0, /:[0-9]+/) + print substr($0, RSTART+1, RLENGTH-1) + exit + } + ' +} + +dump_diagnostics() { + local label="${1:-unknown}" + info "=== Diagnostics for ${label} ===" + info "nemoclaw list:" + "${NEMOCLAW_CMD[@]}" list 2>&1 | sed 's/^/ /' || true + info "openshell sandbox list:" + openshell sandbox list 2>&1 | sed 's/^/ /' || true + info "openshell forward list:" + openshell forward list 2>&1 | sed 's/^/ /' || true + info "docker ps -a:" + docker ps -a --format 'table {{.Names}}\t{{.Status}}' 2>&1 | sed 's/^/ /' || true + info "ss -ltn (gateway/dashboard ports):" + ss -ltn 2>&1 | grep -E ":(${GATEWAY_PORT_A}|${GATEWAY_PORT_B}|1878[0-9]|1879[0-9])" | sed 's/^/ /' || true +} + +onboard_sandbox() { + local name="$1" + local gateway_port="$2" + local label="onboard-${name}" + local start_time + start_time="$(date +%s)" + info "Starting onboard of '${name}' with NEMOCLAW_GATEWAY_PORT=${gateway_port}" + if COMPATIBLE_API_KEY=dummy \ + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + NEMOCLAW_PROVIDER=custom \ + NEMOCLAW_ENDPOINT_URL="${FAKE_BASE_URL}" \ + NEMOCLAW_MODEL=test-model \ + NEMOCLAW_POLICY_MODE=skip \ + NEMOCLAW_DASHBOARD_PORT='' \ + CHAT_UI_URL='' \ + NEMOCLAW_GATEWAY_PORT="${gateway_port}" \ + NEMOCLAW_SANDBOX_NAME="${name}" \ + timeout "${PHASE_TIMEOUT}" "${NEMOCLAW_CMD[@]}" onboard --non-interactive \ + >"/tmp/${name}-onboard.log" 2>&1; then + local elapsed + elapsed=$(($(date +%s) - start_time)) + pass "${label} completed in ${elapsed}s" + return 0 + fi + fail "${label} did not complete within ${PHASE_TIMEOUT}s" + dump_diagnostics "${label}" + tail -200 "/tmp/${name}-onboard.log" | sed 's/^/ /' + return 1 +} + +destroy_default_install_sandbox() { + local default_name + default_name="$("${NEMOCLAW_CMD[@]}" list 2>/dev/null \ + | grep -E '^[[:space:]]+[a-zA-Z0-9_-]+ \*' \ + | awk '{print $1}' \ + | head -1 || true)" + if [ -z "${default_name}" ]; then + info "no pre-existing default sandbox to destroy" + return 0 + fi + if [ "${default_name}" = "${SANDBOX_A}" ] || [ "${default_name}" = "${SANDBOX_B}" ]; then + info "default sandbox is one under test (${default_name}); skipping pre-destroy" + return 0 + fi + info "destroying pre-existing default sandbox '${default_name}' (created by install.sh)" + if NEMOCLAW_NON_INTERACTIVE=1 timeout 300 "${NEMOCLAW_CMD[@]}" "${default_name}" destroy --yes \ + >"/tmp/${default_name}-predestroy.log" 2>&1; then + pass "pre-existing default sandbox '${default_name}' destroyed" + else + fail "could not destroy pre-existing default sandbox '${default_name}'" + tail -100 "/tmp/${default_name}-predestroy.log" | sed 's/^/ /' + return 1 + fi +} + +gateway_name_for_port() { + local port="$1" + if [ "${port}" = "8080" ]; then + echo "nemoclaw" + else + echo "nemoclaw-${port}" + fi +} + +sandbox_phase() { + local name="$1" + local gateway="${2:-}" + local args=("sandbox" "list") + if [ -n "${gateway}" ]; then + args+=("-g" "${gateway}") + fi + openshell "${args[@]}" 2>/dev/null \ + | sed 's/\x1b\[[0-9;]*m//g' \ + | awk -v want="${name}" '$1 == want { print $NF; exit }' +} + +verify_sandbox_alive() { + local name="$1" + local label="${2:-${name} alive}" + local gateway="${3:-}" + local retries="${4:-12}" + local phase="" + for _ in $(seq 1 "${retries}"); do + phase="$(sandbox_phase "${name}" "${gateway}")" + case "${phase}" in + Ready | Running) + pass "${label} (phase=${phase})" + return 0 + ;; + Error | Failed | CrashLoopBackOff) + fail "${label} terminal (phase='${phase}')" + return 1 + ;; + esac + sleep 5 + done + fail "${label} did not reach Ready/Running within ${retries} polls (last phase='${phase:-missing}')" + return 1 +} + +# === Scenario === + +section "Stage 0: prepare fake inference endpoint" +start_fake_openai + +section "Stage 0.5: destroy default sandbox created by install.sh (if any)" +destroy_default_install_sandbox || exit 1 + +section "Stage 1: onboard sandbox A on default gateway port (${GATEWAY_PORT_A})" +GATEWAY_A_NAME="$(gateway_name_for_port "${GATEWAY_PORT_A}")" +GATEWAY_B_NAME="$(gateway_name_for_port "${GATEWAY_PORT_B}")" +onboard_sandbox "${SANDBOX_A}" "${GATEWAY_PORT_A}" || exit 1 +verify_sandbox_alive "${SANDBOX_A}" "Sandbox A reaches Ready/Running on default port" "${GATEWAY_A_NAME}" + +DASHBOARD_A="$(dashboard_port_from_list "${SANDBOX_A}")" +if [ -n "${DASHBOARD_A}" ] && [ "${DASHBOARD_A}" = "${DASHBOARD_PORT_A}" ]; then + pass "Sandbox A holds default dashboard port ${DASHBOARD_PORT_A}" +else + fail "Sandbox A dashboard port is '${DASHBOARD_A:-missing}', expected ${DASHBOARD_PORT_A}" +fi + +section "Stage 2: onboard sandbox B with NEMOCLAW_GATEWAY_PORT=${GATEWAY_PORT_B}" +onboard_sandbox "${SANDBOX_B}" "${GATEWAY_PORT_B}" || { + info "B onboard failed; capturing pre-fail state of A for diagnostics" + dump_diagnostics "stage-2-onboard-B" + exit 1 +} + +section "Stage 3: assert both sandboxes coexist" +verify_sandbox_alive "${SANDBOX_A}" "Sandbox A still alive after B's onboard" "${GATEWAY_A_NAME}" +verify_sandbox_alive "${SANDBOX_B}" "Sandbox B reaches Ready/Running on per-port gateway" "${GATEWAY_B_NAME}" + +DASHBOARD_B="$(dashboard_port_from_list "${SANDBOX_B}")" +if [ -n "${DASHBOARD_B}" ] && [ "${DASHBOARD_B}" != "${DASHBOARD_A:-${DASHBOARD_PORT_A}}" ]; then + pass "Sandbox B got a distinct dashboard port (A=${DASHBOARD_A:-missing} B=${DASHBOARD_B})" +else + fail "Sandbox B dashboard port collides with A: A=${DASHBOARD_A:-missing} B=${DASHBOARD_B:-missing}" + dump_diagnostics "dashboard-port-collision" +fi + +if ss -ltn 2>/dev/null | grep -qE ":${GATEWAY_PORT_A}\\b"; then + pass "Sandbox A gateway port ${GATEWAY_PORT_A} still listening" +else + fail "Sandbox A gateway port ${GATEWAY_PORT_A} no longer listening — recreate destroyed first gateway" + dump_diagnostics "gateway-port-A-missing" +fi + +if ss -ltn 2>/dev/null | grep -qE ":${GATEWAY_PORT_B}\\b"; then + pass "Sandbox B gateway port ${GATEWAY_PORT_B} listening" +else + fail "Sandbox B gateway port ${GATEWAY_PORT_B} not listening" + dump_diagnostics "gateway-port-B-missing" +fi + +LIST_OUTPUT="$("${NEMOCLAW_CMD[@]}" list 2>&1 || true)" +if echo "${LIST_OUTPUT}" | grep -qE "^[[:space:]]+${SANDBOX_A}( \*)?[[:space:]]*$" \ + && echo "${LIST_OUTPUT}" | grep -qE "^[[:space:]]+${SANDBOX_B}( \*)?[[:space:]]*$"; then + pass "nemoclaw list shows both sandbox A and B" +else + fail "nemoclaw list missing one of A/B" + # shellcheck disable=SC2001 + echo "${LIST_OUTPUT}" | sed 's/^/ /' +fi + +section "Stage 4: destroy sandbox B; assert sandbox A still healthy" +if NEMOCLAW_NON_INTERACTIVE=1 timeout 300 "${NEMOCLAW_CMD[@]}" "${SANDBOX_B}" destroy --yes \ + >"/tmp/${SANDBOX_B}-destroy.log" 2>&1; then + pass "Sandbox B destroyed" +else + fail "Sandbox B destroy timed out or failed" + tail -100 "/tmp/${SANDBOX_B}-destroy.log" | sed 's/^/ /' +fi +verify_sandbox_alive "${SANDBOX_A}" "Sandbox A still alive after B's destroy" "${GATEWAY_A_NAME}" + +section "Summary: PASS=${PASS} FAIL=${FAIL} TOTAL=${TOTAL}" +if [ "${FAIL}" -gt 0 ]; then + exit 1 +fi +exit 0