diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index a839a1df908..21431954210 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -1784,6 +1784,94 @@ jobs: if-no-files-found: ignore retention-days: 14 + concurrent-gateway-ports-vitest: + needs: generate-matrix + if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',concurrent-gateway-ports-vitest,') || contains(format(',{0},', inputs.scenarios), ',concurrent-gateway-ports,') }} + runs-on: ubuntu-latest + timeout-minutes: 90 + env: + FREE_STANDING_VITEST_JOB: "1" + FREE_STANDING_SCENARIO_ID: "concurrent-gateway-ports" + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/concurrent-gateway-ports + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_RUN_E2E_SCENARIOS: "1" + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Authenticate to Docker Hub + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + shell: bash + run: | + set -euo pipefail + if [[ -z "${DOCKERHUB_USERNAME}" || -z "${DOCKERHUB_TOKEN}" ]]; then + echo "::notice::Docker Hub credentials not configured; continuing with anonymous pulls." + exit 0 + fi + login_succeeded=0 + for attempt in 1 2 3; do + if echo "${DOCKERHUB_TOKEN}" | timeout 30s docker login docker.io --username "${DOCKERHUB_USERNAME}" --password-stdin; then + login_succeeded=1 + break + fi + if [[ "$attempt" -lt 3 ]]; then + echo "::warning::Docker Hub login attempt ${attempt} failed; retrying." + sleep 5 + fi + done + if [[ "$login_succeeded" -ne 1 ]]; then + echo "::warning::Docker Hub login failed after 3 attempts; continuing with anonymous pulls." + fi + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Build CLI + run: npm run build:cli + + - name: Install OpenShell CLI + run: bash scripts/install-openshell.sh + + - name: Run concurrent gateway ports live Vitest test + run: | + set -euo pipefail + export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" + if command -v openshell >/dev/null 2>&1; then + OPENSHELL_BIN="$(command -v openshell)" + elif [ -x "$HOME/.local/bin/openshell" ]; then + OPENSHELL_BIN="$HOME/.local/bin/openshell" + else + echo "::error::OpenShell CLI not found after install" + ls -la /usr/local/bin/openshell "$HOME/.local/bin/openshell" 2>&1 || true + exit 1 + fi + export OPENSHELL_BIN + echo "Using OPENSHELL_BIN=$OPENSHELL_BIN" + "$OPENSHELL_BIN" --version + npx vitest run --project e2e-scenarios-live \ + test/e2e-scenario/live/concurrent-gateway-ports.test.ts \ + --silent=false --reporter=default + + - name: Upload concurrent gateway ports Vitest artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-vitest-scenarios-concurrent-gateway-ports + path: e2e-artifacts/vitest/concurrent-gateway-ports/ + include-hidden-files: false + if-no-files-found: ignore + retention-days: 14 onboard-resume-vitest: needs: generate-matrix @@ -3464,6 +3552,7 @@ jobs: messaging-providers-vitest, launchable-smoke-vitest, double-onboard-vitest, + concurrent-gateway-ports-vitest, full-e2e-vitest, cloud-onboard-vitest, gpu-double-onboard-vitest, diff --git a/test/e2e-scenario/live/concurrent-gateway-ports.test.ts b/test/e2e-scenario/live/concurrent-gateway-ports.test.ts new file mode 100644 index 00000000000..2d8bd8c201e --- /dev/null +++ b/test/e2e-scenario/live/concurrent-gateway-ports.test.ts @@ -0,0 +1,418 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Live Vitest replacement for test/e2e/test-concurrent-gateway-ports.sh. + * + * Preserves the legacy real-system boundaries: two NemoClaw onboards on one + * host, per-port OpenShell Docker-driver gateways, dashboard forward + * allocation, `nemoclaw list`, OpenShell sandbox discovery, host socket probes, + * and destroy/health cleanup. + */ + +import fs from "node:fs"; +import path from "node:path"; + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import type { HostCliClient } from "../fixtures/clients/host.ts"; +import type { SandboxClient } from "../fixtures/clients/sandbox.ts"; +import { validateSandboxName } from "../fixtures/clients/sandbox.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import { startFakeOpenAiCompatibleServer } from "../fixtures/fake-openai-compatible.ts"; +import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const CLI_ENTRYPOINT = path.join(REPO_ROOT, "bin", "nemoclaw.js"); +const CLI_DIST_ENTRYPOINT = path.join(REPO_ROOT, "dist", "nemoclaw.js"); +const SANDBOX_A = process.env.NEMOCLAW_CGP_SANDBOX_A ?? "e2e-cgp-a"; +const SANDBOX_B = process.env.NEMOCLAW_CGP_SANDBOX_B ?? "e2e-cgp-b"; +const GATEWAY_PORT_A = process.env.NEMOCLAW_E2E_GATEWAY_PORT_A ?? "8080"; +const GATEWAY_PORT_B = process.env.NEMOCLAW_E2E_GATEWAY_PORT_B ?? "18080"; +const DASHBOARD_PORT_A = process.env.NEMOCLAW_E2E_DASHBOARD_PORT_A ?? "18789"; +const PHASE_TIMEOUT_MS = Number(process.env.NEMOCLAW_E2E_PHASE_TIMEOUT_MS ?? 1_200) * 1_000; +const PROBE_ATTEMPTS = Number(process.env.NEMOCLAW_E2E_PROBE_ATTEMPTS ?? 12); +const PROBE_DELAY_MS = Number(process.env.NEMOCLAW_E2E_PROBE_DELAY_SECONDS ?? 5) * 1_000; +const TEST_TIMEOUT_MS = 90 * 60_000; +const liveTest = shouldRunLiveE2EScenarios() ? test : test.skip; + +process.env.NEMOCLAW_CLI_BIN ??= CLI_ENTRYPOINT; +validateSandboxName(SANDBOX_A); +validateSandboxName(SANDBOX_B); + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function resultText(result: { stdout: string; stderr: string }): string { + return [result.stdout, result.stderr].filter(Boolean).join("\n"); +} + +function commandEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + return { + ...buildAvailabilityProbeEnv(), + ...extra, + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + }; +} + +function gatewayNameForPort(port: string): string { + return port === "8080" ? "nemoclaw" : `nemoclaw-${port}`; +} + +function openshellEnvForGateway(gatewayName: string): NodeJS.ProcessEnv { + return commandEnv({ OPENSHELL_GATEWAY: gatewayName }); +} + +function onboardEnv( + sandboxName: string, + gatewayPort: string, + fakeBaseUrl: string, +): NodeJS.ProcessEnv { + return commandEnv({ + CHAT_UI_URL: "", + COMPATIBLE_API_KEY: "dummy", + NEMOCLAW_DASHBOARD_PORT: "", + NEMOCLAW_ENDPOINT_URL: fakeBaseUrl, + NEMOCLAW_GATEWAY_PORT: gatewayPort, + NEMOCLAW_MODEL: "test-model", + NEMOCLAW_POLICY_MODE: "skip", + NEMOCLAW_PROVIDER: "custom", + NEMOCLAW_SANDBOX_NAME: sandboxName, + }); +} + +async function command( + host: HostCliClient, + args: string[], + options: { + artifactName: string; + env?: NodeJS.ProcessEnv; + timeoutMs?: number; + }, +): Promise { + return await host.command(process.execPath, [CLI_ENTRYPOINT, ...args], { + artifactName: options.artifactName, + env: options.env ?? commandEnv(), + timeoutMs: options.timeoutMs, + }); +} + +async function runOnboard( + host: HostCliClient, + sandboxName: string, + gatewayPort: string, + fakeBaseUrl: string, + artifactName: string, +): Promise { + return await command(host, ["onboard", "--non-interactive"], { + artifactName, + env: onboardEnv(sandboxName, gatewayPort, fakeBaseUrl), + timeoutMs: PHASE_TIMEOUT_MS, + }); +} + +function dashboardPortFromList(output: string, sandboxName: string): string | undefined { + let current: string | undefined; + for (const line of output.split("\n")) { + if (/^\s{4}\S/.test(line) && !/^\s{6}/.test(line)) { + const stripped = line.trim(); + current = stripped ? stripped.split(/\s+/)[0] : undefined; + continue; + } + if (current === sandboxName) { + const match = line.match(/dashboard:\s+http:\/\/[0-9.]+:(\d+)\/?/); + if (match) return match[1]; + } + } + return undefined; +} + +function outputIncludesSandbox(output: string, sandboxName: string): boolean { + return new RegExp(`^\\s+${sandboxName}(?: \\*)?\\s*$`, "m").test(output); +} + +function sandboxPhaseFromList(output: string, sandboxName: string): string | undefined { + for (const line of output.replace(/\x1B\[[0-9;]*m/g, "").split("\n")) { + const parts = line.trim().split(/\s+/).filter(Boolean); + if (parts[0] === sandboxName) return parts.at(-1); + } + return undefined; +} + +async function waitForSandboxReady( + sandbox: SandboxClient, + sandboxName: string, + gatewayName: string, + artifactPrefix: string, +): Promise { + let lastPhase = "missing"; + let lastOutput = ""; + for (let attempt = 1; attempt <= PROBE_ATTEMPTS; attempt += 1) { + const result = await sandbox.openshell(["sandbox", "list", "-g", gatewayName], { + artifactName: `${artifactPrefix}-attempt-${attempt}`, + env: openshellEnvForGateway(gatewayName), + timeoutMs: 30_000, + }); + lastOutput = resultText(result); + const phase = sandboxPhaseFromList(lastOutput, sandboxName); + if (phase) lastPhase = phase; + if (phase === "Ready" || phase === "Running") return phase; + if (phase === "Error" || phase === "Failed" || phase === "CrashLoopBackOff") { + throw new Error(`${sandboxName} reached terminal phase '${phase}' on ${gatewayName}`); + } + if (attempt < PROBE_ATTEMPTS) await sleep(PROBE_DELAY_MS); + } + throw new Error( + `${sandboxName} did not reach Ready/Running on ${gatewayName}; last phase '${lastPhase}'\n${lastOutput}`, + ); +} + +async function expectPortListening( + host: HostCliClient, + port: string, + artifactName: string, +): Promise { + const result = await host.command("bash", ["-lc", `ss -ltn | grep -Eq '[:.]${port}\\b'`], { + artifactName, + env: commandEnv(), + timeoutMs: 30_000, + }); + expect(result.exitCode, resultText(result)).toBe(0); + return result; +} + +async function prerequisiteOrSkip( + host: HostCliClient, + skip: (message: string) => never, + commandName: string, + args: string[], + artifactName: string, +): Promise { + const result = await host.command(commandName, args, { + artifactName, + env: commandEnv(), + timeoutMs: 30_000, + }); + if (result.exitCode === 0) return result; + const message = `${commandName} ${args.join(" ")} is required for concurrent gateway ports E2E: ${resultText( + result, + )}`; + if (process.env.GITHUB_ACTIONS === "true") throw new Error(message); + skip(message); +} + +async function bestEffortCleanup( + host: HostCliClient, + sandbox: SandboxClient, + gatewayA: string, + gatewayB: string, +): Promise { + for (const [name, gateway, port] of [ + [SANDBOX_B, gatewayB, GATEWAY_PORT_B], + [SANDBOX_A, gatewayA, GATEWAY_PORT_A], + ] as const) { + try { + await command(host, [name, "destroy", "--yes"], { + artifactName: `cleanup-destroy-${name}`, + env: commandEnv({ NEMOCLAW_GATEWAY_PORT: port }), + timeoutMs: 5 * 60_000, + }); + } catch { + // best effort + } + try { + await sandbox.openshell(["sandbox", "delete", name, "-g", gateway], { + artifactName: `cleanup-openshell-delete-${name}`, + env: openshellEnvForGateway(gateway), + timeoutMs: 60_000, + }); + } catch { + // best effort + } + } + for (const port of [ + "18789", + "18790", + "18791", + "18792", + "18793", + "18794", + "18795", + "18796", + "18797", + "18798", + "18799", + ]) { + try { + await sandbox.openshell(["forward", "stop", port], { + artifactName: `cleanup-forward-stop-${port}`, + env: commandEnv(), + timeoutMs: 15_000, + }); + } catch { + // best effort + } + } + for (const gateway of [gatewayB, gatewayA]) { + try { + await sandbox.openshell(["gateway", "destroy", "-g", gateway], { + artifactName: `cleanup-gateway-destroy-${gateway}`, + env: openshellEnvForGateway(gateway), + timeoutMs: 60_000, + }); + } catch { + // best effort + } + } +} + +liveTest( + "concurrent gateway ports: onboards two sandboxes on isolated gateways and dashboards", + { timeout: TEST_TIMEOUT_MS }, + async ({ artifacts, cleanup, host, sandbox, skip }) => { + expect( + fs.existsSync(CLI_DIST_ENTRYPOINT), + "run `npm run build:cli` before live repo CLI scenarios", + ).toBe(true); + + await prerequisiteOrSkip(host, skip, "docker", ["info"], "prereq-docker-info"); + await prerequisiteOrSkip( + host, + skip, + "bash", + ["-lc", "command -v openshell"], + "prereq-openshell", + ); + await prerequisiteOrSkip( + host, + skip, + process.execPath, + [CLI_ENTRYPOINT, "--version"], + "prereq-nemoclaw-version", + ); + + const gatewayA = gatewayNameForPort(GATEWAY_PORT_A); + const gatewayB = gatewayNameForPort(GATEWAY_PORT_B); + const fake = await startFakeOpenAiCompatibleServer({ + port: Number(process.env.NEMOCLAW_E2E_FAKE_PORT ?? 0), + }); + await artifacts.writeJson("scenario.json", { + id: "concurrent-gateway-ports", + runner: "vitest", + boundary: "direct-cli-docker-openshell-multiple-gateways-dashboard-forwards", + legacySource: "test/e2e/test-concurrent-gateway-ports.sh", + contract: [ + "sandbox A onboards on the default NemoClaw gateway and dashboard port", + "sandbox B onboards with NEMOCLAW_GATEWAY_PORT on a non-default gateway", + "both sandboxes, gateways, and dashboard forwards coexist without port collision", + "destroying sandbox B leaves sandbox A healthy on the default gateway", + ], + gatewayA, + gatewayB, + fakeBaseUrl: fake.baseUrl, + }); + cleanup.add("close fake OpenAI-compatible endpoint", async () => { + await artifacts.writeJson("fake-openai-requests.json", fake.requests()); + await fake.close(); + }); + cleanup.add("remove concurrent gateway sandboxes and gateways", async () => { + await bestEffortCleanup(host, sandbox, gatewayA, gatewayB); + }); + + await bestEffortCleanup(host, sandbox, gatewayA, gatewayB); + + const onboardA = await runOnboard( + host, + SANDBOX_A, + GATEWAY_PORT_A, + fake.baseUrl, + "phase-1-onboard-sandbox-a", + ); + expect(onboardA.exitCode, resultText(onboardA)).toBe(0); + const phaseA = await waitForSandboxReady( + sandbox, + SANDBOX_A, + gatewayA, + "phase-1-sandbox-a-ready", + ); + expect(["Ready", "Running"]).toContain(phaseA); + + const listAfterA = await command(host, ["list"], { + artifactName: "phase-1-nemoclaw-list-after-a", + timeoutMs: 60_000, + }); + expect(listAfterA.exitCode, resultText(listAfterA)).toBe(0); + const dashboardA = dashboardPortFromList(listAfterA.stdout, SANDBOX_A); + expect(dashboardA, listAfterA.stdout).toBe(DASHBOARD_PORT_A); + await expectPortListening(host, GATEWAY_PORT_A, "phase-1-gateway-port-a-listening"); + + const onboardB = await runOnboard( + host, + SANDBOX_B, + GATEWAY_PORT_B, + fake.baseUrl, + "phase-2-onboard-sandbox-b", + ); + expect(onboardB.exitCode, resultText(onboardB)).toBe(0); + + const phaseAAfterB = await waitForSandboxReady( + sandbox, + SANDBOX_A, + gatewayA, + "phase-3-sandbox-a-still-ready", + ); + const phaseBAfterB = await waitForSandboxReady( + sandbox, + SANDBOX_B, + gatewayB, + "phase-3-sandbox-b-ready", + ); + expect(["Ready", "Running"]).toContain(phaseAAfterB); + expect(["Ready", "Running"]).toContain(phaseBAfterB); + await expectPortListening(host, GATEWAY_PORT_A, "phase-3-gateway-port-a-still-listening"); + await expectPortListening(host, GATEWAY_PORT_B, "phase-3-gateway-port-b-listening"); + + const listBoth = await command(host, ["list"], { + artifactName: "phase-3-nemoclaw-list-both-sandboxes", + timeoutMs: 60_000, + }); + expect(listBoth.exitCode, resultText(listBoth)).toBe(0); + expect(outputIncludesSandbox(listBoth.stdout, SANDBOX_A), listBoth.stdout).toBe(true); + expect(outputIncludesSandbox(listBoth.stdout, SANDBOX_B), listBoth.stdout).toBe(true); + const dashboardAAfterB = dashboardPortFromList(listBoth.stdout, SANDBOX_A); + const dashboardB = dashboardPortFromList(listBoth.stdout, SANDBOX_B); + expect(dashboardAAfterB, listBoth.stdout).toBe(dashboardA); + expect(dashboardB, listBoth.stdout).toBeTruthy(); + expect(dashboardB).not.toBe(dashboardA); + + const destroyB = await command(host, [SANDBOX_B, "destroy", "--yes"], { + artifactName: "phase-4-destroy-sandbox-b", + env: commandEnv({ NEMOCLAW_GATEWAY_PORT: GATEWAY_PORT_B }), + timeoutMs: 5 * 60_000, + }); + expect(destroyB.exitCode, resultText(destroyB)).toBe(0); + + const phaseAAfterDestroyB = await waitForSandboxReady( + sandbox, + SANDBOX_A, + gatewayA, + "phase-4-sandbox-a-still-ready-after-b-destroy", + ); + expect(["Ready", "Running"]).toContain(phaseAAfterDestroyB); + await expectPortListening(host, GATEWAY_PORT_A, "phase-4-gateway-port-a-still-listening"); + + await artifacts.writeJson("scenario-result.json", { + id: "concurrent-gateway-ports", + assertions: { + sandboxAOnboarded: onboardA.exitCode === 0, + sandboxBOnboarded: onboardB.exitCode === 0, + sandboxAPreserved: ["Ready", "Running"].includes(phaseAAfterB), + sandboxBReady: ["Ready", "Running"].includes(phaseBAfterB), + dashboardPortsDistinct: Boolean(dashboardA && dashboardB && dashboardA !== dashboardB), + sandboxAPreservedAfterDestroyB: ["Ready", "Running"].includes(phaseAAfterDestroyB), + }, + }); + }, +); diff --git a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts index 50c8209fa40..235ace24923 100644 --- a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts @@ -503,6 +503,22 @@ describe("e2e-vitest-scenarios workflow boundary", () => { selectedFreeStandingJobs: ["gateway-health-honest-vitest"], registryScenarios: [], }); + expect( + evaluateE2eVitestWorkflowDispatchSelectors({ scenarios: "concurrent-gateway-ports" }), + ).toMatchObject({ + valid: true, + liveScenariosRuns: false, + selectedFreeStandingJobs: ["concurrent-gateway-ports-vitest"], + registryScenarios: [], + }); + expect( + evaluateE2eVitestWorkflowDispatchSelectors({ jobs: "concurrent-gateway-ports-vitest" }), + ).toMatchObject({ + valid: true, + liveScenariosRuns: false, + selectedFreeStandingJobs: ["concurrent-gateway-ports-vitest"], + registryScenarios: [], + }); expect( evaluateE2eVitestWorkflowDispatchSelectors({ scenarios: "channels-add-remove" }), ).toMatchObject({ diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts index 5fd96ea0d14..36599009158 100644 --- a/tools/e2e-scenarios/workflow-boundary.mts +++ b/tools/e2e-scenarios/workflow-boundary.mts @@ -3694,6 +3694,13 @@ export function validateE2eVitestScenariosWorkflowBoundary( "gateway-health-honest", ); + validateFreeStandingJobSelector( + errors, + jobs, + "concurrent-gateway-ports-vitest", + "concurrent-gateway-ports", + ); + validateChannelsAddRemoveVitestJob(errors, jobs); const reportToPr = asRecord(jobs["report-to-pr"]);