diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 7fd74817e19..e23eb79240b 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -40,7 +40,7 @@ jobs: SCENARIOS: ${{ inputs.scenarios }} run: | set -euo pipefail - allowed_jobs="openshell-version-pin-vitest,onboard-negative-paths-vitest,network-policy-vitest,token-rotation-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery" + allowed_jobs="openshell-version-pin-vitest,onboard-negative-paths-vitest,hermes-e2e-vitest,network-policy-vitest,token-rotation-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery" if [ -n "${JOBS}" ] && [ -n "${SCENARIOS}" ]; then echo "::error::Use either scenarios or jobs, not both." >&2 exit 1 @@ -71,6 +71,7 @@ jobs: runs-on: ubuntu-latest outputs: matrix: ${{ steps.matrix.outputs.matrix }} + hermes_selected: ${{ steps.matrix.outputs.hermes_selected }} steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: @@ -88,12 +89,16 @@ jobs: - id: matrix name: Generate Vitest scenario matrix env: + JOBS: ${{ inputs.jobs }} SCENARIOS: ${{ inputs.scenarios }} run: | set -euo pipefail + allowed_jobs="openshell-version-pin-vitest,onboard-negative-paths-vitest,hermes-e2e-vitest,network-policy-vitest,token-rotation-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery" args=(--emit-live-matrix) + matrix="" + hermes_selected=false registry_scenarios=() - free_standing_scenarios=(openshell-version-pin onboard-negative-paths network-policy token-rotation openclaw-tui-chat-correlation) + free_standing_scenarios=(openshell-version-pin onboard-negative-paths hermes-e2e network-policy token-rotation openclaw-tui-chat-correlation) is_free_standing_scenario() { local id="$1" local known @@ -104,13 +109,41 @@ jobs: done return 1 } - if [ -n "${SCENARIOS}" ]; then + if [ -n "${JOBS}" ] && [ -n "${SCENARIOS}" ]; then + echo "::error::Use either scenarios or jobs, not both." >&2 + exit 1 + fi + if [ -n "${JOBS}" ]; then + if [[ ! "${JOBS}" =~ ^[A-Za-z0-9_-]+(,[A-Za-z0-9_-]+)*$ ]]; then + echo "::error::Invalid jobs input; use comma-separated job ids" >&2 + exit 1 + fi + IFS=',' read -r -a selected_jobs <<< "${JOBS}" + for job in "${selected_jobs[@]}"; do + if [[ ",${allowed_jobs}," != *",${job},"* ]]; then + echo "::error::Unknown free-standing Vitest job: ${job}" >&2 + echo "::error::Allowed jobs: ${allowed_jobs}" >&2 + exit 1 + fi + case "${job}" in + hermes-e2e-vitest) + hermes_selected=true + ;; + esac + done + matrix="[]" + elif [ -n "${SCENARIOS}" ]; then if [[ ! "${SCENARIOS}" =~ ^[A-Za-z0-9_-]+(,[A-Za-z0-9_-]+)*$ ]]; then echo "::error::Invalid scenario input; use comma-separated scenario ids containing only letters, numbers, underscores, and hyphens." >&2 exit 1 fi IFS=',' read -r -a requested_scenarios <<< "${SCENARIOS}" for scenario in "${requested_scenarios[@]}"; do + case "${scenario}" in + hermes-e2e) + hermes_selected=true + ;; + esac if is_free_standing_scenario "${scenario}"; then continue fi @@ -119,14 +152,16 @@ jobs: if [ "${#registry_scenarios[@]}" -gt 0 ]; then registry_csv="$(IFS=,; echo "${registry_scenarios[*]}")" args+=(--scenarios "${registry_csv}") + matrix="$(npx tsx test/e2e-scenario/scenarios/run.ts "${args[@]}")" + else + matrix="[]" fi - fi - if [ -n "${SCENARIOS}" ] && [ "${#registry_scenarios[@]}" -eq 0 ]; then - matrix="[]" else + hermes_selected=true matrix="$(npx tsx test/e2e-scenario/scenarios/run.ts "${args[@]}")" fi echo "matrix=${matrix}" >> "$GITHUB_OUTPUT" + echo "hermes_selected=${hermes_selected}" >> "$GITHUB_OUTPUT" MATRIX_JSON="${matrix}" python - <<'PY' >> "$GITHUB_STEP_SUMMARY" import json import os @@ -328,6 +363,58 @@ jobs: if-no-files-found: ignore retention-days: 14 + hermes-e2e-vitest: + needs: [validate-jobs, generate-matrix] + if: ${{ needs.generate-matrix.outputs.hermes_selected == 'true' }} + runs-on: ubuntu-latest + timeout-minutes: 75 + env: + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/hermes-e2e + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_RUN_E2E_SCENARIOS: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_AGENT: hermes + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_RECREATE_SANDBOX: "1" + NEMOCLAW_SANDBOX_NAME: e2e-hermes + NEMOCLAW_MODEL: minimaxai/minimax-m2.7 + NEMOCLAW_ONBOARD_VALIDATION_TIMEOUT_SECONDS: "60" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Build CLI + run: npm run build:cli + + - name: Run Hermes live Vitest test + env: + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} + run: | + set -euo pipefail + npx vitest run --project e2e-scenarios-live \ + test/e2e-scenario/live/hermes-e2e.test.ts \ + --silent=false --reporter=default + + - name: Upload Hermes live Vitest artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-vitest-scenarios-hermes-e2e + path: e2e-artifacts/vitest/hermes-e2e/ + include-hidden-files: false + if-no-files-found: ignore + retention-days: 14 + network-policy-vitest: needs: [validate-jobs, generate-matrix] if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',network-policy-vitest,') || contains(format(',{0},', inputs.scenarios), ',network-policy,') }} @@ -656,6 +743,7 @@ jobs: live-scenarios, openshell-version-pin-vitest, onboard-negative-paths-vitest, + hermes-e2e-vitest, network-policy-vitest, token-rotation-vitest, openclaw-tui-chat-correlation-vitest, diff --git a/test/e2e-scenario/live/hermes-e2e.test.ts b/test/e2e-scenario/live/hermes-e2e.test.ts new file mode 100644 index 00000000000..e086f52fc41 --- /dev/null +++ b/test/e2e-scenario/live/hermes-e2e.test.ts @@ -0,0 +1,611 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { setTimeout as sleep } from "node:timers/promises"; + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { trustedProviderEndpoint } from "../fixtures/clients/provider.ts"; +import { trustedSandboxShellScript, validateSandboxName } from "../fixtures/clients/sandbox.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; + +// Migrated from test/e2e/test-hermes-e2e.sh. +// +// This is intentionally a direct live Vitest test, not a new registry layer: +// the legacy contract is the real installer/onboard/runtime boundary for Hermes. +// Vitest owns artifacts, cleanup, redaction, and timeouts while still spawning +// `bash install.sh --non-interactive`, `nemoclaw`, `openshell`, sandbox exec, +// direct NVIDIA Endpoints curl, and inference.local probes. + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-hermes"; +validateSandboxName(SANDBOX_NAME); +const HERMES_HEALTH_URL = "http://localhost:8642/health"; +const HERMES_HOST_HEALTH_URL = "http://127.0.0.1:8642/health"; +const HERMES_DASHBOARD_PORT = process.env.NEMOCLAW_DASHBOARD_PORT ?? "18789"; +const HERMES_DASHBOARD_INTERNAL_PORT = + process.env.NEMOCLAW_HERMES_DASHBOARD_INTERNAL_PORT ?? "19119"; +const SESSION_FILE = path.join(os.homedir(), ".nemoclaw", "onboard-session.json"); +const REGISTRY_FILE = path.join(os.homedir(), ".nemoclaw", "sandboxes.json"); +const LIVE_TIMEOUT_MS = 70 * 60_000; +const CHAT_MODEL = process.env.NEMOCLAW_MODEL ?? "nvidia/nemotron-3-super-120b-a12b"; +const ONBOARD_VALIDATION_TIMEOUT_SECONDS = + process.env.NEMOCLAW_ONBOARD_VALIDATION_TIMEOUT_SECONDS ?? "60"; + +interface OpenAiChoiceLike { + message?: { + content?: unknown; + reasoning_content?: unknown; + }; + text?: unknown; + finish_reason?: unknown; +} + +interface OpenAiChatLike { + choices?: OpenAiChoiceLike[]; +} + +function resultText(result: Pick): string { + return [result.stdout, result.stderr].filter(Boolean).join("\n"); +} + +function truthyEnv(value: string | undefined): boolean { + return ["1", "true", "yes", "on"].includes(value?.trim().toLowerCase() ?? ""); +} + +function hermesDashboardE2eEnabled(): boolean { + return ( + truthyEnv(process.env.NEMOCLAW_E2E_HERMES_DASHBOARD) || + truthyEnv(process.env.NEMOCLAW_HERMES_DASHBOARD) + ); +} + +function commandEnv(apiKey?: string): NodeJS.ProcessEnv { + const env: NodeJS.ProcessEnv = { + ...buildAvailabilityProbeEnv(), + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_AGENT: "hermes", + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_RECREATE_SANDBOX: "1", + NEMOCLAW_MODEL: CHAT_MODEL, + NEMOCLAW_ONBOARD_VALIDATION_TIMEOUT_SECONDS: ONBOARD_VALIDATION_TIMEOUT_SECONDS, + NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, + }; + if (apiKey) env.NVIDIA_API_KEY = apiKey; + if (process.env.NEMOCLAW_E2E_HERMES_DASHBOARD) { + env.NEMOCLAW_E2E_HERMES_DASHBOARD = process.env.NEMOCLAW_E2E_HERMES_DASHBOARD; + } + if (process.env.NEMOCLAW_HERMES_DASHBOARD) { + env.NEMOCLAW_HERMES_DASHBOARD = process.env.NEMOCLAW_HERMES_DASHBOARD; + } + if (process.env.NEMOCLAW_HERMES_DASHBOARD_TUI) { + env.NEMOCLAW_HERMES_DASHBOARD_TUI = process.env.NEMOCLAW_HERMES_DASHBOARD_TUI; + } + if (process.env.NEMOCLAW_DASHBOARD_PORT) { + env.NEMOCLAW_DASHBOARD_PORT = process.env.NEMOCLAW_DASHBOARD_PORT; + } + if (process.env.NEMOCLAW_HERMES_DASHBOARD_INTERNAL_PORT) { + env.NEMOCLAW_HERMES_DASHBOARD_INTERNAL_PORT = + process.env.NEMOCLAW_HERMES_DASHBOARD_INTERNAL_PORT; + } + return env; +} + +function chatPayload(prompt: string, maxTokens = 256): string { + return JSON.stringify({ + model: CHAT_MODEL, + messages: [{ role: "user", content: prompt }], + max_tokens: maxTokens, + }); +} + +function chatContent(response: unknown): string { + if (!response || typeof response !== "object") return ""; + const choices = (response as OpenAiChatLike).choices; + if (!Array.isArray(choices)) return ""; + for (const choice of choices) { + const message = choice?.message; + if (message) { + if (typeof message.content === "string" && message.content.trim()) { + return message.content.trim(); + } + if (typeof message.reasoning_content === "string" && message.reasoning_content.trim()) { + return message.reasoning_content.trim(); + } + } + if (typeof choice?.text === "string" && choice.text.trim()) return choice.text.trim(); + } + return ""; +} + +function firstChoice(response: unknown): OpenAiChoiceLike | undefined { + if (!response || typeof response !== "object") return undefined; + const choices = (response as OpenAiChatLike).choices; + if (!Array.isArray(choices)) return undefined; + return choices.find((choice) => choice && typeof choice === "object"); +} + +function shouldRetryForReasoningBudget(response: unknown): boolean { + const content = chatContent(response); + if (/PONG/i.test(content)) return false; + const choice = firstChoice(response); + const message = choice?.message; + return ( + choice?.finish_reason === "length" && + typeof message?.reasoning_content === "string" && + message.reasoning_content.trim().length > 0 + ); +} + +function expectPong(label: string, response: unknown): void { + const content = chatContent(response); + expect( + content, + `${label} expected PONG; response=${JSON.stringify(response).slice(0, 500)}`, + ).toMatch(/PONG/i); +} + +function readJsonFile(filePath: string): unknown { + return JSON.parse(fs.readFileSync(filePath, "utf8")); +} + +function registryEntry(name: string): Record | undefined { + if (!fs.existsSync(REGISTRY_FILE)) return undefined; + const registry = readJsonFile(REGISTRY_FILE); + if (!registry || typeof registry !== "object") return undefined; + const sandboxes = (registry as { sandboxes?: unknown }).sandboxes; + if (!sandboxes || typeof sandboxes !== "object") return undefined; + const entry = (sandboxes as Record)[name]; + return entry && typeof entry === "object" ? (entry as Record) : undefined; +} + +function httpStatusOk(status: string): boolean { + return /^[23][0-9][0-9]$/.test(status.trim()); +} + +function stripAnsi(value: string): string { + return value.replace(/\x1B(?:\[[0-?]*[ -/]*[@-~]|\][^\x07]*(?:\x07|\x1B\\)|[@-_])/g, ""); +} + +function forwardListHasRunningPort(output: string, sandboxName: string, port: string): boolean { + return output + .split("\n") + .map(stripAnsi) + .some((line) => { + const parts = line.trim().split(/\s+/); + return ( + parts.length >= 5 && + parts[0] === sandboxName && + parts[2] === port && + ["running", "active"].includes(parts.at(-1)?.toLowerCase() ?? "") + ); + }); +} + +async function bestEffort(run: () => Promise): Promise { + try { + await run(); + } catch { + // Cleanup is best-effort because the pre-install path may not have + // nemoclaw/openshell available yet. + } +} + +async function retryHostedInference( + label: string, + run: (attempt: number) => Promise, +): Promise { + let lastError: unknown; + for (let attempt = 1; attempt <= 3; attempt += 1) { + try { + return await run(attempt); + } catch (error) { + lastError = error; + if (attempt < 3) await sleep(5_000 * attempt); + } + } + throw new Error( + `${label} failed after retries: ${ + lastError instanceof Error ? lastError.message : String(lastError) + }`, + ); +} + +test.skipIf(!shouldRunLiveE2EScenarios())( + "hermes-e2e: install.sh onboards Hermes and proves health plus live inference", + { timeout: LIVE_TIMEOUT_MS }, + async ({ artifacts, cleanup, host, provider, sandbox, secrets }) => { + const apiKey = secrets.required("NVIDIA_API_KEY"); + expect(apiKey.startsWith("nvapi-"), "NVIDIA_API_KEY must start with nvapi-").toBe(true); + + await artifacts.writeJson("scenario.json", { + id: "hermes-e2e", + runner: "vitest", + migratedFrom: "test/e2e/test-hermes-e2e.sh", + boundary: "install.sh --non-interactive + Hermes sandbox runtime", + sandboxName: SANDBOX_NAME, + dashboardEnabled: hermesDashboardE2eEnabled(), + }); + + const env = commandEnv(apiKey); + const redactionValues = [apiKey]; + + const cleanupHermes = async (label: string) => { + await bestEffort(() => + host.command("nemoclaw", [SANDBOX_NAME, "destroy", "--yes"], { + artifactName: `${label}-nemoclaw-destroy`, + env: commandEnv(), + timeoutMs: 120_000, + }), + ); + await bestEffort(() => + sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { + artifactName: `${label}-openshell-sandbox-delete`, + env: commandEnv(), + timeoutMs: 60_000, + }), + ); + await bestEffort(() => + sandbox.openshell(["gateway", "destroy", "-g", "nemoclaw"], { + artifactName: `${label}-openshell-gateway-destroy`, + env: commandEnv(), + timeoutMs: 60_000, + }), + ); + }; + + cleanup.add(`destroy Hermes sandbox ${SANDBOX_NAME}`, async () => { + await cleanupHermes("cleanup"); + }); + + // Phase 0: pre-cleanup, after the secret gate so local skipped runs do not + // mutate host state. + await cleanupHermes("pre-cleanup"); + + // Phase 1: prerequisites. + const dockerInfo = await host.command("docker", ["info"], { + artifactName: "phase-1-docker-info", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + expect(dockerInfo.exitCode, resultText(dockerInfo)).toBe(0); + + expect(fs.existsSync(path.join(REPO_ROOT, "agents", "hermes", "manifest.yaml"))).toBe(true); + + const providerModels = await provider.requestJson( + trustedProviderEndpoint("https://integrate.api.nvidia.com/v1/models", { + allowedHosts: ["integrate.api.nvidia.com"], + }), + { + artifactName: "phase-1-integrate-models", + curlMaxTimeSeconds: 15, + headers: [`Authorization: Bearer ${apiKey}`], + env: buildAvailabilityProbeEnv(), + redactionValues, + timeoutMs: 30_000, + }, + ); + expect(providerModels.json).toBeTruthy(); + + // Phase 2: real installer + non-interactive Hermes onboard. + const install = await host.command("bash", ["install.sh", "--non-interactive"], { + artifactName: "phase-2-install-hermes", + cwd: REPO_ROOT, + env, + redactionValues, + timeoutMs: 60 * 60_000, + }); + expect(install.exitCode, resultText(install)).toBe(0); + + const cliProbe = await host.command( + "bash", + ["-lc", "command -v nemoclaw && command -v openshell"], + { + artifactName: "phase-2-cli-probe", + env: commandEnv(), + timeoutMs: 30_000, + }, + ); + expect(cliProbe.exitCode, resultText(cliProbe)).toBe(0); + expect(cliProbe.stdout).toContain("nemoclaw"); + expect(cliProbe.stdout).toContain("openshell"); + + const help = await host.command("nemoclaw", ["--help"], { + artifactName: "phase-2-nemoclaw-help", + env: commandEnv(), + timeoutMs: 30_000, + }); + expect(help.exitCode, resultText(help)).toBe(0); + + if (hermesDashboardE2eEnabled()) { + expect(resultText(install)).toContain( + "Deployment verified — gateway and dashboard are healthy.", + ); + expect(resultText(install)).toContain("Hermes Agent Dashboard"); + expect(resultText(install)).toContain(`http://127.0.0.1:${HERMES_DASHBOARD_PORT}/`); + } + + // Phase 3: sandbox verification. + const list = await host.command("nemoclaw", ["list"], { + artifactName: "phase-3-nemoclaw-list", + env: commandEnv(), + timeoutMs: 30_000, + }); + expect(list.exitCode, resultText(list)).toBe(0); + expect(resultText(list)).toContain(SANDBOX_NAME); + + const status = await host.command("nemoclaw", [SANDBOX_NAME, "status"], { + artifactName: "phase-3-nemoclaw-status", + env: commandEnv(), + timeoutMs: 60_000, + }); + expect(status.exitCode, resultText(status)).toBe(0); + + expect(fs.existsSync(SESSION_FILE), `${SESSION_FILE} missing`).toBe(true); + expect(readJsonFile(SESSION_FILE)).toMatchObject({ agent: "hermes" }); + + const inference = await sandbox.openshell(["inference", "get"], { + artifactName: "phase-3-openshell-inference-get", + env: commandEnv(), + timeoutMs: 30_000, + }); + expect(inference.exitCode, resultText(inference)).toBe(0); + expect(resultText(inference)).toMatch(/nvidia-prod/i); + + const policy = await sandbox.openshell(["policy", "get", "--full", SANDBOX_NAME], { + artifactName: "phase-3-openshell-policy-get", + env: commandEnv(), + timeoutMs: 30_000, + }); + expect(policy.exitCode, resultText(policy)).toBe(0); + expect(resultText(policy)).toMatch(/network_policies/i); + + // Phase 4: Hermes health and sandbox state. + let health: ShellProbeResult | undefined; + for (let attempt = 1; attempt <= 15; attempt += 1) { + health = await sandbox.exec(SANDBOX_NAME, ["curl", "-sf", HERMES_HEALTH_URL], { + artifactName: `phase-4-hermes-health-attempt-${attempt}`, + env: commandEnv(), + timeoutMs: 20_000, + }); + if (health.exitCode === 0 && /"ok"/i.test(resultText(health))) break; + await sleep(4_000); + } + expect(health, "Hermes health probe did not run").toBeTruthy(); + expect(health?.exitCode, health ? resultText(health) : "missing health result").toBe(0); + expect(resultText(health!)).toMatch(/"ok"/i); + + const hermesVersion = await sandbox.exec(SANDBOX_NAME, ["hermes", "--version"], { + artifactName: "phase-4-hermes-version", + env: commandEnv(), + timeoutMs: 30_000, + }); + expect(hermesVersion.exitCode, resultText(hermesVersion)).toBe(0); + expect(resultText(hermesVersion)).not.toMatch(/MISSING|not found|No such file/i); + + const configProbe = await sandbox.execShell( + SANDBOX_NAME, + trustedSandboxShellScript( + "test -f /sandbox/.hermes/config.yaml && test -d /sandbox/.hermes && touch /sandbox/.hermes/test-write && rm -f /sandbox/.hermes/test-write && echo OK", + ), + { + artifactName: "phase-4-hermes-config-state", + env: commandEnv(), + timeoutMs: 30_000, + }, + ); + expect(configProbe.exitCode, resultText(configProbe)).toBe(0); + expect(configProbe.stdout).toContain("OK"); + + if (hermesDashboardE2eEnabled()) { + const entry = registryEntry(SANDBOX_NAME); + expect(entry, `registry missing ${SANDBOX_NAME}`).toBeTruthy(); + expect(entry).toMatchObject({ + agent: "hermes", + dashboardPort: Number(HERMES_DASHBOARD_PORT), + }); + + const forwardList = await sandbox.openshell(["forward", "list"], { + artifactName: "phase-4-dashboard-forward-list", + env: commandEnv(), + timeoutMs: 30_000, + }); + expect(forwardList.exitCode, resultText(forwardList)).toBe(0); + expect(forwardListHasRunningPort(forwardList.stdout, SANDBOX_NAME, "8642")).toBe(true); + expect( + forwardListHasRunningPort(forwardList.stdout, SANDBOX_NAME, HERMES_DASHBOARD_PORT), + ).toBe(true); + + const hostDashboard = await host.command( + "curl", + [ + "-sS", + "-L", + "--max-time", + "10", + "-o", + "/tmp/hermes-dashboard-vitest-body", + "-w", + "%{http_code}", + `http://127.0.0.1:${HERMES_DASHBOARD_PORT}/`, + ], + { + artifactName: "phase-4-dashboard-host-probe", + env: commandEnv(), + timeoutMs: 30_000, + }, + ); + expect(hostDashboard.exitCode, resultText(hostDashboard)).toBe(0); + expect(httpStatusOk(hostDashboard.stdout)).toBe(true); + + const hostHealth = await host.command( + "curl", + ["-sf", "--max-time", "10", HERMES_HOST_HEALTH_URL], + { + artifactName: "phase-4-hermes-host-health", + env: commandEnv(), + timeoutMs: 30_000, + }, + ); + expect(hostHealth.exitCode, resultText(hostHealth)).toBe(0); + expect(resultText(hostHealth)).toMatch(/"ok"/i); + + const dashboardInternal = await sandbox.exec( + SANDBOX_NAME, + [ + "curl", + "-sS", + "-L", + "--max-time", + "10", + "-o", + "/tmp/hermes-dashboard-vitest-body", + "-w", + "%{http_code}", + `http://127.0.0.1:${HERMES_DASHBOARD_INTERNAL_PORT}/`, + ], + { + artifactName: "phase-4-dashboard-sandbox-probe", + env: commandEnv(), + timeoutMs: 30_000, + }, + ); + expect(dashboardInternal.exitCode, resultText(dashboardInternal)).toBe(0); + expect(httpStatusOk(dashboardInternal.stdout)).toBe(true); + } + + // Phase 5: live inference through both the external provider and the + // sandbox's inference.local route. + const directChat = await retryHostedInference( + "direct NVIDIA Endpoints chat", + async (attempt) => { + const response = await provider.requestJson( + trustedProviderEndpoint("https://integrate.api.nvidia.com/v1/chat/completions", { + allowedHosts: ["integrate.api.nvidia.com"], + }), + { + artifactName: `phase-5-direct-nvidia-chat-attempt-${attempt}`, + body: chatPayload("Reply with exactly one word: PONG", attempt === 1 ? 256 : 1024), + curlMaxTimeSeconds: 90, + headers: ["Content-Type: application/json", `Authorization: Bearer ${apiKey}`], + env: buildAvailabilityProbeEnv(), + redactionValues, + timeoutMs: 120_000, + }, + ); + if (shouldRetryForReasoningBudget(response.json)) { + throw new Error("direct chat exhausted response budget while reasoning before PONG"); + } + return response; + }, + ); + expectPong("direct NVIDIA Endpoints chat", directChat.json); + + const sandboxChatJson = await retryHostedInference( + "Hermes sandbox inference.local chat", + async (attempt) => { + const result = await sandbox.exec( + SANDBOX_NAME, + [ + "curl", + "-fsS", + "--max-time", + "90", + "-H", + "Content-Type: application/json", + "--data-raw", + chatPayload("Reply with exactly one word: PONG", attempt === 1 ? 256 : 1024), + "https://inference.local/v1/chat/completions", + ], + { + artifactName: `phase-5-inference-local-chat-attempt-${attempt}`, + env: commandEnv(), + timeoutMs: 120_000, + }, + ); + if (result.exitCode !== 0) throw new Error(resultText(result)); + let parsed: unknown; + try { + parsed = JSON.parse(result.stdout) as unknown; + } catch (error) { + throw new Error( + `Hermes sandbox inference.local chat response was not JSON: ${ + error instanceof Error ? error.message : String(error) + }; body=${result.stdout.slice(0, 500)}`, + ); + } + if (shouldRetryForReasoningBudget(parsed)) { + throw new Error("sandbox chat exhausted response budget while reasoning before PONG"); + } + return parsed; + }, + ); + expectPong("Hermes sandbox inference.local chat", sandboxChatJson); + + // Phase 6: CLI operations and agent manifest regression. + const logs = await host.command("nemoclaw", [SANDBOX_NAME, "logs"], { + artifactName: "phase-6-nemoclaw-logs", + env: commandEnv(), + timeoutMs: 60_000, + }); + expect(logs.exitCode, resultText(logs)).toBe(0); + expect(resultText(logs).trim().length).toBeGreaterThan(0); + + const manifestCheck = await host.command( + "node", + [ + "-e", + `const { loadAgent, listAgents } = require(${JSON.stringify(path.join(REPO_ROOT, "bin", "lib", "agent-defs"))});\n` + + `const agents = listAgents();\n` + + `console.log('agents:', agents.join(', '));\n` + + `console.log('openclaw_display:', loadAgent('openclaw').displayName);\n` + + `console.log('hermes_display:', loadAgent('hermes').displayName);`, + ], + { + artifactName: "phase-6-agent-manifest-check", + env: commandEnv(), + timeoutMs: 30_000, + }, + ); + expect(manifestCheck.exitCode, resultText(manifestCheck)).toBe(0); + expect(manifestCheck.stdout).toMatch(/openclaw_display:.*OpenClaw/); + expect(manifestCheck.stdout).toMatch(/hermes_display:.*Hermes/); + expect(manifestCheck.stdout).toMatch(/agents:.*(openclaw.*hermes|hermes.*openclaw)/); + + // Phase 8: explicit cleanup and post-destroy registry proof. + if (process.env.NEMOCLAW_E2E_KEEP_SANDBOX !== "1") { + const destroy = await host.command("nemoclaw", [SANDBOX_NAME, "destroy", "--yes"], { + artifactName: "phase-8-nemoclaw-destroy", + env: commandEnv(), + timeoutMs: 120_000, + }); + expect(destroy.exitCode, resultText(destroy)).toBe(0); + await bestEffort(() => + sandbox.openshell(["gateway", "destroy", "-g", "nemoclaw"], { + artifactName: "phase-8-openshell-gateway-destroy", + env: commandEnv(), + timeoutMs: 60_000, + }), + ); + expect( + registryEntry(SANDBOX_NAME), + `${SANDBOX_NAME} still in ${REGISTRY_FILE}`, + ).toBeUndefined(); + } + + await artifacts.writeJson("scenario-result.json", { + id: "hermes-e2e", + assertions: { + installShNonInteractiveHermes: true, + sandboxListedAndHealthy: true, + directProviderInferencePong: true, + sandboxInferenceLocalPong: true, + dashboardChecked: hermesDashboardE2eEnabled(), + }, + }); + }, +); diff --git a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts index 94f0e556713..2a1385c2e60 100644 --- a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts @@ -1,16 +1,70 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; +import YAML from "yaml"; import { evaluateE2eVitestWorkflowDispatchSelectors, validateE2eVitestScenariosWorkflowBoundary, } from "../../../tools/e2e-scenarios/workflow-boundary.mts"; +function readWorkflow(): Record { + return YAML.parse( + fs.readFileSync( + path.join(process.cwd(), ".github/workflows/e2e-vitest-scenarios.yaml"), + "utf-8", + ), + ) as Record; +} + +function generateMatrixForDispatch(env: { + JOBS: string; + SCENARIOS: string; +}): Record { + const workflow = readWorkflow(); + const jobs = workflow.jobs as Record> }>; + const generateStep = jobs["generate-matrix"]?.steps?.find( + (step) => step.name === "Generate Vitest scenario matrix", + ); + expect(generateStep?.run).toEqual(expect.any(String)); + + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-vitest-matrix-")); + const outputPath = path.join(tmp, "github-output"); + const summaryPath = path.join(tmp, "github-summary"); + try { + const result = spawnSync("bash", ["-c", generateStep?.run as string], { + cwd: process.cwd(), + encoding: "utf-8", + timeout: 120_000, + killSignal: "SIGKILL", + env: { + ...process.env, + GITHUB_OUTPUT: outputPath, + GITHUB_STEP_SUMMARY: summaryPath, + JOBS: env.JOBS, + SCENARIOS: env.SCENARIOS, + }, + }); + expect(result.signal).toBeNull(); + expect(result.stderr).toBe(""); + expect(result.status).toBe(0); + return Object.fromEntries( + fs + .readFileSync(outputPath, "utf-8") + .trim() + .split("\n") + .map((line) => line.split(/=(.*)/s).slice(0, 2)), + ); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } +} + describe("e2e-vitest-scenarios workflow boundary", () => { it("keeps the live Vitest scenario workflow manual, pinned, and artifact-safe", () => { expect(validateE2eVitestScenariosWorkflowBoundary()).toEqual([]); @@ -60,6 +114,35 @@ describe("e2e-vitest-scenarios workflow boundary", () => { selectedFreeStandingJobs: ["openshell-version-pin-vitest"], registryScenarios: [], }); + expect(evaluateE2eVitestWorkflowDispatchSelectors({ scenarios: "hermes-e2e" })).toMatchObject({ + valid: true, + liveScenariosRuns: false, + selectedFreeStandingJobs: ["hermes-e2e-vitest"], + registryScenarios: [], + }); + }); + + it("keeps jobs-only dispatches from selecting the Hermes secret-bearing job", () => { + expect( + generateMatrixForDispatch({ JOBS: "openshell-version-pin-vitest", SCENARIOS: "" }), + ).toMatchObject({ + hermes_selected: "false", + matrix: "[]", + }); + expect(generateMatrixForDispatch({ JOBS: "hermes-e2e-vitest", SCENARIOS: "" })).toMatchObject({ + hermes_selected: "true", + matrix: "[]", + }); + expect( + generateMatrixForDispatch({ JOBS: "network-policy-vitest", SCENARIOS: "" }), + ).toMatchObject({ + hermes_selected: "false", + matrix: "[]", + }); + expect(generateMatrixForDispatch({ JOBS: "", SCENARIOS: "hermes-e2e" })).toMatchObject({ + hermes_selected: "true", + matrix: "[]", + }); }); it("flags direct dispatch-input interpolation and unsafe artifact upload", () => { @@ -237,10 +320,12 @@ jobs: "step 'Validate free-standing job selector' run script must include Use either scenarios or jobs, not both", "step 'Validate free-standing job selector' run script must include Invalid scenario input; use comma-separated scenario ids", "step 'Validate free-standing job selector' run script must include allowed_jobs=", + "step 'Validate free-standing job selector' run script must include hermes-e2e-vitest", "step 'Validate free-standing job selector' run script must include Invalid jobs input; use comma-separated job ids", "step 'Validate free-standing job selector' run script must not include Invalid jobs input: ${JOBS}", "step 'Validate free-standing job selector' run script must include Unknown free-standing Vitest job", "workflow missing generate-matrix job", + "generate-matrix job must expose hermes_selected output", "generate-matrix job must run on ubuntu-latest", "live-scenarios job must run on the matrix runner", "live-scenarios job must depend on generate-matrix", @@ -345,6 +430,8 @@ jobs: "network-policy-vitest artifact upload must ignore missing fixture artifacts", "network-policy-vitest artifact upload retention-days must be 14", "report-to-pr job must wait for network-policy-vitest", + "workflow missing hermes-e2e-vitest job", + "report-to-pr job must wait for hermes-e2e-vitest", "openclaw-tui-chat-correlation-vitest job must depend on validate-jobs and generate-matrix", "openclaw-tui-chat-correlation-vitest job must use the shared jobs selector condition", "gateway-guard-recovery job must depend on validate-jobs", @@ -366,4 +453,32 @@ jobs: fs.rmSync(tmp, { recursive: true, force: true }); } }); + + it("rejects raw jobs selector echo from matrix generation", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-vitest-workflow-")); + const workflowPath = path.join(tmp, "workflow.yaml"); + const workflow = fs.readFileSync( + path.join(process.cwd(), ".github/workflows/e2e-vitest-scenarios.yaml"), + "utf8", + ); + fs.writeFileSync( + workflowPath, + workflow.replace( + 'echo "::error::Invalid jobs input; use comma-separated job ids" >&2', + 'echo "::error::Invalid jobs input: ${JOBS}" >&2', + ), + ); + + try { + const errors = validateE2eVitestScenariosWorkflowBoundary(workflowPath); + expect(errors).toEqual( + expect.arrayContaining([ + "step 'Generate Vitest scenario matrix' run script must include Invalid jobs input; use comma-separated job ids", + "step 'Generate Vitest scenario matrix' run script must not include Invalid jobs input: ${JOBS}", + ]), + ); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); }); diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts index 3f0d41e1f7f..48f4e3dc85b 100644 --- a/tools/e2e-scenarios/workflow-boundary.mts +++ b/tools/e2e-scenarios/workflow-boundary.mts @@ -21,6 +21,7 @@ const SELECTOR_PATTERN = /^[A-Za-z0-9_-]+(,[A-Za-z0-9_-]+)*$/; const FREE_STANDING_SCENARIO_JOBS = new Map([ ["openshell-version-pin", "openshell-version-pin-vitest"], ["onboard-negative-paths", "onboard-negative-paths-vitest"], + ["hermes-e2e", "hermes-e2e-vitest"], ["network-policy", "network-policy-vitest"], ["token-rotation", "token-rotation-vitest"], ["openclaw-tui-chat-correlation", "openclaw-tui-chat-correlation-vitest"], @@ -273,6 +274,7 @@ function validateJobsSelector(errors: string[], jobs: WorkflowRecord): void { requireRunContains(errors, validate, "allowed_jobs="); requireRunContains(errors, validate, "openshell-version-pin-vitest"); requireRunContains(errors, validate, "onboard-negative-paths-vitest"); + requireRunContains(errors, validate, "hermes-e2e-vitest"); requireRunContains(errors, validate, "network-policy-vitest"); requireRunContains(errors, validate, "token-rotation-vitest"); requireRunContains(errors, validate, "openclaw-tui-chat-correlation-vitest"); @@ -700,6 +702,107 @@ function validateOnboardNegativePathsVitestJob(errors: string[], jobs: WorkflowR } } +function validateHermesE2EVitestJob(errors: string[], jobs: WorkflowRecord): void { + const jobName = "hermes-e2e-vitest"; + const job = asRecord(jobs[jobName]); + if (Object.keys(job).length === 0) { + errors.push("workflow missing hermes-e2e-vitest job"); + return; + } + + if (job["runs-on"] !== "ubuntu-latest") { + errors.push("hermes-e2e-vitest job must run on ubuntu-latest"); + } + const needs = Array.isArray(job.needs) ? job.needs : []; + if (!needs.includes("validate-jobs") || !needs.includes("generate-matrix")) { + errors.push("hermes-e2e-vitest job must depend on validate-jobs and generate-matrix validation"); + } + if (job.if !== "${{ needs.generate-matrix.outputs.hermes_selected == 'true' }}") { + errors.push("hermes-e2e-vitest job must use validated hermes_selected output"); + } + if (stringValue(job.if).includes("inputs.scenarios")) { + errors.push("hermes-e2e-vitest job must not inspect raw workflow dispatch scenarios"); + } + + const jobEnv = asRecord(job.env); + if (jobEnv.NEMOCLAW_RUN_E2E_SCENARIOS !== "1") { + errors.push("hermes-e2e-vitest job must set NEMOCLAW_RUN_E2E_SCENARIOS=1"); + } + if (jobEnv.NEMOCLAW_CLI_BIN !== "${{ github.workspace }}/bin/nemoclaw.js") { + errors.push("hermes-e2e-vitest job must point NEMOCLAW_CLI_BIN at the repo CLI"); + } + if (jobEnv.E2E_ARTIFACT_DIR !== "${{ github.workspace }}/e2e-artifacts/vitest/hermes-e2e") { + errors.push("hermes-e2e-vitest job must write artifacts under e2e-artifacts/vitest/hermes-e2e"); + } + if (jobEnv.NEMOCLAW_AGENT !== "hermes") { + errors.push("hermes-e2e-vitest job must set NEMOCLAW_AGENT=hermes"); + } + if (jobEnv.NEMOCLAW_MODEL !== "minimaxai/minimax-m2.7") { + errors.push("hermes-e2e-vitest job must pin the CI-safe Hermes model"); + } + if (jobEnv.NEMOCLAW_ONBOARD_VALIDATION_TIMEOUT_SECONDS !== "60") { + errors.push("hermes-e2e-vitest job must give hosted endpoint validation a CI-safe timeout"); + } + requireEnvDoesNotExposeSecret(errors, "hermes-e2e-vitest job", jobEnv, "NVIDIA_API_KEY"); + + const steps = asSteps(job.steps); + requireNoDispatchInputInterpolation(errors, steps); + for (const step of steps) { + if (step.name !== "Run Hermes live Vitest test") { + requireEnvDoesNotExposeSecret( + errors, + `hermes-e2e-vitest step '${step.name ?? step.uses ?? ""}'`, + asRecord(step.env), + "NVIDIA_API_KEY", + ); + } + } + + const checkout = steps.find((step) => stringValue(step.uses).startsWith("actions/checkout@")); + if (!checkout) errors.push("hermes-e2e-vitest job missing checkout step"); + requireFullShaAction(errors, checkout, "hermes-e2e-vitest checkout"); + if (asRecord(checkout?.with)["persist-credentials"] !== false) { + errors.push("hermes-e2e-vitest checkout step must set persist-credentials=false"); + } + + const setupNode = namedStep(steps, "Set up Node"); + if (!setupNode) errors.push("hermes-e2e-vitest job missing step: Set up Node"); + requireFullShaAction(errors, setupNode, "hermes-e2e-vitest setup-node"); + + const installRootDependencies = requireJobStep(errors, jobName, steps, "Install root dependencies"); + requireRunContains(errors, installRootDependencies, "npm ci --ignore-scripts"); + + const buildCli = requireJobStep(errors, jobName, steps, "Build CLI"); + requireRunContains(errors, buildCli, "npm run build:cli"); + + const runVitest = requireJobStep(errors, jobName, steps, "Run Hermes live Vitest test"); + const runVitestEnv = asRecord(runVitest?.env); + if (runVitestEnv.NVIDIA_API_KEY !== "${{ secrets.NVIDIA_API_KEY }}") { + errors.push("hermes-e2e-vitest Vitest step must receive NVIDIA_API_KEY from secrets"); + } + requireRunContains(errors, runVitest, "npx vitest run --project e2e-scenarios-live"); + requireRunContains(errors, runVitest, "test/e2e-scenario/live/hermes-e2e.test.ts"); + requireRunDoesNotContain(errors, runVitest, "${{ inputs."); + + const upload = requireJobStep(errors, jobName, steps, "Upload Hermes live Vitest artifacts"); + requireFullShaAction(errors, upload, "hermes-e2e-vitest upload-artifact"); + const uploadWith = asRecord(upload?.with); + if (uploadWith.name !== "e2e-vitest-scenarios-hermes-e2e") { + errors.push("hermes-e2e-vitest artifact upload name must be stable"); + } + const uploadPath = stringValue(uploadWith.path); + requireUploadPathContains(errors, uploadPath, "e2e-artifacts/vitest/hermes-e2e/"); + if (uploadWith["include-hidden-files"] !== false) { + errors.push("hermes-e2e-vitest artifact upload must set include-hidden-files: false"); + } + if (uploadWith["if-no-files-found"] !== "ignore") { + errors.push("hermes-e2e-vitest artifact upload must ignore missing fixture artifacts"); + } + if (uploadWith["retention-days"] !== 14) { + errors.push("hermes-e2e-vitest artifact upload retention-days must be 14"); + } +} + export function validateE2eVitestScenariosWorkflowBoundary( workflowPath = DEFAULT_VITEST_WORKFLOW_PATH, ): string[] { @@ -728,6 +831,13 @@ export function validateE2eVitestScenariosWorkflowBoundary( if (generateMatrix["runs-on"] !== "ubuntu-latest") { errors.push("generate-matrix job must run on ubuntu-latest"); } + const generateOutputs = asRecord(generateMatrix.outputs); + if (generateOutputs.matrix !== "${{ steps.matrix.outputs.matrix }}") { + errors.push("generate-matrix job must expose matrix output"); + } + if (generateOutputs.hermes_selected !== "${{ steps.matrix.outputs.hermes_selected }}") { + errors.push("generate-matrix job must expose hermes_selected output"); + } const generateSteps = asSteps(generateMatrix.steps); requireNoDispatchInputInterpolation(errors, generateSteps); const generateCheckout = generateSteps.find((step) => stringValue(step.uses).startsWith("actions/checkout@")); @@ -741,16 +851,31 @@ export function validateE2eVitestScenariosWorkflowBoundary( requireFullShaAction(errors, generateSetupNode, "generate-matrix setup-node"); const generate = requireStep(errors, generateSteps, "Generate Vitest scenario matrix"); const generateEnv = asRecord(generate?.env); + if (generateEnv.JOBS !== "${{ inputs.jobs }}") { + errors.push("matrix generation step must pass jobs through JOBS env"); + } if (generateEnv.SCENARIOS !== "${{ inputs.scenarios }}") { errors.push("matrix generation step must pass scenarios through SCENARIOS env"); } + requireRunContains(errors, generate, "allowed_jobs="); + requireRunContains(errors, generate, "Use either scenarios or jobs, not both"); + requireRunContains(errors, generate, "Unknown free-standing Vitest job"); + requireRunContains(errors, generate, "hermes-e2e-vitest"); + requireRunContains(errors, generate, "network-policy-vitest"); + requireRunContains(errors, generate, "token-rotation-vitest"); + requireRunContains(errors, generate, 'matrix="[]"'); requireRunContains(errors, generate, "npx tsx test/e2e-scenario/scenarios/run.ts"); requireRunContains(errors, generate, "--emit-live-matrix"); requireRunContains(errors, generate, "--scenarios"); requireRunContains(errors, generate, "^[A-Za-z0-9_-]+(,[A-Za-z0-9_-]+)*$"); requireRunContains(errors, generate, "Invalid scenario input; use comma-separated scenario ids"); + requireRunContains(errors, generate, "Invalid jobs input; use comma-separated job ids"); + requireRunDoesNotContain(errors, generate, "Invalid jobs input: ${JOBS}"); requireRunDoesNotContain(errors, generate, "Invalid scenario input: ${SCENARIOS}"); requireRunDoesNotContain(errors, generate, "^[A-Za-z0-9._-]+"); + requireRunContains(errors, generate, "hermes_selected=false"); + requireRunContains(errors, generate, "hermes_selected=true"); + requireRunContains(errors, generate, 'echo "hermes_selected=${hermes_selected}" >> "$GITHUB_OUTPUT"'); requireRunContains(errors, generate, "## Vitest E2E Scenario Matrix"); requireRunContains(errors, generate, "| Scenario | Runner | Label |"); @@ -890,6 +1015,7 @@ export function validateE2eVitestScenariosWorkflowBoundary( validateOpenShellVersionPinVitestJob(errors, jobs); validateOnboardNegativePathsVitestJob(errors, jobs); + validateHermesE2EVitestJob(errors, jobs); validateNetworkPolicyVitestJob(errors, jobs); validateTokenRotationVitestJob(errors, jobs); validateFreeStandingJobSelector( @@ -911,6 +1037,7 @@ export function validateE2eVitestScenariosWorkflowBoundary( "live-scenarios", "openshell-version-pin-vitest", "onboard-negative-paths-vitest", + "hermes-e2e-vitest", "network-policy-vitest", "token-rotation-vitest", "openclaw-tui-chat-correlation-vitest",