diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index bd32b225b4a..5cc5eb06d1c 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -1756,6 +1756,61 @@ jobs: if-no-files-found: ignore retention-days: 14 + hermes-dashboard-vitest: + needs: generate-matrix + if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',hermes-dashboard-vitest,') || contains(format(',{0},', inputs.scenarios), ',hermes-dashboard,') }} + runs-on: ubuntu-latest + timeout-minutes: 75 + env: + FREE_STANDING_VITEST_JOB: "1" + FREE_STANDING_SCENARIO_ID: "hermes-dashboard" + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/hermes-dashboard + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_RUN_E2E_SCENARIOS: "1" + NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_AGENT: hermes + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_RECREATE_SANDBOX: "1" + NEMOCLAW_SANDBOX_NAME: e2e-hermes-dashboard + NEMOCLAW_E2E_HERMES_DASHBOARD: "1" + NEMOCLAW_ONBOARD_VALIDATION_TIMEOUT_SECONDS: "60" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Build CLI + run: npm run build:cli + + - name: Run Hermes dashboard live Vitest test + env: + NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} + run: | + set -euo pipefail + npx vitest run --project e2e-scenarios-live \ + test/e2e-scenario/live/hermes-e2e.test.ts \ + --silent=false --reporter=default + + - name: Upload Hermes dashboard live Vitest artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-vitest-scenarios-hermes-dashboard + path: e2e-artifacts/vitest/hermes-dashboard/ + include-hidden-files: false + if-no-files-found: ignore + retention-days: 14 + hermes-discord-vitest: needs: generate-matrix if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',hermes-discord-vitest,') || contains(format(',{0},', inputs.scenarios), ',hermes-discord,') }} @@ -5674,6 +5729,7 @@ jobs: sessions-agents-cli-vitest, runtime-overrides-vitest, hermes-e2e-vitest, + hermes-dashboard-vitest, hermes-slack-vitest, hermes-discord-vitest, hermes-root-entrypoint-smoke-vitest, diff --git a/test/e2e-scenario/support-tests/hermes-dashboard-workflow-boundary.test.ts b/test/e2e-scenario/support-tests/hermes-dashboard-workflow-boundary.test.ts new file mode 100644 index 00000000000..6fbf4ab5bf0 --- /dev/null +++ b/test/e2e-scenario/support-tests/hermes-dashboard-workflow-boundary.test.ts @@ -0,0 +1,57 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import { + readHermesDashboardWorkflow, + validateHermesDashboardWorkflow, + validateHermesDashboardWorkflowBoundary, +} from "../../../tools/e2e-scenarios/hermes-dashboard-workflow-boundary.mts"; +import { + evaluateE2eVitestWorkflowDispatchSelectors, + readFreeStandingJobsInventory, +} from "../../../tools/e2e-scenarios/workflow-boundary.mts"; + +describe("Hermes dashboard workflow boundary", () => { + it("runs by default and through either selective dispatch input", () => { + const inventory = readFreeStandingJobsInventory(); + expect(validateHermesDashboardWorkflowBoundary()).toEqual([]); + expect(inventory.scenarioToJob.get("hermes-dashboard")).toBe("hermes-dashboard-vitest"); + + for (const selector of [ + { scenarios: "hermes-dashboard" }, + { jobs: "hermes-dashboard-vitest" }, + ]) { + expect(evaluateE2eVitestWorkflowDispatchSelectors(selector)).toMatchObject({ + valid: true, + liveScenariosRuns: false, + selectedFreeStandingJobs: ["hermes-dashboard-vitest"], + }); + } + expect(evaluateE2eVitestWorkflowDispatchSelectors({}).selectedFreeStandingJobs).toContain( + "hermes-dashboard-vitest", + ); + }); + + it("rejects dashboard mode, execution, and reporting drift", () => { + const dashboardMode = readHermesDashboardWorkflow(); + dashboardMode.jobs["hermes-dashboard-vitest"].env!.NEMOCLAW_E2E_HERMES_DASHBOARD = "0"; + expect(validateHermesDashboardWorkflow(dashboardMode)).toContain( + "hermes-dashboard-vitest must enable Hermes dashboard coverage", + ); + + const execution = readHermesDashboardWorkflow(); + execution.jobs["hermes-dashboard-vitest"].steps!.find( + (step) => step.name === "Run Hermes dashboard live Vitest test", + )!.run = "echo skipped"; + expect(validateHermesDashboardWorkflow(execution)).toContain( + "hermes-dashboard-vitest must run the live Vitest project", + ); + + const reporting = readHermesDashboardWorkflow(); + reporting.jobs["report-to-pr"].needs = []; + expect(validateHermesDashboardWorkflow(reporting)).toContain( + "report-to-pr must wait for hermes-dashboard-vitest", + ); + }); +}); diff --git a/tools/e2e-scenarios/hermes-dashboard-workflow-boundary.mts b/tools/e2e-scenarios/hermes-dashboard-workflow-boundary.mts new file mode 100644 index 00000000000..7f4ee7c98fc --- /dev/null +++ b/tools/e2e-scenarios/hermes-dashboard-workflow-boundary.mts @@ -0,0 +1,114 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import YAML from "yaml"; + +const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); +const DEFAULT_WORKFLOW_PATH = join(REPO_ROOT, ".github", "workflows", "e2e-vitest-scenarios.yaml"); +const JOB_NAME = "hermes-dashboard-vitest"; + +type WorkflowStep = { + env?: Record; + name?: string; + run?: string; + uses?: string; + with?: Record; +}; + +type WorkflowJob = { + env?: Record; + if?: string; + needs?: string[] | string; + steps?: WorkflowStep[]; + "runs-on"?: string; + "timeout-minutes"?: number; +}; + +export type HermesDashboardWorkflow = { + jobs: Record; +}; + +export function readHermesDashboardWorkflow( + workflowPath = DEFAULT_WORKFLOW_PATH, +): HermesDashboardWorkflow { + return YAML.parse(readFileSync(workflowPath, "utf8")) as HermesDashboardWorkflow; +} + +function requireEqual(errors: string[], actual: unknown, expected: unknown, message: string): void { + if (actual !== expected) errors.push(message); +} + +function findStep(job: WorkflowJob, name: string): WorkflowStep { + return job.steps?.find((step) => step.name === name) ?? {}; +} + +export function validateHermesDashboardWorkflow(workflow: HermesDashboardWorkflow): string[] { + const errors: string[] = []; + const job = workflow.jobs[JOB_NAME] ?? {}; + const env = job.env ?? {}; + + requireEqual(errors, job.needs, "generate-matrix", `${JOB_NAME} must depend on generate-matrix`); + requireEqual(errors, job["runs-on"], "ubuntu-latest", `${JOB_NAME} must run on ubuntu-latest`); + requireEqual(errors, job["timeout-minutes"], 75, `${JOB_NAME} timeout must be 75 minutes`); + requireEqual(errors, env.FREE_STANDING_VITEST_JOB, "1", `${JOB_NAME} must be free-standing`); + requireEqual( + errors, + env.FREE_STANDING_SCENARIO_ID, + "hermes-dashboard", + `${JOB_NAME} must publish the hermes-dashboard selector`, + ); + requireEqual( + errors, + env.NEMOCLAW_E2E_HERMES_DASHBOARD, + "1", + `${JOB_NAME} must enable Hermes dashboard coverage`, + ); + requireEqual(errors, env.NEMOCLAW_AGENT, "hermes", `${JOB_NAME} must run Hermes`); + requireEqual( + errors, + env.NEMOCLAW_SANDBOX_NAME, + "e2e-hermes-dashboard", + `${JOB_NAME} must use an isolated sandbox`, + ); + + const run = findStep(job, "Run Hermes dashboard live Vitest test"); + if (!run.run?.includes("npx vitest run --project e2e-scenarios-live")) { + errors.push(`${JOB_NAME} must run the live Vitest project`); + } + if (!run.run?.includes("test/e2e-scenario/live/hermes-e2e.test.ts")) { + errors.push(`${JOB_NAME} must run the Hermes live test`); + } + requireEqual( + errors, + run.env?.NVIDIA_INFERENCE_API_KEY, + "${{ secrets.NVIDIA_INFERENCE_API_KEY }}", + `${JOB_NAME} must pass the inference key through step env`, + ); + + const upload = findStep(job, "Upload Hermes dashboard live Vitest artifacts"); + if (!/^actions\/upload-artifact@[0-9a-f]{40}$/u.test(upload.uses ?? "")) { + errors.push(`${JOB_NAME} artifact upload must pin a full action SHA`); + } + requireEqual( + errors, + upload.with?.path, + "e2e-artifacts/vitest/hermes-dashboard/", + `${JOB_NAME} must upload its dashboard artifacts`, + ); + + const reportNeeds = workflow.jobs["report-to-pr"]?.needs; + if (!Array.isArray(reportNeeds) || !reportNeeds.includes(JOB_NAME)) { + errors.push(`report-to-pr must wait for ${JOB_NAME}`); + } + + return errors; +} + +export function validateHermesDashboardWorkflowBoundary( + workflowPath = DEFAULT_WORKFLOW_PATH, +): string[] { + return validateHermesDashboardWorkflow(readHermesDashboardWorkflow(workflowPath)); +}