diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 63c32c614b0..461d1a10e8f 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -203,7 +203,7 @@ jobs: fi larger_runner="${LARGER_RUNNER_LABEL}" fi - runner_routing="$(jq -cn --arg standard "ubuntu-latest" --arg larger "${larger_runner}" '{"channels-stop-start-hermes":$larger,"channels-stop-start-openclaw":$standard,"common-egress-agent":$larger,"hermes-dashboard":$larger,"hermes-discord":$larger,"hermes-e2e":$larger,"hermes-inference-switch":$larger,"hermes-shields-config":$larger,"mcp-bridge-deepagents":$larger,"mcp-bridge-hermes":$larger,"mcp-bridge-openclaw":$standard,"rebuild-hermes":$larger,"rebuild-hermes-stale-base":$larger,"security-posture-hermes":$larger,"security-posture-openclaw":$standard}')" + runner_routing="$(jq -cn --arg standard "ubuntu-latest" --arg larger "${larger_runner}" '{"channels-stop-start-hermes":$larger,"channels-stop-start-openclaw":$standard,"common-egress-agent":$larger,"hermes-discord":$larger,"hermes-e2e":$larger,"hermes-inference-switch":$larger,"hermes-shields-config":$larger,"mcp-bridge-deepagents":$larger,"mcp-bridge-hermes":$larger,"mcp-bridge-openclaw":$standard,"rebuild-hermes":$larger,"rebuild-hermes-stale-base":$larger,"security-posture-hermes":$larger,"security-posture-openclaw":$standard}')" printf 'runner_routing=%s\n' "${runner_routing}" >> "${GITHUB_OUTPUT}" - name: Authenticate controller dispatch @@ -2170,6 +2170,7 @@ jobs: NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_LIVE_E2E: "1" NEMOCLAW_E2E_INFERENCE_MODE: ${{ inputs.inference_mode || 'mock' }} + NEMOCLAW_E2E_HERMES_DASHBOARD: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" NEMOCLAW_AGENT: hermes NEMOCLAW_NON_INTERACTIVE: "1" @@ -2179,7 +2180,7 @@ jobs: steps: - id: trusted_hermes_swap name: Provision trusted Hermes E2E swap - if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && (github.event_name == 'schedule' || inputs.checkout_sha == '' || (github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && (contains(format(',{0},', inputs.jobs), ',hermes-e2e,') || contains(format(',{0},', inputs.targets), ',hermes-e2e,')))) }} + if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && (github.event_name == 'schedule' || inputs.checkout_sha == '' || (github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && (contains(format(',{0},', inputs.jobs), ',hermes-e2e,') || contains(format(',{0},', inputs.targets), ',hermes-e2e,') || contains(format(',{0},', inputs.jobs), ',hermes-dashboard,') || contains(format(',{0},', inputs.targets), ',hermes-dashboard,')))) }} shell: /bin/bash --noprofile --norc -e -o pipefail {0} env: BASH_ENV: /dev/null @@ -2499,85 +2500,6 @@ jobs: shell: bash run: bash .github/scripts/docker-auth-cleanup.sh - hermes-dashboard: - needs: generate-matrix - if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',hermes-dashboard,') || contains(format(',{0},', inputs.targets), ',hermes-dashboard,') }} - runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['hermes-dashboard'] }} - timeout-minutes: 85 - env: - E2E_JOB: "1" - E2E_TARGET_ID: "hermes-dashboard" - E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/hermes-dashboard - NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js - NEMOCLAW_RUN_LIVE_E2E: "1" - NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" - NEMOCLAW_AGENT: hermes - NEMOCLAW_NON_INTERACTIVE: "1" - NEMOCLAW_RECREATE_SANDBOX: "1" - NEMOCLAW_SANDBOX_NAME: e2e-hermes-dashboard - NEMOCLAW_E2E_HERMES_DASHBOARD: "1" - NEMOCLAW_ONBOARD_VALIDATION_TIMEOUT_SECONDS: "60" - steps: - - id: trusted_hermes_swap - name: Provision trusted Hermes E2E swap - if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }} - shell: /bin/bash --noprofile --norc -e -o pipefail {0} - env: - BASH_ENV: /dev/null - CHECKOUT_SHA: ${{ inputs.checkout_sha }} - DISPATCH_SHA: ${{ github.sha }} - ENV: /dev/null - EVENT_NAME: ${{ github.event_name }} - EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }} - LC_ALL: C - REF: ${{ github.ref }} - REPOSITORY: ${{ github.repository }} - RUNNER_ARCH_KIND: ${{ runner.arch }} - RUNNER_ENVIRONMENT_KIND: ${{ runner.environment }} - RUNNER_OS_KIND: ${{ runner.os }} - WORKFLOW_SHA: ${{ github.workflow_sha }} - run: *trusted-hermes-e2e-swap - - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - repository: ${{ inputs.checkout_repository || github.repository }} - ref: ${{ inputs.checkout_sha || github.sha }} - persist-credentials: false - - - *dockerhub-auth - - - name: Prepare E2E workspace - uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75 - - - name: Initialize runner comparison telemetry - if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' }} - continue-on-error: true - shell: bash - run: npx tsx tools/e2e/runner-comparison.mts initialize - - - name: Run Hermes dashboard live Vitest test - env: - NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} - run: | - set -euo pipefail - npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/hermes-e2e.test.ts - - - name: Finalize runner comparison telemetry - if: ${{ always() && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' }} - continue-on-error: true - shell: bash - run: npx tsx tools/e2e/runner-comparison.mts finalize - - - name: Upload Hermes dashboard live Vitest artifacts - if: always() - uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 - - - name: Clean up Docker auth - if: always() - shell: bash - run: bash .github/scripts/docker-auth-cleanup.sh - hermes-discord: needs: generate-matrix if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',hermes-discord,') || contains(format(',{0},', inputs.targets), ',hermes-discord,') }} @@ -5978,7 +5900,6 @@ jobs: sessions-agents-cli, hermes-e2e, hermes-gpu-startup, - hermes-dashboard, hermes-slack, hermes-discord, network-policy, @@ -6046,6 +5967,7 @@ jobs: persist-credentials: false sparse-checkout: | tools/e2e/report-e2e-results.mts + tools/e2e/selector-aliases.mts sparse-checkout-cone-mode: false - name: Post E2E target results to PR uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 diff --git a/test/e2e/README.md b/test/e2e/README.md index a32a15b0a74..465049655b0 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -77,6 +77,12 @@ discovery command locally to inspect the generated test matrix: npx tsx tools/e2e/credential-free-tests.mts ``` +The retired `hermes-dashboard` selector remains a compatibility alias for +`hermes-e2e` in both selector inputs. Reports use the canonical +`hermes-e2e` name. That lane always enables dashboard coverage while preserving +the manually selected `mock`, `internal-nvidia`, or `public-nvidia` inference +mode. + ## Larger-runner routing The larger-runner experiment is inactive while the configuration variable @@ -117,7 +123,7 @@ Successful state is discarded with the ephemeral runner. The fallback covers agent-turn latency, Hermes inference switch and shields, the Hermes Bedrock and stable MCP shards, the Hermes common-egress and channel -stop/start shards, and the `hermes-e2e`, `hermes-dashboard`, `hermes-discord`, +stop/start shards, the dashboard-bearing `hermes-e2e` lane, `hermes-discord`, and Hermes security-posture tests. Rebuild lanes with workflow-managed swap, dedicated-runner lanes, `mcp-bridge-dev`, and non-Hermes shards do not use it. Candidate-authored workflow definitions and fork-owned runs cannot reach it. @@ -135,7 +141,7 @@ The eligible set is limited to the measured or repeatedly interrupted heavy lanes: - `common-egress-agent`; -- `hermes-e2e`, `hermes-dashboard`, and `hermes-discord`; +- `hermes-e2e`, including dashboard coverage, and `hermes-discord`; - both `hermes-inference-switch` modes; - `hermes-shields-config`; - the Hermes shards of `security-posture` and `channels-stop-start`; @@ -260,7 +266,7 @@ window. ### Runner comparison telemetry Trusted `main` runs without an alternate checkout SHA record runner-comparison -telemetry for 14 routed workflow lane identities / 17 +telemetry for 13 routed workflow lane identities / 16 concrete job executions. - `agent-turn-latency`, spanning its sequential OpenClaw and Hermes setup @@ -272,9 +278,8 @@ concrete job executions. - `mcp-bridge` with the `hermes` shard - `mcp-bridge` with the `deepagents` shard - `channels-stop-start` with the `hermes` shard -- `hermes-dashboard` - `hermes-discord` -- `hermes-e2e` +- `hermes-e2e`, including dashboard coverage - `hermes-inference-switch` with the `hosted` and `anthropic` modes - `hermes-shields-config` - `security-posture` with the `hermes` shard diff --git a/test/e2e/support/e2e-progress-fixture.test.ts b/test/e2e/support/e2e-progress-fixture.test.ts index 220536eea7c..a9e74277092 100644 --- a/test/e2e/support/e2e-progress-fixture.test.ts +++ b/test/e2e/support/e2e-progress-fixture.test.ts @@ -29,7 +29,7 @@ it.each([ }); it.each([ - "hermes-dashboard", + "hermes-e2e", "hermes-discord", "hermes-shields-config", null, @@ -40,7 +40,7 @@ it.each([ it.each([ ["rebuild-hermes", 15_000], ["rebuild-hermes-stale-base", 15_000], - ["hermes-dashboard", 60_000], + ["hermes-e2e", 60_000], ] as const)("wires the live %s comparison cadence into progress options (#7144)", (targetId, intervalMs) => { const samples: Array<{ kind: string; phase: string }> = []; const options = runnerComparisonProgressOptions( diff --git a/test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts b/test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts index 43d610b5438..a52440b09e3 100644 --- a/test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts @@ -398,6 +398,44 @@ it("renders comment content from job evidence without a live GitHub mutation", ( expect(report.body).toContain("All requested tests passed"); }); +it.each([ + { + env: { JOB_TARGETS: "", JOBS: "hermes-dashboard" }, + label: "test ID", + requestedLine: "**Requested test IDs:** `hermes-e2e`", + }, + { + env: { JOB_TARGETS: "hermes-dashboard", JOBS: "" }, + label: "target", + requestedLine: "**Requested targets:** `hermes-e2e`", + }, +])("reports the canonical Hermes result for a retired dashboard $label selector", ({ + env, + requestedLine, +}) => { + const report = renderE2eReport({ + needs: { + "generate-matrix": { result: "success" }, + "hermes-e2e": { result: "success" }, + }, + env: { + EXPLICIT_ONLY_JOBS: "", + TEST_MATRIX: "[]", + JOB_PR_NUMBER: "42", + ...env, + }, + apiJobs: [{ conclusion: "success", name: "hermes-e2e", status: "completed" }], + apiJobsLoaded: true, + context: REPORT_CONTEXT, + }); + + expect(report.fatal).toBeUndefined(); + expect(report.body).toContain(requestedLine); + expect(report.body).toContain("| hermes-e2e | ✅ success | — |"); + expect(report.body).not.toContain("| hermes-dashboard |"); + expect(report.body).not.toContain("not reported"); +}); + it("fails closed on an invalid test matrix without rendering a comment", () => { const report = renderE2eReport({ needs: { diff --git a/test/e2e/support/e2e-workflow.test.ts b/test/e2e/support/e2e-workflow.test.ts index 8033354fdce..e18961d8306 100644 --- a/test/e2e/support/e2e-workflow.test.ts +++ b/test/e2e/support/e2e-workflow.test.ts @@ -645,6 +645,14 @@ describe("e2e workflow boundary", () => { selectedFreeStandingJobs: ["network-policy"], registryTargets: ["ubuntu-repo-cloud-openclaw"], }); + for (const selectors of [{ jobs: "hermes-dashboard" }, { targets: "hermes-dashboard" }]) { + expect(evaluateE2eWorkflowDispatchSelectors(selectors)).toMatchObject({ + valid: true, + liveTargetsRun: false, + selectedFreeStandingJobs: ["hermes-e2e"], + registryTargets: [], + }); + } }, ); diff --git a/test/e2e/support/hermes-dashboard-workflow-boundary.test.ts b/test/e2e/support/hermes-dashboard-workflow-boundary.test.ts index 778660aea3f..45db20053ce 100644 --- a/test/e2e/support/hermes-dashboard-workflow-boundary.test.ts +++ b/test/e2e/support/hermes-dashboard-workflow-boundary.test.ts @@ -7,73 +7,49 @@ import { validateHermesDashboardWorkflow, validateHermesDashboardWorkflowBoundary, } from "../../../tools/e2e/hermes-dashboard-workflow-boundary.mts"; -import { - HERMES_DASHBOARD_JOB_TIMEOUT_MAX_MINUTES, - HERMES_DASHBOARD_JOB_TIMEOUT_MINUTES, -} from "../../../tools/e2e/hermes-timeout-contract.mts"; describe("Hermes dashboard workflow boundary", () => { - it("accepts the checked-in workflow and rejects dashboard mode, execution, and reporting drift", () => { + it("keeps dashboard coverage in the canonical Hermes lane", () => { expect(validateHermesDashboardWorkflowBoundary()).toEqual([]); - const dashboardMode = readHermesDashboardWorkflow(); - const dashboardJob = dashboardMode.jobs["hermes-dashboard"]; - dashboardJob["timeout-minutes"] = 30; - dashboardJob.env!.E2E_ARTIFACT_DIR = "/tmp/hermes-dashboard"; - dashboardJob.env!.NEMOCLAW_E2E_HERMES_DASHBOARD = "0"; - dashboardJob.env!.NVIDIA_INFERENCE_API_KEY = "${{ secrets.NVIDIA_INFERENCE_API_KEY }}"; - const checkout = dashboardJob.steps!.find((step) => + + const workflow = readHermesDashboardWorkflow(); + const canonicalJob = workflow.jobs["hermes-e2e"]; + canonicalJob.env!.NEMOCLAW_E2E_HERMES_DASHBOARD = "0"; + canonicalJob.env!.NEMOCLAW_E2E_INFERENCE_MODE = "mock"; + canonicalJob.env!.E2E_TARGET_ID = "hermes-dashboard"; + canonicalJob.steps!.find((step) => step.name === "Run Hermes live Vitest test")!.run = + "echo skipped"; + workflow.jobs["hermes-dashboard"] = structuredClone(canonicalJob); + workflow.jobs["hermes-discord"].env!.NEMOCLAW_E2E_HERMES_DASHBOARD = "1"; + workflow.jobs["report-to-pr"].needs = ["hermes-dashboard"]; + + expect(validateHermesDashboardWorkflow(workflow)).toEqual( + expect.arrayContaining([ + "hermes-dashboard must remain consolidated into hermes-e2e", + "only hermes-e2e may enable Hermes dashboard E2E coverage (found on hermes-discord)", + "hermes-e2e must enable Hermes dashboard coverage", + "hermes-e2e must preserve manual inference-mode selection", + "hermes-e2e must publish its canonical selector", + "hermes-e2e must run the live Vitest project", + "report-to-pr must wait for hermes-e2e", + "report-to-pr must not wait for retired hermes-dashboard", + ]), + ); + }); + + it("keeps the canonical checkout trust boundary", () => { + const workflow = readHermesDashboardWorkflow(); + const checkout = workflow.jobs["hermes-e2e"].steps!.find((step) => step.uses?.startsWith("actions/checkout@"), )!; checkout.uses = "actions/checkout@v6"; checkout.with!["persist-credentials"] = true; - expect(validateHermesDashboardWorkflow(dashboardMode)).toEqual( + + expect(validateHermesDashboardWorkflow(workflow)).toEqual( expect.arrayContaining([ - `hermes-dashboard timeout must be between ${HERMES_DASHBOARD_JOB_TIMEOUT_MINUTES} and ${HERMES_DASHBOARD_JOB_TIMEOUT_MAX_MINUTES} minutes`, - "hermes-dashboard must use its isolated artifact directory", - "hermes-dashboard must enable Hermes dashboard coverage", - "hermes-dashboard must not expose the inference key at job scope", - "hermes-dashboard checkout must pin a full action SHA", - "hermes-dashboard checkout must disable persisted credentials", + "hermes-e2e checkout must pin a full action SHA", + "hermes-e2e checkout must disable persisted credentials", ]), ); - - const misplacedDashboardMode = readHermesDashboardWorkflow(); - misplacedDashboardMode.jobs["hermes-e2e"].env!.NEMOCLAW_E2E_HERMES_DASHBOARD = "1"; - expect(validateHermesDashboardWorkflow(misplacedDashboardMode)).toContain( - "only hermes-dashboard may enable Hermes dashboard E2E coverage (found on hermes-e2e)", - ); - - const execution = readHermesDashboardWorkflow(); - execution.jobs["hermes-dashboard"].steps!.find( - (step) => step.name === "Run Hermes dashboard live Vitest test", - )!.run = "echo skipped"; - expect(validateHermesDashboardWorkflow(execution)).toContain( - "hermes-dashboard must run the live Vitest project", - ); - - const reporting = readHermesDashboardWorkflow(); - reporting.jobs["report-to-pr"].needs = []; - expect(validateHermesDashboardWorkflow(reporting)).toContain( - "report-to-pr must wait for hermes-dashboard", - ); - }); - - it("shares the bounded Hermes timeout headroom contract", () => { - const upperBound = readHermesDashboardWorkflow(); - upperBound.jobs["hermes-dashboard"]["timeout-minutes"] = - HERMES_DASHBOARD_JOB_TIMEOUT_MAX_MINUTES; - expect(validateHermesDashboardWorkflow(upperBound)).toEqual([]); - - for (const timeoutMinutes of [ - HERMES_DASHBOARD_JOB_TIMEOUT_MINUTES - 1, - HERMES_DASHBOARD_JOB_TIMEOUT_MINUTES + 0.5, - HERMES_DASHBOARD_JOB_TIMEOUT_MAX_MINUTES + 1, - ]) { - const invalid = readHermesDashboardWorkflow(); - invalid.jobs["hermes-dashboard"]["timeout-minutes"] = timeoutMinutes; - expect(validateHermesDashboardWorkflow(invalid)).toContain( - `hermes-dashboard timeout must be between ${HERMES_DASHBOARD_JOB_TIMEOUT_MINUTES} and ${HERMES_DASHBOARD_JOB_TIMEOUT_MAX_MINUTES} minutes`, - ); - } }); }); diff --git a/test/e2e/support/larger-runner-routing-workflow-boundary.test.ts b/test/e2e/support/larger-runner-routing-workflow-boundary.test.ts index fa625230bdf..d70427639ff 100644 --- a/test/e2e/support/larger-runner-routing-workflow-boundary.test.ts +++ b/test/e2e/support/larger-runner-routing-workflow-boundary.test.ts @@ -68,7 +68,6 @@ const standardRouting = { "channels-stop-start-hermes": "ubuntu-latest", "channels-stop-start-openclaw": "ubuntu-latest", "common-egress-agent": "ubuntu-latest", - "hermes-dashboard": "ubuntu-latest", "hermes-discord": "ubuntu-latest", "hermes-e2e": "ubuntu-latest", "hermes-inference-switch": "ubuntu-latest", @@ -139,7 +138,6 @@ describe("larger-runner workflow routing boundary", () => { ...standardRouting, "channels-stop-start-hermes": largerRunner, "common-egress-agent": largerRunner, - "hermes-dashboard": largerRunner, "hermes-discord": largerRunner, "hermes-e2e": largerRunner, "hermes-inference-switch": largerRunner, diff --git a/test/e2e/support/runner-comparison-workflow-boundary.test.ts b/test/e2e/support/runner-comparison-workflow-boundary.test.ts index a6b8dcb5682..b0d67e558eb 100644 --- a/test/e2e/support/runner-comparison-workflow-boundary.test.ts +++ b/test/e2e/support/runner-comparison-workflow-boundary.test.ts @@ -33,7 +33,6 @@ const JOBS = [ "bedrock-runtime-compatible-anthropic", "channels-stop-start", "common-egress-agent", - "hermes-dashboard", "hermes-discord", "hermes-e2e", "hermes-inference-switch", @@ -62,7 +61,7 @@ function telemetrySteps(workflow: Workflow, jobId: string): WorkflowStep[] { } describe("runner comparison E2E workflow boundary (#7140)", () => { - it("accepts 14 routed workflow lane identities / 17 concrete job executions", () => { + it("accepts 13 routed workflow lane identities / 16 concrete job executions", () => { const workflow = loadWorkflow(); expect(validateRunnerComparisonWorkflowBoundary(workflow)).toEqual([]); @@ -88,11 +87,11 @@ describe("runner comparison E2E workflow boundary (#7140)", () => { const routedLanes = JOBS.length - 1 + mcpLanes; const concreteExecutions = routedLanes + commonEgressScenarios!.length - 1 + inferenceSwitchModes!.length - 1; - expect(routedLanes).toBe(14); - expect(concreteExecutions).toBe(17); + expect(routedLanes).toBe(13); + expect(concreteExecutions).toBe(16); }); - it("locks the matrix topology that produces seventeen concrete executions", () => { + it("locks the matrix topology that produces sixteen concrete executions", () => { const workflow = loadWorkflow(); workflow.jobs["bedrock-runtime-compatible-anthropic"]!.strategy!.matrix!.agent = [ "openclaw", @@ -126,7 +125,7 @@ describe("runner comparison E2E workflow boundary (#7140)", () => { ); }); - it("rejects runner comparison consumers outside the thirteen comparison jobs", () => { + it("rejects runner comparison consumers outside the twelve comparison jobs", () => { const workflow = loadWorkflow(); workflow.jobs["shields-config"]!.steps.push( structuredClone(telemetrySteps(workflow, "common-egress-agent")[0]!), diff --git a/test/e2e/support/trusted-hermes-swap-workflow-boundary.test.ts b/test/e2e/support/trusted-hermes-swap-workflow-boundary.test.ts index d5e1afffcb8..9e2bc4bee18 100644 --- a/test/e2e/support/trusted-hermes-swap-workflow-boundary.test.ts +++ b/test/e2e/support/trusted-hermes-swap-workflow-boundary.test.ts @@ -36,7 +36,6 @@ const PROTECTED_JOBS = [ "bedrock-runtime-compatible-anthropic", "channels-stop-start", "common-egress-agent", - "hermes-dashboard", "hermes-discord", "hermes-e2e", "hermes-inference-switch", @@ -267,6 +266,12 @@ describe("trusted Hermes swap workflow boundary", () => { expect(provision.run?.trimEnd()).toBe(TRUSTED_HERMES_SWAP_SCRIPT); expect(JSON.stringify(provision.env)).not.toContain("secrets."); } + expect(trustedSwapStep(workflow, "hermes-e2e").if).toContain( + "inputs.jobs), ',hermes-dashboard,'", + ); + expect(trustedSwapStep(workflow, "hermes-e2e").if).toContain( + "inputs.targets), ',hermes-dashboard,'", + ); }); it("keeps the trusted program fail-closed, bounded, and syntactically valid (#7145)", () => { @@ -534,7 +539,7 @@ describe("trusted Hermes swap workflow boundary", () => { const hermesE2eProvision = trustedSwapStep(workflow, "hermes-e2e"); hermesE2eProvision.if = hermesE2eProvision.if!.replace( - " && (github.event_name == 'schedule' || inputs.checkout_sha == '' || (github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && (contains(format(',{0},', inputs.jobs), ',hermes-e2e,') || contains(format(',{0},', inputs.targets), ',hermes-e2e,')))", + " && (github.event_name == 'schedule' || inputs.checkout_sha == '' || (github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && (contains(format(',{0},', inputs.jobs), ',hermes-e2e,') || contains(format(',{0},', inputs.targets), ',hermes-e2e,') || contains(format(',{0},', inputs.jobs), ',hermes-dashboard,') || contains(format(',{0},', inputs.targets), ',hermes-dashboard,')))", "", ); diff --git a/test/e2e/support/workflow-plan.test.ts b/test/e2e/support/workflow-plan.test.ts index a5b0663f12a..aa090b32ef6 100644 --- a/test/e2e/support/workflow-plan.test.ts +++ b/test/e2e/support/workflow-plan.test.ts @@ -50,6 +50,35 @@ describe("E2E workflow plan", () => { expect(plan.hermesSelected).toBe(true); }); + it.each([ + "jobs", + "targets", + ] as const)("maps the retired Hermes dashboard %s selector to the canonical lane", (kind) => { + const legacyPlan = buildE2eWorkflowPlan({ [kind]: "hermes-dashboard" }); + const canonicalPlan = buildE2eWorkflowPlan({ [kind]: "hermes-e2e" }); + const directory = mkdtempSync(path.join(tmpdir(), "nemoclaw-workflow-plan-alias-")); + const output = path.join(directory, "github-output"); + const summary = path.join(directory, "summary.md"); + + try { + writeE2eWorkflowPlanCiOutput( + { [kind]: "hermes-dashboard" }, + { + GITHUB_OUTPUT: output, + GITHUB_STEP_SUMMARY: summary, + INFERENCE_MODE: "mock", + }, + ); + + expect(legacyPlan).toEqual(canonicalPlan); + expect(legacyPlan.hermesSelected).toBe(true); + expect(readFileSync(output, "utf8")).toContain("hermes_selected=true\n"); + expect(readFreeStandingJobsInventory().allowedJobs).not.toContain("hermes-dashboard"); + } finally { + rmSync(directory, { force: true, recursive: true }); + } + }); + it("routes a registry target into the live matrix", () => { const registryId = firstId(buildLiveTargetMatrix(), "supported registry target"); const plan = buildE2eWorkflowPlan({ targets: registryId }); @@ -68,10 +97,11 @@ describe("E2E workflow plan", () => { expect(plan.testMatrix.map((row) => row.id)).toEqual([testId]); }); - it("rejects an unknown job", () => { - expect(() => buildE2eWorkflowPlan({ jobs: "definitely-unknown-e2e-job" })).toThrow( - "Unknown E2E test ID: definitely-unknown-e2e-job", - ); + it.each([ + "definitely-unknown-e2e-job", + "constructor", + ])("rejects unknown job %s without consulting inherited alias properties", (job) => { + expect(() => buildE2eWorkflowPlan({ jobs: job })).toThrow(`Unknown E2E test ID: ${job}`); }); it("maps the trusted-main bootstrap job during a PR controller checkout", () => { diff --git a/tools/e2e/hermes-dashboard-workflow-boundary.mts b/tools/e2e/hermes-dashboard-workflow-boundary.mts index e7df7325604..2c9205226a2 100644 --- a/tools/e2e/hermes-dashboard-workflow-boundary.mts +++ b/tools/e2e/hermes-dashboard-workflow-boundary.mts @@ -5,23 +5,13 @@ import { readFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import YAML from "yaml"; -import { - HERMES_DASHBOARD_JOB_TIMEOUT_MAX_MINUTES, - HERMES_DASHBOARD_JOB_TIMEOUT_MINUTES, -} from "./hermes-timeout-contract.mts"; const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); const DEFAULT_WORKFLOW_PATH = join(REPO_ROOT, ".github", "workflows", "e2e.yaml"); -const JOB_NAME = "hermes-dashboard"; +const CANONICAL_JOB = "hermes-e2e"; +const LEGACY_JOB = "hermes-dashboard"; const FULL_SHA_ACTION = /^[^\s@]+@[0-9a-f]{40}$/u; -// Dashboard recovery has a distinct trust boundary: it alone enables the -// dashboard mode, receives the inference secret at step scope, and must report -// its result before the PR scorecard is published. Keep those invariants in a -// focused validator, matching the repository's other security-posture and -// sandbox-operations workflow-boundary helpers, instead of duplicating them in -// an untyped YAML-shape assertion. - type WorkflowStep = { env?: Record; name?: string; @@ -53,100 +43,63 @@ function requireEqual(errors: string[], actual: unknown, expected: unknown, mess if (actual !== expected) errors.push(message); } -function findStep(job: WorkflowJob, name: string): WorkflowStep { - return job.steps?.find((step) => step.name === name) ?? {}; -} - export function validateHermesDashboardWorkflow(workflow: HermesDashboardWorkflow): string[] { const errors: string[] = []; - const job = workflow.jobs[JOB_NAME] ?? {}; + const job = workflow.jobs[CANONICAL_JOB] ?? {}; const env = job.env ?? {}; + if (workflow.jobs[LEGACY_JOB] !== undefined) { + errors.push(`${LEGACY_JOB} must remain consolidated into ${CANONICAL_JOB}`); + } for (const [jobName, candidate] of Object.entries(workflow.jobs)) { - if (jobName !== JOB_NAME && candidate.env?.NEMOCLAW_E2E_HERMES_DASHBOARD !== undefined) { + if (jobName !== CANONICAL_JOB && candidate.env?.NEMOCLAW_E2E_HERMES_DASHBOARD !== undefined) { errors.push( - `only ${JOB_NAME} may enable Hermes dashboard E2E coverage (found on ${jobName})`, + `only ${CANONICAL_JOB} may enable Hermes dashboard E2E coverage (found on ${jobName})`, ); } } - requireEqual(errors, job.needs, "generate-matrix", `${JOB_NAME} must depend on generate-matrix`); - const timeoutMinutes = job["timeout-minutes"]; - if ( - !Number.isInteger(timeoutMinutes) || - (timeoutMinutes as number) < HERMES_DASHBOARD_JOB_TIMEOUT_MINUTES || - (timeoutMinutes as number) > HERMES_DASHBOARD_JOB_TIMEOUT_MAX_MINUTES - ) { - errors.push( - `${JOB_NAME} timeout must be between ${HERMES_DASHBOARD_JOB_TIMEOUT_MINUTES} and ${HERMES_DASHBOARD_JOB_TIMEOUT_MAX_MINUTES} minutes`, - ); - } - requireEqual(errors, env.E2E_JOB, "1", `${JOB_NAME} must be free-standing`); - requireEqual( - errors, - env.E2E_TARGET_ID, - "hermes-dashboard", - `${JOB_NAME} must publish the hermes-dashboard selector`, - ); requireEqual( errors, - env.E2E_ARTIFACT_DIR, - "${{ github.workspace }}/e2e-artifacts/live/hermes-dashboard", - `${JOB_NAME} must use its isolated artifact directory`, + env.NEMOCLAW_E2E_HERMES_DASHBOARD, + "1", + `${CANONICAL_JOB} must enable Hermes dashboard coverage`, ); requireEqual( errors, - env.NEMOCLAW_E2E_HERMES_DASHBOARD, - "1", - `${JOB_NAME} must enable Hermes dashboard coverage`, + env.NEMOCLAW_E2E_INFERENCE_MODE, + "${{ inputs.inference_mode || 'mock' }}", + `${CANONICAL_JOB} must preserve manual inference-mode selection`, ); - requireEqual(errors, env.NEMOCLAW_AGENT, "hermes", `${JOB_NAME} must run Hermes`); requireEqual( errors, - env.NEMOCLAW_SANDBOX_NAME, - "e2e-hermes-dashboard", - `${JOB_NAME} must use an isolated sandbox`, + env.E2E_TARGET_ID, + CANONICAL_JOB, + `${CANONICAL_JOB} must publish its canonical selector`, ); - if (Object.hasOwn(env, "NVIDIA_INFERENCE_API_KEY")) { - errors.push(`${JOB_NAME} must not expose the inference key at job scope`); - } const steps = job.steps ?? []; const checkout = steps.find((step) => step.uses?.startsWith("actions/checkout@")) ?? {}; if (!FULL_SHA_ACTION.test(checkout.uses ?? "")) { - errors.push(`${JOB_NAME} checkout must pin a full action SHA`); + errors.push(`${CANONICAL_JOB} checkout must pin a full action SHA`); } if (checkout.with?.["persist-credentials"] !== false) { - errors.push(`${JOB_NAME} checkout must disable persisted credentials`); + errors.push(`${CANONICAL_JOB} checkout must disable persisted credentials`); } - for (const step of steps.filter((candidate) => candidate.uses)) { - if (!FULL_SHA_ACTION.test(step.uses ?? "")) { - errors.push(`${JOB_NAME} action '${step.name ?? step.uses}' must pin a full SHA`); - } - } - - const run = findStep(job, "Run Hermes dashboard live Vitest test"); + const run = steps.find((step) => step.name === "Run Hermes live Vitest test") ?? {}; if (!run.run?.includes("tools/e2e/live-vitest-invocation.mts run --test-path")) { - errors.push(`${JOB_NAME} must run the live Vitest project`); + errors.push(`${CANONICAL_JOB} must run the live Vitest project`); } if (!run.run?.includes("test/e2e/live/hermes-e2e.test.ts")) { - errors.push(`${JOB_NAME} must run the Hermes live test`); - } - requireEqual( - errors, - run.env?.NVIDIA_INFERENCE_API_KEY, - "${{ secrets.NVIDIA_INFERENCE_API_KEY }}", - `${JOB_NAME} must pass the inference key through step env`, - ); - for (const step of steps.filter((candidate) => candidate !== run)) { - if (step.env?.NVIDIA_INFERENCE_API_KEY !== undefined) { - errors.push(`${JOB_NAME} exposes the inference key outside the live test step`); - } + errors.push(`${CANONICAL_JOB} must run the Hermes live test`); } const reportNeeds = workflow.jobs["report-to-pr"]?.needs; - if (!Array.isArray(reportNeeds) || !reportNeeds.includes(JOB_NAME)) { - errors.push(`report-to-pr must wait for ${JOB_NAME}`); + if (!Array.isArray(reportNeeds) || !reportNeeds.includes(CANONICAL_JOB)) { + errors.push(`report-to-pr must wait for ${CANONICAL_JOB}`); + } + if (Array.isArray(reportNeeds) && reportNeeds.includes(LEGACY_JOB)) { + errors.push(`report-to-pr must not wait for retired ${LEGACY_JOB}`); } return errors; diff --git a/tools/e2e/hermes-timeout-contract.mts b/tools/e2e/hermes-timeout-contract.mts index b9cd68c7e2e..55cee896814 100644 --- a/tools/e2e/hermes-timeout-contract.mts +++ b/tools/e2e/hermes-timeout-contract.mts @@ -7,10 +7,6 @@ export const HERMES_TIMEOUT_HEADROOM_MAX_MINUTES = 30; export const HERMES_E2E_TEST_TIMEOUT_MINUTES = 70; export const HERMES_DISCORD_TEST_TIMEOUT_MINUTES = 75; export const HERMES_SHIELDS_CONFIG_TEST_TIMEOUT_MINUTES = 45; -export const HERMES_DASHBOARD_JOB_TIMEOUT_MINUTES = - HERMES_E2E_TEST_TIMEOUT_MINUTES + HERMES_TIMEOUT_HEADROOM_MINUTES; -export const HERMES_DASHBOARD_JOB_TIMEOUT_MAX_MINUTES = - HERMES_E2E_TEST_TIMEOUT_MINUTES + HERMES_TIMEOUT_HEADROOM_MAX_MINUTES; export const HERMES_E2E_TEST_TIMEOUT_MS = HERMES_E2E_TEST_TIMEOUT_MINUTES * 60_000; export const HERMES_DISCORD_TEST_TIMEOUT_MS = HERMES_DISCORD_TEST_TIMEOUT_MINUTES * 60_000; @@ -24,12 +20,6 @@ export const HERMES_TIMEOUT_CONTRACTS = [ jobName: "hermes-e2e", jobTimeoutMinutes: HERMES_E2E_TEST_TIMEOUT_MINUTES + HERMES_TIMEOUT_HEADROOM_MINUTES, }, - { - innerTest: "test/e2e/live/hermes-e2e.test.ts", - innerTimeoutMinutes: HERMES_E2E_TEST_TIMEOUT_MINUTES, - jobName: "hermes-dashboard", - jobTimeoutMinutes: HERMES_DASHBOARD_JOB_TIMEOUT_MINUTES, - }, { innerTest: "test/e2e/live/hermes-discord.test.ts", innerTimeoutMinutes: HERMES_DISCORD_TEST_TIMEOUT_MINUTES, diff --git a/tools/e2e/report-e2e-results.mts b/tools/e2e/report-e2e-results.mts index 34919009fa7..ba1ab62f001 100644 --- a/tools/e2e/report-e2e-results.mts +++ b/tools/e2e/report-e2e-results.mts @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { normalizeE2eSelectorCsv } from "./selector-aliases.mts"; + export type ReportApiJob = { completed_at?: string | null; conclusion?: string | null; @@ -173,8 +175,12 @@ export function renderE2eReport(input: { const rawRequestedTargets = env.JOB_TARGETS || ""; const rawRequestedTestIds = env.JOBS || ""; const selectorValidationPassed = needs["generate-matrix"]?.result === "success"; - const requestedTargets = selectorValidationPassed ? rawRequestedTargets : ""; - const requestedTestIdsCsv = selectorValidationPassed ? rawRequestedTestIds : ""; + const requestedTargets = selectorValidationPassed + ? normalizeE2eSelectorCsv(rawRequestedTargets) + : ""; + const requestedTestIdsCsv = selectorValidationPassed + ? normalizeE2eSelectorCsv(rawRequestedTestIds) + : ""; const explicitOnlyReasons: Record = { "openshell-gateway-auth-contract": { job: "openshell-gateway-auth-contract", diff --git a/tools/e2e/runner-comparison-workflow-boundary.mts b/tools/e2e/runner-comparison-workflow-boundary.mts index 356054a4d2f..a212cd1d34f 100644 --- a/tools/e2e/runner-comparison-workflow-boundary.mts +++ b/tools/e2e/runner-comparison-workflow-boundary.mts @@ -38,10 +38,6 @@ const COMPARISON_JOBS: ReadonlyMap normalizeE2eSelectorId(selector.trim())).filter(Boolean), + ), + ]; +} + +export function normalizeE2eSelectorCsv(selectors: string): string { + return normalizeE2eSelectorIds(selectors.split(",")).join(","); +} + +export function selectorsForCanonicalE2eId(canonicalSelector: string): string[] { + return [ + canonicalSelector, + ...Object.entries(E2E_SELECTOR_ALIASES) + .filter(([, canonical]) => canonical === canonicalSelector) + .map(([legacy]) => legacy), + ]; +} diff --git a/tools/e2e/trusted-hermes-swap-workflow-boundary.mts b/tools/e2e/trusted-hermes-swap-workflow-boundary.mts index 09ee502b6a2..76472a605c9 100644 --- a/tools/e2e/trusted-hermes-swap-workflow-boundary.mts +++ b/tools/e2e/trusted-hermes-swap-workflow-boundary.mts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { isDeepStrictEqual } from "node:util"; +import { selectorsForCanonicalE2eId } from "./selector-aliases.mts"; type WorkflowRecord = Record; type WorkflowStep = WorkflowRecord & { @@ -19,8 +20,12 @@ export const TRUSTED_HERMES_SWAP_STEP_ID = "trusted_hermes_swap"; const TRUSTED_HERMES_SWAP_IF = "github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')"; -const TRUSTED_HERMES_E2E_SELECTION = - "(contains(format(',{0},', inputs.jobs), ',hermes-e2e,') || contains(format(',{0},', inputs.targets), ',hermes-e2e,'))"; +const TRUSTED_HERMES_E2E_SELECTION = `(${selectorsForCanonicalE2eId("hermes-e2e") + .flatMap((selector) => [ + `contains(format(',{0},', inputs.jobs), ',${selector},')`, + `contains(format(',{0},', inputs.targets), ',${selector},')`, + ]) + .join(" || ")})`; const TRUSTED_HERMES_E2E_ELIGIBILITY = `(github.event_name == 'schedule' || inputs.checkout_sha == '' || (github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && ${TRUSTED_HERMES_E2E_SELECTION}))`; const TRUSTED_HERMES_SWAP_SHELL = "/bin/bash --noprofile --norc -e -o pipefail {0}"; const TRUSTED_HERMES_SWAP_ENV = { @@ -217,7 +222,6 @@ const JOB_CONDITIONS = { "bedrock-runtime-compatible-anthropic": `\${{ ${TRUSTED_HERMES_SWAP_IF} && matrix.agent == 'hermes' }}`, "channels-stop-start": `\${{ ${TRUSTED_HERMES_SWAP_IF} && matrix.agent == 'hermes' }}`, "common-egress-agent": `\${{ ${TRUSTED_HERMES_SWAP_IF} && matrix.scenario == 'hermes-open-reference' }}`, - "hermes-dashboard": `\${{ ${TRUSTED_HERMES_SWAP_IF} }}`, "hermes-discord": `\${{ ${TRUSTED_HERMES_SWAP_IF} }}`, "hermes-e2e": `\${{ ${TRUSTED_HERMES_SWAP_IF} && ${TRUSTED_HERMES_E2E_ELIGIBILITY} }}`, "hermes-inference-switch": `\${{ ${TRUSTED_HERMES_SWAP_IF} }}`, diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index 7f39ab98d33..b66d3769374 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -44,6 +44,7 @@ import { validateRunnerComparisonWorkflowBoundary } from "./runner-comparison-wo import { validateRunnerPressureWorkflow } from "./runner-pressure-workflow-boundary.mts"; import { validateSandboxOperationsWorkflow } from "./sandbox-operations-workflow-boundary.mts"; import { validateSecurityPostureWorkflow } from "./security-posture-workflow-boundary.mts"; +import { normalizeE2eSelectorIds } from "./selector-aliases.mts"; import { validateTrustedHermesSwapHelperSource, validateTrustedHermesSwapWorkflow, @@ -185,14 +186,12 @@ const RUNNER_ROUTING_SCRIPT = [ " fi", ' larger_runner="${LARGER_RUNNER_LABEL}"', "fi", - 'runner_routing="$(jq -cn --arg standard "ubuntu-latest" --arg larger "${larger_runner}" \'{"channels-stop-start-hermes":$larger,"channels-stop-start-openclaw":$standard,"common-egress-agent":$larger,"hermes-dashboard":$larger,"hermes-discord":$larger,"hermes-e2e":$larger,"hermes-inference-switch":$larger,"hermes-shields-config":$larger,"mcp-bridge-deepagents":$larger,"mcp-bridge-hermes":$larger,"mcp-bridge-openclaw":$standard,"rebuild-hermes":$larger,"rebuild-hermes-stale-base":$larger,"security-posture-hermes":$larger,"security-posture-openclaw":$standard}\')"', + 'runner_routing="$(jq -cn --arg standard "ubuntu-latest" --arg larger "${larger_runner}" \'{"channels-stop-start-hermes":$larger,"channels-stop-start-openclaw":$standard,"common-egress-agent":$larger,"hermes-discord":$larger,"hermes-e2e":$larger,"hermes-inference-switch":$larger,"hermes-shields-config":$larger,"mcp-bridge-deepagents":$larger,"mcp-bridge-hermes":$larger,"mcp-bridge-openclaw":$standard,"rebuild-hermes":$larger,"rebuild-hermes-stale-base":$larger,"security-posture-hermes":$larger,"security-posture-openclaw":$standard}\')"', 'printf \'runner_routing=%s\\n\' "${runner_routing}" >> "${GITHUB_OUTPUT}"', ].join("\n"); const ROUTED_JOB_RUNNER_EXPRESSIONS = { "common-egress-agent": "${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['common-egress-agent'] }}", - "hermes-dashboard": - "${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['hermes-dashboard'] }}", "hermes-discord": "${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['hermes-discord'] }}", "hermes-e2e": "${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['hermes-e2e'] }}", @@ -626,18 +625,20 @@ export function evaluateE2eWorkflowDispatchSelectors(input: { const jobs = input.jobs ?? ""; const targets = input.targets ?? ""; const errors: string[] = []; + const jobsMatchSelectorPattern = !jobs || SELECTOR_PATTERN.test(jobs); + const normalizedJobs = jobsMatchSelectorPattern + ? normalizeE2eSelectorIds(splitSelector(jobs)) + : []; if (targets && !SELECTOR_PATTERN.test(targets)) { errors.push("Invalid target input"); } - if (jobs && !SELECTOR_PATTERN.test(jobs)) { + if (!jobsMatchSelectorPattern) { errors.push("Invalid jobs input"); } - if (jobs && SELECTOR_PATTERN.test(jobs)) { - for (const job of splitSelector(jobs)) { - if (!freeStandingJobIds.includes(job)) { - errors.push(`Unknown free-standing E2E job: ${job}`); - } + for (const job of normalizedJobs) { + if (!freeStandingJobIds.includes(job)) { + errors.push(`Unknown free-standing E2E job: ${job}`); } } @@ -663,9 +664,9 @@ export function evaluateE2eWorkflowDispatchSelectors(input: { }; } - const selectedFreeStandingJobs = new Set(splitSelector(jobs)); + const selectedFreeStandingJobs = new Set(normalizedJobs); const registryTargets: string[] = []; - for (const target of splitSelector(targets)) { + for (const target of normalizeE2eSelectorIds(splitSelector(targets))) { const job = freeStandingTargetToJob.get(target); if (job) selectedFreeStandingJobs.add(target); else registryTargets.push(target); @@ -5004,6 +5005,11 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { ) { errors.push("report-to-pr report helper checkout must sparse-checkout the report helper"); } + if ( + !stringValue(checkoutWith["sparse-checkout"]).includes("tools/e2e/selector-aliases.mts") + ) { + errors.push("report-to-pr report helper checkout must sparse-checkout selector aliases"); + } const reportStepIndex = reportSteps.findIndex( (step) => asRecord(step).name === "Post E2E target results to PR", ); diff --git a/tools/e2e/workflow-plan.mts b/tools/e2e/workflow-plan.mts index 73d684afd79..b739ba53207 100644 --- a/tools/e2e/workflow-plan.mts +++ b/tools/e2e/workflow-plan.mts @@ -10,6 +10,7 @@ import { type CredentialFreeTestMatrixRow, discoverCredentialFreeTests, } from "./credential-free-tests.mts"; +import { normalizeE2eSelectorIds } from "./selector-aliases.mts"; import { readFreeStandingJobsInventory } from "./workflow-boundary.mts"; export type WorkflowPlanSelectors = { @@ -118,7 +119,7 @@ function selectorIds(value: string | undefined, label: "jobs" | "targets"): stri `Invalid ${label} input; use comma-separated ids containing only letters, numbers, underscores, and hyphens`, ); } - return value.split(","); + return normalizeE2eSelectorIds(value.split(",")); } function selectTestRows( @@ -208,9 +209,10 @@ export function validateE2eWorkflowPlan(plan: unknown): E2eWorkflowPlan { } function expectedHermesSelection(selectors: WorkflowPlanSelectors): boolean { - const selected = [selectors.jobs, selectors.targets] - .filter((value): value is string => !!value) - .flatMap((value) => value.split(",")); + const selected = [ + ...selectorIds(selectors.jobs, "jobs"), + ...selectorIds(selectors.targets, "targets"), + ]; return selected.length === 0 || selected.includes(HERMES_JOB_ID); }