diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 150382b4d04..dd7f41fa710 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -598,6 +598,52 @@ jobs: run: | docker logout docker.io >/dev/null 2>&1 || true + # Checkout-local CLI/docs parity and local Markdown/MDX link validation. + # This preserves nightly-e2e.yaml:docs-validation-e2e as a discrete, + # default-enabled contract until the focused docs workflows are required + # and demonstrably subsume its full-repository boundary. + docs-validation-vitest: + needs: generate-matrix + if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',docs-validation-vitest,') || contains(format(',{0},', inputs.scenarios), ',docs-validation,') }} + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + FREE_STANDING_VITEST_JOB: "1" + FREE_STANDING_SCENARIO_ID: "docs-validation" + CHECK_DOC_LINKS_REMOTE: "0" + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/docs-validation + NEMOCLAW_RUN_E2E_SCENARIOS: "1" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Run docs validation live Vitest test + run: | + set -euo pipefail + npx vitest run --project e2e-scenarios-live \ + test/e2e-scenario/live/docs-validation.test.ts \ + --silent=false --reporter=default + + - name: Upload docs validation artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-vitest-scenarios-docs-validation + path: e2e-artifacts/vitest/docs-validation/ + include-hidden-files: false + if-no-files-found: ignore + retention-days: 14 + hermes-root-entrypoint-smoke-vitest: needs: generate-matrix if: ${{ needs.generate-matrix.result == 'success' && ((inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',hermes-root-entrypoint-smoke-vitest,') || contains(format(',{0},', inputs.scenarios), ',hermes-root-entrypoint-smoke,')) }} @@ -5981,6 +6027,7 @@ jobs: onboard-negative-paths-vitest, skill-agent-vitest, openclaw-skill-cli-vitest, + docs-validation-vitest, inference-routing-vitest, cloud-inference-vitest, gpu-e2e-vitest, diff --git a/test/e2e-scenario/support-tests/docs-validation-workflow-boundary.test.ts b/test/e2e-scenario/support-tests/docs-validation-workflow-boundary.test.ts new file mode 100644 index 00000000000..c7aa18a4b90 --- /dev/null +++ b/test/e2e-scenario/support-tests/docs-validation-workflow-boundary.test.ts @@ -0,0 +1,78 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import YAML from "yaml"; +import { + readDocsValidationWorkflow, + validateDocsValidationWorkflow, + validateDocsValidationWorkflowBoundary, +} from "../../../tools/e2e-scenarios/docs-validation-workflow-boundary.mts"; +import { + evaluateE2eVitestWorkflowDispatchSelectors, + validateE2eVitestScenariosWorkflowBoundary, +} from "../../../tools/e2e-scenarios/workflow-boundary.mts"; + +describe("docs validation workflow boundary", () => { + it("is default-enabled and selectively dispatchable", () => { + expect(validateDocsValidationWorkflowBoundary()).toEqual([]); + expect(validateE2eVitestScenariosWorkflowBoundary()).toEqual([]); + + for (const selector of [{ scenarios: "docs-validation" }, { jobs: "docs-validation-vitest" }]) { + expect(evaluateE2eVitestWorkflowDispatchSelectors(selector)).toMatchObject({ + valid: true, + liveScenariosRuns: false, + selectedFreeStandingJobs: ["docs-validation-vitest"], + }); + } + expect(evaluateE2eVitestWorkflowDispatchSelectors({}).selectedFreeStandingJobs).toContain( + "docs-validation-vitest", + ); + }); + + it("makes execution, determinism, and aggregation part of the focused ratchet", () => { + const workflow = readDocsValidationWorkflow(); + const job = workflow.jobs["docs-validation-vitest"]; + job.env!.CHECK_DOC_LINKS_REMOTE = "1"; + job.steps!.find((step) => step.name === "Run docs validation live Vitest test")!.run = + "echo skipped"; + workflow.jobs["report-to-pr"].needs = (workflow.jobs["report-to-pr"].needs as string[]).filter( + (name) => name !== "docs-validation-vitest", + ); + + expect(validateDocsValidationWorkflow(workflow)).toEqual( + expect.arrayContaining([ + "docs-validation-vitest must keep link checks deterministic and local-only", + "docs-validation-vitest step Run docs validation live Vitest test must contain: test/e2e-scenario/live/docs-validation.test.ts", + "report-to-pr must wait for docs-validation-vitest", + ]), + ); + + const directory = mkdtempSync(join(tmpdir(), "nemoclaw-docs-validation-workflow-")); + const workflowPath = join(directory, "workflow.yaml"); + try { + writeFileSync(workflowPath, YAML.stringify(workflow)); + expect(validateDocsValidationWorkflowBoundary(workflowPath)).toContain( + "report-to-pr must wait for docs-validation-vitest", + ); + } finally { + rmSync(directory, { force: true, recursive: true }); + } + }); + + it("reports empty workflow input as contract errors instead of throwing", () => { + const directory = mkdtempSync(join(tmpdir(), "nemoclaw-docs-validation-empty-")); + const workflowPath = join(directory, "workflow.yaml"); + try { + writeFileSync(workflowPath, ""); + expect(validateDocsValidationWorkflowBoundary(workflowPath)).toContain( + "docs-validation-vitest must depend on generate-matrix", + ); + } finally { + rmSync(directory, { force: true, recursive: true }); + } + }); +}); diff --git a/tools/e2e-scenarios/docs-validation-workflow-boundary.mts b/tools/e2e-scenarios/docs-validation-workflow-boundary.mts new file mode 100644 index 00000000000..3bde9d450e7 --- /dev/null +++ b/tools/e2e-scenarios/docs-validation-workflow-boundary.mts @@ -0,0 +1,170 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import YAML from "yaml"; + +const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); +const DEFAULT_WORKFLOW_PATH = join(REPO_ROOT, ".github", "workflows", "e2e-vitest-scenarios.yaml"); +const JOB_NAME = "docs-validation-vitest"; + +type WorkflowStep = { + if?: string; + name?: string; + run?: string; + uses?: string; + with?: Record; +}; + +type WorkflowJob = { + env?: Record; + if?: string; + needs?: string[] | string; + steps?: WorkflowStep[]; + "runs-on"?: string; + "timeout-minutes"?: number; +}; + +export type DocsValidationWorkflow = { + jobs: Record; +}; + +export function readDocsValidationWorkflow( + workflowPath = DEFAULT_WORKFLOW_PATH, +): DocsValidationWorkflow { + const parsed: unknown = YAML.parse(readFileSync(workflowPath, "utf8")); + const jobs = + parsed && typeof parsed === "object" && !Array.isArray(parsed) + ? (parsed as { jobs?: unknown }).jobs + : undefined; + return { + jobs: + jobs && typeof jobs === "object" && !Array.isArray(jobs) + ? (jobs as Record) + : {}, + }; +} + +function findStep(job: WorkflowJob, name: string): WorkflowStep { + return job.steps?.find((step) => step.name === name) ?? {}; +} + +function requireEqual(errors: string[], actual: unknown, expected: unknown, message: string): void { + if (actual !== expected) errors.push(message); +} + +function requireRunContains(errors: string[], step: WorkflowStep, fragment: string): void { + if (!step.run?.includes(fragment)) { + errors.push(`${JOB_NAME} step ${step.name ?? ""} must contain: ${fragment}`); + } +} + +export function validateDocsValidationWorkflow(workflow: DocsValidationWorkflow): string[] { + const errors: string[] = []; + const job = workflow.jobs[JOB_NAME] ?? {}; + const env = job.env ?? {}; + + requireEqual(errors, job.needs, "generate-matrix", `${JOB_NAME} must depend on generate-matrix`); + requireEqual( + errors, + job.if, + "${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',docs-validation-vitest,') || contains(format(',{0},', inputs.scenarios), ',docs-validation,') }}", + `${JOB_NAME} must remain default-enabled and selectively dispatchable`, + ); + requireEqual(errors, job["runs-on"], "ubuntu-latest", `${JOB_NAME} must run on ubuntu-latest`); + requireEqual(errors, job["timeout-minutes"], 15, `${JOB_NAME} timeout must remain 15 minutes`); + requireEqual(errors, env.FREE_STANDING_VITEST_JOB, "1", `${JOB_NAME} must be free-standing`); + requireEqual( + errors, + env.FREE_STANDING_SCENARIO_ID, + "docs-validation", + `${JOB_NAME} must publish the docs-validation selector`, + ); + requireEqual( + errors, + env.CHECK_DOC_LINKS_REMOTE, + "0", + `${JOB_NAME} must keep link checks deterministic and local-only`, + ); + requireEqual( + errors, + env.E2E_ARTIFACT_DIR, + "${{ github.workspace }}/e2e-artifacts/vitest/docs-validation", + `${JOB_NAME} must isolate docs-validation artifacts`, + ); + requireEqual( + errors, + env.NEMOCLAW_RUN_E2E_SCENARIOS, + "1", + `${JOB_NAME} must enable live Vitest scenarios`, + ); + + const checkout = job.steps?.find((step) => step.uses?.startsWith("actions/checkout@")); + if (!checkout || !/^actions\/checkout@[0-9a-f]{40}$/u.test(checkout.uses ?? "")) { + errors.push(`${JOB_NAME} checkout must pin a full action SHA`); + } + if (checkout?.with?.["persist-credentials"] !== false) { + errors.push(`${JOB_NAME} checkout must disable persisted credentials`); + } + + const setup = findStep(job, "Set up Node"); + if (!/^actions\/setup-node@[0-9a-f]{40}$/u.test(setup.uses ?? "")) { + errors.push(`${JOB_NAME} setup-node must pin a full action SHA`); + } + requireRunContains(errors, findStep(job, "Install root dependencies"), "npm ci --ignore-scripts"); + + const run = findStep(job, "Run docs validation live Vitest test"); + requireRunContains(errors, run, "npx vitest run --project e2e-scenarios-live"); + requireRunContains(errors, run, "test/e2e-scenario/live/docs-validation.test.ts"); + + const upload = findStep(job, "Upload docs validation artifacts"); + requireEqual(errors, upload.if, "always()", `${JOB_NAME} artifact upload must always run`); + if (!/^actions\/upload-artifact@[0-9a-f]{40}$/u.test(upload.uses ?? "")) { + errors.push(`${JOB_NAME} artifact upload must pin a full action SHA`); + } + requireEqual( + errors, + upload.with?.name, + "e2e-vitest-scenarios-docs-validation", + `${JOB_NAME} artifact name must remain stable`, + ); + requireEqual( + errors, + upload.with?.path, + "e2e-artifacts/vitest/docs-validation/", + `${JOB_NAME} must upload docs-validation artifacts`, + ); + requireEqual( + errors, + upload.with?.["include-hidden-files"], + false, + `${JOB_NAME} artifact upload must exclude hidden files`, + ); + requireEqual( + errors, + upload.with?.["if-no-files-found"], + "ignore", + `${JOB_NAME} artifact upload must tolerate missing failure artifacts`, + ); + requireEqual( + errors, + upload.with?.["retention-days"], + 14, + `${JOB_NAME} artifact retention must remain 14 days`, + ); + + const reportNeeds = workflow.jobs["report-to-pr"]?.needs; + if (!Array.isArray(reportNeeds) || !reportNeeds.includes(JOB_NAME)) { + errors.push(`report-to-pr must wait for ${JOB_NAME}`); + } + + return errors; +} + +export function validateDocsValidationWorkflowBoundary( + workflowPath = DEFAULT_WORKFLOW_PATH, +): string[] { + return validateDocsValidationWorkflow(readDocsValidationWorkflow(workflowPath)); +}