diff --git a/.coderabbit.yaml b/.coderabbit.yaml index 058a1f271fd..ccd3594a107 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -110,5 +110,197 @@ reviews: - NVIDIA must be all caps (not Nvidia, nvidia). - NemoClaw, OpenClaw, and OpenShell must use correct casing. - No emoji in technical prose. + + # ── E2E test recommendations ────────────────────────────────── + # Maps sensitive file paths to the nightly E2E jobs that exercise them. + # CodeRabbit surfaces these as review comments on PRs that touch a + # matched path, so reviewers know which E2E jobs to run before merge. + # + # When adding a new E2E job to nightly-e2e.yaml, add a matching + # path_instructions entry below. The cross-validation test in + # test/validate-e2e-coverage.test.ts verifies consistency. + + - path: "scripts/nemoclaw-start.sh" + instructions: &e2e-entrypoint | + This file is a sandbox entrypoint script. Changes affect every + sandbox boot and are invisible to unit tests (Landlock, non-root + execution, process lifecycle). + + **E2E test recommendation:** + - `sandbox-survival-e2e` — gateway restart recovery + - `sandbox-operations-e2e` — process recovery after gateway kill + - `cloud-e2e` — full onboard + cloud inference + + To run selectively: + ``` + gh workflow run nightly-e2e.yaml --ref -f jobs=sandbox-survival-e2e,sandbox-operations-e2e,cloud-e2e + ``` + + - path: "scripts/lib/sandbox-init.sh" + instructions: *e2e-entrypoint + + - path: "Dockerfile" + instructions: &e2e-dockerfile | + This file affects the sandbox container image. Layer ordering, + permissions, and baked config changes are only testable with a + real container build. + + **E2E test recommendation:** + - `cloud-e2e` — full onboard + cloud inference + - `sandbox-survival-e2e` — gateway restart recovery + - `hermes-e2e` — Hermes agent onboard + inference + - `rebuild-openclaw-e2e` — workspace state survives rebuild + + To run selectively: + ``` + gh workflow run nightly-e2e.yaml --ref -f jobs=cloud-e2e,sandbox-survival-e2e,hermes-e2e,rebuild-openclaw-e2e + ``` + + - path: "Dockerfile.base" + instructions: *e2e-dockerfile + + - path: "nemoclaw-blueprint/scripts/http-proxy-fix.js" + instructions: | + This file is the L7 proxy rewrite script. Changes affect all + inference routing through the proxy. FORWARD-mode path needs + manual validation until a dedicated forward-proxy-e2e exists. + + **E2E test recommendation:** + - `cloud-e2e` — full inference through the proxy chain + - `inference-routing-e2e` — credential isolation + error classification + + To run selectively: + ``` + gh workflow run nightly-e2e.yaml --ref -f jobs=cloud-e2e,inference-routing-e2e + ``` + + - path: "src/lib/onboard.ts" + instructions: | + This file contains core onboarding logic. Changes here affect + the full sandbox creation and configuration flow. + + **E2E test recommendation:** + - `cloud-e2e` — full onboard + cloud inference + - `sandbox-operations-e2e` — multi-sandbox lifecycle + - `rebuild-openclaw-e2e` — workspace state survives rebuild + + To run selectively: + ``` + gh workflow run nightly-e2e.yaml --ref -f jobs=cloud-e2e,sandbox-operations-e2e,rebuild-openclaw-e2e + ``` + + - path: "src/nemoclaw.ts" + instructions: | + This file contains CLI dispatch, status, recovery, and connect + functions. Changes affect sandbox lifecycle commands. + + **E2E test recommendation:** + - `sandbox-survival-e2e` — gateway restart recovery + - `sandbox-operations-e2e` — process recovery after gateway kill + - `skip-permissions-e2e` — permissive policy activation + + To run selectively: + ``` + gh workflow run nightly-e2e.yaml --ref -f jobs=sandbox-survival-e2e,sandbox-operations-e2e,skip-permissions-e2e + ``` + + - path: "src/lib/cluster-image-patch.ts" + instructions: &e2e-overlayfs | + This file handles Docker 26+ overlayfs compatibility. Changes + are only testable under real Docker + K3s execution. + + **E2E test recommendation:** + - `overlayfs-autofix-e2e` — Docker 26+ nested-mount auto-fix + + To run selectively: + ``` + gh workflow run nightly-e2e.yaml --ref -f jobs=overlayfs-autofix-e2e + ``` + + - path: "src/lib/preflight.ts" + instructions: *e2e-overlayfs + + - path: "src/lib/deploy.ts" + instructions: | + This file contains deployment lifecycle logic (start/stop, + cloudflared tunnel, uninstall). + + **E2E test recommendation:** + - `deployment-services-e2e` — backup/restore, start/stop, uninstall + + To run selectively: + ``` + gh workflow run nightly-e2e.yaml --ref -f jobs=deployment-services-e2e + ``` + + - path: "src/lib/sandbox-state.ts" + instructions: | + This file manages sandbox state (backup, restore, rebuild, + snapshot). Changes affect data persistence across sandbox + lifecycle operations. + + **E2E test recommendation:** + - `snapshot-commands-e2e` — snapshot create/list/restore lifecycle + - `rebuild-openclaw-e2e` — workspace state survives rebuild + + To run selectively: + ``` + gh workflow run nightly-e2e.yaml --ref -f jobs=snapshot-commands-e2e,rebuild-openclaw-e2e + ``` + + - path: "src/lib/shields*.ts" + instructions: | + These files control shields down/up, config mutability, audit + trail, and auto-restore timer. + + **E2E test recommendation:** + - `shields-config-e2e` — shields lifecycle + config get/set/rotate + + To run selectively: + ``` + gh workflow run nightly-e2e.yaml --ref -f jobs=shields-config-e2e + ``` + + - path: "agents/hermes/**" + instructions: | + This directory contains the Hermes agent. Changes affect + multi-agent onboarding, health probes, and inference routing. + + **E2E test recommendation:** + - `hermes-e2e` — Hermes onboard + health probe + live inference + - `rebuild-hermes-e2e` — Hermes upgrade path + + To run selectively: + ``` + gh workflow run nightly-e2e.yaml --ref -f jobs=hermes-e2e,rebuild-hermes-e2e + ``` + + - path: "nemoclaw-blueprint/policies/**" + instructions: | + This directory contains network policy definitions and presets. + Changes affect sandbox egress rules and SSRF filtering. + + **E2E test recommendation:** + - `network-policy-e2e` — deny-by-default, whitelist, hot-reload, SSRF + - `skip-permissions-e2e` — permissive policy activation + + To run selectively: + ``` + gh workflow run nightly-e2e.yaml --ref -f jobs=network-policy-e2e,skip-permissions-e2e + ``` + + - path: ".github/workflows/nightly-e2e.yaml" + instructions: | + This is the nightly E2E workflow definition. Changes here affect + which tests run and how they are triggered. + + If a new E2E job is added, verify a corresponding + `path_instructions` entry exists in `.coderabbit.yaml` for the + source files it covers. The cross-validation test in + `test/validate-e2e-coverage.test.ts` checks this automatically. + + If a job is renamed or removed, update the corresponding + `.coderabbit.yaml` path_instructions entries and the + `notify-on-failure` needs list. chat: auto_reply: true diff --git a/.github/workflows/nightly-e2e.yaml b/.github/workflows/nightly-e2e.yaml index 800c0faf016..64acdb8f13d 100644 --- a/.github/workflows/nightly-e2e.yaml +++ b/.github/workflows/nightly-e2e.yaml @@ -35,6 +35,19 @@ on: schedule: - cron: "0 0 * * *" workflow_dispatch: + inputs: + jobs: + description: >- + Comma-separated job names to run (empty = all). + Valid: cloud-e2e, messaging-providers-e2e, token-rotation-e2e, + sandbox-survival-e2e, hermes-e2e, skip-permissions-e2e, + sandbox-operations-e2e, inference-routing-e2e, network-policy-e2e, + deployment-services-e2e, diagnostics-e2e, snapshot-commands-e2e, + shields-config-e2e, rebuild-openclaw-e2e, upgrade-stale-sandbox-e2e, + rebuild-hermes-e2e, overlayfs-autofix-e2e, gpu-e2e + required: false + type: string + default: "" permissions: contents: read @@ -45,7 +58,11 @@ concurrency: jobs: cloud-e2e: - if: github.repository == 'NVIDIA/NemoClaw' + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',cloud-e2e,')) runs-on: ubuntu-latest timeout-minutes: 45 steps: @@ -76,7 +93,11 @@ jobs: # proxy rewrites placeholders and the real API returns 401, proving the # chain works. See: PR #1081 messaging-providers-e2e: - if: github.repository == 'NVIDIA/NemoClaw' + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',messaging-providers-e2e,')) runs-on: ubuntu-latest timeout-minutes: 45 steps: @@ -110,7 +131,11 @@ jobs: # rotation and reused when unchanged. # See: issue #1903 token-rotation-e2e: - if: github.repository == 'NVIDIA/NemoClaw' + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',token-rotation-e2e,')) runs-on: ubuntu-latest timeout-minutes: 45 steps: @@ -140,7 +165,11 @@ jobs: # ── Sandbox survival (gateway restart recovery) ────────────── sandbox-survival-e2e: - if: github.repository == 'NVIDIA/NemoClaw' + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',sandbox-survival-e2e,')) runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -169,7 +198,11 @@ jobs: # verifying the Hermes health probe, and running live inference through the # Hermes sandbox. See: PR #1618 hermes-e2e: - if: github.repository == 'NVIDIA/NemoClaw' + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',hermes-e2e,')) runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -201,7 +234,11 @@ jobs: # Reproduces the exact scenario from the bug report: onboard, verify # policy is Active, curl from inside sandbox must not return 403. skip-permissions-e2e: - if: github.repository == 'NVIDIA/NemoClaw' + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',skip-permissions-e2e,')) runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -231,7 +268,11 @@ jobs: # auto-recovery after docker kill, registry rebuild, process recovery, # multi-sandbox metadata, and cross-sandbox network isolation. sandbox-operations-e2e: - if: github.repository == 'NVIDIA/NemoClaw' + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',sandbox-operations-e2e,')) runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -260,7 +301,11 @@ jobs: # TC-INF-06: invalid API key → classified credential error (PR-safe) # TC-INF-07: unreachable endpoint → classified transport error (PR-safe) inference-routing-e2e: - if: github.repository == 'NVIDIA/NemoClaw' + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',inference-routing-e2e,')) runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -287,7 +332,11 @@ jobs: # TC-NET-01..07, TC-NET-09: deny-by-default, whitelist, live policy-add, # dry-run, hot-reload, inference exemption, permissive mode, SSRF validation. network-policy-e2e: - if: github.repository == 'NVIDIA/NemoClaw' + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',network-policy-e2e,')) runs-on: ubuntu-latest timeout-minutes: 45 steps: @@ -316,7 +365,11 @@ jobs: # TC-DEPLOY-01: nemoclaw start/stop (cloudflared tunnel) # TC-DEPLOY-03: uninstall --keep-openshell (destructive, runs last in script) deployment-services-e2e: - if: github.repository == 'NVIDIA/NemoClaw' + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',deployment-services-e2e,')) runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -344,7 +397,11 @@ jobs: # TC-DIAG-01: debug tarball + credential sanitization, # TC-DIAG-05: sandbox config, TC-DIAG-03: credentials list diagnostics-e2e: - if: github.repository == 'NVIDIA/NemoClaw' + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',diagnostics-e2e,')) runs-on: ubuntu-latest timeout-minutes: 45 steps: @@ -371,7 +428,11 @@ jobs: # Validates snapshot create/list/restore lifecycle: create a snapshot, # list it, delete state, restore from snapshot, verify state recovered. snapshot-commands-e2e: - if: github.repository == 'NVIDIA/NemoClaw' + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',snapshot-commands-e2e,')) runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -399,7 +460,11 @@ jobs: # Validates shields down/up controls config mutability, config get/set/ # rotate-token, audit trail, and auto-restore timer. shields-config-e2e: - if: github.repository == 'NVIDIA/NemoClaw' + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',shields-config-e2e,')) runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -427,7 +492,11 @@ jobs: # Reproduces NVBug 6076156: onboard with an older OpenClaw version, # then rebuild to verify workspace state survives the upgrade. rebuild-openclaw-e2e: - if: github.repository == 'NVIDIA/NemoClaw' + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',rebuild-openclaw-e2e,')) runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -456,7 +525,11 @@ jobs: # NemoClaw, create a sandbox, upgrade to current, verify the old # sandbox is detected as stale and rebuilt with the new image. upgrade-stale-sandbox-e2e: - if: github.repository == 'NVIDIA/NemoClaw' + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',upgrade-stale-sandbox-e2e,')) runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -485,7 +558,11 @@ jobs: # ── Hermes rebuild upgrade E2E ────────────────────────────── # Same upgrade scenario as OpenClaw but for Hermes Agent. rebuild-hermes-e2e: - if: github.repository == 'NVIDIA/NemoClaw' + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',rebuild-hermes-e2e,')) runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -516,7 +593,11 @@ jobs: # same PR that deletes cluster-image-patch.ts when the OpenShell # roadmap migration off k3s (NVIDIA/OpenShell#873) lands. overlayfs-autofix-e2e: - if: github.repository == 'NVIDIA/NemoClaw' + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',overlayfs-autofix-e2e,')) runs-on: ubuntu-latest timeout-minutes: 45 steps: @@ -547,7 +628,12 @@ jobs: # Runs on an NVKS ephemeral GPU runner (RTX Pro 6000, 36 GB VRAM). # Each job gets a fresh VM — no state leakage between runs. gpu-e2e: - if: github.repository == 'NVIDIA/NemoClaw' + if: >- + github.repository == 'NVIDIA/NemoClaw' && + vars.GPU_E2E_ENABLED == 'true' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',gpu-e2e,')) runs-on: linux-amd64-gpu-rtxpro6000-latest-1 timeout-minutes: 30 env: @@ -595,7 +681,12 @@ jobs: # proxy running with a different token than what's persisted to disk. # Runs on its own ephemeral VM — no dependency on gpu-e2e. gpu-double-onboard-e2e: - if: github.repository == 'NVIDIA/NemoClaw' + if: >- + github.repository == 'NVIDIA/NemoClaw' && + vars.GPU_E2E_ENABLED == 'true' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',gpu-double-onboard-e2e,')) runs-on: linux-amd64-gpu-rtxpro6000-latest-1 timeout-minutes: 30 env: @@ -714,3 +805,90 @@ jobs: labels: ['bug', 'CI/CD'], }); } + + report-to-pr: + runs-on: ubuntu-latest + needs: + [ + cloud-e2e, + messaging-providers-e2e, + token-rotation-e2e, + sandbox-survival-e2e, + hermes-e2e, + skip-permissions-e2e, + sandbox-operations-e2e, + inference-routing-e2e, + network-policy-e2e, + deployment-services-e2e, + diagnostics-e2e, + snapshot-commands-e2e, + shields-config-e2e, + rebuild-openclaw-e2e, + upgrade-stale-sandbox-e2e, + rebuild-hermes-e2e, + overlayfs-autofix-e2e, + gpu-e2e, + ] + if: ${{ always() && github.event_name == 'workflow_dispatch' }} + permissions: + pull-requests: write + steps: + - name: Post E2E results to PR + uses: actions/github-script@v7 + with: + script: | + const needs = ${{ toJSON(needs) }}; + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const branch = context.ref.replace('refs/heads/', ''); + const requestedJobs = ${{ toJSON(inputs.jobs) }} || ""; + + // Find open PR for this branch + const { data: prs } = await github.rest.pulls.list({ + owner: context.repo.owner, + repo: context.repo.repo, + head: `${context.repo.owner}:${branch}`, + state: 'open', + }); + + if (prs.length === 0) { + core.info(`No open PR found for branch ${branch} — skipping comment.`); + return; + } + + const pr = prs[0]; + + // Build results table + const emoji = { success: '✅', failure: '❌', cancelled: '⚠️', skipped: '⏭️' }; + const rows = Object.entries(needs) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([name, { result }]) => `| ${name} | ${emoji[result] || '❓'} ${result} |`); + + const ran = Object.entries(needs).filter(([, v]) => v.result !== 'skipped'); + const passed = ran.filter(([, v]) => v.result === 'success'); + const failed = ran.filter(([, v]) => v.result === 'failure'); + + const status = failed.length > 0 ? '❌ Some jobs failed' : '✅ All requested jobs passed'; + + const body = [ + `### Selective E2E Results — ${status}`, + '', + `**Run:** [${context.runId}](${runUrl})`, + `**Branch:** \`${branch}\``, + requestedJobs ? `**Requested jobs:** \`${requestedJobs}\`` : '**Requested jobs:** all (no filter)', + `**Summary:** ${passed.length} passed, ${failed.length} failed, ${Object.entries(needs).filter(([, v]) => v.result === 'skipped').length} skipped`, + '', + '| Job | Result |', + '|-----|--------|', + ...rows, + '', + failed.length > 0 + ? `> **Failed jobs:** ${failed.map(([k]) => k).join(', ')}. Check [run artifacts](${runUrl}) for logs.` + : '', + ].filter((line) => line !== undefined).join('\n'); + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pr.number, + body, + }); diff --git a/test/validate-e2e-coverage.test.ts b/test/validate-e2e-coverage.test.ts new file mode 100644 index 00000000000..e48a04c1a3d --- /dev/null +++ b/test/validate-e2e-coverage.test.ts @@ -0,0 +1,239 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Cross-validate E2E test recommendations in .coderabbit.yaml against the + * actual nightly-e2e.yaml workflow. + * + * Catches: + * - Stale job names in CodeRabbit instructions (job renamed or removed) + * - Stale file path globs in CodeRabbit instructions (file renamed or deleted) + * - Nightly E2E jobs with no CodeRabbit path_instructions coverage (new job + * added but no mapping created) + * - Nightly E2E jobs missing the selective dispatch guard in their `if:` condition + */ + +import { readFileSync, existsSync, readdirSync, statSync } from "node:fs"; +import { join, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, it, expect } from "vitest"; +import YAML from "yaml"; + +const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), ".."); + +function repoPath(...segments: string[]): string { + return join(REPO_ROOT, ...segments); +} + +// ── Helpers ────────────────────────────────────────────────────────────────── + +/** Parse a YAML file and return the raw object. */ +function loadYaml(relPath: string): Record { + const text = readFileSync(repoPath(relPath), "utf-8"); + return YAML.parse(text) as Record; +} + +/** + * Extract all E2E job names from the nightly-e2e.yaml workflow. + * A job is any top-level key under `jobs:` except infrastructure jobs + * (`notify-on-failure`, `report-to-pr`). + */ +function getNightlyJobNames(workflow: Record): string[] { + const jobs = workflow.jobs as Record | undefined; + if (!jobs) return []; + const infra = new Set(["notify-on-failure", "report-to-pr"]); + return Object.keys(jobs).filter((name) => !infra.has(name)); +} + +/** + * Extract the `if:` condition string from a workflow job object. + */ +function getJobIf(job: unknown): string | undefined { + if (typeof job !== "object" || job === null) return undefined; + const record = job as Record; + if (typeof record.if === "string") return record.if; + return undefined; +} + +/** + * Extract all E2E job names referenced inside CodeRabbit path_instructions + * that are part of the E2E recommendation block (contain "-e2e"). + */ +function getReferencedJobNames(coderabbit: Record): Set { + const reviews = coderabbit.reviews as Record | undefined; + if (!reviews) return new Set(); + + const pathInstructions = reviews.path_instructions as + | Array<{ path: string; instructions: string }> + | undefined; + if (!pathInstructions) return new Set(); + + const jobNames = new Set(); + // Match job names inside backticks: `cloud-e2e`, `sandbox-survival-e2e` + // or in the gh workflow run -f jobs= argument. + // This avoids false positives from prose like "nightly-e2e.yaml" or + // "forward-proxy-e2e exists". + const backtickPattern = /`([a-z][-a-z]*-e2e)`/g; + const jobsArgPattern = /-f jobs=([a-z][-a-z,]*-e2e)/g; + + for (const entry of pathInstructions) { + const instructions = + typeof entry.instructions === "string" ? entry.instructions : ""; + for (const match of instructions.matchAll(backtickPattern)) { + jobNames.add(match[1]); + } + for (const match of instructions.matchAll(jobsArgPattern)) { + for (const name of match[1].split(",")) { + jobNames.add(name); + } + } + } + return jobNames; +} + +/** + * Extract E2E path_instructions entries (those containing "-e2e" in instructions). + * Returns the path globs. + */ +function getE2ePathGlobs(coderabbit: Record): string[] { + const reviews = coderabbit.reviews as Record | undefined; + if (!reviews) return []; + + const pathInstructions = reviews.path_instructions as + | Array<{ path: string; instructions: string }> + | undefined; + if (!pathInstructions) return []; + + return pathInstructions + .filter((entry) => { + const instructions = + typeof entry.instructions === "string" ? entry.instructions : ""; + return instructions.includes("-e2e"); + }) + .map((entry) => entry.path); +} + +/** + * Check if a path glob matches at least one file in the repo. + * Handles exact paths, directory patterns (agents/hermes/**), and + * simple wildcards (src/lib/shields*.ts). + * + * Not a full glob implementation — covers the patterns we actually use. + */ +function globMatchesAnyFile(glob: string): boolean { + // Exact file path + if (!glob.includes("*")) { + return existsSync(repoPath(glob)); + } + + // Directory wildcard: "agents/hermes/**" or "nemoclaw-blueprint/policies/**" + if (glob.endsWith("/**")) { + const dir = glob.slice(0, -3); + const fullDir = repoPath(dir); + return existsSync(fullDir) && statSync(fullDir).isDirectory(); + } + + // Simple wildcard in filename: "src/lib/shields*.ts" + const lastSlash = glob.lastIndexOf("/"); + const dir = glob.substring(0, lastSlash); + const pattern = glob.substring(lastSlash + 1); + + const fullDir = repoPath(dir); + if (!existsSync(fullDir) || !statSync(fullDir).isDirectory()) return false; + + // Convert simple glob to regex: "shields*.ts" -> /^shields.*\.ts$/ + const escaped = pattern + .replace(/[.+^${}()|[\]\\]/g, "\\$&") + .replace(/\*/g, ".*"); + const regex = new RegExp(`^${escaped}$`); + + const files = readdirSync(fullDir); + return files.some((f) => regex.test(f)); +} + +// ── Tests ──────────────────────────────────────────────────────────────────── + +describe("E2E coverage cross-validation", () => { + const coderabbit = loadYaml(".coderabbit.yaml"); + const workflow = loadYaml(".github/workflows/nightly-e2e.yaml"); + + const nightlyJobs = getNightlyJobNames(workflow); + const referencedJobs = getReferencedJobNames(coderabbit); + const e2ePathGlobs = getE2ePathGlobs(coderabbit); + + it("every job name in CodeRabbit instructions exists in nightly-e2e.yaml", () => { + const stale = [...referencedJobs].filter( + (name) => !nightlyJobs.includes(name), + ); + expect( + stale, + `Stale E2E job names in .coderabbit.yaml path_instructions ` + + `(not found in nightly-e2e.yaml): ${stale.join(", ")}. ` + + `Update or remove these from the CodeRabbit E2E recommendations.`, + ).toEqual([]); + }); + + it("every E2E path glob in CodeRabbit instructions matches at least one file", () => { + const stale = e2ePathGlobs.filter((glob) => !globMatchesAnyFile(glob)); + expect( + stale, + `Stale file path globs in .coderabbit.yaml E2E path_instructions ` + + `(no matching files): ${stale.join(", ")}. ` + + `The referenced files may have been renamed or deleted.`, + ).toEqual([]); + }); + + it("every nightly E2E job has at least one CodeRabbit path_instructions entry", () => { + const uncovered = nightlyJobs.filter((name) => !referencedJobs.has(name)); + // This is a warning-level check: some jobs (e.g., diagnostics-e2e, + // upgrade-stale-sandbox-e2e) may intentionally lack path-based + // recommendations. We still flag them so maintainers can decide. + if (uncovered.length > 0) { + console.warn( + `⚠️ Nightly E2E jobs with no CodeRabbit path_instructions coverage: ` + + `${uncovered.join(", ")}. ` + + `Consider adding path_instructions entries in .coderabbit.yaml ` + + `for the source files these jobs exercise.`, + ); + } + // Intentionally does not fail — this is advisory. + expect(true).toBe(true); + }); + + it("every nightly E2E job has the selective dispatch guard in its if: condition", () => { + const jobs = workflow.jobs as Record; + const missing: string[] = []; + + for (const name of nightlyJobs) { + const job = jobs[name]; + const condition = getJobIf(job); + if (!condition) { + missing.push(`${name} (no if: condition)`); + continue; + } + // Check for the full selective dispatch pattern: + // (github.event_name != 'workflow_dispatch' || + // inputs.jobs == '' || + // contains(format(',{0},', inputs.jobs), ',,')) + const hasDispatchBypass = condition.includes( + "github.event_name != 'workflow_dispatch'", + ); + const hasEmptySelectionBypass = condition.includes("inputs.jobs == ''"); + const hasExactJobMatch = condition.includes( + `contains(format(',{0},', inputs.jobs), ',${name},')`, + ); + if (!(hasDispatchBypass && hasEmptySelectionBypass && hasExactJobMatch)) { + missing.push(name); + } + } + + expect( + missing, + `Nightly E2E jobs missing the selective dispatch guard in their ` + + `if: condition: ${missing.join(", ")}. Each job needs:\n` + + ` (github.event_name != 'workflow_dispatch' ||\n` + + ` inputs.jobs == '' ||\n` + + ` contains(format(',{0},', inputs.jobs), ',,'))`, + ).toEqual([]); + }); +});