From e7ac28516c608a58394f678471a08cc722a45950 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 28 Apr 2026 07:51:08 -0400 Subject: [PATCH 1/3] feat(ci): CodeRabbit E2E recommendations + selective nightly dispatch (#2564) Add path_instructions to .coderabbit.yaml that map sensitive file changes to recommended nightly E2E jobs. CodeRabbit surfaces these as review comments on PRs touching entrypoint scripts, Dockerfile, proxy rewrite, onboard logic, deploy, shields, Hermes, and network policies. Add a `jobs` input to the nightly-e2e.yaml workflow_dispatch trigger so maintainers can run a subset of nightly jobs on any branch: gh workflow run nightly-e2e.yaml --ref -f jobs=sandbox-survival-e2e,sandbox-operations-e2e Each of the 18 E2E jobs gets a conditional that checks the input, so unselected jobs are skipped. Scheduled runs and empty `jobs` input still run everything. The notify-on-failure job is unaffected (skipped jobs produce result 'skipped', not 'failure'). Add test/validate-e2e-coverage.test.ts to cross-validate: - Every job name in CodeRabbit instructions exists in nightly-e2e.yaml - Every path glob in CodeRabbit instructions matches at least one file - Every nightly job has the selective dispatch guard in its if: condition - Advisory warning for nightly jobs with no CodeRabbit coverage Closes #2564 (Phases 1-3) --- .coderabbit.yaml | 192 ++++++++++++++++++++++++ .github/workflows/nightly-e2e.yaml | 122 ++++++++++++--- test/validate-e2e-coverage.test.ts | 231 +++++++++++++++++++++++++++++ 3 files changed, 527 insertions(+), 18 deletions(-) create mode 100644 test/validate-e2e-coverage.test.ts diff --git a/.coderabbit.yaml b/.coderabbit.yaml index 058a1f271fd..ccd3594a107 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -110,5 +110,197 @@ reviews: - NVIDIA must be all caps (not Nvidia, nvidia). - NemoClaw, OpenClaw, and OpenShell must use correct casing. - No emoji in technical prose. + + # ── E2E test recommendations ────────────────────────────────── + # Maps sensitive file paths to the nightly E2E jobs that exercise them. + # CodeRabbit surfaces these as review comments on PRs that touch a + # matched path, so reviewers know which E2E jobs to run before merge. + # + # When adding a new E2E job to nightly-e2e.yaml, add a matching + # path_instructions entry below. The cross-validation test in + # test/validate-e2e-coverage.test.ts verifies consistency. + + - path: "scripts/nemoclaw-start.sh" + instructions: &e2e-entrypoint | + This file is a sandbox entrypoint script. Changes affect every + sandbox boot and are invisible to unit tests (Landlock, non-root + execution, process lifecycle). + + **E2E test recommendation:** + - `sandbox-survival-e2e` — gateway restart recovery + - `sandbox-operations-e2e` — process recovery after gateway kill + - `cloud-e2e` — full onboard + cloud inference + + To run selectively: + ``` + gh workflow run nightly-e2e.yaml --ref -f jobs=sandbox-survival-e2e,sandbox-operations-e2e,cloud-e2e + ``` + + - path: "scripts/lib/sandbox-init.sh" + instructions: *e2e-entrypoint + + - path: "Dockerfile" + instructions: &e2e-dockerfile | + This file affects the sandbox container image. Layer ordering, + permissions, and baked config changes are only testable with a + real container build. + + **E2E test recommendation:** + - `cloud-e2e` — full onboard + cloud inference + - `sandbox-survival-e2e` — gateway restart recovery + - `hermes-e2e` — Hermes agent onboard + inference + - `rebuild-openclaw-e2e` — workspace state survives rebuild + + To run selectively: + ``` + gh workflow run nightly-e2e.yaml --ref -f jobs=cloud-e2e,sandbox-survival-e2e,hermes-e2e,rebuild-openclaw-e2e + ``` + + - path: "Dockerfile.base" + instructions: *e2e-dockerfile + + - path: "nemoclaw-blueprint/scripts/http-proxy-fix.js" + instructions: | + This file is the L7 proxy rewrite script. Changes affect all + inference routing through the proxy. FORWARD-mode path needs + manual validation until a dedicated forward-proxy-e2e exists. + + **E2E test recommendation:** + - `cloud-e2e` — full inference through the proxy chain + - `inference-routing-e2e` — credential isolation + error classification + + To run selectively: + ``` + gh workflow run nightly-e2e.yaml --ref -f jobs=cloud-e2e,inference-routing-e2e + ``` + + - path: "src/lib/onboard.ts" + instructions: | + This file contains core onboarding logic. Changes here affect + the full sandbox creation and configuration flow. + + **E2E test recommendation:** + - `cloud-e2e` — full onboard + cloud inference + - `sandbox-operations-e2e` — multi-sandbox lifecycle + - `rebuild-openclaw-e2e` — workspace state survives rebuild + + To run selectively: + ``` + gh workflow run nightly-e2e.yaml --ref -f jobs=cloud-e2e,sandbox-operations-e2e,rebuild-openclaw-e2e + ``` + + - path: "src/nemoclaw.ts" + instructions: | + This file contains CLI dispatch, status, recovery, and connect + functions. Changes affect sandbox lifecycle commands. + + **E2E test recommendation:** + - `sandbox-survival-e2e` — gateway restart recovery + - `sandbox-operations-e2e` — process recovery after gateway kill + - `skip-permissions-e2e` — permissive policy activation + + To run selectively: + ``` + gh workflow run nightly-e2e.yaml --ref -f jobs=sandbox-survival-e2e,sandbox-operations-e2e,skip-permissions-e2e + ``` + + - path: "src/lib/cluster-image-patch.ts" + instructions: &e2e-overlayfs | + This file handles Docker 26+ overlayfs compatibility. Changes + are only testable under real Docker + K3s execution. + + **E2E test recommendation:** + - `overlayfs-autofix-e2e` — Docker 26+ nested-mount auto-fix + + To run selectively: + ``` + gh workflow run nightly-e2e.yaml --ref -f jobs=overlayfs-autofix-e2e + ``` + + - path: "src/lib/preflight.ts" + instructions: *e2e-overlayfs + + - path: "src/lib/deploy.ts" + instructions: | + This file contains deployment lifecycle logic (start/stop, + cloudflared tunnel, uninstall). + + **E2E test recommendation:** + - `deployment-services-e2e` — backup/restore, start/stop, uninstall + + To run selectively: + ``` + gh workflow run nightly-e2e.yaml --ref -f jobs=deployment-services-e2e + ``` + + - path: "src/lib/sandbox-state.ts" + instructions: | + This file manages sandbox state (backup, restore, rebuild, + snapshot). Changes affect data persistence across sandbox + lifecycle operations. + + **E2E test recommendation:** + - `snapshot-commands-e2e` — snapshot create/list/restore lifecycle + - `rebuild-openclaw-e2e` — workspace state survives rebuild + + To run selectively: + ``` + gh workflow run nightly-e2e.yaml --ref -f jobs=snapshot-commands-e2e,rebuild-openclaw-e2e + ``` + + - path: "src/lib/shields*.ts" + instructions: | + These files control shields down/up, config mutability, audit + trail, and auto-restore timer. + + **E2E test recommendation:** + - `shields-config-e2e` — shields lifecycle + config get/set/rotate + + To run selectively: + ``` + gh workflow run nightly-e2e.yaml --ref -f jobs=shields-config-e2e + ``` + + - path: "agents/hermes/**" + instructions: | + This directory contains the Hermes agent. Changes affect + multi-agent onboarding, health probes, and inference routing. + + **E2E test recommendation:** + - `hermes-e2e` — Hermes onboard + health probe + live inference + - `rebuild-hermes-e2e` — Hermes upgrade path + + To run selectively: + ``` + gh workflow run nightly-e2e.yaml --ref -f jobs=hermes-e2e,rebuild-hermes-e2e + ``` + + - path: "nemoclaw-blueprint/policies/**" + instructions: | + This directory contains network policy definitions and presets. + Changes affect sandbox egress rules and SSRF filtering. + + **E2E test recommendation:** + - `network-policy-e2e` — deny-by-default, whitelist, hot-reload, SSRF + - `skip-permissions-e2e` — permissive policy activation + + To run selectively: + ``` + gh workflow run nightly-e2e.yaml --ref -f jobs=network-policy-e2e,skip-permissions-e2e + ``` + + - path: ".github/workflows/nightly-e2e.yaml" + instructions: | + This is the nightly E2E workflow definition. Changes here affect + which tests run and how they are triggered. + + If a new E2E job is added, verify a corresponding + `path_instructions` entry exists in `.coderabbit.yaml` for the + source files it covers. The cross-validation test in + `test/validate-e2e-coverage.test.ts` checks this automatically. + + If a job is renamed or removed, update the corresponding + `.coderabbit.yaml` path_instructions entries and the + `notify-on-failure` needs list. chat: auto_reply: true diff --git a/.github/workflows/nightly-e2e.yaml b/.github/workflows/nightly-e2e.yaml index b61a6d3a234..2430f2be0aa 100644 --- a/.github/workflows/nightly-e2e.yaml +++ b/.github/workflows/nightly-e2e.yaml @@ -36,6 +36,19 @@ on: schedule: - cron: "0 0 * * *" workflow_dispatch: + inputs: + jobs: + description: >- + Comma-separated job names to run (empty = all). + Valid: cloud-e2e, messaging-providers-e2e, token-rotation-e2e, + sandbox-survival-e2e, hermes-e2e, skip-permissions-e2e, + sandbox-operations-e2e, inference-routing-e2e, network-policy-e2e, + deployment-services-e2e, diagnostics-e2e, snapshot-commands-e2e, + shields-config-e2e, rebuild-openclaw-e2e, upgrade-stale-sandbox-e2e, + rebuild-hermes-e2e, overlayfs-autofix-e2e, gpu-e2e + required: false + type: string + default: "" permissions: contents: read @@ -46,7 +59,11 @@ concurrency: jobs: cloud-e2e: - if: github.repository == 'NVIDIA/NemoClaw' + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(inputs.jobs, 'cloud-e2e')) runs-on: ubuntu-latest timeout-minutes: 45 steps: @@ -77,7 +94,11 @@ jobs: # proxy rewrites placeholders and the real API returns 401, proving the # chain works. See: PR #1081 messaging-providers-e2e: - if: github.repository == 'NVIDIA/NemoClaw' + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(inputs.jobs, 'messaging-providers-e2e')) runs-on: ubuntu-latest timeout-minutes: 45 steps: @@ -111,7 +132,11 @@ jobs: # rotation and reused when unchanged. # See: issue #1903 token-rotation-e2e: - if: github.repository == 'NVIDIA/NemoClaw' + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(inputs.jobs, 'token-rotation-e2e')) runs-on: ubuntu-latest timeout-minutes: 45 steps: @@ -141,7 +166,11 @@ jobs: # ── Sandbox survival (gateway restart recovery) ────────────── sandbox-survival-e2e: - if: github.repository == 'NVIDIA/NemoClaw' + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(inputs.jobs, 'sandbox-survival-e2e')) runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -170,7 +199,11 @@ jobs: # verifying the Hermes health probe, and running live inference through the # Hermes sandbox. See: PR #1618 hermes-e2e: - if: github.repository == 'NVIDIA/NemoClaw' + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(inputs.jobs, 'hermes-e2e')) runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -202,7 +235,11 @@ jobs: # Reproduces the exact scenario from the bug report: onboard, verify # policy is Active, curl from inside sandbox must not return 403. skip-permissions-e2e: - if: github.repository == 'NVIDIA/NemoClaw' + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(inputs.jobs, 'skip-permissions-e2e')) runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -232,7 +269,11 @@ jobs: # auto-recovery after docker kill, registry rebuild, process recovery, # multi-sandbox metadata, and cross-sandbox network isolation. sandbox-operations-e2e: - if: github.repository == 'NVIDIA/NemoClaw' + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(inputs.jobs, 'sandbox-operations-e2e')) runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -261,7 +302,11 @@ jobs: # TC-INF-06: invalid API key → classified credential error (PR-safe) # TC-INF-07: unreachable endpoint → classified transport error (PR-safe) inference-routing-e2e: - if: github.repository == 'NVIDIA/NemoClaw' + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(inputs.jobs, 'inference-routing-e2e')) runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -288,7 +333,11 @@ jobs: # TC-NET-01..07, TC-NET-09: deny-by-default, whitelist, live policy-add, # dry-run, hot-reload, inference exemption, permissive mode, SSRF validation. network-policy-e2e: - if: github.repository == 'NVIDIA/NemoClaw' + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(inputs.jobs, 'network-policy-e2e')) runs-on: ubuntu-latest timeout-minutes: 45 steps: @@ -317,7 +366,11 @@ jobs: # TC-DEPLOY-01: nemoclaw start/stop (cloudflared tunnel) # TC-DEPLOY-03: uninstall --keep-openshell (destructive, runs last in script) deployment-services-e2e: - if: github.repository == 'NVIDIA/NemoClaw' + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(inputs.jobs, 'deployment-services-e2e')) runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -345,7 +398,11 @@ jobs: # TC-DIAG-01: debug tarball + credential sanitization, # TC-DIAG-05: sandbox config, TC-DIAG-03: credentials list diagnostics-e2e: - if: github.repository == 'NVIDIA/NemoClaw' + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(inputs.jobs, 'diagnostics-e2e')) runs-on: ubuntu-latest timeout-minutes: 45 steps: @@ -372,7 +429,11 @@ jobs: # Validates snapshot create/list/restore lifecycle: create a snapshot, # list it, delete state, restore from snapshot, verify state recovered. snapshot-commands-e2e: - if: github.repository == 'NVIDIA/NemoClaw' + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(inputs.jobs, 'snapshot-commands-e2e')) runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -400,7 +461,11 @@ jobs: # Validates shields down/up controls config mutability, config get/set/ # rotate-token, audit trail, and auto-restore timer. shields-config-e2e: - if: github.repository == 'NVIDIA/NemoClaw' + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(inputs.jobs, 'shields-config-e2e')) runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -428,7 +493,11 @@ jobs: # Reproduces NVBug 6076156: onboard with an older OpenClaw version, # then rebuild to verify workspace state survives the upgrade. rebuild-openclaw-e2e: - if: github.repository == 'NVIDIA/NemoClaw' + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(inputs.jobs, 'rebuild-openclaw-e2e')) runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -457,7 +526,11 @@ jobs: # NemoClaw, create a sandbox, upgrade to current, verify the old # sandbox is detected as stale and rebuilt with the new image. upgrade-stale-sandbox-e2e: - if: github.repository == 'NVIDIA/NemoClaw' + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(inputs.jobs, 'upgrade-stale-sandbox-e2e')) runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -486,7 +559,11 @@ jobs: # ── Hermes rebuild upgrade E2E ────────────────────────────── # Same upgrade scenario as OpenClaw but for Hermes Agent. rebuild-hermes-e2e: - if: github.repository == 'NVIDIA/NemoClaw' + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(inputs.jobs, 'rebuild-hermes-e2e')) runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -517,7 +594,11 @@ jobs: # same PR that deletes cluster-image-patch.ts when the OpenShell # roadmap migration off k3s (NVIDIA/OpenShell#873) lands. overlayfs-autofix-e2e: - if: github.repository == 'NVIDIA/NemoClaw' + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(inputs.jobs, 'overlayfs-autofix-e2e')) runs-on: ubuntu-latest timeout-minutes: 45 steps: @@ -551,7 +632,12 @@ jobs: # Runner labels: using 'self-hosted' for now. Refine to # [self-hosted, linux, x64, gpu] once NVIDIA runner labels are confirmed. gpu-e2e: - if: github.repository == 'NVIDIA/NemoClaw' && vars.GPU_E2E_ENABLED == 'true' + if: >- + github.repository == 'NVIDIA/NemoClaw' && + vars.GPU_E2E_ENABLED == 'true' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(inputs.jobs, 'gpu-e2e')) runs-on: self-hosted timeout-minutes: 60 env: diff --git a/test/validate-e2e-coverage.test.ts b/test/validate-e2e-coverage.test.ts new file mode 100644 index 00000000000..80b8277cc82 --- /dev/null +++ b/test/validate-e2e-coverage.test.ts @@ -0,0 +1,231 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Cross-validate E2E test recommendations in .coderabbit.yaml against the + * actual nightly-e2e.yaml workflow. + * + * Catches: + * - Stale job names in CodeRabbit instructions (job renamed or removed) + * - Stale file path globs in CodeRabbit instructions (file renamed or deleted) + * - Nightly E2E jobs with no CodeRabbit path_instructions coverage (new job + * added but no mapping created) + * - Nightly E2E jobs missing the selective dispatch guard in their `if:` condition + */ + +import { readFileSync, existsSync, readdirSync, statSync } from "node:fs"; +import { join, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, it, expect } from "vitest"; +import YAML from "yaml"; + +const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), ".."); + +function repoPath(...segments: string[]): string { + return join(REPO_ROOT, ...segments); +} + +// ── Helpers ────────────────────────────────────────────────────────────────── + +/** Parse a YAML file and return the raw object. */ +function loadYaml(relPath: string): Record { + const text = readFileSync(repoPath(relPath), "utf-8"); + return YAML.parse(text) as Record; +} + +/** + * Extract all E2E job names from the nightly-e2e.yaml workflow. + * A job is any top-level key under `jobs:` except `notify-on-failure`. + */ +function getNightlyJobNames(workflow: Record): string[] { + const jobs = workflow.jobs as Record | undefined; + if (!jobs) return []; + return Object.keys(jobs).filter((name) => name !== "notify-on-failure"); +} + +/** + * Extract the `if:` condition string from a workflow job object. + */ +function getJobIf(job: unknown): string | undefined { + if (typeof job !== "object" || job === null) return undefined; + const record = job as Record; + if (typeof record.if === "string") return record.if; + return undefined; +} + +/** + * Extract all E2E job names referenced inside CodeRabbit path_instructions + * that are part of the E2E recommendation block (contain "-e2e"). + */ +function getReferencedJobNames(coderabbit: Record): Set { + const reviews = coderabbit.reviews as Record | undefined; + if (!reviews) return new Set(); + + const pathInstructions = reviews.path_instructions as + | Array<{ path: string; instructions: string }> + | undefined; + if (!pathInstructions) return new Set(); + + const jobNames = new Set(); + // Match job names inside backticks: `cloud-e2e`, `sandbox-survival-e2e` + // or in the gh workflow run -f jobs= argument. + // This avoids false positives from prose like "nightly-e2e.yaml" or + // "forward-proxy-e2e exists". + const backtickPattern = /`([a-z][-a-z]*-e2e)`/g; + const jobsArgPattern = /-f jobs=([a-z][-a-z,]*-e2e)/g; + + for (const entry of pathInstructions) { + const instructions = + typeof entry.instructions === "string" ? entry.instructions : ""; + for (const match of instructions.matchAll(backtickPattern)) { + jobNames.add(match[1]); + } + for (const match of instructions.matchAll(jobsArgPattern)) { + for (const name of match[1].split(",")) { + jobNames.add(name); + } + } + } + return jobNames; +} + +/** + * Extract E2E path_instructions entries (those containing "-e2e" in instructions). + * Returns the path globs. + */ +function getE2ePathGlobs(coderabbit: Record): string[] { + const reviews = coderabbit.reviews as Record | undefined; + if (!reviews) return []; + + const pathInstructions = reviews.path_instructions as + | Array<{ path: string; instructions: string }> + | undefined; + if (!pathInstructions) return []; + + return pathInstructions + .filter((entry) => { + const instructions = + typeof entry.instructions === "string" ? entry.instructions : ""; + return instructions.includes("-e2e"); + }) + .map((entry) => entry.path); +} + +/** + * Check if a path glob matches at least one file in the repo. + * Handles exact paths, directory patterns (agents/hermes/**), and + * simple wildcards (src/lib/shields*.ts). + * + * Not a full glob implementation — covers the patterns we actually use. + */ +function globMatchesAnyFile(glob: string): boolean { + // Exact file path + if (!glob.includes("*")) { + return existsSync(repoPath(glob)); + } + + // Directory wildcard: "agents/hermes/**" or "nemoclaw-blueprint/policies/**" + if (glob.endsWith("/**")) { + const dir = glob.slice(0, -3); + const fullDir = repoPath(dir); + return existsSync(fullDir) && statSync(fullDir).isDirectory(); + } + + // Simple wildcard in filename: "src/lib/shields*.ts" + const lastSlash = glob.lastIndexOf("/"); + const dir = glob.substring(0, lastSlash); + const pattern = glob.substring(lastSlash + 1); + + const fullDir = repoPath(dir); + if (!existsSync(fullDir) || !statSync(fullDir).isDirectory()) return false; + + // Convert simple glob to regex: "shields*.ts" -> /^shields.*\.ts$/ + const escaped = pattern + .replace(/[.+^${}()|[\]\\]/g, "\\$&") + .replace(/\*/g, ".*"); + const regex = new RegExp(`^${escaped}$`); + + const files = readdirSync(fullDir); + return files.some((f) => regex.test(f)); +} + +// ── Tests ──────────────────────────────────────────────────────────────────── + +describe("E2E coverage cross-validation", () => { + const coderabbit = loadYaml(".coderabbit.yaml"); + const workflow = loadYaml(".github/workflows/nightly-e2e.yaml"); + + const nightlyJobs = getNightlyJobNames(workflow); + const referencedJobs = getReferencedJobNames(coderabbit); + const e2ePathGlobs = getE2ePathGlobs(coderabbit); + + it("every job name in CodeRabbit instructions exists in nightly-e2e.yaml", () => { + const stale = [...referencedJobs].filter( + (name) => !nightlyJobs.includes(name), + ); + expect( + stale, + `Stale E2E job names in .coderabbit.yaml path_instructions ` + + `(not found in nightly-e2e.yaml): ${stale.join(", ")}. ` + + `Update or remove these from the CodeRabbit E2E recommendations.`, + ).toEqual([]); + }); + + it("every E2E path glob in CodeRabbit instructions matches at least one file", () => { + const stale = e2ePathGlobs.filter((glob) => !globMatchesAnyFile(glob)); + expect( + stale, + `Stale file path globs in .coderabbit.yaml E2E path_instructions ` + + `(no matching files): ${stale.join(", ")}. ` + + `The referenced files may have been renamed or deleted.`, + ).toEqual([]); + }); + + it("every nightly E2E job has at least one CodeRabbit path_instructions entry", () => { + const uncovered = nightlyJobs.filter((name) => !referencedJobs.has(name)); + // This is a warning-level check: some jobs (e.g., diagnostics-e2e, + // upgrade-stale-sandbox-e2e) may intentionally lack path-based + // recommendations. We still flag them so maintainers can decide. + if (uncovered.length > 0) { + console.warn( + `⚠️ Nightly E2E jobs with no CodeRabbit path_instructions coverage: ` + + `${uncovered.join(", ")}. ` + + `Consider adding path_instructions entries in .coderabbit.yaml ` + + `for the source files these jobs exercise.`, + ); + } + // Intentionally does not fail — this is advisory. + expect(true).toBe(true); + }); + + it("every nightly E2E job has the selective dispatch guard in its if: condition", () => { + const jobs = workflow.jobs as Record; + const missing: string[] = []; + + for (const name of nightlyJobs) { + const job = jobs[name]; + const condition = getJobIf(job); + if (!condition) { + missing.push(`${name} (no if: condition)`); + continue; + } + // Check for the selective dispatch pattern: + // inputs.jobs == '' || contains(inputs.jobs, '') + if ( + !condition.includes("inputs.jobs") || + !condition.includes(`contains(inputs.jobs, '${name}')`) + ) { + missing.push(name); + } + } + + expect( + missing, + `Nightly E2E jobs missing the selective dispatch guard in their ` + + `if: condition: ${missing.join(", ")}. Each job needs:\n` + + ` (github.event_name != 'workflow_dispatch' ||\n` + + ` inputs.jobs == '' ||\n` + + ` contains(inputs.jobs, ''))`, + ).toEqual([]); + }); +}); From faa04c6fdf7ef328f7c359a3ff94c688f54b566e Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 28 Apr 2026 08:54:44 -0400 Subject: [PATCH 2/3] fix(ci): use delimiter-wrapped contains() to prevent substring false positives CodeRabbit correctly identified that contains(inputs.jobs, 'cloud-e2e') performs substring matching, so passing 'e2e' would match every job. Replace all 18 job guards with: contains(format(',{0},', inputs.jobs), ',,') This wraps the comma-separated input in delimiters so each job name must match exactly between commas. Update the cross-validation test to check for the new pattern. --- .github/workflows/nightly-e2e.yaml | 36 +++++++++++++++--------------- test/validate-e2e-coverage.test.ts | 6 ++--- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/nightly-e2e.yaml b/.github/workflows/nightly-e2e.yaml index 2430f2be0aa..47242c4f6b2 100644 --- a/.github/workflows/nightly-e2e.yaml +++ b/.github/workflows/nightly-e2e.yaml @@ -63,7 +63,7 @@ jobs: github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || inputs.jobs == '' || - contains(inputs.jobs, 'cloud-e2e')) + contains(format(',{0},', inputs.jobs), ',cloud-e2e,')) runs-on: ubuntu-latest timeout-minutes: 45 steps: @@ -98,7 +98,7 @@ jobs: github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || inputs.jobs == '' || - contains(inputs.jobs, 'messaging-providers-e2e')) + contains(format(',{0},', inputs.jobs), ',messaging-providers-e2e,')) runs-on: ubuntu-latest timeout-minutes: 45 steps: @@ -136,7 +136,7 @@ jobs: github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || inputs.jobs == '' || - contains(inputs.jobs, 'token-rotation-e2e')) + contains(format(',{0},', inputs.jobs), ',token-rotation-e2e,')) runs-on: ubuntu-latest timeout-minutes: 45 steps: @@ -170,7 +170,7 @@ jobs: github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || inputs.jobs == '' || - contains(inputs.jobs, 'sandbox-survival-e2e')) + contains(format(',{0},', inputs.jobs), ',sandbox-survival-e2e,')) runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -203,7 +203,7 @@ jobs: github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || inputs.jobs == '' || - contains(inputs.jobs, 'hermes-e2e')) + contains(format(',{0},', inputs.jobs), ',hermes-e2e,')) runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -239,7 +239,7 @@ jobs: github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || inputs.jobs == '' || - contains(inputs.jobs, 'skip-permissions-e2e')) + contains(format(',{0},', inputs.jobs), ',skip-permissions-e2e,')) runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -273,7 +273,7 @@ jobs: github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || inputs.jobs == '' || - contains(inputs.jobs, 'sandbox-operations-e2e')) + contains(format(',{0},', inputs.jobs), ',sandbox-operations-e2e,')) runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -306,7 +306,7 @@ jobs: github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || inputs.jobs == '' || - contains(inputs.jobs, 'inference-routing-e2e')) + contains(format(',{0},', inputs.jobs), ',inference-routing-e2e,')) runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -337,7 +337,7 @@ jobs: github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || inputs.jobs == '' || - contains(inputs.jobs, 'network-policy-e2e')) + contains(format(',{0},', inputs.jobs), ',network-policy-e2e,')) runs-on: ubuntu-latest timeout-minutes: 45 steps: @@ -370,7 +370,7 @@ jobs: github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || inputs.jobs == '' || - contains(inputs.jobs, 'deployment-services-e2e')) + contains(format(',{0},', inputs.jobs), ',deployment-services-e2e,')) runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -402,7 +402,7 @@ jobs: github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || inputs.jobs == '' || - contains(inputs.jobs, 'diagnostics-e2e')) + contains(format(',{0},', inputs.jobs), ',diagnostics-e2e,')) runs-on: ubuntu-latest timeout-minutes: 45 steps: @@ -433,7 +433,7 @@ jobs: github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || inputs.jobs == '' || - contains(inputs.jobs, 'snapshot-commands-e2e')) + contains(format(',{0},', inputs.jobs), ',snapshot-commands-e2e,')) runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -465,7 +465,7 @@ jobs: github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || inputs.jobs == '' || - contains(inputs.jobs, 'shields-config-e2e')) + contains(format(',{0},', inputs.jobs), ',shields-config-e2e,')) runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -497,7 +497,7 @@ jobs: github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || inputs.jobs == '' || - contains(inputs.jobs, 'rebuild-openclaw-e2e')) + contains(format(',{0},', inputs.jobs), ',rebuild-openclaw-e2e,')) runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -530,7 +530,7 @@ jobs: github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || inputs.jobs == '' || - contains(inputs.jobs, 'upgrade-stale-sandbox-e2e')) + contains(format(',{0},', inputs.jobs), ',upgrade-stale-sandbox-e2e,')) runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -563,7 +563,7 @@ jobs: github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || inputs.jobs == '' || - contains(inputs.jobs, 'rebuild-hermes-e2e')) + contains(format(',{0},', inputs.jobs), ',rebuild-hermes-e2e,')) runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -598,7 +598,7 @@ jobs: github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || inputs.jobs == '' || - contains(inputs.jobs, 'overlayfs-autofix-e2e')) + contains(format(',{0},', inputs.jobs), ',overlayfs-autofix-e2e,')) runs-on: ubuntu-latest timeout-minutes: 45 steps: @@ -637,7 +637,7 @@ jobs: vars.GPU_E2E_ENABLED == 'true' && (github.event_name != 'workflow_dispatch' || inputs.jobs == '' || - contains(inputs.jobs, 'gpu-e2e')) + contains(format(',{0},', inputs.jobs), ',gpu-e2e,')) runs-on: self-hosted timeout-minutes: 60 env: diff --git a/test/validate-e2e-coverage.test.ts b/test/validate-e2e-coverage.test.ts index 80b8277cc82..8d83ff100d1 100644 --- a/test/validate-e2e-coverage.test.ts +++ b/test/validate-e2e-coverage.test.ts @@ -210,10 +210,10 @@ describe("E2E coverage cross-validation", () => { continue; } // Check for the selective dispatch pattern: - // inputs.jobs == '' || contains(inputs.jobs, '') + // inputs.jobs == '' || contains(format(',{0},', inputs.jobs), ',,') if ( !condition.includes("inputs.jobs") || - !condition.includes(`contains(inputs.jobs, '${name}')`) + !condition.includes(`contains(format(',{0},', inputs.jobs), ',${name},')`) ) { missing.push(name); } @@ -225,7 +225,7 @@ describe("E2E coverage cross-validation", () => { `if: condition: ${missing.join(", ")}. Each job needs:\n` + ` (github.event_name != 'workflow_dispatch' ||\n` + ` inputs.jobs == '' ||\n` + - ` contains(inputs.jobs, ''))`, + ` contains(format(',{0},', inputs.jobs), ',,'))`, ).toEqual([]); }); }); From 89313c6507d4360b4a07a48de7020afddf38a11e Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 28 Apr 2026 10:21:28 -0400 Subject: [PATCH 3/3] feat(ci): post selective E2E results as PR comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add report-to-pr job to nightly-e2e.yaml. When the workflow is triggered via workflow_dispatch on a branch with an open PR, the job posts a comment with: - Which jobs were requested - A results table (pass/fail/skipped for every job) - Direct link to the Actions run - Failed job callout with artifacts link Only runs on workflow_dispatch (not nightly schedule). Silently skips if no open PR exists for the branch. Exclude report-to-pr from the cross-validation test's E2E job list (same as notify-on-failure — infrastructure, not a test job). --- .github/workflows/nightly-e2e.yaml | 87 ++++++++++++++++++++++++++++++ test/validate-e2e-coverage.test.ts | 6 ++- 2 files changed, 91 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nightly-e2e.yaml b/.github/workflows/nightly-e2e.yaml index 47242c4f6b2..6abce20e7dc 100644 --- a/.github/workflows/nightly-e2e.yaml +++ b/.github/workflows/nightly-e2e.yaml @@ -747,3 +747,90 @@ jobs: labels: ['bug', 'CI/CD'], }); } + + report-to-pr: + runs-on: ubuntu-latest + needs: + [ + cloud-e2e, + messaging-providers-e2e, + token-rotation-e2e, + sandbox-survival-e2e, + hermes-e2e, + skip-permissions-e2e, + sandbox-operations-e2e, + inference-routing-e2e, + network-policy-e2e, + deployment-services-e2e, + diagnostics-e2e, + snapshot-commands-e2e, + shields-config-e2e, + rebuild-openclaw-e2e, + upgrade-stale-sandbox-e2e, + rebuild-hermes-e2e, + overlayfs-autofix-e2e, + gpu-e2e, + ] + if: ${{ always() && github.event_name == 'workflow_dispatch' }} + permissions: + pull-requests: write + steps: + - name: Post E2E results to PR + uses: actions/github-script@v7 + with: + script: | + const needs = ${{ toJSON(needs) }}; + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const branch = context.ref.replace('refs/heads/', ''); + const requestedJobs = '${{ inputs.jobs }}'; + + // Find open PR for this branch + const { data: prs } = await github.rest.pulls.list({ + owner: context.repo.owner, + repo: context.repo.repo, + head: `${context.repo.owner}:${branch}`, + state: 'open', + }); + + if (prs.length === 0) { + core.info(`No open PR found for branch ${branch} — skipping comment.`); + return; + } + + const pr = prs[0]; + + // Build results table + const emoji = { success: '✅', failure: '❌', cancelled: '⚠️', skipped: '⏭️' }; + const rows = Object.entries(needs) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([name, { result }]) => `| ${name} | ${emoji[result] || '❓'} ${result} |`); + + const ran = Object.entries(needs).filter(([, v]) => v.result !== 'skipped'); + const passed = ran.filter(([, v]) => v.result === 'success'); + const failed = ran.filter(([, v]) => v.result === 'failure'); + + const status = failed.length > 0 ? '❌ Some jobs failed' : '✅ All requested jobs passed'; + + const body = [ + `### Selective E2E Results — ${status}`, + '', + `**Run:** [${context.runId}](${runUrl})`, + `**Branch:** \`${branch}\``, + requestedJobs ? `**Requested jobs:** \`${requestedJobs}\`` : '**Requested jobs:** all (no filter)', + `**Summary:** ${passed.length} passed, ${failed.length} failed, ${Object.entries(needs).filter(([, v]) => v.result === 'skipped').length} skipped`, + '', + '| Job | Result |', + '|-----|--------|', + ...rows, + '', + failed.length > 0 + ? `> **Failed jobs:** ${failed.map(([k]) => k).join(', ')}. Check [run artifacts](${runUrl}) for logs.` + : '', + ].filter((line) => line !== undefined).join('\n'); + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pr.number, + body, + }); diff --git a/test/validate-e2e-coverage.test.ts b/test/validate-e2e-coverage.test.ts index 8d83ff100d1..06c9d297d04 100644 --- a/test/validate-e2e-coverage.test.ts +++ b/test/validate-e2e-coverage.test.ts @@ -35,12 +35,14 @@ function loadYaml(relPath: string): Record { /** * Extract all E2E job names from the nightly-e2e.yaml workflow. - * A job is any top-level key under `jobs:` except `notify-on-failure`. + * A job is any top-level key under `jobs:` except infrastructure jobs + * (`notify-on-failure`, `report-to-pr`). */ function getNightlyJobNames(workflow: Record): string[] { const jobs = workflow.jobs as Record | undefined; if (!jobs) return []; - return Object.keys(jobs).filter((name) => name !== "notify-on-failure"); + const infra = new Set(["notify-on-failure", "report-to-pr"]); + return Object.keys(jobs).filter((name) => !infra.has(name)); } /**