diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 07a3c9d1eb3..8b78fcd5bc7 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -15,7 +15,7 @@ on: default: "" type: string jobs: - description: "Optional comma-separated free-standing live E2E job ids. Empty runs default-enabled jobs only when targets is also empty; explicit-only jobs openshell-gateway-auth-contract, mcp-bridge-dev, hermes-gpu-startup, sandbox-rlimits-connect, and jetson-nvmap-gpu are skipped unless selected." + description: "Optional comma-separated E2E test IDs. Empty runs default-enabled tests only when targets is also empty; explicit-only tests openshell-gateway-auth-contract, mcp-bridge-dev, hermes-gpu-startup, sandbox-rlimits-connect, and jetson-nvmap-gpu are skipped unless selected." required: false default: "" type: string @@ -69,6 +69,7 @@ jobs: runs-on: ubuntu-latest outputs: matrix: ${{ steps.matrix.outputs.matrix }} + test_matrix: ${{ steps.matrix.outputs.test_matrix }} hermes_selected: ${{ steps.matrix.outputs.hermes_selected }} explicit_only_jobs: ${{ steps.matrix.outputs.explicit_only_jobs }} steps: @@ -121,187 +122,98 @@ jobs: TARGETS: ${{ inputs.targets }} run: | set -euo pipefail - inventory_output="$(npx tsx tools/e2e/workflow-inventory.mts --shell)" - allowed_jobs="" - explicit_only_jobs_csv="" - free_standing_targets_csv="" - free_standing_target_jobs_csv="" - seen_inventory_keys="," - while IFS= read -r line || [ -n "${line}" ]; do - line="${line#"${line%%[![:space:]]*}"}" - line="${line%"${line##*[![:space:]]}"}" - if [[ -z "${line}" || "${line}" == \#* ]]; then - continue - fi - if [[ ! "${line}" =~ ^(allowed_jobs|explicit_only_jobs_csv|free_standing_targets_csv|free_standing_target_jobs_csv)=([A-Za-z0-9_:-]+(,[A-Za-z0-9_:-]+)*)?$ ]]; then - echo "::error::free-standing workflow inventory must be data-only key=value" >&2 - exit 1 - fi - inventory_key="${BASH_REMATCH[1]}" - inventory_value="${BASH_REMATCH[2]}" - if [[ "${seen_inventory_keys}" == *",${inventory_key},"* ]]; then - echo "::error::free-standing workflow inventory must not redefine ${inventory_key}" >&2 - exit 1 - fi - seen_inventory_keys="${seen_inventory_keys}${inventory_key}," - case "${inventory_key}" in - allowed_jobs) allowed_jobs="${inventory_value}" ;; - explicit_only_jobs_csv) explicit_only_jobs_csv="${inventory_value}" ;; - free_standing_targets_csv) free_standing_targets_csv="${inventory_value}" ;; - free_standing_target_jobs_csv) free_standing_target_jobs_csv="${inventory_value}" ;; - esac - done <<< "${inventory_output}" - for required_inventory_key in allowed_jobs explicit_only_jobs_csv free_standing_targets_csv free_standing_target_jobs_csv; do - if [[ "${seen_inventory_keys}" != *",${required_inventory_key},"* ]]; then - echo "::error::free-standing workflow inventory missing ${required_inventory_key}" >&2 - exit 1 - fi - done - for required_inventory_key in allowed_jobs free_standing_targets_csv free_standing_target_jobs_csv; do - if [[ -z "${!required_inventory_key:-}" ]]; then - echo "::error::free-standing workflow inventory missing ${required_inventory_key}" >&2 - exit 1 - fi - done - seen_allowed_jobs="," - IFS=',' read -r -a allowed_job_entries <<< "${allowed_jobs}" - for job in "${allowed_job_entries[@]}"; do - if [[ ! "${job}" =~ ^[A-Za-z0-9_-]+$ ]]; then - echo "::error::free-standing workflow inventory contains invalid job id" >&2 - exit 1 - fi - if [[ "${seen_allowed_jobs}" == *",${job},"* ]]; then - echo "::error::free-standing workflow inventory repeats job id" >&2 - exit 1 - fi - seen_allowed_jobs="${seen_allowed_jobs}${job}," - done - seen_explicit_only_jobs="," - if [ -n "${explicit_only_jobs_csv}" ]; then - IFS=',' read -r -a explicit_only_job_entries <<< "${explicit_only_jobs_csv}" - for job in "${explicit_only_job_entries[@]}"; do - if [[ "${seen_allowed_jobs}" != *",${job},"* ]]; then - echo "::error::Explicit-only job is not in allowed jobs" >&2 - exit 1 - fi - if [[ "${seen_explicit_only_jobs}" == *",${job},"* ]]; then - echo "::error::free-standing workflow inventory repeats explicit-only job" >&2 - exit 1 - fi - seen_explicit_only_jobs="${seen_explicit_only_jobs}${job}," - done - fi - seen_free_standing_targets="," - IFS=',' read -r -a free_standing_target_entries <<< "${free_standing_targets_csv}" - for target in "${free_standing_target_entries[@]}"; do - if [[ ! "${target}" =~ ^[A-Za-z0-9_-]+$ ]]; then - echo "::error::free-standing workflow inventory contains invalid target id" >&2 - exit 1 - fi - if [[ "${seen_free_standing_targets}" == *",${target},"* ]]; then - echo "::error::free-standing workflow inventory repeats target id" >&2 - exit 1 - fi - seen_free_standing_targets="${seen_free_standing_targets}${target}," - done - IFS=',' read -r -a target_job_entries <<< "${free_standing_target_jobs_csv}" - seen_target_mappings="," - derived_free_standing_targets=() - for entry in "${target_job_entries[@]}"; do - if [[ ! "${entry}" =~ ^[A-Za-z0-9_-]+:[A-Za-z0-9_-]+$ ]]; then - echo "::error::Invalid free-standing target mapping" >&2 - exit 1 - fi - target="${entry%%:*}" - job="${entry#*:}" - if [[ "${seen_target_mappings}" == *",${target},"* ]]; then - echo "::error::free-standing workflow inventory repeats target mapping" >&2 - exit 1 - fi - seen_target_mappings="${seen_target_mappings}${target}," - if [[ "${seen_allowed_jobs}" != *",${job},"* ]]; then - echo "::error::Free-standing target maps to unknown job" >&2 - exit 1 - fi - derived_free_standing_targets+=("${target}") - done - derived_free_standing_targets_csv="$(IFS=,; echo "${derived_free_standing_targets[*]}")" - if [[ "${free_standing_targets_csv}" != "${derived_free_standing_targets_csv}" ]]; then - echo "::error::free_standing_targets_csv must match target mapping keys" >&2 - exit 1 - fi - args=(--emit-live-matrix) - matrix="" - hermes_selected=false - registry_targets=() - is_free_standing_target() { - [[ ",${free_standing_targets_csv}," == *",$1,"* ]] - } if [ -n "${JOBS}" ] && [ -n "${TARGETS}" ]; then echo "::error::Use either targets or jobs, not both." >&2 exit 1 fi - if [ -n "${JOBS}" ]; then - if [[ ! "${JOBS}" =~ ^[A-Za-z0-9_-]+(,[A-Za-z0-9_-]+)*$ ]]; then - echo "::error::Invalid jobs input; use comma-separated job ids" >&2 + for selector_name in JOBS TARGETS; do + selector_value="${!selector_name}" + if [ -n "${selector_value}" ] && [[ ! "${selector_value}" =~ ^[A-Za-z0-9_-]+(,[A-Za-z0-9_-]+)*$ ]]; then + echo "::error::Invalid ${selector_name,,} input; use comma-separated ids" >&2 exit 1 fi - IFS=',' read -r -a selected_jobs <<< "${JOBS}" - for job in "${selected_jobs[@]}"; do - if [[ ",${allowed_jobs}," != *",${job},"* ]]; then - echo "::error::Unknown free-standing E2E job: ${job}" >&2 - echo "::error::Allowed jobs: ${allowed_jobs}" >&2 - exit 1 - fi - case "${job}" in - hermes-e2e) - hermes_selected=true - ;; - esac - done - matrix="[]" + done + + planner_args=() + if [ -n "${JOBS}" ]; then + planner_args+=(--jobs "${JOBS}") elif [ -n "${TARGETS}" ]; then - if [[ ! "${TARGETS}" =~ ^[A-Za-z0-9_-]+(,[A-Za-z0-9_-]+)*$ ]]; then - echo "::error::Invalid target input; use comma-separated target ids containing only letters, numbers, underscores, and hyphens." >&2 - exit 1 - fi - IFS=',' read -r -a requested_targets <<< "${TARGETS}" - for target in "${requested_targets[@]}"; do - case "${target}" in - hermes-e2e) - hermes_selected=true - ;; - esac - if is_free_standing_target "${target}"; then - continue - fi - registry_targets+=("${target}") - done - if [ "${#registry_targets[@]}" -gt 0 ]; then - registry_csv="$(IFS=,; echo "${registry_targets[*]}")" - args+=(--targets "${registry_csv}") - matrix="$(npx tsx test/e2e/registry/run.ts "${args[@]}")" - else - matrix="[]" - fi - else - hermes_selected=true - matrix="$(npx tsx test/e2e/registry/run.ts "${args[@]}")" + planner_args+=(--targets "${TARGETS}") fi + plan="$(npx tsx tools/e2e/workflow-plan.mts "${planner_args[@]}")" + plan_filter=' + type == "object" and + (keys | sort) == ["explicitOnlyJobs", "hermesSelected", "matrix", "testMatrix"] and + (.matrix | type) == "array" and + all(.matrix[]; + type == "object" and + (keys | sort) == ["expectedStateId", "id", "install", "label", "onboarding", "pendingRuntimeSuites", "platform", "requiredSecrets", "runner", "runtime", "suites", "supportReasons", "supported"] and + (.id | type == "string" and test("^[a-z0-9]+(?:-[a-z0-9]+)*$")) and + (.runner | type == "string" and test("^[A-Za-z0-9_-]+$")) and + (.label | type) == "string" and + (.platform | type) == "string" and + (.install | type) == "string" and + (.runtime | type) == "string" and + (.onboarding | type) == "string" and + (.expectedStateId | type) == "string" and + (.supported | type) == "boolean" and + all(.suites[], .requiredSecrets[], .supportReasons[], .pendingRuntimeSuites[]; type == "string") + ) and + (([.matrix[].id] | length) == ([.matrix[].id] | unique | length)) and + (.testMatrix | type) == "array" and + all(.testMatrix[]; + type == "object" and + (keys | sort) == ["file", "id", "project"] and + (.id | test("^[a-z0-9]+(?:-[a-z0-9]+)*$")) and + (.file | test("^test/(?:[A-Za-z0-9._-]+/)*[A-Za-z0-9._-]+[.]test[.](?:js|ts)$")) and + ((.project == "e2e-live" and (.file | startswith("test/e2e/live/"))) or + (.project == "integration" and (.file | startswith("test/")) and + ((.file | startswith("test/e2e/")) | not))) + ) and + (([.testMatrix[].id] | length) == ([.testMatrix[].id] | unique | length)) and + (.hermesSelected | type) == "boolean" and + (.explicitOnlyJobs | type) == "array" and + all(.explicitOnlyJobs[]; type == "string" and test("^[A-Za-z0-9_-]+$")) and + ((.explicitOnlyJobs | length) == (.explicitOnlyJobs | unique | length)) + ' + if ! jq -e "${plan_filter}" <<< "${plan}" >/dev/null; then + echo "::error::E2E planner returned an invalid output schema" >&2 + exit 1 + fi + + expected_hermes_selected=false + selected_csv="${JOBS:-${TARGETS}}" + if [ -z "${selected_csv}" ] || [[ ",${selected_csv}," == *",hermes-e2e,"* ]]; then + expected_hermes_selected=true + fi + hermes_selected="$(jq -r '.hermesSelected' <<< "${plan}")" + if [ "${hermes_selected}" != "${expected_hermes_selected}" ]; then + echo "::error::E2E planner changed the trusted Hermes selection" >&2 + exit 1 + fi + + matrix="$(jq -c '.matrix' <<< "${plan}")" + test_matrix="$(jq -c '.testMatrix' <<< "${plan}")" + explicit_only_jobs_csv="$(jq -r '.explicitOnlyJobs | join(",")' <<< "${plan}")" echo "matrix=${matrix}" >> "$GITHUB_OUTPUT" + echo "test_matrix=${test_matrix}" >> "$GITHUB_OUTPUT" echo "hermes_selected=${hermes_selected}" >> "$GITHUB_OUTPUT" echo "explicit_only_jobs=${explicit_only_jobs_csv}" >> "$GITHUB_OUTPUT" - MATRIX_JSON="${matrix}" python3 - <<'PY' >> "$GITHUB_STEP_SUMMARY" + + MATRIX_JSON="${matrix}" TEST_MATRIX_JSON="${test_matrix}" python3 - <<'PY' >> "$GITHUB_STEP_SUMMARY" import json import os - rows = json.loads(os.environ["MATRIX_JSON"]) - print("## E2E Target Matrix") + typed_rows = json.loads(os.environ["MATRIX_JSON"]) + test_rows = json.loads(os.environ["TEST_MATRIX_JSON"]) + print("## E2E Execution Plan") print() - print("| Target | Runner | Label |") + print("| Test | Execution | Runner |") print("| --- | --- | --- |") - for row in rows: - print(f"| `{row['id']}` | `{row['runner']}` | {row['label']} |") + for row in typed_rows: + print(f"| `{row['id']}` | typed registry | `{row['runner']}` |") + for row in test_rows: + print(f"| `{row['id']}` | shared E2E job | `ubuntu-latest` |") PY live: @@ -532,19 +444,26 @@ jobs: shell: bash run: bash .github/scripts/docker-auth-cleanup.sh - # Free-standing live tests under test/e2e/live/ that don't fit the - # registry-driven steady-state probe model. Each gets a discrete job here - # because the matrix above only runs registry-targets.test.ts. Modeled on - # #5049's free-standing pattern. - openshell-version-pin: + # Credential-free tests opt in with a tag beside the test. Discovery supplies + # only a validated test ID, file, and Vitest project; this E2E workflow owns + # the shared job's runner, setup, timeout, permissions, and artifact policy. + shared-e2e: + name: Shared E2E (${{ matrix.id }}) needs: generate-matrix - if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',openshell-version-pin,') || contains(format(',{0},', inputs.targets), ',openshell-version-pin,') }} + if: ${{ needs.generate-matrix.outputs.test_matrix != '[]' }} runs-on: ubuntu-latest timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.generate-matrix.outputs.test_matrix) }} env: - E2E_JOB: "1" - E2E_TARGET_ID: "openshell-version-pin" - E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openshell-version-pin + CHECK_DOC_LINKS_REMOTE: "0" + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/${{ matrix.id }} + E2E_TARGET_ID: ${{ matrix.id }} + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_RUN_LIVE_E2E: "1" steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 @@ -554,19 +473,17 @@ jobs: - name: Prepare E2E workspace uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28 - with: - build-cli: "false" - - name: Run OpenShell version-pin live test - # Hermetic - # installer-script behavioral test — no real network, no real install. + - name: Run tagged credential-free test + env: + TEST_FILE: ${{ matrix.file }} + TEST_PROJECT: ${{ matrix.project }} run: | set -euo pipefail - npx vitest run --project e2e-live \ - test/e2e/live/openshell-version-pin.test.ts \ + npx vitest run --project "${TEST_PROJECT}" "${TEST_FILE}" \ --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - - name: Upload OpenShell version-pin artifacts + - name: Upload test artifacts if: always() uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 @@ -843,39 +760,6 @@ jobs: shell: bash run: bash .github/scripts/docker-auth-cleanup.sh - onboard-negative-paths: - needs: generate-matrix - if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',onboard-negative-paths,') || contains(format(',{0},', inputs.targets), ',onboard-negative-paths,') }} - runs-on: ubuntu-latest - timeout-minutes: 15 - env: - E2E_JOB: "1" - E2E_TARGET_ID: "onboard-negative-paths" - E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/onboard-negative-paths - NEMOCLAW_RUN_LIVE_E2E: "1" - steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - ref: ${{ inputs.checkout_sha || github.sha }} - persist-credentials: false - - - name: Prepare E2E workspace - uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28 - - - name: Run onboard negative-paths live test - # Direct E2E coverage - # invalid-key contract. This intentionally bypasses typed registry and - # state-validation machinery because the behavior is CLI exit/output. - run: | - set -euo pipefail - npx vitest run --project e2e-live \ - test/e2e/live/onboard-negative-paths.test.ts \ - --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - - - name: Upload onboard negative-paths artifacts - if: always() - uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 - # Live skill injection and real OpenClaw agent-turn contract. skill-agent: needs: generate-matrix @@ -994,42 +878,6 @@ jobs: shell: bash run: bash .github/scripts/docker-auth-cleanup.sh - # Checkout-local CLI/docs parity and deterministic local Markdown/MDX link - # validation. Keep this discrete until the focused docs workflows are - # required and demonstrably subsume the full-repository boundary. - docs-validation: - needs: generate-matrix - if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',docs-validation,') || contains(format(',{0},', inputs.targets), ',docs-validation,') }} - runs-on: ubuntu-latest - timeout-minutes: 15 - env: - E2E_JOB: "1" - E2E_TARGET_ID: "docs-validation" - CHECK_DOC_LINKS_REMOTE: "0" - E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/docs-validation - NEMOCLAW_RUN_LIVE_E2E: "1" - steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - ref: ${{ inputs.checkout_sha || github.sha }} - persist-credentials: false - - - name: Prepare E2E workspace - uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28 - with: - build-cli: "false" - - - name: Run docs validation live Vitest test - run: | - set -euo pipefail - npx vitest run --project e2e-live \ - test/e2e/live/docs-validation.test.ts \ - --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - - - name: Upload docs validation artifacts - if: always() - uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 - inference-routing: needs: generate-matrix if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',inference-routing,') || contains(format(',{0},', inputs.targets), ',inference-routing,') }} @@ -3918,42 +3766,6 @@ jobs: shell: bash run: bash .github/scripts/docker-auth-cleanup.sh - gateway-drift-preflight: - needs: generate-matrix - if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',gateway-drift-preflight,') || contains(format(',{0},', inputs.targets), ',gateway-drift-preflight,') }} - runs-on: ubuntu-latest - timeout-minutes: 15 - env: - E2E_JOB: "1" - E2E_TARGET_ID: "gateway-drift-preflight" - E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/gateway-drift-preflight - NEMOCLAW_RUN_LIVE_E2E: "1" - NEMOCLAW_NON_INTERACTIVE: "1" - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" - steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - ref: ${{ inputs.checkout_sha || github.sha }} - persist-credentials: false - - - name: Prepare E2E workspace - uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28 - - - name: Run gateway drift preflight Vitest test - # This keeps - # the real repo CLI, PATH-resolved openshell/docker shims, host-process - # marker/PID probes, and process exit behavior while avoiding live - # Docker/OpenShell mutation. - run: | - set -euo pipefail - npx vitest run --project integration \ - test/gateway-drift-preflight.test.ts \ - --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - - - name: Upload gateway drift preflight artifacts - if: always() - uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 - # Scheduled coverage for #6108 / #3513 / #3127. This exercises the complete # managed v0.0.71 runtime with a custom plugin, then proves restart/rebuild # persistence and target-side runtime-dependency replacement across devices. @@ -4968,14 +4780,12 @@ jobs: [ generate-matrix, live, - openshell-version-pin, + shared-e2e, openshell-gateway-auth-contract, mcp-bridge, mcp-bridge-dev, - onboard-negative-paths, skill-agent, openclaw-skill-cli, - docs-validation, inference-routing, cloud-inference, gpu-e2e, @@ -5025,7 +4835,6 @@ jobs: sandbox-survival, diagnostics, snapshot-commands, - gateway-drift-preflight, openclaw-plugin-runtime-exdev, openclaw-tui-chat-correlation, gateway-guard-recovery, @@ -5045,6 +4854,7 @@ jobs: ] if: ${{ always() && github.event_name == 'workflow_dispatch' && inputs.checkout_sha == '' }} permissions: + actions: read # The issue-comment endpoint accepts pull request write permission for PR comments. # Keep issues: write absent so this job cannot restore general issue routing. pull-requests: write @@ -5053,6 +4863,7 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: EXPLICIT_ONLY_JOBS: ${{ needs.generate-matrix.outputs.explicit_only_jobs }} + TEST_MATRIX: ${{ needs.generate-matrix.outputs.test_matrix }} JOB_PR_NUMBER: ${{ inputs.pr_number }} JOB_TARGETS: ${{ inputs.targets }} JOBS: ${{ inputs.jobs }} @@ -5063,10 +4874,10 @@ jobs: const workflowBranch = context.ref.replace('refs/heads/', ''); const prNumberInput = process.env.JOB_PR_NUMBER || ''; const rawRequestedTargets = process.env.JOB_TARGETS || ''; - const rawRequestedJobs = process.env.JOBS || ''; + const rawRequestedTestIds = process.env.JOBS || ''; const selectorValidationPassed = needs['generate-matrix']?.result === 'success'; const requestedTargets = selectorValidationPassed ? rawRequestedTargets : ''; - const requestedJobs = selectorValidationPassed ? rawRequestedJobs : ''; + const requestedTestIdsCsv = selectorValidationPassed ? rawRequestedTestIds : ''; const explicitOnlyReasons = { 'openshell-gateway-auth-contract': { job: 'openshell-gateway-auth-contract', @@ -5098,7 +4909,7 @@ jobs: reason: 'default dispatch excludes this explicit-only job unless selected', }); const targetsRejected = rawRequestedTargets && !selectorValidationPassed; - const jobsRejected = rawRequestedJobs && !selectorValidationPassed; + const testIdsRejected = rawRequestedTestIds && !selectorValidationPassed; let prNumber; if (prNumberInput) { @@ -5142,20 +4953,101 @@ jobs: prNumber = prs[0].number; } - const requestedJobList = requestedJobs + const requestedTestIds = requestedTestIdsCsv .split(',') - .map((job) => job.trim()) + .map((testId) => testId.trim()) .filter(Boolean); - const requestedJobSet = new Set(requestedJobList); + const requestedTestIdSet = new Set(requestedTestIds); const selectiveDispatch = - requestedJobList.length > 0 || Boolean(requestedTargets) || targetsRejected || jobsRejected; + requestedTestIds.length > 0 || Boolean(requestedTargets) || targetsRejected || testIdsRejected; const emoji = { success: '✅', failure: '❌', cancelled: '⚠️', skipped: '⏭️' }; - const allEntries = Object.entries(needs).sort(([a], [b]) => a.localeCompare(b)); - const missingRequested = selectorValidationPassed - ? requestedJobList.filter((job) => !(job in needs)) + const safeSelector = /^[A-Za-z0-9_-]+$/; + let testIds; + try { + const testMatrix = JSON.parse(process.env.TEST_MATRIX || '[]'); + if (!Array.isArray(testMatrix)) throw new Error('matrix must be an array'); + testIds = testMatrix.map((row) => { + if (!row || typeof row !== 'object' || !safeSelector.test(row.id || '')) { + throw new Error('matrix row has an invalid id'); + } + return row.id; + }); + if (new Set(testIds).size !== testIds.length) { + throw new Error('matrix repeats a test id'); + } + } catch (error) { + core.setFailed(`Invalid test matrix: ${error.message}`); + return; + } + + const testResults = new Map(); + let sharedJobResultsLoaded = false; + try { + const apiJobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, { + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId, + filter: 'latest', + per_page: 100, + }); + const selectedTestIds = new Set(testIds); + for (const job of apiJobs) { + const match = /^Shared E2E \(([A-Za-z0-9_-]+)\)$/.exec(job.name || ''); + if (!match || !selectedTestIds.has(match[1])) continue; + const result = + job.status === 'completed' && + ['success', 'failure', 'cancelled', 'skipped'].includes(job.conclusion) + ? job.conclusion + : 'unknown'; + testResults.set(match[1], { result }); + } + sharedJobResultsLoaded = true; + } catch (error) { + // The Actions jobs API is the only source of matrix-child conclusions; + // `needs['shared-e2e']` exposes only their aggregate. Keep missing children + // unknown instead of fabricating per-test evidence. This path is covered + // by e2e-report-to-pr-workflow-boundary.test.ts and can be removed when + // Actions exposes matrix-child conclusions directly to dependent jobs. + core.warning(`Could not load per-test results; reporting them as unknown: ${error.message}`); + } + const missingTestResults = testIds.filter((id) => !testResults.has(id)); + if (sharedJobResultsLoaded && missingTestResults.length > 0) { + core.warning(`Missing per-test results for ${missingTestResults.join(', ')}; reporting them as unknown.`); + } + const sharedJobAggregateResult = needs['shared-e2e']?.result; + const knownTestResults = testIds.map((id) => testResults.get(id)?.result); + const allTestResultsKnown = + knownTestResults.length > 0 && + knownTestResults.every((result) => result && result !== 'unknown'); + const expectedAggregateChildResult = + sharedJobAggregateResult === 'failure' + ? 'failure' + : sharedJobAggregateResult === 'cancelled' + ? 'cancelled' + : undefined; + const testAttributionMismatch = + allTestResultsKnown && + expectedAggregateChildResult && + !knownTestResults.includes(expectedAggregateChildResult); + if (testAttributionMismatch) { + core.warning( + `Per-test conclusions (${knownTestResults.join(', ')}) contradict shared E2E job aggregate ${sharedJobAggregateResult}; reporting child attribution as unknown.`, + ); + for (const id of testIds) testResults.set(id, { result: 'unknown' }); + } + + const allEntries = Object.entries(needs).filter(([name]) => name !== 'shared-e2e'); + if (needs['shared-e2e']) { + allEntries.push( + ...testIds.map((id) => [id, testResults.get(id) ?? { result: 'unknown' }]), + ); + } + allEntries.sort(([a], [b]) => a.localeCompare(b)); + const missingRequestedTestIds = selectorValidationPassed + ? requestedTestIds.filter((testId) => !allEntries.some(([name]) => name === testId)) : []; - const selectedEntries = requestedJobList.length > 0 - ? allEntries.filter(([name]) => requestedJobSet.has(name)) + const selectedEntries = requestedTestIds.length > 0 + ? allEntries.filter(([name]) => requestedTestIdSet.has(name)) : selectiveDispatch ? allEntries.filter( ([name, { result }]) => result !== 'skipped' && name !== 'generate-matrix', @@ -5169,7 +5061,7 @@ jobs: const rows = reportedEntries.map( ([name, { result }]) => `| ${name} | ${emoji[result] || '❓'} ${result} |`, ); - for (const name of missingRequested) { + for (const name of missingRequestedTestIds) { rows.push(`| ${name} | ❓ not reported |`); } @@ -5178,20 +5070,25 @@ jobs: const failed = ran.filter(([, v]) => v.result === 'failure'); const skipped = reportedEntries.filter(([, v]) => v.result === 'skipped'); const cancelled = ran.filter(([, v]) => v.result === 'cancelled'); - const passingStatus = requestedJobList.length > 0 - ? '✅ All requested jobs passed' + const unknown = ran.filter(([, v]) => v.result === 'unknown'); + const sharedJobAggregateFailed = sharedJobAggregateResult === 'failure'; + const sharedJobAggregateCancelled = sharedJobAggregateResult === 'cancelled'; + const passingStatus = requestedTestIds.length > 0 + ? '✅ All requested tests passed' : selectiveDispatch - ? '✅ All selected jobs passed' - : '✅ All default jobs passed'; + ? '✅ All selected tests passed' + : '✅ All default tests passed'; const status = - failed.length > 0 || missingRequested.length > 0 - ? '❌ Some jobs failed' - : cancelled.length > 0 && passed.length === 0 + failed.length > 0 || missingRequestedTestIds.length > 0 || sharedJobAggregateFailed + ? '❌ Some tests failed' + : (cancelled.length > 0 || sharedJobAggregateCancelled) && passed.length === 0 ? '⚠️ Run cancelled — no signal' - : cancelled.length > 0 && passed.length > 0 - ? '⚠️ Some jobs cancelled — partial pass' + : cancelled.length > 0 || sharedJobAggregateCancelled + ? '⚠️ Some tests cancelled — partial pass' + : unknown.length > 0 + ? '⚠️ Per-test results incomplete' : skipped.length > 0 && passed.length === 0 - ? '⚠️ No selected jobs ran' + ? '⚠️ No selected tests ran' : passingStatus; const lines = [ @@ -5204,14 +5101,14 @@ jobs: : requestedTargets ? `**Requested targets:** \`${requestedTargets}\`` : '**Requested targets:** _(default — all supported)_', - jobsRejected - ? '**Requested jobs:** _(selector rejected by workflow validation)_' - : requestedJobs - ? `**Requested jobs:** \`${requestedJobs}\`` - : '**Requested jobs:** _(default — all default-enabled free-standing jobs; explicit-only jobs `openshell-gateway-auth-contract`, `mcp-bridge-dev`, `hermes-gpu-startup`, `sandbox-rlimits-connect`, and `jetson-nvmap-gpu` are skipped unless selected)_', - `**Summary:** ${passed.length} passed, ${failed.length} failed, ${cancelled.length} cancelled, ${skipped.length} skipped`, + testIdsRejected + ? '**Requested test IDs:** _(selector rejected by workflow validation)_' + : requestedTestIdsCsv + ? `**Requested test IDs:** \`${requestedTestIdsCsv}\`` + : '**Requested test IDs:** _(default — all default-enabled tests; explicit-only tests `openshell-gateway-auth-contract`, `mcp-bridge-dev`, `hermes-gpu-startup`, `sandbox-rlimits-connect`, and `jetson-nvmap-gpu` are skipped unless selected)_', + `**Summary:** ${passed.length} passed, ${failed.length} failed, ${cancelled.length} cancelled, ${skipped.length} skipped, ${unknown.length} unknown`, '', - '| Job | Result |', + '| Test | Result |', '|-----|--------|', ...rows, ]; @@ -5226,12 +5123,19 @@ jobs: } if (failed.length > 0) { const failedNames = failed.map(([name]) => name).join(', '); - lines.push('', `> **Failed jobs:** ${failedNames}. Check [run artifacts](${runUrl}) for logs.`); + lines.push('', `> **Failed tests:** ${failedNames}. Check [run artifacts](${runUrl}) for logs.`); + } + if (missingRequestedTestIds.length > 0) { + lines.push( + '', + `> **Missing requested test IDs:** ${missingRequestedTestIds.join(', ')}. The reporting workflow needs to include these tests.`, + ); } - if (missingRequested.length > 0) { + if (unknown.length > 0) { + const unknownNames = unknown.map(([name]) => name).join(', '); lines.push( '', - `> **Missing requested jobs:** ${missingRequested.join(', ')}. The reporting workflow needs to include these jobs.`, + `> **Unknown per-test results:** ${unknownNames}. Shared E2E job aggregate: ${needs['shared-e2e']?.result ?? 'unavailable'}.`, ); } diff --git a/scripts/checks/e2e-mock-parity.ts b/scripts/checks/e2e-mock-parity.ts index febaa1d1a51..b9dbf9e64e6 100644 --- a/scripts/checks/e2e-mock-parity.ts +++ b/scripts/checks/e2e-mock-parity.ts @@ -6,6 +6,8 @@ import fs from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; +import ts from "typescript"; + const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); export const DEFAULT_PARITY_MANIFEST = "test/e2e/mock-parity.json"; @@ -28,6 +30,37 @@ const FAST_TESTS = [ /^test\/(?!e2e\/|package-contract\/).+\.test\.(?:js|ts)$/u, ] as const; +function sourceTokens(source: string): string { + const sourceFile = ts.createSourceFile( + "source.ts", + source, + ts.ScriptTarget.Latest, + true, + ts.ScriptKind.TS, + ); + const tokens: Array<[ts.SyntaxKind, string]> = []; + const visit = (node: ts.Node): void => { + const children = node.getChildren(sourceFile); + if (children.length === 0) { + if (node.kind !== ts.SyntaxKind.EndOfFileToken) { + tokens.push([node.kind, node.getText(sourceFile)]); + } + return; + } + for (const child of children) visit(child); + }; + visit(sourceFile); + return JSON.stringify(tokens); +} + +export function isMockParityRelevantSourceChange( + baseSource: string | null, + headSource: string | null, +): boolean { + if (baseSource === null || headSource === null) return true; + return sourceTokens(baseSource) !== sourceTokens(headSource); +} + function isSafeRepoPath(file: string): boolean { return ( file.length > 0 && @@ -116,13 +149,34 @@ function argument(name: string): string | undefined { return index >= 0 ? process.argv[index + 1] : undefined; } +function sourceAtRef(ref: string, file: string): string | null { + try { + return execFileSync("git", ["show", `${ref}:${file}`], { + cwd: REPO_ROOT, + encoding: "utf8", + maxBuffer: 10 * 1024 * 1024, + }); + } catch { + return null; + } +} + function changedFiles(base: string, head: string): string[] { - return execFileSync("git", ["diff", "--name-only", "--diff-filter=ACMR", `${base}...${head}`], { - cwd: REPO_ROOT, - encoding: "utf8", - }) + const files = execFileSync( + "git", + ["diff", "--name-only", "--diff-filter=ACMR", `${base}...${head}`], + { + cwd: REPO_ROOT, + encoding: "utf8", + }, + ) .split(/\r?\n/u) .filter(Boolean); + return files.filter( + (file) => + !LIVE_TEST.test(file) || + isMockParityRelevantSourceChange(sourceAtRef(base, file), sourceAtRef(head, file)), + ); } function main(): void { diff --git a/test/e2e-advisor-targets.test.ts b/test/e2e-advisor-targets.test.ts index 8ad25d5a313..6ff53b128b5 100644 --- a/test/e2e-advisor-targets.test.ts +++ b/test/e2e-advisor-targets.test.ts @@ -408,7 +408,7 @@ describe("E2E target advisor — normalization contract", () => { ]); }); - it("suppresses fan-out for a new free-standing live test that is not workflow-wired", () => { + it("suppresses fan-out for a new E2E test that is not workflow-wired", () => { const normalized = normalizeE2eTargetAdvisorResult( { required: [ @@ -434,6 +434,67 @@ describe("E2E target advisor — normalization contract", () => { expect(normalized.noTargetE2eReason).toContain("test/e2e/live/rebuild-openclaw.test.ts"); }); + it.each([ + ["test/e2e/live/new-credential-free-proof.test.ts", "new-credential-free-proof"], + ["test/new-credential-free-integration.test.ts", "new-credential-free-integration"], + ])("recognizes a credential-free tag on a newly added test (%s)", (file, id) => { + const normalized = normalizeE2eTargetAdvisorResult( + { + required: [ + { + id: "e2e-all", + workflow: E2E_WORKFLOW, + selectorType: "all", + reason: "model requested fan-out", + }, + ], + optional: [], + confidence: "high", + }, + metadata({ changedFiles: [file] }), + { + changedFileSources: { + [file]: "// @module-tag e2e/credential-free\n", + }, + e2eWorkflowText: "jobs:\n shared-e2e:\n steps: []\n", + }, + ); + + expect(normalized.required.map((item) => item.id)).toContain(id); + expect(normalized.required.map((item) => item.id)).not.toContain("e2e-all"); + expect(normalized.noTargetE2eReason).toBeNull(); + }); + + it.each([ + ["has its credential-free tag removed", "// tag removed\n"], + ["is deleted", null], + ])("treats the analyzed change as authoritative when a tagged test %s", (_case, source) => { + const file = "test/e2e/live/docs-validation.test.ts"; + const normalized = normalizeE2eTargetAdvisorResult( + { + required: [ + { + id: "e2e-all", + workflow: E2E_WORKFLOW, + selectorType: "all", + reason: "model requested fan-out", + }, + ], + optional: [], + confidence: "high", + }, + metadata({ changedFiles: [file] }), + { + changedFileSources: { [file]: source }, + e2eWorkflowText: "jobs:\n shared-e2e:\n steps: []\n", + }, + ); + + expect(normalized.required.map((item) => item.id)).not.toContain("docs-validation"); + expect(normalized.required.map((item) => item.id)).not.toContain("e2e-all"); + expect(normalized.noTargetE2eReason).toContain(file); + }); + it("keeps the deterministic floor while suppressing unwired-test fan-out", () => { const normalized = normalizeE2eTargetAdvisorResult( { diff --git a/test/e2e-mock-parity.test.ts b/test/e2e-mock-parity.test.ts index d02ca15f993..0de972be19b 100644 --- a/test/e2e-mock-parity.test.ts +++ b/test/e2e-mock-parity.test.ts @@ -2,10 +2,15 @@ // SPDX-License-Identifier: Apache-2.0 import { describe, expect, it } from "vitest"; -import { type MockParityManifest, validateMockParity } from "../scripts/checks/e2e-mock-parity"; +import { + isMockParityRelevantSourceChange, + type MockParityManifest, + validateMockParity, +} from "../scripts/checks/e2e-mock-parity"; const live = "test/e2e/live/example.test.ts"; const fast = "test/e2e/support/example.test.ts"; +const TAGGED_NEW_SOURCE = "// @module-tag e2e/credential-free\n"; const exists = (file: string) => file === live || file === fast; function manifest(entries: MockParityManifest["entries"]): MockParityManifest { @@ -13,6 +18,47 @@ function manifest(entries: MockParityManifest["entries"]): MockParityManifest { } describe("changed live E2E mock parity", () => { + it("treats module-tag-only diffs as metadata", () => { + expect( + isMockParityRelevantSourceChange( + "// SPDX-License-Identifier: Apache-2.0\n\nexport {};\n", + "// SPDX-License-Identifier: Apache-2.0\n// @module-tag e2e/credential-free\n\nexport {};\n", + ), + ).toBe(false); + expect( + isMockParityRelevantSourceChange( + `${"// @module"}-tag retired/value\n\nexport {};\n`, + "// @module-tag e2e/credential-free\n\nexport {};\n", + ), + ).toBe(false); + expect( + isMockParityRelevantSourceChange( + "// old terminology\nexport {};\n", + "// current terminology\nexport {};\n", + ), + ).toBe(false); + expect( + isMockParityRelevantSourceChange( + "// @module-tag e2e/credential-free\n\nexport {};\n", + "// @module-tag e2e/credential-free\n\nexport const changed = true;\n", + ), + ).toBe(true); + expect( + isMockParityRelevantSourceChange( + "export const fixture = `before\nafter`;\n", + "export const fixture = `before\n// @module-tag e2e/credential-free\nafter`;\n", + ), + ).toBe(true); + expect( + isMockParityRelevantSourceChange( + "// SPDX-License-Identifier: Apache-2.0\n\nexport {};\n", + "// SPDX-License-Identifier: Apache-2.0\n/* @module-tag e2e/credential-free */\n\nexport {};\n", + ), + ).toBe(false); + expect(isMockParityRelevantSourceChange(null, null)).toBe(true); + expect(isMockParityRelevantSourceChange(null, TAGGED_NEW_SOURCE)).toBe(true); + }); + it("accepts a changed live E2E mapped to a fast PR test", () => { expect( validateMockParity({ diff --git a/test/e2e/README.md b/test/e2e/README.md index 4c7530d5030..01eef24267d 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -21,6 +21,33 @@ The former top-level `test/e2e/test-*.sh` suite has been removed. Keep real shell, installer, process, Docker, OpenShell, `/proc`, and sandbox boundaries in E2E tests when those boundaries are the behavior under test. +## Credential-free tests + +Credential-free tests that can use the standard Ubuntu runner, CLI build, and +artifact policy opt into the shared E2E job with a tag beside the test: + +```typescript +// @module-tag e2e/credential-free +``` + +Discovery reads tagged files from the `e2e-live` and `integration` Vitest +projects. It derives each test ID from the filename and supplies only the ID, +repository-relative file, and Vitest project to the test matrix. Keep the +filename stem unique and lowercase kebab-case. Do not add the test to a separate +catalog or manually maintained workflow matrix. + +The E2E workflow owns the shared job's runner, timeout, setup, permissions, +secrets, and artifact handling. Keep a dedicated workflow job when a test needs +different capabilities, such as credentials, a custom runner, additional setup, +or a different timeout. + +Both `jobs` and `targets` selectors continue to accept the test ID. Run the +discovery command locally to inspect the generated test matrix: + +```bash +npx tsx tools/e2e/credential-free-tests.mts +``` + ## Scheduled operations The consolidated workflow keeps its operational reporting in the same job diff --git a/test/e2e/live/docs-validation.test.ts b/test/e2e/live/docs-validation.test.ts index eb523f24cfe..668f942199e 100644 --- a/test/e2e/live/docs-validation.test.ts +++ b/test/e2e/live/docs-validation.test.ts @@ -1,5 +1,6 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +// @module-tag e2e/credential-free import fs from "node:fs"; import fsp from "node:fs/promises"; diff --git a/test/e2e/live/onboard-negative-paths.test.ts b/test/e2e/live/onboard-negative-paths.test.ts index f8116f1264f..10ecfb29c95 100644 --- a/test/e2e/live/onboard-negative-paths.test.ts +++ b/test/e2e/live/onboard-negative-paths.test.ts @@ -1,5 +1,6 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +// @module-tag e2e/credential-free import fs from "node:fs"; import path from "node:path"; diff --git a/test/e2e/live/openshell-version-pin.test.ts b/test/e2e/live/openshell-version-pin.test.ts index aeb945cacff..fa4fe691d1e 100644 --- a/test/e2e/live/openshell-version-pin.test.ts +++ b/test/e2e/live/openshell-version-pin.test.ts @@ -1,5 +1,6 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +// @module-tag e2e/credential-free import { spawnSync } from "node:child_process"; import fs from "node:fs"; @@ -10,12 +11,12 @@ import { type ArtifactSink } from "../fixtures/artifacts.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; import { REPO_ROOT } from "../fixtures/paths.ts"; -// #3474). The former bash script is a hermetic installer-script behavioral +// #3474). The former bash script is a self-contained installer-script behavioral // test: it runs scripts/install-openshell.sh under a stubbed PATH where the // already-installed openshell reports a too-new version (0.0.73) and the // downloaded archives produce a binary that reports the pinned 0.0.72. // -// This is a free-standing live test (per #5049's pattern) — it does not exercise +// This credential-free E2E test does not exercise // the registry-driven steady-state probe model. There is no OpenClaw instance, // no environment phase, no lifecycle. The test consumes only the `artifacts` // fixture from e2e-test.ts so failures attach the per-target artifact root. @@ -98,7 +99,7 @@ function writeExecutable(target: string, contents: string): void { // Bash helpers shared by the gh and curl stubs: write a fake archive and emit // the same pinned digest lines the real OpenShell v0.0.72 release uses. A fake -// sha256sum below keeps this test hermetic even though the tarball bytes are +// sha256sum below keeps this test self-contained even though the tarball bytes are // synthetic. const SHARED_DOWNLOAD_BASH_HELPERS = `\ write_asset() { diff --git a/test/e2e/live/ubuntu-repo-cli-smoke.test.ts b/test/e2e/live/ubuntu-repo-cli-smoke.test.ts index 4d1c2dad482..f44de2bf3b9 100644 --- a/test/e2e/live/ubuntu-repo-cli-smoke.test.ts +++ b/test/e2e/live/ubuntu-repo-cli-smoke.test.ts @@ -1,5 +1,6 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +// @module-tag e2e/credential-free import fs from "node:fs"; import path from "node:path"; diff --git a/test/e2e/support/credential-free-tests.test.ts b/test/e2e/support/credential-free-tests.test.ts new file mode 100644 index 00000000000..b4f1d2928ae --- /dev/null +++ b/test/e2e/support/credential-free-tests.test.ts @@ -0,0 +1,170 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { + CREDENTIAL_FREE_TEST_TAG, + type CredentialFreeTestModule, + credentialFreeTestProjectForFile, + credentialFreeTestRowFromModule, + discoverCredentialFreeTestRows, + discoverCredentialFreeTests, +} from "../../../tools/e2e/credential-free-tests.mts"; +import { REPO_ROOT } from "../fixtures/paths.ts"; + +const CREDENTIAL_FREE_TESTS_CLI = path.join(REPO_ROOT, "tools", "e2e", "credential-free-tests.mts"); +const TSX = path.join(REPO_ROOT, "node_modules", ".bin", "tsx"); +const TAG_COMMENT = `// @module-tag ${CREDENTIAL_FREE_TEST_TAG}`; + +function module(overrides: Partial = {}): CredentialFreeTestModule { + return { + file: "test/e2e/live/example.test.ts", + project: "e2e-live", + source: TAG_COMMENT, + ...overrides, + }; +} + +describe("credential-free test discovery", () => { + it.each([ + ["test/e2e/live/example.test.ts", "e2e-live"], + ["test/e2e/live/nested/example.test.ts", "e2e-live"], + ["test/example.test.ts", "integration"], + ["test/nested/example.test.js", "integration"], + ["test/e2e/support/example.test.ts", undefined], + ["src/example.test.ts", undefined], + ])("classifies %s in the expected Vitest project", (file, expected) => { + expect(credentialFreeTestProjectForFile(file)).toBe(expected); + }); + + it("derives deterministic safe matrix rows without workflow capabilities", () => { + const rows = discoverCredentialFreeTestRows([ + module({ file: "test/zeta.test.ts", project: "integration" }), + module({ file: "test/e2e/live/alpha.test.ts" }), + ]); + + expect(rows).toEqual([ + { id: "alpha", file: "test/e2e/live/alpha.test.ts", project: "e2e-live" }, + { id: "zeta", file: "test/zeta.test.ts", project: "integration" }, + ]); + expect(Object.keys(rows[0])).toEqual(["id", "file", "project"]); + }); + + it("rejects a candidate without the credential-free tag", () => { + expect(() => credentialFreeTestRowFromModule(module({ source: "// no tag" }))).toThrow( + "must declare exactly one e2e/credential-free module tag; found 0", + ); + }); + + it("rejects unknown tags in the E2E namespace", () => { + expect(() => + credentialFreeTestRowFromModule( + module({ source: `${"// @module"}-tag e2e/credential-bearing` }), + ), + ).toThrow("Unknown E2E test tag 'e2e/credential-bearing' in test/e2e/live/example.test.ts"); + }); + + it("rejects duplicate credential-free tags", () => { + expect(() => + credentialFreeTestRowFromModule(module({ source: `${TAG_COMMENT}\n${TAG_COMMENT}` })), + ).toThrow("must declare exactly one e2e/credential-free module tag; found 2"); + }); + + it("only treats literal module-tag comments as credential-free declarations", () => { + expect(() => + credentialFreeTestRowFromModule( + module({ source: `const example = ${JSON.stringify(TAG_COMMENT)};` }), + ), + ).toThrow("found 0"); + expect(() => + credentialFreeTestRowFromModule( + module({ source: `const example = \`\n${TAG_COMMENT}\n\`;` }), + ), + ).toThrow("found 0"); + expect( + credentialFreeTestRowFromModule(module({ source: `/* ${TAG_COMMENT.slice(3)} */` })), + ).toEqual({ + id: "example", + file: "test/e2e/live/example.test.ts", + project: "e2e-live", + }); + }); + + it("rejects duplicate ids derived from different test files", () => { + expect(() => + discoverCredentialFreeTestRows([ + module({ file: "test/e2e/live/nested/example.test.ts" }), + module({ file: "test/example.test.ts", project: "integration" }), + ]), + ).toThrow( + "Duplicate credential-free test id 'example': test/e2e/live/nested/example.test.ts, test/example.test.ts", + ); + }); + + it.each([ + "../escape.test.ts", + "/tmp/escape.test.ts", + "test/e2e/live/../escape.test.ts", + "test\\e2e\\live\\escape.test.ts", + "test/e2e/live/bad id.test.ts", + ])("rejects unsafe repo-relative test path %s", (file) => { + expect(() => credentialFreeTestRowFromModule(module({ file }))).toThrow( + "must be a safe repo-relative test file", + ); + }); + + it("rejects unsafe ids derived from test filenames", () => { + expect(() => + credentialFreeTestRowFromModule(module({ file: "test/e2e/live/Bad_Name.test.ts" })), + ).toThrow("filename must derive a safe id"); + }); + + it("rejects a file that does not belong to its declared Vitest project", () => { + expect(() => + credentialFreeTestRowFromModule( + module({ file: "test/e2e/live/example.test.ts", project: "integration" }), + ), + ).toThrow("integration credential-free test must not live under test/e2e/"); + }); + + it("discovers the tagged repository files through their real Vitest projects", () => { + const rows = discoverCredentialFreeTests(); + expect(rows.length).toBeGreaterThan(0); + expect(rows).toEqual([...rows].sort((left, right) => left.id.localeCompare(right.id))); + for (const row of rows) { + expect(Object.keys(row)).toEqual(["id", "file", "project"]); + expect(row.file).toMatch(/^test\/.+\.test\.(?:js|ts)$/); + } + }); + + it("prints one compact JSON matrix line from the CLI", () => { + const expected = discoverCredentialFreeTests(); + expect(expected.length).toBeGreaterThan(0); + const result = spawnSync(TSX, [CREDENTIAL_FREE_TESTS_CLI], { + cwd: REPO_ROOT, + encoding: "utf8", + timeout: 30_000, + }); + + expect(result.status, result.stderr || result.stdout).toBe(0); + expect(result.stdout).toBe(`${JSON.stringify(expected)}\n`); + }); + + it("rejects selector arguments owned by the workflow planner", () => { + const result = spawnSync(TSX, [CREDENTIAL_FREE_TESTS_CLI, "--jobs", "docs-validation"], { + cwd: REPO_ROOT, + encoding: "utf8", + timeout: 30_000, + }); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(""); + expect(result.stderr).toContain( + "::error::Credential-free test discovery does not accept selectors; use workflow-plan.mts", + ); + }); +}); diff --git a/test/e2e/support/dockerhub-auth-workflow-boundary.test.ts b/test/e2e/support/dockerhub-auth-workflow-boundary.test.ts index 8cc9598f8ed..389a8e662e9 100644 --- a/test/e2e/support/dockerhub-auth-workflow-boundary.test.ts +++ b/test/e2e/support/dockerhub-auth-workflow-boundary.test.ts @@ -13,13 +13,7 @@ import YAML from "yaml"; import { validateE2eWorkflowBoundary } from "../../../tools/e2e/workflow-boundary.mts"; import { readWorkflow } from "../../helpers/e2e-workflow-contract"; -const NO_IMAGE_E2E_JOBS = [ - "docs-validation", - "gateway-drift-preflight", - "gateway-health-honest", - "onboard-negative-paths", - "openshell-version-pin", -] as const; +const NO_IMAGE_E2E_JOBS = ["gateway-health-honest", "shared-e2e"] as const; const AUTH_STEP_NAME = "Authenticate to Docker Hub"; const CLEANUP_STEP_NAME = "Clean up Docker auth"; const CLEANUP_HELPER_RUN = "bash .github/scripts/docker-auth-cleanup.sh"; diff --git a/test/e2e/support/docs-validation-workflow-boundary.test.ts b/test/e2e/support/docs-validation-workflow-boundary.test.ts deleted file mode 100644 index 9c642d31856..00000000000 --- a/test/e2e/support/docs-validation-workflow-boundary.test.ts +++ /dev/null @@ -1,78 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { describe, expect, it } from "vitest"; -import YAML from "yaml"; -import { - readDocsValidationWorkflow, - validateDocsValidationWorkflow, - validateDocsValidationWorkflowBoundary, -} from "../../../tools/e2e/docs-validation-workflow-boundary.mts"; -import { - evaluateE2eWorkflowDispatchSelectors, - validateE2eWorkflowBoundary, -} from "../../../tools/e2e/workflow-boundary.mts"; - -describe("docs validation workflow boundary", () => { - it("is default-enabled and selectively dispatchable", () => { - expect(validateDocsValidationWorkflowBoundary()).toEqual([]); - expect(validateE2eWorkflowBoundary()).toEqual([]); - - for (const selector of [{ targets: "docs-validation" }, { jobs: "docs-validation" }]) { - expect(evaluateE2eWorkflowDispatchSelectors(selector)).toMatchObject({ - valid: true, - liveTargetsRun: false, - selectedFreeStandingJobs: ["docs-validation"], - }); - } - expect(evaluateE2eWorkflowDispatchSelectors({}).selectedFreeStandingJobs).toContain( - "docs-validation", - ); - }); - - it("makes execution, determinism, and aggregation part of the focused ratchet", () => { - const workflow = readDocsValidationWorkflow(); - const job = workflow.jobs["docs-validation"]; - job.env!.CHECK_DOC_LINKS_REMOTE = "1"; - job.steps!.find((step) => step.name === "Run docs validation live Vitest test")!.run = - "echo skipped"; - workflow.jobs["report-to-pr"].needs = (workflow.jobs["report-to-pr"].needs as string[]).filter( - (name) => name !== "docs-validation", - ); - - expect(validateDocsValidationWorkflow(workflow)).toEqual( - expect.arrayContaining([ - "docs-validation must keep link checks deterministic and local-only", - "docs-validation step Run docs validation live Vitest test must contain: test/e2e/live/docs-validation.test.ts", - "report-to-pr must wait for docs-validation", - ]), - ); - - const directory = mkdtempSync(join(tmpdir(), "nemoclaw-docs-validation-workflow-")); - const workflowPath = join(directory, "workflow.yaml"); - try { - writeFileSync(workflowPath, YAML.stringify(workflow)); - expect(validateDocsValidationWorkflowBoundary(workflowPath)).toContain( - "report-to-pr must wait for docs-validation", - ); - } finally { - rmSync(directory, { force: true, recursive: true }); - } - }); - - it("reports empty workflow input as contract errors instead of throwing", () => { - const directory = mkdtempSync(join(tmpdir(), "nemoclaw-docs-validation-empty-")); - const workflowPath = join(directory, "workflow.yaml"); - try { - writeFileSync(workflowPath, ""); - expect(validateDocsValidationWorkflowBoundary(workflowPath)).toContain( - "docs-validation must depend on generate-matrix", - ); - } finally { - rmSync(directory, { force: true, recursive: true }); - } - }); -}); diff --git a/test/e2e/support/e2e-operations-workflow-boundary.test.ts b/test/e2e/support/e2e-operations-workflow-boundary.test.ts index 44f68f62cc9..7d4f1ebc2d3 100644 --- a/test/e2e/support/e2e-operations-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-operations-workflow-boundary.test.ts @@ -31,7 +31,10 @@ describe("E2E operations workflow boundary", () => { const workflow = readE2eOperationsWorkflow(); const reportNeeds = workflow.jobs["report-to-pr"].needs as string[]; expect(workflow.jobs["notify-on-failure"]).toBeUndefined(); - expect(workflow.jobs["report-to-pr"].permissions).toEqual({ "pull-requests": "write" }); + expect(workflow.jobs["report-to-pr"].permissions).toEqual({ + actions: "read", + "pull-requests": "write", + }); expect(workflow.jobs.scorecard.needs).toEqual(reportNeeds); }); @@ -149,7 +152,7 @@ describe("E2E operations workflow boundary", () => { "cloud-onboard must not hold issues: write", "cloud-onboard must not hold pull-requests: write", "report-to-pr must not hold issues: write", - "report-to-pr must hold only pull-requests: write", + "report-to-pr must hold only actions: read and pull-requests: write", "report-to-pr must run only for manual workflow dispatches", "report-to-pr must contain only its PR-comment step", "report-to-pr must not use issue mutations or generic GitHub write surfaces", diff --git a/test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts b/test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts index 460dcb2e32b..c4cdcce1290 100644 --- a/test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts @@ -1,13 +1,19 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { expect, it } from "vitest"; +import { expect, it, vi } from "vitest"; import YAML from "yaml"; +import { + type CredentialFreeTestMatrixRow, + discoverCredentialFreeTests, +} from "../../../tools/e2e/credential-free-tests.mts"; import { validateE2eWorkflowBoundary } from "../../../tools/e2e/workflow-boundary.mts"; +import { buildE2eWorkflowPlan } from "../../../tools/e2e/workflow-plan.mts"; function readWorkflow(): Record { return YAML.parse( @@ -15,6 +21,178 @@ function readWorkflow(): Record { ) as Record; } +const AsyncFunction = Object.getPrototypeOf(async () => undefined).constructor as new ( + ...parameters: string[] +) => (...args: unknown[]) => Promise; + +function reportScript(): string { + const workflow = readWorkflow() as { + jobs: Record }>; + }; + const step = workflow.jobs["report-to-pr"].steps.find( + (candidate) => candidate.name === "Post E2E target results to PR", + ); + expect(step?.with?.script).toEqual(expect.any(String)); + return String(step!.with!.script); +} + +function generateMatrixScript(): string { + const workflow = readWorkflow() as { + jobs: Record }>; + }; + const step = workflow.jobs["generate-matrix"].steps.find( + (candidate) => candidate.id === "matrix", + ); + expect(step?.run).toEqual(expect.any(String)); + return String(step!.run); +} + +function executeGenerateMatrixWithPlannerOutput(plan: unknown) { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-planner-schema-")); + const binDirectory = path.join(directory, "bin"); + const fakeNpx = path.join(binDirectory, "npx"); + const outputPath = path.join(directory, "github-output"); + fs.mkdirSync(binDirectory); + fs.writeFileSync( + fakeNpx, + [ + "#!/usr/bin/env bash", + '[[ "$#" -eq 2 && "$1" == "tsx" && "$2" == "tools/e2e/workflow-plan.mts" ]] || exit 97', + "printf '%s\\n' \"${FAKE_E2E_PLAN}\"", + "", + ].join("\n"), + { mode: 0o755 }, + ); + try { + return { + result: spawnSync("bash", ["-c", generateMatrixScript()], { + cwd: process.cwd(), + encoding: "utf8", + env: { + ...process.env, + FAKE_E2E_PLAN: JSON.stringify(plan), + GITHUB_OUTPUT: outputPath, + GITHUB_STEP_SUMMARY: path.join(directory, "summary.md"), + JOBS: "", + PATH: `${binDirectory}${path.delimiter}${process.env.PATH ?? ""}`, + TARGETS: "", + }, + timeout: 30_000, + }), + workflowOutput: fs.existsSync(outputPath) ? fs.readFileSync(outputPath, "utf8") : "", + }; + } finally { + fs.rmSync(directory, { force: true, recursive: true }); + } +} + +type ApiJob = { + conclusion: string | null; + name: string; + status: string; +}; + +const DEFAULT_TEST_MATRIX: CredentialFreeTestMatrixRow[] = [ + { + id: "alpha", + file: "test/e2e/live/alpha.test.ts", + project: "e2e-live", + }, + { + id: "beta", + file: "test/e2e/live/beta.test.ts", + project: "e2e-live", + }, +]; + +async function executeReport(options: { + apiJobs?: ApiJob[]; + testMatrix?: CredentialFreeTestMatrixRow[]; + jobs?: string; + needs?: Record; + paginateError?: Error; +}): Promise<{ + body: string; + setFailed: ReturnType; + warning: ReturnType; +}> { + const { + apiJobs = [], + testMatrix = DEFAULT_TEST_MATRIX, + jobs = testMatrix.map(({ id }) => id).join(","), + needs = { + "generate-matrix": { result: "success" }, + "shared-e2e": { result: "failure" }, + live: { result: "skipped" }, + }, + paginateError, + } = options; + const script = reportScript().replace( + "const needs = ${{ toJSON(needs) }};", + `const needs = ${JSON.stringify(needs)};`, + ); + const createComment = vi.fn(async (_input: { body: string }) => undefined); + const setFailed = vi.fn(); + const warning = vi.fn(); + const paginate = paginateError + ? vi.fn(() => Promise.reject(paginateError)) + : vi.fn(async () => apiJobs); + const github = { + paginate, + rest: { + actions: { listJobsForWorkflowRun: Symbol("listJobsForWorkflowRun") }, + issues: { createComment }, + pulls: { + get: vi.fn(async () => ({ data: { state: "open" } })), + list: vi.fn(), + }, + }, + }; + const context = { + ref: "refs/heads/main", + repo: { owner: "NVIDIA", repo: "NemoClaw" }, + runId: 123, + serverUrl: "https://github.com", + }; + const core = { info: vi.fn(), setFailed, warning }; + const processStub = { + env: { + EXPLICIT_ONLY_JOBS: "", + TEST_MATRIX: JSON.stringify(testMatrix), + JOB_PR_NUMBER: "42", + JOB_TARGETS: "", + JOBS: jobs, + }, + }; + + await new AsyncFunction("github", "context", "core", "process", script)( + github, + context, + core, + processStub, + ); + + expect(createComment).toHaveBeenCalledOnce(); + return { + body: createComment.mock.calls[0]?.[0]?.body as string, + setFailed, + warning, + }; +} + +function parseSimpleOutput(output: string): Record { + return Object.fromEntries( + output + .trim() + .split("\n") + .map((line) => { + const separator = line.indexOf("="); + expect(separator).toBeGreaterThan(0); + return [line.slice(0, separator), line.slice(separator + 1)]; + }), + ); +} + it("rejects report-to-pr PR number validation drift", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-workflow-")); const workflowPath = path.join(tmp, "workflow.yaml"); @@ -53,3 +231,167 @@ it("rejects report-to-pr PR number validation drift", () => { fs.rmSync(tmp, { recursive: true, force: true }); } }); + +it("reports matrix children by test ID without fabricating a missing child result", async () => { + const { body, setFailed, warning } = await executeReport({ + apiJobs: [ + { + conclusion: "success", + name: "Shared E2E (alpha)", + status: "completed", + }, + ], + }); + + expect(setFailed).not.toHaveBeenCalled(); + expect(warning).toHaveBeenCalledWith( + "Missing per-test results for beta; reporting them as unknown.", + ); + expect(body).toContain("| alpha | ✅ success |"); + expect(body).toContain("| beta | ❓ unknown |"); + expect(body).toContain("Some tests failed"); + expect(body).toContain("Shared E2E job aggregate: failure"); +}); + +it("reports API lookup failures as unknown rather than copying the aggregate result", async () => { + const { body, setFailed, warning } = await executeReport({ + testMatrix: DEFAULT_TEST_MATRIX.slice(0, 1), + jobs: "alpha", + needs: { + "generate-matrix": { result: "success" }, + "shared-e2e": { result: "success" }, + }, + paginateError: new Error("API unavailable"), + }); + + expect(setFailed).not.toHaveBeenCalled(); + expect(warning).toHaveBeenCalledWith( + "Could not load per-test results; reporting them as unknown: API unavailable", + ); + expect(body).toContain("Per-test results incomplete"); + expect(body).toContain("| alpha | ❓ unknown |"); + expect(body).not.toContain("| alpha | ✅ success |"); +}); + +it("keeps nonterminal API conclusions unknown", async () => { + const { body, setFailed } = await executeReport({ + apiJobs: [ + { + conclusion: null, + name: "Shared E2E (alpha)", + status: "in_progress", + }, + ], + testMatrix: DEFAULT_TEST_MATRIX.slice(0, 1), + jobs: "alpha", + needs: { + "generate-matrix": { result: "success" }, + "shared-e2e": { result: "success" }, + }, + }); + + expect(setFailed).not.toHaveBeenCalled(); + expect(body).toContain("Per-test results incomplete"); + expect(body).toContain("| alpha | ❓ unknown |"); +}); + +it("does not claim child success when complete API results contradict the aggregate", async () => { + const { body, setFailed, warning } = await executeReport({ + apiJobs: [ + { + conclusion: "success", + name: "Shared E2E (alpha)", + status: "completed", + }, + ], + testMatrix: DEFAULT_TEST_MATRIX.slice(0, 1), + jobs: "alpha", + needs: { + "generate-matrix": { result: "success" }, + "shared-e2e": { result: "failure" }, + }, + }); + + expect(setFailed).not.toHaveBeenCalled(); + expect(warning).toHaveBeenCalledWith( + "Per-test conclusions (success) contradict shared E2E job aggregate failure; reporting child attribution as unknown.", + ); + expect(body).toContain("Some tests failed"); + expect(body).toContain("| alpha | ❓ unknown |"); + expect(body).not.toContain("| alpha | ✅ success |"); +}); + +it("carries the generated planner matrix through the workflow output and PR report", async () => { + const [selected] = discoverCredentialFreeTests(); + expect(selected).toBeDefined(); + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-shared-e2e-integration-")); + const outputPath = path.join(directory, "github-output"); + const summaryPath = path.join(directory, "summary.md"); + try { + const generated = spawnSync("bash", ["-c", generateMatrixScript()], { + cwd: process.cwd(), + encoding: "utf8", + env: { + ...process.env, + GITHUB_OUTPUT: outputPath, + GITHUB_STEP_SUMMARY: summaryPath, + JOBS: selected.id, + TARGETS: "", + }, + timeout: 30_000, + }); + expect(generated.status, generated.stderr || generated.stdout).toBe(0); + const outputs = parseSimpleOutput(fs.readFileSync(outputPath, "utf8")); + const testMatrix = JSON.parse(outputs.test_matrix) as CredentialFreeTestMatrixRow[]; + expect(testMatrix).toEqual([selected]); + + const { body, setFailed } = await executeReport({ + apiJobs: [ + { + conclusion: "success", + name: `Shared E2E (${selected.id})`, + status: "completed", + }, + ], + testMatrix, + jobs: selected.id, + needs: { + "generate-matrix": { result: "success" }, + "shared-e2e": { result: "success" }, + }, + }); + + expect(setFailed).not.toHaveBeenCalled(); + expect(body).toContain("All requested tests passed"); + expect(body).toContain(`| ${selected.id} | ✅ success |`); + } finally { + fs.rmSync(directory, { force: true, recursive: true }); + } +}); + +it("fails closed when planner output violates the workflow schema", () => { + const [selected] = discoverCredentialFreeTests(); + expect(selected).toBeDefined(); + const validPlan = buildE2eWorkflowPlan(); + const [registryRow] = validPlan.matrix; + expect(registryRow).toBeDefined(); + const { explicitOnlyJobs: _omitted, ...missingField } = validPlan; + const malformedPlans = [ + ["missing required field", missingField], + ["duplicate matrix id", { ...validPlan, matrix: [...validPlan.matrix, { ...registryRow }] }], + ["invalid test ID", { ...validPlan, testMatrix: [{ ...selected, id: "invalid_id" }] }], + ["nonboolean selection", { ...validPlan, hermesSelected: "false" }], + ] as const; + + for (const [label, plan] of malformedPlans) { + const generated = executeGenerateMatrixWithPlannerOutput(plan); + expect( + generated.result.status, + `${label}: ${generated.result.stderr || generated.result.stdout}`, + ).toBe(1); + expect(generated.result.stderr).toContain( + "::error::E2E planner returned an invalid output schema", + ); + expect(generated.workflowOutput).toBe(""); + } +}); diff --git a/test/e2e/support/e2e-workflow.test.ts b/test/e2e/support/e2e-workflow.test.ts index 449fc6cf0f8..a0c0d3b7cdd 100644 --- a/test/e2e/support/e2e-workflow.test.ts +++ b/test/e2e/support/e2e-workflow.test.ts @@ -643,7 +643,9 @@ describe("e2e workflow boundary", () => { }, ); - it("derives the free-standing inventory from workflow job metadata", { timeout: 60_000 }, () => { + it("derives test selectors from code and workflow jobs from workflow metadata", { + timeout: 60_000, + }, () => { const inventory = readFreeStandingJobsInventory(); expect(validateFreeStandingWorkflowInventory()).toEqual([]); expect(inventory.allowedJobs).toContain("openshell-version-pin"); @@ -654,7 +656,7 @@ describe("e2e workflow boundary", () => { expect(inventory.targetToJob.get("openshell-gateway-auth-contract")).toBe( "openshell-gateway-auth-contract", ); - expect(inventory.targetToJob.get("openshell-version-pin")).toBe("openshell-version-pin"); + expect(inventory.targetToJob.get("openshell-version-pin")).toBe("shared-e2e"); expect(inventory.targetToJob.get("upgrade-stale-sandbox")).toBe("upgrade-stale-sandbox"); expect(inventory.targetToJob.get("credential-migration")).toBe("credential-migration"); expect(inventory.targetToJob.get("launchable-smoke")).toBe("launchable-smoke"); @@ -663,12 +665,48 @@ describe("e2e workflow boundary", () => { "openclaw-plugin-runtime-exdev", ); expect( - inventory.allowedJobs.every((job) => + inventory.workflowJobs.every((job) => Object.keys((readWorkflow().jobs as Record) ?? {}).includes(job), ), ).toBe(true); }); + it("emits the inventory consumed by the current base E2E workflow", { + timeout: 60_000, + }, () => { + const result = spawnSync("npx", ["tsx", "tools/e2e/workflow-inventory.mts", "--shell"], { + cwd: process.cwd(), + encoding: "utf-8", + timeout: 30_000, + killSignal: "SIGKILL", + }); + expect(result.signal).toBeNull(); + expect(result.status).toBe(0); + expect(result.stderr).toBe(""); + const currentBaseWorkflowInventory = result.stdout; + expect( + currentBaseWorkflowInventory + .trim() + .split("\n") + .map((line) => line.slice(0, line.indexOf("="))), + ).toEqual([ + "allowed_jobs", + "explicit_only_jobs_csv", + "free_standing_targets_csv", + "free_standing_target_jobs_csv", + ]); + for (const testId of [ + "docs-validation", + "gateway-drift-preflight", + "onboard-negative-paths", + "openshell-version-pin", + ]) { + expect(currentBaseWorkflowInventory).toContain(`${testId}:${testId}`); + } + expect(currentBaseWorkflowInventory).not.toContain("openshell-version-pin:shared-e2e"); + expect(currentBaseWorkflowInventory).not.toContain("ubuntu-repo-cli-smoke"); + }); + it("rejects malformed free-standing workflow metadata before matrix generation", { timeout: 60_000, }, () => { @@ -788,11 +826,11 @@ jobs: registryTargets: [], }); } - for (const [target, job] of inventory.targetToJob) { + for (const target of inventory.targetToJob.keys()) { expect(evaluateE2eWorkflowDispatchSelectors({ targets: target })).toMatchObject({ valid: true, liveTargetsRun: false, - selectedFreeStandingJobs: [job], + selectedFreeStandingJobs: [target], registryTargets: [], }); } @@ -1020,7 +1058,7 @@ jobs: "artifact upload path must include e2e-artifacts/live/${{ matrix.id }}/cloud-onboard-trace-timing-summary.json", "live must not invoke actions/upload-artifact directly", "live must use upload-e2e-artifacts exactly once", - "openshell-version-pin job must use the shared jobs selector condition", + "workflow missing shared E2E job", "network-policy job env must not include NVIDIA_INFERENCE_API_KEY", "network-policy step 'Install OpenShell' env must not include GITHUB_TOKEN", "double-onboard job env must not include DOCKERHUB_TOKEN", @@ -1033,7 +1071,7 @@ jobs: "report-to-pr job must wait for live", "report-to-pr step must pass jobs through JOBS env", "step 'Post E2E target results to PR' run script must check selector validation before echoing selectors", - "step 'Post E2E target results to PR' run script must omit rejected job selectors", + "step 'Post E2E target results to PR' run script must omit rejected test ID selectors", "step 'Post E2E target results to PR' run script must filter reported entries for selective dispatches", "step 'Post E2E target results to PR' run script must report missing requested jobs", "step 'Post E2E target results to PR' run script must count cancelled jobs", @@ -1384,7 +1422,7 @@ jobs: fs.writeFileSync( workflowPath, workflow.replace( - 'echo "::error::Invalid jobs input; use comma-separated job ids" >&2', + 'echo "::error::Invalid ${selector_name,,} input; use comma-separated ids" >&2', 'echo "::error::Invalid jobs input: ${JOBS}" >&2', ), ); @@ -1393,7 +1431,7 @@ jobs: const errors = validateE2eWorkflowBoundary(workflowPath); expect(errors).toEqual( expect.arrayContaining([ - "step 'Generate E2E target matrix' run script must include Invalid jobs input; use comma-separated job ids", + "step 'Generate E2E target matrix' run script must include Invalid ${selector_name,,} input; use comma-separated ids", "step 'Generate E2E target matrix' run script must not include Invalid jobs input: ${JOBS}", ]), ); diff --git a/test/e2e/support/jetson-workflow-boundary.test.ts b/test/e2e/support/jetson-workflow-boundary.test.ts index c91e3f60580..9775d0681a2 100644 --- a/test/e2e/support/jetson-workflow-boundary.test.ts +++ b/test/e2e/support/jetson-workflow-boundary.test.ts @@ -9,7 +9,6 @@ import { describe, expect, it } from "vitest"; import YAML from "yaml"; import { evaluateE2eWorkflowDispatchSelectors, - formatFreeStandingJobsInventoryForShell, readFreeStandingJobsInventory, validateE2eWorkflowBoundary, validateFreeStandingWorkflowInventory, @@ -50,9 +49,6 @@ describe("Jetson nvmap GPU E2E workflow boundary", () => { expect(validateE2eWorkflowBoundary()).toEqual([]); expect(inventory.allowedJobs).toContain("jetson-nvmap-gpu"); expect(inventory.explicitOnlyJobs).toContain("jetson-nvmap-gpu"); - expect(formatFreeStandingJobsInventoryForShell(inventory)).toContain( - "explicit_only_jobs_csv=openshell-gateway-auth-contract,mcp-bridge-dev,hermes-gpu-startup,sandbox-rlimits-connect,jetson-nvmap-gpu", - ); expect(inventory.targetToJob.get("jetson-nvmap-gpu")).toBe("jetson-nvmap-gpu"); expect(evaluateE2eWorkflowDispatchSelectors({}).selectedFreeStandingJobs).not.toContain( "jetson-nvmap-gpu", diff --git a/test/e2e/support/prepare-e2e-workflow-boundary.test.ts b/test/e2e/support/prepare-e2e-workflow-boundary.test.ts index 119c2bc966b..11bf988e5d9 100644 --- a/test/e2e/support/prepare-e2e-workflow-boundary.test.ts +++ b/test/e2e/support/prepare-e2e-workflow-boundary.test.ts @@ -23,7 +23,7 @@ type WorkflowStep = Record & { }; type Workflow = { - jobs: Record; + jobs: Record; steps?: WorkflowStep[] }>; }; describe("prepare-e2e workflow boundary", () => { @@ -83,10 +83,16 @@ describe("prepare-e2e workflow boundary", () => { run: "npm run build:cli", }); - const noBuildJob = workflow.jobs["docs-validation"]; + const noBuildJob = workflow.jobs["launchable-smoke"]; const noBuildPrepare = noBuildJob.steps!.find((step) => step.uses === PREPARE_E2E_ACTION)!; delete noBuildPrepare.with; + const sharedJob = workflow.jobs["shared-e2e"]; + const sharedPrepare = sharedJob.steps!.find((step) => step.uses === PREPARE_E2E_ACTION)!; + sharedPrepare.with = { "build-cli": "false" }; + sharedJob.env!.E2E_EXECUTION_PROFILE = "credential-free"; + sharedJob.env!.E2E_JOB = "1"; + const untrustedJob = workflow.jobs["inference-routing"]; const untrustedPrepare = untrustedJob.steps!.find((step) => step.uses === PREPARE_E2E_ACTION)!; untrustedPrepare.uses = "./.github/actions/prepare-e2e"; @@ -103,8 +109,12 @@ describe("prepare-e2e workflow boundary", () => { "sandbox-operations prepare-e2e must use the default CLI build", "sandbox-operations prepare-e2e invocation must not override its canonical contract", "sandbox-operations must not duplicate prepare-e2e step 'Build CLI'", - "docs-validation prepare-e2e must set build-cli to false", - "docs-validation prepare-e2e invocation must not override its canonical contract", + "launchable-smoke prepare-e2e must set build-cli to false", + "launchable-smoke prepare-e2e invocation must not override its canonical contract", + "shared-e2e must not declare E2E_EXECUTION_PROFILE", + "shared-e2e must not declare E2E_JOB", + "shared-e2e prepare-e2e must use the default CLI build", + "shared-e2e prepare-e2e invocation must not override its canonical contract", "inference-routing must not load prepare-e2e from the target checkout", "inference-routing must use prepare-e2e exactly once", "network-policy must check out the repository before prepare-e2e", diff --git a/test/e2e/support/shared-e2e-workflow-boundary.test.ts b/test/e2e/support/shared-e2e-workflow-boundary.test.ts new file mode 100644 index 00000000000..1a0b2132663 --- /dev/null +++ b/test/e2e/support/shared-e2e-workflow-boundary.test.ts @@ -0,0 +1,107 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; +import YAML from "yaml"; + +import { + discoverCredentialFreeTests, + stripCredentialFreeTestDeclarations, +} from "../../../tools/e2e/credential-free-tests.mts"; +import { + evaluateE2eWorkflowDispatchSelectors, + validateE2eWorkflowBoundary, +} from "../../../tools/e2e/workflow-boundary.mts"; +import { readWorkflow } from "../../helpers/e2e-workflow-contract"; + +type Workflow = { + jobs: Record< + string, + { + env?: Record; + needs?: string[]; + steps?: Array<{ name?: string; run?: string }>; + } + >; +}; + +function validateMutatedWorkflow(mutator: (workflow: Workflow) => void): string[] { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-shared-e2e-workflow-")); + const workflowPath = path.join(directory, "workflow.yaml"); + const workflow = readWorkflow() as Workflow; + try { + mutator(workflow); + fs.writeFileSync(workflowPath, YAML.stringify(workflow)); + return validateE2eWorkflowBoundary(workflowPath); + } finally { + fs.rmSync(directory, { force: true, recursive: true }); + } +} + +describe("shared E2E workflow boundary", () => { + it("keeps every tagged credential-free test visible to Vitest discovery", () => { + const declaredFiles = fs + .globSync(["**/*.test.js", "**/*.test.ts"], { + cwd: process.cwd(), + exclude: ["**/dist/**", "**/node_modules/**"], + }) + .filter((file) => { + const source = fs.readFileSync(path.join(process.cwd(), file), "utf8"); + return stripCredentialFreeTestDeclarations(source) !== source; + }) + .sort(); + + expect( + discoverCredentialFreeTests() + .map(({ file }) => file) + .sort(), + ).toEqual(declaredFiles); + }); + + it("keeps discovered tests default-enabled and selectively dispatchable", () => { + expect(validateE2eWorkflowBoundary()).toEqual([]); + + for (const { id } of discoverCredentialFreeTests()) { + for (const selector of [{ targets: id }, { jobs: id }]) { + expect(evaluateE2eWorkflowDispatchSelectors(selector)).toMatchObject({ + valid: true, + liveTargetsRun: false, + selectedFreeStandingJobs: [id], + }); + } + expect(evaluateE2eWorkflowDispatchSelectors({}).selectedFreeStandingJobs).toContain(id); + } + }); + + it("ratchets shared setup, tagged test execution, and aggregation", () => { + const errors = validateMutatedWorkflow((workflow) => { + const job = workflow.jobs["shared-e2e"]; + job.env!.CHECK_DOC_LINKS_REMOTE = "1"; + job.steps!.find((step) => step.name === "Run tagged credential-free test")!.run = + "echo skipped"; + workflow.jobs["report-to-pr"].needs = workflow.jobs["report-to-pr"].needs!.filter( + (name) => name !== "shared-e2e", + ); + }); + + expect(errors).toEqual( + expect.arrayContaining([ + "shared E2E job must set CHECK_DOC_LINKS_REMOTE to 0", + 'step \'Run tagged credential-free test\' run script must include npx vitest run --project "${TEST_PROJECT}" "${TEST_FILE}"', + "report-to-pr job must wait for shared-e2e", + ]), + ); + }); + + it("reports a missing shared job as a contract error", () => { + const errors = validateMutatedWorkflow((workflow) => { + delete workflow.jobs["shared-e2e"]; + }); + + expect(errors).toContain("workflow missing shared E2E job"); + }); +}); diff --git a/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts b/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts index 1e2e38d79e0..2c4b8f82442 100644 --- a/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts +++ b/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts @@ -106,7 +106,7 @@ describe("upload-e2e-artifacts workflow boundary", () => { const workflow = mutableWorkflow(); uploadStep(workflow.jobs["inference-routing"]).uses = LOCAL_UPLOAD_ACTION; uploadStep(workflow.jobs["network-policy"]).uses = DIRECT_UPLOAD_ACTION; - uploadStep(workflow.jobs["docs-validation"]).uses = + uploadStep(workflow.jobs["shared-e2e"]).uses = "NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@main"; expect(validateUploadE2eArtifactsInvocations(workflow)).toEqual( @@ -115,15 +115,15 @@ describe("upload-e2e-artifacts workflow boundary", () => { "inference-routing must use upload-e2e-artifacts exactly once", "network-policy must not invoke actions/upload-artifact directly", "network-policy must use upload-e2e-artifacts exactly once", - "docs-validation must use the reviewed immutable upload-e2e-artifacts reference", - "docs-validation must use upload-e2e-artifacts exactly once", + "shared-e2e must use the reviewed immutable upload-e2e-artifacts reference", + "shared-e2e must use upload-e2e-artifacts exactly once", ]), ); }); it("rejects missing and duplicate shared upload invocations", () => { const workflow = mutableWorkflow(); - const missingJob = workflow.jobs["openshell-version-pin"]; + const missingJob = workflow.jobs["shared-e2e"]; missingJob.steps = missingJob.steps!.filter( (step) => step.uses !== UPLOAD_E2E_ARTIFACTS_ACTION, ); @@ -132,7 +132,7 @@ describe("upload-e2e-artifacts workflow boundary", () => { expect(validateUploadE2eArtifactsInvocations(workflow)).toEqual( expect.arrayContaining([ - "openshell-version-pin must use upload-e2e-artifacts exactly once", + "shared-e2e must use upload-e2e-artifacts exactly once", "cloud-inference must use upload-e2e-artifacts exactly once", ]), ); @@ -147,7 +147,10 @@ describe("upload-e2e-artifacts workflow boundary", () => { uploadStep(workflow.jobs["hermes-slack"]).with!.path = "e2e-artifacts/live/hermes-slack/"; uploadStep(workflow.jobs["gpu-e2e"]).if = "success()"; uploadStep(workflow.jobs["mcp-bridge"]).if = "always()"; - uploadStep(workflow.jobs["docs-validation"]).env = { UNEXPECTED: "1" }; + uploadStep(workflow.jobs["shared-e2e"]).env = { UNEXPECTED: "1" }; + workflow.jobs["shared-e2e"].env!.E2E_EXECUTION_PROFILE = "credential-free"; + workflow.jobs["shared-e2e"].env!.E2E_JOB = "1"; + workflow.jobs["shared-e2e"].env!.E2E_TARGET_ID = "shared-e2e"; const orderedJob = workflow.jobs["network-policy"]; const orderedUpload = uploadStep(orderedJob); orderedJob.steps!.splice(orderedJob.steps!.indexOf(orderedUpload), 1); @@ -161,7 +164,10 @@ describe("upload-e2e-artifacts workflow boundary", () => { "hermes-slack upload-e2e-artifacts must preserve its explicit name/path contract", "gpu-e2e upload-e2e-artifacts invocation must run with always()", "mcp-bridge upload-e2e-artifacts invocation must remain gated by its reviewed pre-upload checks", - "docs-validation upload-e2e-artifacts invocation must not override its contract", + "shared-e2e must not declare E2E_EXECUTION_PROFILE", + "shared-e2e must not declare E2E_JOB", + "shared-e2e upload-e2e-artifacts invocation must not override its contract", + "shared-e2e default upload caller E2E_TARGET_ID must be '${{ matrix.id }}'", "network-policy upload-e2e-artifacts invocation must follow artifact producers and precede only Docker auth cleanup", ]), ); @@ -177,8 +183,8 @@ describe("upload-e2e-artifacts workflow boundary", () => { expect(validateUploadE2eArtifactsInvocations(workflow)).toEqual( expect.arrayContaining([ - "upload-e2e-artifacts must cover exactly 75 live and E2E_JOB execution jobs", - "upload-e2e-artifacts must keep exactly 63 default callers", + "upload-e2e-artifacts must cover exactly 72 live, E2E_JOB, and shared E2E jobs", + "upload-e2e-artifacts must keep exactly 60 default callers", ]), ); }); diff --git a/test/e2e/support/workflow-plan.test.ts b/test/e2e/support/workflow-plan.test.ts new file mode 100644 index 00000000000..caa83f96b1d --- /dev/null +++ b/test/e2e/support/workflow-plan.test.ts @@ -0,0 +1,125 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { discoverCredentialFreeTests } from "../../../tools/e2e/credential-free-tests.mts"; +import { readFreeStandingJobsInventory } from "../../../tools/e2e/workflow-boundary.mts"; +import { buildE2eWorkflowPlan, runE2eWorkflowPlanCli } from "../../../tools/e2e/workflow-plan.mts"; +import { REPO_ROOT } from "../fixtures/paths.ts"; +import { buildLiveTargetMatrix } from "../registry/run.ts"; + +const PLANNER_CLI = path.join(REPO_ROOT, "tools", "e2e", "workflow-plan.mts"); +const TSX = path.join(REPO_ROOT, "node_modules", ".bin", "tsx"); + +function firstId(rows: readonly T[], label: string): string { + expect(rows, `expected at least one ${label}`).not.toHaveLength(0); + return rows[0]!.id; +} + +describe("E2E workflow plan", () => { + it("defaults to every supported registry target and tagged credential-free test", () => { + const plan = buildE2eWorkflowPlan(); + + expect(plan).toEqual({ + matrix: buildLiveTargetMatrix(), + testMatrix: discoverCredentialFreeTests(), + hermesSelected: true, + explicitOnlyJobs: readFreeStandingJobsInventory().explicitOnlyJobs, + }); + }); + + it("validates jobs and selects only matching credential-free tests", () => { + const testId = firstId(discoverCredentialFreeTests(), "credential-free test"); + const plan = buildE2eWorkflowPlan({ jobs: `${testId},hermes-e2e` }); + + expect(plan.matrix).toEqual([]); + expect(plan.testMatrix.map((row) => row.id)).toEqual([testId]); + expect(plan.hermesSelected).toBe(true); + }); + + it("routes a registry target into the live matrix", () => { + const registryId = firstId(buildLiveTargetMatrix(), "supported registry target"); + const plan = buildE2eWorkflowPlan({ targets: registryId }); + + expect(plan.matrix.map((row) => row.id)).toEqual([registryId]); + expect(plan.testMatrix).toEqual([]); + expect(plan.hermesSelected).toBe(false); + }); + + it("partitions mixed registry and tagged test targets", () => { + const registryId = firstId(buildLiveTargetMatrix(), "supported registry target"); + const testId = firstId(discoverCredentialFreeTests(), "credential-free test"); + const plan = buildE2eWorkflowPlan({ targets: `${registryId},${testId}` }); + + expect(plan.matrix.map((row) => row.id)).toEqual([registryId]); + expect(plan.testMatrix.map((row) => row.id)).toEqual([testId]); + }); + + it("rejects an unknown job", () => { + expect(() => buildE2eWorkflowPlan({ jobs: "definitely-unknown-e2e-job" })).toThrow( + "Unknown E2E test ID: definitely-unknown-e2e-job", + ); + }); + + it("rejects an unknown target that belongs to neither inventory nor registry", () => { + expect(() => buildE2eWorkflowPlan({ targets: "definitely-unknown-e2e-target" })).toThrow( + "Unknown target 'definitely-unknown-e2e-target'", + ); + }); + + it.each([ + ["jobs", "alpha,,beta"], + ["jobs", "alpha beta"], + ["targets", "../escape"], + ["targets", "alpha,"], + ] as const)("rejects invalid %s input %s", (kind, value) => { + expect(() => buildE2eWorkflowPlan({ [kind]: value })).toThrow(`Invalid ${kind} input`); + }); + + it("rejects simultaneous jobs and targets", () => { + expect(() => buildE2eWorkflowPlan({ jobs: "hermes-e2e", targets: "hermes-e2e" })).toThrow( + "Use either jobs or targets, not both", + ); + }); + + it("emits one compact JSON line with the deterministic workflow-output schema", () => { + let output = ""; + const write = process.stdout.write; + process.stdout.write = ((chunk: string | Uint8Array) => { + output += chunk.toString(); + return true; + }) as typeof process.stdout.write; + try { + runE2eWorkflowPlanCli(["--jobs", "hermes-e2e"]); + } finally { + process.stdout.write = write; + } + + expect(output.endsWith("\n")).toBe(true); + expect(output.trim().split("\n")).toHaveLength(1); + const parsed = JSON.parse(output); + expect(Object.keys(parsed)).toEqual([ + "matrix", + "testMatrix", + "hermesSelected", + "explicitOnlyJobs", + ]); + expect(output).toBe(`${JSON.stringify(parsed)}\n`); + }); + + it("reports CLI failures as workflow annotations", () => { + const result = spawnSync( + TSX, + [PLANNER_CLI, "--jobs", "hermes-e2e", "--targets", "hermes-e2e"], + { cwd: REPO_ROOT, encoding: "utf8", timeout: 30_000 }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(""); + expect(result.stderr).toBe("::error::Use either jobs or targets, not both\n"); + }); +}); diff --git a/test/gateway-drift-preflight.test.ts b/test/gateway-drift-preflight.test.ts index a19294f6c53..df2ab1f4506 100644 --- a/test/gateway-drift-preflight.test.ts +++ b/test/gateway-drift-preflight.test.ts @@ -1,5 +1,6 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +// @module-tag e2e/credential-free import { spawn, spawnSync } from "node:child_process"; import fs from "node:fs"; diff --git a/test/pr-e2e-gate.test.ts b/test/pr-e2e-gate.test.ts index f8758f21a25..39dea292c39 100644 --- a/test/pr-e2e-gate.test.ts +++ b/test/pr-e2e-gate.test.ts @@ -322,6 +322,9 @@ describe("PR E2E controller", () => { "onboard-repair": ["default"], "onboard-resume": ["default"], }); + expect(expectedSignalShards(["docs-validation"])).toEqual({ + "docs-validation": ["default"], + }); const broadPlan = buildRiskPlan({ headSha: HEAD_SHA, changedFiles: BROAD_FILES }); const broadShards = expectedSignalShards(riskPlanRequiredJobIds(broadPlan)); expect(Object.keys(broadShards)).toHaveLength(13); diff --git a/test/pr-risk-plan.test.ts b/test/pr-risk-plan.test.ts index cd47c8573d3..c8794b81d00 100644 --- a/test/pr-risk-plan.test.ts +++ b/test/pr-risk-plan.test.ts @@ -181,7 +181,7 @@ describe("deterministic PR risk plan", () => { expect(result.suggestedTests.join("\n")).toContain("`src/lib/state/registry.ts`"); }); - it("keeps every catalog job wired into the canonical E2E workflow", () => { + it("keeps every discovered test selector wired into the canonical E2E workflow", () => { const allowedJobs = new Set(readFreeStandingJobsInventory().allowedJobs); const configuredJobs = new Set(RISK_RULES.flatMap((rule) => rule.requiredJobs)); diff --git a/tools/e2e-advisor/targets.mts b/tools/e2e-advisor/targets.mts index 008ceb3b6f3..487172020b3 100755 --- a/tools/e2e-advisor/targets.mts +++ b/tools/e2e-advisor/targets.mts @@ -41,6 +41,12 @@ import { type RunAdvisorResult, runReadOnlyAdvisor, } from "../advisors/session.mts"; +import { + CREDENTIAL_FREE_TEST_TAG, + credentialFreeTestProjectForFile, + credentialFreeTestRowFromModule, + discoverCredentialFreeTests, +} from "../e2e/credential-free-tests.mts"; import { readFreeStandingJobsInventory } from "../e2e/workflow-boundary.mts"; const root = process.cwd(); @@ -295,8 +301,8 @@ export function buildSystemPrompt(_schema?: AdvisorSchema): string { "- Required (targeted): fixture, live test, manifest, runtime-support, or target changes that affect a specific subset. Recommend the smallest set of live-supported typed target IDs that exercises the changed surface.", "- Onboarding resume rule: changes to src/lib/onboard/machine live slice orchestration, resume state handling, resume repair policy, session bootstrap, or onboarding state transitions MUST require `onboard-resume`. Also require `onboard-repair` when the change can affect repair/backstop execution from persisted sessions. Do not make repair optional for these state-machine resume paths.", "- Deterministic risk plan: required jobs are a trusted floor. You may add adjacent targets, but never remove or downgrade a listed job.", - "- Required (free-standing job): if a PR wires or changes a discrete live E2E job in `.github/workflows/e2e.yaml` for a specific `test/e2e/live/*.test.ts`, prefer that job over `e2e-all`. Use selectorType=`job`, id=``, workflow=`e2e.yaml`, and dispatchCommand exactly `gh workflow run e2e.yaml --ref --field jobs=`.", - "- Missing wiring: if a PR adds or changes a free-standing live E2E file under `test/e2e/live/*.test.ts` but that file is not referenced by `.github/workflows/e2e.yaml` and is not `registry-targets.test.ts`, do not recommend the fan-out as proof. Return no required/optional recommendations and set `noTargetE2eReason` to say the test must be wired into `e2e.yaml` before it can be dispatched.", + "- Required (E2E test): if a PR changes a test wired by a discrete workflow job or tagged as credential-free, prefer its test ID over `e2e-all`. Use selectorType=`job`, id=``, workflow=`e2e.yaml`, and dispatchCommand exactly `gh workflow run e2e.yaml --ref --field jobs=`.", + "- Missing wiring: if a PR adds or changes an E2E file under `test/e2e/live/*.test.ts` but that file is neither tagged as credential-free nor referenced by `.github/workflows/e2e.yaml`, and is not `registry-targets.test.ts`, do not recommend the fan-out as proof. Return no required/optional recommendations and set `noTargetE2eReason` to say the test must be wired before it can be dispatched.", "- Optional: adjacent targets that exercise the same suite on a different platform/onboarding (e.g. macOS, WSL, GPU) but are not the primary target. Special-runner targets (`gpu-`, `macos-`, `wsl-`, `brev-`) should usually be optional unless they are the only path that exercises the change.", "- None: docs-only, comment-only, tests-only outside `test/e2e/`, or changes that cannot affect E2E target behavior. Set `noTargetE2eReason` and return empty `required`/`optional` arrays.", "", @@ -306,7 +312,7 @@ export function buildSystemPrompt(_schema?: AdvisorSchema): string { "- Each `dispatchCommand` for a single-target recommendation MUST be exactly: `gh workflow run e2e.yaml --ref --field targets=`.", "- Each `dispatchCommand` for a free-standing job recommendation MUST be exactly: `gh workflow run e2e.yaml --ref --field jobs=`.", "- For the fan-out, use exactly: `gh workflow run e2e.yaml --ref ` and set `id`/`workflow`/`selectorType` to `e2e-all`/`e2e.yaml`/`all`.", - "- The normalizer validates targeted IDs against the trusted advisor checkout's registry/runtime-support modules, not PR-local TypeScript. If a PR adds or newly wires a typed registry target that is not live-supported on trusted `main` yet, recommend the `e2e-all` fan-out rather than a targeted dispatch. This fallback does not apply to free-standing live test jobs.", + "- The normalizer validates targeted IDs against the trusted advisor checkout's registry/runtime-support modules, not PR-local TypeScript. If a PR adds or newly wires a typed registry target that is not live-supported on trusted `main` yet, recommend the `e2e-all` fan-out rather than a targeted dispatch. This fallback does not apply to tests wired by a discrete job or a literal credential-free tag; the normalizer reads tag declarations as inert text.", "- A `suiteFilter` may be set on a recommendation as analytical metadata explaining why the target was selected. It must NOT leak into the dispatch command.", "- `relevantChangedFiles` must be the subset of `changedFiles` under `test/e2e/`, `.github/workflows/e2e.yaml`, or other directly target-relevant paths.", "", @@ -398,17 +404,25 @@ Call the real \`${contextToolNames}\` context tools before answering. Treat requ export function normalizeE2eTargetAdvisorResult( result: unknown, metadata: AdvisorMetadata, - options: { e2eWorkflowText?: string; riskPlan?: RiskPlan } = {}, + options: { + changedFileSources?: Readonly>; + e2eWorkflowText?: string; + riskPlan?: RiskPlan; + } = {}, ): E2eTargetAdvisorResult { if (!result || typeof result !== "object" || Array.isArray(result)) { throw new Error("Target advisor returned a non-object result"); } const object = result as Record; - const context = buildE2eTargetNormalizationContext(options.e2eWorkflowText); + const context = buildE2eTargetNormalizationContext( + options.e2eWorkflowText, + metadata.changedFiles, + options.changedFileSources, + ); const unwiredFreeStandingLiveTests = findUnwiredFreeStandingLiveTests( metadata.changedFiles, - context.e2eWorkflowText, + context, ); const suppressFanout = shouldSuppressFanoutForUnwiredLiveTests( metadata.changedFiles, @@ -493,10 +507,18 @@ function readE2eWorkflowText(): string | undefined { function buildE2eTargetNormalizationContext( e2eWorkflowText = readE2eWorkflowText(), + changedFiles: readonly string[] = [], + changedFileSources?: Readonly>, ): E2eTargetNormalizationContext { const freeStandingJobs = extractFreeStandingE2eJobs(e2eWorkflowText ?? ""); const allowedJobIds = new Set(readFreeStandingJobsInventory().allowedJobs); const liveTestToJobs = new Map(); + const changedCredentialFreeTestProjects = new Map( + changedFiles.flatMap((file) => { + const project = credentialFreeTestProjectForFile(file); + return project ? [[file, project] as const] : []; + }), + ); for (const job of freeStandingJobs) { for (const file of job.liveTestFiles) { const jobs = liveTestToJobs.get(file) ?? []; @@ -504,6 +526,39 @@ function buildE2eTargetNormalizationContext( liveTestToJobs.set(file, jobs); } } + for (const row of discoverCredentialFreeTests()) { + if (changedCredentialFreeTestProjects.has(row.file)) { + allowedJobIds.delete(row.id); + continue; + } + const jobs = liveTestToJobs.get(row.file) ?? []; + jobs.push(row.id); + liveTestToJobs.set(row.file, jobs); + } + for (const [file, project] of changedCredentialFreeTestProjects) { + let source: string | undefined; + if (changedFileSources && Object.hasOwn(changedFileSources, file)) { + source = changedFileSources[file] ?? undefined; + if (source === undefined) continue; + } else { + try { + source = fs.readFileSync(path.join(root, file), "utf8"); + } catch { + continue; + } + } + if (!source.includes(`@module-tag ${CREDENTIAL_FREE_TEST_TAG}`)) continue; + try { + const row = credentialFreeTestRowFromModule({ file, project, source }); + const jobs = liveTestToJobs.get(row.file) ?? []; + if (!jobs.includes(row.id)) jobs.push(row.id); + liveTestToJobs.set(row.file, jobs); + allowedJobIds.add(row.id); + } catch { + // Invalid or ambiguous credential-free tags remain unwired so the + // normalizer cannot recommend a selector the workflow would reject. + } + } return { e2eWorkflowText, freeStandingJobs, allowedJobIds, liveTestToJobs }; } @@ -536,13 +591,14 @@ export function extractFreeStandingE2eJobs(workflowText: string): E2eWorkflowJob function findUnwiredFreeStandingLiveTests( changedFiles: string[], - e2eWorkflowText = readE2eWorkflowText(), + context: E2eTargetNormalizationContext, ): string[] { return changedFiles.filter( (file) => FREE_STANDING_LIVE_TEST_PATTERN.test(file) && file !== REGISTRY_LIVE_ENTRYPOINT && - !(e2eWorkflowText ?? "").includes(file), + !context.liveTestToJobs.has(file) && + !(context.e2eWorkflowText ?? "").includes(file), ); } @@ -563,16 +619,14 @@ function isE2eTargetRelevantFile(file: string): boolean { function missingFreeStandingLiveWiringReason(files: string[]): string { const fileList = files.map((file) => `\`${file}\``).join(", "); - return `New free-standing live E2E test ${fileList} is not wired into \`${E2E_WORKFLOW_PATH}\`, so the E2E target workflow cannot dispatch it yet. Add a discrete job or register it as a typed live target before treating the PR as E2E-runnable.`; + return `New E2E test ${fileList} is not wired into \`${E2E_WORKFLOW_PATH}\`, so the E2E workflow cannot dispatch it yet. Add the credential-free tag, a discrete job, or a typed live target before treating the PR as E2E-runnable.`; } function deterministicFreeStandingJobRecommendations( changedFiles: string[], context: E2eTargetNormalizationContext, ): E2eTargetRecommendation[] { - const liveFiles = changedFiles.filter( - (file) => FREE_STANDING_LIVE_FILE_PATTERN.test(file) && file !== REGISTRY_LIVE_ENTRYPOINT, - ); + const liveFiles = changedFiles.filter((file) => context.liveTestToJobs.has(file)); const output: E2eTargetRecommendation[] = []; const seen = new Set(); for (const file of liveFiles) { @@ -584,7 +638,7 @@ function deterministicFreeStandingJobRecommendations( workflow: E2E_WORKFLOW, selectorType: "job", required: true, - reason: `Focused free-standing E2E job wired for changed live test \`${file}\`.`, + reason: `Focused free-standing E2E selector wired for changed test \`${file}\`.`, dispatchCommand: canonicalDispatchCommand(E2E_WORKFLOW, job, "job"), }); } diff --git a/tools/e2e/credential-free-tests.mts b/tools/e2e/credential-free-tests.mts new file mode 100644 index 00000000000..db102a79780 --- /dev/null +++ b/tools/e2e/credential-free-tests.mts @@ -0,0 +1,310 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import ts from "typescript"; + +export const CREDENTIAL_FREE_TEST_TAG = "e2e/credential-free"; +export const SHARED_E2E_JOB_ID = "shared-e2e"; + +export type CredentialFreeTestProject = "e2e-live" | "integration"; + +export type CredentialFreeTestMatrixRow = { + id: string; + file: string; + project: CredentialFreeTestProject; +}; + +export type CredentialFreeTestModule = { + file: string; + project: CredentialFreeTestProject; + source: string; +}; + +type VitestFile = { + file: string; + projectName: string; +}; + +const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); +const MODULE_TAG_BODY_PATTERN = /^@module-tag[\t ]+([A-Za-z0-9/_-]+)$/u; +const SAFE_ID_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/; +const SAFE_PATH_SEGMENT_PATTERN = /^[A-Za-z0-9._-]+$/; +const E2E_LIVE_CREDENTIAL_FREE_TEST_PATTERN = + /^test\/e2e\/live\/(?:[A-Za-z0-9._-]+\/)*[A-Za-z0-9._-]+\.test\.ts$/; +const INTEGRATION_CREDENTIAL_FREE_TEST_PATTERN = + /^test\/(?!e2e\/)(?:[A-Za-z0-9._-]+\/)*[A-Za-z0-9._-]+\.test\.(?:js|ts)$/; +const SUPPORTED_PROJECTS = new Set(["e2e-live", "integration"]); + +export function credentialFreeTestProjectForFile( + file: string, +): CredentialFreeTestProject | undefined { + if (E2E_LIVE_CREDENTIAL_FREE_TEST_PATTERN.test(file)) return "e2e-live"; + if (INTEGRATION_CREDENTIAL_FREE_TEST_PATTERN.test(file)) return "integration"; + return undefined; +} + +function isInside(parent: string, child: string): boolean { + const relative = path.relative(parent, child); + return relative !== "" && !relative.startsWith(`..${path.sep}`) && relative !== ".."; +} + +function normalizeVitestFile( + repoRoot: string, + candidate: VitestFile, +): { + absoluteFile: string; + file: string; + project: CredentialFreeTestProject; +} { + if (!SUPPORTED_PROJECTS.has(candidate.projectName as CredentialFreeTestProject)) { + throw new Error(`Unsupported Vitest project '${candidate.projectName}' for ${candidate.file}`); + } + + const absoluteRoot = fs.realpathSync(repoRoot); + const absoluteFile = fs.realpathSync(candidate.file); + if (!isInside(absoluteRoot, absoluteFile)) { + throw new Error(`Vitest returned a test file outside the repository: ${candidate.file}`); + } + + return { + absoluteFile, + file: path.relative(absoluteRoot, absoluteFile).split(path.sep).join("/"), + project: candidate.projectName as CredentialFreeTestProject, + }; +} + +function validateTestFile(file: string, project: CredentialFreeTestProject): void { + if ( + path.posix.isAbsolute(file) || + path.posix.normalize(file) !== file || + file.includes("\\") || + !file.startsWith("test/") || + !file.split("/").every((segment) => SAFE_PATH_SEGMENT_PATTERN.test(segment)) || + !/\.test\.(?:js|ts)$/.test(file) + ) { + throw new Error(`Credential-free test path must be a safe repo-relative test file: ${file}`); + } + + const inferredProject = credentialFreeTestProjectForFile(file); + if (project === "e2e-live" && inferredProject !== "e2e-live") { + throw new Error(`e2e-live credential-free test must live under test/e2e/live/: ${file}`); + } + if (project === "integration" && inferredProject !== "integration") { + throw new Error(`integration credential-free test must not live under test/e2e/: ${file}`); + } +} + +type ModuleTagDeclaration = { + tag: string; + start: number; + end: number; +}; + +function standaloneModuleTag(comment: string): string | undefined { + const body = comment.startsWith("//") + ? comment.slice(2).trim() + : comment + .slice(2, -2) + .split(/\r?\n/u) + .map((line) => line.replace(/^[\t ]*\**[\t ]?/u, "").trim()) + .filter(Boolean) + .join("\n"); + return MODULE_TAG_BODY_PATTERN.exec(body)?.[1]; +} + +function declarationLineRange( + source: string, + tokenStart: number, + tokenEnd: number, +): Pick | undefined { + const lineStart = source.lastIndexOf("\n", tokenStart - 1) + 1; + const nextNewline = source.indexOf("\n", tokenEnd); + const lineEnd = nextNewline < 0 ? source.length : nextNewline; + if ( + !/^[\t ]*$/u.test(source.slice(lineStart, tokenStart)) || + !/^[\t \r]*$/u.test(source.slice(tokenEnd, lineEnd)) + ) { + return undefined; + } + return { start: lineStart, end: nextNewline < 0 ? source.length : nextNewline + 1 }; +} + +function moduleTagDeclarations(source: string): ModuleTagDeclaration[] { + const scanner = ts.createScanner( + ts.ScriptTarget.Latest, + false, + ts.LanguageVariant.Standard, + source, + ); + const declarations: ModuleTagDeclaration[] = []; + for (let token = scanner.scan(); token !== ts.SyntaxKind.EndOfFileToken; token = scanner.scan()) { + if ( + token !== ts.SyntaxKind.SingleLineCommentTrivia && + token !== ts.SyntaxKind.MultiLineCommentTrivia + ) { + continue; + } + const tag = standaloneModuleTag(scanner.getTokenText()); + const range = declarationLineRange(source, scanner.getTokenPos(), scanner.getTextPos()); + if (tag && range) declarations.push({ tag, ...range }); + } + return declarations; +} + +function credentialFreeTestTags(source: string, file?: string): string[] { + const tags = moduleTagDeclarations(source).map(({ tag }) => tag); + const unknownTag = tags.find((tag) => tag.startsWith("e2e/") && tag !== CREDENTIAL_FREE_TEST_TAG); + if (unknownTag) { + throw new Error(`Unknown E2E test tag '${unknownTag}'${file ? ` in ${file}` : ""}`); + } + return tags.filter((tag) => tag === CREDENTIAL_FREE_TEST_TAG); +} + +function stripDeclarations(source: string, declarations: readonly ModuleTagDeclaration[]): string { + let cursor = 0; + let stripped = ""; + for (const declaration of declarations) { + stripped += source.slice(cursor, declaration.start); + cursor = declaration.end; + } + return stripped + source.slice(cursor); +} + +export function stripCredentialFreeTestDeclarations(source: string): string { + return stripDeclarations( + source, + moduleTagDeclarations(source).filter(({ tag }) => tag === CREDENTIAL_FREE_TEST_TAG), + ); +} + +export function credentialFreeTestRowFromModule( + module: CredentialFreeTestModule, +): CredentialFreeTestMatrixRow { + validateTestFile(module.file, module.project); + const tags = credentialFreeTestTags(module.source, module.file); + if (tags.length !== 1) { + throw new Error( + `${module.file} must declare exactly one ${CREDENTIAL_FREE_TEST_TAG} module tag; found ${tags.length}`, + ); + } + + const id = path.posix.basename(module.file).replace(/\.test\.(?:js|ts)$/, ""); + if (!SAFE_ID_PATTERN.test(id)) { + throw new Error(`Credential-free test filename must derive a safe id: ${module.file}`); + } + + return { id, file: module.file, project: module.project }; +} + +export function discoverCredentialFreeTestRows( + modules: readonly CredentialFreeTestModule[], +): CredentialFreeTestMatrixRow[] { + const rows = modules.map(credentialFreeTestRowFromModule).sort((left, right) => { + return ( + left.id.localeCompare(right.id) || + left.file.localeCompare(right.file) || + left.project.localeCompare(right.project) + ); + }); + const seen = new Map(); + for (const row of rows) { + const previous = seen.get(row.id); + if (previous) { + throw new Error(`Duplicate credential-free test id '${row.id}': ${previous}, ${row.file}`); + } + seen.set(row.id, row.file); + } + return rows; +} + +export function listVitestCredentialFreeTestModules( + repoRoot = REPO_ROOT, +): CredentialFreeTestModule[] { + const vitestEntrypoint = path.join(repoRoot, "node_modules", "vitest", "vitest.mjs"); + const result = spawnSync( + process.execPath, + [ + vitestEntrypoint, + "list", + "--filesOnly", + "--json", + "--project", + "e2e-live", + "--project", + "integration", + ], + { + cwd: repoRoot, + encoding: "utf8", + env: { ...process.env, NEMOCLAW_RUN_LIVE_E2E: "1" }, + maxBuffer: 10 * 1024 * 1024, + timeout: 30_000, + }, + ); + if (result.error) { + throw new Error(`Failed to list Vitest test files: ${result.error.message}`); + } + if (result.status !== 0) { + throw new Error( + `Failed to list Vitest test files (exit ${result.status ?? "unknown"}): ${result.stderr || result.stdout}`, + ); + } + + let candidates: unknown; + try { + candidates = JSON.parse(result.stdout); + } catch (error) { + throw new Error( + `Vitest test-file list was not valid JSON: ${error instanceof Error ? error.message : String(error)}`, + ); + } + if (!Array.isArray(candidates)) { + throw new Error("Vitest test-file list must be a JSON array"); + } + + return candidates.flatMap((candidate): CredentialFreeTestModule[] => { + if ( + !candidate || + typeof candidate !== "object" || + typeof (candidate as VitestFile).file !== "string" || + typeof (candidate as VitestFile).projectName !== "string" + ) { + throw new Error("Vitest test-file list contains an invalid entry"); + } + const normalized = normalizeVitestFile(repoRoot, candidate as VitestFile); + const source = fs.readFileSync(normalized.absoluteFile, "utf8"); + if (!credentialFreeTestTags(source, normalized.file).length) return []; + return [{ file: normalized.file, project: normalized.project, source }]; + }); +} + +const discoveryCache = new Map(); + +export function discoverCredentialFreeTests(repoRoot = REPO_ROOT): CredentialFreeTestMatrixRow[] { + const resolvedRoot = fs.realpathSync(repoRoot); + const cached = discoveryCache.get(resolvedRoot); + if (cached) return cached.map((row) => ({ ...row })); + const rows = discoverCredentialFreeTestRows(listVitestCredentialFreeTestModules(resolvedRoot)); + discoveryCache.set(resolvedRoot, rows); + return rows.map((row) => ({ ...row })); +} + +const invokedFile = process.argv[1] ? path.resolve(process.argv[1]) : ""; +if (invokedFile === fileURLToPath(import.meta.url)) { + try { + if (process.argv.length > 2) { + throw new Error( + "Credential-free test discovery does not accept selectors; use workflow-plan.mts", + ); + } + process.stdout.write(`${JSON.stringify(discoverCredentialFreeTests())}\n`); + } catch (error) { + console.error(`::error::${error instanceof Error ? error.message : String(error)}`); + process.exitCode = 1; + } +} diff --git a/tools/e2e/docs-validation-workflow-boundary.mts b/tools/e2e/docs-validation-workflow-boundary.mts deleted file mode 100644 index cb2fc501ce7..00000000000 --- a/tools/e2e/docs-validation-workflow-boundary.mts +++ /dev/null @@ -1,128 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { readFileSync } from "node:fs"; -import { dirname, join } from "node:path"; -import { fileURLToPath } from "node:url"; -import YAML from "yaml"; -import { PREPARE_E2E_ACTION, PREPARE_E2E_STEP } from "./prepare-e2e-workflow-boundary.mts"; - -const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); -const DEFAULT_WORKFLOW_PATH = join(REPO_ROOT, ".github", "workflows", "e2e.yaml"); -const JOB_NAME = "docs-validation"; - -type WorkflowStep = { - if?: string; - name?: string; - run?: string; - uses?: string; - with?: Record; -}; - -type WorkflowJob = { - env?: Record; - if?: string; - needs?: string[] | string; - steps?: WorkflowStep[]; - "runs-on"?: string; - "timeout-minutes"?: number; -}; - -export type DocsValidationWorkflow = { - jobs: Record; -}; - -export function readDocsValidationWorkflow( - workflowPath = DEFAULT_WORKFLOW_PATH, -): DocsValidationWorkflow { - const parsed: unknown = YAML.parse(readFileSync(workflowPath, "utf8")); - const jobs = - parsed && typeof parsed === "object" && !Array.isArray(parsed) - ? (parsed as { jobs?: unknown }).jobs - : undefined; - return { - jobs: - jobs && typeof jobs === "object" && !Array.isArray(jobs) - ? (jobs as Record) - : {}, - }; -} - -function findStep(job: WorkflowJob, name: string): WorkflowStep { - return job.steps?.find((step) => step.name === name) ?? {}; -} - -function requireEqual(errors: string[], actual: unknown, expected: unknown, message: string): void { - if (actual !== expected) errors.push(message); -} - -function requireRunContains(errors: string[], step: WorkflowStep, fragment: string): void { - if (!step.run?.includes(fragment)) { - errors.push(`${JOB_NAME} step ${step.name ?? ""} must contain: ${fragment}`); - } -} - -export function validateDocsValidationWorkflow(workflow: DocsValidationWorkflow): string[] { - const errors: string[] = []; - const job = workflow.jobs[JOB_NAME] ?? {}; - const env = job.env ?? {}; - - requireEqual(errors, job.needs, "generate-matrix", `${JOB_NAME} must depend on generate-matrix`); - requireEqual( - errors, - job.if, - "${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',docs-validation,') || contains(format(',{0},', inputs.targets), ',docs-validation,') }}", - `${JOB_NAME} must remain default-enabled and selectively dispatchable`, - ); - requireEqual(errors, job["runs-on"], "ubuntu-latest", `${JOB_NAME} must run on ubuntu-latest`); - requireEqual(errors, job["timeout-minutes"], 15, `${JOB_NAME} timeout must remain 15 minutes`); - requireEqual(errors, env.E2E_JOB, "1", `${JOB_NAME} must be free-standing`); - requireEqual( - errors, - env.E2E_TARGET_ID, - "docs-validation", - `${JOB_NAME} must publish the docs-validation selector`, - ); - requireEqual( - errors, - env.CHECK_DOC_LINKS_REMOTE, - "0", - `${JOB_NAME} must keep link checks deterministic and local-only`, - ); - requireEqual( - errors, - env.E2E_ARTIFACT_DIR, - "${{ github.workspace }}/e2e-artifacts/live/docs-validation", - `${JOB_NAME} must isolate docs-validation artifacts`, - ); - requireEqual(errors, env.NEMOCLAW_RUN_LIVE_E2E, "1", `${JOB_NAME} must enable live E2E`); - - const checkout = job.steps?.find((step) => step.uses?.startsWith("actions/checkout@")); - if (!checkout || !/^actions\/checkout@[0-9a-f]{40}$/u.test(checkout.uses ?? "")) { - errors.push(`${JOB_NAME} checkout must pin a full action SHA`); - } - if (checkout?.with?.["persist-credentials"] !== false) { - errors.push(`${JOB_NAME} checkout must disable persisted credentials`); - } - - const prepare = findStep(job, PREPARE_E2E_STEP); - requireEqual(errors, prepare.uses, PREPARE_E2E_ACTION, `${JOB_NAME} must use prepare-e2e`); - requireEqual(errors, prepare.with?.["build-cli"], "false", `${JOB_NAME} must skip the CLI build`); - - const run = findStep(job, "Run docs validation live Vitest test"); - requireRunContains(errors, run, "npx vitest run --project e2e-live"); - requireRunContains(errors, run, "test/e2e/live/docs-validation.test.ts"); - - const reportNeeds = workflow.jobs["report-to-pr"]?.needs; - if (!Array.isArray(reportNeeds) || !reportNeeds.includes(JOB_NAME)) { - errors.push(`report-to-pr must wait for ${JOB_NAME}`); - } - - return errors; -} - -export function validateDocsValidationWorkflowBoundary( - workflowPath = DEFAULT_WORKFLOW_PATH, -): string[] { - return validateDocsValidationWorkflow(readDocsValidationWorkflow(workflowPath)); -} diff --git a/tools/e2e/operations-workflow-boundary.mts b/tools/e2e/operations-workflow-boundary.mts index c44451c0a93..a87f8d08d36 100644 --- a/tools/e2e/operations-workflow-boundary.mts +++ b/tools/e2e/operations-workflow-boundary.mts @@ -269,10 +269,11 @@ function validateIssueRoutingRetirement(errors: string[], workflow: OperationsWo if (name === "report-to-pr") { if ( job.permissions === "write-all" || + permissions.actions !== "read" || permissions["pull-requests"] !== "write" || - Object.keys(permissions).length !== 1 + Object.keys(permissions).length !== 2 ) { - errors.push("report-to-pr must hold only pull-requests: write"); + errors.push("report-to-pr must hold only actions: read and pull-requests: write"); } if ( job.if !== diff --git a/tools/e2e/pr-e2e-gate.mts b/tools/e2e/pr-e2e-gate.mts index 8e69cd0b956..2b0a83e972d 100755 --- a/tools/e2e/pr-e2e-gate.mts +++ b/tools/e2e/pr-e2e-gate.mts @@ -18,6 +18,7 @@ import { type RiskPlan, riskPlanRequiredJobIds, } from "../advisors/risk-plan.mts"; +import { SHARED_E2E_JOB_ID } from "./credential-free-tests.mts"; import { readPrivateRegularFile, writePrivateRegularFile } from "./private-file.ts"; import type { E2eRiskSignal } from "./risk-signal.ts"; import { readFreeStandingJobsInventory } from "./workflow-boundary.mts"; @@ -727,10 +728,20 @@ export function expectedSignalShards( ): Record { const workflow = YAML.parse(fs.readFileSync(workflowPath, "utf8")) as unknown; const jobs = isObjectRecord(workflow) && isObjectRecord(workflow.jobs) ? workflow.jobs : {}; + const inventory = readFreeStandingJobsInventory(workflowPath); return Object.fromEntries( jobIds.map((jobId) => { - if (!isObjectRecord(jobs[jobId])) throw new Error(`E2E workflow does not define ${jobId}`); - const job = jobs[jobId]; + const executionJobId = inventory.targetToJob.get(jobId) ?? jobId; + if (!isObjectRecord(jobs[executionJobId])) { + throw new Error(`E2E workflow does not define ${executionJobId} for ${jobId}`); + } + const job = jobs[executionJobId]; + if (executionJobId !== jobId) { + if (executionJobId !== SHARED_E2E_JOB_ID) { + throw new Error(`${jobId} maps to an unknown shared E2E job`); + } + return [jobId, ["default"]]; + } const strategy = isObjectRecord(job.strategy) ? job.strategy : {}; const matrix = isObjectRecord(strategy.matrix) ? strategy.matrix : null; let shards = ["default"]; diff --git a/tools/e2e/prepare-e2e-workflow-boundary.mts b/tools/e2e/prepare-e2e-workflow-boundary.mts index a9821f240c7..dfc6635f209 100644 --- a/tools/e2e/prepare-e2e-workflow-boundary.mts +++ b/tools/e2e/prepare-e2e-workflow-boundary.mts @@ -7,6 +7,7 @@ import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { isDeepStrictEqual } from "node:util"; import YAML from "yaml"; +import { SHARED_E2E_JOB_ID } from "./credential-free-tests.mts"; const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); const DEFAULT_ACTION_PATH = join(REPO_ROOT, ".github", "actions", "prepare-e2e", "action.yaml"); @@ -22,11 +23,9 @@ export const PREPARE_E2E_STEP = "Prepare E2E workspace"; const CHECKOUT_LOCAL_PREPARE_E2E_ACTION = "./.github/actions/prepare-e2e"; const NO_BUILD_JOBS = new Set([ - "docs-validation", "generate-matrix", "launchable-smoke", "ollama-auth-proxy", - "openshell-version-pin", "rebuild-hermes", "rebuild-hermes-stale-base", "shields-config", @@ -113,6 +112,20 @@ export function validatePrepareE2eInvocations(workflow: WorkflowRecord): string[ .map(([jobName]) => jobName), ); + const sharedE2eJob = jobs[SHARED_E2E_JOB_ID]; + if (sharedE2eJob === undefined) { + errors.push(`prepare-e2e shared job is missing: ${SHARED_E2E_JOB_ID}`); + } else { + expectedJobs.add(SHARED_E2E_JOB_ID); + const env = record(record(sharedE2eJob).env); + if (Object.hasOwn(env, "E2E_JOB")) { + errors.push(`${SHARED_E2E_JOB_ID} must not declare E2E_JOB`); + } + if (Object.hasOwn(env, "E2E_EXECUTION_PROFILE")) { + errors.push(`${SHARED_E2E_JOB_ID} must not declare E2E_EXECUTION_PROFILE`); + } + } + for (const [jobName, value] of Object.entries(jobs)) { const jobSteps = steps(record(value).steps); if (jobSteps.some((step) => step.uses === CHECKOUT_LOCAL_PREPARE_E2E_ACTION)) { diff --git a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts index 02b1a5d3651..bcce668a08a 100644 --- a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts +++ b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts @@ -7,6 +7,7 @@ import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { isDeepStrictEqual } from "node:util"; import YAML from "yaml"; +import { SHARED_E2E_JOB_ID } from "./credential-free-tests.mts"; const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); const DEFAULT_ACTION_PATH = join( @@ -34,8 +35,12 @@ const CALLER_ALWAYS = "always()"; const MCP_SCANNED_UPLOAD_CONDITION = "${{ always() && steps.mcp_artifact_secret_scan.outcome == 'success' }}"; const TARGET_ID_PATTERN = /^[A-Za-z0-9_-]+$/; -const EXPECTED_UPLOAD_JOB_COUNT = 75; -const EXPECTED_DEFAULT_CALLER_COUNT = 63; +const EXPECTED_UPLOAD_JOB_COUNT = 72; +const EXPECTED_DEFAULT_CALLER_COUNT = 60; + +const SHARED_E2E_JOBS: ReadonlyMap = new Map([ + [SHARED_E2E_JOB_ID, { targetId: "${{ matrix.id }}" }], +]); type WorkflowRecord = Record; type WorkflowStep = WorkflowRecord & { @@ -262,7 +267,9 @@ export function validateUploadE2eArtifactsInvocations(workflow: WorkflowRecord): Object.entries(jobs) .filter(([jobName, value]) => { const job = record(value); - return jobName === "live" || record(job.env).E2E_JOB === "1"; + return ( + jobName === "live" || record(job.env).E2E_JOB === "1" || SHARED_E2E_JOBS.has(jobName) + ); }) .map(([jobName]) => jobName), ); @@ -272,7 +279,7 @@ export function validateUploadE2eArtifactsInvocations(workflow: WorkflowRecord): if (expectedJobs.size !== EXPECTED_UPLOAD_JOB_COUNT) { errors.push( - `upload-e2e-artifacts must cover exactly ${EXPECTED_UPLOAD_JOB_COUNT} live and E2E_JOB execution jobs`, + `upload-e2e-artifacts must cover exactly ${EXPECTED_UPLOAD_JOB_COUNT} live, E2E_JOB, and shared E2E jobs`, ); } if (defaultJobs.length !== EXPECTED_DEFAULT_CALLER_COUNT) { @@ -286,6 +293,21 @@ export function validateUploadE2eArtifactsInvocations(workflow: WorkflowRecord): } } + for (const jobName of SHARED_E2E_JOBS.keys()) { + const value = jobs[jobName]; + if (value === undefined) { + errors.push(`upload-e2e-artifacts shared job is missing: ${jobName}`); + continue; + } + const env = record(record(value).env); + if (Object.hasOwn(env, "E2E_JOB")) { + errors.push(`${jobName} must not declare E2E_JOB`); + } + if (Object.hasOwn(env, "E2E_EXECUTION_PROFILE")) { + errors.push(`${jobName} must not declare E2E_EXECUTION_PROFILE`); + } + } + for (const [jobName, value] of Object.entries(jobs)) { const job = record(value); const jobSteps = steps(job.steps); @@ -359,6 +381,15 @@ export function validateUploadE2eArtifactsInvocations(workflow: WorkflowRecord): errors.push(`${jobName} upload-e2e-artifacts must use the action defaults`); } const targetId = record(job.env).E2E_TARGET_ID; + const sharedJobContract = SHARED_E2E_JOBS.get(jobName); + if (sharedJobContract) { + if (targetId !== sharedJobContract.targetId) { + errors.push( + `${jobName} default upload caller E2E_TARGET_ID must be '${sharedJobContract.targetId}'`, + ); + } + continue; + } if (typeof targetId !== "string" || !TARGET_ID_PATTERN.test(targetId)) { errors.push(`${jobName} default upload caller must declare a valid E2E_TARGET_ID`); } else if (targetId !== jobName) { diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index f031c7032cc..f2a021105e8 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -5,7 +5,7 @@ import { readFileSync, statSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import YAML from "yaml"; -import { validateDocsValidationWorkflowBoundary } from "./docs-validation-workflow-boundary.mts"; +import { discoverCredentialFreeTests, SHARED_E2E_JOB_ID } from "./credential-free-tests.mts"; import { validateHermesDashboardWorkflowBoundary } from "./hermes-dashboard-workflow-boundary.mts"; import { validateHermesGpuStartupWorkflowBoundary } from "./hermes-gpu-startup-workflow-boundary.mts"; import { validateInferenceSwitchWorkflowBoundary } from "./inference-switch-workflow-boundary.mts"; @@ -17,7 +17,6 @@ import { validateUploadE2eArtifactsWorkflowBoundary } from "./upload-e2e-artifac const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); const DEFAULT_E2E_WORKFLOW_PATH = join(REPO_ROOT, ".github", "workflows", "e2e.yaml"); -export const FREE_STANDING_WORKFLOW_INVENTORY_SCRIPT = "tools/e2e/workflow-inventory.mts"; type WorkflowRecord = Record; type WorkflowStep = WorkflowRecord & { @@ -29,6 +28,7 @@ type WorkflowStep = WorkflowRecord & { export interface FreeStandingJobsInventory { allowedJobs: string[]; + workflowJobs: string[]; explicitOnlyJobs: string[]; freeStandingTargets: string[]; targetToJob: Map; @@ -57,13 +57,7 @@ const PUBLIC_NVIDIA_ENDPOINT_KEY_JOBS = new Set([ "device-auth-health", "model-router-provider-routed-inference", ]); -const NO_IMAGE_E2E_JOBS = new Set([ - "docs-validation", - "gateway-drift-preflight", - "gateway-health-honest", - "onboard-negative-paths", - "openshell-version-pin", -]); +const NO_IMAGE_E2E_JOBS = new Set(["gateway-health-honest", SHARED_E2E_JOB_ID]); const DOCKER_HUB_AUTH_STEP = "Authenticate to Docker Hub"; const DOCKER_HUB_CLEANUP_STEP = "Clean up Docker auth"; const DOCKER_HUB_CLEANUP_RUN = "bash .github/scripts/docker-auth-cleanup.sh"; @@ -98,6 +92,7 @@ function deriveFreeStandingJobsInventoryFromJobs(jobs: WorkflowRecord): { } { const errors: string[] = []; const allowedJobs: string[] = []; + const workflowJobs: string[] = []; const explicitOnlyJobs: string[] = []; const freeStandingTargets: string[] = []; const targetToJob = new Map(); @@ -105,6 +100,7 @@ function deriveFreeStandingJobsInventoryFromJobs(jobs: WorkflowRecord): { for (const [jobId, rawJob] of Object.entries(jobs)) { const job = asRecord(rawJob); const env = asRecord(job.env); + if (jobId === SHARED_E2E_JOB_ID) continue; const hasJobMarker = Object.hasOwn(env, FREE_STANDING_JOB_MARKER); const hasTargetMarker = Object.hasOwn(env, FREE_STANDING_TARGET_MARKER); if (!hasJobMarker && !hasTargetMarker) continue; @@ -124,6 +120,7 @@ function deriveFreeStandingJobsInventoryFromJobs(jobs: WorkflowRecord): { } allowedJobs.push(jobId); + workflowJobs.push(jobId); if (Object.hasOwn(env, FREE_STANDING_DEFAULT_ENABLED_MARKER)) { if (env[FREE_STANDING_DEFAULT_ENABLED_MARKER] !== "0") { errors.push(`${jobId} job ${FREE_STANDING_DEFAULT_ENABLED_MARKER} must be "0" when set`); @@ -142,12 +139,30 @@ function deriveFreeStandingJobsInventoryFromJobs(jobs: WorkflowRecord): { targetToJob.set(target, jobId); } + if (Object.hasOwn(jobs, SHARED_E2E_JOB_ID)) { + workflowJobs.push(SHARED_E2E_JOB_ID); + try { + for (const row of discoverCredentialFreeTests()) { + allowedJobs.push(row.id); + freeStandingTargets.push(row.id); + targetToJob.set(row.id, SHARED_E2E_JOB_ID); + } + } catch (error) { + errors.push( + `credential-free test discovery failed: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + if (allowedJobs.length === 0) { errors.push("free-standing workflow metadata must declare at least one job"); } for (const duplicate of findDuplicates(allowedJobs)) { errors.push(`free-standing workflow metadata repeats job id: ${duplicate}`); } + for (const duplicate of findDuplicates(workflowJobs)) { + errors.push(`free-standing workflow metadata repeats workflow job id: ${duplicate}`); + } for (const duplicate of findDuplicates(freeStandingTargets)) { errors.push(`free-standing workflow metadata repeats target id: ${duplicate}`); } @@ -156,6 +171,7 @@ function deriveFreeStandingJobsInventoryFromJobs(jobs: WorkflowRecord): { errors, inventory: { allowedJobs, + workflowJobs, explicitOnlyJobs, freeStandingTargets, targetToJob, @@ -174,6 +190,7 @@ function cloneFreeStandingJobsInventory( ): FreeStandingJobsInventory { return { allowedJobs: [...inventory.allowedJobs], + workflowJobs: [...inventory.workflowJobs], explicitOnlyJobs: [...inventory.explicitOnlyJobs], freeStandingTargets: [...inventory.freeStandingTargets], targetToJob: new Map(inventory.targetToJob), @@ -209,19 +226,6 @@ export function readFreeStandingJobsInventory( return inventory; } -export function formatFreeStandingJobsInventoryForShell( - inventory: FreeStandingJobsInventory, -): string { - const targetJobMappings = [...inventory.targetToJob].map(([target, job]) => `${target}:${job}`); - return [ - `allowed_jobs=${inventory.allowedJobs.join(",")}`, - `explicit_only_jobs_csv=${inventory.explicitOnlyJobs.join(",")}`, - `free_standing_targets_csv=${inventory.freeStandingTargets.join(",")}`, - `free_standing_target_jobs_csv=${targetJobMappings.join(",")}`, - "", - ].join("\n"); -} - export interface WorkflowDispatchSelectorEvaluation { valid: boolean; errors: string[]; @@ -311,7 +315,7 @@ export function evaluateE2eWorkflowDispatchSelectors(input: { const registryTargets: string[] = []; for (const target of splitSelector(targets)) { const job = freeStandingTargetToJob.get(target); - if (job) selectedFreeStandingJobs.add(job); + if (job) selectedFreeStandingJobs.add(target); else registryTargets.push(target); } @@ -574,11 +578,11 @@ function validateFreeStandingInventoryBoundary( ): void { const targetByJob = new Map([...inventory.targetToJob].map(([target, job]) => [job, target])); - for (const jobName of inventory.allowedJobs) { + for (const jobName of inventory.workflowJobs) { const job = asRecord(jobs[jobName]); if (Object.keys(job).length === 0) continue; - if (!FREE_STANDING_SELECTOR_SPECIAL_CASES.has(jobName)) { + if (jobName !== SHARED_E2E_JOB_ID && !FREE_STANDING_SELECTOR_SPECIAL_CASES.has(jobName)) { validateFreeStandingJobSelector( errors, jobs, @@ -617,7 +621,7 @@ function validateFreeStandingInventoryCoverage( reportNeeds: readonly unknown[], inventory: FreeStandingJobsInventory, ): void { - for (const jobId of inventory.allowedJobs) { + for (const jobId of inventory.workflowJobs) { if (!Object.hasOwn(jobs, jobId)) { errors.push(`free-standing inventory job missing workflow job: ${jobId}`); } @@ -626,10 +630,11 @@ function validateFreeStandingInventoryCoverage( } } for (const [target, jobId] of inventory.targetToJob) { - if (!inventory.allowedJobs.includes(jobId)) { - errors.push(`free-standing inventory maps ${target} to unknown job ${jobId}`); + if (!inventory.workflowJobs.includes(jobId)) { + errors.push(`free-standing inventory maps ${target} to unknown workflow job ${jobId}`); continue; } + if (jobId === SHARED_E2E_JOB_ID) continue; const job = asRecord(jobs[jobId]); if (Object.keys(job).length === 0) continue; const jobIf = stringValue(job.if); @@ -644,58 +649,104 @@ function validateFreeStandingInventoryCoverage( } } -function validateOpenShellVersionPinJob(errors: string[], jobs: WorkflowRecord): void { - const jobName = "openshell-version-pin"; - const job = asRecord(jobs[jobName]); +function validateSharedE2eJob(errors: string[], jobs: WorkflowRecord): void { + const job = asRecord(jobs[SHARED_E2E_JOB_ID]); if (Object.keys(job).length === 0) { - errors.push("workflow missing openshell-version-pin job"); + errors.push("workflow missing shared E2E job"); return; } + if (job.name !== "Shared E2E (${{ matrix.id }})") { + errors.push("shared E2E job name must expose the test ID"); + } + if (job.needs !== "generate-matrix") { + errors.push("shared E2E job must depend on generate-matrix"); + } + if (job.if !== "${{ needs.generate-matrix.outputs.test_matrix != '[]' }}") { + errors.push("shared E2E job must run only for a non-empty test matrix"); + } if (job["runs-on"] !== "ubuntu-latest") { - errors.push("openshell-version-pin job must run on ubuntu-latest"); + errors.push("shared E2E job must run on ubuntu-latest"); + } + if (job["timeout-minutes"] !== 15) { + errors.push("shared E2E job timeout must remain 15 minutes"); } - validateFreeStandingJobSelector(errors, jobs, jobName, "openshell-version-pin"); - const jobEnv = asRecord(job.env); - if (jobEnv.NEMOCLAW_RUN_LIVE_E2E !== "1") { - errors.push("openshell-version-pin job must set NEMOCLAW_RUN_LIVE_E2E=1"); + const strategy = asRecord(job.strategy); + if (strategy["fail-fast"] !== false) { + errors.push("shared E2E strategy.fail-fast must be false"); } if ( - jobEnv.E2E_ARTIFACT_DIR !== "${{ github.workspace }}/e2e-artifacts/live/openshell-version-pin" + asRecord(strategy.matrix).include !== + "${{ fromJSON(needs.generate-matrix.outputs.test_matrix) }}" ) { - errors.push( - "openshell-version-pin job must write artifacts under e2e-artifacts/live/openshell-version-pin", - ); + errors.push("shared E2E matrix must come from tagged credential-free tests"); + } + + const jobEnv = asRecord(job.env); + const expectedEnv = { + CHECK_DOC_LINKS_REMOTE: "0", + E2E_ARTIFACT_DIR: "${{ github.workspace }}/e2e-artifacts/live/${{ matrix.id }}", + E2E_TARGET_ID: "${{ matrix.id }}", + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_CLI_BIN: "${{ github.workspace }}/bin/nemoclaw.js", + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_RUN_LIVE_E2E: "1", + }; + for (const [name, expected] of Object.entries(expectedEnv)) { + if (jobEnv[name] !== expected) { + errors.push(`shared E2E job must set ${name} to ${expected}`); + } + } + if (Object.hasOwn(jobEnv, FREE_STANDING_JOB_MARKER)) { + errors.push("shared E2E job must not become a jobs selector"); + } + if (Object.hasOwn(jobEnv, "E2E_EXECUTION_PROFILE")) { + errors.push("shared E2E job must not declare E2E_EXECUTION_PROFILE"); + } + for (const secret of COMMON_SECRET_ENV_NAMES) { + requireEnvDoesNotExposeSecret(errors, "shared E2E job", jobEnv, secret); } - requireEnvDoesNotExposeSecret( - errors, - "openshell-version-pin job", - jobEnv, - "NVIDIA_INFERENCE_API_KEY", - ); const steps = asSteps(job.steps); requireNoDispatchInputInterpolation(errors, steps); for (const step of steps) { - requireEnvDoesNotExposeSecret( - errors, - `openshell-version-pin step '${step.name ?? step.uses ?? ""}'`, - asRecord(step.env), - "NVIDIA_INFERENCE_API_KEY", - ); + for (const secret of COMMON_SECRET_ENV_NAMES) { + requireEnvDoesNotExposeSecret( + errors, + `shared E2E step '${step.name ?? step.uses ?? ""}'`, + asRecord(step.env), + secret, + ); + } } const checkout = steps.find((step) => stringValue(step.uses).startsWith("actions/checkout@")); - if (!checkout) errors.push("openshell-version-pin job missing checkout step"); - requireFullShaAction(errors, checkout, "openshell-version-pin checkout"); + if (!checkout) errors.push("shared E2E job missing checkout step"); + requireFullShaAction(errors, checkout, "shared E2E checkout"); if (asRecord(checkout?.with)["persist-credentials"] !== false) { - errors.push("openshell-version-pin checkout step must set persist-credentials=false"); + errors.push("shared E2E checkout must disable persisted credentials"); } - const runVitest = requireJobStep(errors, jobName, steps, "Run OpenShell version-pin live test"); - requireRunContains(errors, runVitest, "npx vitest run --project e2e-live"); - requireRunContains(errors, runVitest, "test/e2e/live/openshell-version-pin.test.ts"); + const runVitest = requireJobStep( + errors, + SHARED_E2E_JOB_ID, + steps, + "Run tagged credential-free test", + ); + const runEnv = asRecord(runVitest?.env); + if (runEnv.TEST_FILE !== "${{ matrix.file }}") { + errors.push("shared E2E test step must pass matrix.file through TEST_FILE"); + } + if (runEnv.TEST_PROJECT !== "${{ matrix.project }}") { + errors.push("shared E2E test step must pass matrix.project through TEST_PROJECT"); + } + requireRunContains( + errors, + runVitest, + 'npx vitest run --project "${TEST_PROJECT}" "${TEST_FILE}"', + ); + requireRunContains(errors, runVitest, "--reporter=test/e2e/risk-signal-reporter.ts"); } function validateSkillAgentJob(errors: string[], jobs: WorkflowRecord): void { @@ -1738,60 +1789,6 @@ function validateMessagingCompatibleEndpointJob(errors: string[], jobs: Workflow requireRunContains(errors, runVitest, "test/e2e/live/messaging-compatible-endpoint.test.ts"); } -function validateOnboardNegativePathsJob(errors: string[], jobs: WorkflowRecord): void { - const jobName = "onboard-negative-paths"; - const job = asRecord(jobs[jobName]); - if (Object.keys(job).length === 0) { - errors.push("workflow missing onboard-negative-paths job"); - return; - } - - if (job["runs-on"] !== "ubuntu-latest") { - errors.push("onboard-negative-paths job must run on ubuntu-latest"); - } - validateFreeStandingJobSelector(errors, jobs, jobName, "onboard-negative-paths"); - - const jobEnv = asRecord(job.env); - if (jobEnv.NEMOCLAW_RUN_LIVE_E2E !== "1") { - errors.push("onboard-negative-paths job must set NEMOCLAW_RUN_LIVE_E2E=1"); - } - if ( - jobEnv.E2E_ARTIFACT_DIR !== "${{ github.workspace }}/e2e-artifacts/live/onboard-negative-paths" - ) { - errors.push( - "onboard-negative-paths job must write artifacts under e2e-artifacts/live/onboard-negative-paths", - ); - } - requireEnvDoesNotExposeSecret( - errors, - "onboard-negative-paths job", - jobEnv, - "NVIDIA_INFERENCE_API_KEY", - ); - - const steps = asSteps(job.steps); - requireNoDispatchInputInterpolation(errors, steps); - for (const step of steps) { - requireEnvDoesNotExposeSecret( - errors, - `onboard-negative-paths step '${step.name ?? step.uses ?? ""}'`, - asRecord(step.env), - "NVIDIA_INFERENCE_API_KEY", - ); - } - - const checkout = steps.find((step) => stringValue(step.uses).startsWith("actions/checkout@")); - if (!checkout) errors.push("onboard-negative-paths job missing checkout step"); - requireFullShaAction(errors, checkout, "onboard-negative-paths checkout"); - if (asRecord(checkout?.with)["persist-credentials"] !== false) { - errors.push("onboard-negative-paths checkout step must set persist-credentials=false"); - } - - const runVitest = requireJobStep(errors, jobName, steps, "Run onboard negative-paths live test"); - requireRunContains(errors, runVitest, "npx vitest run --project e2e-live"); - requireRunContains(errors, runVitest, "test/e2e/live/onboard-negative-paths.test.ts"); -} - function validateCloudInferenceJob(errors: string[], jobs: WorkflowRecord): void { const jobName = "cloud-inference"; const job = asRecord(jobs[jobName]); @@ -2023,7 +2020,10 @@ function requireCanonicalDockerHubCleanupRun( function validateDockerHubAuthBoundary(errors: string[], jobs: WorkflowRecord): void { const e2eJobNames = Object.entries(jobs) - .filter(([, rawJob]) => asRecord(asRecord(rawJob).env).E2E_JOB === "1") + .filter(([jobName, rawJob]) => { + const env = asRecord(asRecord(rawJob).env); + return env.E2E_JOB === "1" || jobName === SHARED_E2E_JOB_ID; + }) .map(([jobName]) => jobName); for (const exemptJobName of NO_IMAGE_E2E_JOBS) { if (!e2eJobNames.includes(exemptJobName)) { @@ -2618,23 +2618,6 @@ function validateModelRouterProviderRoutedInferenceJob( ); } -function validateGatewayDriftPreflightJob(errors: string[], jobs: WorkflowRecord): void { - const jobName = "gateway-drift-preflight"; - const job = asRecord(jobs[jobName]); - validateFreeStandingJobSelector(errors, jobs, jobName, "gateway-drift-preflight"); - if (Object.keys(job).length === 0) return; - - const runVitest = requireJobStep( - errors, - jobName, - asSteps(job.steps), - "Run gateway drift preflight Vitest test", - ); - requireRunContains(errors, runVitest, "npx vitest run --project integration"); - requireRunContains(errors, runVitest, "test/gateway-drift-preflight.test.ts"); - requireRunDoesNotContain(errors, runVitest, "--project cli"); -} - function runContainsCloudflaredAptInstall(run: string): boolean { return /apt-get\s+install[\s\S]*cloudflared|apt\s+install[\s\S]*cloudflared|pkg\.cloudflare\.com\/cloudflared/.test( run, @@ -3506,7 +3489,6 @@ export function validateE2eWorkflowBoundary(workflowPath = DEFAULT_E2E_WORKFLOW_ errors.push(...validateHermesGpuStartupWorkflowBoundary(workflowPath)); errors.push(...validateInferenceSwitchWorkflowBoundary(workflowPath)); errors.push(...validateE2eOperationsWorkflowBoundary(workflowPath)); - errors.push(...validateDocsValidationWorkflowBoundary(workflowPath)); errors.push(...validateSecurityPostureWorkflowBoundary(workflowPath)); const triggers = asRecord(workflow.on ?? workflow[true as unknown as string]); @@ -3519,14 +3501,14 @@ export function validateE2eWorkflowBoundary(workflowPath = DEFAULT_E2E_WORKFLOW_ validateAllowJetsonRunnerQueueInput(errors, dispatchInputs); const jobsInput = requireInput(errors, dispatchInputs, "jobs"); const jobsDescription = stringValue(jobsInput.description); - if (!jobsDescription.includes("default-enabled jobs")) { + if (!jobsDescription.includes("default-enabled tests")) { errors.push( - "workflow_dispatch jobs input description must say empty dispatch runs default-enabled jobs", + "workflow_dispatch jobs input description must say empty dispatch runs default-enabled tests", ); } - if (!jobsDescription.includes("explicit-only jobs")) { + if (!jobsDescription.includes("explicit-only tests")) { errors.push( - "workflow_dispatch jobs input description must say explicit-only jobs are skipped unless selected", + "workflow_dispatch jobs input description must say explicit-only tests are skipped unless selected", ); } if (Object.hasOwn(dispatchInputs, "test_filter")) { @@ -3551,6 +3533,9 @@ export function validateE2eWorkflowBoundary(workflowPath = DEFAULT_E2E_WORKFLOW_ if (generateOutputs.matrix !== "${{ steps.matrix.outputs.matrix }}") { errors.push("generate-matrix job must expose matrix output"); } + if (generateOutputs.test_matrix !== "${{ steps.matrix.outputs.test_matrix }}") { + errors.push("generate-matrix job must expose test_matrix output"); + } if (generateOutputs.hermes_selected !== "${{ steps.matrix.outputs.hermes_selected }}") { errors.push("generate-matrix job must expose hermes_selected output"); } @@ -3575,28 +3560,29 @@ export function validateE2eWorkflowBoundary(workflowPath = DEFAULT_E2E_WORKFLOW_ if (generateEnv.TARGETS !== "${{ inputs.targets }}") { errors.push("matrix generation step must pass targets through TARGETS env"); } - requireRunContains(errors, generate, FREE_STANDING_WORKFLOW_INVENTORY_SCRIPT); - requireRunContains( - errors, - generate, - "free-standing workflow inventory must be data-only key=value", - ); - requireRunContains(errors, generate, "free_standing_targets_csv must match target mapping keys"); - requireRunContains(errors, generate, "Free-standing target maps to unknown job"); + requireRunContains(errors, generate, "npx tsx tools/e2e/workflow-plan.mts"); requireRunContains(errors, generate, "Use either targets or jobs, not both"); - requireRunContains(errors, generate, "Unknown free-standing E2E job"); - requireRunContains(errors, generate, 'matrix="[]"'); - requireRunContains(errors, generate, "npx tsx test/e2e/registry/run.ts"); - requireRunContains(errors, generate, "--emit-live-matrix"); + requireRunContains(errors, generate, "for selector_name in JOBS TARGETS"); + requireRunContains(errors, generate, "Invalid ${selector_name,,} input; use comma-separated ids"); + requireRunContains(errors, generate, 'planner_args+=(--jobs "${JOBS}")'); + requireRunContains(errors, generate, 'planner_args+=(--targets "${TARGETS}")'); requireRunContains(errors, generate, "--targets"); requireRunContains(errors, generate, "^[A-Za-z0-9_-]+(,[A-Za-z0-9_-]+)*$"); - requireRunContains(errors, generate, "Invalid target input; use comma-separated target ids"); - requireRunContains(errors, generate, "Invalid jobs input; use comma-separated job ids"); requireRunDoesNotContain(errors, generate, "Invalid jobs input: ${JOBS}"); requireRunDoesNotContain(errors, generate, "Invalid target input: ${TARGETS}"); requireRunDoesNotContain(errors, generate, "^[A-Za-z0-9._-]+"); - requireRunContains(errors, generate, "hermes_selected=false"); - requireRunContains(errors, generate, "hermes_selected=true"); + requireRunContains( + errors, + generate, + '(keys | sort) == ["explicitOnlyJobs", "hermesSelected", "matrix", "testMatrix"]', + ); + requireRunContains(errors, generate, "([.matrix[].id] | unique | length)"); + requireRunContains(errors, generate, '(keys | sort) == ["file", "id", "project"]'); + requireRunContains(errors, generate, "([.testMatrix[].id] | unique | length)"); + requireRunContains(errors, generate, "E2E planner returned an invalid output schema"); + requireRunContains(errors, generate, "expected_hermes_selected=false"); + requireRunContains(errors, generate, "expected_hermes_selected=true"); + requireRunContains(errors, generate, "E2E planner changed the trusted Hermes selection"); requireRunContains( errors, generate, @@ -3607,8 +3593,9 @@ export function validateE2eWorkflowBoundary(workflowPath = DEFAULT_E2E_WORKFLOW_ generate, 'echo "explicit_only_jobs=${explicit_only_jobs_csv}" >> "$GITHUB_OUTPUT"', ); - requireRunContains(errors, generate, "## E2E Target Matrix"); - requireRunContains(errors, generate, "| Target | Runner | Label |"); + requireRunContains(errors, generate, 'echo "test_matrix=${test_matrix}" >> "$GITHUB_OUTPUT"'); + requireRunContains(errors, generate, "## E2E Execution Plan"); + requireRunContains(errors, generate, "| Test | Execution | Runner |"); const liveTargets = asRecord(jobs["live"]); if (Object.keys(liveTargets).length === 0) errors.push("workflow missing live job"); @@ -3923,8 +3910,7 @@ export function validateE2eWorkflowBoundary(workflowPath = DEFAULT_E2E_WORKFLOW_ errors.push("cloud-onboard DCode TUI host dependencies must precede workspace prep"); } - validateOpenShellVersionPinJob(errors, jobs); - validateOnboardNegativePathsJob(errors, jobs); + validateSharedE2eJob(errors, jobs); validateSkillAgentJob(errors, jobs); validateFreeStandingJobSelector(errors, jobs, "credential-migration", "credential-migration"); validateFreeStandingJobSelector(errors, jobs, "sessions-agents-cli", "sessions-agents-cli"); @@ -3959,8 +3945,6 @@ export function validateE2eWorkflowBoundary(workflowPath = DEFAULT_E2E_WORKFLOW_ validateSnapshotCommandsJob(errors, jobs); errors.push(...validateSandboxOperationsWorkflow({ jobs })); validateSparkInstallJob(errors, jobs); - validateGatewayDriftPreflightJob(errors, jobs); - validateFreeStandingJobSelector( errors, jobs, @@ -4053,6 +4037,9 @@ export function validateE2eWorkflowBoundary(workflowPath = DEFAULT_E2E_WORKFLOW_ if (reportEnv.JOBS !== "${{ inputs.jobs }}") { errors.push("report-to-pr step must pass jobs through JOBS env"); } + if (reportEnv.TEST_MATRIX !== "${{ needs.generate-matrix.outputs.test_matrix }}") { + errors.push("report-to-pr must receive the credential-free test matrix"); + } if (reportEnv.JOB_PR_NUMBER !== "${{ inputs.pr_number }}") { errors.push("report-to-pr step must pass pr_number through JOB_PR_NUMBER env"); } @@ -4109,9 +4096,9 @@ export function validateE2eWorkflowBoundary(workflowPath = DEFAULT_E2E_WORKFLOW_ "step 'Post E2E target results to PR' run script must check selector validation before echoing selectors", ); } - if (!reportScript.includes("jobsRejected")) { + if (!reportScript.includes("testIdsRejected")) { errors.push( - "step 'Post E2E target results to PR' run script must omit rejected job selectors", + "step 'Post E2E target results to PR' run script must omit rejected test ID selectors", ); } if (!reportScript.includes("targetsRejected")) { @@ -4129,12 +4116,21 @@ export function validateE2eWorkflowBoundary(workflowPath = DEFAULT_E2E_WORKFLOW_ "step 'Post E2E target results to PR' run script must report missing requested jobs", ); } + if ( + !reportScript.includes("github.rest.actions.listJobsForWorkflowRun") || + !reportScript.includes("Shared E2E") || + !reportScript.includes("testResults") + ) { + errors.push( + "step 'Post E2E target results to PR' must resolve discovered matrix test results from the jobs API", + ); + } if (!reportScript.includes("cancelled")) { errors.push("step 'Post E2E target results to PR' run script must count cancelled jobs"); } - if (!reportScript.includes("**Requested jobs:**")) { + if (!reportScript.includes("**Requested test IDs:**")) { errors.push( - "step 'Post E2E target results to PR' run script must include **Requested jobs:**", + "step 'Post E2E target results to PR' run script must include **Requested test IDs:**", ); } if (!reportScript.includes("**Requested targets:**")) { @@ -4142,14 +4138,14 @@ export function validateE2eWorkflowBoundary(workflowPath = DEFAULT_E2E_WORKFLOW_ "step 'Post E2E target results to PR' run script must include **Requested targets:**", ); } - if (!reportScript.includes("All default jobs passed")) { + if (!reportScript.includes("All default tests passed")) { errors.push( - "step 'Post E2E target results to PR' run script must label empty dispatch as default jobs passed", + "step 'Post E2E target results to PR' run script must label empty dispatch as default tests passed", ); } - if (!reportScript.includes("default-enabled free-standing jobs")) { + if (!reportScript.includes("default-enabled tests")) { errors.push( - "step 'Post E2E target results to PR' run script must say empty dispatch uses default-enabled free-standing jobs", + "step 'Post E2E target results to PR' run script must say empty dispatch uses default-enabled tests", ); } if (!reportScript.includes("Explicit-only jobs skipped")) { diff --git a/tools/e2e/workflow-inventory.mts b/tools/e2e/workflow-inventory.mts index dbce557e69b..b5c8952b7f0 100644 --- a/tools/e2e/workflow-inventory.mts +++ b/tools/e2e/workflow-inventory.mts @@ -1,28 +1,39 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { execFileSync } from "node:child_process"; + +import YAML from "yaml"; + +import { SHARED_E2E_JOB_ID } from "./credential-free-tests.mts"; import { - formatFreeStandingJobsInventoryForShell, + type FreeStandingJobsInventory, readFreeStandingJobsInventory, } from "./workflow-boundary.mts"; +const E2E_WORKFLOW_PATH = ".github/workflows/e2e.yaml"; +const SHA_PATTERN = /^[a-f0-9]{40}$/u; + function usage(): string { return [ "Usage: npx tsx tools/e2e/workflow-inventory.mts [--shell] [--workflow PATH]", "", - "Derives free-standing E2E selector mappings from workflow job metadata.", + "Derives E2E test IDs from tagged tests and workflow jobs.", + " --shell Emit the four-key inventory consumed by the current base E2E workflow.", ].join("\n"); } function parseArgs(argv: readonly string[]): { - shell: boolean; + baseWorkflowFormat: boolean; workflowPath?: string; } { - const parsed: { shell: boolean; workflowPath?: string } = { shell: false }; + const parsed: { baseWorkflowFormat: boolean; workflowPath?: string } = { + baseWorkflowFormat: false, + }; for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; if (arg === "--shell") { - parsed.shell = true; + parsed.baseWorkflowFormat = true; continue; } if (arg === "--workflow") { @@ -41,16 +52,67 @@ function parseArgs(argv: readonly string[]): { return parsed; } +function currentBaseE2eWorkflowJobIds(): Set { + const eventSha = process.env.GITHUB_SHA; + const baseRef = + process.env.GITHUB_REF === "refs/heads/main" && SHA_PATTERN.test(eventSha ?? "") + ? (eventSha ?? "") + : "origin/main"; + const source = execFileSync("git", ["show", `${baseRef}:${E2E_WORKFLOW_PATH}`], { + encoding: "utf8", + maxBuffer: 10 * 1024 * 1024, + }); + const workflow = YAML.parse(source) as unknown; + if (!workflow || typeof workflow !== "object" || Array.isArray(workflow)) { + throw new Error(`Current base ${E2E_WORKFLOW_PATH} must be an object`); + } + const jobs = (workflow as Record).jobs; + if (!jobs || typeof jobs !== "object" || Array.isArray(jobs)) { + throw new Error(`Current base ${E2E_WORKFLOW_PATH} must define jobs`); + } + return new Set(Object.keys(jobs)); +} + +function formatCurrentBaseE2eWorkflowInventory( + inventory: FreeStandingJobsInventory, + currentBaseJobIds: ReadonlySet, +): string { + // `pr-e2e-gate.mts` dispatches `.github/workflows/e2e.yaml` from `main` + // with `checkout_sha` set to the PR head. The current base workflow + // therefore executes this PR's `workflow-inventory.mts --shell`. + // Keep this four-key output until the planner-based workflow is on + // `main`; then delete this CLI. + // Only tagged tests with discrete jobs in that workflow remain selectable; + // newer tagged tests must wait for the shared job instead of scheduling no work. + const supportedByCurrentBase = (testId: string): boolean => { + const job = inventory.targetToJob.get(testId); + return job !== SHARED_E2E_JOB_ID || currentBaseJobIds.has(testId); + }; + const targetJobMappings = [...inventory.targetToJob] + .filter(([target]) => supportedByCurrentBase(target)) + .map(([target, job]) => `${target}:${job === SHARED_E2E_JOB_ID ? target : job}`); + return [ + `allowed_jobs=${inventory.allowedJobs.filter(supportedByCurrentBase).join(",")}`, + `explicit_only_jobs_csv=${inventory.explicitOnlyJobs.join(",")}`, + `free_standing_targets_csv=${inventory.freeStandingTargets.filter(supportedByCurrentBase).join(",")}`, + `free_standing_target_jobs_csv=${targetJobMappings.join(",")}`, + "", + ].join("\n"); +} + try { const options = parseArgs(process.argv.slice(2)); const inventory = readFreeStandingJobsInventory(options.workflowPath); - if (options.shell) { - process.stdout.write(formatFreeStandingJobsInventoryForShell(inventory)); + if (options.baseWorkflowFormat) { + process.stdout.write( + formatCurrentBaseE2eWorkflowInventory(inventory, currentBaseE2eWorkflowJobIds()), + ); } else { process.stdout.write( `${JSON.stringify( { allowedJobs: inventory.allowedJobs, + workflowJobs: inventory.workflowJobs, explicitOnlyJobs: inventory.explicitOnlyJobs, freeStandingTargets: inventory.freeStandingTargets, targetJobs: Object.fromEntries(inventory.targetToJob), diff --git a/tools/e2e/workflow-plan.mts b/tools/e2e/workflow-plan.mts new file mode 100644 index 00000000000..50fdc120782 --- /dev/null +++ b/tools/e2e/workflow-plan.mts @@ -0,0 +1,123 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { buildLiveTargetMatrix, type LiveTargetMatrixEntry } from "../../test/e2e/registry/run.ts"; +import { + type CredentialFreeTestMatrixRow, + discoverCredentialFreeTests, +} from "./credential-free-tests.mts"; +import { readFreeStandingJobsInventory } from "./workflow-boundary.mts"; + +export type WorkflowPlanSelectors = { + jobs?: string; + targets?: string; +}; + +export type E2eWorkflowPlan = { + matrix: LiveTargetMatrixEntry[]; + testMatrix: CredentialFreeTestMatrixRow[]; + hermesSelected: boolean; + explicitOnlyJobs: string[]; +}; + +const SAFE_SELECTOR_LIST_PATTERN = /^[A-Za-z0-9_-]+(?:,[A-Za-z0-9_-]+)*$/; +const HERMES_JOB_ID = "hermes-e2e"; + +function selectorIds(value: string | undefined, label: "jobs" | "targets"): string[] { + if (!value) return []; + if (!SAFE_SELECTOR_LIST_PATTERN.test(value)) { + throw new Error( + `Invalid ${label} input; use comma-separated ids containing only letters, numbers, underscores, and hyphens`, + ); + } + return value.split(","); +} + +function selectTestRows( + rows: readonly CredentialFreeTestMatrixRow[], + ids: readonly string[], +): CredentialFreeTestMatrixRow[] { + if (ids.length === 0) return [...rows]; + const selected = new Set(ids); + return rows.filter((row) => selected.has(row.id)); +} + +export function buildE2eWorkflowPlan(selectors: WorkflowPlanSelectors = {}): E2eWorkflowPlan { + const jobs = selectorIds(selectors.jobs, "jobs"); + const targets = selectorIds(selectors.targets, "targets"); + if (jobs.length > 0 && targets.length > 0) { + throw new Error("Use either jobs or targets, not both"); + } + + const inventory = readFreeStandingJobsInventory(); + const credentialFreeTests = discoverCredentialFreeTests(); + + if (jobs.length > 0) { + const allowedJobs = new Set(inventory.allowedJobs); + for (const job of jobs) { + if (!allowedJobs.has(job)) { + throw new Error( + `Unknown E2E test ID: ${job}\nAllowed test IDs: ${inventory.allowedJobs.join(",")}`, + ); + } + } + + return { + matrix: [], + testMatrix: selectTestRows(credentialFreeTests, jobs), + hermesSelected: jobs.includes(HERMES_JOB_ID), + explicitOnlyJobs: [...inventory.explicitOnlyJobs], + }; + } + + if (targets.length > 0) { + const registryTargets = targets.filter((target) => !inventory.targetToJob.has(target)); + return { + matrix: registryTargets.length > 0 ? buildLiveTargetMatrix(registryTargets) : [], + testMatrix: selectTestRows(credentialFreeTests, targets), + hermesSelected: targets.includes(HERMES_JOB_ID), + explicitOnlyJobs: [...inventory.explicitOnlyJobs], + }; + } + + return { + matrix: buildLiveTargetMatrix(), + testMatrix: credentialFreeTests, + hermesSelected: true, + explicitOnlyJobs: [...inventory.explicitOnlyJobs], + }; +} + +function parseArgs(argv: readonly string[]): WorkflowPlanSelectors { + const selectors: WorkflowPlanSelectors = {}; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg !== "--jobs" && arg !== "--targets") { + throw new Error(`Unknown argument: ${arg}`); + } + const value = argv[index + 1]; + if (value === undefined) throw new Error(`${arg} requires a value`); + if (arg === "--jobs") selectors.jobs = value; + else selectors.targets = value; + index += 1; + } + return selectors; +} + +export function runE2eWorkflowPlanCli(argv = process.argv.slice(2)): void { + process.stdout.write(`${JSON.stringify(buildE2eWorkflowPlan(parseArgs(argv)))}\n`); +} + +const invokedFile = process.argv[1] ? path.resolve(process.argv[1]) : ""; +if (invokedFile === fileURLToPath(import.meta.url)) { + try { + runE2eWorkflowPlanCli(); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + for (const line of message.split("\n")) console.error(`::error::${line}`); + process.exitCode = 1; + } +} diff --git a/vitest.config.ts b/vitest.config.ts index 9375eac148c..d9404c04cef 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -51,6 +51,12 @@ const integrationProjectScheduling = resolveIntegrationProjectScheduling({ export default defineConfig({ test: { + tags: [ + { + name: "e2e/credential-free", + description: "Runs without external credentials in the shared E2E job", + }, + ], env: { NEMOCLAW_DISABLE_GATEWAY_DRIFT_PREFLIGHT: "1", },