diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index f61b1448388..6ccdf66c607 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -23,6 +23,11 @@ on: required: false type: string default: "" + post_to_slack: + description: Post a selective-dispatch scorecard to the preview Slack route. + required: false + default: false + type: boolean permissions: contents: read @@ -37,6 +42,7 @@ jobs: outputs: matrix: ${{ steps.matrix.outputs.matrix }} hermes_selected: ${{ steps.matrix.outputs.hermes_selected }} + explicit_only_jobs: ${{ steps.matrix.outputs.explicit_only_jobs }} steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: @@ -60,6 +66,7 @@ jobs: set -euo pipefail inventory_output="$(npx tsx tools/e2e/workflow-inventory.mts --shell)" allowed_jobs="" + explicit_only_jobs_csv="" free_standing_targets_csv="" free_standing_target_jobs_csv="" seen_inventory_keys="," @@ -69,7 +76,7 @@ jobs: if [[ -z "${line}" || "${line}" == \#* ]]; then continue fi - if [[ ! "${line}" =~ ^(allowed_jobs|free_standing_targets_csv|free_standing_target_jobs_csv)=([A-Za-z0-9_:-]+(,[A-Za-z0-9_:-]+)*)$ ]]; then + if [[ ! "${line}" =~ ^(allowed_jobs|explicit_only_jobs_csv|free_standing_targets_csv|free_standing_target_jobs_csv)=([A-Za-z0-9_:-]+(,[A-Za-z0-9_:-]+)*)?$ ]]; then echo "::error::free-standing workflow inventory must be data-only key=value" >&2 exit 1 fi @@ -82,10 +89,17 @@ jobs: seen_inventory_keys="${seen_inventory_keys}${inventory_key}," case "${inventory_key}" in allowed_jobs) allowed_jobs="${inventory_value}" ;; + explicit_only_jobs_csv) explicit_only_jobs_csv="${inventory_value}" ;; free_standing_targets_csv) free_standing_targets_csv="${inventory_value}" ;; free_standing_target_jobs_csv) free_standing_target_jobs_csv="${inventory_value}" ;; esac done <<< "${inventory_output}" + for required_inventory_key in allowed_jobs explicit_only_jobs_csv free_standing_targets_csv free_standing_target_jobs_csv; do + if [[ "${seen_inventory_keys}" != *",${required_inventory_key},"* ]]; then + echo "::error::free-standing workflow inventory missing ${required_inventory_key}" >&2 + exit 1 + fi + done for required_inventory_key in allowed_jobs free_standing_targets_csv free_standing_target_jobs_csv; do if [[ -z "${!required_inventory_key:-}" ]]; then echo "::error::free-standing workflow inventory missing ${required_inventory_key}" >&2 @@ -105,6 +119,21 @@ jobs: fi seen_allowed_jobs="${seen_allowed_jobs}${job}," done + seen_explicit_only_jobs="," + if [ -n "${explicit_only_jobs_csv}" ]; then + IFS=',' read -r -a explicit_only_job_entries <<< "${explicit_only_jobs_csv}" + for job in "${explicit_only_job_entries[@]}"; do + if [[ "${seen_allowed_jobs}" != *",${job},"* ]]; then + echo "::error::Explicit-only job is not in allowed jobs" >&2 + exit 1 + fi + if [[ "${seen_explicit_only_jobs}" == *",${job},"* ]]; then + echo "::error::free-standing workflow inventory repeats explicit-only job" >&2 + exit 1 + fi + seen_explicit_only_jobs="${seen_explicit_only_jobs}${job}," + done + fi seen_free_standing_targets="," IFS=',' read -r -a free_standing_target_entries <<< "${free_standing_targets_csv}" for target in "${free_standing_target_entries[@]}"; do @@ -204,6 +233,7 @@ jobs: fi echo "matrix=${matrix}" >> "$GITHUB_OUTPUT" echo "hermes_selected=${hermes_selected}" >> "$GITHUB_OUTPUT" + echo "explicit_only_jobs=${explicit_only_jobs_csv}" >> "$GITHUB_OUTPUT" MATRIX_JSON="${matrix}" python3 - <<'PY' >> "$GITHUB_STEP_SUMMARY" import json import os @@ -2531,6 +2561,7 @@ jobs: timeout-minutes: 60 env: E2E_JOB: "1" + E2E_DEFAULT_ENABLED: "0" E2E_TARGET_ID: "sandbox-rlimits-connect" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/sandbox-rlimits-connect NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js @@ -3007,12 +3038,13 @@ jobs: needs: generate-matrix # Explicit-only until a stable Jetson runner is available; otherwise full-suite dispatches remain queued forever. # Required validation path: dispatch with jobs=jetson-nvmap-gpu or targets=jetson-nvmap-gpu. - # Re-enable default dispatch only after a stable Jetson runner exists, then remove this job from FULL_SUITE_EXCLUDED_FREE_STANDING_JOBS. + # Re-enable default dispatch only after a stable Jetson runner exists, then remove E2E_DEFAULT_ENABLED. if: ${{ contains(format(',{0},', inputs.jobs), ',jetson-nvmap-gpu,') || contains(format(',{0},', inputs.targets), ',jetson-nvmap-gpu,') }} runs-on: ${{ vars.JETSON_E2E_RUNNER_LABEL || 'linux-arm64-gpu-jetson-orin-latest-1' }} timeout-minutes: 60 env: E2E_JOB: "1" + E2E_DEFAULT_ENABLED: "0" E2E_TARGET_ID: "jetson-nvmap-gpu" DOCKER_CONFIG: ${{ github.workspace }}/.docker-config-jetson-nvmap-gpu E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/jetson-nvmap-gpu @@ -3349,6 +3381,12 @@ jobs: with: persist-credentials: false + - name: Configure cloud-onboard trace directory + shell: bash + run: | + set -euo pipefail + printf 'NEMOCLAW_TRACE_DIR=%s\n' "${RUNNER_TEMP}/nemoclaw-cloud-onboard-traces" >> "${GITHUB_ENV}" + - name: Set up Node uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 with: @@ -3384,6 +3422,33 @@ jobs: test/e2e/live/cloud-onboard.test.ts \ --silent=false --reporter=default + - name: Build trusted cloud-onboard timing summary + if: always() + shell: bash + run: | + set -euo pipefail + python3 scripts/e2e/sanitize-trace-timing.py \ + "${NEMOCLAW_TRACE_DIR}" \ + "${E2E_ARTIFACT_DIR}" + + # The target process must emit full local traces for diagnosis, but those + # traces may contain prompts, environment data, and credential material. + # Keep cleanup as a separate always() step so a sanitizer failure cannot + # bypass it. A runner-level termination is contained by the ephemeral + # GitHub-hosted runner.temp boundary. Remove this step only when the trace + # producer itself emits the allowlisted timing-only schema. + - name: Delete raw cloud-onboard traces + if: always() + shell: bash + run: | + set -euo pipefail + expected_trace_dir="${RUNNER_TEMP}/nemoclaw-cloud-onboard-traces" + if [ -z "${RUNNER_TEMP}" ] || [ "${NEMOCLAW_TRACE_DIR}" != "${expected_trace_dir}" ]; then + echo "::error::Refusing to delete unexpected raw trace path" >&2 + exit 1 + fi + rm -rf -- "${NEMOCLAW_TRACE_DIR}" + - name: Upload cloud-onboard artifacts if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -5879,6 +5944,134 @@ jobs: rm -rf -- "${DOCKER_CONFIG}" fi + # ── Scheduled failure routing ────────────────────────────────────────────── + notify-on-failure: + runs-on: ubuntu-latest + needs: + [ + generate-matrix, + live, + openshell-version-pin, + onboard-negative-paths, + skill-agent, + openclaw-skill-cli, + inference-routing, + cloud-inference, + gpu-e2e, + agent-turn-latency, + kimi-inference-compat, + hermes-inference-switch, + brave-search, + ollama-auth-proxy, + cron-preflight-inference-local, + credential-sanitization, + credential-migration, + sessions-agents-cli, + runtime-overrides, + hermes-e2e, + hermes-dashboard, + hermes-slack, + hermes-discord, + hermes-root-entrypoint-smoke, + hermes-sandbox-secret-boundary, + network-policy, + common-egress-agent, + shields-config, + rebuild-openclaw, + rebuild-hermes, + rebuild-hermes-stale-base, + sandbox-rebuild, + sandbox-rlimits-connect, + overlayfs-autofix, + state-backup-restore, + upgrade-stale-sandbox, + openshell-gateway-upgrade, + token-rotation, + messaging-compatible-endpoint, + messaging-providers, + launchable-smoke, + double-onboard, + jetson-nvmap-gpu, + concurrent-gateway-ports, + full-e2e, + cloud-onboard, + gpu-double-onboard, + onboard-repair, + issue-4462-scope-upgrade-approval, + onboard-resume, + model-router-provider-routed-inference, + sandbox-operations, + sandbox-survival, + diagnostics, + snapshot-commands, + gateway-drift-preflight, + openclaw-tui-chat-correlation, + gateway-guard-recovery, + issue-4434-tui-unreachable-inference, + openclaw-inference-switch, + bedrock-runtime-compatible-anthropic, + issue-2478-crash-loop-recovery, + gateway-health-honest, + device-auth-health, + channels-add-remove, + tunnel-lifecycle, + telegram-injection, + openclaw-discord-pairing, + openclaw-slack-pairing, + channels-stop-start, + spark-install, + ] + if: ${{ always() && github.event_name == 'schedule' && (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) }} + permissions: + issues: write + steps: + - name: Create or update scheduled E2E failure issue + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + // Preserve the existing issue thread and its historical title. + const title = 'Nightly E2E failed'; + const needs = ${{ toJSON(needs) }}; + const failed = Object.entries(needs) + .filter(([, value]) => value.result === 'failure') + .map(([name]) => name); + const cancelled = Object.entries(needs) + .filter(([, value]) => value.result === 'cancelled') + .map(([name]) => name); + const summary = [ + failed.length ? `**Failed:** ${failed.join(', ')}` : '', + cancelled.length ? `**Cancelled:** ${cancelled.join(', ')}` : '', + ].filter(Boolean).join('\n'); + const body = `**Run:** ${runUrl}\n${summary}\n**Artifacts:** Check the run artifacts for target logs and structured evidence.`; + + const { data: existing } = await github.rest.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + labels: 'CI/CD', + per_page: 100, + }); + const match = existing.find( + (issue) => !issue.pull_request && issue.title.startsWith(title), + ); + if (match) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: match.number, + body: `Failed again on ${new Date().toISOString().split('T')[0]}.\n\n${body}`, + }); + } else { + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: `${title} — ${new Date().toISOString().split('T')[0]}`, + body: `The scheduled E2E pipeline failed.\n\n${body}`, + labels: ['bug', 'CI/CD'], + }); + } + # ── PR result comment ───────────────────────────────────────────────────── # Posts a results table on the open PR for the dispatching branch (or the # PR identified by `inputs.pr_number`). `if: always()` so the comment lands @@ -5971,6 +6164,7 @@ jobs: - name: Post E2E target results to PR uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: + EXPLICIT_ONLY_JOBS: ${{ needs.generate-matrix.outputs.explicit_only_jobs }} JOB_PR_NUMBER: ${{ inputs.pr_number }} JOB_TARGETS: ${{ inputs.targets }} JOBS: ${{ inputs.jobs }} @@ -5985,18 +6179,26 @@ jobs: const selectorValidationPassed = needs['generate-matrix']?.result === 'success'; const requestedTargets = selectorValidationPassed ? rawRequestedTargets : ''; const requestedJobs = selectorValidationPassed ? rawRequestedJobs : ''; - const explicitOnlySkippedJobs = [ - { + const explicitOnlyReasons = { + 'jetson-nvmap-gpu': { job: 'jetson-nvmap-gpu', target: 'jetson-nvmap-gpu', reason: 'default dispatch excludes Jetson until a stable Jetson runner is available', }, - { + 'sandbox-rlimits-connect': { job: 'sandbox-rlimits-connect', target: 'sandbox-rlimits-connect', reason: 'default dispatch excludes the destructive rlimit fork/connect probe unless selected', }, - ]; + }; + const explicitOnlySkippedJobs = (process.env.EXPLICIT_ONLY_JOBS || '') + .split(',') + .filter(Boolean) + .map((job) => explicitOnlyReasons[job] ?? { + job, + target: job, + reason: 'default dispatch excludes this explicit-only job unless selected', + }); const targetsRejected = rawRequestedTargets && !selectorValidationPassed; const jobsRejected = rawRequestedJobs && !selectorValidationPassed; @@ -6141,3 +6343,260 @@ jobs: issue_number: prNumber, body: lines.join('\n'), }); + + # ── Scheduled/manual scorecard ──────────────────────────────────────────── + scorecard: + runs-on: ubuntu-latest + needs: + [ + generate-matrix, + live, + openshell-version-pin, + onboard-negative-paths, + skill-agent, + openclaw-skill-cli, + inference-routing, + cloud-inference, + gpu-e2e, + agent-turn-latency, + kimi-inference-compat, + hermes-inference-switch, + brave-search, + ollama-auth-proxy, + cron-preflight-inference-local, + credential-sanitization, + credential-migration, + sessions-agents-cli, + runtime-overrides, + hermes-e2e, + hermes-dashboard, + hermes-slack, + hermes-discord, + hermes-root-entrypoint-smoke, + hermes-sandbox-secret-boundary, + network-policy, + common-egress-agent, + shields-config, + rebuild-openclaw, + rebuild-hermes, + rebuild-hermes-stale-base, + sandbox-rebuild, + sandbox-rlimits-connect, + overlayfs-autofix, + state-backup-restore, + upgrade-stale-sandbox, + openshell-gateway-upgrade, + token-rotation, + messaging-compatible-endpoint, + messaging-providers, + launchable-smoke, + double-onboard, + jetson-nvmap-gpu, + concurrent-gateway-ports, + full-e2e, + cloud-onboard, + gpu-double-onboard, + onboard-repair, + issue-4462-scope-upgrade-approval, + onboard-resume, + model-router-provider-routed-inference, + sandbox-operations, + sandbox-survival, + diagnostics, + snapshot-commands, + gateway-drift-preflight, + openclaw-tui-chat-correlation, + gateway-guard-recovery, + issue-4434-tui-unreachable-inference, + openclaw-inference-switch, + bedrock-runtime-compatible-anthropic, + issue-2478-crash-loop-recovery, + gateway-health-honest, + device-auth-health, + channels-add-remove, + tunnel-lifecycle, + telegram-injection, + openclaw-discord-pairing, + openclaw-slack-pairing, + channels-stop-start, + spark-install, + ] + if: ${{ always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }} + permissions: + actions: read + contents: read + steps: + - name: Checkout scorecard builders + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + sparse-checkout: scripts/scorecard + sparse-checkout-cone-mode: false + + - name: Generate E2E scorecard + id: scorecard + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + EXPLICIT_ONLY_JOBS: ${{ needs.generate-matrix.outputs.explicit_only_jobs }} + JOBS: ${{ inputs.jobs }} + TARGETS: ${{ inputs.targets }} + with: + script: | + const path = require('path'); + const traceTiming = require( + path.join(process.env.GITHUB_WORKSPACE, 'scripts/scorecard/analyze-trace-timing.ts'), + ); + const scorecardJobs = require( + path.join(process.env.GITHUB_WORKSPACE, 'scripts/scorecard/summarize-jobs.ts'), + ); + const slackBlocks = require( + path.join(process.env.GITHUB_WORKSPACE, 'scripts/scorecard/build-slack-blocks.ts'), + ); + const needs = ${{ toJSON(needs) }}; + const safeSelector = /^[A-Za-z0-9_-]+$/; + const parseSelectors = (value) => + value.split(',').map((name) => name.trim()).filter((name) => safeSelector.test(name)); + const rawRequestedJobs = process.env.JOBS || ''; + const rawRequestedTargets = process.env.TARGETS || ''; + const requestedJobs = parseSelectors(rawRequestedJobs); + const requestedTargets = parseSelectors(rawRequestedTargets); + const isDispatch = context.eventName === 'workflow_dispatch'; + const isSelectiveDispatch = scorecardJobs.isSelectiveDispatch( + context.eventName, + rawRequestedJobs, + rawRequestedTargets, + ); + const runMode = isSelectiveDispatch + ? 'Selective dispatch' + : isDispatch + ? 'Manual full run' + : 'Scheduled E2E'; + const metaJobs = [ + 'generate-matrix', + 'notify-on-failure', + 'report-to-pr', + 'scorecard', + ]; + const explicitOnly = parseSelectors(process.env.EXPLICIT_ONLY_JOBS || ''); + const explicitlySelected = [...requestedJobs, ...requestedTargets]; + + // GitHub's jobs API is the canonical source because `needs.live` + // collapses every matrix target into one result and has no job URL. + // The typed helper owns and tests the degraded `needs` fallback. + const apiJobs = await scorecardJobs.loadWorkflowRunJobs({ github, context, core }); + + const { cancelled, failedJobs, failure, ran, skipped, success, total } = + scorecardJobs.summarizeJobs({ + apiJobs, + explicitOnlyJobNames: explicitOnly, + explicitlySelected, + metaJobNames: metaJobs, + needs, + }); + const perfect = ran > 0 && failure === 0 && cancelled === 0; + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const today = new Date().toLocaleDateString('en-US', { + month: 'short', + day: 'numeric', + }); + const { traceTimingLine, traceSummaryLines } = + await traceTiming.buildTraceTimingResult({ github, context }); + const lines = [ + `## 🌅 NemoClaw E2E Scorecard — ${today}`, + '', + `**Run mode:** ${runMode}`, + ]; + if (requestedJobs.length > 0) { + lines.push(`**Requested jobs:** ${requestedJobs.map((name) => `\`${name}\``).join(', ')}`); + } + if (requestedTargets.length > 0) { + lines.push(`**Requested targets:** ${requestedTargets.map((name) => `\`${name}\``).join(', ')}`); + } + lines.push( + `**Jobs run:** ${ran} of ${total}`, + ` ✅ ${success} passed`, + ` ❌ ${failure} failed`, + ` 🚫 ${cancelled} cancelled`, + ` ⏭️ ${skipped} skipped`, + ); + if (failedJobs.length > 0) { + lines.push('', '**Failed jobs:**'); + for (const job of failedJobs) { + lines.push(job.url ? ` - [${job.name}](${job.url})` : ` - \`${job.name}\``); + } + } + if (perfect) lines.push('', '🎉 **All jobs passed!**'); + lines.push('', traceTimingLine, ...traceSummaryLines, '', `🔗 [Full run details](${runUrl})`); + await core.summary.addRaw(lines.join('\n')).write(); + const scorecardData = { + today, + runMode, + actor: context.actor || '', + isSelectiveDispatch, + requestedJobs, + requestedTargets, + total, + ran, + success, + failure, + cancelled, + skipped, + perfect, + failedJobs, + traceTimingLine, + runUrl, + }; + core.setOutput('scorecardData', JSON.stringify(scorecardData)); + core.setOutput('slackData', JSON.stringify({ + channel: slackBlocks.getSlackChannel(scorecardData), + payload: { + text: slackBlocks.buildFallbackText(scorecardData), + attachments: [{ + color: slackBlocks.getStatusColor(scorecardData), + blocks: slackBlocks.buildBlocks(scorecardData), + }], + }, + })); + + - name: Post scorecard to Slack + # Webhook secrets never enter branch-dispatched runs. The payload is + # computed in the preceding no-secret step; this fixed publisher does + # not load code from the checked-out workflow ref. + if: ${{ steps.scorecard.outputs.slackData != '' && github.ref == 'refs/heads/main' }} + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + SLACK_WEBHOOK_URL_DAILY: ${{ secrets.SLACK_WEBHOOK_URL_DAILY }} + SLACK_WEBHOOK_URL_FULLRUN: ${{ secrets.SLACK_WEBHOOK_URL_FULLRUN }} + SLACK_WEBHOOK_URL_PREVIEW: ${{ secrets.SLACK_WEBHOOK_URL_PREVIEW }} + SLACK_DATA: ${{ steps.scorecard.outputs.slackData }} + POST_TO_SLACK: ${{ inputs.post_to_slack }} + with: + script: | + const data = JSON.parse(process.env.SLACK_DATA); + const envByChannel = { + daily: 'SLACK_WEBHOOK_URL_DAILY', + fullrun: 'SLACK_WEBHOOK_URL_FULLRUN', + preview: 'SLACK_WEBHOOK_URL_PREVIEW', + }; + const channel = data?.channel; + if (!Object.hasOwn(envByChannel, channel) || !data.payload || typeof data.payload !== 'object') { + core.setFailed('Invalid precomputed Slack payload'); + return; + } + if (channel === 'preview' && process.env.POST_TO_SLACK !== 'true') { + core.info('Selective dispatch without post_to_slack — skipping'); + return; + } + const webhookUrl = process.env[envByChannel[channel]]; + if (!webhookUrl) { + core.info(`Slack webhook for "${channel}" not configured — skipping`); + return; + } + const response = await fetch(webhookUrl, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(data.payload), + }); + if (!response.ok) { + core.setFailed(`Slack webhook returned ${response.status}`); + } diff --git a/scripts/e2e/sanitize-trace-timing.py b/scripts/e2e/sanitize-trace-timing.py new file mode 100755 index 00000000000..6a1237363ae --- /dev/null +++ b/scripts/e2e/sanitize-trace-timing.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Reduce raw NemoClaw traces to a timing-only scorecard artifact. + +The E2E target controls the raw trace directory, so CI must never upload it. +This script accepts only the onboard timing shape needed by the scorecard and +writes a single allowlisted summary without attributes, events, paths, prompts, +environment data, or raw error messages. +""" + +from __future__ import annotations + +import json +import math +import os +import re +import sys +from pathlib import Path +from typing import Any + +SCHEMA_VERSION = "nemoclaw.trace_timing.v1" +OUTPUT_FILE = "cloud-onboard-trace-timing-summary.json" +ONBOARD_ROOT_SPAN = "nemoclaw.onboard" +ONBOARD_PHASE_PREFIX = "nemoclaw.onboard.phase." +ONBOARD_PHASE_NAMES = { + f"{ONBOARD_PHASE_PREFIX}preflight", + f"{ONBOARD_PHASE_PREFIX}gateway", + f"{ONBOARD_PHASE_PREFIX}provider_selection", + f"{ONBOARD_PHASE_PREFIX}inference", + f"{ONBOARD_PHASE_PREFIX}sandbox", +} +MAX_JSON_FILES = 100 +MAX_JSON_BYTES = 2 * 1024 * 1024 +MAX_SLOWEST_SPANS = 10 +TRACE_ID_RE = re.compile(r"^[0-9a-f]{32}$") +STATUS_VALUES = {"OK", "ERROR", "UNSET"} + + +def finite_number(value: Any) -> float | None: + if isinstance(value, bool): + return None + try: + number = float(value) + except (TypeError, ValueError): + return None + if not math.isfinite(number) or number < 0: + return None + return number + + +def safe_status(value: Any) -> str: + return value if isinstance(value, str) and value in STATUS_VALUES else "UNSET" + + +def safe_span_name(value: Any) -> str | None: + if not isinstance(value, str): + return None + if value == ONBOARD_ROOT_SPAN or value in ONBOARD_PHASE_NAMES: + return value + return None + + +def iter_json_files(source: Path) -> list[Path]: + if not source.exists(): + return [] + if source.is_file(): + return [source] if source.suffix == ".json" and not source.is_symlink() else [] + if not source.is_dir() or source.is_symlink(): + return [] + files: list[Path] = [] + for path in sorted(source.rglob("*.json")): + if path.is_file() and not path.is_symlink(): + files.append(path) + if len(files) >= MAX_JSON_FILES: + break + return files + + +def load_json(path: Path) -> Any | None: + try: + if path.stat().st_size > MAX_JSON_BYTES: + return None + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError): + return None + + +def first_dict(values: Any) -> dict[str, Any]: + if isinstance(values, list) and values and isinstance(values[0], dict): + return values[0] + return {} + + +def extract_spans(artifact: Any) -> list[dict[str, Any]]: + if not isinstance(artifact, dict): + return [] + resource = first_dict(artifact.get("resource_spans")) + scope = first_dict(resource.get("scope_spans")) + spans = scope.get("spans", []) + return [span for span in spans if isinstance(span, dict)] if isinstance(spans, list) else [] + + +def extract_candidate(artifact: Any) -> dict[str, Any] | None: + if not isinstance(artifact, dict): + return None + spans = extract_spans(artifact) + if not any(span.get("name") == ONBOARD_ROOT_SPAN for span in spans): + return None + + summary = artifact.get("summary") if isinstance(artifact.get("summary"), dict) else {} + total_ms = finite_number(summary.get("total_duration_ms")) + if total_ms is None: + return None + + phases: dict[str, float] = {} + for span in spans: + name = span.get("name") + duration_ms = finite_number(span.get("duration_ms")) + if name in ONBOARD_PHASE_NAMES and duration_ms is not None: + phases[name] = phases.get(name, 0.0) + duration_ms + if not phases: + return None + + slowest_spans = [] + raw_slowest = summary.get("slowest_spans", []) + for span in raw_slowest if isinstance(raw_slowest, list) else []: + if not isinstance(span, dict): + continue + name = safe_span_name(span.get("name")) + duration_ms = finite_number(span.get("duration_ms")) + if name is None or duration_ms is None: + continue + slowest_spans.append( + { + "name": name, + "duration_ms": round(duration_ms, 3), + "status": safe_status(span.get("status")), + } + ) + if len(slowest_spans) >= MAX_SLOWEST_SPANS: + break + + trace_id = summary.get("trace_id") + return { + "schema_version": SCHEMA_VERSION, + "trace_id": trace_id if isinstance(trace_id, str) and TRACE_ID_RE.fullmatch(trace_id) else None, + "total_duration_ms": round(total_ms, 3), + "phases": {name: round(phases[name], 3) for name in sorted(phases)}, + "slowest_spans": slowest_spans, + } + + +def main(argv: list[str]) -> int: + if len(argv) != 3: + print("usage: sanitize-trace-timing.py ", file=sys.stderr) + return 2 + + source_input = Path(argv[1]).absolute() + if source_input.is_symlink(): + print("trace source must not be a symlink", file=sys.stderr) + return 2 + source = source_input.resolve(strict=False) + output_dir = Path(argv[2]).absolute() + if source == output_dir.resolve(strict=False): + print("trace source and trusted output directory must be distinct", file=sys.stderr) + return 2 + + if output_dir.is_symlink() or (output_dir.exists() and not output_dir.is_dir()): + print("trusted output must be a real directory", file=sys.stderr) + return 2 + output_dir.mkdir(parents=True, exist_ok=True, mode=0o700) + + candidates = [] + for json_file in iter_json_files(source): + candidate = extract_candidate(load_json(json_file)) + if candidate is not None: + candidates.append(candidate) + if not candidates: + print("No valid NemoClaw onboard trace found; no timing summary emitted.") + return 0 + + selected = max(candidates, key=lambda item: item["total_duration_ms"]) + output = output_dir / OUTPUT_FILE + if output.is_symlink(): + print("trusted timing summary must not be a symlink", file=sys.stderr) + return 2 + output.write_text(json.dumps(selected, indent=2, sort_keys=True) + "\n", encoding="utf-8") + os.chmod(output, 0o600) + print(f"Wrote trusted trace timing summary: {output}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/scripts/scorecard/analyze-trace-timing.ts b/scripts/scorecard/analyze-trace-timing.ts new file mode 100644 index 00000000000..b69acbb8e2d --- /dev/null +++ b/scripts/scorecard/analyze-trace-timing.ts @@ -0,0 +1,359 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +const fs = require("node:fs") as typeof import("node:fs"); +const os = require("node:os") as typeof import("node:os"); +const path = require("node:path") as typeof import("node:path"); +const { execFileSync } = require("node:child_process") as typeof import("node:child_process"); + +const WORKFLOW_FILE = "e2e.yaml"; +const TRACE_ARTIFACT_NAME = "e2e-cloud-onboard"; +const TRACE_SUMMARY_FILE = "cloud-onboard-trace-timing-summary.json"; +const ONBOARD_PHASE_PREFIX = "nemoclaw.onboard.phase."; +const ONBOARD_PHASE_ORDER = [ + "nemoclaw.onboard.phase.preflight", + "nemoclaw.onboard.phase.gateway", + "nemoclaw.onboard.phase.provider_selection", + "nemoclaw.onboard.phase.inference", + "nemoclaw.onboard.phase.sandbox", +] as const; +const ONBOARD_PHASE_NAMES = new Set(ONBOARD_PHASE_ORDER); + +type SemverTag = { + major: number; + minor: number; + name: string; + patch: number; +}; + +type ReleaseTag = SemverTag & { sha: string }; + +type TimingSummaryArtifact = { + phases?: unknown; + schema_version?: unknown; + total_duration_ms?: unknown; +}; + +type OnboardTraceSummary = { + artifact: TimingSummaryArtifact; + phases: Record; + totalMs: number; +}; + +type PhaseRow = { + currentMs: number; + deltaAbsMs: number; + deltaMs: number; + label: string; + name: string; + priorMs: number; +}; + +type GitHubDeps = { + context: any; + github: any; +}; + +type TraceTimingResult = { + traceSummaryLines: string[]; + traceTimingLine: string; +}; + +type TraceTimingServices = { + findLatestCompletedE2eRunForReleaseTag: ( + deps: GitHubDeps, + tag: ReleaseTag, + ) => Promise<{ id: number } | null>; + readTraceSummaryFromRun: (deps: GitHubDeps, runId: number) => Promise; + resolvePriorReleaseTag: (deps: GitHubDeps) => Promise; +}; + +function parseSemverTag(name: string): SemverTag | null { + const match = /^v(\d+)\.(\d+)\.(\d+)$/.exec(name); + if (!match) return null; + return { + name, + major: Number(match[1]), + minor: Number(match[2]), + patch: Number(match[3]), + }; +} + +function compareSemverDesc(a: SemverTag, b: SemverTag): number { + return b.major - a.major || b.minor - a.minor || b.patch - a.patch; +} + +function formatDuration(ms: number): string { + if (!Number.isFinite(ms)) return "unknown"; + if (ms < 1000) return `${ms.toFixed(0)}ms`; + const seconds = ms / 1000; + if (seconds < 60) return `${seconds.toFixed(1)}s`; + const minutes = Math.floor(seconds / 60); + const remaining = seconds - minutes * 60; + return `${minutes}m ${remaining.toFixed(1)}s`; +} + +function formatTraceDelta(currentMs: number, priorMs: number): string { + const deltaMs = currentMs - priorMs; + if (Math.abs(deltaMs) < 1) return "unchanged"; + const direction = deltaMs > 0 ? "increased" : "decreased"; + const sign = deltaMs > 0 ? "+" : "-"; + if (priorMs <= 0) { + return `${direction} ${sign}${formatDuration(Math.abs(deltaMs))} (n/a)`; + } + const pct = (deltaMs / priorMs) * 100; + return `${direction} ${sign}${formatDuration(Math.abs(deltaMs))} (${sign}${Math.abs(pct).toFixed(1)}%)`; +} + +function phaseLabel(name: string): string { + return name.replace(ONBOARD_PHASE_PREFIX, "").replace(/_/g, " "); +} + +function formatPhaseDelta(currentMs: number, priorMs: number): string { + const deltaMs = currentMs - priorMs; + if (Math.abs(deltaMs) < 1) return "±0ms"; + const sign = deltaMs > 0 ? "+" : "-"; + return `${sign}${formatDuration(Math.abs(deltaMs))}`; +} + +function traceTimingResult( + traceTimingLine: string, + traceSummaryLines: string[] = [], +): TraceTimingResult { + return { traceTimingLine, traceSummaryLines }; +} + +function normalizePhaseDurations(value: unknown): Record | null { + if (value === null || typeof value !== "object" || Array.isArray(value)) return null; + const phases: Record = {}; + for (const [name, entry] of Object.entries(value)) { + if (!ONBOARD_PHASE_NAMES.has(name)) continue; + const durationMs = Number(entry); + if (!Number.isFinite(durationMs) || durationMs < 0) return null; + phases[name] = durationMs; + } + return phases; +} + +function selectOnboardTrace(jsonTexts: string[]): OnboardTraceSummary | null { + const candidates: OnboardTraceSummary[] = []; + for (const text of jsonTexts) { + try { + const artifact = JSON.parse(text) as TimingSummaryArtifact; + const totalMs = Number(artifact?.total_duration_ms); + const phases = normalizePhaseDurations(artifact?.phases); + if ( + artifact?.schema_version === "nemoclaw.trace_timing.v1" && + Number.isFinite(totalMs) && + totalMs >= 0 && + phases !== null + ) { + candidates.push({ artifact, totalMs, phases }); + } + } catch { + // Missing or malformed summaries must not hide the E2E pass/fail signal. + } + } + candidates.sort((a, b) => b.totalMs - a.totalMs); + return candidates[0] ?? null; +} + +function buildPhaseRows( + currentPhases: Record, + priorPhases: Record, +): PhaseRow[] { + return ONBOARD_PHASE_ORDER.filter( + (name) => currentPhases[name] !== undefined && priorPhases[name] !== undefined, + ).map((name) => { + const currentMs = currentPhases[name]; + const priorMs = priorPhases[name]; + const deltaMs = currentMs - priorMs; + return { + name, + label: phaseLabel(name), + currentMs, + priorMs, + deltaMs, + deltaAbsMs: Math.abs(deltaMs), + }; + }); +} + +function formatTopPhaseChanges(phaseRows: PhaseRow[]): string { + return phaseRows + .slice() + .sort((a, b) => b.deltaAbsMs - a.deltaAbsMs || a.label.localeCompare(b.label)) + .slice(0, 3) + .map((row) => `${row.label} ${formatPhaseDelta(row.currentMs, row.priorMs)}`) + .join("; "); +} + +function buildTraceSummaryLines( + currentTrace: Pick, + priorTrace: Pick, + priorTag: Pick, + phaseRows: PhaseRow[], +): string[] { + if (phaseRows.length === 0) return []; + const lines = [ + "", + "## Cloud Onboard Trace Timing", + "", + `Total: ${formatDuration(currentTrace.totalMs)}, ${formatTraceDelta(currentTrace.totalMs, priorTrace.totalMs)} vs ${priorTag.name}`, + "", + "| Phase | Current | Previous | Delta |", + "| --- | ---: | ---: | ---: |", + ]; + for (const row of phaseRows) { + lines.push( + `| ${row.label} | ${formatDuration(row.currentMs)} | ${formatDuration(row.priorMs)} | ${formatPhaseDelta(row.currentMs, row.priorMs)} |`, + ); + } + lines.push(""); + lines.push(`Trace artifact: \`${TRACE_ARTIFACT_NAME}\``); + lines.push( + `Baseline: latest completed \`${WORKFLOW_FILE}\` run for prior release tag \`${priorTag.name}\``, + ); + return lines; +} + +async function resolvePriorReleaseTag({ github, context }: GitHubDeps): Promise { + const tags = await github.paginate(github.rest.repos.listTags, { + owner: context.repo.owner, + repo: context.repo.repo, + per_page: 100, + }); + const semverTags: ReleaseTag[] = (tags as any[]) + .map((tag: any): ReleaseTag | null => { + const semverTag = parseSemverTag(tag.name); + return semverTag && tag.commit?.sha ? { ...semverTag, sha: tag.commit.sha } : null; + }) + .filter((tag: ReleaseTag | null): tag is ReleaseTag => tag !== null) + .sort(compareSemverDesc); + if (semverTags.length === 0) return null; + + const currentTag = context.ref?.startsWith("refs/tags/") + ? parseSemverTag(context.ref.replace("refs/tags/", "")) + : null; + if (!currentTag) return semverTags[0]; + const index = semverTags.findIndex((tag: ReleaseTag) => tag.name === currentTag.name); + return index >= 0 ? (semverTags[index + 1] ?? null) : semverTags[0]; +} + +async function findLatestCompletedE2eRunForReleaseTag( + { github, context }: GitHubDeps, + tag: ReleaseTag, +): Promise { + for (let page = 1; page <= 10; page += 1) { + const { data } = await github.rest.actions.listWorkflowRuns({ + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: WORKFLOW_FILE, + head_sha: tag.sha, + status: "completed", + per_page: 100, + page, + }); + const run = data.workflow_runs.find( + (candidate: any) => candidate.id !== context.runId && candidate.status === "completed", + ); + if (run) return run; + if (data.workflow_runs.length < 100) break; + } + return null; +} + +async function readTraceSummaryFromRun( + { github, context }: GitHubDeps, + runId: number, +): Promise { + const artifacts = await github.paginate(github.rest.actions.listWorkflowRunArtifacts, { + owner: context.repo.owner, + repo: context.repo.repo, + run_id: runId, + per_page: 100, + }); + const artifact = artifacts.find((item: any) => item.name === TRACE_ARTIFACT_NAME); + if (!artifact) return null; + + const download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: artifact.id, + archive_format: "zip", + }); + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-trace-artifact-")); + try { + const zipPath = path.join(tempDir, `${TRACE_ARTIFACT_NAME}.zip`); + fs.writeFileSync(zipPath, Buffer.from(download.data), { mode: 0o600 }); + const summaryText = execFileSync("unzip", ["-p", zipPath, TRACE_SUMMARY_FILE], { + encoding: "utf8", + maxBuffer: 1024 * 1024, + }); + return selectOnboardTrace([summaryText]); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +} + +async function buildTraceTimingResult( + deps: GitHubDeps, + services: TraceTimingServices = { + findLatestCompletedE2eRunForReleaseTag, + readTraceSummaryFromRun, + resolvePriorReleaseTag, + }, +): Promise { + const { context } = deps; + try { + const currentTrace = await services.readTraceSummaryFromRun(deps, context.runId); + if (currentTrace === null) { + return traceTimingResult(`Trace: ⊘ ${TRACE_ARTIFACT_NAME} timing summary not found`); + } + const priorTag = await services.resolvePriorReleaseTag(deps); + if (!priorTag) { + return traceTimingResult( + `Trace: cloud-onboard total ${formatDuration(currentTrace.totalMs)} (no prior release tag found)`, + ); + } + const priorRun = await services.findLatestCompletedE2eRunForReleaseTag(deps, priorTag); + if (!priorRun) { + return traceTimingResult( + `Trace: cloud-onboard total ${formatDuration(currentTrace.totalMs)} (no e2e.yaml run found for ${priorTag.name})`, + ); + } + const priorTrace = await services.readTraceSummaryFromRun(deps, priorRun.id); + if (priorTrace === null) { + return traceTimingResult( + `Trace: cloud-onboard total ${formatDuration(currentTrace.totalMs)} (no timing summary found for ${priorTag.name})`, + ); + } + const phaseRows = buildPhaseRows(currentTrace.phases, priorTrace.phases); + const traceLine = `Trace: cloud-onboard total ${formatDuration(currentTrace.totalMs)}, ${formatTraceDelta(currentTrace.totalMs, priorTrace.totalMs)} vs ${priorTag.name}.`; + if (phaseRows.length === 0) return traceTimingResult(traceLine); + return traceTimingResult( + [ + traceLine, + `Top phase changes: ${formatTopPhaseChanges(phaseRows)}.`, + "Full phase timing table is in the GitHub run summary.", + ].join(" "), + buildTraceSummaryLines(currentTrace, priorTrace, priorTag, phaseRows), + ); + } catch { + return traceTimingResult("Trace: ⊘ comparison unavailable"); + } +} + +module.exports = { + ONBOARD_PHASE_ORDER, + TRACE_ARTIFACT_NAME, + TRACE_SUMMARY_FILE, + buildPhaseRows, + buildTraceTimingResult, + buildTraceSummaryLines, + findLatestCompletedE2eRunForReleaseTag, + formatTopPhaseChanges, + readTraceSummaryFromRun, + resolvePriorReleaseTag, + selectOnboardTrace, +}; diff --git a/scripts/scorecard/build-slack-blocks.ts b/scripts/scorecard/build-slack-blocks.ts new file mode 100644 index 00000000000..28edd228ca9 --- /dev/null +++ b/scripts/scorecard/build-slack-blocks.ts @@ -0,0 +1,161 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** Pure Slack payload builder for the consolidated E2E scorecard. */ + +type ScorecardRunMode = "Scheduled E2E" | "Manual full run" | "Selective dispatch" | (string & {}); + +type ScorecardData = { + today: string; + runMode: ScorecardRunMode; + actor?: string; + isSelectiveDispatch: boolean; + requestedJobs: string[]; + requestedTargets: string[]; + total: number; + ran: number; + success: number; + failure: number; + cancelled: number; + skipped: number; + perfect: boolean; + failedJobs: { name: string; url: string | null }[]; + traceTimingLine?: string; + runUrl: string; +}; + +type SlackMrkdwnText = { type: "mrkdwn"; text: string }; +type SlackPlainText = { type: "plain_text"; text: string; emoji?: boolean }; +type SlackContextBlock = { type: "context"; elements: SlackMrkdwnText[] }; +type SlackSectionBlock = { type: "section"; text: SlackMrkdwnText }; +type SlackButtonElement = { + type: "button"; + text: SlackPlainText; + url: string; + style?: "primary" | "danger"; +}; +type SlackActionsBlock = { type: "actions"; elements: SlackButtonElement[] }; +type SlackBlock = SlackActionsBlock | SlackContextBlock | SlackSectionBlock; + +function buildBlocks(data: ScorecardData): SlackBlock[] { + const blocks: SlackBlock[] = []; + const showActor = data.runMode !== "Scheduled E2E" && Boolean(data.actor); + const runModeText = showActor ? `${data.runMode} (by *${data.actor}*)` : data.runMode; + const contextElements: SlackMrkdwnText[] = [ + { type: "mrkdwn", text: `*Run mode:* ${runModeText}` }, + ]; + if (data.isSelectiveDispatch) { + const selectors = [ + ...data.requestedJobs.map((name) => `job:\`${name}\``), + ...data.requestedTargets.map((name) => `target:\`${name}\``), + ]; + if (selectors.length > 0) { + contextElements.push({ type: "mrkdwn", text: `*Requested:* ${selectors.join(", ")}` }); + } + } + blocks.push({ type: "context", elements: contextElements }); + + blocks.push({ + type: "section", + text: { + type: "mrkdwn", + text: [ + `*Total ran:* ${data.ran}/${data.total}`, + `:white_check_mark: *Passed:* ${data.success}`, + `:x: *Failed:* ${data.failure}`, + `:no_entry_sign: *Cancelled:* ${data.cancelled}`, + `:fast_forward: *Skipped:* ${data.skipped}`, + ].join(" · "), + }, + }); + + if (data.perfect) { + blocks.push({ + type: "section", + text: { type: "mrkdwn", text: ":tada: *All jobs passed!*" }, + }); + } else if (data.failedJobs.length > 0) { + const list = data.failedJobs + .map((job) => (job.url ? `• <${job.url}|${job.name}>` : `• \`${job.name}\``)) + .join("\n"); + blocks.push({ + type: "section", + text: { + type: "mrkdwn", + text: `*Failed jobs (${data.failedJobs.length}):*\n${list}`, + }, + }); + } + + if (data.traceTimingLine) { + blocks.push({ + type: "section", + text: { + type: "mrkdwn", + text: data.traceTimingLine.replace(/^Trace:\s*/, "*Trace:* "), + }, + }); + } + + const workflowUrl = data.runUrl.replace(/\/runs\/\d+$/, "/workflows/e2e.yaml"); + blocks.push({ + type: "actions", + elements: [ + { + type: "button", + text: { type: "plain_text", text: "View this run", emoji: true }, + url: data.runUrl, + style: data.perfect ? "primary" : "danger", + }, + { + type: "button", + text: { type: "plain_text", text: "All E2E runs", emoji: true }, + url: workflowUrl, + }, + ], + }); + return blocks; +} + +function buildFallbackText(data: ScorecardData): string { + let modeSegment: string; + switch (data.runMode) { + case "Scheduled E2E": + modeSegment = "🗓️ DAILY"; + break; + case "Manual full run": + modeSegment = data.actor ? `🛠 Manual full by ${data.actor}` : "🛠 Manual full"; + break; + case "Selective dispatch": + modeSegment = data.actor ? `🛠 Selective by ${data.actor}` : "🛠 Selective"; + break; + default: + modeSegment = data.runMode; + } + return `🌅 *NemoClaw E2E Scorecard · ${modeSegment} · ${data.today}*`; +} + +type SlackStatusColor = "danger" | "good" | "warning"; + +function getStatusColor(data: ScorecardData): SlackStatusColor { + if (data.failure > 0) return "danger"; + if (data.perfect) return "good"; + return "warning"; +} + +type SlackChannel = "daily" | "fullrun" | "preview"; + +function getSlackChannel(data: ScorecardData): SlackChannel { + if (data.runMode === "Scheduled E2E") return "daily"; + if (data.runMode === "Manual full run") return "fullrun"; + return "preview"; +} + +module.exports = { + buildBlocks, + buildFallbackText, + getSlackChannel, + getStatusColor, +}; + +export type { ScorecardData, SlackBlock, SlackChannel, SlackStatusColor }; diff --git a/scripts/scorecard/summarize-jobs.ts b/scripts/scorecard/summarize-jobs.ts new file mode 100644 index 00000000000..684ecfc7041 --- /dev/null +++ b/scripts/scorecard/summarize-jobs.ts @@ -0,0 +1,165 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +type ApiJob = { + completed_at?: string | null; + conclusion?: string | null; + html_url?: string | null; + name: string; + run_attempt?: number | null; + status?: string | null; +}; + +type NeedResult = { result?: string }; + +type FailedJob = { name: string; url: string | null }; + +export type JobSummary = { + cancelled: number; + failedJobs: FailedJob[]; + failure: number; + ran: number; + skipped: number; + success: number; + total: number; +}; + +export type SummarizeJobsInput = { + apiJobs: ApiJob[] | null; + explicitOnlyJobNames: string[]; + explicitlySelected: string[]; + metaJobNames: string[]; + needs: Record; +}; + +export type WorkflowRunJobsDeps = { + context: { + repo: { owner: string; repo: string }; + runId: number; + }; + core: { warning: (message: string) => void }; + github: { + paginate: (method: unknown, parameters: Record) => Promise; + rest: { actions: { listJobsForWorkflowRun: unknown } }; + }; +}; + +type CountedResult = "cancelled" | "failure" | "skipped" | "success"; + +function isSelectiveDispatch(eventName: string, rawJobs = "", rawTargets = ""): boolean { + return eventName === "workflow_dispatch" && (rawJobs.trim() !== "" || rawTargets.trim() !== ""); +} + +function classifyApiJob(job: ApiJob): CountedResult { + if (job.conclusion === "success") return "success"; + if (job.conclusion === "failure") return "failure"; + if (job.conclusion === "cancelled") return "cancelled"; + if (job.conclusion === "skipped" || job.status !== "completed") return "skipped"; + return "failure"; +} + +function classifyNeed(value: NeedResult): CountedResult { + if (value.result === "success") return "success"; + if (value.result === "failure") return "failure"; + if (value.result === "cancelled") return "cancelled"; + if (value.result === "skipped") return "skipped"; + return "failure"; +} + +function countResults(results: CountedResult[]): Omit { + return { + cancelled: results.filter((result) => result === "cancelled").length, + failure: results.filter((result) => result === "failure").length, + skipped: results.filter((result) => result === "skipped").length, + success: results.filter((result) => result === "success").length, + }; +} + +function preferCandidate(candidate: ApiJob, existing: ApiJob | undefined): boolean { + if (!existing) return true; + const candidateAttempt = candidate.run_attempt ?? 0; + const existingAttempt = existing.run_attempt ?? 0; + if (candidateAttempt !== existingAttempt) return candidateAttempt > existingAttempt; + return (candidate.completed_at ?? "") > (existing.completed_at ?? ""); +} + +function normalizeApiJobs( + apiJobs: ApiJob[], + metaJobs: Set, + explicitOnly: Set, + selected: Set, +): ApiJob[] { + const dedupedByName = new Map(); + for (const job of apiJobs) { + const name = job.name.replace(/ \/ [^/]+$/u, ""); + if (metaJobs.has(name)) continue; + if (explicitOnly.has(name) && !selected.has(name)) continue; + const candidate = { ...job, name }; + if (preferCandidate(candidate, dedupedByName.get(name))) { + dedupedByName.set(name, candidate); + } + } + return [...dedupedByName.values()].sort((left, right) => left.name.localeCompare(right.name)); +} + +async function loadWorkflowRunJobs({ + context, + core, + github, +}: WorkflowRunJobsDeps): Promise { + try { + return await github.paginate(github.rest.actions.listJobsForWorkflowRun, { + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId, + per_page: 100, + }); + } catch (error) { + const status = + error !== null && typeof error === "object" && "status" in error + ? String(error.status) + : "unknown"; + const message = error instanceof Error ? error.message : String(error); + core.warning( + `Could not fetch jobs from API (status ${status}); falling back to needs context. Reason: ${message.slice(0, 200)}`, + ); + return null; + } +} + +function summarizeJobs(input: SummarizeJobsInput): JobSummary { + const metaJobs = new Set(input.metaJobNames); + const explicitOnly = new Set(input.explicitOnlyJobNames); + const selected = new Set(input.explicitlySelected); + + if (input.apiJobs !== null) { + const jobs = normalizeApiJobs(input.apiJobs, metaJobs, explicitOnly, selected); + const classified = jobs.map((job) => ({ job, result: classifyApiJob(job) })); + const counts = countResults(classified.map(({ result }) => result)); + return { + ...counts, + failedJobs: classified + .filter(({ result }) => result === "failure") + .map(({ job }) => ({ name: job.name, url: job.html_url ?? null })), + ran: jobs.length - counts.skipped, + total: jobs.length, + }; + } + + const entries = Object.entries(input.needs) + .filter(([name]) => !metaJobs.has(name)) + .filter(([name]) => !explicitOnly.has(name) || selected.has(name)) + .sort(([left], [right]) => left.localeCompare(right)); + const classified = entries.map(([name, value]) => ({ name, result: classifyNeed(value) })); + const counts = countResults(classified.map(({ result }) => result)); + return { + ...counts, + failedJobs: classified + .filter(({ result }) => result === "failure") + .map(({ name }) => ({ name, url: null })), + ran: entries.length - counts.skipped, + total: entries.length, + }; +} + +module.exports = { isSelectiveDispatch, loadWorkflowRunJobs, summarizeJobs }; diff --git a/test/e2e/README.md b/test/e2e/README.md index ded76c661a5..1cb06ab5717 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -18,3 +18,23 @@ before those targets run; local runners must provide it themselves. The former top-level `test/e2e/test-*.sh` suite has been removed. Keep real shell, installer, process, Docker, OpenShell, `/proc`, and sandbox boundaries in E2E tests when those boundaries are the behavior under test. + +## Scheduled operations + +The consolidated workflow keeps its operational reporting in the same job +graph as the live targets: + +- `notify-on-failure` creates or updates the open `CI/CD` failure issue only + when a scheduled run fails or is cancelled. +- `scorecard` writes the scheduled/manual result summary, compares the trusted + cloud-onboard timing summary with the latest prior-release `e2e.yaml` run, + and posts to the daily or full-run Slack route. +- Selective dispatches remain silent unless they run on `main` with + `post_to_slack=true`, which uses the preview Slack route. Branch-dispatched + runs never receive Slack webhook secrets. + +Raw cloud-onboard traces stay under the runner temporary directory. Before +artifact upload, `scripts/e2e/sanitize-trace-timing.py` reduces them to the +allowlisted `cloud-onboard-trace-timing-summary.json` timing schema and deletes +the raw directory. Aggregation ratchets require `notify-on-failure`, +`report-to-pr`, and `scorecard` to wait for the same execution-job set. diff --git a/test/e2e/fixtures/availability-env.ts b/test/e2e/fixtures/availability-env.ts index 917a031c73a..f3f83c1ba9c 100644 --- a/test/e2e/fixtures/availability-env.ts +++ b/test/e2e/fixtures/availability-env.ts @@ -12,6 +12,7 @@ const AVAILABILITY_PROBE_EXTRA_ENV_KEYS = [ "DOCKER_API_VERSION", "XDG_RUNTIME_DIR", "NEMOCLAW_OLLAMA_PULL_TIMEOUT", + "NEMOCLAW_TRACE_DIR", ]; export function buildAvailabilityProbeEnv( diff --git a/test/e2e/support/e2e-operations-workflow-boundary.test.ts b/test/e2e/support/e2e-operations-workflow-boundary.test.ts new file mode 100644 index 00000000000..698d7f08147 --- /dev/null +++ b/test/e2e/support/e2e-operations-workflow-boundary.test.ts @@ -0,0 +1,182 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { describe, expect, it, vi } from "vitest"; + +import { + readE2eOperationsWorkflow, + validateE2eOperationsWorkflow, + validateE2eOperationsWorkflowBoundary, +} from "../../../tools/e2e/operations-workflow-boundary.mts"; + +const AsyncFunction = Object.getPrototypeOf(async () => undefined).constructor as new ( + ...parameters: string[] +) => (...args: unknown[]) => Promise; + +function workflowScript(jobName: string, stepName: string): string { + const workflow = readE2eOperationsWorkflow(); + const step = workflow.jobs[jobName]?.steps?.find((candidate) => candidate.name === stepName); + expect(step?.with?.script).toEqual(expect.any(String)); + return step?.with?.script as string; +} + +describe("E2E operations workflow boundary", () => { + it("keeps scheduled routing and scorecards aggregated over the report job set", () => { + expect(validateE2eOperationsWorkflowBoundary()).toEqual([]); + + const workflow = readE2eOperationsWorkflow(); + const reportNeeds = workflow.jobs["report-to-pr"].needs as string[]; + expect(workflow.jobs["notify-on-failure"].needs).toEqual(reportNeeds); + expect(workflow.jobs.scorecard.needs).toEqual(reportNeeds); + }); + + it("rejects aggregation, permission, and secret-scope drift", () => { + const workflow = readE2eOperationsWorkflow(); + (workflow.jobs["notify-on-failure"].needs as string[]).pop(); + workflow.jobs["notify-on-failure"].permissions = { contents: "write", issues: "write" }; + workflow.jobs.scorecard.permissions = { + actions: "read", + contents: "read", + issues: "write", + }; + workflow.jobs.scorecard.env = { + SLACK_WEBHOOK_URL_DAILY: "${{ secrets.SLACK_WEBHOOK_URL_DAILY }}", + }; + + expect(validateE2eOperationsWorkflow(workflow)).toEqual( + expect.arrayContaining([ + "notify-on-failure needs must exactly match report-to-pr needs", + "notify-on-failure must hold only issues: write", + "scorecard permissions must be actions: read and contents: read", + "scorecard must not expose credentials at job scope", + ]), + ); + }); + + it("pins the Node 24 helper runtime and separate always-on raw trace cleanup", () => { + const workflow = readE2eOperationsWorkflow(); + workflow.jobs["cloud-onboard"].env!.NEMOCLAW_TRACE_DIR = + "${{ runner.temp }}/nemoclaw-cloud-onboard-traces"; + const scorecard = workflow.jobs.scorecard.steps!.find( + (step) => step.name === "Generate E2E scorecard", + )!; + scorecard.uses = "actions/github-script@0000000000000000000000000000000000000000"; + const cleanup = workflow.jobs["cloud-onboard"].steps!.find( + (step) => step.name === "Delete raw cloud-onboard traces", + )!; + cleanup.if = "success()"; + const slack = workflow.jobs.scorecard.steps!.find( + (step) => step.name === "Post scorecard to Slack", + )!; + slack.if = "${{ steps.scorecard.outputs.slackData != '' }}"; + slack.with!.script = `${String(slack.with!.script)}\nrequire(process.env.GITHUB_WORKSPACE);`; + + expect(validateE2eOperationsWorkflow(workflow)).toEqual( + expect.arrayContaining([ + "cloud-onboard trace directory must not use unavailable job-level contexts", + "scorecard generator must use the pinned Node 24 github-script runtime", + "cloud-onboard raw trace cleanup must always run", + "scorecard Slack publisher must expose webhook secrets only on main", + "scorecard Slack publisher must not execute workflow-ref code via GITHUB_WORKSPACE", + "scorecard Slack publisher must not execute workflow-ref code via require(", + ]), + ); + }); + + it("creates the scheduled failure issue when no historical thread exists", async () => { + const script = workflowScript( + "notify-on-failure", + "Create or update scheduled E2E failure issue", + ).replace( + "${{ toJSON(needs) }}", + JSON.stringify({ cloud: { result: "failure" }, hermes: { result: "cancelled" } }), + ); + const create = vi.fn().mockResolvedValue({ data: { number: 123 } }); + const createComment = vi.fn(); + const github = { + rest: { + issues: { + create, + createComment, + listForRepo: vi.fn().mockResolvedValue({ data: [] }), + }, + }, + }; + const context = { + repo: { owner: "NVIDIA", repo: "NemoClaw" }, + runId: 456, + serverUrl: "https://github.com", + }; + + await new AsyncFunction("github", "context", script)(github, context); + + expect(createComment).not.toHaveBeenCalled(); + expect(create).toHaveBeenCalledWith( + expect.objectContaining({ + body: expect.stringContaining("**Failed:** cloud\n**Cancelled:** hermes"), + labels: ["bug", "CI/CD"], + owner: "NVIDIA", + repo: "NemoClaw", + title: expect.stringMatching(/^Nightly E2E failed — \d{4}-\d{2}-\d{2}$/u), + }), + ); + }); + + it("keeps selective scorecards silent unless Slack posting is explicitly enabled", async () => { + const script = workflowScript("scorecard", "Post scorecard to Slack"); + const info = vi.fn(); + const fetchMock = vi.fn(); + vi.stubEnv( + "SLACK_DATA", + JSON.stringify({ channel: "preview", payload: { text: "safe precomputed payload" } }), + ); + vi.stubEnv("POST_TO_SLACK", "false"); + try { + await new AsyncFunction("process", "core", "fetch", script)( + process, + { info, setFailed: vi.fn() }, + fetchMock, + ); + expect(info).toHaveBeenCalledWith("Selective dispatch without post_to_slack — skipping"); + expect(fetchMock).not.toHaveBeenCalled(); + } finally { + vi.unstubAllEnvs(); + } + }); + + it("rejects raw trace upload ordering and advisor auto-dispatch restoration", () => { + const workflow = readE2eOperationsWorkflow(); + const cloudSteps = workflow.jobs["cloud-onboard"].steps!; + const sanitize = cloudSteps.find( + (step) => step.name === "Build trusted cloud-onboard timing summary", + )!; + sanitize.run = "cp -R raw-traces e2e-artifacts"; + + const directory = mkdtempSync(join(tmpdir(), "nemoclaw-e2e-operations-")); + const advisorPath = join(directory, "advisor.yaml"); + try { + writeFileSync(advisorPath, "permissions: write-all\njobs:\n advisor:\n steps: []\n"); + expect(validateE2eOperationsWorkflow(workflow, advisorPath)).toContain( + "E2E advisor must not hold actions: write", + ); + + writeFileSync( + advisorPath, + 'permissions: read-all\njobs:\n advisor:\n permissions:\n actions: "write"\n steps:\n - run: createWorkflowDispatch()\n', + ); + expect(validateE2eOperationsWorkflow(workflow, advisorPath)).toEqual( + expect.arrayContaining([ + "cloud-onboard trace sanitizer must retain scripts/e2e/sanitize-trace-timing.py", + "E2E advisor must not hold actions: write", + "E2E advisor must not auto-dispatch workflows", + ]), + ); + } finally { + rmSync(directory, { force: true, recursive: true }); + } + }); +}); diff --git a/test/e2e/support/e2e-scorecard.test.ts b/test/e2e/support/e2e-scorecard.test.ts new file mode 100644 index 00000000000..10069b3cfdb --- /dev/null +++ b/test/e2e/support/e2e-scorecard.test.ts @@ -0,0 +1,636 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import { + existsSync, + lstatSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import { createRequire } from "node:module"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { describe, expect, it, vi } from "vitest"; + +import type { ScorecardData } from "../../../scripts/scorecard/build-slack-blocks.ts"; +import type { JobSummary, SummarizeJobsInput } from "../../../scripts/scorecard/summarize-jobs.ts"; + +const require = createRequire(import.meta.url); +const slack = require("../../../scripts/scorecard/build-slack-blocks.ts") as { + buildBlocks: (data: ScorecardData) => Array<{ + elements?: Array<{ text?: { text?: string }; url?: string }>; + text?: { text: string }; + type: string; + }>; + buildFallbackText: (data: ScorecardData) => string; + getSlackChannel: (data: ScorecardData) => string; +}; +const trace = require("../../../scripts/scorecard/analyze-trace-timing.ts") as { + buildPhaseRows: ( + current: Record, + previous: Record, + ) => Array<{ label: string }>; + buildTraceSummaryLines: ( + current: { totalMs: number }, + previous: { totalMs: number }, + tag: { name: string }, + rows: Array<{ label: string }>, + ) => string[]; + buildTraceTimingResult: ( + deps: { context: { runId: number }; github: unknown }, + services?: { + findLatestCompletedE2eRunForReleaseTag: ( + deps: unknown, + tag: { name: string; sha: string }, + ) => Promise<{ id: number } | null>; + readTraceSummaryFromRun: (deps: unknown, runId: number) => Promise; + resolvePriorReleaseTag: (deps: unknown) => Promise<{ + major: number; + minor: number; + name: string; + patch: number; + sha: string; + } | null>; + }, + ) => Promise<{ traceSummaryLines: string[]; traceTimingLine: string }>; + findLatestCompletedE2eRunForReleaseTag: ( + deps: GitHubTraceDeps, + tag: { major: number; minor: number; name: string; patch: number; sha: string }, + ) => Promise<{ id: number } | null>; + formatTopPhaseChanges: (rows: Array<{ label: string }>) => string; + readTraceSummaryFromRun: (deps: GitHubTraceDeps, runId: number) => Promise; + resolvePriorReleaseTag: ( + deps: GitHubTraceDeps, + ) => Promise<{ major: number; minor: number; name: string; patch: number; sha: string } | null>; + selectOnboardTrace: (texts: string[]) => { totalMs: number } | null; +}; +const scorecardJobs = require("../../../scripts/scorecard/summarize-jobs.ts") as { + isSelectiveDispatch: (eventName: string, rawJobs?: string, rawTargets?: string) => boolean; + loadWorkflowRunJobs: (deps: { + context: { repo: { owner: string; repo: string }; runId: number }; + core: { warning: (message: string) => void }; + github: { + paginate: (method: unknown, parameters: Record) => Promise; + rest: { actions: { listJobsForWorkflowRun: unknown } }; + }; + }) => Promise; + summarizeJobs: (input: SummarizeJobsInput) => JobSummary; +}; +const SANITIZER = "scripts/e2e/sanitize-trace-timing.py"; + +type TraceSummary = { + artifact: Record; + phases: Record; + totalMs: number; +}; + +type GitHubTraceDeps = { + context: { ref?: string; repo: { owner: string; repo: string }; runId: number }; + github: { + paginate: (method: unknown, parameters: Record) => Promise; + rest: { + actions: { + downloadArtifact?: unknown; + listWorkflowRunArtifacts: unknown; + listWorkflowRuns: (...args: any[]) => Promise; + }; + repos: { listTags: unknown }; + }; + }; +}; + +function makeRawTrace(totalMs = 1200, preflightMs = 500): Record { + return { + resource_spans: [ + { + scope_spans: [ + { + spans: [ + { name: "nemoclaw.onboard", duration_ms: totalMs }, + { + name: "nemoclaw.onboard.phase.preflight", + duration_ms: preflightMs, + attributes: { api_key: "nvapi-should-never-appear" }, + events: [{ name: "prompt", attributes: { value: "secret" } }], + }, + { + name: "nemoclaw.onboard.phase.nvapi-attacker-controlled", + duration_ms: 900, + }, + ], + }, + ], + }, + ], + summary: { + trace_id: "0123456789abcdef0123456789abcdef", + total_duration_ms: totalMs, + output_path: "/tmp/raw-trace.json", + slowest_spans: [ + { + name: "nemoclaw.onboard.phase.preflight", + duration_ms: preflightMs, + status: "OK", + }, + ], + }, + }; +} + +function runSanitizer(source: string, output: string) { + return spawnSync("python3", [SANITIZER, source, output], { + cwd: process.cwd(), + encoding: "utf8", + }); +} + +function scorecardData(overrides: Partial = {}): ScorecardData { + return { + today: "Jun 29", + runMode: "Scheduled E2E", + actor: "", + isSelectiveDispatch: false, + requestedJobs: [], + requestedTargets: [], + total: 58, + ran: 58, + success: 58, + failure: 0, + cancelled: 0, + skipped: 0, + perfect: true, + failedJobs: [], + traceTimingLine: "Trace: cloud-onboard total 2m 1.0s", + runUrl: "https://github.com/NVIDIA/NemoClaw/actions/runs/123", + ...overrides, + }; +} + +describe("E2E scorecard", () => { + it("classifies malformed non-empty dispatch selectors as selective", () => { + expect(scorecardJobs.isSelectiveDispatch("schedule", "cloud-onboard")).toBe(false); + expect(scorecardJobs.isSelectiveDispatch("workflow_dispatch", " ", "")).toBe(false); + expect(scorecardJobs.isSelectiveDispatch("workflow_dispatch", "bad selector!", "")).toBe(true); + expect(scorecardJobs.isSelectiveDispatch("workflow_dispatch", "", "cloud-onboard")).toBe(true); + }); + + it("loads typed scorecard helpers through the native github-script require boundary", () => { + const script = ` + const path = require('node:path'); + for (const file of ['analyze-trace-timing.ts', 'summarize-jobs.ts', 'build-slack-blocks.ts']) { + const loaded = require(path.join(process.env.GITHUB_WORKSPACE, 'scripts/scorecard', file)); + if (Object.keys(loaded).length === 0) process.exit(2); + } + `; + const result = spawnSync(process.execPath, ["--experimental-strip-types", "-e", script], { + cwd: process.cwd(), + encoding: "utf8", + env: { ...process.env, GITHUB_WORKSPACE: process.cwd() }, + }); + + expect(result.status, result.stderr).toBe(0); + }); + + it("routes scheduled, full, and opt-in selective summaries to distinct Slack channels", () => { + expect(slack.getSlackChannel(scorecardData())).toBe("daily"); + expect(slack.getSlackChannel(scorecardData({ runMode: "Manual full run" }))).toBe("fullrun"); + expect( + slack.getSlackChannel( + scorecardData({ + runMode: "Selective dispatch", + isSelectiveDispatch: true, + requestedJobs: ["cloud-onboard"], + }), + ), + ).toBe("preview"); + }); + + it("links Slack summaries to the consolidated workflow", () => { + const data = scorecardData(); + const actions = slack.buildBlocks(data).find((block) => block.type === "actions"); + expect(actions?.elements?.[1]?.url).toBe( + "https://github.com/NVIDIA/NemoClaw/actions/workflows/e2e.yaml", + ); + expect(slack.buildFallbackText(data)).toContain("NemoClaw E2E Scorecard"); + + const failureUrl = "https://github.com/NVIDIA/NemoClaw/actions/runs/123/job/456"; + const failureSection = slack + .buildBlocks( + scorecardData({ + failure: 1, + perfect: false, + failedJobs: [{ name: "live (openclaw-nvidia)", url: failureUrl }], + }), + ) + .find((block) => block.text?.text.includes("Failed jobs")); + expect(failureSection?.text?.text).toContain(`<${failureUrl}|live (openclaw-nvidia)>`); + }); + + it("compares only allowlisted onboard timing phases", () => { + const rows = trace.buildPhaseRows( + { + "nemoclaw.onboard.phase.preflight": 1_000, + "nemoclaw.onboard.phase.gateway": 5_000, + "nemoclaw.onboard.phase.future": 100_000, + }, + { + "nemoclaw.onboard.phase.preflight": 2_000, + "nemoclaw.onboard.phase.gateway": 3_000, + "nemoclaw.onboard.phase.future": 1, + }, + ); + expect(rows.map((row) => row.label)).toEqual(["preflight", "gateway"]); + expect(trace.formatTopPhaseChanges(rows)).toBe("gateway +2.0s; preflight -1.0s"); + expect( + trace + .buildTraceSummaryLines({ totalMs: 6_000 }, { totalMs: 5_000 }, { name: "v0.0.69" }, rows) + .join("\n"), + ).toContain("latest completed `e2e.yaml` run"); + }); + + it("accepts only the trusted timing-summary schema", () => { + const good = JSON.stringify({ + schema_version: "nemoclaw.trace_timing.v1", + total_duration_ms: 1000, + phases: { "nemoclaw.onboard.phase.preflight": 500 }, + }); + const rawTrace = JSON.stringify({ + summary: { total_duration_ms: 9999 }, + resource_spans: [{ scope_spans: [{ spans: [] }] }], + }); + expect(trace.selectOnboardTrace([rawTrace])).toBeNull(); + expect(trace.selectOnboardTrace([good])?.totalMs).toBe(1000); + expect( + trace.selectOnboardTrace([ + good, + JSON.stringify({ + schema_version: "nemoclaw.trace_timing.v1", + total_duration_ms: 2000, + phases: { "nemoclaw.onboard.phase.preflight": 1000 }, + }), + ])?.totalMs, + ).toBe(2000); + }); + + it("keeps trace comparison fallbacks explicit and non-fatal", async () => { + const current: TraceSummary = { + artifact: {}, + phases: { "nemoclaw.onboard.phase.preflight": 1_000 }, + totalMs: 2_000, + }; + const prior: TraceSummary = { + artifact: {}, + phases: { "nemoclaw.onboard.phase.preflight": 500 }, + totalMs: 1_000, + }; + const tag = { major: 0, minor: 0, name: "v0.0.69", patch: 69, sha: "abc" }; + const deps = { context: { runId: 123 }, github: {} }; + const baseServices = { + findLatestCompletedE2eRunForReleaseTag: vi.fn().mockResolvedValue({ id: 99 }), + readTraceSummaryFromRun: vi.fn().mockResolvedValue(current), + resolvePriorReleaseTag: vi.fn().mockResolvedValue(tag), + }; + + await expect( + trace.buildTraceTimingResult(deps, { + ...baseServices, + readTraceSummaryFromRun: vi.fn().mockResolvedValue(null), + }), + ).resolves.toMatchObject({ + traceTimingLine: "Trace: ⊘ e2e-cloud-onboard timing summary not found", + }); + await expect( + trace.buildTraceTimingResult(deps, { + ...baseServices, + resolvePriorReleaseTag: vi.fn().mockResolvedValue(null), + }), + ).resolves.toMatchObject({ + traceTimingLine: "Trace: cloud-onboard total 2.0s (no prior release tag found)", + }); + await expect( + trace.buildTraceTimingResult(deps, { + ...baseServices, + findLatestCompletedE2eRunForReleaseTag: vi.fn().mockResolvedValue(null), + }), + ).resolves.toMatchObject({ + traceTimingLine: "Trace: cloud-onboard total 2.0s (no e2e.yaml run found for v0.0.69)", + }); + await expect( + trace.buildTraceTimingResult(deps, { + ...baseServices, + readTraceSummaryFromRun: vi.fn().mockResolvedValueOnce(current).mockResolvedValueOnce(null), + }), + ).resolves.toMatchObject({ + traceTimingLine: "Trace: cloud-onboard total 2.0s (no timing summary found for v0.0.69)", + }); + await expect( + trace.buildTraceTimingResult(deps, { + ...baseServices, + readTraceSummaryFromRun: vi.fn().mockRejectedValue(new Error("artifact unavailable")), + }), + ).resolves.toMatchObject({ traceTimingLine: "Trace: ⊘ comparison unavailable" }); + await expect( + trace.buildTraceTimingResult(deps, { + ...baseServices, + readTraceSummaryFromRun: vi + .fn() + .mockResolvedValueOnce(current) + .mockResolvedValueOnce(prior), + }), + ).resolves.toMatchObject({ + traceTimingLine: expect.stringContaining("increased +1.0s (+100.0%) vs v0.0.69"), + traceSummaryLines: expect.arrayContaining(["## Cloud Onboard Trace Timing"]), + }); + await expect( + trace.buildTraceTimingResult(deps, { + ...baseServices, + readTraceSummaryFromRun: vi + .fn() + .mockResolvedValueOnce(current) + .mockResolvedValueOnce({ ...prior, totalMs: 0 }), + }), + ).resolves.toMatchObject({ + traceTimingLine: expect.stringContaining("increased +2.0s (n/a) vs v0.0.69"), + }); + }); + + it("returns null at missing release-run and trace-artifact boundaries", async () => { + const listWorkflowRuns = vi.fn().mockResolvedValue({ data: { workflow_runs: [] } }); + const deps: GitHubTraceDeps = { + context: { repo: { owner: "NVIDIA", repo: "NemoClaw" }, runId: 123 }, + github: { + paginate: vi.fn().mockResolvedValue([]), + rest: { + actions: { listWorkflowRunArtifacts: {}, listWorkflowRuns }, + repos: { listTags: {} }, + }, + }, + }; + + await expect(trace.resolvePriorReleaseTag(deps)).resolves.toBeNull(); + await expect( + trace.findLatestCompletedE2eRunForReleaseTag(deps, { + major: 0, + minor: 0, + name: "v0.0.69", + patch: 69, + sha: "abc", + }), + ).resolves.toBeNull(); + await expect(trace.readTraceSummaryFromRun(deps, 99)).resolves.toBeNull(); + }); + + it("falls back to needs when the GitHub jobs API is unavailable", async () => { + const warning = vi.fn(); + const apiJobs = await scorecardJobs.loadWorkflowRunJobs({ + context: { repo: { owner: "NVIDIA", repo: "NemoClaw" }, runId: 123 }, + core: { warning }, + github: { + paginate: vi + .fn() + .mockRejectedValue(Object.assign(new Error("temporary outage"), { status: 503 })), + rest: { actions: { listJobsForWorkflowRun: {} } }, + }, + }); + + expect(apiJobs).toBeNull(); + expect(warning).toHaveBeenCalledWith( + expect.stringContaining("status 503); falling back to needs context"), + ); + expect( + scorecardJobs.summarizeJobs({ + apiJobs, + explicitOnlyJobNames: [], + explicitlySelected: [], + metaJobNames: ["generate-matrix"], + needs: { + "generate-matrix": { result: "success" }, + live: { result: "success" }, + }, + }), + ).toMatchObject({ failure: 0, ran: 1, success: 1, total: 1 }); + }); + + it("uses canonical API jobs, latest reruns, and direct failure links", () => { + expect( + scorecardJobs.summarizeJobs({ + apiJobs: [ + { conclusion: "success", name: "generate-matrix", status: "completed" }, + { + completed_at: "2026-06-29T00:00:00Z", + conclusion: "failure", + html_url: "https://example.test/old", + name: "live (openclaw)", + run_attempt: 1, + status: "completed", + }, + { + completed_at: "2026-06-29T01:00:00Z", + conclusion: "success", + html_url: "https://example.test/new", + name: "live (openclaw)", + run_attempt: 2, + status: "completed", + }, + { + conclusion: "timed_out", + html_url: "https://example.test/hermes", + name: "live (hermes)", + status: "completed", + }, + { conclusion: "success", name: "cloud / inner", status: "completed" }, + { conclusion: "skipped", name: "jetson-nvmap-gpu", status: "completed" }, + { + conclusion: "success", + name: "sandbox-rlimits-connect", + status: "completed", + }, + { conclusion: "success", name: "report-to-pr", status: "completed" }, + ], + explicitOnlyJobNames: ["jetson-nvmap-gpu", "sandbox-rlimits-connect"], + explicitlySelected: ["sandbox-rlimits-connect"], + metaJobNames: ["generate-matrix", "report-to-pr", "scorecard"], + needs: {}, + }), + ).toEqual({ + cancelled: 0, + failedJobs: [{ name: "live (hermes)", url: "https://example.test/hermes" }], + failure: 1, + ran: 4, + skipped: 0, + success: 3, + total: 4, + }); + }); + + it("falls back to needs without counting unselected explicit-only jobs", () => { + expect( + scorecardJobs.summarizeJobs({ + apiJobs: null, + explicitOnlyJobNames: ["jetson-nvmap-gpu", "sandbox-rlimits-connect"], + explicitlySelected: ["jetson-nvmap-gpu"], + metaJobNames: ["generate-matrix", "report-to-pr", "scorecard"], + needs: { + "generate-matrix": { result: "success" }, + cloud: { result: "success" }, + malformed: { result: "timed_out" }, + "jetson-nvmap-gpu": { result: "skipped" }, + "sandbox-rlimits-connect": { result: "skipped" }, + "report-to-pr": { result: "success" }, + }, + }), + ).toEqual({ + cancelled: 0, + failedJobs: [{ name: "malformed", url: null }], + failure: 1, + ran: 2, + skipped: 1, + success: 1, + total: 3, + }); + }); + + it("sanitizes raw traces into a timing-only artifact", () => { + const directory = mkdtempSync(join(tmpdir(), "nemoclaw-trace-sanitize-")); + const source = join(directory, "raw"); + const output = join(directory, "trusted"); + const rawPath = join(source, "trace.json"); + try { + mkdirSync(source); + mkdirSync(output); + writeFileSync(join(output, "existing-artifact.log"), "preserve me\n"); + writeFileSync(rawPath, JSON.stringify(makeRawTrace())); + writeFileSync(join(source, "environment.txt"), "NVIDIA_API_KEY=nvapi-secret\n"); + writeFileSync( + join(source, "malicious.json"), + JSON.stringify({ summary: { total_duration_ms: 9999 }, token: "ghp_secret" }), + ); + const result = runSanitizer(source, output); + expect(result.status, result.stderr).toBe(0); + const summaryPath = join(output, "cloud-onboard-trace-timing-summary.json"); + const summary = readFileSync(summaryPath, "utf8"); + expect(readFileSync(join(output, "existing-artifact.log"), "utf8")).toBe("preserve me\n"); + expect(JSON.parse(summary)).toEqual({ + phases: { "nemoclaw.onboard.phase.preflight": 500 }, + schema_version: "nemoclaw.trace_timing.v1", + slowest_spans: [ + { duration_ms: 500, name: "nemoclaw.onboard.phase.preflight", status: "OK" }, + ], + total_duration_ms: 1200, + trace_id: "0123456789abcdef0123456789abcdef", + }); + expect(summary).not.toMatch(/api_key|nvapi|ghp_|attributes|events|output_path|raw-trace/u); + expect(lstatSync(summaryPath).mode & 0o777).toBe(0o600); + } finally { + rmSync(directory, { force: true, recursive: true }); + } + }); + + it("emits no timing summary for malformed or non-onboard traces", () => { + const directory = mkdtempSync(join(tmpdir(), "nemoclaw-trace-invalid-")); + const source = join(directory, "raw"); + const output = join(directory, "trusted"); + try { + mkdirSync(source); + writeFileSync(join(source, "malformed.json"), "{not-json"); + writeFileSync( + join(source, "not-onboard.json"), + JSON.stringify({ resource_spans: [], summary: { total_duration_ms: 1 } }), + ); + const result = runSanitizer(source, output); + expect(result.status, result.stderr).toBe(0); + expect(readdirSync(output)).toEqual([]); + } finally { + rmSync(directory, { force: true, recursive: true }); + } + }); + + it("bounds trace input count and file size before parsing", () => { + const directory = mkdtempSync(join(tmpdir(), "nemoclaw-trace-bounds-")); + const source = join(directory, "raw"); + const output = join(directory, "trusted"); + try { + mkdirSync(source); + writeFileSync(join(source, "000-valid.json"), JSON.stringify(makeRawTrace(1_200))); + for (let index = 1; index < 100; index += 1) { + writeFileSync(join(source, `${String(index).padStart(3, "0")}-invalid.json`), "{}"); + } + writeFileSync(join(source, "100-ignored.json"), JSON.stringify(makeRawTrace(9_999))); + writeFileSync(join(source, "101-oversized.json"), " ".repeat(2 * 1024 * 1024 + 1)); + + const result = runSanitizer(source, output); + expect(result.status, result.stderr).toBe(0); + expect( + JSON.parse(readFileSync(join(output, "cloud-onboard-trace-timing-summary.json"), "utf8")), + ).toMatchObject({ total_duration_ms: 1_200 }); + + rmSync(output, { force: true, recursive: true }); + rmSync(join(source, "000-valid.json")); + for (let index = 1; index < 100; index += 1) { + rmSync(join(source, `${String(index).padStart(3, "0")}-invalid.json`)); + } + rmSync(join(source, "100-ignored.json")); + const oversizedOnly = runSanitizer(source, output); + expect(oversizedOnly.status, oversizedOnly.stderr).toBe(0); + expect(readdirSync(output)).toEqual([]); + } finally { + rmSync(directory, { force: true, recursive: true }); + } + }); + + it("rejects symlinked trace sources and trusted output paths", () => { + const directory = mkdtempSync(join(tmpdir(), "nemoclaw-trace-symlink-")); + const source = join(directory, "raw"); + const sourceLink = join(directory, "raw-link"); + const outputTarget = join(directory, "target-controlled"); + const outputLink = join(directory, "trusted-link"); + try { + mkdirSync(source); + mkdirSync(outputTarget); + writeFileSync(join(source, "trace.json"), JSON.stringify(makeRawTrace())); + writeFileSync(join(outputTarget, "secret.txt"), "do not overwrite\n"); + symlinkSync(source, sourceLink, "dir"); + symlinkSync(outputTarget, outputLink, "dir"); + + const sourceResult = runSanitizer(sourceLink, join(directory, "trusted")); + expect(sourceResult.status).toBe(2); + expect(sourceResult.stderr).toContain("trace source must not be a symlink"); + + const outputResult = runSanitizer(source, outputLink); + expect(outputResult.status).toBe(2); + expect(outputResult.stderr).toContain("trusted output must be a real directory"); + expect(readFileSync(join(outputTarget, "secret.txt"), "utf8")).toBe("do not overwrite\n"); + } finally { + rmSync(directory, { force: true, recursive: true }); + } + }); + + it("refuses to follow a pre-created timing-summary symlink", () => { + const directory = mkdtempSync(join(tmpdir(), "nemoclaw-trace-file-symlink-")); + const source = join(directory, "raw"); + const output = join(directory, "trusted"); + const target = join(directory, "target-controlled.txt"); + try { + mkdirSync(source); + mkdirSync(output); + writeFileSync(join(source, "trace.json"), JSON.stringify(makeRawTrace())); + writeFileSync(target, "do not overwrite\n"); + symlinkSync(target, join(output, "cloud-onboard-trace-timing-summary.json")); + + const result = runSanitizer(source, output); + expect(result.status).toBe(2); + expect(result.stderr).toContain("trusted timing summary must not be a symlink"); + expect(readFileSync(target, "utf8")).toBe("do not overwrite\n"); + expect(existsSync(target)).toBe(true); + } finally { + rmSync(directory, { force: true, recursive: true }); + } + }); +}); diff --git a/test/e2e/support/e2e-workflow-contract.test.ts b/test/e2e/support/e2e-workflow-contract.test.ts new file mode 100644 index 00000000000..f84a5511447 --- /dev/null +++ b/test/e2e/support/e2e-workflow-contract.test.ts @@ -0,0 +1,29 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { removeJobNeed } from "../../helpers/e2e-workflow-contract"; + +describe("E2E workflow test helpers", () => { + it("refuses to remove a dependency from a later job", () => { + const workflow = [ + "jobs:", + " owner:", + " needs:", + " [", + " present,", + " ]", + " later:", + " needs:", + " [", + " misplaced,", + " ]", + "", + ].join("\n"); + + expect(() => removeJobNeed(workflow, "owner", "misplaced")).toThrow( + "owner does not need misplaced", + ); + }); +}); diff --git a/test/e2e/support/e2e-workflow.test.ts b/test/e2e/support/e2e-workflow.test.ts index 3fa9b5fcc71..ebb82426331 100644 --- a/test/e2e/support/e2e-workflow.test.ts +++ b/test/e2e/support/e2e-workflow.test.ts @@ -14,15 +14,10 @@ import { validateE2eWorkflowBoundary, validateFreeStandingWorkflowInventory, } from "../../../tools/e2e/workflow-boundary.mts"; +import { readWorkflow, removeJobNeed } from "../../helpers/e2e-workflow-contract"; import { testTimeoutOptions } from "../../helpers/timeouts"; import { assertChannelsStopStartSandboxName } from "../live/channels-stop-start-safety.ts"; -function readWorkflow(): Record { - return YAML.parse( - fs.readFileSync(path.join(process.cwd(), ".github/workflows/e2e.yaml"), "utf-8"), - ) as Record; -} - function generateMatrixScript(): string { const workflow = readWorkflow(); const jobs = workflow.jobs as Record> }>; @@ -1167,7 +1162,10 @@ jobs: renamedWorkflowPath, workflow.replace(/^ runtime-overrides:$/m, " runtime-overrides-missing:"), ); - fs.writeFileSync(missingReportNeedPath, workflow.replace(" runtime-overrides,\n", "")); + fs.writeFileSync( + missingReportNeedPath, + removeJobNeed(workflow, "report-to-pr", "runtime-overrides"), + ); try { expect(validateE2eWorkflowBoundary(renamedWorkflowPath)).toContain( @@ -1297,7 +1295,7 @@ jobs: ); fs.writeFileSync( missingReportNeedPath, - workflow.replace(" messaging-compatible-endpoint,\n", ""), + removeJobNeed(workflow, "report-to-pr", "messaging-compatible-endpoint"), ); try { diff --git a/test/e2e/support/gpu-e2e-helpers.test.ts b/test/e2e/support/gpu-e2e-helpers.test.ts index 1ba0b9eb166..18ee92f757a 100644 --- a/test/e2e/support/gpu-e2e-helpers.test.ts +++ b/test/e2e/support/gpu-e2e-helpers.test.ts @@ -15,4 +15,10 @@ describe("GPU E2E helpers", () => { it("does not synthesize an Ollama model pull timeout outside workflow configuration", () => { expect(env({}, {}).NEMOCLAW_OLLAMA_PULL_TIMEOUT).toBeUndefined(); }); + + it("forwards the workflow-owned trace directory through availability probes", () => { + expect(env({}, { NEMOCLAW_TRACE_DIR: "/tmp/nemoclaw-traces" }).NEMOCLAW_TRACE_DIR).toBe( + "/tmp/nemoclaw-traces", + ); + }); }); diff --git a/test/e2e/support/jetson-workflow-boundary.test.ts b/test/e2e/support/jetson-workflow-boundary.test.ts index 045c28b5a42..d31aa8eda9a 100644 --- a/test/e2e/support/jetson-workflow-boundary.test.ts +++ b/test/e2e/support/jetson-workflow-boundary.test.ts @@ -1,24 +1,52 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + import { describe, expect, it } from "vitest"; +import YAML from "yaml"; import { evaluateE2eWorkflowDispatchSelectors, + formatFreeStandingJobsInventoryForShell, readFreeStandingJobsInventory, validateE2eWorkflowBoundary, + validateFreeStandingWorkflowInventory, } from "../../../tools/e2e/workflow-boundary.mts"; +import { readWorkflow } from "../../helpers/e2e-workflow-contract.ts"; describe("Jetson nvmap GPU E2E workflow boundary", () => { it("keeps Jetson selectable but excluded from full-suite dispatch", () => { const inventory = readFreeStandingJobsInventory(); expect(validateE2eWorkflowBoundary()).toEqual([]); expect(inventory.allowedJobs).toContain("jetson-nvmap-gpu"); + expect(inventory.explicitOnlyJobs).toContain("jetson-nvmap-gpu"); + expect(formatFreeStandingJobsInventoryForShell(inventory)).toContain( + "explicit_only_jobs_csv=sandbox-rlimits-connect,jetson-nvmap-gpu", + ); expect(inventory.targetToJob.get("jetson-nvmap-gpu")).toBe("jetson-nvmap-gpu"); expect(evaluateE2eWorkflowDispatchSelectors({}).selectedFreeStandingJobs).not.toContain( "jetson-nvmap-gpu", ); }); + it("rejects invalid explicit-only workflow metadata", () => { + const workflow = readWorkflow(); + const jobs = workflow.jobs as Record }>; + jobs["jetson-nvmap-gpu"].env!.E2E_DEFAULT_ENABLED = "yes"; + const directory = mkdtempSync(join(tmpdir(), "nemoclaw-explicit-only-")); + const workflowPath = join(directory, "workflow.yaml"); + try { + writeFileSync(workflowPath, YAML.stringify(workflow)); + expect(validateFreeStandingWorkflowInventory(workflowPath)).toContain( + 'jetson-nvmap-gpu job E2E_DEFAULT_ENABLED must be "0" when set', + ); + } finally { + rmSync(directory, { force: true, recursive: true }); + } + }); + it("runs Jetson only when explicitly selected", () => { for (const selector of [{ targets: "jetson-nvmap-gpu" }, { jobs: "jetson-nvmap-gpu" }]) { expect(evaluateE2eWorkflowDispatchSelectors(selector)).toMatchObject({ diff --git a/test/e2e/support/rlimit-connect-workflow-boundary.test.ts b/test/e2e/support/rlimit-connect-workflow-boundary.test.ts index e01115e83ed..c273d2979ac 100644 --- a/test/e2e/support/rlimit-connect-workflow-boundary.test.ts +++ b/test/e2e/support/rlimit-connect-workflow-boundary.test.ts @@ -11,6 +11,7 @@ describe("rlimit connect workflow boundary", () => { it("maps the rlimit connect acceptance selector to its explicit Vitest job", () => { const inventory = readFreeStandingJobsInventory(); expect(inventory.allowedJobs).toContain("sandbox-rlimits-connect"); + expect(inventory.explicitOnlyJobs).toContain("sandbox-rlimits-connect"); expect(inventory.targetToJob.get("sandbox-rlimits-connect")).toBe("sandbox-rlimits-connect"); expect( diff --git a/test/helpers/e2e-workflow-contract.ts b/test/helpers/e2e-workflow-contract.ts index 9891c7a5c46..afaf0ff787a 100644 --- a/test/helpers/e2e-workflow-contract.ts +++ b/test/helpers/e2e-workflow-contract.ts @@ -49,3 +49,25 @@ export type CompositeAction = { export function readYaml(path: string): T { return YAML.parse(readFileSync(join(REPO_ROOT, path), "utf-8")) as T; } + +export function readWorkflow(): Record { + return readYaml(".github/workflows/e2e.yaml"); +} + +export function removeJobNeed(source: string, ownerJob: string, dependency: string): string { + const ownerHeader = ` ${ownerJob}:\n`; + const ownerStart = source.indexOf(ownerHeader); + if (ownerStart < 0) { + throw new Error(`workflow is missing job ${ownerJob}`); + } + const prefix = source.slice(0, ownerStart); + const afterOwnerHeader = ownerStart + ownerHeader.length; + const nextJobOffset = source.slice(afterOwnerHeader).search(/^ [\w-]+:\n/mu); + const ownerEnd = nextJobOffset < 0 ? source.length : afterOwnerHeader + nextJobOffset; + const ownerBlock = source.slice(ownerStart, ownerEnd); + const needle = ` ${dependency},\n`; + if (!ownerBlock.includes(needle)) { + throw new Error(`${ownerJob} does not need ${dependency}`); + } + return prefix + ownerBlock.replace(needle, "") + source.slice(ownerEnd); +} diff --git a/tools/e2e/operations-workflow-boundary.mts b/tools/e2e/operations-workflow-boundary.mts new file mode 100644 index 00000000000..34564e12582 --- /dev/null +++ b/tools/e2e/operations-workflow-boundary.mts @@ -0,0 +1,332 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +import YAML from "yaml"; + +const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); +const DEFAULT_WORKFLOW_PATH = join(REPO_ROOT, ".github", "workflows", "e2e.yaml"); +const DEFAULT_ADVISOR_PATH = join(REPO_ROOT, ".github", "workflows", "e2e-advisor.yaml"); +const META_JOBS = new Set(["notify-on-failure", "report-to-pr", "scorecard"]); +const FULL_SHA_ACTION = /^[^\s@]+@[0-9a-f]{40}$/u; +const GITHUB_SCRIPT_NODE24_ACTION = + "actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3"; + +type WorkflowStep = { + env?: Record; + if?: string; + name?: string; + run?: string; + uses?: string; + with?: Record; +}; + +type WorkflowPermissions = Record | string; + +type WorkflowJob = { + env?: Record; + if?: string; + needs?: unknown; + permissions?: WorkflowPermissions; + steps?: WorkflowStep[]; +}; + +export type OperationsWorkflow = { + jobs: Record; + permissions?: WorkflowPermissions; + on?: { + workflow_dispatch?: { + inputs?: Record>; + }; + }; +}; + +export function readE2eOperationsWorkflow(path = DEFAULT_WORKFLOW_PATH): OperationsWorkflow { + return YAML.parse(readFileSync(path, "utf8")) as OperationsWorkflow; +} + +function needs(job: WorkflowJob): string[] { + return Array.isArray(job.needs) + ? job.needs.filter((name): name is string => typeof name === "string") + : typeof job.needs === "string" + ? [job.needs] + : []; +} + +function sorted(values: Iterable): string[] { + return [...values].sort((left, right) => left.localeCompare(right)); +} + +function sameMembers(left: readonly string[], right: readonly string[]): boolean { + return JSON.stringify(sorted(left)) === JSON.stringify(sorted(right)); +} + +function permissionMap(permissions: WorkflowPermissions | undefined): Record { + return permissions !== null && typeof permissions === "object" ? permissions : {}; +} + +function findStep(job: WorkflowJob, name: string): WorkflowStep { + return job.steps?.find((step) => step.name === name) ?? {}; +} + +function requirePinnedAction(errors: string[], step: WorkflowStep, owner: string): void { + if (!FULL_SHA_ACTION.test(step.uses ?? "")) { + errors.push(`${owner} must pin its action to a full SHA`); + } +} + +function requireNode24GithubScript(errors: string[], step: WorkflowStep, owner: string): void { + requirePinnedAction(errors, step, owner); + if (step.uses !== GITHUB_SCRIPT_NODE24_ACTION) { + errors.push(`${owner} must use the pinned Node 24 github-script runtime`); + } +} + +function validateAggregation(errors: string[], workflow: OperationsWorkflow): void { + const executionJobs = Object.keys(workflow.jobs).filter((name) => !META_JOBS.has(name)); + const reportNeeds = needs(workflow.jobs["report-to-pr"] ?? {}); + for (const name of executionJobs) { + if (!reportNeeds.includes(name)) errors.push(`report-to-pr must wait for ${name}`); + } + for (const name of reportNeeds) { + if (!executionJobs.includes(name)) errors.push(`report-to-pr waits for unknown job ${name}`); + } + for (const aggregate of ["notify-on-failure", "scorecard"]) { + const aggregateNeeds = needs(workflow.jobs[aggregate] ?? {}); + if (!sameMembers(aggregateNeeds, reportNeeds)) { + errors.push(`${aggregate} needs must exactly match report-to-pr needs`); + } + } +} + +function validateNotify(errors: string[], workflow: OperationsWorkflow): void { + const job = workflow.jobs["notify-on-failure"] ?? {}; + if ( + job.if !== + "${{ always() && github.event_name == 'schedule' && (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) }}" + ) { + errors.push("notify-on-failure must run only for failed or cancelled scheduled runs"); + } + const permissions = permissionMap(job.permissions); + if (permissions.issues !== "write" || Object.keys(permissions).length !== 1) { + errors.push("notify-on-failure must hold only issues: write"); + } + const notify = findStep(job, "Create or update scheduled E2E failure issue"); + requirePinnedAction(errors, notify, "notify-on-failure"); + const script = String(notify.with?.script ?? ""); + for (const fragment of [ + "github.rest.issues.listForRepo", + "github.rest.issues.createComment", + "github.rest.issues.create", + "Nightly E2E failed", + "contains(needs.*.result", + ]) { + if (!script.includes(fragment) && !String(job.if ?? "").includes(fragment)) { + errors.push(`notify-on-failure must retain ${fragment}`); + } + } +} + +function validateScorecard(errors: string[], workflow: OperationsWorkflow): void { + const dispatchInput = workflow.on?.workflow_dispatch?.inputs?.post_to_slack; + if (dispatchInput?.type !== "boolean" || dispatchInput.default !== false) { + errors.push("workflow_dispatch post_to_slack must be an opt-in boolean"); + } + + const job = workflow.jobs.scorecard ?? {}; + const permissions = permissionMap(job.permissions); + if ( + job.if !== + "${{ always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }}" + ) { + errors.push("scorecard must run after scheduled and manual E2E executions"); + } + if ( + permissions.actions !== "read" || + permissions.contents !== "read" || + Object.keys(permissions).length !== 2 + ) { + errors.push("scorecard permissions must be actions: read and contents: read"); + } + if (job.env && Object.keys(job.env).length > 0) { + errors.push("scorecard must not expose credentials at job scope"); + } + + const checkout = findStep(job, "Checkout scorecard builders"); + requirePinnedAction(errors, checkout, "scorecard checkout"); + if (checkout.with?.["persist-credentials"] !== false) { + errors.push("scorecard checkout must disable persisted credentials"); + } + if (checkout.with?.["sparse-checkout"] !== "scripts/scorecard") { + errors.push("scorecard checkout must be limited to scripts/scorecard"); + } + + const generate = findStep(job, "Generate E2E scorecard"); + requireNode24GithubScript(errors, generate, "scorecard generator"); + const generateScript = String(generate.with?.script ?? ""); + for (const fragment of [ + "scripts/scorecard/analyze-trace-timing.ts", + "traceTiming.buildTraceTimingResult", + "scripts/scorecard/summarize-jobs.ts", + "scorecardJobs.isSelectiveDispatch", + "scorecardJobs.loadWorkflowRunJobs", + "scorecardJobs.summarizeJobs", + "scripts/scorecard/build-slack-blocks.ts", + "slackBlocks.buildBlocks", + "core.summary", + "scorecardData", + "slackData", + ]) { + if (!generateScript.includes(fragment)) + errors.push(`scorecard generator must retain ${fragment}`); + } + if ( + generate.env?.EXPLICIT_ONLY_JOBS !== "${{ needs.generate-matrix.outputs.explicit_only_jobs }}" + ) { + errors.push("scorecard generator must derive explicit-only jobs from workflow inventory"); + } + + const slack = findStep(job, "Post scorecard to Slack"); + requirePinnedAction(errors, slack, "scorecard Slack publisher"); + if ( + slack.if !== "${{ steps.scorecard.outputs.slackData != '' && github.ref == 'refs/heads/main' }}" + ) { + errors.push("scorecard Slack publisher must expose webhook secrets only on main"); + } + const expectedSlackEnv = [ + "SLACK_WEBHOOK_URL_DAILY", + "SLACK_WEBHOOK_URL_FULLRUN", + "SLACK_WEBHOOK_URL_PREVIEW", + ]; + for (const name of expectedSlackEnv) { + if (!String(slack.env?.[name] ?? "").includes(`secrets.${name}`)) { + errors.push(`scorecard Slack publisher must scope ${name} to its step`); + } + } + if (slack.env?.POST_TO_SLACK !== "${{ inputs.post_to_slack }}") { + errors.push("scorecard Slack publisher must honor the post_to_slack opt-in"); + } + if (slack.env?.SLACK_DATA !== "${{ steps.scorecard.outputs.slackData }}") { + errors.push("scorecard Slack publisher must consume the precomputed Slack payload"); + } + const slackScript = String(slack.with?.script ?? ""); + for (const fragment of [ + "process.env.SLACK_DATA", + "Invalid precomputed Slack payload", + "Selective dispatch without post_to_slack", + "SLACK_WEBHOOK_URL_PREVIEW", + ]) { + if (!slackScript.includes(fragment)) + errors.push(`scorecard Slack publisher must retain ${fragment}`); + } + for (const forbidden of ["GITHUB_WORKSPACE", "require(", "scripts/scorecard/"]) { + if (slackScript.includes(forbidden)) { + errors.push(`scorecard Slack publisher must not execute workflow-ref code via ${forbidden}`); + } + } +} + +function validateTraceTiming(errors: string[], workflow: OperationsWorkflow): void { + const job = workflow.jobs["cloud-onboard"] ?? {}; + if (job.env?.NEMOCLAW_TRACE_DIR !== undefined) { + errors.push("cloud-onboard trace directory must not use unavailable job-level contexts"); + } + const configure = findStep(job, "Configure cloud-onboard trace directory"); + for (const fragment of ['"${RUNNER_TEMP}/nemoclaw-cloud-onboard-traces"', '>> "${GITHUB_ENV}"']) { + if (!String(configure.run ?? "").includes(fragment)) { + errors.push(`cloud-onboard trace directory setup must retain ${fragment}`); + } + } + const sanitize = findStep(job, "Build trusted cloud-onboard timing summary"); + if (sanitize.if !== "always()") { + errors.push("cloud-onboard trace sanitizer must always run"); + } + const script = sanitize.run ?? ""; + for (const fragment of [ + "scripts/e2e/sanitize-trace-timing.py", + '"${NEMOCLAW_TRACE_DIR}"', + '"${E2E_ARTIFACT_DIR}"', + ]) { + if (!script.includes(fragment)) + errors.push(`cloud-onboard trace sanitizer must retain ${fragment}`); + } + const steps = job.steps ?? []; + const configureIndex = steps.findIndex( + (step) => step.name === "Configure cloud-onboard trace directory", + ); + const runIndex = steps.findIndex((step) => step.name === "Run cloud-onboard live Vitest test"); + const sanitizeIndex = steps.findIndex( + (step) => step.name === "Build trusted cloud-onboard timing summary", + ); + const cleanup = findStep(job, "Delete raw cloud-onboard traces"); + const cleanupIndex = steps.findIndex((step) => step.name === "Delete raw cloud-onboard traces"); + const uploadIndex = steps.findIndex((step) => step.name === "Upload cloud-onboard artifacts"); + if (cleanup.if !== "always()") { + errors.push("cloud-onboard raw trace cleanup must always run"); + } + for (const fragment of [ + 'expected_trace_dir="${RUNNER_TEMP}/nemoclaw-cloud-onboard-traces"', + '[ "${NEMOCLAW_TRACE_DIR}" != "${expected_trace_dir}" ]', + 'rm -rf -- "${NEMOCLAW_TRACE_DIR}"', + ]) { + if (!String(cleanup.run ?? "").includes(fragment)) { + errors.push(`cloud-onboard raw trace cleanup must retain ${fragment}`); + } + } + if ( + !( + configureIndex >= 0 && + configureIndex < runIndex && + runIndex < sanitizeIndex && + sanitizeIndex < cleanupIndex && + cleanupIndex < uploadIndex + ) + ) { + errors.push( + "cloud-onboard must test, sanitize raw traces, delete raw traces, then upload trusted artifacts", + ); + } +} + +function validateAdvisorRetirement(errors: string[], advisorPath: string): void { + const source = readFileSync(advisorPath, "utf8"); + const advisor = YAML.parse(source) as OperationsWorkflow; + const permissionBlocks = [ + advisor.permissions, + ...Object.values(advisor.jobs ?? {}).map((job) => job.permissions), + ]; + if ( + permissionBlocks.some( + (permissions) => + permissions === "write-all" || permissionMap(permissions).actions === "write", + ) + ) { + errors.push("E2E advisor must not hold actions: write"); + } + if (/createWorkflowDispatch|workflow_dispatches/u.test(source)) { + errors.push("E2E advisor must not auto-dispatch workflows"); + } +} + +export function validateE2eOperationsWorkflow( + workflow: OperationsWorkflow, + advisorPath = DEFAULT_ADVISOR_PATH, +): string[] { + const errors: string[] = []; + validateAggregation(errors, workflow); + validateNotify(errors, workflow); + validateScorecard(errors, workflow); + validateTraceTiming(errors, workflow); + validateAdvisorRetirement(errors, advisorPath); + return errors; +} + +export function validateE2eOperationsWorkflowBoundary( + workflowPath = DEFAULT_WORKFLOW_PATH, + advisorPath = DEFAULT_ADVISOR_PATH, +): string[] { + return validateE2eOperationsWorkflow(readE2eOperationsWorkflow(workflowPath), advisorPath); +} diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index b9e0be54e48..27f874c1471 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -8,6 +8,7 @@ import YAML from "yaml"; import { validateHermesDashboardWorkflowBoundary } from "./hermes-dashboard-workflow-boundary.mts"; import { validateInferenceSwitchWorkflowBoundary } from "./inference-switch-workflow-boundary.mts"; +import { validateE2eOperationsWorkflowBoundary } from "./operations-workflow-boundary.mts"; import { validateSandboxOperationsWorkflow } from "./sandbox-operations-workflow-boundary.mts"; const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); @@ -24,6 +25,7 @@ type WorkflowStep = WorkflowRecord & { export interface FreeStandingJobsInventory { allowedJobs: string[]; + explicitOnlyJobs: string[]; freeStandingTargets: string[]; targetToJob: Map; } @@ -38,6 +40,7 @@ const SELECTOR_PATTERN = /^[A-Za-z0-9_-]+(,[A-Za-z0-9_-]+)*$/; const SELECTOR_ID_PATTERN = /^[A-Za-z0-9_-]+$/; const FREE_STANDING_JOB_MARKER = "E2E_JOB"; const FREE_STANDING_TARGET_MARKER = "E2E_TARGET_ID"; +const FREE_STANDING_DEFAULT_ENABLED_MARKER = "E2E_DEFAULT_ENABLED"; const COMMON_SECRET_ENV_NAMES = [ "NVIDIA_API_KEY", "NVIDIA_INFERENCE_API_KEY", @@ -49,13 +52,7 @@ const FREE_STANDING_SELECTOR_SPECIAL_CASES = new Set([ "full-e2e", "hermes-e2e", "hermes-root-entrypoint-smoke", - "jetson-nvmap-gpu", "openclaw-tui-chat-correlation", - "sandbox-rlimits-connect", -]); -const FULL_SUITE_EXCLUDED_FREE_STANDING_JOBS = new Set([ - "jetson-nvmap-gpu", - "sandbox-rlimits-connect", ]); const PUBLIC_NVIDIA_ENDPOINT_KEY_JOBS = new Set([ "device-auth-health", @@ -84,6 +81,7 @@ function deriveFreeStandingJobsInventoryFromJobs(jobs: WorkflowRecord): { } { const errors: string[] = []; const allowedJobs: string[] = []; + const explicitOnlyJobs: string[] = []; const freeStandingTargets: string[] = []; const targetToJob = new Map(); @@ -109,6 +107,13 @@ function deriveFreeStandingJobsInventoryFromJobs(jobs: WorkflowRecord): { } allowedJobs.push(jobId); + if (Object.hasOwn(env, FREE_STANDING_DEFAULT_ENABLED_MARKER)) { + if (env[FREE_STANDING_DEFAULT_ENABLED_MARKER] !== "0") { + errors.push(`${jobId} job ${FREE_STANDING_DEFAULT_ENABLED_MARKER} must be "0" when set`); + } else { + explicitOnlyJobs.push(jobId); + } + } if (!hasTargetMarker) continue; const target = env[FREE_STANDING_TARGET_MARKER]; @@ -134,6 +139,7 @@ function deriveFreeStandingJobsInventoryFromJobs(jobs: WorkflowRecord): { errors, inventory: { allowedJobs, + explicitOnlyJobs, freeStandingTargets, targetToJob, }, @@ -151,6 +157,7 @@ function cloneFreeStandingJobsInventory( ): FreeStandingJobsInventory { return { allowedJobs: [...inventory.allowedJobs], + explicitOnlyJobs: [...inventory.explicitOnlyJobs], freeStandingTargets: [...inventory.freeStandingTargets], targetToJob: new Map(inventory.targetToJob), }; @@ -191,6 +198,7 @@ export function formatFreeStandingJobsInventoryForShell( const targetJobMappings = [...inventory.targetToJob].map(([target, job]) => `${target}:${job}`); return [ `allowed_jobs=${inventory.allowedJobs.join(",")}`, + `explicit_only_jobs_csv=${inventory.explicitOnlyJobs.join(",")}`, `free_standing_targets_csv=${inventory.freeStandingTargets.join(",")}`, `free_standing_target_jobs_csv=${targetJobMappings.join(",")}`, "", @@ -265,7 +273,7 @@ export function evaluateE2eWorkflowDispatchSelectors(input: { valid: true, errors: [], selectedFreeStandingJobs: freeStandingJobIds - .filter((job) => !FULL_SUITE_EXCLUDED_FREE_STANDING_JOBS.has(job)) + .filter((job) => !inventory.explicitOnlyJobs.includes(job)) .sort(), registryTargets: [], liveTargetsRun: true, @@ -484,12 +492,16 @@ function validateFreeStandingJobSelector( jobs: WorkflowRecord, jobName: string, targetName?: string, + explicitOnly = false, ): void { const job = asRecord(jobs[jobName]); if (job.needs !== "generate-matrix") { errors.push(`${jobName} job must depend on generate-matrix`); } - if (job.if !== freeStandingJobIf(jobName, targetName)) { + const expected = explicitOnly + ? explicitOnlyFreeStandingJobIf(jobName, targetName) + : freeStandingJobIf(jobName, targetName); + if (job.if !== expected) { errors.push(`${jobName} job must use the shared jobs selector condition`); } } @@ -565,7 +577,13 @@ function validateFreeStandingInventoryBoundary( if (Object.keys(job).length === 0) continue; if (!FREE_STANDING_SELECTOR_SPECIAL_CASES.has(jobName)) { - validateFreeStandingJobSelector(errors, jobs, jobName, targetByJob.get(jobName)); + validateFreeStandingJobSelector( + errors, + jobs, + jobName, + targetByJob.get(jobName), + inventory.explicitOnlyJobs.includes(jobName), + ); } const jobEnv = asRecord(job.env); @@ -4919,6 +4937,7 @@ export function validateE2eWorkflowBoundary(workflowPath = DEFAULT_E2E_WORKFLOW_ const errors: string[] = []; errors.push(...validateHermesDashboardWorkflowBoundary(workflowPath)); errors.push(...validateInferenceSwitchWorkflowBoundary(workflowPath)); + errors.push(...validateE2eOperationsWorkflowBoundary(workflowPath)); const triggers = asRecord(workflow.on ?? workflow[true as unknown as string]); const workflowDispatch = requireWorkflowDispatch(errors, triggers); @@ -4963,6 +4982,9 @@ export function validateE2eWorkflowBoundary(workflowPath = DEFAULT_E2E_WORKFLOW_ if (generateOutputs.hermes_selected !== "${{ steps.matrix.outputs.hermes_selected }}") { errors.push("generate-matrix job must expose hermes_selected output"); } + if (generateOutputs.explicit_only_jobs !== "${{ steps.matrix.outputs.explicit_only_jobs }}") { + errors.push("generate-matrix job must expose explicit_only_jobs output"); + } const generateSteps = asSteps(generateMatrix.steps); requireNoDispatchInputInterpolation(errors, generateSteps); const generateCheckout = generateSteps.find((step) => @@ -5011,6 +5033,11 @@ export function validateE2eWorkflowBoundary(workflowPath = DEFAULT_E2E_WORKFLOW_ generate, 'echo "hermes_selected=${hermes_selected}" >> "$GITHUB_OUTPUT"', ); + requireRunContains( + errors, + generate, + 'echo "explicit_only_jobs=${explicit_only_jobs_csv}" >> "$GITHUB_OUTPUT"', + ); requireRunContains(errors, generate, "## E2E Target Matrix"); requireRunContains(errors, generate, "| Target | Runner | Label |"); @@ -5305,6 +5332,11 @@ export function validateE2eWorkflowBoundary(workflowPath = DEFAULT_E2E_WORKFLOW_ if (reportEnv.JOB_TARGETS !== "${{ inputs.targets }}") { errors.push("report-to-pr step must pass targets through JOB_TARGETS env"); } + if ( + reportEnv.EXPLICIT_ONLY_JOBS !== "${{ needs.generate-matrix.outputs.explicit_only_jobs }}" + ) { + errors.push("report-to-pr must derive explicit-only jobs from workflow inventory"); + } const reportScript = stringValue(asRecord(report?.with).script ?? report?.run); if (!reportScript.includes("process.env.JOBS")) { errors.push("step 'Post E2E target results to PR' run script must include process.env.JOBS"); diff --git a/tools/e2e/workflow-inventory.mts b/tools/e2e/workflow-inventory.mts index d164e8cd3e9..dbce557e69b 100644 --- a/tools/e2e/workflow-inventory.mts +++ b/tools/e2e/workflow-inventory.mts @@ -51,6 +51,7 @@ try { `${JSON.stringify( { allowedJobs: inventory.allowedJobs, + explicitOnlyJobs: inventory.explicitOnlyJobs, freeStandingTargets: inventory.freeStandingTargets, targetJobs: Object.fromEntries(inventory.targetToJob), },