From 45a735324527c6a893618c87d1fc0b2ac69d1aee Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Fri, 24 Jul 2026 10:38:21 -0700 Subject: [PATCH 1/6] ci(e2e): report runner wait and execution time Signed-off-by: Charan Jagwani --- scripts/scorecard/coordinate-scorecard.mts | 16 +++++ scripts/scorecard/summarize-jobs.mts | 64 +++++++++++++++++- test/e2e/README.md | 13 ++-- .../support/e2e-scorecard-coordinator.test.ts | 65 +++++++++++++++++++ test/e2e/support/e2e-scorecard.test.ts | 2 + 5 files changed, 153 insertions(+), 7 deletions(-) diff --git a/scripts/scorecard/coordinate-scorecard.mts b/scripts/scorecard/coordinate-scorecard.mts index b78b63f7d3f..cddc8443068 100644 --- a/scripts/scorecard/coordinate-scorecard.mts +++ b/scripts/scorecard/coordinate-scorecard.mts @@ -113,6 +113,22 @@ function renderSummaryLines(input: { lines.push(job.url ? ` - [${job.name}](${job.url})` : ` - \`${job.name}\``); } } + if (summary.timingRows.length > 0) { + const duration = (milliseconds: number | null) => + milliseconds === null ? "n/a" : `${(milliseconds / 1_000).toFixed(1)}s`; + lines.push( + "", + "### Runner wait vs execution", + "", + "| Job | Runner class | Outcome | Queue | Execution |", + "| --- | --- | --- | ---: | ---: |", + ); + for (const row of summary.timingRows) { + lines.push( + `| ${row.name.replaceAll("|", "\\|")} | ${row.runnerClass} | ${row.outcome} | ${duration(row.queueMs)} | ${duration(row.executionMs)} |`, + ); + } + } if (input.perfect) lines.push("", "🎉 **All jobs passed!**"); lines.push( "", diff --git a/scripts/scorecard/summarize-jobs.mts b/scripts/scorecard/summarize-jobs.mts index e8607e047e9..27881699b24 100644 --- a/scripts/scorecard/summarize-jobs.mts +++ b/scripts/scorecard/summarize-jobs.mts @@ -4,9 +4,12 @@ type ApiJob = { completed_at?: string | null; conclusion?: string | null; + created_at?: string | null; html_url?: string | null; + labels?: string[] | null; name: string; run_attempt?: number | null; + started_at?: string | null; status?: string | null; }; @@ -14,6 +17,16 @@ type NeedResult = { result?: string }; type FailedJob = { name: string; url: string | null }; +type CountedResult = "cancelled" | "failure" | "skipped" | "success"; + +export type JobTimingRow = { + executionMs: number | null; + name: string; + outcome: CountedResult; + queueMs: number | null; + runnerClass: "larger" | "standard" | "unknown"; +}; + export type JobSummary = { cancelled: number; failedJobs: FailedJob[]; @@ -21,6 +34,7 @@ export type JobSummary = { ran: number; skipped: number; success: number; + timingRows: JobTimingRow[]; total: number; }; @@ -44,8 +58,6 @@ export type WorkflowRunJobsDeps = { }; }; -type CountedResult = "cancelled" | "failure" | "skipped" | "success"; - function isSelectiveDispatch(eventName: string, rawJobs = "", rawTargets = ""): boolean { return eventName === "workflow_dispatch" && (rawJobs.trim() !== "" || rawTargets.trim() !== ""); } @@ -66,7 +78,9 @@ function classifyNeed(value: NeedResult): CountedResult { return "failure"; } -function countResults(results: CountedResult[]): Omit { +function countResults( + results: CountedResult[], +): Omit { return { cancelled: results.filter((result) => result === "cancelled").length, failure: results.filter((result) => result === "failure").length, @@ -75,6 +89,48 @@ function countResults(results: CountedResult[]): Omit label.toLowerCase())); + if (normalized.has("ubuntu-latest")) return "standard"; + if (normalized.has("self-hosted")) return "unknown"; + return "larger"; +} + +function summarizeJobTimings(jobs: ApiJob[]): JobTimingRow[] { + return jobs + .map( + (job): JobTimingRow => ({ + executionMs: elapsedMs(job.started_at, job.completed_at), + name: job.name, + outcome: classifyApiJob(job), + queueMs: elapsedMs(job.created_at, job.started_at), + runnerClass: normalizeRunnerClass(job.labels), + }), + ) + .filter((row) => row.executionMs !== null || row.queueMs !== null) + .sort( + (left, right) => + (right.executionMs ?? 0) + + (right.queueMs ?? 0) - + ((left.executionMs ?? 0) + (left.queueMs ?? 0)) || left.name.localeCompare(right.name), + ) + .slice(0, 10); +} + function preferCandidate(candidate: ApiJob, existing: ApiJob | undefined): boolean { if (!existing) return true; const candidateAttempt = candidate.run_attempt ?? 0; @@ -142,6 +198,7 @@ function summarizeJobs(input: SummarizeJobsInput): JobSummary { .filter(({ result }) => result === "failure") .map(({ job }) => ({ name: job.name, url: job.html_url ?? null })), ran: jobs.length - counts.skipped, + timingRows: summarizeJobTimings(jobs), total: jobs.length, }; } @@ -158,6 +215,7 @@ function summarizeJobs(input: SummarizeJobsInput): JobSummary { .filter(({ result }) => result === "failure") .map(({ name }) => ({ name, url: null })), ran: entries.length - counts.skipped, + timingRows: [], total: entries.length, }; } diff --git a/test/e2e/README.md b/test/e2e/README.md index 587467d28ae..58a86ce7aa9 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -147,10 +147,15 @@ graph as the live targets: retired. Any future issue escalation should use a separately reviewed exceptional threshold, such as the same lane failing twice consecutively or remaining broken for 24 hours, rather than posting on every failed schedule. -- `scorecard` writes the scheduled/manual result summary, adds this run's - semantic phase runtime table, compares the trusted cloud-onboard timing - summary with the latest prior-release `e2e.yaml` run, and posts to the daily - or full-run Slack route. +- `scorecard` writes the scheduled/manual result summary and posts it to the + daily or full-run Slack route. The summary: + - separates queue time from execution time for the ten jobs with the longest + combined duration; + - reports the runner class as `standard`, `larger`, or `unknown` without + exposing runner labels; + - adds this run's semantic phase runtime table; and + - compares the trusted cloud-onboard timing summary with the latest + prior-release `e2e.yaml` run. - Selective dispatches remain silent unless they run on `main` with `post_to_slack=true`, which uses the preview Slack route. Branch-dispatched runs never receive Slack webhook secrets. diff --git a/test/e2e/support/e2e-scorecard-coordinator.test.ts b/test/e2e/support/e2e-scorecard-coordinator.test.ts index e9d973b92bb..bb7caef5e1c 100644 --- a/test/e2e/support/e2e-scorecard-coordinator.test.ts +++ b/test/e2e/support/e2e-scorecard-coordinator.test.ts @@ -163,6 +163,71 @@ describe("scorecard coordinator assembly", () => { expect(scorecardData).toMatchObject({ total: 2, success: 1, failure: 1, perfect: false }); expect(scorecardData.failedJobs).toEqual([{ name: "hermes-slack", url: null }]); }); + + it("separates runner queue time from execution time without exposing runner labels", () => { + const { summaryMarkdown } = coordinator.buildScorecard( + coordinatorInput({ + apiJobs: [ + { + completed_at: "2026-07-24T00:01:30Z", + conclusion: "failure", + created_at: "2026-07-24T00:00:00Z", + labels: ["private-larger-runner-label"], + name: "mcp-bridge", + started_at: "2026-07-24T00:00:30Z", + status: "completed", + }, + { + completed_at: "2026-07-24T00:00:25Z", + conclusion: "success", + created_at: "2026-07-24T00:00:00Z", + labels: ["ubuntu-latest"], + name: "cloud-onboard", + started_at: "2026-07-24T00:00:05Z", + status: "completed", + }, + { + completed_at: "2026-07-24T00:00:20Z", + conclusion: "success", + created_at: "invalid", + labels: ["self-hosted", "Linux"], + name: "jetson-nvmap-gpu", + started_at: "2026-07-24T00:00:10Z", + status: "completed", + }, + ], + rawExplicitOnly: "jetson-nvmap-gpu", + rawJobs: "jetson-nvmap-gpu", + }), + ); + + expect(summaryMarkdown).toContain("### Runner wait vs execution"); + expect(summaryMarkdown).toContain("| mcp-bridge | larger | failure | 30.0s | 60.0s |"); + expect(summaryMarkdown).toContain("| cloud-onboard | standard | success | 5.0s | 20.0s |"); + expect(summaryMarkdown).toContain("| jetson-nvmap-gpu | unknown | success | n/a | 10.0s |"); + expect(summaryMarkdown).not.toContain("private-larger-runner-label"); + }); + + it("bounds the job timing table to the ten slowest rows", () => { + const { summaryMarkdown } = coordinator.buildScorecard( + coordinatorInput({ + apiJobs: Array.from({ length: 12 }, (_, index) => ({ + completed_at: new Date(Date.UTC(2026, 6, 24, 0, 0, index + 1)).toISOString(), + conclusion: "success", + created_at: "2026-07-24T00:00:00.000Z", + labels: ["ubuntu-latest"], + name: `job-${String(index + 1).padStart(2, "0")}`, + started_at: "2026-07-24T00:00:00.000Z", + status: "completed", + })), + }), + ); + + expect(summaryMarkdown).toContain("| job-12 | standard | success | 0.0s | 12.0s |"); + expect(summaryMarkdown).toContain("| job-03 | standard | success | 0.0s | 3.0s |"); + expect(summaryMarkdown).not.toContain("| job-02 |"); + expect(summaryMarkdown).not.toContain("| job-01 |"); + }); }); describe("scorecard coordinator Slack payload guard", () => { diff --git a/test/e2e/support/e2e-scorecard.test.ts b/test/e2e/support/e2e-scorecard.test.ts index d6c3fe5701b..5186a629ae1 100644 --- a/test/e2e/support/e2e-scorecard.test.ts +++ b/test/e2e/support/e2e-scorecard.test.ts @@ -474,6 +474,7 @@ describe("E2E scorecard", () => { ran: 4, skipped: 0, success: 3, + timingRows: [], total: 4, }); }); @@ -501,6 +502,7 @@ describe("E2E scorecard", () => { ran: 2, skipped: 1, success: 1, + timingRows: [], total: 3, }); }); From 048c802b9b1cee28782e614433e3c9a3a20020af Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Fri, 24 Jul 2026 10:54:23 -0700 Subject: [PATCH 2/6] fix(e2e): classify mixed self-hosted labels safely Signed-off-by: Charan Jagwani --- scripts/scorecard/summarize-jobs.mts | 2 +- test/e2e/support/e2e-scorecard-coordinator.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/scorecard/summarize-jobs.mts b/scripts/scorecard/summarize-jobs.mts index 27881699b24..634ee9a66e9 100644 --- a/scripts/scorecard/summarize-jobs.mts +++ b/scripts/scorecard/summarize-jobs.mts @@ -105,8 +105,8 @@ function normalizeRunnerClass( ): JobTimingRow["runnerClass"] { if (!labels || labels.length === 0) return "unknown"; const normalized = new Set(labels.map((label) => label.toLowerCase())); - if (normalized.has("ubuntu-latest")) return "standard"; if (normalized.has("self-hosted")) return "unknown"; + if (normalized.has("ubuntu-latest")) return "standard"; return "larger"; } diff --git a/test/e2e/support/e2e-scorecard-coordinator.test.ts b/test/e2e/support/e2e-scorecard-coordinator.test.ts index bb7caef5e1c..e23d36bb22d 100644 --- a/test/e2e/support/e2e-scorecard-coordinator.test.ts +++ b/test/e2e/support/e2e-scorecard-coordinator.test.ts @@ -190,7 +190,7 @@ describe("scorecard coordinator assembly", () => { completed_at: "2026-07-24T00:00:20Z", conclusion: "success", created_at: "invalid", - labels: ["self-hosted", "Linux"], + labels: ["self-hosted", "ubuntu-latest", "Linux"], name: "jetson-nvmap-gpu", started_at: "2026-07-24T00:00:10Z", status: "completed", From 436e5a3588524fbaeeebdc07b80063a4ca223ea0 Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Fri, 24 Jul 2026 11:06:50 -0700 Subject: [PATCH 3/6] ci(e2e): add rolling runtime history Signed-off-by: Charan Jagwani --- .github/workflows/e2e.yaml | 25 +- scripts/audit-test-runtime.mts | 13 + scripts/scorecard/analyze-runtime-history.mts | 346 ++++++++++++++++++ scripts/scorecard/analyze-trace-timing.mts | 154 +------- scripts/scorecard/read-artifact-zip.mts | 180 +++++++++ test/e2e/README.md | 24 +- test/e2e/support/artifact-zip.test.ts | 118 ++++++ .../e2e-operations-workflow-boundary.test.ts | 37 +- test/e2e/support/e2e-runtime-history.test.ts | 176 +++++++++ test/e2e/support/test-runtime-audit.test.ts | 6 + ...ad-e2e-artifacts-workflow-boundary.test.ts | 12 + ...upload-e2e-artifacts-workflow-boundary.mts | 25 +- 12 files changed, 957 insertions(+), 159 deletions(-) create mode 100644 scripts/scorecard/analyze-runtime-history.mts create mode 100644 scripts/scorecard/read-artifact-zip.mts create mode 100644 test/e2e/support/artifact-zip.test.ts create mode 100644 test/e2e/support/e2e-runtime-history.test.ts diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 8f5f9adbd97..a95f765ce77 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -5795,6 +5795,7 @@ jobs: EXPLICIT_ONLY_JOBS: ${{ needs.generate-matrix.outputs.explicit_only_jobs }} JOBS: ${{ inputs.jobs }} RUNTIME_ARTIFACTS: ${{ runner.temp }}/e2e-runtime-audit + RUNTIME_SUMMARY_FILE: ${{ runner.temp }}/e2e-runtime-summary.json TARGETS: ${{ inputs.targets }} with: script: | @@ -5811,6 +5812,9 @@ jobs: const runtimeAudit = require( path.join(process.env.GITHUB_WORKSPACE, 'scripts/audit-test-runtime.mts'), ); + const runtimeHistory = require( + path.join(process.env.GITHUB_WORKSPACE, 'scripts/scorecard/analyze-runtime-history.mts'), + ); const needs = ${{ toJSON(needs) }}; // GitHub's jobs API is the canonical source because `needs.live` @@ -5818,8 +5822,9 @@ jobs: // The typed helper owns and tests the degraded `needs` fallback. const apiJobs = await scorecardJobs.loadWorkflowRunJobs({ github, context, core }); let runtimeSummaryMarkdown; + let runtimeRows = []; try { - const runtimeRows = runtimeAudit.auditTestRuntime([process.env.RUNTIME_ARTIFACTS]); + runtimeRows = runtimeAudit.auditTestRuntime([process.env.RUNTIME_ARTIFACTS]); runtimeSummaryMarkdown = runtimeAudit.formatRuntimeAuditSummary(runtimeRows); } catch { core.warning('E2E test phase runtime summary unavailable: invalid progress artifact'); @@ -5830,6 +5835,11 @@ jobs: '', ].join('\n'); } + const runtimeHistoryMarkdown = await runtimeHistory.buildRuntimeHistory( + { github, context, core }, + runtimeRows, + process.env.RUNTIME_SUMMARY_FILE, + ); const trace = await traceTiming.buildTraceTimingResult({ github, context, core }); if (trace.budgetWarningMessage) core.warning(trace.budgetWarningMessage); @@ -5848,7 +5858,9 @@ jobs: today: new Date().toLocaleDateString('en-US', { month: 'short', day: 'numeric' }), }); - await core.summary.addRaw(`${summaryMarkdown}\n\n${runtimeSummaryMarkdown}`).write(); + await core.summary + .addRaw(`${summaryMarkdown}\n\n${runtimeSummaryMarkdown}\n${runtimeHistoryMarkdown}`) + .write(); core.setOutput('scorecardData', JSON.stringify(scorecardData)); core.setOutput('slackData', JSON.stringify(slackData)); @@ -5909,3 +5921,12 @@ jobs: if (!response.ok) { core.setFailed(`Slack webhook returned ${response.status}`); } + + - name: Upload E2E runtime summary + if: ${{ always() && steps.scorecard.outcome == 'success' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-runtime-summary + path: ${{ runner.temp }}/e2e-runtime-summary.json + if-no-files-found: error + retention-days: 14 diff --git a/scripts/audit-test-runtime.mts b/scripts/audit-test-runtime.mts index c6611a7bf29..58c0d2fe263 100644 --- a/scripts/audit-test-runtime.mts +++ b/scripts/audit-test-runtime.mts @@ -15,6 +15,9 @@ export interface RuntimeAuditRow { p95Ms: number; maxMs: number; variabilityMs: number; + passedRuns: number; + failedRuns: number; + skippedRuns: number; slowestPhase: string; slowestPhaseMs: number; slowestPhaseOutcome: "passed" | "failed" | "skipped"; @@ -78,6 +81,12 @@ function median(sorted: readonly number[]): number { return sorted[middle] ?? 0; } +function summaryOutcome(summary: ProgressSummary): "passed" | "failed" | "skipped" { + if (summary.phases.some((phase) => phase.outcome === "failed")) return "failed"; + if (summary.phases.some((phase) => phase.outcome === "skipped")) return "skipped"; + return "passed"; +} + export function auditTestRuntime(roots: readonly string[]): RuntimeAuditRow[] { const summaries = roots.flatMap(progressFiles).map((file) => { const parsed: unknown = JSON.parse(fs.readFileSync(file, "utf8")); @@ -108,6 +117,7 @@ export function auditTestRuntime(roots: readonly string[]): RuntimeAuditRow[] { ); const medianMs = median(durations); const p95Ms = percentile(durations, 0.95); + const outcomes = runs.map(summaryOutcome); return { target: [first.targetId ?? "unlabeled", first.shardId].filter(Boolean).join("/"), scenario: first.scenario, @@ -116,6 +126,9 @@ export function auditTestRuntime(roots: readonly string[]): RuntimeAuditRow[] { p95Ms, maxMs: durations.at(-1) ?? 0, variabilityMs: Math.max(0, p95Ms - medianMs), + passedRuns: outcomes.filter((outcome) => outcome === "passed").length, + failedRuns: outcomes.filter((outcome) => outcome === "failed").length, + skippedRuns: outcomes.filter((outcome) => outcome === "skipped").length, slowestPhase: slowestPhase.label, slowestPhaseMs: slowestPhase.durationMs, slowestPhaseOutcome: slowestPhase.outcome, diff --git a/scripts/scorecard/analyze-runtime-history.mts b/scripts/scorecard/analyze-runtime-history.mts new file mode 100644 index 00000000000..024350bee6a --- /dev/null +++ b/scripts/scorecard/analyze-runtime-history.mts @@ -0,0 +1,346 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; + +import type { RuntimeAuditRow } from "../audit-test-runtime.mts"; +import { readValidatedArtifactZipEntry } from "./read-artifact-zip.mts"; + +export const RUNTIME_SUMMARY_ARTIFACT = "e2e-runtime-summary"; +export const RUNTIME_SUMMARY_FILE = "e2e-runtime-summary.json"; +const RUNTIME_SUMMARY_SCHEMA = "nemoclaw.e2e_runtime_summary.v1"; +const WORKFLOW_FILE = "e2e.yaml"; +const HISTORY_RUN_LIMIT = 10; +const MAX_SUMMARY_BYTES = 256 * 1024; +const MAX_SUMMARY_ROWS = 200; + +type GitHubDeps = { + github: any; + context: { repo: { owner: string; repo: string }; runId: number }; + core?: { warning?: (message: string) => void }; +}; + +export interface RuntimeSummaryArtifact { + schemaVersion: typeof RUNTIME_SUMMARY_SCHEMA; + runId: number; + createdAt: string; + rows: RuntimeAuditRow[]; +} + +type RuntimeHistoryServices = { + loadPriorNightlySummaries: (deps: GitHubDeps) => Promise; +}; + +function isBoundedString(value: unknown): value is string { + return ( + typeof value === "string" && + value.length > 0 && + value.length <= 500 && + !/[\u0000-\u001f\u007f]/u.test(value) + ); +} + +function isNonNegativeNumber(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value) && value >= 0; +} + +function isNonNegativeInteger(value: unknown): value is number { + return Number.isInteger(value) && isNonNegativeNumber(value); +} + +function hasExactKeys(value: Record, expected: readonly string[]): boolean { + return Object.keys(value).sort().join("\0") === [...expected].sort().join("\0"); +} + +function normalizeRuntimeRow(value: unknown): RuntimeAuditRow | null { + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const row = value as Record; + if ( + !hasExactKeys(row, [ + "target", + "scenario", + "runs", + "medianMs", + "p95Ms", + "maxMs", + "variabilityMs", + "passedRuns", + "failedRuns", + "skippedRuns", + "slowestPhase", + "slowestPhaseMs", + "slowestPhaseOutcome", + ]) || + !isBoundedString(row.target) || + !isBoundedString(row.scenario) || + !isNonNegativeInteger(row.runs) || + row.runs < 1 || + !isNonNegativeNumber(row.medianMs) || + !isNonNegativeNumber(row.p95Ms) || + !isNonNegativeNumber(row.maxMs) || + !isNonNegativeNumber(row.variabilityMs) || + row.medianMs > row.p95Ms || + row.p95Ms > row.maxMs || + row.variabilityMs !== row.p95Ms - row.medianMs || + !isNonNegativeInteger(row.passedRuns) || + !isNonNegativeInteger(row.failedRuns) || + !isNonNegativeInteger(row.skippedRuns) || + row.passedRuns + row.failedRuns + row.skippedRuns !== row.runs || + !isBoundedString(row.slowestPhase) || + !isNonNegativeNumber(row.slowestPhaseMs) || + (row.slowestPhaseOutcome !== "passed" && + row.slowestPhaseOutcome !== "failed" && + row.slowestPhaseOutcome !== "skipped") + ) { + return null; + } + return { + target: row.target, + scenario: row.scenario, + runs: row.runs, + medianMs: row.medianMs, + p95Ms: row.p95Ms, + maxMs: row.maxMs, + variabilityMs: row.variabilityMs, + passedRuns: row.passedRuns, + failedRuns: row.failedRuns, + skippedRuns: row.skippedRuns, + slowestPhase: row.slowestPhase, + slowestPhaseMs: row.slowestPhaseMs, + slowestPhaseOutcome: row.slowestPhaseOutcome, + }; +} + +export function normalizeRuntimeSummary(value: unknown): RuntimeSummaryArtifact | null { + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const summary = value as Record; + if ( + !hasExactKeys(summary, ["schemaVersion", "runId", "createdAt", "rows"]) || + summary.schemaVersion !== RUNTIME_SUMMARY_SCHEMA || + !isNonNegativeInteger(summary.runId) || + summary.runId < 1 || + typeof summary.createdAt !== "string" || + !Number.isFinite(Date.parse(summary.createdAt)) || + !Array.isArray(summary.rows) || + summary.rows.length > MAX_SUMMARY_ROWS + ) { + return null; + } + const rows = summary.rows.map(normalizeRuntimeRow); + if (rows.some((row) => row === null)) return null; + const identities = new Set( + (rows as RuntimeAuditRow[]).map((row) => JSON.stringify([row.target, row.scenario])), + ); + if (identities.size !== rows.length) return null; + return { + schemaVersion: RUNTIME_SUMMARY_SCHEMA, + runId: summary.runId, + createdAt: summary.createdAt, + rows: rows as RuntimeAuditRow[], + }; +} + +export function createRuntimeSummary( + runId: number, + createdAt: string, + rows: readonly RuntimeAuditRow[], +): RuntimeSummaryArtifact { + const summary = normalizeRuntimeSummary({ + schemaVersion: RUNTIME_SUMMARY_SCHEMA, + runId, + createdAt, + rows, + }); + if (summary === null) throw new Error("invalid current E2E runtime summary"); + return summary; +} + +function parseRuntimeSummaryArchive(archive: Buffer): RuntimeSummaryArtifact | null { + try { + const contents = readValidatedArtifactZipEntry(archive, RUNTIME_SUMMARY_FILE, { + maxBytes: MAX_SUMMARY_BYTES, + }); + return contents === null ? null : normalizeRuntimeSummary(JSON.parse(contents)); + } catch { + return null; + } +} + +async function readRuntimeSummaryFromRun( + { github, context }: GitHubDeps, + runId: number, +): Promise { + const artifacts = (await github.paginate(github.rest.actions.listWorkflowRunArtifacts, { + owner: context.repo.owner, + repo: context.repo.repo, + run_id: runId, + per_page: 100, + })) as Array<{ expired?: boolean; id: number; name: string }>; + const artifact = artifacts.find( + (candidate) => candidate.name === RUNTIME_SUMMARY_ARTIFACT && candidate.expired !== true, + ); + if (!artifact) return null; + const download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: artifact.id, + archive_format: "zip", + }); + const summary = parseRuntimeSummaryArchive(Buffer.from(download.data)); + return summary?.runId === runId ? summary : null; +} + +export async function loadPriorNightlySummaries( + deps: GitHubDeps, +): Promise { + const { github, context, core } = deps; + const response = await github.rest.actions.listWorkflowRuns({ + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: WORKFLOW_FILE, + event: "schedule", + status: "completed", + per_page: HISTORY_RUN_LIMIT + 5, + }); + const runs = (response.data.workflow_runs as Array<{ id: number }>) + .filter((run) => run.id !== context.runId) + .slice(0, HISTORY_RUN_LIMIT); + const summaries: RuntimeSummaryArtifact[] = []; + for (const run of runs) { + try { + const summary = await readRuntimeSummaryFromRun(deps, run.id); + if (summary !== null) summaries.push(summary); + } catch { + core?.warning?.( + "One prior nightly runtime summary was unavailable; continuing with less history.", + ); + } + } + return summaries; +} + +function percentile(sorted: readonly number[], fraction: number): number { + return sorted[Math.max(0, Math.ceil(sorted.length * fraction) - 1)] ?? 0; +} + +function median(sorted: readonly number[]): number { + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 + ? ((sorted[middle - 1] ?? 0) + (sorted[middle] ?? 0)) / 2 + : (sorted[middle] ?? 0); +} + +function seconds(milliseconds: number): string { + return `${(milliseconds / 1000).toFixed(1)}s`; +} + +function escapeCell(value: string): string { + return value.replaceAll("|", "\\|").replaceAll("\n", " "); +} + +function formatDelta(currentMs: number, priorMs: number): string { + const deltaMs = currentMs - priorMs; + const sign = deltaMs >= 0 ? "+" : "-"; + const percent = priorMs > 0 ? ` (${sign}${Math.abs((deltaMs / priorMs) * 100).toFixed(1)}%)` : ""; + return `${sign}${seconds(Math.abs(deltaMs))}${percent}`; +} + +function formatOutcome(row: RuntimeAuditRow): string { + if (row.failedRuns > 0) return "failed"; + if (row.skippedRuns > 0) return "skipped"; + return "passed"; +} + +function formatPassRate(rows: readonly RuntimeAuditRow[]): string { + const passed = rows.reduce((total, row) => total + row.passedRuns, 0); + const runs = rows.reduce((total, row) => total + row.runs, 0); + return runs > 0 ? `${((passed / runs) * 100).toFixed(0)}% (${passed}/${runs})` : "n/a"; +} + +function passRate(rows: readonly RuntimeAuditRow[]): number { + const passed = rows.reduce((total, row) => total + row.passedRuns, 0); + const runs = rows.reduce((total, row) => total + row.runs, 0); + return runs > 0 ? passed / runs : 1; +} + +export function formatRuntimeHistory( + currentRows: readonly RuntimeAuditRow[], + priorSummaries: readonly RuntimeSummaryArtifact[], +): string { + const lines = [ + "## E2E Nightly Runtime Trend", + "", + "Current run compared with up to 10 prior completed scheduled runs; manual runs are excluded.", + "Rows prioritize current failures, lower historical pass rates, and larger runtime regressions.", + "", + ]; + if (currentRows.length === 0) { + lines.push("No current runtime rows were available for comparison."); + return `${lines.join("\n")}\n`; + } + if (priorSummaries.length === 0) { + lines.push( + "No prior nightly runtime summaries are available yet; this run starts the history.", + ); + return `${lines.join("\n")}\n`; + } + + lines.push( + "| Target | Scenario | Prior nights | Current median | Prior median | Prior p95 | Delta | Prior pass rate | Current |", + "| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | --- |", + ); + const comparisons = currentRows + .map((current) => { + const priorRows = priorSummaries.flatMap((summary) => + summary.rows.filter( + (row) => row.target === current.target && row.scenario === current.scenario, + ), + ); + const priorMedian = + priorRows.length > 0 + ? median(priorRows.map((row) => row.medianMs).sort((a, b) => a - b)) + : null; + return { current, priorRows, priorMedian }; + }) + .sort( + (a, b) => + Number(b.current.failedRuns > 0) - Number(a.current.failedRuns > 0) || + passRate(a.priorRows) - passRate(b.priorRows) || + Math.max(0, b.priorMedian === null ? 0 : b.current.medianMs - b.priorMedian) - + Math.max(0, a.priorMedian === null ? 0 : a.current.medianMs - a.priorMedian) || + b.current.p95Ms - a.current.p95Ms, + ); + for (const { current, priorRows, priorMedian } of comparisons.slice(0, 10)) { + if (priorRows.length === 0) { + lines.push( + `| ${escapeCell(current.target)} | ${escapeCell(current.scenario)} | 0 | ${seconds(current.medianMs)} | n/a | n/a | n/a | n/a | ${formatOutcome(current)} |`, + ); + continue; + } + const medians = priorRows.map((row) => row.medianMs).sort((a, b) => a - b); + lines.push( + `| ${escapeCell(current.target)} | ${escapeCell(current.scenario)} | ${priorRows.length} | ${seconds(current.medianMs)} | ${seconds(priorMedian ?? 0)} | ${seconds(percentile(medians, 0.95))} | ${formatDelta(current.medianMs, priorMedian ?? 0)} | ${formatPassRate(priorRows)} | ${formatOutcome(current)} |`, + ); + } + return `${lines.join("\n")}\n`; +} + +export async function buildRuntimeHistory( + deps: GitHubDeps, + currentRows: readonly RuntimeAuditRow[], + outputPath: string, + services: RuntimeHistoryServices = { loadPriorNightlySummaries }, + now = new Date(), +): Promise { + const current = createRuntimeSummary(deps.context.runId, now.toISOString(), currentRows); + fs.writeFileSync(outputPath, `${JSON.stringify(current, null, 2)}\n`, { mode: 0o600 }); + try { + const prior = await services.loadPriorNightlySummaries(deps); + return formatRuntimeHistory(currentRows, prior); + } catch { + deps.core?.warning?.( + "Nightly E2E runtime history unavailable; current summary was still saved.", + ); + return formatRuntimeHistory(currentRows, []); + } +} diff --git a/scripts/scorecard/analyze-trace-timing.mts b/scripts/scorecard/analyze-trace-timing.mts index 10eb1acf42d..9f9e55ccb2c 100644 --- a/scripts/scorecard/analyze-trace-timing.mts +++ b/scripts/scorecard/analyze-trace-timing.mts @@ -4,7 +4,8 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import zlib from "node:zlib"; + +import { readValidatedArtifactZipEntry } from "./read-artifact-zip.mts"; type SemverTag = { name: string; major: number; minor: number; patch: number; sha?: string }; type Threshold = { minDeltaMs: number; minPercent: number }; @@ -46,17 +47,6 @@ type TraceTimingResult = { budgetWarningMessage: string | null; budgetStatus: string; }; -type ZipSummaryEntry = { - creatorSystem: number; - flags: number; - compressionMethod: number; - expectedCrc: number; - compressedSize: number; - uncompressedSize: number; - diskStart: number; - externalAttributes: number; - localHeaderOffset: number; -}; type GitHubDeps = { github: any; context: any; core?: { warning?: (message: string) => void } }; type TraceTimingServices = { findLatestCompletedE2eRunForReleaseTag: (deps: GitHubDeps, tag: SemverTag) => Promise; @@ -71,9 +61,6 @@ const MAX_TRACE_SUMMARY_BYTES = 1024 * 1024; const MAX_TRACE_ARCHIVE_ENTRIES = 1000; const TRACE_ARCHIVE_REJECTION_WARNING = "Trace timing artifact ZIP validation failed; ignoring the malformed or unsupported archive."; -const ZIP_CENTRAL_DIRECTORY_SIGNATURE = 0x02014b50; -const ZIP_END_OF_CENTRAL_DIRECTORY_SIGNATURE = 0x06054b50; -const ZIP_LOCAL_FILE_SIGNATURE = 0x04034b50; const ONBOARD_PERFORMANCE_BUDGET_FILE = "ci/onboard-performance-budget.json"; const REPO_ROOT = path.resolve(import.meta.dirname, "..", ".."); const ONBOARD_PHASE_PREFIX = "nemoclaw.onboard.phase."; @@ -531,30 +518,6 @@ async function findLatestCompletedE2eRunForReleaseTag( return null; } -function findZipEndOfCentralDirectory(archive: Buffer): number { - const minimumOffset = Math.max(0, archive.length - 22 - 0xffff); - for (let offset = archive.length - 22; offset >= minimumOffset; offset -= 1) { - if ( - archive.readUInt32LE(offset) === ZIP_END_OF_CENTRAL_DIRECTORY_SIGNATURE && - offset + 22 + archive.readUInt16LE(offset + 20) === archive.length - ) { - return offset; - } - } - return -1; -} - -function crc32(data: Buffer): number { - let crc = 0xffffffff; - for (const byte of data) { - crc ^= byte; - for (let bit = 0; bit < 8; bit += 1) { - crc = (crc >>> 1) ^ (0xedb88320 & -(crc & 1)); - } - } - return (crc ^ 0xffffffff) >>> 0; -} - // GitHub creates the workflow artifact ZIP outside this repository, and the // cloud-onboard artifact intentionally contains diagnostics beside the trusted // timing summary. Parse only the exact root-level summary in-process so the @@ -562,115 +525,10 @@ function crc32(data: Buffer): number { // production-shape multi-entry regression test is the removal guard; retire // this parser if GitHub provides a verified single-file artifact API. function readValidatedTraceSummaryArchive(archive: Buffer): string | null { - const endOffset = findZipEndOfCentralDirectory(archive); - if (endOffset < 0) return null; - - const diskNumber = archive.readUInt16LE(endOffset + 4); - const centralDirectoryDisk = archive.readUInt16LE(endOffset + 6); - const entriesOnDisk = archive.readUInt16LE(endOffset + 8); - const totalEntries = archive.readUInt16LE(endOffset + 10); - const centralDirectorySize = archive.readUInt32LE(endOffset + 12); - const centralDirectoryOffset = archive.readUInt32LE(endOffset + 16); - if ( - diskNumber !== 0 || - centralDirectoryDisk !== 0 || - entriesOnDisk !== totalEntries || - totalEntries < 1 || - totalEntries > MAX_TRACE_ARCHIVE_ENTRIES || - centralDirectoryOffset + centralDirectorySize !== endOffset - ) { - return null; - } - - const expectedFileName = Buffer.from(TRACE_SUMMARY_FILE, "utf8"); - let centralEntryOffset = centralDirectoryOffset; - let target: ZipSummaryEntry | null = null; - for (let entryIndex = 0; entryIndex < totalEntries; entryIndex += 1) { - if ( - centralEntryOffset + 46 > endOffset || - archive.readUInt32LE(centralEntryOffset) !== ZIP_CENTRAL_DIRECTORY_SIGNATURE - ) { - return null; - } - const fileNameLength = archive.readUInt16LE(centralEntryOffset + 28); - const extraLength = archive.readUInt16LE(centralEntryOffset + 30); - const commentLength = archive.readUInt16LE(centralEntryOffset + 32); - const centralEntryEnd = centralEntryOffset + 46 + fileNameLength + extraLength + commentLength; - if (centralEntryEnd > endOffset) return null; - const fileName = archive.subarray( - centralEntryOffset + 46, - centralEntryOffset + 46 + fileNameLength, - ); - if (fileName.equals(expectedFileName)) { - if (target !== null) return null; - target = { - creatorSystem: archive.readUInt8(centralEntryOffset + 5), - flags: archive.readUInt16LE(centralEntryOffset + 8), - compressionMethod: archive.readUInt16LE(centralEntryOffset + 10), - expectedCrc: archive.readUInt32LE(centralEntryOffset + 16), - compressedSize: archive.readUInt32LE(centralEntryOffset + 20), - uncompressedSize: archive.readUInt32LE(centralEntryOffset + 24), - diskStart: archive.readUInt16LE(centralEntryOffset + 34), - externalAttributes: archive.readUInt32LE(centralEntryOffset + 38), - localHeaderOffset: archive.readUInt32LE(centralEntryOffset + 42), - }; - } - centralEntryOffset = centralEntryEnd; - } - if (centralEntryOffset !== endOffset || target === null) return null; - - const { - creatorSystem, - flags, - compressionMethod, - expectedCrc, - compressedSize, - uncompressedSize, - diskStart, - externalAttributes, - localHeaderOffset, - } = target; - const unixFileType = (externalAttributes >>> 16) & 0xf000; - if ( - diskStart !== 0 || - (flags & 0x1) !== 0 || - (compressionMethod !== 0 && compressionMethod !== 8) || - compressedSize > MAX_TRACE_SUMMARY_BYTES || - uncompressedSize > MAX_TRACE_SUMMARY_BYTES || - (creatorSystem !== 0 && creatorSystem !== 3) || - (creatorSystem === 3 && unixFileType !== 0 && unixFileType !== 0x8000) || - localHeaderOffset + 30 > centralDirectoryOffset || - archive.readUInt32LE(localHeaderOffset) !== ZIP_LOCAL_FILE_SIGNATURE - ) { - return null; - } - - const localFlags = archive.readUInt16LE(localHeaderOffset + 6); - const localCompressionMethod = archive.readUInt16LE(localHeaderOffset + 8); - const localFileNameLength = archive.readUInt16LE(localHeaderOffset + 26); - const localExtraLength = archive.readUInt16LE(localHeaderOffset + 28); - const localFileName = archive.subarray( - localHeaderOffset + 30, - localHeaderOffset + 30 + localFileNameLength, - ); - const compressedDataOffset = localHeaderOffset + 30 + localFileNameLength + localExtraLength; - const compressedDataEnd = compressedDataOffset + compressedSize; - if ( - localFlags !== flags || - localCompressionMethod !== compressionMethod || - !localFileName.equals(expectedFileName) || - compressedDataEnd > centralDirectoryOffset - ) { - return null; - } - - const compressedData = archive.subarray(compressedDataOffset, compressedDataEnd); - const summary = - compressionMethod === 0 - ? Buffer.from(compressedData) - : zlib.inflateRawSync(compressedData, { maxOutputLength: MAX_TRACE_SUMMARY_BYTES }); - if (summary.length !== uncompressedSize || crc32(summary) !== expectedCrc) return null; - return summary.toString("utf8"); + return readValidatedArtifactZipEntry(archive, TRACE_SUMMARY_FILE, { + maxBytes: MAX_TRACE_SUMMARY_BYTES, + maxEntries: MAX_TRACE_ARCHIVE_ENTRIES, + }); } function readValidatedTraceSummaryZip( diff --git a/scripts/scorecard/read-artifact-zip.mts b/scripts/scorecard/read-artifact-zip.mts new file mode 100644 index 00000000000..5bb3ba0c09d --- /dev/null +++ b/scripts/scorecard/read-artifact-zip.mts @@ -0,0 +1,180 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import zlib from "node:zlib"; + +type ZipEntry = { + creatorSystem: number; + flags: number; + compressionMethod: number; + expectedCrc: number; + compressedSize: number; + uncompressedSize: number; + diskStart: number; + externalAttributes: number; + localHeaderOffset: number; +}; + +const ZIP_CENTRAL_DIRECTORY_SIGNATURE = 0x02014b50; +const ZIP_END_OF_CENTRAL_DIRECTORY_SIGNATURE = 0x06054b50; +const ZIP_LOCAL_FILE_SIGNATURE = 0x04034b50; + +function findZipEndOfCentralDirectory(archive: Buffer): number { + const minimumOffset = Math.max(0, archive.length - 22 - 0xffff); + for (let offset = archive.length - 22; offset >= minimumOffset; offset -= 1) { + if ( + archive.readUInt32LE(offset) === ZIP_END_OF_CENTRAL_DIRECTORY_SIGNATURE && + offset + 22 + archive.readUInt16LE(offset + 20) === archive.length + ) { + return offset; + } + } + return -1; +} + +function crc32(data: Buffer): number { + let crc = 0xffffffff; + for (const byte of data) { + crc ^= byte; + for (let bit = 0; bit < 8; bit += 1) { + crc = (crc >>> 1) ^ (0xedb88320 & -(crc & 1)); + } + } + return (crc ^ 0xffffffff) >>> 0; +} + +/** + * Reads one exact root-level file from a GitHub artifact ZIP without extracting + * paths to disk. The archive may contain other entries, but duplicate target + * entries, links, encryption, split archives, ZIP64, and oversized payloads are + * rejected. + */ +export function readValidatedArtifactZipEntry( + archive: Buffer, + expectedFile: string, + options: { maxBytes: number; maxEntries?: number }, +): string | null { + const maxEntries = options.maxEntries ?? 1000; + const expectedFileName = Buffer.from(expectedFile, "utf8"); + if ( + expectedFile.length === 0 || + expectedFile.includes("/") || + expectedFile.includes("\\") || + options.maxBytes < 1 || + maxEntries < 1 + ) { + return null; + } + + const endOffset = findZipEndOfCentralDirectory(archive); + if (endOffset < 0) return null; + + const diskNumber = archive.readUInt16LE(endOffset + 4); + const centralDirectoryDisk = archive.readUInt16LE(endOffset + 6); + const entriesOnDisk = archive.readUInt16LE(endOffset + 8); + const totalEntries = archive.readUInt16LE(endOffset + 10); + const centralDirectorySize = archive.readUInt32LE(endOffset + 12); + const centralDirectoryOffset = archive.readUInt32LE(endOffset + 16); + if ( + diskNumber !== 0 || + centralDirectoryDisk !== 0 || + entriesOnDisk !== totalEntries || + totalEntries < 1 || + totalEntries > maxEntries || + centralDirectoryOffset + centralDirectorySize !== endOffset + ) { + return null; + } + + let centralEntryOffset = centralDirectoryOffset; + let target: ZipEntry | null = null; + for (let entryIndex = 0; entryIndex < totalEntries; entryIndex += 1) { + if ( + centralEntryOffset + 46 > endOffset || + archive.readUInt32LE(centralEntryOffset) !== ZIP_CENTRAL_DIRECTORY_SIGNATURE + ) { + return null; + } + const fileNameLength = archive.readUInt16LE(centralEntryOffset + 28); + const extraLength = archive.readUInt16LE(centralEntryOffset + 30); + const commentLength = archive.readUInt16LE(centralEntryOffset + 32); + const centralEntryEnd = centralEntryOffset + 46 + fileNameLength + extraLength + commentLength; + if (centralEntryEnd > endOffset) return null; + const fileName = archive.subarray( + centralEntryOffset + 46, + centralEntryOffset + 46 + fileNameLength, + ); + if (fileName.equals(expectedFileName)) { + if (target !== null) return null; + target = { + creatorSystem: archive.readUInt8(centralEntryOffset + 5), + flags: archive.readUInt16LE(centralEntryOffset + 8), + compressionMethod: archive.readUInt16LE(centralEntryOffset + 10), + expectedCrc: archive.readUInt32LE(centralEntryOffset + 16), + compressedSize: archive.readUInt32LE(centralEntryOffset + 20), + uncompressedSize: archive.readUInt32LE(centralEntryOffset + 24), + diskStart: archive.readUInt16LE(centralEntryOffset + 34), + externalAttributes: archive.readUInt32LE(centralEntryOffset + 38), + localHeaderOffset: archive.readUInt32LE(centralEntryOffset + 42), + }; + } + centralEntryOffset = centralEntryEnd; + } + if (centralEntryOffset !== endOffset || target === null) return null; + + const { + creatorSystem, + flags, + compressionMethod, + expectedCrc, + compressedSize, + uncompressedSize, + diskStart, + externalAttributes, + localHeaderOffset, + } = target; + const unixFileType = (externalAttributes >>> 16) & 0xf000; + if ( + diskStart !== 0 || + (flags & 0x1) !== 0 || + (compressionMethod !== 0 && compressionMethod !== 8) || + compressedSize > options.maxBytes || + uncompressedSize > options.maxBytes || + (creatorSystem !== 0 && creatorSystem !== 3) || + (creatorSystem === 3 && unixFileType !== 0 && unixFileType !== 0x8000) || + localHeaderOffset + 30 > centralDirectoryOffset || + archive.readUInt32LE(localHeaderOffset) !== ZIP_LOCAL_FILE_SIGNATURE + ) { + return null; + } + + const localFlags = archive.readUInt16LE(localHeaderOffset + 6); + const localCompressionMethod = archive.readUInt16LE(localHeaderOffset + 8); + const localFileNameLength = archive.readUInt16LE(localHeaderOffset + 26); + const localExtraLength = archive.readUInt16LE(localHeaderOffset + 28); + const localFileNameEnd = localHeaderOffset + 30 + localFileNameLength; + const compressedDataOffset = localFileNameEnd + localExtraLength; + const compressedDataEnd = compressedDataOffset + compressedSize; + if ( + localFileNameEnd > centralDirectoryOffset || + localFlags !== flags || + localCompressionMethod !== compressionMethod || + !archive.subarray(localHeaderOffset + 30, localFileNameEnd).equals(expectedFileName) || + compressedDataEnd > centralDirectoryOffset + ) { + return null; + } + + const compressedData = archive.subarray(compressedDataOffset, compressedDataEnd); + let contents: Buffer; + try { + contents = + compressionMethod === 0 + ? Buffer.from(compressedData) + : zlib.inflateRawSync(compressedData, { maxOutputLength: options.maxBytes }); + } catch { + return null; + } + if (contents.length !== uncompressedSize || crc32(contents) !== expectedCrc) return null; + return contents.toString("utf8"); +} diff --git a/test/e2e/README.md b/test/e2e/README.md index 58a86ce7aa9..9caeed42108 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -153,9 +153,15 @@ graph as the live targets: combined duration; - reports the runner class as `standard`, `larger`, or `unknown` without exposing runner labels; - - adds this run's semantic phase runtime table; and + - adds this run's semantic phase runtime table; + - compares runtime and pass rate with up to ten prior scheduled summaries; + - orders that history by current failures, lowest prior pass rate, and largest + runtime regression; and - compares the trusted cloud-onboard timing summary with the latest prior-release `e2e.yaml` run. +- The rolling comparison uses only the bounded `e2e-runtime-summary.json` + artifact retained for 14 days. It does not download historical raw test + artifacts or include manual runs in the baseline. - Selective dispatches remain silent unless they run on `main` with `post_to_slack=true`, which uses the preview Slack route. Branch-dispatched runs never receive Slack webhook secrets. @@ -324,12 +330,16 @@ npm run test:runtime-audit -- path/to/run-1 path/to/run-2 The audit groups each test by target and optional shard, ranks the groups by p95 runtime, and reports variability plus the slowest observed phase's duration and outcome. Scheduled and ordinary manual runs include the same table for that -run in the GitHub Actions scorecard summary. Keep phase -labels specific to test behavior, call `progress.phase("literal phase label")` -at the declared boundaries in order, and transition through the final -test-declared phase on every passing path. Both fixtures reject a passing test -that never reaches that phase; only the stateful live fixture enters its -resource-release phase automatically. +run in the GitHub Actions scorecard summary. The scorecard also shows a bounded +nightly trend table with prior median, prior p95, delta, and pass rate; the +first run explicitly starts the history instead of inventing a baseline. Its +ordering puts current failures first, followed by the lowest historical pass +rates and largest runtime regressions. Keep phase labels specific to test +behavior, call `progress.phase("literal phase label")` at the declared +boundaries in order, and transition through the final test-declared phase on +every passing path. +Both fixtures reject a passing test that never reaches that phase; only the +stateful live fixture enters its resource-release phase automatically. Validate phase coverage without executing test bodies with: ```bash diff --git a/test/e2e/support/artifact-zip.test.ts b/test/e2e/support/artifact-zip.test.ts new file mode 100644 index 00000000000..e2177deb872 --- /dev/null +++ b/test/e2e/support/artifact-zip.test.ts @@ -0,0 +1,118 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import zlib from "node:zlib"; + +import { describe, expect, it } from "vitest"; + +import { readValidatedArtifactZipEntry } from "../../../scripts/scorecard/read-artifact-zip.mts"; + +function crc32(data: Buffer): number { + let crc = 0xffffffff; + for (const byte of data) { + crc ^= byte; + for (let bit = 0; bit < 8; bit += 1) { + crc = (crc >>> 1) ^ (0xedb88320 & -(crc & 1)); + } + } + return (crc ^ 0xffffffff) >>> 0; +} + +function artifactZip( + entries: Array<{ name: string; contents: string }>, + compressionMethod = 0, +): Buffer { + const localParts: Buffer[] = []; + const centralParts: Buffer[] = []; + let localOffset = 0; + for (const entry of entries) { + const name = Buffer.from(entry.name, "utf8"); + const contents = Buffer.from(entry.contents, "utf8"); + const compressed = + compressionMethod === 8 ? zlib.deflateRawSync(contents) : Buffer.from(contents); + const checksum = crc32(contents); + const local = Buffer.alloc(30); + local.writeUInt32LE(0x04034b50, 0); + local.writeUInt16LE(20, 4); + local.writeUInt16LE(compressionMethod, 8); + local.writeUInt32LE(checksum, 14); + local.writeUInt32LE(compressed.length, 18); + local.writeUInt32LE(contents.length, 22); + local.writeUInt16LE(name.length, 26); + localParts.push(local, name, compressed); + + const central = Buffer.alloc(46); + central.writeUInt32LE(0x02014b50, 0); + central.writeUInt16LE(0x0314, 4); + central.writeUInt16LE(20, 6); + central.writeUInt16LE(compressionMethod, 10); + central.writeUInt32LE(checksum, 16); + central.writeUInt32LE(compressed.length, 20); + central.writeUInt32LE(contents.length, 24); + central.writeUInt16LE(name.length, 28); + central.writeUInt32LE(0x80000000, 38); + central.writeUInt32LE(localOffset, 42); + centralParts.push(central, name); + localOffset += local.length + name.length + compressed.length; + } + const locals = Buffer.concat(localParts); + const centralDirectory = Buffer.concat(centralParts); + const end = Buffer.alloc(22); + end.writeUInt32LE(0x06054b50, 0); + end.writeUInt16LE(entries.length, 8); + end.writeUInt16LE(entries.length, 10); + end.writeUInt32LE(centralDirectory.length, 12); + end.writeUInt32LE(locals.length, 16); + return Buffer.concat([locals, centralDirectory, end]); +} + +describe("validated GitHub artifact ZIP reader", () => { + it("reads only the exact root-level entry from a multi-entry archive", () => { + const archive = artifactZip([ + { name: "diagnostics/log.txt", contents: "ignored" }, + { name: "summary.json", contents: '{"safe":true}' }, + ]); + + expect(readValidatedArtifactZipEntry(archive, "summary.json", { maxBytes: 1_024 })).toBe( + '{"safe":true}', + ); + expect(readValidatedArtifactZipEntry(archive, "log.txt", { maxBytes: 1_024 })).toBeNull(); + }); + + it("rejects duplicate target entries and payloads over the caller's bound", () => { + expect( + readValidatedArtifactZipEntry( + artifactZip([ + { name: "summary.json", contents: "one" }, + { name: "summary.json", contents: "two" }, + ]), + "summary.json", + { maxBytes: 1_024 }, + ), + ).toBeNull(); + expect( + readValidatedArtifactZipEntry( + artifactZip([{ name: "summary.json", contents: "too large" }]), + "summary.json", + { maxBytes: 2 }, + ), + ).toBeNull(); + }); + + it("reads deflated entries and rejects corrupt compressed data", () => { + const archive = artifactZip([{ name: "summary.json", contents: '{"compressed":true}' }], 8); + + expect(readValidatedArtifactZipEntry(archive, "summary.json", { maxBytes: 1_024 })).toBe( + '{"compressed":true}', + ); + + const corruptArchive = Buffer.from(archive); + const compressedDataOffset = + 30 + corruptArchive.readUInt16LE(26) + corruptArchive.readUInt16LE(28); + const compressedDataEnd = compressedDataOffset + corruptArchive.readUInt32LE(18); + corruptArchive.fill(0, compressedDataOffset, compressedDataEnd); + expect( + readValidatedArtifactZipEntry(corruptArchive, "summary.json", { maxBytes: 1_024 }), + ).toBeNull(); + }); +}); diff --git a/test/e2e/support/e2e-operations-workflow-boundary.test.ts b/test/e2e/support/e2e-operations-workflow-boundary.test.ts index 4f2d77d8b15..2aaf8fd990d 100644 --- a/test/e2e/support/e2e-operations-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-operations-workflow-boundary.test.ts @@ -88,6 +88,24 @@ describe("E2E operations workflow boundary", () => { ); }); + it("retains a bounded runtime summary for future scheduled comparisons", () => { + const workflow = readE2eOperationsWorkflow(); + const upload = workflow.jobs.scorecard.steps!.find( + (step) => step.name === "Upload E2E runtime summary", + ); + + expect(upload).toMatchObject({ + if: "${{ always() && steps.scorecard.outcome == 'success' }}", + uses: "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a", + with: { + name: "e2e-runtime-summary", + path: "${{ runner.temp }}/e2e-runtime-summary.json", + "if-no-files-found": "error", + "retention-days": 14, + }, + }); + }); + it("rejects controller protocol and PR validation drift", () => { const workflow = readE2eOperationsWorkflow(); delete workflow.on?.workflow_dispatch?.inputs?.base_sha; @@ -405,9 +423,13 @@ describe("E2E operations workflow boundary", () => { .fn() .mockReturnValue("## E2E Test Phase Runtime\n\n| Target | Slowest observed phase |"), }; + const runtimeHistory = { + buildRuntimeHistory: vi.fn().mockResolvedValue("## E2E Nightly Runtime Trend"), + }; const runtimeModules = new Map([ ["path", { join: (...parts: string[]) => parts.join("/") }], ["/workspace/scripts/audit-test-runtime.mts", runtimeAudit], + ["/workspace/scripts/scorecard/analyze-runtime-history.mts", runtimeHistory], ["/workspace/scripts/scorecard/coordinate-scorecard.mts", coordinator], ["/workspace/scripts/scorecard/analyze-trace-timing.mts", traceTiming], ["/workspace/scripts/scorecard/summarize-jobs.mts", scorecardJobs], @@ -423,6 +445,7 @@ describe("E2E operations workflow boundary", () => { GITHUB_WORKSPACE: "/workspace", JOBS: "", RUNTIME_ARTIFACTS: "/runner/e2e-runtime-audit", + RUNTIME_SUMMARY_FILE: "/runner/e2e-runtime-summary.json", TARGETS: "", }, }; @@ -445,6 +468,11 @@ describe("E2E operations workflow boundary", () => { expect(traceTiming.buildTraceTimingResult).toHaveBeenCalledWith({ github: {}, context, core }); expect(runtimeAudit.auditTestRuntime).toHaveBeenCalledWith(["/runner/e2e-runtime-audit"]); + expect(runtimeHistory.buildRuntimeHistory).toHaveBeenCalledWith( + { github: {}, context, core }, + [{ target: "full-e2e" }], + "/runner/e2e-runtime-summary.json", + ); expect(runtimeAudit.auditTestRuntime.mock.invocationCallOrder[0]).toBeLessThan( traceTiming.buildTraceTimingResult.mock.invocationCallOrder[0], ); @@ -463,7 +491,9 @@ describe("E2E operations workflow boundary", () => { }), ); expect(summary.addRaw).toHaveBeenCalledWith( - expect.stringMatching(/### Onboard Performance Budget[\s\S]*## E2E Test Phase Runtime/u), + expect.stringMatching( + /### Onboard Performance Budget[\s\S]*## E2E Test Phase Runtime[\s\S]*## E2E Nightly Runtime Trend/u, + ), ); expect(summary.write).toHaveBeenCalledOnce(); expect(setOutput).toHaveBeenCalledWith("scorecardData", expect.any(String)); @@ -488,9 +518,13 @@ describe("E2E operations workflow boundary", () => { }), formatRuntimeAuditSummary: vi.fn(), }; + const runtimeHistory = { + buildRuntimeHistory: vi.fn().mockResolvedValue("## E2E Nightly Runtime Trend"), + }; const runtimeModules = new Map([ ["path", { join: (...parts: string[]) => parts.join("/") }], ["/workspace/scripts/audit-test-runtime.mts", runtimeAudit], + ["/workspace/scripts/scorecard/analyze-runtime-history.mts", runtimeHistory], [ "/workspace/scripts/scorecard/coordinate-scorecard.mts", { @@ -527,6 +561,7 @@ describe("E2E operations workflow boundary", () => { GITHUB_WORKSPACE: "/workspace", JOBS: "", RUNTIME_ARTIFACTS: "/runner/e2e-runtime-audit", + RUNTIME_SUMMARY_FILE: "/runner/e2e-runtime-summary.json", TARGETS: "", }, }; diff --git a/test/e2e/support/e2e-runtime-history.test.ts b/test/e2e/support/e2e-runtime-history.test.ts new file mode 100644 index 00000000000..1496101d0ad --- /dev/null +++ b/test/e2e/support/e2e-runtime-history.test.ts @@ -0,0 +1,176 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it, vi } from "vitest"; + +import type { RuntimeAuditRow } from "../../../scripts/audit-test-runtime.mts"; +import { + buildRuntimeHistory, + createRuntimeSummary, + formatRuntimeHistory, + loadPriorNightlySummaries, + normalizeRuntimeSummary, +} from "../../../scripts/scorecard/analyze-runtime-history.mts"; + +function runtimeRow(overrides: Partial = {}): RuntimeAuditRow { + const medianMs = overrides.medianMs ?? 120_000; + const p95Ms = overrides.p95Ms ?? medianMs; + const maxMs = overrides.maxMs ?? p95Ms; + return { + target: "rebuild-hermes", + scenario: "rebuild Hermes from source", + runs: 1, + medianMs, + p95Ms, + maxMs, + variabilityMs: overrides.variabilityMs ?? p95Ms - medianMs, + passedRuns: 1, + failedRuns: 0, + skippedRuns: 0, + slowestPhase: "build Hermes image", + slowestPhaseMs: 90_000, + slowestPhaseOutcome: "passed", + ...overrides, + }; +} + +describe("E2E rolling runtime history", () => { + it("compares current semantic-test timing with prior scheduled summaries", () => { + const current = runtimeRow({ medianMs: 150_000, p95Ms: 150_000 }); + const prior = [ + createRuntimeSummary(1, "2026-07-20T00:00:00.000Z", [runtimeRow({ medianMs: 100_000 })]), + createRuntimeSummary(2, "2026-07-21T00:00:00.000Z", [ + runtimeRow({ medianMs: 120_000, passedRuns: 0, failedRuns: 1 }), + ]), + ]; + + const markdown = formatRuntimeHistory([current], prior); + + expect(markdown).toContain("| rebuild-hermes | rebuild Hermes from source | 2 |"); + expect(markdown).toContain("150.0s | 110.0s | 120.0s | +40.0s (+36.4%)"); + expect(markdown).toContain("50% (1/2) | passed |"); + }); + + it("prioritizes current failures and historically flaky tests over slow stable tests", () => { + const failed = runtimeRow({ + target: "failed-fast", + scenario: "failed fast", + medianMs: 10_000, + p95Ms: 10_000, + maxMs: 10_000, + passedRuns: 0, + failedRuns: 1, + }); + const flaky = runtimeRow({ + target: "flaky-fast", + scenario: "flaky fast", + medianMs: 20_000, + p95Ms: 20_000, + maxMs: 20_000, + }); + const stable = runtimeRow({ + target: "stable-slow", + scenario: "stable slow", + medianMs: 300_000, + p95Ms: 300_000, + maxMs: 300_000, + }); + const prior = [ + createRuntimeSummary(1, "2026-07-20T00:00:00.000Z", [ + runtimeRow({ + target: "failed-fast", + scenario: "failed fast", + medianMs: 10_000, + p95Ms: 10_000, + maxMs: 10_000, + }), + runtimeRow({ + target: "flaky-fast", + scenario: "flaky fast", + medianMs: 20_000, + p95Ms: 20_000, + maxMs: 20_000, + passedRuns: 0, + failedRuns: 1, + }), + runtimeRow({ + target: "stable-slow", + scenario: "stable slow", + medianMs: 300_000, + p95Ms: 300_000, + maxMs: 300_000, + }), + ]), + ]; + + const markdown = formatRuntimeHistory([stable, flaky, failed], prior); + + expect(markdown.indexOf("| failed-fast |")).toBeLessThan(markdown.indexOf("| flaky-fast |")); + expect(markdown.indexOf("| flaky-fast |")).toBeLessThan(markdown.indexOf("| stable-slow |")); + }); + + it("writes the current bounded summary even when history is unavailable", async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-runtime-history-")); + const output = path.join(directory, "e2e-runtime-summary.json"); + const warning = vi.fn(); + try { + const markdown = await buildRuntimeHistory( + { + github: {}, + context: { repo: { owner: "NVIDIA", repo: "NemoClaw" }, runId: 123 }, + core: { warning }, + }, + [runtimeRow()], + output, + { loadPriorNightlySummaries: vi.fn().mockRejectedValue(new Error("unavailable")) }, + new Date("2026-07-22T00:00:00.000Z"), + ); + + expect(JSON.parse(fs.readFileSync(output, "utf8"))).toMatchObject({ + schemaVersion: "nemoclaw.e2e_runtime_summary.v1", + runId: 123, + }); + expect(markdown).toContain("this run starts the history"); + expect(warning).toHaveBeenCalledOnce(); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + }); + + it("rejects outcome counts that do not match the bounded run count", () => { + const summary = createRuntimeSummary(1, "2026-07-22T00:00:00.000Z", [runtimeRow()]); + summary.rows[0]!.failedRuns = 2; + expect(normalizeRuntimeSummary(summary)).toBeNull(); + }); + + it("queries only prior completed scheduled runs and tolerates missing artifacts", async () => { + const listWorkflowRuns = vi.fn().mockResolvedValue({ + data: { workflow_runs: [{ id: 123 }, { id: 122 }] }, + }); + const paginate = vi.fn().mockResolvedValue([]); + const summaries = await loadPriorNightlySummaries({ + context: { repo: { owner: "NVIDIA", repo: "NemoClaw" }, runId: 123 }, + github: { + paginate, + rest: { + actions: { + downloadArtifact: vi.fn(), + listWorkflowRunArtifacts: {}, + listWorkflowRuns, + }, + }, + }, + }); + + expect(summaries).toEqual([]); + expect(listWorkflowRuns).toHaveBeenCalledWith( + expect.objectContaining({ event: "schedule", status: "completed", workflow_id: "e2e.yaml" }), + ); + expect(paginate).toHaveBeenCalledOnce(); + expect(paginate.mock.calls[0]?.[1]).toMatchObject({ run_id: 122 }); + }); +}); diff --git a/test/e2e/support/test-runtime-audit.test.ts b/test/e2e/support/test-runtime-audit.test.ts index b03e40fa6af..83d01aa3477 100644 --- a/test/e2e/support/test-runtime-audit.test.ts +++ b/test/e2e/support/test-runtime-audit.test.ts @@ -68,6 +68,9 @@ describe("test runtime audit", () => { p95Ms: 50_000, maxMs: 50_000, variabilityMs: 20_000, + passedRuns: 1, + failedRuns: 1, + skippedRuns: 0, slowestPhase: "inference", slowestPhaseMs: 40_000, slowestPhaseOutcome: "failed", @@ -80,6 +83,9 @@ describe("test runtime audit", () => { p95Ms: 20_000, maxMs: 20_000, variabilityMs: 0, + passedRuns: 1, + failedRuns: 0, + skippedRuns: 0, slowestPhase: "sandbox", slowestPhaseMs: 15_000, slowestPhaseOutcome: "passed", diff --git a/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts b/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts index 8a548e566d9..9806186a22f 100644 --- a/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts +++ b/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts @@ -82,6 +82,18 @@ describe("upload-e2e-artifacts workflow boundary", () => { expect(validateUploadE2eArtifactsInvocations(readWorkflow())).toEqual([]); }); + it("rejects scorecard runtime-summary upload drift", () => { + const workflow = mutableWorkflow(); + const scorecardUpload = workflow.jobs.scorecard.steps!.find( + (step) => step.name === "Upload E2E runtime summary", + )!; + scorecardUpload.with!["retention-days"] = 30; + + expect(validateUploadE2eArtifactsInvocations(workflow)).toContain( + "scorecard must preserve its bounded runtime summary upload contract", + ); + }); + it("rejects semantic-neutral action byte drift from the immutable provenance", () => { expect(validateActionSourceMutation((source) => `${source}# unreviewed drift\n`)).toEqual([ "upload-e2e-artifacts content must match the action reviewed at its immutable commit pin", diff --git a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts index 9f503c2f629..e320c275b81 100644 --- a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts +++ b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts @@ -30,6 +30,17 @@ const CHECKOUT_LOCAL_UPLOAD_E2E_ARTIFACTS_ACTION = "./.github/actions/upload-e2e const UPLOAD_E2E_ARTIFACTS_ACTION_PREFIX = "NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@"; const UPLOAD_ARTIFACT_ACTION = "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a"; const UPLOAD_ARTIFACT_ACTION_PREFIX = "actions/upload-artifact@"; +const SCORECARD_RUNTIME_UPLOAD = { + name: "Upload E2E runtime summary", + if: "${{ always() && steps.scorecard.outcome == 'success' }}", + uses: UPLOAD_ARTIFACT_ACTION, + with: { + name: "e2e-runtime-summary", + path: "${{ runner.temp }}/e2e-runtime-summary.json", + "if-no-files-found": "error", + "retention-days": 14, + }, +}; const INNER_ALWAYS = "${{ always() }}"; const CALLER_ALWAYS = "always()"; const MCP_SCANNED_UPLOAD_CONDITION = @@ -309,6 +320,15 @@ export function validateUploadE2eArtifactsAction(actionPath = DEFAULT_ACTION_PAT export function validateUploadE2eArtifactsInvocations(workflow: WorkflowRecord): string[] { const errors: string[] = []; const jobs = record(workflow.jobs); + const scorecardRuntimeUploads = steps(record(jobs.scorecard).steps).filter( + (step) => step.name === SCORECARD_RUNTIME_UPLOAD.name, + ); + if ( + scorecardRuntimeUploads.length !== 1 || + !isDeepStrictEqual(scorecardRuntimeUploads[0], SCORECARD_RUNTIME_UPLOAD) + ) { + errors.push("scorecard must preserve its bounded runtime summary upload contract"); + } const expectedJobs = new Set( Object.entries(jobs) .filter(([jobName, value]) => { @@ -362,7 +382,10 @@ export function validateUploadE2eArtifactsInvocations(workflow: WorkflowRecord): if (uses.startsWith(CHECKOUT_LOCAL_UPLOAD_E2E_ARTIFACTS_ACTION)) { errors.push(`${jobName} must not load upload-e2e-artifacts from the target checkout`); } - if (uses.startsWith(UPLOAD_ARTIFACT_ACTION_PREFIX)) { + if ( + uses.startsWith(UPLOAD_ARTIFACT_ACTION_PREFIX) && + !(jobName === "scorecard" && step.name === SCORECARD_RUNTIME_UPLOAD.name) + ) { errors.push(`${jobName} must not invoke actions/upload-artifact directly`); } if ( From 3344cddb875c4cb6dbdbad5e352fff19b49730d2 Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Fri, 24 Jul 2026 11:10:10 -0700 Subject: [PATCH 4/6] Revert "ci(e2e): add rolling runtime history" This reverts commit 436e5a3588524fbaeeebdc07b80063a4ca223ea0. Signed-off-by: Charan Jagwani --- .github/workflows/e2e.yaml | 25 +- scripts/audit-test-runtime.mts | 13 - scripts/scorecard/analyze-runtime-history.mts | 346 ------------------ scripts/scorecard/analyze-trace-timing.mts | 154 +++++++- scripts/scorecard/read-artifact-zip.mts | 180 --------- test/e2e/README.md | 24 +- test/e2e/support/artifact-zip.test.ts | 118 ------ .../e2e-operations-workflow-boundary.test.ts | 37 +- test/e2e/support/e2e-runtime-history.test.ts | 176 --------- test/e2e/support/test-runtime-audit.test.ts | 6 - ...ad-e2e-artifacts-workflow-boundary.test.ts | 12 - ...upload-e2e-artifacts-workflow-boundary.mts | 25 +- 12 files changed, 159 insertions(+), 957 deletions(-) delete mode 100644 scripts/scorecard/analyze-runtime-history.mts delete mode 100644 scripts/scorecard/read-artifact-zip.mts delete mode 100644 test/e2e/support/artifact-zip.test.ts delete mode 100644 test/e2e/support/e2e-runtime-history.test.ts diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index a95f765ce77..8f5f9adbd97 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -5795,7 +5795,6 @@ jobs: EXPLICIT_ONLY_JOBS: ${{ needs.generate-matrix.outputs.explicit_only_jobs }} JOBS: ${{ inputs.jobs }} RUNTIME_ARTIFACTS: ${{ runner.temp }}/e2e-runtime-audit - RUNTIME_SUMMARY_FILE: ${{ runner.temp }}/e2e-runtime-summary.json TARGETS: ${{ inputs.targets }} with: script: | @@ -5812,9 +5811,6 @@ jobs: const runtimeAudit = require( path.join(process.env.GITHUB_WORKSPACE, 'scripts/audit-test-runtime.mts'), ); - const runtimeHistory = require( - path.join(process.env.GITHUB_WORKSPACE, 'scripts/scorecard/analyze-runtime-history.mts'), - ); const needs = ${{ toJSON(needs) }}; // GitHub's jobs API is the canonical source because `needs.live` @@ -5822,9 +5818,8 @@ jobs: // The typed helper owns and tests the degraded `needs` fallback. const apiJobs = await scorecardJobs.loadWorkflowRunJobs({ github, context, core }); let runtimeSummaryMarkdown; - let runtimeRows = []; try { - runtimeRows = runtimeAudit.auditTestRuntime([process.env.RUNTIME_ARTIFACTS]); + const runtimeRows = runtimeAudit.auditTestRuntime([process.env.RUNTIME_ARTIFACTS]); runtimeSummaryMarkdown = runtimeAudit.formatRuntimeAuditSummary(runtimeRows); } catch { core.warning('E2E test phase runtime summary unavailable: invalid progress artifact'); @@ -5835,11 +5830,6 @@ jobs: '', ].join('\n'); } - const runtimeHistoryMarkdown = await runtimeHistory.buildRuntimeHistory( - { github, context, core }, - runtimeRows, - process.env.RUNTIME_SUMMARY_FILE, - ); const trace = await traceTiming.buildTraceTimingResult({ github, context, core }); if (trace.budgetWarningMessage) core.warning(trace.budgetWarningMessage); @@ -5858,9 +5848,7 @@ jobs: today: new Date().toLocaleDateString('en-US', { month: 'short', day: 'numeric' }), }); - await core.summary - .addRaw(`${summaryMarkdown}\n\n${runtimeSummaryMarkdown}\n${runtimeHistoryMarkdown}`) - .write(); + await core.summary.addRaw(`${summaryMarkdown}\n\n${runtimeSummaryMarkdown}`).write(); core.setOutput('scorecardData', JSON.stringify(scorecardData)); core.setOutput('slackData', JSON.stringify(slackData)); @@ -5921,12 +5909,3 @@ jobs: if (!response.ok) { core.setFailed(`Slack webhook returned ${response.status}`); } - - - name: Upload E2E runtime summary - if: ${{ always() && steps.scorecard.outcome == 'success' }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: e2e-runtime-summary - path: ${{ runner.temp }}/e2e-runtime-summary.json - if-no-files-found: error - retention-days: 14 diff --git a/scripts/audit-test-runtime.mts b/scripts/audit-test-runtime.mts index 58c0d2fe263..c6611a7bf29 100644 --- a/scripts/audit-test-runtime.mts +++ b/scripts/audit-test-runtime.mts @@ -15,9 +15,6 @@ export interface RuntimeAuditRow { p95Ms: number; maxMs: number; variabilityMs: number; - passedRuns: number; - failedRuns: number; - skippedRuns: number; slowestPhase: string; slowestPhaseMs: number; slowestPhaseOutcome: "passed" | "failed" | "skipped"; @@ -81,12 +78,6 @@ function median(sorted: readonly number[]): number { return sorted[middle] ?? 0; } -function summaryOutcome(summary: ProgressSummary): "passed" | "failed" | "skipped" { - if (summary.phases.some((phase) => phase.outcome === "failed")) return "failed"; - if (summary.phases.some((phase) => phase.outcome === "skipped")) return "skipped"; - return "passed"; -} - export function auditTestRuntime(roots: readonly string[]): RuntimeAuditRow[] { const summaries = roots.flatMap(progressFiles).map((file) => { const parsed: unknown = JSON.parse(fs.readFileSync(file, "utf8")); @@ -117,7 +108,6 @@ export function auditTestRuntime(roots: readonly string[]): RuntimeAuditRow[] { ); const medianMs = median(durations); const p95Ms = percentile(durations, 0.95); - const outcomes = runs.map(summaryOutcome); return { target: [first.targetId ?? "unlabeled", first.shardId].filter(Boolean).join("/"), scenario: first.scenario, @@ -126,9 +116,6 @@ export function auditTestRuntime(roots: readonly string[]): RuntimeAuditRow[] { p95Ms, maxMs: durations.at(-1) ?? 0, variabilityMs: Math.max(0, p95Ms - medianMs), - passedRuns: outcomes.filter((outcome) => outcome === "passed").length, - failedRuns: outcomes.filter((outcome) => outcome === "failed").length, - skippedRuns: outcomes.filter((outcome) => outcome === "skipped").length, slowestPhase: slowestPhase.label, slowestPhaseMs: slowestPhase.durationMs, slowestPhaseOutcome: slowestPhase.outcome, diff --git a/scripts/scorecard/analyze-runtime-history.mts b/scripts/scorecard/analyze-runtime-history.mts deleted file mode 100644 index 024350bee6a..00000000000 --- a/scripts/scorecard/analyze-runtime-history.mts +++ /dev/null @@ -1,346 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import fs from "node:fs"; - -import type { RuntimeAuditRow } from "../audit-test-runtime.mts"; -import { readValidatedArtifactZipEntry } from "./read-artifact-zip.mts"; - -export const RUNTIME_SUMMARY_ARTIFACT = "e2e-runtime-summary"; -export const RUNTIME_SUMMARY_FILE = "e2e-runtime-summary.json"; -const RUNTIME_SUMMARY_SCHEMA = "nemoclaw.e2e_runtime_summary.v1"; -const WORKFLOW_FILE = "e2e.yaml"; -const HISTORY_RUN_LIMIT = 10; -const MAX_SUMMARY_BYTES = 256 * 1024; -const MAX_SUMMARY_ROWS = 200; - -type GitHubDeps = { - github: any; - context: { repo: { owner: string; repo: string }; runId: number }; - core?: { warning?: (message: string) => void }; -}; - -export interface RuntimeSummaryArtifact { - schemaVersion: typeof RUNTIME_SUMMARY_SCHEMA; - runId: number; - createdAt: string; - rows: RuntimeAuditRow[]; -} - -type RuntimeHistoryServices = { - loadPriorNightlySummaries: (deps: GitHubDeps) => Promise; -}; - -function isBoundedString(value: unknown): value is string { - return ( - typeof value === "string" && - value.length > 0 && - value.length <= 500 && - !/[\u0000-\u001f\u007f]/u.test(value) - ); -} - -function isNonNegativeNumber(value: unknown): value is number { - return typeof value === "number" && Number.isFinite(value) && value >= 0; -} - -function isNonNegativeInteger(value: unknown): value is number { - return Number.isInteger(value) && isNonNegativeNumber(value); -} - -function hasExactKeys(value: Record, expected: readonly string[]): boolean { - return Object.keys(value).sort().join("\0") === [...expected].sort().join("\0"); -} - -function normalizeRuntimeRow(value: unknown): RuntimeAuditRow | null { - if (!value || typeof value !== "object" || Array.isArray(value)) return null; - const row = value as Record; - if ( - !hasExactKeys(row, [ - "target", - "scenario", - "runs", - "medianMs", - "p95Ms", - "maxMs", - "variabilityMs", - "passedRuns", - "failedRuns", - "skippedRuns", - "slowestPhase", - "slowestPhaseMs", - "slowestPhaseOutcome", - ]) || - !isBoundedString(row.target) || - !isBoundedString(row.scenario) || - !isNonNegativeInteger(row.runs) || - row.runs < 1 || - !isNonNegativeNumber(row.medianMs) || - !isNonNegativeNumber(row.p95Ms) || - !isNonNegativeNumber(row.maxMs) || - !isNonNegativeNumber(row.variabilityMs) || - row.medianMs > row.p95Ms || - row.p95Ms > row.maxMs || - row.variabilityMs !== row.p95Ms - row.medianMs || - !isNonNegativeInteger(row.passedRuns) || - !isNonNegativeInteger(row.failedRuns) || - !isNonNegativeInteger(row.skippedRuns) || - row.passedRuns + row.failedRuns + row.skippedRuns !== row.runs || - !isBoundedString(row.slowestPhase) || - !isNonNegativeNumber(row.slowestPhaseMs) || - (row.slowestPhaseOutcome !== "passed" && - row.slowestPhaseOutcome !== "failed" && - row.slowestPhaseOutcome !== "skipped") - ) { - return null; - } - return { - target: row.target, - scenario: row.scenario, - runs: row.runs, - medianMs: row.medianMs, - p95Ms: row.p95Ms, - maxMs: row.maxMs, - variabilityMs: row.variabilityMs, - passedRuns: row.passedRuns, - failedRuns: row.failedRuns, - skippedRuns: row.skippedRuns, - slowestPhase: row.slowestPhase, - slowestPhaseMs: row.slowestPhaseMs, - slowestPhaseOutcome: row.slowestPhaseOutcome, - }; -} - -export function normalizeRuntimeSummary(value: unknown): RuntimeSummaryArtifact | null { - if (!value || typeof value !== "object" || Array.isArray(value)) return null; - const summary = value as Record; - if ( - !hasExactKeys(summary, ["schemaVersion", "runId", "createdAt", "rows"]) || - summary.schemaVersion !== RUNTIME_SUMMARY_SCHEMA || - !isNonNegativeInteger(summary.runId) || - summary.runId < 1 || - typeof summary.createdAt !== "string" || - !Number.isFinite(Date.parse(summary.createdAt)) || - !Array.isArray(summary.rows) || - summary.rows.length > MAX_SUMMARY_ROWS - ) { - return null; - } - const rows = summary.rows.map(normalizeRuntimeRow); - if (rows.some((row) => row === null)) return null; - const identities = new Set( - (rows as RuntimeAuditRow[]).map((row) => JSON.stringify([row.target, row.scenario])), - ); - if (identities.size !== rows.length) return null; - return { - schemaVersion: RUNTIME_SUMMARY_SCHEMA, - runId: summary.runId, - createdAt: summary.createdAt, - rows: rows as RuntimeAuditRow[], - }; -} - -export function createRuntimeSummary( - runId: number, - createdAt: string, - rows: readonly RuntimeAuditRow[], -): RuntimeSummaryArtifact { - const summary = normalizeRuntimeSummary({ - schemaVersion: RUNTIME_SUMMARY_SCHEMA, - runId, - createdAt, - rows, - }); - if (summary === null) throw new Error("invalid current E2E runtime summary"); - return summary; -} - -function parseRuntimeSummaryArchive(archive: Buffer): RuntimeSummaryArtifact | null { - try { - const contents = readValidatedArtifactZipEntry(archive, RUNTIME_SUMMARY_FILE, { - maxBytes: MAX_SUMMARY_BYTES, - }); - return contents === null ? null : normalizeRuntimeSummary(JSON.parse(contents)); - } catch { - return null; - } -} - -async function readRuntimeSummaryFromRun( - { github, context }: GitHubDeps, - runId: number, -): Promise { - const artifacts = (await github.paginate(github.rest.actions.listWorkflowRunArtifacts, { - owner: context.repo.owner, - repo: context.repo.repo, - run_id: runId, - per_page: 100, - })) as Array<{ expired?: boolean; id: number; name: string }>; - const artifact = artifacts.find( - (candidate) => candidate.name === RUNTIME_SUMMARY_ARTIFACT && candidate.expired !== true, - ); - if (!artifact) return null; - const download = await github.rest.actions.downloadArtifact({ - owner: context.repo.owner, - repo: context.repo.repo, - artifact_id: artifact.id, - archive_format: "zip", - }); - const summary = parseRuntimeSummaryArchive(Buffer.from(download.data)); - return summary?.runId === runId ? summary : null; -} - -export async function loadPriorNightlySummaries( - deps: GitHubDeps, -): Promise { - const { github, context, core } = deps; - const response = await github.rest.actions.listWorkflowRuns({ - owner: context.repo.owner, - repo: context.repo.repo, - workflow_id: WORKFLOW_FILE, - event: "schedule", - status: "completed", - per_page: HISTORY_RUN_LIMIT + 5, - }); - const runs = (response.data.workflow_runs as Array<{ id: number }>) - .filter((run) => run.id !== context.runId) - .slice(0, HISTORY_RUN_LIMIT); - const summaries: RuntimeSummaryArtifact[] = []; - for (const run of runs) { - try { - const summary = await readRuntimeSummaryFromRun(deps, run.id); - if (summary !== null) summaries.push(summary); - } catch { - core?.warning?.( - "One prior nightly runtime summary was unavailable; continuing with less history.", - ); - } - } - return summaries; -} - -function percentile(sorted: readonly number[], fraction: number): number { - return sorted[Math.max(0, Math.ceil(sorted.length * fraction) - 1)] ?? 0; -} - -function median(sorted: readonly number[]): number { - const middle = Math.floor(sorted.length / 2); - return sorted.length % 2 === 0 - ? ((sorted[middle - 1] ?? 0) + (sorted[middle] ?? 0)) / 2 - : (sorted[middle] ?? 0); -} - -function seconds(milliseconds: number): string { - return `${(milliseconds / 1000).toFixed(1)}s`; -} - -function escapeCell(value: string): string { - return value.replaceAll("|", "\\|").replaceAll("\n", " "); -} - -function formatDelta(currentMs: number, priorMs: number): string { - const deltaMs = currentMs - priorMs; - const sign = deltaMs >= 0 ? "+" : "-"; - const percent = priorMs > 0 ? ` (${sign}${Math.abs((deltaMs / priorMs) * 100).toFixed(1)}%)` : ""; - return `${sign}${seconds(Math.abs(deltaMs))}${percent}`; -} - -function formatOutcome(row: RuntimeAuditRow): string { - if (row.failedRuns > 0) return "failed"; - if (row.skippedRuns > 0) return "skipped"; - return "passed"; -} - -function formatPassRate(rows: readonly RuntimeAuditRow[]): string { - const passed = rows.reduce((total, row) => total + row.passedRuns, 0); - const runs = rows.reduce((total, row) => total + row.runs, 0); - return runs > 0 ? `${((passed / runs) * 100).toFixed(0)}% (${passed}/${runs})` : "n/a"; -} - -function passRate(rows: readonly RuntimeAuditRow[]): number { - const passed = rows.reduce((total, row) => total + row.passedRuns, 0); - const runs = rows.reduce((total, row) => total + row.runs, 0); - return runs > 0 ? passed / runs : 1; -} - -export function formatRuntimeHistory( - currentRows: readonly RuntimeAuditRow[], - priorSummaries: readonly RuntimeSummaryArtifact[], -): string { - const lines = [ - "## E2E Nightly Runtime Trend", - "", - "Current run compared with up to 10 prior completed scheduled runs; manual runs are excluded.", - "Rows prioritize current failures, lower historical pass rates, and larger runtime regressions.", - "", - ]; - if (currentRows.length === 0) { - lines.push("No current runtime rows were available for comparison."); - return `${lines.join("\n")}\n`; - } - if (priorSummaries.length === 0) { - lines.push( - "No prior nightly runtime summaries are available yet; this run starts the history.", - ); - return `${lines.join("\n")}\n`; - } - - lines.push( - "| Target | Scenario | Prior nights | Current median | Prior median | Prior p95 | Delta | Prior pass rate | Current |", - "| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | --- |", - ); - const comparisons = currentRows - .map((current) => { - const priorRows = priorSummaries.flatMap((summary) => - summary.rows.filter( - (row) => row.target === current.target && row.scenario === current.scenario, - ), - ); - const priorMedian = - priorRows.length > 0 - ? median(priorRows.map((row) => row.medianMs).sort((a, b) => a - b)) - : null; - return { current, priorRows, priorMedian }; - }) - .sort( - (a, b) => - Number(b.current.failedRuns > 0) - Number(a.current.failedRuns > 0) || - passRate(a.priorRows) - passRate(b.priorRows) || - Math.max(0, b.priorMedian === null ? 0 : b.current.medianMs - b.priorMedian) - - Math.max(0, a.priorMedian === null ? 0 : a.current.medianMs - a.priorMedian) || - b.current.p95Ms - a.current.p95Ms, - ); - for (const { current, priorRows, priorMedian } of comparisons.slice(0, 10)) { - if (priorRows.length === 0) { - lines.push( - `| ${escapeCell(current.target)} | ${escapeCell(current.scenario)} | 0 | ${seconds(current.medianMs)} | n/a | n/a | n/a | n/a | ${formatOutcome(current)} |`, - ); - continue; - } - const medians = priorRows.map((row) => row.medianMs).sort((a, b) => a - b); - lines.push( - `| ${escapeCell(current.target)} | ${escapeCell(current.scenario)} | ${priorRows.length} | ${seconds(current.medianMs)} | ${seconds(priorMedian ?? 0)} | ${seconds(percentile(medians, 0.95))} | ${formatDelta(current.medianMs, priorMedian ?? 0)} | ${formatPassRate(priorRows)} | ${formatOutcome(current)} |`, - ); - } - return `${lines.join("\n")}\n`; -} - -export async function buildRuntimeHistory( - deps: GitHubDeps, - currentRows: readonly RuntimeAuditRow[], - outputPath: string, - services: RuntimeHistoryServices = { loadPriorNightlySummaries }, - now = new Date(), -): Promise { - const current = createRuntimeSummary(deps.context.runId, now.toISOString(), currentRows); - fs.writeFileSync(outputPath, `${JSON.stringify(current, null, 2)}\n`, { mode: 0o600 }); - try { - const prior = await services.loadPriorNightlySummaries(deps); - return formatRuntimeHistory(currentRows, prior); - } catch { - deps.core?.warning?.( - "Nightly E2E runtime history unavailable; current summary was still saved.", - ); - return formatRuntimeHistory(currentRows, []); - } -} diff --git a/scripts/scorecard/analyze-trace-timing.mts b/scripts/scorecard/analyze-trace-timing.mts index 9f9e55ccb2c..10eb1acf42d 100644 --- a/scripts/scorecard/analyze-trace-timing.mts +++ b/scripts/scorecard/analyze-trace-timing.mts @@ -4,8 +4,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; - -import { readValidatedArtifactZipEntry } from "./read-artifact-zip.mts"; +import zlib from "node:zlib"; type SemverTag = { name: string; major: number; minor: number; patch: number; sha?: string }; type Threshold = { minDeltaMs: number; minPercent: number }; @@ -47,6 +46,17 @@ type TraceTimingResult = { budgetWarningMessage: string | null; budgetStatus: string; }; +type ZipSummaryEntry = { + creatorSystem: number; + flags: number; + compressionMethod: number; + expectedCrc: number; + compressedSize: number; + uncompressedSize: number; + diskStart: number; + externalAttributes: number; + localHeaderOffset: number; +}; type GitHubDeps = { github: any; context: any; core?: { warning?: (message: string) => void } }; type TraceTimingServices = { findLatestCompletedE2eRunForReleaseTag: (deps: GitHubDeps, tag: SemverTag) => Promise; @@ -61,6 +71,9 @@ const MAX_TRACE_SUMMARY_BYTES = 1024 * 1024; const MAX_TRACE_ARCHIVE_ENTRIES = 1000; const TRACE_ARCHIVE_REJECTION_WARNING = "Trace timing artifact ZIP validation failed; ignoring the malformed or unsupported archive."; +const ZIP_CENTRAL_DIRECTORY_SIGNATURE = 0x02014b50; +const ZIP_END_OF_CENTRAL_DIRECTORY_SIGNATURE = 0x06054b50; +const ZIP_LOCAL_FILE_SIGNATURE = 0x04034b50; const ONBOARD_PERFORMANCE_BUDGET_FILE = "ci/onboard-performance-budget.json"; const REPO_ROOT = path.resolve(import.meta.dirname, "..", ".."); const ONBOARD_PHASE_PREFIX = "nemoclaw.onboard.phase."; @@ -518,6 +531,30 @@ async function findLatestCompletedE2eRunForReleaseTag( return null; } +function findZipEndOfCentralDirectory(archive: Buffer): number { + const minimumOffset = Math.max(0, archive.length - 22 - 0xffff); + for (let offset = archive.length - 22; offset >= minimumOffset; offset -= 1) { + if ( + archive.readUInt32LE(offset) === ZIP_END_OF_CENTRAL_DIRECTORY_SIGNATURE && + offset + 22 + archive.readUInt16LE(offset + 20) === archive.length + ) { + return offset; + } + } + return -1; +} + +function crc32(data: Buffer): number { + let crc = 0xffffffff; + for (const byte of data) { + crc ^= byte; + for (let bit = 0; bit < 8; bit += 1) { + crc = (crc >>> 1) ^ (0xedb88320 & -(crc & 1)); + } + } + return (crc ^ 0xffffffff) >>> 0; +} + // GitHub creates the workflow artifact ZIP outside this repository, and the // cloud-onboard artifact intentionally contains diagnostics beside the trusted // timing summary. Parse only the exact root-level summary in-process so the @@ -525,10 +562,115 @@ async function findLatestCompletedE2eRunForReleaseTag( // production-shape multi-entry regression test is the removal guard; retire // this parser if GitHub provides a verified single-file artifact API. function readValidatedTraceSummaryArchive(archive: Buffer): string | null { - return readValidatedArtifactZipEntry(archive, TRACE_SUMMARY_FILE, { - maxBytes: MAX_TRACE_SUMMARY_BYTES, - maxEntries: MAX_TRACE_ARCHIVE_ENTRIES, - }); + const endOffset = findZipEndOfCentralDirectory(archive); + if (endOffset < 0) return null; + + const diskNumber = archive.readUInt16LE(endOffset + 4); + const centralDirectoryDisk = archive.readUInt16LE(endOffset + 6); + const entriesOnDisk = archive.readUInt16LE(endOffset + 8); + const totalEntries = archive.readUInt16LE(endOffset + 10); + const centralDirectorySize = archive.readUInt32LE(endOffset + 12); + const centralDirectoryOffset = archive.readUInt32LE(endOffset + 16); + if ( + diskNumber !== 0 || + centralDirectoryDisk !== 0 || + entriesOnDisk !== totalEntries || + totalEntries < 1 || + totalEntries > MAX_TRACE_ARCHIVE_ENTRIES || + centralDirectoryOffset + centralDirectorySize !== endOffset + ) { + return null; + } + + const expectedFileName = Buffer.from(TRACE_SUMMARY_FILE, "utf8"); + let centralEntryOffset = centralDirectoryOffset; + let target: ZipSummaryEntry | null = null; + for (let entryIndex = 0; entryIndex < totalEntries; entryIndex += 1) { + if ( + centralEntryOffset + 46 > endOffset || + archive.readUInt32LE(centralEntryOffset) !== ZIP_CENTRAL_DIRECTORY_SIGNATURE + ) { + return null; + } + const fileNameLength = archive.readUInt16LE(centralEntryOffset + 28); + const extraLength = archive.readUInt16LE(centralEntryOffset + 30); + const commentLength = archive.readUInt16LE(centralEntryOffset + 32); + const centralEntryEnd = centralEntryOffset + 46 + fileNameLength + extraLength + commentLength; + if (centralEntryEnd > endOffset) return null; + const fileName = archive.subarray( + centralEntryOffset + 46, + centralEntryOffset + 46 + fileNameLength, + ); + if (fileName.equals(expectedFileName)) { + if (target !== null) return null; + target = { + creatorSystem: archive.readUInt8(centralEntryOffset + 5), + flags: archive.readUInt16LE(centralEntryOffset + 8), + compressionMethod: archive.readUInt16LE(centralEntryOffset + 10), + expectedCrc: archive.readUInt32LE(centralEntryOffset + 16), + compressedSize: archive.readUInt32LE(centralEntryOffset + 20), + uncompressedSize: archive.readUInt32LE(centralEntryOffset + 24), + diskStart: archive.readUInt16LE(centralEntryOffset + 34), + externalAttributes: archive.readUInt32LE(centralEntryOffset + 38), + localHeaderOffset: archive.readUInt32LE(centralEntryOffset + 42), + }; + } + centralEntryOffset = centralEntryEnd; + } + if (centralEntryOffset !== endOffset || target === null) return null; + + const { + creatorSystem, + flags, + compressionMethod, + expectedCrc, + compressedSize, + uncompressedSize, + diskStart, + externalAttributes, + localHeaderOffset, + } = target; + const unixFileType = (externalAttributes >>> 16) & 0xf000; + if ( + diskStart !== 0 || + (flags & 0x1) !== 0 || + (compressionMethod !== 0 && compressionMethod !== 8) || + compressedSize > MAX_TRACE_SUMMARY_BYTES || + uncompressedSize > MAX_TRACE_SUMMARY_BYTES || + (creatorSystem !== 0 && creatorSystem !== 3) || + (creatorSystem === 3 && unixFileType !== 0 && unixFileType !== 0x8000) || + localHeaderOffset + 30 > centralDirectoryOffset || + archive.readUInt32LE(localHeaderOffset) !== ZIP_LOCAL_FILE_SIGNATURE + ) { + return null; + } + + const localFlags = archive.readUInt16LE(localHeaderOffset + 6); + const localCompressionMethod = archive.readUInt16LE(localHeaderOffset + 8); + const localFileNameLength = archive.readUInt16LE(localHeaderOffset + 26); + const localExtraLength = archive.readUInt16LE(localHeaderOffset + 28); + const localFileName = archive.subarray( + localHeaderOffset + 30, + localHeaderOffset + 30 + localFileNameLength, + ); + const compressedDataOffset = localHeaderOffset + 30 + localFileNameLength + localExtraLength; + const compressedDataEnd = compressedDataOffset + compressedSize; + if ( + localFlags !== flags || + localCompressionMethod !== compressionMethod || + !localFileName.equals(expectedFileName) || + compressedDataEnd > centralDirectoryOffset + ) { + return null; + } + + const compressedData = archive.subarray(compressedDataOffset, compressedDataEnd); + const summary = + compressionMethod === 0 + ? Buffer.from(compressedData) + : zlib.inflateRawSync(compressedData, { maxOutputLength: MAX_TRACE_SUMMARY_BYTES }); + if (summary.length !== uncompressedSize || crc32(summary) !== expectedCrc) return null; + return summary.toString("utf8"); } function readValidatedTraceSummaryZip( diff --git a/scripts/scorecard/read-artifact-zip.mts b/scripts/scorecard/read-artifact-zip.mts deleted file mode 100644 index 5bb3ba0c09d..00000000000 --- a/scripts/scorecard/read-artifact-zip.mts +++ /dev/null @@ -1,180 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import zlib from "node:zlib"; - -type ZipEntry = { - creatorSystem: number; - flags: number; - compressionMethod: number; - expectedCrc: number; - compressedSize: number; - uncompressedSize: number; - diskStart: number; - externalAttributes: number; - localHeaderOffset: number; -}; - -const ZIP_CENTRAL_DIRECTORY_SIGNATURE = 0x02014b50; -const ZIP_END_OF_CENTRAL_DIRECTORY_SIGNATURE = 0x06054b50; -const ZIP_LOCAL_FILE_SIGNATURE = 0x04034b50; - -function findZipEndOfCentralDirectory(archive: Buffer): number { - const minimumOffset = Math.max(0, archive.length - 22 - 0xffff); - for (let offset = archive.length - 22; offset >= minimumOffset; offset -= 1) { - if ( - archive.readUInt32LE(offset) === ZIP_END_OF_CENTRAL_DIRECTORY_SIGNATURE && - offset + 22 + archive.readUInt16LE(offset + 20) === archive.length - ) { - return offset; - } - } - return -1; -} - -function crc32(data: Buffer): number { - let crc = 0xffffffff; - for (const byte of data) { - crc ^= byte; - for (let bit = 0; bit < 8; bit += 1) { - crc = (crc >>> 1) ^ (0xedb88320 & -(crc & 1)); - } - } - return (crc ^ 0xffffffff) >>> 0; -} - -/** - * Reads one exact root-level file from a GitHub artifact ZIP without extracting - * paths to disk. The archive may contain other entries, but duplicate target - * entries, links, encryption, split archives, ZIP64, and oversized payloads are - * rejected. - */ -export function readValidatedArtifactZipEntry( - archive: Buffer, - expectedFile: string, - options: { maxBytes: number; maxEntries?: number }, -): string | null { - const maxEntries = options.maxEntries ?? 1000; - const expectedFileName = Buffer.from(expectedFile, "utf8"); - if ( - expectedFile.length === 0 || - expectedFile.includes("/") || - expectedFile.includes("\\") || - options.maxBytes < 1 || - maxEntries < 1 - ) { - return null; - } - - const endOffset = findZipEndOfCentralDirectory(archive); - if (endOffset < 0) return null; - - const diskNumber = archive.readUInt16LE(endOffset + 4); - const centralDirectoryDisk = archive.readUInt16LE(endOffset + 6); - const entriesOnDisk = archive.readUInt16LE(endOffset + 8); - const totalEntries = archive.readUInt16LE(endOffset + 10); - const centralDirectorySize = archive.readUInt32LE(endOffset + 12); - const centralDirectoryOffset = archive.readUInt32LE(endOffset + 16); - if ( - diskNumber !== 0 || - centralDirectoryDisk !== 0 || - entriesOnDisk !== totalEntries || - totalEntries < 1 || - totalEntries > maxEntries || - centralDirectoryOffset + centralDirectorySize !== endOffset - ) { - return null; - } - - let centralEntryOffset = centralDirectoryOffset; - let target: ZipEntry | null = null; - for (let entryIndex = 0; entryIndex < totalEntries; entryIndex += 1) { - if ( - centralEntryOffset + 46 > endOffset || - archive.readUInt32LE(centralEntryOffset) !== ZIP_CENTRAL_DIRECTORY_SIGNATURE - ) { - return null; - } - const fileNameLength = archive.readUInt16LE(centralEntryOffset + 28); - const extraLength = archive.readUInt16LE(centralEntryOffset + 30); - const commentLength = archive.readUInt16LE(centralEntryOffset + 32); - const centralEntryEnd = centralEntryOffset + 46 + fileNameLength + extraLength + commentLength; - if (centralEntryEnd > endOffset) return null; - const fileName = archive.subarray( - centralEntryOffset + 46, - centralEntryOffset + 46 + fileNameLength, - ); - if (fileName.equals(expectedFileName)) { - if (target !== null) return null; - target = { - creatorSystem: archive.readUInt8(centralEntryOffset + 5), - flags: archive.readUInt16LE(centralEntryOffset + 8), - compressionMethod: archive.readUInt16LE(centralEntryOffset + 10), - expectedCrc: archive.readUInt32LE(centralEntryOffset + 16), - compressedSize: archive.readUInt32LE(centralEntryOffset + 20), - uncompressedSize: archive.readUInt32LE(centralEntryOffset + 24), - diskStart: archive.readUInt16LE(centralEntryOffset + 34), - externalAttributes: archive.readUInt32LE(centralEntryOffset + 38), - localHeaderOffset: archive.readUInt32LE(centralEntryOffset + 42), - }; - } - centralEntryOffset = centralEntryEnd; - } - if (centralEntryOffset !== endOffset || target === null) return null; - - const { - creatorSystem, - flags, - compressionMethod, - expectedCrc, - compressedSize, - uncompressedSize, - diskStart, - externalAttributes, - localHeaderOffset, - } = target; - const unixFileType = (externalAttributes >>> 16) & 0xf000; - if ( - diskStart !== 0 || - (flags & 0x1) !== 0 || - (compressionMethod !== 0 && compressionMethod !== 8) || - compressedSize > options.maxBytes || - uncompressedSize > options.maxBytes || - (creatorSystem !== 0 && creatorSystem !== 3) || - (creatorSystem === 3 && unixFileType !== 0 && unixFileType !== 0x8000) || - localHeaderOffset + 30 > centralDirectoryOffset || - archive.readUInt32LE(localHeaderOffset) !== ZIP_LOCAL_FILE_SIGNATURE - ) { - return null; - } - - const localFlags = archive.readUInt16LE(localHeaderOffset + 6); - const localCompressionMethod = archive.readUInt16LE(localHeaderOffset + 8); - const localFileNameLength = archive.readUInt16LE(localHeaderOffset + 26); - const localExtraLength = archive.readUInt16LE(localHeaderOffset + 28); - const localFileNameEnd = localHeaderOffset + 30 + localFileNameLength; - const compressedDataOffset = localFileNameEnd + localExtraLength; - const compressedDataEnd = compressedDataOffset + compressedSize; - if ( - localFileNameEnd > centralDirectoryOffset || - localFlags !== flags || - localCompressionMethod !== compressionMethod || - !archive.subarray(localHeaderOffset + 30, localFileNameEnd).equals(expectedFileName) || - compressedDataEnd > centralDirectoryOffset - ) { - return null; - } - - const compressedData = archive.subarray(compressedDataOffset, compressedDataEnd); - let contents: Buffer; - try { - contents = - compressionMethod === 0 - ? Buffer.from(compressedData) - : zlib.inflateRawSync(compressedData, { maxOutputLength: options.maxBytes }); - } catch { - return null; - } - if (contents.length !== uncompressedSize || crc32(contents) !== expectedCrc) return null; - return contents.toString("utf8"); -} diff --git a/test/e2e/README.md b/test/e2e/README.md index 9caeed42108..58a86ce7aa9 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -153,15 +153,9 @@ graph as the live targets: combined duration; - reports the runner class as `standard`, `larger`, or `unknown` without exposing runner labels; - - adds this run's semantic phase runtime table; - - compares runtime and pass rate with up to ten prior scheduled summaries; - - orders that history by current failures, lowest prior pass rate, and largest - runtime regression; and + - adds this run's semantic phase runtime table; and - compares the trusted cloud-onboard timing summary with the latest prior-release `e2e.yaml` run. -- The rolling comparison uses only the bounded `e2e-runtime-summary.json` - artifact retained for 14 days. It does not download historical raw test - artifacts or include manual runs in the baseline. - Selective dispatches remain silent unless they run on `main` with `post_to_slack=true`, which uses the preview Slack route. Branch-dispatched runs never receive Slack webhook secrets. @@ -330,16 +324,12 @@ npm run test:runtime-audit -- path/to/run-1 path/to/run-2 The audit groups each test by target and optional shard, ranks the groups by p95 runtime, and reports variability plus the slowest observed phase's duration and outcome. Scheduled and ordinary manual runs include the same table for that -run in the GitHub Actions scorecard summary. The scorecard also shows a bounded -nightly trend table with prior median, prior p95, delta, and pass rate; the -first run explicitly starts the history instead of inventing a baseline. Its -ordering puts current failures first, followed by the lowest historical pass -rates and largest runtime regressions. Keep phase labels specific to test -behavior, call `progress.phase("literal phase label")` at the declared -boundaries in order, and transition through the final test-declared phase on -every passing path. -Both fixtures reject a passing test that never reaches that phase; only the -stateful live fixture enters its resource-release phase automatically. +run in the GitHub Actions scorecard summary. Keep phase +labels specific to test behavior, call `progress.phase("literal phase label")` +at the declared boundaries in order, and transition through the final +test-declared phase on every passing path. Both fixtures reject a passing test +that never reaches that phase; only the stateful live fixture enters its +resource-release phase automatically. Validate phase coverage without executing test bodies with: ```bash diff --git a/test/e2e/support/artifact-zip.test.ts b/test/e2e/support/artifact-zip.test.ts deleted file mode 100644 index e2177deb872..00000000000 --- a/test/e2e/support/artifact-zip.test.ts +++ /dev/null @@ -1,118 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import zlib from "node:zlib"; - -import { describe, expect, it } from "vitest"; - -import { readValidatedArtifactZipEntry } from "../../../scripts/scorecard/read-artifact-zip.mts"; - -function crc32(data: Buffer): number { - let crc = 0xffffffff; - for (const byte of data) { - crc ^= byte; - for (let bit = 0; bit < 8; bit += 1) { - crc = (crc >>> 1) ^ (0xedb88320 & -(crc & 1)); - } - } - return (crc ^ 0xffffffff) >>> 0; -} - -function artifactZip( - entries: Array<{ name: string; contents: string }>, - compressionMethod = 0, -): Buffer { - const localParts: Buffer[] = []; - const centralParts: Buffer[] = []; - let localOffset = 0; - for (const entry of entries) { - const name = Buffer.from(entry.name, "utf8"); - const contents = Buffer.from(entry.contents, "utf8"); - const compressed = - compressionMethod === 8 ? zlib.deflateRawSync(contents) : Buffer.from(contents); - const checksum = crc32(contents); - const local = Buffer.alloc(30); - local.writeUInt32LE(0x04034b50, 0); - local.writeUInt16LE(20, 4); - local.writeUInt16LE(compressionMethod, 8); - local.writeUInt32LE(checksum, 14); - local.writeUInt32LE(compressed.length, 18); - local.writeUInt32LE(contents.length, 22); - local.writeUInt16LE(name.length, 26); - localParts.push(local, name, compressed); - - const central = Buffer.alloc(46); - central.writeUInt32LE(0x02014b50, 0); - central.writeUInt16LE(0x0314, 4); - central.writeUInt16LE(20, 6); - central.writeUInt16LE(compressionMethod, 10); - central.writeUInt32LE(checksum, 16); - central.writeUInt32LE(compressed.length, 20); - central.writeUInt32LE(contents.length, 24); - central.writeUInt16LE(name.length, 28); - central.writeUInt32LE(0x80000000, 38); - central.writeUInt32LE(localOffset, 42); - centralParts.push(central, name); - localOffset += local.length + name.length + compressed.length; - } - const locals = Buffer.concat(localParts); - const centralDirectory = Buffer.concat(centralParts); - const end = Buffer.alloc(22); - end.writeUInt32LE(0x06054b50, 0); - end.writeUInt16LE(entries.length, 8); - end.writeUInt16LE(entries.length, 10); - end.writeUInt32LE(centralDirectory.length, 12); - end.writeUInt32LE(locals.length, 16); - return Buffer.concat([locals, centralDirectory, end]); -} - -describe("validated GitHub artifact ZIP reader", () => { - it("reads only the exact root-level entry from a multi-entry archive", () => { - const archive = artifactZip([ - { name: "diagnostics/log.txt", contents: "ignored" }, - { name: "summary.json", contents: '{"safe":true}' }, - ]); - - expect(readValidatedArtifactZipEntry(archive, "summary.json", { maxBytes: 1_024 })).toBe( - '{"safe":true}', - ); - expect(readValidatedArtifactZipEntry(archive, "log.txt", { maxBytes: 1_024 })).toBeNull(); - }); - - it("rejects duplicate target entries and payloads over the caller's bound", () => { - expect( - readValidatedArtifactZipEntry( - artifactZip([ - { name: "summary.json", contents: "one" }, - { name: "summary.json", contents: "two" }, - ]), - "summary.json", - { maxBytes: 1_024 }, - ), - ).toBeNull(); - expect( - readValidatedArtifactZipEntry( - artifactZip([{ name: "summary.json", contents: "too large" }]), - "summary.json", - { maxBytes: 2 }, - ), - ).toBeNull(); - }); - - it("reads deflated entries and rejects corrupt compressed data", () => { - const archive = artifactZip([{ name: "summary.json", contents: '{"compressed":true}' }], 8); - - expect(readValidatedArtifactZipEntry(archive, "summary.json", { maxBytes: 1_024 })).toBe( - '{"compressed":true}', - ); - - const corruptArchive = Buffer.from(archive); - const compressedDataOffset = - 30 + corruptArchive.readUInt16LE(26) + corruptArchive.readUInt16LE(28); - const compressedDataEnd = compressedDataOffset + corruptArchive.readUInt32LE(18); - corruptArchive.fill(0, compressedDataOffset, compressedDataEnd); - expect( - readValidatedArtifactZipEntry(corruptArchive, "summary.json", { maxBytes: 1_024 }), - ).toBeNull(); - }); -}); diff --git a/test/e2e/support/e2e-operations-workflow-boundary.test.ts b/test/e2e/support/e2e-operations-workflow-boundary.test.ts index 2aaf8fd990d..4f2d77d8b15 100644 --- a/test/e2e/support/e2e-operations-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-operations-workflow-boundary.test.ts @@ -88,24 +88,6 @@ describe("E2E operations workflow boundary", () => { ); }); - it("retains a bounded runtime summary for future scheduled comparisons", () => { - const workflow = readE2eOperationsWorkflow(); - const upload = workflow.jobs.scorecard.steps!.find( - (step) => step.name === "Upload E2E runtime summary", - ); - - expect(upload).toMatchObject({ - if: "${{ always() && steps.scorecard.outcome == 'success' }}", - uses: "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a", - with: { - name: "e2e-runtime-summary", - path: "${{ runner.temp }}/e2e-runtime-summary.json", - "if-no-files-found": "error", - "retention-days": 14, - }, - }); - }); - it("rejects controller protocol and PR validation drift", () => { const workflow = readE2eOperationsWorkflow(); delete workflow.on?.workflow_dispatch?.inputs?.base_sha; @@ -423,13 +405,9 @@ describe("E2E operations workflow boundary", () => { .fn() .mockReturnValue("## E2E Test Phase Runtime\n\n| Target | Slowest observed phase |"), }; - const runtimeHistory = { - buildRuntimeHistory: vi.fn().mockResolvedValue("## E2E Nightly Runtime Trend"), - }; const runtimeModules = new Map([ ["path", { join: (...parts: string[]) => parts.join("/") }], ["/workspace/scripts/audit-test-runtime.mts", runtimeAudit], - ["/workspace/scripts/scorecard/analyze-runtime-history.mts", runtimeHistory], ["/workspace/scripts/scorecard/coordinate-scorecard.mts", coordinator], ["/workspace/scripts/scorecard/analyze-trace-timing.mts", traceTiming], ["/workspace/scripts/scorecard/summarize-jobs.mts", scorecardJobs], @@ -445,7 +423,6 @@ describe("E2E operations workflow boundary", () => { GITHUB_WORKSPACE: "/workspace", JOBS: "", RUNTIME_ARTIFACTS: "/runner/e2e-runtime-audit", - RUNTIME_SUMMARY_FILE: "/runner/e2e-runtime-summary.json", TARGETS: "", }, }; @@ -468,11 +445,6 @@ describe("E2E operations workflow boundary", () => { expect(traceTiming.buildTraceTimingResult).toHaveBeenCalledWith({ github: {}, context, core }); expect(runtimeAudit.auditTestRuntime).toHaveBeenCalledWith(["/runner/e2e-runtime-audit"]); - expect(runtimeHistory.buildRuntimeHistory).toHaveBeenCalledWith( - { github: {}, context, core }, - [{ target: "full-e2e" }], - "/runner/e2e-runtime-summary.json", - ); expect(runtimeAudit.auditTestRuntime.mock.invocationCallOrder[0]).toBeLessThan( traceTiming.buildTraceTimingResult.mock.invocationCallOrder[0], ); @@ -491,9 +463,7 @@ describe("E2E operations workflow boundary", () => { }), ); expect(summary.addRaw).toHaveBeenCalledWith( - expect.stringMatching( - /### Onboard Performance Budget[\s\S]*## E2E Test Phase Runtime[\s\S]*## E2E Nightly Runtime Trend/u, - ), + expect.stringMatching(/### Onboard Performance Budget[\s\S]*## E2E Test Phase Runtime/u), ); expect(summary.write).toHaveBeenCalledOnce(); expect(setOutput).toHaveBeenCalledWith("scorecardData", expect.any(String)); @@ -518,13 +488,9 @@ describe("E2E operations workflow boundary", () => { }), formatRuntimeAuditSummary: vi.fn(), }; - const runtimeHistory = { - buildRuntimeHistory: vi.fn().mockResolvedValue("## E2E Nightly Runtime Trend"), - }; const runtimeModules = new Map([ ["path", { join: (...parts: string[]) => parts.join("/") }], ["/workspace/scripts/audit-test-runtime.mts", runtimeAudit], - ["/workspace/scripts/scorecard/analyze-runtime-history.mts", runtimeHistory], [ "/workspace/scripts/scorecard/coordinate-scorecard.mts", { @@ -561,7 +527,6 @@ describe("E2E operations workflow boundary", () => { GITHUB_WORKSPACE: "/workspace", JOBS: "", RUNTIME_ARTIFACTS: "/runner/e2e-runtime-audit", - RUNTIME_SUMMARY_FILE: "/runner/e2e-runtime-summary.json", TARGETS: "", }, }; diff --git a/test/e2e/support/e2e-runtime-history.test.ts b/test/e2e/support/e2e-runtime-history.test.ts deleted file mode 100644 index 1496101d0ad..00000000000 --- a/test/e2e/support/e2e-runtime-history.test.ts +++ /dev/null @@ -1,176 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -import { describe, expect, it, vi } from "vitest"; - -import type { RuntimeAuditRow } from "../../../scripts/audit-test-runtime.mts"; -import { - buildRuntimeHistory, - createRuntimeSummary, - formatRuntimeHistory, - loadPriorNightlySummaries, - normalizeRuntimeSummary, -} from "../../../scripts/scorecard/analyze-runtime-history.mts"; - -function runtimeRow(overrides: Partial = {}): RuntimeAuditRow { - const medianMs = overrides.medianMs ?? 120_000; - const p95Ms = overrides.p95Ms ?? medianMs; - const maxMs = overrides.maxMs ?? p95Ms; - return { - target: "rebuild-hermes", - scenario: "rebuild Hermes from source", - runs: 1, - medianMs, - p95Ms, - maxMs, - variabilityMs: overrides.variabilityMs ?? p95Ms - medianMs, - passedRuns: 1, - failedRuns: 0, - skippedRuns: 0, - slowestPhase: "build Hermes image", - slowestPhaseMs: 90_000, - slowestPhaseOutcome: "passed", - ...overrides, - }; -} - -describe("E2E rolling runtime history", () => { - it("compares current semantic-test timing with prior scheduled summaries", () => { - const current = runtimeRow({ medianMs: 150_000, p95Ms: 150_000 }); - const prior = [ - createRuntimeSummary(1, "2026-07-20T00:00:00.000Z", [runtimeRow({ medianMs: 100_000 })]), - createRuntimeSummary(2, "2026-07-21T00:00:00.000Z", [ - runtimeRow({ medianMs: 120_000, passedRuns: 0, failedRuns: 1 }), - ]), - ]; - - const markdown = formatRuntimeHistory([current], prior); - - expect(markdown).toContain("| rebuild-hermes | rebuild Hermes from source | 2 |"); - expect(markdown).toContain("150.0s | 110.0s | 120.0s | +40.0s (+36.4%)"); - expect(markdown).toContain("50% (1/2) | passed |"); - }); - - it("prioritizes current failures and historically flaky tests over slow stable tests", () => { - const failed = runtimeRow({ - target: "failed-fast", - scenario: "failed fast", - medianMs: 10_000, - p95Ms: 10_000, - maxMs: 10_000, - passedRuns: 0, - failedRuns: 1, - }); - const flaky = runtimeRow({ - target: "flaky-fast", - scenario: "flaky fast", - medianMs: 20_000, - p95Ms: 20_000, - maxMs: 20_000, - }); - const stable = runtimeRow({ - target: "stable-slow", - scenario: "stable slow", - medianMs: 300_000, - p95Ms: 300_000, - maxMs: 300_000, - }); - const prior = [ - createRuntimeSummary(1, "2026-07-20T00:00:00.000Z", [ - runtimeRow({ - target: "failed-fast", - scenario: "failed fast", - medianMs: 10_000, - p95Ms: 10_000, - maxMs: 10_000, - }), - runtimeRow({ - target: "flaky-fast", - scenario: "flaky fast", - medianMs: 20_000, - p95Ms: 20_000, - maxMs: 20_000, - passedRuns: 0, - failedRuns: 1, - }), - runtimeRow({ - target: "stable-slow", - scenario: "stable slow", - medianMs: 300_000, - p95Ms: 300_000, - maxMs: 300_000, - }), - ]), - ]; - - const markdown = formatRuntimeHistory([stable, flaky, failed], prior); - - expect(markdown.indexOf("| failed-fast |")).toBeLessThan(markdown.indexOf("| flaky-fast |")); - expect(markdown.indexOf("| flaky-fast |")).toBeLessThan(markdown.indexOf("| stable-slow |")); - }); - - it("writes the current bounded summary even when history is unavailable", async () => { - const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-runtime-history-")); - const output = path.join(directory, "e2e-runtime-summary.json"); - const warning = vi.fn(); - try { - const markdown = await buildRuntimeHistory( - { - github: {}, - context: { repo: { owner: "NVIDIA", repo: "NemoClaw" }, runId: 123 }, - core: { warning }, - }, - [runtimeRow()], - output, - { loadPriorNightlySummaries: vi.fn().mockRejectedValue(new Error("unavailable")) }, - new Date("2026-07-22T00:00:00.000Z"), - ); - - expect(JSON.parse(fs.readFileSync(output, "utf8"))).toMatchObject({ - schemaVersion: "nemoclaw.e2e_runtime_summary.v1", - runId: 123, - }); - expect(markdown).toContain("this run starts the history"); - expect(warning).toHaveBeenCalledOnce(); - } finally { - fs.rmSync(directory, { recursive: true, force: true }); - } - }); - - it("rejects outcome counts that do not match the bounded run count", () => { - const summary = createRuntimeSummary(1, "2026-07-22T00:00:00.000Z", [runtimeRow()]); - summary.rows[0]!.failedRuns = 2; - expect(normalizeRuntimeSummary(summary)).toBeNull(); - }); - - it("queries only prior completed scheduled runs and tolerates missing artifacts", async () => { - const listWorkflowRuns = vi.fn().mockResolvedValue({ - data: { workflow_runs: [{ id: 123 }, { id: 122 }] }, - }); - const paginate = vi.fn().mockResolvedValue([]); - const summaries = await loadPriorNightlySummaries({ - context: { repo: { owner: "NVIDIA", repo: "NemoClaw" }, runId: 123 }, - github: { - paginate, - rest: { - actions: { - downloadArtifact: vi.fn(), - listWorkflowRunArtifacts: {}, - listWorkflowRuns, - }, - }, - }, - }); - - expect(summaries).toEqual([]); - expect(listWorkflowRuns).toHaveBeenCalledWith( - expect.objectContaining({ event: "schedule", status: "completed", workflow_id: "e2e.yaml" }), - ); - expect(paginate).toHaveBeenCalledOnce(); - expect(paginate.mock.calls[0]?.[1]).toMatchObject({ run_id: 122 }); - }); -}); diff --git a/test/e2e/support/test-runtime-audit.test.ts b/test/e2e/support/test-runtime-audit.test.ts index 83d01aa3477..b03e40fa6af 100644 --- a/test/e2e/support/test-runtime-audit.test.ts +++ b/test/e2e/support/test-runtime-audit.test.ts @@ -68,9 +68,6 @@ describe("test runtime audit", () => { p95Ms: 50_000, maxMs: 50_000, variabilityMs: 20_000, - passedRuns: 1, - failedRuns: 1, - skippedRuns: 0, slowestPhase: "inference", slowestPhaseMs: 40_000, slowestPhaseOutcome: "failed", @@ -83,9 +80,6 @@ describe("test runtime audit", () => { p95Ms: 20_000, maxMs: 20_000, variabilityMs: 0, - passedRuns: 1, - failedRuns: 0, - skippedRuns: 0, slowestPhase: "sandbox", slowestPhaseMs: 15_000, slowestPhaseOutcome: "passed", diff --git a/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts b/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts index 9806186a22f..8a548e566d9 100644 --- a/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts +++ b/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts @@ -82,18 +82,6 @@ describe("upload-e2e-artifacts workflow boundary", () => { expect(validateUploadE2eArtifactsInvocations(readWorkflow())).toEqual([]); }); - it("rejects scorecard runtime-summary upload drift", () => { - const workflow = mutableWorkflow(); - const scorecardUpload = workflow.jobs.scorecard.steps!.find( - (step) => step.name === "Upload E2E runtime summary", - )!; - scorecardUpload.with!["retention-days"] = 30; - - expect(validateUploadE2eArtifactsInvocations(workflow)).toContain( - "scorecard must preserve its bounded runtime summary upload contract", - ); - }); - it("rejects semantic-neutral action byte drift from the immutable provenance", () => { expect(validateActionSourceMutation((source) => `${source}# unreviewed drift\n`)).toEqual([ "upload-e2e-artifacts content must match the action reviewed at its immutable commit pin", diff --git a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts index e320c275b81..9f503c2f629 100644 --- a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts +++ b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts @@ -30,17 +30,6 @@ const CHECKOUT_LOCAL_UPLOAD_E2E_ARTIFACTS_ACTION = "./.github/actions/upload-e2e const UPLOAD_E2E_ARTIFACTS_ACTION_PREFIX = "NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@"; const UPLOAD_ARTIFACT_ACTION = "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a"; const UPLOAD_ARTIFACT_ACTION_PREFIX = "actions/upload-artifact@"; -const SCORECARD_RUNTIME_UPLOAD = { - name: "Upload E2E runtime summary", - if: "${{ always() && steps.scorecard.outcome == 'success' }}", - uses: UPLOAD_ARTIFACT_ACTION, - with: { - name: "e2e-runtime-summary", - path: "${{ runner.temp }}/e2e-runtime-summary.json", - "if-no-files-found": "error", - "retention-days": 14, - }, -}; const INNER_ALWAYS = "${{ always() }}"; const CALLER_ALWAYS = "always()"; const MCP_SCANNED_UPLOAD_CONDITION = @@ -320,15 +309,6 @@ export function validateUploadE2eArtifactsAction(actionPath = DEFAULT_ACTION_PAT export function validateUploadE2eArtifactsInvocations(workflow: WorkflowRecord): string[] { const errors: string[] = []; const jobs = record(workflow.jobs); - const scorecardRuntimeUploads = steps(record(jobs.scorecard).steps).filter( - (step) => step.name === SCORECARD_RUNTIME_UPLOAD.name, - ); - if ( - scorecardRuntimeUploads.length !== 1 || - !isDeepStrictEqual(scorecardRuntimeUploads[0], SCORECARD_RUNTIME_UPLOAD) - ) { - errors.push("scorecard must preserve its bounded runtime summary upload contract"); - } const expectedJobs = new Set( Object.entries(jobs) .filter(([jobName, value]) => { @@ -382,10 +362,7 @@ export function validateUploadE2eArtifactsInvocations(workflow: WorkflowRecord): if (uses.startsWith(CHECKOUT_LOCAL_UPLOAD_E2E_ARTIFACTS_ACTION)) { errors.push(`${jobName} must not load upload-e2e-artifacts from the target checkout`); } - if ( - uses.startsWith(UPLOAD_ARTIFACT_ACTION_PREFIX) && - !(jobName === "scorecard" && step.name === SCORECARD_RUNTIME_UPLOAD.name) - ) { + if (uses.startsWith(UPLOAD_ARTIFACT_ACTION_PREFIX)) { errors.push(`${jobName} must not invoke actions/upload-artifact directly`); } if ( From ad4eaf48397ff5a86abf5269bdffb0a0764f7697 Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Fri, 24 Jul 2026 12:03:32 -0700 Subject: [PATCH 5/6] test(e2e): cover combined runner timing order Signed-off-by: Charan Jagwani --- test/e2e/support/e2e-scorecard-coordinator.test.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/test/e2e/support/e2e-scorecard-coordinator.test.ts b/test/e2e/support/e2e-scorecard-coordinator.test.ts index e23d36bb22d..d37f2bbfa93 100644 --- a/test/e2e/support/e2e-scorecard-coordinator.test.ts +++ b/test/e2e/support/e2e-scorecard-coordinator.test.ts @@ -208,25 +208,28 @@ describe("scorecard coordinator assembly", () => { expect(summaryMarkdown).not.toContain("private-larger-runner-label"); }); - it("bounds the job timing table to the ten slowest rows", () => { + it("bounds the job timing table by combined queue and execution time", () => { const { summaryMarkdown } = coordinator.buildScorecard( coordinatorInput({ apiJobs: Array.from({ length: 12 }, (_, index) => ({ - completed_at: new Date(Date.UTC(2026, 6, 24, 0, 0, index + 1)).toISOString(), + completed_at: new Date( + Date.UTC(2026, 6, 24, 0, 0, index === 0 ? 21 : index + 1), + ).toISOString(), conclusion: "success", created_at: "2026-07-24T00:00:00.000Z", labels: ["ubuntu-latest"], name: `job-${String(index + 1).padStart(2, "0")}`, - started_at: "2026-07-24T00:00:00.000Z", + started_at: index === 0 ? "2026-07-24T00:00:20.000Z" : "2026-07-24T00:00:00.000Z", status: "completed", })), }), ); + expect(summaryMarkdown).toContain("| job-01 | standard | success | 20.0s | 1.0s |"); expect(summaryMarkdown).toContain("| job-12 | standard | success | 0.0s | 12.0s |"); - expect(summaryMarkdown).toContain("| job-03 | standard | success | 0.0s | 3.0s |"); + expect(summaryMarkdown).toContain("| job-04 | standard | success | 0.0s | 4.0s |"); + expect(summaryMarkdown).not.toContain("| job-03 |"); expect(summaryMarkdown).not.toContain("| job-02 |"); - expect(summaryMarkdown).not.toContain("| job-01 |"); }); }); From fa8f27a72685f8f5d2f1aad356dfc953347d33d8 Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Fri, 24 Jul 2026 17:05:21 -0700 Subject: [PATCH 6/6] fix(e2e): preserve matrix job timing rows Signed-off-by: Charan Jagwani --- scripts/scorecard/summarize-jobs.mts | 17 ++++++-------- test/e2e/support/e2e-scorecard.test.ts | 32 ++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/scripts/scorecard/summarize-jobs.mts b/scripts/scorecard/summarize-jobs.mts index 634ee9a66e9..7e45868a30d 100644 --- a/scripts/scorecard/summarize-jobs.mts +++ b/scripts/scorecard/summarize-jobs.mts @@ -139,17 +139,10 @@ function preferCandidate(candidate: ApiJob, existing: ApiJob | undefined): boole return (candidate.completed_at ?? "") > (existing.completed_at ?? ""); } -function normalizeApiJobs( - apiJobs: ApiJob[], - metaJobs: Set, - explicitOnly: Set, - selected: Set, -): ApiJob[] { +function normalizeApiJobs(apiJobs: ApiJob[]): ApiJob[] { const dedupedByName = new Map(); for (const job of apiJobs) { const name = job.name.replace(/ \/ [^/]+$/u, ""); - if (metaJobs.has(name)) continue; - if (explicitOnly.has(name) && !selected.has(name)) continue; const candidate = { ...job, name }; if (preferCandidate(candidate, dedupedByName.get(name))) { dedupedByName.set(name, candidate); @@ -189,7 +182,11 @@ function summarizeJobs(input: SummarizeJobsInput): JobSummary { const selected = new Set(input.explicitlySelected); if (input.apiJobs !== null) { - const jobs = normalizeApiJobs(input.apiJobs, metaJobs, explicitOnly, selected); + const eligibleJobs = input.apiJobs.filter((job) => { + const name = job.name.replace(/ \/ [^/]+$/u, ""); + return !metaJobs.has(name) && (!explicitOnly.has(name) || selected.has(name)); + }); + const jobs = normalizeApiJobs(eligibleJobs); const classified = jobs.map((job) => ({ job, result: classifyApiJob(job) })); const counts = countResults(classified.map(({ result }) => result)); return { @@ -198,7 +195,7 @@ function summarizeJobs(input: SummarizeJobsInput): JobSummary { .filter(({ result }) => result === "failure") .map(({ job }) => ({ name: job.name, url: job.html_url ?? null })), ran: jobs.length - counts.skipped, - timingRows: summarizeJobTimings(jobs), + timingRows: summarizeJobTimings(eligibleJobs), total: jobs.length, }; } diff --git a/test/e2e/support/e2e-scorecard.test.ts b/test/e2e/support/e2e-scorecard.test.ts index 8b07180cdc2..7c6e52eeebd 100644 --- a/test/e2e/support/e2e-scorecard.test.ts +++ b/test/e2e/support/e2e-scorecard.test.ts @@ -486,6 +486,38 @@ describe("E2E scorecard", () => { }); }); + it("keeps every matrix execution eligible for the timing ranking", () => { + const summary = scorecardJobs.summarizeJobs({ + apiJobs: [ + { + completed_at: "2026-07-24T00:00:20Z", + conclusion: "success", + created_at: "2026-07-24T00:00:00Z", + labels: ["ubuntu-latest"], + name: "matrix / fast", + started_at: "2026-07-24T00:00:05Z", + status: "completed", + }, + { + completed_at: "2026-07-24T00:02:00Z", + conclusion: "success", + created_at: "2026-07-24T00:00:00Z", + labels: ["ubuntu-latest"], + name: "matrix / slow", + started_at: "2026-07-24T00:00:10Z", + status: "completed", + }, + ], + explicitOnlyJobNames: [], + explicitlySelected: [], + metaJobNames: [], + needs: {}, + }); + + expect(summary).toMatchObject({ success: 1, total: 1 }); + expect(summary.timingRows.map(({ name }) => name)).toEqual(["matrix / slow", "matrix / fast"]); + }); + it("falls back to needs without counting unselected explicit-only jobs", () => { expect( scorecardJobs.summarizeJobs({