diff --git a/scripts/scorecard/coordinate-scorecard.mts b/scripts/scorecard/coordinate-scorecard.mts index b78b63f7d3..cddc844306 100644 --- a/scripts/scorecard/coordinate-scorecard.mts +++ b/scripts/scorecard/coordinate-scorecard.mts @@ -113,6 +113,22 @@ function renderSummaryLines(input: { lines.push(job.url ? ` - [${job.name}](${job.url})` : ` - \`${job.name}\``); } } + if (summary.timingRows.length > 0) { + const duration = (milliseconds: number | null) => + milliseconds === null ? "n/a" : `${(milliseconds / 1_000).toFixed(1)}s`; + lines.push( + "", + "### Runner wait vs execution", + "", + "| Job | Runner class | Outcome | Queue | Execution |", + "| --- | --- | --- | ---: | ---: |", + ); + for (const row of summary.timingRows) { + lines.push( + `| ${row.name.replaceAll("|", "\\|")} | ${row.runnerClass} | ${row.outcome} | ${duration(row.queueMs)} | ${duration(row.executionMs)} |`, + ); + } + } if (input.perfect) lines.push("", "🎉 **All jobs passed!**"); lines.push( "", diff --git a/scripts/scorecard/summarize-jobs.mts b/scripts/scorecard/summarize-jobs.mts index e8607e047e..7e45868a30 100644 --- a/scripts/scorecard/summarize-jobs.mts +++ b/scripts/scorecard/summarize-jobs.mts @@ -4,9 +4,12 @@ type ApiJob = { completed_at?: string | null; conclusion?: string | null; + created_at?: string | null; html_url?: string | null; + labels?: string[] | null; name: string; run_attempt?: number | null; + started_at?: string | null; status?: string | null; }; @@ -14,6 +17,16 @@ type NeedResult = { result?: string }; type FailedJob = { name: string; url: string | null }; +type CountedResult = "cancelled" | "failure" | "skipped" | "success"; + +export type JobTimingRow = { + executionMs: number | null; + name: string; + outcome: CountedResult; + queueMs: number | null; + runnerClass: "larger" | "standard" | "unknown"; +}; + export type JobSummary = { cancelled: number; failedJobs: FailedJob[]; @@ -21,6 +34,7 @@ export type JobSummary = { ran: number; skipped: number; success: number; + timingRows: JobTimingRow[]; total: number; }; @@ -44,8 +58,6 @@ export type WorkflowRunJobsDeps = { }; }; -type CountedResult = "cancelled" | "failure" | "skipped" | "success"; - function isSelectiveDispatch(eventName: string, rawJobs = "", rawTargets = ""): boolean { return eventName === "workflow_dispatch" && (rawJobs.trim() !== "" || rawTargets.trim() !== ""); } @@ -66,7 +78,9 @@ function classifyNeed(value: NeedResult): CountedResult { return "failure"; } -function countResults(results: CountedResult[]): Omit { +function countResults( + results: CountedResult[], +): Omit { return { cancelled: results.filter((result) => result === "cancelled").length, failure: results.filter((result) => result === "failure").length, @@ -75,6 +89,48 @@ function countResults(results: CountedResult[]): Omit label.toLowerCase())); + if (normalized.has("self-hosted")) return "unknown"; + if (normalized.has("ubuntu-latest")) return "standard"; + return "larger"; +} + +function summarizeJobTimings(jobs: ApiJob[]): JobTimingRow[] { + return jobs + .map( + (job): JobTimingRow => ({ + executionMs: elapsedMs(job.started_at, job.completed_at), + name: job.name, + outcome: classifyApiJob(job), + queueMs: elapsedMs(job.created_at, job.started_at), + runnerClass: normalizeRunnerClass(job.labels), + }), + ) + .filter((row) => row.executionMs !== null || row.queueMs !== null) + .sort( + (left, right) => + (right.executionMs ?? 0) + + (right.queueMs ?? 0) - + ((left.executionMs ?? 0) + (left.queueMs ?? 0)) || left.name.localeCompare(right.name), + ) + .slice(0, 10); +} + function preferCandidate(candidate: ApiJob, existing: ApiJob | undefined): boolean { if (!existing) return true; const candidateAttempt = candidate.run_attempt ?? 0; @@ -83,17 +139,10 @@ function preferCandidate(candidate: ApiJob, existing: ApiJob | undefined): boole return (candidate.completed_at ?? "") > (existing.completed_at ?? ""); } -function normalizeApiJobs( - apiJobs: ApiJob[], - metaJobs: Set, - explicitOnly: Set, - selected: Set, -): ApiJob[] { +function normalizeApiJobs(apiJobs: ApiJob[]): ApiJob[] { const dedupedByName = new Map(); for (const job of apiJobs) { const name = job.name.replace(/ \/ [^/]+$/u, ""); - if (metaJobs.has(name)) continue; - if (explicitOnly.has(name) && !selected.has(name)) continue; const candidate = { ...job, name }; if (preferCandidate(candidate, dedupedByName.get(name))) { dedupedByName.set(name, candidate); @@ -133,7 +182,11 @@ function summarizeJobs(input: SummarizeJobsInput): JobSummary { const selected = new Set(input.explicitlySelected); if (input.apiJobs !== null) { - const jobs = normalizeApiJobs(input.apiJobs, metaJobs, explicitOnly, selected); + const eligibleJobs = input.apiJobs.filter((job) => { + const name = job.name.replace(/ \/ [^/]+$/u, ""); + return !metaJobs.has(name) && (!explicitOnly.has(name) || selected.has(name)); + }); + const jobs = normalizeApiJobs(eligibleJobs); const classified = jobs.map((job) => ({ job, result: classifyApiJob(job) })); const counts = countResults(classified.map(({ result }) => result)); return { @@ -142,6 +195,7 @@ function summarizeJobs(input: SummarizeJobsInput): JobSummary { .filter(({ result }) => result === "failure") .map(({ job }) => ({ name: job.name, url: job.html_url ?? null })), ran: jobs.length - counts.skipped, + timingRows: summarizeJobTimings(eligibleJobs), total: jobs.length, }; } @@ -158,6 +212,7 @@ function summarizeJobs(input: SummarizeJobsInput): JobSummary { .filter(({ result }) => result === "failure") .map(({ name }) => ({ name, url: null })), ran: entries.length - counts.skipped, + timingRows: [], total: entries.length, }; } diff --git a/test/e2e/README.md b/test/e2e/README.md index cdb7531cc1..9c6cc98278 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -152,6 +152,10 @@ graph as the live targets: remaining broken for 24 hours, rather than posting on every failed schedule. - `scorecard` writes the scheduled/manual result summary and posts it to the daily or full-run Slack route. The summary: + - separates queue time from execution time for the ten jobs with the longest + combined duration; + - reports the runner class as `standard`, `larger`, or `unknown` without + exposing runner labels; - adds this run's semantic phase runtime table; - compares each of the ten slowest current tests with up to ten prior completed scheduled runs; and diff --git a/test/e2e/support/e2e-scorecard-coordinator.test.ts b/test/e2e/support/e2e-scorecard-coordinator.test.ts index e9d973b92b..d37f2bbfa9 100644 --- a/test/e2e/support/e2e-scorecard-coordinator.test.ts +++ b/test/e2e/support/e2e-scorecard-coordinator.test.ts @@ -163,6 +163,74 @@ describe("scorecard coordinator assembly", () => { expect(scorecardData).toMatchObject({ total: 2, success: 1, failure: 1, perfect: false }); expect(scorecardData.failedJobs).toEqual([{ name: "hermes-slack", url: null }]); }); + + it("separates runner queue time from execution time without exposing runner labels", () => { + const { summaryMarkdown } = coordinator.buildScorecard( + coordinatorInput({ + apiJobs: [ + { + completed_at: "2026-07-24T00:01:30Z", + conclusion: "failure", + created_at: "2026-07-24T00:00:00Z", + labels: ["private-larger-runner-label"], + name: "mcp-bridge", + started_at: "2026-07-24T00:00:30Z", + status: "completed", + }, + { + completed_at: "2026-07-24T00:00:25Z", + conclusion: "success", + created_at: "2026-07-24T00:00:00Z", + labels: ["ubuntu-latest"], + name: "cloud-onboard", + started_at: "2026-07-24T00:00:05Z", + status: "completed", + }, + { + completed_at: "2026-07-24T00:00:20Z", + conclusion: "success", + created_at: "invalid", + labels: ["self-hosted", "ubuntu-latest", "Linux"], + name: "jetson-nvmap-gpu", + started_at: "2026-07-24T00:00:10Z", + status: "completed", + }, + ], + rawExplicitOnly: "jetson-nvmap-gpu", + rawJobs: "jetson-nvmap-gpu", + }), + ); + + expect(summaryMarkdown).toContain("### Runner wait vs execution"); + expect(summaryMarkdown).toContain("| mcp-bridge | larger | failure | 30.0s | 60.0s |"); + expect(summaryMarkdown).toContain("| cloud-onboard | standard | success | 5.0s | 20.0s |"); + expect(summaryMarkdown).toContain("| jetson-nvmap-gpu | unknown | success | n/a | 10.0s |"); + expect(summaryMarkdown).not.toContain("private-larger-runner-label"); + }); + + it("bounds the job timing table by combined queue and execution time", () => { + const { summaryMarkdown } = coordinator.buildScorecard( + coordinatorInput({ + apiJobs: Array.from({ length: 12 }, (_, index) => ({ + completed_at: new Date( + Date.UTC(2026, 6, 24, 0, 0, index === 0 ? 21 : index + 1), + ).toISOString(), + conclusion: "success", + created_at: "2026-07-24T00:00:00.000Z", + labels: ["ubuntu-latest"], + name: `job-${String(index + 1).padStart(2, "0")}`, + started_at: index === 0 ? "2026-07-24T00:00:20.000Z" : "2026-07-24T00:00:00.000Z", + status: "completed", + })), + }), + ); + + expect(summaryMarkdown).toContain("| job-01 | standard | success | 20.0s | 1.0s |"); + expect(summaryMarkdown).toContain("| job-12 | standard | success | 0.0s | 12.0s |"); + expect(summaryMarkdown).toContain("| job-04 | standard | success | 0.0s | 4.0s |"); + expect(summaryMarkdown).not.toContain("| job-03 |"); + expect(summaryMarkdown).not.toContain("| job-02 |"); + }); }); describe("scorecard coordinator Slack payload guard", () => { diff --git a/test/e2e/support/e2e-scorecard.test.ts b/test/e2e/support/e2e-scorecard.test.ts index c113efd70d..7c6e52eeeb 100644 --- a/test/e2e/support/e2e-scorecard.test.ts +++ b/test/e2e/support/e2e-scorecard.test.ts @@ -481,10 +481,43 @@ describe("E2E scorecard", () => { ran: 4, skipped: 0, success: 3, + timingRows: [], total: 4, }); }); + it("keeps every matrix execution eligible for the timing ranking", () => { + const summary = scorecardJobs.summarizeJobs({ + apiJobs: [ + { + completed_at: "2026-07-24T00:00:20Z", + conclusion: "success", + created_at: "2026-07-24T00:00:00Z", + labels: ["ubuntu-latest"], + name: "matrix / fast", + started_at: "2026-07-24T00:00:05Z", + status: "completed", + }, + { + completed_at: "2026-07-24T00:02:00Z", + conclusion: "success", + created_at: "2026-07-24T00:00:00Z", + labels: ["ubuntu-latest"], + name: "matrix / slow", + started_at: "2026-07-24T00:00:10Z", + status: "completed", + }, + ], + explicitOnlyJobNames: [], + explicitlySelected: [], + metaJobNames: [], + needs: {}, + }); + + expect(summary).toMatchObject({ success: 1, total: 1 }); + expect(summary.timingRows.map(({ name }) => name)).toEqual(["matrix / slow", "matrix / fast"]); + }); + it("falls back to needs without counting unselected explicit-only jobs", () => { expect( scorecardJobs.summarizeJobs({ @@ -508,6 +541,7 @@ describe("E2E scorecard", () => { ran: 2, skipped: 1, success: 1, + timingRows: [], total: 3, }); });