ci(e2e): report runner wait and execution time

scripts/scorecard/coordinate-scorecard.mts

@@ -113,6 +113,22 @@ function renderSummaryLines(input: {
       lines.push(job.url ? `  - [${job.name}](${job.url})` : `  - \`${job.name}\``);
     }
   }
+  if (summary.timingRows.length > 0) {
+    const duration = (milliseconds: number | null) =>
+      milliseconds === null ? "n/a" : `${(milliseconds / 1_000).toFixed(1)}s`;
+    lines.push(
+      "",
+      "### Runner wait vs execution",
+      "",
+      "| Job | Runner class | Outcome | Queue | Execution |",
+      "| --- | --- | --- | ---: | ---: |",
+    );
+    for (const row of summary.timingRows) {
+      lines.push(
+        `| ${row.name.replaceAll("|", "\\|")} | ${row.runnerClass} | ${row.outcome} | ${duration(row.queueMs)} | ${duration(row.executionMs)} |`,
+      );
+    }
+  }
   if (input.perfect) lines.push("", "🎉 **All jobs passed!**");
   lines.push(
     "",

scripts/scorecard/summarize-jobs.mts

@@ -4,23 +4,37 @@
 type ApiJob = {
   completed_at?: string | null;
   conclusion?: string | null;
+  created_at?: string | null;
   html_url?: string | null;
+  labels?: string[] | null;
   name: string;
   run_attempt?: number | null;
+  started_at?: string | null;
   status?: string | null;
 };
 
 type NeedResult = { result?: string };
 
 type FailedJob = { name: string; url: string | null };
 
+type CountedResult = "cancelled" | "failure" | "skipped" | "success";
+
+export type JobTimingRow = {
+  executionMs: number | null;
+  name: string;
+  outcome: CountedResult;
+  queueMs: number | null;
+  runnerClass: "larger" | "standard" | "unknown";
+};
+
 export type JobSummary = {
   cancelled: number;
   failedJobs: FailedJob[];
   failure: number;
   ran: number;
   skipped: number;
   success: number;
+  timingRows: JobTimingRow[];
   total: number;
 };
 
@@ -44,8 +58,6 @@ export type WorkflowRunJobsDeps = {
   };
 };
 
-type CountedResult = "cancelled" | "failure" | "skipped" | "success";
-
 function isSelectiveDispatch(eventName: string, rawJobs = "", rawTargets = ""): boolean {
   return eventName === "workflow_dispatch" && (rawJobs.trim() !== "" || rawTargets.trim() !== "");
 }
@@ -66,7 +78,9 @@ function classifyNeed(value: NeedResult): CountedResult {
   return "failure";
 }
 
-function countResults(results: CountedResult[]): Omit<JobSummary, "failedJobs" | "ran" | "total"> {
+function countResults(
+  results: CountedResult[],
+): Omit<JobSummary, "failedJobs" | "ran" | "timingRows" | "total"> {
   return {
     cancelled: results.filter((result) => result === "cancelled").length,
     failure: results.filter((result) => result === "failure").length,
@@ -75,6 +89,48 @@ function countResults(results: CountedResult[]): Omit<JobSummary, "failedJobs" |
   };
 }
 
+function elapsedMs(
+  start: string | null | undefined,
+  finish: string | null | undefined,
+): number | null {
+  if (!start || !finish) return null;
+  const startMs = Date.parse(start);
+  const finishMs = Date.parse(finish);
+  if (!Number.isFinite(startMs) || !Number.isFinite(finishMs) || finishMs < startMs) return null;
+  return finishMs - startMs;
+}
+
+function normalizeRunnerClass(
+  labels: string[] | null | undefined,
+): JobTimingRow["runnerClass"] {
+  if (!labels || labels.length === 0) return "unknown";
+  const normalized = new Set(labels.map((label) => label.toLowerCase()));
+  if (normalized.has("self-hosted")) return "unknown";
+  if (normalized.has("ubuntu-latest")) return "standard";
+  return "larger";
+}
+
+function summarizeJobTimings(jobs: ApiJob[]): JobTimingRow[] {
+  return jobs
+    .map(
+      (job): JobTimingRow => ({
+        executionMs: elapsedMs(job.started_at, job.completed_at),
+        name: job.name,
+        outcome: classifyApiJob(job),
+        queueMs: elapsedMs(job.created_at, job.started_at),
+        runnerClass: normalizeRunnerClass(job.labels),
+      }),
+    )
+    .filter((row) => row.executionMs !== null || row.queueMs !== null)
+    .sort(
+      (left, right) =>
+        (right.executionMs ?? 0) +
+          (right.queueMs ?? 0) -
+          ((left.executionMs ?? 0) + (left.queueMs ?? 0)) || left.name.localeCompare(right.name),
+    )
+    .slice(0, 10);
+}
+
 function preferCandidate(candidate: ApiJob, existing: ApiJob | undefined): boolean {
   if (!existing) return true;
   const candidateAttempt = candidate.run_attempt ?? 0;
@@ -83,17 +139,10 @@ function preferCandidate(candidate: ApiJob, existing: ApiJob | undefined): boole
   return (candidate.completed_at ?? "") > (existing.completed_at ?? "");
 }
 
-function normalizeApiJobs(
-  apiJobs: ApiJob[],
-  metaJobs: Set<string>,
-  explicitOnly: Set<string>,
-  selected: Set<string>,
-): ApiJob[] {
+function normalizeApiJobs(apiJobs: ApiJob[]): ApiJob[] {
   const dedupedByName = new Map<string, ApiJob>();
   for (const job of apiJobs) {
     const name = job.name.replace(/ \/ [^/]+$/u, "");
-    if (metaJobs.has(name)) continue;
-    if (explicitOnly.has(name) && !selected.has(name)) continue;
     const candidate = { ...job, name };
     if (preferCandidate(candidate, dedupedByName.get(name))) {
       dedupedByName.set(name, candidate);
@@ -133,7 +182,11 @@ function summarizeJobs(input: SummarizeJobsInput): JobSummary {
   const selected = new Set(input.explicitlySelected);
 
   if (input.apiJobs !== null) {
-    const jobs = normalizeApiJobs(input.apiJobs, metaJobs, explicitOnly, selected);
+    const eligibleJobs = input.apiJobs.filter((job) => {
+      const name = job.name.replace(/ \/ [^/]+$/u, "");
+      return !metaJobs.has(name) && (!explicitOnly.has(name) || selected.has(name));
+    });
+    const jobs = normalizeApiJobs(eligibleJobs);
     const classified = jobs.map((job) => ({ job, result: classifyApiJob(job) }));
     const counts = countResults(classified.map(({ result }) => result));
     return {
@@ -142,6 +195,7 @@ function summarizeJobs(input: SummarizeJobsInput): JobSummary {
         .filter(({ result }) => result === "failure")
         .map(({ job }) => ({ name: job.name, url: job.html_url ?? null })),
       ran: jobs.length - counts.skipped,
+      timingRows: summarizeJobTimings(eligibleJobs),
       total: jobs.length,
     };
   }
@@ -158,6 +212,7 @@ function summarizeJobs(input: SummarizeJobsInput): JobSummary {
       .filter(({ result }) => result === "failure")
       .map(({ name }) => ({ name, url: null })),
     ran: entries.length - counts.skipped,
+    timingRows: [],
     total: entries.length,
   };
 }

test/e2e/README.md

@@ -152,6 +152,10 @@ graph as the live targets:
   remaining broken for 24 hours, rather than posting on every failed schedule.
 - `scorecard` writes the scheduled/manual result summary and posts it to the
   daily or full-run Slack route. The summary:
+  - separates queue time from execution time for the ten jobs with the longest
+    combined duration;
+  - reports the runner class as `standard`, `larger`, or `unknown` without
+    exposing runner labels;
   - adds this run's semantic phase runtime table;
   - compares each of the ten slowest current tests with up to ten prior
     completed scheduled runs; and

test/e2e/support/e2e-scorecard-coordinator.test.ts

@@ -163,6 +163,74 @@ describe("scorecard coordinator assembly", () => {
     expect(scorecardData).toMatchObject({ total: 2, success: 1, failure: 1, perfect: false });
     expect(scorecardData.failedJobs).toEqual([{ name: "hermes-slack", url: null }]);
   });
+
+  it("separates runner queue time from execution time without exposing runner labels", () => {
+    const { summaryMarkdown } = coordinator.buildScorecard(
+      coordinatorInput({
+        apiJobs: [
+          {
+            completed_at: "2026-07-24T00:01:30Z",
+            conclusion: "failure",
+            created_at: "2026-07-24T00:00:00Z",
+            labels: ["private-larger-runner-label"],
+            name: "mcp-bridge",
+            started_at: "2026-07-24T00:00:30Z",
+            status: "completed",
+          },
+          {
+            completed_at: "2026-07-24T00:00:25Z",
+            conclusion: "success",
+            created_at: "2026-07-24T00:00:00Z",
+            labels: ["ubuntu-latest"],
+            name: "cloud-onboard",
+            started_at: "2026-07-24T00:00:05Z",
+            status: "completed",
+          },
+          {
+            completed_at: "2026-07-24T00:00:20Z",
+            conclusion: "success",
+            created_at: "invalid",
+            labels: ["self-hosted", "ubuntu-latest", "Linux"],
+            name: "jetson-nvmap-gpu",
+            started_at: "2026-07-24T00:00:10Z",
+            status: "completed",
+          },
+        ],
+        rawExplicitOnly: "jetson-nvmap-gpu",
+        rawJobs: "jetson-nvmap-gpu",
+      }),
+    );
+
+    expect(summaryMarkdown).toContain("### Runner wait vs execution");
+    expect(summaryMarkdown).toContain("| mcp-bridge | larger | failure | 30.0s | 60.0s |");
+    expect(summaryMarkdown).toContain("| cloud-onboard | standard | success | 5.0s | 20.0s |");
+    expect(summaryMarkdown).toContain("| jetson-nvmap-gpu | unknown | success | n/a | 10.0s |");
+    expect(summaryMarkdown).not.toContain("private-larger-runner-label");
+  });
+
+  it("bounds the job timing table by combined queue and execution time", () => {
+    const { summaryMarkdown } = coordinator.buildScorecard(
+      coordinatorInput({
+        apiJobs: Array.from({ length: 12 }, (_, index) => ({
+          completed_at: new Date(
+            Date.UTC(2026, 6, 24, 0, 0, index === 0 ? 21 : index + 1),
+          ).toISOString(),
+          conclusion: "success",
+          created_at: "2026-07-24T00:00:00.000Z",
+          labels: ["ubuntu-latest"],
+          name: `job-${String(index + 1).padStart(2, "0")}`,
+          started_at: index === 0 ? "2026-07-24T00:00:20.000Z" : "2026-07-24T00:00:00.000Z",
+          status: "completed",
+        })),
+      }),
+    );
+
+    expect(summaryMarkdown).toContain("| job-01 | standard | success | 20.0s | 1.0s |");
+    expect(summaryMarkdown).toContain("| job-12 | standard | success | 0.0s | 12.0s |");
+    expect(summaryMarkdown).toContain("| job-04 | standard | success | 0.0s | 4.0s |");
+    expect(summaryMarkdown).not.toContain("| job-03 |");
+    expect(summaryMarkdown).not.toContain("| job-02 |");
+  });
 });
 
 describe("scorecard coordinator Slack payload guard", () => {

test/e2e/support/e2e-scorecard.test.ts

@@ -481,10 +481,43 @@ describe("E2E scorecard", () => {
       ran: 4,
       skipped: 0,
       success: 3,
+      timingRows: [],
       total: 4,
     });
   });
 
+  it("keeps every matrix execution eligible for the timing ranking", () => {
+    const summary = scorecardJobs.summarizeJobs({
+      apiJobs: [
+        {
+          completed_at: "2026-07-24T00:00:20Z",
+          conclusion: "success",
+          created_at: "2026-07-24T00:00:00Z",
+          labels: ["ubuntu-latest"],
+          name: "matrix / fast",
+          started_at: "2026-07-24T00:00:05Z",
+          status: "completed",
+        },
+        {
+          completed_at: "2026-07-24T00:02:00Z",
+          conclusion: "success",
+          created_at: "2026-07-24T00:00:00Z",
+          labels: ["ubuntu-latest"],
+          name: "matrix / slow",
+          started_at: "2026-07-24T00:00:10Z",
+          status: "completed",
+        },
+      ],
+      explicitOnlyJobNames: [],
+      explicitlySelected: [],
+      metaJobNames: [],
+      needs: {},
+    });
+
+    expect(summary).toMatchObject({ success: 1, total: 1 });
+    expect(summary.timingRows.map(({ name }) => name)).toEqual(["matrix / slow", "matrix / fast"]);
+  });
+
   it("falls back to needs without counting unselected explicit-only jobs", () => {
     expect(
       scorecardJobs.summarizeJobs({
@@ -508,6 +541,7 @@ describe("E2E scorecard", () => {
       ran: 2,
       skipped: 1,
       success: 1,
+      timingRows: [],
       total: 3,
     });
   });