ci(e2e): add runner comparison telemetry

.github/workflows/e2e.yaml

@@ -512,6 +512,23 @@ jobs:
       - name: Prepare E2E workspace
         uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28
 
+      - name: Initialize runner comparison evidence
+        if: ${{ matrix.agent == 'hermes' || matrix.agent == 'deepagents' }}
+        continue-on-error: true
+        shell: bash
+        run: |
+          set -euo pipefail
+          mkdir -p "$E2E_ARTIFACT_DIR"
+          snapshots_file="$E2E_ARTIFACT_DIR/runner-resource-snapshots.jsonl"
+          summary_file="$E2E_ARTIFACT_DIR/runner-resource-summary.json"
+          export E2E_RESOURCE_SNAPSHOTS_FILE="$snapshots_file"
+          export E2E_RESOURCE_SUMMARY_FILE="$summary_file"
+          npx tsx tools/e2e/runner-pressure.mts initialize-measurement
+          {
+            printf 'E2E_RESOURCE_SNAPSHOTS_FILE=%s\n' "$snapshots_file"
+            printf 'E2E_RESOURCE_SUMMARY_FILE=%s\n' "$summary_file"
+          } >> "$GITHUB_ENV"
+
       - name: Install and verify cloudflared prerequisite
         # Update posture: maintainers review upstream cloudflared releases and
         # update the version and reviewed SHA256 together in both explicit MCP
@@ -582,6 +599,16 @@ jobs:
               --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts
           fi
 
+      - name: Summarize runner comparison evidence
+        if: ${{ always() && (matrix.agent == 'hermes' || matrix.agent == 'deepagents') }}
+        continue-on-error: true
+        shell: bash
+        run: |
+          set -euo pipefail
+          export E2E_RESOURCE_SNAPSHOTS_FILE="$E2E_ARTIFACT_DIR/runner-resource-snapshots.jsonl"
+          export E2E_RESOURCE_SUMMARY_FILE="$E2E_ARTIFACT_DIR/runner-resource-summary.json"
+          npx tsx tools/e2e/runner-pressure.mts summarize-measurement
+
       - id: mcp_artifact_secret_scan
         name: Scan MCP artifacts for fixture credentials
         if: always()
@@ -2067,6 +2094,22 @@ jobs:
       - name: Prepare E2E workspace
         uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28
 
+      - name: Initialize runner comparison evidence
+        continue-on-error: true
+        shell: bash
+        run: |
+          set -euo pipefail
+          mkdir -p "$E2E_ARTIFACT_DIR"
+          snapshots_file="$E2E_ARTIFACT_DIR/runner-resource-snapshots.jsonl"
+          summary_file="$E2E_ARTIFACT_DIR/runner-resource-summary.json"
+          export E2E_RESOURCE_SNAPSHOTS_FILE="$snapshots_file"
+          export E2E_RESOURCE_SUMMARY_FILE="$summary_file"
+          npx tsx tools/e2e/runner-pressure.mts initialize-measurement
+          {
+            printf 'E2E_RESOURCE_SNAPSHOTS_FILE=%s\n' "$snapshots_file"
+            printf 'E2E_RESOURCE_SUMMARY_FILE=%s\n' "$summary_file"
+          } >> "$GITHUB_ENV"
+
       - name: Install OpenShell
         # Direct Vitest execution uses bin/nemoclaw.js instead of install.sh,
         # so install OpenShell explicitly before onboard and SSH-agent probes.
@@ -2100,6 +2143,16 @@ jobs:
           "$OPENSHELL_BIN" --version
           npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/common-egress-agent.test.ts
 
+      - name: Summarize runner comparison evidence
+        if: always()
+        continue-on-error: true
+        shell: bash
+        run: |
+          set -euo pipefail
+          export E2E_RESOURCE_SNAPSHOTS_FILE="$E2E_ARTIFACT_DIR/runner-resource-snapshots.jsonl"
+          export E2E_RESOURCE_SUMMARY_FILE="$E2E_ARTIFACT_DIR/runner-resource-summary.json"
+          npx tsx tools/e2e/runner-pressure.mts summarize-measurement
+
       - name: Upload common-egress agent artifacts
         if: always()
         uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
@@ -2295,6 +2348,22 @@ jobs:
       - name: Prepare E2E workspace
         uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28
 
+      - name: Initialize runner comparison evidence
+        continue-on-error: true
+        shell: bash
+        run: |
+          set -euo pipefail
+          mkdir -p "$E2E_ARTIFACT_DIR"
+          snapshots_file="$E2E_ARTIFACT_DIR/runner-resource-snapshots.jsonl"
+          summary_file="$E2E_ARTIFACT_DIR/runner-resource-summary.json"
+          export E2E_RESOURCE_SNAPSHOTS_FILE="$snapshots_file"
+          export E2E_RESOURCE_SUMMARY_FILE="$summary_file"
+          npx tsx tools/e2e/runner-pressure.mts initialize-measurement
+          {
+            printf 'E2E_RESOURCE_SNAPSHOTS_FILE=%s\n' "$snapshots_file"
+            printf 'E2E_RESOURCE_SUMMARY_FILE=%s\n' "$summary_file"
+          } >> "$GITHUB_ENV"
+
       - name: Install OpenShell
         # Full NemoClaw installer coverage remains in hermes-e2e. Rebuild uses
         # the checked-out CLI and installs only its external runtime dependency.
@@ -2354,6 +2423,16 @@ jobs:
           fi
           exit "$test_status"
 
+      - name: Summarize runner comparison evidence
+        if: always()
+        continue-on-error: true
+        shell: bash
+        run: |
+          set -euo pipefail
+          export E2E_RESOURCE_SNAPSHOTS_FILE="$E2E_ARTIFACT_DIR/runner-resource-snapshots.jsonl"
+          export E2E_RESOURCE_SUMMARY_FILE="$E2E_ARTIFACT_DIR/runner-resource-summary.json"
+          npx tsx tools/e2e/runner-pressure.mts summarize-measurement
+
       - name: Upload Hermes rebuild artifacts
         if: always()
         uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
@@ -2397,6 +2476,22 @@ jobs:
       - name: Prepare E2E workspace
         uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28
 
+      - name: Initialize runner comparison evidence
+        continue-on-error: true
+        shell: bash
+        run: |
+          set -euo pipefail
+          mkdir -p "$E2E_ARTIFACT_DIR"
+          snapshots_file="$E2E_ARTIFACT_DIR/runner-resource-snapshots.jsonl"
+          summary_file="$E2E_ARTIFACT_DIR/runner-resource-summary.json"
+          export E2E_RESOURCE_SNAPSHOTS_FILE="$snapshots_file"
+          export E2E_RESOURCE_SUMMARY_FILE="$summary_file"
+          npx tsx tools/e2e/runner-pressure.mts initialize-measurement
+          {
+            printf 'E2E_RESOURCE_SNAPSHOTS_FILE=%s\n' "$snapshots_file"
+            printf 'E2E_RESOURCE_SUMMARY_FILE=%s\n' "$summary_file"
+          } >> "$GITHUB_ENV"
+
       - name: Install OpenShell
         # Full NemoClaw installer coverage remains in hermes-e2e. Rebuild uses
         # the checked-out CLI and installs only its external runtime dependency.
@@ -2456,6 +2551,16 @@ jobs:
           fi
           exit "$test_status"
 
+      - name: Summarize runner comparison evidence
+        if: always()
+        continue-on-error: true
+        shell: bash
+        run: |
+          set -euo pipefail
+          export E2E_RESOURCE_SNAPSHOTS_FILE="$E2E_ARTIFACT_DIR/runner-resource-snapshots.jsonl"
+          export E2E_RESOURCE_SUMMARY_FILE="$E2E_ARTIFACT_DIR/runner-resource-summary.json"
+          npx tsx tools/e2e/runner-pressure.mts summarize-measurement
+
       - name: Upload Hermes stale-base rebuild artifacts
         if: always()
         uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
@@ -5094,6 +5199,7 @@ jobs:
           EXPLICIT_ONLY_JOBS: ${{ needs.generate-matrix.outputs.explicit_only_jobs }}
           JOBS: ${{ inputs.jobs }}
           RUNTIME_ARTIFACTS: ${{ runner.temp }}/e2e-runtime-audit
+          RUNTIME_SUMMARY_FILE: ${{ runner.temp }}/e2e-runtime-summary.json
           TARGETS: ${{ inputs.targets }}
         with:
           script: |
@@ -5110,15 +5216,19 @@ jobs:
             const runtimeAudit = require(
               path.join(process.env.GITHUB_WORKSPACE, 'scripts/audit-test-runtime.mts'),
             );
+            const runtimeHistory = require(
+              path.join(process.env.GITHUB_WORKSPACE, 'scripts/scorecard/analyze-runtime-history.mts'),
+            );
             const needs = ${{ toJSON(needs) }};
 
             // GitHub's jobs API is the canonical source because `needs.live`
             // collapses every matrix target into one result and has no job URL.
             // The typed helper owns and tests the degraded `needs` fallback.
             const apiJobs = await scorecardJobs.loadWorkflowRunJobs({ github, context, core });
             let runtimeSummaryMarkdown;
+            let runtimeRows = [];
             try {
-              const runtimeRows = runtimeAudit.auditTestRuntime([process.env.RUNTIME_ARTIFACTS]);
+              runtimeRows = runtimeAudit.auditTestRuntime([process.env.RUNTIME_ARTIFACTS]);
               runtimeSummaryMarkdown = runtimeAudit.formatRuntimeAuditSummary(runtimeRows);
             } catch {
               core.warning('E2E test phase runtime summary unavailable: invalid progress artifact');
@@ -5129,6 +5239,11 @@ jobs:
                 '',
               ].join('\n');
             }
+            const runtimeHistoryMarkdown = await runtimeHistory.buildRuntimeHistory(
+              { github, context, core },
+              runtimeRows,
+              process.env.RUNTIME_SUMMARY_FILE,
+            );
             const trace = await traceTiming.buildTraceTimingResult({ github, context, core });
             if (trace.budgetWarningMessage) core.warning(trace.budgetWarningMessage);
 
@@ -5147,7 +5262,9 @@ jobs:
               today: new Date().toLocaleDateString('en-US', { month: 'short', day: 'numeric' }),
             });
 
-            await core.summary.addRaw(`${summaryMarkdown}\n\n${runtimeSummaryMarkdown}`).write();
+            await core.summary
+              .addRaw(`${summaryMarkdown}\n\n${runtimeSummaryMarkdown}\n${runtimeHistoryMarkdown}`)
+              .write();
             core.setOutput('scorecardData', JSON.stringify(scorecardData));
             core.setOutput('slackData', JSON.stringify(slackData));
 
@@ -5208,3 +5325,10 @@ jobs:
             if (!response.ok) {
               core.setFailed(`Slack webhook returned ${response.status}`);
             }
+
+      - name: Upload E2E runtime summary
+        if: ${{ always() && steps.scorecard.outcome == 'success' }}
+        uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
+        with:
+          name: e2e-runtime-summary
+          path: ${{ runner.temp }}/e2e-runtime-summary.json

scripts/audit-test-runtime.mts

@@ -15,6 +15,9 @@ export interface RuntimeAuditRow {
   p95Ms: number;
   maxMs: number;
   variabilityMs: number;
+  passedRuns: number;
+  failedRuns: number;
+  skippedRuns: number;
   slowestPhase: string;
   slowestPhaseMs: number;
   slowestPhaseOutcome: "passed" | "failed" | "skipped";
@@ -78,6 +81,12 @@ function median(sorted: readonly number[]): number {
   return sorted[middle] ?? 0;
 }
 
+function summaryOutcome(summary: ProgressSummary): "passed" | "failed" | "skipped" {
+  if (summary.phases.some((phase) => phase.outcome === "failed")) return "failed";
+  if (summary.phases.some((phase) => phase.outcome === "skipped")) return "skipped";
+  return "passed";
+}
+
 export function auditTestRuntime(roots: readonly string[]): RuntimeAuditRow[] {
   const summaries = roots.flatMap(progressFiles).map((file) => {
     const parsed: unknown = JSON.parse(fs.readFileSync(file, "utf8"));
@@ -108,6 +117,7 @@ export function auditTestRuntime(roots: readonly string[]): RuntimeAuditRow[] {
       );
       const medianMs = median(durations);
       const p95Ms = percentile(durations, 0.95);
+      const outcomes = runs.map(summaryOutcome);
       return {
         target: [first.targetId ?? "unlabeled", first.shardId].filter(Boolean).join("/"),
         scenario: first.scenario,
@@ -116,6 +126,9 @@ export function auditTestRuntime(roots: readonly string[]): RuntimeAuditRow[] {
         p95Ms,
         maxMs: durations.at(-1) ?? 0,
         variabilityMs: Math.max(0, p95Ms - medianMs),
+        passedRuns: outcomes.filter((outcome) => outcome === "passed").length,
+        failedRuns: outcomes.filter((outcome) => outcome === "failed").length,
+        skippedRuns: outcomes.filter((outcome) => outcome === "skipped").length,
         slowestPhase: slowestPhase.label,
         slowestPhaseMs: slowestPhase.durationMs,
         slowestPhaseOutcome: slowestPhase.outcome,