diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 041633b6cc7..ba7af66e490 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -512,6 +512,23 @@ jobs: - name: Prepare E2E workspace uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28 + - name: Initialize runner comparison evidence + if: ${{ matrix.agent == 'hermes' || matrix.agent == 'deepagents' }} + continue-on-error: true + shell: bash + run: | + set -euo pipefail + mkdir -p "$E2E_ARTIFACT_DIR" + snapshots_file="$E2E_ARTIFACT_DIR/runner-resource-snapshots.jsonl" + summary_file="$E2E_ARTIFACT_DIR/runner-resource-summary.json" + export E2E_RESOURCE_SNAPSHOTS_FILE="$snapshots_file" + export E2E_RESOURCE_SUMMARY_FILE="$summary_file" + npx tsx tools/e2e/runner-pressure.mts initialize-measurement + { + printf 'E2E_RESOURCE_SNAPSHOTS_FILE=%s\n' "$snapshots_file" + printf 'E2E_RESOURCE_SUMMARY_FILE=%s\n' "$summary_file" + } >> "$GITHUB_ENV" + - name: Install and verify cloudflared prerequisite # Update posture: maintainers review upstream cloudflared releases and # update the version and reviewed SHA256 together in both explicit MCP @@ -582,6 +599,16 @@ jobs: --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts fi + - name: Summarize runner comparison evidence + if: ${{ always() && (matrix.agent == 'hermes' || matrix.agent == 'deepagents') }} + continue-on-error: true + shell: bash + run: | + set -euo pipefail + export E2E_RESOURCE_SNAPSHOTS_FILE="$E2E_ARTIFACT_DIR/runner-resource-snapshots.jsonl" + export E2E_RESOURCE_SUMMARY_FILE="$E2E_ARTIFACT_DIR/runner-resource-summary.json" + npx tsx tools/e2e/runner-pressure.mts summarize-measurement + - id: mcp_artifact_secret_scan name: Scan MCP artifacts for fixture credentials if: always() @@ -2067,6 +2094,22 @@ jobs: - name: Prepare E2E workspace uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28 + - name: Initialize runner comparison evidence + continue-on-error: true + shell: bash + run: | + set -euo pipefail + mkdir -p "$E2E_ARTIFACT_DIR" + snapshots_file="$E2E_ARTIFACT_DIR/runner-resource-snapshots.jsonl" + summary_file="$E2E_ARTIFACT_DIR/runner-resource-summary.json" + export E2E_RESOURCE_SNAPSHOTS_FILE="$snapshots_file" + export E2E_RESOURCE_SUMMARY_FILE="$summary_file" + npx tsx tools/e2e/runner-pressure.mts initialize-measurement + { + printf 'E2E_RESOURCE_SNAPSHOTS_FILE=%s\n' "$snapshots_file" + printf 'E2E_RESOURCE_SUMMARY_FILE=%s\n' "$summary_file" + } >> "$GITHUB_ENV" + - name: Install OpenShell # Direct Vitest execution uses bin/nemoclaw.js instead of install.sh, # so install OpenShell explicitly before onboard and SSH-agent probes. @@ -2100,6 +2143,16 @@ jobs: "$OPENSHELL_BIN" --version npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/common-egress-agent.test.ts + - name: Summarize runner comparison evidence + if: always() + continue-on-error: true + shell: bash + run: | + set -euo pipefail + export E2E_RESOURCE_SNAPSHOTS_FILE="$E2E_ARTIFACT_DIR/runner-resource-snapshots.jsonl" + export E2E_RESOURCE_SUMMARY_FILE="$E2E_ARTIFACT_DIR/runner-resource-summary.json" + npx tsx tools/e2e/runner-pressure.mts summarize-measurement + - name: Upload common-egress agent artifacts if: always() uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 @@ -2295,6 +2348,22 @@ jobs: - name: Prepare E2E workspace uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28 + - name: Initialize runner comparison evidence + continue-on-error: true + shell: bash + run: | + set -euo pipefail + mkdir -p "$E2E_ARTIFACT_DIR" + snapshots_file="$E2E_ARTIFACT_DIR/runner-resource-snapshots.jsonl" + summary_file="$E2E_ARTIFACT_DIR/runner-resource-summary.json" + export E2E_RESOURCE_SNAPSHOTS_FILE="$snapshots_file" + export E2E_RESOURCE_SUMMARY_FILE="$summary_file" + npx tsx tools/e2e/runner-pressure.mts initialize-measurement + { + printf 'E2E_RESOURCE_SNAPSHOTS_FILE=%s\n' "$snapshots_file" + printf 'E2E_RESOURCE_SUMMARY_FILE=%s\n' "$summary_file" + } >> "$GITHUB_ENV" + - name: Install OpenShell # Full NemoClaw installer coverage remains in hermes-e2e. Rebuild uses # the checked-out CLI and installs only its external runtime dependency. @@ -2354,6 +2423,16 @@ jobs: fi exit "$test_status" + - name: Summarize runner comparison evidence + if: always() + continue-on-error: true + shell: bash + run: | + set -euo pipefail + export E2E_RESOURCE_SNAPSHOTS_FILE="$E2E_ARTIFACT_DIR/runner-resource-snapshots.jsonl" + export E2E_RESOURCE_SUMMARY_FILE="$E2E_ARTIFACT_DIR/runner-resource-summary.json" + npx tsx tools/e2e/runner-pressure.mts summarize-measurement + - name: Upload Hermes rebuild artifacts if: always() uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 @@ -2397,6 +2476,22 @@ jobs: - name: Prepare E2E workspace uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28 + - name: Initialize runner comparison evidence + continue-on-error: true + shell: bash + run: | + set -euo pipefail + mkdir -p "$E2E_ARTIFACT_DIR" + snapshots_file="$E2E_ARTIFACT_DIR/runner-resource-snapshots.jsonl" + summary_file="$E2E_ARTIFACT_DIR/runner-resource-summary.json" + export E2E_RESOURCE_SNAPSHOTS_FILE="$snapshots_file" + export E2E_RESOURCE_SUMMARY_FILE="$summary_file" + npx tsx tools/e2e/runner-pressure.mts initialize-measurement + { + printf 'E2E_RESOURCE_SNAPSHOTS_FILE=%s\n' "$snapshots_file" + printf 'E2E_RESOURCE_SUMMARY_FILE=%s\n' "$summary_file" + } >> "$GITHUB_ENV" + - name: Install OpenShell # Full NemoClaw installer coverage remains in hermes-e2e. Rebuild uses # the checked-out CLI and installs only its external runtime dependency. @@ -2456,6 +2551,16 @@ jobs: fi exit "$test_status" + - name: Summarize runner comparison evidence + if: always() + continue-on-error: true + shell: bash + run: | + set -euo pipefail + export E2E_RESOURCE_SNAPSHOTS_FILE="$E2E_ARTIFACT_DIR/runner-resource-snapshots.jsonl" + export E2E_RESOURCE_SUMMARY_FILE="$E2E_ARTIFACT_DIR/runner-resource-summary.json" + npx tsx tools/e2e/runner-pressure.mts summarize-measurement + - name: Upload Hermes stale-base rebuild artifacts if: always() uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 @@ -5094,6 +5199,7 @@ jobs: EXPLICIT_ONLY_JOBS: ${{ needs.generate-matrix.outputs.explicit_only_jobs }} JOBS: ${{ inputs.jobs }} RUNTIME_ARTIFACTS: ${{ runner.temp }}/e2e-runtime-audit + RUNTIME_SUMMARY_FILE: ${{ runner.temp }}/e2e-runtime-summary.json TARGETS: ${{ inputs.targets }} with: script: | @@ -5110,6 +5216,9 @@ jobs: const runtimeAudit = require( path.join(process.env.GITHUB_WORKSPACE, 'scripts/audit-test-runtime.mts'), ); + const runtimeHistory = require( + path.join(process.env.GITHUB_WORKSPACE, 'scripts/scorecard/analyze-runtime-history.mts'), + ); const needs = ${{ toJSON(needs) }}; // GitHub's jobs API is the canonical source because `needs.live` @@ -5117,8 +5226,9 @@ jobs: // The typed helper owns and tests the degraded `needs` fallback. const apiJobs = await scorecardJobs.loadWorkflowRunJobs({ github, context, core }); let runtimeSummaryMarkdown; + let runtimeRows = []; try { - const runtimeRows = runtimeAudit.auditTestRuntime([process.env.RUNTIME_ARTIFACTS]); + runtimeRows = runtimeAudit.auditTestRuntime([process.env.RUNTIME_ARTIFACTS]); runtimeSummaryMarkdown = runtimeAudit.formatRuntimeAuditSummary(runtimeRows); } catch { core.warning('E2E test phase runtime summary unavailable: invalid progress artifact'); @@ -5129,6 +5239,11 @@ jobs: '', ].join('\n'); } + const runtimeHistoryMarkdown = await runtimeHistory.buildRuntimeHistory( + { github, context, core }, + runtimeRows, + process.env.RUNTIME_SUMMARY_FILE, + ); const trace = await traceTiming.buildTraceTimingResult({ github, context, core }); if (trace.budgetWarningMessage) core.warning(trace.budgetWarningMessage); @@ -5147,7 +5262,9 @@ jobs: today: new Date().toLocaleDateString('en-US', { month: 'short', day: 'numeric' }), }); - await core.summary.addRaw(`${summaryMarkdown}\n\n${runtimeSummaryMarkdown}`).write(); + await core.summary + .addRaw(`${summaryMarkdown}\n\n${runtimeSummaryMarkdown}\n${runtimeHistoryMarkdown}`) + .write(); core.setOutput('scorecardData', JSON.stringify(scorecardData)); core.setOutput('slackData', JSON.stringify(slackData)); @@ -5208,3 +5325,10 @@ jobs: if (!response.ok) { core.setFailed(`Slack webhook returned ${response.status}`); } + + - name: Upload E2E runtime summary + if: ${{ always() && steps.scorecard.outcome == 'success' }} + uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 + with: + name: e2e-runtime-summary + path: ${{ runner.temp }}/e2e-runtime-summary.json diff --git a/scripts/audit-test-runtime.mts b/scripts/audit-test-runtime.mts index c6611a7bf29..58c0d2fe263 100644 --- a/scripts/audit-test-runtime.mts +++ b/scripts/audit-test-runtime.mts @@ -15,6 +15,9 @@ export interface RuntimeAuditRow { p95Ms: number; maxMs: number; variabilityMs: number; + passedRuns: number; + failedRuns: number; + skippedRuns: number; slowestPhase: string; slowestPhaseMs: number; slowestPhaseOutcome: "passed" | "failed" | "skipped"; @@ -78,6 +81,12 @@ function median(sorted: readonly number[]): number { return sorted[middle] ?? 0; } +function summaryOutcome(summary: ProgressSummary): "passed" | "failed" | "skipped" { + if (summary.phases.some((phase) => phase.outcome === "failed")) return "failed"; + if (summary.phases.some((phase) => phase.outcome === "skipped")) return "skipped"; + return "passed"; +} + export function auditTestRuntime(roots: readonly string[]): RuntimeAuditRow[] { const summaries = roots.flatMap(progressFiles).map((file) => { const parsed: unknown = JSON.parse(fs.readFileSync(file, "utf8")); @@ -108,6 +117,7 @@ export function auditTestRuntime(roots: readonly string[]): RuntimeAuditRow[] { ); const medianMs = median(durations); const p95Ms = percentile(durations, 0.95); + const outcomes = runs.map(summaryOutcome); return { target: [first.targetId ?? "unlabeled", first.shardId].filter(Boolean).join("/"), scenario: first.scenario, @@ -116,6 +126,9 @@ export function auditTestRuntime(roots: readonly string[]): RuntimeAuditRow[] { p95Ms, maxMs: durations.at(-1) ?? 0, variabilityMs: Math.max(0, p95Ms - medianMs), + passedRuns: outcomes.filter((outcome) => outcome === "passed").length, + failedRuns: outcomes.filter((outcome) => outcome === "failed").length, + skippedRuns: outcomes.filter((outcome) => outcome === "skipped").length, slowestPhase: slowestPhase.label, slowestPhaseMs: slowestPhase.durationMs, slowestPhaseOutcome: slowestPhase.outcome, diff --git a/scripts/scorecard/analyze-runtime-history.mts b/scripts/scorecard/analyze-runtime-history.mts new file mode 100644 index 00000000000..e756a8115be --- /dev/null +++ b/scripts/scorecard/analyze-runtime-history.mts @@ -0,0 +1,324 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; + +import type { RuntimeAuditRow } from "../audit-test-runtime.mts"; +import { readValidatedArtifactZipEntry } from "./read-artifact-zip.mts"; + +export const RUNTIME_SUMMARY_ARTIFACT = "e2e-runtime-summary"; +export const RUNTIME_SUMMARY_FILE = "e2e-runtime-summary.json"; +const RUNTIME_SUMMARY_SCHEMA = "nemoclaw.e2e_runtime_summary.v1"; +const WORKFLOW_FILE = "e2e.yaml"; +const HISTORY_RUN_LIMIT = 10; +const MAX_SUMMARY_BYTES = 256 * 1024; +const MAX_SUMMARY_ROWS = 200; + +type GitHubDeps = { + github: any; + context: { repo: { owner: string; repo: string }; runId: number }; + core?: { warning?: (message: string) => void }; +}; + +export interface RuntimeSummaryArtifact { + schemaVersion: typeof RUNTIME_SUMMARY_SCHEMA; + runId: number; + createdAt: string; + rows: RuntimeAuditRow[]; +} + +type RuntimeHistoryServices = { + loadPriorNightlySummaries: (deps: GitHubDeps) => Promise; +}; + +function isBoundedString(value: unknown): value is string { + return ( + typeof value === "string" && + value.length > 0 && + value.length <= 500 && + !/[\u0000-\u001f\u007f]/u.test(value) + ); +} + +function isNonNegativeNumber(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value) && value >= 0; +} + +function isNonNegativeInteger(value: unknown): value is number { + return Number.isInteger(value) && isNonNegativeNumber(value); +} + +function hasExactKeys(value: Record, expected: readonly string[]): boolean { + return Object.keys(value).sort().join("\0") === [...expected].sort().join("\0"); +} + +function normalizeRuntimeRow(value: unknown): RuntimeAuditRow | null { + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const row = value as Record; + if ( + !hasExactKeys(row, [ + "target", + "scenario", + "runs", + "medianMs", + "p95Ms", + "maxMs", + "variabilityMs", + "passedRuns", + "failedRuns", + "skippedRuns", + "slowestPhase", + "slowestPhaseMs", + "slowestPhaseOutcome", + ]) || + !isBoundedString(row.target) || + !isBoundedString(row.scenario) || + !isNonNegativeInteger(row.runs) || + row.runs < 1 || + !isNonNegativeNumber(row.medianMs) || + !isNonNegativeNumber(row.p95Ms) || + !isNonNegativeNumber(row.maxMs) || + !isNonNegativeNumber(row.variabilityMs) || + row.medianMs > row.p95Ms || + row.p95Ms > row.maxMs || + row.variabilityMs !== row.p95Ms - row.medianMs || + !isNonNegativeInteger(row.passedRuns) || + !isNonNegativeInteger(row.failedRuns) || + !isNonNegativeInteger(row.skippedRuns) || + row.passedRuns + row.failedRuns + row.skippedRuns !== row.runs || + !isBoundedString(row.slowestPhase) || + !isNonNegativeNumber(row.slowestPhaseMs) || + (row.slowestPhaseOutcome !== "passed" && + row.slowestPhaseOutcome !== "failed" && + row.slowestPhaseOutcome !== "skipped") + ) { + return null; + } + return { + target: row.target, + scenario: row.scenario, + runs: row.runs, + medianMs: row.medianMs, + p95Ms: row.p95Ms, + maxMs: row.maxMs, + variabilityMs: row.variabilityMs, + passedRuns: row.passedRuns, + failedRuns: row.failedRuns, + skippedRuns: row.skippedRuns, + slowestPhase: row.slowestPhase, + slowestPhaseMs: row.slowestPhaseMs, + slowestPhaseOutcome: row.slowestPhaseOutcome, + }; +} + +export function normalizeRuntimeSummary(value: unknown): RuntimeSummaryArtifact | null { + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const summary = value as Record; + if ( + !hasExactKeys(summary, ["schemaVersion", "runId", "createdAt", "rows"]) || + summary.schemaVersion !== RUNTIME_SUMMARY_SCHEMA || + !isNonNegativeInteger(summary.runId) || + summary.runId < 1 || + typeof summary.createdAt !== "string" || + !Number.isFinite(Date.parse(summary.createdAt)) || + !Array.isArray(summary.rows) || + summary.rows.length > MAX_SUMMARY_ROWS + ) { + return null; + } + const rows = summary.rows.map(normalizeRuntimeRow); + if (rows.some((row) => row === null)) return null; + const identities = new Set( + (rows as RuntimeAuditRow[]).map((row) => JSON.stringify([row.target, row.scenario])), + ); + if (identities.size !== rows.length) return null; + return { + schemaVersion: RUNTIME_SUMMARY_SCHEMA, + runId: summary.runId, + createdAt: summary.createdAt, + rows: rows as RuntimeAuditRow[], + }; +} + +export function createRuntimeSummary( + runId: number, + createdAt: string, + rows: readonly RuntimeAuditRow[], +): RuntimeSummaryArtifact { + const summary = normalizeRuntimeSummary({ + schemaVersion: RUNTIME_SUMMARY_SCHEMA, + runId, + createdAt, + rows, + }); + if (summary === null) throw new Error("invalid current E2E runtime summary"); + return summary; +} + +function parseRuntimeSummaryArchive(archive: Buffer): RuntimeSummaryArtifact | null { + try { + const contents = readValidatedArtifactZipEntry(archive, RUNTIME_SUMMARY_FILE, { + maxBytes: MAX_SUMMARY_BYTES, + }); + return contents === null ? null : normalizeRuntimeSummary(JSON.parse(contents)); + } catch { + return null; + } +} + +async function readRuntimeSummaryFromRun( + { github, context }: GitHubDeps, + runId: number, +): Promise { + const artifacts = (await github.paginate(github.rest.actions.listWorkflowRunArtifacts, { + owner: context.repo.owner, + repo: context.repo.repo, + run_id: runId, + per_page: 100, + })) as Array<{ expired?: boolean; id: number; name: string }>; + const artifact = artifacts.find( + (candidate) => candidate.name === RUNTIME_SUMMARY_ARTIFACT && candidate.expired !== true, + ); + if (!artifact) return null; + const download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: artifact.id, + archive_format: "zip", + }); + const summary = parseRuntimeSummaryArchive(Buffer.from(download.data)); + return summary?.runId === runId ? summary : null; +} + +export async function loadPriorNightlySummaries( + deps: GitHubDeps, +): Promise { + const { github, context, core } = deps; + const response = await github.rest.actions.listWorkflowRuns({ + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: WORKFLOW_FILE, + event: "schedule", + status: "completed", + per_page: HISTORY_RUN_LIMIT + 5, + }); + const runs = (response.data.workflow_runs as Array<{ id: number }>) + .filter((run) => run.id !== context.runId) + .slice(0, HISTORY_RUN_LIMIT); + const summaries: RuntimeSummaryArtifact[] = []; + for (const run of runs) { + try { + const summary = await readRuntimeSummaryFromRun(deps, run.id); + if (summary !== null) summaries.push(summary); + } catch { + core?.warning?.( + "One prior nightly runtime summary was unavailable; continuing with less history.", + ); + } + } + return summaries; +} + +function percentile(sorted: readonly number[], fraction: number): number { + return sorted[Math.max(0, Math.ceil(sorted.length * fraction) - 1)] ?? 0; +} + +function median(sorted: readonly number[]): number { + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 + ? ((sorted[middle - 1] ?? 0) + (sorted[middle] ?? 0)) / 2 + : (sorted[middle] ?? 0); +} + +function seconds(milliseconds: number): string { + return `${(milliseconds / 1000).toFixed(1)}s`; +} + +function escapeCell(value: string): string { + return value.replaceAll("|", "\\|").replaceAll("\n", " "); +} + +function formatDelta(currentMs: number, priorMs: number): string { + const deltaMs = currentMs - priorMs; + const sign = deltaMs >= 0 ? "+" : "-"; + const percent = priorMs > 0 ? ` (${sign}${Math.abs((deltaMs / priorMs) * 100).toFixed(1)}%)` : ""; + return `${sign}${seconds(Math.abs(deltaMs))}${percent}`; +} + +function formatOutcome(row: RuntimeAuditRow): string { + if (row.failedRuns > 0) return "failed"; + if (row.skippedRuns > 0) return "skipped"; + return "passed"; +} + +function formatPassRate(rows: readonly RuntimeAuditRow[]): string { + const passed = rows.reduce((total, row) => total + row.passedRuns, 0); + const runs = rows.reduce((total, row) => total + row.runs, 0); + return runs > 0 ? `${((passed / runs) * 100).toFixed(0)}% (${passed}/${runs})` : "n/a"; +} + +export function formatRuntimeHistory( + currentRows: readonly RuntimeAuditRow[], + priorSummaries: readonly RuntimeSummaryArtifact[], +): string { + const lines = [ + "## E2E Nightly Runtime Trend", + "", + "Current run compared with up to 10 prior completed scheduled runs; manual runs are excluded.", + "", + ]; + if (currentRows.length === 0) { + lines.push("No current runtime rows were available for comparison."); + return `${lines.join("\n")}\n`; + } + if (priorSummaries.length === 0) { + lines.push( + "No prior nightly runtime summaries are available yet; this run starts the history.", + ); + return `${lines.join("\n")}\n`; + } + + lines.push( + "| Target | Scenario | Prior nights | Current median | Prior median | Prior p95 | Delta | Prior pass rate | Current |", + "| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | --- |", + ); + for (const current of currentRows.slice(0, 10)) { + const priorRows = priorSummaries.flatMap((summary) => + summary.rows.filter( + (row) => row.target === current.target && row.scenario === current.scenario, + ), + ); + if (priorRows.length === 0) { + lines.push( + `| ${escapeCell(current.target)} | ${escapeCell(current.scenario)} | 0 | ${seconds(current.medianMs)} | n/a | n/a | n/a | n/a | ${formatOutcome(current)} |`, + ); + continue; + } + const medians = priorRows.map((row) => row.medianMs).sort((a, b) => a - b); + const priorMedian = median(medians); + lines.push( + `| ${escapeCell(current.target)} | ${escapeCell(current.scenario)} | ${priorRows.length} | ${seconds(current.medianMs)} | ${seconds(priorMedian)} | ${seconds(percentile(medians, 0.95))} | ${formatDelta(current.medianMs, priorMedian)} | ${formatPassRate(priorRows)} | ${formatOutcome(current)} |`, + ); + } + return `${lines.join("\n")}\n`; +} + +export async function buildRuntimeHistory( + deps: GitHubDeps, + currentRows: readonly RuntimeAuditRow[], + outputPath: string, + services: RuntimeHistoryServices = { loadPriorNightlySummaries }, + now = new Date(), +): Promise { + const current = createRuntimeSummary(deps.context.runId, now.toISOString(), currentRows); + fs.writeFileSync(outputPath, `${JSON.stringify(current, null, 2)}\n`, { mode: 0o600 }); + try { + const prior = await services.loadPriorNightlySummaries(deps); + return formatRuntimeHistory(currentRows, prior); + } catch { + deps.core?.warning?.( + "Nightly E2E runtime history unavailable; current summary was still saved.", + ); + return formatRuntimeHistory(currentRows, []); + } +} diff --git a/scripts/scorecard/analyze-trace-timing.mts b/scripts/scorecard/analyze-trace-timing.mts index 10eb1acf42d..9f9e55ccb2c 100644 --- a/scripts/scorecard/analyze-trace-timing.mts +++ b/scripts/scorecard/analyze-trace-timing.mts @@ -4,7 +4,8 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import zlib from "node:zlib"; + +import { readValidatedArtifactZipEntry } from "./read-artifact-zip.mts"; type SemverTag = { name: string; major: number; minor: number; patch: number; sha?: string }; type Threshold = { minDeltaMs: number; minPercent: number }; @@ -46,17 +47,6 @@ type TraceTimingResult = { budgetWarningMessage: string | null; budgetStatus: string; }; -type ZipSummaryEntry = { - creatorSystem: number; - flags: number; - compressionMethod: number; - expectedCrc: number; - compressedSize: number; - uncompressedSize: number; - diskStart: number; - externalAttributes: number; - localHeaderOffset: number; -}; type GitHubDeps = { github: any; context: any; core?: { warning?: (message: string) => void } }; type TraceTimingServices = { findLatestCompletedE2eRunForReleaseTag: (deps: GitHubDeps, tag: SemverTag) => Promise; @@ -71,9 +61,6 @@ const MAX_TRACE_SUMMARY_BYTES = 1024 * 1024; const MAX_TRACE_ARCHIVE_ENTRIES = 1000; const TRACE_ARCHIVE_REJECTION_WARNING = "Trace timing artifact ZIP validation failed; ignoring the malformed or unsupported archive."; -const ZIP_CENTRAL_DIRECTORY_SIGNATURE = 0x02014b50; -const ZIP_END_OF_CENTRAL_DIRECTORY_SIGNATURE = 0x06054b50; -const ZIP_LOCAL_FILE_SIGNATURE = 0x04034b50; const ONBOARD_PERFORMANCE_BUDGET_FILE = "ci/onboard-performance-budget.json"; const REPO_ROOT = path.resolve(import.meta.dirname, "..", ".."); const ONBOARD_PHASE_PREFIX = "nemoclaw.onboard.phase."; @@ -531,30 +518,6 @@ async function findLatestCompletedE2eRunForReleaseTag( return null; } -function findZipEndOfCentralDirectory(archive: Buffer): number { - const minimumOffset = Math.max(0, archive.length - 22 - 0xffff); - for (let offset = archive.length - 22; offset >= minimumOffset; offset -= 1) { - if ( - archive.readUInt32LE(offset) === ZIP_END_OF_CENTRAL_DIRECTORY_SIGNATURE && - offset + 22 + archive.readUInt16LE(offset + 20) === archive.length - ) { - return offset; - } - } - return -1; -} - -function crc32(data: Buffer): number { - let crc = 0xffffffff; - for (const byte of data) { - crc ^= byte; - for (let bit = 0; bit < 8; bit += 1) { - crc = (crc >>> 1) ^ (0xedb88320 & -(crc & 1)); - } - } - return (crc ^ 0xffffffff) >>> 0; -} - // GitHub creates the workflow artifact ZIP outside this repository, and the // cloud-onboard artifact intentionally contains diagnostics beside the trusted // timing summary. Parse only the exact root-level summary in-process so the @@ -562,115 +525,10 @@ function crc32(data: Buffer): number { // production-shape multi-entry regression test is the removal guard; retire // this parser if GitHub provides a verified single-file artifact API. function readValidatedTraceSummaryArchive(archive: Buffer): string | null { - const endOffset = findZipEndOfCentralDirectory(archive); - if (endOffset < 0) return null; - - const diskNumber = archive.readUInt16LE(endOffset + 4); - const centralDirectoryDisk = archive.readUInt16LE(endOffset + 6); - const entriesOnDisk = archive.readUInt16LE(endOffset + 8); - const totalEntries = archive.readUInt16LE(endOffset + 10); - const centralDirectorySize = archive.readUInt32LE(endOffset + 12); - const centralDirectoryOffset = archive.readUInt32LE(endOffset + 16); - if ( - diskNumber !== 0 || - centralDirectoryDisk !== 0 || - entriesOnDisk !== totalEntries || - totalEntries < 1 || - totalEntries > MAX_TRACE_ARCHIVE_ENTRIES || - centralDirectoryOffset + centralDirectorySize !== endOffset - ) { - return null; - } - - const expectedFileName = Buffer.from(TRACE_SUMMARY_FILE, "utf8"); - let centralEntryOffset = centralDirectoryOffset; - let target: ZipSummaryEntry | null = null; - for (let entryIndex = 0; entryIndex < totalEntries; entryIndex += 1) { - if ( - centralEntryOffset + 46 > endOffset || - archive.readUInt32LE(centralEntryOffset) !== ZIP_CENTRAL_DIRECTORY_SIGNATURE - ) { - return null; - } - const fileNameLength = archive.readUInt16LE(centralEntryOffset + 28); - const extraLength = archive.readUInt16LE(centralEntryOffset + 30); - const commentLength = archive.readUInt16LE(centralEntryOffset + 32); - const centralEntryEnd = centralEntryOffset + 46 + fileNameLength + extraLength + commentLength; - if (centralEntryEnd > endOffset) return null; - const fileName = archive.subarray( - centralEntryOffset + 46, - centralEntryOffset + 46 + fileNameLength, - ); - if (fileName.equals(expectedFileName)) { - if (target !== null) return null; - target = { - creatorSystem: archive.readUInt8(centralEntryOffset + 5), - flags: archive.readUInt16LE(centralEntryOffset + 8), - compressionMethod: archive.readUInt16LE(centralEntryOffset + 10), - expectedCrc: archive.readUInt32LE(centralEntryOffset + 16), - compressedSize: archive.readUInt32LE(centralEntryOffset + 20), - uncompressedSize: archive.readUInt32LE(centralEntryOffset + 24), - diskStart: archive.readUInt16LE(centralEntryOffset + 34), - externalAttributes: archive.readUInt32LE(centralEntryOffset + 38), - localHeaderOffset: archive.readUInt32LE(centralEntryOffset + 42), - }; - } - centralEntryOffset = centralEntryEnd; - } - if (centralEntryOffset !== endOffset || target === null) return null; - - const { - creatorSystem, - flags, - compressionMethod, - expectedCrc, - compressedSize, - uncompressedSize, - diskStart, - externalAttributes, - localHeaderOffset, - } = target; - const unixFileType = (externalAttributes >>> 16) & 0xf000; - if ( - diskStart !== 0 || - (flags & 0x1) !== 0 || - (compressionMethod !== 0 && compressionMethod !== 8) || - compressedSize > MAX_TRACE_SUMMARY_BYTES || - uncompressedSize > MAX_TRACE_SUMMARY_BYTES || - (creatorSystem !== 0 && creatorSystem !== 3) || - (creatorSystem === 3 && unixFileType !== 0 && unixFileType !== 0x8000) || - localHeaderOffset + 30 > centralDirectoryOffset || - archive.readUInt32LE(localHeaderOffset) !== ZIP_LOCAL_FILE_SIGNATURE - ) { - return null; - } - - const localFlags = archive.readUInt16LE(localHeaderOffset + 6); - const localCompressionMethod = archive.readUInt16LE(localHeaderOffset + 8); - const localFileNameLength = archive.readUInt16LE(localHeaderOffset + 26); - const localExtraLength = archive.readUInt16LE(localHeaderOffset + 28); - const localFileName = archive.subarray( - localHeaderOffset + 30, - localHeaderOffset + 30 + localFileNameLength, - ); - const compressedDataOffset = localHeaderOffset + 30 + localFileNameLength + localExtraLength; - const compressedDataEnd = compressedDataOffset + compressedSize; - if ( - localFlags !== flags || - localCompressionMethod !== compressionMethod || - !localFileName.equals(expectedFileName) || - compressedDataEnd > centralDirectoryOffset - ) { - return null; - } - - const compressedData = archive.subarray(compressedDataOffset, compressedDataEnd); - const summary = - compressionMethod === 0 - ? Buffer.from(compressedData) - : zlib.inflateRawSync(compressedData, { maxOutputLength: MAX_TRACE_SUMMARY_BYTES }); - if (summary.length !== uncompressedSize || crc32(summary) !== expectedCrc) return null; - return summary.toString("utf8"); + return readValidatedArtifactZipEntry(archive, TRACE_SUMMARY_FILE, { + maxBytes: MAX_TRACE_SUMMARY_BYTES, + maxEntries: MAX_TRACE_ARCHIVE_ENTRIES, + }); } function readValidatedTraceSummaryZip( diff --git a/scripts/scorecard/coordinate-scorecard.mts b/scripts/scorecard/coordinate-scorecard.mts index b78b63f7d3f..dea1a4895fe 100644 --- a/scripts/scorecard/coordinate-scorecard.mts +++ b/scripts/scorecard/coordinate-scorecard.mts @@ -113,6 +113,22 @@ function renderSummaryLines(input: { lines.push(job.url ? ` - [${job.name}](${job.url})` : ` - \`${job.name}\``); } } + if (summary.timingRows.length > 0) { + lines.push( + "", + "### Slowest job timing", + "", + "| Job | Runner class | Queue | Execution |", + "| --- | --- | ---: | ---: |", + ); + for (const row of summary.timingRows) { + const duration = (milliseconds: number | null) => + milliseconds === null ? "n/a" : `${(milliseconds / 1_000).toFixed(1)}s`; + lines.push( + `| ${row.name.replaceAll("|", "\\|")} | ${row.runnerClass} | ${duration(row.queueMs)} | ${duration(row.executionMs)} |`, + ); + } + } if (input.perfect) lines.push("", "🎉 **All jobs passed!**"); lines.push( "", diff --git a/scripts/scorecard/read-artifact-zip.mts b/scripts/scorecard/read-artifact-zip.mts new file mode 100644 index 00000000000..5bb3ba0c09d --- /dev/null +++ b/scripts/scorecard/read-artifact-zip.mts @@ -0,0 +1,180 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import zlib from "node:zlib"; + +type ZipEntry = { + creatorSystem: number; + flags: number; + compressionMethod: number; + expectedCrc: number; + compressedSize: number; + uncompressedSize: number; + diskStart: number; + externalAttributes: number; + localHeaderOffset: number; +}; + +const ZIP_CENTRAL_DIRECTORY_SIGNATURE = 0x02014b50; +const ZIP_END_OF_CENTRAL_DIRECTORY_SIGNATURE = 0x06054b50; +const ZIP_LOCAL_FILE_SIGNATURE = 0x04034b50; + +function findZipEndOfCentralDirectory(archive: Buffer): number { + const minimumOffset = Math.max(0, archive.length - 22 - 0xffff); + for (let offset = archive.length - 22; offset >= minimumOffset; offset -= 1) { + if ( + archive.readUInt32LE(offset) === ZIP_END_OF_CENTRAL_DIRECTORY_SIGNATURE && + offset + 22 + archive.readUInt16LE(offset + 20) === archive.length + ) { + return offset; + } + } + return -1; +} + +function crc32(data: Buffer): number { + let crc = 0xffffffff; + for (const byte of data) { + crc ^= byte; + for (let bit = 0; bit < 8; bit += 1) { + crc = (crc >>> 1) ^ (0xedb88320 & -(crc & 1)); + } + } + return (crc ^ 0xffffffff) >>> 0; +} + +/** + * Reads one exact root-level file from a GitHub artifact ZIP without extracting + * paths to disk. The archive may contain other entries, but duplicate target + * entries, links, encryption, split archives, ZIP64, and oversized payloads are + * rejected. + */ +export function readValidatedArtifactZipEntry( + archive: Buffer, + expectedFile: string, + options: { maxBytes: number; maxEntries?: number }, +): string | null { + const maxEntries = options.maxEntries ?? 1000; + const expectedFileName = Buffer.from(expectedFile, "utf8"); + if ( + expectedFile.length === 0 || + expectedFile.includes("/") || + expectedFile.includes("\\") || + options.maxBytes < 1 || + maxEntries < 1 + ) { + return null; + } + + const endOffset = findZipEndOfCentralDirectory(archive); + if (endOffset < 0) return null; + + const diskNumber = archive.readUInt16LE(endOffset + 4); + const centralDirectoryDisk = archive.readUInt16LE(endOffset + 6); + const entriesOnDisk = archive.readUInt16LE(endOffset + 8); + const totalEntries = archive.readUInt16LE(endOffset + 10); + const centralDirectorySize = archive.readUInt32LE(endOffset + 12); + const centralDirectoryOffset = archive.readUInt32LE(endOffset + 16); + if ( + diskNumber !== 0 || + centralDirectoryDisk !== 0 || + entriesOnDisk !== totalEntries || + totalEntries < 1 || + totalEntries > maxEntries || + centralDirectoryOffset + centralDirectorySize !== endOffset + ) { + return null; + } + + let centralEntryOffset = centralDirectoryOffset; + let target: ZipEntry | null = null; + for (let entryIndex = 0; entryIndex < totalEntries; entryIndex += 1) { + if ( + centralEntryOffset + 46 > endOffset || + archive.readUInt32LE(centralEntryOffset) !== ZIP_CENTRAL_DIRECTORY_SIGNATURE + ) { + return null; + } + const fileNameLength = archive.readUInt16LE(centralEntryOffset + 28); + const extraLength = archive.readUInt16LE(centralEntryOffset + 30); + const commentLength = archive.readUInt16LE(centralEntryOffset + 32); + const centralEntryEnd = centralEntryOffset + 46 + fileNameLength + extraLength + commentLength; + if (centralEntryEnd > endOffset) return null; + const fileName = archive.subarray( + centralEntryOffset + 46, + centralEntryOffset + 46 + fileNameLength, + ); + if (fileName.equals(expectedFileName)) { + if (target !== null) return null; + target = { + creatorSystem: archive.readUInt8(centralEntryOffset + 5), + flags: archive.readUInt16LE(centralEntryOffset + 8), + compressionMethod: archive.readUInt16LE(centralEntryOffset + 10), + expectedCrc: archive.readUInt32LE(centralEntryOffset + 16), + compressedSize: archive.readUInt32LE(centralEntryOffset + 20), + uncompressedSize: archive.readUInt32LE(centralEntryOffset + 24), + diskStart: archive.readUInt16LE(centralEntryOffset + 34), + externalAttributes: archive.readUInt32LE(centralEntryOffset + 38), + localHeaderOffset: archive.readUInt32LE(centralEntryOffset + 42), + }; + } + centralEntryOffset = centralEntryEnd; + } + if (centralEntryOffset !== endOffset || target === null) return null; + + const { + creatorSystem, + flags, + compressionMethod, + expectedCrc, + compressedSize, + uncompressedSize, + diskStart, + externalAttributes, + localHeaderOffset, + } = target; + const unixFileType = (externalAttributes >>> 16) & 0xf000; + if ( + diskStart !== 0 || + (flags & 0x1) !== 0 || + (compressionMethod !== 0 && compressionMethod !== 8) || + compressedSize > options.maxBytes || + uncompressedSize > options.maxBytes || + (creatorSystem !== 0 && creatorSystem !== 3) || + (creatorSystem === 3 && unixFileType !== 0 && unixFileType !== 0x8000) || + localHeaderOffset + 30 > centralDirectoryOffset || + archive.readUInt32LE(localHeaderOffset) !== ZIP_LOCAL_FILE_SIGNATURE + ) { + return null; + } + + const localFlags = archive.readUInt16LE(localHeaderOffset + 6); + const localCompressionMethod = archive.readUInt16LE(localHeaderOffset + 8); + const localFileNameLength = archive.readUInt16LE(localHeaderOffset + 26); + const localExtraLength = archive.readUInt16LE(localHeaderOffset + 28); + const localFileNameEnd = localHeaderOffset + 30 + localFileNameLength; + const compressedDataOffset = localFileNameEnd + localExtraLength; + const compressedDataEnd = compressedDataOffset + compressedSize; + if ( + localFileNameEnd > centralDirectoryOffset || + localFlags !== flags || + localCompressionMethod !== compressionMethod || + !archive.subarray(localHeaderOffset + 30, localFileNameEnd).equals(expectedFileName) || + compressedDataEnd > centralDirectoryOffset + ) { + return null; + } + + const compressedData = archive.subarray(compressedDataOffset, compressedDataEnd); + let contents: Buffer; + try { + contents = + compressionMethod === 0 + ? Buffer.from(compressedData) + : zlib.inflateRawSync(compressedData, { maxOutputLength: options.maxBytes }); + } catch { + return null; + } + if (contents.length !== uncompressedSize || crc32(contents) !== expectedCrc) return null; + return contents.toString("utf8"); +} diff --git a/scripts/scorecard/summarize-jobs.mts b/scripts/scorecard/summarize-jobs.mts index e8607e047e9..aa01003f63c 100644 --- a/scripts/scorecard/summarize-jobs.mts +++ b/scripts/scorecard/summarize-jobs.mts @@ -4,9 +4,12 @@ type ApiJob = { completed_at?: string | null; conclusion?: string | null; + created_at?: string | null; html_url?: string | null; + labels?: string[] | null; name: string; run_attempt?: number | null; + started_at?: string | null; status?: string | null; }; @@ -14,6 +17,13 @@ type NeedResult = { result?: string }; type FailedJob = { name: string; url: string | null }; +export type JobTimingRow = { + executionMs: number | null; + name: string; + queueMs: number | null; + runnerClass: "custom" | "standard" | "unknown"; +}; + export type JobSummary = { cancelled: number; failedJobs: FailedJob[]; @@ -21,6 +31,7 @@ export type JobSummary = { ran: number; skipped: number; success: number; + timingRows: JobTimingRow[]; total: number; }; @@ -66,7 +77,9 @@ function classifyNeed(value: NeedResult): CountedResult { return "failure"; } -function countResults(results: CountedResult[]): Omit { +function countResults( + results: CountedResult[], +): Omit { return { cancelled: results.filter((result) => result === "cancelled").length, failure: results.filter((result) => result === "failure").length, @@ -102,6 +115,43 @@ function normalizeApiJobs( return [...dedupedByName.values()].sort((left, right) => left.name.localeCompare(right.name)); } +function elapsedMs( + start: string | null | undefined, + finish: string | null | undefined, +): number | null { + if (!start || !finish) return null; + const startMs = Date.parse(start); + const finishMs = Date.parse(finish); + return Number.isFinite(startMs) && Number.isFinite(finishMs) && finishMs >= startMs + ? finishMs - startMs + : null; +} + +function runnerClass(labels: string[] | null | undefined): JobTimingRow["runnerClass"] { + if (!labels || labels.length === 0) return "unknown"; + return labels.includes("ubuntu-latest") ? "standard" : "custom"; +} + +function summarizeJobTimings(jobs: ApiJob[]): JobTimingRow[] { + return jobs + .map( + (job): JobTimingRow => ({ + executionMs: elapsedMs(job.started_at, job.completed_at), + name: job.name, + queueMs: elapsedMs(job.created_at, job.started_at), + runnerClass: runnerClass(job.labels), + }), + ) + .filter((row) => row.executionMs !== null || row.queueMs !== null) + .sort( + (left, right) => + (right.executionMs ?? 0) + + (right.queueMs ?? 0) - + ((left.executionMs ?? 0) + (left.queueMs ?? 0)) || left.name.localeCompare(right.name), + ) + .slice(0, 10); +} + async function loadWorkflowRunJobs({ context, core, @@ -142,6 +192,7 @@ function summarizeJobs(input: SummarizeJobsInput): JobSummary { .filter(({ result }) => result === "failure") .map(({ job }) => ({ name: job.name, url: job.html_url ?? null })), ran: jobs.length - counts.skipped, + timingRows: summarizeJobTimings(jobs), total: jobs.length, }; } @@ -158,6 +209,7 @@ function summarizeJobs(input: SummarizeJobsInput): JobSummary { .filter(({ result }) => result === "failure") .map(({ name }) => ({ name, url: null })), ran: entries.length - counts.skipped, + timingRows: [], total: entries.length, }; } diff --git a/test/e2e/README.md b/test/e2e/README.md index 97210e223a1..e68cf3d95c9 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -51,6 +51,44 @@ discovery command locally to inspect the generated test matrix: npx tsx tools/e2e/credential-free-tests.mts ``` +## Heavy-runner comparison evidence + +Issue #7145 requires before-and-after evidence before moving heavy lanes to a +larger runner. The initial measurement set is limited to: + +- `common-egress-agent`; +- `rebuild-hermes`; +- `rebuild-hermes-stale-base`; +- the `hermes` and `deepagents` shards of `mcp-bridge`. + +The `openclaw` MCP shard, `mcp-bridge-dev`, and unrelated jobs do not emit this +comparison artifact. This change records the standard-runner baseline only; a +separate reviewed routing change is required before any lane uses a larger +runner. + +Each eligible job has a best-effort initialization step for +`runner-resource-snapshots.jsonl` immediately after the shared workspace +preparation step. When initialization succeeds, the initial sample, automatic +live-test heartbeats and phase changes, and an `always()`-conditioned +best-effort final sample share that one private ledger. This keeps both +serialized Deep Agents Vitest invocations in one job-level measurement. A +best-effort finalizer writes `runner-resource-summary.json` before artifact +scanning and upload. The summary +contains the measured window, sample and valid CPU-interval counts, logical CPU +capacity, peak sampled host CPU percentage, the highest sampled cgroup +`memory.peak` counter (or a sampled `MemAvailable` fallback), memory capacity, +peak workspace growth, minimum free workspace bytes, and workspace capacity. +The cgroup peak can include activity before the ledger starts because the +kernel counter is cumulative for that cgroup. Resource fields are numeric; the +remaining strings are canonical timestamps and fixed lane or source labels. + +A comparison needs at least two snapshots to form a measured window, and CPU +requires adjacent samples at least one second apart. Treat these values as +sampled evidence rather than exact instantaneous peaks. Queue time, execution +time, and outcome come from the GitHub Actions job metadata in the scorecard. +Retain at least five standard-runner samples per eligible lane before reviewing +a larger-runner routing proposal. + ## Scheduled operations The consolidated workflow keeps its operational reporting in the same job @@ -63,9 +101,13 @@ graph as the live targets: exceptional threshold, such as the same lane failing twice consecutively or remaining broken for 24 hours, rather than posting on every failed schedule. - `scorecard` writes the scheduled/manual result summary, adds this run's - semantic phase runtime table, compares the trusted cloud-onboard timing - summary with the latest prior-release `e2e.yaml` run, and posts to the daily - or full-run Slack route. + semantic phase runtime table, separates job queue time from execution time, + compares the current test medians with up to ten prior scheduled summaries, + compares the trusted cloud-onboard timing summary with the latest + prior-release `e2e.yaml` run, and posts to the daily or full-run Slack route. +- The rolling comparison uses only the bounded `e2e-runtime-summary.json` + artifact retained for 14 days. It does not download historical raw test + artifacts, include manual runs in the baseline, or expose runner names. - Selective dispatches remain silent unless they run on `main` with `post_to_slack=true`, which uses the preview Slack route. Branch-dispatched runs never receive Slack webhook secrets. @@ -123,7 +165,9 @@ npm run test:runtime-audit -- path/to/run-1 path/to/run-2 The audit groups each test by target and optional shard, ranks the groups by p95 runtime, and reports variability plus the slowest observed phase's duration and outcome. Scheduled and ordinary manual runs include the same table for that -run in the GitHub Actions scorecard summary. Keep phase +run in the GitHub Actions scorecard summary. The scorecard also shows a bounded +nightly trend table with prior median, prior p95, delta, and pass rate; the +first run explicitly starts the history instead of inventing a baseline. Keep phase labels specific to test behavior, call `progress.phase("literal phase label")` at the declared boundaries in order, and transition through the final test-declared phase on every passing path. The fixture rejects a passing live diff --git a/test/e2e/fixtures/e2e-test.ts b/test/e2e/fixtures/e2e-test.ts index 8705d59d7aa..72fa0aad20f 100644 --- a/test/e2e/fixtures/e2e-test.ts +++ b/test/e2e/fixtures/e2e-test.ts @@ -8,6 +8,7 @@ import { test as base, expect } from "vitest"; import { appendResourcePhaseBaseline, + appendResourceSnapshot, collectResourceSnapshot, } from "../../../tools/e2e/runner-pressure.mts"; import { renderSnapshotLine } from "../../../tools/e2e/runner-pressure-core.mts"; @@ -113,6 +114,7 @@ export const test = base.extend({ async ({ artifacts, onTestFinished, secrets, task }, use) => { const targetId = process.env.E2E_TARGET_ID; const baselinePath = process.env.E2E_RESOURCE_PHASE_BASELINES_FILE; + const snapshotsPath = process.env.E2E_RESOURCE_SNAPSHOTS_FILE; const phasePlan = task.meta.e2ePhases; assert.ok( task.file.projectName !== "e2e-live" || phasePlan, @@ -135,12 +137,41 @@ export const test = base.extend({ : () => { // Keep fixture and support tests quiet; live runs need phase progress. }, - ...(targetId && baselinePath + ...(targetId && (baselinePath || snapshotsPath) ? { - sampleResourceEvidence: (phase: string) => - renderSnapshotLine(collectResourceSnapshot(resourcePhaseLabel(targetId, phase))), - recordResourceBaseline: (phase: string) => - appendResourcePhaseBaseline(baselinePath, resourcePhaseLabel(targetId, phase)), + sampleResourceEvidence: (phase: string) => { + const snapshot = collectResourceSnapshot(resourcePhaseLabel(targetId, phase), { + includeDiagnostics: Boolean(baselinePath), + }); + if (snapshotsPath) { + try { + appendResourceSnapshot(snapshotsPath, snapshot); + } catch { + // Comparison telemetry must not change the live test result. + } + } + return renderSnapshotLine(snapshot); + }, + ...(snapshotsPath + ? { + recordResourceSnapshot: (phase: string) => + appendResourceSnapshot( + snapshotsPath, + collectResourceSnapshot(resourcePhaseLabel(targetId, phase), { + includeDiagnostics: false, + }), + ), + } + : {}), + ...(baselinePath + ? { + recordResourceBaseline: (phase: string) => + appendResourcePhaseBaseline( + baselinePath, + resourcePhaseLabel(targetId, phase), + ), + } + : {}), } : {}), }); diff --git a/test/e2e/fixtures/progress.ts b/test/e2e/fixtures/progress.ts index a61af119fbc..ce336889fb3 100644 --- a/test/e2e/fixtures/progress.ts +++ b/test/e2e/fixtures/progress.ts @@ -50,6 +50,7 @@ export interface TestProgressOptions { sampleResources?: () => ResourceSnapshot; sampleResourceEvidence?: (phase: string) => string; recordResourceBaseline?: (phase: string) => void; + recordResourceSnapshot?: (phase: string) => void; terminalPhase?: string; taskStatus?: () => { errorCount: number; outcome?: ProgressPhaseOutcome }; } @@ -163,6 +164,7 @@ export function startTestProgress( const sampleResources = options.sampleResources ?? defaultResourceSnapshot; const sampleResourceEvidence = options.sampleResourceEvidence; const recordResourceBaseline = options.recordResourceBaseline; + const recordResourceSnapshot = options.recordResourceSnapshot; const taskStatus = options.taskStatus; const stallThresholdMs = options.stallThresholdMs ?? DEFAULT_STALL_THRESHOLD_MS; const stallReminderIntervalMs = @@ -207,6 +209,14 @@ export function startTestProgress( } }; + const recordSnapshotBestEffort = () => { + try { + recordResourceSnapshot?.(currentPhase()); + } catch { + // Diagnostics must not change the live test result. + } + }; + const logTransitionBestEffort = () => { try { logLine(`${phasePrefix()} ${currentPhase()}`); @@ -371,10 +381,12 @@ export function startTestProgress( }, ); recordBaselineBestEffort(); + recordSnapshotBestEffort(); resetStallTimer(); }; recordBaselineBestEffort(); + recordSnapshotBestEffort(); logTransitionBestEffort(); scheduleStall(stallThresholdMs); diff --git a/test/e2e/support/artifact-zip.test.ts b/test/e2e/support/artifact-zip.test.ts new file mode 100644 index 00000000000..3dbbd30e7c8 --- /dev/null +++ b/test/e2e/support/artifact-zip.test.ts @@ -0,0 +1,121 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import zlib from "node:zlib"; + +import { describe, expect, it } from "vitest"; + +import { readValidatedArtifactZipEntry } from "../../../scripts/scorecard/read-artifact-zip.mts"; + +function crc32(data: Buffer): number { + let crc = 0xffffffff; + for (const byte of data) { + crc ^= byte; + for (let bit = 0; bit < 8; bit += 1) { + crc = (crc >>> 1) ^ (0xedb88320 & -(crc & 1)); + } + } + return (crc ^ 0xffffffff) >>> 0; +} + +function artifactZip( + entries: Array<{ name: string; contents: string }>, + compressionMethod = 0, +): Buffer { + const localParts: Buffer[] = []; + const centralParts: Buffer[] = []; + let localOffset = 0; + for (const entry of entries) { + const name = Buffer.from(entry.name, "utf8"); + const contents = Buffer.from(entry.contents, "utf8"); + const compressed = + compressionMethod === 8 ? zlib.deflateRawSync(contents) : Buffer.from(contents); + const checksum = crc32(contents); + const local = Buffer.alloc(30); + local.writeUInt32LE(0x04034b50, 0); + local.writeUInt16LE(20, 4); + local.writeUInt16LE(compressionMethod, 8); + local.writeUInt32LE(checksum, 14); + local.writeUInt32LE(compressed.length, 18); + local.writeUInt32LE(contents.length, 22); + local.writeUInt16LE(name.length, 26); + localParts.push(local, name, compressed); + + const central = Buffer.alloc(46); + central.writeUInt32LE(0x02014b50, 0); + central.writeUInt16LE(0x0314, 4); + central.writeUInt16LE(20, 6); + central.writeUInt16LE(compressionMethod, 10); + central.writeUInt32LE(checksum, 16); + central.writeUInt32LE(compressed.length, 20); + central.writeUInt32LE(contents.length, 24); + central.writeUInt16LE(name.length, 28); + central.writeUInt32LE(0x80000000, 38); + central.writeUInt32LE(localOffset, 42); + centralParts.push(central, name); + localOffset += local.length + name.length + compressed.length; + } + const locals = Buffer.concat(localParts); + const centralDirectory = Buffer.concat(centralParts); + const end = Buffer.alloc(22); + end.writeUInt32LE(0x06054b50, 0); + end.writeUInt16LE(entries.length, 8); + end.writeUInt16LE(entries.length, 10); + end.writeUInt32LE(centralDirectory.length, 12); + end.writeUInt32LE(locals.length, 16); + return Buffer.concat([locals, centralDirectory, end]); +} + +describe("validated GitHub artifact ZIP reader", () => { + it("reads only the exact root-level entry from a multi-entry archive", () => { + const archive = artifactZip([ + { name: "diagnostics/log.txt", contents: "ignored" }, + { name: "summary.json", contents: '{"safe":true}' }, + ]); + + expect(readValidatedArtifactZipEntry(archive, "summary.json", { maxBytes: 1_024 })).toBe( + '{"safe":true}', + ); + expect(readValidatedArtifactZipEntry(archive, "log.txt", { maxBytes: 1_024 })).toBeNull(); + }); + + it("rejects duplicate target entries and payloads over the caller's bound", () => { + expect( + readValidatedArtifactZipEntry( + artifactZip([ + { name: "summary.json", contents: "one" }, + { name: "summary.json", contents: "two" }, + ]), + "summary.json", + { maxBytes: 1_024 }, + ), + ).toBeNull(); + expect( + readValidatedArtifactZipEntry( + artifactZip([{ name: "summary.json", contents: "too large" }]), + "summary.json", + { maxBytes: 2 }, + ), + ).toBeNull(); + }); + + it("reads deflated entries and rejects corrupt compressed data", () => { + const archive = artifactZip( + [{ name: "summary.json", contents: '{"compressed":true}' }], + 8, + ); + + expect(readValidatedArtifactZipEntry(archive, "summary.json", { maxBytes: 1_024 })).toBe( + '{"compressed":true}', + ); + + const corruptArchive = Buffer.from(archive); + const compressedDataOffset = + 30 + corruptArchive.readUInt16LE(26) + corruptArchive.readUInt16LE(28); + const compressedDataEnd = compressedDataOffset + corruptArchive.readUInt32LE(18); + corruptArchive.fill(0, compressedDataOffset, compressedDataEnd); + expect( + readValidatedArtifactZipEntry(corruptArchive, "summary.json", { maxBytes: 1_024 }), + ).toBeNull(); + }); +}); diff --git a/test/e2e/support/e2e-operations-workflow-boundary.test.ts b/test/e2e/support/e2e-operations-workflow-boundary.test.ts index 4f2d77d8b15..f0177f87b4f 100644 --- a/test/e2e/support/e2e-operations-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-operations-workflow-boundary.test.ts @@ -88,6 +88,18 @@ describe("E2E operations workflow boundary", () => { ); }); + it("pins the bounded rolling runtime summary artifact", () => { + const workflow = readE2eOperationsWorkflow(); + const upload = workflow.jobs.scorecard.steps!.find( + (step) => step.name === "Upload E2E runtime summary", + )!; + upload.with!.path = "${{ runner.temp }}"; + + expect(validateE2eOperationsWorkflow(workflow)).toContain( + "scorecard must upload only the bounded runtime summary for rolling history", + ); + }); + it("rejects controller protocol and PR validation drift", () => { const workflow = readE2eOperationsWorkflow(); delete workflow.on?.workflow_dispatch?.inputs?.base_sha; @@ -405,11 +417,15 @@ describe("E2E operations workflow boundary", () => { .fn() .mockReturnValue("## E2E Test Phase Runtime\n\n| Target | Slowest observed phase |"), }; + const runtimeHistory = { + buildRuntimeHistory: vi.fn().mockResolvedValue("## E2E Nightly Runtime Trend\n"), + }; const runtimeModules = new Map([ ["path", { join: (...parts: string[]) => parts.join("/") }], ["/workspace/scripts/audit-test-runtime.mts", runtimeAudit], ["/workspace/scripts/scorecard/coordinate-scorecard.mts", coordinator], ["/workspace/scripts/scorecard/analyze-trace-timing.mts", traceTiming], + ["/workspace/scripts/scorecard/analyze-runtime-history.mts", runtimeHistory], ["/workspace/scripts/scorecard/summarize-jobs.mts", scorecardJobs], ]); const runtimeRequire = (specifier: string) => { @@ -423,6 +439,7 @@ describe("E2E operations workflow boundary", () => { GITHUB_WORKSPACE: "/workspace", JOBS: "", RUNTIME_ARTIFACTS: "/runner/e2e-runtime-audit", + RUNTIME_SUMMARY_FILE: "/runner/e2e-runtime-summary.json", TARGETS: "", }, }; @@ -445,6 +462,11 @@ describe("E2E operations workflow boundary", () => { expect(traceTiming.buildTraceTimingResult).toHaveBeenCalledWith({ github: {}, context, core }); expect(runtimeAudit.auditTestRuntime).toHaveBeenCalledWith(["/runner/e2e-runtime-audit"]); + expect(runtimeHistory.buildRuntimeHistory).toHaveBeenCalledWith( + { github: {}, context, core }, + [{ target: "full-e2e" }], + "/runner/e2e-runtime-summary.json", + ); expect(runtimeAudit.auditTestRuntime.mock.invocationCallOrder[0]).toBeLessThan( traceTiming.buildTraceTimingResult.mock.invocationCallOrder[0], ); @@ -488,9 +510,13 @@ describe("E2E operations workflow boundary", () => { }), formatRuntimeAuditSummary: vi.fn(), }; + const runtimeHistory = { + buildRuntimeHistory: vi.fn().mockResolvedValue("## E2E Nightly Runtime Trend\n"), + }; const runtimeModules = new Map([ ["path", { join: (...parts: string[]) => parts.join("/") }], ["/workspace/scripts/audit-test-runtime.mts", runtimeAudit], + ["/workspace/scripts/scorecard/analyze-runtime-history.mts", runtimeHistory], [ "/workspace/scripts/scorecard/coordinate-scorecard.mts", { @@ -527,6 +553,7 @@ describe("E2E operations workflow boundary", () => { GITHUB_WORKSPACE: "/workspace", JOBS: "", RUNTIME_ARTIFACTS: "/runner/e2e-runtime-audit", + RUNTIME_SUMMARY_FILE: "/runner/e2e-runtime-summary.json", TARGETS: "", }, }; diff --git a/test/e2e/support/e2e-progress-fixture.test.ts b/test/e2e/support/e2e-progress-fixture.test.ts index 6eac7492cf9..7d7da9dae3d 100644 --- a/test/e2e/support/e2e-progress-fixture.test.ts +++ b/test/e2e/support/e2e-progress-fixture.test.ts @@ -6,14 +6,25 @@ import os from "node:os"; import path from "node:path"; import { afterAll, expect, it, vi } from "vitest"; -import { assertPhaseLabel } from "../../../tools/e2e/runner-pressure-core.mts"; +import { assertPhaseLabel, parseSnapshotLine } from "../../../tools/e2e/runner-pressure-core.mts"; import { E2E_TEARDOWN_PHASE, resourcePhaseLabel, test } from "../fixtures/e2e-test.ts"; import type { ProgressSummary } from "../fixtures/progress.ts"; const artifactRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-progress-fixture-")); +const resourceSnapshots = path.join(artifactRoot, "runner-resource-snapshots.jsonl"); let progressArtifact = ""; -vi.stubEnv("E2E_ARTIFACT_DIR", artifactRoot); +const fixtureEnvironment = { + E2E_ARTIFACT_DIR: artifactRoot, + E2E_TARGET_ID: "fixture-progress-target", + NEMOCLAW_E2E_SHARD: "fixture-progress-shard", + E2E_RESOURCE_SNAPSHOTS_FILE: resourceSnapshots, +} as const; +const previousEnvironment = Object.fromEntries( + Object.keys(fixtureEnvironment).map((key) => [key, process.env[key]]), +) as Record; +Object.assign(process.env, fixtureEnvironment); +fs.writeFileSync(resourceSnapshots, "", { mode: 0o600 }); afterAll(() => { try { @@ -35,8 +46,22 @@ afterAll(() => { label: E2E_TEARDOWN_PHASE, outcome: "passed", }); + const snapshots = fs + .readFileSync(resourceSnapshots, "utf8") + .trim() + .split("\n") + .map(parseSnapshotLine); + expect(snapshots.map((snapshot) => snapshot.phase)).toEqual([ + "fixture-progress-target.prepare-progress-artifact", + "fixture-progress-target.record-final-fixture-phase", + "fixture-progress-target.release-registered-e2e-resources", + ]); } finally { - vi.unstubAllEnvs(); + for (const [key, value] of Object.entries(previousEnvironment)) { + value === undefined + ? Reflect.deleteProperty(process.env, key) + : Reflect.set(process.env, key, value); + } fs.rmSync(artifactRoot, { force: true, recursive: true }); } }); diff --git a/test/e2e/support/e2e-runtime-history.test.ts b/test/e2e/support/e2e-runtime-history.test.ts new file mode 100644 index 00000000000..d680e7f2414 --- /dev/null +++ b/test/e2e/support/e2e-runtime-history.test.ts @@ -0,0 +1,118 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it, vi } from "vitest"; + +import type { RuntimeAuditRow } from "../../../scripts/audit-test-runtime.mts"; +import { + buildRuntimeHistory, + createRuntimeSummary, + formatRuntimeHistory, + loadPriorNightlySummaries, + normalizeRuntimeSummary, +} from "../../../scripts/scorecard/analyze-runtime-history.mts"; + +function runtimeRow(overrides: Partial = {}): RuntimeAuditRow { + const medianMs = overrides.medianMs ?? 120_000; + const p95Ms = overrides.p95Ms ?? medianMs; + const maxMs = overrides.maxMs ?? p95Ms; + return { + target: "rebuild-hermes", + scenario: "rebuild Hermes from source", + runs: 1, + medianMs, + p95Ms, + maxMs, + variabilityMs: overrides.variabilityMs ?? p95Ms - medianMs, + passedRuns: 1, + failedRuns: 0, + skippedRuns: 0, + slowestPhase: "build Hermes image", + slowestPhaseMs: 90_000, + slowestPhaseOutcome: "passed", + ...overrides, + }; +} + +describe("E2E rolling runtime history", () => { + it("compares current semantic-test timing with prior scheduled summaries", () => { + const current = runtimeRow({ medianMs: 150_000, p95Ms: 150_000 }); + const prior = [ + createRuntimeSummary(1, "2026-07-20T00:00:00.000Z", [runtimeRow({ medianMs: 100_000 })]), + createRuntimeSummary(2, "2026-07-21T00:00:00.000Z", [ + runtimeRow({ medianMs: 120_000, passedRuns: 0, failedRuns: 1 }), + ]), + ]; + + const markdown = formatRuntimeHistory([current], prior); + + expect(markdown).toContain("| rebuild-hermes | rebuild Hermes from source | 2 |"); + expect(markdown).toContain("150.0s | 110.0s | 120.0s | +40.0s (+36.4%)"); + expect(markdown).toContain("50% (1/2) | passed |"); + }); + + it("writes the current bounded summary even when history is unavailable", async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-runtime-history-")); + const output = path.join(directory, "e2e-runtime-summary.json"); + const warning = vi.fn(); + try { + const markdown = await buildRuntimeHistory( + { + github: {}, + context: { repo: { owner: "NVIDIA", repo: "NemoClaw" }, runId: 123 }, + core: { warning }, + }, + [runtimeRow()], + output, + { loadPriorNightlySummaries: vi.fn().mockRejectedValue(new Error("unavailable")) }, + new Date("2026-07-22T00:00:00.000Z"), + ); + + expect(JSON.parse(fs.readFileSync(output, "utf8"))).toMatchObject({ + schemaVersion: "nemoclaw.e2e_runtime_summary.v1", + runId: 123, + }); + expect(markdown).toContain("this run starts the history"); + expect(warning).toHaveBeenCalledOnce(); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + }); + + it("rejects outcome counts that do not match the bounded run count", () => { + const summary = createRuntimeSummary(1, "2026-07-22T00:00:00.000Z", [runtimeRow()]); + summary.rows[0]!.failedRuns = 2; + expect(normalizeRuntimeSummary(summary)).toBeNull(); + }); + + it("queries only prior completed scheduled runs and tolerates missing artifacts", async () => { + const listWorkflowRuns = vi.fn().mockResolvedValue({ + data: { workflow_runs: [{ id: 123 }, { id: 122 }] }, + }); + const paginate = vi.fn().mockResolvedValue([]); + const summaries = await loadPriorNightlySummaries({ + context: { repo: { owner: "NVIDIA", repo: "NemoClaw" }, runId: 123 }, + github: { + paginate, + rest: { + actions: { + downloadArtifact: vi.fn(), + listWorkflowRunArtifacts: {}, + listWorkflowRuns, + }, + }, + }, + }); + + expect(summaries).toEqual([]); + expect(listWorkflowRuns).toHaveBeenCalledWith( + expect.objectContaining({ event: "schedule", status: "completed", workflow_id: "e2e.yaml" }), + ); + expect(paginate).toHaveBeenCalledOnce(); + expect(paginate.mock.calls[0]?.[1]).toMatchObject({ run_id: 122 }); + }); +}); diff --git a/test/e2e/support/e2e-scorecard-coordinator.test.ts b/test/e2e/support/e2e-scorecard-coordinator.test.ts index e9d973b92bb..76087c9c466 100644 --- a/test/e2e/support/e2e-scorecard-coordinator.test.ts +++ b/test/e2e/support/e2e-scorecard-coordinator.test.ts @@ -150,6 +150,38 @@ describe("scorecard coordinator assembly", () => { expect(slackData.channel).toBe("fullrun"); }); + it("separates runner queue time from execution time without exposing runner names", () => { + const { summaryMarkdown } = coordinator.buildScorecard( + coordinatorInput({ + apiJobs: [ + { + completed_at: "2026-07-16T00:03:10.000Z", + conclusion: "success", + created_at: "2026-07-16T00:00:00.000Z", + labels: ["ubuntu-latest"], + name: "rebuild-hermes | shard", + started_at: "2026-07-16T00:00:10.000Z", + status: "completed", + }, + { + completed_at: "2026-07-16T00:01:30.000Z", + conclusion: "success", + created_at: "2026-07-16T00:00:00.000Z", + labels: ["larger-linux"], + name: "mcp-bridge", + started_at: "2026-07-16T00:00:30.000Z", + status: "completed", + }, + ], + }), + ); + + expect(summaryMarkdown).toContain("### Slowest job timing"); + expect(summaryMarkdown).toContain("| rebuild-hermes \\| shard | standard | 10.0s | 180.0s |"); + expect(summaryMarkdown).toContain("| mcp-bridge | custom | 30.0s | 60.0s |"); + expect(summaryMarkdown).not.toContain("runner_name"); + }); + it("falls back to needs results when the jobs API is unavailable", () => { const { scorecardData } = coordinator.buildScorecard( coordinatorInput({ diff --git a/test/e2e/support/e2e-scorecard.test.ts b/test/e2e/support/e2e-scorecard.test.ts index d6c3fe5701b..8b07180cdc2 100644 --- a/test/e2e/support/e2e-scorecard.test.ts +++ b/test/e2e/support/e2e-scorecard.test.ts @@ -184,7 +184,14 @@ describe("E2E scorecard", () => { it("loads typed scorecard helpers through the native github-script require boundary", () => { const script = ` const path = require('node:path'); - for (const file of ['analyze-trace-timing.mts', 'summarize-jobs.mts', 'build-slack-blocks.mts', 'coordinate-scorecard.mts']) { + for (const file of [ + 'analyze-runtime-history.mts', + 'analyze-trace-timing.mts', + 'summarize-jobs.mts', + 'build-slack-blocks.mts', + 'coordinate-scorecard.mts', + 'read-artifact-zip.mts', + ]) { const loaded = require(path.join(process.env.GITHUB_WORKSPACE, 'scripts/scorecard', file)); if (Object.keys(loaded).length === 0) process.exit(2); } @@ -474,6 +481,7 @@ describe("E2E scorecard", () => { ran: 4, skipped: 0, success: 3, + timingRows: [], total: 4, }); }); @@ -501,6 +509,7 @@ describe("E2E scorecard", () => { ran: 2, skipped: 1, success: 1, + timingRows: [], total: 3, }); }); diff --git a/test/e2e/support/rebuild-hermes-progress.test.ts b/test/e2e/support/rebuild-hermes-progress.test.ts index 70a681de3eb..8c6763d011f 100644 --- a/test/e2e/support/rebuild-hermes-progress.test.ts +++ b/test/e2e/support/rebuild-hermes-progress.test.ts @@ -92,4 +92,23 @@ describe("Hermes rebuild live progress", () => { }).not.toThrow(); expect(state.clearCalls).toBe(2); }); + + it("keeps the live result independent from a failed measurement append", () => { + const { options, state } = progressHarness(); + options.sampleResourceEvidence = () => { + throw new Error("measurement ledger unavailable"); + }; + + expect(() => { + const progress = startTestProgress( + "rebuild-hermes", + ["build previous Hermes base", "remove previous Hermes base"], + options, + ); + state.clockMs = 301_000; + state.timerCallback?.(); + progress.stop(); + }).not.toThrow(); + expect(state.clearCalls).toBe(1); + }); }); diff --git a/test/e2e/support/runner-pressure-workflow-boundary.test.ts b/test/e2e/support/runner-pressure-workflow-boundary.test.ts index f2a497bdfa6..90857cab269 100644 --- a/test/e2e/support/runner-pressure-workflow-boundary.test.ts +++ b/test/e2e/support/runner-pressure-workflow-boundary.test.ts @@ -6,11 +6,24 @@ import { describe, expect, it } from "vitest"; import { validateRunnerPressureWorkflow } from "../../../tools/e2e/runner-pressure-workflow-boundary.mts"; import { readWorkflow } from "../../helpers/e2e-workflow-contract"; -type WorkflowStep = Record & { if?: string; name?: string; run?: string }; +type WorkflowStep = Record & { + "continue-on-error"?: boolean; + if?: string; + name?: string; + run?: string; +}; type WorkflowJob = { steps: WorkflowStep[] }; type Workflow = { jobs: Record }; const JOBS = ["rebuild-hermes", "rebuild-hermes-stale-base"] as const; +const MEASUREMENT_JOBS = [ + "common-egress-agent", + "rebuild-hermes", + "rebuild-hermes-stale-base", + "mcp-bridge", +] as const; +const INITIALIZE_MEASUREMENT_STEP = "Initialize runner comparison evidence"; +const SUMMARIZE_MEASUREMENT_STEP = "Summarize runner comparison evidence"; function loadWorkflow(): Workflow { return readWorkflow() as Workflow; @@ -26,8 +39,12 @@ function swapStep(workflow: Workflow, jobId: (typeof JOBS)[number]): WorkflowSte )!; } +function namedStep(workflow: Workflow, jobId: string, name: string): WorkflowStep { + return workflow.jobs[jobId]!.steps.find((step) => step.name === name)!; +} + describe("runner-pressure E2E workflow boundary (#7146)", () => { - it("accepts the canonical Hermes heartbeat and terminal-consumer wiring", () => { + it("accepts the canonical pressure classification and runner comparison wiring", () => { expect(validateRunnerPressureWorkflow(loadWorkflow())).toEqual([]); }); @@ -47,6 +64,27 @@ describe("runner-pressure E2E workflow boundary (#7146)", () => { expect(provision.run).toContain("swapon /mnt/nemoclaw-hermes-rebuild.swap"); }); + it("limits job-level comparison evidence to the five larger-runner candidates", () => { + const workflow = loadWorkflow(); + for (const jobId of MEASUREMENT_JOBS) { + expect(namedStep(workflow, jobId, INITIALIZE_MEASUREMENT_STEP)).toBeDefined(); + expect(namedStep(workflow, jobId, SUMMARIZE_MEASUREMENT_STEP)).toBeDefined(); + } + expect(namedStep(workflow, "mcp-bridge", INITIALIZE_MEASUREMENT_STEP).if).toBe( + "${{ matrix.agent == 'hermes' || matrix.agent == 'deepagents' }}", + ); + expect(namedStep(workflow, "mcp-bridge", SUMMARIZE_MEASUREMENT_STEP).if).toBe( + "${{ always() && (matrix.agent == 'hermes' || matrix.agent == 'deepagents') }}", + ); + const unrelatedJobs = Object.entries(workflow.jobs).filter( + ([jobId]) => !(MEASUREMENT_JOBS as readonly string[]).includes(jobId), + ); + for (const [, job] of unrelatedJobs) { + expect(job.steps.some((step) => step.name === INITIALIZE_MEASUREMENT_STEP)).toBe(false); + expect(job.steps.some((step) => step.name === SUMMARIZE_MEASUREMENT_STEP)).toBe(false); + } + }); + it.each([ { label: "snapshot and phase baselines", @@ -118,4 +156,171 @@ describe("runner-pressure E2E workflow boundary (#7146)", () => { JOBS.map((jobId) => `${jobId} must upload runner-pressure evidence after every outcome`), ); }); + + it("rejects a missing initializer in every comparison job", () => { + const workflow = loadWorkflow(); + for (const jobId of MEASUREMENT_JOBS) { + workflow.jobs[jobId]!.steps = workflow.jobs[jobId]!.steps.filter( + (step) => step.name !== INITIALIZE_MEASUREMENT_STEP, + ); + } + + const errors = validateRunnerPressureWorkflow(workflow); + for (const jobId of MEASUREMENT_JOBS) { + expect(errors).toContain( + `${jobId} must initialize one runner comparison ledger after workspace preparation and before execution`, + ); + expect(errors).toContain( + `${jobId} must export the exact private runner comparison paths to later test processes`, + ); + } + }); + + it("rejects mutable or missing comparison artifact paths", () => { + const workflow = loadWorkflow(); + for (const jobId of MEASUREMENT_JOBS) { + const initialize = namedStep(workflow, jobId, INITIALIZE_MEASUREMENT_STEP); + initialize.run = initialize.run!.replaceAll( + "runner-resource-snapshots.jsonl", + "mutable-snapshots.jsonl", + ); + } + + const errors = validateRunnerPressureWorkflow(workflow); + for (const jobId of MEASUREMENT_JOBS) { + expect(errors).toContain( + `${jobId} must export the exact private runner comparison paths to later test processes`, + ); + } + }); + + it("rejects initialization after lane-specific setup has already started", () => { + const workflow = loadWorkflow(); + for (const jobId of MEASUREMENT_JOBS) { + const steps = workflow.jobs[jobId]!.steps; + const initializeIndex = steps.findIndex((step) => step.name === INITIALIZE_MEASUREMENT_STEP); + const [initialize] = steps.splice(initializeIndex, 1); + const runIndex = steps.findIndex((step) => step.name?.startsWith("Run ")); + steps.splice(runIndex, 0, initialize!); + } + + const errors = validateRunnerPressureWorkflow(workflow); + for (const jobId of MEASUREMENT_JOBS) { + expect(errors).toContain( + `${jobId} must initialize one runner comparison ledger after workspace preparation and before execution`, + ); + } + }); + + it("rejects summaries that run after artifact scanning or upload", () => { + const workflow = loadWorkflow(); + const publicationSteps = new Map([ + ["common-egress-agent", "Upload common-egress agent artifacts"], + ["rebuild-hermes", "Upload Hermes rebuild artifacts"], + ["rebuild-hermes-stale-base", "Upload Hermes stale-base rebuild artifacts"], + ["mcp-bridge", "Scan MCP artifacts for fixture credentials"], + ]); + for (const jobId of MEASUREMENT_JOBS) { + const steps = workflow.jobs[jobId]!.steps; + const summaryIndex = steps.findIndex((step) => step.name === SUMMARIZE_MEASUREMENT_STEP); + const [summary] = steps.splice(summaryIndex, 1); + const publicationIndex = steps.findIndex((step) => step.name === publicationSteps.get(jobId)); + steps.splice(publicationIndex + 1, 0, summary!); + } + + const errors = validateRunnerPressureWorkflow(workflow); + for (const jobId of MEASUREMENT_JOBS) { + expect(errors).toContain( + `${jobId} must append a final runner snapshot and summarize before artifact publication`, + ); + } + }); + + it("rejects comparison evidence that can replace live-test outcomes", () => { + const workflow = loadWorkflow(); + for (const jobId of MEASUREMENT_JOBS) { + const initialize = namedStep(workflow, jobId, INITIALIZE_MEASUREMENT_STEP); + const summary = namedStep(workflow, jobId, SUMMARIZE_MEASUREMENT_STEP); + initialize["continue-on-error"] = false; + summary.if = "success()"; + summary["continue-on-error"] = false; + } + + const errors = validateRunnerPressureWorkflow(workflow); + for (const jobId of MEASUREMENT_JOBS.filter((jobId) => jobId !== "mcp-bridge")) { + expect(errors).toContain( + `${jobId} runner comparison evidence must remain best-effort after every outcome`, + ); + } + expect(errors).toContain( + "mcp-bridge runner comparison evidence must be best-effort and limited to Hermes and Deep Agents shards", + ); + }); + + it("rejects comparison wiring in MCP OpenClaw, dev, or unrelated jobs", () => { + const workflow = loadWorkflow(); + namedStep(workflow, "mcp-bridge", INITIALIZE_MEASUREMENT_STEP).if = "always()"; + workflow.jobs["mcp-bridge-dev"]!.steps.push({ + name: INITIALIZE_MEASUREMENT_STEP, + run: "export E2E_RESOURCE_SNAPSHOTS_FILE=unexpected", + }); + workflow.jobs["shields-config"]!.steps.push({ + name: SUMMARIZE_MEASUREMENT_STEP, + run: "npx tsx tools/e2e/runner-pressure.mts summarize-measurement", + }); + + const errors = validateRunnerPressureWorkflow(workflow); + expect(errors).toContain( + "mcp-bridge runner comparison evidence must be best-effort and limited to Hermes and Deep Agents shards", + ); + expect(errors).toContain( + "runner comparison evidence must not be wired to out-of-scope job mcp-bridge-dev", + ); + expect(errors).toContain( + "runner comparison evidence must not be wired to out-of-scope job shields-config", + ); + }); + + it.each([ + { + label: "backgrounding the primary MCP test", + mutate: (script: string) => + script.replace( + "npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/mcp-bridge.test.ts", + "npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/mcp-bridge.test.ts &", + ), + }, + { + label: "backgrounding the primary MCP test without whitespace", + mutate: (script: string) => + script.replace( + "npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/mcp-bridge.test.ts", + "npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/mcp-bridge.test.ts&", + ), + }, + { + label: "parallelizing the Deep Agents follow-up", + mutate: (script: string) => script.replace("--no-file-parallelism", "--file-parallelism"), + }, + { + label: "running the follow-up before the primary test", + mutate: (script: string) => { + const main = + "npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/mcp-bridge.test.ts"; + const followUp = "test/e2e/live/openshell-credential-generation-window.test.ts"; + return script + .replace(main, "PRIMARY_TEST_PLACEHOLDER") + .replace(followUp, main) + .replace("PRIMARY_TEST_PLACEHOLDER", followUp); + }, + }, + ])("rejects $label because the shared ledger requires serialized appends", ({ mutate }) => { + const workflow = loadWorkflow(); + const run = namedStep(workflow, "mcp-bridge", "Run MCP OpenShell provider live test"); + run.run = mutate(run.run!); + + expect(validateRunnerPressureWorkflow(workflow)).toContain( + "mcp-bridge Deep Agents runner comparison tests must remain serialized in one job ledger", + ); + }); }); diff --git a/test/e2e/support/runner-pressure.test.ts b/test/e2e/support/runner-pressure.test.ts index cc5c5d0cca7..0b1bfd10bf9 100644 --- a/test/e2e/support/runner-pressure.test.ts +++ b/test/e2e/support/runner-pressure.test.ts @@ -30,12 +30,14 @@ import { parseCgroupMemoryEvents, parseCgroupScalar, parseClassificationLine, + parseCpuTimes, parseDockerSize, parseDockerStats, parseDockerSystemDf, parseLoadAverages, parseMeminfo, parsePressure, + parseSnapshotLine, parseTopProcesses, type ResourceSnapshot, renderBaselineLine, @@ -44,6 +46,7 @@ import { SNAPSHOT_LINE_MAX_LENGTH, SNAPSHOT_LINE_PREFIX, selectFailureBaseline, + summarizeResourceSnapshots, } from "../../../tools/e2e/runner-pressure-core.mts"; const HELPER = path.resolve( @@ -118,6 +121,7 @@ function baseSnapshot(): ResourceSnapshot { return { phase: "rebuild-hermes.image-build", at: "2026-07-18T00:00:00.000Z", + cpu: { logicalCpuCount: 4, idleTicks: 4_000, totalTicks: 10_000 }, meminfo: parseMeminfo(MEMINFO_FIXTURE), load: { load1: 3.5, load5: 2.1, load15: 1.0 }, cgroup: { @@ -151,6 +155,20 @@ function baseSnapshot(): ResourceSnapshot { } describe("host measurement parsers (#7146)", () => { + it("reads aggregate host CPU counters without double-counting guest time", () => { + expect( + parseCpuTimes( + [ + "cpu 100 20 30 400 50 10 20 5 80 10", + "cpu0 1 2 3 4 5 6 7 8 9 10", + "cpu1 1 2 3 4 5 6 7 8 9 10", + "intr 123", + ].join("\n"), + ), + ).toEqual({ logicalCpuCount: 2, idleTicks: 450, totalTicks: 635 }); + expect(parseCpuTimes("cpu malformed\ncpu0 1 2 3\n")).toBeNull(); + }); + it("reads MemAvailable, Cached, SReclaimable, and swap from /proc/meminfo", () => { const sample = parseMeminfo(MEMINFO_FIXTURE); expect(sample).toEqual({ @@ -261,6 +279,7 @@ describe("bounded secret-safe snapshot line (#7146)", () => { "at", "cgroup", "containers", + "cpu", "disk", "dockerDisk", "ioPressure", @@ -272,6 +291,7 @@ describe("bounded secret-safe snapshot line (#7146)", () => { "v", ]); expect(payload.meminfo.memAvailableKb).toBe(12288000); + expect(payload.cpu).toEqual({ logicalCpuCount: 4, idleTicks: 4000, totalTicks: 10000 }); expect(payload.cgroup.limitBytes).toBeNull(); expect(payload.topProcesses[0]).toEqual({ rank: 1, rssKb: 900000 }); expect(payload.containers[0]).toEqual({ @@ -296,6 +316,17 @@ describe("bounded secret-safe snapshot line (#7146)", () => { expect(line).not.toContain("ghp_container_secret"); }); + it("strictly round-trips the allowlisted snapshot and rejects extra fields", () => { + const line = renderSnapshotLine(baseSnapshot()); + expect(parseSnapshotLine(line)).toEqual(baseSnapshot()); + expect(() => parseSnapshotLine(line.replace('"v":1', '"v":1,"token":"ghp_secret"'))).toThrow( + /unsupported shape/, + ); + expect(() => parseSnapshotLine(line.replace('"idleTicks":4000', '"idleTicks":12000'))).toThrow( + /cannot exceed/, + ); + }); + it("stays within the line bound by limiting ranked lists before rendering", () => { const bloated = baseSnapshot(); bloated.topProcesses = Array.from({ length: 500 }, (_, i) => ({ @@ -336,6 +367,119 @@ describe("bounded secret-safe snapshot line (#7146)", () => { }); }); +describe("job-level runner resource summary (#7145)", () => { + function measuredSnapshot( + at: string, + cpu: ResourceSnapshot["cpu"], + peakBytes: number | null, + availableKb: number, + freeBytes: number, + ): ResourceSnapshot { + const snapshot = baseSnapshot(); + return { + ...snapshot, + at, + cpu, + meminfo: { ...snapshot.meminfo!, memAvailableKb: availableKb }, + cgroup: { ...snapshot.cgroup!, peakBytes }, + disk: { ...snapshot.disk!, freeBytes, totalBytes: 100 * 1024 ** 3 }, + }; + } + + it("reports sampled CPU, cgroup memory peak, and disk growth across one lane", () => { + const snapshots = [ + measuredSnapshot( + "2026-07-18T00:00:00.000Z", + { logicalCpuCount: 2, idleTicks: 400, totalTicks: 1_000 }, + 4 * 1024 ** 3, + 12_000_000, + 80 * 1024 ** 3, + ), + measuredSnapshot( + "2026-07-18T00:00:00.500Z", + { logicalCpuCount: 2, idleTicks: 450, totalTicks: 1_500 }, + 5 * 1024 ** 3, + 11_000_000, + 79 * 1024 ** 3, + ), + measuredSnapshot( + "2026-07-18T00:00:02.000Z", + { logicalCpuCount: 2, idleTicks: 800, totalTicks: 3_000 }, + 8 * 1024 ** 3, + 9_000_000, + 70 * 1024 ** 3, + ), + measuredSnapshot( + "2026-07-18T00:00:04.000Z", + { logicalCpuCount: 2, idleTicks: 20, totalTicks: 100 }, + 7 * 1024 ** 3, + 10_000_000, + 75 * 1024 ** 3, + ), + measuredSnapshot( + "2026-07-18T00:00:06.000Z", + { logicalCpuCount: 2, idleTicks: 220, totalTicks: 1_100 }, + 6 * 1024 ** 3, + 11_000_000, + 60 * 1024 ** 3, + ), + ]; + + expect( + summarizeResourceSnapshots(snapshots, { + targetId: "mcp-bridge", + shardId: "deepagents", + }), + ).toEqual({ + version: 1, + targetId: "mcp-bridge", + shardId: "deepagents", + startedAt: "2026-07-18T00:00:00.000Z", + finishedAt: "2026-07-18T00:00:06.000Z", + durationMs: 6_000, + sampleCount: 5, + cpu: { + logicalCpuCount: 2, + peakSampledPercent: 80, + intervalCount: 2, + }, + memory: { + peakBytes: 8 * 1024 ** 3, + capacityBytes: 16_384_000 * 1024, + source: "cgroup-peak", + }, + disk: { + peakDeltaBytes: 20 * 1024 ** 3, + minimumFreeBytes: 60 * 1024 ** 3, + capacityBytes: 100 * 1024 ** 3, + }, + }); + }); + + it("falls back to sampled MemAvailable and exposes an incomplete CPU interval", () => { + const snapshot = measuredSnapshot( + "2026-07-18T00:00:00.000Z", + null, + null, + 12_000_000, + 80 * 1024 ** 3, + ); + const summary = summarizeResourceSnapshots([snapshot], { targetId: "common-egress-agent" }); + expect(summary.durationMs).toBe(0); + expect(summary.sampleCount).toBe(1); + expect(summary.cpu).toEqual({ + logicalCpuCount: null, + peakSampledPercent: null, + intervalCount: 0, + }); + expect(summary.memory).toEqual({ + peakBytes: (16_384_000 - 12_000_000) * 1024, + capacityBytes: 16_384_000 * 1024, + source: "mem-available", + }); + }); +}); + describe("terminal failure classification (#7146)", () => { it("round-trips a strict numeric/boolean phase baseline", () => { const line = renderBaselineLine({ @@ -664,6 +808,64 @@ describe("runner-pressure CLI fail-closed entrypoint (#7146)", () => { } }, 90_000); + it("initializes and summarizes one private job-level measurement ledger", () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-runner-measurement-")); + const snapshotsPath = path.join(directory, "runner-resource-snapshots.jsonl"); + const summaryPath = path.join(directory, "runner-resource-summary.json"); + const environment = { + E2E_TARGET_ID: "common-egress-agent", + E2E_RESOURCE_SNAPSHOTS_FILE: snapshotsPath, + E2E_RESOURCE_SUMMARY_FILE: summaryPath, + }; + try { + expect(runHelper(["initialize-measurement"], environment).status).toBe(0); + expect(fs.readFileSync(snapshotsPath, "utf8").trim().split("\n")).toHaveLength(1); + expect(fs.readFileSync(summaryPath, "utf8")).toBe(""); + + const result = runHelper(["summarize-measurement"], environment); + expect(result.status).toBe(0); + expect(fs.readFileSync(snapshotsPath, "utf8").trim().split("\n")).toHaveLength(2); + const summary = JSON.parse(fs.readFileSync(summaryPath, "utf8")); + expect(summary).toMatchObject({ + version: 1, + targetId: "common-egress-agent", + sampleCount: 2, + cpu: { intervalCount: expect.any(Number) }, + disk: { minimumFreeBytes: expect.any(Number) }, + }); + expect(Object.keys(summary.memory).sort()).toEqual(["capacityBytes", "peakBytes", "source"]); + for (const file of [snapshotsPath, summaryPath]) { + expect(fs.statSync(file).mode & 0o777).toBe(0o600); + } + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + }, 90_000); + + it.each([ + "symlink", + "hardlink", + ] as const)("rejects a %s-substituted measurement ledger", (linkKind) => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-linked-measurement-")); + const snapshotsPath = path.join(directory, "runner-resource-snapshots.jsonl"); + const summaryPath = path.join(directory, "runner-resource-summary.json"); + const protectedPath = path.join(directory, "protected.jsonl"); + const protectedContents = "protected\n"; + const environment = { + E2E_TARGET_ID: "common-egress-agent", + E2E_RESOURCE_SNAPSHOTS_FILE: snapshotsPath, + E2E_RESOURCE_SUMMARY_FILE: summaryPath, + }; + try { + fs.writeFileSync(protectedPath, protectedContents, { mode: 0o600 }); + LINK_CREATORS[linkKind](protectedPath, snapshotsPath); + expect(runHelper(["initialize-measurement"], environment).status).not.toBe(0); + expect(fs.readFileSync(protectedPath, "utf8")).toBe(protectedContents); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + }, 90_000); + it.each([ "assertion", "timeout", diff --git a/test/e2e/support/test-runtime-audit.test.ts b/test/e2e/support/test-runtime-audit.test.ts index b0c113bd766..e55225288d9 100644 --- a/test/e2e/support/test-runtime-audit.test.ts +++ b/test/e2e/support/test-runtime-audit.test.ts @@ -68,6 +68,9 @@ describe("test runtime audit", () => { p95Ms: 50_000, maxMs: 50_000, variabilityMs: 20_000, + passedRuns: 1, + failedRuns: 1, + skippedRuns: 0, slowestPhase: "inference", slowestPhaseMs: 40_000, slowestPhaseOutcome: "failed", @@ -80,6 +83,9 @@ describe("test runtime audit", () => { p95Ms: 20_000, maxMs: 20_000, variabilityMs: 0, + passedRuns: 1, + failedRuns: 0, + skippedRuns: 0, slowestPhase: "sandbox", slowestPhaseMs: 15_000, slowestPhaseOutcome: "passed", diff --git a/tools/e2e/operations-workflow-boundary.mts b/tools/e2e/operations-workflow-boundary.mts index 36e44bf7b4c..3a11808d80b 100644 --- a/tools/e2e/operations-workflow-boundary.mts +++ b/tools/e2e/operations-workflow-boundary.mts @@ -17,6 +17,8 @@ const GITHUB_SCRIPT_NODE24_ACTION = "actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3"; const DOWNLOAD_ARTIFACT_ACTION = "actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c"; +const UPLOAD_E2E_ARTIFACTS_ACTION = + "NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57"; const PR_GATE_REPORTER = "test/e2e/risk-signal-reporter.ts"; const LIVE_VITEST_HELPER = "tools/e2e/live-vitest-invocation.mts run --test-path"; const E2E_ARTIFACT_ACTION = "NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@"; @@ -462,6 +464,8 @@ function validateScorecard(errors: string[], workflow: OperationsWorkflow): void "scripts/audit-test-runtime.mts", "runtimeAudit.auditTestRuntime", "runtimeAudit.formatRuntimeAuditSummary", + "scripts/scorecard/analyze-runtime-history.mts", + "runtimeHistory.buildRuntimeHistory", "core.summary", "scorecardData", "slackData", @@ -477,6 +481,21 @@ function validateScorecard(errors: string[], workflow: OperationsWorkflow): void if (generate.env?.RUNTIME_ARTIFACTS !== "${{ runner.temp }}/e2e-runtime-audit") { errors.push("scorecard generator must read the downloaded runtime audit directory"); } + if (generate.env?.RUNTIME_SUMMARY_FILE !== "${{ runner.temp }}/e2e-runtime-summary.json") { + errors.push("scorecard generator must write the bounded runtime summary file"); + } + + const upload = findStep(job, "Upload E2E runtime summary"); + requirePinnedAction(errors, upload, "scorecard runtime summary upload"); + if ( + upload.uses !== UPLOAD_E2E_ARTIFACTS_ACTION || + upload.if !== "${{ always() && steps.scorecard.outcome == 'success' }}" || + upload.with?.name !== "e2e-runtime-summary" || + upload.with?.path !== "${{ runner.temp }}/e2e-runtime-summary.json" || + Object.keys(upload.with ?? {}).length !== 2 + ) { + errors.push("scorecard must upload only the bounded runtime summary for rolling history"); + } const slack = findStep(job, "Post scorecard to Slack"); requirePinnedAction(errors, slack, "scorecard Slack publisher"); diff --git a/tools/e2e/runner-pressure-core.mts b/tools/e2e/runner-pressure-core.mts index 11b9729a718..ec8f50550ab 100644 --- a/tools/e2e/runner-pressure-core.mts +++ b/tools/e2e/runner-pressure-core.mts @@ -82,6 +82,32 @@ export interface LoadSample { load15: number; } +export interface CpuTimesSample { + logicalCpuCount: number; + idleTicks: number; + totalTicks: number; +} + +/** + * Parse the aggregate CPU counters and logical processor rows from + * `/proc/stat`. Guest counters are excluded because Linux already includes + * them in user/nice and counting them again would inflate utilization. + */ +export function parseCpuTimes(text: string): CpuTimesSample | null { + const lines = text.split("\n"); + const aggregate = lines.find((line) => /^cpu\s+/u.test(line)); + const logicalCpuCount = lines.filter((line) => /^cpu\d+\s+/u.test(line)).length; + if (!aggregate || logicalCpuCount < 1) return null; + const values = aggregate.trim().split(/\s+/u).slice(1, 9); + if (values.length !== 8 || values.some((value) => !/^\d+$/u.test(value))) return null; + const counters = values.map(Number); + if (counters.some((value) => !Number.isSafeInteger(value) || value < 0)) return null; + const idleTicks = (counters[3] ?? 0) + (counters[4] ?? 0); + const totalTicks = counters.reduce((total, value) => total + value, 0); + if (!Number.isSafeInteger(idleTicks) || !Number.isSafeInteger(totalTicks)) return null; + return { logicalCpuCount, idleTicks, totalTicks }; +} + /** Parse `/proc/loadavg`; null when the shape is unrecognized. */ export function parseLoadAverages(text: string): LoadSample | null { const match = /^(\d+\.\d+)\s+(\d+\.\d+)\s+(\d+\.\d+)\s/u.exec(text.trim()); @@ -271,6 +297,7 @@ export interface DiskSample { export interface ResourceSnapshot { phase: string; at: string; + cpu: CpuTimesSample | null; meminfo: MeminfoSample | null; load: LoadSample | null; cgroup: { @@ -330,6 +357,14 @@ export function renderSnapshotLine(snapshot: ResourceSnapshot): string { v: 1, phase: assertPhaseLabel(snapshot.phase), at: assertCanonicalTimestamp(snapshot.at), + cpu: + snapshot.cpu === null + ? null + : { + logicalCpuCount: number_(snapshot.cpu.logicalCpuCount), + idleTicks: number_(snapshot.cpu.idleTicks), + totalTicks: number_(snapshot.cpu.totalTicks), + }, meminfo: snapshot.meminfo === null ? null @@ -414,6 +449,386 @@ function renderPressure(sample: PressureSample | null): PressureSample | null { }; } +type UnknownRecord = Record; + +function record_(value: unknown, field: string): UnknownRecord { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error(`${field} must be an object`); + } + return value as UnknownRecord; +} + +function exactKeys(record: UnknownRecord, keys: readonly string[], field: string): void { + if (Object.keys(record).sort().join(",") !== [...keys].sort().join(",")) { + throw new Error(`${field} has an unsupported shape`); + } +} + +function nullableNonNegativeNumber(value: unknown, field: string): number | null { + if (value === null) return null; + if (typeof value !== "number" || !Number.isFinite(value) || value < 0) { + throw new Error(`${field} must be null or a non-negative finite number`); + } + return value; +} + +function nullableNonNegativeInteger(value: unknown, field: string): number | null { + const parsed = nullableNonNegativeNumber(value, field); + if (parsed !== null && !Number.isSafeInteger(parsed)) { + throw new Error(`${field} must be null or a non-negative safe integer`); + } + return parsed; +} + +function parseNullableRecord( + value: unknown, + field: string, + parser: (record: UnknownRecord, field: string) => T, +): T | null { + return value === null ? null : parser(record_(value, field), field); +} + +function parseCpuRecord(record: UnknownRecord, field: string): CpuTimesSample { + exactKeys(record, ["logicalCpuCount", "idleTicks", "totalTicks"], field); + const logicalCpuCount = nullableNonNegativeInteger( + record.logicalCpuCount, + `${field}.logicalCpuCount`, + ); + const idleTicks = nullableNonNegativeInteger(record.idleTicks, `${field}.idleTicks`); + const totalTicks = nullableNonNegativeInteger(record.totalTicks, `${field}.totalTicks`); + if ( + logicalCpuCount === null || + logicalCpuCount < 1 || + idleTicks === null || + totalTicks === null + ) { + throw new Error(`${field} must contain complete CPU counters`); + } + if (idleTicks > totalTicks) throw new Error(`${field}.idleTicks cannot exceed totalTicks`); + return { logicalCpuCount, idleTicks, totalTicks }; +} + +function parseMeminfoRecord(record: UnknownRecord, field: string): MeminfoSample { + const keys = [ + "memTotalKb", + "memFreeKb", + "memAvailableKb", + "cachedKb", + "sReclaimableKb", + "swapTotalKb", + "swapFreeKb", + ] as const; + exactKeys(record, keys, field); + return { + memTotalKb: nullableNonNegativeNumber(record.memTotalKb, `${field}.memTotalKb`), + memFreeKb: nullableNonNegativeNumber(record.memFreeKb, `${field}.memFreeKb`), + memAvailableKb: nullableNonNegativeNumber(record.memAvailableKb, `${field}.memAvailableKb`), + cachedKb: nullableNonNegativeNumber(record.cachedKb, `${field}.cachedKb`), + sReclaimableKb: nullableNonNegativeNumber(record.sReclaimableKb, `${field}.sReclaimableKb`), + swapTotalKb: nullableNonNegativeNumber(record.swapTotalKb, `${field}.swapTotalKb`), + swapFreeKb: nullableNonNegativeNumber(record.swapFreeKb, `${field}.swapFreeKb`), + }; +} + +function parseLoadRecord(record: UnknownRecord, field: string): LoadSample { + exactKeys(record, ["load1", "load5", "load15"], field); + const load1 = nullableNonNegativeNumber(record.load1, `${field}.load1`); + const load5 = nullableNonNegativeNumber(record.load5, `${field}.load5`); + const load15 = nullableNonNegativeNumber(record.load15, `${field}.load15`); + if (load1 === null || load5 === null || load15 === null) { + throw new Error(`${field} must contain complete load averages`); + } + return { load1, load5, load15 }; +} + +function parseEventsRecord(record: UnknownRecord, field: string): CgroupMemoryEvents { + exactKeys(record, ["oom", "oomKill"], field); + const oom = nullableNonNegativeInteger(record.oom, `${field}.oom`); + const oomKill = nullableNonNegativeInteger(record.oomKill, `${field}.oomKill`); + if (oom === null || oomKill === null) throw new Error(`${field} must contain complete counters`); + return { oom, oomKill }; +} + +function parseCgroupRecord( + record: UnknownRecord, + field: string, +): NonNullable { + exactKeys(record, ["currentBytes", "peakBytes", "limitBytes", "events"], field); + return { + currentBytes: nullableNonNegativeNumber(record.currentBytes, `${field}.currentBytes`), + peakBytes: nullableNonNegativeNumber(record.peakBytes, `${field}.peakBytes`), + limitBytes: nullableNonNegativeNumber(record.limitBytes, `${field}.limitBytes`), + events: parseNullableRecord(record.events, `${field}.events`, parseEventsRecord), + }; +} + +function parsePressureRecord(record: UnknownRecord, field: string): PressureSample { + exactKeys(record, ["someAvg10", "someAvg60", "fullAvg10", "fullAvg60"], field); + return { + someAvg10: nullableNonNegativeNumber(record.someAvg10, `${field}.someAvg10`), + someAvg60: nullableNonNegativeNumber(record.someAvg60, `${field}.someAvg60`), + fullAvg10: nullableNonNegativeNumber(record.fullAvg10, `${field}.fullAvg10`), + fullAvg60: nullableNonNegativeNumber(record.fullAvg60, `${field}.fullAvg60`), + }; +} + +function parseTopProcessList(value: unknown): ProcessSample[] { + if (!Array.isArray(value) || value.length > TOP_PROCESS_LIMIT) { + throw new Error("snapshot.topProcesses must be a bounded array"); + } + return value.map((entry, index) => { + const record = record_(entry, `snapshot.topProcesses[${index}]`); + exactKeys(record, ["rank", "rssKb"], `snapshot.topProcesses[${index}]`); + if (record.rank !== index + 1) throw new Error("snapshot.topProcesses ranks must be ordered"); + const rssKb = nullableNonNegativeNumber(record.rssKb, `snapshot.topProcesses[${index}].rssKb`); + if (rssKb === null) throw new Error("snapshot.topProcesses.rssKb cannot be null"); + return { rssKb }; + }); +} + +function parseContainerList(value: unknown): ContainerStatSample[] { + if (!Array.isArray(value) || value.length > CONTAINER_STAT_LIMIT) { + throw new Error("snapshot.containers must be a bounded array"); + } + return value.map((entry, index) => { + const record = record_(entry, `snapshot.containers[${index}]`); + exactKeys( + record, + ["rank", "cpuPercent", "memBytes", "memLimitBytes"], + `snapshot.containers[${index}]`, + ); + if (record.rank !== index + 1) throw new Error("snapshot.containers ranks must be ordered"); + return { + cpuPercent: nullableNonNegativeNumber( + record.cpuPercent, + `snapshot.containers[${index}].cpuPercent`, + ), + memBytes: nullableNonNegativeNumber( + record.memBytes, + `snapshot.containers[${index}].memBytes`, + ), + memLimitBytes: nullableNonNegativeNumber( + record.memLimitBytes, + `snapshot.containers[${index}].memLimitBytes`, + ), + }; + }); +} + +function parseDockerDiskRecord(record: UnknownRecord, field: string): DockerDiskSample { + exactKeys(record, ["imagesBytes", "containersBytes", "buildCacheBytes"], field); + return { + imagesBytes: nullableNonNegativeNumber(record.imagesBytes, `${field}.imagesBytes`), + containersBytes: nullableNonNegativeNumber(record.containersBytes, `${field}.containersBytes`), + buildCacheBytes: nullableNonNegativeNumber(record.buildCacheBytes, `${field}.buildCacheBytes`), + }; +} + +function parseDiskRecord(record: UnknownRecord, field: string): DiskSample { + exactKeys(record, ["freeBytes", "totalBytes", "inodesFree", "inodesTotal"], field); + return { + freeBytes: nullableNonNegativeNumber(record.freeBytes, `${field}.freeBytes`), + totalBytes: nullableNonNegativeNumber(record.totalBytes, `${field}.totalBytes`), + inodesFree: nullableNonNegativeNumber(record.inodesFree, `${field}.inodesFree`), + inodesTotal: nullableNonNegativeNumber(record.inodesTotal, `${field}.inodesTotal`), + }; +} + +/** Strictly parse one allowlisted snapshot line; unknown fields are rejected. */ +export function parseSnapshotLine(line: string): ResourceSnapshot { + const trimmed = line.trim(); + if (!trimmed.startsWith(SNAPSHOT_LINE_PREFIX)) { + throw new Error(`resource snapshot must start with ${SNAPSHOT_LINE_PREFIX.trim()}`); + } + let parsed: unknown; + try { + parsed = JSON.parse(trimmed.slice(SNAPSHOT_LINE_PREFIX.length)); + } catch { + throw new Error("resource snapshot must contain valid JSON"); + } + const record = record_(parsed, "snapshot"); + exactKeys( + record, + [ + "v", + "phase", + "at", + "cpu", + "meminfo", + "load", + "cgroup", + "memoryPressure", + "ioPressure", + "topProcesses", + "containers", + "dockerDisk", + "disk", + ], + "snapshot", + ); + if (record.v !== 1) throw new Error("resource snapshot has an unsupported version"); + return { + phase: assertPhaseLabel(typeof record.phase === "string" ? record.phase : undefined), + at: assertCanonicalTimestamp(typeof record.at === "string" ? record.at : undefined), + cpu: parseNullableRecord(record.cpu, "snapshot.cpu", parseCpuRecord), + meminfo: parseNullableRecord(record.meminfo, "snapshot.meminfo", parseMeminfoRecord), + load: parseNullableRecord(record.load, "snapshot.load", parseLoadRecord), + cgroup: parseNullableRecord(record.cgroup, "snapshot.cgroup", parseCgroupRecord), + memoryPressure: parseNullableRecord( + record.memoryPressure, + "snapshot.memoryPressure", + parsePressureRecord, + ), + ioPressure: parseNullableRecord(record.ioPressure, "snapshot.ioPressure", parsePressureRecord), + topProcesses: parseTopProcessList(record.topProcesses), + containers: parseContainerList(record.containers), + dockerDisk: parseNullableRecord( + record.dockerDisk, + "snapshot.dockerDisk", + parseDockerDiskRecord, + ), + disk: parseNullableRecord(record.disk, "snapshot.disk", parseDiskRecord), + }; +} + +export interface RunnerResourceSummary { + version: 1; + targetId: string; + shardId?: string; + startedAt: string; + finishedAt: string; + durationMs: number; + sampleCount: number; + cpu: { + logicalCpuCount: number | null; + peakSampledPercent: number | null; + intervalCount: number; + }; + memory: { + peakBytes: number | null; + capacityBytes: number | null; + source: "cgroup-peak" | "mem-available" | null; + }; + disk: { + peakDeltaBytes: number | null; + minimumFreeBytes: number | null; + capacityBytes: number | null; + }; +} + +function consistentValue(values: readonly number[]): number | null { + const first = values[0]; + return first !== undefined && values.every((value) => value === first) ? first : null; +} + +function maximum(values: readonly number[]): number | null { + return values.length > 0 ? Math.max(...values) : null; +} + +/** Reduce one serialized job ledger into a numeric, runner-comparable summary. */ +export function summarizeResourceSnapshots( + snapshots: readonly ResourceSnapshot[], + identity: { targetId: string; shardId?: string }, +): RunnerResourceSummary { + const first = snapshots[0]; + const last = snapshots.at(-1); + if (!first || !last) throw new Error("resource measurement ledger must contain a snapshot"); + const startedAtMs = Date.parse(first.at); + const finishedAtMs = Date.parse(last.at); + if (finishedAtMs < startedAtMs) { + throw new Error("resource measurement ledger timestamps must be monotonic"); + } + for (let index = 1; index < snapshots.length; index += 1) { + if (Date.parse(snapshots[index]!.at) < Date.parse(snapshots[index - 1]!.at)) { + throw new Error("resource measurement ledger timestamps must be monotonic"); + } + } + + const cpuCounts = snapshots.flatMap((snapshot) => + snapshot.cpu === null ? [] : [snapshot.cpu.logicalCpuCount], + ); + let peakSampledPercent: number | null = null; + let intervalCount = 0; + for (let index = 1; index < snapshots.length; index += 1) { + const previous = snapshots[index - 1]!; + const current = snapshots[index]!; + if ( + previous.cpu === null || + current.cpu === null || + previous.cpu.logicalCpuCount !== current.cpu.logicalCpuCount || + Date.parse(current.at) - Date.parse(previous.at) < 1_000 + ) { + continue; + } + const totalDelta = current.cpu.totalTicks - previous.cpu.totalTicks; + const idleDelta = current.cpu.idleTicks - previous.cpu.idleTicks; + if (totalDelta <= 0 || idleDelta < 0 || idleDelta > totalDelta) continue; + const percent = Math.round(((totalDelta - idleDelta) / totalDelta) * 10_000) / 100; + peakSampledPercent = Math.max(peakSampledPercent ?? 0, percent); + intervalCount += 1; + } + + const hostCapacities = snapshots.flatMap((snapshot) => + snapshot.meminfo?.memTotalKb === null || snapshot.meminfo?.memTotalKb === undefined + ? [] + : [snapshot.meminfo.memTotalKb * 1024], + ); + const cgroupPeaks = snapshots.flatMap((snapshot) => + snapshot.cgroup?.peakBytes === null || snapshot.cgroup?.peakBytes === undefined + ? [] + : [snapshot.cgroup.peakBytes], + ); + const sampledMemoryUsed = snapshots.flatMap((snapshot) => { + const total = snapshot.meminfo?.memTotalKb; + const available = snapshot.meminfo?.memAvailableKb; + return total === null || total === undefined || available === null || available === undefined + ? [] + : [Math.max(0, total - available) * 1024]; + }); + const memorySource = + cgroupPeaks.length > 0 ? "cgroup-peak" : sampledMemoryUsed.length > 0 ? "mem-available" : null; + + const diskSamples = snapshots.flatMap((snapshot) => + snapshot.disk?.freeBytes === null || snapshot.disk?.freeBytes === undefined + ? [] + : [{ freeBytes: snapshot.disk.freeBytes, totalBytes: snapshot.disk.totalBytes }], + ); + const baselineFreeBytes = diskSamples[0]?.freeBytes; + const minimumFreeBytes = maximum(diskSamples.map((sample) => -sample.freeBytes)); + const normalizedMinimumFreeBytes = minimumFreeBytes === null ? null : -minimumFreeBytes; + const diskCapacities = diskSamples.flatMap((sample) => + sample.totalBytes === null ? [] : [sample.totalBytes], + ); + + return { + version: 1, + targetId: assertPhaseLabel(identity.targetId), + ...(identity.shardId ? { shardId: assertPhaseLabel(identity.shardId) } : {}), + startedAt: first.at, + finishedAt: last.at, + durationMs: finishedAtMs - startedAtMs, + sampleCount: snapshots.length, + cpu: { + logicalCpuCount: consistentValue(cpuCounts), + peakSampledPercent, + intervalCount, + }, + memory: { + peakBytes: maximum(cgroupPeaks.length > 0 ? cgroupPeaks : sampledMemoryUsed), + capacityBytes: consistentValue(hostCapacities), + source: memorySource, + }, + disk: { + peakDeltaBytes: + baselineFreeBytes === undefined || normalizedMinimumFreeBytes === null + ? null + : Math.max(0, baselineFreeBytes - normalizedMinimumFreeBytes), + minimumFreeBytes: normalizedMinimumFreeBytes, + capacityBytes: consistentValue(diskCapacities), + }, + }; +} + // ── Terminal classification ────────────────────────────────────────────────── export interface ResourceBaseline { diff --git a/tools/e2e/runner-pressure-workflow-boundary.mts b/tools/e2e/runner-pressure-workflow-boundary.mts index ad30e94e0f1..d1b09b8e734 100644 --- a/tools/e2e/runner-pressure-workflow-boundary.mts +++ b/tools/e2e/runner-pressure-workflow-boundary.mts @@ -2,13 +2,49 @@ // SPDX-License-Identifier: Apache-2.0 type WorkflowRecord = Record; -type WorkflowStep = WorkflowRecord & { if?: string; name?: string; run?: string }; +type WorkflowStep = WorkflowRecord & { + "continue-on-error"?: boolean; + if?: string; + name?: string; + run?: string; +}; const RUNNER_PRESSURE_JOBS = [ { id: "rebuild-hermes", runStep: "Run Hermes rebuild live test" }, { id: "rebuild-hermes-stale-base", runStep: "Run Hermes stale-base rebuild live test" }, ] as const; +const INITIALIZE_MEASUREMENT_STEP = "Initialize runner comparison evidence"; +const SUMMARIZE_MEASUREMENT_STEP = "Summarize runner comparison evidence"; +const MCP_MEASUREMENT_CONDITION = "${{ matrix.agent == 'hermes' || matrix.agent == 'deepagents' }}"; +const MCP_SUMMARY_CONDITION = + "${{ always() && (matrix.agent == 'hermes' || matrix.agent == 'deepagents') }}"; +const RUNNER_MEASUREMENT_JOBS = [ + { + id: "common-egress-agent", + publicationStep: "Upload common-egress agent artifacts", + runStep: "Run common-egress agent live test", + }, + { + id: "rebuild-hermes", + publicationStep: "Upload Hermes rebuild artifacts", + runStep: "Run Hermes rebuild live test", + }, + { + id: "rebuild-hermes-stale-base", + publicationStep: "Upload Hermes stale-base rebuild artifacts", + runStep: "Run Hermes stale-base rebuild live test", + }, + { + id: "mcp-bridge", + publicationStep: "Scan MCP artifacts for fixture credentials", + runStep: "Run MCP OpenShell provider live test", + }, +] as const; +const RUNNER_MEASUREMENT_JOB_IDS = new Set( + RUNNER_MEASUREMENT_JOBS.map((contract) => contract.id), +); + function asRecord(value: unknown): WorkflowRecord { return value && typeof value === "object" && !Array.isArray(value) ? (value as WorkflowRecord) @@ -21,6 +57,132 @@ function stepsFor(workflow: WorkflowRecord, jobId: string): WorkflowStep[] { return Array.isArray(job.steps) ? (job.steps as WorkflowStep[]) : []; } +function stepIndex(steps: readonly WorkflowStep[], name: string): number { + return steps.findIndex((step) => step.name === name); +} + +function containsEvery(script: string, fragments: readonly string[]): boolean { + return fragments.every((fragment) => script.includes(fragment)); +} + +function validateMeasurementWiring(workflow: WorkflowRecord, errors: string[]): void { + const jobs = asRecord(workflow.jobs); + for (const contract of RUNNER_MEASUREMENT_JOBS) { + const steps = stepsFor(workflow, contract.id); + const prepare = stepIndex(steps, "Prepare E2E workspace"); + const initialize = stepIndex(steps, INITIALIZE_MEASUREMENT_STEP); + const run = stepIndex(steps, contract.runStep); + const summarize = stepIndex(steps, SUMMARIZE_MEASUREMENT_STEP); + const publication = stepIndex(steps, contract.publicationStep); + const initializeSteps = steps.filter((step) => step.name === INITIALIZE_MEASUREMENT_STEP); + const summarySteps = steps.filter((step) => step.name === SUMMARIZE_MEASUREMENT_STEP); + const initializeStep = initializeSteps[0]; + const summaryStep = summarySteps[0]; + const initializeScript = typeof initializeStep?.run === "string" ? initializeStep.run : ""; + const summaryScript = typeof summaryStep?.run === "string" ? summaryStep.run : ""; + + if ( + initializeSteps.length !== 1 || + prepare < 0 || + initialize !== prepare + 1 || + run <= initialize + ) { + errors.push( + `${contract.id} must initialize one runner comparison ledger after workspace preparation and before execution`, + ); + } + if ( + !containsEvery(initializeScript, [ + 'snapshots_file="$E2E_ARTIFACT_DIR/runner-resource-snapshots.jsonl"', + 'summary_file="$E2E_ARTIFACT_DIR/runner-resource-summary.json"', + 'export E2E_RESOURCE_SNAPSHOTS_FILE="$snapshots_file"', + 'export E2E_RESOURCE_SUMMARY_FILE="$summary_file"', + "runner-pressure.mts initialize-measurement", + "E2E_RESOURCE_SNAPSHOTS_FILE=%s", + "E2E_RESOURCE_SUMMARY_FILE=%s", + '>> "$GITHUB_ENV"', + ]) + ) { + errors.push( + `${contract.id} must export the exact private runner comparison paths to later test processes`, + ); + } + if ( + summarySteps.length !== 1 || + summarize !== run + 1 || + publication <= summarize || + !containsEvery(summaryScript, [ + 'export E2E_RESOURCE_SNAPSHOTS_FILE="$E2E_ARTIFACT_DIR/runner-resource-snapshots.jsonl"', + 'export E2E_RESOURCE_SUMMARY_FILE="$E2E_ARTIFACT_DIR/runner-resource-summary.json"', + "runner-pressure.mts summarize-measurement", + ]) + ) { + errors.push( + `${contract.id} must append a final runner snapshot and summarize before artifact publication`, + ); + } + + const expectedInitializeCondition = + contract.id === "mcp-bridge" ? MCP_MEASUREMENT_CONDITION : undefined; + const expectedSummaryCondition = + contract.id === "mcp-bridge" ? MCP_SUMMARY_CONDITION : "always()"; + if ( + initializeStep?.if !== expectedInitializeCondition || + summaryStep?.if !== expectedSummaryCondition || + initializeStep?.["continue-on-error"] !== true || + summaryStep?.["continue-on-error"] !== true + ) { + errors.push( + contract.id === "mcp-bridge" + ? "mcp-bridge runner comparison evidence must be best-effort and limited to Hermes and Deep Agents shards" + : `${contract.id} runner comparison evidence must remain best-effort after every outcome`, + ); + } + } + + for (const [jobId, jobValue] of Object.entries(jobs)) { + if (RUNNER_MEASUREMENT_JOB_IDS.has(jobId)) continue; + const serialized = JSON.stringify(jobValue); + if ( + serialized.includes(INITIALIZE_MEASUREMENT_STEP) || + serialized.includes(SUMMARIZE_MEASUREMENT_STEP) || + serialized.includes("E2E_RESOURCE_SNAPSHOTS_FILE") || + serialized.includes("E2E_RESOURCE_SUMMARY_FILE") || + serialized.includes("initialize-measurement") || + serialized.includes("summarize-measurement") + ) { + errors.push(`runner comparison evidence must not be wired to out-of-scope job ${jobId}`); + } + } + + const mcpRunStep = stepsFor(workflow, "mcp-bridge").find( + (step) => step.name === "Run MCP OpenShell provider live test", + ); + const mcpScript = typeof mcpRunStep?.run === "string" ? mcpRunStep.run : ""; + const mainTest = mcpScript.indexOf( + "live-vitest-invocation.mts run --test-path test/e2e/live/mcp-bridge.test.ts", + ); + const deepAgentsGuard = mcpScript.indexOf( + 'if [[ "$NEMOCLAW_MCP_BRIDGE_AGENT" == "deepagents" ]]', + ); + const credentialWindow = mcpScript.indexOf( + "test/e2e/live/openshell-credential-generation-window.test.ts", + ); + const serializedRegion = + mainTest >= 0 && credentialWindow > mainTest ? mcpScript.slice(mainTest, credentialWindow) : ""; + if ( + mainTest < 0 || + deepAgentsGuard <= mainTest || + credentialWindow <= deepAgentsGuard || + !mcpScript.slice(credentialWindow).includes("--no-file-parallelism") || + /(?:^|[^&])&(?!&)/u.test(serializedRegion) + ) { + errors.push( + "mcp-bridge Deep Agents runner comparison tests must remain serialized in one job ledger", + ); + } +} + /** * Protect the canonical #7101 Hermes heartbeat integration from becoming an * emit-only diagnostic. Ordinary failures must produce, consume, and upload @@ -91,5 +253,6 @@ export function validateRunnerPressureWorkflow(workflowValue: unknown): string[] errors.push(`${contract.id} must upload runner-pressure evidence after every outcome`); } } + validateMeasurementWiring(workflow, errors); return errors; } diff --git a/tools/e2e/runner-pressure.mts b/tools/e2e/runner-pressure.mts index cea357cfa91..65cfedc9507 100644 --- a/tools/e2e/runner-pressure.mts +++ b/tools/e2e/runner-pressure.mts @@ -24,6 +24,10 @@ * descriptor when `E2E_TERMINAL_CLASSIFICATION_FILE` is * configured. Requires the baseline file named by * `E2E_RESOURCE_BASELINE_FILE`. + * - `initialize-measurement` — create one private job-level resource ledger + * and summary, then append the initial lightweight sample. + * - `summarize-measurement` — append the final lightweight sample and reduce + * the ledger into the numeric comparison artifact. * - `validate-classification` — fail closed unless the named classification * artifact contains exactly one canonical terminal line. * - `decide-retry` — print a JSON retry decision from `E2E_RUNNER_LOSS`, @@ -53,11 +57,13 @@ import { parseCgroupMemoryEvents, parseCgroupScalar, parseClassificationLine, + parseCpuTimes, parseDockerStats, parseDockerSystemDf, parseLoadAverages, parseMeminfo, parsePressure, + parseSnapshotLine, parseTopProcesses, type ResourceBaseline, type ResourceSnapshot, @@ -65,6 +71,7 @@ import { renderClassificationLine, renderSnapshotLine, selectFailureBaseline, + summarizeResourceSnapshots, TERMINAL_CLASSIFICATIONS, type TerminalClassification, } from "./runner-pressure-core.mts"; @@ -74,6 +81,8 @@ const CONTAINER_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/u; const BASELINE_FILE_MAX_BYTES = 4096; const PHASE_BASELINES_FILE_MAX_BYTES = 128 * BASELINE_FILE_MAX_BYTES; const CLASSIFICATION_FILE_MAX_BYTES = 2048; +const RESOURCE_SNAPSHOTS_FILE_MAX_BYTES = 512 * 1024; +const RESOURCE_SNAPSHOT_MAX_LINES = 128; function readTextOrNull(path: string): string | null { try { @@ -106,21 +115,34 @@ function collectDisk(): ResourceSnapshot["disk"] { } } -export function collectResourceSnapshot(phase: string): ResourceSnapshot { +export function collectResourceSnapshot( + phase: string, + options: { includeDiagnostics?: boolean } = {}, +): ResourceSnapshot { + const includeDiagnostics = options.includeDiagnostics ?? true; + const at = new Date().toISOString(); + const cpuText = readTextOrNull("/proc/stat"); const meminfoText = readTextOrNull("/proc/meminfo"); - const loadText = readTextOrNull("/proc/loadavg"); + const loadText = includeDiagnostics ? readTextOrNull("/proc/loadavg") : null; const current = readTextOrNull(`${CGROUP_ROOT}/memory.current`); const peak = readTextOrNull(`${CGROUP_ROOT}/memory.peak`); const limit = readTextOrNull(`${CGROUP_ROOT}/memory.max`); const events = readTextOrNull(`${CGROUP_ROOT}/memory.events`); - const memoryPressure = readTextOrNull(`${CGROUP_ROOT}/memory.pressure`); - const ioPressure = readTextOrNull(`${CGROUP_ROOT}/io.pressure`); - const psText = runOrNull("ps", ["-eo", "rss="]); - const statsText = runOrNull("docker", ["stats", "--no-stream", "--format", "{{json .}}"]); - const dfText = runOrNull("docker", ["system", "df", "--format", "{{json .}}"]); + const memoryPressure = includeDiagnostics + ? readTextOrNull(`${CGROUP_ROOT}/memory.pressure`) + : null; + const ioPressure = includeDiagnostics ? readTextOrNull(`${CGROUP_ROOT}/io.pressure`) : null; + const psText = includeDiagnostics ? runOrNull("ps", ["-eo", "rss="]) : null; + const statsText = includeDiagnostics + ? runOrNull("docker", ["stats", "--no-stream", "--format", "{{json .}}"]) + : null; + const dfText = includeDiagnostics + ? runOrNull("docker", ["system", "df", "--format", "{{json .}}"]) + : null; return { phase, - at: new Date().toISOString(), + at, + cpu: cpuText === null ? null : parseCpuTimes(cpuText), meminfo: meminfoText === null ? null : parseMeminfo(meminfoText), load: loadText === null ? null : parseLoadAverages(loadText), cgroup: @@ -146,6 +168,69 @@ function runSnapshot(): void { console.log(renderSnapshotLine(collectResourceSnapshot(phase))); } +/** Append one already-collected snapshot without collecting diagnostics twice. */ +export function appendResourceSnapshot(path: string, snapshot: ResourceSnapshot): void { + appendPrivateRegularFile( + assertEvidencePath(path, "E2E_RESOURCE_SNAPSHOTS_FILE"), + `${renderSnapshotLine(snapshot)}\n`, + { maxBytes: RESOURCE_SNAPSHOTS_FILE_MAX_BYTES }, + ); +} + +function measurementPhase(suffix: "start" | "finish"): string { + const targetId = assertPhaseLabel(process.env.E2E_TARGET_ID); + return assertPhaseLabel(`${targetId}.measurement-${suffix}`); +} + +function runInitializeMeasurement(): void { + const snapshotsPath = assertEvidencePath( + process.env.E2E_RESOURCE_SNAPSHOTS_FILE, + "E2E_RESOURCE_SNAPSHOTS_FILE", + ); + const summaryPath = assertEvidencePath( + process.env.E2E_RESOURCE_SUMMARY_FILE, + "E2E_RESOURCE_SUMMARY_FILE", + ); + writePrivateRegularFile(snapshotsPath, ""); + writePrivateRegularFile(summaryPath, ""); + appendResourceSnapshot( + snapshotsPath, + collectResourceSnapshot(measurementPhase("start"), { includeDiagnostics: false }), + ); +} + +function runSummarizeMeasurement(): void { + const snapshotsPath = assertEvidencePath( + process.env.E2E_RESOURCE_SNAPSHOTS_FILE, + "E2E_RESOURCE_SNAPSHOTS_FILE", + ); + const summaryPath = assertEvidencePath( + process.env.E2E_RESOURCE_SUMMARY_FILE, + "E2E_RESOURCE_SUMMARY_FILE", + ); + appendResourceSnapshot( + snapshotsPath, + collectResourceSnapshot(measurementPhase("finish"), { includeDiagnostics: false }), + ); + const text = readPrivateRegularFile(snapshotsPath, { + maxBytes: RESOURCE_SNAPSHOTS_FILE_MAX_BYTES, + }); + if (text === null) throw new Error("E2E_RESOURCE_SNAPSHOTS_FILE could not be read"); + const lines = text.split(/\r?\n/u).filter((line) => line.length > 0); + if (lines.length < 2 || lines.length > RESOURCE_SNAPSHOT_MAX_LINES) { + throw new Error( + `resource measurement ledger must contain 2-${RESOURCE_SNAPSHOT_MAX_LINES} snapshots`, + ); + } + const targetId = assertPhaseLabel(process.env.E2E_TARGET_ID); + const shardId = process.env.NEMOCLAW_E2E_SHARD; + const summary = summarizeResourceSnapshots(lines.map(parseSnapshotLine), { + targetId, + ...(shardId ? { shardId: assertPhaseLabel(shardId) } : {}), + }); + writePrivateRegularFile(summaryPath, `${JSON.stringify(summary, null, 2)}\n`); +} + function containerOomKilled(name: string | undefined): boolean { if (!name) return false; if (!CONTAINER_NAME_PATTERN.test(name)) { @@ -319,6 +404,12 @@ function main(): number { case "initialize-evidence": runInitializeEvidence(); return 0; + case "initialize-measurement": + runInitializeMeasurement(); + return 0; + case "summarize-measurement": + runSummarizeMeasurement(); + return 0; case "classify": runClassify(); return 0; @@ -330,7 +421,7 @@ function main(): number { return 0; default: console.error( - "usage: runner-pressure.mts ", + "usage: runner-pressure.mts ", ); return 2; } diff --git a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts index fb78f585949..b491d1bdb12 100644 --- a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts +++ b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts @@ -54,6 +54,13 @@ type ExplicitUploadContract = { }; const EXPLICIT_UPLOAD_CONTRACTS = new Map([ + [ + "scorecard", + { + name: "e2e-runtime-summary", + path: "${{ runner.temp }}/e2e-runtime-summary.json", + }, + ], [ "live", { @@ -174,6 +181,7 @@ const EXPLICIT_UPLOAD_CONTRACTS = new Map([ const EXPLICIT_CALLER_CONDITIONS = new Map([ ["mcp-bridge", MCP_SCANNED_UPLOAD_CONDITION], ["mcp-bridge-dev", MCP_SCANNED_UPLOAD_CONDITION], + ["scorecard", "${{ always() && steps.scorecard.outcome == 'success' }}"], ]); const EXPECTED_ACTION_INPUTS = { @@ -279,6 +287,7 @@ export function validateUploadE2eArtifactsInvocations(workflow: WorkflowRecord): const env = record(job.env); return ( jobName === "live" || + jobName === "scorecard" || env.E2E_JOB === "1" || env.NEMOCLAW_RUN_LIVE_E2E === "1" || SHARED_E2E_JOBS.has(jobName) ||