From aa712209b05dc67478804206c287c72c7fcf4fdf Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Fri, 21 Aug 2026 07:29:50 -0700 Subject: [PATCH 1/4] fix(e2e): bound sandbox phase latency tails Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- .github/workflows/e2e.yaml | 10 +- ci/onboard-performance-budget.json | 3 +- schemas/onboard-config.schema.json | 6 + .../scorecard/analyze-first-turn-latency.mts | 45 ++- scripts/scorecard/analyze-runtime-history.mts | 122 ++++++-- .../scorecard/analyze-sandbox-phase-tail.mts | 286 ++++++++++++++++++ test/e2e/README.md | 24 +- test/e2e/fixtures/onboard-performance.ts | 32 +- test/e2e/live/full-e2e.test.ts | 17 +- .../e2e-first-turn-latency-history.test.ts | 17 +- .../e2e-operations-workflow-boundary.test.ts | 28 +- test/e2e/support/e2e-runtime-history.test.ts | 111 ++++++- .../e2e-sandbox-phase-tail-history.test.ts | 225 ++++++++++++++ test/e2e/support/onboard-performance.test.ts | 62 ++++ .../onboard-performance-config-schema.test.ts | 20 +- tools/e2e/operations-workflow-boundary.mts | 8 +- 16 files changed, 956 insertions(+), 60 deletions(-) create mode 100644 scripts/scorecard/analyze-sandbox-phase-tail.mts create mode 100644 test/e2e/support/e2e-sandbox-phase-tail-history.test.ts diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 46297c6107..99d2febafe 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -2776,6 +2776,7 @@ jobs: e2e-artifacts/live/${{ matrix.id }}/state-validation.result.json e2e-artifacts/live/${{ matrix.id }}/dcode-base-image.json e2e-artifacts/live/${{ matrix.id }}/cloud-onboard-trace-timing-summary.json + e2e-artifacts/live/${{ matrix.id }}/onboard-progress-budget.json e2e-artifacts/live/risk-signal.json e2e-artifacts/live/${{ matrix.id }}/actions/ e2e-artifacts/live/${{ matrix.id }}/logs/ @@ -5703,6 +5704,9 @@ jobs: const firstTurnLatency = require( path.join(process.env.GITHUB_WORKSPACE, 'scripts/scorecard/analyze-first-turn-latency.mts'), ); + const sandboxPhaseTail = require( + path.join(process.env.GITHUB_WORKSPACE, 'scripts/scorecard/analyze-sandbox-phase-tail.mts'), + ); const needs = JSON.parse(process.env.NEEDS_JSON || '{}'); // GitHub's jobs API is the canonical source because `needs.live` @@ -5742,7 +5746,11 @@ jobs: firstTurnLatency.readCurrentFirstTurnLatencySample( process.env.RUNTIME_ARTIFACTS, ), - loadPriorPushSummaries: runtimeHistory.loadPriorPushSummaries, + currentSandboxPhaseTail: + sandboxPhaseTail.readCurrentSandboxPhaseTailSample( + process.env.RUNTIME_ARTIFACTS, + ), + loadPriorPushHistory: runtimeHistory.loadPriorPushHistory, }, ); const trace = await traceTiming.buildTraceTimingResult({ github, context, core }); diff --git a/ci/onboard-performance-budget.json b/ci/onboard-performance-budget.json index 40afc27268..f71fe496df 100644 --- a/ci/onboard-performance-budget.json +++ b/ci/onboard-performance-budget.json @@ -1,5 +1,5 @@ { - "$comment": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\nSPDX-License-Identifier: Apache-2.0\n\nInitial advisory budget for the cloud-onboard-e2e warm-system trace signal. Profiling traces from #3769 were not available in durable CI artifacts when #3776 was implemented. The latest three release tags only exposed one tag-matching trace artifact, from a failed v0.0.66 nightly run, so this cap is calibrated from the latest ten distinct successful main full-trace samples available on 2026-06-23. Those samples had total durations of 298250 ms, 296926 ms, 304190 ms, 294859 ms, 305013 ms, 316147 ms, 300843 ms, 292702 ms, 201332 ms, and 206250 ms; the cap uses p95 via linear interpolation (index 8.55 between samples 8 and 9) plus 25 percent, rounded up to the nearest 30 seconds.\n\nThe full-e2e cold-path baseline budgets are derived from the five current-main samples recorded in ci/full-e2e-cold-path-calibration.json. For each interval and phase independently, the baseline is nearest-rank p95 plus the larger of 5 seconds or 10 percent, rounded up to the nearest second. The calibration records bounded post-change adjustments for the supported sandbox image and the restart-safe OpenClaw Docker startup lifecycle. Each adjustment raises only the root-start and sandbox-phase caps to its observed maximum plus 10 percent, rounded up to one second, and is retired when five successful samples from one eligible head replace the baseline. Independent phase caps are diagnostic regression gates, not additive portions of the root-start interval. The authoritative local base-build allowance is derived separately from exact PR evidence recorded in ci/full-e2e-cold-path-calibration.json. It applies only when full-e2e observes the exact local-build reason and adjusts only the root-start and sandbox-phase caps; published-image runs retain the normal budgets.", + "$comment": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\nSPDX-License-Identifier: Apache-2.0\n\nInitial advisory budget for the cloud-onboard-e2e warm-system trace signal. Profiling traces from #3769 were not available in durable CI artifacts when #3776 was implemented. The latest three release tags only exposed one tag-matching trace artifact, from a failed v0.0.66 nightly run, so this cap is calibrated from the latest ten distinct successful main full-trace samples available on 2026-06-23. Those samples had total durations of 298250 ms, 296926 ms, 304190 ms, 294859 ms, 305013 ms, 316147 ms, 300843 ms, 292702 ms, 201332 ms, and 206250 ms; the cap uses p95 via linear interpolation (index 8.55 between samples 8 and 9) plus 25 percent, rounded up to the nearest 30 seconds.\n\nThe full-e2e cold-path baseline budgets are derived from the five current-main samples recorded in ci/full-e2e-cold-path-calibration.json. For each interval and phase independently, the baseline is nearest-rank p95 plus the larger of 5 seconds or 10 percent, rounded up to the nearest second. The calibration records bounded post-change adjustments for the supported sandbox image and the restart-safe OpenClaw Docker startup lifecycle. Each adjustment raises only the root-start and sandbox-phase caps to its observed maximum plus 10 percent, rounded up to one second, and is retired when five successful samples from one eligible head replace the baseline. Independent phase caps are diagnostic regression gates, not additive portions of the root-start interval. The authoritative local base-build allowance is derived separately from exact PR evidence recorded in ci/full-e2e-cold-path-calibration.json. It applies only when full-e2e observes the exact local-build reason and adjusts only the root-start and sandbox-phase caps; published-image runs retain the normal budgets. One published-base sandbox-phase overage can use the separate checked-in maximum only through the five-sample recurrence policy in test/e2e/README.md.", "schemaVersion": 1, "mode": "advisory", "scope": "cloud-onboard-e2e warm-system", @@ -14,6 +14,7 @@ }, "fullE2eColdPath": { "authoritativeLocalBaseBuildAllowanceMs": 570000, + "sandboxPhaseSingleObservationMaxOverageMs": 5000, "rootStartToFirstTurnCompletionBudgetMs": 263000, "rootEndToFirstTurnCompletionBudgetMs": 14000, "phaseBudgetsMs": { diff --git a/schemas/onboard-config.schema.json b/schemas/onboard-config.schema.json index 3f9523451a..2b23741357 100644 --- a/schemas/onboard-config.schema.json +++ b/schemas/onboard-config.schema.json @@ -44,6 +44,7 @@ "additionalProperties": false, "required": [ "authoritativeLocalBaseBuildAllowanceMs", + "sandboxPhaseSingleObservationMaxOverageMs", "rootStartToFirstTurnCompletionBudgetMs", "rootEndToFirstTurnCompletionBudgetMs", "phaseBudgetsMs" @@ -54,6 +55,11 @@ "minimum": 0, "description": "Bounded extra time allowed for the root-start and sandbox-phase caps only when full-e2e observes the authoritative local base-image rebuild message." }, + "sandboxPhaseSingleObservationMaxOverageMs": { + "type": "number", + "minimum": 0, + "description": "Maximum sandbox-phase overage that one published-base full-E2E sample can report as a non-blocking anomaly." + }, "rootStartToFirstTurnCompletionBudgetMs": { "type": "number", "minimum": 0, diff --git a/scripts/scorecard/analyze-first-turn-latency.mts b/scripts/scorecard/analyze-first-turn-latency.mts index 64e9226be1..aff2938d9b 100644 --- a/scripts/scorecard/analyze-first-turn-latency.mts +++ b/scripts/scorecard/analyze-first-turn-latency.mts @@ -8,8 +8,9 @@ export const FIRST_TURN_LATENCY_MIN_SAMPLES = 12; export const FIRST_TURN_LATENCY_MAX_ANOMALIES = 1; const FIRST_TURN_ARTIFACT_FILE = "onboard-progress-budget.json"; -const FIRST_TURN_ARTIFACT_SCHEMA = "nemoclaw.full_e2e_cold_performance.v3"; +const FIRST_TURN_ARTIFACT_SCHEMA = "nemoclaw.full_e2e_cold_performance.v4"; const FIRST_TURN_ANOMALY_KIND = "first-turn-latency-tail"; +const SANDBOX_PHASE_ANOMALY_KIND = "sandbox-phase-tail"; const MAX_ARTIFACT_BYTES = 256 * 1024; const MAX_DIRECTORY_DEPTH = 4; const MAX_DIRECTORY_ENTRIES = 10_000; @@ -121,6 +122,23 @@ export function normalizeFirstTurnLatencySample(value: unknown): FirstTurnLatenc }; } +function normalizePerformanceAnomaly(value: unknown): Record | null { + const finding = asRecord(value); + if ( + !finding || + !hasExactKeys(finding, ["budgetMs", "kind", "measurementMs", "overageMs"]) || + (finding.kind !== FIRST_TURN_ANOMALY_KIND && finding.kind !== SANDBOX_PHASE_ANOMALY_KIND) || + !isDuration(finding.budgetMs) || + !isDuration(finding.measurementMs) || + !isDuration(finding.overageMs) || + finding.overageMs !== (finding.measurementMs as number) - (finding.budgetMs as number) || + finding.overageMs <= 0 + ) { + return null; + } + return finding; +} + function findArtifactFiles(root: string): string[] { const matches: string[] = []; let visited = 0; @@ -148,7 +166,7 @@ function findArtifactFiles(root: string): string[] { return matches; } -function readCurrentArtifact(root: string): unknown { +export function readCurrentColdOnboardArtifact(root: string): unknown { const matches = findArtifactFiles(root); if (matches.length !== 1) return null; let descriptor: number | null = null; @@ -165,7 +183,7 @@ function readCurrentArtifact(root: string): unknown { } export function readCurrentFirstTurnLatencySample(root: string): FirstTurnLatencySample | null { - const artifact = asRecord(readCurrentArtifact(root)); + const artifact = asRecord(readCurrentColdOnboardArtifact(root)); if (!artifact) return null; const performance = asRecord(artifact.performance); const phaseMeasurements = asRecord(artifact.phaseMeasurements); @@ -200,19 +218,18 @@ export function readCurrentFirstTurnLatencySample(root: string): FirstTurnLatenc const measurementMs = phaseMeasurements.rootEndToFirstTurnCompletionMs; const budgetMs = budget.rootEndToFirstTurnCompletionBudgetMs; const overageMs = Math.max(0, measurementMs - budgetMs); - const anomaly = performance.anomalies.length === 1; + const findings = performance.anomalies.map(normalizePerformanceAnomaly); + if (findings.some((finding) => finding === null)) return null; + const finding = findings.find((candidate) => candidate?.kind === FIRST_TURN_ANOMALY_KIND); + const anomaly = finding !== undefined; if (anomaly !== overageMs > 0) return null; - if (anomaly) { - const finding = asRecord(performance.anomalies[0]); - if ( - !finding || - finding.kind !== FIRST_TURN_ANOMALY_KIND || - finding.measurementMs !== measurementMs || + if ( + finding && + (finding.measurementMs !== measurementMs || finding.budgetMs !== budgetMs || - finding.overageMs !== overageMs - ) { - return null; - } + finding.overageMs !== overageMs) + ) { + return null; } return { anomaly, budgetMs, cohort, measurementMs, overageMs }; diff --git a/scripts/scorecard/analyze-runtime-history.mts b/scripts/scorecard/analyze-runtime-history.mts index 40ab5cb7d2..8062b0887a 100644 --- a/scripts/scorecard/analyze-runtime-history.mts +++ b/scripts/scorecard/analyze-runtime-history.mts @@ -14,6 +14,12 @@ import { formatFirstTurnLatencyRecurrence, normalizeFirstTurnLatencySample, } from "./analyze-first-turn-latency.mts"; +import { + evaluateSandboxPhaseTailRecurrence, + formatSandboxPhaseTailRecurrence, + normalizeSandboxPhaseTailSample, + type SandboxPhaseTailSample, +} from "./analyze-sandbox-phase-tail.mts"; import { readValidatedArtifactZipEntry } from "./read-artifact-zip.mts"; export const RUNTIME_SUMMARY_ARTIFACT = "e2e-runtime-summary"; @@ -22,7 +28,8 @@ export const RUNTIME_REGRESSION_MIN_DELTA_MS = 30_000; export const RUNTIME_REGRESSION_MIN_PERCENT = 20; const LEGACY_RUNTIME_SUMMARY_SCHEMA = "nemoclaw.e2e_runtime_summary.v1"; -const RUNTIME_SUMMARY_SCHEMA = "nemoclaw.e2e_runtime_summary.v2"; +const PREVIOUS_RUNTIME_SUMMARY_SCHEMA = "nemoclaw.e2e_runtime_summary.v2"; +const RUNTIME_SUMMARY_SCHEMA = "nemoclaw.e2e_runtime_summary.v3"; const WORKFLOW_FILE = "e2e.yaml"; const HISTORY_RUN_LIMIT = 30; const HISTORY_QUERY_LIMIT = 30; @@ -43,18 +50,28 @@ type GitHubDeps = { }; export interface RuntimeSummaryArtifact { - schemaVersion: typeof LEGACY_RUNTIME_SUMMARY_SCHEMA | typeof RUNTIME_SUMMARY_SCHEMA; + schemaVersion: + | typeof LEGACY_RUNTIME_SUMMARY_SCHEMA + | typeof PREVIOUS_RUNTIME_SUMMARY_SCHEMA + | typeof RUNTIME_SUMMARY_SCHEMA; runId: number; createdAt: string; firstTurnLatency: FirstTurnLatencySample | null; + sandboxPhaseTail: SandboxPhaseTailSample | null; rows: RuntimeHistorySample[]; } type RuntimeHistoryServices = { currentFirstTurnLatency?: FirstTurnLatencySample | null; - loadPriorPushSummaries: (deps: GitHubDeps) => Promise; + currentSandboxPhaseTail?: SandboxPhaseTailSample | null; + loadPriorPushHistory: (deps: GitHubDeps) => Promise; }; +export interface PriorPushHistory { + summaries: RuntimeSummaryArtifact[]; + unavailableRuns: number; +} + function hasExactKeys(value: Record, expected: readonly string[]): boolean { return Object.keys(value).sort().join("\0") === [...expected].sort().join("\0"); } @@ -138,14 +155,18 @@ export function normalizeRuntimeSummary(value: unknown): RuntimeSummaryArtifact if (!value || typeof value !== "object" || Array.isArray(value)) return null; const summary = value as Record; const legacy = summary.schemaVersion === LEGACY_RUNTIME_SUMMARY_SCHEMA; + const previous = summary.schemaVersion === PREVIOUS_RUNTIME_SUMMARY_SCHEMA; + const current = summary.schemaVersion === RUNTIME_SUMMARY_SCHEMA; if ( + (!legacy && !previous && !current) || !hasExactKeys( summary, legacy ? ["schemaVersion", "runId", "createdAt", "rows"] - : ["schemaVersion", "runId", "createdAt", "firstTurnLatency", "rows"], + : previous + ? ["schemaVersion", "runId", "createdAt", "firstTurnLatency", "rows"] + : ["schemaVersion", "runId", "createdAt", "firstTurnLatency", "sandboxPhaseTail", "rows"], ) || - (!legacy && summary.schemaVersion !== RUNTIME_SUMMARY_SCHEMA) || !Number.isSafeInteger(summary.runId) || (summary.runId as number) < 1 || !isCanonicalTimestamp(summary.createdAt) || @@ -164,11 +185,21 @@ export function normalizeRuntimeSummary(value: unknown): RuntimeSummaryArtifact ? null : normalizeFirstTurnLatencySample(summary.firstTurnLatency); if (!legacy && summary.firstTurnLatency !== null && firstTurnLatency === null) return null; + const sandboxPhaseTail = + current && summary.sandboxPhaseTail !== null + ? normalizeSandboxPhaseTailSample(summary.sandboxPhaseTail) + : null; + if (current && summary.sandboxPhaseTail !== null && sandboxPhaseTail === null) return null; return { - schemaVersion: legacy ? LEGACY_RUNTIME_SUMMARY_SCHEMA : RUNTIME_SUMMARY_SCHEMA, + schemaVersion: legacy + ? LEGACY_RUNTIME_SUMMARY_SCHEMA + : previous + ? PREVIOUS_RUNTIME_SUMMARY_SCHEMA + : RUNTIME_SUMMARY_SCHEMA, runId: summary.runId as number, createdAt: summary.createdAt, firstTurnLatency, + sandboxPhaseTail, rows: normalizedRows, }; } @@ -178,12 +209,14 @@ export function createRuntimeSummary( createdAt: string, rows: readonly RuntimeHistorySample[], firstTurnLatency: FirstTurnLatencySample | null = null, + sandboxPhaseTail: SandboxPhaseTailSample | null = null, ): RuntimeSummaryArtifact { const summary = normalizeRuntimeSummary({ schemaVersion: RUNTIME_SUMMARY_SCHEMA, runId, createdAt, firstTurnLatency, + sandboxPhaseTail, rows, }); if (summary === null) throw new Error("invalid current E2E runtime summary"); @@ -225,9 +258,7 @@ async function readRuntimeSummaryFromRun( return summary?.runId === runId ? summary : null; } -export async function loadPriorPushSummaries( - deps: GitHubDeps, -): Promise { +export async function loadPriorPushHistory(deps: GitHubDeps): Promise { const { github, context, core } = deps; const response = await github.rest.actions.listWorkflowRuns({ owner: context.repo.owner, @@ -239,19 +270,26 @@ export async function loadPriorPushSummaries( }); const runs = response.data.workflow_runs as Array<{ id: number }>; const summaries: RuntimeSummaryArtifact[] = []; + let unavailableRuns = 0; for (const run of runs) { if (run.id === context.runId) continue; try { const summary = await readRuntimeSummaryFromRun(deps, run.id); - if (summary !== null) summaries.push(summary); + if (summary === null) unavailableRuns += 1; + else summaries.push(summary); } catch { + unavailableRuns += 1; core?.warning?.( "One prior push runtime summary was unavailable; continuing with less history.", ); } if (summaries.length === HISTORY_RUN_LIMIT) break; } - return summaries; + return { summaries, unavailableRuns }; +} + +export async function loadPriorPushSummaries(deps: GitHubDeps): Promise { + return (await loadPriorPushHistory(deps)).summaries; } function percentile(sorted: readonly number[], fraction: number): number { @@ -460,9 +498,7 @@ export function formatRuntimeHistory( return `${lines.join("\n")}\n`; } if (sortedPrior.length === 0) { - lines.push( - "No prior push runtime summaries are available yet; this run starts the history.", - ); + lines.push("No prior push runtime summaries are available yet; this run starts the history."); return `${lines.join("\n")}\n`; } @@ -497,11 +533,16 @@ export async function buildRuntimeHistory( deps: GitHubDeps, currentRows: readonly RuntimeHistorySample[], outputPath: string, - services: RuntimeHistoryServices = { loadPriorPushSummaries }, + services: RuntimeHistoryServices = { loadPriorPushHistory }, now = new Date(), ): Promise { const hasFirstTurnLatency = Object.hasOwn(services, "currentFirstTurnLatency"); + const hasSandboxPhaseTail = Object.hasOwn(services, "currentSandboxPhaseTail"); const currentFirstTurnLatency = services.currentFirstTurnLatency ?? null; + const fullE2EPassed = currentRows.some( + (row) => row.target === "full-e2e" && row.outcome === "passed", + ); + const currentSandboxPhaseTail = fullE2EPassed ? (services.currentSandboxPhaseTail ?? null) : null; let current: RuntimeSummaryArtifact; try { current = createRuntimeSummary( @@ -509,6 +550,7 @@ export async function buildRuntimeHistory( now.toISOString(), currentRows, currentFirstTurnLatency, + currentSandboxPhaseTail, ); const serialized = `${JSON.stringify(current, null, 2)}\n`; if (Buffer.byteLength(serialized) > MAX_SUMMARY_BYTES) { @@ -519,23 +561,49 @@ export async function buildRuntimeHistory( deps.core?.warning?.( "Current E2E runtime summary was invalid or could not be saved; push history is unavailable.", ); + if (currentSandboxPhaseTail?.anomaly) { + deps.core?.setFailed?.( + "Current sandbox phase anomaly could not be saved for recurrence enforcement.", + ); + } return formatRuntimeHistory([], []); } try { - const prior = await services.loadPriorPushSummaries(deps); + const priorHistory = await services.loadPriorPushHistory(deps); + const prior = priorHistory.summaries; const runtimeHistory = formatRuntimeHistory(current.rows, prior); - if (!hasFirstTurnLatency) return runtimeHistory; - const recurrence = evaluateFirstTurnLatencyRecurrence(current.firstTurnLatency, prior); - if (!recurrence.passed && recurrence.message) deps.core?.setFailed?.(recurrence.message); - return `${runtimeHistory}\n${formatFirstTurnLatencyRecurrence(recurrence)}`; + const sections = [runtimeHistory]; + if (hasFirstTurnLatency) { + const recurrence = evaluateFirstTurnLatencyRecurrence(current.firstTurnLatency, prior); + if (!recurrence.passed && recurrence.message) deps.core?.setFailed?.(recurrence.message); + sections.push(formatFirstTurnLatencyRecurrence(recurrence)); + } + if (hasSandboxPhaseTail) { + const recurrence = evaluateSandboxPhaseTailRecurrence( + current.sandboxPhaseTail, + prior, + priorHistory.unavailableRuns === 0, + ); + if (!recurrence.passed && recurrence.message) deps.core?.setFailed?.(recurrence.message); + sections.push(formatSandboxPhaseTailRecurrence(recurrence)); + } + return sections.join("\n"); } catch { - deps.core?.warning?.( - "Push E2E runtime history unavailable; current summary was still saved.", - ); + deps.core?.warning?.("Push E2E runtime history unavailable; current summary was still saved."); const runtimeHistory = formatRuntimeHistory(current.rows, []); - if (!hasFirstTurnLatency) return runtimeHistory; - return `${runtimeHistory}\n${formatFirstTurnLatencyRecurrence( - evaluateFirstTurnLatencyRecurrence(current.firstTurnLatency, []), - )}`; + const sections = [runtimeHistory]; + if (hasFirstTurnLatency) { + sections.push( + formatFirstTurnLatencyRecurrence( + evaluateFirstTurnLatencyRecurrence(current.firstTurnLatency, []), + ), + ); + } + if (hasSandboxPhaseTail) { + const recurrence = evaluateSandboxPhaseTailRecurrence(current.sandboxPhaseTail, [], false); + if (!recurrence.passed && recurrence.message) deps.core?.setFailed?.(recurrence.message); + sections.push(formatSandboxPhaseTailRecurrence(recurrence)); + } + return sections.join("\n"); } } diff --git a/scripts/scorecard/analyze-sandbox-phase-tail.mts b/scripts/scorecard/analyze-sandbox-phase-tail.mts new file mode 100644 index 0000000000..08d7049663 --- /dev/null +++ b/scripts/scorecard/analyze-sandbox-phase-tail.mts @@ -0,0 +1,286 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { readCurrentColdOnboardArtifact } from "./analyze-first-turn-latency.mts"; + +export const SANDBOX_PHASE_TAIL_MIN_SAMPLES = 5; +export const SANDBOX_PHASE_TAIL_MAX_ANOMALIES = 1; + +const COLD_ONBOARD_ARTIFACT_SCHEMA = "nemoclaw.full_e2e_cold_performance.v4"; +const SANDBOX_PHASE = "nemoclaw.onboard.phase.sandbox"; +const SANDBOX_PHASE_ANOMALY_KIND = "sandbox-phase-tail"; +const FIRST_TURN_ANOMALY_KIND = "first-turn-latency-tail"; +const MAX_DURATION_MS = 24 * 60 * 60 * 1000; + +export interface SandboxPhaseCohort { + agent: string; + baseBuildMode: string; + platform: string; + setupMode: string; + workloadKind: string; +} + +export interface SandboxPhaseTailSample { + anomaly: boolean; + budgetMs: number; + cohort: SandboxPhaseCohort; + measurementMs: number; + overageMs: number; +} + +export interface SandboxPhaseTailHistorySummary { + createdAt: string; + runId: number; + sandboxPhaseTail: SandboxPhaseTailSample | null; +} + +export interface SandboxPhaseTailRecurrence { + anomalyCount: number; + cohort: SandboxPhaseCohort | null; + currentAnomaly: boolean; + eligibleSamples: number; + message: string | null; + passed: boolean; +} + +function asRecord(value: unknown): Record | null { + return value !== null && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : null; +} + +function hasExactKeys(value: Record, expected: readonly string[]): boolean { + return Object.keys(value).sort().join("\0") === [...expected].sort().join("\0"); +} + +function isBoundedString(value: unknown): value is string { + return ( + typeof value === "string" && + value.length > 0 && + value.length <= 500 && + !/[\u0000-\u001f\u007f]/u.test(value) + ); +} + +function isDuration(value: unknown): value is number { + return ( + typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= MAX_DURATION_MS + ); +} + +function normalizeCohort(value: unknown): SandboxPhaseCohort | null { + const cohort = asRecord(value); + if ( + !cohort || + !hasExactKeys(cohort, ["agent", "baseBuildMode", "platform", "setupMode", "workloadKind"]) || + !isBoundedString(cohort.agent) || + !isBoundedString(cohort.baseBuildMode) || + !isBoundedString(cohort.platform) || + !isBoundedString(cohort.setupMode) || + !isBoundedString(cohort.workloadKind) + ) { + return null; + } + return { + agent: cohort.agent, + baseBuildMode: cohort.baseBuildMode, + platform: cohort.platform, + setupMode: cohort.setupMode, + workloadKind: cohort.workloadKind, + }; +} + +export function normalizeSandboxPhaseTailSample(value: unknown): SandboxPhaseTailSample | null { + const sample = asRecord(value); + if ( + !sample || + !hasExactKeys(sample, ["anomaly", "budgetMs", "cohort", "measurementMs", "overageMs"]) || + typeof sample.anomaly !== "boolean" || + !isDuration(sample.budgetMs) || + !isDuration(sample.measurementMs) || + !isDuration(sample.overageMs) + ) { + return null; + } + const cohort = normalizeCohort(sample.cohort); + if ( + !cohort || + sample.overageMs !== Math.max(0, sample.measurementMs - sample.budgetMs) || + sample.anomaly !== sample.overageMs > 0 + ) { + return null; + } + return { + anomaly: sample.anomaly, + budgetMs: sample.budgetMs, + cohort, + measurementMs: sample.measurementMs, + overageMs: sample.overageMs, + }; +} + +function validWorkloadEvidence( + artifact: Record, + cohort: SandboxPhaseCohort, +): boolean { + const workload = asRecord(artifact.workload); + if (!workload || workload.kind !== cohort.workloadKind) return false; + if (cohort.workloadKind === "legacy-dockerfile") return artifact.usedBuildKitPrebuild === true; + if (cohort.workloadKind === "managed-image") return artifact.usedBuildKitPrebuild === false; + return false; +} + +function normalizeAnomalyFinding(value: unknown): Record | null { + const finding = asRecord(value); + if ( + !finding || + !hasExactKeys(finding, ["budgetMs", "kind", "measurementMs", "overageMs"]) || + (finding.kind !== SANDBOX_PHASE_ANOMALY_KIND && finding.kind !== FIRST_TURN_ANOMALY_KIND) || + !isDuration(finding.budgetMs) || + !isDuration(finding.measurementMs) || + !isDuration(finding.overageMs) || + finding.overageMs !== (finding.measurementMs as number) - (finding.budgetMs as number) || + finding.overageMs <= 0 + ) { + return null; + } + return finding; +} + +export function readCurrentSandboxPhaseTailSample(root: string): SandboxPhaseTailSample | null { + const artifact = asRecord(readCurrentColdOnboardArtifact(root)); + if (!artifact) return null; + const performance = asRecord(artifact.performance); + const phaseMeasurements = asRecord(artifact.phaseMeasurements); + const tracePhases = asRecord(phaseMeasurements?.tracePhasesMs); + const budget = asRecord(artifact.budget); + const phaseBudgets = asRecord(budget?.phaseBudgetsMs); + const cohort = normalizeCohort(artifact.sandboxPhaseCohort); + const anomalyValues = performance?.anomalies; + if ( + artifact.schemaVersion !== COLD_ONBOARD_ARTIFACT_SCHEMA || + artifact.installExitCode !== 0 || + artifact.firstTurnExitCode !== 0 || + artifact.firstTurnSentinelMatched !== true || + artifact.buildKitFallback !== false || + artifact.classicBuildSteps !== 0 || + !isDuration(artifact.maxSilenceSecs) || + !isDuration(artifact.maxSilenceBudgetSecs) || + artifact.maxSilenceSecs > artifact.maxSilenceBudgetSecs || + !performance || + performance.passed !== true || + performance.usedAuthoritativeLocalBaseBuild !== false || + performance.appliedAuthoritativeLocalBaseBuildAllowanceMs !== 0 || + !Array.isArray(performance.violations) || + performance.violations.length !== 0 || + !Array.isArray(anomalyValues) || + anomalyValues.length > 1 || + !tracePhases || + !budget || + !phaseBudgets || + !cohort || + cohort.agent !== "openclaw" || + cohort.baseBuildMode !== "published-base" || + cohort.setupMode !== "source-install" || + !validWorkloadEvidence(artifact, cohort) || + !isDuration(tracePhases[SANDBOX_PHASE]) || + !isDuration(phaseBudgets[SANDBOX_PHASE]) || + !isDuration(budget.sandboxPhaseSingleObservationMaxOverageMs) + ) { + return null; + } + + const findings = anomalyValues.map(normalizeAnomalyFinding); + if (findings.some((finding) => finding === null)) return null; + const sandboxFinding = findings.find((finding) => finding?.kind === SANDBOX_PHASE_ANOMALY_KIND); + const measurementMs = tracePhases[SANDBOX_PHASE]; + const budgetMs = phaseBudgets[SANDBOX_PHASE]; + const overageMs = Math.max(0, measurementMs - budgetMs); + const anomaly = sandboxFinding !== undefined; + if (anomaly !== overageMs > 0) return null; + if ( + sandboxFinding && + (sandboxFinding.measurementMs !== measurementMs || + sandboxFinding.budgetMs !== budgetMs || + sandboxFinding.overageMs !== overageMs || + overageMs > budget.sandboxPhaseSingleObservationMaxOverageMs) + ) { + return null; + } + + return { anomaly, budgetMs, cohort, measurementMs, overageMs }; +} + +function cohortIdentity(cohort: SandboxPhaseCohort): string { + return JSON.stringify([ + cohort.agent, + cohort.setupMode, + cohort.platform, + cohort.baseBuildMode, + cohort.workloadKind, + ]); +} + +export function evaluateSandboxPhaseTailRecurrence( + current: SandboxPhaseTailSample | null, + priorSummaries: readonly SandboxPhaseTailHistorySummary[], + historyComplete = true, +): SandboxPhaseTailRecurrence { + if (current === null) { + return { + anomalyCount: 0, + cohort: null, + currentAnomaly: false, + eligibleSamples: 0, + message: null, + passed: true, + }; + } + + const identity = cohortIdentity(current.cohort); + const priorSamples = [...priorSummaries] + .sort((left, right) => Date.parse(right.createdAt) - Date.parse(left.createdAt)) + .flatMap((summary) => { + const sample = summary.sandboxPhaseTail; + return sample && cohortIdentity(sample.cohort) === identity ? [sample] : []; + }); + const window = [current, ...priorSamples].slice(0, SANDBOX_PHASE_TAIL_MIN_SAMPLES); + const anomalyCount = window.filter((sample) => sample.anomaly).length; + const enoughHistory = historyComplete && window.length === SANDBOX_PHASE_TAIL_MIN_SAMPLES; + const passed = + !current.anomaly || (enoughHistory && anomalyCount <= SANDBOX_PHASE_TAIL_MAX_ANOMALIES); + const cohortLabel = `${current.cohort.agent}/${current.cohort.setupMode}/${current.cohort.platform}/${current.cohort.baseBuildMode}/${current.cohort.workloadKind}`; + const message = passed + ? null + : !historyComplete + ? `sandbox phase anomaly history is incomplete for ${cohortLabel}: one or more prior push summaries are unavailable` + : !enoughHistory + ? `sandbox phase anomaly history is incomplete for ${cohortLabel}: ${window.length} of ${SANDBOX_PHASE_TAIL_MIN_SAMPLES} eligible same-cohort samples are available` + : `sandbox phase latency recurred for ${cohortLabel}: ${anomalyCount} anomalies in ${window.length} eligible same-cohort samples`; + return { + anomalyCount, + cohort: current.cohort, + currentAnomaly: current.anomaly, + eligibleSamples: window.length, + message, + passed, + }; +} + +export function formatSandboxPhaseTailRecurrence(result: SandboxPhaseTailRecurrence): string { + const lines = ["## Sandbox Phase Latency", ""]; + if (result.cohort === null) { + lines.push("No eligible current sandbox-phase sample was available."); + } else if (!result.passed) { + lines.push(`❌ ${result.message}.`); + } else if (result.currentAnomaly) { + lines.push( + `The current anomaly passed with ${result.anomalyCount} anomaly in ${SANDBOX_PHASE_TAIL_MIN_SAMPLES} eligible same-cohort samples.`, + ); + } else { + lines.push( + `The current sample passed with ${result.anomalyCount} anomalies in ${result.eligibleSamples} eligible same-cohort samples.`, + ); + } + return `${lines.join("\n")}\n`; +} diff --git a/test/e2e/README.md b/test/e2e/README.md index 169c4f7d79..8623e6cfc4 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -1363,8 +1363,25 @@ When every deterministic cold-onboard budget passes and the real first turn exit successfully with the expected sentinel, a sole root-end-to-first-turn overage is recorded as a structured, non-blocking hosted-latency anomaly rather than a PR regression. -The same overage remains blocking when accompanied by a root-start or -phase-budget failure. +The same overage remains blocking when accompanied by a root-start or phase-budget failure. + +The checked-in `nemoclaw.onboard.phase.sandbox` budget remains 208,000 ms. +A sandbox-phase overage qualifies for anomaly classification only when it is the sole performance overage and the run uses the published-base build mode without the authoritative local base-build allowance. +For a qualifying overage of at most 5,000 ms, `full-e2e` records a `sandbox-phase-tail` anomaly instead of a blocking performance violation. +An overage greater than 5,000 ms remains blocking. +A run that applies the authoritative local base-build allowance or has another performance violation also remains blocking. +Every other performance contract remains blocking, as do the existing first-turn command exit, BuildKit, gateway-builder no-fallback, output-silence, sentinel, E2E job outcome, and cleanup contracts. + +For `sandbox-phase-tail`, the trusted push scorecard uses the latest five eligible samples from the same agent, setup mode, platform, base-build mode, and workload kind. +A current anomaly passes only when four valid prior same-cohort samples exist and none contains a sandbox-phase anomaly. +A current anomaly remains blocking in these cases: + +- One or more queried prior push summaries are unavailable. +- Fewer than four valid prior same-cohort samples are available. +- One prior sample in the five-sample window contains a sandbox-phase anomaly. + +The second anomaly in the five-sample window therefore blocks immediately. +These sandbox-phase recurrence rules do not change the hosted first-turn policy described below. The trusted push scorecard stores the current eligible sample in the `e2e-runtime-summary` artifact. @@ -1379,6 +1396,9 @@ Missing, malformed, or functionally unsuccessful samples do not enter the window The scorecard waits for 12 eligible samples when retained history is incomplete. The canonical E2E uploader retains each push summary for 14 days. +The sandbox-phase recurrence rule does not recalibrate the checked-in budget. +Recalibration remains deferred until five successful samples from the same commit are available. + When changed base-image inputs require the authoritative local OpenClaw base build, the target applies the separately calibrated 90-second allowance only to the root-start and sandbox-phase limits. The installer must emit the exact local diff --git a/test/e2e/fixtures/onboard-performance.ts b/test/e2e/fixtures/onboard-performance.ts index f6c786ee7f..80d33073e0 100644 --- a/test/e2e/fixtures/onboard-performance.ts +++ b/test/e2e/fixtures/onboard-performance.ts @@ -24,6 +24,7 @@ const COLD_ONBOARD_BUDGET_KEYS = new Set([ "authoritativeLocalBaseBuildAllowanceMs", "rootStartToFirstTurnCompletionBudgetMs", "rootEndToFirstTurnCompletionBudgetMs", + "sandboxPhaseSingleObservationMaxOverageMs", "phaseBudgetsMs", ]); @@ -39,6 +40,7 @@ export interface ColdOnboardPerformanceBudget { phaseBudgetsMs: Record; rootEndToFirstTurnCompletionBudgetMs: number; rootStartToFirstTurnCompletionBudgetMs: number; + sandboxPhaseSingleObservationMaxOverageMs: number; } export interface ColdOnboardPerformanceEvaluation { @@ -52,7 +54,7 @@ export interface ColdOnboardPerformanceEvaluation { export interface ColdOnboardPerformanceAnomaly { budgetMs: number; - kind: "first-turn-latency-tail"; + kind: "first-turn-latency-tail" | "sandbox-phase-tail"; measurementMs: number; overageMs: number; } @@ -60,6 +62,7 @@ export interface ColdOnboardPerformanceAnomaly { interface ColdOnboardPerformanceFinding { kind: "phase" | "root-end-to-first-turn" | "root-start-to-first-turn"; message: string; + phaseName?: OnboardPhaseName; } interface ParsedSpan { @@ -137,9 +140,13 @@ function asColdOnboardBudget(value: unknown): ColdOnboardPerformanceBudget | nul const authoritativeLocalBaseBuildAllowanceMs = nonNegativeMilliseconds( record.authoritativeLocalBaseBuildAllowanceMs, ); + const sandboxPhaseSingleObservationMaxOverageMs = nonNegativeMilliseconds( + record.sandboxPhaseSingleObservationMaxOverageMs, + ); const phaseBudgets = asRecord(record.phaseBudgetsMs); if ( authoritativeLocalBaseBuildAllowanceMs === null || + sandboxPhaseSingleObservationMaxOverageMs === null || rootStartToFirstTurnCompletionBudgetMs === null || rootEndToFirstTurnCompletionBudgetMs === null || rootEndToFirstTurnCompletionBudgetMs > rootStartToFirstTurnCompletionBudgetMs || @@ -160,6 +167,7 @@ function asColdOnboardBudget(value: unknown): ColdOnboardPerformanceBudget | nul authoritativeLocalBaseBuildAllowanceMs, rootStartToFirstTurnCompletionBudgetMs, rootEndToFirstTurnCompletionBudgetMs, + sandboxPhaseSingleObservationMaxOverageMs, phaseBudgetsMs, }; } @@ -302,6 +310,7 @@ export function evaluateColdOnboardPerformance( findings.push({ kind: "phase", message: `${phaseName} ${phaseDurationMs}ms exceeds ${phaseBudgetMs}ms`, + phaseName, }); } } @@ -317,6 +326,16 @@ export function evaluateColdOnboardPerformance( ) && trace.finishedAtMs - trace.startedAtMs <= rootStartBudgetMs - budget.rootEndToFirstTurnCompletionBudgetMs; + const sandboxPhaseName = "nemoclaw.onboard.phase.sandbox"; + const sandboxPhaseMeasurementMs = trace.phaseDurationsMs[sandboxPhaseName]; + const sandboxPhaseOverageMs = sandboxPhaseMeasurementMs - sandboxBudgetMs; + const onlyBoundedPublishedBaseSandboxTailExceeded = + !authoritativeLocalBaseBuild && + findings.length === 1 && + findings[0]?.kind === "phase" && + findings[0].phaseName === sandboxPhaseName && + sandboxPhaseOverageMs > 0 && + sandboxPhaseOverageMs <= budget.sandboxPhaseSingleObservationMaxOverageMs; const anomalies: ColdOnboardPerformanceAnomaly[] = onlyFirstTurnTailExceeded ? [ { @@ -326,7 +345,16 @@ export function evaluateColdOnboardPerformance( overageMs: rootEndToFirstTurnCompletionMs - budget.rootEndToFirstTurnCompletionBudgetMs, }, ] - : []; + : onlyBoundedPublishedBaseSandboxTailExceeded + ? [ + { + budgetMs: sandboxBudgetMs, + kind: "sandbox-phase-tail", + measurementMs: sandboxPhaseMeasurementMs, + overageMs: sandboxPhaseOverageMs, + }, + ] + : []; const violations = anomalies.length === 0 ? findings.map((finding) => finding.message) : []; return { diff --git a/test/e2e/live/full-e2e.test.ts b/test/e2e/live/full-e2e.test.ts index 0b75849940..695d250563 100644 --- a/test/e2e/live/full-e2e.test.ts +++ b/test/e2e/live/full-e2e.test.ts @@ -300,7 +300,7 @@ async function assertColdOnboardPerformance(input: { const responseChars = assistantReply.length; await input.artifacts.writeJson("onboard-progress-budget.json", { - schemaVersion: "nemoclaw.full_e2e_cold_performance.v3", + schemaVersion: "nemoclaw.full_e2e_cold_performance.v4", sandbox: SANDBOX_NAME, installExitCode: input.install.exitCode, firstTurnExitCode: turn.exitCode, @@ -320,6 +320,15 @@ async function assertColdOnboardPerformance(input: { provider: input.providerName, promptContract: "sentinel-v1", }, + sandboxPhaseCohort: { + agent: "openclaw", + baseBuildMode: usedAuthoritativeLocalBaseBuild + ? "authoritative-local-base-build" + : "published-base", + platform: process.platform, + setupMode: SETUP_MODE, + workloadKind: workload.kind, + }, onboardSecs: Math.ceil(traceWindow.durationMs / 1_000), rootStartToFirstTurnCompletionSecs, budget: input.budget, @@ -354,8 +363,12 @@ async function assertColdOnboardPerformance(input: { `expected the sentinel first agent reply, got: ${turnText}`, ).toBe(true); for (const anomaly of performanceEvaluation.anomalies) { + const title = + anomaly.kind === "first-turn-latency-tail" + ? "Hosted first-turn latency anomaly" + : "Sandbox phase latency anomaly"; console.warn( - `::warning title=Hosted first-turn latency anomaly::root-end-to-first-turn-completion ${anomaly.measurementMs}ms exceeded ${anomaly.budgetMs}ms by ${anomaly.overageMs}ms after all deterministic cold-onboard budgets passed`, + `::warning title=${title}::${anomaly.kind} measured ${anomaly.measurementMs}ms against ${anomaly.budgetMs}ms, an overage of ${anomaly.overageMs}ms`, ); } expect( diff --git a/test/e2e/support/e2e-first-turn-latency-history.test.ts b/test/e2e/support/e2e-first-turn-latency-history.test.ts index a2723f6d87..775531c464 100644 --- a/test/e2e/support/e2e-first-turn-latency-history.test.ts +++ b/test/e2e/support/e2e-first-turn-latency-history.test.ts @@ -51,7 +51,7 @@ function summary( function artifact(anomaly: boolean): Record { const current = sample(anomaly); return { - schemaVersion: "nemoclaw.full_e2e_cold_performance.v3", + schemaVersion: "nemoclaw.full_e2e_cold_performance.v4", installExitCode: 0, firstTurnExitCode: 0, firstTurnSentinelMatched: true, @@ -97,6 +97,21 @@ describe("hosted first-turn latency history", () => { expect(readCurrentFirstTurnLatencySample(directory)).toEqual(sample(true)); + const sandboxTail = artifact(false); + sandboxTail.performance = { + ...(sandboxTail.performance as Record), + anomalies: [ + { + budgetMs: 208_000, + kind: "sandbox-phase-tail", + measurementMs: 208_136, + overageMs: 136, + }, + ], + }; + fs.writeFileSync(artifactFile, JSON.stringify(sandboxTail)); + expect(readCurrentFirstTurnLatencySample(directory)).toEqual(sample(false)); + fs.writeFileSync( artifactFile, JSON.stringify({ ...artifact(true), firstTurnSentinelMatched: false }), diff --git a/test/e2e/support/e2e-operations-workflow-boundary.test.ts b/test/e2e/support/e2e-operations-workflow-boundary.test.ts index b85d4d034f..16e8c1ea6a 100644 --- a/test/e2e/support/e2e-operations-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-operations-workflow-boundary.test.ts @@ -32,6 +32,19 @@ describe("E2E operations workflow", testTimeoutOptions(15_000), () => { expect(validateE2eOperationsWorkflowBoundary()).toEqual([]); }); + it("requires the live job to upload cold-onboard performance evidence (#6660)", () => { + const workflow = readE2eOperationsWorkflow(); + const upload = workflow.jobs.live.steps!.find((step) => step.name === "Upload E2E artifacts")!; + upload.with!.path = String(upload.with!.path) + .split("\n") + .filter((line) => !line.includes("onboard-progress-budget.json")) + .join("\n"); + + expect(validateE2eOperationsWorkflow(workflow)).toContain( + "live E2E must upload cold-onboard performance evidence", + ); + }); + it("requires the scorecard to wait for every reporting dependency", () => { const workflow = readE2eOperationsWorkflow(); workflow.jobs.scorecard.needs = [...(workflow.jobs.scorecard.needs as string[])]; @@ -1144,15 +1157,19 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; buildRuntimeHistory: vi .fn() .mockResolvedValue("## E2E Push Runtime Trend\n\n| Target | Prior median |"), - loadPriorPushSummaries: vi.fn(), + loadPriorPushHistory: vi.fn(), }; const firstTurnLatency = { readCurrentFirstTurnLatencySample: vi.fn().mockReturnValue(null), }; + const sandboxPhaseTail = { + readCurrentSandboxPhaseTailSample: vi.fn().mockReturnValue(null), + }; const runtimeModules = new Map([ ["path", { join: (...parts: string[]) => parts.join("/") }], ["/workspace/scripts/audit-test-runtime.mts", runtimeAudit], ["/workspace/scripts/scorecard/analyze-first-turn-latency.mts", firstTurnLatency], + ["/workspace/scripts/scorecard/analyze-sandbox-phase-tail.mts", sandboxPhaseTail], ["/workspace/scripts/scorecard/analyze-runtime-history.mts", runtimeHistory], ["/workspace/scripts/scorecard/coordinate-scorecard.mts", coordinator], ["/workspace/scripts/scorecard/analyze-trace-timing.mts", traceTiming], @@ -1202,12 +1219,16 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; "/runner/e2e-runtime-summary.json", { currentFirstTurnLatency: null, - loadPriorPushSummaries: runtimeHistory.loadPriorPushSummaries, + currentSandboxPhaseTail: null, + loadPriorPushHistory: runtimeHistory.loadPriorPushHistory, }, ); expect(firstTurnLatency.readCurrentFirstTurnLatencySample).toHaveBeenCalledWith( "/runner/e2e-runtime-audit", ); + expect(sandboxPhaseTail.readCurrentSandboxPhaseTailSample).toHaveBeenCalledWith( + "/runner/e2e-runtime-audit", + ); expect(runtimeAudit.auditTestRuntime.mock.invocationCallOrder[0]).toBeLessThan( traceTiming.buildTraceTimingResult.mock.invocationCallOrder[0], ); @@ -1253,10 +1274,12 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; }; const runtimeHistory = { buildRuntimeHistory: vi.fn() }; const firstTurnLatency = { readCurrentFirstTurnLatencySample: vi.fn() }; + const sandboxPhaseTail = { readCurrentSandboxPhaseTailSample: vi.fn() }; const runtimeModules = new Map([ ["path", { join: (...parts: string[]) => parts.join("/") }], ["/workspace/scripts/audit-test-runtime.mts", runtimeAudit], ["/workspace/scripts/scorecard/analyze-first-turn-latency.mts", firstTurnLatency], + ["/workspace/scripts/scorecard/analyze-sandbox-phase-tail.mts", sandboxPhaseTail], ["/workspace/scripts/scorecard/analyze-runtime-history.mts", runtimeHistory], [ "/workspace/scripts/scorecard/coordinate-scorecard.mts", @@ -1321,6 +1344,7 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; expect(runtimeAudit.formatRuntimeAuditSummary).not.toHaveBeenCalled(); expect(runtimeAudit.collectRuntimeHistorySamples).not.toHaveBeenCalled(); expect(firstTurnLatency.readCurrentFirstTurnLatencySample).not.toHaveBeenCalled(); + expect(sandboxPhaseTail.readCurrentSandboxPhaseTailSample).not.toHaveBeenCalled(); expect(runtimeHistory.buildRuntimeHistory).not.toHaveBeenCalled(); expect(summary.addRaw).toHaveBeenCalledWith( expect.stringMatching( diff --git a/test/e2e/support/e2e-runtime-history.test.ts b/test/e2e/support/e2e-runtime-history.test.ts index 19ac047e04..4cd5a89e07 100644 --- a/test/e2e/support/e2e-runtime-history.test.ts +++ b/test/e2e/support/e2e-runtime-history.test.ts @@ -12,10 +12,12 @@ import { evaluateFirstTurnLatencyRecurrence, type FirstTurnLatencySample, } from "../../../scripts/scorecard/analyze-first-turn-latency.mts"; +import type { SandboxPhaseTailSample } from "../../../scripts/scorecard/analyze-sandbox-phase-tail.mts"; import { buildRuntimeHistory, createRuntimeSummary, formatRuntimeHistory, + loadPriorPushHistory, loadPriorPushSummaries, normalizeRuntimeSummary, RUNTIME_SUMMARY_ARTIFACT, @@ -50,6 +52,22 @@ function firstTurnSample(anomaly: boolean): FirstTurnLatencySample { }; } +function sandboxPhaseSample(anomaly: boolean): SandboxPhaseTailSample { + return { + anomaly, + budgetMs: 208_000, + cohort: { + agent: "openclaw", + baseBuildMode: "published-base", + platform: "linux", + setupMode: "source-install", + workloadKind: "legacy-dockerfile", + }, + measurementMs: anomaly ? 208_136 : 201_808, + overageMs: anomaly ? 136 : 0, + }; +} + describe("E2E rolling runtime history", () => { it("reports runtime distribution, outcomes, failure streaks, and significant phase regressions", () => { const current = runtimeSample({ @@ -156,13 +174,14 @@ describe("E2E rolling runtime history", () => { }, [runtimeSample()], output, - { loadPriorPushSummaries: vi.fn().mockRejectedValue(new Error("unavailable")) }, + { loadPriorPushHistory: vi.fn().mockRejectedValue(new Error("unavailable")) }, new Date("2026-07-24T00:00:00.000Z"), ); expect(JSON.parse(fs.readFileSync(output, "utf8"))).toMatchObject({ - schemaVersion: "nemoclaw.e2e_runtime_summary.v2", + schemaVersion: "nemoclaw.e2e_runtime_summary.v3", firstTurnLatency: null, + sandboxPhaseTail: null, runId: 123, }); expect(fs.statSync(output).mode & 0o777).toBe(0o600); @@ -196,7 +215,10 @@ describe("E2E rolling runtime history", () => { output, { currentFirstTurnLatency: firstTurnSample(true), - loadPriorPushSummaries: vi.fn().mockResolvedValue(prior), + loadPriorPushHistory: vi.fn().mockResolvedValue({ + summaries: prior, + unavailableRuns: 0, + }), }, new Date("2026-07-24T00:00:00.000Z"), ); @@ -213,15 +235,92 @@ describe("E2E rolling runtime history", () => { } }); + it("saves one sandbox anomaly after four passing same-cohort samples (#6660)", async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-runtime-history-")); + const output = path.join(directory, "e2e-runtime-summary.json"); + const setFailed = vi.fn(); + const prior = Array.from({ length: 4 }, (_, index) => + createRuntimeSummary( + index + 1, + new Date(Date.UTC(2026, 7, index + 1)).toISOString(), + [runtimeSample()], + null, + sandboxPhaseSample(false), + ), + ); + try { + const markdown = await buildRuntimeHistory( + { + github: {}, + context: { repo: { owner: "NVIDIA", repo: "NemoClaw" }, runId: 123 }, + core: { setFailed }, + }, + [runtimeSample({ target: "full-e2e", scenario: "cold onboard" })], + output, + { + currentSandboxPhaseTail: sandboxPhaseSample(true), + loadPriorPushHistory: vi.fn().mockResolvedValue({ + summaries: prior, + unavailableRuns: 0, + }), + }, + new Date("2026-08-21T00:00:00.000Z"), + ); + + expect(JSON.parse(fs.readFileSync(output, "utf8"))).toMatchObject({ + sandboxPhaseTail: { anomaly: true, overageMs: 136 }, + }); + expect(markdown).toContain("## Sandbox Phase Latency"); + expect(setFailed).not.toHaveBeenCalled(); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + }); + + it("fails closed when one sandbox anomaly has no readable history (#6660)", async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-runtime-history-")); + const output = path.join(directory, "e2e-runtime-summary.json"); + const setFailed = vi.fn(); + try { + await buildRuntimeHistory( + { + github: {}, + context: { repo: { owner: "NVIDIA", repo: "NemoClaw" }, runId: 123 }, + core: { setFailed, warning: vi.fn() }, + }, + [runtimeSample({ target: "full-e2e", scenario: "cold onboard" })], + output, + { + currentSandboxPhaseTail: sandboxPhaseSample(true), + loadPriorPushHistory: vi.fn().mockRejectedValue(new Error("unavailable")), + }, + new Date("2026-08-21T00:00:00.000Z"), + ); + + expect(setFailed).toHaveBeenCalledWith( + expect.stringContaining("one or more prior push summaries are unavailable"), + ); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + }); + it("rejects duplicate identities, duplicate phases, and extra fields", () => { const summary = createRuntimeSummary(1, "2026-07-24T00:00:00.000Z", [runtimeSample()]); - const { firstTurnLatency: _, ...legacy } = summary; + const { firstTurnLatency: _, sandboxPhaseTail: __, ...legacy } = summary; expect( normalizeRuntimeSummary({ ...legacy, schemaVersion: "nemoclaw.e2e_runtime_summary.v1", }), ).toMatchObject({ firstTurnLatency: null }); + const { sandboxPhaseTail: ___, ...previous } = summary; + expect( + normalizeRuntimeSummary({ + ...previous, + schemaVersion: "nemoclaw.e2e_runtime_summary.v2", + }), + ).toMatchObject({ sandboxPhaseTail: null }); expect(normalizeRuntimeSummary({ ...summary, extra: true })).toBeNull(); expect( normalizeRuntimeSummary({ ...summary, rows: [summary.rows[0], summary.rows[0]] }), @@ -244,7 +343,7 @@ describe("E2E rolling runtime history", () => { data: { workflow_runs: [{ id: 123 }, { id: 122 }] }, }); const paginate = vi.fn().mockResolvedValue([]); - const summaries = await loadPriorPushSummaries({ + const history = await loadPriorPushHistory({ context: { repo: { owner: "NVIDIA", repo: "NemoClaw" }, runId: 123 }, github: { paginate, @@ -258,7 +357,7 @@ describe("E2E rolling runtime history", () => { }, }); - expect(summaries).toEqual([]); + expect(history).toEqual({ summaries: [], unavailableRuns: 1 }); expect(listWorkflowRuns).toHaveBeenCalledWith( expect.objectContaining({ event: "push", diff --git a/test/e2e/support/e2e-sandbox-phase-tail-history.test.ts b/test/e2e/support/e2e-sandbox-phase-tail-history.test.ts new file mode 100644 index 0000000000..266a732c7d --- /dev/null +++ b/test/e2e/support/e2e-sandbox-phase-tail-history.test.ts @@ -0,0 +1,225 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { + evaluateSandboxPhaseTailRecurrence, + formatSandboxPhaseTailRecurrence, + readCurrentSandboxPhaseTailSample, + type SandboxPhaseCohort, + type SandboxPhaseTailHistorySummary, + type SandboxPhaseTailSample, +} from "../../../scripts/scorecard/analyze-sandbox-phase-tail.mts"; + +const COHORT: SandboxPhaseCohort = { + agent: "openclaw", + baseBuildMode: "published-base", + platform: "linux", + setupMode: "source-install", + workloadKind: "legacy-dockerfile", +}; + +function sample(anomaly: boolean, cohort: SandboxPhaseCohort = COHORT): SandboxPhaseTailSample { + const budgetMs = 208_000; + const measurementMs = anomaly ? 208_136 : 201_808; + return { + anomaly, + budgetMs, + cohort, + measurementMs, + overageMs: Math.max(0, measurementMs - budgetMs), + }; +} + +function summary( + runId: number, + sandboxPhaseTail: SandboxPhaseTailSample | null, +): SandboxPhaseTailHistorySummary { + return { + createdAt: new Date(Date.UTC(2026, 7, runId)).toISOString(), + runId, + sandboxPhaseTail, + }; +} + +function artifact(anomaly: boolean): Record { + const current = sample(anomaly); + return { + schemaVersion: "nemoclaw.full_e2e_cold_performance.v4", + installExitCode: 0, + firstTurnExitCode: 0, + firstTurnSentinelMatched: true, + phaseMeasurements: { + tracePhasesMs: { + "nemoclaw.onboard.phase.sandbox": current.measurementMs, + }, + }, + sandboxPhaseCohort: current.cohort, + budget: { + sandboxPhaseSingleObservationMaxOverageMs: 5_000, + phaseBudgetsMs: { + "nemoclaw.onboard.phase.sandbox": current.budgetMs, + }, + }, + performance: { + anomalies: anomaly + ? [ + { + budgetMs: current.budgetMs, + kind: "sandbox-phase-tail", + measurementMs: current.measurementMs, + overageMs: current.overageMs, + }, + ] + : [], + appliedAuthoritativeLocalBaseBuildAllowanceMs: 0, + passed: true, + usedAuthoritativeLocalBaseBuild: false, + violations: [], + }, + workload: { + kind: "legacy-dockerfile", + reference: "nemoclaw-sandbox-local:e2e-full-test", + }, + buildKitFallback: false, + usedBuildKitPrebuild: true, + classicBuildSteps: 0, + maxSilenceSecs: 20, + maxSilenceBudgetSecs: 60, + }; +} + +describe("sandbox-phase latency history", () => { + it("reads one eligible anomaly and rejects local-base or failed functional evidence (#6660)", () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-sandbox-tail-")); + const artifactDirectory = path.join(directory, "e2e-full-e2e"); + const artifactFile = path.join(artifactDirectory, "onboard-progress-budget.json"); + try { + fs.mkdirSync(artifactDirectory, { recursive: true }); + fs.writeFileSync(artifactFile, JSON.stringify(artifact(true))); + expect(readCurrentSandboxPhaseTailSample(directory)).toEqual(sample(true)); + + const firstTurnTail = artifact(false); + firstTurnTail.performance = { + ...(firstTurnTail.performance as Record), + anomalies: [ + { + budgetMs: 14_000, + kind: "first-turn-latency-tail", + measurementMs: 14_500, + overageMs: 500, + }, + ], + }; + fs.writeFileSync(artifactFile, JSON.stringify(firstTurnTail)); + expect(readCurrentSandboxPhaseTailSample(directory)).toEqual(sample(false)); + + const localBase = artifact(true); + localBase.sandboxPhaseCohort = { ...COHORT, baseBuildMode: "authoritative-local-base-build" }; + localBase.performance = { + ...(localBase.performance as Record), + appliedAuthoritativeLocalBaseBuildAllowanceMs: 570_000, + usedAuthoritativeLocalBaseBuild: true, + }; + fs.writeFileSync(artifactFile, JSON.stringify(localBase)); + expect(readCurrentSandboxPhaseTailSample(directory)).toBeNull(); + + fs.writeFileSync( + artifactFile, + JSON.stringify({ ...artifact(true), firstTurnSentinelMatched: false }), + ); + expect(readCurrentSandboxPhaseTailSample(directory)).toBeNull(); + + const excessiveTail = artifact(true); + const tracePhases = (excessiveTail.phaseMeasurements as Record) + .tracePhasesMs as Record; + tracePhases["nemoclaw.onboard.phase.sandbox"] = 213_001; + const anomalies = (excessiveTail.performance as Record).anomalies as Array< + Record + >; + anomalies[0] = { ...anomalies[0], measurementMs: 213_001, overageMs: 5_001 }; + fs.writeFileSync(artifactFile, JSON.stringify(excessiveTail)); + expect(readCurrentSandboxPhaseTailSample(directory)).toBeNull(); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + }); + + it("allows one anomaly after four valid same-cohort samples (#6660)", () => { + const prior = Array.from({ length: 4 }, (_, index) => summary(index + 1, sample(false))); + + const result = evaluateSandboxPhaseTailRecurrence(sample(true), prior); + + expect(result).toMatchObject({ + anomalyCount: 1, + currentAnomaly: true, + eligibleSamples: 5, + message: null, + passed: true, + }); + expect(formatSandboxPhaseTailRecurrence(result)).toContain( + "current anomaly passed with 1 anomaly in 5 eligible same-cohort samples", + ); + }); + + it("blocks one anomaly when valid same-cohort history is incomplete (#6660)", () => { + const prior = Array.from({ length: 3 }, (_, index) => summary(index + 1, sample(false))); + + const result = evaluateSandboxPhaseTailRecurrence(sample(true), prior); + + expect(result).toMatchObject({ + anomalyCount: 1, + eligibleSamples: 4, + passed: false, + }); + expect(result.message).toContain("4 of 5 eligible same-cohort samples are available"); + }); + + it("blocks one anomaly when a prior push summary is unavailable (#6660)", () => { + const prior = Array.from({ length: 4 }, (_, index) => summary(index + 1, sample(false))); + + const result = evaluateSandboxPhaseTailRecurrence(sample(true), prior, false); + + expect(result).toMatchObject({ + anomalyCount: 1, + eligibleSamples: 5, + passed: false, + }); + expect(result.message).toContain("one or more prior push summaries are unavailable"); + }); + + it("blocks the second anomaly in the latest five eligible samples (#6660)", () => { + const prior = Array.from({ length: 4 }, (_, index) => summary(index + 1, sample(index === 0))); + + const result = evaluateSandboxPhaseTailRecurrence(sample(true), prior); + + expect(result).toMatchObject({ + anomalyCount: 2, + eligibleSamples: 5, + passed: false, + }); + expect(result.message).toContain("2 anomalies in 5 eligible same-cohort samples"); + }); + + it("does not use a different cohort to satisfy the history requirement (#6660)", () => { + const otherCohort = { ...COHORT, workloadKind: "managed-image" }; + const prior = [ + ...Array.from({ length: 3 }, (_, index) => summary(index + 1, sample(false))), + summary(4, sample(false, otherCohort)), + ]; + + expect(evaluateSandboxPhaseTailRecurrence(sample(true), prior)).toMatchObject({ + eligibleSamples: 4, + passed: false, + }); + expect(evaluateSandboxPhaseTailRecurrence(sample(false), prior)).toMatchObject({ + eligibleSamples: 4, + passed: true, + }); + }); +}); diff --git a/test/e2e/support/onboard-performance.test.ts b/test/e2e/support/onboard-performance.test.ts index 4a6c459f3b..e300105528 100644 --- a/test/e2e/support/onboard-performance.test.ts +++ b/test/e2e/support/onboard-performance.test.ts @@ -202,6 +202,7 @@ describe("onboard performance evidence", () => { const budget = readColdOnboardPerformanceBudget({ fullE2eColdPath: { authoritativeLocalBaseBuildAllowanceMs: 500, + sandboxPhaseSingleObservationMaxOverageMs: 0, rootStartToFirstTurnCompletionBudgetMs: 5_000, rootEndToFirstTurnCompletionBudgetMs: 1_000, phaseBudgetsMs: completePhaseBudgets(), @@ -246,6 +247,7 @@ describe("onboard performance evidence", () => { const budget = readColdOnboardPerformanceBudget({ fullE2eColdPath: { authoritativeLocalBaseBuildAllowanceMs: 0, + sandboxPhaseSingleObservationMaxOverageMs: 0, rootStartToFirstTurnCompletionBudgetMs: 20_000, rootEndToFirstTurnCompletionBudgetMs: 1_000, phaseBudgetsMs: completePhaseBudgets(), @@ -274,6 +276,7 @@ describe("onboard performance evidence", () => { const budget = readColdOnboardPerformanceBudget({ fullE2eColdPath: { authoritativeLocalBaseBuildAllowanceMs: 0, + sandboxPhaseSingleObservationMaxOverageMs: 0, rootStartToFirstTurnCompletionBudgetMs: 6_000, rootEndToFirstTurnCompletionBudgetMs: 1_000, phaseBudgetsMs: completePhaseBudgets(), @@ -301,6 +304,7 @@ describe("onboard performance evidence", () => { const budget = readColdOnboardPerformanceBudget({ fullE2eColdPath: { authoritativeLocalBaseBuildAllowanceMs: 0, + sandboxPhaseSingleObservationMaxOverageMs: 0, rootStartToFirstTurnCompletionBudgetMs: 20_000, rootEndToFirstTurnCompletionBudgetMs: 1_000, phaseBudgetsMs: completePhaseBudgets(), @@ -317,10 +321,67 @@ describe("onboard performance evidence", () => { }); }); + it("classifies one bounded published-base sandbox overage as an anomaly (#6660)", () => { + const trace = readOnboardTraceWindow(traceArtifact()); + trace.phaseDurationsMs[ONBOARD_PHASE_NAMES[4]] = 1_636; + const budget = readColdOnboardPerformanceBudget({ + fullE2eColdPath: { + authoritativeLocalBaseBuildAllowanceMs: 0, + sandboxPhaseSingleObservationMaxOverageMs: 5_000, + rootStartToFirstTurnCompletionBudgetMs: 20_000, + rootEndToFirstTurnCompletionBudgetMs: 1_000, + phaseBudgetsMs: completePhaseBudgets(), + }, + }); + + expect(evaluateColdOnboardPerformance(trace, 6_500, budget)).toMatchObject({ + anomalies: [ + { + budgetMs: 1_500, + kind: "sandbox-phase-tail", + measurementMs: 1_636, + overageMs: 136, + }, + ], + passed: true, + violations: [], + }); + }); + + it.each([ + ["exceeds the overage limit", false, 6_501, 250], + ["uses the local-base allowance path", true, 1_636, 250], + ["has another phase violation", false, 1_636, 1_501], + ])( + "keeps a sandbox overage blocking when it %s (#6660)", + (_case, localBase, sandboxMs, preflightMs) => { + const trace = readOnboardTraceWindow(traceArtifact()); + trace.phaseDurationsMs[ONBOARD_PHASE_NAMES[4]] = sandboxMs as number; + trace.phaseDurationsMs[ONBOARD_PHASE_NAMES[0]] = preflightMs as number; + const budget = readColdOnboardPerformanceBudget({ + fullE2eColdPath: { + authoritativeLocalBaseBuildAllowanceMs: 0, + sandboxPhaseSingleObservationMaxOverageMs: 5_000, + rootStartToFirstTurnCompletionBudgetMs: 20_000, + rootEndToFirstTurnCompletionBudgetMs: 1_000, + phaseBudgetsMs: completePhaseBudgets(), + }, + }); + + expect( + evaluateColdOnboardPerformance(trace, 6_500, budget, localBase as boolean), + ).toMatchObject({ + anomalies: [], + passed: false, + }); + }, + ); + it("rejects malformed or incomplete cold-path budget configuration", () => { expect(() => readColdOnboardPerformanceBudget({})).toThrow("fullE2eColdPath"); const fullE2eColdPath = { authoritativeLocalBaseBuildAllowanceMs: 0, + sandboxPhaseSingleObservationMaxOverageMs: 0, rootStartToFirstTurnCompletionBudgetMs: 1_000, rootEndToFirstTurnCompletionBudgetMs: 1_001, phaseBudgetsMs: completePhaseBudgets(), @@ -361,6 +422,7 @@ describe("onboard performance evidence", () => { const budget = readColdOnboardPerformanceBudget({ fullE2eColdPath: { authoritativeLocalBaseBuildAllowanceMs: 0, + sandboxPhaseSingleObservationMaxOverageMs: 0, rootStartToFirstTurnCompletionBudgetMs: 5_000, rootEndToFirstTurnCompletionBudgetMs: 1_000, phaseBudgetsMs: completePhaseBudgets(), diff --git a/test/onboard-performance-config-schema.test.ts b/test/onboard-performance-config-schema.test.ts index fae5e25e8e..6b239d1c20 100644 --- a/test/onboard-performance-config-schema.test.ts +++ b/test/onboard-performance-config-schema.test.ts @@ -22,11 +22,15 @@ type PhaseName = (typeof PHASE_NAMES)[number]; type PhaseBudgets = Record; interface ColdPathBudget { authoritativeLocalBaseBuildAllowanceMs: number; + sandboxPhaseSingleObservationMaxOverageMs: number; rootStartToFirstTurnCompletionBudgetMs: number; rootEndToFirstTurnCompletionBudgetMs: number; phaseBudgetsMs: PhaseBudgets; } -type CalibratedColdPathBudget = Omit; +type CalibratedColdPathBudget = Omit< + ColdPathBudget, + "authoritativeLocalBaseBuildAllowanceMs" | "sandboxPhaseSingleObservationMaxOverageMs" +>; interface CalibrationSample { runId: number; runUrl: string; @@ -187,6 +191,7 @@ const validConfig = { phaseRegressionWarning: { minDeltaMs: 0, minPercent: 0 }, fullE2eColdPath: { authoritativeLocalBaseBuildAllowanceMs: 500, + sandboxPhaseSingleObservationMaxOverageMs: 5_000, rootStartToFirstTurnCompletionBudgetMs: 5_000, rootEndToFirstTurnCompletionBudgetMs: 1_000, phaseBudgetsMs, @@ -214,6 +219,17 @@ describe("onboard performance config schema", () => { ).toBe(false); }); + it("requires the sandbox-phase single-observation overage limit", () => { + const { sandboxPhaseSingleObservationMaxOverageMs: _, ...withoutSandboxTailLimit } = + validConfig.fullE2eColdPath; + expect( + validate({ + ...validConfig, + fullE2eColdPath: withoutSandboxTailLimit, + }), + ).toBe(false); + }); + it("enforces the root-end budget against the root-start budget", () => { expect( validate({ @@ -314,6 +330,8 @@ function effectiveBudgets(input: Calibration): ColdPathBudget { return { authoritativeLocalBaseBuildAllowanceMs: input.authoritativeLocalBaseBuildAdjustment.derivedAllowanceMs, + sandboxPhaseSingleObservationMaxOverageMs: + checkedInConfig.fullE2eColdPath.sandboxPhaseSingleObservationMaxOverageMs, ...baseline, rootStartToFirstTurnCompletionBudgetMs: Math.max( baseline.rootStartToFirstTurnCompletionBudgetMs, diff --git a/tools/e2e/operations-workflow-boundary.mts b/tools/e2e/operations-workflow-boundary.mts index d4ee5dcb91..757661cddc 100644 --- a/tools/e2e/operations-workflow-boundary.mts +++ b/tools/e2e/operations-workflow-boundary.mts @@ -684,6 +684,9 @@ export function validateBaseImagePublicationGate(workflow: OperationsWorkflow): ) { errors.push("live DCode must record its immutable base contract before E2E execution"); } + if (!String(upload.with?.path ?? "").includes("onboard-progress-budget.json")) { + errors.push("live E2E must upload cold-onboard performance evidence"); + } if (!sameMembers(needs(workflow.jobs["staging-brev-launchable"] ?? {}), ["generate-matrix"])) { errors.push("staging-brev-launchable must wait only for generate-matrix"); } @@ -1045,7 +1048,10 @@ function validateScorecard(errors: string[], workflow: OperationsWorkflow): void "scripts/scorecard/analyze-first-turn-latency.mts", "firstTurnLatency.readCurrentFirstTurnLatencySample", "currentFirstTurnLatency", - "runtimeHistory.loadPriorPushSummaries", + "scripts/scorecard/analyze-sandbox-phase-tail.mts", + "sandboxPhaseTail.readCurrentSandboxPhaseTailSample", + "currentSandboxPhaseTail", + "runtimeHistory.loadPriorPushHistory", "core.summary", "scorecardData", "slackData", From 2745b9f492035b7cfa843f2501f4fbd6d88f7462 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Fri, 21 Aug 2026 07:58:35 -0700 Subject: [PATCH 2/4] fix(e2e): align artifact upload contract Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- tools/e2e/upload-e2e-artifacts-workflow-boundary.mts | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts index ffb2f231fd..249bbb5b89 100644 --- a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts +++ b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts @@ -157,6 +157,7 @@ const EXPLICIT_UPLOAD_CONTRACTS = new Map([ "e2e-artifacts/live/${{ matrix.id }}/state-validation.result.json", "e2e-artifacts/live/${{ matrix.id }}/dcode-base-image.json", "e2e-artifacts/live/${{ matrix.id }}/cloud-onboard-trace-timing-summary.json", + "e2e-artifacts/live/${{ matrix.id }}/onboard-progress-budget.json", "e2e-artifacts/live/risk-signal.json", "e2e-artifacts/live/${{ matrix.id }}/actions/", "e2e-artifacts/live/${{ matrix.id }}/logs/", From a3fd726b12167ae3f406f814db9fab24e1d557b8 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Fri, 21 Aug 2026 08:06:39 -0700 Subject: [PATCH 3/4] fix(e2e): require exact performance artifact path Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- .../support/e2e-operations-workflow-boundary.test.ts | 10 ++++++++-- tools/e2e/operations-workflow-boundary.mts | 8 +++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/test/e2e/support/e2e-operations-workflow-boundary.test.ts b/test/e2e/support/e2e-operations-workflow-boundary.test.ts index 16e8c1ea6a..a3b676814b 100644 --- a/test/e2e/support/e2e-operations-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-operations-workflow-boundary.test.ts @@ -19,6 +19,8 @@ import { testTimeoutOptions } from "../../helpers/timeouts.ts"; const AsyncFunction = Object.getPrototypeOf(async () => undefined).constructor as new ( ...parameters: string[] ) => (...args: unknown[]) => Promise; +const COLD_ONBOARD_PERFORMANCE_EVIDENCE_PATH = + "e2e-artifacts/live/${{ matrix.id }}/onboard-progress-budget.json"; function workflowScript(jobName: string, stepName: string): string { const workflow = readE2eOperationsWorkflow(); @@ -32,12 +34,16 @@ describe("E2E operations workflow", testTimeoutOptions(15_000), () => { expect(validateE2eOperationsWorkflowBoundary()).toEqual([]); }); - it("requires the live job to upload cold-onboard performance evidence (#6660)", () => { + it("rejects a lookalike live cold-onboard performance artifact path (#6660)", () => { const workflow = readE2eOperationsWorkflow(); const upload = workflow.jobs.live.steps!.find((step) => step.name === "Upload E2E artifacts")!; upload.with!.path = String(upload.with!.path) .split("\n") - .filter((line) => !line.includes("onboard-progress-budget.json")) + .map((line) => + line.trim() === COLD_ONBOARD_PERFORMANCE_EVIDENCE_PATH + ? `${COLD_ONBOARD_PERFORMANCE_EVIDENCE_PATH}.backup` + : line, + ) .join("\n"); expect(validateE2eOperationsWorkflow(workflow)).toContain( diff --git a/tools/e2e/operations-workflow-boundary.mts b/tools/e2e/operations-workflow-boundary.mts index 757661cddc..69abca9615 100644 --- a/tools/e2e/operations-workflow-boundary.mts +++ b/tools/e2e/operations-workflow-boundary.mts @@ -31,6 +31,8 @@ const DOWNLOAD_ARTIFACT_ACTION = const PR_GATE_REPORTER = "test/e2e/risk-signal-reporter.ts"; const LIVE_VITEST_HELPER = "tools/e2e/live-vitest-invocation.mts run --test-path"; const E2E_ARTIFACT_ACTION = "NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@"; +const COLD_ONBOARD_PERFORMANCE_EVIDENCE_PATH = + "e2e-artifacts/live/${{ matrix.id }}/onboard-progress-budget.json"; const PUBLICATION_REQUIRED_CONDITION = "${{ steps.publication_mode.outputs.required == '1' }}"; const PUBLICATION_CLASSIFIER_SCRIPT = [ @@ -673,6 +675,10 @@ export function validateBaseImagePublicationGate(workflow: OperationsWorkflow): } const evidence = findStep(live, "Record immutable Deep Agents Code base evidence"); const upload = findStep(live, "Upload E2E artifacts"); + const uploadPaths = String(upload.with?.path ?? "") + .split("\n") + .map((path) => path.trim()) + .filter(Boolean); const liveSteps = live.steps ?? []; if ( evidence.if !== "${{ matrix.id == 'ubuntu-repo-cloud-langchain-deepagents-code' }}" || @@ -684,7 +690,7 @@ export function validateBaseImagePublicationGate(workflow: OperationsWorkflow): ) { errors.push("live DCode must record its immutable base contract before E2E execution"); } - if (!String(upload.with?.path ?? "").includes("onboard-progress-budget.json")) { + if (!uploadPaths.includes(COLD_ONBOARD_PERFORMANCE_EVIDENCE_PATH)) { errors.push("live E2E must upload cold-onboard performance evidence"); } if (!sameMembers(needs(workflow.jobs["staging-brev-launchable"] ?? {}), ["generate-matrix"])) { From 640d20187aa6efd7feffa6f1d6793d456c36b52d Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Sat, 22 Aug 2026 22:30:57 -0700 Subject: [PATCH 4/4] fix(e2e): cap sandbox tail artifact allowance Reject current-run artifacts that raise the fixed 5-second single-observation overage ceiling. Signed-off-by: Prekshi Vyas --- .../scorecard/analyze-sandbox-phase-tail.mts | 5 +++- .../e2e-sandbox-phase-tail-history.test.ts | 24 +++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/scripts/scorecard/analyze-sandbox-phase-tail.mts b/scripts/scorecard/analyze-sandbox-phase-tail.mts index 08d7049663..b24a381751 100644 --- a/scripts/scorecard/analyze-sandbox-phase-tail.mts +++ b/scripts/scorecard/analyze-sandbox-phase-tail.mts @@ -11,6 +11,7 @@ const SANDBOX_PHASE = "nemoclaw.onboard.phase.sandbox"; const SANDBOX_PHASE_ANOMALY_KIND = "sandbox-phase-tail"; const FIRST_TURN_ANOMALY_KIND = "first-turn-latency-tail"; const MAX_DURATION_MS = 24 * 60 * 60 * 1000; +const SANDBOX_PHASE_SINGLE_OBSERVATION_MAX_OVERAGE_MS = 5_000; export interface SandboxPhaseCohort { agent: string; @@ -185,7 +186,9 @@ export function readCurrentSandboxPhaseTailSample(root: string): SandboxPhaseTai !validWorkloadEvidence(artifact, cohort) || !isDuration(tracePhases[SANDBOX_PHASE]) || !isDuration(phaseBudgets[SANDBOX_PHASE]) || - !isDuration(budget.sandboxPhaseSingleObservationMaxOverageMs) + !isDuration(budget.sandboxPhaseSingleObservationMaxOverageMs) || + budget.sandboxPhaseSingleObservationMaxOverageMs > + SANDBOX_PHASE_SINGLE_OBSERVATION_MAX_OVERAGE_MS ) { return null; } diff --git a/test/e2e/support/e2e-sandbox-phase-tail-history.test.ts b/test/e2e/support/e2e-sandbox-phase-tail-history.test.ts index 266a732c7d..e72ddad414 100644 --- a/test/e2e/support/e2e-sandbox-phase-tail-history.test.ts +++ b/test/e2e/support/e2e-sandbox-phase-tail-history.test.ts @@ -150,6 +150,30 @@ describe("sandbox-phase latency history", () => { } }); + it("rejects an artifact that raises the fixed single-observation overage ceiling (#6660)", () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-sandbox-tail-ceiling-")); + const artifactDirectory = path.join(directory, "e2e-full-e2e"); + const artifactFile = path.join(artifactDirectory, "onboard-progress-budget.json"); + try { + const oversizedAllowance = artifact(true); + const budget = oversizedAllowance.budget as Record; + budget.sandboxPhaseSingleObservationMaxOverageMs = 6_000; + const tracePhases = (oversizedAllowance.phaseMeasurements as Record) + .tracePhasesMs as Record; + tracePhases["nemoclaw.onboard.phase.sandbox"] = 214_000; + const anomalies = (oversizedAllowance.performance as Record) + .anomalies as Array>; + anomalies[0] = { ...anomalies[0], measurementMs: 214_000, overageMs: 6_000 }; + + fs.mkdirSync(artifactDirectory, { recursive: true }); + fs.writeFileSync(artifactFile, JSON.stringify(oversizedAllowance)); + + expect(readCurrentSandboxPhaseTailSample(directory)).toBeNull(); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + }); + it("allows one anomaly after four valid same-cohort samples (#6660)", () => { const prior = Array.from({ length: 4 }, (_, index) => summary(index + 1, sample(false)));