From 935debd2809c2027feca6f695888ae377df2feb9 Mon Sep 17 00:00:00 2001 From: Ho Lim Date: Sun, 26 Jul 2026 13:15:58 -0700 Subject: [PATCH 1/6] test(e2e): classify isolated first-turn latency tails Signed-off-by: Ho Lim --- test/e2e/README.md | 7 +++ test/e2e/fixtures/onboard-performance.ts | 49 +++++++++++++++--- test/e2e/live/full-e2e.test.ts | 17 +++++++ test/e2e/support/onboard-performance.test.ts | 53 ++++++++++++++++++++ 4 files changed, 118 insertions(+), 8 deletions(-) diff --git a/test/e2e/README.md b/test/e2e/README.md index 2f46684dc64..43b46110b09 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -973,6 +973,13 @@ context without a gateway-builder fallback, enforces the calibrated root and phase limits in the budget file, and limits the longest onboard output gap to 60 seconds. A violation fails `full-e2e`, and the target writes its evidence to `onboard-progress-budget.json`. +When every deterministic cold-onboard budget passes and the real first turn +exits successfully with the expected sentinel, a sole root-end-to-first-turn +overage is recorded as a structured, non-blocking hosted-latency anomaly rather +than a PR regression. The same overage remains blocking when accompanied by a +root-start or phase-budget failure. The artifact preserves the measurement, +budget, and overage so recurring same-model, same-mode evidence can calibrate +tail-latency enforcement without weakening functional or deterministic gates. When changed base-image inputs require the authoritative local OpenClaw base build, the target applies the separately calibrated 90-second allowance only to diff --git a/test/e2e/fixtures/onboard-performance.ts b/test/e2e/fixtures/onboard-performance.ts index d4a47e3e7b3..981e1e79cc7 100644 --- a/test/e2e/fixtures/onboard-performance.ts +++ b/test/e2e/fixtures/onboard-performance.ts @@ -43,12 +43,25 @@ export interface ColdOnboardPerformanceBudget { export interface ColdOnboardPerformanceEvaluation { appliedAuthoritativeLocalBaseBuildAllowanceMs: number; + anomalies: ColdOnboardPerformanceAnomaly[]; passed: boolean; rootEndToFirstTurnCompletionMs: number; rootStartToFirstTurnCompletionMs: number; violations: string[]; } +export interface ColdOnboardPerformanceAnomaly { + budgetMs: number; + kind: "first-turn-latency-tail"; + measurementMs: number; + overageMs: number; +} + +interface ColdOnboardPerformanceFinding { + kind: "phase" | "root-end-to-first-turn" | "root-start-to-first-turn"; + message: string; +} + interface ParsedSpan { durationMs: number; endNs: bigint; @@ -266,16 +279,18 @@ export function evaluateColdOnboardPerformance( const sandboxBudgetMs = budget.phaseBudgetsMs["nemoclaw.onboard.phase.sandbox"] + appliedAuthoritativeLocalBaseBuildAllowanceMs; - const violations: string[] = []; + const findings: ColdOnboardPerformanceFinding[] = []; if (rootStartToFirstTurnCompletionMs > rootStartBudgetMs) { - violations.push( - `root-start-to-first-turn-completion ${rootStartToFirstTurnCompletionMs}ms exceeds ${rootStartBudgetMs}ms`, - ); + findings.push({ + kind: "root-start-to-first-turn", + message: `root-start-to-first-turn-completion ${rootStartToFirstTurnCompletionMs}ms exceeds ${rootStartBudgetMs}ms`, + }); } if (rootEndToFirstTurnCompletionMs > budget.rootEndToFirstTurnCompletionBudgetMs) { - violations.push( - `root-end-to-first-turn-completion ${rootEndToFirstTurnCompletionMs}ms exceeds ${budget.rootEndToFirstTurnCompletionBudgetMs}ms`, - ); + findings.push({ + kind: "root-end-to-first-turn", + message: `root-end-to-first-turn-completion ${rootEndToFirstTurnCompletionMs}ms exceeds ${budget.rootEndToFirstTurnCompletionBudgetMs}ms`, + }); } for (const phaseName of ONBOARD_PHASE_NAMES) { const phaseBudgetMs = @@ -284,12 +299,30 @@ export function evaluateColdOnboardPerformance( : budget.phaseBudgetsMs[phaseName]; const phaseDurationMs = trace.phaseDurationsMs[phaseName]; if (phaseBudgetMs !== undefined && phaseDurationMs > phaseBudgetMs) { - violations.push(`${phaseName} ${phaseDurationMs}ms exceeds ${phaseBudgetMs}ms`); + findings.push({ + kind: "phase", + message: `${phaseName} ${phaseDurationMs}ms exceeds ${phaseBudgetMs}ms`, + }); } } + const soleFinding = findings.length === 1 ? findings[0] : null; + const anomalies: ColdOnboardPerformanceAnomaly[] = + soleFinding?.kind === "root-end-to-first-turn" + ? [ + { + budgetMs: budget.rootEndToFirstTurnCompletionBudgetMs, + kind: "first-turn-latency-tail", + measurementMs: rootEndToFirstTurnCompletionMs, + overageMs: rootEndToFirstTurnCompletionMs - budget.rootEndToFirstTurnCompletionBudgetMs, + }, + ] + : []; + const violations = anomalies.length === 0 ? findings.map((finding) => finding.message) : []; + return { appliedAuthoritativeLocalBaseBuildAllowanceMs, + anomalies, passed: violations.length === 0, rootStartToFirstTurnCompletionMs, rootEndToFirstTurnCompletionMs, diff --git a/test/e2e/live/full-e2e.test.ts b/test/e2e/live/full-e2e.test.ts index 6c64995e02d..000e80ef24b 100644 --- a/test/e2e/live/full-e2e.test.ts +++ b/test/e2e/live/full-e2e.test.ts @@ -185,7 +185,9 @@ async function assertColdOnboardPerformance(input: { budget: ColdOnboardPerformanceBudget; install: ShellProbeResult; installCompletedAtMs: number; + model: string; outputEvents: readonly ShellProbeOutputEvent[]; + providerName: string; sandbox: SandboxClient; traceDirectory: string; traceFile: string; @@ -252,10 +254,18 @@ async function assertColdOnboardPerformance(input: { rootEndToFirstTurnCompletionMs: performanceEvaluation.rootEndToFirstTurnCompletionMs, tracePhasesMs: traceWindow.phaseDurationsMs, }, + firstTurnCohort: { + agent: "openclaw", + inferenceMode: "agent-thinking-off", + model: input.model, + provider: input.providerName, + promptContract: "sentinel-v1", + }, onboardSecs: Math.ceil(traceWindow.durationMs / 1_000), rootStartToFirstTurnCompletionSecs, budget: input.budget, performance: { + anomalies: performanceEvaluation.anomalies, passed: performanceEvaluation.passed, violations: performanceEvaluation.violations, usedAuthoritativeLocalBaseBuild, @@ -284,6 +294,11 @@ async function assertColdOnboardPerformance(input: { compactAssistantReply, `expected the sentinel first agent reply, got: ${turnText}`, ).toContain(EXPECTED_FIRST_REPLY); + for (const anomaly of performanceEvaluation.anomalies) { + console.warn( + `::warning title=Hosted first-turn latency anomaly::root-end-to-first-turn-completion ${anomaly.measurementMs}ms exceeded ${anomaly.budgetMs}ms by ${anomaly.overageMs}ms after all deterministic cold-onboard budgets passed`, + ); + } expect( performanceEvaluation.passed, `onboard-root-start-to-first-turn-completion took ${rootStartToFirstTurnCompletionSecs}s; ${performanceEvaluation.violations.join("; ")}`, @@ -399,7 +414,9 @@ test("full e2e: install, onboard, inference, cli operations, and cleanup", { budget: coldOnboardBudget!, install, installCompletedAtMs, + model: hosted.model, outputEvents: coldOnboard.outputEvents, + providerName: hosted.providerName, sandbox, traceDirectory: coldOnboard.traceDirectory, traceFile: coldOnboard.traceFile, diff --git a/test/e2e/support/onboard-performance.test.ts b/test/e2e/support/onboard-performance.test.ts index 77f76e0b12a..a8f59ef0c99 100644 --- a/test/e2e/support/onboard-performance.test.ts +++ b/test/e2e/support/onboard-performance.test.ts @@ -206,6 +206,7 @@ describe("onboard performance evidence", () => { expect(evaluateColdOnboardPerformance(trace, 6_000, budget)).toEqual({ appliedAuthoritativeLocalBaseBuildAllowanceMs: 0, + anomalies: [], passed: true, rootStartToFirstTurnCompletionMs: 5_000, rootEndToFirstTurnCompletionMs: 0, @@ -213,6 +214,7 @@ describe("onboard performance evidence", () => { }); expect(evaluateColdOnboardPerformance(trace, 7_500, budget)).toEqual({ appliedAuthoritativeLocalBaseBuildAllowanceMs: 0, + anomalies: [], passed: false, rootStartToFirstTurnCompletionMs: 6_500, rootEndToFirstTurnCompletionMs: 1_500, @@ -228,12 +230,63 @@ describe("onboard performance evidence", () => { ]); expect(evaluateColdOnboardPerformance(trace, 6_500, budget, true)).toMatchObject({ appliedAuthoritativeLocalBaseBuildAllowanceMs: 500, + anomalies: [], passed: true, rootStartToFirstTurnCompletionMs: 5_500, violations: [], }); }); + it("classifies a sole hosted first-turn tail as a structured non-blocking anomaly (#6660)", () => { + const trace = readOnboardTraceWindow(traceArtifact()); + const budget = readColdOnboardPerformanceBudget({ + fullE2eColdPath: { + authoritativeLocalBaseBuildAllowanceMs: 0, + rootStartToFirstTurnCompletionBudgetMs: 20_000, + rootEndToFirstTurnCompletionBudgetMs: 1_000, + phaseBudgetsMs: completePhaseBudgets(), + }, + }); + + expect(evaluateColdOnboardPerformance(trace, 7_500, budget)).toEqual({ + appliedAuthoritativeLocalBaseBuildAllowanceMs: 0, + anomalies: [ + { + budgetMs: 1_000, + kind: "first-turn-latency-tail", + measurementMs: 1_500, + overageMs: 500, + }, + ], + passed: true, + rootStartToFirstTurnCompletionMs: 6_500, + rootEndToFirstTurnCompletionMs: 1_500, + violations: [], + }); + }); + + it("keeps a first-turn overage blocking when another cold-path budget also fails (#6660)", () => { + const trace = readOnboardTraceWindow(traceArtifact()); + trace.phaseDurationsMs[ONBOARD_PHASE_NAMES[4]] = 1_501; + const budget = readColdOnboardPerformanceBudget({ + fullE2eColdPath: { + authoritativeLocalBaseBuildAllowanceMs: 0, + rootStartToFirstTurnCompletionBudgetMs: 20_000, + rootEndToFirstTurnCompletionBudgetMs: 1_000, + phaseBudgetsMs: completePhaseBudgets(), + }, + }); + + expect(evaluateColdOnboardPerformance(trace, 7_500, budget)).toMatchObject({ + anomalies: [], + passed: false, + violations: [ + "root-end-to-first-turn-completion 1500ms exceeds 1000ms", + "nemoclaw.onboard.phase.sandbox 1501ms exceeds 1500ms", + ], + }); + }); + it("rejects malformed or incomplete cold-path budget configuration", () => { expect(() => readColdOnboardPerformanceBudget({})).toThrow("fullE2eColdPath"); const fullE2eColdPath = { From 500e2592c83f12c49a2277f943da34421b7956d3 Mon Sep 17 00:00:00 2001 From: Ho Lim Date: Wed, 29 Jul 2026 12:41:49 -0700 Subject: [PATCH 2/6] test(e2e): preserve first-turn duration evidence Signed-off-by: Ho Lim --- test/e2e/README.md | 14 ++--- test/e2e/fixtures/onboard-performance.ts | 49 +++--------------- test/e2e/live/agent-turn-latency-helpers.ts | 54 ++++++++++++++++++++ test/e2e/live/full-e2e.test.ts | 16 +++--- test/e2e/support/onboard-performance.test.ts | 52 +++++++++++++------ 5 files changed, 112 insertions(+), 73 deletions(-) diff --git a/test/e2e/README.md b/test/e2e/README.md index 6870beb5a15..86b2daf231a 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -967,13 +967,13 @@ context without a gateway-builder fallback, enforces the calibrated root and phase limits in the budget file, and limits the longest onboard output gap to 60 seconds. A violation fails `full-e2e`, and the target writes its evidence to `onboard-progress-budget.json`. -When every deterministic cold-onboard budget passes and the real first turn -exits successfully with the expected sentinel, a sole root-end-to-first-turn -overage is recorded as a structured, non-blocking hosted-latency anomaly rather -than a PR regression. The same overage remains blocking when accompanied by a -root-start or phase-budget failure. The artifact preserves the measurement, -budget, and overage so recurring same-model, same-mode evidence can calibrate -tail-latency enforcement without weakening functional or deterministic gates. +The artifact records the first-turn command wall clock and OpenClaw's internal +agent duration separately. Older or malformed OpenClaw output records an +explicit unavailable reason instead of fabricating a duration. The artifact +also identifies the model, provider, inference mode, and prompt contract so +same-cohort samples can support a later, documented recurrence policy. All +configured performance violations remain blocking until that policy is +accepted. When changed base-image inputs require the authoritative local OpenClaw base build, the target applies the separately calibrated 90-second allowance only to diff --git a/test/e2e/fixtures/onboard-performance.ts b/test/e2e/fixtures/onboard-performance.ts index 981e1e79cc7..d4a47e3e7b3 100644 --- a/test/e2e/fixtures/onboard-performance.ts +++ b/test/e2e/fixtures/onboard-performance.ts @@ -43,25 +43,12 @@ export interface ColdOnboardPerformanceBudget { export interface ColdOnboardPerformanceEvaluation { appliedAuthoritativeLocalBaseBuildAllowanceMs: number; - anomalies: ColdOnboardPerformanceAnomaly[]; passed: boolean; rootEndToFirstTurnCompletionMs: number; rootStartToFirstTurnCompletionMs: number; violations: string[]; } -export interface ColdOnboardPerformanceAnomaly { - budgetMs: number; - kind: "first-turn-latency-tail"; - measurementMs: number; - overageMs: number; -} - -interface ColdOnboardPerformanceFinding { - kind: "phase" | "root-end-to-first-turn" | "root-start-to-first-turn"; - message: string; -} - interface ParsedSpan { durationMs: number; endNs: bigint; @@ -279,18 +266,16 @@ export function evaluateColdOnboardPerformance( const sandboxBudgetMs = budget.phaseBudgetsMs["nemoclaw.onboard.phase.sandbox"] + appliedAuthoritativeLocalBaseBuildAllowanceMs; - const findings: ColdOnboardPerformanceFinding[] = []; + const violations: string[] = []; if (rootStartToFirstTurnCompletionMs > rootStartBudgetMs) { - findings.push({ - kind: "root-start-to-first-turn", - message: `root-start-to-first-turn-completion ${rootStartToFirstTurnCompletionMs}ms exceeds ${rootStartBudgetMs}ms`, - }); + violations.push( + `root-start-to-first-turn-completion ${rootStartToFirstTurnCompletionMs}ms exceeds ${rootStartBudgetMs}ms`, + ); } if (rootEndToFirstTurnCompletionMs > budget.rootEndToFirstTurnCompletionBudgetMs) { - findings.push({ - kind: "root-end-to-first-turn", - message: `root-end-to-first-turn-completion ${rootEndToFirstTurnCompletionMs}ms exceeds ${budget.rootEndToFirstTurnCompletionBudgetMs}ms`, - }); + violations.push( + `root-end-to-first-turn-completion ${rootEndToFirstTurnCompletionMs}ms exceeds ${budget.rootEndToFirstTurnCompletionBudgetMs}ms`, + ); } for (const phaseName of ONBOARD_PHASE_NAMES) { const phaseBudgetMs = @@ -299,30 +284,12 @@ export function evaluateColdOnboardPerformance( : budget.phaseBudgetsMs[phaseName]; const phaseDurationMs = trace.phaseDurationsMs[phaseName]; if (phaseBudgetMs !== undefined && phaseDurationMs > phaseBudgetMs) { - findings.push({ - kind: "phase", - message: `${phaseName} ${phaseDurationMs}ms exceeds ${phaseBudgetMs}ms`, - }); + violations.push(`${phaseName} ${phaseDurationMs}ms exceeds ${phaseBudgetMs}ms`); } } - const soleFinding = findings.length === 1 ? findings[0] : null; - const anomalies: ColdOnboardPerformanceAnomaly[] = - soleFinding?.kind === "root-end-to-first-turn" - ? [ - { - budgetMs: budget.rootEndToFirstTurnCompletionBudgetMs, - kind: "first-turn-latency-tail", - measurementMs: rootEndToFirstTurnCompletionMs, - overageMs: rootEndToFirstTurnCompletionMs - budget.rootEndToFirstTurnCompletionBudgetMs, - }, - ] - : []; - const violations = anomalies.length === 0 ? findings.map((finding) => finding.message) : []; - return { appliedAuthoritativeLocalBaseBuildAllowanceMs, - anomalies, passed: violations.length === 0, rootStartToFirstTurnCompletionMs, rootEndToFirstTurnCompletionMs, diff --git a/test/e2e/live/agent-turn-latency-helpers.ts b/test/e2e/live/agent-turn-latency-helpers.ts index 4401c3e9be8..9ac983851ef 100644 --- a/test/e2e/live/agent-turn-latency-helpers.ts +++ b/test/e2e/live/agent-turn-latency-helpers.ts @@ -229,6 +229,60 @@ export function extractOpenClawAgentPayloadText(output: string): string { return ""; } +export type OpenClawAgentDurationEvidence = + | { durationMs: number; status: "available" } + | { reason: "malformed" | "missing"; status: "unavailable" }; + +export interface OpenClawFirstTurnLatencyEvidence { + firstTurnAgentDuration: OpenClawAgentDurationEvidence; + firstTurnCommandMs: number; +} + +/** + * Extract OpenClaw's internal agent duration without fabricating a value when + * older or malformed output omits the metadata contract. + */ +export function extractOpenClawAgentDurationEvidence( + output: string, +): OpenClawAgentDurationEvidence { + let malformed = false; + for (let start = output.indexOf("{"); start >= 0; start = output.indexOf("{", start + 1)) { + const parsed = parseJsonObjectAt(output, start); + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) continue; + const result = (parsed as Record).result; + if (!result || typeof result !== "object" || Array.isArray(result)) continue; + const meta = (result as Record).meta; + if (meta === undefined) continue; + if (!meta || typeof meta !== "object" || Array.isArray(meta)) { + malformed = true; + continue; + } + const durationMs = (meta as Record).durationMs; + if (typeof durationMs === "number" && Number.isFinite(durationMs) && durationMs >= 0) { + return { durationMs, status: "available" }; + } + if (durationMs !== undefined) malformed = true; + } + return { reason: malformed ? "malformed" : "missing", status: "unavailable" }; +} + +export function buildOpenClawFirstTurnLatencyEvidence( + output: string, + firstTurnCommandMs: number, +): OpenClawFirstTurnLatencyEvidence { + if ( + typeof firstTurnCommandMs !== "number" || + !Number.isFinite(firstTurnCommandMs) || + firstTurnCommandMs < 0 + ) { + throw new Error("first-turn command duration is invalid"); + } + return { + firstTurnAgentDuration: extractOpenClawAgentDurationEvidence(output), + firstTurnCommandMs, + }; +} + export function responseBodyAndStatus(raw: string): { body: string; status: string } { const match = raw.match(/\n__NEMOCLAW_HTTP_STATUS__=(\d{3})\s*$/u); return { body: match ? raw.slice(0, match.index).trim() : raw, status: match?.[1] ?? "000" }; diff --git a/test/e2e/live/full-e2e.test.ts b/test/e2e/live/full-e2e.test.ts index 000e80ef24b..f93309638e2 100644 --- a/test/e2e/live/full-e2e.test.ts +++ b/test/e2e/live/full-e2e.test.ts @@ -32,7 +32,10 @@ import { securityPostureModeEnv, } from "../fixtures/security-posture.ts"; import type { ShellProbeOutputEvent, ShellProbeResult } from "../fixtures/shell-probe.ts"; -import { extractOpenClawAgentPayloadText } from "./agent-turn-latency-helpers.ts"; +import { + buildOpenClawFirstTurnLatencyEvidence, + extractOpenClawAgentPayloadText, +} from "./agent-turn-latency-helpers.ts"; const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-full"; const SETUP_MODE = process.env.NEMOCLAW_E2E_SETUP_MODE ?? "source-install"; @@ -238,11 +241,12 @@ async function assertColdOnboardPerformance(input: { ); const turnText = resultText(turn); const assistantReply = extractOpenClawAgentPayloadText(turnText).trim(); + const firstTurnLatency = buildOpenClawFirstTurnLatencyEvidence(turnText, firstTurnCommandMs); const compactAssistantReply = assistantReply.replace(/\s+/gu, ""); const responseChars = assistantReply.length; await input.artifacts.writeJson("onboard-progress-budget.json", { - schemaVersion: "nemoclaw.full_e2e_cold_performance.v2", + schemaVersion: "nemoclaw.full_e2e_cold_performance.v3", sandbox: SANDBOX_NAME, installExitCode: input.install.exitCode, firstTurnExitCode: turn.exitCode, @@ -250,7 +254,7 @@ async function assertColdOnboardPerformance(input: { onboardRootMs: traceWindow.durationMs, rootStartToFirstTurnCompletionMs: performanceEvaluation.rootStartToFirstTurnCompletionMs, rootEndToInstallCompletionMs, - firstTurnCommandMs, + ...firstTurnLatency, rootEndToFirstTurnCompletionMs: performanceEvaluation.rootEndToFirstTurnCompletionMs, tracePhasesMs: traceWindow.phaseDurationsMs, }, @@ -265,7 +269,6 @@ async function assertColdOnboardPerformance(input: { rootStartToFirstTurnCompletionSecs, budget: input.budget, performance: { - anomalies: performanceEvaluation.anomalies, passed: performanceEvaluation.passed, violations: performanceEvaluation.violations, usedAuthoritativeLocalBaseBuild, @@ -294,11 +297,6 @@ async function assertColdOnboardPerformance(input: { compactAssistantReply, `expected the sentinel first agent reply, got: ${turnText}`, ).toContain(EXPECTED_FIRST_REPLY); - for (const anomaly of performanceEvaluation.anomalies) { - console.warn( - `::warning title=Hosted first-turn latency anomaly::root-end-to-first-turn-completion ${anomaly.measurementMs}ms exceeded ${anomaly.budgetMs}ms by ${anomaly.overageMs}ms after all deterministic cold-onboard budgets passed`, - ); - } expect( performanceEvaluation.passed, `onboard-root-start-to-first-turn-completion took ${rootStartToFirstTurnCompletionSecs}s; ${performanceEvaluation.violations.join("; ")}`, diff --git a/test/e2e/support/onboard-performance.test.ts b/test/e2e/support/onboard-performance.test.ts index a8f59ef0c99..d124b4905e9 100644 --- a/test/e2e/support/onboard-performance.test.ts +++ b/test/e2e/support/onboard-performance.test.ts @@ -9,7 +9,11 @@ import { readColdOnboardPerformanceBudget, readOnboardTraceWindow, } from "../fixtures/onboard-performance.ts"; -import { extractOpenClawAgentPayloadText } from "../live/agent-turn-latency-helpers.ts"; +import { + buildOpenClawFirstTurnLatencyEvidence, + extractOpenClawAgentDurationEvidence, + extractOpenClawAgentPayloadText, +} from "../live/agent-turn-latency-helpers.ts"; const TRACE_ID = "0123456789abcdef0123456789abcdef"; const FOREIGN_TRACE_ID = "fedcba9876543210fedcba9876543210"; @@ -206,7 +210,6 @@ describe("onboard performance evidence", () => { expect(evaluateColdOnboardPerformance(trace, 6_000, budget)).toEqual({ appliedAuthoritativeLocalBaseBuildAllowanceMs: 0, - anomalies: [], passed: true, rootStartToFirstTurnCompletionMs: 5_000, rootEndToFirstTurnCompletionMs: 0, @@ -214,7 +217,6 @@ describe("onboard performance evidence", () => { }); expect(evaluateColdOnboardPerformance(trace, 7_500, budget)).toEqual({ appliedAuthoritativeLocalBaseBuildAllowanceMs: 0, - anomalies: [], passed: false, rootStartToFirstTurnCompletionMs: 6_500, rootEndToFirstTurnCompletionMs: 1_500, @@ -230,14 +232,13 @@ describe("onboard performance evidence", () => { ]); expect(evaluateColdOnboardPerformance(trace, 6_500, budget, true)).toMatchObject({ appliedAuthoritativeLocalBaseBuildAllowanceMs: 500, - anomalies: [], passed: true, rootStartToFirstTurnCompletionMs: 5_500, violations: [], }); }); - it("classifies a sole hosted first-turn tail as a structured non-blocking anomaly (#6660)", () => { + it("keeps a sole hosted first-turn overage blocking until recurrence policy is defined", () => { const trace = readOnboardTraceWindow(traceArtifact()); const budget = readColdOnboardPerformanceBudget({ fullE2eColdPath: { @@ -250,18 +251,10 @@ describe("onboard performance evidence", () => { expect(evaluateColdOnboardPerformance(trace, 7_500, budget)).toEqual({ appliedAuthoritativeLocalBaseBuildAllowanceMs: 0, - anomalies: [ - { - budgetMs: 1_000, - kind: "first-turn-latency-tail", - measurementMs: 1_500, - overageMs: 500, - }, - ], - passed: true, + passed: false, rootStartToFirstTurnCompletionMs: 6_500, rootEndToFirstTurnCompletionMs: 1_500, - violations: [], + violations: ["root-end-to-first-turn-completion 1500ms exceeds 1000ms"], }); }); @@ -278,7 +271,6 @@ describe("onboard performance evidence", () => { }); expect(evaluateColdOnboardPerformance(trace, 7_500, budget)).toMatchObject({ - anomalies: [], passed: false, violations: [ "root-end-to-first-turn-completion 1500ms exceeds 1000ms", @@ -390,4 +382,32 @@ describe("onboard performance evidence", () => { ), ).toBe("NEMOCLAW_\nE2E_READY_6002"); }); + + it("records OpenClaw internal-agent duration with an explicit availability state", () => { + expect( + buildOpenClawFirstTurnLatencyEvidence( + `progress\n${JSON.stringify({ result: { meta: { durationMs: 8_916 } } })}`, + 10_125, + ), + ).toEqual({ + firstTurnAgentDuration: { durationMs: 8_916, status: "available" }, + firstTurnCommandMs: 10_125, + }); + }); + + it("records missing OpenClaw duration metadata as unavailable", () => { + expect( + extractOpenClawAgentDurationEvidence( + JSON.stringify({ result: { payloads: [{ text: "NEMOCLAW_E2E_READY_6002" }] } }), + ), + ).toEqual({ reason: "missing", status: "unavailable" }); + }); + + it("records malformed OpenClaw duration metadata as unavailable", () => { + expect( + extractOpenClawAgentDurationEvidence( + JSON.stringify({ result: { meta: { durationMs: "8916" } } }), + ), + ).toEqual({ reason: "malformed", status: "unavailable" }); + }); }); From 3d0463c683847563b851bd25e3ba3753b401b964 Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Thu, 30 Jul 2026 10:27:33 -0700 Subject: [PATCH 3/6] test(e2e): restore first-turn anomaly policy --- test/e2e/README.md | 11 +++-- test/e2e/fixtures/onboard-performance.ts | 49 ++++++++++++++++---- test/e2e/live/full-e2e.test.ts | 6 +++ test/e2e/support/onboard-performance.test.ts | 18 +++++-- 4 files changed, 70 insertions(+), 14 deletions(-) diff --git a/test/e2e/README.md b/test/e2e/README.md index cdebd40133c..b86d2e9a290 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -975,9 +975,14 @@ The artifact records the first-turn command wall clock and OpenClaw's internal agent duration separately. Older or malformed OpenClaw output records an explicit unavailable reason instead of fabricating a duration. The artifact also identifies the model, provider, inference mode, and prompt contract so -same-cohort samples can support a later, documented recurrence policy. All -configured performance violations remain blocking until that policy is -accepted. +same-cohort samples can support a later, documented recurrence policy. When +every deterministic cold-onboard budget passes and the real first turn exits +successfully with the expected sentinel, a sole root-end-to-first-turn overage +is recorded as a structured, non-blocking hosted-latency anomaly rather than a +PR regression. The same overage remains blocking when accompanied by a +root-start or phase-budget failure. The artifact preserves the measurement, +budget, and overage so recurring same-model, same-mode evidence can calibrate +tail-latency enforcement without weakening functional or deterministic gates. When changed base-image inputs require the authoritative local OpenClaw base build, the target applies the separately calibrated 90-second allowance only to diff --git a/test/e2e/fixtures/onboard-performance.ts b/test/e2e/fixtures/onboard-performance.ts index d4a47e3e7b3..981e1e79cc7 100644 --- a/test/e2e/fixtures/onboard-performance.ts +++ b/test/e2e/fixtures/onboard-performance.ts @@ -43,12 +43,25 @@ export interface ColdOnboardPerformanceBudget { export interface ColdOnboardPerformanceEvaluation { appliedAuthoritativeLocalBaseBuildAllowanceMs: number; + anomalies: ColdOnboardPerformanceAnomaly[]; passed: boolean; rootEndToFirstTurnCompletionMs: number; rootStartToFirstTurnCompletionMs: number; violations: string[]; } +export interface ColdOnboardPerformanceAnomaly { + budgetMs: number; + kind: "first-turn-latency-tail"; + measurementMs: number; + overageMs: number; +} + +interface ColdOnboardPerformanceFinding { + kind: "phase" | "root-end-to-first-turn" | "root-start-to-first-turn"; + message: string; +} + interface ParsedSpan { durationMs: number; endNs: bigint; @@ -266,16 +279,18 @@ export function evaluateColdOnboardPerformance( const sandboxBudgetMs = budget.phaseBudgetsMs["nemoclaw.onboard.phase.sandbox"] + appliedAuthoritativeLocalBaseBuildAllowanceMs; - const violations: string[] = []; + const findings: ColdOnboardPerformanceFinding[] = []; if (rootStartToFirstTurnCompletionMs > rootStartBudgetMs) { - violations.push( - `root-start-to-first-turn-completion ${rootStartToFirstTurnCompletionMs}ms exceeds ${rootStartBudgetMs}ms`, - ); + findings.push({ + kind: "root-start-to-first-turn", + message: `root-start-to-first-turn-completion ${rootStartToFirstTurnCompletionMs}ms exceeds ${rootStartBudgetMs}ms`, + }); } if (rootEndToFirstTurnCompletionMs > budget.rootEndToFirstTurnCompletionBudgetMs) { - violations.push( - `root-end-to-first-turn-completion ${rootEndToFirstTurnCompletionMs}ms exceeds ${budget.rootEndToFirstTurnCompletionBudgetMs}ms`, - ); + findings.push({ + kind: "root-end-to-first-turn", + message: `root-end-to-first-turn-completion ${rootEndToFirstTurnCompletionMs}ms exceeds ${budget.rootEndToFirstTurnCompletionBudgetMs}ms`, + }); } for (const phaseName of ONBOARD_PHASE_NAMES) { const phaseBudgetMs = @@ -284,12 +299,30 @@ export function evaluateColdOnboardPerformance( : budget.phaseBudgetsMs[phaseName]; const phaseDurationMs = trace.phaseDurationsMs[phaseName]; if (phaseBudgetMs !== undefined && phaseDurationMs > phaseBudgetMs) { - violations.push(`${phaseName} ${phaseDurationMs}ms exceeds ${phaseBudgetMs}ms`); + findings.push({ + kind: "phase", + message: `${phaseName} ${phaseDurationMs}ms exceeds ${phaseBudgetMs}ms`, + }); } } + const soleFinding = findings.length === 1 ? findings[0] : null; + const anomalies: ColdOnboardPerformanceAnomaly[] = + soleFinding?.kind === "root-end-to-first-turn" + ? [ + { + budgetMs: budget.rootEndToFirstTurnCompletionBudgetMs, + kind: "first-turn-latency-tail", + measurementMs: rootEndToFirstTurnCompletionMs, + overageMs: rootEndToFirstTurnCompletionMs - budget.rootEndToFirstTurnCompletionBudgetMs, + }, + ] + : []; + const violations = anomalies.length === 0 ? findings.map((finding) => finding.message) : []; + return { appliedAuthoritativeLocalBaseBuildAllowanceMs, + anomalies, passed: violations.length === 0, rootStartToFirstTurnCompletionMs, rootEndToFirstTurnCompletionMs, diff --git a/test/e2e/live/full-e2e.test.ts b/test/e2e/live/full-e2e.test.ts index f93309638e2..99af33661f5 100644 --- a/test/e2e/live/full-e2e.test.ts +++ b/test/e2e/live/full-e2e.test.ts @@ -269,6 +269,7 @@ async function assertColdOnboardPerformance(input: { rootStartToFirstTurnCompletionSecs, budget: input.budget, performance: { + anomalies: performanceEvaluation.anomalies, passed: performanceEvaluation.passed, violations: performanceEvaluation.violations, usedAuthoritativeLocalBaseBuild, @@ -297,6 +298,11 @@ async function assertColdOnboardPerformance(input: { compactAssistantReply, `expected the sentinel first agent reply, got: ${turnText}`, ).toContain(EXPECTED_FIRST_REPLY); + for (const anomaly of performanceEvaluation.anomalies) { + console.warn( + `::warning title=Hosted first-turn latency anomaly::root-end-to-first-turn-completion ${anomaly.measurementMs}ms exceeded ${anomaly.budgetMs}ms by ${anomaly.overageMs}ms after all deterministic cold-onboard budgets passed`, + ); + } expect( performanceEvaluation.passed, `onboard-root-start-to-first-turn-completion took ${rootStartToFirstTurnCompletionSecs}s; ${performanceEvaluation.violations.join("; ")}`, diff --git a/test/e2e/support/onboard-performance.test.ts b/test/e2e/support/onboard-performance.test.ts index d124b4905e9..c7074c1107e 100644 --- a/test/e2e/support/onboard-performance.test.ts +++ b/test/e2e/support/onboard-performance.test.ts @@ -210,6 +210,7 @@ describe("onboard performance evidence", () => { expect(evaluateColdOnboardPerformance(trace, 6_000, budget)).toEqual({ appliedAuthoritativeLocalBaseBuildAllowanceMs: 0, + anomalies: [], passed: true, rootStartToFirstTurnCompletionMs: 5_000, rootEndToFirstTurnCompletionMs: 0, @@ -217,6 +218,7 @@ describe("onboard performance evidence", () => { }); expect(evaluateColdOnboardPerformance(trace, 7_500, budget)).toEqual({ appliedAuthoritativeLocalBaseBuildAllowanceMs: 0, + anomalies: [], passed: false, rootStartToFirstTurnCompletionMs: 6_500, rootEndToFirstTurnCompletionMs: 1_500, @@ -232,13 +234,14 @@ describe("onboard performance evidence", () => { ]); expect(evaluateColdOnboardPerformance(trace, 6_500, budget, true)).toMatchObject({ appliedAuthoritativeLocalBaseBuildAllowanceMs: 500, + anomalies: [], passed: true, rootStartToFirstTurnCompletionMs: 5_500, violations: [], }); }); - it("keeps a sole hosted first-turn overage blocking until recurrence policy is defined", () => { + it("classifies a sole hosted first-turn tail as a structured non-blocking anomaly (#6660)", () => { const trace = readOnboardTraceWindow(traceArtifact()); const budget = readColdOnboardPerformanceBudget({ fullE2eColdPath: { @@ -251,10 +254,18 @@ describe("onboard performance evidence", () => { expect(evaluateColdOnboardPerformance(trace, 7_500, budget)).toEqual({ appliedAuthoritativeLocalBaseBuildAllowanceMs: 0, - passed: false, + anomalies: [ + { + budgetMs: 1_000, + kind: "first-turn-latency-tail", + measurementMs: 1_500, + overageMs: 500, + }, + ], + passed: true, rootStartToFirstTurnCompletionMs: 6_500, rootEndToFirstTurnCompletionMs: 1_500, - violations: ["root-end-to-first-turn-completion 1500ms exceeds 1000ms"], + violations: [], }); }); @@ -271,6 +282,7 @@ describe("onboard performance evidence", () => { }); expect(evaluateColdOnboardPerformance(trace, 7_500, budget)).toMatchObject({ + anomalies: [], passed: false, violations: [ "root-end-to-first-turn-completion 1500ms exceeds 1000ms", From f1811a1e92d35e848a35c3e39289917503649721 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Thu, 30 Jul 2026 14:41:04 -0700 Subject: [PATCH 4/6] test(e2e): enforce recurring first-turn latency Signed-off-by: Prekshi Vyas --- .github/workflows/e2e.yaml | 12 +- .../scorecard/analyze-first-turn-latency.mts | 281 ++++++++++++++++++ scripts/scorecard/analyze-runtime-history.mts | 70 ++++- test/e2e/README.md | 27 +- test/e2e/live/full-e2e.test.ts | 6 +- .../e2e-first-turn-latency-history.test.ts | 161 ++++++++++ .../e2e-operations-workflow-boundary.test.ts | 15 + test/e2e/support/e2e-runtime-history.test.ts | 127 +++++++- tools/e2e/operations-workflow-boundary.mts | 7 +- ...upload-e2e-artifacts-workflow-boundary.mts | 2 +- 10 files changed, 679 insertions(+), 29 deletions(-) create mode 100644 scripts/scorecard/analyze-first-turn-latency.mts create mode 100644 test/e2e/support/e2e-first-turn-latency-history.test.ts diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 770c9f12dd8..7789b5afacd 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -6134,6 +6134,9 @@ jobs: const runtimeHistory = require( path.join(process.env.GITHUB_WORKSPACE, 'scripts/scorecard/analyze-runtime-history.mts'), ); + const firstTurnLatency = require( + path.join(process.env.GITHUB_WORKSPACE, 'scripts/scorecard/analyze-first-turn-latency.mts'), + ); const needs = JSON.parse(process.env.NEEDS_JSON || '{}'); // GitHub's jobs API is the canonical source because `needs.live` @@ -6168,6 +6171,13 @@ jobs: { github, context, core }, runtimeHistoryRows, process.env.RUNTIME_SUMMARY_FILE, + { + currentFirstTurnLatency: + firstTurnLatency.readCurrentFirstTurnLatencySample( + process.env.RUNTIME_ARTIFACTS, + ), + loadPriorNightlySummaries: runtimeHistory.loadPriorNightlySummaries, + }, ); const trace = await traceTiming.buildTraceTimingResult({ github, context, core }); if (trace.budgetWarningMessage) core.warning(trace.budgetWarningMessage); @@ -6252,7 +6262,7 @@ jobs: } - name: Upload E2E runtime summary - if: ${{ always() && github.event_name == 'schedule' && steps.scorecard.outcome == 'success' }} + if: ${{ always() && github.event_name == 'schedule' }} uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 with: name: e2e-runtime-summary diff --git a/scripts/scorecard/analyze-first-turn-latency.mts b/scripts/scorecard/analyze-first-turn-latency.mts new file mode 100644 index 00000000000..4e239814f1b --- /dev/null +++ b/scripts/scorecard/analyze-first-turn-latency.mts @@ -0,0 +1,281 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; + +export const FIRST_TURN_LATENCY_MIN_SAMPLES = 12; +export const FIRST_TURN_LATENCY_MAX_ANOMALIES = 1; + +const FIRST_TURN_ARTIFACT_FILE = "onboard-progress-budget.json"; +const FIRST_TURN_ARTIFACT_SCHEMA = "nemoclaw.full_e2e_cold_performance.v3"; +const FIRST_TURN_ANOMALY_KIND = "first-turn-latency-tail"; +const MAX_ARTIFACT_BYTES = 256 * 1024; +const MAX_DIRECTORY_DEPTH = 4; +const MAX_DIRECTORY_ENTRIES = 10_000; +const MAX_DURATION_MS = 24 * 60 * 60 * 1000; + +export interface FirstTurnCohort { + agent: string; + inferenceMode: string; + model: string; + promptContract: string; + provider: string; +} + +export interface FirstTurnLatencySample { + anomaly: boolean; + budgetMs: number; + cohort: FirstTurnCohort; + measurementMs: number; + overageMs: number; +} + +export interface FirstTurnLatencyHistorySummary { + createdAt: string; + firstTurnLatency: FirstTurnLatencySample | null; + runId: number; +} + +export interface FirstTurnLatencyRecurrence { + anomalyCount: number; + cohort: FirstTurnCohort | null; + eligibleSamples: number; + message: string | null; + passed: boolean; +} + +function asRecord(value: unknown): Record | null { + return value !== null && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : null; +} + +function hasExactKeys(value: Record, expected: readonly string[]): boolean { + return Object.keys(value).sort().join("\0") === [...expected].sort().join("\0"); +} + +function isBoundedString(value: unknown): value is string { + return ( + typeof value === "string" && + value.length > 0 && + value.length <= 500 && + !/[\u0000-\u001f\u007f]/u.test(value) + ); +} + +function isDuration(value: unknown): value is number { + return ( + typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= MAX_DURATION_MS + ); +} + +function normalizeCohort(value: unknown): FirstTurnCohort | null { + const cohort = asRecord(value); + if ( + !cohort || + !hasExactKeys(cohort, ["agent", "inferenceMode", "model", "promptContract", "provider"]) || + !isBoundedString(cohort.agent) || + !isBoundedString(cohort.inferenceMode) || + !isBoundedString(cohort.model) || + !isBoundedString(cohort.promptContract) || + !isBoundedString(cohort.provider) + ) { + return null; + } + return { + agent: cohort.agent, + inferenceMode: cohort.inferenceMode, + model: cohort.model, + promptContract: cohort.promptContract, + provider: cohort.provider, + }; +} + +export function normalizeFirstTurnLatencySample(value: unknown): FirstTurnLatencySample | null { + const sample = asRecord(value); + if ( + !sample || + !hasExactKeys(sample, ["anomaly", "budgetMs", "cohort", "measurementMs", "overageMs"]) || + typeof sample.anomaly !== "boolean" || + !isDuration(sample.budgetMs) || + !isDuration(sample.measurementMs) || + !isDuration(sample.overageMs) + ) { + return null; + } + const cohort = normalizeCohort(sample.cohort); + if ( + !cohort || + sample.overageMs !== Math.max(0, sample.measurementMs - sample.budgetMs) || + sample.anomaly !== sample.overageMs > 0 + ) { + return null; + } + return { + anomaly: sample.anomaly, + budgetMs: sample.budgetMs, + cohort, + measurementMs: sample.measurementMs, + overageMs: sample.overageMs, + }; +} + +function findArtifactFiles(root: string): string[] { + const matches: string[] = []; + let visited = 0; + const visit = (directory: string, depth: number): void => { + if (depth > MAX_DIRECTORY_DEPTH || visited > MAX_DIRECTORY_ENTRIES) return; + let entries: fs.Dirent[]; + try { + entries = fs.readdirSync(directory, { withFileTypes: true }); + } catch { + return; + } + for (const entry of entries) { + visited += 1; + if (visited > MAX_DIRECTORY_ENTRIES) return; + if (entry.isSymbolicLink()) continue; + const candidate = path.join(directory, entry.name); + if (entry.isDirectory()) { + visit(candidate, depth + 1); + } else if (entry.isFile() && entry.name === FIRST_TURN_ARTIFACT_FILE) { + matches.push(candidate); + } + } + }; + visit(root, 0); + return matches; +} + +function readCurrentArtifact(root: string): unknown { + const matches = findArtifactFiles(root); + if (matches.length !== 1) return null; + try { + const stat = fs.statSync(matches[0]!); + if (!stat.isFile() || stat.size < 1 || stat.size > MAX_ARTIFACT_BYTES) return null; + return JSON.parse(fs.readFileSync(matches[0]!, "utf8")); + } catch { + return null; + } +} + +export function readCurrentFirstTurnLatencySample(root: string): FirstTurnLatencySample | null { + const artifact = asRecord(readCurrentArtifact(root)); + const performance = asRecord(artifact?.performance); + const phaseMeasurements = asRecord(artifact?.phaseMeasurements); + const budget = asRecord(artifact?.budget); + const cohort = normalizeCohort(artifact?.firstTurnCohort); + if ( + artifact?.schemaVersion !== FIRST_TURN_ARTIFACT_SCHEMA || + artifact.installExitCode !== 0 || + artifact.firstTurnExitCode !== 0 || + artifact.firstTurnSentinelMatched !== true || + artifact.buildKitFallback !== false || + artifact.usedBuildKitPrebuild !== true || + artifact.classicBuildSteps !== 0 || + !isDuration(artifact.maxSilenceSecs) || + !isDuration(artifact.maxSilenceBudgetSecs) || + artifact.maxSilenceSecs > artifact.maxSilenceBudgetSecs || + !performance || + performance.passed !== true || + !Array.isArray(performance.violations) || + performance.violations.length !== 0 || + !Array.isArray(performance.anomalies) || + performance.anomalies.length > 1 || + !phaseMeasurements || + !budget || + !cohort || + !isDuration(phaseMeasurements.rootEndToFirstTurnCompletionMs) || + !isDuration(budget.rootEndToFirstTurnCompletionBudgetMs) + ) { + return null; + } + + const measurementMs = phaseMeasurements.rootEndToFirstTurnCompletionMs; + const budgetMs = budget.rootEndToFirstTurnCompletionBudgetMs; + const overageMs = Math.max(0, measurementMs - budgetMs); + const anomaly = performance.anomalies.length === 1; + if (anomaly !== overageMs > 0) return null; + if (anomaly) { + const finding = asRecord(performance.anomalies[0]); + if ( + !finding || + finding.kind !== FIRST_TURN_ANOMALY_KIND || + finding.measurementMs !== measurementMs || + finding.budgetMs !== budgetMs || + finding.overageMs !== overageMs + ) { + return null; + } + } + + return { anomaly, budgetMs, cohort, measurementMs, overageMs }; +} + +function cohortIdentity(cohort: FirstTurnCohort): string { + return JSON.stringify([ + cohort.agent, + cohort.inferenceMode, + cohort.model, + cohort.provider, + cohort.promptContract, + ]); +} + +export function evaluateFirstTurnLatencyRecurrence( + current: FirstTurnLatencySample | null, + priorSummaries: readonly FirstTurnLatencyHistorySummary[], +): FirstTurnLatencyRecurrence { + if (current === null) { + return { + anomalyCount: 0, + cohort: null, + eligibleSamples: 0, + message: null, + passed: true, + }; + } + + const identity = cohortIdentity(current.cohort); + const priorSamples = [...priorSummaries] + .sort((left, right) => Date.parse(right.createdAt) - Date.parse(left.createdAt)) + .flatMap((summary) => { + const sample = summary.firstTurnLatency; + return sample && cohortIdentity(sample.cohort) === identity ? [sample] : []; + }); + const window = [current, ...priorSamples].slice(0, FIRST_TURN_LATENCY_MIN_SAMPLES); + const anomalyCount = window.filter((sample) => sample.anomaly).length; + const passed = + window.length < FIRST_TURN_LATENCY_MIN_SAMPLES || + !current.anomaly || + anomalyCount <= FIRST_TURN_LATENCY_MAX_ANOMALIES; + const cohortLabel = `${current.cohort.provider}/${current.cohort.model}/${current.cohort.inferenceMode}/${current.cohort.promptContract}`; + return { + anomalyCount, + cohort: current.cohort, + eligibleSamples: window.length, + message: passed + ? null + : `hosted first-turn latency recurred for ${cohortLabel}: ${anomalyCount} anomalies in ${window.length} eligible same-cohort samples`, + passed, + }; +} + +export function formatFirstTurnLatencyRecurrence(result: FirstTurnLatencyRecurrence): string { + const lines = ["## Hosted First-Turn Latency", ""]; + if (result.cohort === null) { + lines.push("No eligible current first-turn sample was available."); + } else if (result.eligibleSamples < FIRST_TURN_LATENCY_MIN_SAMPLES) { + lines.push( + `${result.eligibleSamples} of ${FIRST_TURN_LATENCY_MIN_SAMPLES} eligible same-cohort samples are available. Recurrence enforcement starts after the window is full.`, + ); + } else if (result.passed) { + lines.push( + `The current sample passed the recurrence rule with ${result.anomalyCount} anomalous samples in the ${FIRST_TURN_LATENCY_MIN_SAMPLES}-sample same-cohort window.`, + ); + } else { + lines.push(`❌ ${result.message}.`); + } + return `${lines.join("\n")}\n`; +} diff --git a/scripts/scorecard/analyze-runtime-history.mts b/scripts/scorecard/analyze-runtime-history.mts index 177b1f66ac1..fd533e171f2 100644 --- a/scripts/scorecard/analyze-runtime-history.mts +++ b/scripts/scorecard/analyze-runtime-history.mts @@ -8,6 +8,12 @@ import type { RuntimeHistorySample, RuntimeOutcome, } from "../audit-test-runtime.mts"; +import { + evaluateFirstTurnLatencyRecurrence, + type FirstTurnLatencySample, + formatFirstTurnLatencyRecurrence, + normalizeFirstTurnLatencySample, +} from "./analyze-first-turn-latency.mts"; import { readValidatedArtifactZipEntry } from "./read-artifact-zip.mts"; export const RUNTIME_SUMMARY_ARTIFACT = "e2e-runtime-summary"; @@ -15,10 +21,12 @@ export const RUNTIME_SUMMARY_FILE = "e2e-runtime-summary.json"; export const RUNTIME_REGRESSION_MIN_DELTA_MS = 30_000; export const RUNTIME_REGRESSION_MIN_PERCENT = 20; -const RUNTIME_SUMMARY_SCHEMA = "nemoclaw.e2e_runtime_summary.v1"; +const LEGACY_RUNTIME_SUMMARY_SCHEMA = "nemoclaw.e2e_runtime_summary.v1"; +const RUNTIME_SUMMARY_SCHEMA = "nemoclaw.e2e_runtime_summary.v2"; const WORKFLOW_FILE = "e2e.yaml"; -const HISTORY_RUN_LIMIT = 10; -const HISTORY_QUERY_LIMIT = 20; +const HISTORY_RUN_LIMIT = 30; +const HISTORY_QUERY_LIMIT = 30; +const RUNTIME_TREND_LIMIT = 10; const FLAKE_WATCH_LIMIT = 5; const MAX_SUMMARY_BYTES = 512 * 1024; const MAX_SUMMARY_ROWS = 200; @@ -28,17 +36,22 @@ const MAX_DURATION_MS = 7 * 24 * 60 * 60 * 1000; type GitHubDeps = { github: any; context: { repo: { owner: string; repo: string }; runId: number }; - core?: { warning?: (message: string) => void }; + core?: { + setFailed?: (message: string) => void; + warning?: (message: string) => void; + }; }; export interface RuntimeSummaryArtifact { - schemaVersion: typeof RUNTIME_SUMMARY_SCHEMA; + schemaVersion: typeof LEGACY_RUNTIME_SUMMARY_SCHEMA | typeof RUNTIME_SUMMARY_SCHEMA; runId: number; createdAt: string; + firstTurnLatency: FirstTurnLatencySample | null; rows: RuntimeHistorySample[]; } type RuntimeHistoryServices = { + currentFirstTurnLatency?: FirstTurnLatencySample | null; loadPriorNightlySummaries: (deps: GitHubDeps) => Promise; }; @@ -124,9 +137,15 @@ function isCanonicalTimestamp(value: unknown): value is string { export function normalizeRuntimeSummary(value: unknown): RuntimeSummaryArtifact | null { if (!value || typeof value !== "object" || Array.isArray(value)) return null; const summary = value as Record; + const legacy = summary.schemaVersion === LEGACY_RUNTIME_SUMMARY_SCHEMA; if ( - !hasExactKeys(summary, ["schemaVersion", "runId", "createdAt", "rows"]) || - summary.schemaVersion !== RUNTIME_SUMMARY_SCHEMA || + !hasExactKeys( + summary, + legacy + ? ["schemaVersion", "runId", "createdAt", "rows"] + : ["schemaVersion", "runId", "createdAt", "firstTurnLatency", "rows"], + ) || + (!legacy && summary.schemaVersion !== RUNTIME_SUMMARY_SCHEMA) || !Number.isSafeInteger(summary.runId) || (summary.runId as number) < 1 || !isCanonicalTimestamp(summary.createdAt) || @@ -140,10 +159,16 @@ export function normalizeRuntimeSummary(value: unknown): RuntimeSummaryArtifact const normalizedRows = rows as RuntimeHistorySample[]; const identities = normalizedRows.map((row) => JSON.stringify([row.target, row.scenario])); if (new Set(identities).size !== identities.length) return null; + const firstTurnLatency = + legacy || summary.firstTurnLatency === null + ? null + : normalizeFirstTurnLatencySample(summary.firstTurnLatency); + if (!legacy && summary.firstTurnLatency !== null && firstTurnLatency === null) return null; return { - schemaVersion: RUNTIME_SUMMARY_SCHEMA, + schemaVersion: legacy ? LEGACY_RUNTIME_SUMMARY_SCHEMA : RUNTIME_SUMMARY_SCHEMA, runId: summary.runId as number, createdAt: summary.createdAt, + firstTurnLatency, rows: normalizedRows, }; } @@ -152,11 +177,13 @@ export function createRuntimeSummary( runId: number, createdAt: string, rows: readonly RuntimeHistorySample[], + firstTurnLatency: FirstTurnLatencySample | null = null, ): RuntimeSummaryArtifact { const summary = normalizeRuntimeSummary({ schemaVersion: RUNTIME_SUMMARY_SCHEMA, runId, createdAt, + firstTurnLatency, rows, }); if (summary === null) throw new Error("invalid current E2E runtime summary"); @@ -418,9 +445,9 @@ export function formatRuntimeHistory( currentRows: readonly RuntimeHistorySample[], priorSummaries: readonly RuntimeSummaryArtifact[], ): string { - const sortedPrior = [...priorSummaries].sort( - (left, right) => Date.parse(right.createdAt) - Date.parse(left.createdAt), - ); + const sortedPrior = [...priorSummaries] + .sort((left, right) => Date.parse(right.createdAt) - Date.parse(left.createdAt)) + .slice(0, RUNTIME_TREND_LIMIT); const lines = [ "## E2E Nightly Runtime Trend", "", @@ -473,9 +500,16 @@ export async function buildRuntimeHistory( services: RuntimeHistoryServices = { loadPriorNightlySummaries }, now = new Date(), ): Promise { + const hasFirstTurnLatency = Object.hasOwn(services, "currentFirstTurnLatency"); + const currentFirstTurnLatency = services.currentFirstTurnLatency ?? null; let current: RuntimeSummaryArtifact; try { - current = createRuntimeSummary(deps.context.runId, now.toISOString(), currentRows); + current = createRuntimeSummary( + deps.context.runId, + now.toISOString(), + currentRows, + currentFirstTurnLatency, + ); const serialized = `${JSON.stringify(current, null, 2)}\n`; if (Buffer.byteLength(serialized) > MAX_SUMMARY_BYTES) { throw new Error("current E2E runtime summary exceeds its size bound"); @@ -489,11 +523,19 @@ export async function buildRuntimeHistory( } try { const prior = await services.loadPriorNightlySummaries(deps); - return formatRuntimeHistory(current.rows, prior); + const runtimeHistory = formatRuntimeHistory(current.rows, prior); + if (!hasFirstTurnLatency) return runtimeHistory; + const recurrence = evaluateFirstTurnLatencyRecurrence(current.firstTurnLatency, prior); + if (!recurrence.passed && recurrence.message) deps.core?.setFailed?.(recurrence.message); + return `${runtimeHistory}\n${formatFirstTurnLatencyRecurrence(recurrence)}`; } catch { deps.core?.warning?.( "Nightly E2E runtime history unavailable; current summary was still saved.", ); - return formatRuntimeHistory(current.rows, []); + const runtimeHistory = formatRuntimeHistory(current.rows, []); + if (!hasFirstTurnLatency) return runtimeHistory; + return `${runtimeHistory}\n${formatFirstTurnLatencyRecurrence( + evaluateFirstTurnLatencyRecurrence(current.firstTurnLatency, []), + )}`; } } diff --git a/test/e2e/README.md b/test/e2e/README.md index 2ac9e2a50ba..56e5185933a 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -972,16 +972,27 @@ phase limits in the budget file, and limits the longest onboard output gap to `full-e2e`, and the target writes its evidence to `onboard-progress-budget.json`. The artifact records the first-turn command wall clock and OpenClaw's internal agent duration separately. Older or malformed OpenClaw output records an -explicit unavailable reason instead of fabricating a duration. The artifact -also identifies the model, provider, inference mode, and prompt contract so -same-cohort samples can support a later, documented recurrence policy. When -every deterministic cold-onboard budget passes and the real first turn exits +explicit unavailable reason instead of fabricating a duration. +The artifact also identifies the model, provider, inference mode, and prompt contract. +When every deterministic cold-onboard budget passes and the real first turn exits successfully with the expected sentinel, a sole root-end-to-first-turn overage is recorded as a structured, non-blocking hosted-latency anomaly rather than a -PR regression. The same overage remains blocking when accompanied by a -root-start or phase-budget failure. The artifact preserves the measurement, -budget, and overage so recurring same-model, same-mode evidence can calibrate -tail-latency enforcement without weakening functional or deterministic gates. +PR regression. +The same overage remains blocking when accompanied by a root-start or +phase-budget failure. + +The trusted scheduled scorecard stores the current eligible sample in the +`e2e-runtime-summary` artifact. +The scorecard compares only samples with the same agent, provider, model, +inference mode, and prompt contract. +The recurrence window contains the 12 most recent eligible samples from +scheduled `main` runs. +The current anomaly fails the scorecard when the window is full and contains at +least one earlier anomaly. +A current sample without an anomaly does not fail because of an earlier anomaly. +Missing, malformed, or functionally unsuccessful samples do not enter the window. +The scorecard waits for 12 eligible samples when retained history is incomplete. +The canonical E2E uploader retains each nightly summary for 14 days. When changed base-image inputs require the authoritative local OpenClaw base build, the target applies the separately calibrated 90-second allowance only to diff --git a/test/e2e/live/full-e2e.test.ts b/test/e2e/live/full-e2e.test.ts index 99af33661f5..e7422ef615e 100644 --- a/test/e2e/live/full-e2e.test.ts +++ b/test/e2e/live/full-e2e.test.ts @@ -243,6 +243,7 @@ async function assertColdOnboardPerformance(input: { const assistantReply = extractOpenClawAgentPayloadText(turnText).trim(); const firstTurnLatency = buildOpenClawFirstTurnLatencyEvidence(turnText, firstTurnCommandMs); const compactAssistantReply = assistantReply.replace(/\s+/gu, ""); + const firstTurnSentinelMatched = compactAssistantReply.includes(EXPECTED_FIRST_REPLY); const responseChars = assistantReply.length; await input.artifacts.writeJson("onboard-progress-budget.json", { @@ -250,6 +251,7 @@ async function assertColdOnboardPerformance(input: { sandbox: SANDBOX_NAME, installExitCode: input.install.exitCode, firstTurnExitCode: turn.exitCode, + firstTurnSentinelMatched, phaseMeasurements: { onboardRootMs: traceWindow.durationMs, rootStartToFirstTurnCompletionMs: performanceEvaluation.rootStartToFirstTurnCompletionMs, @@ -295,9 +297,9 @@ async function assertColdOnboardPerformance(input: { ).toBeLessThanOrEqual(MAX_SILENCE_SECS); expect(turn.exitCode, turnText).toBe(0); expect( - compactAssistantReply, + firstTurnSentinelMatched, `expected the sentinel first agent reply, got: ${turnText}`, - ).toContain(EXPECTED_FIRST_REPLY); + ).toBe(true); for (const anomaly of performanceEvaluation.anomalies) { console.warn( `::warning title=Hosted first-turn latency anomaly::root-end-to-first-turn-completion ${anomaly.measurementMs}ms exceeded ${anomaly.budgetMs}ms by ${anomaly.overageMs}ms after all deterministic cold-onboard budgets passed`, diff --git a/test/e2e/support/e2e-first-turn-latency-history.test.ts b/test/e2e/support/e2e-first-turn-latency-history.test.ts new file mode 100644 index 00000000000..66f2883c985 --- /dev/null +++ b/test/e2e/support/e2e-first-turn-latency-history.test.ts @@ -0,0 +1,161 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { + evaluateFirstTurnLatencyRecurrence, + FIRST_TURN_LATENCY_MIN_SAMPLES, + type FirstTurnCohort, + type FirstTurnLatencyHistorySummary, + type FirstTurnLatencySample, + formatFirstTurnLatencyRecurrence, + readCurrentFirstTurnLatencySample, +} from "../../../scripts/scorecard/analyze-first-turn-latency.mts"; + +const COHORT: FirstTurnCohort = { + agent: "openclaw", + inferenceMode: "agent-thinking-off", + model: "nvidia/nemotron-3-super-120b-a12b", + promptContract: "sentinel-v1", + provider: "NVIDIA", +}; + +function sample(anomaly: boolean, cohort: FirstTurnCohort = COHORT): FirstTurnLatencySample { + const budgetMs = 14_000; + const measurementMs = anomaly ? 14_500 : 8_000; + return { + anomaly, + budgetMs, + cohort, + measurementMs, + overageMs: Math.max(0, measurementMs - budgetMs), + }; +} + +function summary( + runId: number, + firstTurnLatency: FirstTurnLatencySample | null, +): FirstTurnLatencyHistorySummary { + return { + createdAt: new Date(Date.UTC(2026, 6, runId)).toISOString(), + firstTurnLatency, + runId, + }; +} + +function artifact(anomaly: boolean): Record { + const current = sample(anomaly); + return { + schemaVersion: "nemoclaw.full_e2e_cold_performance.v3", + installExitCode: 0, + firstTurnExitCode: 0, + firstTurnSentinelMatched: true, + phaseMeasurements: { + rootEndToFirstTurnCompletionMs: current.measurementMs, + }, + firstTurnCohort: current.cohort, + budget: { + rootEndToFirstTurnCompletionBudgetMs: current.budgetMs, + }, + performance: { + anomalies: anomaly + ? [ + { + budgetMs: current.budgetMs, + kind: "first-turn-latency-tail", + measurementMs: current.measurementMs, + overageMs: current.overageMs, + }, + ] + : [], + passed: true, + violations: [], + }, + buildKitFallback: false, + usedBuildKitPrebuild: true, + classicBuildSteps: 0, + maxSilenceSecs: 20, + maxSilenceBudgetSecs: 60, + }; +} + +describe("hosted first-turn latency history", () => { + it("reads an eligible current full-E2E sample and rejects failed functional evidence", () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-first-turn-")); + const artifactDirectory = path.join(directory, "e2e-full-e2e"); + const artifactFile = path.join(artifactDirectory, "onboard-progress-budget.json"); + try { + fs.mkdirSync(artifactDirectory, { recursive: true }); + fs.writeFileSync(artifactFile, JSON.stringify(artifact(true))); + + expect(readCurrentFirstTurnLatencySample(directory)).toEqual(sample(true)); + + fs.writeFileSync( + artifactFile, + JSON.stringify({ ...artifact(true), firstTurnSentinelMatched: false }), + ); + expect(readCurrentFirstTurnLatencySample(directory)).toBeNull(); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + }); + + it("keeps an anomaly non-blocking until 12 eligible same-cohort samples exist (#6660)", () => { + const prior = Array.from({ length: FIRST_TURN_LATENCY_MIN_SAMPLES - 2 }, (_, index) => + summary(index + 1, sample(index === 0)), + ); + + const result = evaluateFirstTurnLatencyRecurrence(sample(true), prior); + + expect(result).toMatchObject({ + anomalyCount: 2, + eligibleSamples: FIRST_TURN_LATENCY_MIN_SAMPLES - 1, + message: null, + passed: true, + }); + expect(formatFirstTurnLatencyRecurrence(result)).toContain( + "Recurrence enforcement starts after the window is full.", + ); + }); + + it("blocks a current anomaly that a prior anomaly corroborates in a full window (#6660)", () => { + const prior = Array.from({ length: FIRST_TURN_LATENCY_MIN_SAMPLES - 1 }, (_, index) => + summary(index + 1, sample(index === 0)), + ); + + const result = evaluateFirstTurnLatencyRecurrence(sample(true), prior); + + expect(result).toMatchObject({ + anomalyCount: 2, + eligibleSamples: FIRST_TURN_LATENCY_MIN_SAMPLES, + passed: false, + }); + expect(result.message).toContain("2 anomalies in 12 eligible same-cohort samples"); + }); + + it("does not mix cohorts or fail a current sample without an anomaly (#6660)", () => { + const otherCohort = { ...COHORT, model: "other-model" }; + const prior = [ + summary(1, sample(true, otherCohort)), + ...Array.from({ length: FIRST_TURN_LATENCY_MIN_SAMPLES - 1 }, (_, index) => + summary(index + 2, sample(index < 2)), + ), + ]; + + expect(evaluateFirstTurnLatencyRecurrence(sample(true), prior)).toMatchObject({ + anomalyCount: 3, + eligibleSamples: FIRST_TURN_LATENCY_MIN_SAMPLES, + passed: false, + }); + expect(evaluateFirstTurnLatencyRecurrence(sample(false), prior)).toMatchObject({ + anomalyCount: 2, + eligibleSamples: FIRST_TURN_LATENCY_MIN_SAMPLES, + passed: true, + }); + }); +}); diff --git a/test/e2e/support/e2e-operations-workflow-boundary.test.ts b/test/e2e/support/e2e-operations-workflow-boundary.test.ts index 6af2066da71..41096349647 100644 --- a/test/e2e/support/e2e-operations-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-operations-workflow-boundary.test.ts @@ -676,10 +676,15 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; buildRuntimeHistory: vi .fn() .mockResolvedValue("## E2E Nightly Runtime Trend\n\n| Target | Prior median |"), + loadPriorNightlySummaries: vi.fn(), + }; + const firstTurnLatency = { + readCurrentFirstTurnLatencySample: vi.fn().mockReturnValue(null), }; const runtimeModules = new Map([ ["path", { join: (...parts: string[]) => parts.join("/") }], ["/workspace/scripts/audit-test-runtime.mts", runtimeAudit], + ["/workspace/scripts/scorecard/analyze-first-turn-latency.mts", firstTurnLatency], ["/workspace/scripts/scorecard/analyze-runtime-history.mts", runtimeHistory], ["/workspace/scripts/scorecard/coordinate-scorecard.mts", coordinator], ["/workspace/scripts/scorecard/analyze-trace-timing.mts", traceTiming], @@ -727,6 +732,13 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; { github: {}, context, core }, [{ target: "full-e2e" }], "/runner/e2e-runtime-summary.json", + { + currentFirstTurnLatency: null, + loadPriorNightlySummaries: runtimeHistory.loadPriorNightlySummaries, + }, + ); + expect(firstTurnLatency.readCurrentFirstTurnLatencySample).toHaveBeenCalledWith( + "/runner/e2e-runtime-audit", ); expect(runtimeAudit.auditTestRuntime.mock.invocationCallOrder[0]).toBeLessThan( traceTiming.buildTraceTimingResult.mock.invocationCallOrder[0], @@ -772,9 +784,11 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; formatRuntimeAuditSummary: vi.fn(), }; const runtimeHistory = { buildRuntimeHistory: vi.fn() }; + const firstTurnLatency = { readCurrentFirstTurnLatencySample: vi.fn() }; const runtimeModules = new Map([ ["path", { join: (...parts: string[]) => parts.join("/") }], ["/workspace/scripts/audit-test-runtime.mts", runtimeAudit], + ["/workspace/scripts/scorecard/analyze-first-turn-latency.mts", firstTurnLatency], ["/workspace/scripts/scorecard/analyze-runtime-history.mts", runtimeHistory], [ "/workspace/scripts/scorecard/coordinate-scorecard.mts", @@ -838,6 +852,7 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; ); expect(runtimeAudit.formatRuntimeAuditSummary).not.toHaveBeenCalled(); expect(runtimeAudit.collectRuntimeHistorySamples).not.toHaveBeenCalled(); + expect(firstTurnLatency.readCurrentFirstTurnLatencySample).not.toHaveBeenCalled(); expect(runtimeHistory.buildRuntimeHistory).not.toHaveBeenCalled(); expect(summary.addRaw).toHaveBeenCalledWith( expect.stringMatching( diff --git a/test/e2e/support/e2e-runtime-history.test.ts b/test/e2e/support/e2e-runtime-history.test.ts index 772fee31a42..7dac8589672 100644 --- a/test/e2e/support/e2e-runtime-history.test.ts +++ b/test/e2e/support/e2e-runtime-history.test.ts @@ -8,6 +8,10 @@ import path from "node:path"; import { describe, expect, it, vi } from "vitest"; import type { RuntimeHistorySample } from "../../../scripts/audit-test-runtime.mts"; +import { + evaluateFirstTurnLatencyRecurrence, + type FirstTurnLatencySample, +} from "../../../scripts/scorecard/analyze-first-turn-latency.mts"; import { buildRuntimeHistory, createRuntimeSummary, @@ -30,6 +34,22 @@ function runtimeSample(overrides: Partial = {}): RuntimeHi }; } +function firstTurnSample(anomaly: boolean): FirstTurnLatencySample { + return { + anomaly, + budgetMs: 14_000, + cohort: { + agent: "openclaw", + inferenceMode: "agent-thinking-off", + model: "nvidia/nemotron-3-super-120b-a12b", + promptContract: "sentinel-v1", + provider: "NVIDIA", + }, + measurementMs: anomaly ? 14_500 : 8_000, + overageMs: anomaly ? 500 : 0, + }; +} + describe("E2E rolling runtime history", () => { it("reports runtime distribution, outcomes, failure streaks, and significant phase regressions", () => { const current = runtimeSample({ @@ -141,7 +161,8 @@ describe("E2E rolling runtime history", () => { ); expect(JSON.parse(fs.readFileSync(output, "utf8"))).toMatchObject({ - schemaVersion: "nemoclaw.e2e_runtime_summary.v1", + schemaVersion: "nemoclaw.e2e_runtime_summary.v2", + firstTurnLatency: null, runId: 123, }); expect(fs.statSync(output).mode & 0o777).toBe(0o600); @@ -152,8 +173,55 @@ describe("E2E rolling runtime history", () => { } }); + it("saves current latency evidence and fails a corroborated 12-sample recurrence (#6660)", async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-runtime-history-")); + const output = path.join(directory, "e2e-runtime-summary.json"); + const setFailed = vi.fn(); + const prior = Array.from({ length: 11 }, (_, index) => + createRuntimeSummary( + index + 1, + new Date(Date.UTC(2026, 6, index + 1)).toISOString(), + [runtimeSample()], + firstTurnSample(index === 0), + ), + ); + try { + const markdown = await buildRuntimeHistory( + { + github: {}, + context: { repo: { owner: "NVIDIA", repo: "NemoClaw" }, runId: 123 }, + core: { setFailed }, + }, + [runtimeSample()], + output, + { + currentFirstTurnLatency: firstTurnSample(true), + loadPriorNightlySummaries: vi.fn().mockResolvedValue(prior), + }, + new Date("2026-07-24T00:00:00.000Z"), + ); + + expect(JSON.parse(fs.readFileSync(output, "utf8"))).toMatchObject({ + firstTurnLatency: { anomaly: true, overageMs: 500 }, + }); + expect(markdown).toContain("## Hosted First-Turn Latency"); + expect(setFailed).toHaveBeenCalledWith( + expect.stringContaining("2 anomalies in 12 eligible same-cohort samples"), + ); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + }); + it("rejects duplicate identities, duplicate phases, and extra fields", () => { const summary = createRuntimeSummary(1, "2026-07-24T00:00:00.000Z", [runtimeSample()]); + const { firstTurnLatency: _, ...legacy } = summary; + expect( + normalizeRuntimeSummary({ + ...legacy, + schemaVersion: "nemoclaw.e2e_runtime_summary.v1", + }), + ).toMatchObject({ firstTurnLatency: null }); expect(normalizeRuntimeSummary({ ...summary, extra: true })).toBeNull(); expect( normalizeRuntimeSummary({ ...summary, rows: [summary.rows[0], summary.rows[0]] }), @@ -202,6 +270,63 @@ describe("E2E rolling runtime history", () => { expect(paginate.mock.calls[0]?.[1]).toMatchObject({ run_id: 122 }); }); + it("loads older eligible samples after recent summaries without cohort evidence (#6660)", async () => { + const runIds = Array.from({ length: 12 }, (_, index) => 200 - index); + const summaries = new Map( + runIds.map((runId, index) => [ + runId, + createRuntimeSummary( + runId, + new Date(Date.UTC(2026, 6, 20 - index)).toISOString(), + [runtimeSample()], + index === 0 ? null : firstTurnSample(index === runIds.length - 1), + ), + ]), + ); + const loaded = await loadPriorNightlySummaries({ + context: { repo: { owner: "NVIDIA", repo: "NemoClaw" }, runId: 999 }, + github: { + paginate: vi.fn( + async ( + _method: unknown, + options: { run_id: number }, + ): Promise> => [ + { + expired: false, + id: options.run_id + 1_000, + name: RUNTIME_SUMMARY_ARTIFACT, + }, + ], + ), + rest: { + actions: { + downloadArtifact: vi.fn( + async (options: { artifact_id: number }): Promise<{ data: Buffer }> => ({ + data: artifactZip([ + { + name: RUNTIME_SUMMARY_FILE, + contents: JSON.stringify(summaries.get(options.artifact_id - 1_000)), + }, + ]), + }), + ), + listWorkflowRunArtifacts: {}, + listWorkflowRuns: vi.fn().mockResolvedValue({ + data: { workflow_runs: runIds.map((id) => ({ id })) }, + }), + }, + }, + }, + }); + + expect(loaded).toHaveLength(12); + expect(evaluateFirstTurnLatencyRecurrence(firstTurnSample(true), loaded)).toMatchObject({ + anomalyCount: 2, + eligibleSamples: 12, + passed: false, + }); + }); + it("rejects a runtime summary whose embedded run ID does not match its workflow run", async () => { const summary = createRuntimeSummary(121, "2026-07-23T00:00:00.000Z", [runtimeSample()]); const downloadArtifact = vi.fn().mockResolvedValue({ diff --git a/tools/e2e/operations-workflow-boundary.mts b/tools/e2e/operations-workflow-boundary.mts index adb12602b80..417bfe8273d 100644 --- a/tools/e2e/operations-workflow-boundary.mts +++ b/tools/e2e/operations-workflow-boundary.mts @@ -751,6 +751,10 @@ function validateScorecard(errors: string[], workflow: OperationsWorkflow): void "runtimeAudit.formatRuntimeAuditSummary", "scripts/scorecard/analyze-runtime-history.mts", "runtimeHistory.buildRuntimeHistory", + "scripts/scorecard/analyze-first-turn-latency.mts", + "firstTurnLatency.readCurrentFirstTurnLatencySample", + "currentFirstTurnLatency", + "runtimeHistory.loadPriorNightlySummaries", "core.summary", "scorecardData", "slackData", @@ -815,8 +819,7 @@ function validateScorecard(errors: string[], workflow: OperationsWorkflow): void requirePinnedAction(errors, runtimeUpload, "scorecard runtime summary upload"); if ( !String(runtimeUpload.uses ?? "").startsWith(E2E_ARTIFACT_ACTION) || - runtimeUpload.if !== - "${{ always() && github.event_name == 'schedule' && steps.scorecard.outcome == 'success' }}" || + runtimeUpload.if !== "${{ always() && github.event_name == 'schedule' }}" || runtimeUpload.with?.name !== "e2e-runtime-summary" || runtimeUpload.with?.path !== "${{ runner.temp }}/e2e-runtime-summary.json" ) { diff --git a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts index c0142ad13da..978d3805158 100644 --- a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts +++ b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts @@ -43,7 +43,7 @@ const TARGET_ID_PATTERN = /^[A-Za-z0-9_-]+$/; const SCORECARD_RUNTIME_UPLOAD_CONTRACT: WorkflowStep = { name: "Upload E2E runtime summary", - if: "${{ always() && github.event_name == 'schedule' && steps.scorecard.outcome == 'success' }}", + if: "${{ always() && github.event_name == 'schedule' }}", uses: UPLOAD_E2E_ARTIFACTS_ACTION, with: { name: "e2e-runtime-summary", From 1fb1a269db1820ca3413d292200a181026c5bbab Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Thu, 30 Jul 2026 15:05:33 -0700 Subject: [PATCH 5/6] test(e2e): harden first-turn history parsing Signed-off-by: Prekshi Vyas --- scripts/scorecard/analyze-first-turn-latency.mts | 11 ++++++----- scripts/scorecard/analyze-runtime-history.mts | 2 +- .../support/e2e-first-turn-latency-history.test.ts | 2 ++ 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/scripts/scorecard/analyze-first-turn-latency.mts b/scripts/scorecard/analyze-first-turn-latency.mts index 4e239814f1b..b12a106cfd8 100644 --- a/scripts/scorecard/analyze-first-turn-latency.mts +++ b/scripts/scorecard/analyze-first-turn-latency.mts @@ -162,12 +162,13 @@ function readCurrentArtifact(root: string): unknown { export function readCurrentFirstTurnLatencySample(root: string): FirstTurnLatencySample | null { const artifact = asRecord(readCurrentArtifact(root)); - const performance = asRecord(artifact?.performance); - const phaseMeasurements = asRecord(artifact?.phaseMeasurements); - const budget = asRecord(artifact?.budget); - const cohort = normalizeCohort(artifact?.firstTurnCohort); + if (!artifact) return null; + const performance = asRecord(artifact.performance); + const phaseMeasurements = asRecord(artifact.phaseMeasurements); + const budget = asRecord(artifact.budget); + const cohort = normalizeCohort(artifact.firstTurnCohort); if ( - artifact?.schemaVersion !== FIRST_TURN_ARTIFACT_SCHEMA || + artifact.schemaVersion !== FIRST_TURN_ARTIFACT_SCHEMA || artifact.installExitCode !== 0 || artifact.firstTurnExitCode !== 0 || artifact.firstTurnSentinelMatched !== true || diff --git a/scripts/scorecard/analyze-runtime-history.mts b/scripts/scorecard/analyze-runtime-history.mts index fd533e171f2..0da717b5836 100644 --- a/scripts/scorecard/analyze-runtime-history.mts +++ b/scripts/scorecard/analyze-runtime-history.mts @@ -451,7 +451,7 @@ export function formatRuntimeHistory( const lines = [ "## E2E Nightly Runtime Trend", "", - "Current timing compared with up to 10 prior completed scheduled runs; manual runs are excluded from history.", + `Current timing compared with up to ${RUNTIME_TREND_LIMIT} prior completed scheduled runs; manual runs are excluded from history.`, `Regression warnings require both +${seconds(RUNTIME_REGRESSION_MIN_DELTA_MS)} and +${RUNTIME_REGRESSION_MIN_PERCENT}%.`, "", ]; diff --git a/test/e2e/support/e2e-first-turn-latency-history.test.ts b/test/e2e/support/e2e-first-turn-latency-history.test.ts index 66f2883c985..a2723f6d874 100644 --- a/test/e2e/support/e2e-first-turn-latency-history.test.ts +++ b/test/e2e/support/e2e-first-turn-latency-history.test.ts @@ -90,6 +90,8 @@ describe("hosted first-turn latency history", () => { const artifactDirectory = path.join(directory, "e2e-full-e2e"); const artifactFile = path.join(artifactDirectory, "onboard-progress-budget.json"); try { + expect(readCurrentFirstTurnLatencySample(directory)).toBeNull(); + fs.mkdirSync(artifactDirectory, { recursive: true }); fs.writeFileSync(artifactFile, JSON.stringify(artifact(true))); From 30d7f921851ba4cbd21691e1ca41290a5bffc387 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Thu, 30 Jul 2026 15:37:43 -0700 Subject: [PATCH 6/6] fix(e2e): secure first-turn artifact reads Signed-off-by: Prekshi Vyas --- scripts/scorecard/analyze-first-turn-latency.mts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/scorecard/analyze-first-turn-latency.mts b/scripts/scorecard/analyze-first-turn-latency.mts index b12a106cfd8..64e9226be1f 100644 --- a/scripts/scorecard/analyze-first-turn-latency.mts +++ b/scripts/scorecard/analyze-first-turn-latency.mts @@ -151,12 +151,16 @@ function findArtifactFiles(root: string): string[] { function readCurrentArtifact(root: string): unknown { const matches = findArtifactFiles(root); if (matches.length !== 1) return null; + let descriptor: number | null = null; try { - const stat = fs.statSync(matches[0]!); + descriptor = fs.openSync(matches[0]!, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + const stat = fs.fstatSync(descriptor); if (!stat.isFile() || stat.size < 1 || stat.size > MAX_ARTIFACT_BYTES) return null; - return JSON.parse(fs.readFileSync(matches[0]!, "utf8")); + return JSON.parse(fs.readFileSync(descriptor, "utf8")); } catch { return null; + } finally { + if (descriptor !== null) fs.closeSync(descriptor); } }