From f4e9a471495f30f4fb575a9adae85d52f68520c0 Mon Sep 17 00:00:00 2001 From: San Dang Date: Fri, 31 Jul 2026 17:34:48 +0700 Subject: [PATCH 1/3] test(e2e): calibrate restart-safe startup budget Signed-off-by: San Dang --- ci/full-e2e-cold-path-calibration.json | 154 ++++++++++++++++- ci/onboard-performance-budget.json | 8 +- .../onboard-performance-config-schema.test.ts | 156 ++++++++++++++++-- 3 files changed, 301 insertions(+), 17 deletions(-) diff --git a/ci/full-e2e-cold-path-calibration.json b/ci/full-e2e-cold-path-calibration.json index 0c6652bf53a..b9164b471e7 100644 --- a/ci/full-e2e-cold-path-calibration.json +++ b/ci/full-e2e-cold-path-calibration.json @@ -1,5 +1,5 @@ { - "$comment": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\nSPDX-License-Identifier: Apache-2.0\n\nFive independent workflow_dispatch full-e2e samples of current-main baseline 1a74b8348c2182fbf806726341186f008444e28e with the phase-measurement changes at 4544d07c8bfd500c3b64a74380ef5cd0e62089f5. All runs completed install, BuildKit prebuild without fallback, the silence assertion, and the expected first turn. Each baseline budget is derived independently so phase caps diagnose regressions; they are not portions that must sum to the root-start budget. The separate validation adjustment records four existing full-e2e job observations from three descendant heads after f62c278bd737f4f47be2e85436f65b270d5b4280 added the reviewed WeChat runtime graph to the supported sandbox image. The relevant image-building inputs listed in imageInputPaths were unchanged through 2adc8481ff3053a5a7be37d130cb183e222934ff; repository tests enforce that ancestry and unchanged-input boundary. All four jobs completed install, BuildKit prebuild without fallback, the silence assertion, and the expected first turn; two exceeded the prior root-start and/or sandbox-phase caps. The conclusion fields record the full-e2e job conclusions, not aggregate workflow conclusions. The adjustment raises only those two caps from the maximum observed value plus the existing headroom, rounded up to one second. This is a bounded post-change validation adjustment, not a replacement five-run single-SHA calibration. Retire it by replacing the baseline and removing validationAdjustment after five successful full-e2e samples from one SHA that contains the image change. The authoritative-local-build adjustment records functional PR runs that emitted the required local base-build reason. The latest observation covers the longer source build required when reviewed native security packages change. Its allowance is the maximum excess over the normal root-start or sandbox-phase budget plus an operator-selected stability floor of 27 seconds (or 10 percent, if larger), rounded up to one second. It applies only when that reason is observed and does not alter published-image budgets.", + "$comment": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\nSPDX-License-Identifier: Apache-2.0\n\nFive independent workflow_dispatch full-e2e samples of current-main baseline 1a74b8348c2182fbf806726341186f008444e28e with the phase-measurement changes at 4544d07c8bfd500c3b64a74380ef5cd0e62089f5. All runs completed install, BuildKit prebuild without fallback, the silence assertion, and the expected first turn. Each baseline budget is derived independently so phase caps diagnose regressions; they are not portions that must sum to the root-start budget. The separate validation adjustment records four existing full-e2e job observations from three descendant heads after f62c278bd737f4f47be2e85436f65b270d5b4280 added the reviewed WeChat runtime graph to the supported sandbox image. The relevant image-building inputs listed in imageInputPaths were unchanged through 2adc8481ff3053a5a7be37d130cb183e222934ff; repository tests enforce that ancestry and unchanged-input boundary. All four jobs completed install, BuildKit prebuild without fallback, the silence assertion, and the expected first turn; two exceeded the prior root-start and/or sandbox-phase caps. The restart-safe-startup adjustment records five later functional full-e2e jobs after b675ed361a02d2d602900e1f301ba117af7effaf began recreating OpenShell Docker containers with the persisted OpenClaw startup command. Every run exercised that exact output, completed install and the expected first turn, used BuildKit prebuild without fallback, and stayed within the silence limit; four exceeded the stale root-start and sandbox-phase caps. The adjustment raises only those two caps from the maximum observed value plus 10 percent, rounded up to one second. The conclusion fields record the full-e2e job conclusions, not aggregate workflow conclusions. These bounded post-change adjustments do not replace the five-run single-SHA calibration. Retire them when five successful full-e2e samples from one eligible SHA replace the baseline. The authoritative-local-build adjustment records functional PR runs that emitted the required local base-build reason. The latest observation covers the longer source build required when reviewed native security packages change. Its allowance is the maximum excess over the normal root-start or sandbox-phase budget plus an operator-selected stability floor of 27 seconds (or 10 percent, if larger), rounded up to one second. It applies only when that reason is observed and does not alter published-image budgets.", "schemaVersion": 1, "calibratedAt": "2026-07-13", "baselineMainSha": "1a74b8348c2182fbf806726341186f008444e28e", @@ -327,6 +327,156 @@ "sandboxPhaseBudgetMs": 171000 } }, + "restartSafeStartupAdjustment": { + "validatedAt": "2026-07-31", + "changeSha": "b675ed361a02d2d602900e1f301ba117af7effaf", + "runtimeInputsVerifiedThroughSha": "f8fb820159c4843a19759efc9b0e28d4aa122440", + "runtimeInputPaths": [ + "src/lib/onboard/docker-startup-command-agent.ts", + "src/lib/onboard/docker-gpu-sandbox-create.ts", + "src/lib/onboard/docker-startup-command-sandbox-create.ts", + "src/lib/onboard/sandbox-gpu-create-run-attempt.ts" + ], + "triggerOutput": "Recreating OpenShell Docker sandbox container with restart-safe startup...", + "adjustedMetrics": [ + "rootStartToFirstTurnCompletion", + "nemoclaw.onboard.phase.sandbox" + ], + "derivation": { + "statistic": "maximum", + "minimumHeadroomMs": 5000, + "relativeHeadroomPercent": 10, + "roundUpMs": 1000 + }, + "runs": [ + { + "runId": 30614075121, + "runUrl": "https://github.com/NVIDIA/NemoClaw/actions/runs/30614075121", + "jobId": 91103324359, + "workflowHeadSha": "387cb08644fe030bb85146255f4b77e3c54697d2", + "testedSha": "387cb08644fe030bb85146255f4b77e3c54697d2", + "conclusion": "failure", + "installExitCode": 0, + "firstTurnExitCode": 0, + "firstTurnSentinelMatched": true, + "performancePassed": false, + "usedBuildKitPrebuild": true, + "buildKitFallback": false, + "maxSilenceSecs": 51, + "responseChars": 23, + "triggerEvidence": { + "artifact": "e2e-full-e2e", + "path": "full-e2e-install-onboard-inference-cli-operations-and-cleanup/shell/phase-1-install-sh.stdout.txt", + "output": "Recreating OpenShell Docker sandbox container with restart-safe startup..." + }, + "rootStartToFirstTurnCompletionMs": 238921, + "sandboxPhaseMs": 185161 + }, + { + "runId": 30615995748, + "runUrl": "https://github.com/NVIDIA/NemoClaw/actions/runs/30615995748", + "jobId": 91109377750, + "workflowHeadSha": "387cb08644fe030bb85146255f4b77e3c54697d2", + "testedSha": "915b25c522d982fc7d40d01e583bc8f15bcf975f", + "conclusion": "failure", + "installExitCode": 0, + "firstTurnExitCode": 0, + "firstTurnSentinelMatched": true, + "performancePassed": false, + "usedBuildKitPrebuild": true, + "buildKitFallback": false, + "maxSilenceSecs": 55, + "responseChars": 23, + "triggerEvidence": { + "artifact": "e2e-full-e2e", + "path": "full-e2e-install-onboard-inference-cli-operations-and-cleanup/shell/phase-1-install-sh.stdout.txt", + "output": "Recreating OpenShell Docker sandbox container with restart-safe startup..." + }, + "rootStartToFirstTurnCompletionMs": 236352, + "sandboxPhaseMs": 188905 + }, + { + "runId": 30619965759, + "runUrl": "https://github.com/NVIDIA/NemoClaw/actions/runs/30619965759", + "jobId": 91125282922, + "workflowHeadSha": "f8fb820159c4843a19759efc9b0e28d4aa122440", + "testedSha": "c3106eea0669aa645c0ff6f51adce0badf24477f", + "conclusion": "failure", + "installExitCode": 0, + "firstTurnExitCode": 0, + "firstTurnSentinelMatched": true, + "performancePassed": false, + "usedBuildKitPrebuild": true, + "buildKitFallback": false, + "maxSilenceSecs": 43, + "responseChars": 23, + "triggerEvidence": { + "artifact": "e2e-full-e2e", + "path": "full-e2e-install-onboard-inference-cli-operations-and-cleanup/shell/phase-1-install-sh.stdout.txt", + "output": "Recreating OpenShell Docker sandbox container with restart-safe startup..." + }, + "rootStartToFirstTurnCompletionMs": 238040, + "sandboxPhaseMs": 183471 + }, + { + "runId": 30620296004, + "runUrl": "https://github.com/NVIDIA/NemoClaw/actions/runs/30620296004", + "jobId": 91129295688, + "workflowHeadSha": "f8fb820159c4843a19759efc9b0e28d4aa122440", + "testedSha": "f8fb820159c4843a19759efc9b0e28d4aa122440", + "conclusion": "success", + "installExitCode": 0, + "firstTurnExitCode": 0, + "firstTurnSentinelMatched": true, + "performancePassed": true, + "usedBuildKitPrebuild": true, + "buildKitFallback": false, + "maxSilenceSecs": 28, + "responseChars": 23, + "triggerEvidence": { + "artifact": "e2e-full-e2e", + "path": "full-e2e-install-onboard-inference-cli-operations-and-cleanup/shell/phase-1-install-sh.stdout.txt", + "output": "Recreating OpenShell Docker sandbox container with restart-safe startup..." + }, + "rootStartToFirstTurnCompletionMs": 181679, + "sandboxPhaseMs": 131415 + }, + { + "runId": 30620879680, + "runUrl": "https://github.com/NVIDIA/NemoClaw/actions/runs/30620879680", + "jobId": 91125043079, + "workflowHeadSha": "f8fb820159c4843a19759efc9b0e28d4aa122440", + "testedSha": "d1b24c97c348215574fd8c55435a2e8c5e0d86e1", + "conclusion": "failure", + "installExitCode": 0, + "firstTurnExitCode": 0, + "firstTurnSentinelMatched": true, + "performancePassed": false, + "usedBuildKitPrebuild": true, + "buildKitFallback": false, + "maxSilenceSecs": 45, + "responseChars": 23, + "triggerEvidence": { + "artifact": "e2e-full-e2e", + "path": "full-e2e-install-onboard-inference-cli-operations-and-cleanup/shell/phase-1-install-sh.stdout.txt", + "output": "Recreating OpenShell Docker sandbox container with restart-safe startup..." + }, + "rootStartToFirstTurnCompletionMs": 233979, + "sandboxPhaseMs": 184868 + } + ], + "retirement": { + "trigger": "successful-single-sha-calibration", + "minimumSampleCount": 5, + "allSamplesSameHead": true, + "runtimeChangeMustBeAncestor": true, + "action": "replace-baseline-and-remove-adjustment" + }, + "derivedCapsMs": { + "rootStartToFirstTurnCompletionBudgetMs": 263000, + "sandboxPhaseBudgetMs": 208000 + } + }, "authoritativeLocalBaseBuildAdjustment": { "validatedAt": "2026-07-28", "triggerOutput": "Building OpenClaw sandbox base image locally because no compatible published base image was found.", @@ -393,7 +543,7 @@ "nativeSecurityInputsMustBeUnchanged": true, "action": "replace-baseline-and-remove-adjustment" }, - "derivedAllowanceMs": 610000 + "derivedAllowanceMs": 570000 }, "derivedBudgetsMs": { "rootStartToFirstTurnCompletionBudgetMs": 205000, diff --git a/ci/onboard-performance-budget.json b/ci/onboard-performance-budget.json index 979deb25d37..40afc272688 100644 --- a/ci/onboard-performance-budget.json +++ b/ci/onboard-performance-budget.json @@ -1,5 +1,5 @@ { - "$comment": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\nSPDX-License-Identifier: Apache-2.0\n\nInitial advisory budget for the cloud-onboard-e2e warm-system trace signal. Profiling traces from #3769 were not available in durable CI artifacts when #3776 was implemented. The latest three release tags only exposed one tag-matching trace artifact, from a failed v0.0.66 nightly run, so this cap is calibrated from the latest ten distinct successful main full-trace samples available on 2026-06-23. Those samples had total durations of 298250 ms, 296926 ms, 304190 ms, 294859 ms, 305013 ms, 316147 ms, 300843 ms, 292702 ms, 201332 ms, and 206250 ms; the cap uses p95 via linear interpolation (index 8.55 between samples 8 and 9) plus 25 percent, rounded up to the nearest 30 seconds.\n\nThe full-e2e cold-path baseline budgets are derived from the five current-main samples recorded in ci/full-e2e-cold-path-calibration.json. For each interval and phase independently, the baseline is nearest-rank p95 plus the larger of 5 seconds or 10 percent, rounded up to the nearest second. Four later functional full-e2e jobs completed install, BuildKit prebuild without fallback, and the expected first turn on their listed heads after the supported sandbox image changed; their separately recorded bounded validation adjustment raises only the root-start and sandbox-phase caps to the observed maximum plus 10 percent, rounded up to one second. The adjustment is retired after five successful samples from one head containing the image change, when that calibration replaces the baseline and the adjustment is removed. Independent phase caps are diagnostic regression gates, not additive portions of the root-start interval. The authoritative local base-build allowance is derived separately from exact PR evidence recorded in ci/full-e2e-cold-path-calibration.json. It applies only when full-e2e observes the exact local-build reason and adjusts only the root-start and sandbox-phase caps; published-image runs retain the normal budgets.", + "$comment": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\nSPDX-License-Identifier: Apache-2.0\n\nInitial advisory budget for the cloud-onboard-e2e warm-system trace signal. Profiling traces from #3769 were not available in durable CI artifacts when #3776 was implemented. The latest three release tags only exposed one tag-matching trace artifact, from a failed v0.0.66 nightly run, so this cap is calibrated from the latest ten distinct successful main full-trace samples available on 2026-06-23. Those samples had total durations of 298250 ms, 296926 ms, 304190 ms, 294859 ms, 305013 ms, 316147 ms, 300843 ms, 292702 ms, 201332 ms, and 206250 ms; the cap uses p95 via linear interpolation (index 8.55 between samples 8 and 9) plus 25 percent, rounded up to the nearest 30 seconds.\n\nThe full-e2e cold-path baseline budgets are derived from the five current-main samples recorded in ci/full-e2e-cold-path-calibration.json. For each interval and phase independently, the baseline is nearest-rank p95 plus the larger of 5 seconds or 10 percent, rounded up to the nearest second. The calibration records bounded post-change adjustments for the supported sandbox image and the restart-safe OpenClaw Docker startup lifecycle. Each adjustment raises only the root-start and sandbox-phase caps to its observed maximum plus 10 percent, rounded up to one second, and is retired when five successful samples from one eligible head replace the baseline. Independent phase caps are diagnostic regression gates, not additive portions of the root-start interval. The authoritative local base-build allowance is derived separately from exact PR evidence recorded in ci/full-e2e-cold-path-calibration.json. It applies only when full-e2e observes the exact local-build reason and adjusts only the root-start and sandbox-phase caps; published-image runs retain the normal budgets.", "schemaVersion": 1, "mode": "advisory", "scope": "cloud-onboard-e2e warm-system", @@ -13,15 +13,15 @@ "minPercent": 30 }, "fullE2eColdPath": { - "authoritativeLocalBaseBuildAllowanceMs": 610000, - "rootStartToFirstTurnCompletionBudgetMs": 228000, + "authoritativeLocalBaseBuildAllowanceMs": 570000, + "rootStartToFirstTurnCompletionBudgetMs": 263000, "rootEndToFirstTurnCompletionBudgetMs": 14000, "phaseBudgetsMs": { "nemoclaw.onboard.phase.preflight": 11000, "nemoclaw.onboard.phase.gateway": 6000, "nemoclaw.onboard.phase.provider_selection": 8000, "nemoclaw.onboard.phase.inference": 6000, - "nemoclaw.onboard.phase.sandbox": 171000 + "nemoclaw.onboard.phase.sandbox": 208000 } } } diff --git a/test/onboard-performance-config-schema.test.ts b/test/onboard-performance-config-schema.test.ts index c9180f3ed70..da971e4b83e 100644 --- a/test/onboard-performance-config-schema.test.ts +++ b/test/onboard-performance-config-schema.test.ts @@ -85,6 +85,54 @@ interface Calibration { sandboxPhaseBudgetMs: number; }; }; + restartSafeStartupAdjustment?: { + validatedAt: string; + changeSha: string; + runtimeInputsVerifiedThroughSha: string; + runtimeInputPaths: string[]; + triggerOutput: string; + adjustedMetrics: string[]; + derivation: { + statistic: string; + minimumHeadroomMs: number; + relativeHeadroomPercent: number; + roundUpMs: number; + }; + runs: Array<{ + runId: number; + runUrl: string; + jobId: number; + workflowHeadSha: string; + testedSha: string; + conclusion: string; + installExitCode: number; + firstTurnExitCode: number; + firstTurnSentinelMatched: boolean; + performancePassed: boolean; + usedBuildKitPrebuild: boolean; + buildKitFallback: boolean; + maxSilenceSecs: number; + responseChars: number; + triggerEvidence: { + artifact: string; + path: string; + output: string; + }; + rootStartToFirstTurnCompletionMs: number; + sandboxPhaseMs: number; + }>; + retirement: { + trigger: string; + minimumSampleCount: number; + allSamplesSameHead: boolean; + runtimeChangeMustBeAncestor: boolean; + action: string; + }; + derivedCapsMs: { + rootStartToFirstTurnCompletionBudgetMs: number; + sandboxPhaseBudgetMs: number; + }; + }; authoritativeLocalBaseBuildAdjustment: { validatedAt: string; triggerOutput: string; @@ -261,21 +309,26 @@ function validationThreshold( function effectiveBudgets(input: Calibration): ColdPathBudget { const baseline = input.derivedBudgetsMs; - const adjustment = input.validationAdjustment?.derivedCapsMs; + const imageAdjustment = input.validationAdjustment?.derivedCapsMs; + const startupAdjustment = input.restartSafeStartupAdjustment?.derivedCapsMs; return { authoritativeLocalBaseBuildAllowanceMs: input.authoritativeLocalBaseBuildAdjustment.derivedAllowanceMs, ...baseline, rootStartToFirstTurnCompletionBudgetMs: Math.max( baseline.rootStartToFirstTurnCompletionBudgetMs, - adjustment?.rootStartToFirstTurnCompletionBudgetMs ?? + imageAdjustment?.rootStartToFirstTurnCompletionBudgetMs ?? + baseline.rootStartToFirstTurnCompletionBudgetMs, + startupAdjustment?.rootStartToFirstTurnCompletionBudgetMs ?? baseline.rootStartToFirstTurnCompletionBudgetMs, ), phaseBudgetsMs: { ...baseline.phaseBudgetsMs, "nemoclaw.onboard.phase.sandbox": Math.max( baseline.phaseBudgetsMs["nemoclaw.onboard.phase.sandbox"], - adjustment?.sandboxPhaseBudgetMs ?? + imageAdjustment?.sandboxPhaseBudgetMs ?? + baseline.phaseBudgetsMs["nemoclaw.onboard.phase.sandbox"], + startupAdjustment?.sandboxPhaseBudgetMs ?? baseline.phaseBudgetsMs["nemoclaw.onboard.phase.sandbox"], ), }, @@ -306,11 +359,7 @@ function gitRevision(revision: string): string { }).trim(); } -function changedImageInputs( - fromSha: string, - throughSha: string, - imageInputPaths: string[], -): string[] { +function changedInputs(fromSha: string, throughSha: string, imageInputPaths: string[]): string[] { const output = execFileSync( "git", ["diff", "--name-only", fromSha, throughSha, "--", ...imageInputPaths], @@ -325,7 +374,7 @@ function validationProvenanceViolations( const runHeadsWithChangedImageInputs = validation.runs .map((run) => ({ headSha: run.headSha, - changedPaths: changedImageInputs( + changedPaths: changedInputs( validation.imageChangeSha, run.headSha, validation.imageInputPaths, @@ -340,7 +389,7 @@ function validationProvenanceViolations( .map((run) => run.headSha) .filter((headSha) => !gitIsAncestor(headSha, validation.imageInputsVerifiedThroughSha)), runHeadsWithChangedImageInputs, - changedImageInputsThroughBoundary: changedImageInputs( + changedImageInputsThroughBoundary: changedInputs( validation.imageChangeSha, validation.imageInputsVerifiedThroughSha, validation.imageInputPaths, @@ -498,11 +547,96 @@ describe("full-E2E cold-path calibration", () => { ), }); expect(checkedInConfig.fullE2eColdPath).toEqual(effectiveBudgets(calibration)); - expect(effectiveBudgets({ ...calibration, validationAdjustment: undefined })).toEqual({ + expect( + effectiveBudgets({ + ...calibration, + validationAdjustment: undefined, + restartSafeStartupAdjustment: undefined, + }), + ).toEqual({ authoritativeLocalBaseBuildAllowanceMs: calibration.authoritativeLocalBaseBuildAdjustment.derivedAllowanceMs, ...calibration.derivedBudgetsMs, }); + + const startupAdjustment = calibration.restartSafeStartupAdjustment!; + expect(startupAdjustment.validatedAt).toMatch(/^\d{4}-\d{2}-\d{2}$/u); + expect(startupAdjustment.changeSha).toMatch(/^[0-9a-f]{40}$/u); + expect(startupAdjustment.runtimeInputsVerifiedThroughSha).toMatch(/^[0-9a-f]{40}$/u); + expect(startupAdjustment.triggerOutput).toBe( + "Recreating OpenShell Docker sandbox container with restart-safe startup...", + ); + expect(startupAdjustment.adjustedMetrics).toEqual([ + "rootStartToFirstTurnCompletion", + "nemoclaw.onboard.phase.sandbox", + ]); + expect(startupAdjustment.derivation.statistic).toBe("maximum"); + expect(startupAdjustment.runs).toHaveLength(5); + expect(new Set(startupAdjustment.runs.map((run) => run.runId)).size).toBe(5); + expect(new Set(startupAdjustment.runs.map((run) => run.jobId)).size).toBe(5); + expect( + changedInputs( + startupAdjustment.changeSha, + startupAdjustment.runtimeInputsVerifiedThroughSha, + startupAdjustment.runtimeInputPaths, + ), + ).toEqual([]); + + for (const run of startupAdjustment.runs) { + expect(run.runUrl).toBe(`https://github.com/NVIDIA/NemoClaw/actions/runs/${run.runId}`); + expect(run.workflowHeadSha).toMatch(/^[0-9a-f]{40}$/u); + expect(run.testedSha).toMatch(/^[0-9a-f]{40}$/u); + expect(gitIsAncestor(startupAdjustment.changeSha, run.workflowHeadSha)).toBe(true); + expect( + gitIsAncestor(run.workflowHeadSha, startupAdjustment.runtimeInputsVerifiedThroughSha), + ).toBe(true); + expect(run).toMatchObject({ + installExitCode: 0, + firstTurnExitCode: 0, + firstTurnSentinelMatched: true, + usedBuildKitPrebuild: true, + buildKitFallback: false, + responseChars: 23, + triggerEvidence: { + artifact: "e2e-full-e2e", + path: "full-e2e-install-onboard-inference-cli-operations-and-cleanup/shell/phase-1-install-sh.stdout.txt", + output: startupAdjustment.triggerOutput, + }, + }); + expect(run.maxSilenceSecs).toBeLessThanOrEqual(60); + } + expect(startupAdjustment.runs.map((run) => run.conclusion).sort()).toEqual([ + "failure", + "failure", + "failure", + "failure", + "success", + ]); + expect(startupAdjustment.runs.map((run) => run.performancePassed).sort()).toEqual([ + false, + false, + false, + false, + true, + ]); + expect(startupAdjustment.retirement).toEqual({ + trigger: "successful-single-sha-calibration", + minimumSampleCount: 5, + allSamplesSameHead: true, + runtimeChangeMustBeAncestor: true, + action: "replace-baseline-and-remove-adjustment", + }); + expect(startupAdjustment.derivedCapsMs).toEqual({ + rootStartToFirstTurnCompletionBudgetMs: validationThreshold( + startupAdjustment.runs.map((run) => run.rootStartToFirstTurnCompletionMs), + startupAdjustment.derivation, + ), + sandboxPhaseBudgetMs: validationThreshold( + startupAdjustment.runs.map((run) => run.sandboxPhaseMs), + startupAdjustment.derivation, + ), + }); + expect(checkedInConfig.fullE2eColdPath).toEqual(effectiveBudgets(calibration)); }); // source-shape-contract: compatibility -- Exact PR run evidence keeps the local-build allowance bounded and reproducible From 1491ddee4cd828b98dfa0355a8f52d1722e0036d Mon Sep 17 00:00:00 2001 From: San Dang Date: Fri, 31 Jul 2026 17:51:50 +0700 Subject: [PATCH 2/3] test(e2e): declare cold path budget contract Signed-off-by: San Dang --- test/e2e/live/full-e2e.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/test/e2e/live/full-e2e.test.ts b/test/e2e/live/full-e2e.test.ts index e7422ef615e..bc5a3b16b8a 100644 --- a/test/e2e/live/full-e2e.test.ts +++ b/test/e2e/live/full-e2e.test.ts @@ -336,6 +336,7 @@ test("full e2e: install, onboard, inference, cli operations, and cleanup", { USE_PREINSTALLED_LAUNCHABLE ? "the baked Launchable completes onboarding without installing from source" : "install.sh --non-interactive completes onboarding", + "cold onboarding stays within the checked-in full-E2E performance budgets", "nemoclaw and openshell are installed and usable", "sandbox appears in list/status and has policy/inference configuration", "direct hosted inference and sandbox inference.local both respond", From de59620f5fb64aa98a2dc3c616882a88314f3a2b Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Fri, 31 Jul 2026 04:01:30 -0700 Subject: [PATCH 3/3] test(e2e): bind calibration to tested revisions Signed-off-by: Apurv Kumaria --- test/onboard-performance-config-schema.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/test/onboard-performance-config-schema.test.ts b/test/onboard-performance-config-schema.test.ts index da971e4b83e..a860a00cd89 100644 --- a/test/onboard-performance-config-schema.test.ts +++ b/test/onboard-performance-config-schema.test.ts @@ -587,6 +587,7 @@ describe("full-E2E cold-path calibration", () => { expect(run.workflowHeadSha).toMatch(/^[0-9a-f]{40}$/u); expect(run.testedSha).toMatch(/^[0-9a-f]{40}$/u); expect(gitIsAncestor(startupAdjustment.changeSha, run.workflowHeadSha)).toBe(true); + expect(gitIsAncestor(startupAdjustment.changeSha, run.testedSha)).toBe(true); expect( gitIsAncestor(run.workflowHeadSha, startupAdjustment.runtimeInputsVerifiedThroughSha), ).toBe(true);