fix(e2e): bound sandbox phase latency tails

.github/workflows/e2e.yaml

@@ -2866,6 +2866,7 @@ jobs:
             e2e-artifacts/live/${{ matrix.id }}/state-validation.result.json
             e2e-artifacts/live/${{ matrix.id }}/dcode-base-image.json
             e2e-artifacts/live/${{ matrix.id }}/cloud-onboard-trace-timing-summary.json
+            e2e-artifacts/live/${{ matrix.id }}/onboard-progress-budget.json
             e2e-artifacts/live/risk-signal.json
             e2e-artifacts/live/${{ matrix.id }}/actions/
             e2e-artifacts/live/${{ matrix.id }}/logs/
@@ -5841,6 +5842,9 @@ jobs:
             const firstTurnLatency = require(
               path.join(process.env.GITHUB_WORKSPACE, 'scripts/scorecard/analyze-first-turn-latency.mts'),
             );
+            const sandboxPhaseTail = require(
+              path.join(process.env.GITHUB_WORKSPACE, 'scripts/scorecard/analyze-sandbox-phase-tail.mts'),
+            );
             const needs = JSON.parse(process.env.NEEDS_JSON || '{}');
 
             // GitHub's jobs API is the canonical source because `needs.live`
@@ -5880,7 +5884,11 @@ jobs:
                       firstTurnLatency.readCurrentFirstTurnLatencySample(
                         process.env.RUNTIME_ARTIFACTS,
                       ),
-                    loadPriorPushSummaries: runtimeHistory.loadPriorPushSummaries,
+                    currentSandboxPhaseTail:
+                      sandboxPhaseTail.readCurrentSandboxPhaseTailSample(
+                        process.env.RUNTIME_ARTIFACTS,
+                      ),
+                    loadPriorPushHistory: runtimeHistory.loadPriorPushHistory,
                   },
                 );
             const trace = await traceTiming.buildTraceTimingResult({ github, context, core });

ci/onboard-performance-budget.json

@@ -1,5 +1,5 @@
 {
-  "$comment": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\nSPDX-License-Identifier: Apache-2.0\n\nInitial advisory budget for the cloud-onboard-e2e warm-system trace signal. Profiling traces from #3769 were not available in durable CI artifacts when #3776 was implemented. The latest three release tags only exposed one tag-matching trace artifact, from a failed v0.0.66 nightly run, so this cap is calibrated from the latest ten distinct successful main full-trace samples available on 2026-06-23. Those samples had total durations of 298250 ms, 296926 ms, 304190 ms, 294859 ms, 305013 ms, 316147 ms, 300843 ms, 292702 ms, 201332 ms, and 206250 ms; the cap uses p95 via linear interpolation (index 8.55 between samples 8 and 9) plus 25 percent, rounded up to the nearest 30 seconds.\n\nThe full-e2e cold-path baseline budgets are derived from the five current-main samples recorded in ci/full-e2e-cold-path-calibration.json. For each interval and phase independently, the baseline is nearest-rank p95 plus the larger of 5 seconds or 10 percent, rounded up to the nearest second. The calibration records bounded post-change adjustments for the supported sandbox image and the restart-safe OpenClaw Docker startup lifecycle. Each adjustment raises only the root-start and sandbox-phase caps to its observed maximum plus 10 percent, rounded up to one second, and is retired when five successful samples from one eligible head replace the baseline. Independent phase caps are diagnostic regression gates, not additive portions of the root-start interval. The authoritative local base-build allowance is derived separately from exact PR evidence recorded in ci/full-e2e-cold-path-calibration.json. It applies only when full-e2e observes the exact local-build reason and adjusts only the root-start and sandbox-phase caps; published-image runs retain the normal budgets.",
+  "$comment": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\nSPDX-License-Identifier: Apache-2.0\n\nInitial advisory budget for the cloud-onboard-e2e warm-system trace signal. Profiling traces from #3769 were not available in durable CI artifacts when #3776 was implemented. The latest three release tags only exposed one tag-matching trace artifact, from a failed v0.0.66 nightly run, so this cap is calibrated from the latest ten distinct successful main full-trace samples available on 2026-06-23. Those samples had total durations of 298250 ms, 296926 ms, 304190 ms, 294859 ms, 305013 ms, 316147 ms, 300843 ms, 292702 ms, 201332 ms, and 206250 ms; the cap uses p95 via linear interpolation (index 8.55 between samples 8 and 9) plus 25 percent, rounded up to the nearest 30 seconds.\n\nThe full-e2e cold-path baseline budgets are derived from the five current-main samples recorded in ci/full-e2e-cold-path-calibration.json. For each interval and phase independently, the baseline is nearest-rank p95 plus the larger of 5 seconds or 10 percent, rounded up to the nearest second. The calibration records bounded post-change adjustments for the supported sandbox image and the restart-safe OpenClaw Docker startup lifecycle. Each adjustment raises only the root-start and sandbox-phase caps to its observed maximum plus 10 percent, rounded up to one second, and is retired when five successful samples from one eligible head replace the baseline. Independent phase caps are diagnostic regression gates, not additive portions of the root-start interval. The authoritative local base-build allowance is derived separately from exact PR evidence recorded in ci/full-e2e-cold-path-calibration.json. It applies only when full-e2e observes the exact local-build reason and adjusts only the root-start and sandbox-phase caps; published-image runs retain the normal budgets. One published-base sandbox-phase overage can use the separate checked-in maximum only through the five-sample recurrence policy in test/e2e/README.md.",
   "schemaVersion": 1,
   "mode": "advisory",
   "scope": "cloud-onboard-e2e warm-system",
@@ -14,6 +14,7 @@
   },
   "fullE2eColdPath": {
     "authoritativeLocalBaseBuildAllowanceMs": 570000,
+    "sandboxPhaseSingleObservationMaxOverageMs": 5000,
     "rootStartToFirstTurnCompletionBudgetMs": 263000,
     "rootEndToFirstTurnCompletionBudgetMs": 14000,
     "phaseBudgetsMs": {

schemas/onboard-config.schema.json

@@ -44,6 +44,7 @@
       "additionalProperties": false,
       "required": [
         "authoritativeLocalBaseBuildAllowanceMs",
+        "sandboxPhaseSingleObservationMaxOverageMs",
         "rootStartToFirstTurnCompletionBudgetMs",
         "rootEndToFirstTurnCompletionBudgetMs",
         "phaseBudgetsMs"
@@ -54,6 +55,11 @@
           "minimum": 0,
           "description": "Bounded extra time allowed for the root-start and sandbox-phase caps only when full-e2e observes the authoritative local base-image rebuild message."
         },
+        "sandboxPhaseSingleObservationMaxOverageMs": {
+          "type": "number",
+          "minimum": 0,
+          "description": "Maximum sandbox-phase overage that one published-base full-E2E sample can report as a non-blocking anomaly."
+        },
         "rootStartToFirstTurnCompletionBudgetMs": {
           "type": "number",
           "minimum": 0,

scripts/scorecard/analyze-first-turn-latency.mts

@@ -8,8 +8,9 @@ export const FIRST_TURN_LATENCY_MIN_SAMPLES = 12;
 export const FIRST_TURN_LATENCY_MAX_ANOMALIES = 1;
 
 const FIRST_TURN_ARTIFACT_FILE = "onboard-progress-budget.json";
-const FIRST_TURN_ARTIFACT_SCHEMA = "nemoclaw.full_e2e_cold_performance.v3";
+const FIRST_TURN_ARTIFACT_SCHEMA = "nemoclaw.full_e2e_cold_performance.v4";
 const FIRST_TURN_ANOMALY_KIND = "first-turn-latency-tail";
+const SANDBOX_PHASE_ANOMALY_KIND = "sandbox-phase-tail";
 const MAX_ARTIFACT_BYTES = 256 * 1024;
 const MAX_DIRECTORY_DEPTH = 4;
 const MAX_DIRECTORY_ENTRIES = 10_000;
@@ -121,6 +122,23 @@ export function normalizeFirstTurnLatencySample(value: unknown): FirstTurnLatenc
   };
 }
 
+function normalizePerformanceAnomaly(value: unknown): Record<string, unknown> | null {
+  const finding = asRecord(value);
+  if (
+    !finding ||
+    !hasExactKeys(finding, ["budgetMs", "kind", "measurementMs", "overageMs"]) ||
+    (finding.kind !== FIRST_TURN_ANOMALY_KIND && finding.kind !== SANDBOX_PHASE_ANOMALY_KIND) ||
+    !isDuration(finding.budgetMs) ||
+    !isDuration(finding.measurementMs) ||
+    !isDuration(finding.overageMs) ||
+    finding.overageMs !== (finding.measurementMs as number) - (finding.budgetMs as number) ||
+    finding.overageMs <= 0
+  ) {
+    return null;
+  }
+  return finding;
+}
+
 function findArtifactFiles(root: string): string[] {
   const matches: string[] = [];
   let visited = 0;
@@ -148,7 +166,7 @@ function findArtifactFiles(root: string): string[] {
   return matches;
 }
 
-function readCurrentArtifact(root: string): unknown {
+export function readCurrentColdOnboardArtifact(root: string): unknown {
   const matches = findArtifactFiles(root);
   if (matches.length !== 1) return null;
   let descriptor: number | null = null;
@@ -165,7 +183,7 @@ function readCurrentArtifact(root: string): unknown {
 }
 
 export function readCurrentFirstTurnLatencySample(root: string): FirstTurnLatencySample | null {
-  const artifact = asRecord(readCurrentArtifact(root));
+  const artifact = asRecord(readCurrentColdOnboardArtifact(root));
   if (!artifact) return null;
   const performance = asRecord(artifact.performance);
   const phaseMeasurements = asRecord(artifact.phaseMeasurements);
@@ -200,19 +218,18 @@ export function readCurrentFirstTurnLatencySample(root: string): FirstTurnLatenc
   const measurementMs = phaseMeasurements.rootEndToFirstTurnCompletionMs;
   const budgetMs = budget.rootEndToFirstTurnCompletionBudgetMs;
   const overageMs = Math.max(0, measurementMs - budgetMs);
-  const anomaly = performance.anomalies.length === 1;
+  const findings = performance.anomalies.map(normalizePerformanceAnomaly);
+  if (findings.some((finding) => finding === null)) return null;
+  const finding = findings.find((candidate) => candidate?.kind === FIRST_TURN_ANOMALY_KIND);
+  const anomaly = finding !== undefined;
   if (anomaly !== overageMs > 0) return null;
-  if (anomaly) {
-    const finding = asRecord(performance.anomalies[0]);
-    if (
-      !finding ||
-      finding.kind !== FIRST_TURN_ANOMALY_KIND ||
-      finding.measurementMs !== measurementMs ||
+  if (
+    finding &&
+    (finding.measurementMs !== measurementMs ||
       finding.budgetMs !== budgetMs ||
-      finding.overageMs !== overageMs
-    ) {
-      return null;
-    }
+      finding.overageMs !== overageMs)
+  ) {
+    return null;
   }
 
   return { anomaly, budgetMs, cohort, measurementMs, overageMs };