test(e2e): enforce phase-aware onboard budgets

ci/full-e2e-cold-path-calibration.json

@@ -0,0 +1,164 @@
+{
+  "$comment": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\nSPDX-License-Identifier: Apache-2.0\n\nFive independent workflow_dispatch full-e2e samples of current-main baseline 1a74b8348c2182fbf806726341186f008444e28e with the phase-measurement changes at 4544d07c8bfd500c3b64a74380ef5cd0e62089f5. All runs completed install, BuildKit prebuild without fallback, the silence assertion, and the expected first turn. Each budget is derived independently so phase caps diagnose regressions; they are not portions that must sum to the root-start budget.",
+  "schemaVersion": 1,
+  "calibratedAt": "2026-07-13",
+  "baselineMainSha": "1a74b8348c2182fbf806726341186f008444e28e",
+  "measurementHeadSha": "4544d07c8bfd500c3b64a74380ef5cd0e62089f5",
+  "workflowPath": ".github/workflows/e2e.yaml",
+  "job": "full-e2e",
+  "derivation": {
+    "percentile": 95,
+    "percentileMethod": "nearest-rank",
+    "minimumHeadroomMs": 5000,
+    "relativeHeadroomPercent": 10,
+    "roundUpMs": 1000
+  },
+  "samples": [
+    {
+      "runId": 29285379553,
+      "runUrl": "https://github.com/NVIDIA/NemoClaw/actions/runs/29285379553",
+      "headSha": "4544d07c8bfd500c3b64a74380ef5cd0e62089f5",
+      "conclusion": "success",
+      "installExitCode": 0,
+      "firstTurnExitCode": 0,
+      "performancePassed": true,
+      "usedBuildKitPrebuild": true,
+      "buildKitFallback": false,
+      "maxSilenceSecs": 31,
+      "responseChars": 23,
+      "measurementsMs": {
+        "onboardRoot": 170850,
+        "rootStartToFirstTurnCompletion": 178403,
+        "rootEndToInstallCompletion": 137,
+        "firstTurnCommand": 7413,
+        "rootEndToFirstTurnCompletion": 7553,
+        "phases": {
+          "nemoclaw.onboard.phase.preflight": 1251,
+          "nemoclaw.onboard.phase.gateway": 65,
+          "nemoclaw.onboard.phase.provider_selection": 2312,
+          "nemoclaw.onboard.phase.inference": 669,
+          "nemoclaw.onboard.phase.sandbox": 126615
+        }
+      }
+    },
+    {
+      "runId": 29285384233,
+      "runUrl": "https://github.com/NVIDIA/NemoClaw/actions/runs/29285384233",
+      "headSha": "4544d07c8bfd500c3b64a74380ef5cd0e62089f5",
+      "conclusion": "success",
+      "installExitCode": 0,
+      "firstTurnExitCode": 0,
+      "performancePassed": true,
+      "usedBuildKitPrebuild": true,
+      "buildKitFallback": false,
+      "maxSilenceSecs": 31,
+      "responseChars": 23,
+      "measurementsMs": {
+        "onboardRoot": 149056,
+        "rootStartToFirstTurnCompletion": 155856,
+        "rootEndToInstallCompletion": 135,
+        "firstTurnCommand": 6662,
+        "rootEndToFirstTurnCompletion": 6800,
+        "phases": {
+          "nemoclaw.onboard.phase.preflight": 1132,
+          "nemoclaw.onboard.phase.gateway": 58,
+          "nemoclaw.onboard.phase.provider_selection": 1142,
+          "nemoclaw.onboard.phase.inference": 563,
+          "nemoclaw.onboard.phase.sandbox": 105938
+        }
+      }
+    },
+    {
+      "runId": 29285387079,
+      "runUrl": "https://github.com/NVIDIA/NemoClaw/actions/runs/29285387079",
+      "headSha": "4544d07c8bfd500c3b64a74380ef5cd0e62089f5",
+      "conclusion": "success",
+      "installExitCode": 0,
+      "firstTurnExitCode": 0,
+      "performancePassed": true,
+      "usedBuildKitPrebuild": true,
+      "buildKitFallback": false,
+      "maxSilenceSecs": 32,
+      "responseChars": 23,
+      "measurementsMs": {
+        "onboardRoot": 177334,
+        "rootStartToFirstTurnCompletion": 186018,
+        "rootEndToInstallCompletion": 168,
+        "firstTurnCommand": 8512,
+        "rootEndToFirstTurnCompletion": 8684,
+        "phases": {
+          "nemoclaw.onboard.phase.preflight": 1193,
+          "nemoclaw.onboard.phase.gateway": 51,
+          "nemoclaw.onboard.phase.provider_selection": 1358,
+          "nemoclaw.onboard.phase.inference": 638,
+          "nemoclaw.onboard.phase.sandbox": 132561
+        }
+      }
+    },
+    {
+      "runId": 29285389469,
+      "runUrl": "https://github.com/NVIDIA/NemoClaw/actions/runs/29285389469",
+      "headSha": "4544d07c8bfd500c3b64a74380ef5cd0e62089f5",
+      "conclusion": "success",
+      "installExitCode": 0,
+      "firstTurnExitCode": 0,
+      "performancePassed": true,
+      "usedBuildKitPrebuild": true,
+      "buildKitFallback": false,
+      "maxSilenceSecs": 32,
+      "responseChars": 23,
+      "measurementsMs": {
+        "onboardRoot": 175231,
+        "rootStartToFirstTurnCompletion": 182517,
+        "rootEndToInstallCompletion": 131,
+        "firstTurnCommand": 7152,
+        "rootEndToFirstTurnCompletion": 7286,
+        "phases": {
+          "nemoclaw.onboard.phase.preflight": 1205,
+          "nemoclaw.onboard.phase.gateway": 62,
+          "nemoclaw.onboard.phase.provider_selection": 1377,
+          "nemoclaw.onboard.phase.inference": 805,
+          "nemoclaw.onboard.phase.sandbox": 130203
+        }
+      }
+    },
+    {
+      "runId": 29285391845,
+      "runUrl": "https://github.com/NVIDIA/NemoClaw/actions/runs/29285391845",
+      "headSha": "4544d07c8bfd500c3b64a74380ef5cd0e62089f5",
+      "conclusion": "success",
+      "installExitCode": 0,
+      "firstTurnExitCode": 0,
+      "performancePassed": true,
+      "usedBuildKitPrebuild": true,
+      "buildKitFallback": false,
+      "maxSilenceSecs": 30,
+      "responseChars": 23,
+      "measurementsMs": {
+        "onboardRoot": 155691,
+        "rootStartToFirstTurnCompletion": 160740,
+        "rootEndToInstallCompletion": 104,
+        "firstTurnCommand": 4942,
+        "rootEndToFirstTurnCompletion": 5049,
+        "phases": {
+          "nemoclaw.onboard.phase.preflight": 5101,
+          "nemoclaw.onboard.phase.gateway": 211,
+          "nemoclaw.onboard.phase.provider_selection": 1585,
+          "nemoclaw.onboard.phase.inference": 583,
+          "nemoclaw.onboard.phase.sandbox": 118007
+        }
+      }
+    }
+  ],
+  "derivedBudgetsMs": {
+    "rootStartToFirstTurnCompletionBudgetMs": 205000,
+    "rootEndToFirstTurnCompletionBudgetMs": 14000,
+    "phaseBudgetsMs": {
+      "nemoclaw.onboard.phase.preflight": 11000,
+      "nemoclaw.onboard.phase.gateway": 6000,
+      "nemoclaw.onboard.phase.provider_selection": 8000,
+      "nemoclaw.onboard.phase.inference": 6000,
+      "nemoclaw.onboard.phase.sandbox": 146000
+    }
+  }
+}

ci/onboard-performance-budget.json

@@ -1,5 +1,5 @@
 {
-  "$comment": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\nSPDX-License-Identifier: Apache-2.0\n\nInitial advisory budget for the cloud-onboard-e2e warm-system trace signal. Profiling traces from #3769 were not available in durable CI artifacts when #3776 was implemented. The latest three release tags only exposed one tag-matching trace artifact, from a failed v0.0.66 nightly run, so this cap is calibrated from the latest ten distinct successful main full-trace samples available on 2026-06-23. Those samples had total durations of 298250 ms, 296926 ms, 304190 ms, 294859 ms, 305013 ms, 316147 ms, 300843 ms, 292702 ms, 201332 ms, and 206250 ms; the cap uses p95 via linear interpolation (index 8.55 between samples 8 and 9) plus 25 percent, rounded up to the nearest 30 seconds.",
+  "$comment": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\nSPDX-License-Identifier: Apache-2.0\n\nInitial advisory budget for the cloud-onboard-e2e warm-system trace signal. Profiling traces from #3769 were not available in durable CI artifacts when #3776 was implemented. The latest three release tags only exposed one tag-matching trace artifact, from a failed v0.0.66 nightly run, so this cap is calibrated from the latest ten distinct successful main full-trace samples available on 2026-06-23. Those samples had total durations of 298250 ms, 296926 ms, 304190 ms, 294859 ms, 305013 ms, 316147 ms, 300843 ms, 292702 ms, 201332 ms, and 206250 ms; the cap uses p95 via linear interpolation (index 8.55 between samples 8 and 9) plus 25 percent, rounded up to the nearest 30 seconds.\n\nThe full-e2e cold-path budgets are derived from the five exact-head current-main samples recorded in ci/full-e2e-cold-path-calibration.json. For each interval and phase independently, the budget is nearest-rank p95 plus the larger of 5 seconds or 10 percent, rounded up to the nearest second. Independent phase caps are diagnostic regression gates, not additive portions of the root-start interval.",
   "schemaVersion": 1,
   "mode": "advisory",
   "scope": "cloud-onboard-e2e warm-system",
@@ -11,5 +11,16 @@
   "phaseRegressionWarning": {
     "minDeltaMs": 30000,
     "minPercent": 30
+  },
+  "fullE2eColdPath": {
+    "rootStartToFirstTurnCompletionBudgetMs": 205000,
+    "rootEndToFirstTurnCompletionBudgetMs": 14000,
+    "phaseBudgetsMs": {
+      "nemoclaw.onboard.phase.preflight": 11000,
+      "nemoclaw.onboard.phase.gateway": 6000,
+      "nemoclaw.onboard.phase.provider_selection": 8000,
+      "nemoclaw.onboard.phase.inference": 6000,
+      "nemoclaw.onboard.phase.sandbox": 146000
+    }
   }
 }

ci/source-shape-test-budget.json

@@ -266,6 +266,16 @@
       "test": "keeps the setup docs aligned with the stable default",
       "category": "compatibility"
     },
+    {
+      "file": "test/onboard-performance-config-schema.test.ts",
+      "test": "keeps configured budgets derived from the checked-in samples",
+      "category": "compatibility"
+    },
+    {
+      "file": "test/onboard-performance-config-schema.test.ts",
+      "test": "records five independent successful exact-head samples",
+      "category": "compatibility"
+    },
     {
       "file": "test/openclaw-dependency-review.test.ts",
       "test": "runs and gates the real patched-distribution harness only from trusted main code",

schemas/onboard-config.schema.json

@@ -2,7 +2,7 @@
   "$schema": "https://json-schema.org/draft/2020-12/schema",
   "$id": "https://github.com/NVIDIA/NemoClaw/schemas/onboard-config.schema.json",
   "title": "NemoClaw onboard performance budget config",
-  "description": "Advisory warm-system cloud onboard timing budget consumed by the nightly scorecard.",
+  "description": "Advisory warm-system cloud onboard timing budget and full-E2E cold-path phase budget.",
   "type": "object",
   "additionalProperties": false,
   "required": [
@@ -12,7 +12,8 @@
     "scope",
     "totalBudgetMs",
     "regressionWarning",
-    "phaseRegressionWarning"
+    "phaseRegressionWarning",
+    "fullE2eColdPath"
   ],
   "properties": {
     "$comment": {
@@ -37,6 +38,48 @@
     },
     "phaseRegressionWarning": {
       "$ref": "#/$defs/threshold"
+    },
+    "fullE2eColdPath": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "rootStartToFirstTurnCompletionBudgetMs",
+        "rootEndToFirstTurnCompletionBudgetMs",
+        "phaseBudgetsMs"
+      ],
+      "properties": {
+        "rootStartToFirstTurnCompletionBudgetMs": {
+          "type": "number",
+          "minimum": 0,
+          "description": "Maximum time from the nemoclaw.onboard root start until the first agent-turn command completes."
+        },
+        "rootEndToFirstTurnCompletionBudgetMs": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": {
+            "$data": "1/rootStartToFirstTurnCompletionBudgetMs"
+          },
+          "description": "Maximum time from the nemoclaw.onboard root end until the first agent-turn command completes."
+        },
+        "phaseBudgetsMs": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "nemoclaw.onboard.phase.preflight",
+            "nemoclaw.onboard.phase.gateway",
+            "nemoclaw.onboard.phase.provider_selection",
+            "nemoclaw.onboard.phase.inference",
+            "nemoclaw.onboard.phase.sandbox"
+          ],
+          "properties": {
+            "nemoclaw.onboard.phase.preflight": { "$ref": "#/$defs/milliseconds" },
+            "nemoclaw.onboard.phase.gateway": { "$ref": "#/$defs/milliseconds" },
+            "nemoclaw.onboard.phase.provider_selection": { "$ref": "#/$defs/milliseconds" },
+            "nemoclaw.onboard.phase.inference": { "$ref": "#/$defs/milliseconds" },
+            "nemoclaw.onboard.phase.sandbox": { "$ref": "#/$defs/milliseconds" }
+          }
+        }
+      }
     }
   },
   "$defs": {
@@ -54,6 +97,10 @@
           "minimum": 0
         }
       }
+    },
+    "milliseconds": {
+      "type": "number",
+      "minimum": 0
     }
   }
 }

scripts/validate-configs.ts

@@ -190,6 +190,13 @@ function loadSchema(repoRelative: string): object {
   return schema;
 }
 
+function compileConfigSchema(
+  repoRelative: string,
+  ajv = new Ajv({ allErrors: true, strict: false, $data: true }),
+) {
+  return ajv.compile(loadSchema(repoRelative));
+}
+
 type ValidationParams = { additionalProperty?: string; unevaluatedProperty?: string };
 
 /**
@@ -300,7 +307,7 @@ function main(): void {
     targets = discoverTargets();
   }
 
-  const ajv = new Ajv({ allErrors: true, strict: false });
+  const ajv = new Ajv({ allErrors: true, strict: false, $data: true });
   let totalErrors = 0;
   let totalFiles = 0;
 
@@ -309,8 +316,7 @@ function main(): void {
   for (const target of targets) {
     let validate;
     try {
-      const schema = loadSchema(target.schema);
-      validate = ajv.compile(schema);
+      validate = compileConfigSchema(target.schema, ajv);
     } catch (err) {
       console.error(`FAIL: ${target.schema}`);
       console.error(`  Could not compile schema: ${err}`);
@@ -367,6 +373,7 @@ function main(): void {
 // Export for unit tests without re-running main().
 export {
   DANGEROUS_HOSTS,
+  compileConfigSchema,
   discoverTargets,
   findDangerousHosts,
   findDangerousRouterApiBases,

src/lib/onboard/machine/handlers/gateway.test.ts

@@ -1,12 +1,16 @@
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
 import { describe, expect, it, vi } from "vitest";
-
+import type { GatewayReuseState } from "../../../state/gateway";
 import { createSession, type Session } from "../../../state/onboard-session";
+import { flushTrace, resetTraceForTests, TRACE_FILE_ENV, type TraceArtifact } from "../../../trace";
 import type { GatewayContainerState } from "../../gateway-container-running";
-import type { GatewayReuseState } from "../../../state/gateway";
-import { handleGatewayState, type GatewayStateOptions } from "./gateway";
+import { ONBOARD_TRACE_PHASE_NAMES } from "../../tracing";
+import { type GatewayStateOptions, handleGatewayState } from "./gateway";
 
 type Gpu = { type: string } | null;
 
@@ -119,6 +123,34 @@ describe("handleGatewayState", () => {
     expect(calls.complete).toHaveBeenCalledWith("gateway");
   });
 
+  it("emits one successful gateway phase when reusing a healthy gateway", async () => {
+    const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-gateway-trace-"));
+    const traceFile = path.join(directory, "trace.json");
+    process.env[TRACE_FILE_ENV] = traceFile;
+    resetTraceForTests();
+
+    try {
+      const { deps } = createDeps();
+
+      await handleGatewayState(baseOptions(deps, "healthy"));
+      flushTrace();
+
+      const artifact = JSON.parse(fs.readFileSync(traceFile, "utf8")) as TraceArtifact;
+      const gatewaySpans = artifact.resource_spans[0].scope_spans[0].spans.filter(
+        (span) => span.name === ONBOARD_TRACE_PHASE_NAMES.gateway,
+      );
+      expect(gatewaySpans).toHaveLength(1);
+      expect(gatewaySpans[0]).toMatchObject({
+        status: { code: "OK" },
+        attributes: { reuse_state: "healthy", gpu_passthrough: true },
+      });
+    } finally {
+      delete process.env[TRACE_FILE_ENV];
+      resetTraceForTests();
+      fs.rmSync(directory, { recursive: true, force: true });
+    }
+  });
+
   it("reuses healthy gateways on resume only when the gateway step was complete", async () => {
     const session = createSession();
     session.steps.gateway.status = "complete";