diff --git a/src/lib/onboard/machine/core-flow-phases.test.ts b/src/lib/onboard/machine/core-flow-phases.test.ts index 8ccc72a77cf..7048349d83f 100644 --- a/src/lib/onboard/machine/core-flow-phases.test.ts +++ b/src/lib/onboard/machine/core-flow-phases.test.ts @@ -313,6 +313,79 @@ describe("core onboard flow phases", () => { expect(recorded).toEqual(["sandbox", "openclaw"]); }); + it.each([ + "policies", + "finalizing", + "post_verify", + ] as const)("lets resume sessions at %s pass through core compatibility", async (state) => { + const recorded: string[] = []; + const phases: readonly OnboardSequencePhase[] = [ + { + state: "provider_selection", + run: (ctx) => ({ context: ctx, result: advanceTo("sandbox") }), + }, + { + state: "sandbox", + run: (ctx) => ({ context: ctx, result: advanceTo("openclaw") }), + }, + ]; + + await runCoreOnboardFlowSlice({ + context: context({ resume: true }), + runtime: { + session: async () => + createSession({ + machine: { + version: 1, + state, + stateEnteredAt: "2026-06-09T00:00:00.000Z", + revision: 7, + }, + }), + applyResult: async () => createSession(), + }, + phases, + resume: true, + recordStateResult: async (result) => { + recorded.push((result as ReturnType).next); + }, + }); + + expect(recorded).toEqual(["sandbox", "openclaw"]); + }); + + it.each([ + "complete", + "failed", + ] as const)("rejects terminal %s sessions before core compatibility side effects", async (state) => { + const phase: OnboardSequencePhase = { + state: "provider_selection", + run: vi.fn((ctx) => ({ context: ctx, result: advanceTo("sandbox") })), + }; + + await expect( + runCoreOnboardFlowSlice({ + context: context({ resume: true }), + runtime: { + session: async () => + createSession({ + machine: { + version: 1, + state, + stateEnteredAt: "2026-06-09T00:00:00.000Z", + revision: 7, + }, + }), + applyResult: async () => createSession(), + }, + phases: [phase], + resume: true, + recordStateResult: async () => undefined, + }), + ).rejects.toThrow("Unexpected onboarding live flow state before slice entry"); + expect(phase.run).not.toHaveBeenCalled(); + }); + it("keeps non-resume ahead-state sessions on the compatibility path", async () => { const calls: string[] = []; const skipped: string[] = []; diff --git a/src/lib/onboard/machine/core-flow-phases.ts b/src/lib/onboard/machine/core-flow-phases.ts index 10fcacbfdc1..5f5e0abe85d 100644 --- a/src/lib/onboard/machine/core-flow-phases.ts +++ b/src/lib/onboard/machine/core-flow-phases.ts @@ -147,13 +147,16 @@ export async function runCoreOnboardFlowSlice; }): Promise> { - // Compatibility bridge for live host glue while legacy step helpers remain a - // second machine snapshot writer. OnboardRuntimeBoundary records skipped - // stale/already-reached transition results from handlers whose source state - // was advanced by markStepStarted()/markStepComplete() in the durable session. - // Keep resume and ahead-state sessions here so provider/sandbox repair checks - // still run. Remove this path once legacy step helpers no longer advance - // session.machine and handler FSM results are the only transition source. + // Compatibility bridge for live resume repair while legacy step helpers and + // OnboardRuntimeBoundary compatibility replay can leave the durable machine + // snapshot already downstream of this slice. The tolerated downstream family + // includes sandbox branch states and the final slice handoff states: openclaw, + // agent_setup, policies, finalizing, and post_verify. Resume still needs to + // re-run provider and sandbox repair/backstop checks before policy or final + // verification handling observes the session. This PR does not fix the + // broader persistence contract because those repairs are not strict FSM states + // yet. Remove this fallback once resume repairs are strict FSM states, or once + // direct legacy step helpers no longer write session.machine. return runLiveOnboardFlowSlice({ context: options.context, runtime: options.runtime, @@ -166,6 +169,9 @@ export async function runCoreOnboardFlowSlice { expect(recorded).toEqual(["gateway", "provider_selection"]); }); + it.each([ + "inference", + "sandbox", + "openclaw", + "agent_setup", + "policies", + "finalizing", + "post_verify", + ] as const)("lets resume sessions at %s pass through initial compatibility", async (state) => { + const recorded: string[] = []; + const phases: readonly OnboardSequencePhase[] = [ + { + state: "preflight", + run: (ctx) => ({ context: ctx, result: advanceTo("gateway") }), + }, + { + state: "gateway", + run: (ctx) => ({ context: ctx, result: advanceTo("provider_selection") }), + }, + ]; + + await runInitialOnboardFlowSlice({ + context: context({ resume: true }), + runtime: runtime( + createSession({ + machine: { + version: 1, + state, + stateEnteredAt: "2026-06-09T00:00:00.000Z", + revision: 7, + }, + }), + ), + phases, + resume: true, + recordStateResult: async (stateResult) => { + recorded.push((stateResult as ReturnType).next); + }, + }); + + expect(recorded).toEqual(["gateway", "provider_selection"]); + }); + + it.each([ + "complete", + "failed", + ] as const)("rejects terminal %s sessions before initial compatibility side effects", async (state) => { + const phase: OnboardSequencePhase = { + state: "preflight", + run: vi.fn((ctx) => ({ context: ctx, result: advanceTo("gateway") })), + }; + + await expect( + runInitialOnboardFlowSlice({ + context: context({ resume: true }), + runtime: runtime( + createSession({ + machine: { + version: 1, + state, + stateEnteredAt: "2026-06-09T00:00:00.000Z", + revision: 7, + }, + }), + ), + phases: [phase], + resume: true, + recordStateResult: async () => undefined, + }), + ).rejects.toThrow("Unexpected onboarding live flow state before slice entry"); + expect(phase.run).not.toHaveBeenCalled(); + }); + it("uses the strict runner for fresh init sessions", async () => { const order: string[] = []; const session = createSession(); diff --git a/src/lib/onboard/machine/initial-flow-phases.ts b/src/lib/onboard/machine/initial-flow-phases.ts index b9dd5ddf3c3..08d0d3abe9f 100644 --- a/src/lib/onboard/machine/initial-flow-phases.ts +++ b/src/lib/onboard/machine/initial-flow-phases.ts @@ -190,17 +190,36 @@ export async function runInitialOnboardFlowSlice; }): Promise> { - // Keep resume on the compatibility path for now: resume intentionally re-runs - // preflight/gateway backstops even when the saved machine is already ahead. - // Remove this fallback only after resume repairs are modeled as strict FSM - // transitions that preserve these safety checks before later phases run. + // Compatibility bridge for live resume repair while legacy step helpers and + // OnboardRuntimeBoundary compatibility replay can leave the durable machine + // snapshot already downstream of this slice. The tolerated downstream family + // is every nonterminal state after the initial slice: inference, sandbox, + // openclaw/agent_setup, policies, finalizing, and post_verify. Resume still + // needs to re-run preflight/gateway host backstops before later provider, + // sandbox, policy, or verification handling observes the session. This PR + // does not fix the broader persistence contract because strict FSM repair + // states must preserve those safety checks first. Remove this fallback once + // resume repairs are strict FSM states, or once direct legacy step helpers no + // longer write session.machine. return runLiveOnboardFlowSlice({ context: options.context, runtime: options.runtime, phases: options.phases, resume: options.resume, runWhenState: ["init", "preflight"], - compatibilityWhenState: ["init", "preflight", "gateway", "provider_selection"], + compatibilityWhenState: [ + "init", + "preflight", + "gateway", + "provider_selection", + "inference", + "sandbox", + "openclaw", + "agent_setup", + "policies", + "finalizing", + "post_verify", + ], runSlice: runInitialOnboardFlowSequence, applyCompatibleResult: options.recordStateResult, }); diff --git a/test/e2e-advisor.test.ts b/test/e2e-advisor.test.ts new file mode 100644 index 00000000000..00bf58ba9ec --- /dev/null +++ b/test/e2e-advisor.test.ts @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { buildSystemPrompt } from "../tools/e2e-advisor/analyze.mts"; + +describe("E2E recommendation advisor prompt", () => { + it("requires resume and repair E2E for onboarding machine compatibility changes", () => { + const prompt = buildSystemPrompt(); + + expect(prompt).toContain("Onboarding resume compatibility rule"); + expect(prompt).toContain("onboard-resume-e2e"); + expect(prompt).toContain("onboard-repair-e2e"); + expect(prompt).toContain("src/lib/onboard/machine"); + }); +}); diff --git a/test/e2e-scenario-advisor.test.ts b/test/e2e-scenario-advisor.test.ts index 57e97c0ae5c..28a33fa3aa2 100644 --- a/test/e2e-scenario-advisor.test.ts +++ b/test/e2e-scenario-advisor.test.ts @@ -83,6 +83,8 @@ describe("Vitest E2E scenario advisor — prompt construction", () => { expect(systemPrompt).toContain("trusted advisor checkout"); expect(systemPrompt).toContain("recommend the `e2e-scenarios-all` fan-out"); expect(systemPrompt).toContain("single NemoClaw E2E system"); + expect(systemPrompt).toContain("onboard-resume-vitest"); + expect(systemPrompt).toContain("onboard-repair-vitest"); expect(systemPrompt).not.toContain("non-scenario E2E"); expect(systemPrompt).not.toContain("e2e-scenarios-all.yaml"); expect(systemPrompt).not.toContain("e2e-scenarios.yaml"); diff --git a/tools/e2e-advisor/analyze.mts b/tools/e2e-advisor/analyze.mts index a8d4917bbd7..8824e56cb02 100755 --- a/tools/e2e-advisor/analyze.mts +++ b/tools/e2e-advisor/analyze.mts @@ -222,7 +222,7 @@ function logProgress(message: string): void { console.log(`[e2e-advisor] ${new Date().toISOString()} ${message}`); } -function buildSystemPrompt(): string { +export function buildSystemPrompt(): string { return [ "You are the NemoClaw E2E recommendation advisor for CI.", "", @@ -236,6 +236,7 @@ function buildSystemPrompt(): string { "", "Decision policy:", "- Required E2E: changes that can affect installer/onboarding, sandbox lifecycle, credentials, security boundaries, network policy, inference routing, deployment, or real assistant user flows.", + "- Onboarding resume compatibility rule: changes to src/lib/onboard/machine live slice orchestration, resume compatibility states, resume repair policy, session bootstrap, or onboarding state transitions MUST require both `onboard-resume-e2e` and `onboard-repair-e2e` unless the PR is tests-only. If the change can also affect full hosted onboarding, require `cloud-onboard-e2e`. Do not rely only on unit/runtime-boundary tests for these state-machine resume paths.", "- Optional E2E: useful confidence checks for adjacent behavior, but not merge-blocking.", "- No E2E: safe docs, tests-only, comments, refactors, or tooling changes that cannot affect runtime/user flows; explain in noE2eReason.", "- Missing coverage: use newE2eRecommendations. Do not invent existing test names.", diff --git a/tools/e2e-advisor/scenarios.mts b/tools/e2e-advisor/scenarios.mts index a05f6f4b648..a5cd4f68099 100755 --- a/tools/e2e-advisor/scenarios.mts +++ b/tools/e2e-advisor/scenarios.mts @@ -280,6 +280,7 @@ export function buildSystemPrompt(_schema?: AdvisorSchema): string { "Decision policy:", "- Required (all scenarios): changes to scenario registry, matrix emission, expected-state metadata, live support classification, shared fixtures, or the shared Vitest scenario workflow machinery. Recommend the `e2e-scenarios-all` fan-out through `e2e-vitest-scenarios.yaml`.", "- Required (targeted): fixture, live test, manifest, runtime-support, or scenario changes that affect a specific subset. Recommend the smallest set of live-supported typed scenario IDs that exercises the changed surface.", + "- Onboarding resume compatibility rule: changes to src/lib/onboard/machine live slice orchestration, resume compatibility states, resume repair policy, session bootstrap, or onboarding state transitions MUST require `onboard-resume-vitest`. Also require `onboard-repair-vitest` when the change can affect repair/backstop execution from persisted sessions. Do not make repair optional for these state-machine resume paths.", "- Required (free-standing job): if a PR wires or changes a discrete live Vitest job in `.github/workflows/e2e-vitest-scenarios.yaml` for a specific `test/e2e-scenario/live/*.test.ts`, prefer that job over `e2e-scenarios-all`. Use selectorType=`job`, id=``, workflow=`e2e-vitest-scenarios.yaml`, and dispatchCommand exactly `gh workflow run e2e-vitest-scenarios.yaml --ref --field jobs=`.", "- Missing wiring: if a PR adds or changes a free-standing live Vitest file under `test/e2e-scenario/live/*.test.ts` but that file is not referenced by `.github/workflows/e2e-vitest-scenarios.yaml` and is not `registry-scenarios.test.ts`, do not recommend the fan-out as proof. Return no required/optional recommendations and set `noScenarioE2eReason` to say the test must be wired into `e2e-vitest-scenarios.yaml` before it can be dispatched.", "- Optional: adjacent scenarios that exercise the same suite on a different platform/onboarding (e.g. macOS, WSL, GPU) but are not the primary target. Special-runner scenarios (`gpu-`, `macos-`, `wsl-`, `brev-`) should usually be optional unless they are the only path that exercises the change.",