Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 73 additions & 0 deletions src/lib/onboard/machine/core-flow-phases.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -313,6 +313,79 @@ describe("core onboard flow phases", () => {
expect(recorded).toEqual(["sandbox", "openclaw"]);
});

it.each([
"policies",
"finalizing",
"post_verify",
] as const)("lets resume sessions at %s pass through core compatibility", async (state) => {
const recorded: string[] = [];
const phases: readonly OnboardSequencePhase<CoreContext>[] = [
{
state: "provider_selection",
run: (ctx) => ({ context: ctx, result: advanceTo("sandbox") }),
},
{
state: "sandbox",
run: (ctx) => ({ context: ctx, result: advanceTo("openclaw") }),
},
];

await runCoreOnboardFlowSlice({
context: context({ resume: true }),
runtime: {
session: async () =>
createSession({
machine: {
version: 1,
state,
stateEnteredAt: "2026-06-09T00:00:00.000Z",
revision: 7,
},
}),
applyResult: async () => createSession(),
},
phases,
resume: true,
recordStateResult: async (result) => {
recorded.push((result as ReturnType<typeof advanceTo>).next);
},
});

expect(recorded).toEqual(["sandbox", "openclaw"]);
});

it.each([
"complete",
"failed",
] as const)("rejects terminal %s sessions before core compatibility side effects", async (state) => {
const phase: OnboardSequencePhase<CoreContext> = {
state: "provider_selection",
run: vi.fn((ctx) => ({ context: ctx, result: advanceTo("sandbox") })),
};

await expect(
runCoreOnboardFlowSlice({
context: context({ resume: true }),
runtime: {
session: async () =>
createSession({
machine: {
version: 1,
state,
stateEnteredAt: "2026-06-09T00:00:00.000Z",
revision: 7,
},
}),
applyResult: async () => createSession(),
},
phases: [phase],
resume: true,
recordStateResult: async () => undefined,
}),
).rejects.toThrow("Unexpected onboarding live flow state before slice entry");
expect(phase.run).not.toHaveBeenCalled();
});

it("keeps non-resume ahead-state sessions on the compatibility path", async () => {
const calls: string[] = [];
const skipped: string[] = [];
Expand Down
20 changes: 13 additions & 7 deletions src/lib/onboard/machine/core-flow-phases.ts
Original file line number Diff line number Diff line change
Expand Up @@ -147,13 +147,16 @@ export async function runCoreOnboardFlowSlice<Context extends OnboardFlowContext
resume: boolean;
recordStateResult(result: OnboardStateResult): Promise<unknown>;
}): Promise<OnboardMachineRunnerResult<Context>> {
// Compatibility bridge for live host glue while legacy step helpers remain a
// second machine snapshot writer. OnboardRuntimeBoundary records skipped
// stale/already-reached transition results from handlers whose source state
// was advanced by markStepStarted()/markStepComplete() in the durable session.
// Keep resume and ahead-state sessions here so provider/sandbox repair checks
// still run. Remove this path once legacy step helpers no longer advance
// session.machine and handler FSM results are the only transition source.
// Compatibility bridge for live resume repair while legacy step helpers and
// OnboardRuntimeBoundary compatibility replay can leave the durable machine
// snapshot already downstream of this slice. The tolerated downstream family
// includes sandbox branch states and the final slice handoff states: openclaw,
// agent_setup, policies, finalizing, and post_verify. Resume still needs to
// re-run provider and sandbox repair/backstop checks before policy or final
// verification handling observes the session. This PR does not fix the
// broader persistence contract because those repairs are not strict FSM states
// yet. Remove this fallback once resume repairs are strict FSM states, or once
// direct legacy step helpers no longer write session.machine.
return runLiveOnboardFlowSlice({
context: options.context,
runtime: options.runtime,
Expand All @@ -166,6 +169,9 @@ export async function runCoreOnboardFlowSlice<Context extends OnboardFlowContext
"sandbox",
"openclaw",
"agent_setup",
"policies",
"finalizing",
"post_verify",
],
runSlice: runCoreOnboardFlowSequence,
applyCompatibleResult: options.recordStateResult,
Expand Down
73 changes: 73 additions & 0 deletions src/lib/onboard/machine/initial-flow-phases.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -420,6 +420,79 @@ describe("initial onboard flow phases", () => {
expect(recorded).toEqual(["gateway", "provider_selection"]);
});

it.each([
"inference",
"sandbox",
"openclaw",
"agent_setup",
"policies",
"finalizing",
"post_verify",
] as const)("lets resume sessions at %s pass through initial compatibility", async (state) => {
const recorded: string[] = [];
const phases: readonly OnboardSequencePhase<Context>[] = [
{
state: "preflight",
run: (ctx) => ({ context: ctx, result: advanceTo("gateway") }),
},
{
state: "gateway",
run: (ctx) => ({ context: ctx, result: advanceTo("provider_selection") }),
},
];

await runInitialOnboardFlowSlice({
context: context({ resume: true }),
runtime: runtime(
createSession({
machine: {
version: 1,
state,
stateEnteredAt: "2026-06-09T00:00:00.000Z",
revision: 7,
},
}),
),
phases,
resume: true,
recordStateResult: async (stateResult) => {
recorded.push((stateResult as ReturnType<typeof advanceTo>).next);
},
});

expect(recorded).toEqual(["gateway", "provider_selection"]);
});

it.each([
"complete",
"failed",
] as const)("rejects terminal %s sessions before initial compatibility side effects", async (state) => {
const phase: OnboardSequencePhase<Context> = {
state: "preflight",
run: vi.fn((ctx) => ({ context: ctx, result: advanceTo("gateway") })),
};

await expect(
runInitialOnboardFlowSlice({
context: context({ resume: true }),
runtime: runtime(
createSession({
machine: {
version: 1,
state,
stateEnteredAt: "2026-06-09T00:00:00.000Z",
revision: 7,
},
}),
),
phases: [phase],
resume: true,
recordStateResult: async () => undefined,
}),
).rejects.toThrow("Unexpected onboarding live flow state before slice entry");
expect(phase.run).not.toHaveBeenCalled();
});

it("uses the strict runner for fresh init sessions", async () => {
const order: string[] = [];
const session = createSession();
Expand Down
29 changes: 24 additions & 5 deletions src/lib/onboard/machine/initial-flow-phases.ts
Original file line number Diff line number Diff line change
Expand Up @@ -190,17 +190,36 @@ export async function runInitialOnboardFlowSlice<Context extends OnboardFlowCont
resume: boolean;
recordStateResult(result: OnboardStateResult): Promise<unknown>;
}): Promise<OnboardMachineRunnerResult<Context>> {
// Keep resume on the compatibility path for now: resume intentionally re-runs
// preflight/gateway backstops even when the saved machine is already ahead.
// Remove this fallback only after resume repairs are modeled as strict FSM
// transitions that preserve these safety checks before later phases run.
// Compatibility bridge for live resume repair while legacy step helpers and
// OnboardRuntimeBoundary compatibility replay can leave the durable machine
// snapshot already downstream of this slice. The tolerated downstream family
// is every nonterminal state after the initial slice: inference, sandbox,
// openclaw/agent_setup, policies, finalizing, and post_verify. Resume still
// needs to re-run preflight/gateway host backstops before later provider,
// sandbox, policy, or verification handling observes the session. This PR
// does not fix the broader persistence contract because strict FSM repair
// states must preserve those safety checks first. Remove this fallback once
// resume repairs are strict FSM states, or once direct legacy step helpers no
// longer write session.machine.
return runLiveOnboardFlowSlice({
context: options.context,
runtime: options.runtime,
phases: options.phases,
resume: options.resume,
runWhenState: ["init", "preflight"],
compatibilityWhenState: ["init", "preflight", "gateway", "provider_selection"],
compatibilityWhenState: [
"init",
"preflight",
"gateway",
"provider_selection",
"inference",
"sandbox",
"openclaw",
"agent_setup",
"policies",
"finalizing",
"post_verify",
],
runSlice: runInitialOnboardFlowSequence,
applyCompatibleResult: options.recordStateResult,
});
Expand Down
17 changes: 17 additions & 0 deletions test/e2e-advisor.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { describe, expect, it } from "vitest";

import { buildSystemPrompt } from "../tools/e2e-advisor/analyze.mts";

describe("E2E recommendation advisor prompt", () => {
it("requires resume and repair E2E for onboarding machine compatibility changes", () => {
const prompt = buildSystemPrompt();

expect(prompt).toContain("Onboarding resume compatibility rule");
expect(prompt).toContain("onboard-resume-e2e");
expect(prompt).toContain("onboard-repair-e2e");
expect(prompt).toContain("src/lib/onboard/machine");
});
});
2 changes: 2 additions & 0 deletions test/e2e-scenario-advisor.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,8 @@ describe("Vitest E2E scenario advisor — prompt construction", () => {
expect(systemPrompt).toContain("trusted advisor checkout");
expect(systemPrompt).toContain("recommend the `e2e-scenarios-all` fan-out");
expect(systemPrompt).toContain("single NemoClaw E2E system");
expect(systemPrompt).toContain("onboard-resume-vitest");
expect(systemPrompt).toContain("onboard-repair-vitest");
expect(systemPrompt).not.toContain("non-scenario E2E");
expect(systemPrompt).not.toContain("e2e-scenarios-all.yaml");
expect(systemPrompt).not.toContain("e2e-scenarios.yaml");
Expand Down
3 changes: 2 additions & 1 deletion tools/e2e-advisor/analyze.mts
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,7 @@ function logProgress(message: string): void {
console.log(`[e2e-advisor] ${new Date().toISOString()} ${message}`);
}

function buildSystemPrompt(): string {
export function buildSystemPrompt(): string {
return [
"You are the NemoClaw E2E recommendation advisor for CI.",
"",
Expand All @@ -236,6 +236,7 @@ function buildSystemPrompt(): string {
"",
"Decision policy:",
"- Required E2E: changes that can affect installer/onboarding, sandbox lifecycle, credentials, security boundaries, network policy, inference routing, deployment, or real assistant user flows.",
"- Onboarding resume compatibility rule: changes to src/lib/onboard/machine live slice orchestration, resume compatibility states, resume repair policy, session bootstrap, or onboarding state transitions MUST require both `onboard-resume-e2e` and `onboard-repair-e2e` unless the PR is tests-only. If the change can also affect full hosted onboarding, require `cloud-onboard-e2e`. Do not rely only on unit/runtime-boundary tests for these state-machine resume paths.",
"- Optional E2E: useful confidence checks for adjacent behavior, but not merge-blocking.",
"- No E2E: safe docs, tests-only, comments, refactors, or tooling changes that cannot affect runtime/user flows; explain in noE2eReason.",
"- Missing coverage: use newE2eRecommendations. Do not invent existing test names.",
Expand Down
1 change: 1 addition & 0 deletions tools/e2e-advisor/scenarios.mts
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,7 @@ export function buildSystemPrompt(_schema?: AdvisorSchema): string {
"Decision policy:",
"- Required (all scenarios): changes to scenario registry, matrix emission, expected-state metadata, live support classification, shared fixtures, or the shared Vitest scenario workflow machinery. Recommend the `e2e-scenarios-all` fan-out through `e2e-vitest-scenarios.yaml`.",
"- Required (targeted): fixture, live test, manifest, runtime-support, or scenario changes that affect a specific subset. Recommend the smallest set of live-supported typed scenario IDs that exercises the changed surface.",
"- Onboarding resume compatibility rule: changes to src/lib/onboard/machine live slice orchestration, resume compatibility states, resume repair policy, session bootstrap, or onboarding state transitions MUST require `onboard-resume-vitest`. Also require `onboard-repair-vitest` when the change can affect repair/backstop execution from persisted sessions. Do not make repair optional for these state-machine resume paths.",
"- Required (free-standing job): if a PR wires or changes a discrete live Vitest job in `.github/workflows/e2e-vitest-scenarios.yaml` for a specific `test/e2e-scenario/live/*.test.ts`, prefer that job over `e2e-scenarios-all`. Use selectorType=`job`, id=`<job-id>`, workflow=`e2e-vitest-scenarios.yaml`, and dispatchCommand exactly `gh workflow run e2e-vitest-scenarios.yaml --ref <pr-head-ref> --field jobs=<job-id>`.",
"- Missing wiring: if a PR adds or changes a free-standing live Vitest file under `test/e2e-scenario/live/*.test.ts` but that file is not referenced by `.github/workflows/e2e-vitest-scenarios.yaml` and is not `registry-scenarios.test.ts`, do not recommend the fan-out as proof. Return no required/optional recommendations and set `noScenarioE2eReason` to say the test must be wired into `e2e-vitest-scenarios.yaml` before it can be dispatched.",
"- Optional: adjacent scenarios that exercise the same suite on a different platform/onboarding (e.g. macOS, WSL, GPU) but are not the primary target. Special-runner scenarios (`gpu-`, `macos-`, `wsl-`, `brev-`) should usually be optional unless they are the only path that exercises the change.",
Expand Down