Skip to content
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
49c18e4
test(e2e): add interactive onboard policy-preset step-ordering test (…
wakqasahmed Aug 8, 2026
6b2fe72
fix(e2e): use the shared docker fixture instead of an inline Docker c…
wakqasahmed Aug 8, 2026
9ce0eda
fix(e2e): redact the generated API key from onboard test assertion me…
wakqasahmed Aug 8, 2026
a79fdb7
Merge branch 'main' into test/onboard-policy-preset-sequencing-6042
cv Aug 8, 2026
4b0ea3d
Merge branch 'main' into test/onboard-policy-preset-sequencing-6042
cv Aug 9, 2026
eb5cfb4
fix(e2e): address maintainer review on the onboard PTY regression
wakqasahmed Aug 9, 2026
abe9231
merge(main): refresh PR #8618
apurvvkumaria Aug 11, 2026
b237902
merge(main): refresh PR #8618
apurvvkumaria Aug 11, 2026
6aa218c
merge(main): refresh PR #8618
apurvvkumaria Aug 11, 2026
38b5d26
test(e2e): complete onboard PTY review coverage
apurvvkumaria Aug 11, 2026
006518e
merge: refresh onboard policy sequencing base
apurvvkumaria Aug 11, 2026
7868381
test(e2e): terminate the PTY child session
prekshivyas Aug 11, 2026
b19cbeb
merge(main): refresh PR #8618
apurvvkumaria Aug 11, 2026
a27049e
Merge branch 'main' into test/onboard-policy-preset-sequencing-6042
cv Aug 11, 2026
ffaecff
Merge branch 'main' into test/onboard-policy-preset-sequencing-6042
cv Aug 11, 2026
20df452
Merge branch 'main' into test/onboard-policy-preset-sequencing-6042
cv Aug 12, 2026
747579b
merge(e2e): update #8618 with main
apurvvkumaria Aug 13, 2026
df8299c
Merge branch 'main' into test/onboard-policy-preset-sequencing-6042
cv Aug 13, 2026
b8cface
merge(main): refresh PR #8618
cv Aug 13, 2026
c4b5e05
ci(e2e): incorporate current main checks
cv Aug 13, 2026
9172a61
ci(e2e): incorporate current main checks
cv Aug 13, 2026
9bf0d0b
merge(e2e): retain formatted current-main checks
cv Aug 13, 2026
3895ca5
merge(main): resolve PR #8618 conflict
cv Aug 13, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
168 changes: 168 additions & 0 deletions test/e2e/live/onboard-interactive-pty.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import type { ChildProcessProgress } from "../fixtures/observed-child-process.ts";
import { spawnObservedChild } from "../fixtures/observed-child-process.ts";

// Drives an interactive CLI through a real PTY, the same technique
// `test/helpers/installer-express-prompt-pty-harness.ts` uses for the
// installer's express prompt. The real onboard wizard behaves differently
// under a piped, non-TTY stdin than under a real terminal (raw-mode
// keypress selectors, `isTTY`-gated prompts), so a faithful regression test
// for interactive-only behavior must drive a real PTY rather than pipe
// stdin.
//
// Rules fire independently and out of order: a rule whose trigger never
// appears (for example, a first-run license notice already accepted on a
// prior run) must not block a later rule from firing when its own trigger
// appears.
//
// The child process itself is launched through the shared
// `spawnObservedChild` boundary (the suite's single audited asynchronous
// child-process call) so it still tracks a content-free progress activity
// and canonical lifecycle checkpoints; this module attaches its own
// listeners on top to capture output and match rule triggers.

export interface InteractiveCommandRule {
readonly trigger: string;
readonly response: string;
}

export interface InteractiveCommandResult {
readonly exitCode: number;
readonly output: string;
readonly firedTriggers: readonly string[];
readonly timedOut: boolean;
}

export interface DriveInteractiveCommandOptions {
readonly activityLabel: string;
readonly cmd: readonly [string, ...string[]];
readonly cwd?: string;
readonly env: NodeJS.ProcessEnv;
readonly progress: ChildProcessProgress;
readonly rules: readonly InteractiveCommandRule[];
readonly timeoutMs: number;
}

// Runs inside a Python child so it can fork a real pseudo-terminal;
// Node has no built-in PTY primitive and this repo does not depend on
// node-pty. The child's own deadline is generous; the Node-side timer
// below is the enforced hard bound and SIGKILLs the whole process tree.
const PTY_DRIVER_SCRIPT = `
import json, os, pty, select, signal, sys, time

payload = json.loads(sys.argv[1])
cmd = payload["cmd"]
rules = payload["rules"]
timeout_s = payload["timeoutSeconds"]

pid, fd = pty.fork()
if pid == 0:
os.execvp(cmd[0], cmd)

output = bytearray()
os.set_blocking(fd, False)
deadline = time.monotonic() + timeout_s
fired = [False] * len(rules)
exit_code = None
while time.monotonic() < deadline:
ready, _, _ = select.select([fd], [], [], 0.2)
if ready:
try:
chunk = os.read(fd, 65536)
except OSError:
break
if not chunk:
break
output.extend(chunk)
sys.stdout.buffer.write(chunk)
sys.stdout.flush()
text = output.decode("utf-8", errors="ignore")
for i, rule in enumerate(rules):
if fired[i]:
continue
if rule["trigger"] in text:
os.write(fd, rule["response"].encode())
sys.stderr.write("FIRED\\t" + rule["trigger"] + "\\n")
fired[i] = True
waited = os.waitpid(pid, os.WNOHANG)
if waited[0] == pid:
exit_code = os.waitstatus_to_exitcode(waited[1])
break
if exit_code is None:
try:
os.kill(pid, signal.SIGKILL)
except ProcessLookupError:
pass
os.waitpid(pid, 0)
sys.stderr.write("DRIVER_TIMEOUT\\n")
sys.exit(124)
sys.exit(exit_code)
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
`;

function resolvePython(): string {
return process.env.NEMOCLAW_E2E_PYTHON3_BIN || "python3";
}

export function driveInteractiveCommand(
options: DriveInteractiveCommandOptions,
): Promise<InteractiveCommandResult> {
const payload = JSON.stringify({
cmd: options.cmd,
rules: options.rules.map((rule) => ({ trigger: rule.trigger, response: rule.response })),
// Comfortably longer than the Node-side hard timeout below so the
// driver's own bookkeeping never races the enforced bound.
timeoutSeconds: Math.ceil(options.timeoutMs / 1000) + 30,
});
// Kept directly in this function's body, not inside the Promise executor
// below, so the sole audited async child-process boundary stays attached
// to a named, reviewed callsite.
const child = spawnObservedChild(resolvePython(), ["-c", PTY_DRIVER_SCRIPT, payload], {
activityLabel: options.activityLabel,
progress: options.progress,
spawn: { cwd: options.cwd, env: options.env },
});

return new Promise((resolve, reject) => {
let output = "";
const firedTriggers: string[] = [];
let timedOut = false;
let settled = false;

const timer = setTimeout(() => {
timedOut = true;
child.kill("SIGKILL");
}, options.timeoutMs);
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved

// Additional listeners alongside spawnObservedChild's own content-free
// observer; this module needs the real transcript to match rule
// triggers and to report ordered step evidence.
child.stdout?.on("data", (chunk: Buffer) => {
output += chunk.toString("utf-8");
});
child.stderr?.on("data", (chunk: Buffer) => {
for (const line of chunk.toString("utf-8").split("\n")) {
const fired = line.match(/^FIRED\t(.*)$/);
if (fired) firedTriggers.push(fired[1]);
}
});
child.once("error", (error) => {
if (settled) return;
settled = true;
clearTimeout(timer);
reject(error);
});
child.once("close", (code) => {
if (settled) return;
settled = true;
clearTimeout(timer);
resolve({
exitCode: timedOut ? 124 : (code ?? 1),
output,
firedTriggers,
timedOut,
});
});
});
}
164 changes: 164 additions & 0 deletions test/e2e/live/onboard-policy-preset-sequencing.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,164 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { randomBytes } from "node:crypto";

import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts";
import { validateSandboxName } from "../fixtures/clients/sandbox.ts";
import { expect, test } from "../fixtures/e2e-test.ts";
import { startFakeOpenAiCompatibleServer } from "../fixtures/fake-openai-compatible.ts";
import { CLI_ENTRYPOINT, REPO_ROOT } from "../fixtures/paths.ts";
import { redactString } from "../fixtures/redaction.ts";
import { driveInteractiveCommand } from "./onboard-interactive-pty.ts";

// Regression coverage for #6042: "interactive onboard wizard skips Policy
// Presets TUI step". Three independent investigations could not reproduce a
// skip — the onboard state machine has no transition from any earlier state
// directly to `complete`, every path passes through `policies` — but no
// checked-in test drove the real interactive TUI through a PTY to prove it.
// This test is that proof: it answers every interactive prompt in the
// compatible-endpoint journey through a real pseudo-terminal (piped stdin
// does not reproduce the raw-mode selectors this wizard uses) and asserts
// the ordered step markers appear in order, ending with `[8/8] Policy
// presets`, before the wizard can report completion.
//
// This is a hermetic, mock-provider variant of the reporter's journey
// (`nemoclaw onboard` with "Other OpenAI-compatible endpoint"), so it needs
// no NVIDIA credential and runs in ordinary CI, not just a live-inference
// lane.

const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-policy-order";
validateSandboxName(SANDBOX_NAME);
const ONBOARD_TIMEOUT_MS = 40 * 60_000;
const MODEL = "test-model";
// A Docker network namespace cannot reach host loopback directly; bind the
// fake endpoint on all interfaces and advertise the OpenShell host alias so
// both the host-side onboard validation and the sandbox's own inference
// route can reach it (matches the shared E2E inference adapter's mock mode).
const SANDBOX_HOST_ALIAS = "host.openshell.internal";

// The ordered, observable step headers the real interactive wizard prints.
// Each must appear strictly after the previous one; `[8/8] Policy presets`
// is the step the issue claims gets skipped.
const ORDERED_STEP_MARKERS = [
"[1/8] Preflight checks",
"[2/8] Starting OpenShell gateway",
"[3/8] Configuring inference provider",
"[4/8] Setting up inference provider",
"[5/8] Messaging channels",
"[6/8] Creating sandbox",
"[7/8] Setting up OpenClaw inside sandbox",
"[8/8] Policy presets",
] as const;

test("interactive onboard wizard reaches Policy presets in step order (#6042)", {
timeout: ONBOARD_TIMEOUT_MS,
meta: {
e2ePhases: [
"start the local compatible-endpoint fake server",
"drive the interactive onboard wizard through a real PTY",
"confirm every ordered onboarding step appears in order",
"confirm Policy presets is reached before completion",
],
},
}, async ({ artifacts, cleanup, docker, host, progress }) => {
await docker.requireDocker();

progress.phase("start the local compatible-endpoint fake server");
const apiKey = `e2e-6042-${randomBytes(16).toString("hex")}`;
const fake = await startFakeOpenAiCompatibleServer({
apiKey,
chatContent: "PONG",
host: "0.0.0.0",
model: MODEL,
progress,
publicHost: SANDBOX_HOST_ALIAS,
requireAuth: true,
responseText: "PONG",
});
artifacts.addRedactionValues([apiKey]);
cleanup.trackDisposable("close fake compatible-endpoint server", () => fake.close());
cleanup.trackSandbox(host, SANDBOX_NAME, {
artifactName: "cleanup-nemoclaw-destroy-onboard-policy-order",
env: buildAvailabilityProbeEnv(),
redactionValues: [apiKey],
timeoutMs: 120_000,
});

progress.phase("drive the interactive onboard wizard through a real PTY");
const result = await driveInteractiveCommand({
activityLabel: "command: onboard-interactive-pty",
progress,
cmd: [
"node",
CLI_ENTRYPOINT,
"onboard",
"--fresh",
"--agent",
"openclaw",
"--name",
SANDBOX_NAME,
"--yes-i-accept-third-party-software",
],
cwd: REPO_ROOT,
env: buildAvailabilityProbeEnv(),
rules: [
// First-run license notice; already accepted on a reused host, so
// this rule may simply never fire.
{ trigger: "Type 'yes' to accept", response: "yes\n" },
// Only appears when the preflight resource check warns; skipped on
// an adequately provisioned CI runner.
{ trigger: "Continue with onboarding?", response: "y\n" },
// "Other OpenAI-compatible endpoint" — position depends on
// src/lib/onboard/providers.ts's provider list for the openclaw agent.
{ trigger: "Select your inference provider:", response: "4\n" },
{ trigger: "OpenAI-compatible base URL", response: `${fake.baseUrl}\n` },
{ trigger: "API key:", response: `${apiKey}\n` },
{ trigger: "endpoint model", response: `${MODEL}\n` },
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
{ trigger: "Apply this configuration?", response: "y\n" },
{ trigger: "Enable web search", response: "1\n" },
// Raw-mode messaging-channel selector; Enter with none toggled skips.
{ trigger: "Press 1-7 to toggle", response: "\r" },
{ trigger: "Resource profiles:", response: "6\n" },
// Raw-mode Policy tier selector; Enter confirms the pre-selected
// default (Balanced). This is the exact prompt the issue claims the
// wizard never reaches.
{ trigger: "Policy tier", response: "\r" },
// A second raw-mode selector follows immediately: individual preset
// inclusion/rw toggles, pre-populated from the chosen tier. Enter
// confirms the Balanced defaults.
{ trigger: "Presets (", response: "\r" },
],
timeoutMs: ONBOARD_TIMEOUT_MS - 5 * 60_000,
});
await artifacts.writeText("onboard-transcript.txt", result.output);

progress.phase("confirm every ordered onboarding step appears in order");
let searchFrom = 0;
for (const marker of ORDERED_STEP_MARKERS) {
const index = result.output.indexOf(marker, searchFrom);
expect(
index,
`expected step marker ${JSON.stringify(marker)} after offset ${searchFrom} in the transcript; see onboard-transcript.txt`,
).toBeGreaterThanOrEqual(searchFrom);
searchFrom = index + marker.length;
}

progress.phase("confirm Policy presets is reached before completion");
const policyIndex = result.output.indexOf("[8/8] Policy presets");
const abortedIndex = result.output.search(/Onboarding did not finish/i);
const redactedTranscript = redactString(result.output, [apiKey]);
expect(policyIndex, "Policy presets step must be observed").toBeGreaterThanOrEqual(0);
expect(
abortedIndex,
`onboarding must not abort after reaching Policy presets; see onboard-transcript.txt:\n${redactedTranscript}`,
).not.toBeGreaterThanOrEqual(0);
expect(
result.timedOut,
`onboard command timed out; see onboard-transcript.txt:\n${redactedTranscript}`,
).toBe(false);
expect(
result.exitCode,
`onboard command exited non-zero; see onboard-transcript.txt:\n${redactedTranscript}`,
).toBe(0);
});
4 changes: 4 additions & 0 deletions tools/e2e/check-semantic-phases.mts
Original file line number Diff line number Diff line change
Expand Up @@ -403,6 +403,10 @@ const OBSERVED_CHILD_PROGRESS_POLICIES = new Map<string, ObservedChildProgressPo
"test/e2e/live/windows-mxc-openclaw-process-container-helpers.ts#runWindowsMxcOpenClawProcessContainerQualification",
{ kind: "path", path: "progress" },
],
[
"test/e2e/live/onboard-interactive-pty.ts#driveInteractiveCommand",
{ kind: "path", path: "options.progress" },
],
]);

interface DirectChildProcessBindings {
Expand Down